accel/tcg/softmmu_template.h

   1 /*
   2  *  Software MMU support
   3  *
   4  * Generate helpers used by TCG for qemu_ld/st ops and code load
   5  * functions.
   6  *
   7  * Included from target op helpers and exec.c.
   8  *
   9  *  Copyright (c) 2003 Fabrice Bellard
  10  *
  11  * This library is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation; either
  14  * version 2 of the License, or (at your option) any later version.
  15  *
  16  * This library is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  23  */
  24 #if DATA_SIZE == 8
  25 #define SUFFIX q
  26 #define LSUFFIX q
  27 #define SDATA_TYPE  int64_t
  28 #define DATA_TYPE  uint64_t
  29 #elif DATA_SIZE == 4
  30 #define SUFFIX l
  31 #define LSUFFIX l
  32 #define SDATA_TYPE  int32_t
  33 #define DATA_TYPE  uint32_t
  34 #elif DATA_SIZE == 2
  35 #define SUFFIX w
  36 #define LSUFFIX uw
  37 #define SDATA_TYPE  int16_t
  38 #define DATA_TYPE  uint16_t
  39 #elif DATA_SIZE == 1
  40 #define SUFFIX b
  41 #define LSUFFIX ub
  42 #define SDATA_TYPE  int8_t
  43 #define DATA_TYPE  uint8_t
  44 #else
  45 #error unsupported data size
  46 #endif
  47
  48
  49 /* For the benefit of TCG generated code, we want to avoid the complication
  50    of ABI-specific return type promotion and always return a value extended
  51    to the register size of the host.  This is tcg_target_long, except in the
  52    case of a 32-bit host and 64-bit data, and for that we always have
  53    uint64_t.  Don't bother with this widened value for SOFTMMU_CODE_ACCESS.  */
  54 #if defined(SOFTMMU_CODE_ACCESS) || DATA_SIZE == 8
  55 # define WORD_TYPE  DATA_TYPE
  56 # define USUFFIX    SUFFIX
  57 #else
  58 # define WORD_TYPE  tcg_target_ulong
  59 # define USUFFIX    glue(u, SUFFIX)
  60 # define SSUFFIX    glue(s, SUFFIX)
  61 #endif
  62
  63 #ifdef SOFTMMU_CODE_ACCESS
  64 #define READ_ACCESS_TYPE MMU_INST_FETCH
  65 #define ADDR_READ addr_code
  66 #else
  67 #define READ_ACCESS_TYPE MMU_DATA_LOAD
  68 #define ADDR_READ addr_read
  69 #endif
  70
  71 #if DATA_SIZE == 8
  72 # define BSWAP(X)  bswap64(X)
  73 #elif DATA_SIZE == 4
  74 # define BSWAP(X)  bswap32(X)
  75 #elif DATA_SIZE == 2
  76 # define BSWAP(X)  bswap16(X)
  77 #else
  78 # define BSWAP(X)  (X)
  79 #endif
  80
  81 #if DATA_SIZE == 1
  82 # define helper_le_ld_name  glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)
  83 # define helper_be_ld_name  helper_le_ld_name
  84 # define helper_le_lds_name glue(glue(helper_ret_ld, SSUFFIX), MMUSUFFIX)
  85 # define helper_be_lds_name helper_le_lds_name
  86 # define helper_le_st_name  glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)
  87 # define helper_be_st_name  helper_le_st_name
  88 #else
  89 # define helper_le_ld_name  glue(glue(helper_le_ld, USUFFIX), MMUSUFFIX)
  90 # define helper_be_ld_name  glue(glue(helper_be_ld, USUFFIX), MMUSUFFIX)
  91 # define helper_le_lds_name glue(glue(helper_le_ld, SSUFFIX), MMUSUFFIX)
  92 # define helper_be_lds_name glue(glue(helper_be_ld, SSUFFIX), MMUSUFFIX)
  93 # define helper_le_st_name  glue(glue(helper_le_st, SUFFIX), MMUSUFFIX)
  94 # define helper_be_st_name  glue(glue(helper_be_st, SUFFIX), MMUSUFFIX)
  95 #endif
  96
  97 #ifndef SOFTMMU_CODE_ACCESS
  98 static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
  99                                               size_t mmu_idx, size_t index,
 100                                               target_ulong addr,
 101                                               uintptr_t retaddr,
 102                                               bool recheck)
 103 {
 104     CPUIOTLBEntry *iotlbentry = &env->iotlb[mmu_idx][index];
 105     return io_readx(env, iotlbentry, mmu_idx, addr, retaddr, recheck,
 106                     DATA_SIZE);
 107 }
 108 #endif
 109
 110 WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
 111                             TCGMemOpIdx oi, uintptr_t retaddr)
 112 {
 113     unsigned mmu_idx = get_mmuidx(oi);
 114     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 115     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 116     unsigned a_bits = get_alignment_bits(get_memop(oi));
 117     uintptr_t haddr;
 118     DATA_TYPE res;
 119
 120     if (addr & ((1 << a_bits) - 1)) {
 121         cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
 122                              mmu_idx, retaddr);
 123     }
 124
 125     /* If the TLB entry is for a different page, reload and try again.  */
 126     if ((addr & TARGET_PAGE_MASK)
 127          != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 128         if (!VICTIM_TLB_HIT(ADDR_READ, addr)) {
 129             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,
 130                      mmu_idx, retaddr);
 131         }
 132         tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 133     }
 134
 135     /* Handle an IO access.  */
 136     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 137         if ((addr & (DATA_SIZE - 1)) != 0) {
 138             goto do_unaligned_access;
 139         }
 140
 141         /* ??? Note that the io helpers always read data in the target
 142            byte ordering.  We should push the LE/BE request down into io.  */
 143         res = glue(io_read, SUFFIX)(env, mmu_idx, index, addr, retaddr,
 144                                     tlb_addr & TLB_RECHECK);
 145         res = TGT_LE(res);
 146         return res;
 147     }
 148
 149     /* Handle slow unaligned access (it spans two pages or IO).  */
 150     if (DATA_SIZE > 1
 151         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 152                     >= TARGET_PAGE_SIZE)) {
 153         target_ulong addr1, addr2;
 154         DATA_TYPE res1, res2;
 155         unsigned shift;
 156     do_unaligned_access:
 157         addr1 = addr & ~(DATA_SIZE - 1);
 158         addr2 = addr1 + DATA_SIZE;
 159         res1 = helper_le_ld_name(env, addr1, oi, retaddr);
 160         res2 = helper_le_ld_name(env, addr2, oi, retaddr);
 161         shift = (addr & (DATA_SIZE - 1)) * 8;
 162
 163         /* Little-endian combine.  */
 164         res = (res1 >> shift) | (res2 << ((DATA_SIZE * 8) - shift));
 165         return res;
 166     }
 167
 168     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 169 #if DATA_SIZE == 1
 170     res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);
 171 #else
 172     res = glue(glue(ld, LSUFFIX), _le_p)((uint8_t *)haddr);
 173 #endif
 174     return res;
 175 }
 176
 177 #if DATA_SIZE > 1
 178 WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
 179                             TCGMemOpIdx oi, uintptr_t retaddr)
 180 {
 181     unsigned mmu_idx = get_mmuidx(oi);
 182     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 183     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 184     unsigned a_bits = get_alignment_bits(get_memop(oi));
 185     uintptr_t haddr;
 186     DATA_TYPE res;
 187
 188     if (addr & ((1 << a_bits) - 1)) {
 189         cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
 190                              mmu_idx, retaddr);
 191     }
 192
 193     /* If the TLB entry is for a different page, reload and try again.  */
 194     if ((addr & TARGET_PAGE_MASK)
 195          != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 196         if (!VICTIM_TLB_HIT(ADDR_READ, addr)) {
 197             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,
 198                      mmu_idx, retaddr);
 199         }
 200         tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 201     }
 202
 203     /* Handle an IO access.  */
 204     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 205         if ((addr & (DATA_SIZE - 1)) != 0) {
 206             goto do_unaligned_access;
 207         }
 208
 209         /* ??? Note that the io helpers always read data in the target
 210            byte ordering.  We should push the LE/BE request down into io.  */
 211         res = glue(io_read, SUFFIX)(env, mmu_idx, index, addr, retaddr,
 212                                     tlb_addr & TLB_RECHECK);
 213         res = TGT_BE(res);
 214         return res;
 215     }
 216
 217     /* Handle slow unaligned access (it spans two pages or IO).  */
 218     if (DATA_SIZE > 1
 219         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 220                     >= TARGET_PAGE_SIZE)) {
 221         target_ulong addr1, addr2;
 222         DATA_TYPE res1, res2;
 223         unsigned shift;
 224     do_unaligned_access:
 225         addr1 = addr & ~(DATA_SIZE - 1);
 226         addr2 = addr1 + DATA_SIZE;
 227         res1 = helper_be_ld_name(env, addr1, oi, retaddr);
 228         res2 = helper_be_ld_name(env, addr2, oi, retaddr);
 229         shift = (addr & (DATA_SIZE - 1)) * 8;
 230
 231         /* Big-endian combine.  */
 232         res = (res1 << shift) | (res2 >> ((DATA_SIZE * 8) - shift));
 233         return res;
 234     }
 235
 236     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 237     res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);
 238     return res;
 239 }
 240 #endif /* DATA_SIZE > 1 */
 241
 242 #ifndef SOFTMMU_CODE_ACCESS
 243
 244 /* Provide signed versions of the load routines as well.  We can of course
 245    avoid this for 64-bit data, or for 32-bit data on 32-bit host.  */
 246 #if DATA_SIZE * 8 < TCG_TARGET_REG_BITS
 247 WORD_TYPE helper_le_lds_name(CPUArchState *env, target_ulong addr,
 248                              TCGMemOpIdx oi, uintptr_t retaddr)
 249 {
 250     return (SDATA_TYPE)helper_le_ld_name(env, addr, oi, retaddr);
 251 }
 252
 253 # if DATA_SIZE > 1
 254 WORD_TYPE helper_be_lds_name(CPUArchState *env, target_ulong addr,
 255                              TCGMemOpIdx oi, uintptr_t retaddr)
 256 {
 257     return (SDATA_TYPE)helper_be_ld_name(env, addr, oi, retaddr);
 258 }
 259 # endif
 260 #endif
 261
 262 static inline void glue(io_write, SUFFIX)(CPUArchState *env,
 263                                           size_t mmu_idx, size_t index,
 264                                           DATA_TYPE val,
 265                                           target_ulong addr,
 266                                           uintptr_t retaddr,
 267                                           bool recheck)
 268 {
 269     CPUIOTLBEntry *iotlbentry = &env->iotlb[mmu_idx][index];
 270     return io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr,
 271                      recheck, DATA_SIZE);
 272 }
 273
 274 void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
 275                        TCGMemOpIdx oi, uintptr_t retaddr)
 276 {
 277     unsigned mmu_idx = get_mmuidx(oi);
 278     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 279     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 280     unsigned a_bits = get_alignment_bits(get_memop(oi));
 281     uintptr_t haddr;
 282
 283     if (addr & ((1 << a_bits) - 1)) {
 284         cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
 285                              mmu_idx, retaddr);
 286     }
 287
 288     /* If the TLB entry is for a different page, reload and try again.  */
 289     if ((addr & TARGET_PAGE_MASK)
 290         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 291         if (!VICTIM_TLB_HIT(addr_write, addr)) {
 292             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
 293                      mmu_idx, retaddr);
 294         }
 295         tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK;
 296     }
 297
 298     /* Handle an IO access.  */
 299     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 300         if ((addr & (DATA_SIZE - 1)) != 0) {
 301             goto do_unaligned_access;
 302         }
 303
 304         /* ??? Note that the io helpers always read data in the target
 305            byte ordering.  We should push the LE/BE request down into io.  */
 306         val = TGT_LE(val);
 307         glue(io_write, SUFFIX)(env, mmu_idx, index, val, addr,
 308                                retaddr, tlb_addr & TLB_RECHECK);
 309         return;
 310     }
 311
 312     /* Handle slow unaligned access (it spans two pages or IO).  */
 313     if (DATA_SIZE > 1
 314         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 315                      >= TARGET_PAGE_SIZE)) {
 316         int i, index2;
 317         target_ulong page2, tlb_addr2;
 318     do_unaligned_access:
 319         /* Ensure the second page is in the TLB.  Note that the first page
 320            is already guaranteed to be filled, and that the second page
 321            cannot evict the first.  */
 322         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
 323         index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 324         tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
 325         if (page2 != (tlb_addr2 & (TARGET_PAGE_MASK | TLB_INVALID_MASK))
 326             && !VICTIM_TLB_HIT(addr_write, page2)) {
 327             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
 328                      mmu_idx, retaddr);
 329         }
 330
 331         /* XXX: not efficient, but simple.  */
 332         /* This loop must go in the forward direction to avoid issues
 333            with self-modifying code in Windows 64-bit.  */
 334         for (i = 0; i < DATA_SIZE; ++i) {
 335             /* Little-endian extract.  */
 336             uint8_t val8 = val >> (i * 8);
 337             glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
 338                                             oi, retaddr);
 339         }
 340         return;
 341     }
 342
 343     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 344 #if DATA_SIZE == 1
 345     glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
 346 #else
 347     glue(glue(st, SUFFIX), _le_p)((uint8_t *)haddr, val);
 348 #endif
 349 }
 350
 351 #if DATA_SIZE > 1
 352 void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
 353                        TCGMemOpIdx oi, uintptr_t retaddr)
 354 {
 355     unsigned mmu_idx = get_mmuidx(oi);
 356     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 357     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 358     unsigned a_bits = get_alignment_bits(get_memop(oi));
 359     uintptr_t haddr;
 360
 361     if (addr & ((1 << a_bits) - 1)) {
 362         cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
 363                              mmu_idx, retaddr);
 364     }
 365
 366     /* If the TLB entry is for a different page, reload and try again.  */
 367     if ((addr & TARGET_PAGE_MASK)
 368         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 369         if (!VICTIM_TLB_HIT(addr_write, addr)) {
 370             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
 371                      mmu_idx, retaddr);
 372         }
 373         tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK;
 374     }
 375
 376     /* Handle an IO access.  */
 377     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 378         if ((addr & (DATA_SIZE - 1)) != 0) {
 379             goto do_unaligned_access;
 380         }
 381
 382         /* ??? Note that the io helpers always read data in the target
 383            byte ordering.  We should push the LE/BE request down into io.  */
 384         val = TGT_BE(val);
 385         glue(io_write, SUFFIX)(env, mmu_idx, index, val, addr, retaddr,
 386                                tlb_addr & TLB_RECHECK);
 387         return;
 388     }
 389
 390     /* Handle slow unaligned access (it spans two pages or IO).  */
 391     if (DATA_SIZE > 1
 392         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 393                      >= TARGET_PAGE_SIZE)) {
 394         int i, index2;
 395         target_ulong page2, tlb_addr2;
 396     do_unaligned_access:
 397         /* Ensure the second page is in the TLB.  Note that the first page
 398            is already guaranteed to be filled, and that the second page
 399            cannot evict the first.  */
 400         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
 401         index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 402         tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
 403         if (page2 != (tlb_addr2 & (TARGET_PAGE_MASK | TLB_INVALID_MASK))
 404             && !VICTIM_TLB_HIT(addr_write, page2)) {
 405             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
 406                      mmu_idx, retaddr);
 407         }
 408
 409         /* XXX: not efficient, but simple */
 410         /* This loop must go in the forward direction to avoid issues
 411            with self-modifying code.  */
 412         for (i = 0; i < DATA_SIZE; ++i) {
 413             /* Big-endian extract.  */
 414             uint8_t val8 = val >> (((DATA_SIZE - 1) * 8) - (i * 8));
 415             glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
 416                                             oi, retaddr);
 417         }
 418         return;
 419     }
 420
 421     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 422     glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);
 423 }
 424 #endif /* DATA_SIZE > 1 */
 425 #endif /* !defined(SOFTMMU_CODE_ACCESS) */
 426
 427 #undef READ_ACCESS_TYPE
 428 #undef DATA_TYPE
 429 #undef SUFFIX
 430 #undef LSUFFIX
 431 #undef DATA_SIZE
 432 #undef ADDR_READ
 433 #undef WORD_TYPE
 434 #undef SDATA_TYPE
 435 #undef USUFFIX
 436 #undef SSUFFIX
 437 #undef BSWAP
 438 #undef helper_le_ld_name
 439 #undef helper_be_ld_name
 440 #undef helper_le_lds_name
 441 #undef helper_be_lds_name
 442 #undef helper_le_st_name
 443 #undef helper_be_st_name