accel/tcg/softmmu_template.h

   1 /*
   2  *  Software MMU support
   3  *
   4  * Generate helpers used by TCG for qemu_ld/st ops and code load
   5  * functions.
   6  *
   7  * Included from target op helpers and exec.c.
   8  *
   9  *  Copyright (c) 2003 Fabrice Bellard
  10  *
  11  * This library is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation; either
  14  * version 2 of the License, or (at your option) any later version.
  15  *
  16  * This library is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  23  */
  24 #if DATA_SIZE == 8
  25 #define SUFFIX q
  26 #define LSUFFIX q
  27 #define SDATA_TYPE  int64_t
  28 #define DATA_TYPE  uint64_t
  29 #elif DATA_SIZE == 4
  30 #define SUFFIX l
  31 #define LSUFFIX l
  32 #define SDATA_TYPE  int32_t
  33 #define DATA_TYPE  uint32_t
  34 #elif DATA_SIZE == 2
  35 #define SUFFIX w
  36 #define LSUFFIX uw
  37 #define SDATA_TYPE  int16_t
  38 #define DATA_TYPE  uint16_t
  39 #elif DATA_SIZE == 1
  40 #define SUFFIX b
  41 #define LSUFFIX ub
  42 #define SDATA_TYPE  int8_t
  43 #define DATA_TYPE  uint8_t
  44 #else
  45 #error unsupported data size
  46 #endif
  47
  48
  49 /* For the benefit of TCG generated code, we want to avoid the complication
  50    of ABI-specific return type promotion and always return a value extended
  51    to the register size of the host.  This is tcg_target_long, except in the
  52    case of a 32-bit host and 64-bit data, and for that we always have
  53    uint64_t.  Don't bother with this widened value for SOFTMMU_CODE_ACCESS.  */
  54 #if defined(SOFTMMU_CODE_ACCESS) || DATA_SIZE == 8
  55 # define WORD_TYPE  DATA_TYPE
  56 # define USUFFIX    SUFFIX
  57 #else
  58 # define WORD_TYPE  tcg_target_ulong
  59 # define USUFFIX    glue(u, SUFFIX)
  60 # define SSUFFIX    glue(s, SUFFIX)
  61 #endif
  62
  63 #ifdef SOFTMMU_CODE_ACCESS
  64 #define READ_ACCESS_TYPE MMU_INST_FETCH
  65 #define ADDR_READ addr_code
  66 #else
  67 #define READ_ACCESS_TYPE MMU_DATA_LOAD
  68 #define ADDR_READ addr_read
  69 #endif
  70
  71 #if DATA_SIZE == 8
  72 # define BSWAP(X)  bswap64(X)
  73 #elif DATA_SIZE == 4
  74 # define BSWAP(X)  bswap32(X)
  75 #elif DATA_SIZE == 2
  76 # define BSWAP(X)  bswap16(X)
  77 #else
  78 # define BSWAP(X)  (X)
  79 #endif
  80
  81 #if DATA_SIZE == 1
  82 # define helper_le_ld_name  glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)
  83 # define helper_be_ld_name  helper_le_ld_name
  84 # define helper_le_lds_name glue(glue(helper_ret_ld, SSUFFIX), MMUSUFFIX)
  85 # define helper_be_lds_name helper_le_lds_name
  86 # define helper_le_st_name  glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)
  87 # define helper_be_st_name  helper_le_st_name
  88 #else
  89 # define helper_le_ld_name  glue(glue(helper_le_ld, USUFFIX), MMUSUFFIX)
  90 # define helper_be_ld_name  glue(glue(helper_be_ld, USUFFIX), MMUSUFFIX)
  91 # define helper_le_lds_name glue(glue(helper_le_ld, SSUFFIX), MMUSUFFIX)
  92 # define helper_be_lds_name glue(glue(helper_be_ld, SSUFFIX), MMUSUFFIX)
  93 # define helper_le_st_name  glue(glue(helper_le_st, SUFFIX), MMUSUFFIX)
  94 # define helper_be_st_name  glue(glue(helper_be_st, SUFFIX), MMUSUFFIX)
  95 #endif
  96
  97 #ifndef SOFTMMU_CODE_ACCESS
  98 static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
  99                                               size_t mmu_idx, size_t index,
 100                                               target_ulong addr,
 101                                               uintptr_t retaddr)
 102 {
 103     CPUIOTLBEntry *iotlbentry = &env->iotlb[mmu_idx][index];
 104     return io_readx(env, iotlbentry, mmu_idx, addr, retaddr, DATA_SIZE);
 105 }
 106 #endif
 107
 108 WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
 109                             TCGMemOpIdx oi, uintptr_t retaddr)
 110 {
 111     unsigned mmu_idx = get_mmuidx(oi);
 112     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 113     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 114     unsigned a_bits = get_alignment_bits(get_memop(oi));
 115     uintptr_t haddr;
 116     DATA_TYPE res;
 117
 118     if (addr & ((1 << a_bits) - 1)) {
 119         cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
 120                              mmu_idx, retaddr);
 121     }
 122
 123     /* If the TLB entry is for a different page, reload and try again.  */
 124     if ((addr & TARGET_PAGE_MASK)
 125          != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 126         if (!VICTIM_TLB_HIT(ADDR_READ, addr)) {
 127             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,
 128                      mmu_idx, retaddr);
 129         }
 130         tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 131     }
 132
 133     /* Handle an IO access.  */
 134     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 135         if ((addr & (DATA_SIZE - 1)) != 0) {
 136             goto do_unaligned_access;
 137         }
 138
 139         /* ??? Note that the io helpers always read data in the target
 140            byte ordering.  We should push the LE/BE request down into io.  */
 141         res = glue(io_read, SUFFIX)(env, mmu_idx, index, addr, retaddr);
 142         res = TGT_LE(res);
 143         return res;
 144     }
 145
 146     /* Handle slow unaligned access (it spans two pages or IO).  */
 147     if (DATA_SIZE > 1
 148         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 149                     >= TARGET_PAGE_SIZE)) {
 150         target_ulong addr1, addr2;
 151         DATA_TYPE res1, res2;
 152         unsigned shift;
 153     do_unaligned_access:
 154         addr1 = addr & ~(DATA_SIZE - 1);
 155         addr2 = addr1 + DATA_SIZE;
 156         res1 = helper_le_ld_name(env, addr1, oi, retaddr);
 157         res2 = helper_le_ld_name(env, addr2, oi, retaddr);
 158         shift = (addr & (DATA_SIZE - 1)) * 8;
 159
 160         /* Little-endian combine.  */
 161         res = (res1 >> shift) | (res2 << ((DATA_SIZE * 8) - shift));
 162         return res;
 163     }
 164
 165     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 166 #if DATA_SIZE == 1
 167     res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);
 168 #else
 169     res = glue(glue(ld, LSUFFIX), _le_p)((uint8_t *)haddr);
 170 #endif
 171     return res;
 172 }
 173
 174 #if DATA_SIZE > 1
 175 WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
 176                             TCGMemOpIdx oi, uintptr_t retaddr)
 177 {
 178     unsigned mmu_idx = get_mmuidx(oi);
 179     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 180     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 181     unsigned a_bits = get_alignment_bits(get_memop(oi));
 182     uintptr_t haddr;
 183     DATA_TYPE res;
 184
 185     if (addr & ((1 << a_bits) - 1)) {
 186         cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
 187                              mmu_idx, retaddr);
 188     }
 189
 190     /* If the TLB entry is for a different page, reload and try again.  */
 191     if ((addr & TARGET_PAGE_MASK)
 192          != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 193         if (!VICTIM_TLB_HIT(ADDR_READ, addr)) {
 194             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,
 195                      mmu_idx, retaddr);
 196         }
 197         tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 198     }
 199
 200     /* Handle an IO access.  */
 201     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 202         if ((addr & (DATA_SIZE - 1)) != 0) {
 203             goto do_unaligned_access;
 204         }
 205
 206         /* ??? Note that the io helpers always read data in the target
 207            byte ordering.  We should push the LE/BE request down into io.  */
 208         res = glue(io_read, SUFFIX)(env, mmu_idx, index, addr, retaddr);
 209         res = TGT_BE(res);
 210         return res;
 211     }
 212
 213     /* Handle slow unaligned access (it spans two pages or IO).  */
 214     if (DATA_SIZE > 1
 215         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 216                     >= TARGET_PAGE_SIZE)) {
 217         target_ulong addr1, addr2;
 218         DATA_TYPE res1, res2;
 219         unsigned shift;
 220     do_unaligned_access:
 221         addr1 = addr & ~(DATA_SIZE - 1);
 222         addr2 = addr1 + DATA_SIZE;
 223         res1 = helper_be_ld_name(env, addr1, oi, retaddr);
 224         res2 = helper_be_ld_name(env, addr2, oi, retaddr);
 225         shift = (addr & (DATA_SIZE - 1)) * 8;
 226
 227         /* Big-endian combine.  */
 228         res = (res1 << shift) | (res2 >> ((DATA_SIZE * 8) - shift));
 229         return res;
 230     }
 231
 232     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 233     res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);
 234     return res;
 235 }
 236 #endif /* DATA_SIZE > 1 */
 237
 238 #ifndef SOFTMMU_CODE_ACCESS
 239
 240 /* Provide signed versions of the load routines as well.  We can of course
 241    avoid this for 64-bit data, or for 32-bit data on 32-bit host.  */
 242 #if DATA_SIZE * 8 < TCG_TARGET_REG_BITS
 243 WORD_TYPE helper_le_lds_name(CPUArchState *env, target_ulong addr,
 244                              TCGMemOpIdx oi, uintptr_t retaddr)
 245 {
 246     return (SDATA_TYPE)helper_le_ld_name(env, addr, oi, retaddr);
 247 }
 248
 249 # if DATA_SIZE > 1
 250 WORD_TYPE helper_be_lds_name(CPUArchState *env, target_ulong addr,
 251                              TCGMemOpIdx oi, uintptr_t retaddr)
 252 {
 253     return (SDATA_TYPE)helper_be_ld_name(env, addr, oi, retaddr);
 254 }
 255 # endif
 256 #endif
 257
 258 static inline void glue(io_write, SUFFIX)(CPUArchState *env,
 259                                           size_t mmu_idx, size_t index,
 260                                           DATA_TYPE val,
 261                                           target_ulong addr,
 262                                           uintptr_t retaddr)
 263 {
 264     CPUIOTLBEntry *iotlbentry = &env->iotlb[mmu_idx][index];
 265     return io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr, DATA_SIZE);
 266 }
 267
 268 void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
 269                        TCGMemOpIdx oi, uintptr_t retaddr)
 270 {
 271     unsigned mmu_idx = get_mmuidx(oi);
 272     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 273     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 274     unsigned a_bits = get_alignment_bits(get_memop(oi));
 275     uintptr_t haddr;
 276
 277     if (addr & ((1 << a_bits) - 1)) {
 278         cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
 279                              mmu_idx, retaddr);
 280     }
 281
 282     /* If the TLB entry is for a different page, reload and try again.  */
 283     if ((addr & TARGET_PAGE_MASK)
 284         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 285         if (!VICTIM_TLB_HIT(addr_write, addr)) {
 286             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
 287                      mmu_idx, retaddr);
 288         }
 289         tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK;
 290     }
 291
 292     /* Handle an IO access.  */
 293     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 294         if ((addr & (DATA_SIZE - 1)) != 0) {
 295             goto do_unaligned_access;
 296         }
 297
 298         /* ??? Note that the io helpers always read data in the target
 299            byte ordering.  We should push the LE/BE request down into io.  */
 300         val = TGT_LE(val);
 301         glue(io_write, SUFFIX)(env, mmu_idx, index, val, addr, retaddr);
 302         return;
 303     }
 304
 305     /* Handle slow unaligned access (it spans two pages or IO).  */
 306     if (DATA_SIZE > 1
 307         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 308                      >= TARGET_PAGE_SIZE)) {
 309         int i, index2;
 310         target_ulong page2, tlb_addr2;
 311     do_unaligned_access:
 312         /* Ensure the second page is in the TLB.  Note that the first page
 313            is already guaranteed to be filled, and that the second page
 314            cannot evict the first.  */
 315         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
 316         index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 317         tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
 318         if (page2 != (tlb_addr2 & (TARGET_PAGE_MASK | TLB_INVALID_MASK))
 319             && !VICTIM_TLB_HIT(addr_write, page2)) {
 320             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
 321                      mmu_idx, retaddr);
 322         }
 323
 324         /* XXX: not efficient, but simple.  */
 325         /* This loop must go in the forward direction to avoid issues
 326            with self-modifying code in Windows 64-bit.  */
 327         for (i = 0; i < DATA_SIZE; ++i) {
 328             /* Little-endian extract.  */
 329             uint8_t val8 = val >> (i * 8);
 330             glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
 331                                             oi, retaddr);
 332         }
 333         return;
 334     }
 335
 336     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 337 #if DATA_SIZE == 1
 338     glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
 339 #else
 340     glue(glue(st, SUFFIX), _le_p)((uint8_t *)haddr, val);
 341 #endif
 342 }
 343
 344 #if DATA_SIZE > 1
 345 void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
 346                        TCGMemOpIdx oi, uintptr_t retaddr)
 347 {
 348     unsigned mmu_idx = get_mmuidx(oi);
 349     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 350     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 351     unsigned a_bits = get_alignment_bits(get_memop(oi));
 352     uintptr_t haddr;
 353
 354     if (addr & ((1 << a_bits) - 1)) {
 355         cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
 356                              mmu_idx, retaddr);
 357     }
 358
 359     /* If the TLB entry is for a different page, reload and try again.  */
 360     if ((addr & TARGET_PAGE_MASK)
 361         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 362         if (!VICTIM_TLB_HIT(addr_write, addr)) {
 363             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
 364                      mmu_idx, retaddr);
 365         }
 366         tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK;
 367     }
 368
 369     /* Handle an IO access.  */
 370     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 371         if ((addr & (DATA_SIZE - 1)) != 0) {
 372             goto do_unaligned_access;
 373         }
 374
 375         /* ??? Note that the io helpers always read data in the target
 376            byte ordering.  We should push the LE/BE request down into io.  */
 377         val = TGT_BE(val);
 378         glue(io_write, SUFFIX)(env, mmu_idx, index, val, addr, retaddr);
 379         return;
 380     }
 381
 382     /* Handle slow unaligned access (it spans two pages or IO).  */
 383     if (DATA_SIZE > 1
 384         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 385                      >= TARGET_PAGE_SIZE)) {
 386         int i, index2;
 387         target_ulong page2, tlb_addr2;
 388     do_unaligned_access:
 389         /* Ensure the second page is in the TLB.  Note that the first page
 390            is already guaranteed to be filled, and that the second page
 391            cannot evict the first.  */
 392         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
 393         index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 394         tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
 395         if (page2 != (tlb_addr2 & (TARGET_PAGE_MASK | TLB_INVALID_MASK))
 396             && !VICTIM_TLB_HIT(addr_write, page2)) {
 397             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
 398                      mmu_idx, retaddr);
 399         }
 400
 401         /* XXX: not efficient, but simple */
 402         /* This loop must go in the forward direction to avoid issues
 403            with self-modifying code.  */
 404         for (i = 0; i < DATA_SIZE; ++i) {
 405             /* Big-endian extract.  */
 406             uint8_t val8 = val >> (((DATA_SIZE - 1) * 8) - (i * 8));
 407             glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
 408                                             oi, retaddr);
 409         }
 410         return;
 411     }
 412
 413     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 414     glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);
 415 }
 416 #endif /* DATA_SIZE > 1 */
 417 #endif /* !defined(SOFTMMU_CODE_ACCESS) */
 418
 419 #undef READ_ACCESS_TYPE
 420 #undef DATA_TYPE
 421 #undef SUFFIX
 422 #undef LSUFFIX
 423 #undef DATA_SIZE
 424 #undef ADDR_READ
 425 #undef WORD_TYPE
 426 #undef SDATA_TYPE
 427 #undef USUFFIX
 428 #undef SSUFFIX
 429 #undef BSWAP
 430 #undef helper_le_ld_name
 431 #undef helper_be_ld_name
 432 #undef helper_le_lds_name
 433 #undef helper_be_lds_name
 434 #undef helper_le_st_name
 435 #undef helper_be_st_name