softmmu_template.h

   1 /*
   2  *  Software MMU support
   3  *
   4  * Generate helpers used by TCG for qemu_ld/st ops and code load
   5  * functions.
   6  *
   7  * Included from target op helpers and exec.c.
   8  *
   9  *  Copyright (c) 2003 Fabrice Bellard
  10  *
  11  * This library is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation; either
  14  * version 2 of the License, or (at your option) any later version.
  15  *
  16  * This library is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  23  */
  24 #if DATA_SIZE == 8
  25 #define SUFFIX q
  26 #define LSUFFIX q
  27 #define SDATA_TYPE  int64_t
  28 #define DATA_TYPE  uint64_t
  29 #elif DATA_SIZE == 4
  30 #define SUFFIX l
  31 #define LSUFFIX l
  32 #define SDATA_TYPE  int32_t
  33 #define DATA_TYPE  uint32_t
  34 #elif DATA_SIZE == 2
  35 #define SUFFIX w
  36 #define LSUFFIX uw
  37 #define SDATA_TYPE  int16_t
  38 #define DATA_TYPE  uint16_t
  39 #elif DATA_SIZE == 1
  40 #define SUFFIX b
  41 #define LSUFFIX ub
  42 #define SDATA_TYPE  int8_t
  43 #define DATA_TYPE  uint8_t
  44 #else
  45 #error unsupported data size
  46 #endif
  47
  48
  49 /* For the benefit of TCG generated code, we want to avoid the complication
  50    of ABI-specific return type promotion and always return a value extended
  51    to the register size of the host.  This is tcg_target_long, except in the
  52    case of a 32-bit host and 64-bit data, and for that we always have
  53    uint64_t.  Don't bother with this widened value for SOFTMMU_CODE_ACCESS.  */
  54 #if defined(SOFTMMU_CODE_ACCESS) || DATA_SIZE == 8
  55 # define WORD_TYPE  DATA_TYPE
  56 # define USUFFIX    SUFFIX
  57 #else
  58 # define WORD_TYPE  tcg_target_ulong
  59 # define USUFFIX    glue(u, SUFFIX)
  60 # define SSUFFIX    glue(s, SUFFIX)
  61 #endif
  62
  63 #ifdef SOFTMMU_CODE_ACCESS
  64 #define READ_ACCESS_TYPE MMU_INST_FETCH
  65 #define ADDR_READ addr_code
  66 #else
  67 #define READ_ACCESS_TYPE MMU_DATA_LOAD
  68 #define ADDR_READ addr_read
  69 #endif
  70
  71 #if DATA_SIZE == 8
  72 # define BSWAP(X)  bswap64(X)
  73 #elif DATA_SIZE == 4
  74 # define BSWAP(X)  bswap32(X)
  75 #elif DATA_SIZE == 2
  76 # define BSWAP(X)  bswap16(X)
  77 #else
  78 # define BSWAP(X)  (X)
  79 #endif
  80
  81 #ifdef TARGET_WORDS_BIGENDIAN
  82 # define TGT_BE(X)  (X)
  83 # define TGT_LE(X)  BSWAP(X)
  84 #else
  85 # define TGT_BE(X)  BSWAP(X)
  86 # define TGT_LE(X)  (X)
  87 #endif
  88
  89 #if DATA_SIZE == 1
  90 # define helper_le_ld_name  glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)
  91 # define helper_be_ld_name  helper_le_ld_name
  92 # define helper_le_lds_name glue(glue(helper_ret_ld, SSUFFIX), MMUSUFFIX)
  93 # define helper_be_lds_name helper_le_lds_name
  94 # define helper_le_st_name  glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)
  95 # define helper_be_st_name  helper_le_st_name
  96 #else
  97 # define helper_le_ld_name  glue(glue(helper_le_ld, USUFFIX), MMUSUFFIX)
  98 # define helper_be_ld_name  glue(glue(helper_be_ld, USUFFIX), MMUSUFFIX)
  99 # define helper_le_lds_name glue(glue(helper_le_ld, SSUFFIX), MMUSUFFIX)
 100 # define helper_be_lds_name glue(glue(helper_be_ld, SSUFFIX), MMUSUFFIX)
 101 # define helper_le_st_name  glue(glue(helper_le_st, SUFFIX), MMUSUFFIX)
 102 # define helper_be_st_name  glue(glue(helper_be_st, SUFFIX), MMUSUFFIX)
 103 #endif
 104
 105 #ifdef TARGET_WORDS_BIGENDIAN
 106 # define helper_te_ld_name  helper_be_ld_name
 107 # define helper_te_st_name  helper_be_st_name
 108 #else
 109 # define helper_te_ld_name  helper_le_ld_name
 110 # define helper_te_st_name  helper_le_st_name
 111 #endif
 112
 113 #ifndef SOFTMMU_CODE_ACCESS
 114 static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
 115                                               size_t mmu_idx, size_t index,
 116                                               target_ulong addr,
 117                                               uintptr_t retaddr)
 118 {
 119     CPUIOTLBEntry *iotlbentry = &env->iotlb[mmu_idx][index];
 120     return io_readx(env, iotlbentry, addr, retaddr, DATA_SIZE);
 121 }
 122 #endif
 123
 124 WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
 125                             TCGMemOpIdx oi, uintptr_t retaddr)
 126 {
 127     unsigned mmu_idx = get_mmuidx(oi);
 128     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 129     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 130     unsigned a_bits = get_alignment_bits(get_memop(oi));
 131     uintptr_t haddr;
 132     DATA_TYPE res;
 133
 134     if (addr & ((1 << a_bits) - 1)) {
 135         cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
 136                              mmu_idx, retaddr);
 137     }
 138
 139     /* If the TLB entry is for a different page, reload and try again.  */
 140     if ((addr & TARGET_PAGE_MASK)
 141          != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 142         if (!VICTIM_TLB_HIT(ADDR_READ, addr)) {
 143             tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
 144                      mmu_idx, retaddr);
 145         }
 146         tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 147     }
 148
 149     /* Handle an IO access.  */
 150     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 151         if ((addr & (DATA_SIZE - 1)) != 0) {
 152             goto do_unaligned_access;
 153         }
 154
 155         /* ??? Note that the io helpers always read data in the target
 156            byte ordering.  We should push the LE/BE request down into io.  */
 157         res = glue(io_read, SUFFIX)(env, mmu_idx, index, addr, retaddr);
 158         res = TGT_LE(res);
 159         return res;
 160     }
 161
 162     /* Handle slow unaligned access (it spans two pages or IO).  */
 163     if (DATA_SIZE > 1
 164         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 165                     >= TARGET_PAGE_SIZE)) {
 166         target_ulong addr1, addr2;
 167         DATA_TYPE res1, res2;
 168         unsigned shift;
 169     do_unaligned_access:
 170         addr1 = addr & ~(DATA_SIZE - 1);
 171         addr2 = addr1 + DATA_SIZE;
 172         res1 = helper_le_ld_name(env, addr1, oi, retaddr);
 173         res2 = helper_le_ld_name(env, addr2, oi, retaddr);
 174         shift = (addr & (DATA_SIZE - 1)) * 8;
 175
 176         /* Little-endian combine.  */
 177         res = (res1 >> shift) | (res2 << ((DATA_SIZE * 8) - shift));
 178         return res;
 179     }
 180
 181     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 182 #if DATA_SIZE == 1
 183     res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);
 184 #else
 185     res = glue(glue(ld, LSUFFIX), _le_p)((uint8_t *)haddr);
 186 #endif
 187     return res;
 188 }
 189
 190 #if DATA_SIZE > 1
 191 WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
 192                             TCGMemOpIdx oi, uintptr_t retaddr)
 193 {
 194     unsigned mmu_idx = get_mmuidx(oi);
 195     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 196     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 197     unsigned a_bits = get_alignment_bits(get_memop(oi));
 198     uintptr_t haddr;
 199     DATA_TYPE res;
 200
 201     if (addr & ((1 << a_bits) - 1)) {
 202         cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
 203                              mmu_idx, retaddr);
 204     }
 205
 206     /* If the TLB entry is for a different page, reload and try again.  */
 207     if ((addr & TARGET_PAGE_MASK)
 208          != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 209         if (!VICTIM_TLB_HIT(ADDR_READ, addr)) {
 210             tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
 211                      mmu_idx, retaddr);
 212         }
 213         tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
 214     }
 215
 216     /* Handle an IO access.  */
 217     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 218         if ((addr & (DATA_SIZE - 1)) != 0) {
 219             goto do_unaligned_access;
 220         }
 221
 222         /* ??? Note that the io helpers always read data in the target
 223            byte ordering.  We should push the LE/BE request down into io.  */
 224         res = glue(io_read, SUFFIX)(env, mmu_idx, index, addr, retaddr);
 225         res = TGT_BE(res);
 226         return res;
 227     }
 228
 229     /* Handle slow unaligned access (it spans two pages or IO).  */
 230     if (DATA_SIZE > 1
 231         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 232                     >= TARGET_PAGE_SIZE)) {
 233         target_ulong addr1, addr2;
 234         DATA_TYPE res1, res2;
 235         unsigned shift;
 236     do_unaligned_access:
 237         addr1 = addr & ~(DATA_SIZE - 1);
 238         addr2 = addr1 + DATA_SIZE;
 239         res1 = helper_be_ld_name(env, addr1, oi, retaddr);
 240         res2 = helper_be_ld_name(env, addr2, oi, retaddr);
 241         shift = (addr & (DATA_SIZE - 1)) * 8;
 242
 243         /* Big-endian combine.  */
 244         res = (res1 << shift) | (res2 >> ((DATA_SIZE * 8) - shift));
 245         return res;
 246     }
 247
 248     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 249     res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);
 250     return res;
 251 }
 252 #endif /* DATA_SIZE > 1 */
 253
 254 #ifndef SOFTMMU_CODE_ACCESS
 255
 256 /* Provide signed versions of the load routines as well.  We can of course
 257    avoid this for 64-bit data, or for 32-bit data on 32-bit host.  */
 258 #if DATA_SIZE * 8 < TCG_TARGET_REG_BITS
 259 WORD_TYPE helper_le_lds_name(CPUArchState *env, target_ulong addr,
 260                              TCGMemOpIdx oi, uintptr_t retaddr)
 261 {
 262     return (SDATA_TYPE)helper_le_ld_name(env, addr, oi, retaddr);
 263 }
 264
 265 # if DATA_SIZE > 1
 266 WORD_TYPE helper_be_lds_name(CPUArchState *env, target_ulong addr,
 267                              TCGMemOpIdx oi, uintptr_t retaddr)
 268 {
 269     return (SDATA_TYPE)helper_be_ld_name(env, addr, oi, retaddr);
 270 }
 271 # endif
 272 #endif
 273
 274 static inline void glue(io_write, SUFFIX)(CPUArchState *env,
 275                                           size_t mmu_idx, size_t index,
 276                                           DATA_TYPE val,
 277                                           target_ulong addr,
 278                                           uintptr_t retaddr)
 279 {
 280     CPUIOTLBEntry *iotlbentry = &env->iotlb[mmu_idx][index];
 281     return io_writex(env, iotlbentry, val, addr, retaddr, DATA_SIZE);
 282 }
 283
 284 void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
 285                        TCGMemOpIdx oi, uintptr_t retaddr)
 286 {
 287     unsigned mmu_idx = get_mmuidx(oi);
 288     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 289     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 290     unsigned a_bits = get_alignment_bits(get_memop(oi));
 291     uintptr_t haddr;
 292
 293     if (addr & ((1 << a_bits) - 1)) {
 294         cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
 295                              mmu_idx, retaddr);
 296     }
 297
 298     /* If the TLB entry is for a different page, reload and try again.  */
 299     if ((addr & TARGET_PAGE_MASK)
 300         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 301         if (!VICTIM_TLB_HIT(addr_write, addr)) {
 302             tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
 303         }
 304         tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 305     }
 306
 307     /* Handle an IO access.  */
 308     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 309         if ((addr & (DATA_SIZE - 1)) != 0) {
 310             goto do_unaligned_access;
 311         }
 312
 313         /* ??? Note that the io helpers always read data in the target
 314            byte ordering.  We should push the LE/BE request down into io.  */
 315         val = TGT_LE(val);
 316         glue(io_write, SUFFIX)(env, mmu_idx, index, val, addr, retaddr);
 317         return;
 318     }
 319
 320     /* Handle slow unaligned access (it spans two pages or IO).  */
 321     if (DATA_SIZE > 1
 322         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 323                      >= TARGET_PAGE_SIZE)) {
 324         int i, index2;
 325         target_ulong page2, tlb_addr2;
 326     do_unaligned_access:
 327         /* Ensure the second page is in the TLB.  Note that the first page
 328            is already guaranteed to be filled, and that the second page
 329            cannot evict the first.  */
 330         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
 331         index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 332         tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
 333         if (page2 != (tlb_addr2 & (TARGET_PAGE_MASK | TLB_INVALID_MASK))
 334             && !VICTIM_TLB_HIT(addr_write, page2)) {
 335             tlb_fill(ENV_GET_CPU(env), page2, MMU_DATA_STORE,
 336                      mmu_idx, retaddr);
 337         }
 338
 339         /* XXX: not efficient, but simple.  */
 340         /* This loop must go in the forward direction to avoid issues
 341            with self-modifying code in Windows 64-bit.  */
 342         for (i = 0; i < DATA_SIZE; ++i) {
 343             /* Little-endian extract.  */
 344             uint8_t val8 = val >> (i * 8);
 345             glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
 346                                             oi, retaddr);
 347         }
 348         return;
 349     }
 350
 351     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 352 #if DATA_SIZE == 1
 353     glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
 354 #else
 355     glue(glue(st, SUFFIX), _le_p)((uint8_t *)haddr, val);
 356 #endif
 357 }
 358
 359 #if DATA_SIZE > 1
 360 void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
 361                        TCGMemOpIdx oi, uintptr_t retaddr)
 362 {
 363     unsigned mmu_idx = get_mmuidx(oi);
 364     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 365     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 366     unsigned a_bits = get_alignment_bits(get_memop(oi));
 367     uintptr_t haddr;
 368
 369     if (addr & ((1 << a_bits) - 1)) {
 370         cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
 371                              mmu_idx, retaddr);
 372     }
 373
 374     /* If the TLB entry is for a different page, reload and try again.  */
 375     if ((addr & TARGET_PAGE_MASK)
 376         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 377         if (!VICTIM_TLB_HIT(addr_write, addr)) {
 378             tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
 379         }
 380         tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 381     }
 382
 383     /* Handle an IO access.  */
 384     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 385         if ((addr & (DATA_SIZE - 1)) != 0) {
 386             goto do_unaligned_access;
 387         }
 388
 389         /* ??? Note that the io helpers always read data in the target
 390            byte ordering.  We should push the LE/BE request down into io.  */
 391         val = TGT_BE(val);
 392         glue(io_write, SUFFIX)(env, mmu_idx, index, val, addr, retaddr);
 393         return;
 394     }
 395
 396     /* Handle slow unaligned access (it spans two pages or IO).  */
 397     if (DATA_SIZE > 1
 398         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 399                      >= TARGET_PAGE_SIZE)) {
 400         int i, index2;
 401         target_ulong page2, tlb_addr2;
 402     do_unaligned_access:
 403         /* Ensure the second page is in the TLB.  Note that the first page
 404            is already guaranteed to be filled, and that the second page
 405            cannot evict the first.  */
 406         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
 407         index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 408         tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
 409         if (page2 != (tlb_addr2 & (TARGET_PAGE_MASK | TLB_INVALID_MASK))
 410             && !VICTIM_TLB_HIT(addr_write, page2)) {
 411             tlb_fill(ENV_GET_CPU(env), page2, MMU_DATA_STORE,
 412                      mmu_idx, retaddr);
 413         }
 414
 415         /* XXX: not efficient, but simple */
 416         /* This loop must go in the forward direction to avoid issues
 417            with self-modifying code.  */
 418         for (i = 0; i < DATA_SIZE; ++i) {
 419             /* Big-endian extract.  */
 420             uint8_t val8 = val >> (((DATA_SIZE - 1) * 8) - (i * 8));
 421             glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
 422                                             oi, retaddr);
 423         }
 424         return;
 425     }
 426
 427     haddr = addr + env->tlb_table[mmu_idx][index].addend;
 428     glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);
 429 }
 430 #endif /* DATA_SIZE > 1 */
 431 #endif /* !defined(SOFTMMU_CODE_ACCESS) */
 432
 433 #undef READ_ACCESS_TYPE
 434 #undef DATA_TYPE
 435 #undef SUFFIX
 436 #undef LSUFFIX
 437 #undef DATA_SIZE
 438 #undef ADDR_READ
 439 #undef WORD_TYPE
 440 #undef SDATA_TYPE
 441 #undef USUFFIX
 442 #undef SSUFFIX
 443 #undef BSWAP
 444 #undef TGT_BE
 445 #undef TGT_LE
 446 #undef CPU_BE
 447 #undef CPU_LE
 448 #undef helper_le_ld_name
 449 #undef helper_be_ld_name
 450 #undef helper_le_lds_name
 451 #undef helper_be_lds_name
 452 #undef helper_le_st_name
 453 #undef helper_be_st_name
 454 #undef helper_te_ld_name
 455 #undef helper_te_st_name