accel/tcg/softmmu_template.h

   1 /*
   2  *  Software MMU support
   3  *
   4  * Generate helpers used by TCG for qemu_ld/st ops and code load
   5  * functions.
   6  *
   7  * Included from target op helpers and exec.c.
   8  *
   9  *  Copyright (c) 2003 Fabrice Bellard
  10  *
  11  * This library is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation; either
  14  * version 2 of the License, or (at your option) any later version.
  15  *
  16  * This library is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  23  */
  24 #if DATA_SIZE == 8
  25 #define SUFFIX q
  26 #define LSUFFIX q
  27 #define SDATA_TYPE  int64_t
  28 #define DATA_TYPE  uint64_t
  29 #elif DATA_SIZE == 4
  30 #define SUFFIX l
  31 #define LSUFFIX l
  32 #define SDATA_TYPE  int32_t
  33 #define DATA_TYPE  uint32_t
  34 #elif DATA_SIZE == 2
  35 #define SUFFIX w
  36 #define LSUFFIX uw
  37 #define SDATA_TYPE  int16_t
  38 #define DATA_TYPE  uint16_t
  39 #elif DATA_SIZE == 1
  40 #define SUFFIX b
  41 #define LSUFFIX ub
  42 #define SDATA_TYPE  int8_t
  43 #define DATA_TYPE  uint8_t
  44 #else
  45 #error unsupported data size
  46 #endif
  47
  48
  49 /* For the benefit of TCG generated code, we want to avoid the complication
  50    of ABI-specific return type promotion and always return a value extended
  51    to the register size of the host.  This is tcg_target_long, except in the
  52    case of a 32-bit host and 64-bit data, and for that we always have
  53    uint64_t.  Don't bother with this widened value for SOFTMMU_CODE_ACCESS.  */
  54 #if defined(SOFTMMU_CODE_ACCESS) || DATA_SIZE == 8
  55 # define WORD_TYPE  DATA_TYPE
  56 # define USUFFIX    SUFFIX
  57 #else
  58 # define WORD_TYPE  tcg_target_ulong
  59 # define USUFFIX    glue(u, SUFFIX)
  60 # define SSUFFIX    glue(s, SUFFIX)
  61 #endif
  62
  63 #ifdef SOFTMMU_CODE_ACCESS
  64 #define READ_ACCESS_TYPE MMU_INST_FETCH
  65 #define ADDR_READ addr_code
  66 #else
  67 #define READ_ACCESS_TYPE MMU_DATA_LOAD
  68 #define ADDR_READ addr_read
  69 #endif
  70
  71 #if DATA_SIZE == 8
  72 # define BSWAP(X)  bswap64(X)
  73 #elif DATA_SIZE == 4
  74 # define BSWAP(X)  bswap32(X)
  75 #elif DATA_SIZE == 2
  76 # define BSWAP(X)  bswap16(X)
  77 #else
  78 # define BSWAP(X)  (X)
  79 #endif
  80
  81 #if DATA_SIZE == 1
  82 # define helper_le_ld_name  glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)
  83 # define helper_be_ld_name  helper_le_ld_name
  84 # define helper_le_lds_name glue(glue(helper_ret_ld, SSUFFIX), MMUSUFFIX)
  85 # define helper_be_lds_name helper_le_lds_name
  86 # define helper_le_st_name  glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)
  87 # define helper_be_st_name  helper_le_st_name
  88 #else
  89 # define helper_le_ld_name  glue(glue(helper_le_ld, USUFFIX), MMUSUFFIX)
  90 # define helper_be_ld_name  glue(glue(helper_be_ld, USUFFIX), MMUSUFFIX)
  91 # define helper_le_lds_name glue(glue(helper_le_ld, SSUFFIX), MMUSUFFIX)
  92 # define helper_be_lds_name glue(glue(helper_be_ld, SSUFFIX), MMUSUFFIX)
  93 # define helper_le_st_name  glue(glue(helper_le_st, SUFFIX), MMUSUFFIX)
  94 # define helper_be_st_name  glue(glue(helper_be_st, SUFFIX), MMUSUFFIX)
  95 #endif
  96
  97 #ifndef SOFTMMU_CODE_ACCESS
  98 static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
  99                                               size_t mmu_idx, size_t index,
 100                                               target_ulong addr,
 101                                               uintptr_t retaddr,
 102                                               bool recheck,
 103                                               MMUAccessType access_type)
 104 {
 105     CPUIOTLBEntry *iotlbentry = &env->iotlb[mmu_idx][index];
 106     return io_readx(env, iotlbentry, mmu_idx, addr, retaddr, recheck,
 107                     access_type, DATA_SIZE);
 108 }
 109 #endif
 110
 111 WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
 112                             TCGMemOpIdx oi, uintptr_t retaddr)
 113 {
 114     uintptr_t mmu_idx = get_mmuidx(oi);
 115     uintptr_t index = tlb_index(env, mmu_idx, addr);
 116     CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
 117     target_ulong tlb_addr = entry->ADDR_READ;
 118     unsigned a_bits = get_alignment_bits(get_memop(oi));
 119     uintptr_t haddr;
 120     DATA_TYPE res;
 121
 122     if (addr & ((1 << a_bits) - 1)) {
 123         cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
 124                              mmu_idx, retaddr);
 125     }
 126
 127     /* If the TLB entry is for a different page, reload and try again.  */
 128     if (!tlb_hit(tlb_addr, addr)) {
 129         if (!VICTIM_TLB_HIT(ADDR_READ, addr)) {
 130             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,
 131                      mmu_idx, retaddr);
 132         }
 133         tlb_addr = entry->ADDR_READ;
 134     }
 135
 136     /* Handle an IO access.  */
 137     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 138         if ((addr & (DATA_SIZE - 1)) != 0) {
 139             goto do_unaligned_access;
 140         }
 141
 142         /* ??? Note that the io helpers always read data in the target
 143            byte ordering.  We should push the LE/BE request down into io.  */
 144         res = glue(io_read, SUFFIX)(env, mmu_idx, index, addr, retaddr,
 145                                     tlb_addr & TLB_RECHECK,
 146                                     READ_ACCESS_TYPE);
 147         res = TGT_LE(res);
 148         return res;
 149     }
 150
 151     /* Handle slow unaligned access (it spans two pages or IO).  */
 152     if (DATA_SIZE > 1
 153         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 154                     >= TARGET_PAGE_SIZE)) {
 155         target_ulong addr1, addr2;
 156         DATA_TYPE res1, res2;
 157         unsigned shift;
 158     do_unaligned_access:
 159         addr1 = addr & ~(DATA_SIZE - 1);
 160         addr2 = addr1 + DATA_SIZE;
 161         res1 = helper_le_ld_name(env, addr1, oi, retaddr);
 162         res2 = helper_le_ld_name(env, addr2, oi, retaddr);
 163         shift = (addr & (DATA_SIZE - 1)) * 8;
 164
 165         /* Little-endian combine.  */
 166         res = (res1 >> shift) | (res2 << ((DATA_SIZE * 8) - shift));
 167         return res;
 168     }
 169
 170     haddr = addr + entry->addend;
 171 #if DATA_SIZE == 1
 172     res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);
 173 #else
 174     res = glue(glue(ld, LSUFFIX), _le_p)((uint8_t *)haddr);
 175 #endif
 176     return res;
 177 }
 178
 179 #if DATA_SIZE > 1
 180 WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
 181                             TCGMemOpIdx oi, uintptr_t retaddr)
 182 {
 183     uintptr_t mmu_idx = get_mmuidx(oi);
 184     uintptr_t index = tlb_index(env, mmu_idx, addr);
 185     CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
 186     target_ulong tlb_addr = entry->ADDR_READ;
 187     unsigned a_bits = get_alignment_bits(get_memop(oi));
 188     uintptr_t haddr;
 189     DATA_TYPE res;
 190
 191     if (addr & ((1 << a_bits) - 1)) {
 192         cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
 193                              mmu_idx, retaddr);
 194     }
 195
 196     /* If the TLB entry is for a different page, reload and try again.  */
 197     if (!tlb_hit(tlb_addr, addr)) {
 198         if (!VICTIM_TLB_HIT(ADDR_READ, addr)) {
 199             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,
 200                      mmu_idx, retaddr);
 201         }
 202         tlb_addr = entry->ADDR_READ;
 203     }
 204
 205     /* Handle an IO access.  */
 206     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 207         if ((addr & (DATA_SIZE - 1)) != 0) {
 208             goto do_unaligned_access;
 209         }
 210
 211         /* ??? Note that the io helpers always read data in the target
 212            byte ordering.  We should push the LE/BE request down into io.  */
 213         res = glue(io_read, SUFFIX)(env, mmu_idx, index, addr, retaddr,
 214                                     tlb_addr & TLB_RECHECK,
 215                                     READ_ACCESS_TYPE);
 216         res = TGT_BE(res);
 217         return res;
 218     }
 219
 220     /* Handle slow unaligned access (it spans two pages or IO).  */
 221     if (DATA_SIZE > 1
 222         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 223                     >= TARGET_PAGE_SIZE)) {
 224         target_ulong addr1, addr2;
 225         DATA_TYPE res1, res2;
 226         unsigned shift;
 227     do_unaligned_access:
 228         addr1 = addr & ~(DATA_SIZE - 1);
 229         addr2 = addr1 + DATA_SIZE;
 230         res1 = helper_be_ld_name(env, addr1, oi, retaddr);
 231         res2 = helper_be_ld_name(env, addr2, oi, retaddr);
 232         shift = (addr & (DATA_SIZE - 1)) * 8;
 233
 234         /* Big-endian combine.  */
 235         res = (res1 << shift) | (res2 >> ((DATA_SIZE * 8) - shift));
 236         return res;
 237     }
 238
 239     haddr = addr + entry->addend;
 240     res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);
 241     return res;
 242 }
 243 #endif /* DATA_SIZE > 1 */
 244
 245 #ifndef SOFTMMU_CODE_ACCESS
 246
 247 /* Provide signed versions of the load routines as well.  We can of course
 248    avoid this for 64-bit data, or for 32-bit data on 32-bit host.  */
 249 #if DATA_SIZE * 8 < TCG_TARGET_REG_BITS
 250 WORD_TYPE helper_le_lds_name(CPUArchState *env, target_ulong addr,
 251                              TCGMemOpIdx oi, uintptr_t retaddr)
 252 {
 253     return (SDATA_TYPE)helper_le_ld_name(env, addr, oi, retaddr);
 254 }
 255
 256 # if DATA_SIZE > 1
 257 WORD_TYPE helper_be_lds_name(CPUArchState *env, target_ulong addr,
 258                              TCGMemOpIdx oi, uintptr_t retaddr)
 259 {
 260     return (SDATA_TYPE)helper_be_ld_name(env, addr, oi, retaddr);
 261 }
 262 # endif
 263 #endif
 264
 265 static inline void glue(io_write, SUFFIX)(CPUArchState *env,
 266                                           size_t mmu_idx, size_t index,
 267                                           DATA_TYPE val,
 268                                           target_ulong addr,
 269                                           uintptr_t retaddr,
 270                                           bool recheck)
 271 {
 272     CPUIOTLBEntry *iotlbentry = &env->iotlb[mmu_idx][index];
 273     return io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr,
 274                      recheck, DATA_SIZE);
 275 }
 276
 277 void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
 278                        TCGMemOpIdx oi, uintptr_t retaddr)
 279 {
 280     uintptr_t mmu_idx = get_mmuidx(oi);
 281     uintptr_t index = tlb_index(env, mmu_idx, addr);
 282     CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
 283     target_ulong tlb_addr = tlb_addr_write(entry);
 284     unsigned a_bits = get_alignment_bits(get_memop(oi));
 285     uintptr_t haddr;
 286
 287     if (addr & ((1 << a_bits) - 1)) {
 288         cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
 289                              mmu_idx, retaddr);
 290     }
 291
 292     /* If the TLB entry is for a different page, reload and try again.  */
 293     if (!tlb_hit(tlb_addr, addr)) {
 294         if (!VICTIM_TLB_HIT(addr_write, addr)) {
 295             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
 296                      mmu_idx, retaddr);
 297         }
 298         tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK;
 299     }
 300
 301     /* Handle an IO access.  */
 302     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 303         if ((addr & (DATA_SIZE - 1)) != 0) {
 304             goto do_unaligned_access;
 305         }
 306
 307         /* ??? Note that the io helpers always read data in the target
 308            byte ordering.  We should push the LE/BE request down into io.  */
 309         val = TGT_LE(val);
 310         glue(io_write, SUFFIX)(env, mmu_idx, index, val, addr,
 311                                retaddr, tlb_addr & TLB_RECHECK);
 312         return;
 313     }
 314
 315     /* Handle slow unaligned access (it spans two pages or IO).  */
 316     if (DATA_SIZE > 1
 317         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 318                      >= TARGET_PAGE_SIZE)) {
 319         int i;
 320         target_ulong page2;
 321         CPUTLBEntry *entry2;
 322     do_unaligned_access:
 323         /* Ensure the second page is in the TLB.  Note that the first page
 324            is already guaranteed to be filled, and that the second page
 325            cannot evict the first.  */
 326         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
 327         entry2 = tlb_entry(env, mmu_idx, page2);
 328         if (!tlb_hit_page(tlb_addr_write(entry2), page2)
 329             && !VICTIM_TLB_HIT(addr_write, page2)) {
 330             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
 331                      mmu_idx, retaddr);
 332         }
 333
 334         /* XXX: not efficient, but simple.  */
 335         /* This loop must go in the forward direction to avoid issues
 336            with self-modifying code in Windows 64-bit.  */
 337         for (i = 0; i < DATA_SIZE; ++i) {
 338             /* Little-endian extract.  */
 339             uint8_t val8 = val >> (i * 8);
 340             glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
 341                                             oi, retaddr);
 342         }
 343         return;
 344     }
 345
 346     haddr = addr + entry->addend;
 347 #if DATA_SIZE == 1
 348     glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
 349 #else
 350     glue(glue(st, SUFFIX), _le_p)((uint8_t *)haddr, val);
 351 #endif
 352 }
 353
 354 #if DATA_SIZE > 1
 355 void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
 356                        TCGMemOpIdx oi, uintptr_t retaddr)
 357 {
 358     uintptr_t mmu_idx = get_mmuidx(oi);
 359     uintptr_t index = tlb_index(env, mmu_idx, addr);
 360     CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
 361     target_ulong tlb_addr = tlb_addr_write(entry);
 362     unsigned a_bits = get_alignment_bits(get_memop(oi));
 363     uintptr_t haddr;
 364
 365     if (addr & ((1 << a_bits) - 1)) {
 366         cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
 367                              mmu_idx, retaddr);
 368     }
 369
 370     /* If the TLB entry is for a different page, reload and try again.  */
 371     if (!tlb_hit(tlb_addr, addr)) {
 372         if (!VICTIM_TLB_HIT(addr_write, addr)) {
 373             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
 374                      mmu_idx, retaddr);
 375         }
 376         tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK;
 377     }
 378
 379     /* Handle an IO access.  */
 380     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 381         if ((addr & (DATA_SIZE - 1)) != 0) {
 382             goto do_unaligned_access;
 383         }
 384
 385         /* ??? Note that the io helpers always read data in the target
 386            byte ordering.  We should push the LE/BE request down into io.  */
 387         val = TGT_BE(val);
 388         glue(io_write, SUFFIX)(env, mmu_idx, index, val, addr, retaddr,
 389                                tlb_addr & TLB_RECHECK);
 390         return;
 391     }
 392
 393     /* Handle slow unaligned access (it spans two pages or IO).  */
 394     if (DATA_SIZE > 1
 395         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
 396                      >= TARGET_PAGE_SIZE)) {
 397         int i;
 398         target_ulong page2;
 399         CPUTLBEntry *entry2;
 400     do_unaligned_access:
 401         /* Ensure the second page is in the TLB.  Note that the first page
 402            is already guaranteed to be filled, and that the second page
 403            cannot evict the first.  */
 404         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
 405         entry2 = tlb_entry(env, mmu_idx, page2);
 406         if (!tlb_hit_page(tlb_addr_write(entry2), page2)
 407             && !VICTIM_TLB_HIT(addr_write, page2)) {
 408             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
 409                      mmu_idx, retaddr);
 410         }
 411
 412         /* XXX: not efficient, but simple */
 413         /* This loop must go in the forward direction to avoid issues
 414            with self-modifying code.  */
 415         for (i = 0; i < DATA_SIZE; ++i) {
 416             /* Big-endian extract.  */
 417             uint8_t val8 = val >> (((DATA_SIZE - 1) * 8) - (i * 8));
 418             glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
 419                                             oi, retaddr);
 420         }
 421         return;
 422     }
 423
 424     haddr = addr + entry->addend;
 425     glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);
 426 }
 427 #endif /* DATA_SIZE > 1 */
 428 #endif /* !defined(SOFTMMU_CODE_ACCESS) */
 429
 430 #undef READ_ACCESS_TYPE
 431 #undef DATA_TYPE
 432 #undef SUFFIX
 433 #undef LSUFFIX
 434 #undef DATA_SIZE
 435 #undef ADDR_READ
 436 #undef WORD_TYPE
 437 #undef SDATA_TYPE
 438 #undef USUFFIX
 439 #undef SSUFFIX
 440 #undef BSWAP
 441 #undef helper_le_ld_name
 442 #undef helper_be_ld_name
 443 #undef helper_le_lds_name
 444 #undef helper_be_lds_name
 445 #undef helper_le_st_name
 446 #undef helper_be_st_name