accel/tcg/user-exec.c

   1 /*
   2  *  User emulator execution
   3  *
   4  *  Copyright (c) 2003-2005 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "qemu/osdep.h"
  20 #include "cpu.h"
  21 #include "disas/disas.h"
  22 #include "exec/exec-all.h"
  23 #include "tcg.h"
  24 #include "qemu/bitops.h"
  25 #include "exec/cpu_ldst.h"
  26 #include "translate-all.h"
  27 #include "exec/helper-proto.h"
  28
  29 #undef EAX
  30 #undef ECX
  31 #undef EDX
  32 #undef EBX
  33 #undef ESP
  34 #undef EBP
  35 #undef ESI
  36 #undef EDI
  37 #undef EIP
  38 #ifdef __linux__
  39 #include <sys/ucontext.h>
  40 #endif
  41
  42 //#define DEBUG_SIGNAL
  43
  44 /* exit the current TB from a signal handler. The host registers are
  45    restored in a state compatible with the CPU emulator
  46  */
  47 static void cpu_exit_tb_from_sighandler(CPUState *cpu, sigset_t *old_set)
  48 {
  49     /* XXX: use siglongjmp ? */
  50     sigprocmask(SIG_SETMASK, old_set, NULL);
  51     cpu_loop_exit_noexc(cpu);
  52 }
  53
  54 /* 'pc' is the host PC at which the exception was raised. 'address' is
  55    the effective address of the memory exception. 'is_write' is 1 if a
  56    write caused the exception and otherwise 0'. 'old_set' is the
  57    signal set which should be restored */
  58 static inline int handle_cpu_signal(uintptr_t pc, unsigned long address,
  59                                     int is_write, sigset_t *old_set)
  60 {
  61     CPUState *cpu = current_cpu;
  62     CPUClass *cc;
  63     int ret;
  64
  65     /* For synchronous signals we expect to be coming from the vCPU
  66      * thread (so current_cpu should be valid) and either from running
  67      * code or during translation which can fault as we cross pages.
  68      *
  69      * If neither is true then something has gone wrong and we should
  70      * abort rather than try and restart the vCPU execution.
  71      */
  72     if (!cpu || !cpu->running) {
  73         printf("qemu:%s received signal outside vCPU context @ pc=0x%"
  74                PRIxPTR "\n",  __func__, pc);
  75         abort();
  76     }
  77
  78 #if defined(DEBUG_SIGNAL)
  79     printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
  80            pc, address, is_write, *(unsigned long *)old_set);
  81 #endif
  82     /* XXX: locking issue */
  83     if (is_write && h2g_valid(address)) {
  84         switch (page_unprotect(h2g(address), pc)) {
  85         case 0:
  86             /* Fault not caused by a page marked unwritable to protect
  87              * cached translations, must be the guest binary's problem
  88              */
  89             break;
  90         case 1:
  91             /* Fault caused by protection of cached translation; TBs
  92              * invalidated, so resume execution
  93              */
  94             return 1;
  95         case 2:
  96             /* Fault caused by protection of cached translation, and the
  97              * currently executing TB was modified and must be exited
  98              * immediately.
  99              */
 100             cpu_exit_tb_from_sighandler(cpu, old_set);
 101             g_assert_not_reached();
 102         default:
 103             g_assert_not_reached();
 104         }
 105     }
 106
 107     /* Convert forcefully to guest address space, invalid addresses
 108        are still valid segv ones */
 109     address = h2g_nocheck(address);
 110
 111     cc = CPU_GET_CLASS(cpu);
 112     /* see if it is an MMU fault */
 113     g_assert(cc->handle_mmu_fault);
 114     ret = cc->handle_mmu_fault(cpu, address, is_write, MMU_USER_IDX);
 115     if (ret < 0) {
 116         return 0; /* not an MMU fault */
 117     }
 118     if (ret == 0) {
 119         return 1; /* the MMU fault was handled without causing real CPU fault */
 120     }
 121
 122     /* Now we have a real cpu fault.  Since this is the exact location of
 123      * the exception, we must undo the adjustment done by cpu_restore_state
 124      * for handling call return addresses.  */
 125     cpu_restore_state(cpu, pc + GETPC_ADJ);
 126
 127     sigprocmask(SIG_SETMASK, old_set, NULL);
 128     cpu_loop_exit(cpu);
 129
 130     /* never comes here */
 131     return 1;
 132 }
 133
 134 #if defined(__i386__)
 135
 136 #if defined(__NetBSD__)
 137 #include <ucontext.h>
 138
 139 #define EIP_sig(context)     ((context)->uc_mcontext.__gregs[_REG_EIP])
 140 #define TRAP_sig(context)    ((context)->uc_mcontext.__gregs[_REG_TRAPNO])
 141 #define ERROR_sig(context)   ((context)->uc_mcontext.__gregs[_REG_ERR])
 142 #define MASK_sig(context)    ((context)->uc_sigmask)
 143 #elif defined(__FreeBSD__) || defined(__DragonFly__)
 144 #include <ucontext.h>
 145
 146 #define EIP_sig(context)  (*((unsigned long *)&(context)->uc_mcontext.mc_eip))
 147 #define TRAP_sig(context)    ((context)->uc_mcontext.mc_trapno)
 148 #define ERROR_sig(context)   ((context)->uc_mcontext.mc_err)
 149 #define MASK_sig(context)    ((context)->uc_sigmask)
 150 #elif defined(__OpenBSD__)
 151 #define EIP_sig(context)     ((context)->sc_eip)
 152 #define TRAP_sig(context)    ((context)->sc_trapno)
 153 #define ERROR_sig(context)   ((context)->sc_err)
 154 #define MASK_sig(context)    ((context)->sc_mask)
 155 #else
 156 #define EIP_sig(context)     ((context)->uc_mcontext.gregs[REG_EIP])
 157 #define TRAP_sig(context)    ((context)->uc_mcontext.gregs[REG_TRAPNO])
 158 #define ERROR_sig(context)   ((context)->uc_mcontext.gregs[REG_ERR])
 159 #define MASK_sig(context)    ((context)->uc_sigmask)
 160 #endif
 161
 162 int cpu_signal_handler(int host_signum, void *pinfo,
 163                        void *puc)
 164 {
 165     siginfo_t *info = pinfo;
 166 #if defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__)
 167     ucontext_t *uc = puc;
 168 #elif defined(__OpenBSD__)
 169     struct sigcontext *uc = puc;
 170 #else
 171     ucontext_t *uc = puc;
 172 #endif
 173     unsigned long pc;
 174     int trapno;
 175
 176 #ifndef REG_EIP
 177 /* for glibc 2.1 */
 178 #define REG_EIP    EIP
 179 #define REG_ERR    ERR
 180 #define REG_TRAPNO TRAPNO
 181 #endif
 182     pc = EIP_sig(uc);
 183     trapno = TRAP_sig(uc);
 184     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
 185                              trapno == 0xe ?
 186                              (ERROR_sig(uc) >> 1) & 1 : 0,
 187                              &MASK_sig(uc));
 188 }
 189
 190 #elif defined(__x86_64__)
 191
 192 #ifdef __NetBSD__
 193 #define PC_sig(context)       _UC_MACHINE_PC(context)
 194 #define TRAP_sig(context)     ((context)->uc_mcontext.__gregs[_REG_TRAPNO])
 195 #define ERROR_sig(context)    ((context)->uc_mcontext.__gregs[_REG_ERR])
 196 #define MASK_sig(context)     ((context)->uc_sigmask)
 197 #elif defined(__OpenBSD__)
 198 #define PC_sig(context)       ((context)->sc_rip)
 199 #define TRAP_sig(context)     ((context)->sc_trapno)
 200 #define ERROR_sig(context)    ((context)->sc_err)
 201 #define MASK_sig(context)     ((context)->sc_mask)
 202 #elif defined(__FreeBSD__) || defined(__DragonFly__)
 203 #include <ucontext.h>
 204
 205 #define PC_sig(context)  (*((unsigned long *)&(context)->uc_mcontext.mc_rip))
 206 #define TRAP_sig(context)     ((context)->uc_mcontext.mc_trapno)
 207 #define ERROR_sig(context)    ((context)->uc_mcontext.mc_err)
 208 #define MASK_sig(context)     ((context)->uc_sigmask)
 209 #else
 210 #define PC_sig(context)       ((context)->uc_mcontext.gregs[REG_RIP])
 211 #define TRAP_sig(context)     ((context)->uc_mcontext.gregs[REG_TRAPNO])
 212 #define ERROR_sig(context)    ((context)->uc_mcontext.gregs[REG_ERR])
 213 #define MASK_sig(context)     ((context)->uc_sigmask)
 214 #endif
 215
 216 int cpu_signal_handler(int host_signum, void *pinfo,
 217                        void *puc)
 218 {
 219     siginfo_t *info = pinfo;
 220     unsigned long pc;
 221 #if defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__)
 222     ucontext_t *uc = puc;
 223 #elif defined(__OpenBSD__)
 224     struct sigcontext *uc = puc;
 225 #else
 226     ucontext_t *uc = puc;
 227 #endif
 228
 229     pc = PC_sig(uc);
 230     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
 231                              TRAP_sig(uc) == 0xe ?
 232                              (ERROR_sig(uc) >> 1) & 1 : 0,
 233                              &MASK_sig(uc));
 234 }
 235
 236 #elif defined(_ARCH_PPC)
 237
 238 /***********************************************************************
 239  * signal context platform-specific definitions
 240  * From Wine
 241  */
 242 #ifdef linux
 243 /* All Registers access - only for local access */
 244 #define REG_sig(reg_name, context)              \
 245     ((context)->uc_mcontext.regs->reg_name)
 246 /* Gpr Registers access  */
 247 #define GPR_sig(reg_num, context)              REG_sig(gpr[reg_num], context)
 248 /* Program counter */
 249 #define IAR_sig(context)                       REG_sig(nip, context)
 250 /* Machine State Register (Supervisor) */
 251 #define MSR_sig(context)                       REG_sig(msr, context)
 252 /* Count register */
 253 #define CTR_sig(context)                       REG_sig(ctr, context)
 254 /* User's integer exception register */
 255 #define XER_sig(context)                       REG_sig(xer, context)
 256 /* Link register */
 257 #define LR_sig(context)                        REG_sig(link, context)
 258 /* Condition register */
 259 #define CR_sig(context)                        REG_sig(ccr, context)
 260
 261 /* Float Registers access  */
 262 #define FLOAT_sig(reg_num, context)                                     \
 263     (((double *)((char *)((context)->uc_mcontext.regs + 48 * 4)))[reg_num])
 264 #define FPSCR_sig(context) \
 265     (*(int *)((char *)((context)->uc_mcontext.regs + (48 + 32 * 2) * 4)))
 266 /* Exception Registers access */
 267 #define DAR_sig(context)                       REG_sig(dar, context)
 268 #define DSISR_sig(context)                     REG_sig(dsisr, context)
 269 #define TRAP_sig(context)                      REG_sig(trap, context)
 270 #endif /* linux */
 271
 272 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 273 #include <ucontext.h>
 274 #define IAR_sig(context)               ((context)->uc_mcontext.mc_srr0)
 275 #define MSR_sig(context)               ((context)->uc_mcontext.mc_srr1)
 276 #define CTR_sig(context)               ((context)->uc_mcontext.mc_ctr)
 277 #define XER_sig(context)               ((context)->uc_mcontext.mc_xer)
 278 #define LR_sig(context)                ((context)->uc_mcontext.mc_lr)
 279 #define CR_sig(context)                ((context)->uc_mcontext.mc_cr)
 280 /* Exception Registers access */
 281 #define DAR_sig(context)               ((context)->uc_mcontext.mc_dar)
 282 #define DSISR_sig(context)             ((context)->uc_mcontext.mc_dsisr)
 283 #define TRAP_sig(context)              ((context)->uc_mcontext.mc_exc)
 284 #endif /* __FreeBSD__|| __FreeBSD_kernel__ */
 285
 286 int cpu_signal_handler(int host_signum, void *pinfo,
 287                        void *puc)
 288 {
 289     siginfo_t *info = pinfo;
 290 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 291     ucontext_t *uc = puc;
 292 #else
 293     ucontext_t *uc = puc;
 294 #endif
 295     unsigned long pc;
 296     int is_write;
 297
 298     pc = IAR_sig(uc);
 299     is_write = 0;
 300 #if 0
 301     /* ppc 4xx case */
 302     if (DSISR_sig(uc) & 0x00800000) {
 303         is_write = 1;
 304     }
 305 #else
 306     if (TRAP_sig(uc) != 0x400 && (DSISR_sig(uc) & 0x02000000)) {
 307         is_write = 1;
 308     }
 309 #endif
 310     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
 311                              is_write, &uc->uc_sigmask);
 312 }
 313
 314 #elif defined(__alpha__)
 315
 316 int cpu_signal_handler(int host_signum, void *pinfo,
 317                            void *puc)
 318 {
 319     siginfo_t *info = pinfo;
 320     ucontext_t *uc = puc;
 321     uint32_t *pc = uc->uc_mcontext.sc_pc;
 322     uint32_t insn = *pc;
 323     int is_write = 0;
 324
 325     /* XXX: need kernel patch to get write flag faster */
 326     switch (insn >> 26) {
 327     case 0x0d: /* stw */
 328     case 0x0e: /* stb */
 329     case 0x0f: /* stq_u */
 330     case 0x24: /* stf */
 331     case 0x25: /* stg */
 332     case 0x26: /* sts */
 333     case 0x27: /* stt */
 334     case 0x2c: /* stl */
 335     case 0x2d: /* stq */
 336     case 0x2e: /* stl_c */
 337     case 0x2f: /* stq_c */
 338         is_write = 1;
 339     }
 340
 341     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
 342                              is_write, &uc->uc_sigmask);
 343 }
 344 #elif defined(__sparc__)
 345
 346 int cpu_signal_handler(int host_signum, void *pinfo,
 347                        void *puc)
 348 {
 349     siginfo_t *info = pinfo;
 350     int is_write;
 351     uint32_t insn;
 352 #if !defined(__arch64__) || defined(CONFIG_SOLARIS)
 353     uint32_t *regs = (uint32_t *)(info + 1);
 354     void *sigmask = (regs + 20);
 355     /* XXX: is there a standard glibc define ? */
 356     unsigned long pc = regs[1];
 357 #else
 358 #ifdef __linux__
 359     struct sigcontext *sc = puc;
 360     unsigned long pc = sc->sigc_regs.tpc;
 361     void *sigmask = (void *)sc->sigc_mask;
 362 #elif defined(__OpenBSD__)
 363     struct sigcontext *uc = puc;
 364     unsigned long pc = uc->sc_pc;
 365     void *sigmask = (void *)(long)uc->sc_mask;
 366 #elif defined(__NetBSD__)
 367     ucontext_t *uc = puc;
 368     unsigned long pc = _UC_MACHINE_PC(uc);
 369     void *sigmask = (void *)&uc->uc_sigmask;
 370 #endif
 371 #endif
 372
 373     /* XXX: need kernel patch to get write flag faster */
 374     is_write = 0;
 375     insn = *(uint32_t *)pc;
 376     if ((insn >> 30) == 3) {
 377         switch ((insn >> 19) & 0x3f) {
 378         case 0x05: /* stb */
 379         case 0x15: /* stba */
 380         case 0x06: /* sth */
 381         case 0x16: /* stha */
 382         case 0x04: /* st */
 383         case 0x14: /* sta */
 384         case 0x07: /* std */
 385         case 0x17: /* stda */
 386         case 0x0e: /* stx */
 387         case 0x1e: /* stxa */
 388         case 0x24: /* stf */
 389         case 0x34: /* stfa */
 390         case 0x27: /* stdf */
 391         case 0x37: /* stdfa */
 392         case 0x26: /* stqf */
 393         case 0x36: /* stqfa */
 394         case 0x25: /* stfsr */
 395         case 0x3c: /* casa */
 396         case 0x3e: /* casxa */
 397             is_write = 1;
 398             break;
 399         }
 400     }
 401     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
 402                              is_write, sigmask);
 403 }
 404
 405 #elif defined(__arm__)
 406
 407 #if defined(__NetBSD__)
 408 #include <ucontext.h>
 409 #endif
 410
 411 int cpu_signal_handler(int host_signum, void *pinfo,
 412                        void *puc)
 413 {
 414     siginfo_t *info = pinfo;
 415 #if defined(__NetBSD__)
 416     ucontext_t *uc = puc;
 417 #else
 418     ucontext_t *uc = puc;
 419 #endif
 420     unsigned long pc;
 421     int is_write;
 422
 423 #if defined(__NetBSD__)
 424     pc = uc->uc_mcontext.__gregs[_REG_R15];
 425 #elif defined(__GLIBC__) && (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 3))
 426     pc = uc->uc_mcontext.gregs[R15];
 427 #else
 428     pc = uc->uc_mcontext.arm_pc;
 429 #endif
 430
 431     /* error_code is the FSR value, in which bit 11 is WnR (assuming a v6 or
 432      * later processor; on v5 we will always report this as a read).
 433      */
 434     is_write = extract32(uc->uc_mcontext.error_code, 11, 1);
 435     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
 436                              is_write,
 437                              &uc->uc_sigmask);
 438 }
 439
 440 #elif defined(__aarch64__)
 441
 442 int cpu_signal_handler(int host_signum, void *pinfo, void *puc)
 443 {
 444     siginfo_t *info = pinfo;
 445     ucontext_t *uc = puc;
 446     uintptr_t pc = uc->uc_mcontext.pc;
 447     uint32_t insn = *(uint32_t *)pc;
 448     bool is_write;
 449
 450     /* XXX: need kernel patch to get write flag faster.  */
 451     is_write = (   (insn & 0xbfff0000) == 0x0c000000   /* C3.3.1 */
 452                 || (insn & 0xbfe00000) == 0x0c800000   /* C3.3.2 */
 453                 || (insn & 0xbfdf0000) == 0x0d000000   /* C3.3.3 */
 454                 || (insn & 0xbfc00000) == 0x0d800000   /* C3.3.4 */
 455                 || (insn & 0x3f400000) == 0x08000000   /* C3.3.6 */
 456                 || (insn & 0x3bc00000) == 0x39000000   /* C3.3.13 */
 457                 || (insn & 0x3fc00000) == 0x3d800000   /* ... 128bit */
 458                 /* Ingore bits 10, 11 & 21, controlling indexing.  */
 459                 || (insn & 0x3bc00000) == 0x38000000   /* C3.3.8-12 */
 460                 || (insn & 0x3fe00000) == 0x3c800000   /* ... 128bit */
 461                 /* Ignore bits 23 & 24, controlling indexing.  */
 462                 || (insn & 0x3a400000) == 0x28000000); /* C3.3.7,14-16 */
 463
 464     return handle_cpu_signal(pc, (uintptr_t)info->si_addr,
 465                              is_write, &uc->uc_sigmask);
 466 }
 467
 468 #elif defined(__ia64)
 469
 470 #ifndef __ISR_VALID
 471   /* This ought to be in <bits/siginfo.h>... */
 472 # define __ISR_VALID    1
 473 #endif
 474
 475 int cpu_signal_handler(int host_signum, void *pinfo, void *puc)
 476 {
 477     siginfo_t *info = pinfo;
 478     ucontext_t *uc = puc;
 479     unsigned long ip;
 480     int is_write = 0;
 481
 482     ip = uc->uc_mcontext.sc_ip;
 483     switch (host_signum) {
 484     case SIGILL:
 485     case SIGFPE:
 486     case SIGSEGV:
 487     case SIGBUS:
 488     case SIGTRAP:
 489         if (info->si_code && (info->si_segvflags & __ISR_VALID)) {
 490             /* ISR.W (write-access) is bit 33:  */
 491             is_write = (info->si_isr >> 33) & 1;
 492         }
 493         break;
 494
 495     default:
 496         break;
 497     }
 498     return handle_cpu_signal(ip, (unsigned long)info->si_addr,
 499                              is_write,
 500                              (sigset_t *)&uc->uc_sigmask);
 501 }
 502
 503 #elif defined(__s390__)
 504
 505 int cpu_signal_handler(int host_signum, void *pinfo,
 506                        void *puc)
 507 {
 508     siginfo_t *info = pinfo;
 509     ucontext_t *uc = puc;
 510     unsigned long pc;
 511     uint16_t *pinsn;
 512     int is_write = 0;
 513
 514     pc = uc->uc_mcontext.psw.addr;
 515
 516     /* ??? On linux, the non-rt signal handler has 4 (!) arguments instead
 517        of the normal 2 arguments.  The 3rd argument contains the "int_code"
 518        from the hardware which does in fact contain the is_write value.
 519        The rt signal handler, as far as I can tell, does not give this value
 520        at all.  Not that we could get to it from here even if it were.  */
 521     /* ??? This is not even close to complete, since it ignores all
 522        of the read-modify-write instructions.  */
 523     pinsn = (uint16_t *)pc;
 524     switch (pinsn[0] >> 8) {
 525     case 0x50: /* ST */
 526     case 0x42: /* STC */
 527     case 0x40: /* STH */
 528         is_write = 1;
 529         break;
 530     case 0xc4: /* RIL format insns */
 531         switch (pinsn[0] & 0xf) {
 532         case 0xf: /* STRL */
 533         case 0xb: /* STGRL */
 534         case 0x7: /* STHRL */
 535             is_write = 1;
 536         }
 537         break;
 538     case 0xe3: /* RXY format insns */
 539         switch (pinsn[2] & 0xff) {
 540         case 0x50: /* STY */
 541         case 0x24: /* STG */
 542         case 0x72: /* STCY */
 543         case 0x70: /* STHY */
 544         case 0x8e: /* STPQ */
 545         case 0x3f: /* STRVH */
 546         case 0x3e: /* STRV */
 547         case 0x2f: /* STRVG */
 548             is_write = 1;
 549         }
 550         break;
 551     }
 552     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
 553                              is_write, &uc->uc_sigmask);
 554 }
 555
 556 #elif defined(__mips__)
 557
 558 int cpu_signal_handler(int host_signum, void *pinfo,
 559                        void *puc)
 560 {
 561     siginfo_t *info = pinfo;
 562     ucontext_t *uc = puc;
 563     greg_t pc = uc->uc_mcontext.pc;
 564     int is_write;
 565
 566     /* XXX: compute is_write */
 567     is_write = 0;
 568     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
 569                              is_write, &uc->uc_sigmask);
 570 }
 571
 572 #else
 573
 574 #error host CPU specific signal handler needed
 575
 576 #endif
 577
 578 /* The softmmu versions of these helpers are in cputlb.c.  */
 579
 580 /* Do not allow unaligned operations to proceed.  Return the host address.  */
 581 static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 582                                int size, uintptr_t retaddr)
 583 {
 584     /* Enforce qemu required alignment.  */
 585     if (unlikely(addr & (size - 1))) {
 586         cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);
 587     }
 588     return g2h(addr);
 589 }
 590
 591 /* Macro to call the above, with local variables from the use context.  */
 592 #define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, DATA_SIZE, GETPC())
 593
 594 #define ATOMIC_NAME(X)   HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
 595 #define EXTRA_ARGS
 596
 597 #define DATA_SIZE 1
 598 #include "atomic_template.h"
 599
 600 #define DATA_SIZE 2
 601 #include "atomic_template.h"
 602
 603 #define DATA_SIZE 4
 604 #include "atomic_template.h"
 605
 606 #ifdef CONFIG_ATOMIC64
 607 #define DATA_SIZE 8
 608 #include "atomic_template.h"
 609 #endif
 610
 611 /* The following is only callable from other helpers, and matches up
 612    with the softmmu version.  */
 613
 614 #ifdef CONFIG_ATOMIC128
 615
 616 #undef EXTRA_ARGS
 617 #undef ATOMIC_NAME
 618 #undef ATOMIC_MMU_LOOKUP
 619
 620 #define EXTRA_ARGS     , TCGMemOpIdx oi, uintptr_t retaddr
 621 #define ATOMIC_NAME(X) \
 622     HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
 623 #define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, DATA_SIZE, retaddr)
 624
 625 #define DATA_SIZE 16
 626 #include "atomic_template.h"
 627 #endif /* CONFIG_ATOMIC128 */