qemu/accel/tcg/user-exec.c
<<
>>
Prefs
   1/*
   2 *  User emulator execution
   3 *
   4 *  Copyright (c) 2003-2005 Fabrice Bellard
   5 *
   6 * This library is free software; you can redistribute it and/or
   7 * modify it under the terms of the GNU Lesser General Public
   8 * License as published by the Free Software Foundation; either
   9 * version 2 of the License, or (at your option) any later version.
  10 *
  11 * This library is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 * Lesser General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU Lesser General Public
  17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19#include "qemu/osdep.h"
  20#include "cpu.h"
  21#include "disas/disas.h"
  22#include "exec/exec-all.h"
  23#include "tcg.h"
  24#include "qemu/bitops.h"
  25#include "exec/cpu_ldst.h"
  26#include "translate-all.h"
  27#include "exec/helper-proto.h"
  28
  29#undef EAX
  30#undef ECX
  31#undef EDX
  32#undef EBX
  33#undef ESP
  34#undef EBP
  35#undef ESI
  36#undef EDI
  37#undef EIP
  38#ifdef __linux__
  39#include <sys/ucontext.h>
  40#endif
  41
  42__thread uintptr_t helper_retaddr;
  43
  44//#define DEBUG_SIGNAL
  45
  46/* exit the current TB from a signal handler. The host registers are
  47   restored in a state compatible with the CPU emulator
  48 */
  49static void cpu_exit_tb_from_sighandler(CPUState *cpu, sigset_t *old_set)
  50{
  51    /* XXX: use siglongjmp ? */
  52    sigprocmask(SIG_SETMASK, old_set, NULL);
  53    cpu_loop_exit_noexc(cpu);
  54}
  55
  56/* 'pc' is the host PC at which the exception was raised. 'address' is
  57   the effective address of the memory exception. 'is_write' is 1 if a
  58   write caused the exception and otherwise 0'. 'old_set' is the
  59   signal set which should be restored */
  60static inline int handle_cpu_signal(uintptr_t pc, siginfo_t *info,
  61                                    int is_write, sigset_t *old_set)
  62{
  63    CPUState *cpu = current_cpu;
  64    CPUClass *cc;
  65    int ret;
  66    unsigned long address = (unsigned long)info->si_addr;
  67
  68    /* We must handle PC addresses from two different sources:
  69     * a call return address and a signal frame address.
  70     *
  71     * Within cpu_restore_state_from_tb we assume the former and adjust
  72     * the address by -GETPC_ADJ so that the address is within the call
  73     * insn so that addr does not accidentally match the beginning of the
  74     * next guest insn.
  75     *
  76     * However, when the PC comes from the signal frame, it points to
  77     * the actual faulting host insn and not a call insn.  Subtracting
  78     * GETPC_ADJ in that case may accidentally match the previous guest insn.
  79     *
  80     * So for the later case, adjust forward to compensate for what
  81     * will be done later by cpu_restore_state_from_tb.
  82     */
  83    if (helper_retaddr) {
  84        pc = helper_retaddr;
  85    } else {
  86        pc += GETPC_ADJ;
  87    }
  88
  89    /* For synchronous signals we expect to be coming from the vCPU
  90     * thread (so current_cpu should be valid) and either from running
  91     * code or during translation which can fault as we cross pages.
  92     *
  93     * If neither is true then something has gone wrong and we should
  94     * abort rather than try and restart the vCPU execution.
  95     */
  96    if (!cpu || !cpu->running) {
  97        printf("qemu:%s received signal outside vCPU context @ pc=0x%"
  98               PRIxPTR "\n",  __func__, pc);
  99        abort();
 100    }
 101
 102#if defined(DEBUG_SIGNAL)
 103    printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
 104           pc, address, is_write, *(unsigned long *)old_set);
 105#endif
 106    /* XXX: locking issue */
 107    /* Note that it is important that we don't call page_unprotect() unless
 108     * this is really a "write to nonwriteable page" fault, because
 109     * page_unprotect() assumes that if it is called for an access to
 110     * a page that's writeable this means we had two threads racing and
 111     * another thread got there first and already made the page writeable;
 112     * so we will retry the access. If we were to call page_unprotect()
 113     * for some other kind of fault that should really be passed to the
 114     * guest, we'd end up in an infinite loop of retrying the faulting
 115     * access.
 116     */
 117    if (is_write && info->si_signo == SIGSEGV && info->si_code == SEGV_ACCERR &&
 118        h2g_valid(address)) {
 119        switch (page_unprotect(h2g(address), pc)) {
 120        case 0:
 121            /* Fault not caused by a page marked unwritable to protect
 122             * cached translations, must be the guest binary's problem.
 123             */
 124            break;
 125        case 1:
 126            /* Fault caused by protection of cached translation; TBs
 127             * invalidated, so resume execution.  Retain helper_retaddr
 128             * for a possible second fault.
 129             */
 130            return 1;
 131        case 2:
 132            /* Fault caused by protection of cached translation, and the
 133             * currently executing TB was modified and must be exited
 134             * immediately.  Clear helper_retaddr for next execution.
 135             */
 136            helper_retaddr = 0;
 137            cpu_exit_tb_from_sighandler(cpu, old_set);
 138            /* NORETURN */
 139
 140        default:
 141            g_assert_not_reached();
 142        }
 143    }
 144
 145    /* Convert forcefully to guest address space, invalid addresses
 146       are still valid segv ones */
 147    address = h2g_nocheck(address);
 148
 149    cc = CPU_GET_CLASS(cpu);
 150    /* see if it is an MMU fault */
 151    g_assert(cc->handle_mmu_fault);
 152    ret = cc->handle_mmu_fault(cpu, address, 0, is_write, MMU_USER_IDX);
 153
 154    if (ret == 0) {
 155        /* The MMU fault was handled without causing real CPU fault.
 156         *  Retain helper_retaddr for a possible second fault.
 157         */
 158        return 1;
 159    }
 160
 161    /* All other paths lead to cpu_exit; clear helper_retaddr
 162     * for next execution.
 163     */
 164    helper_retaddr = 0;
 165
 166    if (ret < 0) {
 167        return 0; /* not an MMU fault */
 168    }
 169
 170    /* Now we have a real cpu fault.  */
 171    cpu_restore_state(cpu, pc, true);
 172
 173    sigprocmask(SIG_SETMASK, old_set, NULL);
 174    cpu_loop_exit(cpu);
 175
 176    /* never comes here */
 177    return 1;
 178}
 179
 180#if defined(__i386__)
 181
 182#if defined(__NetBSD__)
 183#include <ucontext.h>
 184
 185#define EIP_sig(context)     ((context)->uc_mcontext.__gregs[_REG_EIP])
 186#define TRAP_sig(context)    ((context)->uc_mcontext.__gregs[_REG_TRAPNO])
 187#define ERROR_sig(context)   ((context)->uc_mcontext.__gregs[_REG_ERR])
 188#define MASK_sig(context)    ((context)->uc_sigmask)
 189#elif defined(__FreeBSD__) || defined(__DragonFly__)
 190#include <ucontext.h>
 191
 192#define EIP_sig(context)  (*((unsigned long *)&(context)->uc_mcontext.mc_eip))
 193#define TRAP_sig(context)    ((context)->uc_mcontext.mc_trapno)
 194#define ERROR_sig(context)   ((context)->uc_mcontext.mc_err)
 195#define MASK_sig(context)    ((context)->uc_sigmask)
 196#elif defined(__OpenBSD__)
 197#define EIP_sig(context)     ((context)->sc_eip)
 198#define TRAP_sig(context)    ((context)->sc_trapno)
 199#define ERROR_sig(context)   ((context)->sc_err)
 200#define MASK_sig(context)    ((context)->sc_mask)
 201#else
 202#define EIP_sig(context)     ((context)->uc_mcontext.gregs[REG_EIP])
 203#define TRAP_sig(context)    ((context)->uc_mcontext.gregs[REG_TRAPNO])
 204#define ERROR_sig(context)   ((context)->uc_mcontext.gregs[REG_ERR])
 205#define MASK_sig(context)    ((context)->uc_sigmask)
 206#endif
 207
 208int cpu_signal_handler(int host_signum, void *pinfo,
 209                       void *puc)
 210{
 211    siginfo_t *info = pinfo;
 212#if defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__)
 213    ucontext_t *uc = puc;
 214#elif defined(__OpenBSD__)
 215    struct sigcontext *uc = puc;
 216#else
 217    ucontext_t *uc = puc;
 218#endif
 219    unsigned long pc;
 220    int trapno;
 221
 222#ifndef REG_EIP
 223/* for glibc 2.1 */
 224#define REG_EIP    EIP
 225#define REG_ERR    ERR
 226#define REG_TRAPNO TRAPNO
 227#endif
 228    pc = EIP_sig(uc);
 229    trapno = TRAP_sig(uc);
 230    return handle_cpu_signal(pc, info,
 231                             trapno == 0xe ? (ERROR_sig(uc) >> 1) & 1 : 0,
 232                             &MASK_sig(uc));
 233}
 234
 235#elif defined(__x86_64__)
 236
 237#ifdef __NetBSD__
 238#define PC_sig(context)       _UC_MACHINE_PC(context)
 239#define TRAP_sig(context)     ((context)->uc_mcontext.__gregs[_REG_TRAPNO])
 240#define ERROR_sig(context)    ((context)->uc_mcontext.__gregs[_REG_ERR])
 241#define MASK_sig(context)     ((context)->uc_sigmask)
 242#elif defined(__OpenBSD__)
 243#define PC_sig(context)       ((context)->sc_rip)
 244#define TRAP_sig(context)     ((context)->sc_trapno)
 245#define ERROR_sig(context)    ((context)->sc_err)
 246#define MASK_sig(context)     ((context)->sc_mask)
 247#elif defined(__FreeBSD__) || defined(__DragonFly__)
 248#include <ucontext.h>
 249
 250#define PC_sig(context)  (*((unsigned long *)&(context)->uc_mcontext.mc_rip))
 251#define TRAP_sig(context)     ((context)->uc_mcontext.mc_trapno)
 252#define ERROR_sig(context)    ((context)->uc_mcontext.mc_err)
 253#define MASK_sig(context)     ((context)->uc_sigmask)
 254#else
 255#define PC_sig(context)       ((context)->uc_mcontext.gregs[REG_RIP])
 256#define TRAP_sig(context)     ((context)->uc_mcontext.gregs[REG_TRAPNO])
 257#define ERROR_sig(context)    ((context)->uc_mcontext.gregs[REG_ERR])
 258#define MASK_sig(context)     ((context)->uc_sigmask)
 259#endif
 260
 261int cpu_signal_handler(int host_signum, void *pinfo,
 262                       void *puc)
 263{
 264    siginfo_t *info = pinfo;
 265    unsigned long pc;
 266#if defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__)
 267    ucontext_t *uc = puc;
 268#elif defined(__OpenBSD__)
 269    struct sigcontext *uc = puc;
 270#else
 271    ucontext_t *uc = puc;
 272#endif
 273
 274    pc = PC_sig(uc);
 275    return handle_cpu_signal(pc, info,
 276                             TRAP_sig(uc) == 0xe ? (ERROR_sig(uc) >> 1) & 1 : 0,
 277                             &MASK_sig(uc));
 278}
 279
 280#elif defined(_ARCH_PPC)
 281
 282/***********************************************************************
 283 * signal context platform-specific definitions
 284 * From Wine
 285 */
 286#ifdef linux
 287/* All Registers access - only for local access */
 288#define REG_sig(reg_name, context)              \
 289    ((context)->uc_mcontext.regs->reg_name)
 290/* Gpr Registers access  */
 291#define GPR_sig(reg_num, context)              REG_sig(gpr[reg_num], context)
 292/* Program counter */
 293#define IAR_sig(context)                       REG_sig(nip, context)
 294/* Machine State Register (Supervisor) */
 295#define MSR_sig(context)                       REG_sig(msr, context)
 296/* Count register */
 297#define CTR_sig(context)                       REG_sig(ctr, context)
 298/* User's integer exception register */
 299#define XER_sig(context)                       REG_sig(xer, context)
 300/* Link register */
 301#define LR_sig(context)                        REG_sig(link, context)
 302/* Condition register */
 303#define CR_sig(context)                        REG_sig(ccr, context)
 304
 305/* Float Registers access  */
 306#define FLOAT_sig(reg_num, context)                                     \
 307    (((double *)((char *)((context)->uc_mcontext.regs + 48 * 4)))[reg_num])
 308#define FPSCR_sig(context) \
 309    (*(int *)((char *)((context)->uc_mcontext.regs + (48 + 32 * 2) * 4)))
 310/* Exception Registers access */
 311#define DAR_sig(context)                       REG_sig(dar, context)
 312#define DSISR_sig(context)                     REG_sig(dsisr, context)
 313#define TRAP_sig(context)                      REG_sig(trap, context)
 314#endif /* linux */
 315
 316#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 317#include <ucontext.h>
 318#define IAR_sig(context)               ((context)->uc_mcontext.mc_srr0)
 319#define MSR_sig(context)               ((context)->uc_mcontext.mc_srr1)
 320#define CTR_sig(context)               ((context)->uc_mcontext.mc_ctr)
 321#define XER_sig(context)               ((context)->uc_mcontext.mc_xer)
 322#define LR_sig(context)                ((context)->uc_mcontext.mc_lr)
 323#define CR_sig(context)                ((context)->uc_mcontext.mc_cr)
 324/* Exception Registers access */
 325#define DAR_sig(context)               ((context)->uc_mcontext.mc_dar)
 326#define DSISR_sig(context)             ((context)->uc_mcontext.mc_dsisr)
 327#define TRAP_sig(context)              ((context)->uc_mcontext.mc_exc)
 328#endif /* __FreeBSD__|| __FreeBSD_kernel__ */
 329
 330int cpu_signal_handler(int host_signum, void *pinfo,
 331                       void *puc)
 332{
 333    siginfo_t *info = pinfo;
 334#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 335    ucontext_t *uc = puc;
 336#else
 337    ucontext_t *uc = puc;
 338#endif
 339    unsigned long pc;
 340    int is_write;
 341
 342    pc = IAR_sig(uc);
 343    is_write = 0;
 344#if 0
 345    /* ppc 4xx case */
 346    if (DSISR_sig(uc) & 0x00800000) {
 347        is_write = 1;
 348    }
 349#else
 350    if (TRAP_sig(uc) != 0x400 && (DSISR_sig(uc) & 0x02000000)) {
 351        is_write = 1;
 352    }
 353#endif
 354    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 355}
 356
 357#elif defined(__alpha__)
 358
 359int cpu_signal_handler(int host_signum, void *pinfo,
 360                           void *puc)
 361{
 362    siginfo_t *info = pinfo;
 363    ucontext_t *uc = puc;
 364    uint32_t *pc = uc->uc_mcontext.sc_pc;
 365    uint32_t insn = *pc;
 366    int is_write = 0;
 367
 368    /* XXX: need kernel patch to get write flag faster */
 369    switch (insn >> 26) {
 370    case 0x0d: /* stw */
 371    case 0x0e: /* stb */
 372    case 0x0f: /* stq_u */
 373    case 0x24: /* stf */
 374    case 0x25: /* stg */
 375    case 0x26: /* sts */
 376    case 0x27: /* stt */
 377    case 0x2c: /* stl */
 378    case 0x2d: /* stq */
 379    case 0x2e: /* stl_c */
 380    case 0x2f: /* stq_c */
 381        is_write = 1;
 382    }
 383
 384    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 385}
 386#elif defined(__sparc__)
 387
 388int cpu_signal_handler(int host_signum, void *pinfo,
 389                       void *puc)
 390{
 391    siginfo_t *info = pinfo;
 392    int is_write;
 393    uint32_t insn;
 394#if !defined(__arch64__) || defined(CONFIG_SOLARIS)
 395    uint32_t *regs = (uint32_t *)(info + 1);
 396    void *sigmask = (regs + 20);
 397    /* XXX: is there a standard glibc define ? */
 398    unsigned long pc = regs[1];
 399#else
 400#ifdef __linux__
 401    struct sigcontext *sc = puc;
 402    unsigned long pc = sc->sigc_regs.tpc;
 403    void *sigmask = (void *)sc->sigc_mask;
 404#elif defined(__OpenBSD__)
 405    struct sigcontext *uc = puc;
 406    unsigned long pc = uc->sc_pc;
 407    void *sigmask = (void *)(long)uc->sc_mask;
 408#elif defined(__NetBSD__)
 409    ucontext_t *uc = puc;
 410    unsigned long pc = _UC_MACHINE_PC(uc);
 411    void *sigmask = (void *)&uc->uc_sigmask;
 412#endif
 413#endif
 414
 415    /* XXX: need kernel patch to get write flag faster */
 416    is_write = 0;
 417    insn = *(uint32_t *)pc;
 418    if ((insn >> 30) == 3) {
 419        switch ((insn >> 19) & 0x3f) {
 420        case 0x05: /* stb */
 421        case 0x15: /* stba */
 422        case 0x06: /* sth */
 423        case 0x16: /* stha */
 424        case 0x04: /* st */
 425        case 0x14: /* sta */
 426        case 0x07: /* std */
 427        case 0x17: /* stda */
 428        case 0x0e: /* stx */
 429        case 0x1e: /* stxa */
 430        case 0x24: /* stf */
 431        case 0x34: /* stfa */
 432        case 0x27: /* stdf */
 433        case 0x37: /* stdfa */
 434        case 0x26: /* stqf */
 435        case 0x36: /* stqfa */
 436        case 0x25: /* stfsr */
 437        case 0x3c: /* casa */
 438        case 0x3e: /* casxa */
 439            is_write = 1;
 440            break;
 441        }
 442    }
 443    return handle_cpu_signal(pc, info, is_write, sigmask);
 444}
 445
 446#elif defined(__arm__)
 447
 448#if defined(__NetBSD__)
 449#include <ucontext.h>
 450#endif
 451
 452int cpu_signal_handler(int host_signum, void *pinfo,
 453                       void *puc)
 454{
 455    siginfo_t *info = pinfo;
 456#if defined(__NetBSD__)
 457    ucontext_t *uc = puc;
 458#else
 459    ucontext_t *uc = puc;
 460#endif
 461    unsigned long pc;
 462    int is_write;
 463
 464#if defined(__NetBSD__)
 465    pc = uc->uc_mcontext.__gregs[_REG_R15];
 466#elif defined(__GLIBC__) && (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 3))
 467    pc = uc->uc_mcontext.gregs[R15];
 468#else
 469    pc = uc->uc_mcontext.arm_pc;
 470#endif
 471
 472    /* error_code is the FSR value, in which bit 11 is WnR (assuming a v6 or
 473     * later processor; on v5 we will always report this as a read).
 474     */
 475    is_write = extract32(uc->uc_mcontext.error_code, 11, 1);
 476    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 477}
 478
 479#elif defined(__aarch64__)
 480
 481int cpu_signal_handler(int host_signum, void *pinfo, void *puc)
 482{
 483    siginfo_t *info = pinfo;
 484    ucontext_t *uc = puc;
 485    uintptr_t pc = uc->uc_mcontext.pc;
 486    uint32_t insn = *(uint32_t *)pc;
 487    bool is_write;
 488
 489    /* XXX: need kernel patch to get write flag faster.  */
 490    is_write = (   (insn & 0xbfff0000) == 0x0c000000   /* C3.3.1 */
 491                || (insn & 0xbfe00000) == 0x0c800000   /* C3.3.2 */
 492                || (insn & 0xbfdf0000) == 0x0d000000   /* C3.3.3 */
 493                || (insn & 0xbfc00000) == 0x0d800000   /* C3.3.4 */
 494                || (insn & 0x3f400000) == 0x08000000   /* C3.3.6 */
 495                || (insn & 0x3bc00000) == 0x39000000   /* C3.3.13 */
 496                || (insn & 0x3fc00000) == 0x3d800000   /* ... 128bit */
 497                /* Ingore bits 10, 11 & 21, controlling indexing.  */
 498                || (insn & 0x3bc00000) == 0x38000000   /* C3.3.8-12 */
 499                || (insn & 0x3fe00000) == 0x3c800000   /* ... 128bit */
 500                /* Ignore bits 23 & 24, controlling indexing.  */
 501                || (insn & 0x3a400000) == 0x28000000); /* C3.3.7,14-16 */
 502
 503    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 504}
 505
 506#elif defined(__s390__)
 507
 508int cpu_signal_handler(int host_signum, void *pinfo,
 509                       void *puc)
 510{
 511    siginfo_t *info = pinfo;
 512    ucontext_t *uc = puc;
 513    unsigned long pc;
 514    uint16_t *pinsn;
 515    int is_write = 0;
 516
 517    pc = uc->uc_mcontext.psw.addr;
 518
 519    /* ??? On linux, the non-rt signal handler has 4 (!) arguments instead
 520       of the normal 2 arguments.  The 3rd argument contains the "int_code"
 521       from the hardware which does in fact contain the is_write value.
 522       The rt signal handler, as far as I can tell, does not give this value
 523       at all.  Not that we could get to it from here even if it were.  */
 524    /* ??? This is not even close to complete, since it ignores all
 525       of the read-modify-write instructions.  */
 526    pinsn = (uint16_t *)pc;
 527    switch (pinsn[0] >> 8) {
 528    case 0x50: /* ST */
 529    case 0x42: /* STC */
 530    case 0x40: /* STH */
 531        is_write = 1;
 532        break;
 533    case 0xc4: /* RIL format insns */
 534        switch (pinsn[0] & 0xf) {
 535        case 0xf: /* STRL */
 536        case 0xb: /* STGRL */
 537        case 0x7: /* STHRL */
 538            is_write = 1;
 539        }
 540        break;
 541    case 0xe3: /* RXY format insns */
 542        switch (pinsn[2] & 0xff) {
 543        case 0x50: /* STY */
 544        case 0x24: /* STG */
 545        case 0x72: /* STCY */
 546        case 0x70: /* STHY */
 547        case 0x8e: /* STPQ */
 548        case 0x3f: /* STRVH */
 549        case 0x3e: /* STRV */
 550        case 0x2f: /* STRVG */
 551            is_write = 1;
 552        }
 553        break;
 554    }
 555    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 556}
 557
 558#elif defined(__mips__)
 559
 560int cpu_signal_handler(int host_signum, void *pinfo,
 561                       void *puc)
 562{
 563    siginfo_t *info = pinfo;
 564    ucontext_t *uc = puc;
 565    greg_t pc = uc->uc_mcontext.pc;
 566    int is_write;
 567
 568    /* XXX: compute is_write */
 569    is_write = 0;
 570    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 571}
 572
 573#else
 574
 575#error host CPU specific signal handler needed
 576
 577#endif
 578
 579/* The softmmu versions of these helpers are in cputlb.c.  */
 580
 581/* Do not allow unaligned operations to proceed.  Return the host address.  */
 582static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 583                               int size, uintptr_t retaddr)
 584{
 585    /* Enforce qemu required alignment.  */
 586    if (unlikely(addr & (size - 1))) {
 587        cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);
 588    }
 589    helper_retaddr = retaddr;
 590    return g2h(addr);
 591}
 592
 593/* Macro to call the above, with local variables from the use context.  */
 594#define ATOMIC_MMU_DECLS do {} while (0)
 595#define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, DATA_SIZE, GETPC())
 596#define ATOMIC_MMU_CLEANUP do { helper_retaddr = 0; } while (0)
 597
 598#define ATOMIC_NAME(X)   HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
 599#define EXTRA_ARGS
 600
 601#define DATA_SIZE 1
 602#include "atomic_template.h"
 603
 604#define DATA_SIZE 2
 605#include "atomic_template.h"
 606
 607#define DATA_SIZE 4
 608#include "atomic_template.h"
 609
 610#ifdef CONFIG_ATOMIC64
 611#define DATA_SIZE 8
 612#include "atomic_template.h"
 613#endif
 614
 615/* The following is only callable from other helpers, and matches up
 616   with the softmmu version.  */
 617
 618#ifdef CONFIG_ATOMIC128
 619
 620#undef EXTRA_ARGS
 621#undef ATOMIC_NAME
 622#undef ATOMIC_MMU_LOOKUP
 623
 624#define EXTRA_ARGS     , TCGMemOpIdx oi, uintptr_t retaddr
 625#define ATOMIC_NAME(X) \
 626    HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
 627#define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, DATA_SIZE, retaddr)
 628
 629#define DATA_SIZE 16
 630#include "atomic_template.h"
 631#endif /* CONFIG_ATOMIC128 */
 632