qemu/accel/tcg/user-exec.c
<<
>>
Prefs
   1/*
   2 *  User emulator execution
   3 *
   4 *  Copyright (c) 2003-2005 Fabrice Bellard
   5 *
   6 * This library is free software; you can redistribute it and/or
   7 * modify it under the terms of the GNU Lesser General Public
   8 * License as published by the Free Software Foundation; either
   9 * version 2 of the License, or (at your option) any later version.
  10 *
  11 * This library is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 * Lesser General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU Lesser General Public
  17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19#include "qemu/osdep.h"
  20#include "cpu.h"
  21#include "disas/disas.h"
  22#include "exec/exec-all.h"
  23#include "tcg.h"
  24#include "qemu/bitops.h"
  25#include "exec/cpu_ldst.h"
  26#include "translate-all.h"
  27#include "exec/helper-proto.h"
  28#include "qemu/atomic128.h"
  29
  30#undef EAX
  31#undef ECX
  32#undef EDX
  33#undef EBX
  34#undef ESP
  35#undef EBP
  36#undef ESI
  37#undef EDI
  38#undef EIP
  39#ifdef __linux__
  40#include <sys/ucontext.h>
  41#endif
  42
  43__thread uintptr_t helper_retaddr;
  44
  45//#define DEBUG_SIGNAL
  46
  47/* exit the current TB from a signal handler. The host registers are
  48   restored in a state compatible with the CPU emulator
  49 */
  50static void cpu_exit_tb_from_sighandler(CPUState *cpu, sigset_t *old_set)
  51{
  52    /* XXX: use siglongjmp ? */
  53    sigprocmask(SIG_SETMASK, old_set, NULL);
  54    cpu_loop_exit_noexc(cpu);
  55}
  56
  57/* 'pc' is the host PC at which the exception was raised. 'address' is
  58   the effective address of the memory exception. 'is_write' is 1 if a
  59   write caused the exception and otherwise 0'. 'old_set' is the
  60   signal set which should be restored */
  61static inline int handle_cpu_signal(uintptr_t pc, siginfo_t *info,
  62                                    int is_write, sigset_t *old_set)
  63{
  64    CPUState *cpu = current_cpu;
  65    CPUClass *cc;
  66    int ret;
  67    unsigned long address = (unsigned long)info->si_addr;
  68
  69    /* We must handle PC addresses from two different sources:
  70     * a call return address and a signal frame address.
  71     *
  72     * Within cpu_restore_state_from_tb we assume the former and adjust
  73     * the address by -GETPC_ADJ so that the address is within the call
  74     * insn so that addr does not accidentally match the beginning of the
  75     * next guest insn.
  76     *
  77     * However, when the PC comes from the signal frame, it points to
  78     * the actual faulting host insn and not a call insn.  Subtracting
  79     * GETPC_ADJ in that case may accidentally match the previous guest insn.
  80     *
  81     * So for the later case, adjust forward to compensate for what
  82     * will be done later by cpu_restore_state_from_tb.
  83     */
  84    if (helper_retaddr) {
  85        pc = helper_retaddr;
  86    } else {
  87        pc += GETPC_ADJ;
  88    }
  89
  90    /* For synchronous signals we expect to be coming from the vCPU
  91     * thread (so current_cpu should be valid) and either from running
  92     * code or during translation which can fault as we cross pages.
  93     *
  94     * If neither is true then something has gone wrong and we should
  95     * abort rather than try and restart the vCPU execution.
  96     */
  97    if (!cpu || !cpu->running) {
  98        printf("qemu:%s received signal outside vCPU context @ pc=0x%"
  99               PRIxPTR "\n",  __func__, pc);
 100        abort();
 101    }
 102
 103#if defined(DEBUG_SIGNAL)
 104    printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
 105           pc, address, is_write, *(unsigned long *)old_set);
 106#endif
 107    /* XXX: locking issue */
 108    /* Note that it is important that we don't call page_unprotect() unless
 109     * this is really a "write to nonwriteable page" fault, because
 110     * page_unprotect() assumes that if it is called for an access to
 111     * a page that's writeable this means we had two threads racing and
 112     * another thread got there first and already made the page writeable;
 113     * so we will retry the access. If we were to call page_unprotect()
 114     * for some other kind of fault that should really be passed to the
 115     * guest, we'd end up in an infinite loop of retrying the faulting
 116     * access.
 117     */
 118    if (is_write && info->si_signo == SIGSEGV && info->si_code == SEGV_ACCERR &&
 119        h2g_valid(address)) {
 120        switch (page_unprotect(h2g(address), pc)) {
 121        case 0:
 122            /* Fault not caused by a page marked unwritable to protect
 123             * cached translations, must be the guest binary's problem.
 124             */
 125            break;
 126        case 1:
 127            /* Fault caused by protection of cached translation; TBs
 128             * invalidated, so resume execution.  Retain helper_retaddr
 129             * for a possible second fault.
 130             */
 131            return 1;
 132        case 2:
 133            /* Fault caused by protection of cached translation, and the
 134             * currently executing TB was modified and must be exited
 135             * immediately.  Clear helper_retaddr for next execution.
 136             */
 137            helper_retaddr = 0;
 138            cpu_exit_tb_from_sighandler(cpu, old_set);
 139            /* NORETURN */
 140
 141        default:
 142            g_assert_not_reached();
 143        }
 144    }
 145
 146    /* Convert forcefully to guest address space, invalid addresses
 147       are still valid segv ones */
 148    address = h2g_nocheck(address);
 149
 150    cc = CPU_GET_CLASS(cpu);
 151    /* see if it is an MMU fault */
 152    g_assert(cc->handle_mmu_fault);
 153    ret = cc->handle_mmu_fault(cpu, address, 0, is_write, MMU_USER_IDX);
 154
 155    if (ret == 0) {
 156        /* The MMU fault was handled without causing real CPU fault.
 157         *  Retain helper_retaddr for a possible second fault.
 158         */
 159        return 1;
 160    }
 161
 162    /* All other paths lead to cpu_exit; clear helper_retaddr
 163     * for next execution.
 164     */
 165    helper_retaddr = 0;
 166
 167    if (ret < 0) {
 168        return 0; /* not an MMU fault */
 169    }
 170
 171    /* Now we have a real cpu fault.  */
 172    cpu_restore_state(cpu, pc, true);
 173
 174    sigprocmask(SIG_SETMASK, old_set, NULL);
 175    cpu_loop_exit(cpu);
 176
 177    /* never comes here */
 178    return 1;
 179}
 180
 181#if defined(__i386__)
 182
 183#if defined(__NetBSD__)
 184#include <ucontext.h>
 185
 186#define EIP_sig(context)     ((context)->uc_mcontext.__gregs[_REG_EIP])
 187#define TRAP_sig(context)    ((context)->uc_mcontext.__gregs[_REG_TRAPNO])
 188#define ERROR_sig(context)   ((context)->uc_mcontext.__gregs[_REG_ERR])
 189#define MASK_sig(context)    ((context)->uc_sigmask)
 190#elif defined(__FreeBSD__) || defined(__DragonFly__)
 191#include <ucontext.h>
 192
 193#define EIP_sig(context)  (*((unsigned long *)&(context)->uc_mcontext.mc_eip))
 194#define TRAP_sig(context)    ((context)->uc_mcontext.mc_trapno)
 195#define ERROR_sig(context)   ((context)->uc_mcontext.mc_err)
 196#define MASK_sig(context)    ((context)->uc_sigmask)
 197#elif defined(__OpenBSD__)
 198#define EIP_sig(context)     ((context)->sc_eip)
 199#define TRAP_sig(context)    ((context)->sc_trapno)
 200#define ERROR_sig(context)   ((context)->sc_err)
 201#define MASK_sig(context)    ((context)->sc_mask)
 202#else
 203#define EIP_sig(context)     ((context)->uc_mcontext.gregs[REG_EIP])
 204#define TRAP_sig(context)    ((context)->uc_mcontext.gregs[REG_TRAPNO])
 205#define ERROR_sig(context)   ((context)->uc_mcontext.gregs[REG_ERR])
 206#define MASK_sig(context)    ((context)->uc_sigmask)
 207#endif
 208
 209int cpu_signal_handler(int host_signum, void *pinfo,
 210                       void *puc)
 211{
 212    siginfo_t *info = pinfo;
 213#if defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__)
 214    ucontext_t *uc = puc;
 215#elif defined(__OpenBSD__)
 216    struct sigcontext *uc = puc;
 217#else
 218    ucontext_t *uc = puc;
 219#endif
 220    unsigned long pc;
 221    int trapno;
 222
 223#ifndef REG_EIP
 224/* for glibc 2.1 */
 225#define REG_EIP    EIP
 226#define REG_ERR    ERR
 227#define REG_TRAPNO TRAPNO
 228#endif
 229    pc = EIP_sig(uc);
 230    trapno = TRAP_sig(uc);
 231    return handle_cpu_signal(pc, info,
 232                             trapno == 0xe ? (ERROR_sig(uc) >> 1) & 1 : 0,
 233                             &MASK_sig(uc));
 234}
 235
 236#elif defined(__x86_64__)
 237
 238#ifdef __NetBSD__
 239#define PC_sig(context)       _UC_MACHINE_PC(context)
 240#define TRAP_sig(context)     ((context)->uc_mcontext.__gregs[_REG_TRAPNO])
 241#define ERROR_sig(context)    ((context)->uc_mcontext.__gregs[_REG_ERR])
 242#define MASK_sig(context)     ((context)->uc_sigmask)
 243#elif defined(__OpenBSD__)
 244#define PC_sig(context)       ((context)->sc_rip)
 245#define TRAP_sig(context)     ((context)->sc_trapno)
 246#define ERROR_sig(context)    ((context)->sc_err)
 247#define MASK_sig(context)     ((context)->sc_mask)
 248#elif defined(__FreeBSD__) || defined(__DragonFly__)
 249#include <ucontext.h>
 250
 251#define PC_sig(context)  (*((unsigned long *)&(context)->uc_mcontext.mc_rip))
 252#define TRAP_sig(context)     ((context)->uc_mcontext.mc_trapno)
 253#define ERROR_sig(context)    ((context)->uc_mcontext.mc_err)
 254#define MASK_sig(context)     ((context)->uc_sigmask)
 255#else
 256#define PC_sig(context)       ((context)->uc_mcontext.gregs[REG_RIP])
 257#define TRAP_sig(context)     ((context)->uc_mcontext.gregs[REG_TRAPNO])
 258#define ERROR_sig(context)    ((context)->uc_mcontext.gregs[REG_ERR])
 259#define MASK_sig(context)     ((context)->uc_sigmask)
 260#endif
 261
 262int cpu_signal_handler(int host_signum, void *pinfo,
 263                       void *puc)
 264{
 265    siginfo_t *info = pinfo;
 266    unsigned long pc;
 267#if defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__)
 268    ucontext_t *uc = puc;
 269#elif defined(__OpenBSD__)
 270    struct sigcontext *uc = puc;
 271#else
 272    ucontext_t *uc = puc;
 273#endif
 274
 275    pc = PC_sig(uc);
 276    return handle_cpu_signal(pc, info,
 277                             TRAP_sig(uc) == 0xe ? (ERROR_sig(uc) >> 1) & 1 : 0,
 278                             &MASK_sig(uc));
 279}
 280
 281#elif defined(_ARCH_PPC)
 282
 283/***********************************************************************
 284 * signal context platform-specific definitions
 285 * From Wine
 286 */
 287#ifdef linux
 288/* All Registers access - only for local access */
 289#define REG_sig(reg_name, context)              \
 290    ((context)->uc_mcontext.regs->reg_name)
 291/* Gpr Registers access  */
 292#define GPR_sig(reg_num, context)              REG_sig(gpr[reg_num], context)
 293/* Program counter */
 294#define IAR_sig(context)                       REG_sig(nip, context)
 295/* Machine State Register (Supervisor) */
 296#define MSR_sig(context)                       REG_sig(msr, context)
 297/* Count register */
 298#define CTR_sig(context)                       REG_sig(ctr, context)
 299/* User's integer exception register */
 300#define XER_sig(context)                       REG_sig(xer, context)
 301/* Link register */
 302#define LR_sig(context)                        REG_sig(link, context)
 303/* Condition register */
 304#define CR_sig(context)                        REG_sig(ccr, context)
 305
 306/* Float Registers access  */
 307#define FLOAT_sig(reg_num, context)                                     \
 308    (((double *)((char *)((context)->uc_mcontext.regs + 48 * 4)))[reg_num])
 309#define FPSCR_sig(context) \
 310    (*(int *)((char *)((context)->uc_mcontext.regs + (48 + 32 * 2) * 4)))
 311/* Exception Registers access */
 312#define DAR_sig(context)                       REG_sig(dar, context)
 313#define DSISR_sig(context)                     REG_sig(dsisr, context)
 314#define TRAP_sig(context)                      REG_sig(trap, context)
 315#endif /* linux */
 316
 317#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 318#include <ucontext.h>
 319#define IAR_sig(context)               ((context)->uc_mcontext.mc_srr0)
 320#define MSR_sig(context)               ((context)->uc_mcontext.mc_srr1)
 321#define CTR_sig(context)               ((context)->uc_mcontext.mc_ctr)
 322#define XER_sig(context)               ((context)->uc_mcontext.mc_xer)
 323#define LR_sig(context)                ((context)->uc_mcontext.mc_lr)
 324#define CR_sig(context)                ((context)->uc_mcontext.mc_cr)
 325/* Exception Registers access */
 326#define DAR_sig(context)               ((context)->uc_mcontext.mc_dar)
 327#define DSISR_sig(context)             ((context)->uc_mcontext.mc_dsisr)
 328#define TRAP_sig(context)              ((context)->uc_mcontext.mc_exc)
 329#endif /* __FreeBSD__|| __FreeBSD_kernel__ */
 330
 331int cpu_signal_handler(int host_signum, void *pinfo,
 332                       void *puc)
 333{
 334    siginfo_t *info = pinfo;
 335#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 336    ucontext_t *uc = puc;
 337#else
 338    ucontext_t *uc = puc;
 339#endif
 340    unsigned long pc;
 341    int is_write;
 342
 343    pc = IAR_sig(uc);
 344    is_write = 0;
 345#if 0
 346    /* ppc 4xx case */
 347    if (DSISR_sig(uc) & 0x00800000) {
 348        is_write = 1;
 349    }
 350#else
 351    if (TRAP_sig(uc) != 0x400 && (DSISR_sig(uc) & 0x02000000)) {
 352        is_write = 1;
 353    }
 354#endif
 355    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 356}
 357
 358#elif defined(__alpha__)
 359
 360int cpu_signal_handler(int host_signum, void *pinfo,
 361                           void *puc)
 362{
 363    siginfo_t *info = pinfo;
 364    ucontext_t *uc = puc;
 365    uint32_t *pc = uc->uc_mcontext.sc_pc;
 366    uint32_t insn = *pc;
 367    int is_write = 0;
 368
 369    /* XXX: need kernel patch to get write flag faster */
 370    switch (insn >> 26) {
 371    case 0x0d: /* stw */
 372    case 0x0e: /* stb */
 373    case 0x0f: /* stq_u */
 374    case 0x24: /* stf */
 375    case 0x25: /* stg */
 376    case 0x26: /* sts */
 377    case 0x27: /* stt */
 378    case 0x2c: /* stl */
 379    case 0x2d: /* stq */
 380    case 0x2e: /* stl_c */
 381    case 0x2f: /* stq_c */
 382        is_write = 1;
 383    }
 384
 385    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 386}
 387#elif defined(__sparc__)
 388
 389int cpu_signal_handler(int host_signum, void *pinfo,
 390                       void *puc)
 391{
 392    siginfo_t *info = pinfo;
 393    int is_write;
 394    uint32_t insn;
 395#if !defined(__arch64__) || defined(CONFIG_SOLARIS)
 396    uint32_t *regs = (uint32_t *)(info + 1);
 397    void *sigmask = (regs + 20);
 398    /* XXX: is there a standard glibc define ? */
 399    unsigned long pc = regs[1];
 400#else
 401#ifdef __linux__
 402    struct sigcontext *sc = puc;
 403    unsigned long pc = sc->sigc_regs.tpc;
 404    void *sigmask = (void *)sc->sigc_mask;
 405#elif defined(__OpenBSD__)
 406    struct sigcontext *uc = puc;
 407    unsigned long pc = uc->sc_pc;
 408    void *sigmask = (void *)(long)uc->sc_mask;
 409#elif defined(__NetBSD__)
 410    ucontext_t *uc = puc;
 411    unsigned long pc = _UC_MACHINE_PC(uc);
 412    void *sigmask = (void *)&uc->uc_sigmask;
 413#endif
 414#endif
 415
 416    /* XXX: need kernel patch to get write flag faster */
 417    is_write = 0;
 418    insn = *(uint32_t *)pc;
 419    if ((insn >> 30) == 3) {
 420        switch ((insn >> 19) & 0x3f) {
 421        case 0x05: /* stb */
 422        case 0x15: /* stba */
 423        case 0x06: /* sth */
 424        case 0x16: /* stha */
 425        case 0x04: /* st */
 426        case 0x14: /* sta */
 427        case 0x07: /* std */
 428        case 0x17: /* stda */
 429        case 0x0e: /* stx */
 430        case 0x1e: /* stxa */
 431        case 0x24: /* stf */
 432        case 0x34: /* stfa */
 433        case 0x27: /* stdf */
 434        case 0x37: /* stdfa */
 435        case 0x26: /* stqf */
 436        case 0x36: /* stqfa */
 437        case 0x25: /* stfsr */
 438        case 0x3c: /* casa */
 439        case 0x3e: /* casxa */
 440            is_write = 1;
 441            break;
 442        }
 443    }
 444    return handle_cpu_signal(pc, info, is_write, sigmask);
 445}
 446
 447#elif defined(__arm__)
 448
 449#if defined(__NetBSD__)
 450#include <ucontext.h>
 451#endif
 452
 453int cpu_signal_handler(int host_signum, void *pinfo,
 454                       void *puc)
 455{
 456    siginfo_t *info = pinfo;
 457#if defined(__NetBSD__)
 458    ucontext_t *uc = puc;
 459#else
 460    ucontext_t *uc = puc;
 461#endif
 462    unsigned long pc;
 463    int is_write;
 464
 465#if defined(__NetBSD__)
 466    pc = uc->uc_mcontext.__gregs[_REG_R15];
 467#elif defined(__GLIBC__) && (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 3))
 468    pc = uc->uc_mcontext.gregs[R15];
 469#else
 470    pc = uc->uc_mcontext.arm_pc;
 471#endif
 472
 473    /* error_code is the FSR value, in which bit 11 is WnR (assuming a v6 or
 474     * later processor; on v5 we will always report this as a read).
 475     */
 476    is_write = extract32(uc->uc_mcontext.error_code, 11, 1);
 477    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 478}
 479
 480#elif defined(__aarch64__)
 481
 482int cpu_signal_handler(int host_signum, void *pinfo, void *puc)
 483{
 484    siginfo_t *info = pinfo;
 485    ucontext_t *uc = puc;
 486    uintptr_t pc = uc->uc_mcontext.pc;
 487    uint32_t insn = *(uint32_t *)pc;
 488    bool is_write;
 489
 490    /* XXX: need kernel patch to get write flag faster.  */
 491    is_write = (   (insn & 0xbfff0000) == 0x0c000000   /* C3.3.1 */
 492                || (insn & 0xbfe00000) == 0x0c800000   /* C3.3.2 */
 493                || (insn & 0xbfdf0000) == 0x0d000000   /* C3.3.3 */
 494                || (insn & 0xbfc00000) == 0x0d800000   /* C3.3.4 */
 495                || (insn & 0x3f400000) == 0x08000000   /* C3.3.6 */
 496                || (insn & 0x3bc00000) == 0x39000000   /* C3.3.13 */
 497                || (insn & 0x3fc00000) == 0x3d800000   /* ... 128bit */
 498                /* Ingore bits 10, 11 & 21, controlling indexing.  */
 499                || (insn & 0x3bc00000) == 0x38000000   /* C3.3.8-12 */
 500                || (insn & 0x3fe00000) == 0x3c800000   /* ... 128bit */
 501                /* Ignore bits 23 & 24, controlling indexing.  */
 502                || (insn & 0x3a400000) == 0x28000000); /* C3.3.7,14-16 */
 503
 504    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 505}
 506
 507#elif defined(__s390__)
 508
 509int cpu_signal_handler(int host_signum, void *pinfo,
 510                       void *puc)
 511{
 512    siginfo_t *info = pinfo;
 513    ucontext_t *uc = puc;
 514    unsigned long pc;
 515    uint16_t *pinsn;
 516    int is_write = 0;
 517
 518    pc = uc->uc_mcontext.psw.addr;
 519
 520    /* ??? On linux, the non-rt signal handler has 4 (!) arguments instead
 521       of the normal 2 arguments.  The 3rd argument contains the "int_code"
 522       from the hardware which does in fact contain the is_write value.
 523       The rt signal handler, as far as I can tell, does not give this value
 524       at all.  Not that we could get to it from here even if it were.  */
 525    /* ??? This is not even close to complete, since it ignores all
 526       of the read-modify-write instructions.  */
 527    pinsn = (uint16_t *)pc;
 528    switch (pinsn[0] >> 8) {
 529    case 0x50: /* ST */
 530    case 0x42: /* STC */
 531    case 0x40: /* STH */
 532        is_write = 1;
 533        break;
 534    case 0xc4: /* RIL format insns */
 535        switch (pinsn[0] & 0xf) {
 536        case 0xf: /* STRL */
 537        case 0xb: /* STGRL */
 538        case 0x7: /* STHRL */
 539            is_write = 1;
 540        }
 541        break;
 542    case 0xe3: /* RXY format insns */
 543        switch (pinsn[2] & 0xff) {
 544        case 0x50: /* STY */
 545        case 0x24: /* STG */
 546        case 0x72: /* STCY */
 547        case 0x70: /* STHY */
 548        case 0x8e: /* STPQ */
 549        case 0x3f: /* STRVH */
 550        case 0x3e: /* STRV */
 551        case 0x2f: /* STRVG */
 552            is_write = 1;
 553        }
 554        break;
 555    }
 556    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 557}
 558
 559#elif defined(__mips__)
 560
 561int cpu_signal_handler(int host_signum, void *pinfo,
 562                       void *puc)
 563{
 564    siginfo_t *info = pinfo;
 565    ucontext_t *uc = puc;
 566    greg_t pc = uc->uc_mcontext.pc;
 567    int is_write;
 568
 569    /* XXX: compute is_write */
 570    is_write = 0;
 571    return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 572}
 573
 574#else
 575
 576#error host CPU specific signal handler needed
 577
 578#endif
 579
 580/* The softmmu versions of these helpers are in cputlb.c.  */
 581
 582/* Do not allow unaligned operations to proceed.  Return the host address.  */
 583static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 584                               int size, uintptr_t retaddr)
 585{
 586    /* Enforce qemu required alignment.  */
 587    if (unlikely(addr & (size - 1))) {
 588        cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);
 589    }
 590    helper_retaddr = retaddr;
 591    return g2h(addr);
 592}
 593
 594/* Macro to call the above, with local variables from the use context.  */
 595#define ATOMIC_MMU_DECLS do {} while (0)
 596#define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, DATA_SIZE, GETPC())
 597#define ATOMIC_MMU_CLEANUP do { helper_retaddr = 0; } while (0)
 598
 599#define ATOMIC_NAME(X)   HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
 600#define EXTRA_ARGS
 601
 602#define DATA_SIZE 1
 603#include "atomic_template.h"
 604
 605#define DATA_SIZE 2
 606#include "atomic_template.h"
 607
 608#define DATA_SIZE 4
 609#include "atomic_template.h"
 610
 611#ifdef CONFIG_ATOMIC64
 612#define DATA_SIZE 8
 613#include "atomic_template.h"
 614#endif
 615
 616/* The following is only callable from other helpers, and matches up
 617   with the softmmu version.  */
 618
 619#if HAVE_ATOMIC128 || HAVE_CMPXCHG128
 620
 621#undef EXTRA_ARGS
 622#undef ATOMIC_NAME
 623#undef ATOMIC_MMU_LOOKUP
 624
 625#define EXTRA_ARGS     , TCGMemOpIdx oi, uintptr_t retaddr
 626#define ATOMIC_NAME(X) \
 627    HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
 628#define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, DATA_SIZE, retaddr)
 629
 630#define DATA_SIZE 16
 631#include "atomic_template.h"
 632#endif
 633