LXR linux/tools/testing/selftests/rseq/rseq-x86.h

   1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2/*
   3 * rseq-x86.h
   4 *
   5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6 */
   7
   8#include <stdint.h>
   9
  10/*
  11 * RSEQ_SIG is used with the following reserved undefined instructions, which
  12 * trap in user-space:
  13 *
  14 * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
  15 * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
  16 */
  17#define RSEQ_SIG        0x53053053
  18
  19/*
  20 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
  21 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
  22 * address through a "r" input operand.
  23 */
  24
  25/* Offset of cpu_id and rseq_cs fields in struct rseq. */
  26#define RSEQ_CPU_ID_OFFSET      4
  27#define RSEQ_CS_OFFSET          8
  28
  29#ifdef __x86_64__
  30
  31#define rseq_smp_mb()   \
  32        __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
  33#define rseq_smp_rmb()  rseq_barrier()
  34#define rseq_smp_wmb()  rseq_barrier()
  35
  36#define rseq_smp_load_acquire(p)                                        \
  37__extension__ ({                                                        \
  38        __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  39        rseq_barrier();                                                 \
  40        ____p1;                                                         \
  41})
  42
  43#define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  44
  45#define rseq_smp_store_release(p, v)                                    \
  46do {                                                                    \
  47        rseq_barrier();                                                 \
  48        RSEQ_WRITE_ONCE(*p, v);                                         \
  49} while (0)
  50
  51#ifdef RSEQ_SKIP_FASTPATH
  52#include "rseq-skip.h"
  53#else /* !RSEQ_SKIP_FASTPATH */
  54
  55#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  56                                start_ip, post_commit_offset, abort_ip) \
  57                ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  58                ".balign 32\n\t"                                        \
  59                __rseq_str(label) ":\n\t"                               \
  60                ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  61                ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  62                ".popsection\n\t"                                       \
  63                ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  64                ".quad " __rseq_str(label) "b\n\t"                      \
  65                ".popsection\n\t"
  66
  67
  68#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
  69        __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
  70                                (post_commit_ip - start_ip), abort_ip)
  71
  72/*
  73 * Exit points of a rseq critical section consist of all instructions outside
  74 * of the critical section where a critical section can either branch to or
  75 * reach through the normal course of its execution. The abort IP and the
  76 * post-commit IP are already part of the __rseq_cs section and should not be
  77 * explicitly defined as additional exit points. Knowing all exit points is
  78 * useful to assist debuggers stepping over the critical section.
  79 */
  80#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  81                ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  82                ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  83                ".popsection\n\t"
  84
  85#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
  86                RSEQ_INJECT_ASM(1)                                      \
  87                "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"       \
  88                "movq %%rax, " __rseq_str(rseq_cs) "\n\t"               \
  89                __rseq_str(label) ":\n\t"
  90
  91#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
  92                RSEQ_INJECT_ASM(2)                                      \
  93                "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
  94                "jnz " __rseq_str(label) "\n\t"
  95
  96#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
  97                ".pushsection __rseq_failure, \"ax\"\n\t"               \
  98                /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
  99                ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 100                ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 101                __rseq_str(label) ":\n\t"                               \
 102                teardown                                                \
 103                "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 104                ".popsection\n\t"
 105
 106#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 107                ".pushsection __rseq_failure, \"ax\"\n\t"               \
 108                __rseq_str(label) ":\n\t"                               \
 109                teardown                                                \
 110                "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
 111                ".popsection\n\t"
 112
 113static inline __attribute__((always_inline))
 114int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 115{
 116        RSEQ_INJECT_C(9)
 117
 118        __asm__ __volatile__ goto (
 119                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 120                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 121#ifdef RSEQ_COMPARE_TWICE
 122                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 123                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 124#endif
 125                /* Start rseq by storing table entry pointer into rseq_cs. */
 126                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 127                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 128                RSEQ_INJECT_ASM(3)
 129                "cmpq %[v], %[expect]\n\t"
 130                "jnz %l[cmpfail]\n\t"
 131                RSEQ_INJECT_ASM(4)
 132#ifdef RSEQ_COMPARE_TWICE
 133                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 134                "cmpq %[v], %[expect]\n\t"
 135                "jnz %l[error2]\n\t"
 136#endif
 137                /* final store */
 138                "movq %[newv], %[v]\n\t"
 139                "2:\n\t"
 140                RSEQ_INJECT_ASM(5)
 141                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 142                : /* gcc asm goto does not allow outputs */
 143                : [cpu_id]              "r" (cpu),
 144                  [rseq_abi]            "r" (&__rseq_abi),
 145                  [v]                   "m" (*v),
 146                  [expect]              "r" (expect),
 147                  [newv]                "r" (newv)
 148                : "memory", "cc", "rax"
 149                  RSEQ_INJECT_CLOBBER
 150                : abort, cmpfail
 151#ifdef RSEQ_COMPARE_TWICE
 152                  , error1, error2
 153#endif
 154        );
 155        return 0;
 156abort:
 157        RSEQ_INJECT_FAILED
 158        return -1;
 159cmpfail:
 160        return 1;
 161#ifdef RSEQ_COMPARE_TWICE
 162error1:
 163        rseq_bug("cpu_id comparison failed");
 164error2:
 165        rseq_bug("expected value comparison failed");
 166#endif
 167}
 168
 169/*
 170 * Compare @v against @expectnot. When it does _not_ match, load @v
 171 * into @load, and store the content of *@v + voffp into @v.
 172 */
 173static inline __attribute__((always_inline))
 174int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 175                               off_t voffp, intptr_t *load, int cpu)
 176{
 177        RSEQ_INJECT_C(9)
 178
 179        __asm__ __volatile__ goto (
 180                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 181                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 182#ifdef RSEQ_COMPARE_TWICE
 183                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 184                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 185#endif
 186                /* Start rseq by storing table entry pointer into rseq_cs. */
 187                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 188                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 189                RSEQ_INJECT_ASM(3)
 190                "movq %[v], %%rbx\n\t"
 191                "cmpq %%rbx, %[expectnot]\n\t"
 192                "je %l[cmpfail]\n\t"
 193                RSEQ_INJECT_ASM(4)
 194#ifdef RSEQ_COMPARE_TWICE
 195                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 196                "movq %[v], %%rbx\n\t"
 197                "cmpq %%rbx, %[expectnot]\n\t"
 198                "je %l[error2]\n\t"
 199#endif
 200                "movq %%rbx, %[load]\n\t"
 201                "addq %[voffp], %%rbx\n\t"
 202                "movq (%%rbx), %%rbx\n\t"
 203                /* final store */
 204                "movq %%rbx, %[v]\n\t"
 205                "2:\n\t"
 206                RSEQ_INJECT_ASM(5)
 207                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 208                : /* gcc asm goto does not allow outputs */
 209                : [cpu_id]              "r" (cpu),
 210                  [rseq_abi]            "r" (&__rseq_abi),
 211                  /* final store input */
 212                  [v]                   "m" (*v),
 213                  [expectnot]           "r" (expectnot),
 214                  [voffp]               "er" (voffp),
 215                  [load]                "m" (*load)
 216                : "memory", "cc", "rax", "rbx"
 217                  RSEQ_INJECT_CLOBBER
 218                : abort, cmpfail
 219#ifdef RSEQ_COMPARE_TWICE
 220                  , error1, error2
 221#endif
 222        );
 223        return 0;
 224abort:
 225        RSEQ_INJECT_FAILED
 226        return -1;
 227cmpfail:
 228        return 1;
 229#ifdef RSEQ_COMPARE_TWICE
 230error1:
 231        rseq_bug("cpu_id comparison failed");
 232error2:
 233        rseq_bug("expected value comparison failed");
 234#endif
 235}
 236
 237static inline __attribute__((always_inline))
 238int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 239{
 240        RSEQ_INJECT_C(9)
 241
 242        __asm__ __volatile__ goto (
 243                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 244#ifdef RSEQ_COMPARE_TWICE
 245                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 246#endif
 247                /* Start rseq by storing table entry pointer into rseq_cs. */
 248                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 249                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 250                RSEQ_INJECT_ASM(3)
 251#ifdef RSEQ_COMPARE_TWICE
 252                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 253#endif
 254                /* final store */
 255                "addq %[count], %[v]\n\t"
 256                "2:\n\t"
 257                RSEQ_INJECT_ASM(4)
 258                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 259                : /* gcc asm goto does not allow outputs */
 260                : [cpu_id]              "r" (cpu),
 261                  [rseq_abi]            "r" (&__rseq_abi),
 262                  /* final store input */
 263                  [v]                   "m" (*v),
 264                  [count]               "er" (count)
 265                : "memory", "cc", "rax"
 266                  RSEQ_INJECT_CLOBBER
 267                : abort
 268#ifdef RSEQ_COMPARE_TWICE
 269                  , error1
 270#endif
 271        );
 272        return 0;
 273abort:
 274        RSEQ_INJECT_FAILED
 275        return -1;
 276#ifdef RSEQ_COMPARE_TWICE
 277error1:
 278        rseq_bug("cpu_id comparison failed");
 279#endif
 280}
 281
 282#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
 283
 284/*
 285 *   pval = *(ptr+off)
 286 *  *pval += inc;
 287 */
 288static inline __attribute__((always_inline))
 289int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
 290{
 291        RSEQ_INJECT_C(9)
 292
 293        __asm__ __volatile__ goto (
 294                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 295#ifdef RSEQ_COMPARE_TWICE
 296                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 297#endif
 298                /* Start rseq by storing table entry pointer into rseq_cs. */
 299                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 300                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 301                RSEQ_INJECT_ASM(3)
 302#ifdef RSEQ_COMPARE_TWICE
 303                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 304#endif
 305                /* get p+v */
 306                "movq %[ptr], %%rbx\n\t"
 307                "addq %[off], %%rbx\n\t"
 308                /* get pv */
 309                "movq (%%rbx), %%rcx\n\t"
 310                /* *pv += inc */
 311                "addq %[inc], (%%rcx)\n\t"
 312                "2:\n\t"
 313                RSEQ_INJECT_ASM(4)
 314                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 315                : /* gcc asm goto does not allow outputs */
 316                : [cpu_id]              "r" (cpu),
 317                  [rseq_abi]            "r" (&__rseq_abi),
 318                  /* final store input */
 319                  [ptr]                 "m" (*ptr),
 320                  [off]                 "er" (off),
 321                  [inc]                 "er" (inc)
 322                : "memory", "cc", "rax", "rbx", "rcx"
 323                  RSEQ_INJECT_CLOBBER
 324                : abort
 325#ifdef RSEQ_COMPARE_TWICE
 326                  , error1
 327#endif
 328        );
 329        return 0;
 330abort:
 331        RSEQ_INJECT_FAILED
 332        return -1;
 333#ifdef RSEQ_COMPARE_TWICE
 334error1:
 335        rseq_bug("cpu_id comparison failed");
 336#endif
 337}
 338
 339static inline __attribute__((always_inline))
 340int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 341                                 intptr_t *v2, intptr_t newv2,
 342                                 intptr_t newv, int cpu)
 343{
 344        RSEQ_INJECT_C(9)
 345
 346        __asm__ __volatile__ goto (
 347                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 348                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 349#ifdef RSEQ_COMPARE_TWICE
 350                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 351                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 352#endif
 353                /* Start rseq by storing table entry pointer into rseq_cs. */
 354                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 355                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 356                RSEQ_INJECT_ASM(3)
 357                "cmpq %[v], %[expect]\n\t"
 358                "jnz %l[cmpfail]\n\t"
 359                RSEQ_INJECT_ASM(4)
 360#ifdef RSEQ_COMPARE_TWICE
 361                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 362                "cmpq %[v], %[expect]\n\t"
 363                "jnz %l[error2]\n\t"
 364#endif
 365                /* try store */
 366                "movq %[newv2], %[v2]\n\t"
 367                RSEQ_INJECT_ASM(5)
 368                /* final store */
 369                "movq %[newv], %[v]\n\t"
 370                "2:\n\t"
 371                RSEQ_INJECT_ASM(6)
 372                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 373                : /* gcc asm goto does not allow outputs */
 374                : [cpu_id]              "r" (cpu),
 375                  [rseq_abi]            "r" (&__rseq_abi),
 376                  /* try store input */
 377                  [v2]                  "m" (*v2),
 378                  [newv2]               "r" (newv2),
 379                  /* final store input */
 380                  [v]                   "m" (*v),
 381                  [expect]              "r" (expect),
 382                  [newv]                "r" (newv)
 383                : "memory", "cc", "rax"
 384                  RSEQ_INJECT_CLOBBER
 385                : abort, cmpfail
 386#ifdef RSEQ_COMPARE_TWICE
 387                  , error1, error2
 388#endif
 389        );
 390        return 0;
 391abort:
 392        RSEQ_INJECT_FAILED
 393        return -1;
 394cmpfail:
 395        return 1;
 396#ifdef RSEQ_COMPARE_TWICE
 397error1:
 398        rseq_bug("cpu_id comparison failed");
 399error2:
 400        rseq_bug("expected value comparison failed");
 401#endif
 402}
 403
 404/* x86-64 is TSO. */
 405static inline __attribute__((always_inline))
 406int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 407                                         intptr_t *v2, intptr_t newv2,
 408                                         intptr_t newv, int cpu)
 409{
 410        return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
 411}
 412
 413static inline __attribute__((always_inline))
 414int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 415                              intptr_t *v2, intptr_t expect2,
 416                              intptr_t newv, int cpu)
 417{
 418        RSEQ_INJECT_C(9)
 419
 420        __asm__ __volatile__ goto (
 421                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 422                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 423#ifdef RSEQ_COMPARE_TWICE
 424                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 425                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 426                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 427#endif
 428                /* Start rseq by storing table entry pointer into rseq_cs. */
 429                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 430                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 431                RSEQ_INJECT_ASM(3)
 432                "cmpq %[v], %[expect]\n\t"
 433                "jnz %l[cmpfail]\n\t"
 434                RSEQ_INJECT_ASM(4)
 435                "cmpq %[v2], %[expect2]\n\t"
 436                "jnz %l[cmpfail]\n\t"
 437                RSEQ_INJECT_ASM(5)
 438#ifdef RSEQ_COMPARE_TWICE
 439                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 440                "cmpq %[v], %[expect]\n\t"
 441                "jnz %l[error2]\n\t"
 442                "cmpq %[v2], %[expect2]\n\t"
 443                "jnz %l[error3]\n\t"
 444#endif
 445                /* final store */
 446                "movq %[newv], %[v]\n\t"
 447                "2:\n\t"
 448                RSEQ_INJECT_ASM(6)
 449                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 450                : /* gcc asm goto does not allow outputs */
 451                : [cpu_id]              "r" (cpu),
 452                  [rseq_abi]            "r" (&__rseq_abi),
 453                  /* cmp2 input */
 454                  [v2]                  "m" (*v2),
 455                  [expect2]             "r" (expect2),
 456                  /* final store input */
 457                  [v]                   "m" (*v),
 458                  [expect]              "r" (expect),
 459                  [newv]                "r" (newv)
 460                : "memory", "cc", "rax"
 461                  RSEQ_INJECT_CLOBBER
 462                : abort, cmpfail
 463#ifdef RSEQ_COMPARE_TWICE
 464                  , error1, error2, error3
 465#endif
 466        );
 467        return 0;
 468abort:
 469        RSEQ_INJECT_FAILED
 470        return -1;
 471cmpfail:
 472        return 1;
 473#ifdef RSEQ_COMPARE_TWICE
 474error1:
 475        rseq_bug("cpu_id comparison failed");
 476error2:
 477        rseq_bug("1st expected value comparison failed");
 478error3:
 479        rseq_bug("2nd expected value comparison failed");
 480#endif
 481}
 482
 483static inline __attribute__((always_inline))
 484int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 485                                 void *dst, void *src, size_t len,
 486                                 intptr_t newv, int cpu)
 487{
 488        uint64_t rseq_scratch[3];
 489
 490        RSEQ_INJECT_C(9)
 491
 492        __asm__ __volatile__ goto (
 493                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 494                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 495#ifdef RSEQ_COMPARE_TWICE
 496                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 497                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 498#endif
 499                "movq %[src], %[rseq_scratch0]\n\t"
 500                "movq %[dst], %[rseq_scratch1]\n\t"
 501                "movq %[len], %[rseq_scratch2]\n\t"
 502                /* Start rseq by storing table entry pointer into rseq_cs. */
 503                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 504                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 505                RSEQ_INJECT_ASM(3)
 506                "cmpq %[v], %[expect]\n\t"
 507                "jnz 5f\n\t"
 508                RSEQ_INJECT_ASM(4)
 509#ifdef RSEQ_COMPARE_TWICE
 510                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
 511                "cmpq %[v], %[expect]\n\t"
 512                "jnz 7f\n\t"
 513#endif
 514                /* try memcpy */
 515                "test %[len], %[len]\n\t" \
 516                "jz 333f\n\t" \
 517                "222:\n\t" \
 518                "movb (%[src]), %%al\n\t" \
 519                "movb %%al, (%[dst])\n\t" \
 520                "inc %[src]\n\t" \
 521                "inc %[dst]\n\t" \
 522                "dec %[len]\n\t" \
 523                "jnz 222b\n\t" \
 524                "333:\n\t" \
 525                RSEQ_INJECT_ASM(5)
 526                /* final store */
 527                "movq %[newv], %[v]\n\t"
 528                "2:\n\t"
 529                RSEQ_INJECT_ASM(6)
 530                /* teardown */
 531                "movq %[rseq_scratch2], %[len]\n\t"
 532                "movq %[rseq_scratch1], %[dst]\n\t"
 533                "movq %[rseq_scratch0], %[src]\n\t"
 534                RSEQ_ASM_DEFINE_ABORT(4,
 535                        "movq %[rseq_scratch2], %[len]\n\t"
 536                        "movq %[rseq_scratch1], %[dst]\n\t"
 537                        "movq %[rseq_scratch0], %[src]\n\t",
 538                        abort)
 539                RSEQ_ASM_DEFINE_CMPFAIL(5,
 540                        "movq %[rseq_scratch2], %[len]\n\t"
 541                        "movq %[rseq_scratch1], %[dst]\n\t"
 542                        "movq %[rseq_scratch0], %[src]\n\t",
 543                        cmpfail)
 544#ifdef RSEQ_COMPARE_TWICE
 545                RSEQ_ASM_DEFINE_CMPFAIL(6,
 546                        "movq %[rseq_scratch2], %[len]\n\t"
 547                        "movq %[rseq_scratch1], %[dst]\n\t"
 548                        "movq %[rseq_scratch0], %[src]\n\t",
 549                        error1)
 550                RSEQ_ASM_DEFINE_CMPFAIL(7,
 551                        "movq %[rseq_scratch2], %[len]\n\t"
 552                        "movq %[rseq_scratch1], %[dst]\n\t"
 553                        "movq %[rseq_scratch0], %[src]\n\t",
 554                        error2)
 555#endif
 556                : /* gcc asm goto does not allow outputs */
 557                : [cpu_id]              "r" (cpu),
 558                  [rseq_abi]            "r" (&__rseq_abi),
 559                  /* final store input */
 560                  [v]                   "m" (*v),
 561                  [expect]              "r" (expect),
 562                  [newv]                "r" (newv),
 563                  /* try memcpy input */
 564                  [dst]                 "r" (dst),
 565                  [src]                 "r" (src),
 566                  [len]                 "r" (len),
 567                  [rseq_scratch0]       "m" (rseq_scratch[0]),
 568                  [rseq_scratch1]       "m" (rseq_scratch[1]),
 569                  [rseq_scratch2]       "m" (rseq_scratch[2])
 570                : "memory", "cc", "rax"
 571                  RSEQ_INJECT_CLOBBER
 572                : abort, cmpfail
 573#ifdef RSEQ_COMPARE_TWICE
 574                  , error1, error2
 575#endif
 576        );
 577        return 0;
 578abort:
 579        RSEQ_INJECT_FAILED
 580        return -1;
 581cmpfail:
 582        return 1;
 583#ifdef RSEQ_COMPARE_TWICE
 584error1:
 585        rseq_bug("cpu_id comparison failed");
 586error2:
 587        rseq_bug("expected value comparison failed");
 588#endif
 589}
 590
 591/* x86-64 is TSO. */
 592static inline __attribute__((always_inline))
 593int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 594                                         void *dst, void *src, size_t len,
 595                                         intptr_t newv, int cpu)
 596{
 597        return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
 598                                            newv, cpu);
 599}
 600
 601#endif /* !RSEQ_SKIP_FASTPATH */
 602
 603#elif __i386__
 604
 605#define rseq_smp_mb()   \
 606        __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 607#define rseq_smp_rmb()  \
 608        __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 609#define rseq_smp_wmb()  \
 610        __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 611
 612#define rseq_smp_load_acquire(p)                                        \
 613__extension__ ({                                                        \
 614        __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
 615        rseq_smp_mb();                                                  \
 616        ____p1;                                                         \
 617})
 618
 619#define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
 620
 621#define rseq_smp_store_release(p, v)                                    \
 622do {                                                                    \
 623        rseq_smp_mb();                                                  \
 624        RSEQ_WRITE_ONCE(*p, v);                                         \
 625} while (0)
 626
 627#ifdef RSEQ_SKIP_FASTPATH
 628#include "rseq-skip.h"
 629#else /* !RSEQ_SKIP_FASTPATH */
 630
 631/*
 632 * Use eax as scratch register and take memory operands as input to
 633 * lessen register pressure. Especially needed when compiling in O0.
 634 */
 635#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
 636                                start_ip, post_commit_offset, abort_ip) \
 637                ".pushsection __rseq_cs, \"aw\"\n\t"                    \
 638                ".balign 32\n\t"                                        \
 639                __rseq_str(label) ":\n\t"                               \
 640                ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 641                ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
 642                ".popsection\n\t"                                       \
 643                ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 644                ".long " __rseq_str(label) "b, 0x0\n\t"                 \
 645                ".popsection\n\t"
 646
 647#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 648        __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 649                                (post_commit_ip - start_ip), abort_ip)
 650
 651/*
 652 * Exit points of a rseq critical section consist of all instructions outside
 653 * of the critical section where a critical section can either branch to or
 654 * reach through the normal course of its execution. The abort IP and the
 655 * post-commit IP are already part of the __rseq_cs section and should not be
 656 * explicitly defined as additional exit points. Knowing all exit points is
 657 * useful to assist debuggers stepping over the critical section.
 658 */
 659#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 660                ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
 661                ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
 662                ".popsection\n\t"
 663
 664#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 665                RSEQ_INJECT_ASM(1)                                      \
 666                "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"   \
 667                __rseq_str(label) ":\n\t"
 668
 669#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 670                RSEQ_INJECT_ASM(2)                                      \
 671                "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
 672                "jnz " __rseq_str(label) "\n\t"
 673
 674#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 675                ".pushsection __rseq_failure, \"ax\"\n\t"               \
 676                /* Disassembler-friendly signature: ud1 <sig>,%edi. */  \
 677                ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 678                ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 679                __rseq_str(label) ":\n\t"                               \
 680                teardown                                                \
 681                "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 682                ".popsection\n\t"
 683
 684#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 685                ".pushsection __rseq_failure, \"ax\"\n\t"               \
 686                __rseq_str(label) ":\n\t"                               \
 687                teardown                                                \
 688                "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
 689                ".popsection\n\t"
 690
 691static inline __attribute__((always_inline))
 692int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 693{
 694        RSEQ_INJECT_C(9)
 695
 696        __asm__ __volatile__ goto (
 697                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 698                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 699#ifdef RSEQ_COMPARE_TWICE
 700                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 701                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 702#endif
 703                /* Start rseq by storing table entry pointer into rseq_cs. */
 704                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 705                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 706                RSEQ_INJECT_ASM(3)
 707                "cmpl %[v], %[expect]\n\t"
 708                "jnz %l[cmpfail]\n\t"
 709                RSEQ_INJECT_ASM(4)
 710#ifdef RSEQ_COMPARE_TWICE
 711                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 712                "cmpl %[v], %[expect]\n\t"
 713                "jnz %l[error2]\n\t"
 714#endif
 715                /* final store */
 716                "movl %[newv], %[v]\n\t"
 717                "2:\n\t"
 718                RSEQ_INJECT_ASM(5)
 719                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 720                : /* gcc asm goto does not allow outputs */
 721                : [cpu_id]              "r" (cpu),
 722                  [rseq_abi]            "r" (&__rseq_abi),
 723                  [v]                   "m" (*v),
 724                  [expect]              "r" (expect),
 725                  [newv]                "r" (newv)
 726                : "memory", "cc", "eax"
 727                  RSEQ_INJECT_CLOBBER
 728                : abort, cmpfail
 729#ifdef RSEQ_COMPARE_TWICE
 730                  , error1, error2
 731#endif
 732        );
 733        return 0;
 734abort:
 735        RSEQ_INJECT_FAILED
 736        return -1;
 737cmpfail:
 738        return 1;
 739#ifdef RSEQ_COMPARE_TWICE
 740error1:
 741        rseq_bug("cpu_id comparison failed");
 742error2:
 743        rseq_bug("expected value comparison failed");
 744#endif
 745}
 746
 747/*
 748 * Compare @v against @expectnot. When it does _not_ match, load @v
 749 * into @load, and store the content of *@v + voffp into @v.
 750 */
 751static inline __attribute__((always_inline))
 752int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 753                               off_t voffp, intptr_t *load, int cpu)
 754{
 755        RSEQ_INJECT_C(9)
 756
 757        __asm__ __volatile__ goto (
 758                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 759                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 760#ifdef RSEQ_COMPARE_TWICE
 761                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 762                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 763#endif
 764                /* Start rseq by storing table entry pointer into rseq_cs. */
 765                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 766                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 767                RSEQ_INJECT_ASM(3)
 768                "movl %[v], %%ebx\n\t"
 769                "cmpl %%ebx, %[expectnot]\n\t"
 770                "je %l[cmpfail]\n\t"
 771                RSEQ_INJECT_ASM(4)
 772#ifdef RSEQ_COMPARE_TWICE
 773                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 774                "movl %[v], %%ebx\n\t"
 775                "cmpl %%ebx, %[expectnot]\n\t"
 776                "je %l[error2]\n\t"
 777#endif
 778                "movl %%ebx, %[load]\n\t"
 779                "addl %[voffp], %%ebx\n\t"
 780                "movl (%%ebx), %%ebx\n\t"
 781                /* final store */
 782                "movl %%ebx, %[v]\n\t"
 783                "2:\n\t"
 784                RSEQ_INJECT_ASM(5)
 785                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 786                : /* gcc asm goto does not allow outputs */
 787                : [cpu_id]              "r" (cpu),
 788                  [rseq_abi]            "r" (&__rseq_abi),
 789                  /* final store input */
 790                  [v]                   "m" (*v),
 791                  [expectnot]           "r" (expectnot),
 792                  [voffp]               "ir" (voffp),
 793                  [load]                "m" (*load)
 794                : "memory", "cc", "eax", "ebx"
 795                  RSEQ_INJECT_CLOBBER
 796                : abort, cmpfail
 797#ifdef RSEQ_COMPARE_TWICE
 798                  , error1, error2
 799#endif
 800        );
 801        return 0;
 802abort:
 803        RSEQ_INJECT_FAILED
 804        return -1;
 805cmpfail:
 806        return 1;
 807#ifdef RSEQ_COMPARE_TWICE
 808error1:
 809        rseq_bug("cpu_id comparison failed");
 810error2:
 811        rseq_bug("expected value comparison failed");
 812#endif
 813}
 814
 815static inline __attribute__((always_inline))
 816int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 817{
 818        RSEQ_INJECT_C(9)
 819
 820        __asm__ __volatile__ goto (
 821                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 822#ifdef RSEQ_COMPARE_TWICE
 823                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 824#endif
 825                /* Start rseq by storing table entry pointer into rseq_cs. */
 826                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 827                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 828                RSEQ_INJECT_ASM(3)
 829#ifdef RSEQ_COMPARE_TWICE
 830                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 831#endif
 832                /* final store */
 833                "addl %[count], %[v]\n\t"
 834                "2:\n\t"
 835                RSEQ_INJECT_ASM(4)
 836                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 837                : /* gcc asm goto does not allow outputs */
 838                : [cpu_id]              "r" (cpu),
 839                  [rseq_abi]            "r" (&__rseq_abi),
 840                  /* final store input */
 841                  [v]                   "m" (*v),
 842                  [count]               "ir" (count)
 843                : "memory", "cc", "eax"
 844                  RSEQ_INJECT_CLOBBER
 845                : abort
 846#ifdef RSEQ_COMPARE_TWICE
 847                  , error1
 848#endif
 849        );
 850        return 0;
 851abort:
 852        RSEQ_INJECT_FAILED
 853        return -1;
 854#ifdef RSEQ_COMPARE_TWICE
 855error1:
 856        rseq_bug("cpu_id comparison failed");
 857#endif
 858}
 859
 860static inline __attribute__((always_inline))
 861int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 862                                 intptr_t *v2, intptr_t newv2,
 863                                 intptr_t newv, int cpu)
 864{
 865        RSEQ_INJECT_C(9)
 866
 867        __asm__ __volatile__ goto (
 868                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 869                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 870#ifdef RSEQ_COMPARE_TWICE
 871                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 872                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 873#endif
 874                /* Start rseq by storing table entry pointer into rseq_cs. */
 875                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 876                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 877                RSEQ_INJECT_ASM(3)
 878                "cmpl %[v], %[expect]\n\t"
 879                "jnz %l[cmpfail]\n\t"
 880                RSEQ_INJECT_ASM(4)
 881#ifdef RSEQ_COMPARE_TWICE
 882                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 883                "cmpl %[v], %[expect]\n\t"
 884                "jnz %l[error2]\n\t"
 885#endif
 886                /* try store */
 887                "movl %[newv2], %%eax\n\t"
 888                "movl %%eax, %[v2]\n\t"
 889                RSEQ_INJECT_ASM(5)
 890                /* final store */
 891                "movl %[newv], %[v]\n\t"
 892                "2:\n\t"
 893                RSEQ_INJECT_ASM(6)
 894                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 895                : /* gcc asm goto does not allow outputs */
 896                : [cpu_id]              "r" (cpu),
 897                  [rseq_abi]            "r" (&__rseq_abi),
 898                  /* try store input */
 899                  [v2]                  "m" (*v2),
 900                  [newv2]               "m" (newv2),
 901                  /* final store input */
 902                  [v]                   "m" (*v),
 903                  [expect]              "r" (expect),
 904                  [newv]                "r" (newv)
 905                : "memory", "cc", "eax"
 906                  RSEQ_INJECT_CLOBBER
 907                : abort, cmpfail
 908#ifdef RSEQ_COMPARE_TWICE
 909                  , error1, error2
 910#endif
 911        );
 912        return 0;
 913abort:
 914        RSEQ_INJECT_FAILED
 915        return -1;
 916cmpfail:
 917        return 1;
 918#ifdef RSEQ_COMPARE_TWICE
 919error1:
 920        rseq_bug("cpu_id comparison failed");
 921error2:
 922        rseq_bug("expected value comparison failed");
 923#endif
 924}
 925
 926static inline __attribute__((always_inline))
 927int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 928                                         intptr_t *v2, intptr_t newv2,
 929                                         intptr_t newv, int cpu)
 930{
 931        RSEQ_INJECT_C(9)
 932
 933        __asm__ __volatile__ goto (
 934                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 935                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 936#ifdef RSEQ_COMPARE_TWICE
 937                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 938                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 939#endif
 940                /* Start rseq by storing table entry pointer into rseq_cs. */
 941                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 942                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 943                RSEQ_INJECT_ASM(3)
 944                "movl %[expect], %%eax\n\t"
 945                "cmpl %[v], %%eax\n\t"
 946                "jnz %l[cmpfail]\n\t"
 947                RSEQ_INJECT_ASM(4)
 948#ifdef RSEQ_COMPARE_TWICE
 949                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 950                "movl %[expect], %%eax\n\t"
 951                "cmpl %[v], %%eax\n\t"
 952                "jnz %l[error2]\n\t"
 953#endif
 954                /* try store */
 955                "movl %[newv2], %[v2]\n\t"
 956                RSEQ_INJECT_ASM(5)
 957                "lock; addl $0,-128(%%esp)\n\t"
 958                /* final store */
 959                "movl %[newv], %[v]\n\t"
 960                "2:\n\t"
 961                RSEQ_INJECT_ASM(6)
 962                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 963                : /* gcc asm goto does not allow outputs */
 964                : [cpu_id]              "r" (cpu),
 965                  [rseq_abi]            "r" (&__rseq_abi),
 966                  /* try store input */
 967                  [v2]                  "m" (*v2),
 968                  [newv2]               "r" (newv2),
 969                  /* final store input */
 970                  [v]                   "m" (*v),
 971                  [expect]              "m" (expect),
 972                  [newv]                "r" (newv)
 973                : "memory", "cc", "eax"
 974                  RSEQ_INJECT_CLOBBER
 975                : abort, cmpfail
 976#ifdef RSEQ_COMPARE_TWICE
 977                  , error1, error2
 978#endif
 979        );
 980        return 0;
 981abort:
 982        RSEQ_INJECT_FAILED
 983        return -1;
 984cmpfail:
 985        return 1;
 986#ifdef RSEQ_COMPARE_TWICE
 987error1:
 988        rseq_bug("cpu_id comparison failed");
 989error2:
 990        rseq_bug("expected value comparison failed");
 991#endif
 992
 993}
 994
 995static inline __attribute__((always_inline))
 996int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 997                              intptr_t *v2, intptr_t expect2,
 998                              intptr_t newv, int cpu)
 999{
1000        RSEQ_INJECT_C(9)

1001
1002        __asm__ __volatile__ goto (
1003                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1004                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1005#ifdef RSEQ_COMPARE_TWICE
1006                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1007                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1008                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1009#endif
1010                /* Start rseq by storing table entry pointer into rseq_cs. */
1011                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1012                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1013                RSEQ_INJECT_ASM(3)
1014                "cmpl %[v], %[expect]\n\t"
1015                "jnz %l[cmpfail]\n\t"
1016                RSEQ_INJECT_ASM(4)
1017                "cmpl %[expect2], %[v2]\n\t"
1018                "jnz %l[cmpfail]\n\t"
1019                RSEQ_INJECT_ASM(5)
1020#ifdef RSEQ_COMPARE_TWICE
1021                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
1022                "cmpl %[v], %[expect]\n\t"
1023                "jnz %l[error2]\n\t"
1024                "cmpl %[expect2], %[v2]\n\t"
1025                "jnz %l[error3]\n\t"
1026#endif
1027                "movl %[newv], %%eax\n\t"
1028                /* final store */
1029                "movl %%eax, %[v]\n\t"
1030                "2:\n\t"
1031                RSEQ_INJECT_ASM(6)
1032                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1033                : /* gcc asm goto does not allow outputs */
1034                : [cpu_id]              "r" (cpu),
1035                  [rseq_abi]            "r" (&__rseq_abi),
1036                  /* cmp2 input */
1037                  [v2]                  "m" (*v2),
1038                  [expect2]             "r" (expect2),
1039                  /* final store input */
1040                  [v]                   "m" (*v),
1041                  [expect]              "r" (expect),
1042                  [newv]                "m" (newv)
1043                : "memory", "cc", "eax"
1044                  RSEQ_INJECT_CLOBBER
1045                : abort, cmpfail
1046#ifdef RSEQ_COMPARE_TWICE
1047                  , error1, error2, error3
1048#endif
1049        );
1050        return 0;
1051abort:
1052        RSEQ_INJECT_FAILED
1053        return -1;
1054cmpfail:
1055        return 1;
1056#ifdef RSEQ_COMPARE_TWICE
1057error1:
1058        rseq_bug("cpu_id comparison failed");
1059error2:
1060        rseq_bug("1st expected value comparison failed");
1061error3:
1062        rseq_bug("2nd expected value comparison failed");
1063#endif
1064}
1065
1066/* TODO: implement a faster memcpy. */
1067static inline __attribute__((always_inline))
1068int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1069                                 void *dst, void *src, size_t len,
1070                                 intptr_t newv, int cpu)
1071{
1072        uint32_t rseq_scratch[3];
1073
1074        RSEQ_INJECT_C(9)
1075
1076        __asm__ __volatile__ goto (
1077                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1078                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1079#ifdef RSEQ_COMPARE_TWICE
1080                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1081                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1082#endif
1083                "movl %[src], %[rseq_scratch0]\n\t"
1084                "movl %[dst], %[rseq_scratch1]\n\t"
1085                "movl %[len], %[rseq_scratch2]\n\t"
1086                /* Start rseq by storing table entry pointer into rseq_cs. */
1087                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1088                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1089                RSEQ_INJECT_ASM(3)
1090                "movl %[expect], %%eax\n\t"
1091                "cmpl %%eax, %[v]\n\t"
1092                "jnz 5f\n\t"
1093                RSEQ_INJECT_ASM(4)
1094#ifdef RSEQ_COMPARE_TWICE
1095                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1096                "movl %[expect], %%eax\n\t"
1097                "cmpl %%eax, %[v]\n\t"
1098                "jnz 7f\n\t"
1099#endif
1100                /* try memcpy */
1101                "test %[len], %[len]\n\t" \
1102                "jz 333f\n\t" \
1103                "222:\n\t" \
1104                "movb (%[src]), %%al\n\t" \
1105                "movb %%al, (%[dst])\n\t" \
1106                "inc %[src]\n\t" \
1107                "inc %[dst]\n\t" \
1108                "dec %[len]\n\t" \
1109                "jnz 222b\n\t" \
1110                "333:\n\t" \
1111                RSEQ_INJECT_ASM(5)
1112                "movl %[newv], %%eax\n\t"
1113                /* final store */
1114                "movl %%eax, %[v]\n\t"
1115                "2:\n\t"
1116                RSEQ_INJECT_ASM(6)
1117                /* teardown */
1118                "movl %[rseq_scratch2], %[len]\n\t"
1119                "movl %[rseq_scratch1], %[dst]\n\t"
1120                "movl %[rseq_scratch0], %[src]\n\t"
1121                RSEQ_ASM_DEFINE_ABORT(4,
1122                        "movl %[rseq_scratch2], %[len]\n\t"
1123                        "movl %[rseq_scratch1], %[dst]\n\t"
1124                        "movl %[rseq_scratch0], %[src]\n\t",
1125                        abort)
1126                RSEQ_ASM_DEFINE_CMPFAIL(5,
1127                        "movl %[rseq_scratch2], %[len]\n\t"
1128                        "movl %[rseq_scratch1], %[dst]\n\t"
1129                        "movl %[rseq_scratch0], %[src]\n\t",
1130                        cmpfail)
1131#ifdef RSEQ_COMPARE_TWICE
1132                RSEQ_ASM_DEFINE_CMPFAIL(6,
1133                        "movl %[rseq_scratch2], %[len]\n\t"
1134                        "movl %[rseq_scratch1], %[dst]\n\t"
1135                        "movl %[rseq_scratch0], %[src]\n\t",
1136                        error1)
1137                RSEQ_ASM_DEFINE_CMPFAIL(7,
1138                        "movl %[rseq_scratch2], %[len]\n\t"
1139                        "movl %[rseq_scratch1], %[dst]\n\t"
1140                        "movl %[rseq_scratch0], %[src]\n\t",
1141                        error2)
1142#endif
1143                : /* gcc asm goto does not allow outputs */
1144                : [cpu_id]              "r" (cpu),
1145                  [rseq_abi]            "r" (&__rseq_abi),
1146                  /* final store input */
1147                  [v]                   "m" (*v),
1148                  [expect]              "m" (expect),
1149                  [newv]                "m" (newv),
1150                  /* try memcpy input */
1151                  [dst]                 "r" (dst),
1152                  [src]                 "r" (src),
1153                  [len]                 "r" (len),
1154                  [rseq_scratch0]       "m" (rseq_scratch[0]),
1155                  [rseq_scratch1]       "m" (rseq_scratch[1]),
1156                  [rseq_scratch2]       "m" (rseq_scratch[2])
1157                : "memory", "cc", "eax"
1158                  RSEQ_INJECT_CLOBBER
1159                : abort, cmpfail
1160#ifdef RSEQ_COMPARE_TWICE
1161                  , error1, error2
1162#endif
1163        );
1164        return 0;
1165abort:
1166        RSEQ_INJECT_FAILED
1167        return -1;
1168cmpfail:
1169        return 1;
1170#ifdef RSEQ_COMPARE_TWICE
1171error1:
1172        rseq_bug("cpu_id comparison failed");
1173error2:
1174        rseq_bug("expected value comparison failed");
1175#endif
1176}
1177
1178/* TODO: implement a faster memcpy. */
1179static inline __attribute__((always_inline))
1180int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1181                                         void *dst, void *src, size_t len,
1182                                         intptr_t newv, int cpu)
1183{
1184        uint32_t rseq_scratch[3];
1185
1186        RSEQ_INJECT_C(9)
1187
1188        __asm__ __volatile__ goto (
1189                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1190                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1191#ifdef RSEQ_COMPARE_TWICE
1192                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1193                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1194#endif
1195                "movl %[src], %[rseq_scratch0]\n\t"
1196                "movl %[dst], %[rseq_scratch1]\n\t"
1197                "movl %[len], %[rseq_scratch2]\n\t"
1198                /* Start rseq by storing table entry pointer into rseq_cs. */
1199                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1200                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1201                RSEQ_INJECT_ASM(3)
1202                "movl %[expect], %%eax\n\t"
1203                "cmpl %%eax, %[v]\n\t"
1204                "jnz 5f\n\t"
1205                RSEQ_INJECT_ASM(4)
1206#ifdef RSEQ_COMPARE_TWICE
1207                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1208                "movl %[expect], %%eax\n\t"
1209                "cmpl %%eax, %[v]\n\t"
1210                "jnz 7f\n\t"
1211#endif
1212                /* try memcpy */
1213                "test %[len], %[len]\n\t" \
1214                "jz 333f\n\t" \
1215                "222:\n\t" \
1216                "movb (%[src]), %%al\n\t" \
1217                "movb %%al, (%[dst])\n\t" \
1218                "inc %[src]\n\t" \
1219                "inc %[dst]\n\t" \
1220                "dec %[len]\n\t" \
1221                "jnz 222b\n\t" \
1222                "333:\n\t" \
1223                RSEQ_INJECT_ASM(5)
1224                "lock; addl $0,-128(%%esp)\n\t"
1225                "movl %[newv], %%eax\n\t"
1226                /* final store */
1227                "movl %%eax, %[v]\n\t"
1228                "2:\n\t"
1229                RSEQ_INJECT_ASM(6)
1230                /* teardown */
1231                "movl %[rseq_scratch2], %[len]\n\t"
1232                "movl %[rseq_scratch1], %[dst]\n\t"
1233                "movl %[rseq_scratch0], %[src]\n\t"
1234                RSEQ_ASM_DEFINE_ABORT(4,
1235                        "movl %[rseq_scratch2], %[len]\n\t"
1236                        "movl %[rseq_scratch1], %[dst]\n\t"
1237                        "movl %[rseq_scratch0], %[src]\n\t",
1238                        abort)
1239                RSEQ_ASM_DEFINE_CMPFAIL(5,
1240                        "movl %[rseq_scratch2], %[len]\n\t"
1241                        "movl %[rseq_scratch1], %[dst]\n\t"
1242                        "movl %[rseq_scratch0], %[src]\n\t",
1243                        cmpfail)
1244#ifdef RSEQ_COMPARE_TWICE
1245                RSEQ_ASM_DEFINE_CMPFAIL(6,
1246                        "movl %[rseq_scratch2], %[len]\n\t"
1247                        "movl %[rseq_scratch1], %[dst]\n\t"
1248                        "movl %[rseq_scratch0], %[src]\n\t",
1249                        error1)
1250                RSEQ_ASM_DEFINE_CMPFAIL(7,
1251                        "movl %[rseq_scratch2], %[len]\n\t"
1252                        "movl %[rseq_scratch1], %[dst]\n\t"
1253                        "movl %[rseq_scratch0], %[src]\n\t",
1254                        error2)
1255#endif
1256                : /* gcc asm goto does not allow outputs */
1257                : [cpu_id]              "r" (cpu),
1258                  [rseq_abi]            "r" (&__rseq_abi),
1259                  /* final store input */
1260                  [v]                   "m" (*v),
1261                  [expect]              "m" (expect),
1262                  [newv]                "m" (newv),
1263                  /* try memcpy input */
1264                  [dst]                 "r" (dst),
1265                  [src]                 "r" (src),
1266                  [len]                 "r" (len),
1267                  [rseq_scratch0]       "m" (rseq_scratch[0]),
1268                  [rseq_scratch1]       "m" (rseq_scratch[1]),
1269                  [rseq_scratch2]       "m" (rseq_scratch[2])
1270                : "memory", "cc", "eax"
1271                  RSEQ_INJECT_CLOBBER
1272                : abort, cmpfail
1273#ifdef RSEQ_COMPARE_TWICE
1274                  , error1, error2
1275#endif
1276        );
1277        return 0;
1278abort:
1279        RSEQ_INJECT_FAILED
1280        return -1;
1281cmpfail:
1282        return 1;
1283#ifdef RSEQ_COMPARE_TWICE
1284error1:
1285        rseq_bug("cpu_id comparison failed");
1286error2:
1287        rseq_bug("expected value comparison failed");
1288#endif
1289}
1290
1291#endif /* !RSEQ_SKIP_FASTPATH */
1292
1293#endif
1294