LXR linux/tools/testing/selftests/rseq/rseq-x86.h

   1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2/*
   3 * rseq-x86.h
   4 *
   5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6 */
   7
   8#include <stdint.h>
   9
  10/*
  11 * RSEQ_SIG is used with the following reserved undefined instructions, which
  12 * trap in user-space:
  13 *
  14 * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
  15 * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
  16 */
  17#define RSEQ_SIG        0x53053053
  18
  19/*
  20 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
  21 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
  22 * address through a "r" input operand.
  23 */
  24
  25/* Offset of cpu_id and rseq_cs fields in struct rseq. */
  26#define RSEQ_CPU_ID_OFFSET      4
  27#define RSEQ_CS_OFFSET          8
  28
  29#ifdef __x86_64__
  30
  31#define rseq_smp_mb()   \
  32        __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
  33#define rseq_smp_rmb()  rseq_barrier()
  34#define rseq_smp_wmb()  rseq_barrier()
  35
  36#define rseq_smp_load_acquire(p)                                        \
  37__extension__ ({                                                        \
  38        __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  39        rseq_barrier();                                                 \
  40        ____p1;                                                         \
  41})
  42
  43#define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  44
  45#define rseq_smp_store_release(p, v)                                    \
  46do {                                                                    \
  47        rseq_barrier();                                                 \
  48        RSEQ_WRITE_ONCE(*p, v);                                         \
  49} while (0)
  50
  51#ifdef RSEQ_SKIP_FASTPATH
  52#include "rseq-skip.h"
  53#else /* !RSEQ_SKIP_FASTPATH */
  54
  55#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  56                                start_ip, post_commit_offset, abort_ip) \
  57                ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  58                ".balign 32\n\t"                                        \
  59                __rseq_str(label) ":\n\t"                               \
  60                ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  61                ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  62                ".popsection\n\t"                                       \
  63                ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  64                ".quad " __rseq_str(label) "b\n\t"                      \
  65                ".popsection\n\t"
  66
  67
  68#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
  69        __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
  70                                (post_commit_ip - start_ip), abort_ip)
  71
  72/*
  73 * Exit points of a rseq critical section consist of all instructions outside
  74 * of the critical section where a critical section can either branch to or
  75 * reach through the normal course of its execution. The abort IP and the
  76 * post-commit IP are already part of the __rseq_cs section and should not be
  77 * explicitly defined as additional exit points. Knowing all exit points is
  78 * useful to assist debuggers stepping over the critical section.
  79 */
  80#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  81                ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  82                ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  83                ".popsection\n\t"
  84
  85#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
  86                RSEQ_INJECT_ASM(1)                                      \
  87                "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"       \
  88                "movq %%rax, " __rseq_str(rseq_cs) "\n\t"               \
  89                __rseq_str(label) ":\n\t"
  90
  91#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
  92                RSEQ_INJECT_ASM(2)                                      \
  93                "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
  94                "jnz " __rseq_str(label) "\n\t"
  95
  96#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
  97                ".pushsection __rseq_failure, \"ax\"\n\t"               \
  98                /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
  99                ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 100                ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 101                __rseq_str(label) ":\n\t"                               \
 102                teardown                                                \
 103                "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 104                ".popsection\n\t"
 105
 106#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 107                ".pushsection __rseq_failure, \"ax\"\n\t"               \
 108                __rseq_str(label) ":\n\t"                               \
 109                teardown                                                \
 110                "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
 111                ".popsection\n\t"
 112
 113static inline __attribute__((always_inline))
 114int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 115{
 116        RSEQ_INJECT_C(9)
 117
 118        __asm__ __volatile__ goto (
 119                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 120                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 121#ifdef RSEQ_COMPARE_TWICE
 122                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 123                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 124#endif
 125                /* Start rseq by storing table entry pointer into rseq_cs. */
 126                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 127                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 128                RSEQ_INJECT_ASM(3)
 129                "cmpq %[v], %[expect]\n\t"
 130                "jnz %l[cmpfail]\n\t"
 131                RSEQ_INJECT_ASM(4)
 132#ifdef RSEQ_COMPARE_TWICE
 133                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 134                "cmpq %[v], %[expect]\n\t"
 135                "jnz %l[error2]\n\t"
 136#endif
 137                /* final store */
 138                "movq %[newv], %[v]\n\t"
 139                "2:\n\t"
 140                RSEQ_INJECT_ASM(5)
 141                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 142                : /* gcc asm goto does not allow outputs */
 143                : [cpu_id]              "r" (cpu),
 144                  [rseq_abi]            "r" (&__rseq_abi),
 145                  [v]                   "m" (*v),
 146                  [expect]              "r" (expect),
 147                  [newv]                "r" (newv)
 148                : "memory", "cc", "rax"
 149                  RSEQ_INJECT_CLOBBER
 150                : abort, cmpfail
 151#ifdef RSEQ_COMPARE_TWICE
 152                  , error1, error2
 153#endif
 154        );
 155        return 0;
 156abort:
 157        RSEQ_INJECT_FAILED
 158        return -1;
 159cmpfail:
 160        return 1;
 161#ifdef RSEQ_COMPARE_TWICE
 162error1:
 163        rseq_bug("cpu_id comparison failed");
 164error2:
 165        rseq_bug("expected value comparison failed");
 166#endif
 167}
 168
 169/*
 170 * Compare @v against @expectnot. When it does _not_ match, load @v
 171 * into @load, and store the content of *@v + voffp into @v.
 172 */
 173static inline __attribute__((always_inline))
 174int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 175                               off_t voffp, intptr_t *load, int cpu)
 176{
 177        RSEQ_INJECT_C(9)
 178
 179        __asm__ __volatile__ goto (
 180                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 181                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 182#ifdef RSEQ_COMPARE_TWICE
 183                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 184                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 185#endif
 186                /* Start rseq by storing table entry pointer into rseq_cs. */
 187                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 188                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 189                RSEQ_INJECT_ASM(3)
 190                "movq %[v], %%rbx\n\t"
 191                "cmpq %%rbx, %[expectnot]\n\t"
 192                "je %l[cmpfail]\n\t"
 193                RSEQ_INJECT_ASM(4)
 194#ifdef RSEQ_COMPARE_TWICE
 195                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 196                "movq %[v], %%rbx\n\t"
 197                "cmpq %%rbx, %[expectnot]\n\t"
 198                "je %l[error2]\n\t"
 199#endif
 200                "movq %%rbx, %[load]\n\t"
 201                "addq %[voffp], %%rbx\n\t"
 202                "movq (%%rbx), %%rbx\n\t"
 203                /* final store */
 204                "movq %%rbx, %[v]\n\t"
 205                "2:\n\t"
 206                RSEQ_INJECT_ASM(5)
 207                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 208                : /* gcc asm goto does not allow outputs */
 209                : [cpu_id]              "r" (cpu),
 210                  [rseq_abi]            "r" (&__rseq_abi),
 211                  /* final store input */
 212                  [v]                   "m" (*v),
 213                  [expectnot]           "r" (expectnot),
 214                  [voffp]               "er" (voffp),
 215                  [load]                "m" (*load)
 216                : "memory", "cc", "rax", "rbx"
 217                  RSEQ_INJECT_CLOBBER
 218                : abort, cmpfail
 219#ifdef RSEQ_COMPARE_TWICE
 220                  , error1, error2
 221#endif
 222        );
 223        return 0;
 224abort:
 225        RSEQ_INJECT_FAILED
 226        return -1;
 227cmpfail:
 228        return 1;
 229#ifdef RSEQ_COMPARE_TWICE
 230error1:
 231        rseq_bug("cpu_id comparison failed");
 232error2:
 233        rseq_bug("expected value comparison failed");
 234#endif
 235}
 236
 237static inline __attribute__((always_inline))
 238int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 239{
 240        RSEQ_INJECT_C(9)
 241
 242        __asm__ __volatile__ goto (
 243                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 244#ifdef RSEQ_COMPARE_TWICE
 245                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 246#endif
 247                /* Start rseq by storing table entry pointer into rseq_cs. */
 248                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 249                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 250                RSEQ_INJECT_ASM(3)
 251#ifdef RSEQ_COMPARE_TWICE
 252                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 253#endif
 254                /* final store */
 255                "addq %[count], %[v]\n\t"
 256                "2:\n\t"
 257                RSEQ_INJECT_ASM(4)
 258                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 259                : /* gcc asm goto does not allow outputs */
 260                : [cpu_id]              "r" (cpu),
 261                  [rseq_abi]            "r" (&__rseq_abi),
 262                  /* final store input */
 263                  [v]                   "m" (*v),
 264                  [count]               "er" (count)
 265                : "memory", "cc", "rax"
 266                  RSEQ_INJECT_CLOBBER
 267                : abort
 268#ifdef RSEQ_COMPARE_TWICE
 269                  , error1
 270#endif
 271        );
 272        return 0;
 273abort:
 274        RSEQ_INJECT_FAILED
 275        return -1;
 276#ifdef RSEQ_COMPARE_TWICE
 277error1:
 278        rseq_bug("cpu_id comparison failed");
 279#endif
 280}
 281
 282static inline __attribute__((always_inline))
 283int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 284                                 intptr_t *v2, intptr_t newv2,
 285                                 intptr_t newv, int cpu)
 286{
 287        RSEQ_INJECT_C(9)
 288
 289        __asm__ __volatile__ goto (
 290                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 291                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 292#ifdef RSEQ_COMPARE_TWICE
 293                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 294                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 295#endif
 296                /* Start rseq by storing table entry pointer into rseq_cs. */
 297                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 298                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 299                RSEQ_INJECT_ASM(3)
 300                "cmpq %[v], %[expect]\n\t"
 301                "jnz %l[cmpfail]\n\t"
 302                RSEQ_INJECT_ASM(4)
 303#ifdef RSEQ_COMPARE_TWICE
 304                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 305                "cmpq %[v], %[expect]\n\t"
 306                "jnz %l[error2]\n\t"
 307#endif
 308                /* try store */
 309                "movq %[newv2], %[v2]\n\t"
 310                RSEQ_INJECT_ASM(5)
 311                /* final store */
 312                "movq %[newv], %[v]\n\t"
 313                "2:\n\t"
 314                RSEQ_INJECT_ASM(6)
 315                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 316                : /* gcc asm goto does not allow outputs */
 317                : [cpu_id]              "r" (cpu),
 318                  [rseq_abi]            "r" (&__rseq_abi),
 319                  /* try store input */
 320                  [v2]                  "m" (*v2),
 321                  [newv2]               "r" (newv2),
 322                  /* final store input */
 323                  [v]                   "m" (*v),
 324                  [expect]              "r" (expect),
 325                  [newv]                "r" (newv)
 326                : "memory", "cc", "rax"
 327                  RSEQ_INJECT_CLOBBER
 328                : abort, cmpfail
 329#ifdef RSEQ_COMPARE_TWICE
 330                  , error1, error2
 331#endif
 332        );
 333        return 0;
 334abort:
 335        RSEQ_INJECT_FAILED
 336        return -1;
 337cmpfail:
 338        return 1;
 339#ifdef RSEQ_COMPARE_TWICE
 340error1:
 341        rseq_bug("cpu_id comparison failed");
 342error2:
 343        rseq_bug("expected value comparison failed");
 344#endif
 345}
 346
 347/* x86-64 is TSO. */
 348static inline __attribute__((always_inline))
 349int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 350                                         intptr_t *v2, intptr_t newv2,
 351                                         intptr_t newv, int cpu)
 352{
 353        return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
 354}
 355
 356static inline __attribute__((always_inline))
 357int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 358                              intptr_t *v2, intptr_t expect2,
 359                              intptr_t newv, int cpu)
 360{
 361        RSEQ_INJECT_C(9)
 362
 363        __asm__ __volatile__ goto (
 364                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 365                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 366#ifdef RSEQ_COMPARE_TWICE
 367                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 368                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 369                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 370#endif
 371                /* Start rseq by storing table entry pointer into rseq_cs. */
 372                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 373                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 374                RSEQ_INJECT_ASM(3)
 375                "cmpq %[v], %[expect]\n\t"
 376                "jnz %l[cmpfail]\n\t"
 377                RSEQ_INJECT_ASM(4)
 378                "cmpq %[v2], %[expect2]\n\t"
 379                "jnz %l[cmpfail]\n\t"
 380                RSEQ_INJECT_ASM(5)
 381#ifdef RSEQ_COMPARE_TWICE
 382                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 383                "cmpq %[v], %[expect]\n\t"
 384                "jnz %l[error2]\n\t"
 385                "cmpq %[v2], %[expect2]\n\t"
 386                "jnz %l[error3]\n\t"
 387#endif
 388                /* final store */
 389                "movq %[newv], %[v]\n\t"
 390                "2:\n\t"
 391                RSEQ_INJECT_ASM(6)
 392                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 393                : /* gcc asm goto does not allow outputs */
 394                : [cpu_id]              "r" (cpu),
 395                  [rseq_abi]            "r" (&__rseq_abi),
 396                  /* cmp2 input */
 397                  [v2]                  "m" (*v2),
 398                  [expect2]             "r" (expect2),
 399                  /* final store input */
 400                  [v]                   "m" (*v),
 401                  [expect]              "r" (expect),
 402                  [newv]                "r" (newv)
 403                : "memory", "cc", "rax"
 404                  RSEQ_INJECT_CLOBBER
 405                : abort, cmpfail
 406#ifdef RSEQ_COMPARE_TWICE
 407                  , error1, error2, error3
 408#endif
 409        );
 410        return 0;
 411abort:
 412        RSEQ_INJECT_FAILED
 413        return -1;
 414cmpfail:
 415        return 1;
 416#ifdef RSEQ_COMPARE_TWICE
 417error1:
 418        rseq_bug("cpu_id comparison failed");
 419error2:
 420        rseq_bug("1st expected value comparison failed");
 421error3:
 422        rseq_bug("2nd expected value comparison failed");
 423#endif
 424}
 425
 426static inline __attribute__((always_inline))
 427int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 428                                 void *dst, void *src, size_t len,
 429                                 intptr_t newv, int cpu)
 430{
 431        uint64_t rseq_scratch[3];
 432
 433        RSEQ_INJECT_C(9)
 434
 435        __asm__ __volatile__ goto (
 436                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 437                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 438#ifdef RSEQ_COMPARE_TWICE
 439                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 440                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 441#endif
 442                "movq %[src], %[rseq_scratch0]\n\t"
 443                "movq %[dst], %[rseq_scratch1]\n\t"
 444                "movq %[len], %[rseq_scratch2]\n\t"
 445                /* Start rseq by storing table entry pointer into rseq_cs. */
 446                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 447                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 448                RSEQ_INJECT_ASM(3)
 449                "cmpq %[v], %[expect]\n\t"
 450                "jnz 5f\n\t"
 451                RSEQ_INJECT_ASM(4)
 452#ifdef RSEQ_COMPARE_TWICE
 453                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
 454                "cmpq %[v], %[expect]\n\t"
 455                "jnz 7f\n\t"
 456#endif
 457                /* try memcpy */
 458                "test %[len], %[len]\n\t" \
 459                "jz 333f\n\t" \
 460                "222:\n\t" \
 461                "movb (%[src]), %%al\n\t" \
 462                "movb %%al, (%[dst])\n\t" \
 463                "inc %[src]\n\t" \
 464                "inc %[dst]\n\t" \
 465                "dec %[len]\n\t" \
 466                "jnz 222b\n\t" \
 467                "333:\n\t" \
 468                RSEQ_INJECT_ASM(5)
 469                /* final store */
 470                "movq %[newv], %[v]\n\t"
 471                "2:\n\t"
 472                RSEQ_INJECT_ASM(6)
 473                /* teardown */
 474                "movq %[rseq_scratch2], %[len]\n\t"
 475                "movq %[rseq_scratch1], %[dst]\n\t"
 476                "movq %[rseq_scratch0], %[src]\n\t"
 477                RSEQ_ASM_DEFINE_ABORT(4,
 478                        "movq %[rseq_scratch2], %[len]\n\t"
 479                        "movq %[rseq_scratch1], %[dst]\n\t"
 480                        "movq %[rseq_scratch0], %[src]\n\t",
 481                        abort)
 482                RSEQ_ASM_DEFINE_CMPFAIL(5,
 483                        "movq %[rseq_scratch2], %[len]\n\t"
 484                        "movq %[rseq_scratch1], %[dst]\n\t"
 485                        "movq %[rseq_scratch0], %[src]\n\t",
 486                        cmpfail)
 487#ifdef RSEQ_COMPARE_TWICE
 488                RSEQ_ASM_DEFINE_CMPFAIL(6,
 489                        "movq %[rseq_scratch2], %[len]\n\t"
 490                        "movq %[rseq_scratch1], %[dst]\n\t"
 491                        "movq %[rseq_scratch0], %[src]\n\t",
 492                        error1)
 493                RSEQ_ASM_DEFINE_CMPFAIL(7,
 494                        "movq %[rseq_scratch2], %[len]\n\t"
 495                        "movq %[rseq_scratch1], %[dst]\n\t"
 496                        "movq %[rseq_scratch0], %[src]\n\t",
 497                        error2)
 498#endif
 499                : /* gcc asm goto does not allow outputs */
 500                : [cpu_id]              "r" (cpu),
 501                  [rseq_abi]            "r" (&__rseq_abi),
 502                  /* final store input */
 503                  [v]                   "m" (*v),
 504                  [expect]              "r" (expect),
 505                  [newv]                "r" (newv),
 506                  /* try memcpy input */
 507                  [dst]                 "r" (dst),
 508                  [src]                 "r" (src),
 509                  [len]                 "r" (len),
 510                  [rseq_scratch0]       "m" (rseq_scratch[0]),
 511                  [rseq_scratch1]       "m" (rseq_scratch[1]),
 512                  [rseq_scratch2]       "m" (rseq_scratch[2])
 513                : "memory", "cc", "rax"
 514                  RSEQ_INJECT_CLOBBER
 515                : abort, cmpfail
 516#ifdef RSEQ_COMPARE_TWICE
 517                  , error1, error2
 518#endif
 519        );
 520        return 0;
 521abort:
 522        RSEQ_INJECT_FAILED
 523        return -1;
 524cmpfail:
 525        return 1;
 526#ifdef RSEQ_COMPARE_TWICE
 527error1:
 528        rseq_bug("cpu_id comparison failed");
 529error2:
 530        rseq_bug("expected value comparison failed");
 531#endif
 532}
 533
 534/* x86-64 is TSO. */
 535static inline __attribute__((always_inline))
 536int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 537                                         void *dst, void *src, size_t len,
 538                                         intptr_t newv, int cpu)
 539{
 540        return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
 541                                            newv, cpu);
 542}
 543
 544#endif /* !RSEQ_SKIP_FASTPATH */
 545
 546#elif __i386__
 547
 548#define rseq_smp_mb()   \
 549        __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 550#define rseq_smp_rmb()  \
 551        __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 552#define rseq_smp_wmb()  \
 553        __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 554
 555#define rseq_smp_load_acquire(p)                                        \
 556__extension__ ({                                                        \
 557        __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
 558        rseq_smp_mb();                                                  \
 559        ____p1;                                                         \
 560})
 561
 562#define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
 563
 564#define rseq_smp_store_release(p, v)                                    \
 565do {                                                                    \
 566        rseq_smp_mb();                                                  \
 567        RSEQ_WRITE_ONCE(*p, v);                                         \
 568} while (0)
 569
 570#ifdef RSEQ_SKIP_FASTPATH
 571#include "rseq-skip.h"
 572#else /* !RSEQ_SKIP_FASTPATH */
 573
 574/*
 575 * Use eax as scratch register and take memory operands as input to
 576 * lessen register pressure. Especially needed when compiling in O0.
 577 */
 578#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
 579                                start_ip, post_commit_offset, abort_ip) \
 580                ".pushsection __rseq_cs, \"aw\"\n\t"                    \
 581                ".balign 32\n\t"                                        \
 582                __rseq_str(label) ":\n\t"                               \
 583                ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 584                ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
 585                ".popsection\n\t"                                       \
 586                ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 587                ".long " __rseq_str(label) "b, 0x0\n\t"                 \
 588                ".popsection\n\t"
 589
 590#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 591        __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 592                                (post_commit_ip - start_ip), abort_ip)
 593
 594/*
 595 * Exit points of a rseq critical section consist of all instructions outside
 596 * of the critical section where a critical section can either branch to or
 597 * reach through the normal course of its execution. The abort IP and the
 598 * post-commit IP are already part of the __rseq_cs section and should not be
 599 * explicitly defined as additional exit points. Knowing all exit points is
 600 * useful to assist debuggers stepping over the critical section.
 601 */
 602#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 603                ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
 604                ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
 605                ".popsection\n\t"
 606
 607#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 608                RSEQ_INJECT_ASM(1)                                      \
 609                "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"   \
 610                __rseq_str(label) ":\n\t"
 611
 612#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 613                RSEQ_INJECT_ASM(2)                                      \
 614                "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
 615                "jnz " __rseq_str(label) "\n\t"
 616
 617#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 618                ".pushsection __rseq_failure, \"ax\"\n\t"               \
 619                /* Disassembler-friendly signature: ud1 <sig>,%edi. */  \
 620                ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 621                ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 622                __rseq_str(label) ":\n\t"                               \
 623                teardown                                                \
 624                "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 625                ".popsection\n\t"
 626
 627#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 628                ".pushsection __rseq_failure, \"ax\"\n\t"               \
 629                __rseq_str(label) ":\n\t"                               \
 630                teardown                                                \
 631                "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
 632                ".popsection\n\t"
 633
 634static inline __attribute__((always_inline))
 635int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 636{
 637        RSEQ_INJECT_C(9)
 638
 639        __asm__ __volatile__ goto (
 640                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 641                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 642#ifdef RSEQ_COMPARE_TWICE
 643                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 644                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 645#endif
 646                /* Start rseq by storing table entry pointer into rseq_cs. */
 647                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 648                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 649                RSEQ_INJECT_ASM(3)
 650                "cmpl %[v], %[expect]\n\t"
 651                "jnz %l[cmpfail]\n\t"
 652                RSEQ_INJECT_ASM(4)
 653#ifdef RSEQ_COMPARE_TWICE
 654                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 655                "cmpl %[v], %[expect]\n\t"
 656                "jnz %l[error2]\n\t"
 657#endif
 658                /* final store */
 659                "movl %[newv], %[v]\n\t"
 660                "2:\n\t"
 661                RSEQ_INJECT_ASM(5)
 662                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 663                : /* gcc asm goto does not allow outputs */
 664                : [cpu_id]              "r" (cpu),
 665                  [rseq_abi]            "r" (&__rseq_abi),
 666                  [v]                   "m" (*v),
 667                  [expect]              "r" (expect),
 668                  [newv]                "r" (newv)
 669                : "memory", "cc", "eax"
 670                  RSEQ_INJECT_CLOBBER
 671                : abort, cmpfail
 672#ifdef RSEQ_COMPARE_TWICE
 673                  , error1, error2
 674#endif
 675        );
 676        return 0;
 677abort:
 678        RSEQ_INJECT_FAILED
 679        return -1;
 680cmpfail:
 681        return 1;
 682#ifdef RSEQ_COMPARE_TWICE
 683error1:
 684        rseq_bug("cpu_id comparison failed");
 685error2:
 686        rseq_bug("expected value comparison failed");
 687#endif
 688}
 689
 690/*
 691 * Compare @v against @expectnot. When it does _not_ match, load @v
 692 * into @load, and store the content of *@v + voffp into @v.
 693 */
 694static inline __attribute__((always_inline))
 695int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 696                               off_t voffp, intptr_t *load, int cpu)
 697{
 698        RSEQ_INJECT_C(9)
 699
 700        __asm__ __volatile__ goto (
 701                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 702                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 703#ifdef RSEQ_COMPARE_TWICE
 704                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 705                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 706#endif
 707                /* Start rseq by storing table entry pointer into rseq_cs. */
 708                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 709                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 710                RSEQ_INJECT_ASM(3)
 711                "movl %[v], %%ebx\n\t"
 712                "cmpl %%ebx, %[expectnot]\n\t"
 713                "je %l[cmpfail]\n\t"
 714                RSEQ_INJECT_ASM(4)
 715#ifdef RSEQ_COMPARE_TWICE
 716                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 717                "movl %[v], %%ebx\n\t"
 718                "cmpl %%ebx, %[expectnot]\n\t"
 719                "je %l[error2]\n\t"
 720#endif
 721                "movl %%ebx, %[load]\n\t"
 722                "addl %[voffp], %%ebx\n\t"
 723                "movl (%%ebx), %%ebx\n\t"
 724                /* final store */
 725                "movl %%ebx, %[v]\n\t"
 726                "2:\n\t"
 727                RSEQ_INJECT_ASM(5)
 728                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 729                : /* gcc asm goto does not allow outputs */
 730                : [cpu_id]              "r" (cpu),
 731                  [rseq_abi]            "r" (&__rseq_abi),
 732                  /* final store input */
 733                  [v]                   "m" (*v),
 734                  [expectnot]           "r" (expectnot),
 735                  [voffp]               "ir" (voffp),
 736                  [load]                "m" (*load)
 737                : "memory", "cc", "eax", "ebx"
 738                  RSEQ_INJECT_CLOBBER
 739                : abort, cmpfail
 740#ifdef RSEQ_COMPARE_TWICE
 741                  , error1, error2
 742#endif
 743        );
 744        return 0;
 745abort:
 746        RSEQ_INJECT_FAILED
 747        return -1;
 748cmpfail:
 749        return 1;
 750#ifdef RSEQ_COMPARE_TWICE
 751error1:
 752        rseq_bug("cpu_id comparison failed");
 753error2:
 754        rseq_bug("expected value comparison failed");
 755#endif
 756}
 757
 758static inline __attribute__((always_inline))
 759int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 760{
 761        RSEQ_INJECT_C(9)
 762
 763        __asm__ __volatile__ goto (
 764                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 765#ifdef RSEQ_COMPARE_TWICE
 766                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 767#endif
 768                /* Start rseq by storing table entry pointer into rseq_cs. */
 769                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 770                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 771                RSEQ_INJECT_ASM(3)
 772#ifdef RSEQ_COMPARE_TWICE
 773                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 774#endif
 775                /* final store */
 776                "addl %[count], %[v]\n\t"
 777                "2:\n\t"
 778                RSEQ_INJECT_ASM(4)
 779                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 780                : /* gcc asm goto does not allow outputs */
 781                : [cpu_id]              "r" (cpu),
 782                  [rseq_abi]            "r" (&__rseq_abi),
 783                  /* final store input */
 784                  [v]                   "m" (*v),
 785                  [count]               "ir" (count)
 786                : "memory", "cc", "eax"
 787                  RSEQ_INJECT_CLOBBER
 788                : abort
 789#ifdef RSEQ_COMPARE_TWICE
 790                  , error1
 791#endif
 792        );
 793        return 0;
 794abort:
 795        RSEQ_INJECT_FAILED
 796        return -1;
 797#ifdef RSEQ_COMPARE_TWICE
 798error1:
 799        rseq_bug("cpu_id comparison failed");
 800#endif
 801}
 802
 803static inline __attribute__((always_inline))
 804int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 805                                 intptr_t *v2, intptr_t newv2,
 806                                 intptr_t newv, int cpu)
 807{
 808        RSEQ_INJECT_C(9)
 809
 810        __asm__ __volatile__ goto (
 811                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 812                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 813#ifdef RSEQ_COMPARE_TWICE
 814                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 815                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 816#endif
 817                /* Start rseq by storing table entry pointer into rseq_cs. */
 818                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 819                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 820                RSEQ_INJECT_ASM(3)
 821                "cmpl %[v], %[expect]\n\t"
 822                "jnz %l[cmpfail]\n\t"
 823                RSEQ_INJECT_ASM(4)
 824#ifdef RSEQ_COMPARE_TWICE
 825                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 826                "cmpl %[v], %[expect]\n\t"
 827                "jnz %l[error2]\n\t"
 828#endif
 829                /* try store */
 830                "movl %[newv2], %%eax\n\t"
 831                "movl %%eax, %[v2]\n\t"
 832                RSEQ_INJECT_ASM(5)
 833                /* final store */
 834                "movl %[newv], %[v]\n\t"
 835                "2:\n\t"
 836                RSEQ_INJECT_ASM(6)
 837                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 838                : /* gcc asm goto does not allow outputs */
 839                : [cpu_id]              "r" (cpu),
 840                  [rseq_abi]            "r" (&__rseq_abi),
 841                  /* try store input */
 842                  [v2]                  "m" (*v2),
 843                  [newv2]               "m" (newv2),
 844                  /* final store input */
 845                  [v]                   "m" (*v),
 846                  [expect]              "r" (expect),
 847                  [newv]                "r" (newv)
 848                : "memory", "cc", "eax"
 849                  RSEQ_INJECT_CLOBBER
 850                : abort, cmpfail
 851#ifdef RSEQ_COMPARE_TWICE
 852                  , error1, error2
 853#endif
 854        );
 855        return 0;
 856abort:
 857        RSEQ_INJECT_FAILED
 858        return -1;
 859cmpfail:
 860        return 1;
 861#ifdef RSEQ_COMPARE_TWICE
 862error1:
 863        rseq_bug("cpu_id comparison failed");
 864error2:
 865        rseq_bug("expected value comparison failed");
 866#endif
 867}
 868
 869static inline __attribute__((always_inline))
 870int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 871                                         intptr_t *v2, intptr_t newv2,
 872                                         intptr_t newv, int cpu)
 873{
 874        RSEQ_INJECT_C(9)
 875
 876        __asm__ __volatile__ goto (
 877                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 878                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 879#ifdef RSEQ_COMPARE_TWICE
 880                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 881                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 882#endif
 883                /* Start rseq by storing table entry pointer into rseq_cs. */
 884                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 885                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 886                RSEQ_INJECT_ASM(3)
 887                "movl %[expect], %%eax\n\t"
 888                "cmpl %[v], %%eax\n\t"
 889                "jnz %l[cmpfail]\n\t"
 890                RSEQ_INJECT_ASM(4)
 891#ifdef RSEQ_COMPARE_TWICE
 892                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 893                "movl %[expect], %%eax\n\t"
 894                "cmpl %[v], %%eax\n\t"
 895                "jnz %l[error2]\n\t"
 896#endif
 897                /* try store */
 898                "movl %[newv2], %[v2]\n\t"
 899                RSEQ_INJECT_ASM(5)
 900                "lock; addl $0,-128(%%esp)\n\t"
 901                /* final store */
 902                "movl %[newv], %[v]\n\t"
 903                "2:\n\t"
 904                RSEQ_INJECT_ASM(6)
 905                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 906                : /* gcc asm goto does not allow outputs */
 907                : [cpu_id]              "r" (cpu),
 908                  [rseq_abi]            "r" (&__rseq_abi),
 909                  /* try store input */
 910                  [v2]                  "m" (*v2),
 911                  [newv2]               "r" (newv2),
 912                  /* final store input */
 913                  [v]                   "m" (*v),
 914                  [expect]              "m" (expect),
 915                  [newv]                "r" (newv)
 916                : "memory", "cc", "eax"
 917                  RSEQ_INJECT_CLOBBER
 918                : abort, cmpfail
 919#ifdef RSEQ_COMPARE_TWICE
 920                  , error1, error2
 921#endif
 922        );
 923        return 0;
 924abort:
 925        RSEQ_INJECT_FAILED
 926        return -1;
 927cmpfail:
 928        return 1;
 929#ifdef RSEQ_COMPARE_TWICE
 930error1:
 931        rseq_bug("cpu_id comparison failed");
 932error2:
 933        rseq_bug("expected value comparison failed");
 934#endif
 935
 936}
 937
 938static inline __attribute__((always_inline))
 939int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 940                              intptr_t *v2, intptr_t expect2,
 941                              intptr_t newv, int cpu)
 942{
 943        RSEQ_INJECT_C(9)
 944
 945        __asm__ __volatile__ goto (
 946                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 947                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 948#ifdef RSEQ_COMPARE_TWICE
 949                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 950                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 951                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 952#endif
 953                /* Start rseq by storing table entry pointer into rseq_cs. */
 954                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 955                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 956                RSEQ_INJECT_ASM(3)
 957                "cmpl %[v], %[expect]\n\t"
 958                "jnz %l[cmpfail]\n\t"
 959                RSEQ_INJECT_ASM(4)
 960                "cmpl %[expect2], %[v2]\n\t"
 961                "jnz %l[cmpfail]\n\t"
 962                RSEQ_INJECT_ASM(5)
 963#ifdef RSEQ_COMPARE_TWICE
 964                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 965                "cmpl %[v], %[expect]\n\t"
 966                "jnz %l[error2]\n\t"
 967                "cmpl %[expect2], %[v2]\n\t"
 968                "jnz %l[error3]\n\t"
 969#endif
 970                "movl %[newv], %%eax\n\t"
 971                /* final store */
 972                "movl %%eax, %[v]\n\t"
 973                "2:\n\t"
 974                RSEQ_INJECT_ASM(6)
 975                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 976                : /* gcc asm goto does not allow outputs */
 977                : [cpu_id]              "r" (cpu),
 978                  [rseq_abi]            "r" (&__rseq_abi),
 979                  /* cmp2 input */
 980                  [v2]                  "m" (*v2),
 981                  [expect2]             "r" (expect2),
 982                  /* final store input */
 983                  [v]                   "m" (*v),
 984                  [expect]              "r" (expect),
 985                  [newv]                "m" (newv)
 986                : "memory", "cc", "eax"
 987                  RSEQ_INJECT_CLOBBER
 988                : abort, cmpfail
 989#ifdef RSEQ_COMPARE_TWICE
 990                  , error1, error2, error3
 991#endif
 992        );
 993        return 0;
 994abort:
 995        RSEQ_INJECT_FAILED
 996        return -1;
 997cmpfail:
 998        return 1;
 999#ifdef RSEQ_COMPARE_TWICE
1000error1:

1001        rseq_bug("cpu_id comparison failed");
1002error2:
1003        rseq_bug("1st expected value comparison failed");
1004error3:
1005        rseq_bug("2nd expected value comparison failed");
1006#endif
1007}
1008
1009/* TODO: implement a faster memcpy. */
1010static inline __attribute__((always_inline))
1011int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1012                                 void *dst, void *src, size_t len,
1013                                 intptr_t newv, int cpu)
1014{
1015        uint32_t rseq_scratch[3];
1016
1017        RSEQ_INJECT_C(9)
1018
1019        __asm__ __volatile__ goto (
1020                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1021                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1022#ifdef RSEQ_COMPARE_TWICE
1023                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1024                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1025#endif
1026                "movl %[src], %[rseq_scratch0]\n\t"
1027                "movl %[dst], %[rseq_scratch1]\n\t"
1028                "movl %[len], %[rseq_scratch2]\n\t"
1029                /* Start rseq by storing table entry pointer into rseq_cs. */
1030                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1031                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1032                RSEQ_INJECT_ASM(3)
1033                "movl %[expect], %%eax\n\t"
1034                "cmpl %%eax, %[v]\n\t"
1035                "jnz 5f\n\t"
1036                RSEQ_INJECT_ASM(4)
1037#ifdef RSEQ_COMPARE_TWICE
1038                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1039                "movl %[expect], %%eax\n\t"
1040                "cmpl %%eax, %[v]\n\t"
1041                "jnz 7f\n\t"
1042#endif
1043                /* try memcpy */
1044                "test %[len], %[len]\n\t" \
1045                "jz 333f\n\t" \
1046                "222:\n\t" \
1047                "movb (%[src]), %%al\n\t" \
1048                "movb %%al, (%[dst])\n\t" \
1049                "inc %[src]\n\t" \
1050                "inc %[dst]\n\t" \
1051                "dec %[len]\n\t" \
1052                "jnz 222b\n\t" \
1053                "333:\n\t" \
1054                RSEQ_INJECT_ASM(5)
1055                "movl %[newv], %%eax\n\t"
1056                /* final store */
1057                "movl %%eax, %[v]\n\t"
1058                "2:\n\t"
1059                RSEQ_INJECT_ASM(6)
1060                /* teardown */
1061                "movl %[rseq_scratch2], %[len]\n\t"
1062                "movl %[rseq_scratch1], %[dst]\n\t"
1063                "movl %[rseq_scratch0], %[src]\n\t"
1064                RSEQ_ASM_DEFINE_ABORT(4,
1065                        "movl %[rseq_scratch2], %[len]\n\t"
1066                        "movl %[rseq_scratch1], %[dst]\n\t"
1067                        "movl %[rseq_scratch0], %[src]\n\t",
1068                        abort)
1069                RSEQ_ASM_DEFINE_CMPFAIL(5,
1070                        "movl %[rseq_scratch2], %[len]\n\t"
1071                        "movl %[rseq_scratch1], %[dst]\n\t"
1072                        "movl %[rseq_scratch0], %[src]\n\t",
1073                        cmpfail)
1074#ifdef RSEQ_COMPARE_TWICE
1075                RSEQ_ASM_DEFINE_CMPFAIL(6,
1076                        "movl %[rseq_scratch2], %[len]\n\t"
1077                        "movl %[rseq_scratch1], %[dst]\n\t"
1078                        "movl %[rseq_scratch0], %[src]\n\t",
1079                        error1)
1080                RSEQ_ASM_DEFINE_CMPFAIL(7,
1081                        "movl %[rseq_scratch2], %[len]\n\t"
1082                        "movl %[rseq_scratch1], %[dst]\n\t"
1083                        "movl %[rseq_scratch0], %[src]\n\t",
1084                        error2)
1085#endif
1086                : /* gcc asm goto does not allow outputs */
1087                : [cpu_id]              "r" (cpu),
1088                  [rseq_abi]            "r" (&__rseq_abi),
1089                  /* final store input */
1090                  [v]                   "m" (*v),
1091                  [expect]              "m" (expect),
1092                  [newv]                "m" (newv),
1093                  /* try memcpy input */
1094                  [dst]                 "r" (dst),
1095                  [src]                 "r" (src),
1096                  [len]                 "r" (len),
1097                  [rseq_scratch0]       "m" (rseq_scratch[0]),
1098                  [rseq_scratch1]       "m" (rseq_scratch[1]),
1099                  [rseq_scratch2]       "m" (rseq_scratch[2])
1100                : "memory", "cc", "eax"
1101                  RSEQ_INJECT_CLOBBER
1102                : abort, cmpfail
1103#ifdef RSEQ_COMPARE_TWICE
1104                  , error1, error2
1105#endif
1106        );
1107        return 0;
1108abort:
1109        RSEQ_INJECT_FAILED
1110        return -1;
1111cmpfail:
1112        return 1;
1113#ifdef RSEQ_COMPARE_TWICE
1114error1:
1115        rseq_bug("cpu_id comparison failed");
1116error2:
1117        rseq_bug("expected value comparison failed");
1118#endif
1119}
1120
1121/* TODO: implement a faster memcpy. */
1122static inline __attribute__((always_inline))
1123int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1124                                         void *dst, void *src, size_t len,
1125                                         intptr_t newv, int cpu)
1126{
1127        uint32_t rseq_scratch[3];
1128
1129        RSEQ_INJECT_C(9)
1130
1131        __asm__ __volatile__ goto (
1132                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1133                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1134#ifdef RSEQ_COMPARE_TWICE
1135                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1136                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1137#endif
1138                "movl %[src], %[rseq_scratch0]\n\t"
1139                "movl %[dst], %[rseq_scratch1]\n\t"
1140                "movl %[len], %[rseq_scratch2]\n\t"
1141                /* Start rseq by storing table entry pointer into rseq_cs. */
1142                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1143                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1144                RSEQ_INJECT_ASM(3)
1145                "movl %[expect], %%eax\n\t"
1146                "cmpl %%eax, %[v]\n\t"
1147                "jnz 5f\n\t"
1148                RSEQ_INJECT_ASM(4)
1149#ifdef RSEQ_COMPARE_TWICE
1150                RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1151                "movl %[expect], %%eax\n\t"
1152                "cmpl %%eax, %[v]\n\t"
1153                "jnz 7f\n\t"
1154#endif
1155                /* try memcpy */
1156                "test %[len], %[len]\n\t" \
1157                "jz 333f\n\t" \
1158                "222:\n\t" \
1159                "movb (%[src]), %%al\n\t" \
1160                "movb %%al, (%[dst])\n\t" \
1161                "inc %[src]\n\t" \
1162                "inc %[dst]\n\t" \
1163                "dec %[len]\n\t" \
1164                "jnz 222b\n\t" \
1165                "333:\n\t" \
1166                RSEQ_INJECT_ASM(5)
1167                "lock; addl $0,-128(%%esp)\n\t"
1168                "movl %[newv], %%eax\n\t"
1169                /* final store */
1170                "movl %%eax, %[v]\n\t"
1171                "2:\n\t"
1172                RSEQ_INJECT_ASM(6)
1173                /* teardown */
1174                "movl %[rseq_scratch2], %[len]\n\t"
1175                "movl %[rseq_scratch1], %[dst]\n\t"
1176                "movl %[rseq_scratch0], %[src]\n\t"
1177                RSEQ_ASM_DEFINE_ABORT(4,
1178                        "movl %[rseq_scratch2], %[len]\n\t"
1179                        "movl %[rseq_scratch1], %[dst]\n\t"
1180                        "movl %[rseq_scratch0], %[src]\n\t",
1181                        abort)
1182                RSEQ_ASM_DEFINE_CMPFAIL(5,
1183                        "movl %[rseq_scratch2], %[len]\n\t"
1184                        "movl %[rseq_scratch1], %[dst]\n\t"
1185                        "movl %[rseq_scratch0], %[src]\n\t",
1186                        cmpfail)
1187#ifdef RSEQ_COMPARE_TWICE
1188                RSEQ_ASM_DEFINE_CMPFAIL(6,
1189                        "movl %[rseq_scratch2], %[len]\n\t"
1190                        "movl %[rseq_scratch1], %[dst]\n\t"
1191                        "movl %[rseq_scratch0], %[src]\n\t",
1192                        error1)
1193                RSEQ_ASM_DEFINE_CMPFAIL(7,
1194                        "movl %[rseq_scratch2], %[len]\n\t"
1195                        "movl %[rseq_scratch1], %[dst]\n\t"
1196                        "movl %[rseq_scratch0], %[src]\n\t",
1197                        error2)
1198#endif
1199                : /* gcc asm goto does not allow outputs */
1200                : [cpu_id]              "r" (cpu),
1201                  [rseq_abi]            "r" (&__rseq_abi),
1202                  /* final store input */
1203                  [v]                   "m" (*v),
1204                  [expect]              "m" (expect),
1205                  [newv]                "m" (newv),
1206                  /* try memcpy input */
1207                  [dst]                 "r" (dst),
1208                  [src]                 "r" (src),
1209                  [len]                 "r" (len),
1210                  [rseq_scratch0]       "m" (rseq_scratch[0]),
1211                  [rseq_scratch1]       "m" (rseq_scratch[1]),
1212                  [rseq_scratch2]       "m" (rseq_scratch[2])
1213                : "memory", "cc", "eax"
1214                  RSEQ_INJECT_CLOBBER
1215                : abort, cmpfail
1216#ifdef RSEQ_COMPARE_TWICE
1217                  , error1, error2
1218#endif
1219        );
1220        return 0;
1221abort:
1222        RSEQ_INJECT_FAILED
1223        return -1;
1224cmpfail:
1225        return 1;
1226#ifdef RSEQ_COMPARE_TWICE
1227error1:
1228        rseq_bug("cpu_id comparison failed");
1229error2:
1230        rseq_bug("expected value comparison failed");
1231#endif
1232}
1233
1234#endif /* !RSEQ_SKIP_FASTPATH */
1235
1236#endif
1237