LXR linux/tools/testing/selftests/rseq/rseq-ppc.h

   1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2/*
   3 * rseq-ppc.h
   4 *
   5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
   7 */
   8
   9/*
  10 * RSEQ_SIG is used with the following trap instruction:
  11 *
  12 * powerpc-be:    0f e5 00 0b           twui   r5,11
  13 * powerpc64-le:  0b 00 e5 0f           twui   r5,11
  14 * powerpc64-be:  0f e5 00 0b           twui   r5,11
  15 */
  16
  17#define RSEQ_SIG        0x0fe5000b
  18
  19#define rseq_smp_mb()           __asm__ __volatile__ ("sync"    ::: "memory", "cc")
  20#define rseq_smp_lwsync()       __asm__ __volatile__ ("lwsync"  ::: "memory", "cc")
  21#define rseq_smp_rmb()          rseq_smp_lwsync()
  22#define rseq_smp_wmb()          rseq_smp_lwsync()
  23
  24#define rseq_smp_load_acquire(p)                                        \
  25__extension__ ({                                                        \
  26        __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  27        rseq_smp_lwsync();                                              \
  28        ____p1;                                                         \
  29})
  30
  31#define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_lwsync()
  32
  33#define rseq_smp_store_release(p, v)                                    \
  34do {                                                                    \
  35        rseq_smp_lwsync();                                              \
  36        RSEQ_WRITE_ONCE(*p, v);                                         \
  37} while (0)
  38
  39#ifdef RSEQ_SKIP_FASTPATH
  40#include "rseq-skip.h"
  41#else /* !RSEQ_SKIP_FASTPATH */
  42
  43/*
  44 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
  45 * better handle single-stepping through the restartable critical sections.
  46 */
  47
  48#ifdef __PPC64__
  49
  50#define STORE_WORD      "std "
  51#define LOAD_WORD       "ld "
  52#define LOADX_WORD      "ldx "
  53#define CMP_WORD        "cmpd "
  54
  55#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                          \
  56                        start_ip, post_commit_offset, abort_ip)                 \
  57                ".pushsection __rseq_cs, \"aw\"\n\t"                            \
  58                ".balign 32\n\t"                                                \
  59                __rseq_str(label) ":\n\t"                                       \
  60                ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"      \
  61                ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  62                ".popsection\n\t"                                               \
  63                ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"                  \
  64                ".quad " __rseq_str(label) "b\n\t"                              \
  65                ".popsection\n\t"
  66
  67#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
  68                RSEQ_INJECT_ASM(1)                                              \
  69                "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t"             \
  70                "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t"       \
  71                "rldicr %%r17, %%r17, 32, 31\n\t"                               \
  72                "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t"        \
  73                "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"            \
  74                "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t"                     \
  75                __rseq_str(label) ":\n\t"
  76
  77/*
  78 * Exit points of a rseq critical section consist of all instructions outside
  79 * of the critical section where a critical section can either branch to or
  80 * reach through the normal course of its execution. The abort IP and the
  81 * post-commit IP are already part of the __rseq_cs section and should not be
  82 * explicitly defined as additional exit points. Knowing all exit points is
  83 * useful to assist debuggers stepping over the critical section.
  84 */
  85#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  86                ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  87                ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  88                ".popsection\n\t"
  89
  90#else /* #ifdef __PPC64__ */
  91
  92#define STORE_WORD      "stw "
  93#define LOAD_WORD       "lwz "
  94#define LOADX_WORD      "lwzx "
  95#define CMP_WORD        "cmpw "
  96
  97#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                          \
  98                        start_ip, post_commit_offset, abort_ip)                 \
  99                ".pushsection __rseq_cs, \"aw\"\n\t"                            \
 100                ".balign 32\n\t"                                                \
 101                __rseq_str(label) ":\n\t"                                       \
 102                ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"      \
 103                /* 32-bit only supported on BE */                               \
 104                ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
 105                ".popsection\n\t"                                       \
 106                ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 107                ".long 0x0, " __rseq_str(label) "b\n\t"                 \
 108                ".popsection\n\t"
 109
 110/*
 111 * Exit points of a rseq critical section consist of all instructions outside
 112 * of the critical section where a critical section can either branch to or
 113 * reach through the normal course of its execution. The abort IP and the
 114 * post-commit IP are already part of the __rseq_cs section and should not be
 115 * explicitly defined as additional exit points. Knowing all exit points is
 116 * useful to assist debuggers stepping over the critical section.
 117 */
 118#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                           \
 119                ".pushsection __rseq_exit_point_array, \"aw\"\n\t"              \
 120                /* 32-bit only supported on BE */                               \
 121                ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
 122                ".popsection\n\t"
 123
 124#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
 125                RSEQ_INJECT_ASM(1)                                              \
 126                "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t"                  \
 127                "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"           \
 128                "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t"                     \
 129                __rseq_str(label) ":\n\t"
 130
 131#endif /* #ifdef __PPC64__ */
 132
 133#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)        \
 134                __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 135                                        (post_commit_ip - start_ip), abort_ip)
 136
 137#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)                      \
 138                RSEQ_INJECT_ASM(2)                                              \
 139                "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t"              \
 140                "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t"                \
 141                "bne- cr7, " __rseq_str(label) "\n\t"
 142
 143#define RSEQ_ASM_DEFINE_ABORT(label, abort_label)                               \
 144                ".pushsection __rseq_failure, \"ax\"\n\t"                       \
 145                ".long " __rseq_str(RSEQ_SIG) "\n\t"                            \
 146                __rseq_str(label) ":\n\t"                                       \
 147                "b %l[" __rseq_str(abort_label) "]\n\t"                         \
 148                ".popsection\n\t"
 149
 150/*
 151 * RSEQ_ASM_OPs: asm operations for rseq
 152 *      RSEQ_ASM_OP_R_*: has hard-code registers in it
 153 *      RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
 154 */
 155#define RSEQ_ASM_OP_CMPEQ(var, expect, label)                                   \
 156                LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                   \
 157                CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t"            \
 158                "bne- cr7, " __rseq_str(label) "\n\t"
 159
 160#define RSEQ_ASM_OP_CMPNE(var, expectnot, label)                                \
 161                LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                   \
 162                CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t"         \
 163                "beq- cr7, " __rseq_str(label) "\n\t"
 164
 165#define RSEQ_ASM_OP_STORE(value, var)                                           \
 166                STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
 167
 168/* Load @var to r17 */
 169#define RSEQ_ASM_OP_R_LOAD(var)                                                 \
 170                LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
 171
 172/* Store r17 to @var */
 173#define RSEQ_ASM_OP_R_STORE(var)                                                \
 174                STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
 175
 176/* Add @count to r17 */
 177#define RSEQ_ASM_OP_R_ADD(count)                                                \
 178                "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
 179
 180/* Load (r17 + voffp) to r17 */
 181#define RSEQ_ASM_OP_R_LOADX(voffp)                                              \
 182                LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
 183
 184/* TODO: implement a faster memcpy. */
 185#define RSEQ_ASM_OP_R_MEMCPY() \
 186                "cmpdi %%r19, 0\n\t" \
 187                "beq 333f\n\t" \
 188                "addi %%r20, %%r20, -1\n\t" \
 189                "addi %%r21, %%r21, -1\n\t" \
 190                "222:\n\t" \
 191                "lbzu %%r18, 1(%%r20)\n\t" \
 192                "stbu %%r18, 1(%%r21)\n\t" \
 193                "addi %%r19, %%r19, -1\n\t" \
 194                "cmpdi %%r19, 0\n\t" \
 195                "bne 222b\n\t" \
 196                "333:\n\t" \
 197
 198#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)                       \
 199                STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                  \
 200                __rseq_str(post_commit_label) ":\n\t"
 201
 202#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)                  \
 203                STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
 204                __rseq_str(post_commit_label) ":\n\t"
 205
 206static inline __attribute__((always_inline))
 207int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 208{
 209        RSEQ_INJECT_C(9)
 210
 211        __asm__ __volatile__ goto (
 212                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 213                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 214#ifdef RSEQ_COMPARE_TWICE
 215                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 216                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 217#endif
 218                /* Start rseq by storing table entry pointer into rseq_cs. */
 219                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 220                /* cmp cpuid */
 221                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 222                RSEQ_INJECT_ASM(3)
 223                /* cmp @v equal to @expect */
 224                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 225                RSEQ_INJECT_ASM(4)
 226#ifdef RSEQ_COMPARE_TWICE
 227                /* cmp cpuid */
 228                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 229                /* cmp @v equal to @expect */
 230                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 231#endif
 232                /* final store */
 233                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 234                RSEQ_INJECT_ASM(5)
 235                RSEQ_ASM_DEFINE_ABORT(4, abort)
 236                : /* gcc asm goto does not allow outputs */
 237                : [cpu_id]              "r" (cpu),
 238                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 239                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 240                  [v]                   "m" (*v),
 241                  [expect]              "r" (expect),
 242                  [newv]                "r" (newv)
 243                  RSEQ_INJECT_INPUT
 244                : "memory", "cc", "r17"
 245                  RSEQ_INJECT_CLOBBER
 246                : abort, cmpfail
 247#ifdef RSEQ_COMPARE_TWICE
 248                  , error1, error2
 249#endif
 250        );
 251        return 0;
 252abort:
 253        RSEQ_INJECT_FAILED
 254        return -1;
 255cmpfail:
 256        return 1;
 257#ifdef RSEQ_COMPARE_TWICE
 258error1:
 259        rseq_bug("cpu_id comparison failed");
 260error2:
 261        rseq_bug("expected value comparison failed");
 262#endif
 263}
 264
 265static inline __attribute__((always_inline))
 266int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 267                               off_t voffp, intptr_t *load, int cpu)
 268{
 269        RSEQ_INJECT_C(9)
 270
 271        __asm__ __volatile__ goto (
 272                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 273                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 274#ifdef RSEQ_COMPARE_TWICE
 275                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 276                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 277#endif
 278                /* Start rseq by storing table entry pointer into rseq_cs. */
 279                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 280                /* cmp cpuid */
 281                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 282                RSEQ_INJECT_ASM(3)
 283                /* cmp @v not equal to @expectnot */
 284                RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
 285                RSEQ_INJECT_ASM(4)
 286#ifdef RSEQ_COMPARE_TWICE
 287                /* cmp cpuid */
 288                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 289                /* cmp @v not equal to @expectnot */
 290                RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
 291#endif
 292                /* load the value of @v */
 293                RSEQ_ASM_OP_R_LOAD(v)
 294                /* store it in @load */
 295                RSEQ_ASM_OP_R_STORE(load)
 296                /* dereference voffp(v) */
 297                RSEQ_ASM_OP_R_LOADX(voffp)
 298                /* final store the value at voffp(v) */
 299                RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
 300                RSEQ_INJECT_ASM(5)
 301                RSEQ_ASM_DEFINE_ABORT(4, abort)
 302                : /* gcc asm goto does not allow outputs */
 303                : [cpu_id]              "r" (cpu),
 304                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 305                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 306                  /* final store input */
 307                  [v]                   "m" (*v),
 308                  [expectnot]           "r" (expectnot),
 309                  [voffp]               "b" (voffp),
 310                  [load]                "m" (*load)
 311                  RSEQ_INJECT_INPUT
 312                : "memory", "cc", "r17"
 313                  RSEQ_INJECT_CLOBBER
 314                : abort, cmpfail
 315#ifdef RSEQ_COMPARE_TWICE
 316                  , error1, error2
 317#endif
 318        );
 319        return 0;
 320abort:
 321        RSEQ_INJECT_FAILED
 322        return -1;
 323cmpfail:
 324        return 1;
 325#ifdef RSEQ_COMPARE_TWICE
 326error1:
 327        rseq_bug("cpu_id comparison failed");
 328error2:
 329        rseq_bug("expected value comparison failed");
 330#endif
 331}
 332
 333static inline __attribute__((always_inline))
 334int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 335{
 336        RSEQ_INJECT_C(9)
 337
 338        __asm__ __volatile__ goto (
 339                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 340#ifdef RSEQ_COMPARE_TWICE
 341                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 342#endif
 343                /* Start rseq by storing table entry pointer into rseq_cs. */
 344                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 345                /* cmp cpuid */
 346                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 347                RSEQ_INJECT_ASM(3)
 348#ifdef RSEQ_COMPARE_TWICE
 349                /* cmp cpuid */
 350                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 351#endif
 352                /* load the value of @v */
 353                RSEQ_ASM_OP_R_LOAD(v)
 354                /* add @count to it */
 355                RSEQ_ASM_OP_R_ADD(count)
 356                /* final store */
 357                RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
 358                RSEQ_INJECT_ASM(4)
 359                RSEQ_ASM_DEFINE_ABORT(4, abort)
 360                : /* gcc asm goto does not allow outputs */
 361                : [cpu_id]              "r" (cpu),
 362                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 363                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 364                  /* final store input */
 365                  [v]                   "m" (*v),
 366                  [count]               "r" (count)
 367                  RSEQ_INJECT_INPUT
 368                : "memory", "cc", "r17"
 369                  RSEQ_INJECT_CLOBBER
 370                : abort
 371#ifdef RSEQ_COMPARE_TWICE
 372                  , error1
 373#endif
 374        );
 375        return 0;
 376abort:
 377        RSEQ_INJECT_FAILED
 378        return -1;
 379#ifdef RSEQ_COMPARE_TWICE
 380error1:
 381        rseq_bug("cpu_id comparison failed");
 382#endif
 383}
 384
 385static inline __attribute__((always_inline))
 386int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 387                                 intptr_t *v2, intptr_t newv2,
 388                                 intptr_t newv, int cpu)
 389{
 390        RSEQ_INJECT_C(9)
 391
 392        __asm__ __volatile__ goto (
 393                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 394                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 395#ifdef RSEQ_COMPARE_TWICE
 396                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 397                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 398#endif
 399                /* Start rseq by storing table entry pointer into rseq_cs. */
 400                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 401                /* cmp cpuid */
 402                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 403                RSEQ_INJECT_ASM(3)
 404                /* cmp @v equal to @expect */
 405                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 406                RSEQ_INJECT_ASM(4)
 407#ifdef RSEQ_COMPARE_TWICE
 408                /* cmp cpuid */
 409                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 410                /* cmp @v equal to @expect */
 411                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 412#endif
 413                /* try store */
 414                RSEQ_ASM_OP_STORE(newv2, v2)
 415                RSEQ_INJECT_ASM(5)
 416                /* final store */
 417                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 418                RSEQ_INJECT_ASM(6)
 419                RSEQ_ASM_DEFINE_ABORT(4, abort)
 420                : /* gcc asm goto does not allow outputs */
 421                : [cpu_id]              "r" (cpu),
 422                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 423                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 424                  /* try store input */
 425                  [v2]                  "m" (*v2),
 426                  [newv2]               "r" (newv2),
 427                  /* final store input */
 428                  [v]                   "m" (*v),
 429                  [expect]              "r" (expect),
 430                  [newv]                "r" (newv)
 431                  RSEQ_INJECT_INPUT
 432                : "memory", "cc", "r17"
 433                  RSEQ_INJECT_CLOBBER
 434                : abort, cmpfail
 435#ifdef RSEQ_COMPARE_TWICE
 436                  , error1, error2
 437#endif
 438        );
 439        return 0;
 440abort:
 441        RSEQ_INJECT_FAILED
 442        return -1;
 443cmpfail:
 444        return 1;
 445#ifdef RSEQ_COMPARE_TWICE
 446error1:
 447        rseq_bug("cpu_id comparison failed");
 448error2:
 449        rseq_bug("expected value comparison failed");
 450#endif
 451}
 452
 453static inline __attribute__((always_inline))
 454int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 455                                         intptr_t *v2, intptr_t newv2,
 456                                         intptr_t newv, int cpu)
 457{
 458        RSEQ_INJECT_C(9)
 459
 460        __asm__ __volatile__ goto (
 461                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 462                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 463#ifdef RSEQ_COMPARE_TWICE
 464                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 465                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 466#endif
 467                /* Start rseq by storing table entry pointer into rseq_cs. */
 468                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 469                /* cmp cpuid */
 470                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 471                RSEQ_INJECT_ASM(3)
 472                /* cmp @v equal to @expect */
 473                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 474                RSEQ_INJECT_ASM(4)
 475#ifdef RSEQ_COMPARE_TWICE
 476                /* cmp cpuid */
 477                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 478                /* cmp @v equal to @expect */
 479                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 480#endif
 481                /* try store */
 482                RSEQ_ASM_OP_STORE(newv2, v2)
 483                RSEQ_INJECT_ASM(5)
 484                /* for 'release' */
 485                "lwsync\n\t"
 486                /* final store */
 487                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 488                RSEQ_INJECT_ASM(6)
 489                RSEQ_ASM_DEFINE_ABORT(4, abort)
 490                : /* gcc asm goto does not allow outputs */
 491                : [cpu_id]              "r" (cpu),
 492                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 493                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 494                  /* try store input */
 495                  [v2]                  "m" (*v2),
 496                  [newv2]               "r" (newv2),
 497                  /* final store input */
 498                  [v]                   "m" (*v),
 499                  [expect]              "r" (expect),
 500                  [newv]                "r" (newv)
 501                  RSEQ_INJECT_INPUT
 502                : "memory", "cc", "r17"
 503                  RSEQ_INJECT_CLOBBER
 504                : abort, cmpfail
 505#ifdef RSEQ_COMPARE_TWICE
 506                  , error1, error2
 507#endif
 508        );
 509        return 0;
 510abort:
 511        RSEQ_INJECT_FAILED
 512        return -1;
 513cmpfail:
 514        return 1;
 515#ifdef RSEQ_COMPARE_TWICE
 516error1:
 517        rseq_bug("cpu_id comparison failed");
 518error2:
 519        rseq_bug("expected value comparison failed");
 520#endif
 521}
 522
 523static inline __attribute__((always_inline))
 524int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 525                              intptr_t *v2, intptr_t expect2,
 526                              intptr_t newv, int cpu)
 527{
 528        RSEQ_INJECT_C(9)
 529
 530        __asm__ __volatile__ goto (
 531                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 532                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 533#ifdef RSEQ_COMPARE_TWICE
 534                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 535                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 536                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 537#endif
 538                /* Start rseq by storing table entry pointer into rseq_cs. */
 539                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 540                /* cmp cpuid */
 541                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 542                RSEQ_INJECT_ASM(3)
 543                /* cmp @v equal to @expect */
 544                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 545                RSEQ_INJECT_ASM(4)
 546                /* cmp @v2 equal to @expct2 */
 547                RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
 548                RSEQ_INJECT_ASM(5)
 549#ifdef RSEQ_COMPARE_TWICE
 550                /* cmp cpuid */
 551                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 552                /* cmp @v equal to @expect */
 553                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 554                /* cmp @v2 equal to @expct2 */
 555                RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
 556#endif
 557                /* final store */
 558                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 559                RSEQ_INJECT_ASM(6)
 560                RSEQ_ASM_DEFINE_ABORT(4, abort)
 561                : /* gcc asm goto does not allow outputs */
 562                : [cpu_id]              "r" (cpu),
 563                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 564                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 565                  /* cmp2 input */
 566                  [v2]                  "m" (*v2),
 567                  [expect2]             "r" (expect2),
 568                  /* final store input */
 569                  [v]                   "m" (*v),
 570                  [expect]              "r" (expect),
 571                  [newv]                "r" (newv)
 572                  RSEQ_INJECT_INPUT
 573                : "memory", "cc", "r17"
 574                  RSEQ_INJECT_CLOBBER
 575                : abort, cmpfail
 576#ifdef RSEQ_COMPARE_TWICE
 577                  , error1, error2, error3
 578#endif
 579        );
 580        return 0;
 581abort:
 582        RSEQ_INJECT_FAILED
 583        return -1;
 584cmpfail:
 585        return 1;
 586#ifdef RSEQ_COMPARE_TWICE
 587error1:
 588        rseq_bug("cpu_id comparison failed");
 589error2:
 590        rseq_bug("1st expected value comparison failed");
 591error3:
 592        rseq_bug("2nd expected value comparison failed");
 593#endif
 594}
 595
 596static inline __attribute__((always_inline))
 597int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 598                                 void *dst, void *src, size_t len,
 599                                 intptr_t newv, int cpu)
 600{
 601        RSEQ_INJECT_C(9)
 602
 603        __asm__ __volatile__ goto (
 604                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 605                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 606#ifdef RSEQ_COMPARE_TWICE
 607                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 608                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 609#endif
 610                /* setup for mempcy */
 611                "mr %%r19, %[len]\n\t"
 612                "mr %%r20, %[src]\n\t"
 613                "mr %%r21, %[dst]\n\t"
 614                /* Start rseq by storing table entry pointer into rseq_cs. */
 615                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 616                /* cmp cpuid */
 617                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 618                RSEQ_INJECT_ASM(3)
 619                /* cmp @v equal to @expect */
 620                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 621                RSEQ_INJECT_ASM(4)
 622#ifdef RSEQ_COMPARE_TWICE
 623                /* cmp cpuid */
 624                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 625                /* cmp @v equal to @expect */
 626                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 627#endif
 628                /* try memcpy */
 629                RSEQ_ASM_OP_R_MEMCPY()
 630                RSEQ_INJECT_ASM(5)
 631                /* final store */
 632                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 633                RSEQ_INJECT_ASM(6)
 634                /* teardown */
 635                RSEQ_ASM_DEFINE_ABORT(4, abort)
 636                : /* gcc asm goto does not allow outputs */
 637                : [cpu_id]              "r" (cpu),
 638                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 639                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 640                  /* final store input */
 641                  [v]                   "m" (*v),
 642                  [expect]              "r" (expect),
 643                  [newv]                "r" (newv),
 644                  /* try memcpy input */
 645                  [dst]                 "r" (dst),
 646                  [src]                 "r" (src),
 647                  [len]                 "r" (len)
 648                  RSEQ_INJECT_INPUT
 649                : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
 650                  RSEQ_INJECT_CLOBBER
 651                : abort, cmpfail
 652#ifdef RSEQ_COMPARE_TWICE
 653                  , error1, error2
 654#endif
 655        );
 656        return 0;
 657abort:
 658        RSEQ_INJECT_FAILED
 659        return -1;
 660cmpfail:
 661        return 1;
 662#ifdef RSEQ_COMPARE_TWICE
 663error1:
 664        rseq_bug("cpu_id comparison failed");
 665error2:
 666        rseq_bug("expected value comparison failed");
 667#endif
 668}
 669
 670static inline __attribute__((always_inline))
 671int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 672                                         void *dst, void *src, size_t len,
 673                                         intptr_t newv, int cpu)
 674{
 675        RSEQ_INJECT_C(9)
 676
 677        __asm__ __volatile__ goto (
 678                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 679                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 680#ifdef RSEQ_COMPARE_TWICE
 681                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 682                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 683#endif
 684                /* setup for mempcy */
 685                "mr %%r19, %[len]\n\t"
 686                "mr %%r20, %[src]\n\t"
 687                "mr %%r21, %[dst]\n\t"
 688                /* Start rseq by storing table entry pointer into rseq_cs. */
 689                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 690                /* cmp cpuid */
 691                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 692                RSEQ_INJECT_ASM(3)
 693                /* cmp @v equal to @expect */
 694                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 695                RSEQ_INJECT_ASM(4)
 696#ifdef RSEQ_COMPARE_TWICE
 697                /* cmp cpuid */
 698                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 699                /* cmp @v equal to @expect */
 700                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 701#endif
 702                /* try memcpy */
 703                RSEQ_ASM_OP_R_MEMCPY()
 704                RSEQ_INJECT_ASM(5)
 705                /* for 'release' */
 706                "lwsync\n\t"
 707                /* final store */
 708                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 709                RSEQ_INJECT_ASM(6)
 710                /* teardown */
 711                RSEQ_ASM_DEFINE_ABORT(4, abort)
 712                : /* gcc asm goto does not allow outputs */
 713                : [cpu_id]              "r" (cpu),
 714                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 715                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 716                  /* final store input */
 717                  [v]                   "m" (*v),
 718                  [expect]              "r" (expect),
 719                  [newv]                "r" (newv),
 720                  /* try memcpy input */
 721                  [dst]                 "r" (dst),
 722                  [src]                 "r" (src),
 723                  [len]                 "r" (len)
 724                  RSEQ_INJECT_INPUT
 725                : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
 726                  RSEQ_INJECT_CLOBBER
 727                : abort, cmpfail
 728#ifdef RSEQ_COMPARE_TWICE
 729                  , error1, error2
 730#endif
 731        );
 732        return 0;
 733abort:
 734        RSEQ_INJECT_FAILED
 735        return -1;
 736cmpfail:
 737        return 1;
 738#ifdef RSEQ_COMPARE_TWICE
 739error1:
 740        rseq_bug("cpu_id comparison failed");
 741error2:
 742        rseq_bug("expected value comparison failed");
 743#endif
 744}
 745
 746#undef STORE_WORD
 747#undef LOAD_WORD
 748#undef LOADX_WORD
 749#undef CMP_WORD
 750
 751#endif /* !RSEQ_SKIP_FASTPATH */
 752