LXR linux/tools/testing/selftests/rseq/rseq-ppc.h

   1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2/*
   3 * rseq-ppc.h
   4 *
   5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
   7 */
   8
   9/*
  10 * RSEQ_SIG is used with the following trap instruction:
  11 *
  12 * powerpc-be:    0f e5 00 0b           twui   r5,11
  13 * powerpc64-le:  0b 00 e5 0f           twui   r5,11
  14 * powerpc64-be:  0f e5 00 0b           twui   r5,11
  15 */
  16
  17#define RSEQ_SIG        0x0fe5000b
  18
  19#define rseq_smp_mb()           __asm__ __volatile__ ("sync"    ::: "memory", "cc")
  20#define rseq_smp_lwsync()       __asm__ __volatile__ ("lwsync"  ::: "memory", "cc")
  21#define rseq_smp_rmb()          rseq_smp_lwsync()
  22#define rseq_smp_wmb()          rseq_smp_lwsync()
  23
  24#define rseq_smp_load_acquire(p)                                        \
  25__extension__ ({                                                        \
  26        __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  27        rseq_smp_lwsync();                                              \
  28        ____p1;                                                         \
  29})
  30
  31#define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_lwsync()
  32
  33#define rseq_smp_store_release(p, v)                                    \
  34do {                                                                    \
  35        rseq_smp_lwsync();                                              \
  36        RSEQ_WRITE_ONCE(*p, v);                                         \
  37} while (0)
  38
  39#ifdef RSEQ_SKIP_FASTPATH
  40#include "rseq-skip.h"
  41#else /* !RSEQ_SKIP_FASTPATH */
  42
  43/*
  44 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
  45 * better handle single-stepping through the restartable critical sections.
  46 */
  47
  48#ifdef __PPC64__
  49
  50#define RSEQ_STORE_LONG(arg)    "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "    /* To memory ("m" constraint) */
  51#define RSEQ_STORE_INT(arg)     "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "    /* To memory ("m" constraint) */
  52#define RSEQ_LOAD_LONG(arg)     "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "     /* From memory ("m" constraint) */
  53#define RSEQ_LOAD_INT(arg)      "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "    /* From memory ("m" constraint) */
  54#define RSEQ_LOADX_LONG         "ldx "                                                  /* From base register ("b" constraint) */
  55#define RSEQ_CMP_LONG           "cmpd "
  56#define RSEQ_CMP_LONG_INT       "cmpdi "
  57
  58#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                          \
  59                        start_ip, post_commit_offset, abort_ip)                 \
  60                ".pushsection __rseq_cs, \"aw\"\n\t"                            \
  61                ".balign 32\n\t"                                                \
  62                __rseq_str(label) ":\n\t"                                       \
  63                ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"      \
  64                ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  65                ".popsection\n\t"                                               \
  66                ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"                  \
  67                ".quad " __rseq_str(label) "b\n\t"                              \
  68                ".popsection\n\t"
  69
  70#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
  71                RSEQ_INJECT_ASM(1)                                              \
  72                "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t"             \
  73                "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t"       \
  74                "rldicr %%r17, %%r17, 32, 31\n\t"                               \
  75                "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t"        \
  76                "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"            \
  77                "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t"                     \
  78                __rseq_str(label) ":\n\t"
  79
  80/*
  81 * Exit points of a rseq critical section consist of all instructions outside
  82 * of the critical section where a critical section can either branch to or
  83 * reach through the normal course of its execution. The abort IP and the
  84 * post-commit IP are already part of the __rseq_cs section and should not be
  85 * explicitly defined as additional exit points. Knowing all exit points is
  86 * useful to assist debuggers stepping over the critical section.
  87 */
  88#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  89                ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  90                ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  91                ".popsection\n\t"
  92
  93#else /* #ifdef __PPC64__ */
  94
  95#define RSEQ_STORE_LONG(arg)    "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "    /* To memory ("m" constraint) */
  96#define RSEQ_STORE_INT(arg)     RSEQ_STORE_LONG(arg)                                    /* To memory ("m" constraint) */
  97#define RSEQ_LOAD_LONG(arg)     "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "    /* From memory ("m" constraint) */
  98#define RSEQ_LOAD_INT(arg)      RSEQ_LOAD_LONG(arg)                                     /* From memory ("m" constraint) */
  99#define RSEQ_LOADX_LONG         "lwzx "                                                 /* From base register ("b" constraint) */
 100#define RSEQ_CMP_LONG           "cmpw "
 101#define RSEQ_CMP_LONG_INT       "cmpwi "
 102
 103#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                          \
 104                        start_ip, post_commit_offset, abort_ip)                 \
 105                ".pushsection __rseq_cs, \"aw\"\n\t"                            \
 106                ".balign 32\n\t"                                                \
 107                __rseq_str(label) ":\n\t"                                       \
 108                ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"      \
 109                /* 32-bit only supported on BE */                               \
 110                ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
 111                ".popsection\n\t"                                       \
 112                ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 113                ".long 0x0, " __rseq_str(label) "b\n\t"                 \
 114                ".popsection\n\t"
 115
 116/*
 117 * Exit points of a rseq critical section consist of all instructions outside
 118 * of the critical section where a critical section can either branch to or
 119 * reach through the normal course of its execution. The abort IP and the
 120 * post-commit IP are already part of the __rseq_cs section and should not be
 121 * explicitly defined as additional exit points. Knowing all exit points is
 122 * useful to assist debuggers stepping over the critical section.
 123 */
 124#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                           \
 125                ".pushsection __rseq_exit_point_array, \"aw\"\n\t"              \
 126                /* 32-bit only supported on BE */                               \
 127                ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
 128                ".popsection\n\t"
 129
 130#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
 131                RSEQ_INJECT_ASM(1)                                              \
 132                "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t"                  \
 133                "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"           \
 134                RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
 135                __rseq_str(label) ":\n\t"
 136
 137#endif /* #ifdef __PPC64__ */
 138
 139#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)        \
 140                __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 141                                        (post_commit_ip - start_ip), abort_ip)
 142
 143#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)                      \
 144                RSEQ_INJECT_ASM(2)                                              \
 145                RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
 146                "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t"                \
 147                "bne- cr7, " __rseq_str(label) "\n\t"
 148
 149#define RSEQ_ASM_DEFINE_ABORT(label, abort_label)                               \
 150                ".pushsection __rseq_failure, \"ax\"\n\t"                       \
 151                ".long " __rseq_str(RSEQ_SIG) "\n\t"                            \
 152                __rseq_str(label) ":\n\t"                                       \
 153                "b %l[" __rseq_str(abort_label) "]\n\t"                         \
 154                ".popsection\n\t"
 155
 156/*
 157 * RSEQ_ASM_OPs: asm operations for rseq
 158 *      RSEQ_ASM_OP_R_*: has hard-code registers in it
 159 *      RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
 160 */
 161#define RSEQ_ASM_OP_CMPEQ(var, expect, label)                                   \
 162                RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"         \
 163                RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t"               \
 164                "bne- cr7, " __rseq_str(label) "\n\t"
 165
 166#define RSEQ_ASM_OP_CMPNE(var, expectnot, label)                                \
 167                RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"         \
 168                RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t"            \
 169                "beq- cr7, " __rseq_str(label) "\n\t"
 170
 171#define RSEQ_ASM_OP_STORE(value, var)                                           \
 172                RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
 173
 174/* Load @var to r17 */
 175#define RSEQ_ASM_OP_R_LOAD(var)                                                 \
 176                RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
 177
 178/* Store r17 to @var */
 179#define RSEQ_ASM_OP_R_STORE(var)                                                \
 180                RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
 181
 182/* Add @count to r17 */
 183#define RSEQ_ASM_OP_R_ADD(count)                                                \
 184                "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
 185
 186/* Load (r17 + voffp) to r17 */
 187#define RSEQ_ASM_OP_R_LOADX(voffp)                                              \
 188                RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
 189
 190/* TODO: implement a faster memcpy. */
 191#define RSEQ_ASM_OP_R_MEMCPY() \
 192                RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
 193                "beq 333f\n\t" \
 194                "addi %%r20, %%r20, -1\n\t" \
 195                "addi %%r21, %%r21, -1\n\t" \
 196                "222:\n\t" \
 197                "lbzu %%r18, 1(%%r20)\n\t" \
 198                "stbu %%r18, 1(%%r21)\n\t" \
 199                "addi %%r19, %%r19, -1\n\t" \
 200                RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
 201                "bne 222b\n\t" \
 202                "333:\n\t" \
 203
 204#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)                       \
 205                RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"                        \
 206                __rseq_str(post_commit_label) ":\n\t"
 207
 208#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)                  \
 209                RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
 210                __rseq_str(post_commit_label) ":\n\t"
 211
 212static inline __attribute__((always_inline))
 213int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 214{
 215        RSEQ_INJECT_C(9)
 216
 217        __asm__ __volatile__ goto (
 218                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 219                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 220#ifdef RSEQ_COMPARE_TWICE
 221                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 222                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 223#endif
 224                /* Start rseq by storing table entry pointer into rseq_cs. */
 225                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 226                /* cmp cpuid */
 227                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 228                RSEQ_INJECT_ASM(3)
 229                /* cmp @v equal to @expect */
 230                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 231                RSEQ_INJECT_ASM(4)
 232#ifdef RSEQ_COMPARE_TWICE
 233                /* cmp cpuid */
 234                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 235                /* cmp @v equal to @expect */
 236                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 237#endif
 238                /* final store */
 239                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 240                RSEQ_INJECT_ASM(5)
 241                RSEQ_ASM_DEFINE_ABORT(4, abort)
 242                : /* gcc asm goto does not allow outputs */
 243                : [cpu_id]              "r" (cpu),
 244                  [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 245                  [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 246                  [v]                   "m" (*v),
 247                  [expect]              "r" (expect),
 248                  [newv]                "r" (newv)
 249                  RSEQ_INJECT_INPUT
 250                : "memory", "cc", "r17"
 251                  RSEQ_INJECT_CLOBBER
 252                : abort, cmpfail
 253#ifdef RSEQ_COMPARE_TWICE
 254                  , error1, error2
 255#endif
 256        );
 257        rseq_after_asm_goto();
 258        return 0;
 259abort:
 260        rseq_after_asm_goto();
 261        RSEQ_INJECT_FAILED
 262        return -1;
 263cmpfail:
 264        rseq_after_asm_goto();
 265        return 1;
 266#ifdef RSEQ_COMPARE_TWICE
 267error1:
 268        rseq_after_asm_goto();
 269        rseq_bug("cpu_id comparison failed");
 270error2:
 271        rseq_after_asm_goto();
 272        rseq_bug("expected value comparison failed");
 273#endif
 274}
 275
 276static inline __attribute__((always_inline))
 277int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 278                               long voffp, intptr_t *load, int cpu)
 279{
 280        RSEQ_INJECT_C(9)
 281
 282        __asm__ __volatile__ goto (
 283                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 284                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 285#ifdef RSEQ_COMPARE_TWICE
 286                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 287                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 288#endif
 289                /* Start rseq by storing table entry pointer into rseq_cs. */
 290                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 291                /* cmp cpuid */
 292                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 293                RSEQ_INJECT_ASM(3)
 294                /* cmp @v not equal to @expectnot */
 295                RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
 296                RSEQ_INJECT_ASM(4)
 297#ifdef RSEQ_COMPARE_TWICE
 298                /* cmp cpuid */
 299                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 300                /* cmp @v not equal to @expectnot */
 301                RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
 302#endif
 303                /* load the value of @v */
 304                RSEQ_ASM_OP_R_LOAD(v)
 305                /* store it in @load */
 306                RSEQ_ASM_OP_R_STORE(load)
 307                /* dereference voffp(v) */
 308                RSEQ_ASM_OP_R_LOADX(voffp)
 309                /* final store the value at voffp(v) */
 310                RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
 311                RSEQ_INJECT_ASM(5)
 312                RSEQ_ASM_DEFINE_ABORT(4, abort)
 313                : /* gcc asm goto does not allow outputs */
 314                : [cpu_id]              "r" (cpu),
 315                  [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 316                  [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 317                  /* final store input */
 318                  [v]                   "m" (*v),
 319                  [expectnot]           "r" (expectnot),
 320                  [voffp]               "b" (voffp),
 321                  [load]                "m" (*load)
 322                  RSEQ_INJECT_INPUT
 323                : "memory", "cc", "r17"
 324                  RSEQ_INJECT_CLOBBER
 325                : abort, cmpfail
 326#ifdef RSEQ_COMPARE_TWICE
 327                  , error1, error2
 328#endif
 329        );
 330        rseq_after_asm_goto();
 331        return 0;
 332abort:
 333        rseq_after_asm_goto();
 334        RSEQ_INJECT_FAILED
 335        return -1;
 336cmpfail:
 337        rseq_after_asm_goto();
 338        return 1;
 339#ifdef RSEQ_COMPARE_TWICE
 340error1:
 341        rseq_after_asm_goto();
 342        rseq_bug("cpu_id comparison failed");
 343error2:
 344        rseq_after_asm_goto();
 345        rseq_bug("expected value comparison failed");
 346#endif
 347}
 348
 349static inline __attribute__((always_inline))
 350int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 351{
 352        RSEQ_INJECT_C(9)
 353
 354        __asm__ __volatile__ goto (
 355                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 356#ifdef RSEQ_COMPARE_TWICE
 357                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 358#endif
 359                /* Start rseq by storing table entry pointer into rseq_cs. */
 360                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 361                /* cmp cpuid */
 362                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 363                RSEQ_INJECT_ASM(3)
 364#ifdef RSEQ_COMPARE_TWICE
 365                /* cmp cpuid */
 366                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 367#endif
 368                /* load the value of @v */
 369                RSEQ_ASM_OP_R_LOAD(v)
 370                /* add @count to it */
 371                RSEQ_ASM_OP_R_ADD(count)
 372                /* final store */
 373                RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
 374                RSEQ_INJECT_ASM(4)
 375                RSEQ_ASM_DEFINE_ABORT(4, abort)
 376                : /* gcc asm goto does not allow outputs */
 377                : [cpu_id]              "r" (cpu),
 378                  [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 379                  [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 380                  /* final store input */
 381                  [v]                   "m" (*v),
 382                  [count]               "r" (count)
 383                  RSEQ_INJECT_INPUT
 384                : "memory", "cc", "r17"
 385                  RSEQ_INJECT_CLOBBER
 386                : abort
 387#ifdef RSEQ_COMPARE_TWICE
 388                  , error1
 389#endif
 390        );
 391        rseq_after_asm_goto();
 392        return 0;
 393abort:
 394        rseq_after_asm_goto();
 395        RSEQ_INJECT_FAILED
 396        return -1;
 397#ifdef RSEQ_COMPARE_TWICE
 398error1:
 399        rseq_after_asm_goto();
 400        rseq_bug("cpu_id comparison failed");
 401#endif
 402}
 403
 404static inline __attribute__((always_inline))
 405int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 406                                 intptr_t *v2, intptr_t newv2,
 407                                 intptr_t newv, int cpu)
 408{
 409        RSEQ_INJECT_C(9)
 410
 411        __asm__ __volatile__ goto (
 412                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 413                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 414#ifdef RSEQ_COMPARE_TWICE
 415                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 416                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 417#endif
 418                /* Start rseq by storing table entry pointer into rseq_cs. */
 419                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 420                /* cmp cpuid */
 421                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 422                RSEQ_INJECT_ASM(3)
 423                /* cmp @v equal to @expect */
 424                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 425                RSEQ_INJECT_ASM(4)
 426#ifdef RSEQ_COMPARE_TWICE
 427                /* cmp cpuid */
 428                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 429                /* cmp @v equal to @expect */
 430                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 431#endif
 432                /* try store */
 433                RSEQ_ASM_OP_STORE(newv2, v2)
 434                RSEQ_INJECT_ASM(5)
 435                /* final store */
 436                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 437                RSEQ_INJECT_ASM(6)
 438                RSEQ_ASM_DEFINE_ABORT(4, abort)
 439                : /* gcc asm goto does not allow outputs */
 440                : [cpu_id]              "r" (cpu),
 441                  [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 442                  [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 443                  /* try store input */
 444                  [v2]                  "m" (*v2),
 445                  [newv2]               "r" (newv2),
 446                  /* final store input */
 447                  [v]                   "m" (*v),
 448                  [expect]              "r" (expect),
 449                  [newv]                "r" (newv)
 450                  RSEQ_INJECT_INPUT
 451                : "memory", "cc", "r17"
 452                  RSEQ_INJECT_CLOBBER
 453                : abort, cmpfail
 454#ifdef RSEQ_COMPARE_TWICE
 455                  , error1, error2
 456#endif
 457        );
 458        rseq_after_asm_goto();
 459        return 0;
 460abort:
 461        rseq_after_asm_goto();
 462        RSEQ_INJECT_FAILED
 463        return -1;
 464cmpfail:
 465        rseq_after_asm_goto();
 466        return 1;
 467#ifdef RSEQ_COMPARE_TWICE
 468error1:
 469        rseq_after_asm_goto();
 470        rseq_bug("cpu_id comparison failed");
 471error2:
 472        rseq_after_asm_goto();
 473        rseq_bug("expected value comparison failed");
 474#endif
 475}
 476
 477static inline __attribute__((always_inline))
 478int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 479                                         intptr_t *v2, intptr_t newv2,
 480                                         intptr_t newv, int cpu)
 481{
 482        RSEQ_INJECT_C(9)
 483
 484        __asm__ __volatile__ goto (
 485                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 486                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 487#ifdef RSEQ_COMPARE_TWICE
 488                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 489                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 490#endif
 491                /* Start rseq by storing table entry pointer into rseq_cs. */
 492                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 493                /* cmp cpuid */
 494                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 495                RSEQ_INJECT_ASM(3)
 496                /* cmp @v equal to @expect */
 497                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 498                RSEQ_INJECT_ASM(4)
 499#ifdef RSEQ_COMPARE_TWICE
 500                /* cmp cpuid */
 501                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 502                /* cmp @v equal to @expect */
 503                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 504#endif
 505                /* try store */
 506                RSEQ_ASM_OP_STORE(newv2, v2)
 507                RSEQ_INJECT_ASM(5)
 508                /* for 'release' */
 509                "lwsync\n\t"
 510                /* final store */
 511                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 512                RSEQ_INJECT_ASM(6)
 513                RSEQ_ASM_DEFINE_ABORT(4, abort)
 514                : /* gcc asm goto does not allow outputs */
 515                : [cpu_id]              "r" (cpu),
 516                  [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 517                  [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 518                  /* try store input */
 519                  [v2]                  "m" (*v2),
 520                  [newv2]               "r" (newv2),
 521                  /* final store input */
 522                  [v]                   "m" (*v),
 523                  [expect]              "r" (expect),
 524                  [newv]                "r" (newv)
 525                  RSEQ_INJECT_INPUT
 526                : "memory", "cc", "r17"
 527                  RSEQ_INJECT_CLOBBER
 528                : abort, cmpfail
 529#ifdef RSEQ_COMPARE_TWICE
 530                  , error1, error2
 531#endif
 532        );
 533        rseq_after_asm_goto();
 534        return 0;
 535abort:
 536        rseq_after_asm_goto();
 537        RSEQ_INJECT_FAILED
 538        return -1;
 539cmpfail:
 540        rseq_after_asm_goto();
 541        return 1;
 542#ifdef RSEQ_COMPARE_TWICE
 543error1:
 544        rseq_after_asm_goto();
 545        rseq_bug("cpu_id comparison failed");
 546error2:
 547        rseq_after_asm_goto();
 548        rseq_bug("expected value comparison failed");
 549#endif
 550}
 551
 552static inline __attribute__((always_inline))
 553int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 554                              intptr_t *v2, intptr_t expect2,
 555                              intptr_t newv, int cpu)
 556{
 557        RSEQ_INJECT_C(9)
 558
 559        __asm__ __volatile__ goto (
 560                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 561                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 562#ifdef RSEQ_COMPARE_TWICE
 563                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 564                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 565                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 566#endif
 567                /* Start rseq by storing table entry pointer into rseq_cs. */
 568                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 569                /* cmp cpuid */
 570                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 571                RSEQ_INJECT_ASM(3)
 572                /* cmp @v equal to @expect */
 573                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 574                RSEQ_INJECT_ASM(4)
 575                /* cmp @v2 equal to @expct2 */
 576                RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
 577                RSEQ_INJECT_ASM(5)
 578#ifdef RSEQ_COMPARE_TWICE
 579                /* cmp cpuid */
 580                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 581                /* cmp @v equal to @expect */
 582                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 583                /* cmp @v2 equal to @expct2 */
 584                RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
 585#endif
 586                /* final store */
 587                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 588                RSEQ_INJECT_ASM(6)
 589                RSEQ_ASM_DEFINE_ABORT(4, abort)
 590                : /* gcc asm goto does not allow outputs */
 591                : [cpu_id]              "r" (cpu),
 592                  [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 593                  [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 594                  /* cmp2 input */
 595                  [v2]                  "m" (*v2),
 596                  [expect2]             "r" (expect2),
 597                  /* final store input */
 598                  [v]                   "m" (*v),
 599                  [expect]              "r" (expect),
 600                  [newv]                "r" (newv)
 601                  RSEQ_INJECT_INPUT
 602                : "memory", "cc", "r17"
 603                  RSEQ_INJECT_CLOBBER
 604                : abort, cmpfail
 605#ifdef RSEQ_COMPARE_TWICE
 606                  , error1, error2, error3
 607#endif
 608        );
 609        rseq_after_asm_goto();
 610        return 0;
 611abort:
 612        rseq_after_asm_goto();
 613        RSEQ_INJECT_FAILED
 614        return -1;
 615cmpfail:
 616        rseq_after_asm_goto();
 617        return 1;
 618#ifdef RSEQ_COMPARE_TWICE
 619error1:
 620        rseq_after_asm_goto();
 621        rseq_bug("cpu_id comparison failed");
 622error2:
 623        rseq_after_asm_goto();
 624        rseq_bug("1st expected value comparison failed");
 625error3:
 626        rseq_after_asm_goto();
 627        rseq_bug("2nd expected value comparison failed");
 628#endif
 629}
 630
 631static inline __attribute__((always_inline))
 632int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 633                                 void *dst, void *src, size_t len,
 634                                 intptr_t newv, int cpu)
 635{
 636        RSEQ_INJECT_C(9)
 637
 638        __asm__ __volatile__ goto (
 639                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 640                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 641#ifdef RSEQ_COMPARE_TWICE
 642                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 643                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 644#endif
 645                /* setup for mempcy */
 646                "mr %%r19, %[len]\n\t"
 647                "mr %%r20, %[src]\n\t"
 648                "mr %%r21, %[dst]\n\t"
 649                /* Start rseq by storing table entry pointer into rseq_cs. */
 650                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 651                /* cmp cpuid */
 652                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 653                RSEQ_INJECT_ASM(3)
 654                /* cmp @v equal to @expect */
 655                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 656                RSEQ_INJECT_ASM(4)
 657#ifdef RSEQ_COMPARE_TWICE
 658                /* cmp cpuid */
 659                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 660                /* cmp @v equal to @expect */
 661                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 662#endif
 663                /* try memcpy */
 664                RSEQ_ASM_OP_R_MEMCPY()
 665                RSEQ_INJECT_ASM(5)
 666                /* final store */
 667                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 668                RSEQ_INJECT_ASM(6)
 669                /* teardown */
 670                RSEQ_ASM_DEFINE_ABORT(4, abort)
 671                : /* gcc asm goto does not allow outputs */
 672                : [cpu_id]              "r" (cpu),
 673                  [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 674                  [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 675                  /* final store input */
 676                  [v]                   "m" (*v),
 677                  [expect]              "r" (expect),
 678                  [newv]                "r" (newv),
 679                  /* try memcpy input */
 680                  [dst]                 "r" (dst),
 681                  [src]                 "r" (src),
 682                  [len]                 "r" (len)
 683                  RSEQ_INJECT_INPUT
 684                : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
 685                  RSEQ_INJECT_CLOBBER
 686                : abort, cmpfail
 687#ifdef RSEQ_COMPARE_TWICE
 688                  , error1, error2
 689#endif
 690        );
 691        rseq_after_asm_goto();
 692        return 0;
 693abort:
 694        rseq_after_asm_goto();
 695        RSEQ_INJECT_FAILED
 696        return -1;
 697cmpfail:
 698        rseq_after_asm_goto();
 699        return 1;
 700#ifdef RSEQ_COMPARE_TWICE
 701error1:
 702        rseq_after_asm_goto();
 703        rseq_bug("cpu_id comparison failed");
 704error2:
 705        rseq_after_asm_goto();
 706        rseq_bug("expected value comparison failed");
 707#endif
 708}
 709
 710static inline __attribute__((always_inline))
 711int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 712                                         void *dst, void *src, size_t len,
 713                                         intptr_t newv, int cpu)
 714{
 715        RSEQ_INJECT_C(9)
 716
 717        __asm__ __volatile__ goto (
 718                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 719                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 720#ifdef RSEQ_COMPARE_TWICE
 721                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 722                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 723#endif
 724                /* setup for mempcy */
 725                "mr %%r19, %[len]\n\t"
 726                "mr %%r20, %[src]\n\t"
 727                "mr %%r21, %[dst]\n\t"
 728                /* Start rseq by storing table entry pointer into rseq_cs. */
 729                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 730                /* cmp cpuid */
 731                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 732                RSEQ_INJECT_ASM(3)
 733                /* cmp @v equal to @expect */
 734                RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 735                RSEQ_INJECT_ASM(4)
 736#ifdef RSEQ_COMPARE_TWICE
 737                /* cmp cpuid */
 738                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 739                /* cmp @v equal to @expect */
 740                RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 741#endif
 742                /* try memcpy */
 743                RSEQ_ASM_OP_R_MEMCPY()
 744                RSEQ_INJECT_ASM(5)
 745                /* for 'release' */
 746                "lwsync\n\t"
 747                /* final store */
 748                RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 749                RSEQ_INJECT_ASM(6)
 750                /* teardown */
 751                RSEQ_ASM_DEFINE_ABORT(4, abort)
 752                : /* gcc asm goto does not allow outputs */
 753                : [cpu_id]              "r" (cpu),
 754                  [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 755                  [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 756                  /* final store input */
 757                  [v]                   "m" (*v),
 758                  [expect]              "r" (expect),
 759                  [newv]                "r" (newv),
 760                  /* try memcpy input */
 761                  [dst]                 "r" (dst),
 762                  [src]                 "r" (src),
 763                  [len]                 "r" (len)
 764                  RSEQ_INJECT_INPUT
 765                : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
 766                  RSEQ_INJECT_CLOBBER
 767                : abort, cmpfail
 768#ifdef RSEQ_COMPARE_TWICE
 769                  , error1, error2
 770#endif
 771        );
 772        rseq_after_asm_goto();
 773        return 0;
 774abort:
 775        rseq_after_asm_goto();
 776        RSEQ_INJECT_FAILED
 777        return -1;
 778cmpfail:
 779        rseq_after_asm_goto();
 780        return 1;
 781#ifdef RSEQ_COMPARE_TWICE
 782error1:
 783        rseq_after_asm_goto();
 784        rseq_bug("cpu_id comparison failed");
 785error2:
 786        rseq_after_asm_goto();
 787        rseq_bug("expected value comparison failed");
 788#endif
 789}
 790
 791#endif /* !RSEQ_SKIP_FASTPATH */
 792