LXR linux/tools/testing/selftests/rseq/rseq-s390.h

   1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2
   3/*
   4 * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
   5 * access-register mode nor the linkage stack this instruction will always
   6 * cause a special-operation exception (the trap-enabled bit in the DUCT
   7 * is and will stay 0). The instruction pattern is
   8 *      b2 ff 0f ff     trap4   4095(%r0)
   9 */
  10#define RSEQ_SIG        0xB2FF0FFF
  11
  12#define rseq_smp_mb()   __asm__ __volatile__ ("bcr 15,0" ::: "memory")
  13#define rseq_smp_rmb()  rseq_smp_mb()
  14#define rseq_smp_wmb()  rseq_smp_mb()
  15
  16#define rseq_smp_load_acquire(p)                                        \
  17__extension__ ({                                                        \
  18        __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  19        rseq_barrier();                                                 \
  20        ____p1;                                                         \
  21})
  22
  23#define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  24
  25#define rseq_smp_store_release(p, v)                                    \
  26do {                                                                    \
  27        rseq_barrier();                                                 \
  28        RSEQ_WRITE_ONCE(*p, v);                                         \
  29} while (0)
  30
  31#ifdef RSEQ_SKIP_FASTPATH
  32#include "rseq-skip.h"
  33#else /* !RSEQ_SKIP_FASTPATH */
  34
  35#ifdef __s390x__
  36
  37#define LONG_L                  "lg"
  38#define LONG_S                  "stg"
  39#define LONG_LT_R               "ltgr"
  40#define LONG_CMP                "cg"
  41#define LONG_CMP_R              "cgr"
  42#define LONG_ADDI               "aghi"
  43#define LONG_ADD_R              "agr"
  44
  45#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  46                                start_ip, post_commit_offset, abort_ip) \
  47                ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  48                ".balign 32\n\t"                                        \
  49                __rseq_str(label) ":\n\t"                               \
  50                ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  51                ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  52                ".popsection\n\t"                                       \
  53                ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  54                ".quad " __rseq_str(label) "b\n\t"                      \
  55                ".popsection\n\t"
  56
  57/*
  58 * Exit points of a rseq critical section consist of all instructions outside
  59 * of the critical section where a critical section can either branch to or
  60 * reach through the normal course of its execution. The abort IP and the
  61 * post-commit IP are already part of the __rseq_cs section and should not be
  62 * explicitly defined as additional exit points. Knowing all exit points is
  63 * useful to assist debuggers stepping over the critical section.
  64 */
  65#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  66                ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  67                ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  68                ".popsection\n\t"
  69
  70#elif __s390__
  71
  72#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  73                                start_ip, post_commit_offset, abort_ip) \
  74                ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  75                ".balign 32\n\t"                                        \
  76                __rseq_str(label) ":\n\t"                               \
  77                ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  78                ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
  79                ".popsection\n\t"                                       \
  80                ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  81                ".long 0x0, " __rseq_str(label) "b\n\t"                 \
  82                ".popsection\n\t"
  83
  84/*
  85 * Exit points of a rseq critical section consist of all instructions outside
  86 * of the critical section where a critical section can either branch to or
  87 * reach through the normal course of its execution. The abort IP and the
  88 * post-commit IP are already part of the __rseq_cs section and should not be
  89 * explicitly defined as additional exit points. Knowing all exit points is
  90 * useful to assist debuggers stepping over the critical section.
  91 */
  92#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  93                ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  94                ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
  95                ".popsection\n\t"
  96
  97#define LONG_L                  "l"
  98#define LONG_S                  "st"
  99#define LONG_LT_R               "ltr"
 100#define LONG_CMP                "c"
 101#define LONG_CMP_R              "cr"
 102#define LONG_ADDI               "ahi"
 103#define LONG_ADD_R              "ar"
 104
 105#endif
 106
 107#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 108        __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 109                                (post_commit_ip - start_ip), abort_ip)
 110
 111#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 112                RSEQ_INJECT_ASM(1)                                      \
 113                "larl %%r0, " __rseq_str(cs_label) "\n\t"               \
 114                LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t"          \
 115                __rseq_str(label) ":\n\t"
 116
 117#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 118                RSEQ_INJECT_ASM(2)                                      \
 119                "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
 120                "jnz " __rseq_str(label) "\n\t"
 121
 122#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 123                ".pushsection __rseq_failure, \"ax\"\n\t"               \
 124                ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 125                __rseq_str(label) ":\n\t"                               \
 126                teardown                                                \
 127                "jg %l[" __rseq_str(abort_label) "]\n\t"                \
 128                ".popsection\n\t"
 129
 130#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 131                ".pushsection __rseq_failure, \"ax\"\n\t"               \
 132                __rseq_str(label) ":\n\t"                               \
 133                teardown                                                \
 134                "jg %l[" __rseq_str(cmpfail_label) "]\n\t"              \
 135                ".popsection\n\t"
 136
 137static inline __attribute__((always_inline))
 138int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 139{
 140        RSEQ_INJECT_C(9)
 141
 142        __asm__ __volatile__ goto (
 143                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 144                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 145#ifdef RSEQ_COMPARE_TWICE
 146                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 147                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 148#endif
 149                /* Start rseq by storing table entry pointer into rseq_cs. */
 150                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 151                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 152                RSEQ_INJECT_ASM(3)
 153                LONG_CMP " %[expect], %[v]\n\t"
 154                "jnz %l[cmpfail]\n\t"
 155                RSEQ_INJECT_ASM(4)
 156#ifdef RSEQ_COMPARE_TWICE
 157                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 158                LONG_CMP " %[expect], %[v]\n\t"
 159                "jnz %l[error2]\n\t"
 160#endif
 161                /* final store */
 162                LONG_S " %[newv], %[v]\n\t"
 163                "2:\n\t"
 164                RSEQ_INJECT_ASM(5)
 165                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 166                : /* gcc asm goto does not allow outputs */
 167                : [cpu_id]              "r" (cpu),
 168                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 169                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 170                  [v]                   "m" (*v),
 171                  [expect]              "r" (expect),
 172                  [newv]                "r" (newv)
 173                  RSEQ_INJECT_INPUT
 174                : "memory", "cc", "r0"
 175                  RSEQ_INJECT_CLOBBER
 176                : abort, cmpfail
 177#ifdef RSEQ_COMPARE_TWICE
 178                  , error1, error2
 179#endif
 180        );
 181        return 0;
 182abort:
 183        RSEQ_INJECT_FAILED
 184        return -1;
 185cmpfail:
 186        return 1;
 187#ifdef RSEQ_COMPARE_TWICE
 188error1:
 189        rseq_bug("cpu_id comparison failed");
 190error2:
 191        rseq_bug("expected value comparison failed");
 192#endif
 193}
 194
 195/*
 196 * Compare @v against @expectnot. When it does _not_ match, load @v
 197 * into @load, and store the content of *@v + voffp into @v.
 198 */
 199static inline __attribute__((always_inline))
 200int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 201                               off_t voffp, intptr_t *load, int cpu)
 202{
 203        RSEQ_INJECT_C(9)
 204
 205        __asm__ __volatile__ goto (
 206                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 207                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 208#ifdef RSEQ_COMPARE_TWICE
 209                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 210                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 211#endif
 212                /* Start rseq by storing table entry pointer into rseq_cs. */
 213                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 214                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 215                RSEQ_INJECT_ASM(3)
 216                LONG_L " %%r1, %[v]\n\t"
 217                LONG_CMP_R " %%r1, %[expectnot]\n\t"
 218                "je %l[cmpfail]\n\t"
 219                RSEQ_INJECT_ASM(4)
 220#ifdef RSEQ_COMPARE_TWICE
 221                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 222                LONG_L " %%r1, %[v]\n\t"
 223                LONG_CMP_R " %%r1, %[expectnot]\n\t"
 224                "je %l[error2]\n\t"
 225#endif
 226                LONG_S " %%r1, %[load]\n\t"
 227                LONG_ADD_R " %%r1, %[voffp]\n\t"
 228                LONG_L " %%r1, 0(%%r1)\n\t"
 229                /* final store */
 230                LONG_S " %%r1, %[v]\n\t"
 231                "2:\n\t"
 232                RSEQ_INJECT_ASM(5)
 233                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 234                : /* gcc asm goto does not allow outputs */
 235                : [cpu_id]              "r" (cpu),
 236                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 237                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 238                  /* final store input */
 239                  [v]                   "m" (*v),
 240                  [expectnot]           "r" (expectnot),
 241                  [voffp]               "r" (voffp),
 242                  [load]                "m" (*load)
 243                  RSEQ_INJECT_INPUT
 244                : "memory", "cc", "r0", "r1"
 245                  RSEQ_INJECT_CLOBBER
 246                : abort, cmpfail
 247#ifdef RSEQ_COMPARE_TWICE
 248                  , error1, error2
 249#endif
 250        );
 251        return 0;
 252abort:
 253        RSEQ_INJECT_FAILED
 254        return -1;
 255cmpfail:
 256        return 1;
 257#ifdef RSEQ_COMPARE_TWICE
 258error1:
 259        rseq_bug("cpu_id comparison failed");
 260error2:
 261        rseq_bug("expected value comparison failed");
 262#endif
 263}
 264
 265static inline __attribute__((always_inline))
 266int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 267{
 268        RSEQ_INJECT_C(9)
 269
 270        __asm__ __volatile__ goto (
 271                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 272#ifdef RSEQ_COMPARE_TWICE
 273                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 274#endif
 275                /* Start rseq by storing table entry pointer into rseq_cs. */
 276                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 277                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 278                RSEQ_INJECT_ASM(3)
 279#ifdef RSEQ_COMPARE_TWICE
 280                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 281#endif
 282                LONG_L " %%r0, %[v]\n\t"
 283                LONG_ADD_R " %%r0, %[count]\n\t"
 284                /* final store */
 285                LONG_S " %%r0, %[v]\n\t"
 286                "2:\n\t"
 287                RSEQ_INJECT_ASM(4)
 288                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 289                : /* gcc asm goto does not allow outputs */
 290                : [cpu_id]              "r" (cpu),
 291                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 292                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 293                  /* final store input */
 294                  [v]                   "m" (*v),
 295                  [count]               "r" (count)
 296                  RSEQ_INJECT_INPUT
 297                : "memory", "cc", "r0"
 298                  RSEQ_INJECT_CLOBBER
 299                : abort
 300#ifdef RSEQ_COMPARE_TWICE
 301                  , error1
 302#endif
 303        );
 304        return 0;
 305abort:
 306        RSEQ_INJECT_FAILED
 307        return -1;
 308#ifdef RSEQ_COMPARE_TWICE
 309error1:
 310        rseq_bug("cpu_id comparison failed");
 311#endif
 312}
 313
 314static inline __attribute__((always_inline))
 315int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 316                                 intptr_t *v2, intptr_t newv2,
 317                                 intptr_t newv, int cpu)
 318{
 319        RSEQ_INJECT_C(9)
 320
 321        __asm__ __volatile__ goto (
 322                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 323                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 324#ifdef RSEQ_COMPARE_TWICE
 325                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 326                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 327#endif
 328                /* Start rseq by storing table entry pointer into rseq_cs. */
 329                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 330                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 331                RSEQ_INJECT_ASM(3)
 332                LONG_CMP " %[expect], %[v]\n\t"
 333                "jnz %l[cmpfail]\n\t"
 334                RSEQ_INJECT_ASM(4)
 335#ifdef RSEQ_COMPARE_TWICE
 336                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 337                LONG_CMP " %[expect], %[v]\n\t"
 338                "jnz %l[error2]\n\t"
 339#endif
 340                /* try store */
 341                LONG_S " %[newv2], %[v2]\n\t"
 342                RSEQ_INJECT_ASM(5)
 343                /* final store */
 344                LONG_S " %[newv], %[v]\n\t"
 345                "2:\n\t"
 346                RSEQ_INJECT_ASM(6)
 347                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 348                : /* gcc asm goto does not allow outputs */
 349                : [cpu_id]              "r" (cpu),
 350                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 351                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 352                  /* try store input */
 353                  [v2]                  "m" (*v2),
 354                  [newv2]               "r" (newv2),
 355                  /* final store input */
 356                  [v]                   "m" (*v),
 357                  [expect]              "r" (expect),
 358                  [newv]                "r" (newv)
 359                  RSEQ_INJECT_INPUT
 360                : "memory", "cc", "r0"
 361                  RSEQ_INJECT_CLOBBER
 362                : abort, cmpfail
 363#ifdef RSEQ_COMPARE_TWICE
 364                  , error1, error2
 365#endif
 366        );
 367        return 0;
 368abort:
 369        RSEQ_INJECT_FAILED
 370        return -1;
 371cmpfail:
 372        return 1;
 373#ifdef RSEQ_COMPARE_TWICE
 374error1:
 375        rseq_bug("cpu_id comparison failed");
 376error2:
 377        rseq_bug("expected value comparison failed");
 378#endif
 379}
 380
 381/* s390 is TSO. */
 382static inline __attribute__((always_inline))
 383int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 384                                         intptr_t *v2, intptr_t newv2,
 385                                         intptr_t newv, int cpu)
 386{
 387        return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
 388}
 389
 390static inline __attribute__((always_inline))
 391int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 392                              intptr_t *v2, intptr_t expect2,
 393                              intptr_t newv, int cpu)
 394{
 395        RSEQ_INJECT_C(9)
 396
 397        __asm__ __volatile__ goto (
 398                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 399                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 400#ifdef RSEQ_COMPARE_TWICE
 401                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 402                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 403                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 404#endif
 405                /* Start rseq by storing table entry pointer into rseq_cs. */
 406                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 407                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 408                RSEQ_INJECT_ASM(3)
 409                LONG_CMP " %[expect], %[v]\n\t"
 410                "jnz %l[cmpfail]\n\t"
 411                RSEQ_INJECT_ASM(4)
 412                LONG_CMP " %[expect2], %[v2]\n\t"
 413                "jnz %l[cmpfail]\n\t"
 414                RSEQ_INJECT_ASM(5)
 415#ifdef RSEQ_COMPARE_TWICE
 416                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 417                LONG_CMP " %[expect], %[v]\n\t"
 418                "jnz %l[error2]\n\t"
 419                LONG_CMP " %[expect2], %[v2]\n\t"
 420                "jnz %l[error3]\n\t"
 421#endif
 422                /* final store */
 423                LONG_S " %[newv], %[v]\n\t"
 424                "2:\n\t"
 425                RSEQ_INJECT_ASM(6)
 426                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 427                : /* gcc asm goto does not allow outputs */
 428                : [cpu_id]              "r" (cpu),
 429                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 430                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 431                  /* cmp2 input */
 432                  [v2]                  "m" (*v2),
 433                  [expect2]             "r" (expect2),
 434                  /* final store input */
 435                  [v]                   "m" (*v),
 436                  [expect]              "r" (expect),
 437                  [newv]                "r" (newv)
 438                  RSEQ_INJECT_INPUT
 439                : "memory", "cc", "r0"
 440                  RSEQ_INJECT_CLOBBER
 441                : abort, cmpfail
 442#ifdef RSEQ_COMPARE_TWICE
 443                  , error1, error2, error3
 444#endif
 445        );
 446        return 0;
 447abort:
 448        RSEQ_INJECT_FAILED
 449        return -1;
 450cmpfail:
 451        return 1;
 452#ifdef RSEQ_COMPARE_TWICE
 453error1:
 454        rseq_bug("cpu_id comparison failed");
 455error2:
 456        rseq_bug("1st expected value comparison failed");
 457error3:
 458        rseq_bug("2nd expected value comparison failed");
 459#endif
 460}
 461
 462static inline __attribute__((always_inline))
 463int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 464                                 void *dst, void *src, size_t len,
 465                                 intptr_t newv, int cpu)
 466{
 467        uint64_t rseq_scratch[3];
 468
 469        RSEQ_INJECT_C(9)
 470
 471        __asm__ __volatile__ goto (
 472                RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 473                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 474#ifdef RSEQ_COMPARE_TWICE
 475                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 476                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 477#endif
 478                LONG_S " %[src], %[rseq_scratch0]\n\t"
 479                LONG_S " %[dst], %[rseq_scratch1]\n\t"
 480                LONG_S " %[len], %[rseq_scratch2]\n\t"
 481                /* Start rseq by storing table entry pointer into rseq_cs. */
 482                RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 483                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 484                RSEQ_INJECT_ASM(3)
 485                LONG_CMP " %[expect], %[v]\n\t"
 486                "jnz 5f\n\t"
 487                RSEQ_INJECT_ASM(4)
 488#ifdef RSEQ_COMPARE_TWICE
 489                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 490                LONG_CMP " %[expect], %[v]\n\t"
 491                "jnz 7f\n\t"
 492#endif
 493                /* try memcpy */
 494                LONG_LT_R " %[len], %[len]\n\t"
 495                "jz 333f\n\t"
 496                "222:\n\t"
 497                "ic %%r0,0(%[src])\n\t"
 498                "stc %%r0,0(%[dst])\n\t"
 499                LONG_ADDI " %[src], 1\n\t"
 500                LONG_ADDI " %[dst], 1\n\t"
 501                LONG_ADDI " %[len], -1\n\t"
 502                "jnz 222b\n\t"
 503                "333:\n\t"
 504                RSEQ_INJECT_ASM(5)
 505                /* final store */
 506                LONG_S " %[newv], %[v]\n\t"
 507                "2:\n\t"
 508                RSEQ_INJECT_ASM(6)
 509                /* teardown */
 510                LONG_L " %[len], %[rseq_scratch2]\n\t"
 511                LONG_L " %[dst], %[rseq_scratch1]\n\t"
 512                LONG_L " %[src], %[rseq_scratch0]\n\t"
 513                RSEQ_ASM_DEFINE_ABORT(4,
 514                        LONG_L " %[len], %[rseq_scratch2]\n\t"
 515                        LONG_L " %[dst], %[rseq_scratch1]\n\t"
 516                        LONG_L " %[src], %[rseq_scratch0]\n\t",
 517                        abort)
 518                RSEQ_ASM_DEFINE_CMPFAIL(5,
 519                        LONG_L " %[len], %[rseq_scratch2]\n\t"
 520                        LONG_L " %[dst], %[rseq_scratch1]\n\t"
 521                        LONG_L " %[src], %[rseq_scratch0]\n\t",
 522                        cmpfail)
 523#ifdef RSEQ_COMPARE_TWICE
 524                RSEQ_ASM_DEFINE_CMPFAIL(6,
 525                        LONG_L " %[len], %[rseq_scratch2]\n\t"
 526                        LONG_L " %[dst], %[rseq_scratch1]\n\t"
 527                        LONG_L " %[src], %[rseq_scratch0]\n\t",
 528                        error1)
 529                RSEQ_ASM_DEFINE_CMPFAIL(7,
 530                        LONG_L " %[len], %[rseq_scratch2]\n\t"
 531                        LONG_L " %[dst], %[rseq_scratch1]\n\t"
 532                        LONG_L " %[src], %[rseq_scratch0]\n\t",
 533                        error2)
 534#endif
 535                : /* gcc asm goto does not allow outputs */
 536                : [cpu_id]              "r" (cpu),
 537                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 538                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 539                  /* final store input */
 540                  [v]                   "m" (*v),
 541                  [expect]              "r" (expect),
 542                  [newv]                "r" (newv),
 543                  /* try memcpy input */
 544                  [dst]                 "r" (dst),
 545                  [src]                 "r" (src),
 546                  [len]                 "r" (len),
 547                  [rseq_scratch0]       "m" (rseq_scratch[0]),
 548                  [rseq_scratch1]       "m" (rseq_scratch[1]),
 549                  [rseq_scratch2]       "m" (rseq_scratch[2])
 550                  RSEQ_INJECT_INPUT
 551                : "memory", "cc", "r0"
 552                  RSEQ_INJECT_CLOBBER
 553                : abort, cmpfail
 554#ifdef RSEQ_COMPARE_TWICE
 555                  , error1, error2
 556#endif
 557        );
 558        return 0;
 559abort:
 560        RSEQ_INJECT_FAILED
 561        return -1;
 562cmpfail:
 563        return 1;
 564#ifdef RSEQ_COMPARE_TWICE
 565error1:
 566        rseq_bug("cpu_id comparison failed");
 567error2:
 568        rseq_bug("expected value comparison failed");
 569#endif
 570}
 571
 572/* s390 is TSO. */
 573static inline __attribute__((always_inline))
 574int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 575                                         void *dst, void *src, size_t len,
 576                                         intptr_t newv, int cpu)
 577{
 578        return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
 579                                            newv, cpu);
 580}
 581#endif /* !RSEQ_SKIP_FASTPATH */
 582