1
2
3
4
5
6
7
8#define RSEQ_SIG 0x53053053
9
10#define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
11#define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
12#define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
13
14#define rseq_smp_load_acquire(p) \
15__extension__ ({ \
16 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
17 rseq_smp_mb(); \
18 ____p1; \
19})
20
21#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
22
23#define rseq_smp_store_release(p, v) \
24do { \
25 rseq_smp_mb(); \
26 RSEQ_WRITE_ONCE(*p, v); \
27} while (0)
28
29#ifdef RSEQ_SKIP_FASTPATH
30#include "rseq-skip.h"
31#else
32
33#define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \
34 post_commit_offset, abort_ip) \
35 ".pushsection __rseq_table, \"aw\"\n\t" \
36 ".balign 32\n\t" \
37 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
38 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
39 ".popsection\n\t"
40
41#define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \
42 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \
43 (post_commit_ip - start_ip), abort_ip)
44
45#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
46 RSEQ_INJECT_ASM(1) \
47 "adr r0, " __rseq_str(cs_label) "\n\t" \
48 "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \
49 __rseq_str(label) ":\n\t"
50
51#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
52 RSEQ_INJECT_ASM(2) \
53 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
54 "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \
55 "bne " __rseq_str(label) "\n\t"
56
57#define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
58 abort_label, version, flags, \
59 start_ip, post_commit_offset, abort_ip) \
60 ".balign 32\n\t" \
61 __rseq_str(table_label) ":\n\t" \
62 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
63 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
64 ".word " __rseq_str(RSEQ_SIG) "\n\t" \
65 __rseq_str(label) ":\n\t" \
66 teardown \
67 "b %l[" __rseq_str(abort_label) "]\n\t"
68
69#define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
70 start_ip, post_commit_ip, abort_ip) \
71 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
72 abort_label, 0x0, 0x0, start_ip, \
73 (post_commit_ip - start_ip), abort_ip)
74
75#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
76 __rseq_str(label) ":\n\t" \
77 teardown \
78 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
79
80#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("")
81
82static inline __attribute__((always_inline))
83int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
84{
85 RSEQ_INJECT_C(9)
86
87 rseq_workaround_gcc_asm_size_guess();
88 __asm__ __volatile__ goto (
89 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f)
90
91 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
92 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
93 RSEQ_INJECT_ASM(3)
94 "ldr r0, %[v]\n\t"
95 "cmp %[expect], r0\n\t"
96 "bne %l[cmpfail]\n\t"
97 RSEQ_INJECT_ASM(4)
98#ifdef RSEQ_COMPARE_TWICE
99 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
100 "ldr r0, %[v]\n\t"
101 "cmp %[expect], r0\n\t"
102 "bne %l[error2]\n\t"
103#endif
104
105 "str %[newv], %[v]\n\t"
106 "2:\n\t"
107 RSEQ_INJECT_ASM(5)
108 "b 5f\n\t"
109 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
110 "5:\n\t"
111 :
112 : [cpu_id] "r" (cpu),
113 [current_cpu_id] "m" (__rseq_abi.cpu_id),
114 [rseq_cs] "m" (__rseq_abi.rseq_cs),
115 [v] "m" (*v),
116 [expect] "r" (expect),
117 [newv] "r" (newv)
118 RSEQ_INJECT_INPUT
119 : "r0", "memory", "cc"
120 RSEQ_INJECT_CLOBBER
121 : abort, cmpfail
122#ifdef RSEQ_COMPARE_TWICE
123 , error1, error2
124#endif
125 );
126 rseq_workaround_gcc_asm_size_guess();
127 return 0;
128abort:
129 rseq_workaround_gcc_asm_size_guess();
130 RSEQ_INJECT_FAILED
131 return -1;
132cmpfail:
133 rseq_workaround_gcc_asm_size_guess();
134 return 1;
135#ifdef RSEQ_COMPARE_TWICE
136error1:
137 rseq_bug("cpu_id comparison failed");
138error2:
139 rseq_bug("expected value comparison failed");
140#endif
141}
142
143static inline __attribute__((always_inline))
144int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
145 off_t voffp, intptr_t *load, int cpu)
146{
147 RSEQ_INJECT_C(9)
148
149 rseq_workaround_gcc_asm_size_guess();
150 __asm__ __volatile__ goto (
151 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f)
152
153 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
154 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
155 RSEQ_INJECT_ASM(3)
156 "ldr r0, %[v]\n\t"
157 "cmp %[expectnot], r0\n\t"
158 "beq %l[cmpfail]\n\t"
159 RSEQ_INJECT_ASM(4)
160#ifdef RSEQ_COMPARE_TWICE
161 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
162 "ldr r0, %[v]\n\t"
163 "cmp %[expectnot], r0\n\t"
164 "beq %l[error2]\n\t"
165#endif
166 "str r0, %[load]\n\t"
167 "add r0, %[voffp]\n\t"
168 "ldr r0, [r0]\n\t"
169
170 "str r0, %[v]\n\t"
171 "2:\n\t"
172 RSEQ_INJECT_ASM(5)
173 "b 5f\n\t"
174 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
175 "5:\n\t"
176 :
177 : [cpu_id] "r" (cpu),
178 [current_cpu_id] "m" (__rseq_abi.cpu_id),
179 [rseq_cs] "m" (__rseq_abi.rseq_cs),
180
181 [v] "m" (*v),
182 [expectnot] "r" (expectnot),
183 [voffp] "Ir" (voffp),
184 [load] "m" (*load)
185 RSEQ_INJECT_INPUT
186 : "r0", "memory", "cc"
187 RSEQ_INJECT_CLOBBER
188 : abort, cmpfail
189#ifdef RSEQ_COMPARE_TWICE
190 , error1, error2
191#endif
192 );
193 rseq_workaround_gcc_asm_size_guess();
194 return 0;
195abort:
196 rseq_workaround_gcc_asm_size_guess();
197 RSEQ_INJECT_FAILED
198 return -1;
199cmpfail:
200 rseq_workaround_gcc_asm_size_guess();
201 return 1;
202#ifdef RSEQ_COMPARE_TWICE
203error1:
204 rseq_bug("cpu_id comparison failed");
205error2:
206 rseq_bug("expected value comparison failed");
207#endif
208}
209
210static inline __attribute__((always_inline))
211int rseq_addv(intptr_t *v, intptr_t count, int cpu)
212{
213 RSEQ_INJECT_C(9)
214
215 rseq_workaround_gcc_asm_size_guess();
216 __asm__ __volatile__ goto (
217 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f)
218
219 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
221 RSEQ_INJECT_ASM(3)
222#ifdef RSEQ_COMPARE_TWICE
223 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
224#endif
225 "ldr r0, %[v]\n\t"
226 "add r0, %[count]\n\t"
227
228 "str r0, %[v]\n\t"
229 "2:\n\t"
230 RSEQ_INJECT_ASM(4)
231 "b 5f\n\t"
232 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
233 "5:\n\t"
234 :
235 : [cpu_id] "r" (cpu),
236 [current_cpu_id] "m" (__rseq_abi.cpu_id),
237 [rseq_cs] "m" (__rseq_abi.rseq_cs),
238 [v] "m" (*v),
239 [count] "Ir" (count)
240 RSEQ_INJECT_INPUT
241 : "r0", "memory", "cc"
242 RSEQ_INJECT_CLOBBER
243 : abort
244#ifdef RSEQ_COMPARE_TWICE
245 , error1
246#endif
247 );
248 rseq_workaround_gcc_asm_size_guess();
249 return 0;
250abort:
251 rseq_workaround_gcc_asm_size_guess();
252 RSEQ_INJECT_FAILED
253 return -1;
254#ifdef RSEQ_COMPARE_TWICE
255error1:
256 rseq_bug("cpu_id comparison failed");
257#endif
258}
259
260static inline __attribute__((always_inline))
261int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
262 intptr_t *v2, intptr_t newv2,
263 intptr_t newv, int cpu)
264{
265 RSEQ_INJECT_C(9)
266
267 rseq_workaround_gcc_asm_size_guess();
268 __asm__ __volatile__ goto (
269 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f)
270
271 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
272 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
273 RSEQ_INJECT_ASM(3)
274 "ldr r0, %[v]\n\t"
275 "cmp %[expect], r0\n\t"
276 "bne %l[cmpfail]\n\t"
277 RSEQ_INJECT_ASM(4)
278#ifdef RSEQ_COMPARE_TWICE
279 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
280 "ldr r0, %[v]\n\t"
281 "cmp %[expect], r0\n\t"
282 "bne %l[error2]\n\t"
283#endif
284
285 "str %[newv2], %[v2]\n\t"
286 RSEQ_INJECT_ASM(5)
287
288 "str %[newv], %[v]\n\t"
289 "2:\n\t"
290 RSEQ_INJECT_ASM(6)
291 "b 5f\n\t"
292 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
293 "5:\n\t"
294 :
295 : [cpu_id] "r" (cpu),
296 [current_cpu_id] "m" (__rseq_abi.cpu_id),
297 [rseq_cs] "m" (__rseq_abi.rseq_cs),
298
299 [v2] "m" (*v2),
300 [newv2] "r" (newv2),
301
302 [v] "m" (*v),
303 [expect] "r" (expect),
304 [newv] "r" (newv)
305 RSEQ_INJECT_INPUT
306 : "r0", "memory", "cc"
307 RSEQ_INJECT_CLOBBER
308 : abort, cmpfail
309#ifdef RSEQ_COMPARE_TWICE
310 , error1, error2
311#endif
312 );
313 rseq_workaround_gcc_asm_size_guess();
314 return 0;
315abort:
316 rseq_workaround_gcc_asm_size_guess();
317 RSEQ_INJECT_FAILED
318 return -1;
319cmpfail:
320 rseq_workaround_gcc_asm_size_guess();
321 return 1;
322#ifdef RSEQ_COMPARE_TWICE
323error1:
324 rseq_bug("cpu_id comparison failed");
325error2:
326 rseq_bug("expected value comparison failed");
327#endif
328}
329
330static inline __attribute__((always_inline))
331int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
332 intptr_t *v2, intptr_t newv2,
333 intptr_t newv, int cpu)
334{
335 RSEQ_INJECT_C(9)
336
337 rseq_workaround_gcc_asm_size_guess();
338 __asm__ __volatile__ goto (
339 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f)
340
341 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
342 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
343 RSEQ_INJECT_ASM(3)
344 "ldr r0, %[v]\n\t"
345 "cmp %[expect], r0\n\t"
346 "bne %l[cmpfail]\n\t"
347 RSEQ_INJECT_ASM(4)
348#ifdef RSEQ_COMPARE_TWICE
349 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
350 "ldr r0, %[v]\n\t"
351 "cmp %[expect], r0\n\t"
352 "bne %l[error2]\n\t"
353#endif
354
355 "str %[newv2], %[v2]\n\t"
356 RSEQ_INJECT_ASM(5)
357 "dmb\n\t"
358
359 "str %[newv], %[v]\n\t"
360 "2:\n\t"
361 RSEQ_INJECT_ASM(6)
362 "b 5f\n\t"
363 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
364 "5:\n\t"
365 :
366 : [cpu_id] "r" (cpu),
367 [current_cpu_id] "m" (__rseq_abi.cpu_id),
368 [rseq_cs] "m" (__rseq_abi.rseq_cs),
369
370 [v2] "m" (*v2),
371 [newv2] "r" (newv2),
372
373 [v] "m" (*v),
374 [expect] "r" (expect),
375 [newv] "r" (newv)
376 RSEQ_INJECT_INPUT
377 : "r0", "memory", "cc"
378 RSEQ_INJECT_CLOBBER
379 : abort, cmpfail
380#ifdef RSEQ_COMPARE_TWICE
381 , error1, error2
382#endif
383 );
384 rseq_workaround_gcc_asm_size_guess();
385 return 0;
386abort:
387 rseq_workaround_gcc_asm_size_guess();
388 RSEQ_INJECT_FAILED
389 return -1;
390cmpfail:
391 rseq_workaround_gcc_asm_size_guess();
392 return 1;
393#ifdef RSEQ_COMPARE_TWICE
394error1:
395 rseq_bug("cpu_id comparison failed");
396error2:
397 rseq_bug("expected value comparison failed");
398#endif
399}
400
401static inline __attribute__((always_inline))
402int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
403 intptr_t *v2, intptr_t expect2,
404 intptr_t newv, int cpu)
405{
406 RSEQ_INJECT_C(9)
407
408 rseq_workaround_gcc_asm_size_guess();
409 __asm__ __volatile__ goto (
410 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f)
411
412 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
413 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
414 RSEQ_INJECT_ASM(3)
415 "ldr r0, %[v]\n\t"
416 "cmp %[expect], r0\n\t"
417 "bne %l[cmpfail]\n\t"
418 RSEQ_INJECT_ASM(4)
419 "ldr r0, %[v2]\n\t"
420 "cmp %[expect2], r0\n\t"
421 "bne %l[cmpfail]\n\t"
422 RSEQ_INJECT_ASM(5)
423#ifdef RSEQ_COMPARE_TWICE
424 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
425 "ldr r0, %[v]\n\t"
426 "cmp %[expect], r0\n\t"
427 "bne %l[error2]\n\t"
428 "ldr r0, %[v2]\n\t"
429 "cmp %[expect2], r0\n\t"
430 "bne %l[error3]\n\t"
431#endif
432
433 "str %[newv], %[v]\n\t"
434 "2:\n\t"
435 RSEQ_INJECT_ASM(6)
436 "b 5f\n\t"
437 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
438 "5:\n\t"
439 :
440 : [cpu_id] "r" (cpu),
441 [current_cpu_id] "m" (__rseq_abi.cpu_id),
442 [rseq_cs] "m" (__rseq_abi.rseq_cs),
443
444 [v2] "m" (*v2),
445 [expect2] "r" (expect2),
446
447 [v] "m" (*v),
448 [expect] "r" (expect),
449 [newv] "r" (newv)
450 RSEQ_INJECT_INPUT
451 : "r0", "memory", "cc"
452 RSEQ_INJECT_CLOBBER
453 : abort, cmpfail
454#ifdef RSEQ_COMPARE_TWICE
455 , error1, error2, error3
456#endif
457 );
458 rseq_workaround_gcc_asm_size_guess();
459 return 0;
460abort:
461 rseq_workaround_gcc_asm_size_guess();
462 RSEQ_INJECT_FAILED
463 return -1;
464cmpfail:
465 rseq_workaround_gcc_asm_size_guess();
466 return 1;
467#ifdef RSEQ_COMPARE_TWICE
468error1:
469 rseq_bug("cpu_id comparison failed");
470error2:
471 rseq_bug("1st expected value comparison failed");
472error3:
473 rseq_bug("2nd expected value comparison failed");
474#endif
475}
476
477static inline __attribute__((always_inline))
478int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
479 void *dst, void *src, size_t len,
480 intptr_t newv, int cpu)
481{
482 uint32_t rseq_scratch[3];
483
484 RSEQ_INJECT_C(9)
485
486 rseq_workaround_gcc_asm_size_guess();
487 __asm__ __volatile__ goto (
488 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f)
489 "str %[src], %[rseq_scratch0]\n\t"
490 "str %[dst], %[rseq_scratch1]\n\t"
491 "str %[len], %[rseq_scratch2]\n\t"
492
493 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
494 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
495 RSEQ_INJECT_ASM(3)
496 "ldr r0, %[v]\n\t"
497 "cmp %[expect], r0\n\t"
498 "bne 5f\n\t"
499 RSEQ_INJECT_ASM(4)
500#ifdef RSEQ_COMPARE_TWICE
501 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
502 "ldr r0, %[v]\n\t"
503 "cmp %[expect], r0\n\t"
504 "bne 7f\n\t"
505#endif
506
507 "cmp %[len], #0\n\t" \
508 "beq 333f\n\t" \
509 "222:\n\t" \
510 "ldrb %%r0, [%[src]]\n\t" \
511 "strb %%r0, [%[dst]]\n\t" \
512 "adds %[src], #1\n\t" \
513 "adds %[dst], #1\n\t" \
514 "subs %[len], #1\n\t" \
515 "bne 222b\n\t" \
516 "333:\n\t" \
517 RSEQ_INJECT_ASM(5)
518
519 "str %[newv], %[v]\n\t"
520 "2:\n\t"
521 RSEQ_INJECT_ASM(6)
522
523 "ldr %[len], %[rseq_scratch2]\n\t"
524 "ldr %[dst], %[rseq_scratch1]\n\t"
525 "ldr %[src], %[rseq_scratch0]\n\t"
526 "b 8f\n\t"
527 RSEQ_ASM_DEFINE_ABORT(3, 4,
528
529 "ldr %[len], %[rseq_scratch2]\n\t"
530 "ldr %[dst], %[rseq_scratch1]\n\t"
531 "ldr %[src], %[rseq_scratch0]\n\t",
532 abort, 1b, 2b, 4f)
533 RSEQ_ASM_DEFINE_CMPFAIL(5,
534
535 "ldr %[len], %[rseq_scratch2]\n\t"
536 "ldr %[dst], %[rseq_scratch1]\n\t"
537 "ldr %[src], %[rseq_scratch0]\n\t",
538 cmpfail)
539#ifdef RSEQ_COMPARE_TWICE
540 RSEQ_ASM_DEFINE_CMPFAIL(6,
541
542 "ldr %[len], %[rseq_scratch2]\n\t"
543 "ldr %[dst], %[rseq_scratch1]\n\t"
544 "ldr %[src], %[rseq_scratch0]\n\t",
545 error1)
546 RSEQ_ASM_DEFINE_CMPFAIL(7,
547
548 "ldr %[len], %[rseq_scratch2]\n\t"
549 "ldr %[dst], %[rseq_scratch1]\n\t"
550 "ldr %[src], %[rseq_scratch0]\n\t",
551 error2)
552#endif
553 "8:\n\t"
554 :
555 : [cpu_id] "r" (cpu),
556 [current_cpu_id] "m" (__rseq_abi.cpu_id),
557 [rseq_cs] "m" (__rseq_abi.rseq_cs),
558
559 [v] "m" (*v),
560 [expect] "r" (expect),
561 [newv] "r" (newv),
562
563 [dst] "r" (dst),
564 [src] "r" (src),
565 [len] "r" (len),
566 [rseq_scratch0] "m" (rseq_scratch[0]),
567 [rseq_scratch1] "m" (rseq_scratch[1]),
568 [rseq_scratch2] "m" (rseq_scratch[2])
569 RSEQ_INJECT_INPUT
570 : "r0", "memory", "cc"
571 RSEQ_INJECT_CLOBBER
572 : abort, cmpfail
573#ifdef RSEQ_COMPARE_TWICE
574 , error1, error2
575#endif
576 );
577 rseq_workaround_gcc_asm_size_guess();
578 return 0;
579abort:
580 rseq_workaround_gcc_asm_size_guess();
581 RSEQ_INJECT_FAILED
582 return -1;
583cmpfail:
584 rseq_workaround_gcc_asm_size_guess();
585 return 1;
586#ifdef RSEQ_COMPARE_TWICE
587error1:
588 rseq_workaround_gcc_asm_size_guess();
589 rseq_bug("cpu_id comparison failed");
590error2:
591 rseq_workaround_gcc_asm_size_guess();
592 rseq_bug("expected value comparison failed");
593#endif
594}
595
596static inline __attribute__((always_inline))
597int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
598 void *dst, void *src, size_t len,
599 intptr_t newv, int cpu)
600{
601 uint32_t rseq_scratch[3];
602
603 RSEQ_INJECT_C(9)
604
605 rseq_workaround_gcc_asm_size_guess();
606 __asm__ __volatile__ goto (
607 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f)
608 "str %[src], %[rseq_scratch0]\n\t"
609 "str %[dst], %[rseq_scratch1]\n\t"
610 "str %[len], %[rseq_scratch2]\n\t"
611
612 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
613 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
614 RSEQ_INJECT_ASM(3)
615 "ldr r0, %[v]\n\t"
616 "cmp %[expect], r0\n\t"
617 "bne 5f\n\t"
618 RSEQ_INJECT_ASM(4)
619#ifdef RSEQ_COMPARE_TWICE
620 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
621 "ldr r0, %[v]\n\t"
622 "cmp %[expect], r0\n\t"
623 "bne 7f\n\t"
624#endif
625
626 "cmp %[len], #0\n\t" \
627 "beq 333f\n\t" \
628 "222:\n\t" \
629 "ldrb %%r0, [%[src]]\n\t" \
630 "strb %%r0, [%[dst]]\n\t" \
631 "adds %[src], #1\n\t" \
632 "adds %[dst], #1\n\t" \
633 "subs %[len], #1\n\t" \
634 "bne 222b\n\t" \
635 "333:\n\t" \
636 RSEQ_INJECT_ASM(5)
637 "dmb\n\t"
638
639 "str %[newv], %[v]\n\t"
640 "2:\n\t"
641 RSEQ_INJECT_ASM(6)
642
643 "ldr %[len], %[rseq_scratch2]\n\t"
644 "ldr %[dst], %[rseq_scratch1]\n\t"
645 "ldr %[src], %[rseq_scratch0]\n\t"
646 "b 8f\n\t"
647 RSEQ_ASM_DEFINE_ABORT(3, 4,
648
649 "ldr %[len], %[rseq_scratch2]\n\t"
650 "ldr %[dst], %[rseq_scratch1]\n\t"
651 "ldr %[src], %[rseq_scratch0]\n\t",
652 abort, 1b, 2b, 4f)
653 RSEQ_ASM_DEFINE_CMPFAIL(5,
654
655 "ldr %[len], %[rseq_scratch2]\n\t"
656 "ldr %[dst], %[rseq_scratch1]\n\t"
657 "ldr %[src], %[rseq_scratch0]\n\t",
658 cmpfail)
659#ifdef RSEQ_COMPARE_TWICE
660 RSEQ_ASM_DEFINE_CMPFAIL(6,
661
662 "ldr %[len], %[rseq_scratch2]\n\t"
663 "ldr %[dst], %[rseq_scratch1]\n\t"
664 "ldr %[src], %[rseq_scratch0]\n\t",
665 error1)
666 RSEQ_ASM_DEFINE_CMPFAIL(7,
667
668 "ldr %[len], %[rseq_scratch2]\n\t"
669 "ldr %[dst], %[rseq_scratch1]\n\t"
670 "ldr %[src], %[rseq_scratch0]\n\t",
671 error2)
672#endif
673 "8:\n\t"
674 :
675 : [cpu_id] "r" (cpu),
676 [current_cpu_id] "m" (__rseq_abi.cpu_id),
677 [rseq_cs] "m" (__rseq_abi.rseq_cs),
678
679 [v] "m" (*v),
680 [expect] "r" (expect),
681 [newv] "r" (newv),
682
683 [dst] "r" (dst),
684 [src] "r" (src),
685 [len] "r" (len),
686 [rseq_scratch0] "m" (rseq_scratch[0]),
687 [rseq_scratch1] "m" (rseq_scratch[1]),
688 [rseq_scratch2] "m" (rseq_scratch[2])
689 RSEQ_INJECT_INPUT
690 : "r0", "memory", "cc"
691 RSEQ_INJECT_CLOBBER
692 : abort, cmpfail
693#ifdef RSEQ_COMPARE_TWICE
694 , error1, error2
695#endif
696 );
697 rseq_workaround_gcc_asm_size_guess();
698 return 0;
699abort:
700 rseq_workaround_gcc_asm_size_guess();
701 RSEQ_INJECT_FAILED
702 return -1;
703cmpfail:
704 rseq_workaround_gcc_asm_size_guess();
705 return 1;
706#ifdef RSEQ_COMPARE_TWICE
707error1:
708 rseq_workaround_gcc_asm_size_guess();
709 rseq_bug("cpu_id comparison failed");
710error2:
711 rseq_workaround_gcc_asm_size_guess();
712 rseq_bug("expected value comparison failed");
713#endif
714}
715
716#endif
717
