1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#ifndef _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
18#define _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
19
20#include <linux/build_bug.h>
21#include <linux/compiler.h>
22#include <linux/instrumented.h>
23
24static __always_inline int
25atomic_read(const atomic_t *v)
26{
27 instrument_atomic_read(v, sizeof(*v));
28 return arch_atomic_read(v);
29}
30#define atomic_read atomic_read
31
32#if defined(arch_atomic_read_acquire)
33static __always_inline int
34atomic_read_acquire(const atomic_t *v)
35{
36 instrument_atomic_read(v, sizeof(*v));
37 return arch_atomic_read_acquire(v);
38}
39#define atomic_read_acquire atomic_read_acquire
40#endif
41
42static __always_inline void
43atomic_set(atomic_t *v, int i)
44{
45 instrument_atomic_write(v, sizeof(*v));
46 arch_atomic_set(v, i);
47}
48#define atomic_set atomic_set
49
50#if defined(arch_atomic_set_release)
51static __always_inline void
52atomic_set_release(atomic_t *v, int i)
53{
54 instrument_atomic_write(v, sizeof(*v));
55 arch_atomic_set_release(v, i);
56}
57#define atomic_set_release atomic_set_release
58#endif
59
60static __always_inline void
61atomic_add(int i, atomic_t *v)
62{
63 instrument_atomic_read_write(v, sizeof(*v));
64 arch_atomic_add(i, v);
65}
66#define atomic_add atomic_add
67
68#if !defined(arch_atomic_add_return_relaxed) || defined(arch_atomic_add_return)
69static __always_inline int
70atomic_add_return(int i, atomic_t *v)
71{
72 instrument_atomic_read_write(v, sizeof(*v));
73 return arch_atomic_add_return(i, v);
74}
75#define atomic_add_return atomic_add_return
76#endif
77
78#if defined(arch_atomic_add_return_acquire)
79static __always_inline int
80atomic_add_return_acquire(int i, atomic_t *v)
81{
82 instrument_atomic_read_write(v, sizeof(*v));
83 return arch_atomic_add_return_acquire(i, v);
84}
85#define atomic_add_return_acquire atomic_add_return_acquire
86#endif
87
88#if defined(arch_atomic_add_return_release)
89static __always_inline int
90atomic_add_return_release(int i, atomic_t *v)
91{
92 instrument_atomic_read_write(v, sizeof(*v));
93 return arch_atomic_add_return_release(i, v);
94}
95#define atomic_add_return_release atomic_add_return_release
96#endif
97
98#if defined(arch_atomic_add_return_relaxed)
99static __always_inline int
100atomic_add_return_relaxed(int i, atomic_t *v)
101{
102 instrument_atomic_read_write(v, sizeof(*v));
103 return arch_atomic_add_return_relaxed(i, v);
104}
105#define atomic_add_return_relaxed atomic_add_return_relaxed
106#endif
107
108#if !defined(arch_atomic_fetch_add_relaxed) || defined(arch_atomic_fetch_add)
109static __always_inline int
110atomic_fetch_add(int i, atomic_t *v)
111{
112 instrument_atomic_read_write(v, sizeof(*v));
113 return arch_atomic_fetch_add(i, v);
114}
115#define atomic_fetch_add atomic_fetch_add
116#endif
117
118#if defined(arch_atomic_fetch_add_acquire)
119static __always_inline int
120atomic_fetch_add_acquire(int i, atomic_t *v)
121{
122 instrument_atomic_read_write(v, sizeof(*v));
123 return arch_atomic_fetch_add_acquire(i, v);
124}
125#define atomic_fetch_add_acquire atomic_fetch_add_acquire
126#endif
127
128#if defined(arch_atomic_fetch_add_release)
129static __always_inline int
130atomic_fetch_add_release(int i, atomic_t *v)
131{
132 instrument_atomic_read_write(v, sizeof(*v));
133 return arch_atomic_fetch_add_release(i, v);
134}
135#define atomic_fetch_add_release atomic_fetch_add_release
136#endif
137
138#if defined(arch_atomic_fetch_add_relaxed)
139static __always_inline int
140atomic_fetch_add_relaxed(int i, atomic_t *v)
141{
142 instrument_atomic_read_write(v, sizeof(*v));
143 return arch_atomic_fetch_add_relaxed(i, v);
144}
145#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
146#endif
147
148static __always_inline void
149atomic_sub(int i, atomic_t *v)
150{
151 instrument_atomic_read_write(v, sizeof(*v));
152 arch_atomic_sub(i, v);
153}
154#define atomic_sub atomic_sub
155
156#if !defined(arch_atomic_sub_return_relaxed) || defined(arch_atomic_sub_return)
157static __always_inline int
158atomic_sub_return(int i, atomic_t *v)
159{
160 instrument_atomic_read_write(v, sizeof(*v));
161 return arch_atomic_sub_return(i, v);
162}
163#define atomic_sub_return atomic_sub_return
164#endif
165
166#if defined(arch_atomic_sub_return_acquire)
167static __always_inline int
168atomic_sub_return_acquire(int i, atomic_t *v)
169{
170 instrument_atomic_read_write(v, sizeof(*v));
171 return arch_atomic_sub_return_acquire(i, v);
172}
173#define atomic_sub_return_acquire atomic_sub_return_acquire
174#endif
175
176#if defined(arch_atomic_sub_return_release)
177static __always_inline int
178atomic_sub_return_release(int i, atomic_t *v)
179{
180 instrument_atomic_read_write(v, sizeof(*v));
181 return arch_atomic_sub_return_release(i, v);
182}
183#define atomic_sub_return_release atomic_sub_return_release
184#endif
185
186#if defined(arch_atomic_sub_return_relaxed)
187static __always_inline int
188atomic_sub_return_relaxed(int i, atomic_t *v)
189{
190 instrument_atomic_read_write(v, sizeof(*v));
191 return arch_atomic_sub_return_relaxed(i, v);
192}
193#define atomic_sub_return_relaxed atomic_sub_return_relaxed
194#endif
195
196#if !defined(arch_atomic_fetch_sub_relaxed) || defined(arch_atomic_fetch_sub)
197static __always_inline int
198atomic_fetch_sub(int i, atomic_t *v)
199{
200 instrument_atomic_read_write(v, sizeof(*v));
201 return arch_atomic_fetch_sub(i, v);
202}
203#define atomic_fetch_sub atomic_fetch_sub
204#endif
205
206#if defined(arch_atomic_fetch_sub_acquire)
207static __always_inline int
208atomic_fetch_sub_acquire(int i, atomic_t *v)
209{
210 instrument_atomic_read_write(v, sizeof(*v));
211 return arch_atomic_fetch_sub_acquire(i, v);
212}
213#define atomic_fetch_sub_acquire atomic_fetch_sub_acquire
214#endif
215
216#if defined(arch_atomic_fetch_sub_release)
217static __always_inline int
218atomic_fetch_sub_release(int i, atomic_t *v)
219{
220 instrument_atomic_read_write(v, sizeof(*v));
221 return arch_atomic_fetch_sub_release(i, v);
222}
223#define atomic_fetch_sub_release atomic_fetch_sub_release
224#endif
225
226#if defined(arch_atomic_fetch_sub_relaxed)
227static __always_inline int
228atomic_fetch_sub_relaxed(int i, atomic_t *v)
229{
230 instrument_atomic_read_write(v, sizeof(*v));
231 return arch_atomic_fetch_sub_relaxed(i, v);
232}
233#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
234#endif
235
236#if defined(arch_atomic_inc)
237static __always_inline void
238atomic_inc(atomic_t *v)
239{
240 instrument_atomic_read_write(v, sizeof(*v));
241 arch_atomic_inc(v);
242}
243#define atomic_inc atomic_inc
244#endif
245
246#if defined(arch_atomic_inc_return)
247static __always_inline int
248atomic_inc_return(atomic_t *v)
249{
250 instrument_atomic_read_write(v, sizeof(*v));
251 return arch_atomic_inc_return(v);
252}
253#define atomic_inc_return atomic_inc_return
254#endif
255
256#if defined(arch_atomic_inc_return_acquire)
257static __always_inline int
258atomic_inc_return_acquire(atomic_t *v)
259{
260 instrument_atomic_read_write(v, sizeof(*v));
261 return arch_atomic_inc_return_acquire(v);
262}
263#define atomic_inc_return_acquire atomic_inc_return_acquire
264#endif
265
266#if defined(arch_atomic_inc_return_release)
267static __always_inline int
268atomic_inc_return_release(atomic_t *v)
269{
270 instrument_atomic_read_write(v, sizeof(*v));
271 return arch_atomic_inc_return_release(v);
272}
273#define atomic_inc_return_release atomic_inc_return_release
274#endif
275
276#if defined(arch_atomic_inc_return_relaxed)
277static __always_inline int
278atomic_inc_return_relaxed(atomic_t *v)
279{
280 instrument_atomic_read_write(v, sizeof(*v));
281 return arch_atomic_inc_return_relaxed(v);
282}
283#define atomic_inc_return_relaxed atomic_inc_return_relaxed
284#endif
285
286#if defined(arch_atomic_fetch_inc)
287static __always_inline int
288atomic_fetch_inc(atomic_t *v)
289{
290 instrument_atomic_read_write(v, sizeof(*v));
291 return arch_atomic_fetch_inc(v);
292}
293#define atomic_fetch_inc atomic_fetch_inc
294#endif
295
296#if defined(arch_atomic_fetch_inc_acquire)
297static __always_inline int
298atomic_fetch_inc_acquire(atomic_t *v)
299{
300 instrument_atomic_read_write(v, sizeof(*v));
301 return arch_atomic_fetch_inc_acquire(v);
302}
303#define atomic_fetch_inc_acquire atomic_fetch_inc_acquire
304#endif
305
306#if defined(arch_atomic_fetch_inc_release)
307static __always_inline int
308atomic_fetch_inc_release(atomic_t *v)
309{
310 instrument_atomic_read_write(v, sizeof(*v));
311 return arch_atomic_fetch_inc_release(v);
312}
313#define atomic_fetch_inc_release atomic_fetch_inc_release
314#endif
315
316#if defined(arch_atomic_fetch_inc_relaxed)
317static __always_inline int
318atomic_fetch_inc_relaxed(atomic_t *v)
319{
320 instrument_atomic_read_write(v, sizeof(*v));
321 return arch_atomic_fetch_inc_relaxed(v);
322}
323#define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed
324#endif
325
326#if defined(arch_atomic_dec)
327static __always_inline void
328atomic_dec(atomic_t *v)
329{
330 instrument_atomic_read_write(v, sizeof(*v));
331 arch_atomic_dec(v);
332}
333#define atomic_dec atomic_dec
334#endif
335
336#if defined(arch_atomic_dec_return)
337static __always_inline int
338atomic_dec_return(atomic_t *v)
339{
340 instrument_atomic_read_write(v, sizeof(*v));
341 return arch_atomic_dec_return(v);
342}
343#define atomic_dec_return atomic_dec_return
344#endif
345
346#if defined(arch_atomic_dec_return_acquire)
347static __always_inline int
348atomic_dec_return_acquire(atomic_t *v)
349{
350 instrument_atomic_read_write(v, sizeof(*v));
351 return arch_atomic_dec_return_acquire(v);
352}
353#define atomic_dec_return_acquire atomic_dec_return_acquire
354#endif
355
356#if defined(arch_atomic_dec_return_release)
357static __always_inline int
358atomic_dec_return_release(atomic_t *v)
359{
360 instrument_atomic_read_write(v, sizeof(*v));
361 return arch_atomic_dec_return_release(v);
362}
363#define atomic_dec_return_release atomic_dec_return_release
364#endif
365
366#if defined(arch_atomic_dec_return_relaxed)
367static __always_inline int
368atomic_dec_return_relaxed(atomic_t *v)
369{
370 instrument_atomic_read_write(v, sizeof(*v));
371 return arch_atomic_dec_return_relaxed(v);
372}
373#define atomic_dec_return_relaxed atomic_dec_return_relaxed
374#endif
375
376#if defined(arch_atomic_fetch_dec)
377static __always_inline int
378atomic_fetch_dec(atomic_t *v)
379{
380 instrument_atomic_read_write(v, sizeof(*v));
381 return arch_atomic_fetch_dec(v);
382}
383#define atomic_fetch_dec atomic_fetch_dec
384#endif
385
386#if defined(arch_atomic_fetch_dec_acquire)
387static __always_inline int
388atomic_fetch_dec_acquire(atomic_t *v)
389{
390 instrument_atomic_read_write(v, sizeof(*v));
391 return arch_atomic_fetch_dec_acquire(v);
392}
393#define atomic_fetch_dec_acquire atomic_fetch_dec_acquire
394#endif
395
396#if defined(arch_atomic_fetch_dec_release)
397static __always_inline int
398atomic_fetch_dec_release(atomic_t *v)
399{
400 instrument_atomic_read_write(v, sizeof(*v));
401 return arch_atomic_fetch_dec_release(v);
402}
403#define atomic_fetch_dec_release atomic_fetch_dec_release
404#endif
405
406#if defined(arch_atomic_fetch_dec_relaxed)
407static __always_inline int
408atomic_fetch_dec_relaxed(atomic_t *v)
409{
410 instrument_atomic_read_write(v, sizeof(*v));
411 return arch_atomic_fetch_dec_relaxed(v);
412}
413#define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed
414#endif
415
416static __always_inline void
417atomic_and(int i, atomic_t *v)
418{
419 instrument_atomic_read_write(v, sizeof(*v));
420 arch_atomic_and(i, v);
421}
422#define atomic_and atomic_and
423
424#if !defined(arch_atomic_fetch_and_relaxed) || defined(arch_atomic_fetch_and)
425static __always_inline int
426atomic_fetch_and(int i, atomic_t *v)
427{
428 instrument_atomic_read_write(v, sizeof(*v));
429 return arch_atomic_fetch_and(i, v);
430}
431#define atomic_fetch_and atomic_fetch_and
432#endif
433
434#if defined(arch_atomic_fetch_and_acquire)
435static __always_inline int
436atomic_fetch_and_acquire(int i, atomic_t *v)
437{
438 instrument_atomic_read_write(v, sizeof(*v));
439 return arch_atomic_fetch_and_acquire(i, v);
440}
441#define atomic_fetch_and_acquire atomic_fetch_and_acquire
442#endif
443
444#if defined(arch_atomic_fetch_and_release)
445static __always_inline int
446atomic_fetch_and_release(int i, atomic_t *v)
447{
448 instrument_atomic_read_write(v, sizeof(*v));
449 return arch_atomic_fetch_and_release(i, v);
450}
451#define atomic_fetch_and_release atomic_fetch_and_release
452#endif
453
454#if defined(arch_atomic_fetch_and_relaxed)
455static __always_inline int
456atomic_fetch_and_relaxed(int i, atomic_t *v)
457{
458 instrument_atomic_read_write(v, sizeof(*v));
459 return arch_atomic_fetch_and_relaxed(i, v);
460}
461#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
462#endif
463
464#if defined(arch_atomic_andnot)
465static __always_inline void
466atomic_andnot(int i, atomic_t *v)
467{
468 instrument_atomic_read_write(v, sizeof(*v));
469 arch_atomic_andnot(i, v);
470}
471#define atomic_andnot atomic_andnot
472#endif
473
474#if defined(arch_atomic_fetch_andnot)
475static __always_inline int
476atomic_fetch_andnot(int i, atomic_t *v)
477{
478 instrument_atomic_read_write(v, sizeof(*v));
479 return arch_atomic_fetch_andnot(i, v);
480}
481#define atomic_fetch_andnot atomic_fetch_andnot
482#endif
483
484#if defined(arch_atomic_fetch_andnot_acquire)
485static __always_inline int
486atomic_fetch_andnot_acquire(int i, atomic_t *v)
487{
488 instrument_atomic_read_write(v, sizeof(*v));
489 return arch_atomic_fetch_andnot_acquire(i, v);
490}
491#define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
492#endif
493
494#if defined(arch_atomic_fetch_andnot_release)
495static __always_inline int
496atomic_fetch_andnot_release(int i, atomic_t *v)
497{
498 instrument_atomic_read_write(v, sizeof(*v));
499 return arch_atomic_fetch_andnot_release(i, v);
500}
501#define atomic_fetch_andnot_release atomic_fetch_andnot_release
502#endif
503
504#if defined(arch_atomic_fetch_andnot_relaxed)
505static __always_inline int
506atomic_fetch_andnot_relaxed(int i, atomic_t *v)
507{
508 instrument_atomic_read_write(v, sizeof(*v));
509 return arch_atomic_fetch_andnot_relaxed(i, v);
510}
511#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed
512#endif
513
514static __always_inline void
515atomic_or(int i, atomic_t *v)
516{
517 instrument_atomic_read_write(v, sizeof(*v));
518 arch_atomic_or(i, v);
519}
520#define atomic_or atomic_or
521
522#if !defined(arch_atomic_fetch_or_relaxed) || defined(arch_atomic_fetch_or)
523static __always_inline int
524atomic_fetch_or(int i, atomic_t *v)
525{
526 instrument_atomic_read_write(v, sizeof(*v));
527 return arch_atomic_fetch_or(i, v);
528}
529#define atomic_fetch_or atomic_fetch_or
530#endif
531
532#if defined(arch_atomic_fetch_or_acquire)
533static __always_inline int
534atomic_fetch_or_acquire(int i, atomic_t *v)
535{
536 instrument_atomic_read_write(v, sizeof(*v));
537 return arch_atomic_fetch_or_acquire(i, v);
538}
539#define atomic_fetch_or_acquire atomic_fetch_or_acquire
540#endif
541
542#if defined(arch_atomic_fetch_or_release)
543static __always_inline int
544atomic_fetch_or_release(int i, atomic_t *v)
545{
546 instrument_atomic_read_write(v, sizeof(*v));
547 return arch_atomic_fetch_or_release(i, v);
548}
549#define atomic_fetch_or_release atomic_fetch_or_release
550#endif
551
552#if defined(arch_atomic_fetch_or_relaxed)
553static __always_inline int
554atomic_fetch_or_relaxed(int i, atomic_t *v)
555{
556 instrument_atomic_read_write(v, sizeof(*v));
557 return arch_atomic_fetch_or_relaxed(i, v);
558}
559#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed
560#endif
561
562static __always_inline void
563atomic_xor(int i, atomic_t *v)
564{
565 instrument_atomic_read_write(v, sizeof(*v));
566 arch_atomic_xor(i, v);
567}
568#define atomic_xor atomic_xor
569
570#if !defined(arch_atomic_fetch_xor_relaxed) || defined(arch_atomic_fetch_xor)
571static __always_inline int
572atomic_fetch_xor(int i, atomic_t *v)
573{
574 instrument_atomic_read_write(v, sizeof(*v));
575 return arch_atomic_fetch_xor(i, v);
576}
577#define atomic_fetch_xor atomic_fetch_xor
578#endif
579
580#if defined(arch_atomic_fetch_xor_acquire)
581static __always_inline int
582atomic_fetch_xor_acquire(int i, atomic_t *v)
583{
584 instrument_atomic_read_write(v, sizeof(*v));
585 return arch_atomic_fetch_xor_acquire(i, v);
586}
587#define atomic_fetch_xor_acquire atomic_fetch_xor_acquire
588#endif
589
590#if defined(arch_atomic_fetch_xor_release)
591static __always_inline int
592atomic_fetch_xor_release(int i, atomic_t *v)
593{
594 instrument_atomic_read_write(v, sizeof(*v));
595 return arch_atomic_fetch_xor_release(i, v);
596}
597#define atomic_fetch_xor_release atomic_fetch_xor_release
598#endif
599
600#if defined(arch_atomic_fetch_xor_relaxed)
601static __always_inline int
602atomic_fetch_xor_relaxed(int i, atomic_t *v)
603{
604 instrument_atomic_read_write(v, sizeof(*v));
605 return arch_atomic_fetch_xor_relaxed(i, v);
606}
607#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
608#endif
609
610#if !defined(arch_atomic_xchg_relaxed) || defined(arch_atomic_xchg)
611static __always_inline int
612atomic_xchg(atomic_t *v, int i)
613{
614 instrument_atomic_read_write(v, sizeof(*v));
615 return arch_atomic_xchg(v, i);
616}
617#define atomic_xchg atomic_xchg
618#endif
619
620#if defined(arch_atomic_xchg_acquire)
621static __always_inline int
622atomic_xchg_acquire(atomic_t *v, int i)
623{
624 instrument_atomic_read_write(v, sizeof(*v));
625 return arch_atomic_xchg_acquire(v, i);
626}
627#define atomic_xchg_acquire atomic_xchg_acquire
628#endif
629
630#if defined(arch_atomic_xchg_release)
631static __always_inline int
632atomic_xchg_release(atomic_t *v, int i)
633{
634 instrument_atomic_read_write(v, sizeof(*v));
635 return arch_atomic_xchg_release(v, i);
636}
637#define atomic_xchg_release atomic_xchg_release
638#endif
639
640#if defined(arch_atomic_xchg_relaxed)
641static __always_inline int
642atomic_xchg_relaxed(atomic_t *v, int i)
643{
644 instrument_atomic_read_write(v, sizeof(*v));
645 return arch_atomic_xchg_relaxed(v, i);
646}
647#define atomic_xchg_relaxed atomic_xchg_relaxed
648#endif
649
650#if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg)
651static __always_inline int
652atomic_cmpxchg(atomic_t *v, int old, int new)
653{
654 instrument_atomic_read_write(v, sizeof(*v));
655 return arch_atomic_cmpxchg(v, old, new);
656}
657#define atomic_cmpxchg atomic_cmpxchg
658#endif
659
660#if defined(arch_atomic_cmpxchg_acquire)
661static __always_inline int
662atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
663{
664 instrument_atomic_read_write(v, sizeof(*v));
665 return arch_atomic_cmpxchg_acquire(v, old, new);
666}
667#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire
668#endif
669
670#if defined(arch_atomic_cmpxchg_release)
671static __always_inline int
672atomic_cmpxchg_release(atomic_t *v, int old, int new)
673{
674 instrument_atomic_read_write(v, sizeof(*v));
675 return arch_atomic_cmpxchg_release(v, old, new);
676}
677#define atomic_cmpxchg_release atomic_cmpxchg_release
678#endif
679
680#if defined(arch_atomic_cmpxchg_relaxed)
681static __always_inline int
682atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
683{
684 instrument_atomic_read_write(v, sizeof(*v));
685 return arch_atomic_cmpxchg_relaxed(v, old, new);
686}
687#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed
688#endif
689
690#if defined(arch_atomic_try_cmpxchg)
691static __always_inline bool
692atomic_try_cmpxchg(atomic_t *v, int *old, int new)
693{
694 instrument_atomic_read_write(v, sizeof(*v));
695 instrument_atomic_read_write(old, sizeof(*old));
696 return arch_atomic_try_cmpxchg(v, old, new);
697}
698#define atomic_try_cmpxchg atomic_try_cmpxchg
699#endif
700
701#if defined(arch_atomic_try_cmpxchg_acquire)
702static __always_inline bool
703atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
704{
705 instrument_atomic_read_write(v, sizeof(*v));
706 instrument_atomic_read_write(old, sizeof(*old));
707 return arch_atomic_try_cmpxchg_acquire(v, old, new);
708}
709#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire
710#endif
711
712#if defined(arch_atomic_try_cmpxchg_release)
713static __always_inline bool
714atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
715{
716 instrument_atomic_read_write(v, sizeof(*v));
717 instrument_atomic_read_write(old, sizeof(*old));
718 return arch_atomic_try_cmpxchg_release(v, old, new);
719}
720#define atomic_try_cmpxchg_release atomic_try_cmpxchg_release
721#endif
722
723#if defined(arch_atomic_try_cmpxchg_relaxed)
724static __always_inline bool
725atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
726{
727 instrument_atomic_read_write(v, sizeof(*v));
728 instrument_atomic_read_write(old, sizeof(*old));
729 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
730}
731#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed
732#endif
733
734#if defined(arch_atomic_sub_and_test)
735static __always_inline bool
736atomic_sub_and_test(int i, atomic_t *v)
737{
738 instrument_atomic_read_write(v, sizeof(*v));
739 return arch_atomic_sub_and_test(i, v);
740}
741#define atomic_sub_and_test atomic_sub_and_test
742#endif
743
744#if defined(arch_atomic_dec_and_test)
745static __always_inline bool
746atomic_dec_and_test(atomic_t *v)
747{
748 instrument_atomic_read_write(v, sizeof(*v));
749 return arch_atomic_dec_and_test(v);
750}
751#define atomic_dec_and_test atomic_dec_and_test
752#endif
753
754#if defined(arch_atomic_inc_and_test)
755static __always_inline bool
756atomic_inc_and_test(atomic_t *v)
757{
758 instrument_atomic_read_write(v, sizeof(*v));
759 return arch_atomic_inc_and_test(v);
760}
761#define atomic_inc_and_test atomic_inc_and_test
762#endif
763
764#if defined(arch_atomic_add_negative)
765static __always_inline bool
766atomic_add_negative(int i, atomic_t *v)
767{
768 instrument_atomic_read_write(v, sizeof(*v));
769 return arch_atomic_add_negative(i, v);
770}
771#define atomic_add_negative atomic_add_negative
772#endif
773
774#if defined(arch_atomic_fetch_add_unless)
775static __always_inline int
776atomic_fetch_add_unless(atomic_t *v, int a, int u)
777{
778 instrument_atomic_read_write(v, sizeof(*v));
779 return arch_atomic_fetch_add_unless(v, a, u);
780}
781#define atomic_fetch_add_unless atomic_fetch_add_unless
782#endif
783
784#if defined(arch_atomic_add_unless)
785static __always_inline bool
786atomic_add_unless(atomic_t *v, int a, int u)
787{
788 instrument_atomic_read_write(v, sizeof(*v));
789 return arch_atomic_add_unless(v, a, u);
790}
791#define atomic_add_unless atomic_add_unless
792#endif
793
794#if defined(arch_atomic_inc_not_zero)
795static __always_inline bool
796atomic_inc_not_zero(atomic_t *v)
797{
798 instrument_atomic_read_write(v, sizeof(*v));
799 return arch_atomic_inc_not_zero(v);
800}
801#define atomic_inc_not_zero atomic_inc_not_zero
802#endif
803
804#if defined(arch_atomic_inc_unless_negative)
805static __always_inline bool
806atomic_inc_unless_negative(atomic_t *v)
807{
808 instrument_atomic_read_write(v, sizeof(*v));
809 return arch_atomic_inc_unless_negative(v);
810}
811#define atomic_inc_unless_negative atomic_inc_unless_negative
812#endif
813
814#if defined(arch_atomic_dec_unless_positive)
815static __always_inline bool
816atomic_dec_unless_positive(atomic_t *v)
817{
818 instrument_atomic_read_write(v, sizeof(*v));
819 return arch_atomic_dec_unless_positive(v);
820}
821#define atomic_dec_unless_positive atomic_dec_unless_positive
822#endif
823
824#if defined(arch_atomic_dec_if_positive)
825static __always_inline int
826atomic_dec_if_positive(atomic_t *v)
827{
828 instrument_atomic_read_write(v, sizeof(*v));
829 return arch_atomic_dec_if_positive(v);
830}
831#define atomic_dec_if_positive atomic_dec_if_positive
832#endif
833
834static __always_inline s64
835atomic64_read(const atomic64_t *v)
836{
837 instrument_atomic_read(v, sizeof(*v));
838 return arch_atomic64_read(v);
839}
840#define atomic64_read atomic64_read
841
842#if defined(arch_atomic64_read_acquire)
843static __always_inline s64
844atomic64_read_acquire(const atomic64_t *v)
845{
846 instrument_atomic_read(v, sizeof(*v));
847 return arch_atomic64_read_acquire(v);
848}
849#define atomic64_read_acquire atomic64_read_acquire
850#endif
851
852static __always_inline void
853atomic64_set(atomic64_t *v, s64 i)
854{
855 instrument_atomic_write(v, sizeof(*v));
856 arch_atomic64_set(v, i);
857}
858#define atomic64_set atomic64_set
859
860#if defined(arch_atomic64_set_release)
861static __always_inline void
862atomic64_set_release(atomic64_t *v, s64 i)
863{
864 instrument_atomic_write(v, sizeof(*v));
865 arch_atomic64_set_release(v, i);
866}
867#define atomic64_set_release atomic64_set_release
868#endif
869
870static __always_inline void
871atomic64_add(s64 i, atomic64_t *v)
872{
873 instrument_atomic_read_write(v, sizeof(*v));
874 arch_atomic64_add(i, v);
875}
876#define atomic64_add atomic64_add
877
878#if !defined(arch_atomic64_add_return_relaxed) || defined(arch_atomic64_add_return)
879static __always_inline s64
880atomic64_add_return(s64 i, atomic64_t *v)
881{
882 instrument_atomic_read_write(v, sizeof(*v));
883 return arch_atomic64_add_return(i, v);
884}
885#define atomic64_add_return atomic64_add_return
886#endif
887
888#if defined(arch_atomic64_add_return_acquire)
889static __always_inline s64
890atomic64_add_return_acquire(s64 i, atomic64_t *v)
891{
892 instrument_atomic_read_write(v, sizeof(*v));
893 return arch_atomic64_add_return_acquire(i, v);
894}
895#define atomic64_add_return_acquire atomic64_add_return_acquire
896#endif
897
898#if defined(arch_atomic64_add_return_release)
899static __always_inline s64
900atomic64_add_return_release(s64 i, atomic64_t *v)
901{
902 instrument_atomic_read_write(v, sizeof(*v));
903 return arch_atomic64_add_return_release(i, v);
904}
905#define atomic64_add_return_release atomic64_add_return_release
906#endif
907
908#if defined(arch_atomic64_add_return_relaxed)
909static __always_inline s64
910atomic64_add_return_relaxed(s64 i, atomic64_t *v)
911{
912 instrument_atomic_read_write(v, sizeof(*v));
913 return arch_atomic64_add_return_relaxed(i, v);
914}
915#define atomic64_add_return_relaxed atomic64_add_return_relaxed
916#endif
917
918#if !defined(arch_atomic64_fetch_add_relaxed) || defined(arch_atomic64_fetch_add)
919static __always_inline s64
920atomic64_fetch_add(s64 i, atomic64_t *v)
921{
922 instrument_atomic_read_write(v, sizeof(*v));
923 return arch_atomic64_fetch_add(i, v);
924}
925#define atomic64_fetch_add atomic64_fetch_add
926#endif
927
928#if defined(arch_atomic64_fetch_add_acquire)
929static __always_inline s64
930atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
931{
932 instrument_atomic_read_write(v, sizeof(*v));
933 return arch_atomic64_fetch_add_acquire(i, v);
934}
935#define atomic64_fetch_add_acquire atomic64_fetch_add_acquire
936#endif
937
938#if defined(arch_atomic64_fetch_add_release)
939static __always_inline s64
940atomic64_fetch_add_release(s64 i, atomic64_t *v)
941{
942 instrument_atomic_read_write(v, sizeof(*v));
943 return arch_atomic64_fetch_add_release(i, v);
944}
945#define atomic64_fetch_add_release atomic64_fetch_add_release
946#endif
947
948#if defined(arch_atomic64_fetch_add_relaxed)
949static __always_inline s64
950atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
951{
952 instrument_atomic_read_write(v, sizeof(*v));
953 return arch_atomic64_fetch_add_relaxed(i, v);
954}
955#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
956#endif
957
958static __always_inline void
959atomic64_sub(s64 i, atomic64_t *v)
960{
961 instrument_atomic_read_write(v, sizeof(*v));
962 arch_atomic64_sub(i, v);
963}
964#define atomic64_sub atomic64_sub
965
966#if !defined(arch_atomic64_sub_return_relaxed) || defined(arch_atomic64_sub_return)
967static __always_inline s64
968atomic64_sub_return(s64 i, atomic64_t *v)
969{
970 instrument_atomic_read_write(v, sizeof(*v));
971 return arch_atomic64_sub_return(i, v);
972}
973#define atomic64_sub_return atomic64_sub_return
974#endif
975
976#if defined(arch_atomic64_sub_return_acquire)
977static __always_inline s64
978atomic64_sub_return_acquire(s64 i, atomic64_t *v)
979{
980 instrument_atomic_read_write(v, sizeof(*v));
981 return arch_atomic64_sub_return_acquire(i, v);
982}
983#define atomic64_sub_return_acquire atomic64_sub_return_acquire
984#endif
985
986#if defined(arch_atomic64_sub_return_release)
987static __always_inline s64
988atomic64_sub_return_release(s64 i, atomic64_t *v)
989{
990 instrument_atomic_read_write(v, sizeof(*v));
991 return arch_atomic64_sub_return_release(i, v);
992}
993#define atomic64_sub_return_release atomic64_sub_return_release
994#endif
995
996#if defined(arch_atomic64_sub_return_relaxed)
997static __always_inline s64
998atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
999{
1000 instrument_atomic_read_write(v, sizeof(*v));
1001 return arch_atomic64_sub_return_relaxed(i, v);
1002}
1003#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
1004#endif
1005
1006#if !defined(arch_atomic64_fetch_sub_relaxed) || defined(arch_atomic64_fetch_sub)
1007static __always_inline s64
1008atomic64_fetch_sub(s64 i, atomic64_t *v)
1009{
1010 instrument_atomic_read_write(v, sizeof(*v));
1011 return arch_atomic64_fetch_sub(i, v);
1012}
1013#define atomic64_fetch_sub atomic64_fetch_sub
1014#endif
1015
1016#if defined(arch_atomic64_fetch_sub_acquire)
1017static __always_inline s64
1018atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1019{
1020 instrument_atomic_read_write(v, sizeof(*v));
1021 return arch_atomic64_fetch_sub_acquire(i, v);
1022}
1023#define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire
1024#endif
1025
1026#if defined(arch_atomic64_fetch_sub_release)
1027static __always_inline s64
1028atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1029{
1030 instrument_atomic_read_write(v, sizeof(*v));
1031 return arch_atomic64_fetch_sub_release(i, v);
1032}
1033#define atomic64_fetch_sub_release atomic64_fetch_sub_release
1034#endif
1035
1036#if defined(arch_atomic64_fetch_sub_relaxed)
1037static __always_inline s64
1038atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
1039{
1040 instrument_atomic_read_write(v, sizeof(*v));
1041 return arch_atomic64_fetch_sub_relaxed(i, v);
1042}
1043#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
1044#endif
1045
1046#if defined(arch_atomic64_inc)
1047static __always_inline void
1048atomic64_inc(atomic64_t *v)
1049{
1050 instrument_atomic_read_write(v, sizeof(*v));
1051 arch_atomic64_inc(v);
1052}
1053#define atomic64_inc atomic64_inc
1054#endif
1055
1056#if defined(arch_atomic64_inc_return)
1057static __always_inline s64
1058atomic64_inc_return(atomic64_t *v)
1059{
1060 instrument_atomic_read_write(v, sizeof(*v));
1061 return arch_atomic64_inc_return(v);
1062}
1063#define atomic64_inc_return atomic64_inc_return
1064#endif
1065
1066#if defined(arch_atomic64_inc_return_acquire)
1067static __always_inline s64
1068atomic64_inc_return_acquire(atomic64_t *v)
1069{
1070 instrument_atomic_read_write(v, sizeof(*v));
1071 return arch_atomic64_inc_return_acquire(v);
1072}
1073#define atomic64_inc_return_acquire atomic64_inc_return_acquire
1074#endif
1075
1076#if defined(arch_atomic64_inc_return_release)
1077static __always_inline s64
1078atomic64_inc_return_release(atomic64_t *v)
1079{
1080 instrument_atomic_read_write(v, sizeof(*v));
1081 return arch_atomic64_inc_return_release(v);
1082}
1083#define atomic64_inc_return_release atomic64_inc_return_release
1084#endif
1085
1086#if defined(arch_atomic64_inc_return_relaxed)
1087static __always_inline s64
1088atomic64_inc_return_relaxed(atomic64_t *v)
1089{
1090 instrument_atomic_read_write(v, sizeof(*v));
1091 return arch_atomic64_inc_return_relaxed(v);
1092}
1093#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
1094#endif
1095
1096#if defined(arch_atomic64_fetch_inc)
1097static __always_inline s64
1098atomic64_fetch_inc(atomic64_t *v)
1099{
1100 instrument_atomic_read_write(v, sizeof(*v));
1101 return arch_atomic64_fetch_inc(v);
1102}
1103#define atomic64_fetch_inc atomic64_fetch_inc
1104#endif
1105
1106#if defined(arch_atomic64_fetch_inc_acquire)
1107static __always_inline s64
1108atomic64_fetch_inc_acquire(atomic64_t *v)
1109{
1110 instrument_atomic_read_write(v, sizeof(*v));
1111 return arch_atomic64_fetch_inc_acquire(v);
1112}
1113#define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire
1114#endif
1115
1116#if defined(arch_atomic64_fetch_inc_release)
1117static __always_inline s64
1118atomic64_fetch_inc_release(atomic64_t *v)
1119{
1120 instrument_atomic_read_write(v, sizeof(*v));
1121 return arch_atomic64_fetch_inc_release(v);
1122}
1123#define atomic64_fetch_inc_release atomic64_fetch_inc_release
1124#endif
1125
1126#if defined(arch_atomic64_fetch_inc_relaxed)
1127static __always_inline s64
1128atomic64_fetch_inc_relaxed(atomic64_t *v)
1129{
1130 instrument_atomic_read_write(v, sizeof(*v));
1131 return arch_atomic64_fetch_inc_relaxed(v);
1132}
1133#define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed
1134#endif
1135
1136#if defined(arch_atomic64_dec)
1137static __always_inline void
1138atomic64_dec(atomic64_t *v)
1139{
1140 instrument_atomic_read_write(v, sizeof(*v));
1141 arch_atomic64_dec(v);
1142}
1143#define atomic64_dec atomic64_dec
1144#endif
1145
1146#if defined(arch_atomic64_dec_return)
1147static __always_inline s64
1148atomic64_dec_return(atomic64_t *v)
1149{
1150 instrument_atomic_read_write(v, sizeof(*v));
1151 return arch_atomic64_dec_return(v);
1152}
1153#define atomic64_dec_return atomic64_dec_return
1154#endif
1155
1156#if defined(arch_atomic64_dec_return_acquire)
1157static __always_inline s64
1158atomic64_dec_return_acquire(atomic64_t *v)
1159{
1160 instrument_atomic_read_write(v, sizeof(*v));
1161 return arch_atomic64_dec_return_acquire(v);
1162}
1163#define atomic64_dec_return_acquire atomic64_dec_return_acquire
1164#endif
1165
1166#if defined(arch_atomic64_dec_return_release)
1167static __always_inline s64
1168atomic64_dec_return_release(atomic64_t *v)
1169{
1170 instrument_atomic_read_write(v, sizeof(*v));
1171 return arch_atomic64_dec_return_release(v);
1172}
1173#define atomic64_dec_return_release atomic64_dec_return_release
1174#endif
1175
1176#if defined(arch_atomic64_dec_return_relaxed)
1177static __always_inline s64
1178atomic64_dec_return_relaxed(atomic64_t *v)
1179{
1180 instrument_atomic_read_write(v, sizeof(*v));
1181 return arch_atomic64_dec_return_relaxed(v);
1182}
1183#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
1184#endif
1185
1186#if defined(arch_atomic64_fetch_dec)
1187static __always_inline s64
1188atomic64_fetch_dec(atomic64_t *v)
1189{
1190 instrument_atomic_read_write(v, sizeof(*v));
1191 return arch_atomic64_fetch_dec(v);
1192}
1193#define atomic64_fetch_dec atomic64_fetch_dec
1194#endif
1195
1196#if defined(arch_atomic64_fetch_dec_acquire)
1197static __always_inline s64
1198atomic64_fetch_dec_acquire(atomic64_t *v)
1199{
1200 instrument_atomic_read_write(v, sizeof(*v));
1201 return arch_atomic64_fetch_dec_acquire(v);
1202}
1203#define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire
1204#endif
1205
1206#if defined(arch_atomic64_fetch_dec_release)
1207static __always_inline s64
1208atomic64_fetch_dec_release(atomic64_t *v)
1209{
1210 instrument_atomic_read_write(v, sizeof(*v));
1211 return arch_atomic64_fetch_dec_release(v);
1212}
1213#define atomic64_fetch_dec_release atomic64_fetch_dec_release
1214#endif
1215
1216#if defined(arch_atomic64_fetch_dec_relaxed)
1217static __always_inline s64
1218atomic64_fetch_dec_relaxed(atomic64_t *v)
1219{
1220 instrument_atomic_read_write(v, sizeof(*v));
1221 return arch_atomic64_fetch_dec_relaxed(v);
1222}
1223#define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed
1224#endif
1225
1226static __always_inline void
1227atomic64_and(s64 i, atomic64_t *v)
1228{
1229 instrument_atomic_read_write(v, sizeof(*v));
1230 arch_atomic64_and(i, v);
1231}
1232#define atomic64_and atomic64_and
1233
1234#if !defined(arch_atomic64_fetch_and_relaxed) || defined(arch_atomic64_fetch_and)
1235static __always_inline s64
1236atomic64_fetch_and(s64 i, atomic64_t *v)
1237{
1238 instrument_atomic_read_write(v, sizeof(*v));
1239 return arch_atomic64_fetch_and(i, v);
1240}
1241#define atomic64_fetch_and atomic64_fetch_and
1242#endif
1243
1244#if defined(arch_atomic64_fetch_and_acquire)
1245static __always_inline s64
1246atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1247{
1248 instrument_atomic_read_write(v, sizeof(*v));
1249 return arch_atomic64_fetch_and_acquire(i, v);
1250}
1251#define atomic64_fetch_and_acquire atomic64_fetch_and_acquire
1252#endif
1253
1254#if defined(arch_atomic64_fetch_and_release)
1255static __always_inline s64
1256atomic64_fetch_and_release(s64 i, atomic64_t *v)
1257{
1258 instrument_atomic_read_write(v, sizeof(*v));
1259 return arch_atomic64_fetch_and_release(i, v);
1260}
1261#define atomic64_fetch_and_release atomic64_fetch_and_release
1262#endif
1263
1264#if defined(arch_atomic64_fetch_and_relaxed)
1265static __always_inline s64
1266atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
1267{
1268 instrument_atomic_read_write(v, sizeof(*v));
1269 return arch_atomic64_fetch_and_relaxed(i, v);
1270}
1271#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
1272#endif
1273
1274#if defined(arch_atomic64_andnot)
1275static __always_inline void
1276atomic64_andnot(s64 i, atomic64_t *v)
1277{
1278 instrument_atomic_read_write(v, sizeof(*v));
1279 arch_atomic64_andnot(i, v);
1280}
1281#define atomic64_andnot atomic64_andnot
1282#endif
1283
1284#if defined(arch_atomic64_fetch_andnot)
1285static __always_inline s64
1286atomic64_fetch_andnot(s64 i, atomic64_t *v)
1287{
1288 instrument_atomic_read_write(v, sizeof(*v));
1289 return arch_atomic64_fetch_andnot(i, v);
1290}
1291#define atomic64_fetch_andnot atomic64_fetch_andnot
1292#endif
1293
1294#if defined(arch_atomic64_fetch_andnot_acquire)
1295static __always_inline s64
1296atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1297{
1298 instrument_atomic_read_write(v, sizeof(*v));
1299 return arch_atomic64_fetch_andnot_acquire(i, v);
1300}
1301#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
1302#endif
1303
1304#if defined(arch_atomic64_fetch_andnot_release)
1305static __always_inline s64
1306atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1307{
1308 instrument_atomic_read_write(v, sizeof(*v));
1309 return arch_atomic64_fetch_andnot_release(i, v);
1310}
1311#define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
1312#endif
1313
1314#if defined(arch_atomic64_fetch_andnot_relaxed)
1315static __always_inline s64
1316atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1317{
1318 instrument_atomic_read_write(v, sizeof(*v));
1319 return arch_atomic64_fetch_andnot_relaxed(i, v);
1320}
1321#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed
1322#endif
1323
1324static __always_inline void
1325atomic64_or(s64 i, atomic64_t *v)
1326{
1327 instrument_atomic_read_write(v, sizeof(*v));
1328 arch_atomic64_or(i, v);
1329}
1330#define atomic64_or atomic64_or
1331
1332#if !defined(arch_atomic64_fetch_or_relaxed) || defined(arch_atomic64_fetch_or)
1333static __always_inline s64
1334atomic64_fetch_or(s64 i, atomic64_t *v)
1335{
1336 instrument_atomic_read_write(v, sizeof(*v));
1337 return arch_atomic64_fetch_or(i, v);
1338}
1339#define atomic64_fetch_or atomic64_fetch_or
1340#endif
1341
1342#if defined(arch_atomic64_fetch_or_acquire)
1343static __always_inline s64
1344atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1345{
1346 instrument_atomic_read_write(v, sizeof(*v));
1347 return arch_atomic64_fetch_or_acquire(i, v);
1348}
1349#define atomic64_fetch_or_acquire atomic64_fetch_or_acquire
1350#endif
1351
1352#if defined(arch_atomic64_fetch_or_release)
1353static __always_inline s64
1354atomic64_fetch_or_release(s64 i, atomic64_t *v)
1355{
1356 instrument_atomic_read_write(v, sizeof(*v));
1357 return arch_atomic64_fetch_or_release(i, v);
1358}
1359#define atomic64_fetch_or_release atomic64_fetch_or_release
1360#endif
1361
1362#if defined(arch_atomic64_fetch_or_relaxed)
1363static __always_inline s64
1364atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1365{
1366 instrument_atomic_read_write(v, sizeof(*v));
1367 return arch_atomic64_fetch_or_relaxed(i, v);
1368}
1369#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed
1370#endif
1371
1372static __always_inline void
1373atomic64_xor(s64 i, atomic64_t *v)
1374{
1375 instrument_atomic_read_write(v, sizeof(*v));
1376 arch_atomic64_xor(i, v);
1377}
1378#define atomic64_xor atomic64_xor
1379
1380#if !defined(arch_atomic64_fetch_xor_relaxed) || defined(arch_atomic64_fetch_xor)
1381static __always_inline s64
1382atomic64_fetch_xor(s64 i, atomic64_t *v)
1383{
1384 instrument_atomic_read_write(v, sizeof(*v));
1385 return arch_atomic64_fetch_xor(i, v);
1386}
1387#define atomic64_fetch_xor atomic64_fetch_xor
1388#endif
1389
1390#if defined(arch_atomic64_fetch_xor_acquire)
1391static __always_inline s64
1392atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1393{
1394 instrument_atomic_read_write(v, sizeof(*v));
1395 return arch_atomic64_fetch_xor_acquire(i, v);
1396}
1397#define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire
1398#endif
1399
1400#if defined(arch_atomic64_fetch_xor_release)
1401static __always_inline s64
1402atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1403{
1404 instrument_atomic_read_write(v, sizeof(*v));
1405 return arch_atomic64_fetch_xor_release(i, v);
1406}
1407#define atomic64_fetch_xor_release atomic64_fetch_xor_release
1408#endif
1409
1410#if defined(arch_atomic64_fetch_xor_relaxed)
1411static __always_inline s64
1412atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1413{
1414 instrument_atomic_read_write(v, sizeof(*v));
1415 return arch_atomic64_fetch_xor_relaxed(i, v);
1416}
1417#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
1418#endif
1419
1420#if !defined(arch_atomic64_xchg_relaxed) || defined(arch_atomic64_xchg)
1421static __always_inline s64
1422atomic64_xchg(atomic64_t *v, s64 i)
1423{
1424 instrument_atomic_read_write(v, sizeof(*v));
1425 return arch_atomic64_xchg(v, i);
1426}
1427#define atomic64_xchg atomic64_xchg
1428#endif
1429
1430#if defined(arch_atomic64_xchg_acquire)
1431static __always_inline s64
1432atomic64_xchg_acquire(atomic64_t *v, s64 i)
1433{
1434 instrument_atomic_read_write(v, sizeof(*v));
1435 return arch_atomic64_xchg_acquire(v, i);
1436}
1437#define atomic64_xchg_acquire atomic64_xchg_acquire
1438#endif
1439
1440#if defined(arch_atomic64_xchg_release)
1441static __always_inline s64
1442atomic64_xchg_release(atomic64_t *v, s64 i)
1443{
1444 instrument_atomic_read_write(v, sizeof(*v));
1445 return arch_atomic64_xchg_release(v, i);
1446}
1447#define atomic64_xchg_release atomic64_xchg_release
1448#endif
1449
1450#if defined(arch_atomic64_xchg_relaxed)
1451static __always_inline s64
1452atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1453{
1454 instrument_atomic_read_write(v, sizeof(*v));
1455 return arch_atomic64_xchg_relaxed(v, i);
1456}
1457#define atomic64_xchg_relaxed atomic64_xchg_relaxed
1458#endif
1459
1460#if !defined(arch_atomic64_cmpxchg_relaxed) || defined(arch_atomic64_cmpxchg)
1461static __always_inline s64
1462atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1463{
1464 instrument_atomic_read_write(v, sizeof(*v));
1465 return arch_atomic64_cmpxchg(v, old, new);
1466}
1467#define atomic64_cmpxchg atomic64_cmpxchg
1468#endif
1469
1470#if defined(arch_atomic64_cmpxchg_acquire)
1471static __always_inline s64
1472atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1473{
1474 instrument_atomic_read_write(v, sizeof(*v));
1475 return arch_atomic64_cmpxchg_acquire(v, old, new);
1476}
1477#define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire
1478#endif
1479
1480#if defined(arch_atomic64_cmpxchg_release)
1481static __always_inline s64
1482atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1483{
1484 instrument_atomic_read_write(v, sizeof(*v));
1485 return arch_atomic64_cmpxchg_release(v, old, new);
1486}
1487#define atomic64_cmpxchg_release atomic64_cmpxchg_release
1488#endif
1489
1490#if defined(arch_atomic64_cmpxchg_relaxed)
1491static __always_inline s64
1492atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1493{
1494 instrument_atomic_read_write(v, sizeof(*v));
1495 return arch_atomic64_cmpxchg_relaxed(v, old, new);
1496}
1497#define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed
1498#endif
1499
1500#if defined(arch_atomic64_try_cmpxchg)
1501static __always_inline bool
1502atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1503{
1504 instrument_atomic_read_write(v, sizeof(*v));
1505 instrument_atomic_read_write(old, sizeof(*old));
1506 return arch_atomic64_try_cmpxchg(v, old, new);
1507}
1508#define atomic64_try_cmpxchg atomic64_try_cmpxchg
1509#endif
1510
1511#if defined(arch_atomic64_try_cmpxchg_acquire)
1512static __always_inline bool
1513atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1514{
1515 instrument_atomic_read_write(v, sizeof(*v));
1516 instrument_atomic_read_write(old, sizeof(*old));
1517 return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1518}
1519#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire
1520#endif
1521
1522#if defined(arch_atomic64_try_cmpxchg_release)
1523static __always_inline bool
1524atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1525{
1526 instrument_atomic_read_write(v, sizeof(*v));
1527 instrument_atomic_read_write(old, sizeof(*old));
1528 return arch_atomic64_try_cmpxchg_release(v, old, new);
1529}
1530#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release
1531#endif
1532
1533#if defined(arch_atomic64_try_cmpxchg_relaxed)
1534static __always_inline bool
1535atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1536{
1537 instrument_atomic_read_write(v, sizeof(*v));
1538 instrument_atomic_read_write(old, sizeof(*old));
1539 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1540}
1541#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed
1542#endif
1543
1544#if defined(arch_atomic64_sub_and_test)
1545static __always_inline bool
1546atomic64_sub_and_test(s64 i, atomic64_t *v)
1547{
1548 instrument_atomic_read_write(v, sizeof(*v));
1549 return arch_atomic64_sub_and_test(i, v);
1550}
1551#define atomic64_sub_and_test atomic64_sub_and_test
1552#endif
1553
1554#if defined(arch_atomic64_dec_and_test)
1555static __always_inline bool
1556atomic64_dec_and_test(atomic64_t *v)
1557{
1558 instrument_atomic_read_write(v, sizeof(*v));
1559 return arch_atomic64_dec_and_test(v);
1560}
1561#define atomic64_dec_and_test atomic64_dec_and_test
1562#endif
1563
1564#if defined(arch_atomic64_inc_and_test)
1565static __always_inline bool
1566atomic64_inc_and_test(atomic64_t *v)
1567{
1568 instrument_atomic_read_write(v, sizeof(*v));
1569 return arch_atomic64_inc_and_test(v);
1570}
1571#define atomic64_inc_and_test atomic64_inc_and_test
1572#endif
1573
1574#if defined(arch_atomic64_add_negative)
1575static __always_inline bool
1576atomic64_add_negative(s64 i, atomic64_t *v)
1577{
1578 instrument_atomic_read_write(v, sizeof(*v));
1579 return arch_atomic64_add_negative(i, v);
1580}
1581#define atomic64_add_negative atomic64_add_negative
1582#endif
1583
1584#if defined(arch_atomic64_fetch_add_unless)
1585static __always_inline s64
1586atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1587{
1588 instrument_atomic_read_write(v, sizeof(*v));
1589 return arch_atomic64_fetch_add_unless(v, a, u);
1590}
1591#define atomic64_fetch_add_unless atomic64_fetch_add_unless
1592#endif
1593
1594#if defined(arch_atomic64_add_unless)
1595static __always_inline bool
1596atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1597{
1598 instrument_atomic_read_write(v, sizeof(*v));
1599 return arch_atomic64_add_unless(v, a, u);
1600}
1601#define atomic64_add_unless atomic64_add_unless
1602#endif
1603
1604#if defined(arch_atomic64_inc_not_zero)
1605static __always_inline bool
1606atomic64_inc_not_zero(atomic64_t *v)
1607{
1608 instrument_atomic_read_write(v, sizeof(*v));
1609 return arch_atomic64_inc_not_zero(v);
1610}
1611#define atomic64_inc_not_zero atomic64_inc_not_zero
1612#endif
1613
1614#if defined(arch_atomic64_inc_unless_negative)
1615static __always_inline bool
1616atomic64_inc_unless_negative(atomic64_t *v)
1617{
1618 instrument_atomic_read_write(v, sizeof(*v));
1619 return arch_atomic64_inc_unless_negative(v);
1620}
1621#define atomic64_inc_unless_negative atomic64_inc_unless_negative
1622#endif
1623
1624#if defined(arch_atomic64_dec_unless_positive)
1625static __always_inline bool
1626atomic64_dec_unless_positive(atomic64_t *v)
1627{
1628 instrument_atomic_read_write(v, sizeof(*v));
1629 return arch_atomic64_dec_unless_positive(v);
1630}
1631#define atomic64_dec_unless_positive atomic64_dec_unless_positive
1632#endif
1633
1634#if defined(arch_atomic64_dec_if_positive)
1635static __always_inline s64
1636atomic64_dec_if_positive(atomic64_t *v)
1637{
1638 instrument_atomic_read_write(v, sizeof(*v));
1639 return arch_atomic64_dec_if_positive(v);
1640}
1641#define atomic64_dec_if_positive atomic64_dec_if_positive
1642#endif
1643
1644#if !defined(arch_xchg_relaxed) || defined(arch_xchg)
1645#define xchg(ptr, ...) \
1646({ \
1647 typeof(ptr) __ai_ptr = (ptr); \
1648 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1649 arch_xchg(__ai_ptr, __VA_ARGS__); \
1650})
1651#endif
1652
1653#if defined(arch_xchg_acquire)
1654#define xchg_acquire(ptr, ...) \
1655({ \
1656 typeof(ptr) __ai_ptr = (ptr); \
1657 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1658 arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1659})
1660#endif
1661
1662#if defined(arch_xchg_release)
1663#define xchg_release(ptr, ...) \
1664({ \
1665 typeof(ptr) __ai_ptr = (ptr); \
1666 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1667 arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1668})
1669#endif
1670
1671#if defined(arch_xchg_relaxed)
1672#define xchg_relaxed(ptr, ...) \
1673({ \
1674 typeof(ptr) __ai_ptr = (ptr); \
1675 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1676 arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1677})
1678#endif
1679
1680#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg)
1681#define cmpxchg(ptr, ...) \
1682({ \
1683 typeof(ptr) __ai_ptr = (ptr); \
1684 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1685 arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1686})
1687#endif
1688
1689#if defined(arch_cmpxchg_acquire)
1690#define cmpxchg_acquire(ptr, ...) \
1691({ \
1692 typeof(ptr) __ai_ptr = (ptr); \
1693 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1694 arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1695})
1696#endif
1697
1698#if defined(arch_cmpxchg_release)
1699#define cmpxchg_release(ptr, ...) \
1700({ \
1701 typeof(ptr) __ai_ptr = (ptr); \
1702 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1703 arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1704})
1705#endif
1706
1707#if defined(arch_cmpxchg_relaxed)
1708#define cmpxchg_relaxed(ptr, ...) \
1709({ \
1710 typeof(ptr) __ai_ptr = (ptr); \
1711 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1712 arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1713})
1714#endif
1715
1716#if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64)
1717#define cmpxchg64(ptr, ...) \
1718({ \
1719 typeof(ptr) __ai_ptr = (ptr); \
1720 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1721 arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1722})
1723#endif
1724
1725#if defined(arch_cmpxchg64_acquire)
1726#define cmpxchg64_acquire(ptr, ...) \
1727({ \
1728 typeof(ptr) __ai_ptr = (ptr); \
1729 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1730 arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1731})
1732#endif
1733
1734#if defined(arch_cmpxchg64_release)
1735#define cmpxchg64_release(ptr, ...) \
1736({ \
1737 typeof(ptr) __ai_ptr = (ptr); \
1738 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1739 arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1740})
1741#endif
1742
1743#if defined(arch_cmpxchg64_relaxed)
1744#define cmpxchg64_relaxed(ptr, ...) \
1745({ \
1746 typeof(ptr) __ai_ptr = (ptr); \
1747 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1748 arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1749})
1750#endif
1751
1752#if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg)
1753#define try_cmpxchg(ptr, oldp, ...) \
1754({ \
1755 typeof(ptr) __ai_ptr = (ptr); \
1756 typeof(oldp) __ai_oldp = (oldp); \
1757 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1758 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1759 arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1760})
1761#endif
1762
1763#if defined(arch_try_cmpxchg_acquire)
1764#define try_cmpxchg_acquire(ptr, oldp, ...) \
1765({ \
1766 typeof(ptr) __ai_ptr = (ptr); \
1767 typeof(oldp) __ai_oldp = (oldp); \
1768 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1769 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1770 arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1771})
1772#endif
1773
1774#if defined(arch_try_cmpxchg_release)
1775#define try_cmpxchg_release(ptr, oldp, ...) \
1776({ \
1777 typeof(ptr) __ai_ptr = (ptr); \
1778 typeof(oldp) __ai_oldp = (oldp); \
1779 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1780 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1781 arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1782})
1783#endif
1784
1785#if defined(arch_try_cmpxchg_relaxed)
1786#define try_cmpxchg_relaxed(ptr, oldp, ...) \
1787({ \
1788 typeof(ptr) __ai_ptr = (ptr); \
1789 typeof(oldp) __ai_oldp = (oldp); \
1790 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1791 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1792 arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1793})
1794#endif
1795
1796#define cmpxchg_local(ptr, ...) \
1797({ \
1798 typeof(ptr) __ai_ptr = (ptr); \
1799 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1800 arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
1801})
1802
1803#define cmpxchg64_local(ptr, ...) \
1804({ \
1805 typeof(ptr) __ai_ptr = (ptr); \
1806 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1807 arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
1808})
1809
1810#define sync_cmpxchg(ptr, ...) \
1811({ \
1812 typeof(ptr) __ai_ptr = (ptr); \
1813 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1814 arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
1815})
1816
1817#define cmpxchg_double(ptr, ...) \
1818({ \
1819 typeof(ptr) __ai_ptr = (ptr); \
1820 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1821 arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
1822})
1823
1824
1825#define cmpxchg_double_local(ptr, ...) \
1826({ \
1827 typeof(ptr) __ai_ptr = (ptr); \
1828 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1829 arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
1830})
1831
1832#endif
1833
1834
