1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18#define _LINUX_ATOMIC_INSTRUMENTED_H
19
20#include <linux/build_bug.h>
21#include <linux/compiler.h>
22#include <linux/instrumented.h>
23
24static __always_inline int
25atomic_read(const atomic_t *v)
26{
27 instrument_atomic_read(v, sizeof(*v));
28 return arch_atomic_read(v);
29}
30
31static __always_inline int
32atomic_read_acquire(const atomic_t *v)
33{
34 instrument_atomic_read(v, sizeof(*v));
35 return arch_atomic_read_acquire(v);
36}
37
38static __always_inline void
39atomic_set(atomic_t *v, int i)
40{
41 instrument_atomic_write(v, sizeof(*v));
42 arch_atomic_set(v, i);
43}
44
45static __always_inline void
46atomic_set_release(atomic_t *v, int i)
47{
48 instrument_atomic_write(v, sizeof(*v));
49 arch_atomic_set_release(v, i);
50}
51
52static __always_inline void
53atomic_add(int i, atomic_t *v)
54{
55 instrument_atomic_read_write(v, sizeof(*v));
56 arch_atomic_add(i, v);
57}
58
59static __always_inline int
60atomic_add_return(int i, atomic_t *v)
61{
62 instrument_atomic_read_write(v, sizeof(*v));
63 return arch_atomic_add_return(i, v);
64}
65
66static __always_inline int
67atomic_add_return_acquire(int i, atomic_t *v)
68{
69 instrument_atomic_read_write(v, sizeof(*v));
70 return arch_atomic_add_return_acquire(i, v);
71}
72
73static __always_inline int
74atomic_add_return_release(int i, atomic_t *v)
75{
76 instrument_atomic_read_write(v, sizeof(*v));
77 return arch_atomic_add_return_release(i, v);
78}
79
80static __always_inline int
81atomic_add_return_relaxed(int i, atomic_t *v)
82{
83 instrument_atomic_read_write(v, sizeof(*v));
84 return arch_atomic_add_return_relaxed(i, v);
85}
86
87static __always_inline int
88atomic_fetch_add(int i, atomic_t *v)
89{
90 instrument_atomic_read_write(v, sizeof(*v));
91 return arch_atomic_fetch_add(i, v);
92}
93
94static __always_inline int
95atomic_fetch_add_acquire(int i, atomic_t *v)
96{
97 instrument_atomic_read_write(v, sizeof(*v));
98 return arch_atomic_fetch_add_acquire(i, v);
99}
100
101static __always_inline int
102atomic_fetch_add_release(int i, atomic_t *v)
103{
104 instrument_atomic_read_write(v, sizeof(*v));
105 return arch_atomic_fetch_add_release(i, v);
106}
107
108static __always_inline int
109atomic_fetch_add_relaxed(int i, atomic_t *v)
110{
111 instrument_atomic_read_write(v, sizeof(*v));
112 return arch_atomic_fetch_add_relaxed(i, v);
113}
114
115static __always_inline void
116atomic_sub(int i, atomic_t *v)
117{
118 instrument_atomic_read_write(v, sizeof(*v));
119 arch_atomic_sub(i, v);
120}
121
122static __always_inline int
123atomic_sub_return(int i, atomic_t *v)
124{
125 instrument_atomic_read_write(v, sizeof(*v));
126 return arch_atomic_sub_return(i, v);
127}
128
129static __always_inline int
130atomic_sub_return_acquire(int i, atomic_t *v)
131{
132 instrument_atomic_read_write(v, sizeof(*v));
133 return arch_atomic_sub_return_acquire(i, v);
134}
135
136static __always_inline int
137atomic_sub_return_release(int i, atomic_t *v)
138{
139 instrument_atomic_read_write(v, sizeof(*v));
140 return arch_atomic_sub_return_release(i, v);
141}
142
143static __always_inline int
144atomic_sub_return_relaxed(int i, atomic_t *v)
145{
146 instrument_atomic_read_write(v, sizeof(*v));
147 return arch_atomic_sub_return_relaxed(i, v);
148}
149
150static __always_inline int
151atomic_fetch_sub(int i, atomic_t *v)
152{
153 instrument_atomic_read_write(v, sizeof(*v));
154 return arch_atomic_fetch_sub(i, v);
155}
156
157static __always_inline int
158atomic_fetch_sub_acquire(int i, atomic_t *v)
159{
160 instrument_atomic_read_write(v, sizeof(*v));
161 return arch_atomic_fetch_sub_acquire(i, v);
162}
163
164static __always_inline int
165atomic_fetch_sub_release(int i, atomic_t *v)
166{
167 instrument_atomic_read_write(v, sizeof(*v));
168 return arch_atomic_fetch_sub_release(i, v);
169}
170
171static __always_inline int
172atomic_fetch_sub_relaxed(int i, atomic_t *v)
173{
174 instrument_atomic_read_write(v, sizeof(*v));
175 return arch_atomic_fetch_sub_relaxed(i, v);
176}
177
178static __always_inline void
179atomic_inc(atomic_t *v)
180{
181 instrument_atomic_read_write(v, sizeof(*v));
182 arch_atomic_inc(v);
183}
184
185static __always_inline int
186atomic_inc_return(atomic_t *v)
187{
188 instrument_atomic_read_write(v, sizeof(*v));
189 return arch_atomic_inc_return(v);
190}
191
192static __always_inline int
193atomic_inc_return_acquire(atomic_t *v)
194{
195 instrument_atomic_read_write(v, sizeof(*v));
196 return arch_atomic_inc_return_acquire(v);
197}
198
199static __always_inline int
200atomic_inc_return_release(atomic_t *v)
201{
202 instrument_atomic_read_write(v, sizeof(*v));
203 return arch_atomic_inc_return_release(v);
204}
205
206static __always_inline int
207atomic_inc_return_relaxed(atomic_t *v)
208{
209 instrument_atomic_read_write(v, sizeof(*v));
210 return arch_atomic_inc_return_relaxed(v);
211}
212
213static __always_inline int
214atomic_fetch_inc(atomic_t *v)
215{
216 instrument_atomic_read_write(v, sizeof(*v));
217 return arch_atomic_fetch_inc(v);
218}
219
220static __always_inline int
221atomic_fetch_inc_acquire(atomic_t *v)
222{
223 instrument_atomic_read_write(v, sizeof(*v));
224 return arch_atomic_fetch_inc_acquire(v);
225}
226
227static __always_inline int
228atomic_fetch_inc_release(atomic_t *v)
229{
230 instrument_atomic_read_write(v, sizeof(*v));
231 return arch_atomic_fetch_inc_release(v);
232}
233
234static __always_inline int
235atomic_fetch_inc_relaxed(atomic_t *v)
236{
237 instrument_atomic_read_write(v, sizeof(*v));
238 return arch_atomic_fetch_inc_relaxed(v);
239}
240
241static __always_inline void
242atomic_dec(atomic_t *v)
243{
244 instrument_atomic_read_write(v, sizeof(*v));
245 arch_atomic_dec(v);
246}
247
248static __always_inline int
249atomic_dec_return(atomic_t *v)
250{
251 instrument_atomic_read_write(v, sizeof(*v));
252 return arch_atomic_dec_return(v);
253}
254
255static __always_inline int
256atomic_dec_return_acquire(atomic_t *v)
257{
258 instrument_atomic_read_write(v, sizeof(*v));
259 return arch_atomic_dec_return_acquire(v);
260}
261
262static __always_inline int
263atomic_dec_return_release(atomic_t *v)
264{
265 instrument_atomic_read_write(v, sizeof(*v));
266 return arch_atomic_dec_return_release(v);
267}
268
269static __always_inline int
270atomic_dec_return_relaxed(atomic_t *v)
271{
272 instrument_atomic_read_write(v, sizeof(*v));
273 return arch_atomic_dec_return_relaxed(v);
274}
275
276static __always_inline int
277atomic_fetch_dec(atomic_t *v)
278{
279 instrument_atomic_read_write(v, sizeof(*v));
280 return arch_atomic_fetch_dec(v);
281}
282
283static __always_inline int
284atomic_fetch_dec_acquire(atomic_t *v)
285{
286 instrument_atomic_read_write(v, sizeof(*v));
287 return arch_atomic_fetch_dec_acquire(v);
288}
289
290static __always_inline int
291atomic_fetch_dec_release(atomic_t *v)
292{
293 instrument_atomic_read_write(v, sizeof(*v));
294 return arch_atomic_fetch_dec_release(v);
295}
296
297static __always_inline int
298atomic_fetch_dec_relaxed(atomic_t *v)
299{
300 instrument_atomic_read_write(v, sizeof(*v));
301 return arch_atomic_fetch_dec_relaxed(v);
302}
303
304static __always_inline void
305atomic_and(int i, atomic_t *v)
306{
307 instrument_atomic_read_write(v, sizeof(*v));
308 arch_atomic_and(i, v);
309}
310
311static __always_inline int
312atomic_fetch_and(int i, atomic_t *v)
313{
314 instrument_atomic_read_write(v, sizeof(*v));
315 return arch_atomic_fetch_and(i, v);
316}
317
318static __always_inline int
319atomic_fetch_and_acquire(int i, atomic_t *v)
320{
321 instrument_atomic_read_write(v, sizeof(*v));
322 return arch_atomic_fetch_and_acquire(i, v);
323}
324
325static __always_inline int
326atomic_fetch_and_release(int i, atomic_t *v)
327{
328 instrument_atomic_read_write(v, sizeof(*v));
329 return arch_atomic_fetch_and_release(i, v);
330}
331
332static __always_inline int
333atomic_fetch_and_relaxed(int i, atomic_t *v)
334{
335 instrument_atomic_read_write(v, sizeof(*v));
336 return arch_atomic_fetch_and_relaxed(i, v);
337}
338
339static __always_inline void
340atomic_andnot(int i, atomic_t *v)
341{
342 instrument_atomic_read_write(v, sizeof(*v));
343 arch_atomic_andnot(i, v);
344}
345
346static __always_inline int
347atomic_fetch_andnot(int i, atomic_t *v)
348{
349 instrument_atomic_read_write(v, sizeof(*v));
350 return arch_atomic_fetch_andnot(i, v);
351}
352
353static __always_inline int
354atomic_fetch_andnot_acquire(int i, atomic_t *v)
355{
356 instrument_atomic_read_write(v, sizeof(*v));
357 return arch_atomic_fetch_andnot_acquire(i, v);
358}
359
360static __always_inline int
361atomic_fetch_andnot_release(int i, atomic_t *v)
362{
363 instrument_atomic_read_write(v, sizeof(*v));
364 return arch_atomic_fetch_andnot_release(i, v);
365}
366
367static __always_inline int
368atomic_fetch_andnot_relaxed(int i, atomic_t *v)
369{
370 instrument_atomic_read_write(v, sizeof(*v));
371 return arch_atomic_fetch_andnot_relaxed(i, v);
372}
373
374static __always_inline void
375atomic_or(int i, atomic_t *v)
376{
377 instrument_atomic_read_write(v, sizeof(*v));
378 arch_atomic_or(i, v);
379}
380
381static __always_inline int
382atomic_fetch_or(int i, atomic_t *v)
383{
384 instrument_atomic_read_write(v, sizeof(*v));
385 return arch_atomic_fetch_or(i, v);
386}
387
388static __always_inline int
389atomic_fetch_or_acquire(int i, atomic_t *v)
390{
391 instrument_atomic_read_write(v, sizeof(*v));
392 return arch_atomic_fetch_or_acquire(i, v);
393}
394
395static __always_inline int
396atomic_fetch_or_release(int i, atomic_t *v)
397{
398 instrument_atomic_read_write(v, sizeof(*v));
399 return arch_atomic_fetch_or_release(i, v);
400}
401
402static __always_inline int
403atomic_fetch_or_relaxed(int i, atomic_t *v)
404{
405 instrument_atomic_read_write(v, sizeof(*v));
406 return arch_atomic_fetch_or_relaxed(i, v);
407}
408
409static __always_inline void
410atomic_xor(int i, atomic_t *v)
411{
412 instrument_atomic_read_write(v, sizeof(*v));
413 arch_atomic_xor(i, v);
414}
415
416static __always_inline int
417atomic_fetch_xor(int i, atomic_t *v)
418{
419 instrument_atomic_read_write(v, sizeof(*v));
420 return arch_atomic_fetch_xor(i, v);
421}
422
423static __always_inline int
424atomic_fetch_xor_acquire(int i, atomic_t *v)
425{
426 instrument_atomic_read_write(v, sizeof(*v));
427 return arch_atomic_fetch_xor_acquire(i, v);
428}
429
430static __always_inline int
431atomic_fetch_xor_release(int i, atomic_t *v)
432{
433 instrument_atomic_read_write(v, sizeof(*v));
434 return arch_atomic_fetch_xor_release(i, v);
435}
436
437static __always_inline int
438atomic_fetch_xor_relaxed(int i, atomic_t *v)
439{
440 instrument_atomic_read_write(v, sizeof(*v));
441 return arch_atomic_fetch_xor_relaxed(i, v);
442}
443
444static __always_inline int
445atomic_xchg(atomic_t *v, int i)
446{
447 instrument_atomic_read_write(v, sizeof(*v));
448 return arch_atomic_xchg(v, i);
449}
450
451static __always_inline int
452atomic_xchg_acquire(atomic_t *v, int i)
453{
454 instrument_atomic_read_write(v, sizeof(*v));
455 return arch_atomic_xchg_acquire(v, i);
456}
457
458static __always_inline int
459atomic_xchg_release(atomic_t *v, int i)
460{
461 instrument_atomic_read_write(v, sizeof(*v));
462 return arch_atomic_xchg_release(v, i);
463}
464
465static __always_inline int
466atomic_xchg_relaxed(atomic_t *v, int i)
467{
468 instrument_atomic_read_write(v, sizeof(*v));
469 return arch_atomic_xchg_relaxed(v, i);
470}
471
472static __always_inline int
473atomic_cmpxchg(atomic_t *v, int old, int new)
474{
475 instrument_atomic_read_write(v, sizeof(*v));
476 return arch_atomic_cmpxchg(v, old, new);
477}
478
479static __always_inline int
480atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
481{
482 instrument_atomic_read_write(v, sizeof(*v));
483 return arch_atomic_cmpxchg_acquire(v, old, new);
484}
485
486static __always_inline int
487atomic_cmpxchg_release(atomic_t *v, int old, int new)
488{
489 instrument_atomic_read_write(v, sizeof(*v));
490 return arch_atomic_cmpxchg_release(v, old, new);
491}
492
493static __always_inline int
494atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
495{
496 instrument_atomic_read_write(v, sizeof(*v));
497 return arch_atomic_cmpxchg_relaxed(v, old, new);
498}
499
500static __always_inline bool
501atomic_try_cmpxchg(atomic_t *v, int *old, int new)
502{
503 instrument_atomic_read_write(v, sizeof(*v));
504 instrument_atomic_read_write(old, sizeof(*old));
505 return arch_atomic_try_cmpxchg(v, old, new);
506}
507
508static __always_inline bool
509atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
510{
511 instrument_atomic_read_write(v, sizeof(*v));
512 instrument_atomic_read_write(old, sizeof(*old));
513 return arch_atomic_try_cmpxchg_acquire(v, old, new);
514}
515
516static __always_inline bool
517atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
518{
519 instrument_atomic_read_write(v, sizeof(*v));
520 instrument_atomic_read_write(old, sizeof(*old));
521 return arch_atomic_try_cmpxchg_release(v, old, new);
522}
523
524static __always_inline bool
525atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
526{
527 instrument_atomic_read_write(v, sizeof(*v));
528 instrument_atomic_read_write(old, sizeof(*old));
529 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
530}
531
532static __always_inline bool
533atomic_sub_and_test(int i, atomic_t *v)
534{
535 instrument_atomic_read_write(v, sizeof(*v));
536 return arch_atomic_sub_and_test(i, v);
537}
538
539static __always_inline bool
540atomic_dec_and_test(atomic_t *v)
541{
542 instrument_atomic_read_write(v, sizeof(*v));
543 return arch_atomic_dec_and_test(v);
544}
545
546static __always_inline bool
547atomic_inc_and_test(atomic_t *v)
548{
549 instrument_atomic_read_write(v, sizeof(*v));
550 return arch_atomic_inc_and_test(v);
551}
552
553static __always_inline bool
554atomic_add_negative(int i, atomic_t *v)
555{
556 instrument_atomic_read_write(v, sizeof(*v));
557 return arch_atomic_add_negative(i, v);
558}
559
560static __always_inline int
561atomic_fetch_add_unless(atomic_t *v, int a, int u)
562{
563 instrument_atomic_read_write(v, sizeof(*v));
564 return arch_atomic_fetch_add_unless(v, a, u);
565}
566
567static __always_inline bool
568atomic_add_unless(atomic_t *v, int a, int u)
569{
570 instrument_atomic_read_write(v, sizeof(*v));
571 return arch_atomic_add_unless(v, a, u);
572}
573
574static __always_inline bool
575atomic_inc_not_zero(atomic_t *v)
576{
577 instrument_atomic_read_write(v, sizeof(*v));
578 return arch_atomic_inc_not_zero(v);
579}
580
581static __always_inline bool
582atomic_inc_unless_negative(atomic_t *v)
583{
584 instrument_atomic_read_write(v, sizeof(*v));
585 return arch_atomic_inc_unless_negative(v);
586}
587
588static __always_inline bool
589atomic_dec_unless_positive(atomic_t *v)
590{
591 instrument_atomic_read_write(v, sizeof(*v));
592 return arch_atomic_dec_unless_positive(v);
593}
594
595static __always_inline int
596atomic_dec_if_positive(atomic_t *v)
597{
598 instrument_atomic_read_write(v, sizeof(*v));
599 return arch_atomic_dec_if_positive(v);
600}
601
602static __always_inline s64
603atomic64_read(const atomic64_t *v)
604{
605 instrument_atomic_read(v, sizeof(*v));
606 return arch_atomic64_read(v);
607}
608
609static __always_inline s64
610atomic64_read_acquire(const atomic64_t *v)
611{
612 instrument_atomic_read(v, sizeof(*v));
613 return arch_atomic64_read_acquire(v);
614}
615
616static __always_inline void
617atomic64_set(atomic64_t *v, s64 i)
618{
619 instrument_atomic_write(v, sizeof(*v));
620 arch_atomic64_set(v, i);
621}
622
623static __always_inline void
624atomic64_set_release(atomic64_t *v, s64 i)
625{
626 instrument_atomic_write(v, sizeof(*v));
627 arch_atomic64_set_release(v, i);
628}
629
630static __always_inline void
631atomic64_add(s64 i, atomic64_t *v)
632{
633 instrument_atomic_read_write(v, sizeof(*v));
634 arch_atomic64_add(i, v);
635}
636
637static __always_inline s64
638atomic64_add_return(s64 i, atomic64_t *v)
639{
640 instrument_atomic_read_write(v, sizeof(*v));
641 return arch_atomic64_add_return(i, v);
642}
643
644static __always_inline s64
645atomic64_add_return_acquire(s64 i, atomic64_t *v)
646{
647 instrument_atomic_read_write(v, sizeof(*v));
648 return arch_atomic64_add_return_acquire(i, v);
649}
650
651static __always_inline s64
652atomic64_add_return_release(s64 i, atomic64_t *v)
653{
654 instrument_atomic_read_write(v, sizeof(*v));
655 return arch_atomic64_add_return_release(i, v);
656}
657
658static __always_inline s64
659atomic64_add_return_relaxed(s64 i, atomic64_t *v)
660{
661 instrument_atomic_read_write(v, sizeof(*v));
662 return arch_atomic64_add_return_relaxed(i, v);
663}
664
665static __always_inline s64
666atomic64_fetch_add(s64 i, atomic64_t *v)
667{
668 instrument_atomic_read_write(v, sizeof(*v));
669 return arch_atomic64_fetch_add(i, v);
670}
671
672static __always_inline s64
673atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
674{
675 instrument_atomic_read_write(v, sizeof(*v));
676 return arch_atomic64_fetch_add_acquire(i, v);
677}
678
679static __always_inline s64
680atomic64_fetch_add_release(s64 i, atomic64_t *v)
681{
682 instrument_atomic_read_write(v, sizeof(*v));
683 return arch_atomic64_fetch_add_release(i, v);
684}
685
686static __always_inline s64
687atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
688{
689 instrument_atomic_read_write(v, sizeof(*v));
690 return arch_atomic64_fetch_add_relaxed(i, v);
691}
692
693static __always_inline void
694atomic64_sub(s64 i, atomic64_t *v)
695{
696 instrument_atomic_read_write(v, sizeof(*v));
697 arch_atomic64_sub(i, v);
698}
699
700static __always_inline s64
701atomic64_sub_return(s64 i, atomic64_t *v)
702{
703 instrument_atomic_read_write(v, sizeof(*v));
704 return arch_atomic64_sub_return(i, v);
705}
706
707static __always_inline s64
708atomic64_sub_return_acquire(s64 i, atomic64_t *v)
709{
710 instrument_atomic_read_write(v, sizeof(*v));
711 return arch_atomic64_sub_return_acquire(i, v);
712}
713
714static __always_inline s64
715atomic64_sub_return_release(s64 i, atomic64_t *v)
716{
717 instrument_atomic_read_write(v, sizeof(*v));
718 return arch_atomic64_sub_return_release(i, v);
719}
720
721static __always_inline s64
722atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
723{
724 instrument_atomic_read_write(v, sizeof(*v));
725 return arch_atomic64_sub_return_relaxed(i, v);
726}
727
728static __always_inline s64
729atomic64_fetch_sub(s64 i, atomic64_t *v)
730{
731 instrument_atomic_read_write(v, sizeof(*v));
732 return arch_atomic64_fetch_sub(i, v);
733}
734
735static __always_inline s64
736atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
737{
738 instrument_atomic_read_write(v, sizeof(*v));
739 return arch_atomic64_fetch_sub_acquire(i, v);
740}
741
742static __always_inline s64
743atomic64_fetch_sub_release(s64 i, atomic64_t *v)
744{
745 instrument_atomic_read_write(v, sizeof(*v));
746 return arch_atomic64_fetch_sub_release(i, v);
747}
748
749static __always_inline s64
750atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
751{
752 instrument_atomic_read_write(v, sizeof(*v));
753 return arch_atomic64_fetch_sub_relaxed(i, v);
754}
755
756static __always_inline void
757atomic64_inc(atomic64_t *v)
758{
759 instrument_atomic_read_write(v, sizeof(*v));
760 arch_atomic64_inc(v);
761}
762
763static __always_inline s64
764atomic64_inc_return(atomic64_t *v)
765{
766 instrument_atomic_read_write(v, sizeof(*v));
767 return arch_atomic64_inc_return(v);
768}
769
770static __always_inline s64
771atomic64_inc_return_acquire(atomic64_t *v)
772{
773 instrument_atomic_read_write(v, sizeof(*v));
774 return arch_atomic64_inc_return_acquire(v);
775}
776
777static __always_inline s64
778atomic64_inc_return_release(atomic64_t *v)
779{
780 instrument_atomic_read_write(v, sizeof(*v));
781 return arch_atomic64_inc_return_release(v);
782}
783
784static __always_inline s64
785atomic64_inc_return_relaxed(atomic64_t *v)
786{
787 instrument_atomic_read_write(v, sizeof(*v));
788 return arch_atomic64_inc_return_relaxed(v);
789}
790
791static __always_inline s64
792atomic64_fetch_inc(atomic64_t *v)
793{
794 instrument_atomic_read_write(v, sizeof(*v));
795 return arch_atomic64_fetch_inc(v);
796}
797
798static __always_inline s64
799atomic64_fetch_inc_acquire(atomic64_t *v)
800{
801 instrument_atomic_read_write(v, sizeof(*v));
802 return arch_atomic64_fetch_inc_acquire(v);
803}
804
805static __always_inline s64
806atomic64_fetch_inc_release(atomic64_t *v)
807{
808 instrument_atomic_read_write(v, sizeof(*v));
809 return arch_atomic64_fetch_inc_release(v);
810}
811
812static __always_inline s64
813atomic64_fetch_inc_relaxed(atomic64_t *v)
814{
815 instrument_atomic_read_write(v, sizeof(*v));
816 return arch_atomic64_fetch_inc_relaxed(v);
817}
818
819static __always_inline void
820atomic64_dec(atomic64_t *v)
821{
822 instrument_atomic_read_write(v, sizeof(*v));
823 arch_atomic64_dec(v);
824}
825
826static __always_inline s64
827atomic64_dec_return(atomic64_t *v)
828{
829 instrument_atomic_read_write(v, sizeof(*v));
830 return arch_atomic64_dec_return(v);
831}
832
833static __always_inline s64
834atomic64_dec_return_acquire(atomic64_t *v)
835{
836 instrument_atomic_read_write(v, sizeof(*v));
837 return arch_atomic64_dec_return_acquire(v);
838}
839
840static __always_inline s64
841atomic64_dec_return_release(atomic64_t *v)
842{
843 instrument_atomic_read_write(v, sizeof(*v));
844 return arch_atomic64_dec_return_release(v);
845}
846
847static __always_inline s64
848atomic64_dec_return_relaxed(atomic64_t *v)
849{
850 instrument_atomic_read_write(v, sizeof(*v));
851 return arch_atomic64_dec_return_relaxed(v);
852}
853
854static __always_inline s64
855atomic64_fetch_dec(atomic64_t *v)
856{
857 instrument_atomic_read_write(v, sizeof(*v));
858 return arch_atomic64_fetch_dec(v);
859}
860
861static __always_inline s64
862atomic64_fetch_dec_acquire(atomic64_t *v)
863{
864 instrument_atomic_read_write(v, sizeof(*v));
865 return arch_atomic64_fetch_dec_acquire(v);
866}
867
868static __always_inline s64
869atomic64_fetch_dec_release(atomic64_t *v)
870{
871 instrument_atomic_read_write(v, sizeof(*v));
872 return arch_atomic64_fetch_dec_release(v);
873}
874
875static __always_inline s64
876atomic64_fetch_dec_relaxed(atomic64_t *v)
877{
878 instrument_atomic_read_write(v, sizeof(*v));
879 return arch_atomic64_fetch_dec_relaxed(v);
880}
881
882static __always_inline void
883atomic64_and(s64 i, atomic64_t *v)
884{
885 instrument_atomic_read_write(v, sizeof(*v));
886 arch_atomic64_and(i, v);
887}
888
889static __always_inline s64
890atomic64_fetch_and(s64 i, atomic64_t *v)
891{
892 instrument_atomic_read_write(v, sizeof(*v));
893 return arch_atomic64_fetch_and(i, v);
894}
895
896static __always_inline s64
897atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
898{
899 instrument_atomic_read_write(v, sizeof(*v));
900 return arch_atomic64_fetch_and_acquire(i, v);
901}
902
903static __always_inline s64
904atomic64_fetch_and_release(s64 i, atomic64_t *v)
905{
906 instrument_atomic_read_write(v, sizeof(*v));
907 return arch_atomic64_fetch_and_release(i, v);
908}
909
910static __always_inline s64
911atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
912{
913 instrument_atomic_read_write(v, sizeof(*v));
914 return arch_atomic64_fetch_and_relaxed(i, v);
915}
916
917static __always_inline void
918atomic64_andnot(s64 i, atomic64_t *v)
919{
920 instrument_atomic_read_write(v, sizeof(*v));
921 arch_atomic64_andnot(i, v);
922}
923
924static __always_inline s64
925atomic64_fetch_andnot(s64 i, atomic64_t *v)
926{
927 instrument_atomic_read_write(v, sizeof(*v));
928 return arch_atomic64_fetch_andnot(i, v);
929}
930
931static __always_inline s64
932atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
933{
934 instrument_atomic_read_write(v, sizeof(*v));
935 return arch_atomic64_fetch_andnot_acquire(i, v);
936}
937
938static __always_inline s64
939atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
940{
941 instrument_atomic_read_write(v, sizeof(*v));
942 return arch_atomic64_fetch_andnot_release(i, v);
943}
944
945static __always_inline s64
946atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
947{
948 instrument_atomic_read_write(v, sizeof(*v));
949 return arch_atomic64_fetch_andnot_relaxed(i, v);
950}
951
952static __always_inline void
953atomic64_or(s64 i, atomic64_t *v)
954{
955 instrument_atomic_read_write(v, sizeof(*v));
956 arch_atomic64_or(i, v);
957}
958
959static __always_inline s64
960atomic64_fetch_or(s64 i, atomic64_t *v)
961{
962 instrument_atomic_read_write(v, sizeof(*v));
963 return arch_atomic64_fetch_or(i, v);
964}
965
966static __always_inline s64
967atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
968{
969 instrument_atomic_read_write(v, sizeof(*v));
970 return arch_atomic64_fetch_or_acquire(i, v);
971}
972
973static __always_inline s64
974atomic64_fetch_or_release(s64 i, atomic64_t *v)
975{
976 instrument_atomic_read_write(v, sizeof(*v));
977 return arch_atomic64_fetch_or_release(i, v);
978}
979
980static __always_inline s64
981atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
982{
983 instrument_atomic_read_write(v, sizeof(*v));
984 return arch_atomic64_fetch_or_relaxed(i, v);
985}
986
987static __always_inline void
988atomic64_xor(s64 i, atomic64_t *v)
989{
990 instrument_atomic_read_write(v, sizeof(*v));
991 arch_atomic64_xor(i, v);
992}
993
994static __always_inline s64
995atomic64_fetch_xor(s64 i, atomic64_t *v)
996{
997 instrument_atomic_read_write(v, sizeof(*v));
998 return arch_atomic64_fetch_xor(i, v);
999}
1000
1001static __always_inline s64
1002atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1003{
1004 instrument_atomic_read_write(v, sizeof(*v));
1005 return arch_atomic64_fetch_xor_acquire(i, v);
1006}
1007
1008static __always_inline s64
1009atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1010{
1011 instrument_atomic_read_write(v, sizeof(*v));
1012 return arch_atomic64_fetch_xor_release(i, v);
1013}
1014
1015static __always_inline s64
1016atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1017{
1018 instrument_atomic_read_write(v, sizeof(*v));
1019 return arch_atomic64_fetch_xor_relaxed(i, v);
1020}
1021
1022static __always_inline s64
1023atomic64_xchg(atomic64_t *v, s64 i)
1024{
1025 instrument_atomic_read_write(v, sizeof(*v));
1026 return arch_atomic64_xchg(v, i);
1027}
1028
1029static __always_inline s64
1030atomic64_xchg_acquire(atomic64_t *v, s64 i)
1031{
1032 instrument_atomic_read_write(v, sizeof(*v));
1033 return arch_atomic64_xchg_acquire(v, i);
1034}
1035
1036static __always_inline s64
1037atomic64_xchg_release(atomic64_t *v, s64 i)
1038{
1039 instrument_atomic_read_write(v, sizeof(*v));
1040 return arch_atomic64_xchg_release(v, i);
1041}
1042
1043static __always_inline s64
1044atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1045{
1046 instrument_atomic_read_write(v, sizeof(*v));
1047 return arch_atomic64_xchg_relaxed(v, i);
1048}
1049
1050static __always_inline s64
1051atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1052{
1053 instrument_atomic_read_write(v, sizeof(*v));
1054 return arch_atomic64_cmpxchg(v, old, new);
1055}
1056
1057static __always_inline s64
1058atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1059{
1060 instrument_atomic_read_write(v, sizeof(*v));
1061 return arch_atomic64_cmpxchg_acquire(v, old, new);
1062}
1063
1064static __always_inline s64
1065atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1066{
1067 instrument_atomic_read_write(v, sizeof(*v));
1068 return arch_atomic64_cmpxchg_release(v, old, new);
1069}
1070
1071static __always_inline s64
1072atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1073{
1074 instrument_atomic_read_write(v, sizeof(*v));
1075 return arch_atomic64_cmpxchg_relaxed(v, old, new);
1076}
1077
1078static __always_inline bool
1079atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1080{
1081 instrument_atomic_read_write(v, sizeof(*v));
1082 instrument_atomic_read_write(old, sizeof(*old));
1083 return arch_atomic64_try_cmpxchg(v, old, new);
1084}
1085
1086static __always_inline bool
1087atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1088{
1089 instrument_atomic_read_write(v, sizeof(*v));
1090 instrument_atomic_read_write(old, sizeof(*old));
1091 return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1092}
1093
1094static __always_inline bool
1095atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1096{
1097 instrument_atomic_read_write(v, sizeof(*v));
1098 instrument_atomic_read_write(old, sizeof(*old));
1099 return arch_atomic64_try_cmpxchg_release(v, old, new);
1100}
1101
1102static __always_inline bool
1103atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1104{
1105 instrument_atomic_read_write(v, sizeof(*v));
1106 instrument_atomic_read_write(old, sizeof(*old));
1107 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1108}
1109
1110static __always_inline bool
1111atomic64_sub_and_test(s64 i, atomic64_t *v)
1112{
1113 instrument_atomic_read_write(v, sizeof(*v));
1114 return arch_atomic64_sub_and_test(i, v);
1115}
1116
1117static __always_inline bool
1118atomic64_dec_and_test(atomic64_t *v)
1119{
1120 instrument_atomic_read_write(v, sizeof(*v));
1121 return arch_atomic64_dec_and_test(v);
1122}
1123
1124static __always_inline bool
1125atomic64_inc_and_test(atomic64_t *v)
1126{
1127 instrument_atomic_read_write(v, sizeof(*v));
1128 return arch_atomic64_inc_and_test(v);
1129}
1130
1131static __always_inline bool
1132atomic64_add_negative(s64 i, atomic64_t *v)
1133{
1134 instrument_atomic_read_write(v, sizeof(*v));
1135 return arch_atomic64_add_negative(i, v);
1136}
1137
1138static __always_inline s64
1139atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1140{
1141 instrument_atomic_read_write(v, sizeof(*v));
1142 return arch_atomic64_fetch_add_unless(v, a, u);
1143}
1144
1145static __always_inline bool
1146atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1147{
1148 instrument_atomic_read_write(v, sizeof(*v));
1149 return arch_atomic64_add_unless(v, a, u);
1150}
1151
1152static __always_inline bool
1153atomic64_inc_not_zero(atomic64_t *v)
1154{
1155 instrument_atomic_read_write(v, sizeof(*v));
1156 return arch_atomic64_inc_not_zero(v);
1157}
1158
1159static __always_inline bool
1160atomic64_inc_unless_negative(atomic64_t *v)
1161{
1162 instrument_atomic_read_write(v, sizeof(*v));
1163 return arch_atomic64_inc_unless_negative(v);
1164}
1165
1166static __always_inline bool
1167atomic64_dec_unless_positive(atomic64_t *v)
1168{
1169 instrument_atomic_read_write(v, sizeof(*v));
1170 return arch_atomic64_dec_unless_positive(v);
1171}
1172
1173static __always_inline s64
1174atomic64_dec_if_positive(atomic64_t *v)
1175{
1176 instrument_atomic_read_write(v, sizeof(*v));
1177 return arch_atomic64_dec_if_positive(v);
1178}
1179
1180static __always_inline long
1181atomic_long_read(const atomic_long_t *v)
1182{
1183 instrument_atomic_read(v, sizeof(*v));
1184 return arch_atomic_long_read(v);
1185}
1186
1187static __always_inline long
1188atomic_long_read_acquire(const atomic_long_t *v)
1189{
1190 instrument_atomic_read(v, sizeof(*v));
1191 return arch_atomic_long_read_acquire(v);
1192}
1193
1194static __always_inline void
1195atomic_long_set(atomic_long_t *v, long i)
1196{
1197 instrument_atomic_write(v, sizeof(*v));
1198 arch_atomic_long_set(v, i);
1199}
1200
1201static __always_inline void
1202atomic_long_set_release(atomic_long_t *v, long i)
1203{
1204 instrument_atomic_write(v, sizeof(*v));
1205 arch_atomic_long_set_release(v, i);
1206}
1207
1208static __always_inline void
1209atomic_long_add(long i, atomic_long_t *v)
1210{
1211 instrument_atomic_read_write(v, sizeof(*v));
1212 arch_atomic_long_add(i, v);
1213}
1214
1215static __always_inline long
1216atomic_long_add_return(long i, atomic_long_t *v)
1217{
1218 instrument_atomic_read_write(v, sizeof(*v));
1219 return arch_atomic_long_add_return(i, v);
1220}
1221
1222static __always_inline long
1223atomic_long_add_return_acquire(long i, atomic_long_t *v)
1224{
1225 instrument_atomic_read_write(v, sizeof(*v));
1226 return arch_atomic_long_add_return_acquire(i, v);
1227}
1228
1229static __always_inline long
1230atomic_long_add_return_release(long i, atomic_long_t *v)
1231{
1232 instrument_atomic_read_write(v, sizeof(*v));
1233 return arch_atomic_long_add_return_release(i, v);
1234}
1235
1236static __always_inline long
1237atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1238{
1239 instrument_atomic_read_write(v, sizeof(*v));
1240 return arch_atomic_long_add_return_relaxed(i, v);
1241}
1242
1243static __always_inline long
1244atomic_long_fetch_add(long i, atomic_long_t *v)
1245{
1246 instrument_atomic_read_write(v, sizeof(*v));
1247 return arch_atomic_long_fetch_add(i, v);
1248}
1249
1250static __always_inline long
1251atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1252{
1253 instrument_atomic_read_write(v, sizeof(*v));
1254 return arch_atomic_long_fetch_add_acquire(i, v);
1255}
1256
1257static __always_inline long
1258atomic_long_fetch_add_release(long i, atomic_long_t *v)
1259{
1260 instrument_atomic_read_write(v, sizeof(*v));
1261 return arch_atomic_long_fetch_add_release(i, v);
1262}
1263
1264static __always_inline long
1265atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1266{
1267 instrument_atomic_read_write(v, sizeof(*v));
1268 return arch_atomic_long_fetch_add_relaxed(i, v);
1269}
1270
1271static __always_inline void
1272atomic_long_sub(long i, atomic_long_t *v)
1273{
1274 instrument_atomic_read_write(v, sizeof(*v));
1275 arch_atomic_long_sub(i, v);
1276}
1277
1278static __always_inline long
1279atomic_long_sub_return(long i, atomic_long_t *v)
1280{
1281 instrument_atomic_read_write(v, sizeof(*v));
1282 return arch_atomic_long_sub_return(i, v);
1283}
1284
1285static __always_inline long
1286atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1287{
1288 instrument_atomic_read_write(v, sizeof(*v));
1289 return arch_atomic_long_sub_return_acquire(i, v);
1290}
1291
1292static __always_inline long
1293atomic_long_sub_return_release(long i, atomic_long_t *v)
1294{
1295 instrument_atomic_read_write(v, sizeof(*v));
1296 return arch_atomic_long_sub_return_release(i, v);
1297}
1298
1299static __always_inline long
1300atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1301{
1302 instrument_atomic_read_write(v, sizeof(*v));
1303 return arch_atomic_long_sub_return_relaxed(i, v);
1304}
1305
1306static __always_inline long
1307atomic_long_fetch_sub(long i, atomic_long_t *v)
1308{
1309 instrument_atomic_read_write(v, sizeof(*v));
1310 return arch_atomic_long_fetch_sub(i, v);
1311}
1312
1313static __always_inline long
1314atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1315{
1316 instrument_atomic_read_write(v, sizeof(*v));
1317 return arch_atomic_long_fetch_sub_acquire(i, v);
1318}
1319
1320static __always_inline long
1321atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1322{
1323 instrument_atomic_read_write(v, sizeof(*v));
1324 return arch_atomic_long_fetch_sub_release(i, v);
1325}
1326
1327static __always_inline long
1328atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1329{
1330 instrument_atomic_read_write(v, sizeof(*v));
1331 return arch_atomic_long_fetch_sub_relaxed(i, v);
1332}
1333
1334static __always_inline void
1335atomic_long_inc(atomic_long_t *v)
1336{
1337 instrument_atomic_read_write(v, sizeof(*v));
1338 arch_atomic_long_inc(v);
1339}
1340
1341static __always_inline long
1342atomic_long_inc_return(atomic_long_t *v)
1343{
1344 instrument_atomic_read_write(v, sizeof(*v));
1345 return arch_atomic_long_inc_return(v);
1346}
1347
1348static __always_inline long
1349atomic_long_inc_return_acquire(atomic_long_t *v)
1350{
1351 instrument_atomic_read_write(v, sizeof(*v));
1352 return arch_atomic_long_inc_return_acquire(v);
1353}
1354
1355static __always_inline long
1356atomic_long_inc_return_release(atomic_long_t *v)
1357{
1358 instrument_atomic_read_write(v, sizeof(*v));
1359 return arch_atomic_long_inc_return_release(v);
1360}
1361
1362static __always_inline long
1363atomic_long_inc_return_relaxed(atomic_long_t *v)
1364{
1365 instrument_atomic_read_write(v, sizeof(*v));
1366 return arch_atomic_long_inc_return_relaxed(v);
1367}
1368
1369static __always_inline long
1370atomic_long_fetch_inc(atomic_long_t *v)
1371{
1372 instrument_atomic_read_write(v, sizeof(*v));
1373 return arch_atomic_long_fetch_inc(v);
1374}
1375
1376static __always_inline long
1377atomic_long_fetch_inc_acquire(atomic_long_t *v)
1378{
1379 instrument_atomic_read_write(v, sizeof(*v));
1380 return arch_atomic_long_fetch_inc_acquire(v);
1381}
1382
1383static __always_inline long
1384atomic_long_fetch_inc_release(atomic_long_t *v)
1385{
1386 instrument_atomic_read_write(v, sizeof(*v));
1387 return arch_atomic_long_fetch_inc_release(v);
1388}
1389
1390static __always_inline long
1391atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1392{
1393 instrument_atomic_read_write(v, sizeof(*v));
1394 return arch_atomic_long_fetch_inc_relaxed(v);
1395}
1396
1397static __always_inline void
1398atomic_long_dec(atomic_long_t *v)
1399{
1400 instrument_atomic_read_write(v, sizeof(*v));
1401 arch_atomic_long_dec(v);
1402}
1403
1404static __always_inline long
1405atomic_long_dec_return(atomic_long_t *v)
1406{
1407 instrument_atomic_read_write(v, sizeof(*v));
1408 return arch_atomic_long_dec_return(v);
1409}
1410
1411static __always_inline long
1412atomic_long_dec_return_acquire(atomic_long_t *v)
1413{
1414 instrument_atomic_read_write(v, sizeof(*v));
1415 return arch_atomic_long_dec_return_acquire(v);
1416}
1417
1418static __always_inline long
1419atomic_long_dec_return_release(atomic_long_t *v)
1420{
1421 instrument_atomic_read_write(v, sizeof(*v));
1422 return arch_atomic_long_dec_return_release(v);
1423}
1424
1425static __always_inline long
1426atomic_long_dec_return_relaxed(atomic_long_t *v)
1427{
1428 instrument_atomic_read_write(v, sizeof(*v));
1429 return arch_atomic_long_dec_return_relaxed(v);
1430}
1431
1432static __always_inline long
1433atomic_long_fetch_dec(atomic_long_t *v)
1434{
1435 instrument_atomic_read_write(v, sizeof(*v));
1436 return arch_atomic_long_fetch_dec(v);
1437}
1438
1439static __always_inline long
1440atomic_long_fetch_dec_acquire(atomic_long_t *v)
1441{
1442 instrument_atomic_read_write(v, sizeof(*v));
1443 return arch_atomic_long_fetch_dec_acquire(v);
1444}
1445
1446static __always_inline long
1447atomic_long_fetch_dec_release(atomic_long_t *v)
1448{
1449 instrument_atomic_read_write(v, sizeof(*v));
1450 return arch_atomic_long_fetch_dec_release(v);
1451}
1452
1453static __always_inline long
1454atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1455{
1456 instrument_atomic_read_write(v, sizeof(*v));
1457 return arch_atomic_long_fetch_dec_relaxed(v);
1458}
1459
1460static __always_inline void
1461atomic_long_and(long i, atomic_long_t *v)
1462{
1463 instrument_atomic_read_write(v, sizeof(*v));
1464 arch_atomic_long_and(i, v);
1465}
1466
1467static __always_inline long
1468atomic_long_fetch_and(long i, atomic_long_t *v)
1469{
1470 instrument_atomic_read_write(v, sizeof(*v));
1471 return arch_atomic_long_fetch_and(i, v);
1472}
1473
1474static __always_inline long
1475atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1476{
1477 instrument_atomic_read_write(v, sizeof(*v));
1478 return arch_atomic_long_fetch_and_acquire(i, v);
1479}
1480
1481static __always_inline long
1482atomic_long_fetch_and_release(long i, atomic_long_t *v)
1483{
1484 instrument_atomic_read_write(v, sizeof(*v));
1485 return arch_atomic_long_fetch_and_release(i, v);
1486}
1487
1488static __always_inline long
1489atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1490{
1491 instrument_atomic_read_write(v, sizeof(*v));
1492 return arch_atomic_long_fetch_and_relaxed(i, v);
1493}
1494
1495static __always_inline void
1496atomic_long_andnot(long i, atomic_long_t *v)
1497{
1498 instrument_atomic_read_write(v, sizeof(*v));
1499 arch_atomic_long_andnot(i, v);
1500}
1501
1502static __always_inline long
1503atomic_long_fetch_andnot(long i, atomic_long_t *v)
1504{
1505 instrument_atomic_read_write(v, sizeof(*v));
1506 return arch_atomic_long_fetch_andnot(i, v);
1507}
1508
1509static __always_inline long
1510atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1511{
1512 instrument_atomic_read_write(v, sizeof(*v));
1513 return arch_atomic_long_fetch_andnot_acquire(i, v);
1514}
1515
1516static __always_inline long
1517atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1518{
1519 instrument_atomic_read_write(v, sizeof(*v));
1520 return arch_atomic_long_fetch_andnot_release(i, v);
1521}
1522
1523static __always_inline long
1524atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1525{
1526 instrument_atomic_read_write(v, sizeof(*v));
1527 return arch_atomic_long_fetch_andnot_relaxed(i, v);
1528}
1529
1530static __always_inline void
1531atomic_long_or(long i, atomic_long_t *v)
1532{
1533 instrument_atomic_read_write(v, sizeof(*v));
1534 arch_atomic_long_or(i, v);
1535}
1536
1537static __always_inline long
1538atomic_long_fetch_or(long i, atomic_long_t *v)
1539{
1540 instrument_atomic_read_write(v, sizeof(*v));
1541 return arch_atomic_long_fetch_or(i, v);
1542}
1543
1544static __always_inline long
1545atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1546{
1547 instrument_atomic_read_write(v, sizeof(*v));
1548 return arch_atomic_long_fetch_or_acquire(i, v);
1549}
1550
1551static __always_inline long
1552atomic_long_fetch_or_release(long i, atomic_long_t *v)
1553{
1554 instrument_atomic_read_write(v, sizeof(*v));
1555 return arch_atomic_long_fetch_or_release(i, v);
1556}
1557
1558static __always_inline long
1559atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1560{
1561 instrument_atomic_read_write(v, sizeof(*v));
1562 return arch_atomic_long_fetch_or_relaxed(i, v);
1563}
1564
1565static __always_inline void
1566atomic_long_xor(long i, atomic_long_t *v)
1567{
1568 instrument_atomic_read_write(v, sizeof(*v));
1569 arch_atomic_long_xor(i, v);
1570}
1571
1572static __always_inline long
1573atomic_long_fetch_xor(long i, atomic_long_t *v)
1574{
1575 instrument_atomic_read_write(v, sizeof(*v));
1576 return arch_atomic_long_fetch_xor(i, v);
1577}
1578
1579static __always_inline long
1580atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1581{
1582 instrument_atomic_read_write(v, sizeof(*v));
1583 return arch_atomic_long_fetch_xor_acquire(i, v);
1584}
1585
1586static __always_inline long
1587atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1588{
1589 instrument_atomic_read_write(v, sizeof(*v));
1590 return arch_atomic_long_fetch_xor_release(i, v);
1591}
1592
1593static __always_inline long
1594atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1595{
1596 instrument_atomic_read_write(v, sizeof(*v));
1597 return arch_atomic_long_fetch_xor_relaxed(i, v);
1598}
1599
1600static __always_inline long
1601atomic_long_xchg(atomic_long_t *v, long i)
1602{
1603 instrument_atomic_read_write(v, sizeof(*v));
1604 return arch_atomic_long_xchg(v, i);
1605}
1606
1607static __always_inline long
1608atomic_long_xchg_acquire(atomic_long_t *v, long i)
1609{
1610 instrument_atomic_read_write(v, sizeof(*v));
1611 return arch_atomic_long_xchg_acquire(v, i);
1612}
1613
1614static __always_inline long
1615atomic_long_xchg_release(atomic_long_t *v, long i)
1616{
1617 instrument_atomic_read_write(v, sizeof(*v));
1618 return arch_atomic_long_xchg_release(v, i);
1619}
1620
1621static __always_inline long
1622atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1623{
1624 instrument_atomic_read_write(v, sizeof(*v));
1625 return arch_atomic_long_xchg_relaxed(v, i);
1626}
1627
1628static __always_inline long
1629atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1630{
1631 instrument_atomic_read_write(v, sizeof(*v));
1632 return arch_atomic_long_cmpxchg(v, old, new);
1633}
1634
1635static __always_inline long
1636atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1637{
1638 instrument_atomic_read_write(v, sizeof(*v));
1639 return arch_atomic_long_cmpxchg_acquire(v, old, new);
1640}
1641
1642static __always_inline long
1643atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1644{
1645 instrument_atomic_read_write(v, sizeof(*v));
1646 return arch_atomic_long_cmpxchg_release(v, old, new);
1647}
1648
1649static __always_inline long
1650atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1651{
1652 instrument_atomic_read_write(v, sizeof(*v));
1653 return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1654}
1655
1656static __always_inline bool
1657atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1658{
1659 instrument_atomic_read_write(v, sizeof(*v));
1660 instrument_atomic_read_write(old, sizeof(*old));
1661 return arch_atomic_long_try_cmpxchg(v, old, new);
1662}
1663
1664static __always_inline bool
1665atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1666{
1667 instrument_atomic_read_write(v, sizeof(*v));
1668 instrument_atomic_read_write(old, sizeof(*old));
1669 return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1670}
1671
1672static __always_inline bool
1673atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1674{
1675 instrument_atomic_read_write(v, sizeof(*v));
1676 instrument_atomic_read_write(old, sizeof(*old));
1677 return arch_atomic_long_try_cmpxchg_release(v, old, new);
1678}
1679
1680static __always_inline bool
1681atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1682{
1683 instrument_atomic_read_write(v, sizeof(*v));
1684 instrument_atomic_read_write(old, sizeof(*old));
1685 return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1686}
1687
1688static __always_inline bool
1689atomic_long_sub_and_test(long i, atomic_long_t *v)
1690{
1691 instrument_atomic_read_write(v, sizeof(*v));
1692 return arch_atomic_long_sub_and_test(i, v);
1693}
1694
1695static __always_inline bool
1696atomic_long_dec_and_test(atomic_long_t *v)
1697{
1698 instrument_atomic_read_write(v, sizeof(*v));
1699 return arch_atomic_long_dec_and_test(v);
1700}
1701
1702static __always_inline bool
1703atomic_long_inc_and_test(atomic_long_t *v)
1704{
1705 instrument_atomic_read_write(v, sizeof(*v));
1706 return arch_atomic_long_inc_and_test(v);
1707}
1708
1709static __always_inline bool
1710atomic_long_add_negative(long i, atomic_long_t *v)
1711{
1712 instrument_atomic_read_write(v, sizeof(*v));
1713 return arch_atomic_long_add_negative(i, v);
1714}
1715
1716static __always_inline long
1717atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1718{
1719 instrument_atomic_read_write(v, sizeof(*v));
1720 return arch_atomic_long_fetch_add_unless(v, a, u);
1721}
1722
1723static __always_inline bool
1724atomic_long_add_unless(atomic_long_t *v, long a, long u)
1725{
1726 instrument_atomic_read_write(v, sizeof(*v));
1727 return arch_atomic_long_add_unless(v, a, u);
1728}
1729
1730static __always_inline bool
1731atomic_long_inc_not_zero(atomic_long_t *v)
1732{
1733 instrument_atomic_read_write(v, sizeof(*v));
1734 return arch_atomic_long_inc_not_zero(v);
1735}
1736
1737static __always_inline bool
1738atomic_long_inc_unless_negative(atomic_long_t *v)
1739{
1740 instrument_atomic_read_write(v, sizeof(*v));
1741 return arch_atomic_long_inc_unless_negative(v);
1742}
1743
1744static __always_inline bool
1745atomic_long_dec_unless_positive(atomic_long_t *v)
1746{
1747 instrument_atomic_read_write(v, sizeof(*v));
1748 return arch_atomic_long_dec_unless_positive(v);
1749}
1750
1751static __always_inline long
1752atomic_long_dec_if_positive(atomic_long_t *v)
1753{
1754 instrument_atomic_read_write(v, sizeof(*v));
1755 return arch_atomic_long_dec_if_positive(v);
1756}
1757
1758#define xchg(ptr, ...) \
1759({ \
1760 typeof(ptr) __ai_ptr = (ptr); \
1761 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1762 arch_xchg(__ai_ptr, __VA_ARGS__); \
1763})
1764
1765#define xchg_acquire(ptr, ...) \
1766({ \
1767 typeof(ptr) __ai_ptr = (ptr); \
1768 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1769 arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1770})
1771
1772#define xchg_release(ptr, ...) \
1773({ \
1774 typeof(ptr) __ai_ptr = (ptr); \
1775 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1776 arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1777})
1778
1779#define xchg_relaxed(ptr, ...) \
1780({ \
1781 typeof(ptr) __ai_ptr = (ptr); \
1782 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1783 arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1784})
1785
1786#define cmpxchg(ptr, ...) \
1787({ \
1788 typeof(ptr) __ai_ptr = (ptr); \
1789 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1790 arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1791})
1792
1793#define cmpxchg_acquire(ptr, ...) \
1794({ \
1795 typeof(ptr) __ai_ptr = (ptr); \
1796 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1797 arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1798})
1799
1800#define cmpxchg_release(ptr, ...) \
1801({ \
1802 typeof(ptr) __ai_ptr = (ptr); \
1803 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1804 arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1805})
1806
1807#define cmpxchg_relaxed(ptr, ...) \
1808({ \
1809 typeof(ptr) __ai_ptr = (ptr); \
1810 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1811 arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1812})
1813
1814#define cmpxchg64(ptr, ...) \
1815({ \
1816 typeof(ptr) __ai_ptr = (ptr); \
1817 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1818 arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1819})
1820
1821#define cmpxchg64_acquire(ptr, ...) \
1822({ \
1823 typeof(ptr) __ai_ptr = (ptr); \
1824 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1825 arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1826})
1827
1828#define cmpxchg64_release(ptr, ...) \
1829({ \
1830 typeof(ptr) __ai_ptr = (ptr); \
1831 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1832 arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1833})
1834
1835#define cmpxchg64_relaxed(ptr, ...) \
1836({ \
1837 typeof(ptr) __ai_ptr = (ptr); \
1838 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1839 arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1840})
1841
1842#define try_cmpxchg(ptr, oldp, ...) \
1843({ \
1844 typeof(ptr) __ai_ptr = (ptr); \
1845 typeof(oldp) __ai_oldp = (oldp); \
1846 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1847 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1848 arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1849})
1850
1851#define try_cmpxchg_acquire(ptr, oldp, ...) \
1852({ \
1853 typeof(ptr) __ai_ptr = (ptr); \
1854 typeof(oldp) __ai_oldp = (oldp); \
1855 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1856 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1857 arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1858})
1859
1860#define try_cmpxchg_release(ptr, oldp, ...) \
1861({ \
1862 typeof(ptr) __ai_ptr = (ptr); \
1863 typeof(oldp) __ai_oldp = (oldp); \
1864 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1865 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1866 arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1867})
1868
1869#define try_cmpxchg_relaxed(ptr, oldp, ...) \
1870({ \
1871 typeof(ptr) __ai_ptr = (ptr); \
1872 typeof(oldp) __ai_oldp = (oldp); \
1873 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1874 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1875 arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1876})
1877
1878#define cmpxchg_local(ptr, ...) \
1879({ \
1880 typeof(ptr) __ai_ptr = (ptr); \
1881 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1882 arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
1883})
1884
1885#define cmpxchg64_local(ptr, ...) \
1886({ \
1887 typeof(ptr) __ai_ptr = (ptr); \
1888 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1889 arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
1890})
1891
1892#define sync_cmpxchg(ptr, ...) \
1893({ \
1894 typeof(ptr) __ai_ptr = (ptr); \
1895 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1896 arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
1897})
1898
1899#define cmpxchg_double(ptr, ...) \
1900({ \
1901 typeof(ptr) __ai_ptr = (ptr); \
1902 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1903 arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
1904})
1905
1906
1907#define cmpxchg_double_local(ptr, ...) \
1908({ \
1909 typeof(ptr) __ai_ptr = (ptr); \
1910 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1911 arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
1912})
1913
1914#endif
1915
1916
