1#ifndef __LINUX_COMPILER_H
2#define __LINUX_COMPILER_H
3
4#ifndef __ASSEMBLY__
5
6#ifdef __CHECKER__
7# define __user __attribute__((noderef, address_space(1)))
8# define __kernel __attribute__((address_space(0)))
9# define __safe __attribute__((safe))
10# define __force __attribute__((force))
11# define __nocast __attribute__((nocast))
12# define __iomem __attribute__((noderef, address_space(2)))
13# define __must_hold(x) __attribute__((context(x,1,1)))
14# define __acquires(x) __attribute__((context(x,0,1)))
15# define __releases(x) __attribute__((context(x,1,0)))
16# define __acquire(x) __context__(x,1)
17# define __release(x) __context__(x,-1)
18# define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0)
19# define __percpu __attribute__((noderef, address_space(3)))
20# define __pmem __attribute__((noderef, address_space(5)))
21#ifdef CONFIG_SPARSE_RCU_POINTER
22# define __rcu __attribute__((noderef, address_space(4)))
23#else
24# define __rcu
25#endif
26# define __private __attribute__((noderef))
27extern void __chk_user_ptr(const volatile void __user *);
28extern void __chk_io_ptr(const volatile void __iomem *);
29# define ACCESS_PRIVATE(p, member) (*((typeof((p)->member) __force *) &(p)->member))
30#else
31# define __user
32# define __kernel
33# define __safe
34# define __force
35# define __nocast
36# define __iomem
37# define __chk_user_ptr(x) (void)0
38# define __chk_io_ptr(x) (void)0
39# define __builtin_warning(x, y...) (1)
40# define __must_hold(x)
41# define __acquires(x)
42# define __releases(x)
43# define __acquire(x) (void)0
44# define __release(x) (void)0
45# define __cond_lock(x,c) (c)
46# define __percpu
47# define __rcu
48# define __pmem
49# define __private
50# define ACCESS_PRIVATE(p, member) ((p)->member)
51#endif
52
53
54#define ___PASTE(a,b) a##b
55#define __PASTE(a,b) ___PASTE(a,b)
56
57#ifdef __KERNEL__
58
59#ifdef __GNUC__
60#include <linux/compiler-gcc.h>
61#endif
62
63#if defined(CC_USING_HOTPATCH) && !defined(__CHECKER__)
64#define notrace __attribute__((hotpatch(0,0)))
65#else
66#define notrace __attribute__((no_instrument_function))
67#endif
68
69
70
71
72#ifdef __INTEL_COMPILER
73# include <linux/compiler-intel.h>
74#endif
75
76
77
78
79#ifdef __clang__
80#include <linux/compiler-clang.h>
81#endif
82
83
84
85
86
87
88
89struct ftrace_branch_data {
90 const char *func;
91 const char *file;
92 unsigned line;
93 union {
94 struct {
95 unsigned long correct;
96 unsigned long incorrect;
97 };
98 struct {
99 unsigned long miss;
100 unsigned long hit;
101 };
102 unsigned long miss_hit[2];
103 };
104};
105
106
107
108
109
110#if defined(CONFIG_TRACE_BRANCH_PROFILING) \
111 && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__)
112void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);
113
114#define likely_notrace(x) __builtin_expect(!!(x), 1)
115#define unlikely_notrace(x) __builtin_expect(!!(x), 0)
116
117#define __branch_check__(x, expect) ({ \
118 int ______r; \
119 static struct ftrace_branch_data \
120 __attribute__((__aligned__(4))) \
121 __attribute__((section("_ftrace_annotated_branch"))) \
122 ______f = { \
123 .func = __func__, \
124 .file = __FILE__, \
125 .line = __LINE__, \
126 }; \
127 ______r = likely_notrace(x); \
128 ftrace_likely_update(&______f, ______r, expect); \
129 ______r; \
130 })
131
132
133
134
135
136
137# ifndef likely
138# define likely(x) (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 1))
139# endif
140# ifndef unlikely
141# define unlikely(x) (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 0))
142# endif
143
144#ifdef CONFIG_PROFILE_ALL_BRANCHES
145
146
147
148
149#define if(cond, ...) __trace_if( (cond , ## __VA_ARGS__) )
150#define __trace_if(cond) \
151 if (__builtin_constant_p(!!(cond)) ? !!(cond) : \
152 ({ \
153 int ______r; \
154 static struct ftrace_branch_data \
155 __attribute__((__aligned__(4))) \
156 __attribute__((section("_ftrace_branch"))) \
157 ______f = { \
158 .func = __func__, \
159 .file = __FILE__, \
160 .line = __LINE__, \
161 }; \
162 ______r = !!(cond); \
163 ______f.miss_hit[______r]++; \
164 ______r; \
165 }))
166#endif
167
168#else
169# define likely(x) __builtin_expect(!!(x), 1)
170# define unlikely(x) __builtin_expect(!!(x), 0)
171#endif
172
173
174#ifndef barrier
175# define barrier() __memory_barrier()
176#endif
177
178#ifndef barrier_data
179# define barrier_data(ptr) barrier()
180#endif
181
182
183#ifndef unreachable
184# define unreachable() do { } while (1)
185#endif
186
187#ifndef RELOC_HIDE
188# define RELOC_HIDE(ptr, off) \
189 ({ unsigned long __ptr; \
190 __ptr = (unsigned long) (ptr); \
191 (typeof(ptr)) (__ptr + (off)); })
192#endif
193
194#ifndef OPTIMIZER_HIDE_VAR
195#define OPTIMIZER_HIDE_VAR(var) barrier()
196#endif
197
198
199#ifndef __UNIQUE_ID
200# define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__)
201#endif
202
203#include <uapi/linux/types.h>
204
205#define __READ_ONCE_SIZE \
206({ \
207 switch (size) { \
208 case 1: *(__u8 *)res = *(volatile __u8 *)p; break; \
209 case 2: *(__u16 *)res = *(volatile __u16 *)p; break; \
210 case 4: *(__u32 *)res = *(volatile __u32 *)p; break; \
211 case 8: *(__u64 *)res = *(volatile __u64 *)p; break; \
212 default: \
213 barrier(); \
214 __builtin_memcpy((void *)res, (const void *)p, size); \
215 barrier(); \
216 } \
217})
218
219static __always_inline
220void __read_once_size(const volatile void *p, void *res, int size)
221{
222 __READ_ONCE_SIZE;
223}
224
225#ifdef CONFIG_KASAN
226
227
228
229
230
231
232static __no_sanitize_address __maybe_unused
233void __read_once_size_nocheck(const volatile void *p, void *res, int size)
234{
235 __READ_ONCE_SIZE;
236}
237#else
238static __always_inline
239void __read_once_size_nocheck(const volatile void *p, void *res, int size)
240{
241 __READ_ONCE_SIZE;
242}
243#endif
244
245static __always_inline void __write_once_size(volatile void *p, void *res, int size)
246{
247 switch (size) {
248 case 1: *(volatile __u8 *)p = *(__u8 *)res; break;
249 case 2: *(volatile __u16 *)p = *(__u16 *)res; break;
250 case 4: *(volatile __u32 *)p = *(__u32 *)res; break;
251 case 8: *(volatile __u64 *)p = *(__u64 *)res; break;
252 default:
253 barrier();
254 __builtin_memcpy((void *)p, (const void *)res, size);
255 barrier();
256 }
257}
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282#define __READ_ONCE(x, check) \
283({ \
284 union { typeof(x) __val; char __c[1]; } __u; \
285 if (check) \
286 __read_once_size(&(x), __u.__c, sizeof(x)); \
287 else \
288 __read_once_size_nocheck(&(x), __u.__c, sizeof(x)); \
289 __u.__val; \
290})
291#define READ_ONCE(x) __READ_ONCE(x, 1)
292
293
294
295
296
297#define READ_ONCE_NOCHECK(x) __READ_ONCE(x, 0)
298
299#define WRITE_ONCE(x, val) \
300({ \
301 union { typeof(x) __val; char __c[1]; } __u = \
302 { .__val = (__force typeof(x)) (val) }; \
303 __write_once_size(&(x), __u.__c, sizeof(x)); \
304 __u.__val; \
305})
306
307
308
309
310
311
312
313
314
315
316
317
318#define smp_cond_acquire(cond) do { \
319 while (!(cond)) \
320 cpu_relax(); \
321 smp_rmb(); \
322} while (0)
323
324#endif
325
326#endif
327
328#ifdef __KERNEL__
329
330
331
332
333
334
335#ifndef __deprecated
336# define __deprecated
337#endif
338
339#ifdef MODULE
340#define __deprecated_for_modules __deprecated
341#else
342#define __deprecated_for_modules
343#endif
344
345#ifndef __must_check
346#define __must_check
347#endif
348
349#ifndef CONFIG_ENABLE_MUST_CHECK
350#undef __must_check
351#define __must_check
352#endif
353#ifndef CONFIG_ENABLE_WARN_DEPRECATED
354#undef __deprecated
355#undef __deprecated_for_modules
356#define __deprecated
357#define __deprecated_for_modules
358#endif
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378#ifndef __used
379# define __used
380#endif
381
382#ifndef __maybe_unused
383# define __maybe_unused
384#endif
385
386#ifndef __always_unused
387# define __always_unused
388#endif
389
390#ifndef noinline
391#define noinline
392#endif
393
394
395
396
397
398#define noinline_for_stack noinline
399
400#ifndef __always_inline
401#define __always_inline inline
402#endif
403
404#endif
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420#ifndef __attribute_const__
421# define __attribute_const__
422#endif
423
424
425
426
427
428
429#ifndef __cold
430#define __cold
431#endif
432
433
434#ifndef __section
435# define __section(S) __attribute__ ((__section__(#S)))
436#endif
437
438#ifndef __visible
439#define __visible
440#endif
441
442
443
444
445#ifndef __assume_aligned
446#define __assume_aligned(a, ...)
447#endif
448
449
450
451#ifndef __same_type
452# define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b))
453#endif
454
455
456#ifndef __native_word
457# define __native_word(t) (sizeof(t) == sizeof(char) || sizeof(t) == sizeof(short) || sizeof(t) == sizeof(int) || sizeof(t) == sizeof(long))
458#endif
459
460
461#ifndef __compiletime_object_size
462# define __compiletime_object_size(obj) -1
463#endif
464#ifndef __compiletime_warning
465# define __compiletime_warning(message)
466#endif
467#ifndef __compiletime_error
468# define __compiletime_error(message)
469
470
471
472
473
474
475# ifndef __CHECKER__
476# define __compiletime_error_fallback(condition) \
477 do { ((void)sizeof(char[1 - 2 * condition])); } while (0)
478# endif
479#endif
480#ifndef __compiletime_error_fallback
481# define __compiletime_error_fallback(condition) do { } while (0)
482#endif
483
484#define __compiletime_assert(condition, msg, prefix, suffix) \
485 do { \
486 bool __cond = !(condition); \
487 extern void prefix ## suffix(void) __compiletime_error(msg); \
488 if (__cond) \
489 prefix ## suffix(); \
490 __compiletime_error_fallback(__cond); \
491 } while (0)
492
493#define _compiletime_assert(condition, msg, prefix, suffix) \
494 __compiletime_assert(condition, msg, prefix, suffix)
495
496
497
498
499
500
501
502
503
504
505#define compiletime_assert(condition, msg) \
506 _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
507
508#define compiletime_assert_atomic_type(t) \
509 compiletime_assert(__native_word(t), \
510 "Need native word sized stores/loads for atomicity.")
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532#define __ACCESS_ONCE(x) ({ \
533 __maybe_unused typeof(x) __var = (__force typeof(x)) 0; \
534 (volatile typeof(x) *)&(x); })
535#define ACCESS_ONCE(x) (*__ACCESS_ONCE(x))
536
537
538
539
540
541
542
543
544
545#define lockless_dereference(p) \
546({ \
547 typeof(p) _________p1 = READ_ONCE(p); \
548 smp_read_barrier_depends(); \
549 (_________p1); \
550})
551
552
553#ifdef CONFIG_KPROBES
554# define __kprobes __attribute__((__section__(".kprobes.text")))
555# define nokprobe_inline __always_inline
556#else
557# define __kprobes
558# define nokprobe_inline inline
559#endif
560#endif
561