1/* SPDX-License-Identifier: GPL-2.0 */ 2 3#ifndef _LINUX_KCSAN_CHECKS_H 4#define _LINUX_KCSAN_CHECKS_H 5 6/* Note: Only include what is already included by compiler.h. */ 7#include <linux/compiler_attributes.h> 8#include <linux/types.h> 9 10/* Access types -- if KCSAN_ACCESS_WRITE is not set, the access is a read. */ 11#define KCSAN_ACCESS_WRITE (1 << 0) /* Access is a write. */ 12#define KCSAN_ACCESS_COMPOUND (1 << 1) /* Compounded read-write instrumentation. */ 13#define KCSAN_ACCESS_ATOMIC (1 << 2) /* Access is atomic. */ 14/* The following are special, and never due to compiler instrumentation. */ 15#define KCSAN_ACCESS_ASSERT (1 << 3) /* Access is an assertion. */ 16#define KCSAN_ACCESS_SCOPED (1 << 4) /* Access is a scoped access. */ 17 18/* 19 * __kcsan_*: Always calls into the runtime when KCSAN is enabled. This may be used 20 * even in compilation units that selectively disable KCSAN, but must use KCSAN 21 * to validate access to an address. Never use these in header files! 22 */ 23#ifdef CONFIG_KCSAN 24/** 25 * __kcsan_check_access - check generic access for races 26 * 27 * @ptr: address of access 28 * @size: size of access 29 * @type: access type modifier 30 */ 31void __kcsan_check_access(const volatile void *ptr, size_t size, int type); 32 33/** 34 * kcsan_disable_current - disable KCSAN for the current context 35 * 36 * Supports nesting. 37 */ 38void kcsan_disable_current(void); 39 40/** 41 * kcsan_enable_current - re-enable KCSAN for the current context 42 * 43 * Supports nesting. 44 */ 45void kcsan_enable_current(void); 46void kcsan_enable_current_nowarn(void); /* Safe in uaccess regions. */ 47 48/** 49 * kcsan_nestable_atomic_begin - begin nestable atomic region 50 * 51 * Accesses within the atomic region may appear to race with other accesses but 52 * should be considered atomic. 53 */ 54void kcsan_nestable_atomic_begin(void); 55 56/** 57 * kcsan_nestable_atomic_end - end nestable atomic region 58 */ 59void kcsan_nestable_atomic_end(void); 60 61/** 62 * kcsan_flat_atomic_begin - begin flat atomic region 63 * 64 * Accesses within the atomic region may appear to race with other accesses but 65 * should be considered atomic. 66 */ 67void kcsan_flat_atomic_begin(void); 68 69/** 70 * kcsan_flat_atomic_end - end flat atomic region 71 */ 72void kcsan_flat_atomic_end(void); 73 74/** 75 * kcsan_atomic_next - consider following accesses as atomic 76 * 77 * Force treating the next n memory accesses for the current context as atomic 78 * operations. 79 * 80 * @n: number of following memory accesses to treat as atomic. 81 */ 82void kcsan_atomic_next(int n); 83 84/** 85 * kcsan_set_access_mask - set access mask 86 * 87 * Set the access mask for all accesses for the current context if non-zero. 88 * Only value changes to bits set in the mask will be reported. 89 * 90 * @mask: bitmask 91 */ 92void kcsan_set_access_mask(unsigned long mask); 93 94/* Scoped access information. */ 95struct kcsan_scoped_access { 96 struct list_head list; 97 const volatile void *ptr; 98 size_t size; 99 int type; 100}; 101/* 102 * Automatically call kcsan_end_scoped_access() when kcsan_scoped_access goes 103 * out of scope; relies on attribute "cleanup", which is supported by all 104 * compilers that support KCSAN. 105 */ 106#define __kcsan_cleanup_scoped \ 107 __maybe_unused __attribute__((__cleanup__(kcsan_end_scoped_access))) 108 109/** 110 * kcsan_begin_scoped_access - begin scoped access 111 * 112 * Begin scoped access and initialize @sa, which will cause KCSAN to 113 * continuously check the memory range in the current thread until 114 * kcsan_end_scoped_access() is called for @sa. 115 * 116 * Scoped accesses are implemented by appending @sa to an internal list for the 117 * current execution context, and then checked on every call into the KCSAN 118 * runtime. 119 * 120 * @ptr: address of access 121 * @size: size of access 122 * @type: access type modifier 123 * @sa: struct kcsan_scoped_access to use for the scope of the access 124 */ 125struct kcsan_scoped_access * 126kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type, 127 struct kcsan_scoped_access *sa); 128 129/** 130 * kcsan_end_scoped_access - end scoped access 131 * 132 * End a scoped access, which will stop KCSAN checking the memory range. 133 * Requires that kcsan_begin_scoped_access() was previously called once for @sa. 134 * 135 * @sa: a previously initialized struct kcsan_scoped_access 136 */ 137void kcsan_end_scoped_access(struct kcsan_scoped_access *sa); 138 139 140#else /* CONFIG_KCSAN */ 141 142static inline void __kcsan_check_access(const volatile void *ptr, size_t size, 143 int type) { } 144 145static inline void kcsan_disable_current(void) { } 146static inline void kcsan_enable_current(void) { } 147static inline void kcsan_enable_current_nowarn(void) { } 148static inline void kcsan_nestable_atomic_begin(void) { } 149static inline void kcsan_nestable_atomic_end(void) { } 150static inline void kcsan_flat_atomic_begin(void) { } 151static inline void kcsan_flat_atomic_end(void) { } 152static inline void kcsan_atomic_next(int n) { } 153static inline void kcsan_set_access_mask(unsigned long mask) { } 154 155struct kcsan_scoped_access { }; 156#define __kcsan_cleanup_scoped __maybe_unused 157static inline struct kcsan_scoped_access * 158kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type, 159 struct kcsan_scoped_access *sa) { return sa; } 160static inline void kcsan_end_scoped_access(struct kcsan_scoped_access *sa) { } 161 162#endif /* CONFIG_KCSAN */ 163 164#ifdef __SANITIZE_THREAD__ 165/* 166 * Only calls into the runtime when the particular compilation unit has KCSAN 167 * instrumentation enabled. May be used in header files. 168 */ 169#define kcsan_check_access __kcsan_check_access 170 171/* 172 * Only use these to disable KCSAN for accesses in the current compilation unit; 173 * calls into libraries may still perform KCSAN checks. 174 */ 175#define __kcsan_disable_current kcsan_disable_current 176#define __kcsan_enable_current kcsan_enable_current_nowarn 177#else 178static inline void kcsan_check_access(const volatile void *ptr, size_t size, 179 int type) { } 180static inline void __kcsan_enable_current(void) { } 181static inline void __kcsan_disable_current(void) { } 182#endif 183 184/** 185 * __kcsan_check_read - check regular read access for races 186 * 187 * @ptr: address of access 188 * @size: size of access 189 */ 190#define __kcsan_check_read(ptr, size) __kcsan_check_access(ptr, size, 0) 191 192/** 193 * __kcsan_check_write - check regular write access for races 194 * 195 * @ptr: address of access 196 * @size: size of access 197 */ 198#define __kcsan_check_write(ptr, size) \ 199 __kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE) 200 201/** 202 * __kcsan_check_read_write - check regular read-write access for races 203 * 204 * @ptr: address of access 205 * @size: size of access 206 */ 207#define __kcsan_check_read_write(ptr, size) \ 208 __kcsan_check_access(ptr, size, KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE) 209 210/** 211 * kcsan_check_read - check regular read access for races 212 * 213 * @ptr: address of access 214 * @size: size of access 215 */ 216#define kcsan_check_read(ptr, size) kcsan_check_access(ptr, size, 0) 217 218/** 219 * kcsan_check_write - check regular write access for races 220 * 221 * @ptr: address of access 222 * @size: size of access 223 */ 224#define kcsan_check_write(ptr, size) \ 225 kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE) 226 227/** 228 * kcsan_check_read_write - check regular read-write access for races 229 * 230 * @ptr: address of access 231 * @size: size of access 232 */ 233#define kcsan_check_read_write(ptr, size) \ 234 kcsan_check_access(ptr, size, KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE) 235 236/* 237 * Check for atomic accesses: if atomic accesses are not ignored, this simply 238 * aliases to kcsan_check_access(), otherwise becomes a no-op. 239 */ 240#ifdef CONFIG_KCSAN_IGNORE_ATOMICS 241#define kcsan_check_atomic_read(...) do { } while (0) 242#define kcsan_check_atomic_write(...) do { } while (0) 243#define kcsan_check_atomic_read_write(...) do { } while (0) 244#else 245#define kcsan_check_atomic_read(ptr, size) \ 246 kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC) 247#define kcsan_check_atomic_write(ptr, size) \ 248 kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE) 249#define kcsan_check_atomic_read_write(ptr, size) \ 250 kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_COMPOUND) 251#endif 252 253/** 254 * ASSERT_EXCLUSIVE_WRITER - assert no concurrent writes to @var 255 * 256 * Assert that there are no concurrent writes to @var; other readers are 257 * allowed. This assertion can be used to specify properties of concurrent code, 258 * where violation cannot be detected as a normal data race. 259 * 260 * For example, if we only have a single writer, but multiple concurrent 261 * readers, to avoid data races, all these accesses must be marked; even 262 * concurrent marked writes racing with the single writer are bugs. 263 * Unfortunately, due to being marked, they are no longer data races. For cases 264 * like these, we can use the macro as follows: 265 * 266 * .. code-block:: c 267 * 268 * void writer(void) { 269 * spin_lock(&update_foo_lock); 270 * ASSERT_EXCLUSIVE_WRITER(shared_foo); 271 * WRITE_ONCE(shared_foo, ...); 272 * spin_unlock(&update_foo_lock); 273 * } 274 * void reader(void) { 275 * // update_foo_lock does not need to be held! 276 * ... = READ_ONCE(shared_foo); 277 * } 278 * 279 * Note: ASSERT_EXCLUSIVE_WRITER_SCOPED(), if applicable, performs more thorough 280 * checking if a clear scope where no concurrent writes are expected exists. 281 * 282 * @var: variable to assert on 283 */ 284#define ASSERT_EXCLUSIVE_WRITER(var) \ 285 __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT) 286 287/* 288 * Helper macros for implementation of for ASSERT_EXCLUSIVE_*_SCOPED(). @id is 289 * expected to be unique for the scope in which instances of kcsan_scoped_access 290 * are declared. 291 */ 292#define __kcsan_scoped_name(c, suffix) __kcsan_scoped_##c##suffix 293#define __ASSERT_EXCLUSIVE_SCOPED(var, type, id) \ 294 struct kcsan_scoped_access __kcsan_scoped_name(id, _) \ 295 __kcsan_cleanup_scoped; \ 296 struct kcsan_scoped_access *__kcsan_scoped_name(id, _dummy_p) \ 297 __maybe_unused = kcsan_begin_scoped_access( \ 298 &(var), sizeof(var), KCSAN_ACCESS_SCOPED | (type), \ 299 &__kcsan_scoped_name(id, _)) 300 301/** 302 * ASSERT_EXCLUSIVE_WRITER_SCOPED - assert no concurrent writes to @var in scope 303 * 304 * Scoped variant of ASSERT_EXCLUSIVE_WRITER(). 305 * 306 * Assert that there are no concurrent writes to @var for the duration of the 307 * scope in which it is introduced. This provides a better way to fully cover 308 * the enclosing scope, compared to multiple ASSERT_EXCLUSIVE_WRITER(), and 309 * increases the likelihood for KCSAN to detect racing accesses. 310 * 311 * For example, it allows finding race-condition bugs that only occur due to 312 * state changes within the scope itself: 313 * 314 * .. code-block:: c 315 * 316 * void writer(void) { 317 * spin_lock(&update_foo_lock); 318 * { 319 * ASSERT_EXCLUSIVE_WRITER_SCOPED(shared_foo); 320 * WRITE_ONCE(shared_foo, 42); 321 * ... 322 * // shared_foo should still be 42 here! 323 * } 324 * spin_unlock(&update_foo_lock); 325 * } 326 * void buggy(void) { 327 * if (READ_ONCE(shared_foo) == 42) 328 * WRITE_ONCE(shared_foo, 1); // bug! 329 * } 330 * 331 * @var: variable to assert on 332 */ 333#define ASSERT_EXCLUSIVE_WRITER_SCOPED(var) \ 334 __ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_ASSERT, __COUNTER__) 335 336/** 337 * ASSERT_EXCLUSIVE_ACCESS - assert no concurrent accesses to @var 338 * 339 * Assert that there are no concurrent accesses to @var (no readers nor 340 * writers). This assertion can be used to specify properties of concurrent 341 * code, where violation cannot be detected as a normal data race. 342 * 343 * For example, where exclusive access is expected after determining no other 344 * users of an object are left, but the object is not actually freed. We can 345 * check that this property actually holds as follows: 346 * 347 * .. code-block:: c 348 * 349 * if (refcount_dec_and_test(&obj->refcnt)) { 350 * ASSERT_EXCLUSIVE_ACCESS(*obj); 351 * do_some_cleanup(obj); 352 * release_for_reuse(obj); 353 * } 354 * 355 * Note: 356 * 357 * 1. ASSERT_EXCLUSIVE_ACCESS_SCOPED(), if applicable, performs more thorough 358 * checking if a clear scope where no concurrent accesses are expected exists. 359 * 360 * 2. For cases where the object is freed, `KASAN <kasan.html>`_ is a better 361 * fit to detect use-after-free bugs. 362 * 363 * @var: variable to assert on 364 */ 365#define ASSERT_EXCLUSIVE_ACCESS(var) \ 366 __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT) 367 368/** 369 * ASSERT_EXCLUSIVE_ACCESS_SCOPED - assert no concurrent accesses to @var in scope 370 * 371 * Scoped variant of ASSERT_EXCLUSIVE_ACCESS(). 372 * 373 * Assert that there are no concurrent accesses to @var (no readers nor writers) 374 * for the entire duration of the scope in which it is introduced. This provides 375 * a better way to fully cover the enclosing scope, compared to multiple 376 * ASSERT_EXCLUSIVE_ACCESS(), and increases the likelihood for KCSAN to detect 377 * racing accesses. 378 * 379 * @var: variable to assert on 380 */ 381#define ASSERT_EXCLUSIVE_ACCESS_SCOPED(var) \ 382 __ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT, __COUNTER__) 383 384/** 385 * ASSERT_EXCLUSIVE_BITS - assert no concurrent writes to subset of bits in @var 386 * 387 * Bit-granular variant of ASSERT_EXCLUSIVE_WRITER(). 388 * 389 * Assert that there are no concurrent writes to a subset of bits in @var; 390 * concurrent readers are permitted. This assertion captures more detailed 391 * bit-level properties, compared to the other (word granularity) assertions. 392 * Only the bits set in @mask are checked for concurrent modifications, while 393 * ignoring the remaining bits, i.e. concurrent writes (or reads) to ~mask bits 394 * are ignored. 395 * 396 * Use this for variables, where some bits must not be modified concurrently, 397 * yet other bits are expected to be modified concurrently. 398 * 399 * For example, variables where, after initialization, some bits are read-only, 400 * but other bits may still be modified concurrently. A reader may wish to 401 * assert that this is true as follows: 402 * 403 * .. code-block:: c 404 * 405 * ASSERT_EXCLUSIVE_BITS(flags, READ_ONLY_MASK); 406 * foo = (READ_ONCE(flags) & READ_ONLY_MASK) >> READ_ONLY_SHIFT; 407 * 408 * Note: The access that immediately follows ASSERT_EXCLUSIVE_BITS() is assumed 409 * to access the masked bits only, and KCSAN optimistically assumes it is 410 * therefore safe, even in the presence of data races, and marking it with 411 * READ_ONCE() is optional from KCSAN's point-of-view. We caution, however, that 412 * it may still be advisable to do so, since we cannot reason about all compiler 413 * optimizations when it comes to bit manipulations (on the reader and writer 414 * side). If you are sure nothing can go wrong, we can write the above simply 415 * as: 416 * 417 * .. code-block:: c 418 * 419 * ASSERT_EXCLUSIVE_BITS(flags, READ_ONLY_MASK); 420 * foo = (flags & READ_ONLY_MASK) >> READ_ONLY_SHIFT; 421 * 422 * Another example, where this may be used, is when certain bits of @var may 423 * only be modified when holding the appropriate lock, but other bits may still 424 * be modified concurrently. Writers, where other bits may change concurrently, 425 * could use the assertion as follows: 426 * 427 * .. code-block:: c 428 * 429 * spin_lock(&foo_lock); 430 * ASSERT_EXCLUSIVE_BITS(flags, FOO_MASK); 431 * old_flags = flags; 432 * new_flags = (old_flags & ~FOO_MASK) | (new_foo << FOO_SHIFT); 433 * if (cmpxchg(&flags, old_flags, new_flags) != old_flags) { ... } 434 * spin_unlock(&foo_lock); 435 * 436 * @var: variable to assert on 437 * @mask: only check for modifications to bits set in @mask 438 */ 439#define ASSERT_EXCLUSIVE_BITS(var, mask) \ 440 do { \ 441 kcsan_set_access_mask(mask); \ 442 __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT);\ 443 kcsan_set_access_mask(0); \ 444 kcsan_atomic_next(1); \ 445 } while (0) 446 447#endif /* _LINUX_KCSAN_CHECKS_H */ 448