LXR linux/include/asm-generic/bitops/instrumented-non-atomic.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2
   3/*
   4 * This file provides wrappers with sanitizer instrumentation for non-atomic
   5 * bit operations.
   6 *
   7 * To use this functionality, an arch's bitops.h file needs to define each of
   8 * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
   9 * arch___set_bit(), etc.).
  10 */
  11#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
  12#define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
  13
  14#include <linux/instrumented.h>
  15
  16/**
  17 * __set_bit - Set a bit in memory
  18 * @nr: the bit to set
  19 * @addr: the address to start counting from
  20 *
  21 * Unlike set_bit(), this function is non-atomic. If it is called on the same
  22 * region of memory concurrently, the effect may be that only one operation
  23 * succeeds.
  24 */
  25static inline void __set_bit(long nr, volatile unsigned long *addr)
  26{
  27        instrument_write(addr + BIT_WORD(nr), sizeof(long));
  28        arch___set_bit(nr, addr);
  29}
  30
  31/**
  32 * __clear_bit - Clears a bit in memory
  33 * @nr: the bit to clear
  34 * @addr: the address to start counting from
  35 *
  36 * Unlike clear_bit(), this function is non-atomic. If it is called on the same
  37 * region of memory concurrently, the effect may be that only one operation
  38 * succeeds.
  39 */
  40static inline void __clear_bit(long nr, volatile unsigned long *addr)
  41{
  42        instrument_write(addr + BIT_WORD(nr), sizeof(long));
  43        arch___clear_bit(nr, addr);
  44}
  45
  46/**
  47 * __change_bit - Toggle a bit in memory
  48 * @nr: the bit to change
  49 * @addr: the address to start counting from
  50 *
  51 * Unlike change_bit(), this function is non-atomic. If it is called on the same
  52 * region of memory concurrently, the effect may be that only one operation
  53 * succeeds.
  54 */
  55static inline void __change_bit(long nr, volatile unsigned long *addr)
  56{
  57        instrument_write(addr + BIT_WORD(nr), sizeof(long));
  58        arch___change_bit(nr, addr);
  59}
  60
  61static inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
  62{
  63        if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
  64                /*
  65                 * We treat non-atomic read-write bitops a little more special.
  66                 * Given the operations here only modify a single bit, assuming
  67                 * non-atomicity of the writer is sufficient may be reasonable
  68                 * for certain usage (and follows the permissible nature of the
  69                 * assume-plain-writes-atomic rule):
  70                 * 1. report read-modify-write races -> check read;
  71                 * 2. do not report races with marked readers, but do report
  72                 *    races with unmarked readers -> check "atomic" write.
  73                 */
  74                kcsan_check_read(addr + BIT_WORD(nr), sizeof(long));
  75                /*
  76                 * Use generic write instrumentation, in case other sanitizers
  77                 * or tools are enabled alongside KCSAN.
  78                 */
  79                instrument_write(addr + BIT_WORD(nr), sizeof(long));
  80        } else {
  81                instrument_read_write(addr + BIT_WORD(nr), sizeof(long));
  82        }
  83}
  84
  85/**
  86 * __test_and_set_bit - Set a bit and return its old value
  87 * @nr: Bit to set
  88 * @addr: Address to count from
  89 *
  90 * This operation is non-atomic. If two instances of this operation race, one
  91 * can appear to succeed but actually fail.
  92 */
  93static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
  94{
  95        __instrument_read_write_bitop(nr, addr);
  96        return arch___test_and_set_bit(nr, addr);
  97}
  98
  99/**
 100 * __test_and_clear_bit - Clear a bit and return its old value
 101 * @nr: Bit to clear
 102 * @addr: Address to count from
 103 *
 104 * This operation is non-atomic. If two instances of this operation race, one
 105 * can appear to succeed but actually fail.
 106 */
 107static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
 108{
 109        __instrument_read_write_bitop(nr, addr);
 110        return arch___test_and_clear_bit(nr, addr);
 111}
 112
 113/**
 114 * __test_and_change_bit - Change a bit and return its old value
 115 * @nr: Bit to change
 116 * @addr: Address to count from
 117 *
 118 * This operation is non-atomic. If two instances of this operation race, one
 119 * can appear to succeed but actually fail.
 120 */
 121static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
 122{
 123        __instrument_read_write_bitop(nr, addr);
 124        return arch___test_and_change_bit(nr, addr);
 125}
 126
 127/**
 128 * test_bit - Determine whether a bit is set
 129 * @nr: bit number to test
 130 * @addr: Address to start counting from
 131 */
 132static inline bool test_bit(long nr, const volatile unsigned long *addr)
 133{
 134        instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
 135        return arch_test_bit(nr, addr);
 136}
 137
 138#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */
 139