LXR linux/arch/arm/include/asm/bitops.h

   1/*
   2 * Copyright 1995, Russell King.
   3 * Various bits and pieces copyrights include:
   4 *  Linus Torvalds (test_bit).
   5 * Big endian support: Copyright 2001, Nicolas Pitre
   6 *  reworked by rmk.
   7 *
   8 * bit 0 is the LSB of an "unsigned long" quantity.
   9 *
  10 * Please note that the code in this file should never be included
  11 * from user space.  Many of these are not implemented in assembler
  12 * since they would be too costly.  Also, they require privileged
  13 * instructions (which are not available from user mode) to ensure
  14 * that they are atomic.
  15 */
  16
  17#ifndef __ASM_ARM_BITOPS_H
  18#define __ASM_ARM_BITOPS_H
  19
  20#ifdef __KERNEL__
  21
  22#ifndef _LINUX_BITOPS_H
  23#error only <linux/bitops.h> can be included directly
  24#endif
  25
  26#include <linux/compiler.h>
  27#include <asm/system.h>
  28
  29#define smp_mb__before_clear_bit()      mb()
  30#define smp_mb__after_clear_bit()       mb()
  31
  32/*
  33 * These functions are the basis of our bit ops.
  34 *
  35 * First, the atomic bitops. These use native endian.
  36 */
  37static inline void ____atomic_set_bit(unsigned int bit, volatile unsigned long *p)
  38{
  39        unsigned long flags;
  40        unsigned long mask = 1UL << (bit & 31);
  41
  42        p += bit >> 5;
  43
  44        raw_local_irq_save(flags);
  45        *p |= mask;
  46        raw_local_irq_restore(flags);
  47}
  48
  49static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long *p)
  50{
  51        unsigned long flags;
  52        unsigned long mask = 1UL << (bit & 31);
  53
  54        p += bit >> 5;
  55
  56        raw_local_irq_save(flags);
  57        *p &= ~mask;
  58        raw_local_irq_restore(flags);
  59}
  60
  61static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned long *p)
  62{
  63        unsigned long flags;
  64        unsigned long mask = 1UL << (bit & 31);
  65
  66        p += bit >> 5;
  67
  68        raw_local_irq_save(flags);
  69        *p ^= mask;
  70        raw_local_irq_restore(flags);
  71}
  72
  73static inline int
  74____atomic_test_and_set_bit(unsigned int bit, volatile unsigned long *p)
  75{
  76        unsigned long flags;
  77        unsigned int res;
  78        unsigned long mask = 1UL << (bit & 31);
  79
  80        p += bit >> 5;
  81
  82        raw_local_irq_save(flags);
  83        res = *p;
  84        *p = res | mask;
  85        raw_local_irq_restore(flags);
  86
  87        return (res & mask) != 0;
  88}
  89
  90static inline int
  91____atomic_test_and_clear_bit(unsigned int bit, volatile unsigned long *p)
  92{
  93        unsigned long flags;
  94        unsigned int res;
  95        unsigned long mask = 1UL << (bit & 31);
  96
  97        p += bit >> 5;
  98
  99        raw_local_irq_save(flags);
 100        res = *p;
 101        *p = res & ~mask;
 102        raw_local_irq_restore(flags);
 103
 104        return (res & mask) != 0;
 105}
 106
 107static inline int
 108____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p)
 109{
 110        unsigned long flags;
 111        unsigned int res;
 112        unsigned long mask = 1UL << (bit & 31);
 113
 114        p += bit >> 5;
 115
 116        raw_local_irq_save(flags);
 117        res = *p;
 118        *p = res ^ mask;
 119        raw_local_irq_restore(flags);
 120
 121        return (res & mask) != 0;
 122}
 123
 124#include <asm-generic/bitops/non-atomic.h>
 125
 126/*
 127 *  A note about Endian-ness.
 128 *  -------------------------
 129 *
 130 * When the ARM is put into big endian mode via CR15, the processor
 131 * merely swaps the order of bytes within words, thus:
 132 *
 133 *          ------------ physical data bus bits -----------
 134 *          D31 ... D24  D23 ... D16  D15 ... D8  D7 ... D0
 135 * little     byte 3       byte 2       byte 1      byte 0
 136 * big        byte 0       byte 1       byte 2      byte 3
 137 *
 138 * This means that reading a 32-bit word at address 0 returns the same
 139 * value irrespective of the endian mode bit.
 140 *
 141 * Peripheral devices should be connected with the data bus reversed in
 142 * "Big Endian" mode.  ARM Application Note 61 is applicable, and is
 143 * available from http://www.arm.com/.
 144 *
 145 * The following assumes that the data bus connectivity for big endian
 146 * mode has been followed.
 147 *
 148 * Note that bit 0 is defined to be 32-bit word bit 0, not byte 0 bit 0.
 149 */
 150
 151/*
 152 * Little endian assembly bitops.  nr = 0 -> byte 0 bit 0.
 153 */
 154extern void _set_bit_le(int nr, volatile unsigned long * p);
 155extern void _clear_bit_le(int nr, volatile unsigned long * p);
 156extern void _change_bit_le(int nr, volatile unsigned long * p);
 157extern int _test_and_set_bit_le(int nr, volatile unsigned long * p);
 158extern int _test_and_clear_bit_le(int nr, volatile unsigned long * p);
 159extern int _test_and_change_bit_le(int nr, volatile unsigned long * p);
 160extern int _find_first_zero_bit_le(const void * p, unsigned size);
 161extern int _find_next_zero_bit_le(const void * p, int size, int offset);
 162extern int _find_first_bit_le(const unsigned long *p, unsigned size);
 163extern int _find_next_bit_le(const unsigned long *p, int size, int offset);
 164
 165/*
 166 * Big endian assembly bitops.  nr = 0 -> byte 3 bit 0.
 167 */
 168extern void _set_bit_be(int nr, volatile unsigned long * p);
 169extern void _clear_bit_be(int nr, volatile unsigned long * p);
 170extern void _change_bit_be(int nr, volatile unsigned long * p);
 171extern int _test_and_set_bit_be(int nr, volatile unsigned long * p);
 172extern int _test_and_clear_bit_be(int nr, volatile unsigned long * p);
 173extern int _test_and_change_bit_be(int nr, volatile unsigned long * p);
 174extern int _find_first_zero_bit_be(const void * p, unsigned size);
 175extern int _find_next_zero_bit_be(const void * p, int size, int offset);
 176extern int _find_first_bit_be(const unsigned long *p, unsigned size);
 177extern int _find_next_bit_be(const unsigned long *p, int size, int offset);
 178
 179#ifndef CONFIG_SMP
 180/*
 181 * The __* form of bitops are non-atomic and may be reordered.
 182 */
 183#define ATOMIC_BITOP_LE(name,nr,p)              \
 184        (__builtin_constant_p(nr) ?             \
 185         ____atomic_##name(nr, p) :             \
 186         _##name##_le(nr,p))
 187
 188#define ATOMIC_BITOP_BE(name,nr,p)              \
 189        (__builtin_constant_p(nr) ?             \
 190         ____atomic_##name(nr, p) :             \
 191         _##name##_be(nr,p))
 192#else
 193#define ATOMIC_BITOP_LE(name,nr,p)      _##name##_le(nr,p)
 194#define ATOMIC_BITOP_BE(name,nr,p)      _##name##_be(nr,p)
 195#endif
 196
 197#define NONATOMIC_BITOP(name,nr,p)              \
 198        (____nonatomic_##name(nr, p))
 199
 200#ifndef __ARMEB__
 201/*
 202 * These are the little endian, atomic definitions.
 203 */
 204#define set_bit(nr,p)                   ATOMIC_BITOP_LE(set_bit,nr,p)
 205#define clear_bit(nr,p)                 ATOMIC_BITOP_LE(clear_bit,nr,p)
 206#define change_bit(nr,p)                ATOMIC_BITOP_LE(change_bit,nr,p)
 207#define test_and_set_bit(nr,p)          ATOMIC_BITOP_LE(test_and_set_bit,nr,p)
 208#define test_and_clear_bit(nr,p)        ATOMIC_BITOP_LE(test_and_clear_bit,nr,p)
 209#define test_and_change_bit(nr,p)       ATOMIC_BITOP_LE(test_and_change_bit,nr,p)
 210#define find_first_zero_bit(p,sz)       _find_first_zero_bit_le(p,sz)
 211#define find_next_zero_bit(p,sz,off)    _find_next_zero_bit_le(p,sz,off)
 212#define find_first_bit(p,sz)            _find_first_bit_le(p,sz)
 213#define find_next_bit(p,sz,off)         _find_next_bit_le(p,sz,off)
 214
 215#define WORD_BITOFF_TO_LE(x)            ((x))
 216
 217#else
 218
 219/*
 220 * These are the big endian, atomic definitions.
 221 */
 222#define set_bit(nr,p)                   ATOMIC_BITOP_BE(set_bit,nr,p)
 223#define clear_bit(nr,p)                 ATOMIC_BITOP_BE(clear_bit,nr,p)
 224#define change_bit(nr,p)                ATOMIC_BITOP_BE(change_bit,nr,p)
 225#define test_and_set_bit(nr,p)          ATOMIC_BITOP_BE(test_and_set_bit,nr,p)
 226#define test_and_clear_bit(nr,p)        ATOMIC_BITOP_BE(test_and_clear_bit,nr,p)
 227#define test_and_change_bit(nr,p)       ATOMIC_BITOP_BE(test_and_change_bit,nr,p)
 228#define find_first_zero_bit(p,sz)       _find_first_zero_bit_be(p,sz)
 229#define find_next_zero_bit(p,sz,off)    _find_next_zero_bit_be(p,sz,off)
 230#define find_first_bit(p,sz)            _find_first_bit_be(p,sz)
 231#define find_next_bit(p,sz,off)         _find_next_bit_be(p,sz,off)
 232
 233#define WORD_BITOFF_TO_LE(x)            ((x) ^ 0x18)
 234
 235#endif
 236
 237#if __LINUX_ARM_ARCH__ < 5
 238
 239#include <asm-generic/bitops/ffz.h>
 240#include <asm-generic/bitops/__fls.h>
 241#include <asm-generic/bitops/__ffs.h>
 242#include <asm-generic/bitops/fls.h>
 243#include <asm-generic/bitops/ffs.h>
 244
 245#else
 246
 247static inline int constant_fls(int x)
 248{
 249        int r = 32;
 250
 251        if (!x)
 252                return 0;
 253        if (!(x & 0xffff0000u)) {
 254                x <<= 16;
 255                r -= 16;
 256        }
 257        if (!(x & 0xff000000u)) {
 258                x <<= 8;
 259                r -= 8;
 260        }
 261        if (!(x & 0xf0000000u)) {
 262                x <<= 4;
 263                r -= 4;
 264        }
 265        if (!(x & 0xc0000000u)) {
 266                x <<= 2;
 267                r -= 2;
 268        }
 269        if (!(x & 0x80000000u)) {
 270                x <<= 1;
 271                r -= 1;
 272        }
 273        return r;
 274}
 275
 276/*
 277 * On ARMv5 and above those functions can be implemented around
 278 * the clz instruction for much better code efficiency.
 279 */
 280
 281static inline int fls(int x)
 282{
 283        int ret;
 284
 285        if (__builtin_constant_p(x))
 286               return constant_fls(x);
 287
 288        asm("clz\t%0, %1" : "=r" (ret) : "r" (x) : "cc");
 289        ret = 32 - ret;
 290        return ret;
 291}
 292
 293#define __fls(x) (fls(x) - 1)
 294#define ffs(x) ({ unsigned long __t = (x); fls(__t & -__t); })
 295#define __ffs(x) (ffs(x) - 1)
 296#define ffz(x) __ffs( ~(x) )
 297
 298#endif
 299
 300#include <asm-generic/bitops/fls64.h>
 301
 302#include <asm-generic/bitops/sched.h>
 303#include <asm-generic/bitops/hweight.h>
 304#include <asm-generic/bitops/lock.h>
 305
 306/*
 307 * Ext2 is defined to use little-endian byte ordering.
 308 * These do not need to be atomic.
 309 */
 310#define ext2_set_bit(nr,p)                      \
 311                __test_and_set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 312#define ext2_set_bit_atomic(lock,nr,p)          \
 313                test_and_set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 314#define ext2_clear_bit(nr,p)                    \
 315                __test_and_clear_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 316#define ext2_clear_bit_atomic(lock,nr,p)        \
 317                test_and_clear_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 318#define ext2_test_bit(nr,p)                     \
 319                test_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 320#define ext2_find_first_zero_bit(p,sz)          \
 321                _find_first_zero_bit_le(p,sz)
 322#define ext2_find_next_zero_bit(p,sz,off)       \
 323                _find_next_zero_bit_le(p,sz,off)
 324#define ext2_find_next_bit(p, sz, off) \
 325                _find_next_bit_le(p, sz, off)
 326
 327/*
 328 * Minix is defined to use little-endian byte ordering.
 329 * These do not need to be atomic.
 330 */
 331#define minix_set_bit(nr,p)                     \
 332                __set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 333#define minix_test_bit(nr,p)                    \
 334                test_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 335#define minix_test_and_set_bit(nr,p)            \
 336                __test_and_set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 337#define minix_test_and_clear_bit(nr,p)          \
 338                __test_and_clear_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 339#define minix_find_first_zero_bit(p,sz)         \
 340                _find_first_zero_bit_le(p,sz)
 341
 342#endif /* __KERNEL__ */
 343
 344#endif /* _ARM_BITOPS_H */
 345