LXR linux/lib/atomic64.c

   1/*
   2 * Generic implementation of 64-bit atomics using spinlocks,
   3 * useful on processors that don't have 64-bit atomic instructions.
   4 *
   5 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
   6 *
   7 * This program is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU General Public License
   9 * as published by the Free Software Foundation; either version
  10 * 2 of the License, or (at your option) any later version.
  11 */
  12#include <linux/types.h>
  13#include <linux/cache.h>
  14#include <linux/spinlock.h>
  15#include <linux/init.h>
  16#include <linux/export.h>
  17#include <linux/atomic.h>
  18
  19/*
  20 * We use a hashed array of spinlocks to provide exclusive access
  21 * to each atomic64_t variable.  Since this is expected to used on
  22 * systems with small numbers of CPUs (<= 4 or so), we use a
  23 * relatively small array of 16 spinlocks to avoid wasting too much
  24 * memory on the spinlock array.
  25 */
  26#define NR_LOCKS        16
  27
  28/*
  29 * Ensure each lock is in a separate cacheline.
  30 */
  31static union {
  32        raw_spinlock_t lock;
  33        char pad[L1_CACHE_BYTES];
  34} atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp;
  35
  36static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
  37{
  38        unsigned long addr = (unsigned long) v;
  39
  40        addr >>= L1_CACHE_SHIFT;
  41        addr ^= (addr >> 8) ^ (addr >> 16);
  42        return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
  43}
  44
  45long long atomic64_read(const atomic64_t *v)
  46{
  47        unsigned long flags;
  48        raw_spinlock_t *lock = lock_addr(v);
  49        long long val;
  50
  51        raw_spin_lock_irqsave(lock, flags);
  52        val = v->counter;
  53        raw_spin_unlock_irqrestore(lock, flags);
  54        return val;
  55}
  56EXPORT_SYMBOL(atomic64_read);
  57
  58void atomic64_set(atomic64_t *v, long long i)
  59{
  60        unsigned long flags;
  61        raw_spinlock_t *lock = lock_addr(v);
  62
  63        raw_spin_lock_irqsave(lock, flags);
  64        v->counter = i;
  65        raw_spin_unlock_irqrestore(lock, flags);
  66}
  67EXPORT_SYMBOL(atomic64_set);
  68
  69void atomic64_add(long long a, atomic64_t *v)
  70{
  71        unsigned long flags;
  72        raw_spinlock_t *lock = lock_addr(v);
  73
  74        raw_spin_lock_irqsave(lock, flags);
  75        v->counter += a;
  76        raw_spin_unlock_irqrestore(lock, flags);
  77}
  78EXPORT_SYMBOL(atomic64_add);
  79
  80long long atomic64_add_return(long long a, atomic64_t *v)
  81{
  82        unsigned long flags;
  83        raw_spinlock_t *lock = lock_addr(v);
  84        long long val;
  85
  86        raw_spin_lock_irqsave(lock, flags);
  87        val = v->counter += a;
  88        raw_spin_unlock_irqrestore(lock, flags);
  89        return val;
  90}
  91EXPORT_SYMBOL(atomic64_add_return);
  92
  93void atomic64_sub(long long a, atomic64_t *v)
  94{
  95        unsigned long flags;
  96        raw_spinlock_t *lock = lock_addr(v);
  97
  98        raw_spin_lock_irqsave(lock, flags);
  99        v->counter -= a;
 100        raw_spin_unlock_irqrestore(lock, flags);
 101}
 102EXPORT_SYMBOL(atomic64_sub);
 103
 104long long atomic64_sub_return(long long a, atomic64_t *v)
 105{
 106        unsigned long flags;
 107        raw_spinlock_t *lock = lock_addr(v);
 108        long long val;
 109
 110        raw_spin_lock_irqsave(lock, flags);
 111        val = v->counter -= a;
 112        raw_spin_unlock_irqrestore(lock, flags);
 113        return val;
 114}
 115EXPORT_SYMBOL(atomic64_sub_return);
 116
 117long long atomic64_dec_if_positive(atomic64_t *v)
 118{
 119        unsigned long flags;
 120        raw_spinlock_t *lock = lock_addr(v);
 121        long long val;
 122
 123        raw_spin_lock_irqsave(lock, flags);
 124        val = v->counter - 1;
 125        if (val >= 0)
 126                v->counter = val;
 127        raw_spin_unlock_irqrestore(lock, flags);
 128        return val;
 129}
 130EXPORT_SYMBOL(atomic64_dec_if_positive);
 131
 132long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n)
 133{
 134        unsigned long flags;
 135        raw_spinlock_t *lock = lock_addr(v);
 136        long long val;
 137
 138        raw_spin_lock_irqsave(lock, flags);
 139        val = v->counter;
 140        if (val == o)
 141                v->counter = n;
 142        raw_spin_unlock_irqrestore(lock, flags);
 143        return val;
 144}
 145EXPORT_SYMBOL(atomic64_cmpxchg);
 146
 147long long atomic64_xchg(atomic64_t *v, long long new)
 148{
 149        unsigned long flags;
 150        raw_spinlock_t *lock = lock_addr(v);
 151        long long val;
 152
 153        raw_spin_lock_irqsave(lock, flags);
 154        val = v->counter;
 155        v->counter = new;
 156        raw_spin_unlock_irqrestore(lock, flags);
 157        return val;
 158}
 159EXPORT_SYMBOL(atomic64_xchg);
 160
 161int atomic64_add_unless(atomic64_t *v, long long a, long long u)
 162{
 163        unsigned long flags;
 164        raw_spinlock_t *lock = lock_addr(v);
 165        int ret = 0;
 166
 167        raw_spin_lock_irqsave(lock, flags);
 168        if (v->counter != u) {
 169                v->counter += a;
 170                ret = 1;
 171        }
 172        raw_spin_unlock_irqrestore(lock, flags);
 173        return ret;
 174}
 175EXPORT_SYMBOL(atomic64_add_unless);
 176
 177static int init_atomic64_lock(void)
 178{
 179        int i;
 180
 181        for (i = 0; i < NR_LOCKS; ++i)
 182                raw_spin_lock_init(&atomic64_lock[i].lock);
 183        return 0;
 184}
 185
 186pure_initcall(init_atomic64_lock);
 187