LXR linux/arch/s390/lib/spinlock.c

   1/*
   2 *    Out of line spinlock code.
   3 *
   4 *    Copyright IBM Corp. 2004, 2006
   5 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
   6 */
   7
   8#include <linux/types.h>
   9#include <linux/module.h>
  10#include <linux/spinlock.h>
  11#include <linux/init.h>
  12#include <linux/smp.h>
  13#include <asm/io.h>
  14
  15int spin_retry = -1;
  16
  17static int __init spin_retry_init(void)
  18{
  19        if (spin_retry < 0)
  20                spin_retry = MACHINE_HAS_CAD ? 10 : 1000;
  21        return 0;
  22}
  23early_initcall(spin_retry_init);
  24
  25/**
  26 * spin_retry= parameter
  27 */
  28static int __init spin_retry_setup(char *str)
  29{
  30        spin_retry = simple_strtoul(str, &str, 0);
  31        return 1;
  32}
  33__setup("spin_retry=", spin_retry_setup);
  34
  35static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old)
  36{
  37        asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock));
  38}
  39
  40void arch_spin_lock_wait(arch_spinlock_t *lp)
  41{
  42        unsigned int cpu = SPINLOCK_LOCKVAL;
  43        unsigned int owner;
  44        int count;
  45
  46        while (1) {
  47                owner = ACCESS_ONCE(lp->lock);
  48                /* Try to get the lock if it is free. */
  49                if (!owner) {
  50                        if (_raw_compare_and_swap(&lp->lock, 0, cpu))
  51                                return;
  52                        continue;
  53                }
  54                /* Check if the lock owner is running. */
  55                if (!smp_vcpu_scheduled(~owner)) {
  56                        smp_yield_cpu(~owner);
  57                        continue;
  58                }
  59                /* Loop for a while on the lock value. */
  60                count = spin_retry;
  61                do {
  62                        if (MACHINE_HAS_CAD)
  63                                _raw_compare_and_delay(&lp->lock, owner);
  64                        owner = ACCESS_ONCE(lp->lock);
  65                } while (owner && count-- > 0);
  66                if (!owner)
  67                        continue;
  68                /*
  69                 * For multiple layers of hypervisors, e.g. z/VM + LPAR
  70                 * yield the CPU if the lock is still unavailable.
  71                 */
  72                if (!MACHINE_IS_LPAR)
  73                        smp_yield_cpu(~owner);
  74        }
  75}
  76EXPORT_SYMBOL(arch_spin_lock_wait);
  77
  78void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
  79{
  80        unsigned int cpu = SPINLOCK_LOCKVAL;
  81        unsigned int owner;
  82        int count;
  83
  84        local_irq_restore(flags);
  85        while (1) {
  86                owner = ACCESS_ONCE(lp->lock);
  87                /* Try to get the lock if it is free. */
  88                if (!owner) {
  89                        local_irq_disable();
  90                        if (_raw_compare_and_swap(&lp->lock, 0, cpu))
  91                                return;
  92                        local_irq_restore(flags);
  93                }
  94                /* Check if the lock owner is running. */
  95                if (!smp_vcpu_scheduled(~owner)) {
  96                        smp_yield_cpu(~owner);
  97                        continue;
  98                }
  99                /* Loop for a while on the lock value. */
 100                count = spin_retry;
 101                do {
 102                        if (MACHINE_HAS_CAD)
 103                                _raw_compare_and_delay(&lp->lock, owner);
 104                        owner = ACCESS_ONCE(lp->lock);
 105                } while (owner && count-- > 0);
 106                if (!owner)
 107                        continue;
 108                /*
 109                 * For multiple layers of hypervisors, e.g. z/VM + LPAR
 110                 * yield the CPU if the lock is still unavailable.
 111                 */
 112                if (!MACHINE_IS_LPAR)
 113                        smp_yield_cpu(~owner);
 114        }
 115}
 116EXPORT_SYMBOL(arch_spin_lock_wait_flags);
 117
 118int arch_spin_trylock_retry(arch_spinlock_t *lp)
 119{
 120        unsigned int cpu = SPINLOCK_LOCKVAL;
 121        unsigned int owner;
 122        int count;
 123
 124        for (count = spin_retry; count > 0; count--) {
 125                owner = ACCESS_ONCE(lp->lock);
 126                /* Try to get the lock if it is free. */
 127                if (!owner) {
 128                        if (_raw_compare_and_swap(&lp->lock, 0, cpu))
 129                                return 1;
 130                } else if (MACHINE_HAS_CAD)
 131                        _raw_compare_and_delay(&lp->lock, owner);
 132        }
 133        return 0;
 134}
 135EXPORT_SYMBOL(arch_spin_trylock_retry);
 136
 137void _raw_read_lock_wait(arch_rwlock_t *rw)
 138{
 139        unsigned int owner, old;
 140        int count = spin_retry;
 141
 142#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 143        __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD);
 144#endif
 145        owner = 0;
 146        while (1) {
 147                if (count-- <= 0) {
 148                        if (owner && !smp_vcpu_scheduled(~owner))
 149                                smp_yield_cpu(~owner);
 150                        count = spin_retry;
 151                }
 152                old = ACCESS_ONCE(rw->lock);
 153                owner = ACCESS_ONCE(rw->owner);
 154                if ((int) old < 0) {
 155                        if (MACHINE_HAS_CAD)
 156                                _raw_compare_and_delay(&rw->lock, old);
 157                        continue;
 158                }
 159                if (_raw_compare_and_swap(&rw->lock, old, old + 1))
 160                        return;
 161        }
 162}
 163EXPORT_SYMBOL(_raw_read_lock_wait);
 164
 165int _raw_read_trylock_retry(arch_rwlock_t *rw)
 166{
 167        unsigned int old;
 168        int count = spin_retry;
 169
 170        while (count-- > 0) {
 171                old = ACCESS_ONCE(rw->lock);
 172                if ((int) old < 0) {
 173                        if (MACHINE_HAS_CAD)
 174                                _raw_compare_and_delay(&rw->lock, old);
 175                        continue;
 176                }
 177                if (_raw_compare_and_swap(&rw->lock, old, old + 1))
 178                        return 1;
 179        }
 180        return 0;
 181}
 182EXPORT_SYMBOL(_raw_read_trylock_retry);
 183
 184#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 185
 186void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev)
 187{
 188        unsigned int owner, old;
 189        int count = spin_retry;
 190
 191        owner = 0;
 192        while (1) {
 193                if (count-- <= 0) {
 194                        if (owner && !smp_vcpu_scheduled(~owner))
 195                                smp_yield_cpu(~owner);
 196                        count = spin_retry;
 197                }
 198                old = ACCESS_ONCE(rw->lock);
 199                owner = ACCESS_ONCE(rw->owner);
 200                smp_rmb();
 201                if ((int) old >= 0) {
 202                        prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR);
 203                        old = prev;
 204                }
 205                if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
 206                        break;
 207                if (MACHINE_HAS_CAD)
 208                        _raw_compare_and_delay(&rw->lock, old);
 209        }
 210}
 211EXPORT_SYMBOL(_raw_write_lock_wait);
 212
 213#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
 214
 215void _raw_write_lock_wait(arch_rwlock_t *rw)
 216{
 217        unsigned int owner, old, prev;
 218        int count = spin_retry;
 219
 220        prev = 0x80000000;
 221        owner = 0;
 222        while (1) {
 223                if (count-- <= 0) {
 224                        if (owner && !smp_vcpu_scheduled(~owner))
 225                                smp_yield_cpu(~owner);
 226                        count = spin_retry;
 227                }
 228                old = ACCESS_ONCE(rw->lock);
 229                owner = ACCESS_ONCE(rw->owner);
 230                if ((int) old >= 0 &&
 231                    _raw_compare_and_swap(&rw->lock, old, old | 0x80000000))
 232                        prev = old;
 233                else
 234                        smp_rmb();
 235                if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
 236                        break;
 237                if (MACHINE_HAS_CAD)
 238                        _raw_compare_and_delay(&rw->lock, old);
 239        }
 240}
 241EXPORT_SYMBOL(_raw_write_lock_wait);
 242
 243#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
 244
 245int _raw_write_trylock_retry(arch_rwlock_t *rw)
 246{
 247        unsigned int old;
 248        int count = spin_retry;
 249
 250        while (count-- > 0) {
 251                old = ACCESS_ONCE(rw->lock);
 252                if (old) {
 253                        if (MACHINE_HAS_CAD)
 254                                _raw_compare_and_delay(&rw->lock, old);
 255                        continue;
 256                }
 257                if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
 258                        return 1;
 259        }
 260        return 0;
 261}
 262EXPORT_SYMBOL(_raw_write_trylock_retry);
 263
 264void arch_lock_relax(unsigned int cpu)
 265{
 266        if (!cpu)
 267                return;
 268        if (MACHINE_IS_LPAR && smp_vcpu_scheduled(~cpu))
 269                return;
 270        smp_yield_cpu(~cpu);
 271}
 272EXPORT_SYMBOL(arch_lock_relax);
 273