1 /* 2 * Out of line spinlock code. 3 * 4 * Copyright IBM Corp. 2004, 2006 5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) 6 */ 7 8 #include <linux/types.h> 9 #include <linux/module.h> 10 #include <linux/spinlock.h> 11 #include <linux/init.h> 12 #include <linux/smp.h> 13 #include <asm/io.h> 14 15 int spin_retry = -1; 16 17 static int __init spin_retry_init(void) 18 { 19 if (spin_retry < 0) 20 spin_retry = MACHINE_HAS_CAD ? 10 : 1000; 21 return 0; 22 } 23 early_initcall(spin_retry_init); 24 25 /** 26 * spin_retry= parameter 27 */ 28 static int __init spin_retry_setup(char *str) 29 { 30 spin_retry = simple_strtoul(str, &str, 0); 31 return 1; 32 } 33 __setup("spin_retry=", spin_retry_setup); 34 35 static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old) 36 { 37 asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock)); 38 } 39 40 void arch_spin_lock_wait(arch_spinlock_t *lp) 41 { 42 unsigned int cpu = SPINLOCK_LOCKVAL; 43 unsigned int owner; 44 int count; 45 46 while (1) { 47 owner = ACCESS_ONCE(lp->lock); 48 /* Try to get the lock if it is free. */ 49 if (!owner) { 50 if (_raw_compare_and_swap(&lp->lock, 0, cpu)) 51 return; 52 continue; 53 } 54 /* Check if the lock owner is running. */ 55 if (!smp_vcpu_scheduled(~owner)) { 56 smp_yield_cpu(~owner); 57 continue; 58 } 59 /* Loop for a while on the lock value. */ 60 count = spin_retry; 61 do { 62 if (MACHINE_HAS_CAD) 63 _raw_compare_and_delay(&lp->lock, owner); 64 owner = ACCESS_ONCE(lp->lock); 65 } while (owner && count-- > 0); 66 if (!owner) 67 continue; 68 /* 69 * For multiple layers of hypervisors, e.g. z/VM + LPAR 70 * yield the CPU if the lock is still unavailable. 71 */ 72 if (!MACHINE_IS_LPAR) 73 smp_yield_cpu(~owner); 74 } 75 } 76 EXPORT_SYMBOL(arch_spin_lock_wait); 77 78 void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) 79 { 80 unsigned int cpu = SPINLOCK_LOCKVAL; 81 unsigned int owner; 82 int count; 83 84 local_irq_restore(flags); 85 while (1) { 86 owner = ACCESS_ONCE(lp->lock); 87 /* Try to get the lock if it is free. */ 88 if (!owner) { 89 local_irq_disable(); 90 if (_raw_compare_and_swap(&lp->lock, 0, cpu)) 91 return; 92 local_irq_restore(flags); 93 } 94 /* Check if the lock owner is running. */ 95 if (!smp_vcpu_scheduled(~owner)) { 96 smp_yield_cpu(~owner); 97 continue; 98 } 99 /* Loop for a while on the lock value. */ 100 count = spin_retry; 101 do { 102 if (MACHINE_HAS_CAD) 103 _raw_compare_and_delay(&lp->lock, owner); 104 owner = ACCESS_ONCE(lp->lock); 105 } while (owner && count-- > 0); 106 if (!owner) 107 continue; 108 /* 109 * For multiple layers of hypervisors, e.g. z/VM + LPAR 110 * yield the CPU if the lock is still unavailable. 111 */ 112 if (!MACHINE_IS_LPAR) 113 smp_yield_cpu(~owner); 114 } 115 } 116 EXPORT_SYMBOL(arch_spin_lock_wait_flags); 117 118 int arch_spin_trylock_retry(arch_spinlock_t *lp) 119 { 120 unsigned int cpu = SPINLOCK_LOCKVAL; 121 unsigned int owner; 122 int count; 123 124 for (count = spin_retry; count > 0; count--) { 125 owner = ACCESS_ONCE(lp->lock); 126 /* Try to get the lock if it is free. */ 127 if (!owner) { 128 if (_raw_compare_and_swap(&lp->lock, 0, cpu)) 129 return 1; 130 } else if (MACHINE_HAS_CAD) 131 _raw_compare_and_delay(&lp->lock, owner); 132 } 133 return 0; 134 } 135 EXPORT_SYMBOL(arch_spin_trylock_retry); 136 137 void _raw_read_lock_wait(arch_rwlock_t *rw) 138 { 139 unsigned int owner, old; 140 int count = spin_retry; 141 142 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES 143 __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD); 144 #endif 145 owner = 0; 146 while (1) { 147 if (count-- <= 0) { 148 if (owner && !smp_vcpu_scheduled(~owner)) 149 smp_yield_cpu(~owner); 150 count = spin_retry; 151 } 152 old = ACCESS_ONCE(rw->lock); 153 owner = ACCESS_ONCE(rw->owner); 154 if ((int) old < 0) { 155 if (MACHINE_HAS_CAD) 156 _raw_compare_and_delay(&rw->lock, old); 157 continue; 158 } 159 if (_raw_compare_and_swap(&rw->lock, old, old + 1)) 160 return; 161 } 162 } 163 EXPORT_SYMBOL(_raw_read_lock_wait); 164 165 int _raw_read_trylock_retry(arch_rwlock_t *rw) 166 { 167 unsigned int old; 168 int count = spin_retry; 169 170 while (count-- > 0) { 171 old = ACCESS_ONCE(rw->lock); 172 if ((int) old < 0) { 173 if (MACHINE_HAS_CAD) 174 _raw_compare_and_delay(&rw->lock, old); 175 continue; 176 } 177 if (_raw_compare_and_swap(&rw->lock, old, old + 1)) 178 return 1; 179 } 180 return 0; 181 } 182 EXPORT_SYMBOL(_raw_read_trylock_retry); 183 184 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES 185 186 void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev) 187 { 188 unsigned int owner, old; 189 int count = spin_retry; 190 191 owner = 0; 192 while (1) { 193 if (count-- <= 0) { 194 if (owner && !smp_vcpu_scheduled(~owner)) 195 smp_yield_cpu(~owner); 196 count = spin_retry; 197 } 198 old = ACCESS_ONCE(rw->lock); 199 owner = ACCESS_ONCE(rw->owner); 200 smp_rmb(); 201 if ((int) old >= 0) { 202 prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR); 203 old = prev; 204 } 205 if ((old & 0x7fffffff) == 0 && (int) prev >= 0) 206 break; 207 if (MACHINE_HAS_CAD) 208 _raw_compare_and_delay(&rw->lock, old); 209 } 210 } 211 EXPORT_SYMBOL(_raw_write_lock_wait); 212 213 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 214 215 void _raw_write_lock_wait(arch_rwlock_t *rw) 216 { 217 unsigned int owner, old, prev; 218 int count = spin_retry; 219 220 prev = 0x80000000; 221 owner = 0; 222 while (1) { 223 if (count-- <= 0) { 224 if (owner && !smp_vcpu_scheduled(~owner)) 225 smp_yield_cpu(~owner); 226 count = spin_retry; 227 } 228 old = ACCESS_ONCE(rw->lock); 229 owner = ACCESS_ONCE(rw->owner); 230 if ((int) old >= 0 && 231 _raw_compare_and_swap(&rw->lock, old, old | 0x80000000)) 232 prev = old; 233 else 234 smp_rmb(); 235 if ((old & 0x7fffffff) == 0 && (int) prev >= 0) 236 break; 237 if (MACHINE_HAS_CAD) 238 _raw_compare_and_delay(&rw->lock, old); 239 } 240 } 241 EXPORT_SYMBOL(_raw_write_lock_wait); 242 243 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 244 245 int _raw_write_trylock_retry(arch_rwlock_t *rw) 246 { 247 unsigned int old; 248 int count = spin_retry; 249 250 while (count-- > 0) { 251 old = ACCESS_ONCE(rw->lock); 252 if (old) { 253 if (MACHINE_HAS_CAD) 254 _raw_compare_and_delay(&rw->lock, old); 255 continue; 256 } 257 if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000)) 258 return 1; 259 } 260 return 0; 261 } 262 EXPORT_SYMBOL(_raw_write_trylock_retry); 263 264 void arch_lock_relax(unsigned int cpu) 265 { 266 if (!cpu) 267 return; 268 if (MACHINE_IS_LPAR && smp_vcpu_scheduled(~cpu)) 269 return; 270 smp_yield_cpu(~cpu); 271 } 272 EXPORT_SYMBOL(arch_lock_relax); 273