1 /* 2 * Out of line spinlock code. 3 * 4 * Copyright IBM Corp. 2004, 2006 5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) 6 */ 7 8 #include <linux/types.h> 9 #include <linux/export.h> 10 #include <linux/spinlock.h> 11 #include <linux/init.h> 12 #include <linux/smp.h> 13 #include <asm/io.h> 14 15 int spin_retry = -1; 16 17 static int __init spin_retry_init(void) 18 { 19 if (spin_retry < 0) 20 spin_retry = MACHINE_HAS_CAD ? 10 : 1000; 21 return 0; 22 } 23 early_initcall(spin_retry_init); 24 25 /** 26 * spin_retry= parameter 27 */ 28 static int __init spin_retry_setup(char *str) 29 { 30 spin_retry = simple_strtoul(str, &str, 0); 31 return 1; 32 } 33 __setup("spin_retry=", spin_retry_setup); 34 35 static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old) 36 { 37 asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock)); 38 } 39 40 void arch_spin_lock_wait(arch_spinlock_t *lp) 41 { 42 unsigned int cpu = SPINLOCK_LOCKVAL; 43 unsigned int owner; 44 int count, first_diag; 45 46 first_diag = 1; 47 while (1) { 48 owner = ACCESS_ONCE(lp->lock); 49 /* Try to get the lock if it is free. */ 50 if (!owner) { 51 if (_raw_compare_and_swap(&lp->lock, 0, cpu)) 52 return; 53 continue; 54 } 55 /* First iteration: check if the lock owner is running. */ 56 if (first_diag && arch_vcpu_is_preempted(~owner)) { 57 smp_yield_cpu(~owner); 58 first_diag = 0; 59 continue; 60 } 61 /* Loop for a while on the lock value. */ 62 count = spin_retry; 63 do { 64 if (MACHINE_HAS_CAD) 65 _raw_compare_and_delay(&lp->lock, owner); 66 owner = ACCESS_ONCE(lp->lock); 67 } while (owner && count-- > 0); 68 if (!owner) 69 continue; 70 /* 71 * For multiple layers of hypervisors, e.g. z/VM + LPAR 72 * yield the CPU unconditionally. For LPAR rely on the 73 * sense running status. 74 */ 75 if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner)) { 76 smp_yield_cpu(~owner); 77 first_diag = 0; 78 } 79 } 80 } 81 EXPORT_SYMBOL(arch_spin_lock_wait); 82 83 void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) 84 { 85 unsigned int cpu = SPINLOCK_LOCKVAL; 86 unsigned int owner; 87 int count, first_diag; 88 89 local_irq_restore(flags); 90 first_diag = 1; 91 while (1) { 92 owner = ACCESS_ONCE(lp->lock); 93 /* Try to get the lock if it is free. */ 94 if (!owner) { 95 local_irq_disable(); 96 if (_raw_compare_and_swap(&lp->lock, 0, cpu)) 97 return; 98 local_irq_restore(flags); 99 continue; 100 } 101 /* Check if the lock owner is running. */ 102 if (first_diag && arch_vcpu_is_preempted(~owner)) { 103 smp_yield_cpu(~owner); 104 first_diag = 0; 105 continue; 106 } 107 /* Loop for a while on the lock value. */ 108 count = spin_retry; 109 do { 110 if (MACHINE_HAS_CAD) 111 _raw_compare_and_delay(&lp->lock, owner); 112 owner = ACCESS_ONCE(lp->lock); 113 } while (owner && count-- > 0); 114 if (!owner) 115 continue; 116 /* 117 * For multiple layers of hypervisors, e.g. z/VM + LPAR 118 * yield the CPU unconditionally. For LPAR rely on the 119 * sense running status. 120 */ 121 if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner)) { 122 smp_yield_cpu(~owner); 123 first_diag = 0; 124 } 125 } 126 } 127 EXPORT_SYMBOL(arch_spin_lock_wait_flags); 128 129 int arch_spin_trylock_retry(arch_spinlock_t *lp) 130 { 131 unsigned int cpu = SPINLOCK_LOCKVAL; 132 unsigned int owner; 133 int count; 134 135 for (count = spin_retry; count > 0; count--) { 136 owner = READ_ONCE(lp->lock); 137 /* Try to get the lock if it is free. */ 138 if (!owner) { 139 if (_raw_compare_and_swap(&lp->lock, 0, cpu)) 140 return 1; 141 } else if (MACHINE_HAS_CAD) 142 _raw_compare_and_delay(&lp->lock, owner); 143 } 144 return 0; 145 } 146 EXPORT_SYMBOL(arch_spin_trylock_retry); 147 148 void _raw_read_lock_wait(arch_rwlock_t *rw) 149 { 150 unsigned int owner, old; 151 int count = spin_retry; 152 153 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES 154 __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD); 155 #endif 156 owner = 0; 157 while (1) { 158 if (count-- <= 0) { 159 if (owner && arch_vcpu_is_preempted(~owner)) 160 smp_yield_cpu(~owner); 161 count = spin_retry; 162 } 163 old = ACCESS_ONCE(rw->lock); 164 owner = ACCESS_ONCE(rw->owner); 165 if ((int) old < 0) { 166 if (MACHINE_HAS_CAD) 167 _raw_compare_and_delay(&rw->lock, old); 168 continue; 169 } 170 if (_raw_compare_and_swap(&rw->lock, old, old + 1)) 171 return; 172 } 173 } 174 EXPORT_SYMBOL(_raw_read_lock_wait); 175 176 int _raw_read_trylock_retry(arch_rwlock_t *rw) 177 { 178 unsigned int old; 179 int count = spin_retry; 180 181 while (count-- > 0) { 182 old = ACCESS_ONCE(rw->lock); 183 if ((int) old < 0) { 184 if (MACHINE_HAS_CAD) 185 _raw_compare_and_delay(&rw->lock, old); 186 continue; 187 } 188 if (_raw_compare_and_swap(&rw->lock, old, old + 1)) 189 return 1; 190 } 191 return 0; 192 } 193 EXPORT_SYMBOL(_raw_read_trylock_retry); 194 195 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES 196 197 void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev) 198 { 199 unsigned int owner, old; 200 int count = spin_retry; 201 202 owner = 0; 203 while (1) { 204 if (count-- <= 0) { 205 if (owner && arch_vcpu_is_preempted(~owner)) 206 smp_yield_cpu(~owner); 207 count = spin_retry; 208 } 209 old = ACCESS_ONCE(rw->lock); 210 owner = ACCESS_ONCE(rw->owner); 211 smp_mb(); 212 if ((int) old >= 0) { 213 prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR); 214 old = prev; 215 } 216 if ((old & 0x7fffffff) == 0 && (int) prev >= 0) 217 break; 218 if (MACHINE_HAS_CAD) 219 _raw_compare_and_delay(&rw->lock, old); 220 } 221 } 222 EXPORT_SYMBOL(_raw_write_lock_wait); 223 224 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 225 226 void _raw_write_lock_wait(arch_rwlock_t *rw) 227 { 228 unsigned int owner, old, prev; 229 int count = spin_retry; 230 231 prev = 0x80000000; 232 owner = 0; 233 while (1) { 234 if (count-- <= 0) { 235 if (owner && arch_vcpu_is_preempted(~owner)) 236 smp_yield_cpu(~owner); 237 count = spin_retry; 238 } 239 old = ACCESS_ONCE(rw->lock); 240 owner = ACCESS_ONCE(rw->owner); 241 if ((int) old >= 0 && 242 _raw_compare_and_swap(&rw->lock, old, old | 0x80000000)) 243 prev = old; 244 else 245 smp_mb(); 246 if ((old & 0x7fffffff) == 0 && (int) prev >= 0) 247 break; 248 if (MACHINE_HAS_CAD) 249 _raw_compare_and_delay(&rw->lock, old); 250 } 251 } 252 EXPORT_SYMBOL(_raw_write_lock_wait); 253 254 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 255 256 int _raw_write_trylock_retry(arch_rwlock_t *rw) 257 { 258 unsigned int old; 259 int count = spin_retry; 260 261 while (count-- > 0) { 262 old = ACCESS_ONCE(rw->lock); 263 if (old) { 264 if (MACHINE_HAS_CAD) 265 _raw_compare_and_delay(&rw->lock, old); 266 continue; 267 } 268 if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000)) 269 return 1; 270 } 271 return 0; 272 } 273 EXPORT_SYMBOL(_raw_write_trylock_retry); 274 275 void arch_lock_relax(unsigned int cpu) 276 { 277 if (!cpu) 278 return; 279 if (MACHINE_IS_LPAR && !arch_vcpu_is_preempted(~cpu)) 280 return; 281 smp_yield_cpu(~cpu); 282 } 283 EXPORT_SYMBOL(arch_lock_relax); 284