1 /* 2 * Split spinlock implementation out into its own file, so it can be 3 * compiled in a FTRACE-compatible way. 4 */ 5 #include <linux/kernel_stat.h> 6 #include <linux/spinlock.h> 7 #include <linux/debugfs.h> 8 #include <linux/log2.h> 9 #include <linux/gfp.h> 10 #include <linux/slab.h> 11 12 #include <asm/paravirt.h> 13 14 #include <xen/interface/xen.h> 15 #include <xen/events.h> 16 17 #include "xen-ops.h" 18 #include "debugfs.h" 19 20 #ifdef CONFIG_XEN_DEBUG_FS 21 static struct xen_spinlock_stats 22 { 23 u64 taken; 24 u32 taken_slow; 25 u32 taken_slow_nested; 26 u32 taken_slow_pickup; 27 u32 taken_slow_spurious; 28 u32 taken_slow_irqenable; 29 30 u64 released; 31 u32 released_slow; 32 u32 released_slow_kicked; 33 34 #define HISTO_BUCKETS 30 35 u32 histo_spin_total[HISTO_BUCKETS+1]; 36 u32 histo_spin_spinning[HISTO_BUCKETS+1]; 37 u32 histo_spin_blocked[HISTO_BUCKETS+1]; 38 39 u64 time_total; 40 u64 time_spinning; 41 u64 time_blocked; 42 } spinlock_stats; 43 44 static u8 zero_stats; 45 46 static unsigned lock_timeout = 1 << 10; 47 #define TIMEOUT lock_timeout 48 49 static inline void check_zero(void) 50 { 51 if (unlikely(zero_stats)) { 52 memset(&spinlock_stats, 0, sizeof(spinlock_stats)); 53 zero_stats = 0; 54 } 55 } 56 57 #define ADD_STATS(elem, val) \ 58 do { check_zero(); spinlock_stats.elem += (val); } while(0) 59 60 static inline u64 spin_time_start(void) 61 { 62 return xen_clocksource_read(); 63 } 64 65 static void __spin_time_accum(u64 delta, u32 *array) 66 { 67 unsigned index = ilog2(delta); 68 69 check_zero(); 70 71 if (index < HISTO_BUCKETS) 72 array[index]++; 73 else 74 array[HISTO_BUCKETS]++; 75 } 76 77 static inline void spin_time_accum_spinning(u64 start) 78 { 79 u32 delta = xen_clocksource_read() - start; 80 81 __spin_time_accum(delta, spinlock_stats.histo_spin_spinning); 82 spinlock_stats.time_spinning += delta; 83 } 84 85 static inline void spin_time_accum_total(u64 start) 86 { 87 u32 delta = xen_clocksource_read() - start; 88 89 __spin_time_accum(delta, spinlock_stats.histo_spin_total); 90 spinlock_stats.time_total += delta; 91 } 92 93 static inline void spin_time_accum_blocked(u64 start) 94 { 95 u32 delta = xen_clocksource_read() - start; 96 97 __spin_time_accum(delta, spinlock_stats.histo_spin_blocked); 98 spinlock_stats.time_blocked += delta; 99 } 100 #else /* !CONFIG_XEN_DEBUG_FS */ 101 #define TIMEOUT (1 << 10) 102 #define ADD_STATS(elem, val) do { (void)(val); } while(0) 103 104 static inline u64 spin_time_start(void) 105 { 106 return 0; 107 } 108 109 static inline void spin_time_accum_total(u64 start) 110 { 111 } 112 static inline void spin_time_accum_spinning(u64 start) 113 { 114 } 115 static inline void spin_time_accum_blocked(u64 start) 116 { 117 } 118 #endif /* CONFIG_XEN_DEBUG_FS */ 119 120 /* 121 * Size struct xen_spinlock so it's the same as arch_spinlock_t. 122 */ 123 #if NR_CPUS < 256 124 typedef u8 xen_spinners_t; 125 # define inc_spinners(xl) \ 126 asm(LOCK_PREFIX " incb %0" : "+m" ((xl)->spinners) : : "memory"); 127 # define dec_spinners(xl) \ 128 asm(LOCK_PREFIX " decb %0" : "+m" ((xl)->spinners) : : "memory"); 129 #else 130 typedef u16 xen_spinners_t; 131 # define inc_spinners(xl) \ 132 asm(LOCK_PREFIX " incw %0" : "+m" ((xl)->spinners) : : "memory"); 133 # define dec_spinners(xl) \ 134 asm(LOCK_PREFIX " decw %0" : "+m" ((xl)->spinners) : : "memory"); 135 #endif 136 137 struct xen_spinlock { 138 unsigned char lock; /* 0 -> free; 1 -> locked */ 139 xen_spinners_t spinners; /* count of waiting cpus */ 140 }; 141 142 static int xen_spin_is_locked(struct arch_spinlock *lock) 143 { 144 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 145 146 return xl->lock != 0; 147 } 148 149 static int xen_spin_is_contended(struct arch_spinlock *lock) 150 { 151 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 152 153 /* Not strictly true; this is only the count of contended 154 lock-takers entering the slow path. */ 155 return xl->spinners != 0; 156 } 157 158 static int xen_spin_trylock(struct arch_spinlock *lock) 159 { 160 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 161 u8 old = 1; 162 163 asm("xchgb %b0,%1" 164 : "+q" (old), "+m" (xl->lock) : : "memory"); 165 166 return old == 0; 167 } 168 169 static DEFINE_PER_CPU(char *, irq_name); 170 static DEFINE_PER_CPU(int, lock_kicker_irq) = -1; 171 static DEFINE_PER_CPU(struct xen_spinlock *, lock_spinners); 172 173 /* 174 * Mark a cpu as interested in a lock. Returns the CPU's previous 175 * lock of interest, in case we got preempted by an interrupt. 176 */ 177 static inline struct xen_spinlock *spinning_lock(struct xen_spinlock *xl) 178 { 179 struct xen_spinlock *prev; 180 181 prev = __this_cpu_read(lock_spinners); 182 __this_cpu_write(lock_spinners, xl); 183 184 wmb(); /* set lock of interest before count */ 185 186 inc_spinners(xl); 187 188 return prev; 189 } 190 191 /* 192 * Mark a cpu as no longer interested in a lock. Restores previous 193 * lock of interest (NULL for none). 194 */ 195 static inline void unspinning_lock(struct xen_spinlock *xl, struct xen_spinlock *prev) 196 { 197 dec_spinners(xl); 198 wmb(); /* decrement count before restoring lock */ 199 __this_cpu_write(lock_spinners, prev); 200 } 201 202 static noinline int xen_spin_lock_slow(struct arch_spinlock *lock, bool irq_enable) 203 { 204 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 205 struct xen_spinlock *prev; 206 int irq = __this_cpu_read(lock_kicker_irq); 207 int ret; 208 u64 start; 209 210 /* If kicker interrupts not initialized yet, just spin */ 211 if (irq == -1) 212 return 0; 213 214 start = spin_time_start(); 215 216 /* announce we're spinning */ 217 prev = spinning_lock(xl); 218 219 ADD_STATS(taken_slow, 1); 220 ADD_STATS(taken_slow_nested, prev != NULL); 221 222 do { 223 unsigned long flags; 224 225 /* clear pending */ 226 xen_clear_irq_pending(irq); 227 228 /* check again make sure it didn't become free while 229 we weren't looking */ 230 ret = xen_spin_trylock(lock); 231 if (ret) { 232 ADD_STATS(taken_slow_pickup, 1); 233 234 /* 235 * If we interrupted another spinlock while it 236 * was blocking, make sure it doesn't block 237 * without rechecking the lock. 238 */ 239 if (prev != NULL) 240 xen_set_irq_pending(irq); 241 goto out; 242 } 243 244 flags = arch_local_save_flags(); 245 if (irq_enable) { 246 ADD_STATS(taken_slow_irqenable, 1); 247 raw_local_irq_enable(); 248 } 249 250 /* 251 * Block until irq becomes pending. If we're 252 * interrupted at this point (after the trylock but 253 * before entering the block), then the nested lock 254 * handler guarantees that the irq will be left 255 * pending if there's any chance the lock became free; 256 * xen_poll_irq() returns immediately if the irq is 257 * pending. 258 */ 259 xen_poll_irq(irq); 260 261 raw_local_irq_restore(flags); 262 263 ADD_STATS(taken_slow_spurious, !xen_test_irq_pending(irq)); 264 } while (!xen_test_irq_pending(irq)); /* check for spurious wakeups */ 265 266 kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq)); 267 268 out: 269 unspinning_lock(xl, prev); 270 spin_time_accum_blocked(start); 271 272 return ret; 273 } 274 275 static inline void __xen_spin_lock(struct arch_spinlock *lock, bool irq_enable) 276 { 277 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 278 unsigned timeout; 279 u8 oldval; 280 u64 start_spin; 281 282 ADD_STATS(taken, 1); 283 284 start_spin = spin_time_start(); 285 286 do { 287 u64 start_spin_fast = spin_time_start(); 288 289 timeout = TIMEOUT; 290 291 asm("1: xchgb %1,%0\n" 292 " testb %1,%1\n" 293 " jz 3f\n" 294 "2: rep;nop\n" 295 " cmpb $0,%0\n" 296 " je 1b\n" 297 " dec %2\n" 298 " jnz 2b\n" 299 "3:\n" 300 : "+m" (xl->lock), "=q" (oldval), "+r" (timeout) 301 : "1" (1) 302 : "memory"); 303 304 spin_time_accum_spinning(start_spin_fast); 305 306 } while (unlikely(oldval != 0 && 307 (TIMEOUT == ~0 || !xen_spin_lock_slow(lock, irq_enable)))); 308 309 spin_time_accum_total(start_spin); 310 } 311 312 static void xen_spin_lock(struct arch_spinlock *lock) 313 { 314 __xen_spin_lock(lock, false); 315 } 316 317 static void xen_spin_lock_flags(struct arch_spinlock *lock, unsigned long flags) 318 { 319 __xen_spin_lock(lock, !raw_irqs_disabled_flags(flags)); 320 } 321 322 static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl) 323 { 324 int cpu; 325 326 ADD_STATS(released_slow, 1); 327 328 for_each_online_cpu(cpu) { 329 /* XXX should mix up next cpu selection */ 330 if (per_cpu(lock_spinners, cpu) == xl) { 331 ADD_STATS(released_slow_kicked, 1); 332 xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); 333 } 334 } 335 } 336 337 static void xen_spin_unlock(struct arch_spinlock *lock) 338 { 339 struct xen_spinlock *xl = (struct xen_spinlock *)lock; 340 341 ADD_STATS(released, 1); 342 343 smp_wmb(); /* make sure no writes get moved after unlock */ 344 xl->lock = 0; /* release lock */ 345 346 /* 347 * Make sure unlock happens before checking for waiting 348 * spinners. We need a strong barrier to enforce the 349 * write-read ordering to different memory locations, as the 350 * CPU makes no implied guarantees about their ordering. 351 */ 352 mb(); 353 354 if (unlikely(xl->spinners)) 355 xen_spin_unlock_slow(xl); 356 } 357 358 static irqreturn_t dummy_handler(int irq, void *dev_id) 359 { 360 BUG(); 361 return IRQ_HANDLED; 362 } 363 364 void __cpuinit xen_init_lock_cpu(int cpu) 365 { 366 int irq; 367 char *name; 368 369 WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n", 370 cpu, per_cpu(lock_kicker_irq, cpu)); 371 372 /* 373 * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23 374 * (xen: disable PV spinlocks on HVM) 375 */ 376 if (xen_hvm_domain()) 377 return; 378 379 name = kasprintf(GFP_KERNEL, "spinlock%d", cpu); 380 irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR, 381 cpu, 382 dummy_handler, 383 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING, 384 name, 385 NULL); 386 387 if (irq >= 0) { 388 disable_irq(irq); /* make sure it's never delivered */ 389 per_cpu(lock_kicker_irq, cpu) = irq; 390 per_cpu(irq_name, cpu) = name; 391 } 392 393 printk("cpu %d spinlock event irq %d\n", cpu, irq); 394 } 395 396 void xen_uninit_lock_cpu(int cpu) 397 { 398 /* 399 * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23 400 * (xen: disable PV spinlocks on HVM) 401 */ 402 if (xen_hvm_domain()) 403 return; 404 405 unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL); 406 per_cpu(lock_kicker_irq, cpu) = -1; 407 kfree(per_cpu(irq_name, cpu)); 408 per_cpu(irq_name, cpu) = NULL; 409 } 410 411 void __init xen_init_spinlocks(void) 412 { 413 /* 414 * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23 415 * (xen: disable PV spinlocks on HVM) 416 */ 417 if (xen_hvm_domain()) 418 return; 419 420 BUILD_BUG_ON(sizeof(struct xen_spinlock) > sizeof(arch_spinlock_t)); 421 422 pv_lock_ops.spin_is_locked = xen_spin_is_locked; 423 pv_lock_ops.spin_is_contended = xen_spin_is_contended; 424 pv_lock_ops.spin_lock = xen_spin_lock; 425 pv_lock_ops.spin_lock_flags = xen_spin_lock_flags; 426 pv_lock_ops.spin_trylock = xen_spin_trylock; 427 pv_lock_ops.spin_unlock = xen_spin_unlock; 428 } 429 430 #ifdef CONFIG_XEN_DEBUG_FS 431 432 static struct dentry *d_spin_debug; 433 434 static int __init xen_spinlock_debugfs(void) 435 { 436 struct dentry *d_xen = xen_init_debugfs(); 437 438 if (d_xen == NULL) 439 return -ENOMEM; 440 441 d_spin_debug = debugfs_create_dir("spinlocks", d_xen); 442 443 debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats); 444 445 debugfs_create_u32("timeout", 0644, d_spin_debug, &lock_timeout); 446 447 debugfs_create_u64("taken", 0444, d_spin_debug, &spinlock_stats.taken); 448 debugfs_create_u32("taken_slow", 0444, d_spin_debug, 449 &spinlock_stats.taken_slow); 450 debugfs_create_u32("taken_slow_nested", 0444, d_spin_debug, 451 &spinlock_stats.taken_slow_nested); 452 debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug, 453 &spinlock_stats.taken_slow_pickup); 454 debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug, 455 &spinlock_stats.taken_slow_spurious); 456 debugfs_create_u32("taken_slow_irqenable", 0444, d_spin_debug, 457 &spinlock_stats.taken_slow_irqenable); 458 459 debugfs_create_u64("released", 0444, d_spin_debug, &spinlock_stats.released); 460 debugfs_create_u32("released_slow", 0444, d_spin_debug, 461 &spinlock_stats.released_slow); 462 debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug, 463 &spinlock_stats.released_slow_kicked); 464 465 debugfs_create_u64("time_spinning", 0444, d_spin_debug, 466 &spinlock_stats.time_spinning); 467 debugfs_create_u64("time_blocked", 0444, d_spin_debug, 468 &spinlock_stats.time_blocked); 469 debugfs_create_u64("time_total", 0444, d_spin_debug, 470 &spinlock_stats.time_total); 471 472 debugfs_create_u32_array("histo_total", 0444, d_spin_debug, 473 spinlock_stats.histo_spin_total, HISTO_BUCKETS + 1); 474 debugfs_create_u32_array("histo_spinning", 0444, d_spin_debug, 475 spinlock_stats.histo_spin_spinning, HISTO_BUCKETS + 1); 476 debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug, 477 spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1); 478 479 return 0; 480 } 481 fs_initcall(xen_spinlock_debugfs); 482 483 #endif /* CONFIG_XEN_DEBUG_FS */ 484