1 /* 2 * Split spinlock implementation out into its own file, so it can be 3 * compiled in a FTRACE-compatible way. 4 */ 5 #include <linux/kernel_stat.h> 6 #include <linux/spinlock.h> 7 #include <linux/debugfs.h> 8 #include <linux/log2.h> 9 #include <linux/gfp.h> 10 #include <linux/slab.h> 11 12 #include <asm/paravirt.h> 13 14 #include <xen/interface/xen.h> 15 #include <xen/events.h> 16 17 #include "xen-ops.h" 18 #include "debugfs.h" 19 20 enum xen_contention_stat { 21 TAKEN_SLOW, 22 TAKEN_SLOW_PICKUP, 23 TAKEN_SLOW_SPURIOUS, 24 RELEASED_SLOW, 25 RELEASED_SLOW_KICKED, 26 NR_CONTENTION_STATS 27 }; 28 29 30 #ifdef CONFIG_XEN_DEBUG_FS 31 #define HISTO_BUCKETS 30 32 static struct xen_spinlock_stats 33 { 34 u32 contention_stats[NR_CONTENTION_STATS]; 35 u32 histo_spin_blocked[HISTO_BUCKETS+1]; 36 u64 time_blocked; 37 } spinlock_stats; 38 39 static u8 zero_stats; 40 41 static inline void check_zero(void) 42 { 43 u8 ret; 44 u8 old = ACCESS_ONCE(zero_stats); 45 if (unlikely(old)) { 46 ret = cmpxchg(&zero_stats, old, 0); 47 /* This ensures only one fellow resets the stat */ 48 if (ret == old) 49 memset(&spinlock_stats, 0, sizeof(spinlock_stats)); 50 } 51 } 52 53 static inline void add_stats(enum xen_contention_stat var, u32 val) 54 { 55 check_zero(); 56 spinlock_stats.contention_stats[var] += val; 57 } 58 59 static inline u64 spin_time_start(void) 60 { 61 return xen_clocksource_read(); 62 } 63 64 static void __spin_time_accum(u64 delta, u32 *array) 65 { 66 unsigned index = ilog2(delta); 67 68 check_zero(); 69 70 if (index < HISTO_BUCKETS) 71 array[index]++; 72 else 73 array[HISTO_BUCKETS]++; 74 } 75 76 static inline void spin_time_accum_blocked(u64 start) 77 { 78 u32 delta = xen_clocksource_read() - start; 79 80 __spin_time_accum(delta, spinlock_stats.histo_spin_blocked); 81 spinlock_stats.time_blocked += delta; 82 } 83 #else /* !CONFIG_XEN_DEBUG_FS */ 84 #define TIMEOUT (1 << 10) 85 static inline void add_stats(enum xen_contention_stat var, u32 val) 86 { 87 } 88 89 static inline u64 spin_time_start(void) 90 { 91 return 0; 92 } 93 94 static inline void spin_time_accum_blocked(u64 start) 95 { 96 } 97 #endif /* CONFIG_XEN_DEBUG_FS */ 98 99 /* 100 * Size struct xen_spinlock so it's the same as arch_spinlock_t. 101 */ 102 #if NR_CPUS < 256 103 typedef u8 xen_spinners_t; 104 # define inc_spinners(xl) \ 105 asm(LOCK_PREFIX " incb %0" : "+m" ((xl)->spinners) : : "memory"); 106 # define dec_spinners(xl) \ 107 asm(LOCK_PREFIX " decb %0" : "+m" ((xl)->spinners) : : "memory"); 108 #else 109 typedef u16 xen_spinners_t; 110 # define inc_spinners(xl) \ 111 asm(LOCK_PREFIX " incw %0" : "+m" ((xl)->spinners) : : "memory"); 112 # define dec_spinners(xl) \ 113 asm(LOCK_PREFIX " decw %0" : "+m" ((xl)->spinners) : : "memory"); 114 #endif 115 116 struct xen_lock_waiting { 117 struct arch_spinlock *lock; 118 __ticket_t want; 119 }; 120 121 static DEFINE_PER_CPU(int, lock_kicker_irq) = -1; 122 static DEFINE_PER_CPU(char *, irq_name); 123 static DEFINE_PER_CPU(struct xen_lock_waiting, lock_waiting); 124 static cpumask_t waiting_cpus; 125 126 static void xen_lock_spinning(struct arch_spinlock *lock, __ticket_t want) 127 { 128 int irq = __this_cpu_read(lock_kicker_irq); 129 struct xen_lock_waiting *w = &__get_cpu_var(lock_waiting); 130 int cpu = smp_processor_id(); 131 u64 start; 132 unsigned long flags; 133 134 /* If kicker interrupts not initialized yet, just spin */ 135 if (irq == -1) 136 return; 137 138 start = spin_time_start(); 139 140 /* 141 * Make sure an interrupt handler can't upset things in a 142 * partially setup state. 143 */ 144 local_irq_save(flags); 145 /* 146 * We don't really care if we're overwriting some other 147 * (lock,want) pair, as that would mean that we're currently 148 * in an interrupt context, and the outer context had 149 * interrupts enabled. That has already kicked the VCPU out 150 * of xen_poll_irq(), so it will just return spuriously and 151 * retry with newly setup (lock,want). 152 * 153 * The ordering protocol on this is that the "lock" pointer 154 * may only be set non-NULL if the "want" ticket is correct. 155 * If we're updating "want", we must first clear "lock". 156 */ 157 w->lock = NULL; 158 smp_wmb(); 159 w->want = want; 160 smp_wmb(); 161 w->lock = lock; 162 163 /* This uses set_bit, which atomic and therefore a barrier */ 164 cpumask_set_cpu(cpu, &waiting_cpus); 165 add_stats(TAKEN_SLOW, 1); 166 167 /* clear pending */ 168 xen_clear_irq_pending(irq); 169 170 /* Only check lock once pending cleared */ 171 barrier(); 172 173 /* 174 * Mark entry to slowpath before doing the pickup test to make 175 * sure we don't deadlock with an unlocker. 176 */ 177 __ticket_enter_slowpath(lock); 178 179 /* 180 * check again make sure it didn't become free while 181 * we weren't looking 182 */ 183 if (ACCESS_ONCE(lock->tickets.head) == want) { 184 add_stats(TAKEN_SLOW_PICKUP, 1); 185 goto out; 186 } 187 188 /* Allow interrupts while blocked */ 189 local_irq_restore(flags); 190 191 /* 192 * If an interrupt happens here, it will leave the wakeup irq 193 * pending, which will cause xen_poll_irq() to return 194 * immediately. 195 */ 196 197 /* Block until irq becomes pending (or perhaps a spurious wakeup) */ 198 xen_poll_irq(irq); 199 add_stats(TAKEN_SLOW_SPURIOUS, !xen_test_irq_pending(irq)); 200 201 local_irq_save(flags); 202 203 kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq)); 204 out: 205 cpumask_clear_cpu(cpu, &waiting_cpus); 206 w->lock = NULL; 207 208 local_irq_restore(flags); 209 210 spin_time_accum_blocked(start); 211 } 212 PV_CALLEE_SAVE_REGS_THUNK(xen_lock_spinning); 213 214 static void xen_unlock_kick(struct arch_spinlock *lock, __ticket_t next) 215 { 216 int cpu; 217 218 add_stats(RELEASED_SLOW, 1); 219 220 for_each_cpu(cpu, &waiting_cpus) { 221 const struct xen_lock_waiting *w = &per_cpu(lock_waiting, cpu); 222 223 /* Make sure we read lock before want */ 224 if (ACCESS_ONCE(w->lock) == lock && 225 ACCESS_ONCE(w->want) == next) { 226 add_stats(RELEASED_SLOW_KICKED, 1); 227 xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); 228 break; 229 } 230 } 231 } 232 233 static irqreturn_t dummy_handler(int irq, void *dev_id) 234 { 235 BUG(); 236 return IRQ_HANDLED; 237 } 238 239 void xen_init_lock_cpu(int cpu) 240 { 241 int irq; 242 char *name; 243 244 WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n", 245 cpu, per_cpu(lock_kicker_irq, cpu)); 246 247 /* 248 * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23 249 * (xen: disable PV spinlocks on HVM) 250 */ 251 if (xen_hvm_domain()) 252 return; 253 254 name = kasprintf(GFP_KERNEL, "spinlock%d", cpu); 255 irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR, 256 cpu, 257 dummy_handler, 258 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING, 259 name, 260 NULL); 261 262 if (irq >= 0) { 263 disable_irq(irq); /* make sure it's never delivered */ 264 per_cpu(lock_kicker_irq, cpu) = irq; 265 per_cpu(irq_name, cpu) = name; 266 } 267 268 printk("cpu %d spinlock event irq %d\n", cpu, irq); 269 } 270 271 void xen_uninit_lock_cpu(int cpu) 272 { 273 /* 274 * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23 275 * (xen: disable PV spinlocks on HVM) 276 */ 277 if (xen_hvm_domain()) 278 return; 279 280 unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL); 281 per_cpu(lock_kicker_irq, cpu) = -1; 282 kfree(per_cpu(irq_name, cpu)); 283 per_cpu(irq_name, cpu) = NULL; 284 } 285 286 static bool xen_pvspin __initdata = true; 287 288 void __init xen_init_spinlocks(void) 289 { 290 /* 291 * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23 292 * (xen: disable PV spinlocks on HVM) 293 */ 294 if (xen_hvm_domain()) 295 return; 296 297 if (!xen_pvspin) { 298 printk(KERN_DEBUG "xen: PV spinlocks disabled\n"); 299 return; 300 } 301 302 static_key_slow_inc(¶virt_ticketlocks_enabled); 303 304 pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(xen_lock_spinning); 305 pv_lock_ops.unlock_kick = xen_unlock_kick; 306 } 307 308 static __init int xen_parse_nopvspin(char *arg) 309 { 310 xen_pvspin = false; 311 return 0; 312 } 313 early_param("xen_nopvspin", xen_parse_nopvspin); 314 315 #ifdef CONFIG_XEN_DEBUG_FS 316 317 static struct dentry *d_spin_debug; 318 319 static int __init xen_spinlock_debugfs(void) 320 { 321 struct dentry *d_xen = xen_init_debugfs(); 322 323 if (d_xen == NULL) 324 return -ENOMEM; 325 326 d_spin_debug = debugfs_create_dir("spinlocks", d_xen); 327 328 debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats); 329 330 debugfs_create_u32("taken_slow", 0444, d_spin_debug, 331 &spinlock_stats.contention_stats[TAKEN_SLOW]); 332 debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug, 333 &spinlock_stats.contention_stats[TAKEN_SLOW_PICKUP]); 334 debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug, 335 &spinlock_stats.contention_stats[TAKEN_SLOW_SPURIOUS]); 336 337 debugfs_create_u32("released_slow", 0444, d_spin_debug, 338 &spinlock_stats.contention_stats[RELEASED_SLOW]); 339 debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug, 340 &spinlock_stats.contention_stats[RELEASED_SLOW_KICKED]); 341 342 debugfs_create_u64("time_blocked", 0444, d_spin_debug, 343 &spinlock_stats.time_blocked); 344 345 debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug, 346 spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1); 347 348 return 0; 349 } 350 fs_initcall(xen_spinlock_debugfs); 351 352 #endif /* CONFIG_XEN_DEBUG_FS */ 353