1 /* 2 * Copyright (C) 1991, 1992 Linus Torvalds 3 * Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs 4 * Copyright (C) 2011 Don Zickus Red Hat, Inc. 5 * 6 * Pentium III FXSR, SSE support 7 * Gareth Hughes <gareth@valinux.com>, May 2000 8 */ 9 10 /* 11 * Handle hardware traps and faults. 12 */ 13 #include <linux/spinlock.h> 14 #include <linux/kprobes.h> 15 #include <linux/kdebug.h> 16 #include <linux/nmi.h> 17 #include <linux/delay.h> 18 #include <linux/hardirq.h> 19 #include <linux/slab.h> 20 #include <linux/export.h> 21 22 #include <linux/mca.h> 23 24 #if defined(CONFIG_EDAC) 25 #include <linux/edac.h> 26 #endif 27 28 #include <linux/atomic.h> 29 #include <asm/traps.h> 30 #include <asm/mach_traps.h> 31 #include <asm/nmi.h> 32 33 #define NMI_MAX_NAMELEN 16 34 struct nmiaction { 35 struct list_head list; 36 nmi_handler_t handler; 37 unsigned int flags; 38 char *name; 39 }; 40 41 struct nmi_desc { 42 spinlock_t lock; 43 struct list_head head; 44 }; 45 46 static struct nmi_desc nmi_desc[NMI_MAX] = 47 { 48 { 49 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[0].lock), 50 .head = LIST_HEAD_INIT(nmi_desc[0].head), 51 }, 52 { 53 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[1].lock), 54 .head = LIST_HEAD_INIT(nmi_desc[1].head), 55 }, 56 57 }; 58 59 struct nmi_stats { 60 unsigned int normal; 61 unsigned int unknown; 62 unsigned int external; 63 unsigned int swallow; 64 }; 65 66 static DEFINE_PER_CPU(struct nmi_stats, nmi_stats); 67 68 static int ignore_nmis; 69 70 int unknown_nmi_panic; 71 /* 72 * Prevent NMI reason port (0x61) being accessed simultaneously, can 73 * only be used in NMI handler. 74 */ 75 static DEFINE_RAW_SPINLOCK(nmi_reason_lock); 76 77 static int __init setup_unknown_nmi_panic(char *str) 78 { 79 unknown_nmi_panic = 1; 80 return 1; 81 } 82 __setup("unknown_nmi_panic", setup_unknown_nmi_panic); 83 84 #define nmi_to_desc(type) (&nmi_desc[type]) 85 86 static int notrace __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b) 87 { 88 struct nmi_desc *desc = nmi_to_desc(type); 89 struct nmiaction *a; 90 int handled=0; 91 92 rcu_read_lock(); 93 94 /* 95 * NMIs are edge-triggered, which means if you have enough 96 * of them concurrently, you can lose some because only one 97 * can be latched at any given time. Walk the whole list 98 * to handle those situations. 99 */ 100 list_for_each_entry_rcu(a, &desc->head, list) 101 handled += a->handler(type, regs); 102 103 rcu_read_unlock(); 104 105 /* return total number of NMI events handled */ 106 return handled; 107 } 108 109 static int __setup_nmi(unsigned int type, struct nmiaction *action) 110 { 111 struct nmi_desc *desc = nmi_to_desc(type); 112 unsigned long flags; 113 114 spin_lock_irqsave(&desc->lock, flags); 115 116 /* 117 * most handlers of type NMI_UNKNOWN never return because 118 * they just assume the NMI is theirs. Just a sanity check 119 * to manage expectations 120 */ 121 WARN_ON_ONCE(type == NMI_UNKNOWN && !list_empty(&desc->head)); 122 123 /* 124 * some handlers need to be executed first otherwise a fake 125 * event confuses some handlers (kdump uses this flag) 126 */ 127 if (action->flags & NMI_FLAG_FIRST) 128 list_add_rcu(&action->list, &desc->head); 129 else 130 list_add_tail_rcu(&action->list, &desc->head); 131 132 spin_unlock_irqrestore(&desc->lock, flags); 133 return 0; 134 } 135 136 static struct nmiaction *__free_nmi(unsigned int type, const char *name) 137 { 138 struct nmi_desc *desc = nmi_to_desc(type); 139 struct nmiaction *n; 140 unsigned long flags; 141 142 spin_lock_irqsave(&desc->lock, flags); 143 144 list_for_each_entry_rcu(n, &desc->head, list) { 145 /* 146 * the name passed in to describe the nmi handler 147 * is used as the lookup key 148 */ 149 if (!strcmp(n->name, name)) { 150 WARN(in_nmi(), 151 "Trying to free NMI (%s) from NMI context!\n", n->name); 152 list_del_rcu(&n->list); 153 break; 154 } 155 } 156 157 spin_unlock_irqrestore(&desc->lock, flags); 158 synchronize_rcu(); 159 return (n); 160 } 161 162 int register_nmi_handler(unsigned int type, nmi_handler_t handler, 163 unsigned long nmiflags, const char *devname) 164 { 165 struct nmiaction *action; 166 int retval = -ENOMEM; 167 168 if (!handler) 169 return -EINVAL; 170 171 action = kzalloc(sizeof(struct nmiaction), GFP_KERNEL); 172 if (!action) 173 goto fail_action; 174 175 action->handler = handler; 176 action->flags = nmiflags; 177 action->name = kstrndup(devname, NMI_MAX_NAMELEN, GFP_KERNEL); 178 if (!action->name) 179 goto fail_action_name; 180 181 retval = __setup_nmi(type, action); 182 183 if (retval) 184 goto fail_setup_nmi; 185 186 return retval; 187 188 fail_setup_nmi: 189 kfree(action->name); 190 fail_action_name: 191 kfree(action); 192 fail_action: 193 194 return retval; 195 } 196 EXPORT_SYMBOL_GPL(register_nmi_handler); 197 198 void unregister_nmi_handler(unsigned int type, const char *name) 199 { 200 struct nmiaction *a; 201 202 a = __free_nmi(type, name); 203 if (a) { 204 kfree(a->name); 205 kfree(a); 206 } 207 } 208 209 EXPORT_SYMBOL_GPL(unregister_nmi_handler); 210 211 static notrace __kprobes void 212 pci_serr_error(unsigned char reason, struct pt_regs *regs) 213 { 214 pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n", 215 reason, smp_processor_id()); 216 217 /* 218 * On some machines, PCI SERR line is used to report memory 219 * errors. EDAC makes use of it. 220 */ 221 #if defined(CONFIG_EDAC) 222 if (edac_handler_set()) { 223 edac_atomic_assert_error(); 224 return; 225 } 226 #endif 227 228 if (panic_on_unrecovered_nmi) 229 panic("NMI: Not continuing"); 230 231 pr_emerg("Dazed and confused, but trying to continue\n"); 232 233 /* Clear and disable the PCI SERR error line. */ 234 reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR; 235 outb(reason, NMI_REASON_PORT); 236 } 237 238 static notrace __kprobes void 239 io_check_error(unsigned char reason, struct pt_regs *regs) 240 { 241 unsigned long i; 242 243 pr_emerg( 244 "NMI: IOCK error (debug interrupt?) for reason %02x on CPU %d.\n", 245 reason, smp_processor_id()); 246 show_registers(regs); 247 248 if (panic_on_io_nmi) 249 panic("NMI IOCK error: Not continuing"); 250 251 /* Re-enable the IOCK line, wait for a few seconds */ 252 reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK; 253 outb(reason, NMI_REASON_PORT); 254 255 i = 20000; 256 while (--i) { 257 touch_nmi_watchdog(); 258 udelay(100); 259 } 260 261 reason &= ~NMI_REASON_CLEAR_IOCHK; 262 outb(reason, NMI_REASON_PORT); 263 } 264 265 static notrace __kprobes void 266 unknown_nmi_error(unsigned char reason, struct pt_regs *regs) 267 { 268 int handled; 269 270 /* 271 * Use 'false' as back-to-back NMIs are dealt with one level up. 272 * Of course this makes having multiple 'unknown' handlers useless 273 * as only the first one is ever run (unless it can actually determine 274 * if it caused the NMI) 275 */ 276 handled = nmi_handle(NMI_UNKNOWN, regs, false); 277 if (handled) { 278 __this_cpu_add(nmi_stats.unknown, handled); 279 return; 280 } 281 282 __this_cpu_add(nmi_stats.unknown, 1); 283 284 #ifdef CONFIG_MCA 285 /* 286 * Might actually be able to figure out what the guilty party 287 * is: 288 */ 289 if (MCA_bus) { 290 mca_handle_nmi(); 291 return; 292 } 293 #endif 294 pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n", 295 reason, smp_processor_id()); 296 297 pr_emerg("Do you have a strange power saving mode enabled?\n"); 298 if (unknown_nmi_panic || panic_on_unrecovered_nmi) 299 panic("NMI: Not continuing"); 300 301 pr_emerg("Dazed and confused, but trying to continue\n"); 302 } 303 304 static DEFINE_PER_CPU(bool, swallow_nmi); 305 static DEFINE_PER_CPU(unsigned long, last_nmi_rip); 306 307 static notrace __kprobes void default_do_nmi(struct pt_regs *regs) 308 { 309 unsigned char reason = 0; 310 int handled; 311 bool b2b = false; 312 313 /* 314 * CPU-specific NMI must be processed before non-CPU-specific 315 * NMI, otherwise we may lose it, because the CPU-specific 316 * NMI can not be detected/processed on other CPUs. 317 */ 318 319 /* 320 * Back-to-back NMIs are interesting because they can either 321 * be two NMI or more than two NMIs (any thing over two is dropped 322 * due to NMI being edge-triggered). If this is the second half 323 * of the back-to-back NMI, assume we dropped things and process 324 * more handlers. Otherwise reset the 'swallow' NMI behaviour 325 */ 326 if (regs->ip == __this_cpu_read(last_nmi_rip)) 327 b2b = true; 328 else 329 __this_cpu_write(swallow_nmi, false); 330 331 __this_cpu_write(last_nmi_rip, regs->ip); 332 333 handled = nmi_handle(NMI_LOCAL, regs, b2b); 334 __this_cpu_add(nmi_stats.normal, handled); 335 if (handled) { 336 /* 337 * There are cases when a NMI handler handles multiple 338 * events in the current NMI. One of these events may 339 * be queued for in the next NMI. Because the event is 340 * already handled, the next NMI will result in an unknown 341 * NMI. Instead lets flag this for a potential NMI to 342 * swallow. 343 */ 344 if (handled > 1) 345 __this_cpu_write(swallow_nmi, true); 346 return; 347 } 348 349 /* Non-CPU-specific NMI: NMI sources can be processed on any CPU */ 350 raw_spin_lock(&nmi_reason_lock); 351 reason = get_nmi_reason(); 352 353 if (reason & NMI_REASON_MASK) { 354 if (reason & NMI_REASON_SERR) 355 pci_serr_error(reason, regs); 356 else if (reason & NMI_REASON_IOCHK) 357 io_check_error(reason, regs); 358 #ifdef CONFIG_X86_32 359 /* 360 * Reassert NMI in case it became active 361 * meanwhile as it's edge-triggered: 362 */ 363 reassert_nmi(); 364 #endif 365 __this_cpu_add(nmi_stats.external, 1); 366 raw_spin_unlock(&nmi_reason_lock); 367 return; 368 } 369 raw_spin_unlock(&nmi_reason_lock); 370 371 /* 372 * Only one NMI can be latched at a time. To handle 373 * this we may process multiple nmi handlers at once to 374 * cover the case where an NMI is dropped. The downside 375 * to this approach is we may process an NMI prematurely, 376 * while its real NMI is sitting latched. This will cause 377 * an unknown NMI on the next run of the NMI processing. 378 * 379 * We tried to flag that condition above, by setting the 380 * swallow_nmi flag when we process more than one event. 381 * This condition is also only present on the second half 382 * of a back-to-back NMI, so we flag that condition too. 383 * 384 * If both are true, we assume we already processed this 385 * NMI previously and we swallow it. Otherwise we reset 386 * the logic. 387 * 388 * There are scenarios where we may accidentally swallow 389 * a 'real' unknown NMI. For example, while processing 390 * a perf NMI another perf NMI comes in along with a 391 * 'real' unknown NMI. These two NMIs get combined into 392 * one (as descibed above). When the next NMI gets 393 * processed, it will be flagged by perf as handled, but 394 * noone will know that there was a 'real' unknown NMI sent 395 * also. As a result it gets swallowed. Or if the first 396 * perf NMI returns two events handled then the second 397 * NMI will get eaten by the logic below, again losing a 398 * 'real' unknown NMI. But this is the best we can do 399 * for now. 400 */ 401 if (b2b && __this_cpu_read(swallow_nmi)) 402 __this_cpu_add(nmi_stats.swallow, 1); 403 else 404 unknown_nmi_error(reason, regs); 405 } 406 407 dotraplinkage notrace __kprobes void 408 do_nmi(struct pt_regs *regs, long error_code) 409 { 410 nmi_enter(); 411 412 inc_irq_stat(__nmi_count); 413 414 if (!ignore_nmis) 415 default_do_nmi(regs); 416 417 nmi_exit(); 418 } 419 420 void stop_nmi(void) 421 { 422 ignore_nmis++; 423 } 424 425 void restart_nmi(void) 426 { 427 ignore_nmis--; 428 } 429 430 /* reset the back-to-back NMI logic */ 431 void local_touch_nmi(void) 432 { 433 __this_cpu_write(last_nmi_rip, 0); 434 } 435