xref: /linux/arch/x86/kernel/cpu/mce/core.c (revision a7f7f6248d9740d710fd6bd190293fe5e16410ac)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Machine check handler.
4  *
5  * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
6  * Rest from unknown author(s).
7  * 2004 Andi Kleen. Rewrote most of it.
8  * Copyright 2008 Intel Corporation
9  * Author: Andi Kleen
10  */
11 
12 #include <linux/thread_info.h>
13 #include <linux/capability.h>
14 #include <linux/miscdevice.h>
15 #include <linux/ratelimit.h>
16 #include <linux/rcupdate.h>
17 #include <linux/kobject.h>
18 #include <linux/uaccess.h>
19 #include <linux/kdebug.h>
20 #include <linux/kernel.h>
21 #include <linux/percpu.h>
22 #include <linux/string.h>
23 #include <linux/device.h>
24 #include <linux/syscore_ops.h>
25 #include <linux/delay.h>
26 #include <linux/ctype.h>
27 #include <linux/sched.h>
28 #include <linux/sysfs.h>
29 #include <linux/types.h>
30 #include <linux/slab.h>
31 #include <linux/init.h>
32 #include <linux/kmod.h>
33 #include <linux/poll.h>
34 #include <linux/nmi.h>
35 #include <linux/cpu.h>
36 #include <linux/ras.h>
37 #include <linux/smp.h>
38 #include <linux/fs.h>
39 #include <linux/mm.h>
40 #include <linux/debugfs.h>
41 #include <linux/irq_work.h>
42 #include <linux/export.h>
43 #include <linux/jump_label.h>
44 #include <linux/set_memory.h>
45 #include <linux/task_work.h>
46 #include <linux/hardirq.h>
47 
48 #include <asm/intel-family.h>
49 #include <asm/processor.h>
50 #include <asm/traps.h>
51 #include <asm/tlbflush.h>
52 #include <asm/mce.h>
53 #include <asm/msr.h>
54 #include <asm/reboot.h>
55 
56 #include "internal.h"
57 
58 /* sysfs synchronization */
59 static DEFINE_MUTEX(mce_sysfs_mutex);
60 
61 #define CREATE_TRACE_POINTS
62 #include <trace/events/mce.h>
63 
64 #define SPINUNIT		100	/* 100ns */
65 
66 DEFINE_PER_CPU(unsigned, mce_exception_count);
67 
68 DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks);
69 
70 struct mce_bank {
71 	u64			ctl;			/* subevents to enable */
72 	bool			init;			/* initialise bank? */
73 };
74 static DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array);
75 
76 #define ATTR_LEN               16
77 /* One object for each MCE bank, shared by all CPUs */
78 struct mce_bank_dev {
79 	struct device_attribute	attr;			/* device attribute */
80 	char			attrname[ATTR_LEN];	/* attribute name */
81 	u8			bank;			/* bank number */
82 };
83 static struct mce_bank_dev mce_bank_devs[MAX_NR_BANKS];
84 
85 struct mce_vendor_flags mce_flags __read_mostly;
86 
87 struct mca_config mca_cfg __read_mostly = {
88 	.bootlog  = -1,
89 	/*
90 	 * Tolerant levels:
91 	 * 0: always panic on uncorrected errors, log corrected errors
92 	 * 1: panic or SIGBUS on uncorrected errors, log corrected errors
93 	 * 2: SIGBUS or log uncorrected errors (if possible), log corr. errors
94 	 * 3: never panic or SIGBUS, log all errors (for testing only)
95 	 */
96 	.tolerant = 1,
97 	.monarch_timeout = -1
98 };
99 
100 static DEFINE_PER_CPU(struct mce, mces_seen);
101 static unsigned long mce_need_notify;
102 static int cpu_missing;
103 
104 /*
105  * MCA banks polled by the period polling timer for corrected events.
106  * With Intel CMCI, this only has MCA banks which do not support CMCI (if any).
107  */
108 DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
109 	[0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
110 };
111 
112 /*
113  * MCA banks controlled through firmware first for corrected errors.
114  * This is a global list of banks for which we won't enable CMCI and we
115  * won't poll. Firmware controls these banks and is responsible for
116  * reporting corrected errors through GHES. Uncorrected/recoverable
117  * errors are still notified through a machine check.
118  */
119 mce_banks_t mce_banks_ce_disabled;
120 
121 static struct work_struct mce_work;
122 static struct irq_work mce_irq_work;
123 
124 static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs);
125 
126 /*
127  * CPU/chipset specific EDAC code can register a notifier call here to print
128  * MCE errors in a human-readable form.
129  */
130 BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain);
131 
132 /* Do initial initialization of a struct mce */
133 void mce_setup(struct mce *m)
134 {
135 	memset(m, 0, sizeof(struct mce));
136 	m->cpu = m->extcpu = smp_processor_id();
137 	/* need the internal __ version to avoid deadlocks */
138 	m->time = __ktime_get_real_seconds();
139 	m->cpuvendor = boot_cpu_data.x86_vendor;
140 	m->cpuid = cpuid_eax(1);
141 	m->socketid = cpu_data(m->extcpu).phys_proc_id;
142 	m->apicid = cpu_data(m->extcpu).initial_apicid;
143 	rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);
144 
145 	if (this_cpu_has(X86_FEATURE_INTEL_PPIN))
146 		rdmsrl(MSR_PPIN, m->ppin);
147 	else if (this_cpu_has(X86_FEATURE_AMD_PPIN))
148 		rdmsrl(MSR_AMD_PPIN, m->ppin);
149 
150 	m->microcode = boot_cpu_data.microcode;
151 }
152 
153 DEFINE_PER_CPU(struct mce, injectm);
154 EXPORT_PER_CPU_SYMBOL_GPL(injectm);
155 
156 void mce_log(struct mce *m)
157 {
158 	if (!mce_gen_pool_add(m))
159 		irq_work_queue(&mce_irq_work);
160 }
161 EXPORT_SYMBOL_GPL(mce_log);
162 
163 /*
164  * We run the default notifier if we have only the UC, the first and the
165  * default notifier registered. I.e., the mandatory NUM_DEFAULT_NOTIFIERS
166  * notifiers registered on the chain.
167  */
168 #define NUM_DEFAULT_NOTIFIERS	3
169 static atomic_t num_notifiers;
170 
171 void mce_register_decode_chain(struct notifier_block *nb)
172 {
173 	if (WARN_ON(nb->priority > MCE_PRIO_MCELOG && nb->priority < MCE_PRIO_EDAC))
174 		return;
175 
176 	atomic_inc(&num_notifiers);
177 
178 	blocking_notifier_chain_register(&x86_mce_decoder_chain, nb);
179 }
180 EXPORT_SYMBOL_GPL(mce_register_decode_chain);
181 
182 void mce_unregister_decode_chain(struct notifier_block *nb)
183 {
184 	atomic_dec(&num_notifiers);
185 
186 	blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb);
187 }
188 EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);
189 
190 static inline u32 ctl_reg(int bank)
191 {
192 	return MSR_IA32_MCx_CTL(bank);
193 }
194 
195 static inline u32 status_reg(int bank)
196 {
197 	return MSR_IA32_MCx_STATUS(bank);
198 }
199 
200 static inline u32 addr_reg(int bank)
201 {
202 	return MSR_IA32_MCx_ADDR(bank);
203 }
204 
205 static inline u32 misc_reg(int bank)
206 {
207 	return MSR_IA32_MCx_MISC(bank);
208 }
209 
210 static inline u32 smca_ctl_reg(int bank)
211 {
212 	return MSR_AMD64_SMCA_MCx_CTL(bank);
213 }
214 
215 static inline u32 smca_status_reg(int bank)
216 {
217 	return MSR_AMD64_SMCA_MCx_STATUS(bank);
218 }
219 
220 static inline u32 smca_addr_reg(int bank)
221 {
222 	return MSR_AMD64_SMCA_MCx_ADDR(bank);
223 }
224 
225 static inline u32 smca_misc_reg(int bank)
226 {
227 	return MSR_AMD64_SMCA_MCx_MISC(bank);
228 }
229 
230 struct mca_msr_regs msr_ops = {
231 	.ctl	= ctl_reg,
232 	.status	= status_reg,
233 	.addr	= addr_reg,
234 	.misc	= misc_reg
235 };
236 
237 static void __print_mce(struct mce *m)
238 {
239 	pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n",
240 		 m->extcpu,
241 		 (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""),
242 		 m->mcgstatus, m->bank, m->status);
243 
244 	if (m->ip) {
245 		pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
246 			!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
247 			m->cs, m->ip);
248 
249 		if (m->cs == __KERNEL_CS)
250 			pr_cont("{%pS}", (void *)(unsigned long)m->ip);
251 		pr_cont("\n");
252 	}
253 
254 	pr_emerg(HW_ERR "TSC %llx ", m->tsc);
255 	if (m->addr)
256 		pr_cont("ADDR %llx ", m->addr);
257 	if (m->misc)
258 		pr_cont("MISC %llx ", m->misc);
259 
260 	if (mce_flags.smca) {
261 		if (m->synd)
262 			pr_cont("SYND %llx ", m->synd);
263 		if (m->ipid)
264 			pr_cont("IPID %llx ", m->ipid);
265 	}
266 
267 	pr_cont("\n");
268 	/*
269 	 * Note this output is parsed by external tools and old fields
270 	 * should not be changed.
271 	 */
272 	pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
273 		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
274 		m->microcode);
275 }
276 
277 static void print_mce(struct mce *m)
278 {
279 	__print_mce(m);
280 
281 	if (m->cpuvendor != X86_VENDOR_AMD && m->cpuvendor != X86_VENDOR_HYGON)
282 		pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
283 }
284 
285 #define PANIC_TIMEOUT 5 /* 5 seconds */
286 
287 static atomic_t mce_panicked;
288 
289 static int fake_panic;
290 static atomic_t mce_fake_panicked;
291 
292 /* Panic in progress. Enable interrupts and wait for final IPI */
293 static void wait_for_panic(void)
294 {
295 	long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
296 
297 	preempt_disable();
298 	local_irq_enable();
299 	while (timeout-- > 0)
300 		udelay(1);
301 	if (panic_timeout == 0)
302 		panic_timeout = mca_cfg.panic_timeout;
303 	panic("Panicing machine check CPU died");
304 }
305 
306 static void mce_panic(const char *msg, struct mce *final, char *exp)
307 {
308 	int apei_err = 0;
309 	struct llist_node *pending;
310 	struct mce_evt_llist *l;
311 
312 	if (!fake_panic) {
313 		/*
314 		 * Make sure only one CPU runs in machine check panic
315 		 */
316 		if (atomic_inc_return(&mce_panicked) > 1)
317 			wait_for_panic();
318 		barrier();
319 
320 		bust_spinlocks(1);
321 		console_verbose();
322 	} else {
323 		/* Don't log too much for fake panic */
324 		if (atomic_inc_return(&mce_fake_panicked) > 1)
325 			return;
326 	}
327 	pending = mce_gen_pool_prepare_records();
328 	/* First print corrected ones that are still unlogged */
329 	llist_for_each_entry(l, pending, llnode) {
330 		struct mce *m = &l->mce;
331 		if (!(m->status & MCI_STATUS_UC)) {
332 			print_mce(m);
333 			if (!apei_err)
334 				apei_err = apei_write_mce(m);
335 		}
336 	}
337 	/* Now print uncorrected but with the final one last */
338 	llist_for_each_entry(l, pending, llnode) {
339 		struct mce *m = &l->mce;
340 		if (!(m->status & MCI_STATUS_UC))
341 			continue;
342 		if (!final || mce_cmp(m, final)) {
343 			print_mce(m);
344 			if (!apei_err)
345 				apei_err = apei_write_mce(m);
346 		}
347 	}
348 	if (final) {
349 		print_mce(final);
350 		if (!apei_err)
351 			apei_err = apei_write_mce(final);
352 	}
353 	if (cpu_missing)
354 		pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n");
355 	if (exp)
356 		pr_emerg(HW_ERR "Machine check: %s\n", exp);
357 	if (!fake_panic) {
358 		if (panic_timeout == 0)
359 			panic_timeout = mca_cfg.panic_timeout;
360 		panic(msg);
361 	} else
362 		pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
363 }
364 
365 /* Support code for software error injection */
366 
367 static int msr_to_offset(u32 msr)
368 {
369 	unsigned bank = __this_cpu_read(injectm.bank);
370 
371 	if (msr == mca_cfg.rip_msr)
372 		return offsetof(struct mce, ip);
373 	if (msr == msr_ops.status(bank))
374 		return offsetof(struct mce, status);
375 	if (msr == msr_ops.addr(bank))
376 		return offsetof(struct mce, addr);
377 	if (msr == msr_ops.misc(bank))
378 		return offsetof(struct mce, misc);
379 	if (msr == MSR_IA32_MCG_STATUS)
380 		return offsetof(struct mce, mcgstatus);
381 	return -1;
382 }
383 
384 /* MSR access wrappers used for error injection */
385 static u64 mce_rdmsrl(u32 msr)
386 {
387 	u64 v;
388 
389 	if (__this_cpu_read(injectm.finished)) {
390 		int offset = msr_to_offset(msr);
391 
392 		if (offset < 0)
393 			return 0;
394 		return *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
395 	}
396 
397 	if (rdmsrl_safe(msr, &v)) {
398 		WARN_ONCE(1, "mce: Unable to read MSR 0x%x!\n", msr);
399 		/*
400 		 * Return zero in case the access faulted. This should
401 		 * not happen normally but can happen if the CPU does
402 		 * something weird, or if the code is buggy.
403 		 */
404 		v = 0;
405 	}
406 
407 	return v;
408 }
409 
410 static void mce_wrmsrl(u32 msr, u64 v)
411 {
412 	if (__this_cpu_read(injectm.finished)) {
413 		int offset = msr_to_offset(msr);
414 
415 		if (offset >= 0)
416 			*(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
417 		return;
418 	}
419 	wrmsrl(msr, v);
420 }
421 
422 /*
423  * Collect all global (w.r.t. this processor) status about this machine
424  * check into our "mce" struct so that we can use it later to assess
425  * the severity of the problem as we read per-bank specific details.
426  */
427 static inline void mce_gather_info(struct mce *m, struct pt_regs *regs)
428 {
429 	mce_setup(m);
430 
431 	m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
432 	if (regs) {
433 		/*
434 		 * Get the address of the instruction at the time of
435 		 * the machine check error.
436 		 */
437 		if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
438 			m->ip = regs->ip;
439 			m->cs = regs->cs;
440 
441 			/*
442 			 * When in VM86 mode make the cs look like ring 3
443 			 * always. This is a lie, but it's better than passing
444 			 * the additional vm86 bit around everywhere.
445 			 */
446 			if (v8086_mode(regs))
447 				m->cs |= 3;
448 		}
449 		/* Use accurate RIP reporting if available. */
450 		if (mca_cfg.rip_msr)
451 			m->ip = mce_rdmsrl(mca_cfg.rip_msr);
452 	}
453 }
454 
455 int mce_available(struct cpuinfo_x86 *c)
456 {
457 	if (mca_cfg.disabled)
458 		return 0;
459 	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
460 }
461 
462 static void mce_schedule_work(void)
463 {
464 	if (!mce_gen_pool_empty())
465 		schedule_work(&mce_work);
466 }
467 
468 static void mce_irq_work_cb(struct irq_work *entry)
469 {
470 	mce_schedule_work();
471 }
472 
473 /*
474  * Check if the address reported by the CPU is in a format we can parse.
475  * It would be possible to add code for most other cases, but all would
476  * be somewhat complicated (e.g. segment offset would require an instruction
477  * parser). So only support physical addresses up to page granuality for now.
478  */
479 int mce_usable_address(struct mce *m)
480 {
481 	if (!(m->status & MCI_STATUS_ADDRV))
482 		return 0;
483 
484 	/* Checks after this one are Intel/Zhaoxin-specific: */
485 	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL &&
486 	    boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN)
487 		return 1;
488 
489 	if (!(m->status & MCI_STATUS_MISCV))
490 		return 0;
491 
492 	if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
493 		return 0;
494 
495 	if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
496 		return 0;
497 
498 	return 1;
499 }
500 EXPORT_SYMBOL_GPL(mce_usable_address);
501 
502 bool mce_is_memory_error(struct mce *m)
503 {
504 	switch (m->cpuvendor) {
505 	case X86_VENDOR_AMD:
506 	case X86_VENDOR_HYGON:
507 		return amd_mce_is_memory_error(m);
508 
509 	case X86_VENDOR_INTEL:
510 	case X86_VENDOR_ZHAOXIN:
511 		/*
512 		 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
513 		 *
514 		 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for
515 		 * indicating a memory error. Bit 8 is used for indicating a
516 		 * cache hierarchy error. The combination of bit 2 and bit 3
517 		 * is used for indicating a `generic' cache hierarchy error
518 		 * But we can't just blindly check the above bits, because if
519 		 * bit 11 is set, then it is a bus/interconnect error - and
520 		 * either way the above bits just gives more detail on what
521 		 * bus/interconnect error happened. Note that bit 12 can be
522 		 * ignored, as it's the "filter" bit.
523 		 */
524 		return (m->status & 0xef80) == BIT(7) ||
525 		       (m->status & 0xef00) == BIT(8) ||
526 		       (m->status & 0xeffc) == 0xc;
527 
528 	default:
529 		return false;
530 	}
531 }
532 EXPORT_SYMBOL_GPL(mce_is_memory_error);
533 
534 bool mce_is_correctable(struct mce *m)
535 {
536 	if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED)
537 		return false;
538 
539 	if (m->cpuvendor == X86_VENDOR_HYGON && m->status & MCI_STATUS_DEFERRED)
540 		return false;
541 
542 	if (m->status & MCI_STATUS_UC)
543 		return false;
544 
545 	return true;
546 }
547 EXPORT_SYMBOL_GPL(mce_is_correctable);
548 
549 static bool cec_add_mce(struct mce *m)
550 {
551 	if (!m)
552 		return false;
553 
554 	/* We eat only correctable DRAM errors with usable addresses. */
555 	if (mce_is_memory_error(m) &&
556 	    mce_is_correctable(m)  &&
557 	    mce_usable_address(m))
558 		if (!cec_add_elem(m->addr >> PAGE_SHIFT))
559 			return true;
560 
561 	return false;
562 }
563 
564 static int mce_first_notifier(struct notifier_block *nb, unsigned long val,
565 			      void *data)
566 {
567 	struct mce *m = (struct mce *)data;
568 
569 	if (!m)
570 		return NOTIFY_DONE;
571 
572 	if (cec_add_mce(m))
573 		return NOTIFY_STOP;
574 
575 	/* Emit the trace record: */
576 	trace_mce_record(m);
577 
578 	set_bit(0, &mce_need_notify);
579 
580 	mce_notify_irq();
581 
582 	return NOTIFY_DONE;
583 }
584 
585 static struct notifier_block first_nb = {
586 	.notifier_call	= mce_first_notifier,
587 	.priority	= MCE_PRIO_FIRST,
588 };
589 
590 static int uc_decode_notifier(struct notifier_block *nb, unsigned long val,
591 			      void *data)
592 {
593 	struct mce *mce = (struct mce *)data;
594 	unsigned long pfn;
595 
596 	if (!mce || !mce_usable_address(mce))
597 		return NOTIFY_DONE;
598 
599 	if (mce->severity != MCE_AO_SEVERITY &&
600 	    mce->severity != MCE_DEFERRED_SEVERITY)
601 		return NOTIFY_DONE;
602 
603 	pfn = mce->addr >> PAGE_SHIFT;
604 	if (!memory_failure(pfn, 0))
605 		set_mce_nospec(pfn);
606 
607 	return NOTIFY_OK;
608 }
609 
610 static struct notifier_block mce_uc_nb = {
611 	.notifier_call	= uc_decode_notifier,
612 	.priority	= MCE_PRIO_UC,
613 };
614 
615 static int mce_default_notifier(struct notifier_block *nb, unsigned long val,
616 				void *data)
617 {
618 	struct mce *m = (struct mce *)data;
619 
620 	if (!m)
621 		return NOTIFY_DONE;
622 
623 	if (atomic_read(&num_notifiers) > NUM_DEFAULT_NOTIFIERS)
624 		return NOTIFY_DONE;
625 
626 	__print_mce(m);
627 
628 	return NOTIFY_DONE;
629 }
630 
631 static struct notifier_block mce_default_nb = {
632 	.notifier_call	= mce_default_notifier,
633 	/* lowest prio, we want it to run last. */
634 	.priority	= MCE_PRIO_LOWEST,
635 };
636 
637 /*
638  * Read ADDR and MISC registers.
639  */
640 static void mce_read_aux(struct mce *m, int i)
641 {
642 	if (m->status & MCI_STATUS_MISCV)
643 		m->misc = mce_rdmsrl(msr_ops.misc(i));
644 
645 	if (m->status & MCI_STATUS_ADDRV) {
646 		m->addr = mce_rdmsrl(msr_ops.addr(i));
647 
648 		/*
649 		 * Mask the reported address by the reported granularity.
650 		 */
651 		if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {
652 			u8 shift = MCI_MISC_ADDR_LSB(m->misc);
653 			m->addr >>= shift;
654 			m->addr <<= shift;
655 		}
656 
657 		/*
658 		 * Extract [55:<lsb>] where lsb is the least significant
659 		 * *valid* bit of the address bits.
660 		 */
661 		if (mce_flags.smca) {
662 			u8 lsb = (m->addr >> 56) & 0x3f;
663 
664 			m->addr &= GENMASK_ULL(55, lsb);
665 		}
666 	}
667 
668 	if (mce_flags.smca) {
669 		m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i));
670 
671 		if (m->status & MCI_STATUS_SYNDV)
672 			m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i));
673 	}
674 }
675 
676 DEFINE_PER_CPU(unsigned, mce_poll_count);
677 
678 /*
679  * Poll for corrected events or events that happened before reset.
680  * Those are just logged through /dev/mcelog.
681  *
682  * This is executed in standard interrupt context.
683  *
684  * Note: spec recommends to panic for fatal unsignalled
685  * errors here. However this would be quite problematic --
686  * we would need to reimplement the Monarch handling and
687  * it would mess up the exclusion between exception handler
688  * and poll handler -- * so we skip this for now.
689  * These cases should not happen anyways, or only when the CPU
690  * is already totally * confused. In this case it's likely it will
691  * not fully execute the machine check handler either.
692  */
693 bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
694 {
695 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
696 	bool error_seen = false;
697 	struct mce m;
698 	int i;
699 
700 	this_cpu_inc(mce_poll_count);
701 
702 	mce_gather_info(&m, NULL);
703 
704 	if (flags & MCP_TIMESTAMP)
705 		m.tsc = rdtsc();
706 
707 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
708 		if (!mce_banks[i].ctl || !test_bit(i, *b))
709 			continue;
710 
711 		m.misc = 0;
712 		m.addr = 0;
713 		m.bank = i;
714 
715 		barrier();
716 		m.status = mce_rdmsrl(msr_ops.status(i));
717 
718 		/* If this entry is not valid, ignore it */
719 		if (!(m.status & MCI_STATUS_VAL))
720 			continue;
721 
722 		/*
723 		 * If we are logging everything (at CPU online) or this
724 		 * is a corrected error, then we must log it.
725 		 */
726 		if ((flags & MCP_UC) || !(m.status & MCI_STATUS_UC))
727 			goto log_it;
728 
729 		/*
730 		 * Newer Intel systems that support software error
731 		 * recovery need to make additional checks. Other
732 		 * CPUs should skip over uncorrected errors, but log
733 		 * everything else.
734 		 */
735 		if (!mca_cfg.ser) {
736 			if (m.status & MCI_STATUS_UC)
737 				continue;
738 			goto log_it;
739 		}
740 
741 		/* Log "not enabled" (speculative) errors */
742 		if (!(m.status & MCI_STATUS_EN))
743 			goto log_it;
744 
745 		/*
746 		 * Log UCNA (SDM: 15.6.3 "UCR Error Classification")
747 		 * UC == 1 && PCC == 0 && S == 0
748 		 */
749 		if (!(m.status & MCI_STATUS_PCC) && !(m.status & MCI_STATUS_S))
750 			goto log_it;
751 
752 		/*
753 		 * Skip anything else. Presumption is that our read of this
754 		 * bank is racing with a machine check. Leave the log alone
755 		 * for do_machine_check() to deal with it.
756 		 */
757 		continue;
758 
759 log_it:
760 		error_seen = true;
761 
762 		if (flags & MCP_DONTLOG)
763 			goto clear_it;
764 
765 		mce_read_aux(&m, i);
766 		m.severity = mce_severity(&m, mca_cfg.tolerant, NULL, false);
767 		/*
768 		 * Don't get the IP here because it's unlikely to
769 		 * have anything to do with the actual error location.
770 		 */
771 
772 		if (mca_cfg.dont_log_ce && !mce_usable_address(&m))
773 			goto clear_it;
774 
775 		mce_log(&m);
776 
777 clear_it:
778 		/*
779 		 * Clear state for this bank.
780 		 */
781 		mce_wrmsrl(msr_ops.status(i), 0);
782 	}
783 
784 	/*
785 	 * Don't clear MCG_STATUS here because it's only defined for
786 	 * exceptions.
787 	 */
788 
789 	sync_core();
790 
791 	return error_seen;
792 }
793 EXPORT_SYMBOL_GPL(machine_check_poll);
794 
795 /*
796  * Do a quick check if any of the events requires a panic.
797  * This decides if we keep the events around or clear them.
798  */
799 static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
800 			  struct pt_regs *regs)
801 {
802 	char *tmp = *msg;
803 	int i;
804 
805 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
806 		m->status = mce_rdmsrl(msr_ops.status(i));
807 		if (!(m->status & MCI_STATUS_VAL))
808 			continue;
809 
810 		__set_bit(i, validp);
811 		if (quirk_no_way_out)
812 			quirk_no_way_out(i, m, regs);
813 
814 		m->bank = i;
815 		if (mce_severity(m, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) {
816 			mce_read_aux(m, i);
817 			*msg = tmp;
818 			return 1;
819 		}
820 	}
821 	return 0;
822 }
823 
824 /*
825  * Variable to establish order between CPUs while scanning.
826  * Each CPU spins initially until executing is equal its number.
827  */
828 static atomic_t mce_executing;
829 
830 /*
831  * Defines order of CPUs on entry. First CPU becomes Monarch.
832  */
833 static atomic_t mce_callin;
834 
835 /*
836  * Check if a timeout waiting for other CPUs happened.
837  */
838 static int mce_timed_out(u64 *t, const char *msg)
839 {
840 	/*
841 	 * The others already did panic for some reason.
842 	 * Bail out like in a timeout.
843 	 * rmb() to tell the compiler that system_state
844 	 * might have been modified by someone else.
845 	 */
846 	rmb();
847 	if (atomic_read(&mce_panicked))
848 		wait_for_panic();
849 	if (!mca_cfg.monarch_timeout)
850 		goto out;
851 	if ((s64)*t < SPINUNIT) {
852 		if (mca_cfg.tolerant <= 1)
853 			mce_panic(msg, NULL, NULL);
854 		cpu_missing = 1;
855 		return 1;
856 	}
857 	*t -= SPINUNIT;
858 out:
859 	touch_nmi_watchdog();
860 	return 0;
861 }
862 
863 /*
864  * The Monarch's reign.  The Monarch is the CPU who entered
865  * the machine check handler first. It waits for the others to
866  * raise the exception too and then grades them. When any
867  * error is fatal panic. Only then let the others continue.
868  *
869  * The other CPUs entering the MCE handler will be controlled by the
870  * Monarch. They are called Subjects.
871  *
872  * This way we prevent any potential data corruption in a unrecoverable case
873  * and also makes sure always all CPU's errors are examined.
874  *
875  * Also this detects the case of a machine check event coming from outer
876  * space (not detected by any CPUs) In this case some external agent wants
877  * us to shut down, so panic too.
878  *
879  * The other CPUs might still decide to panic if the handler happens
880  * in a unrecoverable place, but in this case the system is in a semi-stable
881  * state and won't corrupt anything by itself. It's ok to let the others
882  * continue for a bit first.
883  *
884  * All the spin loops have timeouts; when a timeout happens a CPU
885  * typically elects itself to be Monarch.
886  */
887 static void mce_reign(void)
888 {
889 	int cpu;
890 	struct mce *m = NULL;
891 	int global_worst = 0;
892 	char *msg = NULL;
893 	char *nmsg = NULL;
894 
895 	/*
896 	 * This CPU is the Monarch and the other CPUs have run
897 	 * through their handlers.
898 	 * Grade the severity of the errors of all the CPUs.
899 	 */
900 	for_each_possible_cpu(cpu) {
901 		int severity = mce_severity(&per_cpu(mces_seen, cpu),
902 					    mca_cfg.tolerant,
903 					    &nmsg, true);
904 		if (severity > global_worst) {
905 			msg = nmsg;
906 			global_worst = severity;
907 			m = &per_cpu(mces_seen, cpu);
908 		}
909 	}
910 
911 	/*
912 	 * Cannot recover? Panic here then.
913 	 * This dumps all the mces in the log buffer and stops the
914 	 * other CPUs.
915 	 */
916 	if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
917 		mce_panic("Fatal machine check", m, msg);
918 
919 	/*
920 	 * For UC somewhere we let the CPU who detects it handle it.
921 	 * Also must let continue the others, otherwise the handling
922 	 * CPU could deadlock on a lock.
923 	 */
924 
925 	/*
926 	 * No machine check event found. Must be some external
927 	 * source or one CPU is hung. Panic.
928 	 */
929 	if (global_worst <= MCE_KEEP_SEVERITY && mca_cfg.tolerant < 3)
930 		mce_panic("Fatal machine check from unknown source", NULL, NULL);
931 
932 	/*
933 	 * Now clear all the mces_seen so that they don't reappear on
934 	 * the next mce.
935 	 */
936 	for_each_possible_cpu(cpu)
937 		memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
938 }
939 
940 static atomic_t global_nwo;
941 
942 /*
943  * Start of Monarch synchronization. This waits until all CPUs have
944  * entered the exception handler and then determines if any of them
945  * saw a fatal event that requires panic. Then it executes them
946  * in the entry order.
947  * TBD double check parallel CPU hotunplug
948  */
949 static int mce_start(int *no_way_out)
950 {
951 	int order;
952 	int cpus = num_online_cpus();
953 	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
954 
955 	if (!timeout)
956 		return -1;
957 
958 	atomic_add(*no_way_out, &global_nwo);
959 	/*
960 	 * Rely on the implied barrier below, such that global_nwo
961 	 * is updated before mce_callin.
962 	 */
963 	order = atomic_inc_return(&mce_callin);
964 
965 	/*
966 	 * Wait for everyone.
967 	 */
968 	while (atomic_read(&mce_callin) != cpus) {
969 		if (mce_timed_out(&timeout,
970 				  "Timeout: Not all CPUs entered broadcast exception handler")) {
971 			atomic_set(&global_nwo, 0);
972 			return -1;
973 		}
974 		ndelay(SPINUNIT);
975 	}
976 
977 	/*
978 	 * mce_callin should be read before global_nwo
979 	 */
980 	smp_rmb();
981 
982 	if (order == 1) {
983 		/*
984 		 * Monarch: Starts executing now, the others wait.
985 		 */
986 		atomic_set(&mce_executing, 1);
987 	} else {
988 		/*
989 		 * Subject: Now start the scanning loop one by one in
990 		 * the original callin order.
991 		 * This way when there are any shared banks it will be
992 		 * only seen by one CPU before cleared, avoiding duplicates.
993 		 */
994 		while (atomic_read(&mce_executing) < order) {
995 			if (mce_timed_out(&timeout,
996 					  "Timeout: Subject CPUs unable to finish machine check processing")) {
997 				atomic_set(&global_nwo, 0);
998 				return -1;
999 			}
1000 			ndelay(SPINUNIT);
1001 		}
1002 	}
1003 
1004 	/*
1005 	 * Cache the global no_way_out state.
1006 	 */
1007 	*no_way_out = atomic_read(&global_nwo);
1008 
1009 	return order;
1010 }
1011 
1012 /*
1013  * Synchronize between CPUs after main scanning loop.
1014  * This invokes the bulk of the Monarch processing.
1015  */
1016 static int mce_end(int order)
1017 {
1018 	int ret = -1;
1019 	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
1020 
1021 	if (!timeout)
1022 		goto reset;
1023 	if (order < 0)
1024 		goto reset;
1025 
1026 	/*
1027 	 * Allow others to run.
1028 	 */
1029 	atomic_inc(&mce_executing);
1030 
1031 	if (order == 1) {
1032 		/* CHECKME: Can this race with a parallel hotplug? */
1033 		int cpus = num_online_cpus();
1034 
1035 		/*
1036 		 * Monarch: Wait for everyone to go through their scanning
1037 		 * loops.
1038 		 */
1039 		while (atomic_read(&mce_executing) <= cpus) {
1040 			if (mce_timed_out(&timeout,
1041 					  "Timeout: Monarch CPU unable to finish machine check processing"))
1042 				goto reset;
1043 			ndelay(SPINUNIT);
1044 		}
1045 
1046 		mce_reign();
1047 		barrier();
1048 		ret = 0;
1049 	} else {
1050 		/*
1051 		 * Subject: Wait for Monarch to finish.
1052 		 */
1053 		while (atomic_read(&mce_executing) != 0) {
1054 			if (mce_timed_out(&timeout,
1055 					  "Timeout: Monarch CPU did not finish machine check processing"))
1056 				goto reset;
1057 			ndelay(SPINUNIT);
1058 		}
1059 
1060 		/*
1061 		 * Don't reset anything. That's done by the Monarch.
1062 		 */
1063 		return 0;
1064 	}
1065 
1066 	/*
1067 	 * Reset all global state.
1068 	 */
1069 reset:
1070 	atomic_set(&global_nwo, 0);
1071 	atomic_set(&mce_callin, 0);
1072 	barrier();
1073 
1074 	/*
1075 	 * Let others run again.
1076 	 */
1077 	atomic_set(&mce_executing, 0);
1078 	return ret;
1079 }
1080 
1081 static void mce_clear_state(unsigned long *toclear)
1082 {
1083 	int i;
1084 
1085 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1086 		if (test_bit(i, toclear))
1087 			mce_wrmsrl(msr_ops.status(i), 0);
1088 	}
1089 }
1090 
1091 /*
1092  * Cases where we avoid rendezvous handler timeout:
1093  * 1) If this CPU is offline.
1094  *
1095  * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to
1096  *  skip those CPUs which remain looping in the 1st kernel - see
1097  *  crash_nmi_callback().
1098  *
1099  * Note: there still is a small window between kexec-ing and the new,
1100  * kdump kernel establishing a new #MC handler where a broadcasted MCE
1101  * might not get handled properly.
1102  */
1103 static bool __mc_check_crashing_cpu(int cpu)
1104 {
1105 	if (cpu_is_offline(cpu) ||
1106 	    (crashing_cpu != -1 && crashing_cpu != cpu)) {
1107 		u64 mcgstatus;
1108 
1109 		mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
1110 
1111 		if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) {
1112 			if (mcgstatus & MCG_STATUS_LMCES)
1113 				return false;
1114 		}
1115 
1116 		if (mcgstatus & MCG_STATUS_RIPV) {
1117 			mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1118 			return true;
1119 		}
1120 	}
1121 	return false;
1122 }
1123 
1124 static void __mc_scan_banks(struct mce *m, struct mce *final,
1125 			    unsigned long *toclear, unsigned long *valid_banks,
1126 			    int no_way_out, int *worst)
1127 {
1128 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1129 	struct mca_config *cfg = &mca_cfg;
1130 	int severity, i;
1131 
1132 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1133 		__clear_bit(i, toclear);
1134 		if (!test_bit(i, valid_banks))
1135 			continue;
1136 
1137 		if (!mce_banks[i].ctl)
1138 			continue;
1139 
1140 		m->misc = 0;
1141 		m->addr = 0;
1142 		m->bank = i;
1143 
1144 		m->status = mce_rdmsrl(msr_ops.status(i));
1145 		if (!(m->status & MCI_STATUS_VAL))
1146 			continue;
1147 
1148 		/*
1149 		 * Corrected or non-signaled errors are handled by
1150 		 * machine_check_poll(). Leave them alone, unless this panics.
1151 		 */
1152 		if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
1153 			!no_way_out)
1154 			continue;
1155 
1156 		/* Set taint even when machine check was not enabled. */
1157 		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
1158 
1159 		severity = mce_severity(m, cfg->tolerant, NULL, true);
1160 
1161 		/*
1162 		 * When machine check was for corrected/deferred handler don't
1163 		 * touch, unless we're panicking.
1164 		 */
1165 		if ((severity == MCE_KEEP_SEVERITY ||
1166 		     severity == MCE_UCNA_SEVERITY) && !no_way_out)
1167 			continue;
1168 
1169 		__set_bit(i, toclear);
1170 
1171 		/* Machine check event was not enabled. Clear, but ignore. */
1172 		if (severity == MCE_NO_SEVERITY)
1173 			continue;
1174 
1175 		mce_read_aux(m, i);
1176 
1177 		/* assuming valid severity level != 0 */
1178 		m->severity = severity;
1179 
1180 		mce_log(m);
1181 
1182 		if (severity > *worst) {
1183 			*final = *m;
1184 			*worst = severity;
1185 		}
1186 	}
1187 
1188 	/* mce_clear_state will clear *final, save locally for use later */
1189 	*m = *final;
1190 }
1191 
1192 static void kill_me_now(struct callback_head *ch)
1193 {
1194 	force_sig(SIGBUS);
1195 }
1196 
1197 static void kill_me_maybe(struct callback_head *cb)
1198 {
1199 	struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me);
1200 	int flags = MF_ACTION_REQUIRED;
1201 
1202 	pr_err("Uncorrected hardware memory error in user-access at %llx", p->mce_addr);
1203 	if (!(p->mce_status & MCG_STATUS_RIPV))
1204 		flags |= MF_MUST_KILL;
1205 
1206 	if (!memory_failure(p->mce_addr >> PAGE_SHIFT, flags)) {
1207 		set_mce_nospec(p->mce_addr >> PAGE_SHIFT);
1208 		return;
1209 	}
1210 
1211 	pr_err("Memory error not recovered");
1212 	kill_me_now(cb);
1213 }
1214 
1215 /*
1216  * The actual machine check handler. This only handles real
1217  * exceptions when something got corrupted coming in through int 18.
1218  *
1219  * This is executed in NMI context not subject to normal locking rules. This
1220  * implies that most kernel services cannot be safely used. Don't even
1221  * think about putting a printk in there!
1222  *
1223  * On Intel systems this is entered on all CPUs in parallel through
1224  * MCE broadcast. However some CPUs might be broken beyond repair,
1225  * so be always careful when synchronizing with others.
1226  *
1227  * Tracing and kprobes are disabled: if we interrupted a kernel context
1228  * with IF=1, we need to minimize stack usage.  There are also recursion
1229  * issues: if the machine check was due to a failure of the memory
1230  * backing the user stack, tracing that reads the user stack will cause
1231  * potentially infinite recursion.
1232  */
1233 void noinstr do_machine_check(struct pt_regs *regs, long error_code)
1234 {
1235 	DECLARE_BITMAP(valid_banks, MAX_NR_BANKS);
1236 	DECLARE_BITMAP(toclear, MAX_NR_BANKS);
1237 	struct mca_config *cfg = &mca_cfg;
1238 	int cpu = smp_processor_id();
1239 	struct mce m, *final;
1240 	char *msg = NULL;
1241 	int worst = 0;
1242 
1243 	/*
1244 	 * Establish sequential order between the CPUs entering the machine
1245 	 * check handler.
1246 	 */
1247 	int order = -1;
1248 
1249 	/*
1250 	 * If no_way_out gets set, there is no safe way to recover from this
1251 	 * MCE.  If mca_cfg.tolerant is cranked up, we'll try anyway.
1252 	 */
1253 	int no_way_out = 0;
1254 
1255 	/*
1256 	 * If kill_it gets set, there might be a way to recover from this
1257 	 * error.
1258 	 */
1259 	int kill_it = 0;
1260 
1261 	/*
1262 	 * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES
1263 	 * on Intel.
1264 	 */
1265 	int lmce = 1;
1266 
1267 	if (__mc_check_crashing_cpu(cpu))
1268 		return;
1269 
1270 	nmi_enter();
1271 
1272 	this_cpu_inc(mce_exception_count);
1273 
1274 	mce_gather_info(&m, regs);
1275 	m.tsc = rdtsc();
1276 
1277 	final = this_cpu_ptr(&mces_seen);
1278 	*final = m;
1279 
1280 	memset(valid_banks, 0, sizeof(valid_banks));
1281 	no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs);
1282 
1283 	barrier();
1284 
1285 	/*
1286 	 * When no restart IP might need to kill or panic.
1287 	 * Assume the worst for now, but if we find the
1288 	 * severity is MCE_AR_SEVERITY we have other options.
1289 	 */
1290 	if (!(m.mcgstatus & MCG_STATUS_RIPV))
1291 		kill_it = 1;
1292 
1293 	/*
1294 	 * Check if this MCE is signaled to only this logical processor,
1295 	 * on Intel, Zhaoxin only.
1296 	 */
1297 	if (m.cpuvendor == X86_VENDOR_INTEL ||
1298 	    m.cpuvendor == X86_VENDOR_ZHAOXIN)
1299 		lmce = m.mcgstatus & MCG_STATUS_LMCES;
1300 
1301 	/*
1302 	 * Local machine check may already know that we have to panic.
1303 	 * Broadcast machine check begins rendezvous in mce_start()
1304 	 * Go through all banks in exclusion of the other CPUs. This way we
1305 	 * don't report duplicated events on shared banks because the first one
1306 	 * to see it will clear it.
1307 	 */
1308 	if (lmce) {
1309 		if (no_way_out)
1310 			mce_panic("Fatal local machine check", &m, msg);
1311 	} else {
1312 		order = mce_start(&no_way_out);
1313 	}
1314 
1315 	__mc_scan_banks(&m, final, toclear, valid_banks, no_way_out, &worst);
1316 
1317 	if (!no_way_out)
1318 		mce_clear_state(toclear);
1319 
1320 	/*
1321 	 * Do most of the synchronization with other CPUs.
1322 	 * When there's any problem use only local no_way_out state.
1323 	 */
1324 	if (!lmce) {
1325 		if (mce_end(order) < 0)
1326 			no_way_out = worst >= MCE_PANIC_SEVERITY;
1327 	} else {
1328 		/*
1329 		 * If there was a fatal machine check we should have
1330 		 * already called mce_panic earlier in this function.
1331 		 * Since we re-read the banks, we might have found
1332 		 * something new. Check again to see if we found a
1333 		 * fatal error. We call "mce_severity()" again to
1334 		 * make sure we have the right "msg".
1335 		 */
1336 		if (worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) {
1337 			mce_severity(&m, cfg->tolerant, &msg, true);
1338 			mce_panic("Local fatal machine check!", &m, msg);
1339 		}
1340 	}
1341 
1342 	/*
1343 	 * If tolerant is at an insane level we drop requests to kill
1344 	 * processes and continue even when there is no way out.
1345 	 */
1346 	if (cfg->tolerant == 3)
1347 		kill_it = 0;
1348 	else if (no_way_out)
1349 		mce_panic("Fatal machine check on current CPU", &m, msg);
1350 
1351 	if (worst > 0)
1352 		irq_work_queue(&mce_irq_work);
1353 
1354 	mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1355 
1356 	sync_core();
1357 
1358 	if (worst != MCE_AR_SEVERITY && !kill_it)
1359 		goto out_ist;
1360 
1361 	/* Fault was in user mode and we need to take some action */
1362 	if ((m.cs & 3) == 3) {
1363 		/* If this triggers there is no way to recover. Die hard. */
1364 		BUG_ON(!on_thread_stack() || !user_mode(regs));
1365 
1366 		current->mce_addr = m.addr;
1367 		current->mce_status = m.mcgstatus;
1368 		current->mce_kill_me.func = kill_me_maybe;
1369 		if (kill_it)
1370 			current->mce_kill_me.func = kill_me_now;
1371 		task_work_add(current, &current->mce_kill_me, true);
1372 	} else {
1373 		if (!fixup_exception(regs, X86_TRAP_MC, error_code, 0))
1374 			mce_panic("Failed kernel mode recovery", &m, msg);
1375 	}
1376 
1377 out_ist:
1378 	nmi_exit();
1379 }
1380 EXPORT_SYMBOL_GPL(do_machine_check);
1381 
1382 #ifndef CONFIG_MEMORY_FAILURE
1383 int memory_failure(unsigned long pfn, int flags)
1384 {
1385 	/* mce_severity() should not hand us an ACTION_REQUIRED error */
1386 	BUG_ON(flags & MF_ACTION_REQUIRED);
1387 	pr_err("Uncorrected memory error in page 0x%lx ignored\n"
1388 	       "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n",
1389 	       pfn);
1390 
1391 	return 0;
1392 }
1393 #endif
1394 
1395 /*
1396  * Periodic polling timer for "silent" machine check errors.  If the
1397  * poller finds an MCE, poll 2x faster.  When the poller finds no more
1398  * errors, poll 2x slower (up to check_interval seconds).
1399  */
1400 static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
1401 
1402 static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */
1403 static DEFINE_PER_CPU(struct timer_list, mce_timer);
1404 
1405 static unsigned long mce_adjust_timer_default(unsigned long interval)
1406 {
1407 	return interval;
1408 }
1409 
1410 static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default;
1411 
1412 static void __start_timer(struct timer_list *t, unsigned long interval)
1413 {
1414 	unsigned long when = jiffies + interval;
1415 	unsigned long flags;
1416 
1417 	local_irq_save(flags);
1418 
1419 	if (!timer_pending(t) || time_before(when, t->expires))
1420 		mod_timer(t, round_jiffies(when));
1421 
1422 	local_irq_restore(flags);
1423 }
1424 
1425 static void mce_timer_fn(struct timer_list *t)
1426 {
1427 	struct timer_list *cpu_t = this_cpu_ptr(&mce_timer);
1428 	unsigned long iv;
1429 
1430 	WARN_ON(cpu_t != t);
1431 
1432 	iv = __this_cpu_read(mce_next_interval);
1433 
1434 	if (mce_available(this_cpu_ptr(&cpu_info))) {
1435 		machine_check_poll(0, this_cpu_ptr(&mce_poll_banks));
1436 
1437 		if (mce_intel_cmci_poll()) {
1438 			iv = mce_adjust_timer(iv);
1439 			goto done;
1440 		}
1441 	}
1442 
1443 	/*
1444 	 * Alert userspace if needed. If we logged an MCE, reduce the polling
1445 	 * interval, otherwise increase the polling interval.
1446 	 */
1447 	if (mce_notify_irq())
1448 		iv = max(iv / 2, (unsigned long) HZ/100);
1449 	else
1450 		iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
1451 
1452 done:
1453 	__this_cpu_write(mce_next_interval, iv);
1454 	__start_timer(t, iv);
1455 }
1456 
1457 /*
1458  * Ensure that the timer is firing in @interval from now.
1459  */
1460 void mce_timer_kick(unsigned long interval)
1461 {
1462 	struct timer_list *t = this_cpu_ptr(&mce_timer);
1463 	unsigned long iv = __this_cpu_read(mce_next_interval);
1464 
1465 	__start_timer(t, interval);
1466 
1467 	if (interval < iv)
1468 		__this_cpu_write(mce_next_interval, interval);
1469 }
1470 
1471 /* Must not be called in IRQ context where del_timer_sync() can deadlock */
1472 static void mce_timer_delete_all(void)
1473 {
1474 	int cpu;
1475 
1476 	for_each_online_cpu(cpu)
1477 		del_timer_sync(&per_cpu(mce_timer, cpu));
1478 }
1479 
1480 /*
1481  * Notify the user(s) about new machine check events.
1482  * Can be called from interrupt context, but not from machine check/NMI
1483  * context.
1484  */
1485 int mce_notify_irq(void)
1486 {
1487 	/* Not more than two messages every minute */
1488 	static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);
1489 
1490 	if (test_and_clear_bit(0, &mce_need_notify)) {
1491 		mce_work_trigger();
1492 
1493 		if (__ratelimit(&ratelimit))
1494 			pr_info(HW_ERR "Machine check events logged\n");
1495 
1496 		return 1;
1497 	}
1498 	return 0;
1499 }
1500 EXPORT_SYMBOL_GPL(mce_notify_irq);
1501 
1502 static void __mcheck_cpu_mce_banks_init(void)
1503 {
1504 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1505 	u8 n_banks = this_cpu_read(mce_num_banks);
1506 	int i;
1507 
1508 	for (i = 0; i < n_banks; i++) {
1509 		struct mce_bank *b = &mce_banks[i];
1510 
1511 		/*
1512 		 * Init them all, __mcheck_cpu_apply_quirks() is going to apply
1513 		 * the required vendor quirks before
1514 		 * __mcheck_cpu_init_clear_banks() does the final bank setup.
1515 		 */
1516 		b->ctl = -1ULL;
1517 		b->init = 1;
1518 	}
1519 }
1520 
1521 /*
1522  * Initialize Machine Checks for a CPU.
1523  */
1524 static void __mcheck_cpu_cap_init(void)
1525 {
1526 	u64 cap;
1527 	u8 b;
1528 
1529 	rdmsrl(MSR_IA32_MCG_CAP, cap);
1530 
1531 	b = cap & MCG_BANKCNT_MASK;
1532 
1533 	if (b > MAX_NR_BANKS) {
1534 		pr_warn("CPU%d: Using only %u machine check banks out of %u\n",
1535 			smp_processor_id(), MAX_NR_BANKS, b);
1536 		b = MAX_NR_BANKS;
1537 	}
1538 
1539 	this_cpu_write(mce_num_banks, b);
1540 
1541 	__mcheck_cpu_mce_banks_init();
1542 
1543 	/* Use accurate RIP reporting if available. */
1544 	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1545 		mca_cfg.rip_msr = MSR_IA32_MCG_EIP;
1546 
1547 	if (cap & MCG_SER_P)
1548 		mca_cfg.ser = 1;
1549 }
1550 
1551 static void __mcheck_cpu_init_generic(void)
1552 {
1553 	enum mcp_flags m_fl = 0;
1554 	mce_banks_t all_banks;
1555 	u64 cap;
1556 
1557 	if (!mca_cfg.bootlog)
1558 		m_fl = MCP_DONTLOG;
1559 
1560 	/*
1561 	 * Log the machine checks left over from the previous reset.
1562 	 */
1563 	bitmap_fill(all_banks, MAX_NR_BANKS);
1564 	machine_check_poll(MCP_UC | m_fl, &all_banks);
1565 
1566 	cr4_set_bits(X86_CR4_MCE);
1567 
1568 	rdmsrl(MSR_IA32_MCG_CAP, cap);
1569 	if (cap & MCG_CTL_P)
1570 		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
1571 }
1572 
1573 static void __mcheck_cpu_init_clear_banks(void)
1574 {
1575 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1576 	int i;
1577 
1578 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1579 		struct mce_bank *b = &mce_banks[i];
1580 
1581 		if (!b->init)
1582 			continue;
1583 		wrmsrl(msr_ops.ctl(i), b->ctl);
1584 		wrmsrl(msr_ops.status(i), 0);
1585 	}
1586 }
1587 
1588 /*
1589  * Do a final check to see if there are any unused/RAZ banks.
1590  *
1591  * This must be done after the banks have been initialized and any quirks have
1592  * been applied.
1593  *
1594  * Do not call this from any user-initiated flows, e.g. CPU hotplug or sysfs.
1595  * Otherwise, a user who disables a bank will not be able to re-enable it
1596  * without a system reboot.
1597  */
1598 static void __mcheck_cpu_check_banks(void)
1599 {
1600 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1601 	u64 msrval;
1602 	int i;
1603 
1604 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1605 		struct mce_bank *b = &mce_banks[i];
1606 
1607 		if (!b->init)
1608 			continue;
1609 
1610 		rdmsrl(msr_ops.ctl(i), msrval);
1611 		b->init = !!msrval;
1612 	}
1613 }
1614 
1615 /*
1616  * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and
1617  * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM
1618  * Vol 3B Table 15-20). But this confuses both the code that determines
1619  * whether the machine check occurred in kernel or user mode, and also
1620  * the severity assessment code. Pretend that EIPV was set, and take the
1621  * ip/cs values from the pt_regs that mce_gather_info() ignored earlier.
1622  */
1623 static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
1624 {
1625 	if (bank != 0)
1626 		return;
1627 	if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)
1628 		return;
1629 	if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|
1630 		          MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV|
1631 			  MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR|
1632 			  MCACOD)) !=
1633 			 (MCI_STATUS_UC|MCI_STATUS_EN|
1634 			  MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S|
1635 			  MCI_STATUS_AR|MCACOD_INSTR))
1636 		return;
1637 
1638 	m->mcgstatus |= MCG_STATUS_EIPV;
1639 	m->ip = regs->ip;
1640 	m->cs = regs->cs;
1641 }
1642 
1643 /* Add per CPU specific workarounds here */
1644 static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1645 {
1646 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1647 	struct mca_config *cfg = &mca_cfg;
1648 
1649 	if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
1650 		pr_info("unknown CPU type - not enabling MCE support\n");
1651 		return -EOPNOTSUPP;
1652 	}
1653 
1654 	/* This should be disabled by the BIOS, but isn't always */
1655 	if (c->x86_vendor == X86_VENDOR_AMD) {
1656 		if (c->x86 == 15 && this_cpu_read(mce_num_banks) > 4) {
1657 			/*
1658 			 * disable GART TBL walk error reporting, which
1659 			 * trips off incorrectly with the IOMMU & 3ware
1660 			 * & Cerberus:
1661 			 */
1662 			clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
1663 		}
1664 		if (c->x86 < 0x11 && cfg->bootlog < 0) {
1665 			/*
1666 			 * Lots of broken BIOS around that don't clear them
1667 			 * by default and leave crap in there. Don't log:
1668 			 */
1669 			cfg->bootlog = 0;
1670 		}
1671 		/*
1672 		 * Various K7s with broken bank 0 around. Always disable
1673 		 * by default.
1674 		 */
1675 		if (c->x86 == 6 && this_cpu_read(mce_num_banks) > 0)
1676 			mce_banks[0].ctl = 0;
1677 
1678 		/*
1679 		 * overflow_recov is supported for F15h Models 00h-0fh
1680 		 * even though we don't have a CPUID bit for it.
1681 		 */
1682 		if (c->x86 == 0x15 && c->x86_model <= 0xf)
1683 			mce_flags.overflow_recov = 1;
1684 
1685 	}
1686 
1687 	if (c->x86_vendor == X86_VENDOR_INTEL) {
1688 		/*
1689 		 * SDM documents that on family 6 bank 0 should not be written
1690 		 * because it aliases to another special BIOS controlled
1691 		 * register.
1692 		 * But it's not aliased anymore on model 0x1a+
1693 		 * Don't ignore bank 0 completely because there could be a
1694 		 * valid event later, merely don't write CTL0.
1695 		 */
1696 
1697 		if (c->x86 == 6 && c->x86_model < 0x1A && this_cpu_read(mce_num_banks) > 0)
1698 			mce_banks[0].init = 0;
1699 
1700 		/*
1701 		 * All newer Intel systems support MCE broadcasting. Enable
1702 		 * synchronization with a one second timeout.
1703 		 */
1704 		if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
1705 			cfg->monarch_timeout < 0)
1706 			cfg->monarch_timeout = USEC_PER_SEC;
1707 
1708 		/*
1709 		 * There are also broken BIOSes on some Pentium M and
1710 		 * earlier systems:
1711 		 */
1712 		if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0)
1713 			cfg->bootlog = 0;
1714 
1715 		if (c->x86 == 6 && c->x86_model == 45)
1716 			quirk_no_way_out = quirk_sandybridge_ifu;
1717 	}
1718 
1719 	if (c->x86_vendor == X86_VENDOR_ZHAOXIN) {
1720 		/*
1721 		 * All newer Zhaoxin CPUs support MCE broadcasting. Enable
1722 		 * synchronization with a one second timeout.
1723 		 */
1724 		if (c->x86 > 6 || (c->x86_model == 0x19 || c->x86_model == 0x1f)) {
1725 			if (cfg->monarch_timeout < 0)
1726 				cfg->monarch_timeout = USEC_PER_SEC;
1727 		}
1728 	}
1729 
1730 	if (cfg->monarch_timeout < 0)
1731 		cfg->monarch_timeout = 0;
1732 	if (cfg->bootlog != 0)
1733 		cfg->panic_timeout = 30;
1734 
1735 	return 0;
1736 }
1737 
1738 static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1739 {
1740 	if (c->x86 != 5)
1741 		return 0;
1742 
1743 	switch (c->x86_vendor) {
1744 	case X86_VENDOR_INTEL:
1745 		intel_p5_mcheck_init(c);
1746 		return 1;
1747 		break;
1748 	case X86_VENDOR_CENTAUR:
1749 		winchip_mcheck_init(c);
1750 		return 1;
1751 		break;
1752 	default:
1753 		return 0;
1754 	}
1755 
1756 	return 0;
1757 }
1758 
1759 /*
1760  * Init basic CPU features needed for early decoding of MCEs.
1761  */
1762 static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c)
1763 {
1764 	if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) {
1765 		mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV);
1766 		mce_flags.succor	 = !!cpu_has(c, X86_FEATURE_SUCCOR);
1767 		mce_flags.smca		 = !!cpu_has(c, X86_FEATURE_SMCA);
1768 
1769 		if (mce_flags.smca) {
1770 			msr_ops.ctl	= smca_ctl_reg;
1771 			msr_ops.status	= smca_status_reg;
1772 			msr_ops.addr	= smca_addr_reg;
1773 			msr_ops.misc	= smca_misc_reg;
1774 		}
1775 	}
1776 }
1777 
1778 static void mce_centaur_feature_init(struct cpuinfo_x86 *c)
1779 {
1780 	struct mca_config *cfg = &mca_cfg;
1781 
1782 	 /*
1783 	  * All newer Centaur CPUs support MCE broadcasting. Enable
1784 	  * synchronization with a one second timeout.
1785 	  */
1786 	if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) ||
1787 	     c->x86 > 6) {
1788 		if (cfg->monarch_timeout < 0)
1789 			cfg->monarch_timeout = USEC_PER_SEC;
1790 	}
1791 }
1792 
1793 static void mce_zhaoxin_feature_init(struct cpuinfo_x86 *c)
1794 {
1795 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1796 
1797 	/*
1798 	 * These CPUs have MCA bank 8 which reports only one error type called
1799 	 * SVAD (System View Address Decoder). The reporting of that error is
1800 	 * controlled by IA32_MC8.CTL.0.
1801 	 *
1802 	 * If enabled, prefetching on these CPUs will cause SVAD MCE when
1803 	 * virtual machines start and result in a system  panic. Always disable
1804 	 * bank 8 SVAD error by default.
1805 	 */
1806 	if ((c->x86 == 7 && c->x86_model == 0x1b) ||
1807 	    (c->x86_model == 0x19 || c->x86_model == 0x1f)) {
1808 		if (this_cpu_read(mce_num_banks) > 8)
1809 			mce_banks[8].ctl = 0;
1810 	}
1811 
1812 	intel_init_cmci();
1813 	intel_init_lmce();
1814 	mce_adjust_timer = cmci_intel_adjust_timer;
1815 }
1816 
1817 static void mce_zhaoxin_feature_clear(struct cpuinfo_x86 *c)
1818 {
1819 	intel_clear_lmce();
1820 }
1821 
1822 static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
1823 {
1824 	switch (c->x86_vendor) {
1825 	case X86_VENDOR_INTEL:
1826 		mce_intel_feature_init(c);
1827 		mce_adjust_timer = cmci_intel_adjust_timer;
1828 		break;
1829 
1830 	case X86_VENDOR_AMD: {
1831 		mce_amd_feature_init(c);
1832 		break;
1833 		}
1834 
1835 	case X86_VENDOR_HYGON:
1836 		mce_hygon_feature_init(c);
1837 		break;
1838 
1839 	case X86_VENDOR_CENTAUR:
1840 		mce_centaur_feature_init(c);
1841 		break;
1842 
1843 	case X86_VENDOR_ZHAOXIN:
1844 		mce_zhaoxin_feature_init(c);
1845 		break;
1846 
1847 	default:
1848 		break;
1849 	}
1850 }
1851 
1852 static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c)
1853 {
1854 	switch (c->x86_vendor) {
1855 	case X86_VENDOR_INTEL:
1856 		mce_intel_feature_clear(c);
1857 		break;
1858 
1859 	case X86_VENDOR_ZHAOXIN:
1860 		mce_zhaoxin_feature_clear(c);
1861 		break;
1862 
1863 	default:
1864 		break;
1865 	}
1866 }
1867 
1868 static void mce_start_timer(struct timer_list *t)
1869 {
1870 	unsigned long iv = check_interval * HZ;
1871 
1872 	if (mca_cfg.ignore_ce || !iv)
1873 		return;
1874 
1875 	this_cpu_write(mce_next_interval, iv);
1876 	__start_timer(t, iv);
1877 }
1878 
1879 static void __mcheck_cpu_setup_timer(void)
1880 {
1881 	struct timer_list *t = this_cpu_ptr(&mce_timer);
1882 
1883 	timer_setup(t, mce_timer_fn, TIMER_PINNED);
1884 }
1885 
1886 static void __mcheck_cpu_init_timer(void)
1887 {
1888 	struct timer_list *t = this_cpu_ptr(&mce_timer);
1889 
1890 	timer_setup(t, mce_timer_fn, TIMER_PINNED);
1891 	mce_start_timer(t);
1892 }
1893 
1894 bool filter_mce(struct mce *m)
1895 {
1896 	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1897 		return amd_filter_mce(m);
1898 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
1899 		return intel_filter_mce(m);
1900 
1901 	return false;
1902 }
1903 
1904 /* Handle unconfigured int18 (should never happen) */
1905 static void unexpected_machine_check(struct pt_regs *regs, long error_code)
1906 {
1907 	pr_err("CPU#%d: Unexpected int18 (Machine Check)\n",
1908 	       smp_processor_id());
1909 }
1910 
1911 /* Call the installed machine check handler for this CPU setup. */
1912 void (*machine_check_vector)(struct pt_regs *, long error_code) =
1913 						unexpected_machine_check;
1914 
1915 dotraplinkage notrace void do_mce(struct pt_regs *regs, long error_code)
1916 {
1917 	machine_check_vector(regs, error_code);
1918 }
1919 NOKPROBE_SYMBOL(do_mce);
1920 
1921 /*
1922  * Called for each booted CPU to set up machine checks.
1923  * Must be called with preempt off:
1924  */
1925 void mcheck_cpu_init(struct cpuinfo_x86 *c)
1926 {
1927 	if (mca_cfg.disabled)
1928 		return;
1929 
1930 	if (__mcheck_cpu_ancient_init(c))
1931 		return;
1932 
1933 	if (!mce_available(c))
1934 		return;
1935 
1936 	__mcheck_cpu_cap_init();
1937 
1938 	if (__mcheck_cpu_apply_quirks(c) < 0) {
1939 		mca_cfg.disabled = 1;
1940 		return;
1941 	}
1942 
1943 	if (mce_gen_pool_init()) {
1944 		mca_cfg.disabled = 1;
1945 		pr_emerg("Couldn't allocate MCE records pool!\n");
1946 		return;
1947 	}
1948 
1949 	machine_check_vector = do_machine_check;
1950 
1951 	__mcheck_cpu_init_early(c);
1952 	__mcheck_cpu_init_generic();
1953 	__mcheck_cpu_init_vendor(c);
1954 	__mcheck_cpu_init_clear_banks();
1955 	__mcheck_cpu_check_banks();
1956 	__mcheck_cpu_setup_timer();
1957 }
1958 
1959 /*
1960  * Called for each booted CPU to clear some machine checks opt-ins
1961  */
1962 void mcheck_cpu_clear(struct cpuinfo_x86 *c)
1963 {
1964 	if (mca_cfg.disabled)
1965 		return;
1966 
1967 	if (!mce_available(c))
1968 		return;
1969 
1970 	/*
1971 	 * Possibly to clear general settings generic to x86
1972 	 * __mcheck_cpu_clear_generic(c);
1973 	 */
1974 	__mcheck_cpu_clear_vendor(c);
1975 
1976 }
1977 
1978 static void __mce_disable_bank(void *arg)
1979 {
1980 	int bank = *((int *)arg);
1981 	__clear_bit(bank, this_cpu_ptr(mce_poll_banks));
1982 	cmci_disable_bank(bank);
1983 }
1984 
1985 void mce_disable_bank(int bank)
1986 {
1987 	if (bank >= this_cpu_read(mce_num_banks)) {
1988 		pr_warn(FW_BUG
1989 			"Ignoring request to disable invalid MCA bank %d.\n",
1990 			bank);
1991 		return;
1992 	}
1993 	set_bit(bank, mce_banks_ce_disabled);
1994 	on_each_cpu(__mce_disable_bank, &bank, 1);
1995 }
1996 
1997 /*
1998  * mce=off Disables machine check
1999  * mce=no_cmci Disables CMCI
2000  * mce=no_lmce Disables LMCE
2001  * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
2002  * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
2003  * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
2004  *	monarchtimeout is how long to wait for other CPUs on machine
2005  *	check, or 0 to not wait
2006  * mce=bootlog Log MCEs from before booting. Disabled by default on AMD Fam10h
2007 	and older.
2008  * mce=nobootlog Don't log MCEs from before booting.
2009  * mce=bios_cmci_threshold Don't program the CMCI threshold
2010  * mce=recovery force enable memcpy_mcsafe()
2011  */
2012 static int __init mcheck_enable(char *str)
2013 {
2014 	struct mca_config *cfg = &mca_cfg;
2015 
2016 	if (*str == 0) {
2017 		enable_p5_mce();
2018 		return 1;
2019 	}
2020 	if (*str == '=')
2021 		str++;
2022 	if (!strcmp(str, "off"))
2023 		cfg->disabled = 1;
2024 	else if (!strcmp(str, "no_cmci"))
2025 		cfg->cmci_disabled = true;
2026 	else if (!strcmp(str, "no_lmce"))
2027 		cfg->lmce_disabled = 1;
2028 	else if (!strcmp(str, "dont_log_ce"))
2029 		cfg->dont_log_ce = true;
2030 	else if (!strcmp(str, "ignore_ce"))
2031 		cfg->ignore_ce = true;
2032 	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
2033 		cfg->bootlog = (str[0] == 'b');
2034 	else if (!strcmp(str, "bios_cmci_threshold"))
2035 		cfg->bios_cmci_threshold = 1;
2036 	else if (!strcmp(str, "recovery"))
2037 		cfg->recovery = 1;
2038 	else if (isdigit(str[0])) {
2039 		if (get_option(&str, &cfg->tolerant) == 2)
2040 			get_option(&str, &(cfg->monarch_timeout));
2041 	} else {
2042 		pr_info("mce argument %s ignored. Please use /sys\n", str);
2043 		return 0;
2044 	}
2045 	return 1;
2046 }
2047 __setup("mce", mcheck_enable);
2048 
2049 int __init mcheck_init(void)
2050 {
2051 	mcheck_intel_therm_init();
2052 	mce_register_decode_chain(&first_nb);
2053 	mce_register_decode_chain(&mce_uc_nb);
2054 	mce_register_decode_chain(&mce_default_nb);
2055 	mcheck_vendor_init_severity();
2056 
2057 	INIT_WORK(&mce_work, mce_gen_pool_process);
2058 	init_irq_work(&mce_irq_work, mce_irq_work_cb);
2059 
2060 	return 0;
2061 }
2062 
2063 /*
2064  * mce_syscore: PM support
2065  */
2066 
2067 /*
2068  * Disable machine checks on suspend and shutdown. We can't really handle
2069  * them later.
2070  */
2071 static void mce_disable_error_reporting(void)
2072 {
2073 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2074 	int i;
2075 
2076 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
2077 		struct mce_bank *b = &mce_banks[i];
2078 
2079 		if (b->init)
2080 			wrmsrl(msr_ops.ctl(i), 0);
2081 	}
2082 	return;
2083 }
2084 
2085 static void vendor_disable_error_reporting(void)
2086 {
2087 	/*
2088 	 * Don't clear on Intel or AMD or Hygon or Zhaoxin CPUs. Some of these
2089 	 * MSRs are socket-wide. Disabling them for just a single offlined CPU
2090 	 * is bad, since it will inhibit reporting for all shared resources on
2091 	 * the socket like the last level cache (LLC), the integrated memory
2092 	 * controller (iMC), etc.
2093 	 */
2094 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL ||
2095 	    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ||
2096 	    boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
2097 	    boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN)
2098 		return;
2099 
2100 	mce_disable_error_reporting();
2101 }
2102 
2103 static int mce_syscore_suspend(void)
2104 {
2105 	vendor_disable_error_reporting();
2106 	return 0;
2107 }
2108 
2109 static void mce_syscore_shutdown(void)
2110 {
2111 	vendor_disable_error_reporting();
2112 }
2113 
2114 /*
2115  * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
2116  * Only one CPU is active at this time, the others get re-added later using
2117  * CPU hotplug:
2118  */
2119 static void mce_syscore_resume(void)
2120 {
2121 	__mcheck_cpu_init_generic();
2122 	__mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info));
2123 	__mcheck_cpu_init_clear_banks();
2124 }
2125 
2126 static struct syscore_ops mce_syscore_ops = {
2127 	.suspend	= mce_syscore_suspend,
2128 	.shutdown	= mce_syscore_shutdown,
2129 	.resume		= mce_syscore_resume,
2130 };
2131 
2132 /*
2133  * mce_device: Sysfs support
2134  */
2135 
2136 static void mce_cpu_restart(void *data)
2137 {
2138 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2139 		return;
2140 	__mcheck_cpu_init_generic();
2141 	__mcheck_cpu_init_clear_banks();
2142 	__mcheck_cpu_init_timer();
2143 }
2144 
2145 /* Reinit MCEs after user configuration changes */
2146 static void mce_restart(void)
2147 {
2148 	mce_timer_delete_all();
2149 	on_each_cpu(mce_cpu_restart, NULL, 1);
2150 }
2151 
2152 /* Toggle features for corrected errors */
2153 static void mce_disable_cmci(void *data)
2154 {
2155 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2156 		return;
2157 	cmci_clear();
2158 }
2159 
2160 static void mce_enable_ce(void *all)
2161 {
2162 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2163 		return;
2164 	cmci_reenable();
2165 	cmci_recheck();
2166 	if (all)
2167 		__mcheck_cpu_init_timer();
2168 }
2169 
2170 static struct bus_type mce_subsys = {
2171 	.name		= "machinecheck",
2172 	.dev_name	= "machinecheck",
2173 };
2174 
2175 DEFINE_PER_CPU(struct device *, mce_device);
2176 
2177 static inline struct mce_bank_dev *attr_to_bank(struct device_attribute *attr)
2178 {
2179 	return container_of(attr, struct mce_bank_dev, attr);
2180 }
2181 
2182 static ssize_t show_bank(struct device *s, struct device_attribute *attr,
2183 			 char *buf)
2184 {
2185 	u8 bank = attr_to_bank(attr)->bank;
2186 	struct mce_bank *b;
2187 
2188 	if (bank >= per_cpu(mce_num_banks, s->id))
2189 		return -EINVAL;
2190 
2191 	b = &per_cpu(mce_banks_array, s->id)[bank];
2192 
2193 	if (!b->init)
2194 		return -ENODEV;
2195 
2196 	return sprintf(buf, "%llx\n", b->ctl);
2197 }
2198 
2199 static ssize_t set_bank(struct device *s, struct device_attribute *attr,
2200 			const char *buf, size_t size)
2201 {
2202 	u8 bank = attr_to_bank(attr)->bank;
2203 	struct mce_bank *b;
2204 	u64 new;
2205 
2206 	if (kstrtou64(buf, 0, &new) < 0)
2207 		return -EINVAL;
2208 
2209 	if (bank >= per_cpu(mce_num_banks, s->id))
2210 		return -EINVAL;
2211 
2212 	b = &per_cpu(mce_banks_array, s->id)[bank];
2213 
2214 	if (!b->init)
2215 		return -ENODEV;
2216 
2217 	b->ctl = new;
2218 	mce_restart();
2219 
2220 	return size;
2221 }
2222 
2223 static ssize_t set_ignore_ce(struct device *s,
2224 			     struct device_attribute *attr,
2225 			     const char *buf, size_t size)
2226 {
2227 	u64 new;
2228 
2229 	if (kstrtou64(buf, 0, &new) < 0)
2230 		return -EINVAL;
2231 
2232 	mutex_lock(&mce_sysfs_mutex);
2233 	if (mca_cfg.ignore_ce ^ !!new) {
2234 		if (new) {
2235 			/* disable ce features */
2236 			mce_timer_delete_all();
2237 			on_each_cpu(mce_disable_cmci, NULL, 1);
2238 			mca_cfg.ignore_ce = true;
2239 		} else {
2240 			/* enable ce features */
2241 			mca_cfg.ignore_ce = false;
2242 			on_each_cpu(mce_enable_ce, (void *)1, 1);
2243 		}
2244 	}
2245 	mutex_unlock(&mce_sysfs_mutex);
2246 
2247 	return size;
2248 }
2249 
2250 static ssize_t set_cmci_disabled(struct device *s,
2251 				 struct device_attribute *attr,
2252 				 const char *buf, size_t size)
2253 {
2254 	u64 new;
2255 
2256 	if (kstrtou64(buf, 0, &new) < 0)
2257 		return -EINVAL;
2258 
2259 	mutex_lock(&mce_sysfs_mutex);
2260 	if (mca_cfg.cmci_disabled ^ !!new) {
2261 		if (new) {
2262 			/* disable cmci */
2263 			on_each_cpu(mce_disable_cmci, NULL, 1);
2264 			mca_cfg.cmci_disabled = true;
2265 		} else {
2266 			/* enable cmci */
2267 			mca_cfg.cmci_disabled = false;
2268 			on_each_cpu(mce_enable_ce, NULL, 1);
2269 		}
2270 	}
2271 	mutex_unlock(&mce_sysfs_mutex);
2272 
2273 	return size;
2274 }
2275 
2276 static ssize_t store_int_with_restart(struct device *s,
2277 				      struct device_attribute *attr,
2278 				      const char *buf, size_t size)
2279 {
2280 	unsigned long old_check_interval = check_interval;
2281 	ssize_t ret = device_store_ulong(s, attr, buf, size);
2282 
2283 	if (check_interval == old_check_interval)
2284 		return ret;
2285 
2286 	mutex_lock(&mce_sysfs_mutex);
2287 	mce_restart();
2288 	mutex_unlock(&mce_sysfs_mutex);
2289 
2290 	return ret;
2291 }
2292 
2293 static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant);
2294 static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
2295 static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
2296 
2297 static struct dev_ext_attribute dev_attr_check_interval = {
2298 	__ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
2299 	&check_interval
2300 };
2301 
2302 static struct dev_ext_attribute dev_attr_ignore_ce = {
2303 	__ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
2304 	&mca_cfg.ignore_ce
2305 };
2306 
2307 static struct dev_ext_attribute dev_attr_cmci_disabled = {
2308 	__ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
2309 	&mca_cfg.cmci_disabled
2310 };
2311 
2312 static struct device_attribute *mce_device_attrs[] = {
2313 	&dev_attr_tolerant.attr,
2314 	&dev_attr_check_interval.attr,
2315 #ifdef CONFIG_X86_MCELOG_LEGACY
2316 	&dev_attr_trigger,
2317 #endif
2318 	&dev_attr_monarch_timeout.attr,
2319 	&dev_attr_dont_log_ce.attr,
2320 	&dev_attr_ignore_ce.attr,
2321 	&dev_attr_cmci_disabled.attr,
2322 	NULL
2323 };
2324 
2325 static cpumask_var_t mce_device_initialized;
2326 
2327 static void mce_device_release(struct device *dev)
2328 {
2329 	kfree(dev);
2330 }
2331 
2332 /* Per CPU device init. All of the CPUs still share the same bank device: */
2333 static int mce_device_create(unsigned int cpu)
2334 {
2335 	struct device *dev;
2336 	int err;
2337 	int i, j;
2338 
2339 	if (!mce_available(&boot_cpu_data))
2340 		return -EIO;
2341 
2342 	dev = per_cpu(mce_device, cpu);
2343 	if (dev)
2344 		return 0;
2345 
2346 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2347 	if (!dev)
2348 		return -ENOMEM;
2349 	dev->id  = cpu;
2350 	dev->bus = &mce_subsys;
2351 	dev->release = &mce_device_release;
2352 
2353 	err = device_register(dev);
2354 	if (err) {
2355 		put_device(dev);
2356 		return err;
2357 	}
2358 
2359 	for (i = 0; mce_device_attrs[i]; i++) {
2360 		err = device_create_file(dev, mce_device_attrs[i]);
2361 		if (err)
2362 			goto error;
2363 	}
2364 	for (j = 0; j < per_cpu(mce_num_banks, cpu); j++) {
2365 		err = device_create_file(dev, &mce_bank_devs[j].attr);
2366 		if (err)
2367 			goto error2;
2368 	}
2369 	cpumask_set_cpu(cpu, mce_device_initialized);
2370 	per_cpu(mce_device, cpu) = dev;
2371 
2372 	return 0;
2373 error2:
2374 	while (--j >= 0)
2375 		device_remove_file(dev, &mce_bank_devs[j].attr);
2376 error:
2377 	while (--i >= 0)
2378 		device_remove_file(dev, mce_device_attrs[i]);
2379 
2380 	device_unregister(dev);
2381 
2382 	return err;
2383 }
2384 
2385 static void mce_device_remove(unsigned int cpu)
2386 {
2387 	struct device *dev = per_cpu(mce_device, cpu);
2388 	int i;
2389 
2390 	if (!cpumask_test_cpu(cpu, mce_device_initialized))
2391 		return;
2392 
2393 	for (i = 0; mce_device_attrs[i]; i++)
2394 		device_remove_file(dev, mce_device_attrs[i]);
2395 
2396 	for (i = 0; i < per_cpu(mce_num_banks, cpu); i++)
2397 		device_remove_file(dev, &mce_bank_devs[i].attr);
2398 
2399 	device_unregister(dev);
2400 	cpumask_clear_cpu(cpu, mce_device_initialized);
2401 	per_cpu(mce_device, cpu) = NULL;
2402 }
2403 
2404 /* Make sure there are no machine checks on offlined CPUs. */
2405 static void mce_disable_cpu(void)
2406 {
2407 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2408 		return;
2409 
2410 	if (!cpuhp_tasks_frozen)
2411 		cmci_clear();
2412 
2413 	vendor_disable_error_reporting();
2414 }
2415 
2416 static void mce_reenable_cpu(void)
2417 {
2418 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2419 	int i;
2420 
2421 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2422 		return;
2423 
2424 	if (!cpuhp_tasks_frozen)
2425 		cmci_reenable();
2426 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
2427 		struct mce_bank *b = &mce_banks[i];
2428 
2429 		if (b->init)
2430 			wrmsrl(msr_ops.ctl(i), b->ctl);
2431 	}
2432 }
2433 
2434 static int mce_cpu_dead(unsigned int cpu)
2435 {
2436 	mce_intel_hcpu_update(cpu);
2437 
2438 	/* intentionally ignoring frozen here */
2439 	if (!cpuhp_tasks_frozen)
2440 		cmci_rediscover();
2441 	return 0;
2442 }
2443 
2444 static int mce_cpu_online(unsigned int cpu)
2445 {
2446 	struct timer_list *t = this_cpu_ptr(&mce_timer);
2447 	int ret;
2448 
2449 	mce_device_create(cpu);
2450 
2451 	ret = mce_threshold_create_device(cpu);
2452 	if (ret) {
2453 		mce_device_remove(cpu);
2454 		return ret;
2455 	}
2456 	mce_reenable_cpu();
2457 	mce_start_timer(t);
2458 	return 0;
2459 }
2460 
2461 static int mce_cpu_pre_down(unsigned int cpu)
2462 {
2463 	struct timer_list *t = this_cpu_ptr(&mce_timer);
2464 
2465 	mce_disable_cpu();
2466 	del_timer_sync(t);
2467 	mce_threshold_remove_device(cpu);
2468 	mce_device_remove(cpu);
2469 	return 0;
2470 }
2471 
2472 static __init void mce_init_banks(void)
2473 {
2474 	int i;
2475 
2476 	for (i = 0; i < MAX_NR_BANKS; i++) {
2477 		struct mce_bank_dev *b = &mce_bank_devs[i];
2478 		struct device_attribute *a = &b->attr;
2479 
2480 		b->bank = i;
2481 
2482 		sysfs_attr_init(&a->attr);
2483 		a->attr.name	= b->attrname;
2484 		snprintf(b->attrname, ATTR_LEN, "bank%d", i);
2485 
2486 		a->attr.mode	= 0644;
2487 		a->show		= show_bank;
2488 		a->store	= set_bank;
2489 	}
2490 }
2491 
2492 static __init int mcheck_init_device(void)
2493 {
2494 	int err;
2495 
2496 	/*
2497 	 * Check if we have a spare virtual bit. This will only become
2498 	 * a problem if/when we move beyond 5-level page tables.
2499 	 */
2500 	MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63);
2501 
2502 	if (!mce_available(&boot_cpu_data)) {
2503 		err = -EIO;
2504 		goto err_out;
2505 	}
2506 
2507 	if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
2508 		err = -ENOMEM;
2509 		goto err_out;
2510 	}
2511 
2512 	mce_init_banks();
2513 
2514 	err = subsys_system_register(&mce_subsys, NULL);
2515 	if (err)
2516 		goto err_out_mem;
2517 
2518 	err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL,
2519 				mce_cpu_dead);
2520 	if (err)
2521 		goto err_out_mem;
2522 
2523 	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
2524 				mce_cpu_online, mce_cpu_pre_down);
2525 	if (err < 0)
2526 		goto err_out_online;
2527 
2528 	register_syscore_ops(&mce_syscore_ops);
2529 
2530 	return 0;
2531 
2532 err_out_online:
2533 	cpuhp_remove_state(CPUHP_X86_MCE_DEAD);
2534 
2535 err_out_mem:
2536 	free_cpumask_var(mce_device_initialized);
2537 
2538 err_out:
2539 	pr_err("Unable to init MCE device (rc: %d)\n", err);
2540 
2541 	return err;
2542 }
2543 device_initcall_sync(mcheck_init_device);
2544 
2545 /*
2546  * Old style boot options parsing. Only for compatibility.
2547  */
2548 static int __init mcheck_disable(char *str)
2549 {
2550 	mca_cfg.disabled = 1;
2551 	return 1;
2552 }
2553 __setup("nomce", mcheck_disable);
2554 
2555 #ifdef CONFIG_DEBUG_FS
2556 struct dentry *mce_get_debugfs_dir(void)
2557 {
2558 	static struct dentry *dmce;
2559 
2560 	if (!dmce)
2561 		dmce = debugfs_create_dir("mce", NULL);
2562 
2563 	return dmce;
2564 }
2565 
2566 static void mce_reset(void)
2567 {
2568 	cpu_missing = 0;
2569 	atomic_set(&mce_fake_panicked, 0);
2570 	atomic_set(&mce_executing, 0);
2571 	atomic_set(&mce_callin, 0);
2572 	atomic_set(&global_nwo, 0);
2573 }
2574 
2575 static int fake_panic_get(void *data, u64 *val)
2576 {
2577 	*val = fake_panic;
2578 	return 0;
2579 }
2580 
2581 static int fake_panic_set(void *data, u64 val)
2582 {
2583 	mce_reset();
2584 	fake_panic = val;
2585 	return 0;
2586 }
2587 
2588 DEFINE_DEBUGFS_ATTRIBUTE(fake_panic_fops, fake_panic_get, fake_panic_set,
2589 			 "%llu\n");
2590 
2591 static void __init mcheck_debugfs_init(void)
2592 {
2593 	struct dentry *dmce;
2594 
2595 	dmce = mce_get_debugfs_dir();
2596 	debugfs_create_file_unsafe("fake_panic", 0444, dmce, NULL,
2597 				   &fake_panic_fops);
2598 }
2599 #else
2600 static void __init mcheck_debugfs_init(void) { }
2601 #endif
2602 
2603 DEFINE_STATIC_KEY_FALSE(mcsafe_key);
2604 EXPORT_SYMBOL_GPL(mcsafe_key);
2605 
2606 static int __init mcheck_late_init(void)
2607 {
2608 	if (mca_cfg.recovery)
2609 		static_branch_inc(&mcsafe_key);
2610 
2611 	mcheck_debugfs_init();
2612 	cec_init();
2613 
2614 	/*
2615 	 * Flush out everything that has been logged during early boot, now that
2616 	 * everything has been initialized (workqueues, decoders, ...).
2617 	 */
2618 	mce_schedule_work();
2619 
2620 	return 0;
2621 }
2622 late_initcall(mcheck_late_init);
2623