1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Machine check handler 4 * 5 * Copyright IBM Corp. 2000, 2009 6 * Author(s): Ingo Adlung <adlung@de.ibm.com>, 7 * Martin Schwidefsky <schwidefsky@de.ibm.com>, 8 * Cornelia Huck <cornelia.huck@de.ibm.com>, 9 */ 10 11 #include <linux/kernel_stat.h> 12 #include <linux/cpufeature.h> 13 #include <linux/init.h> 14 #include <linux/errno.h> 15 #include <linux/entry-common.h> 16 #include <linux/hardirq.h> 17 #include <linux/log2.h> 18 #include <linux/kprobes.h> 19 #include <linux/kmemleak.h> 20 #include <linux/time.h> 21 #include <linux/module.h> 22 #include <linux/sched/signal.h> 23 #include <linux/kvm_host.h> 24 #include <linux/export.h> 25 #include <asm/lowcore.h> 26 #include <asm/ctlreg.h> 27 #include <asm/fpu.h> 28 #include <asm/smp.h> 29 #include <asm/stp.h> 30 #include <asm/cputime.h> 31 #include <asm/nmi.h> 32 #include <asm/crw.h> 33 #include <asm/asm-offsets.h> 34 #include <asm/pai.h> 35 #include <asm/vtime.h> 36 37 struct mcck_struct { 38 unsigned int kill_task : 1; 39 unsigned int channel_report : 1; 40 unsigned int warning : 1; 41 unsigned int stp_queue : 1; 42 unsigned long mcck_code; 43 }; 44 45 static DEFINE_PER_CPU(struct mcck_struct, cpu_mcck); 46 47 static inline int nmi_needs_mcesa(void) 48 { 49 return cpu_has_vx() || cpu_has_gs(); 50 } 51 52 /* 53 * The initial machine check extended save area for the boot CPU. 54 * It will be replaced on the boot CPU reinit with an allocated 55 * structure. The structure is required for machine check happening 56 * early in the boot process. 57 */ 58 static struct mcesa boot_mcesa __aligned(MCESA_MAX_SIZE); 59 60 void __init nmi_alloc_mcesa_early(u64 *mcesad) 61 { 62 if (!nmi_needs_mcesa()) 63 return; 64 *mcesad = __pa(&boot_mcesa); 65 if (cpu_has_gs()) 66 *mcesad |= ilog2(MCESA_MAX_SIZE); 67 } 68 69 int nmi_alloc_mcesa(u64 *mcesad) 70 { 71 unsigned long size; 72 void *origin; 73 74 *mcesad = 0; 75 if (!nmi_needs_mcesa()) 76 return 0; 77 size = cpu_has_gs() ? MCESA_MAX_SIZE : MCESA_MIN_SIZE; 78 origin = kmalloc(size, GFP_KERNEL); 79 if (!origin) 80 return -ENOMEM; 81 /* The pointer is stored with mcesa_bits ORed in */ 82 kmemleak_not_leak(origin); 83 *mcesad = __pa(origin); 84 if (cpu_has_gs()) 85 *mcesad |= ilog2(MCESA_MAX_SIZE); 86 return 0; 87 } 88 89 void nmi_free_mcesa(u64 *mcesad) 90 { 91 if (!nmi_needs_mcesa()) 92 return; 93 kfree(__va(*mcesad & MCESA_ORIGIN_MASK)); 94 } 95 96 static __always_inline char *nmi_puts(char *dest, const char *src) 97 { 98 while (*src) 99 *dest++ = *src++; 100 *dest = 0; 101 return dest; 102 } 103 104 static __always_inline char *u64_to_hex(char *dest, u64 val) 105 { 106 int i, num; 107 108 for (i = 1; i <= 16; i++) { 109 num = (val >> (64 - 4 * i)) & 0xf; 110 if (num >= 10) 111 *dest++ = 'A' + num - 10; 112 else 113 *dest++ = '0' + num; 114 } 115 *dest = 0; 116 return dest; 117 } 118 119 static notrace void s390_handle_damage(void) 120 { 121 struct lowcore *lc = get_lowcore(); 122 union ctlreg0 cr0, cr0_new; 123 char message[100]; 124 psw_t psw_save; 125 char *ptr; 126 127 smp_emergency_stop(); 128 diag_amode31_ops.diag308_reset(); 129 ptr = nmi_puts(message, "System stopped due to unrecoverable machine check, code: 0x"); 130 u64_to_hex(ptr, lc->mcck_interruption_code); 131 132 /* 133 * Disable low address protection and make machine check new PSW a 134 * disabled wait PSW. Any additional machine check cannot be handled. 135 */ 136 local_ctl_store(0, &cr0.reg); 137 cr0_new = cr0; 138 cr0_new.lap = 0; 139 local_ctl_load(0, &cr0_new.reg); 140 psw_save = lc->mcck_new_psw; 141 psw_bits(lc->mcck_new_psw).io = 0; 142 psw_bits(lc->mcck_new_psw).ext = 0; 143 psw_bits(lc->mcck_new_psw).wait = 1; 144 sclp_emergency_printk(message); 145 146 /* 147 * Restore machine check new PSW and control register 0 to original 148 * values. This makes possible system dump analysis easier. 149 */ 150 lc->mcck_new_psw = psw_save; 151 local_ctl_load(0, &cr0.reg); 152 disabled_wait(); 153 while (1); 154 } 155 NOKPROBE_SYMBOL(s390_handle_damage); 156 157 /* 158 * Main machine check handler function. Will be called with interrupts disabled 159 * and machine checks enabled. 160 */ 161 void s390_handle_mcck(void) 162 { 163 struct mcck_struct mcck; 164 unsigned long mflags; 165 166 /* 167 * Disable machine checks and get the current state of accumulated 168 * machine checks. Afterwards delete the old state and enable machine 169 * checks again. 170 */ 171 local_mcck_save(mflags); 172 mcck = *this_cpu_ptr(&cpu_mcck); 173 memset(this_cpu_ptr(&cpu_mcck), 0, sizeof(mcck)); 174 local_mcck_restore(mflags); 175 176 if (mcck.channel_report) 177 crw_handle_channel_report(); 178 /* 179 * A warning may remain for a prolonged period on the bare iron. 180 * (actually until the machine is powered off, or the problem is gone) 181 * So we just stop listening for the WARNING MCH and avoid continuously 182 * being interrupted. One caveat is however, that we must do this per 183 * processor and cannot use the smp version of ctl_clear_bit(). 184 * On VM we only get one interrupt per virtally presented machinecheck. 185 * Though one suffices, we may get one interrupt per (virtual) cpu. 186 */ 187 if (mcck.warning) { /* WARNING pending ? */ 188 static int mchchk_wng_posted = 0; 189 190 /* Use single cpu clear, as we cannot handle smp here. */ 191 local_ctl_clear_bit(14, CR14_WARNING_SUBMASK_BIT); 192 if (xchg(&mchchk_wng_posted, 1) == 0) 193 kill_cad_pid(SIGPWR, 1); 194 } 195 if (mcck.stp_queue) 196 stp_queue_work(); 197 if (mcck.kill_task) { 198 printk(KERN_EMERG "mcck: Terminating task because of machine " 199 "malfunction (code 0x%016lx).\n", mcck.mcck_code); 200 printk(KERN_EMERG "mcck: task: %s, pid: %d.\n", 201 current->comm, current->pid); 202 if (is_global_init(current)) 203 panic("mcck: Attempting to kill init!\n"); 204 do_send_sig_info(SIGKILL, SEND_SIG_PRIV, current, PIDTYPE_PID); 205 } 206 } 207 208 /** 209 * nmi_registers_valid - verify if registers are valid 210 * @mci: machine check interruption code 211 * 212 * Inspect a machine check interruption code and verify if all required 213 * registers are valid. For some registers the corresponding validity bit is 214 * ignored and the registers are set to the expected value. 215 * Returns true if all registers are valid, otherwise false. 216 */ 217 static bool notrace nmi_registers_valid(union mci mci) 218 { 219 union ctlreg2 cr2; 220 221 /* 222 * The getcpu vdso syscall reads the CPU number from the programmable 223 * field of the TOD clock. Disregard the TOD programmable register 224 * validity bit and load the CPU number into the TOD programmable field 225 * unconditionally. 226 */ 227 set_tod_programmable_field(raw_smp_processor_id()); 228 /* 229 * Set the clock comparator register to the next expected value. 230 */ 231 set_clock_comparator(get_lowcore()->clock_comparator); 232 if (!mci.gr || !mci.fp || !mci.fc) 233 return false; 234 /* 235 * The vector validity must only be checked if not running a 236 * KVM guest. For KVM guests the machine check is forwarded by 237 * KVM and it is the responsibility of the guest to take 238 * appropriate actions. The host vector or FPU values have been 239 * saved by KVM and will be restored by KVM. 240 */ 241 if (!mci.vr && !test_cpu_flag(CIF_MCCK_GUEST)) 242 return false; 243 if (!mci.ar) 244 return false; 245 /* 246 * Two cases for guarded storage registers: 247 * - machine check in kernel or userspace 248 * - machine check while running SIE (KVM guest) 249 * For kernel or userspace the userspace values of guarded storage 250 * control can not be recreated, the process must be terminated. 251 * For SIE the guest values of guarded storage can not be recreated. 252 * This is either due to a bug or due to GS being disabled in the 253 * guest. The guest will be notified by KVM code and the guests machine 254 * check handling must take care of this. The host values are saved by 255 * KVM and are not affected. 256 */ 257 cr2.reg = get_lowcore()->cregs_save_area[2]; 258 if (cr2.gse && !mci.gs && !test_cpu_flag(CIF_MCCK_GUEST)) 259 return false; 260 if (!mci.ms || !mci.pm || !mci.ia) 261 return false; 262 return true; 263 } 264 NOKPROBE_SYMBOL(nmi_registers_valid); 265 266 /* 267 * Backup the guest's machine check info to its description block 268 */ 269 static void notrace s390_backup_mcck_info(struct pt_regs *regs) 270 { 271 struct mcck_volatile_info *mcck_backup; 272 struct sie_page *sie_page; 273 274 /* r14 contains the sie block, which was set in sie64a */ 275 struct kvm_s390_sie_block *sie_block = phys_to_virt(regs->gprs[14]); 276 277 if (sie_block == NULL) 278 /* Something's seriously wrong, stop system. */ 279 s390_handle_damage(); 280 281 sie_page = container_of(sie_block, struct sie_page, sie_block); 282 mcck_backup = &sie_page->mcck_info; 283 mcck_backup->mcic = get_lowcore()->mcck_interruption_code & 284 ~(MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE); 285 mcck_backup->ext_damage_code = get_lowcore()->external_damage_code; 286 mcck_backup->failing_storage_address = get_lowcore()->failing_storage_address; 287 } 288 NOKPROBE_SYMBOL(s390_backup_mcck_info); 289 290 #define MAX_IPD_COUNT 29 291 #define MAX_IPD_TIME (5 * 60 * USEC_PER_SEC) /* 5 minutes */ 292 293 #define ED_STP_ISLAND 6 /* External damage STP island check */ 294 #define ED_STP_SYNC 7 /* External damage STP sync check */ 295 296 #define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE) 297 298 /* 299 * machine check handler. 300 */ 301 void notrace s390_do_machine_check(struct pt_regs *regs) 302 { 303 static int ipd_count; 304 static DEFINE_SPINLOCK(ipd_lock); 305 static unsigned long long last_ipd; 306 struct lowcore *lc = get_lowcore(); 307 struct mcck_struct *mcck; 308 unsigned long long tmp; 309 irqentry_state_t irq_state; 310 union mci mci; 311 unsigned long mcck_dam_code; 312 int mcck_pending = 0; 313 314 irq_state = irqentry_nmi_enter(regs); 315 316 if (user_mode(regs)) 317 update_timer_mcck(); 318 inc_irq_stat(NMI_NMI); 319 mci.val = lc->mcck_interruption_code; 320 mcck = this_cpu_ptr(&cpu_mcck); 321 322 /* 323 * Reinject the instruction processing damages' machine checks 324 * including Delayed Access Exception into the guest 325 * instead of damaging the host if they happen in the guest. 326 */ 327 if (mci.pd && !test_cpu_flag(CIF_MCCK_GUEST)) { 328 if (mci.b) { 329 /* Processing backup -> verify if we can survive this */ 330 u64 z_mcic, o_mcic, t_mcic; 331 z_mcic = (1ULL<<63 | 1ULL<<59 | 1ULL<<29); 332 o_mcic = (1ULL<<43 | 1ULL<<42 | 1ULL<<41 | 1ULL<<40 | 333 1ULL<<36 | 1ULL<<35 | 1ULL<<34 | 1ULL<<32 | 334 1ULL<<30 | 1ULL<<21 | 1ULL<<20 | 1ULL<<17 | 335 1ULL<<16); 336 t_mcic = mci.val; 337 338 if (((t_mcic & z_mcic) != 0) || 339 ((t_mcic & o_mcic) != o_mcic)) { 340 s390_handle_damage(); 341 } 342 343 /* 344 * Nullifying exigent condition, therefore we might 345 * retry this instruction. 346 */ 347 spin_lock(&ipd_lock); 348 tmp = get_tod_clock(); 349 if (((tmp - last_ipd) >> 12) < MAX_IPD_TIME) 350 ipd_count++; 351 else 352 ipd_count = 1; 353 last_ipd = tmp; 354 if (ipd_count == MAX_IPD_COUNT) 355 s390_handle_damage(); 356 spin_unlock(&ipd_lock); 357 } else { 358 /* Processing damage -> stopping machine */ 359 s390_handle_damage(); 360 } 361 } 362 if (!nmi_registers_valid(mci)) { 363 if (!user_mode(regs)) 364 s390_handle_damage(); 365 /* 366 * Couldn't restore all register contents for the 367 * user space process -> mark task for termination. 368 */ 369 mcck->kill_task = 1; 370 mcck->mcck_code = mci.val; 371 mcck_pending = 1; 372 } 373 374 /* 375 * Backup the machine check's info if it happens when the guest 376 * is running. 377 */ 378 if (test_cpu_flag(CIF_MCCK_GUEST)) 379 s390_backup_mcck_info(regs); 380 381 if (mci.cd) { 382 /* Timing facility damage */ 383 s390_handle_damage(); 384 } 385 if (mci.ed && mci.ec) { 386 /* External damage */ 387 if (lc->external_damage_code & (1U << ED_STP_SYNC)) 388 mcck->stp_queue |= stp_sync_check(); 389 if (lc->external_damage_code & (1U << ED_STP_ISLAND)) 390 mcck->stp_queue |= stp_island_check(); 391 mcck_pending = 1; 392 } 393 /* 394 * Reinject storage related machine checks into the guest if they 395 * happen when the guest is running. 396 */ 397 if (!test_cpu_flag(CIF_MCCK_GUEST)) { 398 /* Storage error uncorrected */ 399 if (mci.se) 400 s390_handle_damage(); 401 /* Storage key-error uncorrected */ 402 if (mci.ke) 403 s390_handle_damage(); 404 /* Storage degradation */ 405 if (mci.ds && mci.fa) 406 s390_handle_damage(); 407 } 408 if (mci.cp) { 409 /* Channel report word pending */ 410 mcck->channel_report = 1; 411 mcck_pending = 1; 412 } 413 if (mci.w) { 414 /* Warning pending */ 415 mcck->warning = 1; 416 mcck_pending = 1; 417 } 418 419 /* 420 * If there are only Channel Report Pending and External Damage 421 * machine checks, they will not be reinjected into the guest 422 * because they refer to host conditions only. 423 */ 424 mcck_dam_code = (mci.val & MCIC_SUBCLASS_MASK); 425 if (test_cpu_flag(CIF_MCCK_GUEST) && 426 (mcck_dam_code & MCCK_CODE_NO_GUEST) != mcck_dam_code) { 427 /* Set exit reason code for host's later handling */ 428 *((long *)(regs->gprs[15] + __SF_SIE_REASON)) = -EINTR; 429 } 430 clear_cpu_flag(CIF_MCCK_GUEST); 431 432 if (mcck_pending) 433 schedule_mcck_handler(); 434 435 irqentry_nmi_exit(regs, irq_state); 436 } 437 NOKPROBE_SYMBOL(s390_do_machine_check); 438 439 static int __init machine_check_init(void) 440 { 441 system_ctl_set_bit(14, CR14_EXTERNAL_DAMAGE_SUBMASK_BIT); 442 system_ctl_set_bit(14, CR14_RECOVERY_SUBMASK_BIT); 443 system_ctl_set_bit(14, CR14_WARNING_SUBMASK_BIT); 444 return 0; 445 } 446 early_initcall(machine_check_init); 447