1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * S390 version 4 * Copyright IBM Corp. 1999, 2000 5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 6 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), 7 * 8 * Derived from "arch/i386/kernel/traps.c" 9 * Copyright (C) 1991, 1992 Linus Torvalds 10 */ 11 12 #include <linux/capability.h> 13 #include <linux/cpufeature.h> 14 #include <linux/debugfs.h> 15 #include <linux/kprobes.h> 16 #include <linux/kdebug.h> 17 #include <linux/randomize_kstack.h> 18 #include <linux/extable.h> 19 #include <linux/ptrace.h> 20 #include <linux/sched.h> 21 #include <linux/sched/debug.h> 22 #include <linux/mm.h> 23 #include <linux/slab.h> 24 #include <linux/uaccess.h> 25 #include <linux/cpu.h> 26 #include <linux/entry-common.h> 27 #include <linux/kmsan.h> 28 #include <linux/bug.h> 29 #include <linux/panic.h> 30 #include <asm/pgmcheck-defs.h> 31 #include <asm/entry-percpu.h> 32 #include <asm/asm-extable.h> 33 #include <asm/irqflags.h> 34 #include <asm/ptrace.h> 35 #include <asm/vtime.h> 36 #include <asm/fpu.h> 37 #include <asm/fault.h> 38 #include <asm/processor.h> 39 #include "entry.h" 40 41 struct pgm_stat { 42 unsigned int count[128]; 43 }; 44 45 static DEFINE_PER_CPU_SHARED_ALIGNED(struct pgm_stat, pgm_stat); 46 47 unsigned long get_trap_ip(struct pt_regs *regs) 48 { 49 static const unsigned long nullified[2] = { PGM_CHECKS_NULLIFIED }; 50 psw_t psw = regs->psw; 51 52 if (test_pt_regs_flag(regs, PIF_PSW_ADDR_ADJUSTED)) 53 return psw.addr; 54 if (regs->int_code & 0x200) 55 psw.addr = current->thread.trap_tdb.data[3]; 56 if (test_bit(regs->int_code & PGM_INT_CODE_MASK, nullified)) 57 return psw.addr; 58 psw.addr = __rewind_psw(psw, regs->int_code >> 16); 59 return psw.addr; 60 } 61 NOKPROBE_SYMBOL(get_trap_ip); 62 63 #ifdef CONFIG_GENERIC_BUG 64 int is_valid_bugaddr(unsigned long addr) 65 { 66 return 1; 67 } 68 #endif 69 70 void do_report_trap(struct pt_regs *regs, int si_signo, int si_code, char *str) 71 { 72 if (user_mode(regs)) { 73 force_sig_fault(si_signo, si_code, (void __user *)get_trap_ip(regs)); 74 report_user_fault(regs, si_signo, 0); 75 } else { 76 if (!fixup_exception(regs)) 77 die(regs, str); 78 } 79 } 80 81 static void do_trap(struct pt_regs *regs, int si_signo, int si_code, char *str) 82 { 83 if (notify_die(DIE_TRAP, str, regs, 0, regs->int_code, si_signo) == NOTIFY_STOP) 84 return; 85 do_report_trap(regs, si_signo, si_code, str); 86 } 87 NOKPROBE_SYMBOL(do_trap); 88 89 void do_per_trap(struct pt_regs *regs) 90 { 91 if (notify_die(DIE_SSTEP, "sstep", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) 92 return; 93 if (!current->ptrace) 94 return; 95 force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __force __user *)current->thread.per_event.address); 96 } 97 NOKPROBE_SYMBOL(do_per_trap); 98 99 static void default_trap_handler(struct pt_regs *regs) 100 { 101 if (user_mode(regs)) { 102 report_user_fault(regs, SIGSEGV, 0); 103 force_exit_sig(SIGSEGV); 104 } else 105 die(regs, "Unknown program exception"); 106 } 107 108 #define DO_ERROR_INFO(name, signr, sicode, str) \ 109 static void name(struct pt_regs *regs) \ 110 { \ 111 do_trap(regs, signr, sicode, str); \ 112 } 113 114 DO_ERROR_INFO(addressing_exception, SIGILL, ILL_ILLADR, "addressing exception") 115 DO_ERROR_INFO(divide_exception, SIGFPE, FPE_INTDIV, "fixpoint divide exception") 116 DO_ERROR_INFO(execute_exception, SIGILL, ILL_ILLOPN, "execute exception") 117 DO_ERROR_INFO(hfp_divide_exception, SIGFPE, FPE_FLTDIV, "HFP divide exception") 118 DO_ERROR_INFO(hfp_overflow_exception, SIGFPE, FPE_FLTOVF, "HFP overflow exception") 119 DO_ERROR_INFO(hfp_significance_exception, SIGFPE, FPE_FLTRES, "HFP significance exception") 120 DO_ERROR_INFO(hfp_sqrt_exception, SIGFPE, FPE_FLTINV, "HFP square root exception") 121 DO_ERROR_INFO(hfp_underflow_exception, SIGFPE, FPE_FLTUND, "HFP underflow exception") 122 DO_ERROR_INFO(operand_exception, SIGILL, ILL_ILLOPN, "operand exception") 123 DO_ERROR_INFO(overflow_exception, SIGFPE, FPE_INTOVF, "fixpoint overflow exception") 124 DO_ERROR_INFO(privileged_op, SIGILL, ILL_PRVOPC, "privileged operation") 125 DO_ERROR_INFO(special_op_exception, SIGILL, ILL_ILLOPN, "special operation exception") 126 DO_ERROR_INFO(specification_exception, SIGILL, ILL_ILLOPN, "specification exception"); 127 DO_ERROR_INFO(transaction_exception, SIGILL, ILL_ILLOPN, "transaction constraint exception") 128 129 static inline void do_fp_trap(struct pt_regs *regs, __u32 fpc) 130 { 131 int si_code = 0; 132 133 /* FPC[2] is Data Exception Code */ 134 if ((fpc & 0x00000300) == 0) { 135 /* bits 6 and 7 of DXC are 0 iff IEEE exception */ 136 if (fpc & 0x8000) /* invalid fp operation */ 137 si_code = FPE_FLTINV; 138 else if (fpc & 0x4000) /* div by 0 */ 139 si_code = FPE_FLTDIV; 140 else if (fpc & 0x2000) /* overflow */ 141 si_code = FPE_FLTOVF; 142 else if (fpc & 0x1000) /* underflow */ 143 si_code = FPE_FLTUND; 144 else if (fpc & 0x0800) /* inexact */ 145 si_code = FPE_FLTRES; 146 } 147 do_trap(regs, SIGFPE, si_code, "floating point exception"); 148 } 149 150 static void translation_specification_exception(struct pt_regs *regs) 151 { 152 /* May never happen. */ 153 panic("Translation-Specification Exception"); 154 } 155 156 static void illegal_op(struct pt_regs *regs) 157 { 158 int is_uprobe_insn = 0; 159 int signal = 0; 160 u16 opcode; 161 162 if (user_mode(regs)) { 163 u16 __user *location = (void __user *)get_trap_ip(regs); 164 165 if (get_user(opcode, location)) 166 return; 167 if (opcode == S390_BREAKPOINT_U16) { 168 if (current->ptrace) 169 force_sig_fault(SIGTRAP, TRAP_BRKPT, location); 170 else 171 signal = SIGILL; 172 #ifdef CONFIG_UPROBES 173 } else if (opcode == UPROBE_SWBP_INSN) { 174 is_uprobe_insn = 1; 175 #endif 176 } else { 177 signal = SIGILL; 178 } 179 } 180 /* 181 * This is either an illegal op in kernel mode, or user space trapped 182 * on a uprobes illegal instruction. See if kprobes or uprobes picks 183 * it up. If not, SIGILL. 184 */ 185 if (is_uprobe_insn || !user_mode(regs)) { 186 if (notify_die(DIE_BPT, "bpt", regs, 0, 3, SIGTRAP) != NOTIFY_STOP) 187 signal = SIGILL; 188 } 189 if (signal) 190 do_trap(regs, signal, ILL_ILLOPC, "illegal operation"); 191 } 192 NOKPROBE_SYMBOL(illegal_op); 193 194 static void vector_exception(struct pt_regs *regs) 195 { 196 int si_code, vic; 197 198 /* get vector interrupt code from fpc */ 199 save_user_fpu_regs(); 200 vic = (current->thread.ufpu.fpc & 0xf00) >> 8; 201 switch (vic) { 202 case 1: /* invalid vector operation */ 203 si_code = FPE_FLTINV; 204 break; 205 case 2: /* division by zero */ 206 si_code = FPE_FLTDIV; 207 break; 208 case 3: /* overflow */ 209 si_code = FPE_FLTOVF; 210 break; 211 case 4: /* underflow */ 212 si_code = FPE_FLTUND; 213 break; 214 case 5: /* inexact */ 215 si_code = FPE_FLTRES; 216 break; 217 default: /* unknown cause */ 218 si_code = 0; 219 } 220 do_trap(regs, SIGFPE, si_code, "vector exception"); 221 } 222 223 static void data_exception(struct pt_regs *regs) 224 { 225 save_user_fpu_regs(); 226 if (current->thread.ufpu.fpc & FPC_DXC_MASK) 227 do_fp_trap(regs, current->thread.ufpu.fpc); 228 else 229 do_trap(regs, SIGILL, ILL_ILLOPN, "data exception"); 230 } 231 232 static void space_switch_exception(struct pt_regs *regs) 233 { 234 /* Set user psw back to home space mode. */ 235 if (user_mode(regs)) 236 regs->psw.mask |= PSW_ASC_HOME; 237 /* Send SIGILL. */ 238 do_trap(regs, SIGILL, ILL_PRVOPC, "space switch event"); 239 } 240 241 #if defined(CONFIG_BUG) && defined(CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS) 242 243 void *__warn_args(struct arch_va_list *args, struct pt_regs *regs) 244 { 245 struct stack_frame *stack_frame; 246 247 /* 248 * Generate va_list from pt_regs. See ELF Application Binary Interface 249 * s390x Supplement documentation for details. 250 * 251 * - __overflow_arg_area needs to point to the parameter area, which 252 * is right above the standard stack frame (160 bytes) 253 * 254 * - __reg_save_area needs to point to a register save area where 255 * general registers (%r2 - %r6) can be found at offset 16. Which 256 * means that the gprs save area of pt_regs can be used 257 * 258 * - __gpr must be set to one, since the first parameter has been 259 * processed (pointer to bug_entry) 260 */ 261 stack_frame = (struct stack_frame *)regs->gprs[15]; 262 args->__overflow_arg_area = stack_frame + 1; 263 args->__reg_save_area = regs->gprs; 264 args->__gpr = 1; 265 return args; 266 } 267 268 #endif /* CONFIG_BUG && CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS */ 269 270 static void monitor_event_exception(struct pt_regs *regs) 271 { 272 enum bug_trap_type btt; 273 274 if (user_mode(regs)) 275 return; 276 if (regs->monitor_code == MONCODE_BUG_ARG) { 277 regs->psw.addr = regs->gprs[14]; 278 set_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED); 279 btt = report_bug_entry((struct bug_entry *)regs->gprs[2], regs); 280 } else { 281 btt = report_bug(regs->psw.addr - (regs->int_code >> 16), regs); 282 } 283 switch (btt) { 284 case BUG_TRAP_TYPE_NONE: 285 fixup_exception(regs); 286 break; 287 case BUG_TRAP_TYPE_WARN: 288 break; 289 case BUG_TRAP_TYPE_BUG: 290 die(regs, "monitor event"); 291 break; 292 } 293 } 294 295 void arch_do_panic(void) 296 { 297 disabled_wait(); 298 } 299 300 /* 301 * Content of the faulting CPU's lowcore, copied by stack_invalid(). 302 */ 303 struct lowcore stack_invalid_lowcore; 304 int stack_invalid_lowcore_used; 305 306 void kernel_stack_invalid(struct pt_regs *regs) 307 { 308 /* 309 * Normally regs are unpoisoned by the generic entry code, but 310 * kernel_stack_overflow() is a rare case that is called bypassing it. 311 */ 312 kmsan_unpoison_entry_regs(regs); 313 bust_spinlocks(1); 314 pr_emerg("Kernel stack pointer invalid\n"); 315 show_regs(regs); 316 bust_spinlocks(0); 317 panic("Invalid kernel stack pointer, cannot continue"); 318 } 319 NOKPROBE_SYMBOL(kernel_stack_invalid); 320 321 static void __init test_monitor_call(void) 322 { 323 int val = 1; 324 325 if (!IS_ENABLED(CONFIG_BUG)) 326 return; 327 asm_inline volatile( 328 "0: mc %[monc](%%r0),0\n" 329 " lhi %[val],0\n" 330 "1:\n" 331 EX_TABLE_INSN(0b, 1b) 332 : [val] "+d" (val) 333 : [monc] "i" (MONCODE_BUG)); 334 if (!val) 335 panic("Monitor call doesn't work!\n"); 336 } 337 338 void __init trap_init(void) 339 { 340 struct lowcore *lc = get_lowcore(); 341 unsigned long flags; 342 struct ctlreg cr0; 343 344 local_irq_save(flags); 345 cr0 = local_ctl_clear_bit(0, CR0_LOW_ADDRESS_PROTECTION_BIT); 346 psw_bits(lc->external_new_psw).mcheck = 1; 347 psw_bits(lc->program_new_psw).mcheck = 1; 348 psw_bits(lc->svc_new_psw).mcheck = 1; 349 psw_bits(lc->io_new_psw).mcheck = 1; 350 local_ctl_load(0, &cr0); 351 local_irq_restore(flags); 352 local_mcck_enable(); 353 test_monitor_call(); 354 } 355 356 static void (*pgm_check_table[128])(struct pt_regs *regs); 357 358 void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags) 359 { 360 struct lowcore *lc = get_lowcore(); 361 irqentry_state_t state; 362 struct pgm_stat *stat; 363 unsigned int trapnr; 364 union teid teid; 365 366 teid.val = lc->trans_exc_code; 367 regs->int_code = lc->pgm_int_code; 368 regs->int_parm_long = teid.val; 369 regs->monitor_code = lc->monitor_code; 370 set_pt_regs_flag(regs, PIF_PGM_CHECK); 371 372 trapnr = regs->int_code & PGM_INT_CODE_MASK; 373 stat = this_cpu_ptr(&pgm_stat); 374 stat->count[trapnr]++; 375 /* 376 * In case of a guest fault, short-circuit the fault handler and return. 377 * This way the sie64a() function will return 0; fault address and 378 * other relevant bits are saved in current->thread.gmap_teid, and 379 * the fault number in current->thread.gmap_int_code. KVM will be 380 * able to use this information to handle the fault. 381 */ 382 if (flags & PGM_FLAG_GUEST_FAULT) { 383 current->thread.gmap_teid.val = regs->int_parm_long; 384 current->thread.gmap_int_code = regs->int_code & 0xffff; 385 return; 386 } 387 percpu_entry(regs); 388 state = irqentry_enter(regs); 389 if (user_mode(regs)) { 390 update_timer_sys(); 391 if (!cpu_has_bear()) { 392 if (regs->last_break < 4096) 393 regs->last_break = 1; 394 } 395 current->thread.last_break = regs->last_break; 396 } 397 if (lc->pgm_code & 0x0200) { 398 /* transaction abort */ 399 current->thread.trap_tdb = lc->pgm_tdb; 400 } 401 if (lc->pgm_code & PGM_INT_CODE_PER) { 402 if (user_mode(regs)) { 403 struct per_event *ev = ¤t->thread.per_event; 404 405 set_thread_flag(TIF_PER_TRAP); 406 ev->address = lc->per_address; 407 ev->cause = lc->per_code_combined; 408 ev->paid = lc->per_access_id; 409 } else { 410 /* PER event in kernel is kprobes */ 411 __arch_local_irq_ssm(regs->psw.mask & ~PSW_MASK_PER); 412 do_per_trap(regs); 413 goto out; 414 } 415 } 416 if (!irqs_disabled_flags(regs->psw.mask)) 417 trace_hardirqs_on(); 418 __arch_local_irq_ssm(regs->psw.mask & ~PSW_MASK_PER); 419 if (trapnr) 420 pgm_check_table[trapnr](regs); 421 out: 422 local_irq_disable(); 423 irqentry_exit(regs, state); 424 percpu_exit(regs); 425 } 426 427 static int pgm_check_stat_show(struct seq_file *p, void *v) 428 { 429 int i, cpu; 430 431 cpus_read_lock(); 432 seq_puts(p, " "); 433 for_each_online_cpu(cpu) 434 seq_printf(p, "CPU%-8d", cpu); 435 seq_putc(p, '\n'); 436 for (i = 0; i < 128; i++) { 437 seq_printf(p, "%02x: ", i); 438 for_each_online_cpu(cpu) 439 seq_printf(p, "%10u ", per_cpu(pgm_stat, cpu).count[i]); 440 seq_putc(p, '\n'); 441 } 442 cpus_read_unlock(); 443 return 0; 444 } 445 DEFINE_SHOW_ATTRIBUTE(pgm_check_stat); 446 447 static int __init debugfs_pgm_check_init(void) 448 { 449 debugfs_create_file("exceptions", 0400, arch_debugfs_dir, NULL, &pgm_check_stat_fops); 450 return 0; 451 } 452 late_initcall(debugfs_pgm_check_init); 453 454 /* 455 * The program check table contains exactly 128 (0x00-0x7f) entries. Each 456 * line defines the function to be called corresponding to the program check 457 * interruption code. 458 */ 459 static void (*pgm_check_table[128])(struct pt_regs *regs) = { 460 [0x00] = default_trap_handler, 461 [0x01] = illegal_op, 462 [0x02] = privileged_op, 463 [0x03] = execute_exception, 464 [0x04] = do_protection_exception, 465 [0x05] = addressing_exception, 466 [0x06] = specification_exception, 467 [0x07] = data_exception, 468 [0x08] = overflow_exception, 469 [0x09] = divide_exception, 470 [0x0a] = overflow_exception, 471 [0x0b] = divide_exception, 472 [0x0c] = hfp_overflow_exception, 473 [0x0d] = hfp_underflow_exception, 474 [0x0e] = hfp_significance_exception, 475 [0x0f] = hfp_divide_exception, 476 [0x10] = do_dat_exception, 477 [0x11] = do_dat_exception, 478 [0x12] = translation_specification_exception, 479 [0x13] = special_op_exception, 480 [0x14] = default_trap_handler, 481 [0x15] = operand_exception, 482 [0x16] = default_trap_handler, 483 [0x17] = default_trap_handler, 484 [0x18] = transaction_exception, 485 [0x19] = default_trap_handler, 486 [0x1a] = default_trap_handler, 487 [0x1b] = vector_exception, 488 [0x1c] = space_switch_exception, 489 [0x1d] = hfp_sqrt_exception, 490 [0x1e ... 0x37] = default_trap_handler, 491 [0x38] = do_dat_exception, 492 [0x39] = do_dat_exception, 493 [0x3a] = do_dat_exception, 494 [0x3b] = do_dat_exception, 495 [0x3c] = default_trap_handler, 496 [0x3d] = do_secure_storage_access, 497 [0x3e] = default_trap_handler, 498 [0x3f] = default_trap_handler, 499 [0x40] = monitor_event_exception, 500 [0x41 ... 0x7f] = default_trap_handler, 501 }; 502 503 #define COND_TRAP(x) asm( \ 504 ".weak " __stringify(x) "\n\t" \ 505 ".set " __stringify(x) "," \ 506 __stringify(default_trap_handler)) 507 508 COND_TRAP(do_secure_storage_access); 509