1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Based on arch/arm/kernel/traps.c 4 * 5 * Copyright (C) 1995-2009 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 9 #include <linux/bug.h> 10 #include <linux/context_tracking.h> 11 #include <linux/signal.h> 12 #include <linux/kallsyms.h> 13 #include <linux/kprobes.h> 14 #include <linux/spinlock.h> 15 #include <linux/uaccess.h> 16 #include <linux/hardirq.h> 17 #include <linux/kdebug.h> 18 #include <linux/module.h> 19 #include <linux/kexec.h> 20 #include <linux/delay.h> 21 #include <linux/init.h> 22 #include <linux/sched/signal.h> 23 #include <linux/sched/debug.h> 24 #include <linux/sched/task_stack.h> 25 #include <linux/sizes.h> 26 #include <linux/syscalls.h> 27 #include <linux/mm_types.h> 28 #include <linux/kasan.h> 29 #include <linux/cfi.h> 30 31 #include <asm/atomic.h> 32 #include <asm/bug.h> 33 #include <asm/cpufeature.h> 34 #include <asm/daifflags.h> 35 #include <asm/debug-monitors.h> 36 #include <asm/esr.h> 37 #include <asm/exception.h> 38 #include <asm/extable.h> 39 #include <asm/insn.h> 40 #include <asm/kprobes.h> 41 #include <asm/patching.h> 42 #include <asm/traps.h> 43 #include <asm/smp.h> 44 #include <asm/stack_pointer.h> 45 #include <asm/stacktrace.h> 46 #include <asm/system_misc.h> 47 #include <asm/sysreg.h> 48 49 static bool __kprobes __check_eq(unsigned long pstate) 50 { 51 return (pstate & PSR_Z_BIT) != 0; 52 } 53 54 static bool __kprobes __check_ne(unsigned long pstate) 55 { 56 return (pstate & PSR_Z_BIT) == 0; 57 } 58 59 static bool __kprobes __check_cs(unsigned long pstate) 60 { 61 return (pstate & PSR_C_BIT) != 0; 62 } 63 64 static bool __kprobes __check_cc(unsigned long pstate) 65 { 66 return (pstate & PSR_C_BIT) == 0; 67 } 68 69 static bool __kprobes __check_mi(unsigned long pstate) 70 { 71 return (pstate & PSR_N_BIT) != 0; 72 } 73 74 static bool __kprobes __check_pl(unsigned long pstate) 75 { 76 return (pstate & PSR_N_BIT) == 0; 77 } 78 79 static bool __kprobes __check_vs(unsigned long pstate) 80 { 81 return (pstate & PSR_V_BIT) != 0; 82 } 83 84 static bool __kprobes __check_vc(unsigned long pstate) 85 { 86 return (pstate & PSR_V_BIT) == 0; 87 } 88 89 static bool __kprobes __check_hi(unsigned long pstate) 90 { 91 pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ 92 return (pstate & PSR_C_BIT) != 0; 93 } 94 95 static bool __kprobes __check_ls(unsigned long pstate) 96 { 97 pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ 98 return (pstate & PSR_C_BIT) == 0; 99 } 100 101 static bool __kprobes __check_ge(unsigned long pstate) 102 { 103 pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ 104 return (pstate & PSR_N_BIT) == 0; 105 } 106 107 static bool __kprobes __check_lt(unsigned long pstate) 108 { 109 pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ 110 return (pstate & PSR_N_BIT) != 0; 111 } 112 113 static bool __kprobes __check_gt(unsigned long pstate) 114 { 115 /*PSR_N_BIT ^= PSR_V_BIT */ 116 unsigned long temp = pstate ^ (pstate << 3); 117 118 temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ 119 return (temp & PSR_N_BIT) == 0; 120 } 121 122 static bool __kprobes __check_le(unsigned long pstate) 123 { 124 /*PSR_N_BIT ^= PSR_V_BIT */ 125 unsigned long temp = pstate ^ (pstate << 3); 126 127 temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ 128 return (temp & PSR_N_BIT) != 0; 129 } 130 131 static bool __kprobes __check_al(unsigned long pstate) 132 { 133 return true; 134 } 135 136 /* 137 * Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that 138 * it behaves identically to 0b1110 ("al"). 139 */ 140 pstate_check_t * const aarch32_opcode_cond_checks[16] = { 141 __check_eq, __check_ne, __check_cs, __check_cc, 142 __check_mi, __check_pl, __check_vs, __check_vc, 143 __check_hi, __check_ls, __check_ge, __check_lt, 144 __check_gt, __check_le, __check_al, __check_al 145 }; 146 147 int show_unhandled_signals = 0; 148 149 static void dump_kernel_instr(const char *lvl, struct pt_regs *regs) 150 { 151 unsigned long addr = instruction_pointer(regs); 152 char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str; 153 int i; 154 155 if (user_mode(regs)) 156 return; 157 158 for (i = -4; i < 1; i++) { 159 unsigned int val, bad; 160 161 bad = aarch64_insn_read(&((u32 *)addr)[i], &val); 162 163 if (!bad) 164 p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val); 165 else 166 p += sprintf(p, i == 0 ? "(????????) " : "???????? "); 167 } 168 169 printk("%sCode: %s\n", lvl, str); 170 } 171 172 #ifdef CONFIG_PREEMPT 173 #define S_PREEMPT " PREEMPT" 174 #elif defined(CONFIG_PREEMPT_RT) 175 #define S_PREEMPT " PREEMPT_RT" 176 #else 177 #define S_PREEMPT "" 178 #endif 179 180 #define S_SMP " SMP" 181 182 static int __die(const char *str, long err, struct pt_regs *regs) 183 { 184 static int die_counter; 185 int ret; 186 187 pr_emerg("Internal error: %s: %016lx [#%d]" S_PREEMPT S_SMP "\n", 188 str, err, ++die_counter); 189 190 /* trap and error numbers are mostly meaningless on ARM */ 191 ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV); 192 if (ret == NOTIFY_STOP) 193 return ret; 194 195 print_modules(); 196 show_regs(regs); 197 198 dump_kernel_instr(KERN_EMERG, regs); 199 200 return ret; 201 } 202 203 static DEFINE_RAW_SPINLOCK(die_lock); 204 205 /* 206 * This function is protected against re-entrancy. 207 */ 208 void die(const char *str, struct pt_regs *regs, long err) 209 { 210 int ret; 211 unsigned long flags; 212 213 raw_spin_lock_irqsave(&die_lock, flags); 214 215 oops_enter(); 216 217 console_verbose(); 218 bust_spinlocks(1); 219 ret = __die(str, err, regs); 220 221 if (regs && kexec_should_crash(current)) 222 crash_kexec(regs); 223 224 bust_spinlocks(0); 225 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 226 oops_exit(); 227 228 if (in_interrupt()) 229 panic("%s: Fatal exception in interrupt", str); 230 if (panic_on_oops) 231 panic("%s: Fatal exception", str); 232 233 raw_spin_unlock_irqrestore(&die_lock, flags); 234 235 if (ret != NOTIFY_STOP) 236 make_task_dead(SIGSEGV); 237 } 238 239 static void arm64_show_signal(int signo, const char *str) 240 { 241 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 242 DEFAULT_RATELIMIT_BURST); 243 struct task_struct *tsk = current; 244 unsigned long esr = tsk->thread.fault_code; 245 struct pt_regs *regs = task_pt_regs(tsk); 246 247 /* Leave if the signal won't be shown */ 248 if (!show_unhandled_signals || 249 !unhandled_signal(tsk, signo) || 250 !__ratelimit(&rs)) 251 return; 252 253 pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk)); 254 if (esr) 255 pr_cont("%s, ESR 0x%016lx, ", esr_get_class_string(esr), esr); 256 257 pr_cont("%s", str); 258 print_vma_addr(KERN_CONT " in ", regs->pc); 259 pr_cont("\n"); 260 __show_regs(regs); 261 } 262 263 void arm64_force_sig_fault(int signo, int code, unsigned long far, 264 const char *str) 265 { 266 arm64_show_signal(signo, str); 267 if (signo == SIGKILL) 268 force_sig(SIGKILL); 269 else 270 force_sig_fault(signo, code, (void __user *)far); 271 } 272 273 void arm64_force_sig_mceerr(int code, unsigned long far, short lsb, 274 const char *str) 275 { 276 arm64_show_signal(SIGBUS, str); 277 force_sig_mceerr(code, (void __user *)far, lsb); 278 } 279 280 void arm64_force_sig_ptrace_errno_trap(int errno, unsigned long far, 281 const char *str) 282 { 283 arm64_show_signal(SIGTRAP, str); 284 force_sig_ptrace_errno_trap(errno, (void __user *)far); 285 } 286 287 void arm64_notify_die(const char *str, struct pt_regs *regs, 288 int signo, int sicode, unsigned long far, 289 unsigned long err) 290 { 291 if (user_mode(regs)) { 292 WARN_ON(regs != current_pt_regs()); 293 current->thread.fault_address = 0; 294 current->thread.fault_code = err; 295 296 arm64_force_sig_fault(signo, sicode, far, str); 297 } else { 298 die(str, regs, err); 299 } 300 } 301 302 #ifdef CONFIG_COMPAT 303 #define PSTATE_IT_1_0_SHIFT 25 304 #define PSTATE_IT_1_0_MASK (0x3 << PSTATE_IT_1_0_SHIFT) 305 #define PSTATE_IT_7_2_SHIFT 10 306 #define PSTATE_IT_7_2_MASK (0x3f << PSTATE_IT_7_2_SHIFT) 307 308 static u32 compat_get_it_state(struct pt_regs *regs) 309 { 310 u32 it, pstate = regs->pstate; 311 312 it = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT; 313 it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2; 314 315 return it; 316 } 317 318 static void compat_set_it_state(struct pt_regs *regs, u32 it) 319 { 320 u32 pstate_it; 321 322 pstate_it = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK; 323 pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK; 324 325 regs->pstate &= ~PSR_AA32_IT_MASK; 326 regs->pstate |= pstate_it; 327 } 328 329 static void advance_itstate(struct pt_regs *regs) 330 { 331 u32 it; 332 333 /* ARM mode */ 334 if (!(regs->pstate & PSR_AA32_T_BIT) || 335 !(regs->pstate & PSR_AA32_IT_MASK)) 336 return; 337 338 it = compat_get_it_state(regs); 339 340 /* 341 * If this is the last instruction of the block, wipe the IT 342 * state. Otherwise advance it. 343 */ 344 if (!(it & 7)) 345 it = 0; 346 else 347 it = (it & 0xe0) | ((it << 1) & 0x1f); 348 349 compat_set_it_state(regs, it); 350 } 351 #else 352 static void advance_itstate(struct pt_regs *regs) 353 { 354 } 355 #endif 356 357 void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size) 358 { 359 regs->pc += size; 360 361 /* 362 * If we were single stepping, we want to get the step exception after 363 * we return from the trap. 364 */ 365 if (user_mode(regs)) 366 user_fastforward_single_step(current); 367 368 if (compat_user_mode(regs)) 369 advance_itstate(regs); 370 else 371 regs->pstate &= ~PSR_BTYPE_MASK; 372 } 373 374 static int user_insn_read(struct pt_regs *regs, u32 *insnp) 375 { 376 u32 instr; 377 unsigned long pc = instruction_pointer(regs); 378 379 if (compat_thumb_mode(regs)) { 380 /* 16-bit Thumb instruction */ 381 __le16 instr_le; 382 if (get_user(instr_le, (__le16 __user *)pc)) 383 return -EFAULT; 384 instr = le16_to_cpu(instr_le); 385 if (aarch32_insn_is_wide(instr)) { 386 u32 instr2; 387 388 if (get_user(instr_le, (__le16 __user *)(pc + 2))) 389 return -EFAULT; 390 instr2 = le16_to_cpu(instr_le); 391 instr = (instr << 16) | instr2; 392 } 393 } else { 394 /* 32-bit ARM instruction */ 395 __le32 instr_le; 396 if (get_user(instr_le, (__le32 __user *)pc)) 397 return -EFAULT; 398 instr = le32_to_cpu(instr_le); 399 } 400 401 *insnp = instr; 402 return 0; 403 } 404 405 void force_signal_inject(int signal, int code, unsigned long address, unsigned long err) 406 { 407 const char *desc; 408 struct pt_regs *regs = current_pt_regs(); 409 410 if (WARN_ON(!user_mode(regs))) 411 return; 412 413 switch (signal) { 414 case SIGILL: 415 desc = "undefined instruction"; 416 break; 417 case SIGSEGV: 418 desc = "illegal memory access"; 419 break; 420 default: 421 desc = "unknown or unrecoverable error"; 422 break; 423 } 424 425 /* Force signals we don't understand to SIGKILL */ 426 if (WARN_ON(signal != SIGKILL && 427 siginfo_layout(signal, code) != SIL_FAULT)) { 428 signal = SIGKILL; 429 } 430 431 arm64_notify_die(desc, regs, signal, code, address, err); 432 } 433 434 /* 435 * Set up process info to signal segmentation fault - called on access error. 436 */ 437 void arm64_notify_segfault(unsigned long addr) 438 { 439 int code; 440 441 mmap_read_lock(current->mm); 442 if (find_vma(current->mm, untagged_addr(addr)) == NULL) 443 code = SEGV_MAPERR; 444 else 445 code = SEGV_ACCERR; 446 mmap_read_unlock(current->mm); 447 448 force_signal_inject(SIGSEGV, code, addr, 0); 449 } 450 451 void do_el0_undef(struct pt_regs *regs, unsigned long esr) 452 { 453 u32 insn; 454 455 /* check for AArch32 breakpoint instructions */ 456 if (!aarch32_break_handler(regs)) 457 return; 458 459 if (user_insn_read(regs, &insn)) 460 goto out_err; 461 462 if (try_emulate_mrs(regs, insn)) 463 return; 464 465 if (try_emulate_armv8_deprecated(regs, insn)) 466 return; 467 468 out_err: 469 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); 470 } 471 472 void do_el1_undef(struct pt_regs *regs, unsigned long esr) 473 { 474 u32 insn; 475 476 if (aarch64_insn_read((void *)regs->pc, &insn)) 477 goto out_err; 478 479 if (try_emulate_el1_ssbs(regs, insn)) 480 return; 481 482 out_err: 483 die("Oops - Undefined instruction", regs, esr); 484 } 485 486 void do_el0_bti(struct pt_regs *regs) 487 { 488 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); 489 } 490 491 void do_el1_bti(struct pt_regs *regs, unsigned long esr) 492 { 493 die("Oops - BTI", regs, esr); 494 } 495 496 void do_el0_fpac(struct pt_regs *regs, unsigned long esr) 497 { 498 force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr); 499 } 500 501 void do_el1_fpac(struct pt_regs *regs, unsigned long esr) 502 { 503 /* 504 * Unexpected FPAC exception in the kernel: kill the task before it 505 * does any more harm. 506 */ 507 die("Oops - FPAC", regs, esr); 508 } 509 510 #define __user_cache_maint(insn, address, res) \ 511 if (address >= TASK_SIZE_MAX) { \ 512 res = -EFAULT; \ 513 } else { \ 514 uaccess_ttbr0_enable(); \ 515 asm volatile ( \ 516 "1: " insn ", %1\n" \ 517 " mov %w0, #0\n" \ 518 "2:\n" \ 519 _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) \ 520 : "=r" (res) \ 521 : "r" (address)); \ 522 uaccess_ttbr0_disable(); \ 523 } 524 525 static void user_cache_maint_handler(unsigned long esr, struct pt_regs *regs) 526 { 527 unsigned long tagged_address, address; 528 int rt = ESR_ELx_SYS64_ISS_RT(esr); 529 int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; 530 int ret = 0; 531 532 tagged_address = pt_regs_read_reg(regs, rt); 533 address = untagged_addr(tagged_address); 534 535 switch (crm) { 536 case ESR_ELx_SYS64_ISS_CRM_DC_CVAU: /* DC CVAU, gets promoted */ 537 __user_cache_maint("dc civac", address, ret); 538 break; 539 case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */ 540 __user_cache_maint("dc civac", address, ret); 541 break; 542 case ESR_ELx_SYS64_ISS_CRM_DC_CVADP: /* DC CVADP */ 543 __user_cache_maint("sys 3, c7, c13, 1", address, ret); 544 break; 545 case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */ 546 __user_cache_maint("sys 3, c7, c12, 1", address, ret); 547 break; 548 case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */ 549 __user_cache_maint("dc civac", address, ret); 550 break; 551 case ESR_ELx_SYS64_ISS_CRM_IC_IVAU: /* IC IVAU */ 552 __user_cache_maint("ic ivau", address, ret); 553 break; 554 default: 555 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); 556 return; 557 } 558 559 if (ret) 560 arm64_notify_segfault(tagged_address); 561 else 562 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 563 } 564 565 static void ctr_read_handler(unsigned long esr, struct pt_regs *regs) 566 { 567 int rt = ESR_ELx_SYS64_ISS_RT(esr); 568 unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0); 569 570 if (cpus_have_const_cap(ARM64_WORKAROUND_1542419)) { 571 /* Hide DIC so that we can trap the unnecessary maintenance...*/ 572 val &= ~BIT(CTR_EL0_DIC_SHIFT); 573 574 /* ... and fake IminLine to reduce the number of traps. */ 575 val &= ~CTR_EL0_IminLine_MASK; 576 val |= (PAGE_SHIFT - 2) & CTR_EL0_IminLine_MASK; 577 } 578 579 pt_regs_write_reg(regs, rt, val); 580 581 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 582 } 583 584 static void cntvct_read_handler(unsigned long esr, struct pt_regs *regs) 585 { 586 int rt = ESR_ELx_SYS64_ISS_RT(esr); 587 588 pt_regs_write_reg(regs, rt, arch_timer_read_counter()); 589 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 590 } 591 592 static void cntfrq_read_handler(unsigned long esr, struct pt_regs *regs) 593 { 594 int rt = ESR_ELx_SYS64_ISS_RT(esr); 595 596 pt_regs_write_reg(regs, rt, arch_timer_get_rate()); 597 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 598 } 599 600 static void mrs_handler(unsigned long esr, struct pt_regs *regs) 601 { 602 u32 sysreg, rt; 603 604 rt = ESR_ELx_SYS64_ISS_RT(esr); 605 sysreg = esr_sys64_to_sysreg(esr); 606 607 if (do_emulate_mrs(regs, sysreg, rt) != 0) 608 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); 609 } 610 611 static void wfi_handler(unsigned long esr, struct pt_regs *regs) 612 { 613 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 614 } 615 616 struct sys64_hook { 617 unsigned long esr_mask; 618 unsigned long esr_val; 619 void (*handler)(unsigned long esr, struct pt_regs *regs); 620 }; 621 622 static const struct sys64_hook sys64_hooks[] = { 623 { 624 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK, 625 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL, 626 .handler = user_cache_maint_handler, 627 }, 628 { 629 /* Trap read access to CTR_EL0 */ 630 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 631 .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ, 632 .handler = ctr_read_handler, 633 }, 634 { 635 /* Trap read access to CNTVCT_EL0 */ 636 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 637 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT, 638 .handler = cntvct_read_handler, 639 }, 640 { 641 /* Trap read access to CNTVCTSS_EL0 */ 642 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 643 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCTSS, 644 .handler = cntvct_read_handler, 645 }, 646 { 647 /* Trap read access to CNTFRQ_EL0 */ 648 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, 649 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ, 650 .handler = cntfrq_read_handler, 651 }, 652 { 653 /* Trap read access to CPUID registers */ 654 .esr_mask = ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK, 655 .esr_val = ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL, 656 .handler = mrs_handler, 657 }, 658 { 659 /* Trap WFI instructions executed in userspace */ 660 .esr_mask = ESR_ELx_WFx_MASK, 661 .esr_val = ESR_ELx_WFx_WFI_VAL, 662 .handler = wfi_handler, 663 }, 664 {}, 665 }; 666 667 #ifdef CONFIG_COMPAT 668 static bool cp15_cond_valid(unsigned long esr, struct pt_regs *regs) 669 { 670 int cond; 671 672 /* Only a T32 instruction can trap without CV being set */ 673 if (!(esr & ESR_ELx_CV)) { 674 u32 it; 675 676 it = compat_get_it_state(regs); 677 if (!it) 678 return true; 679 680 cond = it >> 4; 681 } else { 682 cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT; 683 } 684 685 return aarch32_opcode_cond_checks[cond](regs->pstate); 686 } 687 688 static void compat_cntfrq_read_handler(unsigned long esr, struct pt_regs *regs) 689 { 690 int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT; 691 692 pt_regs_write_reg(regs, reg, arch_timer_get_rate()); 693 arm64_skip_faulting_instruction(regs, 4); 694 } 695 696 static const struct sys64_hook cp15_32_hooks[] = { 697 { 698 .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK, 699 .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ, 700 .handler = compat_cntfrq_read_handler, 701 }, 702 {}, 703 }; 704 705 static void compat_cntvct_read_handler(unsigned long esr, struct pt_regs *regs) 706 { 707 int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT; 708 int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT; 709 u64 val = arch_timer_read_counter(); 710 711 pt_regs_write_reg(regs, rt, lower_32_bits(val)); 712 pt_regs_write_reg(regs, rt2, upper_32_bits(val)); 713 arm64_skip_faulting_instruction(regs, 4); 714 } 715 716 static const struct sys64_hook cp15_64_hooks[] = { 717 { 718 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK, 719 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT, 720 .handler = compat_cntvct_read_handler, 721 }, 722 { 723 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK, 724 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCTSS, 725 .handler = compat_cntvct_read_handler, 726 }, 727 {}, 728 }; 729 730 void do_el0_cp15(unsigned long esr, struct pt_regs *regs) 731 { 732 const struct sys64_hook *hook, *hook_base; 733 734 if (!cp15_cond_valid(esr, regs)) { 735 /* 736 * There is no T16 variant of a CP access, so we 737 * always advance PC by 4 bytes. 738 */ 739 arm64_skip_faulting_instruction(regs, 4); 740 return; 741 } 742 743 switch (ESR_ELx_EC(esr)) { 744 case ESR_ELx_EC_CP15_32: 745 hook_base = cp15_32_hooks; 746 break; 747 case ESR_ELx_EC_CP15_64: 748 hook_base = cp15_64_hooks; 749 break; 750 default: 751 do_el0_undef(regs, esr); 752 return; 753 } 754 755 for (hook = hook_base; hook->handler; hook++) 756 if ((hook->esr_mask & esr) == hook->esr_val) { 757 hook->handler(esr, regs); 758 return; 759 } 760 761 /* 762 * New cp15 instructions may previously have been undefined at 763 * EL0. Fall back to our usual undefined instruction handler 764 * so that we handle these consistently. 765 */ 766 do_el0_undef(regs, esr); 767 } 768 #endif 769 770 void do_el0_sys(unsigned long esr, struct pt_regs *regs) 771 { 772 const struct sys64_hook *hook; 773 774 for (hook = sys64_hooks; hook->handler; hook++) 775 if ((hook->esr_mask & esr) == hook->esr_val) { 776 hook->handler(esr, regs); 777 return; 778 } 779 780 /* 781 * New SYS instructions may previously have been undefined at EL0. Fall 782 * back to our usual undefined instruction handler so that we handle 783 * these consistently. 784 */ 785 do_el0_undef(regs, esr); 786 } 787 788 static const char *esr_class_str[] = { 789 [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", 790 [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized", 791 [ESR_ELx_EC_WFx] = "WFI/WFE", 792 [ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC", 793 [ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC", 794 [ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC", 795 [ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC", 796 [ESR_ELx_EC_FP_ASIMD] = "ASIMD", 797 [ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS", 798 [ESR_ELx_EC_PAC] = "PAC", 799 [ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC", 800 [ESR_ELx_EC_BTI] = "BTI", 801 [ESR_ELx_EC_ILL] = "PSTATE.IL", 802 [ESR_ELx_EC_SVC32] = "SVC (AArch32)", 803 [ESR_ELx_EC_HVC32] = "HVC (AArch32)", 804 [ESR_ELx_EC_SMC32] = "SMC (AArch32)", 805 [ESR_ELx_EC_SVC64] = "SVC (AArch64)", 806 [ESR_ELx_EC_HVC64] = "HVC (AArch64)", 807 [ESR_ELx_EC_SMC64] = "SMC (AArch64)", 808 [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)", 809 [ESR_ELx_EC_SVE] = "SVE", 810 [ESR_ELx_EC_ERET] = "ERET/ERETAA/ERETAB", 811 [ESR_ELx_EC_FPAC] = "FPAC", 812 [ESR_ELx_EC_SME] = "SME", 813 [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF", 814 [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)", 815 [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)", 816 [ESR_ELx_EC_PC_ALIGN] = "PC Alignment", 817 [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)", 818 [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)", 819 [ESR_ELx_EC_SP_ALIGN] = "SP Alignment", 820 [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)", 821 [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)", 822 [ESR_ELx_EC_SERROR] = "SError", 823 [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)", 824 [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)", 825 [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)", 826 [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)", 827 [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)", 828 [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)", 829 [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)", 830 [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)", 831 [ESR_ELx_EC_BRK64] = "BRK (AArch64)", 832 }; 833 834 const char *esr_get_class_string(unsigned long esr) 835 { 836 return esr_class_str[ESR_ELx_EC(esr)]; 837 } 838 839 /* 840 * bad_el0_sync handles unexpected, but potentially recoverable synchronous 841 * exceptions taken from EL0. 842 */ 843 void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr) 844 { 845 unsigned long pc = instruction_pointer(regs); 846 847 current->thread.fault_address = 0; 848 current->thread.fault_code = esr; 849 850 arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc, 851 "Bad EL0 synchronous exception"); 852 } 853 854 #ifdef CONFIG_VMAP_STACK 855 856 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) 857 __aligned(16); 858 859 void panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far) 860 { 861 unsigned long tsk_stk = (unsigned long)current->stack; 862 unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); 863 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); 864 865 console_verbose(); 866 pr_emerg("Insufficient stack space to handle exception!"); 867 868 pr_emerg("ESR: 0x%016lx -- %s\n", esr, esr_get_class_string(esr)); 869 pr_emerg("FAR: 0x%016lx\n", far); 870 871 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", 872 tsk_stk, tsk_stk + THREAD_SIZE); 873 pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n", 874 irq_stk, irq_stk + IRQ_STACK_SIZE); 875 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", 876 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); 877 878 __show_regs(regs); 879 880 /* 881 * We use nmi_panic to limit the potential for recusive overflows, and 882 * to get a better stack trace. 883 */ 884 nmi_panic(NULL, "kernel stack overflow"); 885 cpu_park_loop(); 886 } 887 #endif 888 889 void __noreturn arm64_serror_panic(struct pt_regs *regs, unsigned long esr) 890 { 891 console_verbose(); 892 893 pr_crit("SError Interrupt on CPU%d, code 0x%016lx -- %s\n", 894 smp_processor_id(), esr, esr_get_class_string(esr)); 895 if (regs) 896 __show_regs(regs); 897 898 nmi_panic(regs, "Asynchronous SError Interrupt"); 899 900 cpu_park_loop(); 901 unreachable(); 902 } 903 904 bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned long esr) 905 { 906 unsigned long aet = arm64_ras_serror_get_severity(esr); 907 908 switch (aet) { 909 case ESR_ELx_AET_CE: /* corrected error */ 910 case ESR_ELx_AET_UEO: /* restartable, not yet consumed */ 911 /* 912 * The CPU can make progress. We may take UEO again as 913 * a more severe error. 914 */ 915 return false; 916 917 case ESR_ELx_AET_UEU: /* Uncorrected Unrecoverable */ 918 case ESR_ELx_AET_UER: /* Uncorrected Recoverable */ 919 /* 920 * The CPU can't make progress. The exception may have 921 * been imprecise. 922 * 923 * Neoverse-N1 #1349291 means a non-KVM SError reported as 924 * Unrecoverable should be treated as Uncontainable. We 925 * call arm64_serror_panic() in both cases. 926 */ 927 return true; 928 929 case ESR_ELx_AET_UC: /* Uncontainable or Uncategorized error */ 930 default: 931 /* Error has been silently propagated */ 932 arm64_serror_panic(regs, esr); 933 } 934 } 935 936 void do_serror(struct pt_regs *regs, unsigned long esr) 937 { 938 /* non-RAS errors are not containable */ 939 if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr)) 940 arm64_serror_panic(regs, esr); 941 } 942 943 /* GENERIC_BUG traps */ 944 945 int is_valid_bugaddr(unsigned long addr) 946 { 947 /* 948 * bug_handler() only called for BRK #BUG_BRK_IMM. 949 * So the answer is trivial -- any spurious instances with no 950 * bug table entry will be rejected by report_bug() and passed 951 * back to the debug-monitors code and handled as a fatal 952 * unexpected debug exception. 953 */ 954 return 1; 955 } 956 957 static int bug_handler(struct pt_regs *regs, unsigned long esr) 958 { 959 switch (report_bug(regs->pc, regs)) { 960 case BUG_TRAP_TYPE_BUG: 961 die("Oops - BUG", regs, esr); 962 break; 963 964 case BUG_TRAP_TYPE_WARN: 965 break; 966 967 default: 968 /* unknown/unrecognised bug trap type */ 969 return DBG_HOOK_ERROR; 970 } 971 972 /* If thread survives, skip over the BUG instruction and continue: */ 973 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 974 return DBG_HOOK_HANDLED; 975 } 976 977 static struct break_hook bug_break_hook = { 978 .fn = bug_handler, 979 .imm = BUG_BRK_IMM, 980 }; 981 982 #ifdef CONFIG_CFI_CLANG 983 static int cfi_handler(struct pt_regs *regs, unsigned long esr) 984 { 985 unsigned long target; 986 u32 type; 987 988 target = pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TARGET, esr)); 989 type = (u32)pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TYPE, esr)); 990 991 switch (report_cfi_failure(regs, regs->pc, &target, type)) { 992 case BUG_TRAP_TYPE_BUG: 993 die("Oops - CFI", regs, 0); 994 break; 995 996 case BUG_TRAP_TYPE_WARN: 997 break; 998 999 default: 1000 return DBG_HOOK_ERROR; 1001 } 1002 1003 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 1004 return DBG_HOOK_HANDLED; 1005 } 1006 1007 static struct break_hook cfi_break_hook = { 1008 .fn = cfi_handler, 1009 .imm = CFI_BRK_IMM_BASE, 1010 .mask = CFI_BRK_IMM_MASK, 1011 }; 1012 #endif /* CONFIG_CFI_CLANG */ 1013 1014 static int reserved_fault_handler(struct pt_regs *regs, unsigned long esr) 1015 { 1016 pr_err("%s generated an invalid instruction at %pS!\n", 1017 "Kernel text patching", 1018 (void *)instruction_pointer(regs)); 1019 1020 /* We cannot handle this */ 1021 return DBG_HOOK_ERROR; 1022 } 1023 1024 static struct break_hook fault_break_hook = { 1025 .fn = reserved_fault_handler, 1026 .imm = FAULT_BRK_IMM, 1027 }; 1028 1029 #ifdef CONFIG_KASAN_SW_TAGS 1030 1031 #define KASAN_ESR_RECOVER 0x20 1032 #define KASAN_ESR_WRITE 0x10 1033 #define KASAN_ESR_SIZE_MASK 0x0f 1034 #define KASAN_ESR_SIZE(esr) (1 << ((esr) & KASAN_ESR_SIZE_MASK)) 1035 1036 static int kasan_handler(struct pt_regs *regs, unsigned long esr) 1037 { 1038 bool recover = esr & KASAN_ESR_RECOVER; 1039 bool write = esr & KASAN_ESR_WRITE; 1040 size_t size = KASAN_ESR_SIZE(esr); 1041 u64 addr = regs->regs[0]; 1042 u64 pc = regs->pc; 1043 1044 kasan_report(addr, size, write, pc); 1045 1046 /* 1047 * The instrumentation allows to control whether we can proceed after 1048 * a crash was detected. This is done by passing the -recover flag to 1049 * the compiler. Disabling recovery allows to generate more compact 1050 * code. 1051 * 1052 * Unfortunately disabling recovery doesn't work for the kernel right 1053 * now. KASAN reporting is disabled in some contexts (for example when 1054 * the allocator accesses slab object metadata; this is controlled by 1055 * current->kasan_depth). All these accesses are detected by the tool, 1056 * even though the reports for them are not printed. 1057 * 1058 * This is something that might be fixed at some point in the future. 1059 */ 1060 if (!recover) 1061 die("Oops - KASAN", regs, esr); 1062 1063 /* If thread survives, skip over the brk instruction and continue: */ 1064 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 1065 return DBG_HOOK_HANDLED; 1066 } 1067 1068 static struct break_hook kasan_break_hook = { 1069 .fn = kasan_handler, 1070 .imm = KASAN_BRK_IMM, 1071 .mask = KASAN_BRK_MASK, 1072 }; 1073 #endif 1074 1075 1076 #define esr_comment(esr) ((esr) & ESR_ELx_BRK64_ISS_COMMENT_MASK) 1077 1078 /* 1079 * Initial handler for AArch64 BRK exceptions 1080 * This handler only used until debug_traps_init(). 1081 */ 1082 int __init early_brk64(unsigned long addr, unsigned long esr, 1083 struct pt_regs *regs) 1084 { 1085 #ifdef CONFIG_CFI_CLANG 1086 if ((esr_comment(esr) & ~CFI_BRK_IMM_MASK) == CFI_BRK_IMM_BASE) 1087 return cfi_handler(regs, esr) != DBG_HOOK_HANDLED; 1088 #endif 1089 #ifdef CONFIG_KASAN_SW_TAGS 1090 if ((esr_comment(esr) & ~KASAN_BRK_MASK) == KASAN_BRK_IMM) 1091 return kasan_handler(regs, esr) != DBG_HOOK_HANDLED; 1092 #endif 1093 return bug_handler(regs, esr) != DBG_HOOK_HANDLED; 1094 } 1095 1096 void __init trap_init(void) 1097 { 1098 register_kernel_break_hook(&bug_break_hook); 1099 #ifdef CONFIG_CFI_CLANG 1100 register_kernel_break_hook(&cfi_break_hook); 1101 #endif 1102 register_kernel_break_hook(&fault_break_hook); 1103 #ifdef CONFIG_KASAN_SW_TAGS 1104 register_kernel_break_hook(&kasan_break_hook); 1105 #endif 1106 debug_traps_init(); 1107 } 1108