1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Exception handling code 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 */ 7 8 #include <linux/context_tracking.h> 9 #include <linux/irq-entry-common.h> 10 #include <linux/kasan.h> 11 #include <linux/linkage.h> 12 #include <linux/livepatch.h> 13 #include <linux/lockdep.h> 14 #include <linux/ptrace.h> 15 #include <linux/resume_user_mode.h> 16 #include <linux/sched.h> 17 #include <linux/sched/debug.h> 18 #include <linux/thread_info.h> 19 20 #include <asm/cpufeature.h> 21 #include <asm/daifflags.h> 22 #include <asm/esr.h> 23 #include <asm/exception.h> 24 #include <asm/fpsimd.h> 25 #include <asm/irq_regs.h> 26 #include <asm/kprobes.h> 27 #include <asm/mmu.h> 28 #include <asm/percpu.h> 29 #include <asm/processor.h> 30 #include <asm/sdei.h> 31 #include <asm/stacktrace.h> 32 #include <asm/sysreg.h> 33 #include <asm/system_misc.h> 34 35 /* 36 * Where the context being returned to had an active percpu GPR critical 37 * section, ensure that the offset and address GPRs are updated to match the 38 * current CPU. 39 * 40 * For simplicity we always update the GPRs when a critical section is active 41 * and preemption was *possible*, regardless of whether preemption actually 42 * occurred. Where preemption did not occur, the updates are redundant but not 43 * harmful. 44 */ 45 static __always_inline void irqentry_exit_pcpu_adjust(struct pt_regs *regs) 46 { 47 int reg_pcp, reg_off, reg_addr; 48 unsigned long pcp, off, addr; 49 u16 gprs = regs->pcpu_gprs; 50 51 /* 52 * Zero means no active PCPU GPRs. As the PCPU GPRs must be distinct, 53 * a PCPU critical section cannot possibly use {x0,x0,x0}. 54 */ 55 if (likely(!gprs)) 56 return; 57 58 reg_pcp = FIELD_GET(PCPU_GPR_PCP, gprs); 59 reg_off = FIELD_GET(PCPU_GPR_OFF, gprs); 60 reg_addr = FIELD_GET(PCPU_GPR_ADDR, gprs); 61 62 pcp = pt_regs_read_reg(regs, reg_pcp); 63 64 off = __kern_my_cpu_offset(); 65 pt_regs_write_reg(regs, reg_off, off); 66 67 addr = pcp + off; 68 pt_regs_write_reg(regs, reg_addr, addr); 69 } 70 71 /* 72 * Handle IRQ/context state management when entering from kernel mode. 73 * Before this function is called it is not safe to call regular kernel code, 74 * instrumentable code, or any code which may trigger an exception. 75 */ 76 static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *regs) 77 { 78 irqentry_state_t state; 79 80 state = irqentry_enter_from_kernel_mode(regs); 81 mte_check_tfsr_entry(); 82 mte_disable_tco_entry(current); 83 84 return state; 85 } 86 87 /* 88 * Handle IRQ/context state management when exiting to kernel mode. 89 * After this function returns it is not safe to call regular kernel code, 90 * instrumentable code, or any code which may trigger an exception. 91 */ 92 static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs, 93 irqentry_state_t state) 94 { 95 local_daif_mask(); 96 irqentry_exit_pcpu_adjust(regs); 97 mte_check_tfsr_exit(); 98 irqentry_exit_to_kernel_mode_after_preempt(regs, state); 99 } 100 101 /* 102 * We are returning from the context which allows involuntary kernel preemption 103 */ 104 static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs, 105 irqentry_state_t state) 106 { 107 irqentry_exit_to_kernel_mode_preempt(regs, state); 108 __arm64_exit_to_kernel_mode(regs, state); 109 } 110 111 static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, 112 irqentry_state_t state) 113 { 114 if (!regs_irqs_disabled(regs)) { 115 local_irq_disable(); 116 arm64_exit_to_kernel_mode_preempt(regs, state); 117 return; 118 } 119 120 __arm64_exit_to_kernel_mode(regs, state); 121 } 122 123 static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs) 124 { 125 enter_from_user_mode(regs); 126 mte_disable_tco_entry(current); 127 sme_enter_from_user_mode(); 128 } 129 130 /* 131 * Handle IRQ/context state management when entering from user mode. 132 * Before this function is called it is not safe to call regular kernel code, 133 * instrumentable code, or any code which may trigger an exception. 134 */ 135 static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs) 136 { 137 enter_from_user_mode(regs); 138 rseq_note_user_irq_entry(); 139 mte_disable_tco_entry(current); 140 sme_enter_from_user_mode(); 141 } 142 143 static __always_inline void arm64_syscall_exit_to_user_mode(struct pt_regs *regs) 144 { 145 local_irq_disable(); 146 syscall_exit_to_user_mode_prepare(regs); 147 local_daif_mask(); 148 sme_exit_to_user_mode(); 149 mte_check_tfsr_exit(); 150 exit_to_user_mode(); 151 } 152 153 /* 154 * Handle IRQ/context state management when exiting to user mode. 155 * After this function returns it is not safe to call regular kernel code, 156 * instrumentable code, or any code which may trigger an exception. 157 */ 158 static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs) 159 { 160 local_irq_disable(); 161 irqentry_exit_to_user_mode_prepare(regs); 162 local_daif_mask(); 163 sme_exit_to_user_mode(); 164 mte_check_tfsr_exit(); 165 exit_to_user_mode(); 166 } 167 168 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) 169 { 170 arm64_syscall_exit_to_user_mode(regs); 171 } 172 173 /* 174 * Handle IRQ/context state management when entering a debug exception from 175 * kernel mode. Before this function is called it is not safe to call regular 176 * kernel code, instrumentable code, or any code which may trigger an exception. 177 */ 178 static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs) 179 { 180 irqentry_state_t state; 181 182 state.lockdep = lockdep_hardirqs_enabled(); 183 184 lockdep_hardirqs_off(CALLER_ADDR0); 185 ct_nmi_enter(); 186 187 trace_hardirqs_off_finish(); 188 189 return state; 190 } 191 192 /* 193 * Handle IRQ/context state management when exiting a debug exception from 194 * kernel mode. After this function returns it is not safe to call regular 195 * kernel code, instrumentable code, or any code which may trigger an exception. 196 */ 197 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs, 198 irqentry_state_t state) 199 { 200 if (state.lockdep) { 201 trace_hardirqs_on_prepare(); 202 lockdep_hardirqs_on_prepare(); 203 } 204 205 ct_nmi_exit(); 206 if (state.lockdep) 207 lockdep_hardirqs_on(CALLER_ADDR0); 208 } 209 210 static void do_interrupt_handler(struct pt_regs *regs, 211 void (*handler)(struct pt_regs *)) 212 { 213 struct pt_regs *old_regs = set_irq_regs(regs); 214 215 if (on_thread_stack()) 216 call_on_irq_stack(regs, handler); 217 else 218 handler(regs); 219 220 set_irq_regs(old_regs); 221 } 222 223 extern void (*handle_arch_irq)(struct pt_regs *); 224 extern void (*handle_arch_fiq)(struct pt_regs *); 225 226 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, 227 unsigned long esr) 228 { 229 irqentry_nmi_enter(regs); 230 231 console_verbose(); 232 233 pr_crit("Unhandled %s exception on CPU%d, ESR 0x%016lx -- %s\n", 234 vector, smp_processor_id(), esr, 235 esr_get_class_string(esr)); 236 237 __show_regs(regs); 238 panic("Unhandled exception"); 239 } 240 241 #define UNHANDLED(el, regsize, vector) \ 242 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \ 243 { \ 244 const char *desc = #regsize "-bit " #el " " #vector; \ 245 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \ 246 } 247 248 #ifdef CONFIG_ARM64_ERRATUM_1463225 249 static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa); 250 251 static void cortex_a76_erratum_1463225_svc_handler(void) 252 { 253 u64 reg, val; 254 255 if (!unlikely(test_thread_flag(TIF_SINGLESTEP))) 256 return; 257 258 if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225))) 259 return; 260 261 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1); 262 reg = read_sysreg(mdscr_el1); 263 val = reg | MDSCR_EL1_SS | MDSCR_EL1_KDE; 264 write_sysreg(val, mdscr_el1); 265 asm volatile("msr daifclr, #8"); 266 isb(); 267 268 /* We will have taken a single-step exception by this point */ 269 270 write_sysreg(reg, mdscr_el1); 271 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0); 272 } 273 274 static __always_inline bool 275 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 276 { 277 if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa)) 278 return false; 279 280 /* 281 * We've taken a dummy step exception from the kernel to ensure 282 * that interrupts are re-enabled on the syscall path. Return back 283 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions 284 * masked so that we can safely restore the mdscr and get on with 285 * handling the syscall. 286 */ 287 regs->pstate |= PSR_D_BIT; 288 return true; 289 } 290 #else /* CONFIG_ARM64_ERRATUM_1463225 */ 291 static void cortex_a76_erratum_1463225_svc_handler(void) { } 292 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 293 { 294 return false; 295 } 296 #endif /* CONFIG_ARM64_ERRATUM_1463225 */ 297 298 /* 299 * As per the ABI exit SME streaming mode and clear the SVE state not 300 * shared with FPSIMD on syscall entry. 301 */ 302 static inline void fpsimd_syscall_enter(void) 303 { 304 /* Ensure PSTATE.SM is clear, but leave PSTATE.ZA as-is. */ 305 if (system_supports_sme()) 306 sme_smstop_sm(); 307 308 /* 309 * The CPU is not in streaming mode. If non-streaming SVE is not 310 * supported, there is no SVE state that needs to be discarded. 311 */ 312 if (!system_supports_sve()) 313 return; 314 315 if (test_thread_flag(TIF_SVE)) 316 sve_flush_live(); 317 318 /* 319 * Any live non-FPSIMD SVE state has been zeroed. Allow 320 * fpsimd_save_user_state() to lazily discard SVE state until either 321 * the live state is unbound or fpsimd_syscall_exit() is called. 322 */ 323 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_FPSIMD); 324 } 325 326 static __always_inline void fpsimd_syscall_exit(void) 327 { 328 if (!system_supports_sve()) 329 return; 330 331 /* 332 * The current task's user FPSIMD/SVE/SME state is now bound to this 333 * CPU. The fpsimd_last_state.to_save value is either: 334 * 335 * - FP_STATE_FPSIMD, if the state has not been reloaded on this CPU 336 * since fpsimd_syscall_enter(). 337 * 338 * - FP_STATE_CURRENT, if the state has been reloaded on this CPU at 339 * any point. 340 * 341 * Reset this to FP_STATE_CURRENT to stop lazy discarding. 342 */ 343 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT); 344 } 345 346 /* 347 * In debug exception context, we explicitly disable preemption despite 348 * having interrupts disabled. 349 * This serves two purposes: it makes it much less likely that we would 350 * accidentally schedule in exception context and it will force a warning 351 * if we somehow manage to schedule by accident. 352 */ 353 static void debug_exception_enter(struct pt_regs *regs) 354 { 355 preempt_disable(); 356 357 /* This code is a bit fragile. Test it. */ 358 RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work"); 359 } 360 NOKPROBE_SYMBOL(debug_exception_enter); 361 362 static void debug_exception_exit(struct pt_regs *regs) 363 { 364 preempt_enable_no_resched(); 365 } 366 NOKPROBE_SYMBOL(debug_exception_exit); 367 368 UNHANDLED(el1t, 64, sync) 369 UNHANDLED(el1t, 64, irq) 370 UNHANDLED(el1t, 64, fiq) 371 UNHANDLED(el1t, 64, error) 372 373 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) 374 { 375 unsigned long far = read_sysreg(far_el1); 376 irqentry_state_t state; 377 378 state = arm64_enter_from_kernel_mode(regs); 379 local_daif_inherit(regs); 380 do_mem_abort(far, esr, regs); 381 arm64_exit_to_kernel_mode(regs, state); 382 } 383 384 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) 385 { 386 unsigned long far = read_sysreg(far_el1); 387 irqentry_state_t state; 388 389 state = arm64_enter_from_kernel_mode(regs); 390 local_daif_inherit(regs); 391 do_sp_pc_abort(far, esr, regs); 392 arm64_exit_to_kernel_mode(regs, state); 393 } 394 395 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) 396 { 397 irqentry_state_t state; 398 399 state = arm64_enter_from_kernel_mode(regs); 400 local_daif_inherit(regs); 401 do_el1_undef(regs, esr); 402 arm64_exit_to_kernel_mode(regs, state); 403 } 404 405 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) 406 { 407 irqentry_state_t state; 408 409 state = arm64_enter_from_kernel_mode(regs); 410 local_daif_inherit(regs); 411 do_el1_bti(regs, esr); 412 arm64_exit_to_kernel_mode(regs, state); 413 } 414 415 static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr) 416 { 417 irqentry_state_t state; 418 419 state = arm64_enter_from_kernel_mode(regs); 420 local_daif_inherit(regs); 421 do_el1_gcs(regs, esr); 422 arm64_exit_to_kernel_mode(regs, state); 423 } 424 425 static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr) 426 { 427 irqentry_state_t state; 428 429 state = arm64_enter_from_kernel_mode(regs); 430 local_daif_inherit(regs); 431 do_el1_mops(regs, esr); 432 arm64_exit_to_kernel_mode(regs, state); 433 } 434 435 static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) 436 { 437 irqentry_state_t state; 438 439 state = arm64_enter_el1_dbg(regs); 440 debug_exception_enter(regs); 441 do_breakpoint(esr, regs); 442 debug_exception_exit(regs); 443 arm64_exit_el1_dbg(regs, state); 444 } 445 446 static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) 447 { 448 irqentry_state_t state; 449 450 state = arm64_enter_el1_dbg(regs); 451 if (!cortex_a76_erratum_1463225_debug_handler(regs)) { 452 debug_exception_enter(regs); 453 /* 454 * After handling a breakpoint, we suspend the breakpoint 455 * and use single-step to move to the next instruction. 456 * If we are stepping a suspended breakpoint there's nothing more to do: 457 * the single-step is complete. 458 */ 459 if (!try_step_suspended_breakpoints(regs)) 460 do_el1_softstep(esr, regs); 461 debug_exception_exit(regs); 462 } 463 arm64_exit_el1_dbg(regs, state); 464 } 465 466 static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) 467 { 468 /* Watchpoints are the only debug exception to write FAR_EL1 */ 469 unsigned long far = read_sysreg(far_el1); 470 irqentry_state_t state; 471 472 state = arm64_enter_el1_dbg(regs); 473 debug_exception_enter(regs); 474 do_watchpoint(far, esr, regs); 475 debug_exception_exit(regs); 476 arm64_exit_el1_dbg(regs, state); 477 } 478 479 static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr) 480 { 481 irqentry_state_t state; 482 483 state = arm64_enter_el1_dbg(regs); 484 debug_exception_enter(regs); 485 do_el1_brk64(esr, regs); 486 debug_exception_exit(regs); 487 arm64_exit_el1_dbg(regs, state); 488 } 489 490 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) 491 { 492 irqentry_state_t state; 493 494 state = arm64_enter_from_kernel_mode(regs); 495 local_daif_inherit(regs); 496 do_el1_fpac(regs, esr); 497 arm64_exit_to_kernel_mode(regs, state); 498 } 499 500 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) 501 { 502 unsigned long esr = read_sysreg(esr_el1); 503 504 switch (ESR_ELx_EC(esr)) { 505 case ESR_ELx_EC_DABT_CUR: 506 case ESR_ELx_EC_IABT_CUR: 507 el1_abort(regs, esr); 508 break; 509 /* 510 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a 511 * recursive exception when trying to push the initial pt_regs. 512 */ 513 case ESR_ELx_EC_PC_ALIGN: 514 el1_pc(regs, esr); 515 break; 516 case ESR_ELx_EC_SYS64: 517 case ESR_ELx_EC_UNKNOWN: 518 el1_undef(regs, esr); 519 break; 520 case ESR_ELx_EC_BTI: 521 el1_bti(regs, esr); 522 break; 523 case ESR_ELx_EC_GCS: 524 el1_gcs(regs, esr); 525 break; 526 case ESR_ELx_EC_MOPS: 527 el1_mops(regs, esr); 528 break; 529 case ESR_ELx_EC_BREAKPT_CUR: 530 el1_breakpt(regs, esr); 531 break; 532 case ESR_ELx_EC_SOFTSTP_CUR: 533 el1_softstp(regs, esr); 534 break; 535 case ESR_ELx_EC_WATCHPT_CUR: 536 el1_watchpt(regs, esr); 537 break; 538 case ESR_ELx_EC_BRK64: 539 el1_brk64(regs, esr); 540 break; 541 case ESR_ELx_EC_FPAC: 542 el1_fpac(regs, esr); 543 break; 544 default: 545 __panic_unhandled(regs, "64-bit el1h sync", esr); 546 } 547 } 548 549 static __always_inline void __el1_pnmi(struct pt_regs *regs, 550 void (*handler)(struct pt_regs *)) 551 { 552 irqentry_state_t state; 553 554 state = irqentry_nmi_enter(regs); 555 do_interrupt_handler(regs, handler); 556 local_daif_mask(); 557 irqentry_nmi_exit(regs, state); 558 } 559 560 static __always_inline void __el1_irq(struct pt_regs *regs, 561 void (*handler)(struct pt_regs *)) 562 { 563 irqentry_state_t state; 564 565 state = arm64_enter_from_kernel_mode(regs); 566 567 irq_enter_rcu(); 568 do_interrupt_handler(regs, handler); 569 irq_exit_rcu(); 570 571 arm64_exit_to_kernel_mode_preempt(regs, state); 572 } 573 static void noinstr el1_interrupt(struct pt_regs *regs, 574 void (*handler)(struct pt_regs *)) 575 { 576 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 577 578 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs)) 579 __el1_pnmi(regs, handler); 580 else 581 __el1_irq(regs, handler); 582 } 583 584 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) 585 { 586 el1_interrupt(regs, handle_arch_irq); 587 } 588 589 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) 590 { 591 el1_interrupt(regs, handle_arch_fiq); 592 } 593 594 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) 595 { 596 unsigned long esr = read_sysreg(esr_el1); 597 irqentry_state_t state; 598 599 local_daif_restore(DAIF_ERRCTX); 600 state = irqentry_nmi_enter(regs); 601 do_serror(regs, esr); 602 local_daif_mask(); 603 irqentry_nmi_exit(regs, state); 604 } 605 606 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) 607 { 608 unsigned long far = read_sysreg(far_el1); 609 610 arm64_enter_from_user_mode(regs); 611 local_daif_restore(DAIF_PROCCTX); 612 do_mem_abort(far, esr, regs); 613 arm64_exit_to_user_mode(regs); 614 } 615 616 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr) 617 { 618 unsigned long far = read_sysreg(far_el1); 619 620 /* 621 * We've taken an instruction abort from userspace and not yet 622 * re-enabled IRQs. If the address is a kernel address, apply 623 * BP hardening prior to enabling IRQs and pre-emption. 624 */ 625 if (!is_ttbr0_addr(far)) 626 arm64_apply_bp_hardening(); 627 628 arm64_enter_from_user_mode(regs); 629 local_daif_restore(DAIF_PROCCTX); 630 do_mem_abort(far, esr, regs); 631 arm64_exit_to_user_mode(regs); 632 } 633 634 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) 635 { 636 arm64_enter_from_user_mode(regs); 637 local_daif_restore(DAIF_PROCCTX); 638 do_fpsimd_acc(esr, regs); 639 arm64_exit_to_user_mode(regs); 640 } 641 642 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr) 643 { 644 arm64_enter_from_user_mode(regs); 645 local_daif_restore(DAIF_PROCCTX); 646 do_sve_acc(esr, regs); 647 arm64_exit_to_user_mode(regs); 648 } 649 650 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr) 651 { 652 arm64_enter_from_user_mode(regs); 653 local_daif_restore(DAIF_PROCCTX); 654 do_sme_acc(esr, regs); 655 arm64_exit_to_user_mode(regs); 656 } 657 658 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) 659 { 660 arm64_enter_from_user_mode(regs); 661 local_daif_restore(DAIF_PROCCTX); 662 do_fpsimd_exc(esr, regs); 663 arm64_exit_to_user_mode(regs); 664 } 665 666 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr) 667 { 668 arm64_enter_from_user_mode(regs); 669 local_daif_restore(DAIF_PROCCTX); 670 do_el0_sys(esr, regs); 671 arm64_exit_to_user_mode(regs); 672 } 673 674 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr) 675 { 676 unsigned long far = read_sysreg(far_el1); 677 678 if (!is_ttbr0_addr(instruction_pointer(regs))) 679 arm64_apply_bp_hardening(); 680 681 arm64_enter_from_user_mode(regs); 682 local_daif_restore(DAIF_PROCCTX); 683 do_sp_pc_abort(far, esr, regs); 684 arm64_exit_to_user_mode(regs); 685 } 686 687 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr) 688 { 689 arm64_enter_from_user_mode(regs); 690 local_daif_restore(DAIF_PROCCTX); 691 do_sp_pc_abort(regs->sp, esr, regs); 692 arm64_exit_to_user_mode(regs); 693 } 694 695 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr) 696 { 697 arm64_enter_from_user_mode(regs); 698 local_daif_restore(DAIF_PROCCTX); 699 do_el0_undef(regs, esr); 700 arm64_exit_to_user_mode(regs); 701 } 702 703 static void noinstr el0_bti(struct pt_regs *regs) 704 { 705 arm64_enter_from_user_mode(regs); 706 local_daif_restore(DAIF_PROCCTX); 707 do_el0_bti(regs); 708 arm64_exit_to_user_mode(regs); 709 } 710 711 static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr) 712 { 713 arm64_enter_from_user_mode(regs); 714 local_daif_restore(DAIF_PROCCTX); 715 do_el0_mops(regs, esr); 716 arm64_exit_to_user_mode(regs); 717 } 718 719 static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr) 720 { 721 arm64_enter_from_user_mode(regs); 722 local_daif_restore(DAIF_PROCCTX); 723 do_el0_gcs(regs, esr); 724 arm64_exit_to_user_mode(regs); 725 } 726 727 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr) 728 { 729 arm64_enter_from_user_mode(regs); 730 local_daif_restore(DAIF_PROCCTX); 731 bad_el0_sync(regs, 0, esr); 732 arm64_exit_to_user_mode(regs); 733 } 734 735 static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr) 736 { 737 if (!is_ttbr0_addr(regs->pc)) 738 arm64_apply_bp_hardening(); 739 740 arm64_enter_from_user_mode(regs); 741 debug_exception_enter(regs); 742 do_breakpoint(esr, regs); 743 debug_exception_exit(regs); 744 local_daif_restore(DAIF_PROCCTX); 745 arm64_exit_to_user_mode(regs); 746 } 747 748 static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr) 749 { 750 bool step_done; 751 752 if (!is_ttbr0_addr(regs->pc)) 753 arm64_apply_bp_hardening(); 754 755 arm64_enter_from_user_mode(regs); 756 /* 757 * After handling a breakpoint, we suspend the breakpoint 758 * and use single-step to move to the next instruction. 759 * If we are stepping a suspended breakpoint there's nothing more to do: 760 * the single-step is complete. 761 */ 762 step_done = try_step_suspended_breakpoints(regs); 763 local_daif_restore(DAIF_PROCCTX); 764 if (!step_done) 765 do_el0_softstep(esr, regs); 766 arm64_exit_to_user_mode(regs); 767 } 768 769 static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr) 770 { 771 /* Watchpoints are the only debug exception to write FAR_EL1 */ 772 unsigned long far = read_sysreg(far_el1); 773 774 arm64_enter_from_user_mode(regs); 775 debug_exception_enter(regs); 776 do_watchpoint(far, esr, regs); 777 debug_exception_exit(regs); 778 local_daif_restore(DAIF_PROCCTX); 779 arm64_exit_to_user_mode(regs); 780 } 781 782 static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr) 783 { 784 arm64_enter_from_user_mode(regs); 785 local_daif_restore(DAIF_PROCCTX); 786 do_el0_brk64(esr, regs); 787 arm64_exit_to_user_mode(regs); 788 } 789 790 static void noinstr el0_svc(struct pt_regs *regs) 791 { 792 arm64_syscall_enter_from_user_mode(regs); 793 cortex_a76_erratum_1463225_svc_handler(); 794 fpsimd_syscall_enter(); 795 local_daif_restore(DAIF_PROCCTX); 796 do_el0_svc(regs); 797 arm64_syscall_exit_to_user_mode(regs); 798 fpsimd_syscall_exit(); 799 } 800 801 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) 802 { 803 arm64_enter_from_user_mode(regs); 804 local_daif_restore(DAIF_PROCCTX); 805 do_el0_fpac(regs, esr); 806 arm64_exit_to_user_mode(regs); 807 } 808 809 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) 810 { 811 unsigned long esr = read_sysreg(esr_el1); 812 813 switch (ESR_ELx_EC(esr)) { 814 case ESR_ELx_EC_SVC64: 815 el0_svc(regs); 816 break; 817 case ESR_ELx_EC_DABT_LOW: 818 el0_da(regs, esr); 819 break; 820 case ESR_ELx_EC_IABT_LOW: 821 el0_ia(regs, esr); 822 break; 823 case ESR_ELx_EC_FP_ASIMD: 824 el0_fpsimd_acc(regs, esr); 825 break; 826 case ESR_ELx_EC_SVE: 827 el0_sve_acc(regs, esr); 828 break; 829 case ESR_ELx_EC_SME: 830 el0_sme_acc(regs, esr); 831 break; 832 case ESR_ELx_EC_FP_EXC64: 833 el0_fpsimd_exc(regs, esr); 834 break; 835 case ESR_ELx_EC_SYS64: 836 case ESR_ELx_EC_WFx: 837 el0_sys(regs, esr); 838 break; 839 case ESR_ELx_EC_SP_ALIGN: 840 el0_sp(regs, esr); 841 break; 842 case ESR_ELx_EC_PC_ALIGN: 843 el0_pc(regs, esr); 844 break; 845 case ESR_ELx_EC_UNKNOWN: 846 el0_undef(regs, esr); 847 break; 848 case ESR_ELx_EC_BTI: 849 el0_bti(regs); 850 break; 851 case ESR_ELx_EC_MOPS: 852 el0_mops(regs, esr); 853 break; 854 case ESR_ELx_EC_GCS: 855 el0_gcs(regs, esr); 856 break; 857 case ESR_ELx_EC_BREAKPT_LOW: 858 el0_breakpt(regs, esr); 859 break; 860 case ESR_ELx_EC_SOFTSTP_LOW: 861 el0_softstp(regs, esr); 862 break; 863 case ESR_ELx_EC_WATCHPT_LOW: 864 el0_watchpt(regs, esr); 865 break; 866 case ESR_ELx_EC_BRK64: 867 el0_brk64(regs, esr); 868 break; 869 case ESR_ELx_EC_FPAC: 870 el0_fpac(regs, esr); 871 break; 872 default: 873 el0_inv(regs, esr); 874 } 875 } 876 877 static void noinstr el0_interrupt(struct pt_regs *regs, 878 void (*handler)(struct pt_regs *)) 879 { 880 arm64_enter_from_user_mode(regs); 881 882 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 883 884 if (regs->pc & BIT(55)) 885 arm64_apply_bp_hardening(); 886 887 irq_enter_rcu(); 888 do_interrupt_handler(regs, handler); 889 irq_exit_rcu(); 890 891 arm64_exit_to_user_mode(regs); 892 } 893 894 static void noinstr __el0_irq_handler_common(struct pt_regs *regs) 895 { 896 el0_interrupt(regs, handle_arch_irq); 897 } 898 899 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs) 900 { 901 __el0_irq_handler_common(regs); 902 } 903 904 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs) 905 { 906 el0_interrupt(regs, handle_arch_fiq); 907 } 908 909 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs) 910 { 911 __el0_fiq_handler_common(regs); 912 } 913 914 static void noinstr __el0_error_handler_common(struct pt_regs *regs) 915 { 916 unsigned long esr = read_sysreg(esr_el1); 917 irqentry_state_t state; 918 919 arm64_enter_from_user_mode(regs); 920 local_daif_restore(DAIF_ERRCTX); 921 state = irqentry_nmi_enter(regs); 922 do_serror(regs, esr); 923 irqentry_nmi_exit(regs, state); 924 local_daif_restore(DAIF_PROCCTX); 925 arm64_exit_to_user_mode(regs); 926 } 927 928 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) 929 { 930 __el0_error_handler_common(regs); 931 } 932 933 #ifdef CONFIG_COMPAT 934 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) 935 { 936 arm64_enter_from_user_mode(regs); 937 local_daif_restore(DAIF_PROCCTX); 938 do_el0_cp15(esr, regs); 939 arm64_exit_to_user_mode(regs); 940 } 941 942 static void noinstr el0_svc_compat(struct pt_regs *regs) 943 { 944 arm64_syscall_enter_from_user_mode(regs); 945 cortex_a76_erratum_1463225_svc_handler(); 946 local_daif_restore(DAIF_PROCCTX); 947 do_el0_svc_compat(regs); 948 arm64_syscall_exit_to_user_mode(regs); 949 } 950 951 static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr) 952 { 953 arm64_enter_from_user_mode(regs); 954 local_daif_restore(DAIF_PROCCTX); 955 do_bkpt32(esr, regs); 956 arm64_exit_to_user_mode(regs); 957 } 958 959 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) 960 { 961 unsigned long esr = read_sysreg(esr_el1); 962 963 switch (ESR_ELx_EC(esr)) { 964 case ESR_ELx_EC_SVC32: 965 el0_svc_compat(regs); 966 break; 967 case ESR_ELx_EC_DABT_LOW: 968 el0_da(regs, esr); 969 break; 970 case ESR_ELx_EC_IABT_LOW: 971 el0_ia(regs, esr); 972 break; 973 case ESR_ELx_EC_FP_ASIMD: 974 el0_fpsimd_acc(regs, esr); 975 break; 976 case ESR_ELx_EC_FP_EXC32: 977 el0_fpsimd_exc(regs, esr); 978 break; 979 case ESR_ELx_EC_PC_ALIGN: 980 el0_pc(regs, esr); 981 break; 982 case ESR_ELx_EC_UNKNOWN: 983 case ESR_ELx_EC_CP14_MR: 984 case ESR_ELx_EC_CP14_LS: 985 case ESR_ELx_EC_CP14_64: 986 el0_undef(regs, esr); 987 break; 988 case ESR_ELx_EC_CP15_32: 989 case ESR_ELx_EC_CP15_64: 990 el0_cp15(regs, esr); 991 break; 992 case ESR_ELx_EC_BREAKPT_LOW: 993 el0_breakpt(regs, esr); 994 break; 995 case ESR_ELx_EC_SOFTSTP_LOW: 996 el0_softstp(regs, esr); 997 break; 998 case ESR_ELx_EC_WATCHPT_LOW: 999 el0_watchpt(regs, esr); 1000 break; 1001 case ESR_ELx_EC_BKPT32: 1002 el0_bkpt32(regs, esr); 1003 break; 1004 default: 1005 el0_inv(regs, esr); 1006 } 1007 } 1008 1009 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) 1010 { 1011 __el0_irq_handler_common(regs); 1012 } 1013 1014 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) 1015 { 1016 __el0_fiq_handler_common(regs); 1017 } 1018 1019 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) 1020 { 1021 __el0_error_handler_common(regs); 1022 } 1023 #else /* CONFIG_COMPAT */ 1024 UNHANDLED(el0t, 32, sync) 1025 UNHANDLED(el0t, 32, irq) 1026 UNHANDLED(el0t, 32, fiq) 1027 UNHANDLED(el0t, 32, error) 1028 #endif /* CONFIG_COMPAT */ 1029 1030 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) 1031 { 1032 unsigned long esr = read_sysreg(esr_el1); 1033 unsigned long far = read_sysreg(far_el1); 1034 1035 irqentry_nmi_enter(regs); 1036 panic_bad_stack(regs, esr, far); 1037 } 1038 1039 #ifdef CONFIG_ARM_SDE_INTERFACE 1040 asmlinkage noinstr unsigned long 1041 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) 1042 { 1043 irqentry_state_t state; 1044 unsigned long ret; 1045 1046 /* 1047 * We didn't take an exception to get here, so the HW hasn't 1048 * set/cleared bits in PSTATE that we may rely on. 1049 * 1050 * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to 1051 * whether PSTATE bits are inherited unchanged or generated from 1052 * scratch, and the TF-A implementation always clears PAN and always 1053 * clears UAO. There are no other known implementations. 1054 * 1055 * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how 1056 * PSTATE is modified upon architectural exceptions, and so PAN is 1057 * either inherited or set per SCTLR_ELx.SPAN, and UAO is always 1058 * cleared. 1059 * 1060 * We must explicitly reset PAN to the expected state, including 1061 * clearing it when the host isn't using it, in case a VM had it set. 1062 */ 1063 if (system_uses_hw_pan()) 1064 set_pstate_pan(1); 1065 else if (cpu_has_pan()) 1066 set_pstate_pan(0); 1067 1068 state = irqentry_nmi_enter(regs); 1069 ret = do_sdei_event(regs, arg); 1070 irqentry_nmi_exit(regs, state); 1071 1072 return ret; 1073 } 1074 #endif /* CONFIG_ARM_SDE_INTERFACE */ 1075