1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Exception handling code 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 */ 7 8 #include <linux/context_tracking.h> 9 #include <linux/irq-entry-common.h> 10 #include <linux/kasan.h> 11 #include <linux/linkage.h> 12 #include <linux/livepatch.h> 13 #include <linux/lockdep.h> 14 #include <linux/ptrace.h> 15 #include <linux/resume_user_mode.h> 16 #include <linux/sched.h> 17 #include <linux/sched/debug.h> 18 #include <linux/thread_info.h> 19 20 #include <asm/cpufeature.h> 21 #include <asm/daifflags.h> 22 #include <asm/esr.h> 23 #include <asm/exception.h> 24 #include <asm/fpsimd.h> 25 #include <asm/irq_regs.h> 26 #include <asm/kprobes.h> 27 #include <asm/mmu.h> 28 #include <asm/processor.h> 29 #include <asm/sdei.h> 30 #include <asm/stacktrace.h> 31 #include <asm/sysreg.h> 32 #include <asm/system_misc.h> 33 34 /* 35 * Handle IRQ/context state management when entering from kernel mode. 36 * Before this function is called it is not safe to call regular kernel code, 37 * instrumentable code, or any code which may trigger an exception. 38 */ 39 static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *regs) 40 { 41 irqentry_state_t state; 42 43 state = irqentry_enter_from_kernel_mode(regs); 44 mte_check_tfsr_entry(); 45 mte_disable_tco_entry(current); 46 47 return state; 48 } 49 50 /* 51 * Handle IRQ/context state management when exiting to kernel mode. 52 * After this function returns it is not safe to call regular kernel code, 53 * instrumentable code, or any code which may trigger an exception. 54 */ 55 static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs, 56 irqentry_state_t state) 57 { 58 local_daif_mask(); 59 mte_check_tfsr_exit(); 60 irqentry_exit_to_kernel_mode_after_preempt(regs, state); 61 } 62 63 /* 64 * We are returning from the context which allows involuntary kernel preemption 65 */ 66 static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs, 67 irqentry_state_t state) 68 { 69 irqentry_exit_to_kernel_mode_preempt(regs, state); 70 __arm64_exit_to_kernel_mode(regs, state); 71 } 72 73 static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, 74 irqentry_state_t state) 75 { 76 if (!regs_irqs_disabled(regs)) { 77 local_irq_disable(); 78 arm64_exit_to_kernel_mode_preempt(regs, state); 79 return; 80 } 81 82 __arm64_exit_to_kernel_mode(regs, state); 83 } 84 85 static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs) 86 { 87 enter_from_user_mode(regs); 88 mte_disable_tco_entry(current); 89 sme_enter_from_user_mode(); 90 } 91 92 /* 93 * Handle IRQ/context state management when entering from user mode. 94 * Before this function is called it is not safe to call regular kernel code, 95 * instrumentable code, or any code which may trigger an exception. 96 */ 97 static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs) 98 { 99 enter_from_user_mode(regs); 100 rseq_note_user_irq_entry(); 101 mte_disable_tco_entry(current); 102 sme_enter_from_user_mode(); 103 } 104 105 static __always_inline void arm64_syscall_exit_to_user_mode(struct pt_regs *regs) 106 { 107 local_irq_disable(); 108 syscall_exit_to_user_mode_prepare(regs); 109 local_daif_mask(); 110 sme_exit_to_user_mode(); 111 mte_check_tfsr_exit(); 112 exit_to_user_mode(); 113 } 114 115 /* 116 * Handle IRQ/context state management when exiting to user mode. 117 * After this function returns it is not safe to call regular kernel code, 118 * instrumentable code, or any code which may trigger an exception. 119 */ 120 static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs) 121 { 122 local_irq_disable(); 123 irqentry_exit_to_user_mode_prepare(regs); 124 local_daif_mask(); 125 sme_exit_to_user_mode(); 126 mte_check_tfsr_exit(); 127 exit_to_user_mode(); 128 } 129 130 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) 131 { 132 arm64_syscall_exit_to_user_mode(regs); 133 } 134 135 /* 136 * Handle IRQ/context state management when entering a debug exception from 137 * kernel mode. Before this function is called it is not safe to call regular 138 * kernel code, instrumentable code, or any code which may trigger an exception. 139 */ 140 static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs) 141 { 142 irqentry_state_t state; 143 144 state.lockdep = lockdep_hardirqs_enabled(); 145 146 lockdep_hardirqs_off(CALLER_ADDR0); 147 ct_nmi_enter(); 148 149 trace_hardirqs_off_finish(); 150 151 return state; 152 } 153 154 /* 155 * Handle IRQ/context state management when exiting a debug exception from 156 * kernel mode. After this function returns it is not safe to call regular 157 * kernel code, instrumentable code, or any code which may trigger an exception. 158 */ 159 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs, 160 irqentry_state_t state) 161 { 162 if (state.lockdep) { 163 trace_hardirqs_on_prepare(); 164 lockdep_hardirqs_on_prepare(); 165 } 166 167 ct_nmi_exit(); 168 if (state.lockdep) 169 lockdep_hardirqs_on(CALLER_ADDR0); 170 } 171 172 static void do_interrupt_handler(struct pt_regs *regs, 173 void (*handler)(struct pt_regs *)) 174 { 175 struct pt_regs *old_regs = set_irq_regs(regs); 176 177 if (on_thread_stack()) 178 call_on_irq_stack(regs, handler); 179 else 180 handler(regs); 181 182 set_irq_regs(old_regs); 183 } 184 185 extern void (*handle_arch_irq)(struct pt_regs *); 186 extern void (*handle_arch_fiq)(struct pt_regs *); 187 188 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, 189 unsigned long esr) 190 { 191 irqentry_nmi_enter(regs); 192 193 console_verbose(); 194 195 pr_crit("Unhandled %s exception on CPU%d, ESR 0x%016lx -- %s\n", 196 vector, smp_processor_id(), esr, 197 esr_get_class_string(esr)); 198 199 __show_regs(regs); 200 panic("Unhandled exception"); 201 } 202 203 #define UNHANDLED(el, regsize, vector) \ 204 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \ 205 { \ 206 const char *desc = #regsize "-bit " #el " " #vector; \ 207 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \ 208 } 209 210 #ifdef CONFIG_ARM64_ERRATUM_1463225 211 static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa); 212 213 static void cortex_a76_erratum_1463225_svc_handler(void) 214 { 215 u64 reg, val; 216 217 if (!unlikely(test_thread_flag(TIF_SINGLESTEP))) 218 return; 219 220 if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225))) 221 return; 222 223 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1); 224 reg = read_sysreg(mdscr_el1); 225 val = reg | MDSCR_EL1_SS | MDSCR_EL1_KDE; 226 write_sysreg(val, mdscr_el1); 227 asm volatile("msr daifclr, #8"); 228 isb(); 229 230 /* We will have taken a single-step exception by this point */ 231 232 write_sysreg(reg, mdscr_el1); 233 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0); 234 } 235 236 static __always_inline bool 237 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 238 { 239 if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa)) 240 return false; 241 242 /* 243 * We've taken a dummy step exception from the kernel to ensure 244 * that interrupts are re-enabled on the syscall path. Return back 245 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions 246 * masked so that we can safely restore the mdscr and get on with 247 * handling the syscall. 248 */ 249 regs->pstate |= PSR_D_BIT; 250 return true; 251 } 252 #else /* CONFIG_ARM64_ERRATUM_1463225 */ 253 static void cortex_a76_erratum_1463225_svc_handler(void) { } 254 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 255 { 256 return false; 257 } 258 #endif /* CONFIG_ARM64_ERRATUM_1463225 */ 259 260 /* 261 * As per the ABI exit SME streaming mode and clear the SVE state not 262 * shared with FPSIMD on syscall entry. 263 */ 264 static inline void fpsimd_syscall_enter(void) 265 { 266 /* Ensure PSTATE.SM is clear, but leave PSTATE.ZA as-is. */ 267 if (system_supports_sme()) 268 sme_smstop_sm(); 269 270 /* 271 * The CPU is not in streaming mode. If non-streaming SVE is not 272 * supported, there is no SVE state that needs to be discarded. 273 */ 274 if (!system_supports_sve()) 275 return; 276 277 if (test_thread_flag(TIF_SVE)) 278 sve_flush_live(); 279 280 /* 281 * Any live non-FPSIMD SVE state has been zeroed. Allow 282 * fpsimd_save_user_state() to lazily discard SVE state until either 283 * the live state is unbound or fpsimd_syscall_exit() is called. 284 */ 285 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_FPSIMD); 286 } 287 288 static __always_inline void fpsimd_syscall_exit(void) 289 { 290 if (!system_supports_sve()) 291 return; 292 293 /* 294 * The current task's user FPSIMD/SVE/SME state is now bound to this 295 * CPU. The fpsimd_last_state.to_save value is either: 296 * 297 * - FP_STATE_FPSIMD, if the state has not been reloaded on this CPU 298 * since fpsimd_syscall_enter(). 299 * 300 * - FP_STATE_CURRENT, if the state has been reloaded on this CPU at 301 * any point. 302 * 303 * Reset this to FP_STATE_CURRENT to stop lazy discarding. 304 */ 305 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT); 306 } 307 308 /* 309 * In debug exception context, we explicitly disable preemption despite 310 * having interrupts disabled. 311 * This serves two purposes: it makes it much less likely that we would 312 * accidentally schedule in exception context and it will force a warning 313 * if we somehow manage to schedule by accident. 314 */ 315 static void debug_exception_enter(struct pt_regs *regs) 316 { 317 preempt_disable(); 318 319 /* This code is a bit fragile. Test it. */ 320 RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work"); 321 } 322 NOKPROBE_SYMBOL(debug_exception_enter); 323 324 static void debug_exception_exit(struct pt_regs *regs) 325 { 326 preempt_enable_no_resched(); 327 } 328 NOKPROBE_SYMBOL(debug_exception_exit); 329 330 UNHANDLED(el1t, 64, sync) 331 UNHANDLED(el1t, 64, irq) 332 UNHANDLED(el1t, 64, fiq) 333 UNHANDLED(el1t, 64, error) 334 335 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) 336 { 337 unsigned long far = read_sysreg(far_el1); 338 irqentry_state_t state; 339 340 state = arm64_enter_from_kernel_mode(regs); 341 local_daif_inherit(regs); 342 do_mem_abort(far, esr, regs); 343 arm64_exit_to_kernel_mode(regs, state); 344 } 345 346 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) 347 { 348 unsigned long far = read_sysreg(far_el1); 349 irqentry_state_t state; 350 351 state = arm64_enter_from_kernel_mode(regs); 352 local_daif_inherit(regs); 353 do_sp_pc_abort(far, esr, regs); 354 arm64_exit_to_kernel_mode(regs, state); 355 } 356 357 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) 358 { 359 irqentry_state_t state; 360 361 state = arm64_enter_from_kernel_mode(regs); 362 local_daif_inherit(regs); 363 do_el1_undef(regs, esr); 364 arm64_exit_to_kernel_mode(regs, state); 365 } 366 367 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) 368 { 369 irqentry_state_t state; 370 371 state = arm64_enter_from_kernel_mode(regs); 372 local_daif_inherit(regs); 373 do_el1_bti(regs, esr); 374 arm64_exit_to_kernel_mode(regs, state); 375 } 376 377 static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr) 378 { 379 irqentry_state_t state; 380 381 state = arm64_enter_from_kernel_mode(regs); 382 local_daif_inherit(regs); 383 do_el1_gcs(regs, esr); 384 arm64_exit_to_kernel_mode(regs, state); 385 } 386 387 static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr) 388 { 389 irqentry_state_t state; 390 391 state = arm64_enter_from_kernel_mode(regs); 392 local_daif_inherit(regs); 393 do_el1_mops(regs, esr); 394 arm64_exit_to_kernel_mode(regs, state); 395 } 396 397 static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) 398 { 399 irqentry_state_t state; 400 401 state = arm64_enter_el1_dbg(regs); 402 debug_exception_enter(regs); 403 do_breakpoint(esr, regs); 404 debug_exception_exit(regs); 405 arm64_exit_el1_dbg(regs, state); 406 } 407 408 static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) 409 { 410 irqentry_state_t state; 411 412 state = arm64_enter_el1_dbg(regs); 413 if (!cortex_a76_erratum_1463225_debug_handler(regs)) { 414 debug_exception_enter(regs); 415 /* 416 * After handling a breakpoint, we suspend the breakpoint 417 * and use single-step to move to the next instruction. 418 * If we are stepping a suspended breakpoint there's nothing more to do: 419 * the single-step is complete. 420 */ 421 if (!try_step_suspended_breakpoints(regs)) 422 do_el1_softstep(esr, regs); 423 debug_exception_exit(regs); 424 } 425 arm64_exit_el1_dbg(regs, state); 426 } 427 428 static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) 429 { 430 /* Watchpoints are the only debug exception to write FAR_EL1 */ 431 unsigned long far = read_sysreg(far_el1); 432 irqentry_state_t state; 433 434 state = arm64_enter_el1_dbg(regs); 435 debug_exception_enter(regs); 436 do_watchpoint(far, esr, regs); 437 debug_exception_exit(regs); 438 arm64_exit_el1_dbg(regs, state); 439 } 440 441 static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr) 442 { 443 irqentry_state_t state; 444 445 state = arm64_enter_el1_dbg(regs); 446 debug_exception_enter(regs); 447 do_el1_brk64(esr, regs); 448 debug_exception_exit(regs); 449 arm64_exit_el1_dbg(regs, state); 450 } 451 452 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) 453 { 454 irqentry_state_t state; 455 456 state = arm64_enter_from_kernel_mode(regs); 457 local_daif_inherit(regs); 458 do_el1_fpac(regs, esr); 459 arm64_exit_to_kernel_mode(regs, state); 460 } 461 462 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) 463 { 464 unsigned long esr = read_sysreg(esr_el1); 465 466 switch (ESR_ELx_EC(esr)) { 467 case ESR_ELx_EC_DABT_CUR: 468 case ESR_ELx_EC_IABT_CUR: 469 el1_abort(regs, esr); 470 break; 471 /* 472 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a 473 * recursive exception when trying to push the initial pt_regs. 474 */ 475 case ESR_ELx_EC_PC_ALIGN: 476 el1_pc(regs, esr); 477 break; 478 case ESR_ELx_EC_SYS64: 479 case ESR_ELx_EC_UNKNOWN: 480 el1_undef(regs, esr); 481 break; 482 case ESR_ELx_EC_BTI: 483 el1_bti(regs, esr); 484 break; 485 case ESR_ELx_EC_GCS: 486 el1_gcs(regs, esr); 487 break; 488 case ESR_ELx_EC_MOPS: 489 el1_mops(regs, esr); 490 break; 491 case ESR_ELx_EC_BREAKPT_CUR: 492 el1_breakpt(regs, esr); 493 break; 494 case ESR_ELx_EC_SOFTSTP_CUR: 495 el1_softstp(regs, esr); 496 break; 497 case ESR_ELx_EC_WATCHPT_CUR: 498 el1_watchpt(regs, esr); 499 break; 500 case ESR_ELx_EC_BRK64: 501 el1_brk64(regs, esr); 502 break; 503 case ESR_ELx_EC_FPAC: 504 el1_fpac(regs, esr); 505 break; 506 default: 507 __panic_unhandled(regs, "64-bit el1h sync", esr); 508 } 509 } 510 511 static __always_inline void __el1_pnmi(struct pt_regs *regs, 512 void (*handler)(struct pt_regs *)) 513 { 514 irqentry_state_t state; 515 516 state = irqentry_nmi_enter(regs); 517 do_interrupt_handler(regs, handler); 518 local_daif_mask(); 519 irqentry_nmi_exit(regs, state); 520 } 521 522 static __always_inline void __el1_irq(struct pt_regs *regs, 523 void (*handler)(struct pt_regs *)) 524 { 525 irqentry_state_t state; 526 527 state = arm64_enter_from_kernel_mode(regs); 528 529 irq_enter_rcu(); 530 do_interrupt_handler(regs, handler); 531 irq_exit_rcu(); 532 533 arm64_exit_to_kernel_mode_preempt(regs, state); 534 } 535 static void noinstr el1_interrupt(struct pt_regs *regs, 536 void (*handler)(struct pt_regs *)) 537 { 538 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 539 540 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs)) 541 __el1_pnmi(regs, handler); 542 else 543 __el1_irq(regs, handler); 544 } 545 546 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) 547 { 548 el1_interrupt(regs, handle_arch_irq); 549 } 550 551 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) 552 { 553 el1_interrupt(regs, handle_arch_fiq); 554 } 555 556 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) 557 { 558 unsigned long esr = read_sysreg(esr_el1); 559 irqentry_state_t state; 560 561 local_daif_restore(DAIF_ERRCTX); 562 state = irqentry_nmi_enter(regs); 563 do_serror(regs, esr); 564 local_daif_mask(); 565 irqentry_nmi_exit(regs, state); 566 } 567 568 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) 569 { 570 unsigned long far = read_sysreg(far_el1); 571 572 arm64_enter_from_user_mode(regs); 573 local_daif_restore(DAIF_PROCCTX); 574 do_mem_abort(far, esr, regs); 575 arm64_exit_to_user_mode(regs); 576 } 577 578 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr) 579 { 580 unsigned long far = read_sysreg(far_el1); 581 582 /* 583 * We've taken an instruction abort from userspace and not yet 584 * re-enabled IRQs. If the address is a kernel address, apply 585 * BP hardening prior to enabling IRQs and pre-emption. 586 */ 587 if (!is_ttbr0_addr(far)) 588 arm64_apply_bp_hardening(); 589 590 arm64_enter_from_user_mode(regs); 591 local_daif_restore(DAIF_PROCCTX); 592 do_mem_abort(far, esr, regs); 593 arm64_exit_to_user_mode(regs); 594 } 595 596 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) 597 { 598 arm64_enter_from_user_mode(regs); 599 local_daif_restore(DAIF_PROCCTX); 600 do_fpsimd_acc(esr, regs); 601 arm64_exit_to_user_mode(regs); 602 } 603 604 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr) 605 { 606 arm64_enter_from_user_mode(regs); 607 local_daif_restore(DAIF_PROCCTX); 608 do_sve_acc(esr, regs); 609 arm64_exit_to_user_mode(regs); 610 } 611 612 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr) 613 { 614 arm64_enter_from_user_mode(regs); 615 local_daif_restore(DAIF_PROCCTX); 616 do_sme_acc(esr, regs); 617 arm64_exit_to_user_mode(regs); 618 } 619 620 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) 621 { 622 arm64_enter_from_user_mode(regs); 623 local_daif_restore(DAIF_PROCCTX); 624 do_fpsimd_exc(esr, regs); 625 arm64_exit_to_user_mode(regs); 626 } 627 628 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr) 629 { 630 arm64_enter_from_user_mode(regs); 631 local_daif_restore(DAIF_PROCCTX); 632 do_el0_sys(esr, regs); 633 arm64_exit_to_user_mode(regs); 634 } 635 636 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr) 637 { 638 unsigned long far = read_sysreg(far_el1); 639 640 if (!is_ttbr0_addr(instruction_pointer(regs))) 641 arm64_apply_bp_hardening(); 642 643 arm64_enter_from_user_mode(regs); 644 local_daif_restore(DAIF_PROCCTX); 645 do_sp_pc_abort(far, esr, regs); 646 arm64_exit_to_user_mode(regs); 647 } 648 649 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr) 650 { 651 arm64_enter_from_user_mode(regs); 652 local_daif_restore(DAIF_PROCCTX); 653 do_sp_pc_abort(regs->sp, esr, regs); 654 arm64_exit_to_user_mode(regs); 655 } 656 657 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr) 658 { 659 arm64_enter_from_user_mode(regs); 660 local_daif_restore(DAIF_PROCCTX); 661 do_el0_undef(regs, esr); 662 arm64_exit_to_user_mode(regs); 663 } 664 665 static void noinstr el0_bti(struct pt_regs *regs) 666 { 667 arm64_enter_from_user_mode(regs); 668 local_daif_restore(DAIF_PROCCTX); 669 do_el0_bti(regs); 670 arm64_exit_to_user_mode(regs); 671 } 672 673 static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr) 674 { 675 arm64_enter_from_user_mode(regs); 676 local_daif_restore(DAIF_PROCCTX); 677 do_el0_mops(regs, esr); 678 arm64_exit_to_user_mode(regs); 679 } 680 681 static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr) 682 { 683 arm64_enter_from_user_mode(regs); 684 local_daif_restore(DAIF_PROCCTX); 685 do_el0_gcs(regs, esr); 686 arm64_exit_to_user_mode(regs); 687 } 688 689 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr) 690 { 691 arm64_enter_from_user_mode(regs); 692 local_daif_restore(DAIF_PROCCTX); 693 bad_el0_sync(regs, 0, esr); 694 arm64_exit_to_user_mode(regs); 695 } 696 697 static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr) 698 { 699 if (!is_ttbr0_addr(regs->pc)) 700 arm64_apply_bp_hardening(); 701 702 arm64_enter_from_user_mode(regs); 703 debug_exception_enter(regs); 704 do_breakpoint(esr, regs); 705 debug_exception_exit(regs); 706 local_daif_restore(DAIF_PROCCTX); 707 arm64_exit_to_user_mode(regs); 708 } 709 710 static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr) 711 { 712 bool step_done; 713 714 if (!is_ttbr0_addr(regs->pc)) 715 arm64_apply_bp_hardening(); 716 717 arm64_enter_from_user_mode(regs); 718 /* 719 * After handling a breakpoint, we suspend the breakpoint 720 * and use single-step to move to the next instruction. 721 * If we are stepping a suspended breakpoint there's nothing more to do: 722 * the single-step is complete. 723 */ 724 step_done = try_step_suspended_breakpoints(regs); 725 local_daif_restore(DAIF_PROCCTX); 726 if (!step_done) 727 do_el0_softstep(esr, regs); 728 arm64_exit_to_user_mode(regs); 729 } 730 731 static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr) 732 { 733 /* Watchpoints are the only debug exception to write FAR_EL1 */ 734 unsigned long far = read_sysreg(far_el1); 735 736 arm64_enter_from_user_mode(regs); 737 debug_exception_enter(regs); 738 do_watchpoint(far, esr, regs); 739 debug_exception_exit(regs); 740 local_daif_restore(DAIF_PROCCTX); 741 arm64_exit_to_user_mode(regs); 742 } 743 744 static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr) 745 { 746 arm64_enter_from_user_mode(regs); 747 local_daif_restore(DAIF_PROCCTX); 748 do_el0_brk64(esr, regs); 749 arm64_exit_to_user_mode(regs); 750 } 751 752 static void noinstr el0_svc(struct pt_regs *regs) 753 { 754 arm64_syscall_enter_from_user_mode(regs); 755 cortex_a76_erratum_1463225_svc_handler(); 756 fpsimd_syscall_enter(); 757 local_daif_restore(DAIF_PROCCTX); 758 do_el0_svc(regs); 759 arm64_syscall_exit_to_user_mode(regs); 760 fpsimd_syscall_exit(); 761 } 762 763 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) 764 { 765 arm64_enter_from_user_mode(regs); 766 local_daif_restore(DAIF_PROCCTX); 767 do_el0_fpac(regs, esr); 768 arm64_exit_to_user_mode(regs); 769 } 770 771 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) 772 { 773 unsigned long esr = read_sysreg(esr_el1); 774 775 switch (ESR_ELx_EC(esr)) { 776 case ESR_ELx_EC_SVC64: 777 el0_svc(regs); 778 break; 779 case ESR_ELx_EC_DABT_LOW: 780 el0_da(regs, esr); 781 break; 782 case ESR_ELx_EC_IABT_LOW: 783 el0_ia(regs, esr); 784 break; 785 case ESR_ELx_EC_FP_ASIMD: 786 el0_fpsimd_acc(regs, esr); 787 break; 788 case ESR_ELx_EC_SVE: 789 el0_sve_acc(regs, esr); 790 break; 791 case ESR_ELx_EC_SME: 792 el0_sme_acc(regs, esr); 793 break; 794 case ESR_ELx_EC_FP_EXC64: 795 el0_fpsimd_exc(regs, esr); 796 break; 797 case ESR_ELx_EC_SYS64: 798 case ESR_ELx_EC_WFx: 799 el0_sys(regs, esr); 800 break; 801 case ESR_ELx_EC_SP_ALIGN: 802 el0_sp(regs, esr); 803 break; 804 case ESR_ELx_EC_PC_ALIGN: 805 el0_pc(regs, esr); 806 break; 807 case ESR_ELx_EC_UNKNOWN: 808 el0_undef(regs, esr); 809 break; 810 case ESR_ELx_EC_BTI: 811 el0_bti(regs); 812 break; 813 case ESR_ELx_EC_MOPS: 814 el0_mops(regs, esr); 815 break; 816 case ESR_ELx_EC_GCS: 817 el0_gcs(regs, esr); 818 break; 819 case ESR_ELx_EC_BREAKPT_LOW: 820 el0_breakpt(regs, esr); 821 break; 822 case ESR_ELx_EC_SOFTSTP_LOW: 823 el0_softstp(regs, esr); 824 break; 825 case ESR_ELx_EC_WATCHPT_LOW: 826 el0_watchpt(regs, esr); 827 break; 828 case ESR_ELx_EC_BRK64: 829 el0_brk64(regs, esr); 830 break; 831 case ESR_ELx_EC_FPAC: 832 el0_fpac(regs, esr); 833 break; 834 default: 835 el0_inv(regs, esr); 836 } 837 } 838 839 static void noinstr el0_interrupt(struct pt_regs *regs, 840 void (*handler)(struct pt_regs *)) 841 { 842 arm64_enter_from_user_mode(regs); 843 844 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 845 846 if (regs->pc & BIT(55)) 847 arm64_apply_bp_hardening(); 848 849 irq_enter_rcu(); 850 do_interrupt_handler(regs, handler); 851 irq_exit_rcu(); 852 853 arm64_exit_to_user_mode(regs); 854 } 855 856 static void noinstr __el0_irq_handler_common(struct pt_regs *regs) 857 { 858 el0_interrupt(regs, handle_arch_irq); 859 } 860 861 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs) 862 { 863 __el0_irq_handler_common(regs); 864 } 865 866 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs) 867 { 868 el0_interrupt(regs, handle_arch_fiq); 869 } 870 871 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs) 872 { 873 __el0_fiq_handler_common(regs); 874 } 875 876 static void noinstr __el0_error_handler_common(struct pt_regs *regs) 877 { 878 unsigned long esr = read_sysreg(esr_el1); 879 irqentry_state_t state; 880 881 arm64_enter_from_user_mode(regs); 882 local_daif_restore(DAIF_ERRCTX); 883 state = irqentry_nmi_enter(regs); 884 do_serror(regs, esr); 885 irqentry_nmi_exit(regs, state); 886 local_daif_restore(DAIF_PROCCTX); 887 arm64_exit_to_user_mode(regs); 888 } 889 890 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) 891 { 892 __el0_error_handler_common(regs); 893 } 894 895 #ifdef CONFIG_COMPAT 896 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) 897 { 898 arm64_enter_from_user_mode(regs); 899 local_daif_restore(DAIF_PROCCTX); 900 do_el0_cp15(esr, regs); 901 arm64_exit_to_user_mode(regs); 902 } 903 904 static void noinstr el0_svc_compat(struct pt_regs *regs) 905 { 906 arm64_syscall_enter_from_user_mode(regs); 907 cortex_a76_erratum_1463225_svc_handler(); 908 local_daif_restore(DAIF_PROCCTX); 909 do_el0_svc_compat(regs); 910 arm64_syscall_exit_to_user_mode(regs); 911 } 912 913 static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr) 914 { 915 arm64_enter_from_user_mode(regs); 916 local_daif_restore(DAIF_PROCCTX); 917 do_bkpt32(esr, regs); 918 arm64_exit_to_user_mode(regs); 919 } 920 921 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) 922 { 923 unsigned long esr = read_sysreg(esr_el1); 924 925 switch (ESR_ELx_EC(esr)) { 926 case ESR_ELx_EC_SVC32: 927 el0_svc_compat(regs); 928 break; 929 case ESR_ELx_EC_DABT_LOW: 930 el0_da(regs, esr); 931 break; 932 case ESR_ELx_EC_IABT_LOW: 933 el0_ia(regs, esr); 934 break; 935 case ESR_ELx_EC_FP_ASIMD: 936 el0_fpsimd_acc(regs, esr); 937 break; 938 case ESR_ELx_EC_FP_EXC32: 939 el0_fpsimd_exc(regs, esr); 940 break; 941 case ESR_ELx_EC_PC_ALIGN: 942 el0_pc(regs, esr); 943 break; 944 case ESR_ELx_EC_UNKNOWN: 945 case ESR_ELx_EC_CP14_MR: 946 case ESR_ELx_EC_CP14_LS: 947 case ESR_ELx_EC_CP14_64: 948 el0_undef(regs, esr); 949 break; 950 case ESR_ELx_EC_CP15_32: 951 case ESR_ELx_EC_CP15_64: 952 el0_cp15(regs, esr); 953 break; 954 case ESR_ELx_EC_BREAKPT_LOW: 955 el0_breakpt(regs, esr); 956 break; 957 case ESR_ELx_EC_SOFTSTP_LOW: 958 el0_softstp(regs, esr); 959 break; 960 case ESR_ELx_EC_WATCHPT_LOW: 961 el0_watchpt(regs, esr); 962 break; 963 case ESR_ELx_EC_BKPT32: 964 el0_bkpt32(regs, esr); 965 break; 966 default: 967 el0_inv(regs, esr); 968 } 969 } 970 971 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) 972 { 973 __el0_irq_handler_common(regs); 974 } 975 976 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) 977 { 978 __el0_fiq_handler_common(regs); 979 } 980 981 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) 982 { 983 __el0_error_handler_common(regs); 984 } 985 #else /* CONFIG_COMPAT */ 986 UNHANDLED(el0t, 32, sync) 987 UNHANDLED(el0t, 32, irq) 988 UNHANDLED(el0t, 32, fiq) 989 UNHANDLED(el0t, 32, error) 990 #endif /* CONFIG_COMPAT */ 991 992 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) 993 { 994 unsigned long esr = read_sysreg(esr_el1); 995 unsigned long far = read_sysreg(far_el1); 996 997 irqentry_nmi_enter(regs); 998 panic_bad_stack(regs, esr, far); 999 } 1000 1001 #ifdef CONFIG_ARM_SDE_INTERFACE 1002 asmlinkage noinstr unsigned long 1003 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) 1004 { 1005 irqentry_state_t state; 1006 unsigned long ret; 1007 1008 /* 1009 * We didn't take an exception to get here, so the HW hasn't 1010 * set/cleared bits in PSTATE that we may rely on. 1011 * 1012 * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to 1013 * whether PSTATE bits are inherited unchanged or generated from 1014 * scratch, and the TF-A implementation always clears PAN and always 1015 * clears UAO. There are no other known implementations. 1016 * 1017 * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how 1018 * PSTATE is modified upon architectural exceptions, and so PAN is 1019 * either inherited or set per SCTLR_ELx.SPAN, and UAO is always 1020 * cleared. 1021 * 1022 * We must explicitly reset PAN to the expected state, including 1023 * clearing it when the host isn't using it, in case a VM had it set. 1024 */ 1025 if (system_uses_hw_pan()) 1026 set_pstate_pan(1); 1027 else if (cpu_has_pan()) 1028 set_pstate_pan(0); 1029 1030 state = irqentry_nmi_enter(regs); 1031 ret = do_sdei_event(regs, arg); 1032 irqentry_nmi_exit(regs, state); 1033 1034 return ret; 1035 } 1036 #endif /* CONFIG_ARM_SDE_INTERFACE */ 1037