1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Exception handling code 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 */ 7 8 #include <linux/context_tracking.h> 9 #include <linux/irq-entry-common.h> 10 #include <linux/kasan.h> 11 #include <linux/linkage.h> 12 #include <linux/livepatch.h> 13 #include <linux/lockdep.h> 14 #include <linux/ptrace.h> 15 #include <linux/resume_user_mode.h> 16 #include <linux/sched.h> 17 #include <linux/sched/debug.h> 18 #include <linux/thread_info.h> 19 20 #include <asm/cpufeature.h> 21 #include <asm/daifflags.h> 22 #include <asm/esr.h> 23 #include <asm/exception.h> 24 #include <asm/fpsimd.h> 25 #include <asm/irq_regs.h> 26 #include <asm/kprobes.h> 27 #include <asm/mmu.h> 28 #include <asm/processor.h> 29 #include <asm/sdei.h> 30 #include <asm/stacktrace.h> 31 #include <asm/sysreg.h> 32 #include <asm/system_misc.h> 33 34 /* 35 * Handle IRQ/context state management when entering from kernel mode. 36 * Before this function is called it is not safe to call regular kernel code, 37 * instrumentable code, or any code which may trigger an exception. 38 */ 39 static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *regs) 40 { 41 irqentry_state_t state; 42 43 state = irqentry_enter_from_kernel_mode(regs); 44 mte_check_tfsr_entry(); 45 mte_disable_tco_entry(current); 46 47 return state; 48 } 49 50 /* 51 * Handle IRQ/context state management when exiting to kernel mode. 52 * After this function returns it is not safe to call regular kernel code, 53 * instrumentable code, or any code which may trigger an exception. 54 */ 55 static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, 56 irqentry_state_t state) 57 { 58 local_irq_disable(); 59 irqentry_exit_to_kernel_mode_preempt(regs, state); 60 local_daif_mask(); 61 mte_check_tfsr_exit(); 62 irqentry_exit_to_kernel_mode_after_preempt(regs, state); 63 } 64 65 static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs) 66 { 67 enter_from_user_mode(regs); 68 mte_disable_tco_entry(current); 69 sme_enter_from_user_mode(); 70 } 71 72 /* 73 * Handle IRQ/context state management when entering from user mode. 74 * Before this function is called it is not safe to call regular kernel code, 75 * instrumentable code, or any code which may trigger an exception. 76 */ 77 static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs) 78 { 79 enter_from_user_mode(regs); 80 rseq_note_user_irq_entry(); 81 mte_disable_tco_entry(current); 82 sme_enter_from_user_mode(); 83 } 84 85 static __always_inline void arm64_syscall_exit_to_user_mode(struct pt_regs *regs) 86 { 87 local_irq_disable(); 88 syscall_exit_to_user_mode_prepare(regs); 89 local_daif_mask(); 90 sme_exit_to_user_mode(); 91 mte_check_tfsr_exit(); 92 exit_to_user_mode(); 93 } 94 95 /* 96 * Handle IRQ/context state management when exiting to user mode. 97 * After this function returns it is not safe to call regular kernel code, 98 * instrumentable code, or any code which may trigger an exception. 99 */ 100 static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs) 101 { 102 local_irq_disable(); 103 irqentry_exit_to_user_mode_prepare(regs); 104 local_daif_mask(); 105 sme_exit_to_user_mode(); 106 mte_check_tfsr_exit(); 107 exit_to_user_mode(); 108 } 109 110 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) 111 { 112 arm64_syscall_exit_to_user_mode(regs); 113 } 114 115 /* 116 * Handle IRQ/context state management when entering a debug exception from 117 * kernel mode. Before this function is called it is not safe to call regular 118 * kernel code, instrumentable code, or any code which may trigger an exception. 119 */ 120 static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs) 121 { 122 irqentry_state_t state; 123 124 state.lockdep = lockdep_hardirqs_enabled(); 125 126 lockdep_hardirqs_off(CALLER_ADDR0); 127 ct_nmi_enter(); 128 129 trace_hardirqs_off_finish(); 130 131 return state; 132 } 133 134 /* 135 * Handle IRQ/context state management when exiting a debug exception from 136 * kernel mode. After this function returns it is not safe to call regular 137 * kernel code, instrumentable code, or any code which may trigger an exception. 138 */ 139 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs, 140 irqentry_state_t state) 141 { 142 if (state.lockdep) { 143 trace_hardirqs_on_prepare(); 144 lockdep_hardirqs_on_prepare(); 145 } 146 147 ct_nmi_exit(); 148 if (state.lockdep) 149 lockdep_hardirqs_on(CALLER_ADDR0); 150 } 151 152 static void do_interrupt_handler(struct pt_regs *regs, 153 void (*handler)(struct pt_regs *)) 154 { 155 struct pt_regs *old_regs = set_irq_regs(regs); 156 157 if (on_thread_stack()) 158 call_on_irq_stack(regs, handler); 159 else 160 handler(regs); 161 162 set_irq_regs(old_regs); 163 } 164 165 extern void (*handle_arch_irq)(struct pt_regs *); 166 extern void (*handle_arch_fiq)(struct pt_regs *); 167 168 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, 169 unsigned long esr) 170 { 171 irqentry_nmi_enter(regs); 172 173 console_verbose(); 174 175 pr_crit("Unhandled %s exception on CPU%d, ESR 0x%016lx -- %s\n", 176 vector, smp_processor_id(), esr, 177 esr_get_class_string(esr)); 178 179 __show_regs(regs); 180 panic("Unhandled exception"); 181 } 182 183 #define UNHANDLED(el, regsize, vector) \ 184 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \ 185 { \ 186 const char *desc = #regsize "-bit " #el " " #vector; \ 187 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \ 188 } 189 190 #ifdef CONFIG_ARM64_ERRATUM_1463225 191 static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa); 192 193 static void cortex_a76_erratum_1463225_svc_handler(void) 194 { 195 u64 reg, val; 196 197 if (!unlikely(test_thread_flag(TIF_SINGLESTEP))) 198 return; 199 200 if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225))) 201 return; 202 203 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1); 204 reg = read_sysreg(mdscr_el1); 205 val = reg | MDSCR_EL1_SS | MDSCR_EL1_KDE; 206 write_sysreg(val, mdscr_el1); 207 asm volatile("msr daifclr, #8"); 208 isb(); 209 210 /* We will have taken a single-step exception by this point */ 211 212 write_sysreg(reg, mdscr_el1); 213 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0); 214 } 215 216 static __always_inline bool 217 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 218 { 219 if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa)) 220 return false; 221 222 /* 223 * We've taken a dummy step exception from the kernel to ensure 224 * that interrupts are re-enabled on the syscall path. Return back 225 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions 226 * masked so that we can safely restore the mdscr and get on with 227 * handling the syscall. 228 */ 229 regs->pstate |= PSR_D_BIT; 230 return true; 231 } 232 #else /* CONFIG_ARM64_ERRATUM_1463225 */ 233 static void cortex_a76_erratum_1463225_svc_handler(void) { } 234 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 235 { 236 return false; 237 } 238 #endif /* CONFIG_ARM64_ERRATUM_1463225 */ 239 240 /* 241 * As per the ABI exit SME streaming mode and clear the SVE state not 242 * shared with FPSIMD on syscall entry. 243 */ 244 static inline void fpsimd_syscall_enter(void) 245 { 246 /* Ensure PSTATE.SM is clear, but leave PSTATE.ZA as-is. */ 247 if (system_supports_sme()) 248 sme_smstop_sm(); 249 250 /* 251 * The CPU is not in streaming mode. If non-streaming SVE is not 252 * supported, there is no SVE state that needs to be discarded. 253 */ 254 if (!system_supports_sve()) 255 return; 256 257 if (test_thread_flag(TIF_SVE)) 258 sve_flush_live(); 259 260 /* 261 * Any live non-FPSIMD SVE state has been zeroed. Allow 262 * fpsimd_save_user_state() to lazily discard SVE state until either 263 * the live state is unbound or fpsimd_syscall_exit() is called. 264 */ 265 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_FPSIMD); 266 } 267 268 static __always_inline void fpsimd_syscall_exit(void) 269 { 270 if (!system_supports_sve()) 271 return; 272 273 /* 274 * The current task's user FPSIMD/SVE/SME state is now bound to this 275 * CPU. The fpsimd_last_state.to_save value is either: 276 * 277 * - FP_STATE_FPSIMD, if the state has not been reloaded on this CPU 278 * since fpsimd_syscall_enter(). 279 * 280 * - FP_STATE_CURRENT, if the state has been reloaded on this CPU at 281 * any point. 282 * 283 * Reset this to FP_STATE_CURRENT to stop lazy discarding. 284 */ 285 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT); 286 } 287 288 /* 289 * In debug exception context, we explicitly disable preemption despite 290 * having interrupts disabled. 291 * This serves two purposes: it makes it much less likely that we would 292 * accidentally schedule in exception context and it will force a warning 293 * if we somehow manage to schedule by accident. 294 */ 295 static void debug_exception_enter(struct pt_regs *regs) 296 { 297 preempt_disable(); 298 299 /* This code is a bit fragile. Test it. */ 300 RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work"); 301 } 302 NOKPROBE_SYMBOL(debug_exception_enter); 303 304 static void debug_exception_exit(struct pt_regs *regs) 305 { 306 preempt_enable_no_resched(); 307 } 308 NOKPROBE_SYMBOL(debug_exception_exit); 309 310 UNHANDLED(el1t, 64, sync) 311 UNHANDLED(el1t, 64, irq) 312 UNHANDLED(el1t, 64, fiq) 313 UNHANDLED(el1t, 64, error) 314 315 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) 316 { 317 unsigned long far = read_sysreg(far_el1); 318 irqentry_state_t state; 319 320 state = arm64_enter_from_kernel_mode(regs); 321 local_daif_inherit(regs); 322 do_mem_abort(far, esr, regs); 323 arm64_exit_to_kernel_mode(regs, state); 324 } 325 326 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) 327 { 328 unsigned long far = read_sysreg(far_el1); 329 irqentry_state_t state; 330 331 state = arm64_enter_from_kernel_mode(regs); 332 local_daif_inherit(regs); 333 do_sp_pc_abort(far, esr, regs); 334 arm64_exit_to_kernel_mode(regs, state); 335 } 336 337 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) 338 { 339 irqentry_state_t state; 340 341 state = arm64_enter_from_kernel_mode(regs); 342 local_daif_inherit(regs); 343 do_el1_undef(regs, esr); 344 arm64_exit_to_kernel_mode(regs, state); 345 } 346 347 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) 348 { 349 irqentry_state_t state; 350 351 state = arm64_enter_from_kernel_mode(regs); 352 local_daif_inherit(regs); 353 do_el1_bti(regs, esr); 354 arm64_exit_to_kernel_mode(regs, state); 355 } 356 357 static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr) 358 { 359 irqentry_state_t state; 360 361 state = arm64_enter_from_kernel_mode(regs); 362 local_daif_inherit(regs); 363 do_el1_gcs(regs, esr); 364 arm64_exit_to_kernel_mode(regs, state); 365 } 366 367 static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr) 368 { 369 irqentry_state_t state; 370 371 state = arm64_enter_from_kernel_mode(regs); 372 local_daif_inherit(regs); 373 do_el1_mops(regs, esr); 374 arm64_exit_to_kernel_mode(regs, state); 375 } 376 377 static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) 378 { 379 irqentry_state_t state; 380 381 state = arm64_enter_el1_dbg(regs); 382 debug_exception_enter(regs); 383 do_breakpoint(esr, regs); 384 debug_exception_exit(regs); 385 arm64_exit_el1_dbg(regs, state); 386 } 387 388 static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) 389 { 390 irqentry_state_t state; 391 392 state = arm64_enter_el1_dbg(regs); 393 if (!cortex_a76_erratum_1463225_debug_handler(regs)) { 394 debug_exception_enter(regs); 395 /* 396 * After handling a breakpoint, we suspend the breakpoint 397 * and use single-step to move to the next instruction. 398 * If we are stepping a suspended breakpoint there's nothing more to do: 399 * the single-step is complete. 400 */ 401 if (!try_step_suspended_breakpoints(regs)) 402 do_el1_softstep(esr, regs); 403 debug_exception_exit(regs); 404 } 405 arm64_exit_el1_dbg(regs, state); 406 } 407 408 static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) 409 { 410 /* Watchpoints are the only debug exception to write FAR_EL1 */ 411 unsigned long far = read_sysreg(far_el1); 412 irqentry_state_t state; 413 414 state = arm64_enter_el1_dbg(regs); 415 debug_exception_enter(regs); 416 do_watchpoint(far, esr, regs); 417 debug_exception_exit(regs); 418 arm64_exit_el1_dbg(regs, state); 419 } 420 421 static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr) 422 { 423 irqentry_state_t state; 424 425 state = arm64_enter_el1_dbg(regs); 426 debug_exception_enter(regs); 427 do_el1_brk64(esr, regs); 428 debug_exception_exit(regs); 429 arm64_exit_el1_dbg(regs, state); 430 } 431 432 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) 433 { 434 irqentry_state_t state; 435 436 state = arm64_enter_from_kernel_mode(regs); 437 local_daif_inherit(regs); 438 do_el1_fpac(regs, esr); 439 arm64_exit_to_kernel_mode(regs, state); 440 } 441 442 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) 443 { 444 unsigned long esr = read_sysreg(esr_el1); 445 446 switch (ESR_ELx_EC(esr)) { 447 case ESR_ELx_EC_DABT_CUR: 448 case ESR_ELx_EC_IABT_CUR: 449 el1_abort(regs, esr); 450 break; 451 /* 452 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a 453 * recursive exception when trying to push the initial pt_regs. 454 */ 455 case ESR_ELx_EC_PC_ALIGN: 456 el1_pc(regs, esr); 457 break; 458 case ESR_ELx_EC_SYS64: 459 case ESR_ELx_EC_UNKNOWN: 460 el1_undef(regs, esr); 461 break; 462 case ESR_ELx_EC_BTI: 463 el1_bti(regs, esr); 464 break; 465 case ESR_ELx_EC_GCS: 466 el1_gcs(regs, esr); 467 break; 468 case ESR_ELx_EC_MOPS: 469 el1_mops(regs, esr); 470 break; 471 case ESR_ELx_EC_BREAKPT_CUR: 472 el1_breakpt(regs, esr); 473 break; 474 case ESR_ELx_EC_SOFTSTP_CUR: 475 el1_softstp(regs, esr); 476 break; 477 case ESR_ELx_EC_WATCHPT_CUR: 478 el1_watchpt(regs, esr); 479 break; 480 case ESR_ELx_EC_BRK64: 481 el1_brk64(regs, esr); 482 break; 483 case ESR_ELx_EC_FPAC: 484 el1_fpac(regs, esr); 485 break; 486 default: 487 __panic_unhandled(regs, "64-bit el1h sync", esr); 488 } 489 } 490 491 static __always_inline void __el1_pnmi(struct pt_regs *regs, 492 void (*handler)(struct pt_regs *)) 493 { 494 irqentry_state_t state; 495 496 state = irqentry_nmi_enter(regs); 497 do_interrupt_handler(regs, handler); 498 irqentry_nmi_exit(regs, state); 499 } 500 501 static __always_inline void __el1_irq(struct pt_regs *regs, 502 void (*handler)(struct pt_regs *)) 503 { 504 irqentry_state_t state; 505 506 state = arm64_enter_from_kernel_mode(regs); 507 508 irq_enter_rcu(); 509 do_interrupt_handler(regs, handler); 510 irq_exit_rcu(); 511 512 arm64_exit_to_kernel_mode(regs, state); 513 } 514 static void noinstr el1_interrupt(struct pt_regs *regs, 515 void (*handler)(struct pt_regs *)) 516 { 517 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 518 519 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs)) 520 __el1_pnmi(regs, handler); 521 else 522 __el1_irq(regs, handler); 523 } 524 525 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) 526 { 527 el1_interrupt(regs, handle_arch_irq); 528 } 529 530 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) 531 { 532 el1_interrupt(regs, handle_arch_fiq); 533 } 534 535 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) 536 { 537 unsigned long esr = read_sysreg(esr_el1); 538 irqentry_state_t state; 539 540 local_daif_restore(DAIF_ERRCTX); 541 state = irqentry_nmi_enter(regs); 542 do_serror(regs, esr); 543 irqentry_nmi_exit(regs, state); 544 } 545 546 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) 547 { 548 unsigned long far = read_sysreg(far_el1); 549 550 arm64_enter_from_user_mode(regs); 551 local_daif_restore(DAIF_PROCCTX); 552 do_mem_abort(far, esr, regs); 553 arm64_exit_to_user_mode(regs); 554 } 555 556 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr) 557 { 558 unsigned long far = read_sysreg(far_el1); 559 560 /* 561 * We've taken an instruction abort from userspace and not yet 562 * re-enabled IRQs. If the address is a kernel address, apply 563 * BP hardening prior to enabling IRQs and pre-emption. 564 */ 565 if (!is_ttbr0_addr(far)) 566 arm64_apply_bp_hardening(); 567 568 arm64_enter_from_user_mode(regs); 569 local_daif_restore(DAIF_PROCCTX); 570 do_mem_abort(far, esr, regs); 571 arm64_exit_to_user_mode(regs); 572 } 573 574 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) 575 { 576 arm64_enter_from_user_mode(regs); 577 local_daif_restore(DAIF_PROCCTX); 578 do_fpsimd_acc(esr, regs); 579 arm64_exit_to_user_mode(regs); 580 } 581 582 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr) 583 { 584 arm64_enter_from_user_mode(regs); 585 local_daif_restore(DAIF_PROCCTX); 586 do_sve_acc(esr, regs); 587 arm64_exit_to_user_mode(regs); 588 } 589 590 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr) 591 { 592 arm64_enter_from_user_mode(regs); 593 local_daif_restore(DAIF_PROCCTX); 594 do_sme_acc(esr, regs); 595 arm64_exit_to_user_mode(regs); 596 } 597 598 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) 599 { 600 arm64_enter_from_user_mode(regs); 601 local_daif_restore(DAIF_PROCCTX); 602 do_fpsimd_exc(esr, regs); 603 arm64_exit_to_user_mode(regs); 604 } 605 606 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr) 607 { 608 arm64_enter_from_user_mode(regs); 609 local_daif_restore(DAIF_PROCCTX); 610 do_el0_sys(esr, regs); 611 arm64_exit_to_user_mode(regs); 612 } 613 614 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr) 615 { 616 unsigned long far = read_sysreg(far_el1); 617 618 if (!is_ttbr0_addr(instruction_pointer(regs))) 619 arm64_apply_bp_hardening(); 620 621 arm64_enter_from_user_mode(regs); 622 local_daif_restore(DAIF_PROCCTX); 623 do_sp_pc_abort(far, esr, regs); 624 arm64_exit_to_user_mode(regs); 625 } 626 627 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr) 628 { 629 arm64_enter_from_user_mode(regs); 630 local_daif_restore(DAIF_PROCCTX); 631 do_sp_pc_abort(regs->sp, esr, regs); 632 arm64_exit_to_user_mode(regs); 633 } 634 635 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr) 636 { 637 arm64_enter_from_user_mode(regs); 638 local_daif_restore(DAIF_PROCCTX); 639 do_el0_undef(regs, esr); 640 arm64_exit_to_user_mode(regs); 641 } 642 643 static void noinstr el0_bti(struct pt_regs *regs) 644 { 645 arm64_enter_from_user_mode(regs); 646 local_daif_restore(DAIF_PROCCTX); 647 do_el0_bti(regs); 648 arm64_exit_to_user_mode(regs); 649 } 650 651 static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr) 652 { 653 arm64_enter_from_user_mode(regs); 654 local_daif_restore(DAIF_PROCCTX); 655 do_el0_mops(regs, esr); 656 arm64_exit_to_user_mode(regs); 657 } 658 659 static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr) 660 { 661 arm64_enter_from_user_mode(regs); 662 local_daif_restore(DAIF_PROCCTX); 663 do_el0_gcs(regs, esr); 664 arm64_exit_to_user_mode(regs); 665 } 666 667 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr) 668 { 669 arm64_enter_from_user_mode(regs); 670 local_daif_restore(DAIF_PROCCTX); 671 bad_el0_sync(regs, 0, esr); 672 arm64_exit_to_user_mode(regs); 673 } 674 675 static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr) 676 { 677 if (!is_ttbr0_addr(regs->pc)) 678 arm64_apply_bp_hardening(); 679 680 arm64_enter_from_user_mode(regs); 681 debug_exception_enter(regs); 682 do_breakpoint(esr, regs); 683 debug_exception_exit(regs); 684 local_daif_restore(DAIF_PROCCTX); 685 arm64_exit_to_user_mode(regs); 686 } 687 688 static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr) 689 { 690 bool step_done; 691 692 if (!is_ttbr0_addr(regs->pc)) 693 arm64_apply_bp_hardening(); 694 695 arm64_enter_from_user_mode(regs); 696 /* 697 * After handling a breakpoint, we suspend the breakpoint 698 * and use single-step to move to the next instruction. 699 * If we are stepping a suspended breakpoint there's nothing more to do: 700 * the single-step is complete. 701 */ 702 step_done = try_step_suspended_breakpoints(regs); 703 local_daif_restore(DAIF_PROCCTX); 704 if (!step_done) 705 do_el0_softstep(esr, regs); 706 arm64_exit_to_user_mode(regs); 707 } 708 709 static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr) 710 { 711 /* Watchpoints are the only debug exception to write FAR_EL1 */ 712 unsigned long far = read_sysreg(far_el1); 713 714 arm64_enter_from_user_mode(regs); 715 debug_exception_enter(regs); 716 do_watchpoint(far, esr, regs); 717 debug_exception_exit(regs); 718 local_daif_restore(DAIF_PROCCTX); 719 arm64_exit_to_user_mode(regs); 720 } 721 722 static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr) 723 { 724 arm64_enter_from_user_mode(regs); 725 local_daif_restore(DAIF_PROCCTX); 726 do_el0_brk64(esr, regs); 727 arm64_exit_to_user_mode(regs); 728 } 729 730 static void noinstr el0_svc(struct pt_regs *regs) 731 { 732 arm64_syscall_enter_from_user_mode(regs); 733 cortex_a76_erratum_1463225_svc_handler(); 734 fpsimd_syscall_enter(); 735 local_daif_restore(DAIF_PROCCTX); 736 do_el0_svc(regs); 737 arm64_syscall_exit_to_user_mode(regs); 738 fpsimd_syscall_exit(); 739 } 740 741 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) 742 { 743 arm64_enter_from_user_mode(regs); 744 local_daif_restore(DAIF_PROCCTX); 745 do_el0_fpac(regs, esr); 746 arm64_exit_to_user_mode(regs); 747 } 748 749 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) 750 { 751 unsigned long esr = read_sysreg(esr_el1); 752 753 switch (ESR_ELx_EC(esr)) { 754 case ESR_ELx_EC_SVC64: 755 el0_svc(regs); 756 break; 757 case ESR_ELx_EC_DABT_LOW: 758 el0_da(regs, esr); 759 break; 760 case ESR_ELx_EC_IABT_LOW: 761 el0_ia(regs, esr); 762 break; 763 case ESR_ELx_EC_FP_ASIMD: 764 el0_fpsimd_acc(regs, esr); 765 break; 766 case ESR_ELx_EC_SVE: 767 el0_sve_acc(regs, esr); 768 break; 769 case ESR_ELx_EC_SME: 770 el0_sme_acc(regs, esr); 771 break; 772 case ESR_ELx_EC_FP_EXC64: 773 el0_fpsimd_exc(regs, esr); 774 break; 775 case ESR_ELx_EC_SYS64: 776 case ESR_ELx_EC_WFx: 777 el0_sys(regs, esr); 778 break; 779 case ESR_ELx_EC_SP_ALIGN: 780 el0_sp(regs, esr); 781 break; 782 case ESR_ELx_EC_PC_ALIGN: 783 el0_pc(regs, esr); 784 break; 785 case ESR_ELx_EC_UNKNOWN: 786 el0_undef(regs, esr); 787 break; 788 case ESR_ELx_EC_BTI: 789 el0_bti(regs); 790 break; 791 case ESR_ELx_EC_MOPS: 792 el0_mops(regs, esr); 793 break; 794 case ESR_ELx_EC_GCS: 795 el0_gcs(regs, esr); 796 break; 797 case ESR_ELx_EC_BREAKPT_LOW: 798 el0_breakpt(regs, esr); 799 break; 800 case ESR_ELx_EC_SOFTSTP_LOW: 801 el0_softstp(regs, esr); 802 break; 803 case ESR_ELx_EC_WATCHPT_LOW: 804 el0_watchpt(regs, esr); 805 break; 806 case ESR_ELx_EC_BRK64: 807 el0_brk64(regs, esr); 808 break; 809 case ESR_ELx_EC_FPAC: 810 el0_fpac(regs, esr); 811 break; 812 default: 813 el0_inv(regs, esr); 814 } 815 } 816 817 static void noinstr el0_interrupt(struct pt_regs *regs, 818 void (*handler)(struct pt_regs *)) 819 { 820 arm64_enter_from_user_mode(regs); 821 822 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 823 824 if (regs->pc & BIT(55)) 825 arm64_apply_bp_hardening(); 826 827 irq_enter_rcu(); 828 do_interrupt_handler(regs, handler); 829 irq_exit_rcu(); 830 831 arm64_exit_to_user_mode(regs); 832 } 833 834 static void noinstr __el0_irq_handler_common(struct pt_regs *regs) 835 { 836 el0_interrupt(regs, handle_arch_irq); 837 } 838 839 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs) 840 { 841 __el0_irq_handler_common(regs); 842 } 843 844 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs) 845 { 846 el0_interrupt(regs, handle_arch_fiq); 847 } 848 849 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs) 850 { 851 __el0_fiq_handler_common(regs); 852 } 853 854 static void noinstr __el0_error_handler_common(struct pt_regs *regs) 855 { 856 unsigned long esr = read_sysreg(esr_el1); 857 irqentry_state_t state; 858 859 arm64_enter_from_user_mode(regs); 860 local_daif_restore(DAIF_ERRCTX); 861 state = irqentry_nmi_enter(regs); 862 do_serror(regs, esr); 863 irqentry_nmi_exit(regs, state); 864 local_daif_restore(DAIF_PROCCTX); 865 arm64_exit_to_user_mode(regs); 866 } 867 868 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) 869 { 870 __el0_error_handler_common(regs); 871 } 872 873 #ifdef CONFIG_COMPAT 874 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) 875 { 876 arm64_enter_from_user_mode(regs); 877 local_daif_restore(DAIF_PROCCTX); 878 do_el0_cp15(esr, regs); 879 arm64_exit_to_user_mode(regs); 880 } 881 882 static void noinstr el0_svc_compat(struct pt_regs *regs) 883 { 884 arm64_syscall_enter_from_user_mode(regs); 885 cortex_a76_erratum_1463225_svc_handler(); 886 local_daif_restore(DAIF_PROCCTX); 887 do_el0_svc_compat(regs); 888 arm64_syscall_exit_to_user_mode(regs); 889 } 890 891 static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr) 892 { 893 arm64_enter_from_user_mode(regs); 894 local_daif_restore(DAIF_PROCCTX); 895 do_bkpt32(esr, regs); 896 arm64_exit_to_user_mode(regs); 897 } 898 899 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) 900 { 901 unsigned long esr = read_sysreg(esr_el1); 902 903 switch (ESR_ELx_EC(esr)) { 904 case ESR_ELx_EC_SVC32: 905 el0_svc_compat(regs); 906 break; 907 case ESR_ELx_EC_DABT_LOW: 908 el0_da(regs, esr); 909 break; 910 case ESR_ELx_EC_IABT_LOW: 911 el0_ia(regs, esr); 912 break; 913 case ESR_ELx_EC_FP_ASIMD: 914 el0_fpsimd_acc(regs, esr); 915 break; 916 case ESR_ELx_EC_FP_EXC32: 917 el0_fpsimd_exc(regs, esr); 918 break; 919 case ESR_ELx_EC_PC_ALIGN: 920 el0_pc(regs, esr); 921 break; 922 case ESR_ELx_EC_UNKNOWN: 923 case ESR_ELx_EC_CP14_MR: 924 case ESR_ELx_EC_CP14_LS: 925 case ESR_ELx_EC_CP14_64: 926 el0_undef(regs, esr); 927 break; 928 case ESR_ELx_EC_CP15_32: 929 case ESR_ELx_EC_CP15_64: 930 el0_cp15(regs, esr); 931 break; 932 case ESR_ELx_EC_BREAKPT_LOW: 933 el0_breakpt(regs, esr); 934 break; 935 case ESR_ELx_EC_SOFTSTP_LOW: 936 el0_softstp(regs, esr); 937 break; 938 case ESR_ELx_EC_WATCHPT_LOW: 939 el0_watchpt(regs, esr); 940 break; 941 case ESR_ELx_EC_BKPT32: 942 el0_bkpt32(regs, esr); 943 break; 944 default: 945 el0_inv(regs, esr); 946 } 947 } 948 949 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) 950 { 951 __el0_irq_handler_common(regs); 952 } 953 954 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) 955 { 956 __el0_fiq_handler_common(regs); 957 } 958 959 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) 960 { 961 __el0_error_handler_common(regs); 962 } 963 #else /* CONFIG_COMPAT */ 964 UNHANDLED(el0t, 32, sync) 965 UNHANDLED(el0t, 32, irq) 966 UNHANDLED(el0t, 32, fiq) 967 UNHANDLED(el0t, 32, error) 968 #endif /* CONFIG_COMPAT */ 969 970 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) 971 { 972 unsigned long esr = read_sysreg(esr_el1); 973 unsigned long far = read_sysreg(far_el1); 974 975 irqentry_nmi_enter(regs); 976 panic_bad_stack(regs, esr, far); 977 } 978 979 #ifdef CONFIG_ARM_SDE_INTERFACE 980 asmlinkage noinstr unsigned long 981 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) 982 { 983 irqentry_state_t state; 984 unsigned long ret; 985 986 /* 987 * We didn't take an exception to get here, so the HW hasn't 988 * set/cleared bits in PSTATE that we may rely on. 989 * 990 * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to 991 * whether PSTATE bits are inherited unchanged or generated from 992 * scratch, and the TF-A implementation always clears PAN and always 993 * clears UAO. There are no other known implementations. 994 * 995 * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how 996 * PSTATE is modified upon architectural exceptions, and so PAN is 997 * either inherited or set per SCTLR_ELx.SPAN, and UAO is always 998 * cleared. 999 * 1000 * We must explicitly reset PAN to the expected state, including 1001 * clearing it when the host isn't using it, in case a VM had it set. 1002 */ 1003 if (system_uses_hw_pan()) 1004 set_pstate_pan(1); 1005 else if (cpu_has_pan()) 1006 set_pstate_pan(0); 1007 1008 state = irqentry_nmi_enter(regs); 1009 ret = do_sdei_event(regs, arg); 1010 irqentry_nmi_exit(regs, state); 1011 1012 return ret; 1013 } 1014 #endif /* CONFIG_ARM_SDE_INTERFACE */ 1015