1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Exception handling code 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 */ 7 8 #include <linux/context_tracking.h> 9 #include <linux/irq-entry-common.h> 10 #include <linux/kasan.h> 11 #include <linux/linkage.h> 12 #include <linux/livepatch.h> 13 #include <linux/lockdep.h> 14 #include <linux/ptrace.h> 15 #include <linux/resume_user_mode.h> 16 #include <linux/sched.h> 17 #include <linux/sched/debug.h> 18 #include <linux/thread_info.h> 19 20 #include <asm/cpufeature.h> 21 #include <asm/daifflags.h> 22 #include <asm/esr.h> 23 #include <asm/exception.h> 24 #include <asm/fpsimd.h> 25 #include <asm/irq_regs.h> 26 #include <asm/kprobes.h> 27 #include <asm/mmu.h> 28 #include <asm/processor.h> 29 #include <asm/sdei.h> 30 #include <asm/stacktrace.h> 31 #include <asm/sysreg.h> 32 #include <asm/system_misc.h> 33 34 /* 35 * Handle IRQ/context state management when entering from kernel mode. 36 * Before this function is called it is not safe to call regular kernel code, 37 * instrumentable code, or any code which may trigger an exception. 38 */ 39 static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *regs) 40 { 41 irqentry_state_t state; 42 43 state = irqentry_enter_from_kernel_mode(regs); 44 mte_check_tfsr_entry(); 45 mte_disable_tco_entry(current); 46 47 return state; 48 } 49 50 /* 51 * Handle IRQ/context state management when exiting to kernel mode. 52 * After this function returns it is not safe to call regular kernel code, 53 * instrumentable code, or any code which may trigger an exception. 54 */ 55 static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, 56 irqentry_state_t state) 57 { 58 local_irq_disable(); 59 irqentry_exit_to_kernel_mode_preempt(regs, state); 60 local_daif_mask(); 61 mte_check_tfsr_exit(); 62 irqentry_exit_to_kernel_mode_after_preempt(regs, state); 63 } 64 65 static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs) 66 { 67 enter_from_user_mode(regs); 68 mte_disable_tco_entry(current); 69 sme_enter_from_user_mode(); 70 } 71 72 /* 73 * Handle IRQ/context state management when entering from user mode. 74 * Before this function is called it is not safe to call regular kernel code, 75 * instrumentable code, or any code which may trigger an exception. 76 */ 77 static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs) 78 { 79 enter_from_user_mode(regs); 80 rseq_note_user_irq_entry(); 81 mte_disable_tco_entry(current); 82 sme_enter_from_user_mode(); 83 } 84 85 static __always_inline void arm64_syscall_exit_to_user_mode(struct pt_regs *regs) 86 { 87 local_irq_disable(); 88 syscall_exit_to_user_mode_prepare(regs); 89 local_daif_mask(); 90 sme_exit_to_user_mode(); 91 mte_check_tfsr_exit(); 92 exit_to_user_mode(); 93 } 94 95 /* 96 * Handle IRQ/context state management when exiting to user mode. 97 * After this function returns it is not safe to call regular kernel code, 98 * instrumentable code, or any code which may trigger an exception. 99 */ 100 static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs) 101 { 102 local_irq_disable(); 103 irqentry_exit_to_user_mode_prepare(regs); 104 local_daif_mask(); 105 sme_exit_to_user_mode(); 106 mte_check_tfsr_exit(); 107 exit_to_user_mode(); 108 } 109 110 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) 111 { 112 arm64_syscall_exit_to_user_mode(regs); 113 } 114 115 /* 116 * Handle IRQ/context state management when entering a debug exception from 117 * kernel mode. Before this function is called it is not safe to call regular 118 * kernel code, instrumentable code, or any code which may trigger an exception. 119 */ 120 static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs) 121 { 122 irqentry_state_t state; 123 124 state.lockdep = lockdep_hardirqs_enabled(); 125 126 lockdep_hardirqs_off(CALLER_ADDR0); 127 ct_nmi_enter(); 128 129 trace_hardirqs_off_finish(); 130 131 return state; 132 } 133 134 /* 135 * Handle IRQ/context state management when exiting a debug exception from 136 * kernel mode. After this function returns it is not safe to call regular 137 * kernel code, instrumentable code, or any code which may trigger an exception. 138 */ 139 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs, 140 irqentry_state_t state) 141 { 142 if (state.lockdep) { 143 trace_hardirqs_on_prepare(); 144 lockdep_hardirqs_on_prepare(); 145 } 146 147 ct_nmi_exit(); 148 if (state.lockdep) 149 lockdep_hardirqs_on(CALLER_ADDR0); 150 } 151 152 static void do_interrupt_handler(struct pt_regs *regs, 153 void (*handler)(struct pt_regs *)) 154 { 155 struct pt_regs *old_regs = set_irq_regs(regs); 156 157 if (on_thread_stack()) 158 call_on_irq_stack(regs, handler); 159 else 160 handler(regs); 161 162 set_irq_regs(old_regs); 163 } 164 165 extern void (*handle_arch_irq)(struct pt_regs *); 166 extern void (*handle_arch_fiq)(struct pt_regs *); 167 168 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, 169 unsigned long esr) 170 { 171 irqentry_nmi_enter(regs); 172 173 console_verbose(); 174 175 pr_crit("Unhandled %s exception on CPU%d, ESR 0x%016lx -- %s\n", 176 vector, smp_processor_id(), esr, 177 esr_get_class_string(esr)); 178 179 __show_regs(regs); 180 panic("Unhandled exception"); 181 } 182 183 #define UNHANDLED(el, regsize, vector) \ 184 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \ 185 { \ 186 const char *desc = #regsize "-bit " #el " " #vector; \ 187 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \ 188 } 189 190 #ifdef CONFIG_ARM64_ERRATUM_1463225 191 static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa); 192 193 static void cortex_a76_erratum_1463225_svc_handler(void) 194 { 195 u64 reg, val; 196 197 if (!unlikely(test_thread_flag(TIF_SINGLESTEP))) 198 return; 199 200 if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225))) 201 return; 202 203 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1); 204 reg = read_sysreg(mdscr_el1); 205 val = reg | MDSCR_EL1_SS | MDSCR_EL1_KDE; 206 write_sysreg(val, mdscr_el1); 207 asm volatile("msr daifclr, #8"); 208 isb(); 209 210 /* We will have taken a single-step exception by this point */ 211 212 write_sysreg(reg, mdscr_el1); 213 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0); 214 } 215 216 static __always_inline bool 217 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 218 { 219 if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa)) 220 return false; 221 222 /* 223 * We've taken a dummy step exception from the kernel to ensure 224 * that interrupts are re-enabled on the syscall path. Return back 225 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions 226 * masked so that we can safely restore the mdscr and get on with 227 * handling the syscall. 228 */ 229 regs->pstate |= PSR_D_BIT; 230 return true; 231 } 232 #else /* CONFIG_ARM64_ERRATUM_1463225 */ 233 static void cortex_a76_erratum_1463225_svc_handler(void) { } 234 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 235 { 236 return false; 237 } 238 #endif /* CONFIG_ARM64_ERRATUM_1463225 */ 239 240 /* 241 * As per the ABI exit SME streaming mode and clear the SVE state not 242 * shared with FPSIMD on syscall entry. 243 */ 244 static inline void fpsimd_syscall_enter(void) 245 { 246 /* Ensure PSTATE.SM is clear, but leave PSTATE.ZA as-is. */ 247 if (system_supports_sme()) 248 sme_smstop_sm(); 249 250 /* 251 * The CPU is not in streaming mode. If non-streaming SVE is not 252 * supported, there is no SVE state that needs to be discarded. 253 */ 254 if (!system_supports_sve()) 255 return; 256 257 if (test_thread_flag(TIF_SVE)) { 258 unsigned int sve_vq_minus_one; 259 260 sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1; 261 sve_flush_live(true, sve_vq_minus_one); 262 } 263 264 /* 265 * Any live non-FPSIMD SVE state has been zeroed. Allow 266 * fpsimd_save_user_state() to lazily discard SVE state until either 267 * the live state is unbound or fpsimd_syscall_exit() is called. 268 */ 269 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_FPSIMD); 270 } 271 272 static __always_inline void fpsimd_syscall_exit(void) 273 { 274 if (!system_supports_sve()) 275 return; 276 277 /* 278 * The current task's user FPSIMD/SVE/SME state is now bound to this 279 * CPU. The fpsimd_last_state.to_save value is either: 280 * 281 * - FP_STATE_FPSIMD, if the state has not been reloaded on this CPU 282 * since fpsimd_syscall_enter(). 283 * 284 * - FP_STATE_CURRENT, if the state has been reloaded on this CPU at 285 * any point. 286 * 287 * Reset this to FP_STATE_CURRENT to stop lazy discarding. 288 */ 289 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT); 290 } 291 292 /* 293 * In debug exception context, we explicitly disable preemption despite 294 * having interrupts disabled. 295 * This serves two purposes: it makes it much less likely that we would 296 * accidentally schedule in exception context and it will force a warning 297 * if we somehow manage to schedule by accident. 298 */ 299 static void debug_exception_enter(struct pt_regs *regs) 300 { 301 preempt_disable(); 302 303 /* This code is a bit fragile. Test it. */ 304 RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work"); 305 } 306 NOKPROBE_SYMBOL(debug_exception_enter); 307 308 static void debug_exception_exit(struct pt_regs *regs) 309 { 310 preempt_enable_no_resched(); 311 } 312 NOKPROBE_SYMBOL(debug_exception_exit); 313 314 UNHANDLED(el1t, 64, sync) 315 UNHANDLED(el1t, 64, irq) 316 UNHANDLED(el1t, 64, fiq) 317 UNHANDLED(el1t, 64, error) 318 319 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) 320 { 321 unsigned long far = read_sysreg(far_el1); 322 irqentry_state_t state; 323 324 state = arm64_enter_from_kernel_mode(regs); 325 local_daif_inherit(regs); 326 do_mem_abort(far, esr, regs); 327 arm64_exit_to_kernel_mode(regs, state); 328 } 329 330 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) 331 { 332 unsigned long far = read_sysreg(far_el1); 333 irqentry_state_t state; 334 335 state = arm64_enter_from_kernel_mode(regs); 336 local_daif_inherit(regs); 337 do_sp_pc_abort(far, esr, regs); 338 arm64_exit_to_kernel_mode(regs, state); 339 } 340 341 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) 342 { 343 irqentry_state_t state; 344 345 state = arm64_enter_from_kernel_mode(regs); 346 local_daif_inherit(regs); 347 do_el1_undef(regs, esr); 348 arm64_exit_to_kernel_mode(regs, state); 349 } 350 351 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) 352 { 353 irqentry_state_t state; 354 355 state = arm64_enter_from_kernel_mode(regs); 356 local_daif_inherit(regs); 357 do_el1_bti(regs, esr); 358 arm64_exit_to_kernel_mode(regs, state); 359 } 360 361 static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr) 362 { 363 irqentry_state_t state; 364 365 state = arm64_enter_from_kernel_mode(regs); 366 local_daif_inherit(regs); 367 do_el1_gcs(regs, esr); 368 arm64_exit_to_kernel_mode(regs, state); 369 } 370 371 static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr) 372 { 373 irqentry_state_t state; 374 375 state = arm64_enter_from_kernel_mode(regs); 376 local_daif_inherit(regs); 377 do_el1_mops(regs, esr); 378 arm64_exit_to_kernel_mode(regs, state); 379 } 380 381 static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) 382 { 383 irqentry_state_t state; 384 385 state = arm64_enter_el1_dbg(regs); 386 debug_exception_enter(regs); 387 do_breakpoint(esr, regs); 388 debug_exception_exit(regs); 389 arm64_exit_el1_dbg(regs, state); 390 } 391 392 static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) 393 { 394 irqentry_state_t state; 395 396 state = arm64_enter_el1_dbg(regs); 397 if (!cortex_a76_erratum_1463225_debug_handler(regs)) { 398 debug_exception_enter(regs); 399 /* 400 * After handling a breakpoint, we suspend the breakpoint 401 * and use single-step to move to the next instruction. 402 * If we are stepping a suspended breakpoint there's nothing more to do: 403 * the single-step is complete. 404 */ 405 if (!try_step_suspended_breakpoints(regs)) 406 do_el1_softstep(esr, regs); 407 debug_exception_exit(regs); 408 } 409 arm64_exit_el1_dbg(regs, state); 410 } 411 412 static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) 413 { 414 /* Watchpoints are the only debug exception to write FAR_EL1 */ 415 unsigned long far = read_sysreg(far_el1); 416 irqentry_state_t state; 417 418 state = arm64_enter_el1_dbg(regs); 419 debug_exception_enter(regs); 420 do_watchpoint(far, esr, regs); 421 debug_exception_exit(regs); 422 arm64_exit_el1_dbg(regs, state); 423 } 424 425 static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr) 426 { 427 irqentry_state_t state; 428 429 state = arm64_enter_el1_dbg(regs); 430 debug_exception_enter(regs); 431 do_el1_brk64(esr, regs); 432 debug_exception_exit(regs); 433 arm64_exit_el1_dbg(regs, state); 434 } 435 436 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) 437 { 438 irqentry_state_t state; 439 440 state = arm64_enter_from_kernel_mode(regs); 441 local_daif_inherit(regs); 442 do_el1_fpac(regs, esr); 443 arm64_exit_to_kernel_mode(regs, state); 444 } 445 446 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) 447 { 448 unsigned long esr = read_sysreg(esr_el1); 449 450 switch (ESR_ELx_EC(esr)) { 451 case ESR_ELx_EC_DABT_CUR: 452 case ESR_ELx_EC_IABT_CUR: 453 el1_abort(regs, esr); 454 break; 455 /* 456 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a 457 * recursive exception when trying to push the initial pt_regs. 458 */ 459 case ESR_ELx_EC_PC_ALIGN: 460 el1_pc(regs, esr); 461 break; 462 case ESR_ELx_EC_SYS64: 463 case ESR_ELx_EC_UNKNOWN: 464 el1_undef(regs, esr); 465 break; 466 case ESR_ELx_EC_BTI: 467 el1_bti(regs, esr); 468 break; 469 case ESR_ELx_EC_GCS: 470 el1_gcs(regs, esr); 471 break; 472 case ESR_ELx_EC_MOPS: 473 el1_mops(regs, esr); 474 break; 475 case ESR_ELx_EC_BREAKPT_CUR: 476 el1_breakpt(regs, esr); 477 break; 478 case ESR_ELx_EC_SOFTSTP_CUR: 479 el1_softstp(regs, esr); 480 break; 481 case ESR_ELx_EC_WATCHPT_CUR: 482 el1_watchpt(regs, esr); 483 break; 484 case ESR_ELx_EC_BRK64: 485 el1_brk64(regs, esr); 486 break; 487 case ESR_ELx_EC_FPAC: 488 el1_fpac(regs, esr); 489 break; 490 default: 491 __panic_unhandled(regs, "64-bit el1h sync", esr); 492 } 493 } 494 495 static __always_inline void __el1_pnmi(struct pt_regs *regs, 496 void (*handler)(struct pt_regs *)) 497 { 498 irqentry_state_t state; 499 500 state = irqentry_nmi_enter(regs); 501 do_interrupt_handler(regs, handler); 502 irqentry_nmi_exit(regs, state); 503 } 504 505 static __always_inline void __el1_irq(struct pt_regs *regs, 506 void (*handler)(struct pt_regs *)) 507 { 508 irqentry_state_t state; 509 510 state = arm64_enter_from_kernel_mode(regs); 511 512 irq_enter_rcu(); 513 do_interrupt_handler(regs, handler); 514 irq_exit_rcu(); 515 516 arm64_exit_to_kernel_mode(regs, state); 517 } 518 static void noinstr el1_interrupt(struct pt_regs *regs, 519 void (*handler)(struct pt_regs *)) 520 { 521 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 522 523 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs)) 524 __el1_pnmi(regs, handler); 525 else 526 __el1_irq(regs, handler); 527 } 528 529 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) 530 { 531 el1_interrupt(regs, handle_arch_irq); 532 } 533 534 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) 535 { 536 el1_interrupt(regs, handle_arch_fiq); 537 } 538 539 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) 540 { 541 unsigned long esr = read_sysreg(esr_el1); 542 irqentry_state_t state; 543 544 local_daif_restore(DAIF_ERRCTX); 545 state = irqentry_nmi_enter(regs); 546 do_serror(regs, esr); 547 irqentry_nmi_exit(regs, state); 548 } 549 550 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) 551 { 552 unsigned long far = read_sysreg(far_el1); 553 554 arm64_enter_from_user_mode(regs); 555 local_daif_restore(DAIF_PROCCTX); 556 do_mem_abort(far, esr, regs); 557 arm64_exit_to_user_mode(regs); 558 } 559 560 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr) 561 { 562 unsigned long far = read_sysreg(far_el1); 563 564 /* 565 * We've taken an instruction abort from userspace and not yet 566 * re-enabled IRQs. If the address is a kernel address, apply 567 * BP hardening prior to enabling IRQs and pre-emption. 568 */ 569 if (!is_ttbr0_addr(far)) 570 arm64_apply_bp_hardening(); 571 572 arm64_enter_from_user_mode(regs); 573 local_daif_restore(DAIF_PROCCTX); 574 do_mem_abort(far, esr, regs); 575 arm64_exit_to_user_mode(regs); 576 } 577 578 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) 579 { 580 arm64_enter_from_user_mode(regs); 581 local_daif_restore(DAIF_PROCCTX); 582 do_fpsimd_acc(esr, regs); 583 arm64_exit_to_user_mode(regs); 584 } 585 586 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr) 587 { 588 arm64_enter_from_user_mode(regs); 589 local_daif_restore(DAIF_PROCCTX); 590 do_sve_acc(esr, regs); 591 arm64_exit_to_user_mode(regs); 592 } 593 594 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr) 595 { 596 arm64_enter_from_user_mode(regs); 597 local_daif_restore(DAIF_PROCCTX); 598 do_sme_acc(esr, regs); 599 arm64_exit_to_user_mode(regs); 600 } 601 602 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) 603 { 604 arm64_enter_from_user_mode(regs); 605 local_daif_restore(DAIF_PROCCTX); 606 do_fpsimd_exc(esr, regs); 607 arm64_exit_to_user_mode(regs); 608 } 609 610 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr) 611 { 612 arm64_enter_from_user_mode(regs); 613 local_daif_restore(DAIF_PROCCTX); 614 do_el0_sys(esr, regs); 615 arm64_exit_to_user_mode(regs); 616 } 617 618 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr) 619 { 620 unsigned long far = read_sysreg(far_el1); 621 622 if (!is_ttbr0_addr(instruction_pointer(regs))) 623 arm64_apply_bp_hardening(); 624 625 arm64_enter_from_user_mode(regs); 626 local_daif_restore(DAIF_PROCCTX); 627 do_sp_pc_abort(far, esr, regs); 628 arm64_exit_to_user_mode(regs); 629 } 630 631 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr) 632 { 633 arm64_enter_from_user_mode(regs); 634 local_daif_restore(DAIF_PROCCTX); 635 do_sp_pc_abort(regs->sp, esr, regs); 636 arm64_exit_to_user_mode(regs); 637 } 638 639 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr) 640 { 641 arm64_enter_from_user_mode(regs); 642 local_daif_restore(DAIF_PROCCTX); 643 do_el0_undef(regs, esr); 644 arm64_exit_to_user_mode(regs); 645 } 646 647 static void noinstr el0_bti(struct pt_regs *regs) 648 { 649 arm64_enter_from_user_mode(regs); 650 local_daif_restore(DAIF_PROCCTX); 651 do_el0_bti(regs); 652 arm64_exit_to_user_mode(regs); 653 } 654 655 static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr) 656 { 657 arm64_enter_from_user_mode(regs); 658 local_daif_restore(DAIF_PROCCTX); 659 do_el0_mops(regs, esr); 660 arm64_exit_to_user_mode(regs); 661 } 662 663 static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr) 664 { 665 arm64_enter_from_user_mode(regs); 666 local_daif_restore(DAIF_PROCCTX); 667 do_el0_gcs(regs, esr); 668 arm64_exit_to_user_mode(regs); 669 } 670 671 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr) 672 { 673 arm64_enter_from_user_mode(regs); 674 local_daif_restore(DAIF_PROCCTX); 675 bad_el0_sync(regs, 0, esr); 676 arm64_exit_to_user_mode(regs); 677 } 678 679 static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr) 680 { 681 if (!is_ttbr0_addr(regs->pc)) 682 arm64_apply_bp_hardening(); 683 684 arm64_enter_from_user_mode(regs); 685 debug_exception_enter(regs); 686 do_breakpoint(esr, regs); 687 debug_exception_exit(regs); 688 local_daif_restore(DAIF_PROCCTX); 689 arm64_exit_to_user_mode(regs); 690 } 691 692 static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr) 693 { 694 bool step_done; 695 696 if (!is_ttbr0_addr(regs->pc)) 697 arm64_apply_bp_hardening(); 698 699 arm64_enter_from_user_mode(regs); 700 /* 701 * After handling a breakpoint, we suspend the breakpoint 702 * and use single-step to move to the next instruction. 703 * If we are stepping a suspended breakpoint there's nothing more to do: 704 * the single-step is complete. 705 */ 706 step_done = try_step_suspended_breakpoints(regs); 707 local_daif_restore(DAIF_PROCCTX); 708 if (!step_done) 709 do_el0_softstep(esr, regs); 710 arm64_exit_to_user_mode(regs); 711 } 712 713 static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr) 714 { 715 /* Watchpoints are the only debug exception to write FAR_EL1 */ 716 unsigned long far = read_sysreg(far_el1); 717 718 arm64_enter_from_user_mode(regs); 719 debug_exception_enter(regs); 720 do_watchpoint(far, esr, regs); 721 debug_exception_exit(regs); 722 local_daif_restore(DAIF_PROCCTX); 723 arm64_exit_to_user_mode(regs); 724 } 725 726 static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr) 727 { 728 arm64_enter_from_user_mode(regs); 729 local_daif_restore(DAIF_PROCCTX); 730 do_el0_brk64(esr, regs); 731 arm64_exit_to_user_mode(regs); 732 } 733 734 static void noinstr el0_svc(struct pt_regs *regs) 735 { 736 arm64_syscall_enter_from_user_mode(regs); 737 cortex_a76_erratum_1463225_svc_handler(); 738 fpsimd_syscall_enter(); 739 local_daif_restore(DAIF_PROCCTX); 740 do_el0_svc(regs); 741 arm64_syscall_exit_to_user_mode(regs); 742 fpsimd_syscall_exit(); 743 } 744 745 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) 746 { 747 arm64_enter_from_user_mode(regs); 748 local_daif_restore(DAIF_PROCCTX); 749 do_el0_fpac(regs, esr); 750 arm64_exit_to_user_mode(regs); 751 } 752 753 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) 754 { 755 unsigned long esr = read_sysreg(esr_el1); 756 757 switch (ESR_ELx_EC(esr)) { 758 case ESR_ELx_EC_SVC64: 759 el0_svc(regs); 760 break; 761 case ESR_ELx_EC_DABT_LOW: 762 el0_da(regs, esr); 763 break; 764 case ESR_ELx_EC_IABT_LOW: 765 el0_ia(regs, esr); 766 break; 767 case ESR_ELx_EC_FP_ASIMD: 768 el0_fpsimd_acc(regs, esr); 769 break; 770 case ESR_ELx_EC_SVE: 771 el0_sve_acc(regs, esr); 772 break; 773 case ESR_ELx_EC_SME: 774 el0_sme_acc(regs, esr); 775 break; 776 case ESR_ELx_EC_FP_EXC64: 777 el0_fpsimd_exc(regs, esr); 778 break; 779 case ESR_ELx_EC_SYS64: 780 case ESR_ELx_EC_WFx: 781 el0_sys(regs, esr); 782 break; 783 case ESR_ELx_EC_SP_ALIGN: 784 el0_sp(regs, esr); 785 break; 786 case ESR_ELx_EC_PC_ALIGN: 787 el0_pc(regs, esr); 788 break; 789 case ESR_ELx_EC_UNKNOWN: 790 el0_undef(regs, esr); 791 break; 792 case ESR_ELx_EC_BTI: 793 el0_bti(regs); 794 break; 795 case ESR_ELx_EC_MOPS: 796 el0_mops(regs, esr); 797 break; 798 case ESR_ELx_EC_GCS: 799 el0_gcs(regs, esr); 800 break; 801 case ESR_ELx_EC_BREAKPT_LOW: 802 el0_breakpt(regs, esr); 803 break; 804 case ESR_ELx_EC_SOFTSTP_LOW: 805 el0_softstp(regs, esr); 806 break; 807 case ESR_ELx_EC_WATCHPT_LOW: 808 el0_watchpt(regs, esr); 809 break; 810 case ESR_ELx_EC_BRK64: 811 el0_brk64(regs, esr); 812 break; 813 case ESR_ELx_EC_FPAC: 814 el0_fpac(regs, esr); 815 break; 816 default: 817 el0_inv(regs, esr); 818 } 819 } 820 821 static void noinstr el0_interrupt(struct pt_regs *regs, 822 void (*handler)(struct pt_regs *)) 823 { 824 arm64_enter_from_user_mode(regs); 825 826 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 827 828 if (regs->pc & BIT(55)) 829 arm64_apply_bp_hardening(); 830 831 irq_enter_rcu(); 832 do_interrupt_handler(regs, handler); 833 irq_exit_rcu(); 834 835 arm64_exit_to_user_mode(regs); 836 } 837 838 static void noinstr __el0_irq_handler_common(struct pt_regs *regs) 839 { 840 el0_interrupt(regs, handle_arch_irq); 841 } 842 843 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs) 844 { 845 __el0_irq_handler_common(regs); 846 } 847 848 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs) 849 { 850 el0_interrupt(regs, handle_arch_fiq); 851 } 852 853 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs) 854 { 855 __el0_fiq_handler_common(regs); 856 } 857 858 static void noinstr __el0_error_handler_common(struct pt_regs *regs) 859 { 860 unsigned long esr = read_sysreg(esr_el1); 861 irqentry_state_t state; 862 863 arm64_enter_from_user_mode(regs); 864 local_daif_restore(DAIF_ERRCTX); 865 state = irqentry_nmi_enter(regs); 866 do_serror(regs, esr); 867 irqentry_nmi_exit(regs, state); 868 local_daif_restore(DAIF_PROCCTX); 869 arm64_exit_to_user_mode(regs); 870 } 871 872 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) 873 { 874 __el0_error_handler_common(regs); 875 } 876 877 #ifdef CONFIG_COMPAT 878 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) 879 { 880 arm64_enter_from_user_mode(regs); 881 local_daif_restore(DAIF_PROCCTX); 882 do_el0_cp15(esr, regs); 883 arm64_exit_to_user_mode(regs); 884 } 885 886 static void noinstr el0_svc_compat(struct pt_regs *regs) 887 { 888 arm64_syscall_enter_from_user_mode(regs); 889 cortex_a76_erratum_1463225_svc_handler(); 890 local_daif_restore(DAIF_PROCCTX); 891 do_el0_svc_compat(regs); 892 arm64_syscall_exit_to_user_mode(regs); 893 } 894 895 static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr) 896 { 897 arm64_enter_from_user_mode(regs); 898 local_daif_restore(DAIF_PROCCTX); 899 do_bkpt32(esr, regs); 900 arm64_exit_to_user_mode(regs); 901 } 902 903 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) 904 { 905 unsigned long esr = read_sysreg(esr_el1); 906 907 switch (ESR_ELx_EC(esr)) { 908 case ESR_ELx_EC_SVC32: 909 el0_svc_compat(regs); 910 break; 911 case ESR_ELx_EC_DABT_LOW: 912 el0_da(regs, esr); 913 break; 914 case ESR_ELx_EC_IABT_LOW: 915 el0_ia(regs, esr); 916 break; 917 case ESR_ELx_EC_FP_ASIMD: 918 el0_fpsimd_acc(regs, esr); 919 break; 920 case ESR_ELx_EC_FP_EXC32: 921 el0_fpsimd_exc(regs, esr); 922 break; 923 case ESR_ELx_EC_PC_ALIGN: 924 el0_pc(regs, esr); 925 break; 926 case ESR_ELx_EC_UNKNOWN: 927 case ESR_ELx_EC_CP14_MR: 928 case ESR_ELx_EC_CP14_LS: 929 case ESR_ELx_EC_CP14_64: 930 el0_undef(regs, esr); 931 break; 932 case ESR_ELx_EC_CP15_32: 933 case ESR_ELx_EC_CP15_64: 934 el0_cp15(regs, esr); 935 break; 936 case ESR_ELx_EC_BREAKPT_LOW: 937 el0_breakpt(regs, esr); 938 break; 939 case ESR_ELx_EC_SOFTSTP_LOW: 940 el0_softstp(regs, esr); 941 break; 942 case ESR_ELx_EC_WATCHPT_LOW: 943 el0_watchpt(regs, esr); 944 break; 945 case ESR_ELx_EC_BKPT32: 946 el0_bkpt32(regs, esr); 947 break; 948 default: 949 el0_inv(regs, esr); 950 } 951 } 952 953 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) 954 { 955 __el0_irq_handler_common(regs); 956 } 957 958 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) 959 { 960 __el0_fiq_handler_common(regs); 961 } 962 963 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) 964 { 965 __el0_error_handler_common(regs); 966 } 967 #else /* CONFIG_COMPAT */ 968 UNHANDLED(el0t, 32, sync) 969 UNHANDLED(el0t, 32, irq) 970 UNHANDLED(el0t, 32, fiq) 971 UNHANDLED(el0t, 32, error) 972 #endif /* CONFIG_COMPAT */ 973 974 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) 975 { 976 unsigned long esr = read_sysreg(esr_el1); 977 unsigned long far = read_sysreg(far_el1); 978 979 irqentry_nmi_enter(regs); 980 panic_bad_stack(regs, esr, far); 981 } 982 983 #ifdef CONFIG_ARM_SDE_INTERFACE 984 asmlinkage noinstr unsigned long 985 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) 986 { 987 irqentry_state_t state; 988 unsigned long ret; 989 990 /* 991 * We didn't take an exception to get here, so the HW hasn't 992 * set/cleared bits in PSTATE that we may rely on. 993 * 994 * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to 995 * whether PSTATE bits are inherited unchanged or generated from 996 * scratch, and the TF-A implementation always clears PAN and always 997 * clears UAO. There are no other known implementations. 998 * 999 * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how 1000 * PSTATE is modified upon architectural exceptions, and so PAN is 1001 * either inherited or set per SCTLR_ELx.SPAN, and UAO is always 1002 * cleared. 1003 * 1004 * We must explicitly reset PAN to the expected state, including 1005 * clearing it when the host isn't using it, in case a VM had it set. 1006 */ 1007 if (system_uses_hw_pan()) 1008 set_pstate_pan(1); 1009 else if (cpu_has_pan()) 1010 set_pstate_pan(0); 1011 1012 state = irqentry_nmi_enter(regs); 1013 ret = do_sdei_event(regs, arg); 1014 irqentry_nmi_exit(regs, state); 1015 1016 return ret; 1017 } 1018 #endif /* CONFIG_ARM_SDE_INTERFACE */ 1019