1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Exception handling code 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 */ 7 8 #include <linux/context_tracking.h> 9 #include <linux/irq-entry-common.h> 10 #include <linux/kasan.h> 11 #include <linux/linkage.h> 12 #include <linux/livepatch.h> 13 #include <linux/lockdep.h> 14 #include <linux/ptrace.h> 15 #include <linux/resume_user_mode.h> 16 #include <linux/sched.h> 17 #include <linux/sched/debug.h> 18 #include <linux/thread_info.h> 19 20 #include <asm/cpufeature.h> 21 #include <asm/daifflags.h> 22 #include <asm/esr.h> 23 #include <asm/exception.h> 24 #include <asm/irq_regs.h> 25 #include <asm/kprobes.h> 26 #include <asm/mmu.h> 27 #include <asm/processor.h> 28 #include <asm/sdei.h> 29 #include <asm/stacktrace.h> 30 #include <asm/sysreg.h> 31 #include <asm/system_misc.h> 32 33 /* 34 * Handle IRQ/context state management when entering from kernel mode. 35 * Before this function is called it is not safe to call regular kernel code, 36 * instrumentable code, or any code which may trigger an exception. 37 */ 38 static noinstr irqentry_state_t enter_from_kernel_mode(struct pt_regs *regs) 39 { 40 irqentry_state_t state; 41 42 state = irqentry_enter(regs); 43 mte_check_tfsr_entry(); 44 mte_disable_tco_entry(current); 45 46 return state; 47 } 48 49 /* 50 * Handle IRQ/context state management when exiting to kernel mode. 51 * After this function returns it is not safe to call regular kernel code, 52 * instrumentable code, or any code which may trigger an exception. 53 */ 54 static void noinstr exit_to_kernel_mode(struct pt_regs *regs, 55 irqentry_state_t state) 56 { 57 mte_check_tfsr_exit(); 58 irqentry_exit(regs, state); 59 } 60 61 /* 62 * Handle IRQ/context state management when entering from user mode. 63 * Before this function is called it is not safe to call regular kernel code, 64 * instrumentable code, or any code which may trigger an exception. 65 */ 66 static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs) 67 { 68 enter_from_user_mode(regs); 69 mte_disable_tco_entry(current); 70 } 71 72 /* 73 * Handle IRQ/context state management when exiting to user mode. 74 * After this function returns it is not safe to call regular kernel code, 75 * instrumentable code, or any code which may trigger an exception. 76 */ 77 78 static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs) 79 { 80 local_irq_disable(); 81 exit_to_user_mode_prepare_legacy(regs); 82 local_daif_mask(); 83 mte_check_tfsr_exit(); 84 exit_to_user_mode(); 85 } 86 87 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) 88 { 89 arm64_exit_to_user_mode(regs); 90 } 91 92 /* 93 * Handle IRQ/context state management when entering a debug exception from 94 * kernel mode. Before this function is called it is not safe to call regular 95 * kernel code, instrumentable code, or any code which may trigger an exception. 96 */ 97 static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs) 98 { 99 irqentry_state_t state; 100 101 state.lockdep = lockdep_hardirqs_enabled(); 102 103 lockdep_hardirqs_off(CALLER_ADDR0); 104 ct_nmi_enter(); 105 106 trace_hardirqs_off_finish(); 107 108 return state; 109 } 110 111 /* 112 * Handle IRQ/context state management when exiting a debug exception from 113 * kernel mode. After this function returns it is not safe to call regular 114 * kernel code, instrumentable code, or any code which may trigger an exception. 115 */ 116 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs, 117 irqentry_state_t state) 118 { 119 if (state.lockdep) { 120 trace_hardirqs_on_prepare(); 121 lockdep_hardirqs_on_prepare(); 122 } 123 124 ct_nmi_exit(); 125 if (state.lockdep) 126 lockdep_hardirqs_on(CALLER_ADDR0); 127 } 128 129 static void do_interrupt_handler(struct pt_regs *regs, 130 void (*handler)(struct pt_regs *)) 131 { 132 struct pt_regs *old_regs = set_irq_regs(regs); 133 134 if (on_thread_stack()) 135 call_on_irq_stack(regs, handler); 136 else 137 handler(regs); 138 139 set_irq_regs(old_regs); 140 } 141 142 extern void (*handle_arch_irq)(struct pt_regs *); 143 extern void (*handle_arch_fiq)(struct pt_regs *); 144 145 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, 146 unsigned long esr) 147 { 148 irqentry_nmi_enter(regs); 149 150 console_verbose(); 151 152 pr_crit("Unhandled %s exception on CPU%d, ESR 0x%016lx -- %s\n", 153 vector, smp_processor_id(), esr, 154 esr_get_class_string(esr)); 155 156 __show_regs(regs); 157 panic("Unhandled exception"); 158 } 159 160 #define UNHANDLED(el, regsize, vector) \ 161 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \ 162 { \ 163 const char *desc = #regsize "-bit " #el " " #vector; \ 164 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \ 165 } 166 167 #ifdef CONFIG_ARM64_ERRATUM_1463225 168 static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa); 169 170 static void cortex_a76_erratum_1463225_svc_handler(void) 171 { 172 u64 reg, val; 173 174 if (!unlikely(test_thread_flag(TIF_SINGLESTEP))) 175 return; 176 177 if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225))) 178 return; 179 180 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1); 181 reg = read_sysreg(mdscr_el1); 182 val = reg | MDSCR_EL1_SS | MDSCR_EL1_KDE; 183 write_sysreg(val, mdscr_el1); 184 asm volatile("msr daifclr, #8"); 185 isb(); 186 187 /* We will have taken a single-step exception by this point */ 188 189 write_sysreg(reg, mdscr_el1); 190 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0); 191 } 192 193 static __always_inline bool 194 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 195 { 196 if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa)) 197 return false; 198 199 /* 200 * We've taken a dummy step exception from the kernel to ensure 201 * that interrupts are re-enabled on the syscall path. Return back 202 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions 203 * masked so that we can safely restore the mdscr and get on with 204 * handling the syscall. 205 */ 206 regs->pstate |= PSR_D_BIT; 207 return true; 208 } 209 #else /* CONFIG_ARM64_ERRATUM_1463225 */ 210 static void cortex_a76_erratum_1463225_svc_handler(void) { } 211 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 212 { 213 return false; 214 } 215 #endif /* CONFIG_ARM64_ERRATUM_1463225 */ 216 217 /* 218 * As per the ABI exit SME streaming mode and clear the SVE state not 219 * shared with FPSIMD on syscall entry. 220 */ 221 static inline void fpsimd_syscall_enter(void) 222 { 223 /* Ensure PSTATE.SM is clear, but leave PSTATE.ZA as-is. */ 224 if (system_supports_sme()) 225 sme_smstop_sm(); 226 227 /* 228 * The CPU is not in streaming mode. If non-streaming SVE is not 229 * supported, there is no SVE state that needs to be discarded. 230 */ 231 if (!system_supports_sve()) 232 return; 233 234 if (test_thread_flag(TIF_SVE)) { 235 unsigned int sve_vq_minus_one; 236 237 sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1; 238 sve_flush_live(true, sve_vq_minus_one); 239 } 240 241 /* 242 * Any live non-FPSIMD SVE state has been zeroed. Allow 243 * fpsimd_save_user_state() to lazily discard SVE state until either 244 * the live state is unbound or fpsimd_syscall_exit() is called. 245 */ 246 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_FPSIMD); 247 } 248 249 static __always_inline void fpsimd_syscall_exit(void) 250 { 251 if (!system_supports_sve()) 252 return; 253 254 /* 255 * The current task's user FPSIMD/SVE/SME state is now bound to this 256 * CPU. The fpsimd_last_state.to_save value is either: 257 * 258 * - FP_STATE_FPSIMD, if the state has not been reloaded on this CPU 259 * since fpsimd_syscall_enter(). 260 * 261 * - FP_STATE_CURRENT, if the state has been reloaded on this CPU at 262 * any point. 263 * 264 * Reset this to FP_STATE_CURRENT to stop lazy discarding. 265 */ 266 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT); 267 } 268 269 /* 270 * In debug exception context, we explicitly disable preemption despite 271 * having interrupts disabled. 272 * This serves two purposes: it makes it much less likely that we would 273 * accidentally schedule in exception context and it will force a warning 274 * if we somehow manage to schedule by accident. 275 */ 276 static void debug_exception_enter(struct pt_regs *regs) 277 { 278 preempt_disable(); 279 280 /* This code is a bit fragile. Test it. */ 281 RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work"); 282 } 283 NOKPROBE_SYMBOL(debug_exception_enter); 284 285 static void debug_exception_exit(struct pt_regs *regs) 286 { 287 preempt_enable_no_resched(); 288 } 289 NOKPROBE_SYMBOL(debug_exception_exit); 290 291 UNHANDLED(el1t, 64, sync) 292 UNHANDLED(el1t, 64, irq) 293 UNHANDLED(el1t, 64, fiq) 294 UNHANDLED(el1t, 64, error) 295 296 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) 297 { 298 unsigned long far = read_sysreg(far_el1); 299 irqentry_state_t state; 300 301 state = enter_from_kernel_mode(regs); 302 local_daif_inherit(regs); 303 do_mem_abort(far, esr, regs); 304 local_daif_mask(); 305 exit_to_kernel_mode(regs, state); 306 } 307 308 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) 309 { 310 unsigned long far = read_sysreg(far_el1); 311 irqentry_state_t state; 312 313 state = enter_from_kernel_mode(regs); 314 local_daif_inherit(regs); 315 do_sp_pc_abort(far, esr, regs); 316 local_daif_mask(); 317 exit_to_kernel_mode(regs, state); 318 } 319 320 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) 321 { 322 irqentry_state_t state; 323 324 state = enter_from_kernel_mode(regs); 325 local_daif_inherit(regs); 326 do_el1_undef(regs, esr); 327 local_daif_mask(); 328 exit_to_kernel_mode(regs, state); 329 } 330 331 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) 332 { 333 irqentry_state_t state; 334 335 state = enter_from_kernel_mode(regs); 336 local_daif_inherit(regs); 337 do_el1_bti(regs, esr); 338 local_daif_mask(); 339 exit_to_kernel_mode(regs, state); 340 } 341 342 static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr) 343 { 344 irqentry_state_t state; 345 346 state = enter_from_kernel_mode(regs); 347 local_daif_inherit(regs); 348 do_el1_gcs(regs, esr); 349 local_daif_mask(); 350 exit_to_kernel_mode(regs, state); 351 } 352 353 static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr) 354 { 355 irqentry_state_t state; 356 357 state = enter_from_kernel_mode(regs); 358 local_daif_inherit(regs); 359 do_el1_mops(regs, esr); 360 local_daif_mask(); 361 exit_to_kernel_mode(regs, state); 362 } 363 364 static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) 365 { 366 irqentry_state_t state; 367 368 state = arm64_enter_el1_dbg(regs); 369 debug_exception_enter(regs); 370 do_breakpoint(esr, regs); 371 debug_exception_exit(regs); 372 arm64_exit_el1_dbg(regs, state); 373 } 374 375 static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) 376 { 377 irqentry_state_t state; 378 379 state = arm64_enter_el1_dbg(regs); 380 if (!cortex_a76_erratum_1463225_debug_handler(regs)) { 381 debug_exception_enter(regs); 382 /* 383 * After handling a breakpoint, we suspend the breakpoint 384 * and use single-step to move to the next instruction. 385 * If we are stepping a suspended breakpoint there's nothing more to do: 386 * the single-step is complete. 387 */ 388 if (!try_step_suspended_breakpoints(regs)) 389 do_el1_softstep(esr, regs); 390 debug_exception_exit(regs); 391 } 392 arm64_exit_el1_dbg(regs, state); 393 } 394 395 static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) 396 { 397 /* Watchpoints are the only debug exception to write FAR_EL1 */ 398 unsigned long far = read_sysreg(far_el1); 399 irqentry_state_t state; 400 401 state = arm64_enter_el1_dbg(regs); 402 debug_exception_enter(regs); 403 do_watchpoint(far, esr, regs); 404 debug_exception_exit(regs); 405 arm64_exit_el1_dbg(regs, state); 406 } 407 408 static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr) 409 { 410 irqentry_state_t state; 411 412 state = arm64_enter_el1_dbg(regs); 413 debug_exception_enter(regs); 414 do_el1_brk64(esr, regs); 415 debug_exception_exit(regs); 416 arm64_exit_el1_dbg(regs, state); 417 } 418 419 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) 420 { 421 irqentry_state_t state; 422 423 state = enter_from_kernel_mode(regs); 424 local_daif_inherit(regs); 425 do_el1_fpac(regs, esr); 426 local_daif_mask(); 427 exit_to_kernel_mode(regs, state); 428 } 429 430 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) 431 { 432 unsigned long esr = read_sysreg(esr_el1); 433 434 switch (ESR_ELx_EC(esr)) { 435 case ESR_ELx_EC_DABT_CUR: 436 case ESR_ELx_EC_IABT_CUR: 437 el1_abort(regs, esr); 438 break; 439 /* 440 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a 441 * recursive exception when trying to push the initial pt_regs. 442 */ 443 case ESR_ELx_EC_PC_ALIGN: 444 el1_pc(regs, esr); 445 break; 446 case ESR_ELx_EC_SYS64: 447 case ESR_ELx_EC_UNKNOWN: 448 el1_undef(regs, esr); 449 break; 450 case ESR_ELx_EC_BTI: 451 el1_bti(regs, esr); 452 break; 453 case ESR_ELx_EC_GCS: 454 el1_gcs(regs, esr); 455 break; 456 case ESR_ELx_EC_MOPS: 457 el1_mops(regs, esr); 458 break; 459 case ESR_ELx_EC_BREAKPT_CUR: 460 el1_breakpt(regs, esr); 461 break; 462 case ESR_ELx_EC_SOFTSTP_CUR: 463 el1_softstp(regs, esr); 464 break; 465 case ESR_ELx_EC_WATCHPT_CUR: 466 el1_watchpt(regs, esr); 467 break; 468 case ESR_ELx_EC_BRK64: 469 el1_brk64(regs, esr); 470 break; 471 case ESR_ELx_EC_FPAC: 472 el1_fpac(regs, esr); 473 break; 474 default: 475 __panic_unhandled(regs, "64-bit el1h sync", esr); 476 } 477 } 478 479 static __always_inline void __el1_pnmi(struct pt_regs *regs, 480 void (*handler)(struct pt_regs *)) 481 { 482 irqentry_state_t state; 483 484 state = irqentry_nmi_enter(regs); 485 do_interrupt_handler(regs, handler); 486 irqentry_nmi_exit(regs, state); 487 } 488 489 static __always_inline void __el1_irq(struct pt_regs *regs, 490 void (*handler)(struct pt_regs *)) 491 { 492 irqentry_state_t state; 493 494 state = enter_from_kernel_mode(regs); 495 496 irq_enter_rcu(); 497 do_interrupt_handler(regs, handler); 498 irq_exit_rcu(); 499 500 exit_to_kernel_mode(regs, state); 501 } 502 static void noinstr el1_interrupt(struct pt_regs *regs, 503 void (*handler)(struct pt_regs *)) 504 { 505 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 506 507 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs)) 508 __el1_pnmi(regs, handler); 509 else 510 __el1_irq(regs, handler); 511 } 512 513 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) 514 { 515 el1_interrupt(regs, handle_arch_irq); 516 } 517 518 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) 519 { 520 el1_interrupt(regs, handle_arch_fiq); 521 } 522 523 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) 524 { 525 unsigned long esr = read_sysreg(esr_el1); 526 irqentry_state_t state; 527 528 local_daif_restore(DAIF_ERRCTX); 529 state = irqentry_nmi_enter(regs); 530 do_serror(regs, esr); 531 irqentry_nmi_exit(regs, state); 532 } 533 534 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) 535 { 536 unsigned long far = read_sysreg(far_el1); 537 538 arm64_enter_from_user_mode(regs); 539 local_daif_restore(DAIF_PROCCTX); 540 do_mem_abort(far, esr, regs); 541 arm64_exit_to_user_mode(regs); 542 } 543 544 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr) 545 { 546 unsigned long far = read_sysreg(far_el1); 547 548 /* 549 * We've taken an instruction abort from userspace and not yet 550 * re-enabled IRQs. If the address is a kernel address, apply 551 * BP hardening prior to enabling IRQs and pre-emption. 552 */ 553 if (!is_ttbr0_addr(far)) 554 arm64_apply_bp_hardening(); 555 556 arm64_enter_from_user_mode(regs); 557 local_daif_restore(DAIF_PROCCTX); 558 do_mem_abort(far, esr, regs); 559 arm64_exit_to_user_mode(regs); 560 } 561 562 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) 563 { 564 arm64_enter_from_user_mode(regs); 565 local_daif_restore(DAIF_PROCCTX); 566 do_fpsimd_acc(esr, regs); 567 arm64_exit_to_user_mode(regs); 568 } 569 570 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr) 571 { 572 arm64_enter_from_user_mode(regs); 573 local_daif_restore(DAIF_PROCCTX); 574 do_sve_acc(esr, regs); 575 arm64_exit_to_user_mode(regs); 576 } 577 578 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr) 579 { 580 arm64_enter_from_user_mode(regs); 581 local_daif_restore(DAIF_PROCCTX); 582 do_sme_acc(esr, regs); 583 arm64_exit_to_user_mode(regs); 584 } 585 586 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) 587 { 588 arm64_enter_from_user_mode(regs); 589 local_daif_restore(DAIF_PROCCTX); 590 do_fpsimd_exc(esr, regs); 591 arm64_exit_to_user_mode(regs); 592 } 593 594 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr) 595 { 596 arm64_enter_from_user_mode(regs); 597 local_daif_restore(DAIF_PROCCTX); 598 do_el0_sys(esr, regs); 599 arm64_exit_to_user_mode(regs); 600 } 601 602 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr) 603 { 604 unsigned long far = read_sysreg(far_el1); 605 606 if (!is_ttbr0_addr(instruction_pointer(regs))) 607 arm64_apply_bp_hardening(); 608 609 arm64_enter_from_user_mode(regs); 610 local_daif_restore(DAIF_PROCCTX); 611 do_sp_pc_abort(far, esr, regs); 612 arm64_exit_to_user_mode(regs); 613 } 614 615 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr) 616 { 617 arm64_enter_from_user_mode(regs); 618 local_daif_restore(DAIF_PROCCTX); 619 do_sp_pc_abort(regs->sp, esr, regs); 620 arm64_exit_to_user_mode(regs); 621 } 622 623 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr) 624 { 625 arm64_enter_from_user_mode(regs); 626 local_daif_restore(DAIF_PROCCTX); 627 do_el0_undef(regs, esr); 628 arm64_exit_to_user_mode(regs); 629 } 630 631 static void noinstr el0_bti(struct pt_regs *regs) 632 { 633 arm64_enter_from_user_mode(regs); 634 local_daif_restore(DAIF_PROCCTX); 635 do_el0_bti(regs); 636 arm64_exit_to_user_mode(regs); 637 } 638 639 static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr) 640 { 641 arm64_enter_from_user_mode(regs); 642 local_daif_restore(DAIF_PROCCTX); 643 do_el0_mops(regs, esr); 644 arm64_exit_to_user_mode(regs); 645 } 646 647 static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr) 648 { 649 arm64_enter_from_user_mode(regs); 650 local_daif_restore(DAIF_PROCCTX); 651 do_el0_gcs(regs, esr); 652 arm64_exit_to_user_mode(regs); 653 } 654 655 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr) 656 { 657 arm64_enter_from_user_mode(regs); 658 local_daif_restore(DAIF_PROCCTX); 659 bad_el0_sync(regs, 0, esr); 660 arm64_exit_to_user_mode(regs); 661 } 662 663 static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr) 664 { 665 if (!is_ttbr0_addr(regs->pc)) 666 arm64_apply_bp_hardening(); 667 668 arm64_enter_from_user_mode(regs); 669 debug_exception_enter(regs); 670 do_breakpoint(esr, regs); 671 debug_exception_exit(regs); 672 local_daif_restore(DAIF_PROCCTX); 673 arm64_exit_to_user_mode(regs); 674 } 675 676 static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr) 677 { 678 bool step_done; 679 680 if (!is_ttbr0_addr(regs->pc)) 681 arm64_apply_bp_hardening(); 682 683 arm64_enter_from_user_mode(regs); 684 /* 685 * After handling a breakpoint, we suspend the breakpoint 686 * and use single-step to move to the next instruction. 687 * If we are stepping a suspended breakpoint there's nothing more to do: 688 * the single-step is complete. 689 */ 690 step_done = try_step_suspended_breakpoints(regs); 691 local_daif_restore(DAIF_PROCCTX); 692 if (!step_done) 693 do_el0_softstep(esr, regs); 694 arm64_exit_to_user_mode(regs); 695 } 696 697 static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr) 698 { 699 /* Watchpoints are the only debug exception to write FAR_EL1 */ 700 unsigned long far = read_sysreg(far_el1); 701 702 arm64_enter_from_user_mode(regs); 703 debug_exception_enter(regs); 704 do_watchpoint(far, esr, regs); 705 debug_exception_exit(regs); 706 local_daif_restore(DAIF_PROCCTX); 707 arm64_exit_to_user_mode(regs); 708 } 709 710 static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr) 711 { 712 arm64_enter_from_user_mode(regs); 713 local_daif_restore(DAIF_PROCCTX); 714 do_el0_brk64(esr, regs); 715 arm64_exit_to_user_mode(regs); 716 } 717 718 static void noinstr el0_svc(struct pt_regs *regs) 719 { 720 arm64_enter_from_user_mode(regs); 721 cortex_a76_erratum_1463225_svc_handler(); 722 fpsimd_syscall_enter(); 723 local_daif_restore(DAIF_PROCCTX); 724 do_el0_svc(regs); 725 arm64_exit_to_user_mode(regs); 726 fpsimd_syscall_exit(); 727 } 728 729 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) 730 { 731 arm64_enter_from_user_mode(regs); 732 local_daif_restore(DAIF_PROCCTX); 733 do_el0_fpac(regs, esr); 734 arm64_exit_to_user_mode(regs); 735 } 736 737 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) 738 { 739 unsigned long esr = read_sysreg(esr_el1); 740 741 switch (ESR_ELx_EC(esr)) { 742 case ESR_ELx_EC_SVC64: 743 el0_svc(regs); 744 break; 745 case ESR_ELx_EC_DABT_LOW: 746 el0_da(regs, esr); 747 break; 748 case ESR_ELx_EC_IABT_LOW: 749 el0_ia(regs, esr); 750 break; 751 case ESR_ELx_EC_FP_ASIMD: 752 el0_fpsimd_acc(regs, esr); 753 break; 754 case ESR_ELx_EC_SVE: 755 el0_sve_acc(regs, esr); 756 break; 757 case ESR_ELx_EC_SME: 758 el0_sme_acc(regs, esr); 759 break; 760 case ESR_ELx_EC_FP_EXC64: 761 el0_fpsimd_exc(regs, esr); 762 break; 763 case ESR_ELx_EC_SYS64: 764 case ESR_ELx_EC_WFx: 765 el0_sys(regs, esr); 766 break; 767 case ESR_ELx_EC_SP_ALIGN: 768 el0_sp(regs, esr); 769 break; 770 case ESR_ELx_EC_PC_ALIGN: 771 el0_pc(regs, esr); 772 break; 773 case ESR_ELx_EC_UNKNOWN: 774 el0_undef(regs, esr); 775 break; 776 case ESR_ELx_EC_BTI: 777 el0_bti(regs); 778 break; 779 case ESR_ELx_EC_MOPS: 780 el0_mops(regs, esr); 781 break; 782 case ESR_ELx_EC_GCS: 783 el0_gcs(regs, esr); 784 break; 785 case ESR_ELx_EC_BREAKPT_LOW: 786 el0_breakpt(regs, esr); 787 break; 788 case ESR_ELx_EC_SOFTSTP_LOW: 789 el0_softstp(regs, esr); 790 break; 791 case ESR_ELx_EC_WATCHPT_LOW: 792 el0_watchpt(regs, esr); 793 break; 794 case ESR_ELx_EC_BRK64: 795 el0_brk64(regs, esr); 796 break; 797 case ESR_ELx_EC_FPAC: 798 el0_fpac(regs, esr); 799 break; 800 default: 801 el0_inv(regs, esr); 802 } 803 } 804 805 static void noinstr el0_interrupt(struct pt_regs *regs, 806 void (*handler)(struct pt_regs *)) 807 { 808 arm64_enter_from_user_mode(regs); 809 810 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 811 812 if (regs->pc & BIT(55)) 813 arm64_apply_bp_hardening(); 814 815 irq_enter_rcu(); 816 do_interrupt_handler(regs, handler); 817 irq_exit_rcu(); 818 819 arm64_exit_to_user_mode(regs); 820 } 821 822 static void noinstr __el0_irq_handler_common(struct pt_regs *regs) 823 { 824 el0_interrupt(regs, handle_arch_irq); 825 } 826 827 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs) 828 { 829 __el0_irq_handler_common(regs); 830 } 831 832 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs) 833 { 834 el0_interrupt(regs, handle_arch_fiq); 835 } 836 837 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs) 838 { 839 __el0_fiq_handler_common(regs); 840 } 841 842 static void noinstr __el0_error_handler_common(struct pt_regs *regs) 843 { 844 unsigned long esr = read_sysreg(esr_el1); 845 irqentry_state_t state; 846 847 arm64_enter_from_user_mode(regs); 848 local_daif_restore(DAIF_ERRCTX); 849 state = irqentry_nmi_enter(regs); 850 do_serror(regs, esr); 851 irqentry_nmi_exit(regs, state); 852 local_daif_restore(DAIF_PROCCTX); 853 arm64_exit_to_user_mode(regs); 854 } 855 856 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) 857 { 858 __el0_error_handler_common(regs); 859 } 860 861 #ifdef CONFIG_COMPAT 862 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) 863 { 864 arm64_enter_from_user_mode(regs); 865 local_daif_restore(DAIF_PROCCTX); 866 do_el0_cp15(esr, regs); 867 arm64_exit_to_user_mode(regs); 868 } 869 870 static void noinstr el0_svc_compat(struct pt_regs *regs) 871 { 872 arm64_enter_from_user_mode(regs); 873 cortex_a76_erratum_1463225_svc_handler(); 874 local_daif_restore(DAIF_PROCCTX); 875 do_el0_svc_compat(regs); 876 arm64_exit_to_user_mode(regs); 877 } 878 879 static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr) 880 { 881 arm64_enter_from_user_mode(regs); 882 local_daif_restore(DAIF_PROCCTX); 883 do_bkpt32(esr, regs); 884 arm64_exit_to_user_mode(regs); 885 } 886 887 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) 888 { 889 unsigned long esr = read_sysreg(esr_el1); 890 891 switch (ESR_ELx_EC(esr)) { 892 case ESR_ELx_EC_SVC32: 893 el0_svc_compat(regs); 894 break; 895 case ESR_ELx_EC_DABT_LOW: 896 el0_da(regs, esr); 897 break; 898 case ESR_ELx_EC_IABT_LOW: 899 el0_ia(regs, esr); 900 break; 901 case ESR_ELx_EC_FP_ASIMD: 902 el0_fpsimd_acc(regs, esr); 903 break; 904 case ESR_ELx_EC_FP_EXC32: 905 el0_fpsimd_exc(regs, esr); 906 break; 907 case ESR_ELx_EC_PC_ALIGN: 908 el0_pc(regs, esr); 909 break; 910 case ESR_ELx_EC_UNKNOWN: 911 case ESR_ELx_EC_CP14_MR: 912 case ESR_ELx_EC_CP14_LS: 913 case ESR_ELx_EC_CP14_64: 914 el0_undef(regs, esr); 915 break; 916 case ESR_ELx_EC_CP15_32: 917 case ESR_ELx_EC_CP15_64: 918 el0_cp15(regs, esr); 919 break; 920 case ESR_ELx_EC_BREAKPT_LOW: 921 el0_breakpt(regs, esr); 922 break; 923 case ESR_ELx_EC_SOFTSTP_LOW: 924 el0_softstp(regs, esr); 925 break; 926 case ESR_ELx_EC_WATCHPT_LOW: 927 el0_watchpt(regs, esr); 928 break; 929 case ESR_ELx_EC_BKPT32: 930 el0_bkpt32(regs, esr); 931 break; 932 default: 933 el0_inv(regs, esr); 934 } 935 } 936 937 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) 938 { 939 __el0_irq_handler_common(regs); 940 } 941 942 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) 943 { 944 __el0_fiq_handler_common(regs); 945 } 946 947 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) 948 { 949 __el0_error_handler_common(regs); 950 } 951 #else /* CONFIG_COMPAT */ 952 UNHANDLED(el0t, 32, sync) 953 UNHANDLED(el0t, 32, irq) 954 UNHANDLED(el0t, 32, fiq) 955 UNHANDLED(el0t, 32, error) 956 #endif /* CONFIG_COMPAT */ 957 958 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) 959 { 960 unsigned long esr = read_sysreg(esr_el1); 961 unsigned long far = read_sysreg(far_el1); 962 963 irqentry_nmi_enter(regs); 964 panic_bad_stack(regs, esr, far); 965 } 966 967 #ifdef CONFIG_ARM_SDE_INTERFACE 968 asmlinkage noinstr unsigned long 969 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) 970 { 971 irqentry_state_t state; 972 unsigned long ret; 973 974 /* 975 * We didn't take an exception to get here, so the HW hasn't 976 * set/cleared bits in PSTATE that we may rely on. 977 * 978 * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to 979 * whether PSTATE bits are inherited unchanged or generated from 980 * scratch, and the TF-A implementation always clears PAN and always 981 * clears UAO. There are no other known implementations. 982 * 983 * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how 984 * PSTATE is modified upon architectural exceptions, and so PAN is 985 * either inherited or set per SCTLR_ELx.SPAN, and UAO is always 986 * cleared. 987 * 988 * We must explicitly reset PAN to the expected state, including 989 * clearing it when the host isn't using it, in case a VM had it set. 990 */ 991 if (system_uses_hw_pan()) 992 set_pstate_pan(1); 993 else if (cpu_has_pan()) 994 set_pstate_pan(0); 995 996 state = irqentry_nmi_enter(regs); 997 ret = do_sdei_event(regs, arg); 998 irqentry_nmi_exit(regs, state); 999 1000 return ret; 1001 } 1002 #endif /* CONFIG_ARM_SDE_INTERFACE */ 1003