1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Low-level exception handling code 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 * Authors: Catalin Marinas <catalin.marinas@arm.com> 7 * Will Deacon <will.deacon@arm.com> 8 */ 9 10#include <linux/arm-smccc.h> 11#include <linux/init.h> 12#include <linux/linkage.h> 13 14#include <asm/alternative.h> 15#include <asm/assembler.h> 16#include <asm/asm-offsets.h> 17#include <asm/asm_pointer_auth.h> 18#include <asm/bug.h> 19#include <asm/cpufeature.h> 20#include <asm/errno.h> 21#include <asm/esr.h> 22#include <asm/irq.h> 23#include <asm/memory.h> 24#include <asm/mmu.h> 25#include <asm/processor.h> 26#include <asm/ptrace.h> 27#include <asm/scs.h> 28#include <asm/stacktrace/frame.h> 29#include <asm/thread_info.h> 30#include <asm/asm-uaccess.h> 31#include <asm/unistd.h> 32 33 .macro clear_gp_regs 34 .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29 35 mov x\n, xzr 36 .endr 37 .endm 38 39 .macro kernel_ventry, el:req, ht:req, regsize:req, label:req 40 .align 7 41.Lventry_start\@: 42 .if \el == 0 43 /* 44 * This must be the first instruction of the EL0 vector entries. It is 45 * skipped by the trampoline vectors, to trigger the cleanup. 46 */ 47 b .Lskip_tramp_vectors_cleanup\@ 48 .if \regsize == 64 49 mrs x30, tpidrro_el0 50 msr tpidrro_el0, xzr 51 .else 52 mov x30, xzr 53 .endif 54.Lskip_tramp_vectors_cleanup\@: 55 .endif 56 57 sub sp, sp, #PT_REGS_SIZE 58 /* 59 * Test whether the SP has overflowed, without corrupting a GPR. 60 * Task and IRQ stacks are aligned so that SP & (1 << THREAD_SHIFT) 61 * should always be zero. 62 */ 63 add sp, sp, x0 // sp' = sp + x0 64 sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp 65 tbnz x0, #THREAD_SHIFT, 0f 66 sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0 67 sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp 68 b el\el\ht\()_\regsize\()_\label 69 700: 71 /* 72 * Either we've just detected an overflow, or we've taken an exception 73 * while on the overflow stack. Either way, we won't return to 74 * userspace, and can clobber EL0 registers to free up GPRs. 75 */ 76 77 /* Stash the original SP (minus PT_REGS_SIZE) in tpidr_el0. */ 78 msr tpidr_el0, x0 79 80 /* Recover the original x0 value and stash it in tpidrro_el0 */ 81 sub x0, sp, x0 82 msr tpidrro_el0, x0 83 84 /* Switch to the overflow stack */ 85 adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0 86 87 /* 88 * Check whether we were already on the overflow stack. This may happen 89 * after panic() re-enables interrupts. 90 */ 91 mrs x0, tpidr_el0 // sp of interrupted context 92 sub x0, sp, x0 // delta with top of overflow stack 93 tst x0, #~(OVERFLOW_STACK_SIZE - 1) // within range? 94 b.ne __bad_stack // no? -> bad stack pointer 95 96 /* We were already on the overflow stack. Restore sp/x0 and carry on. */ 97 sub sp, sp, x0 98 mrs x0, tpidrro_el0 99 b el\el\ht\()_\regsize\()_\label 100.org .Lventry_start\@ + 128 // Did we overflow the ventry slot? 101 .endm 102 103 .macro tramp_alias, dst, sym 104 .set .Lalias\@, TRAMP_VALIAS + \sym - .entry.tramp.text 105 movz \dst, :abs_g2_s:.Lalias\@ 106 movk \dst, :abs_g1_nc:.Lalias\@ 107 movk \dst, :abs_g0_nc:.Lalias\@ 108 .endm 109 110 /* 111 * This macro corrupts x0-x3. It is the caller's duty to save/restore 112 * them if required. 113 */ 114 .macro apply_ssbd, state, tmp1, tmp2 115alternative_cb ARM64_ALWAYS_SYSTEM, spectre_v4_patch_fw_mitigation_enable 116 b .L__asm_ssbd_skip\@ // Patched to NOP 117alternative_cb_end 118 ldr_this_cpu \tmp2, arm64_ssbd_callback_required, \tmp1 119 cbz \tmp2, .L__asm_ssbd_skip\@ 120 ldr \tmp2, [tsk, #TSK_TI_FLAGS] 121 tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@ 122 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2 123 mov w1, #\state 124alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit 125 nop // Patched to SMC/HVC #0 126alternative_cb_end 127.L__asm_ssbd_skip\@: 128 .endm 129 130 /* Check for MTE asynchronous tag check faults */ 131 .macro check_mte_async_tcf, tmp, ti_flags, thread_sctlr 132#ifdef CONFIG_ARM64_MTE 133 .arch_extension lse 134alternative_if_not ARM64_MTE 135 b 1f 136alternative_else_nop_endif 137 /* 138 * Asynchronous tag check faults are only possible in ASYNC (2) or 139 * ASYM (3) modes. In each of these modes bit 1 of SCTLR_EL1.TCF0 is 140 * set, so skip the check if it is unset. 141 */ 142 tbz \thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f 143 mrs_s \tmp, SYS_TFSRE0_EL1 144 tbz \tmp, #SYS_TFSR_EL1_TF0_SHIFT, 1f 145 /* Asynchronous TCF occurred for TTBR0 access, set the TI flag */ 146 mov \tmp, #_TIF_MTE_ASYNC_FAULT 147 add \ti_flags, tsk, #TSK_TI_FLAGS 148 stset \tmp, [\ti_flags] 1491: 150#endif 151 .endm 152 153 /* Clear the MTE asynchronous tag check faults */ 154 .macro clear_mte_async_tcf thread_sctlr 155#ifdef CONFIG_ARM64_MTE 156alternative_if ARM64_MTE 157 /* See comment in check_mte_async_tcf above. */ 158 tbz \thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f 159 dsb ish 160 msr_s SYS_TFSRE0_EL1, xzr 1611: 162alternative_else_nop_endif 163#endif 164 .endm 165 166 .macro mte_set_gcr, mte_ctrl, tmp 167#ifdef CONFIG_ARM64_MTE 168 ubfx \tmp, \mte_ctrl, #MTE_CTRL_GCR_USER_EXCL_SHIFT, #16 169 orr \tmp, \tmp, #SYS_GCR_EL1_RRND 170 msr_s SYS_GCR_EL1, \tmp 171#endif 172 .endm 173 174 .macro mte_set_kernel_gcr, tmp, tmp2 175#ifdef CONFIG_KASAN_HW_TAGS 176alternative_cb ARM64_ALWAYS_SYSTEM, kasan_hw_tags_enable 177 b 1f 178alternative_cb_end 179 mov \tmp, KERNEL_GCR_EL1 180 msr_s SYS_GCR_EL1, \tmp 1811: 182#endif 183 .endm 184 185 .macro mte_set_user_gcr, tsk, tmp, tmp2 186#ifdef CONFIG_KASAN_HW_TAGS 187alternative_cb ARM64_ALWAYS_SYSTEM, kasan_hw_tags_enable 188 b 1f 189alternative_cb_end 190 ldr \tmp, [\tsk, #THREAD_MTE_CTRL] 191 192 mte_set_gcr \tmp, \tmp2 1931: 194#endif 195 .endm 196 197 .macro pcpu_gprs_entry, tsk:req, regs:req, tmp:req 198 ldrh w\tmp, [\tsk, #TSK_TI_PCPU_GPRS] 199 strh w\tmp, [\regs, #S_PCPU_GPRS] 200 strh wzr, [\tsk, #TSK_TI_PCPU_GPRS] 201 .endm 202 203 .macro pcpu_gprs_exit, tsk:req, regs:req, tmp:req 204 ldrh w\tmp, [\regs, #S_PCPU_GPRS] 205 strh w\tmp, [\tsk, #TSK_TI_PCPU_GPRS] 206 .endm 207 208 .macro kernel_entry, el, regsize = 64 209 .if \el == 0 210 alternative_insn nop, SET_PSTATE_DIT(1), ARM64_HAS_DIT 211 .endif 212 .if \regsize == 32 213 mov w0, w0 // zero upper 32 bits of x0 214 .endif 215 stp x0, x1, [sp, #16 * 0] 216 stp x2, x3, [sp, #16 * 1] 217 stp x4, x5, [sp, #16 * 2] 218 stp x6, x7, [sp, #16 * 3] 219 stp x8, x9, [sp, #16 * 4] 220 stp x10, x11, [sp, #16 * 5] 221 stp x12, x13, [sp, #16 * 6] 222 stp x14, x15, [sp, #16 * 7] 223 stp x16, x17, [sp, #16 * 8] 224 stp x18, x19, [sp, #16 * 9] 225 stp x20, x21, [sp, #16 * 10] 226 stp x22, x23, [sp, #16 * 11] 227 stp x24, x25, [sp, #16 * 12] 228 stp x26, x27, [sp, #16 * 13] 229 stp x28, x29, [sp, #16 * 14] 230 231 .if \el == 0 232 clear_gp_regs 233 mrs x21, sp_el0 234 ldr_this_cpu tsk, __entry_task, x20 235 msr sp_el0, tsk 236 237 /* 238 * Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions 239 * when scheduling. 240 */ 241 ldr x19, [tsk, #TSK_TI_FLAGS] 242 disable_step_tsk x19, x20 243 244 /* Check for asynchronous tag check faults in user space */ 245 ldr x0, [tsk, THREAD_SCTLR_USER] 246 check_mte_async_tcf x22, x23, x0 247 248#ifdef CONFIG_ARM64_PTR_AUTH 249alternative_if ARM64_HAS_ADDRESS_AUTH 250 /* 251 * Enable IA for in-kernel PAC if the task had it disabled. Although 252 * this could be implemented with an unconditional MRS which would avoid 253 * a load, this was measured to be slower on Cortex-A75 and Cortex-A76. 254 * 255 * Install the kernel IA key only if IA was enabled in the task. If IA 256 * was disabled on kernel exit then we would have left the kernel IA 257 * installed so there is no need to install it again. 258 */ 259 tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f 260 __ptrauth_keys_install_kernel_nosync tsk, x20, x22, x23 261 b 2f 2621: 263 mrs x0, sctlr_el1 264 orr x0, x0, SCTLR_ELx_ENIA 265 msr sctlr_el1, x0 2662: 267alternative_else_nop_endif 268#endif 269 270 apply_ssbd 1, x22, x23 271 272 mte_set_kernel_gcr x22, x23 273 274 /* 275 * Any non-self-synchronizing system register updates required for 276 * kernel entry should be placed before this point. 277 */ 278alternative_if ARM64_MTE 279 isb 280 b 1f 281alternative_else_nop_endif 282alternative_if ARM64_HAS_ADDRESS_AUTH 283 isb 284alternative_else_nop_endif 2851: 286 287 scs_load_current_base 288 .else 289 add x21, sp, #PT_REGS_SIZE 290 get_current_task tsk 291 pcpu_gprs_entry tsk, sp, x0 292 .endif /* \el == 0 */ 293 mrs x22, elr_el1 294 mrs x23, spsr_el1 295 stp lr, x21, [sp, #S_LR] 296 297 /* 298 * Create a metadata frame record. The unwinder will use this to 299 * identify and unwind exception boundaries. 300 */ 301 stp xzr, xzr, [sp, #S_STACKFRAME] 302 .if \el == 0 303 mov x0, #FRAME_META_TYPE_FINAL 304 .else 305 mov x0, #FRAME_META_TYPE_PT_REGS 306 .endif 307 str x0, [sp, #S_STACKFRAME_TYPE] 308 add x29, sp, #S_STACKFRAME 309 310#ifdef CONFIG_ARM64_SW_TTBR0_PAN 311alternative_if_not ARM64_HAS_PAN 312 bl __swpan_entry_el\el 313alternative_else_nop_endif 314#endif 315 316 stp x22, x23, [sp, #S_PC] 317 318 /* Not in a syscall by default (el0_svc overwrites for real syscall) */ 319 .if \el == 0 320 mov w21, #NO_SYSCALL 321 str w21, [sp, #S_SYSCALLNO] 322 .endif 323 324#ifdef CONFIG_ARM64_PSEUDO_NMI 325alternative_if_not ARM64_HAS_GIC_PRIO_MASKING 326 b .Lskip_pmr_save\@ 327alternative_else_nop_endif 328 329 mrs_s x20, SYS_ICC_PMR_EL1 330 str w20, [sp, #S_PMR] 331 mov x20, #GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET 332 msr_s SYS_ICC_PMR_EL1, x20 333 334.Lskip_pmr_save\@: 335#endif 336 337 /* 338 * Registers that may be useful after this macro is invoked: 339 * 340 * x20 - ICC_PMR_EL1 341 * x21 - aborted SP 342 * x22 - aborted PC 343 * x23 - aborted PSTATE 344 */ 345 .endm 346 347 .macro kernel_exit, el 348 .if \el != 0 349 pcpu_gprs_exit tsk, sp, x0 350 .endif 351 352#ifdef CONFIG_ARM64_PSEUDO_NMI 353alternative_if_not ARM64_HAS_GIC_PRIO_MASKING 354 b .Lskip_pmr_restore\@ 355alternative_else_nop_endif 356 357 ldr w20, [sp, #S_PMR] 358 msr_s SYS_ICC_PMR_EL1, x20 359 360 /* Ensure priority change is seen by redistributor */ 361alternative_if_not ARM64_HAS_GIC_PRIO_RELAXED_SYNC 362 dsb sy 363alternative_else_nop_endif 364 365.Lskip_pmr_restore\@: 366#endif 367 368 ldp x21, x22, [sp, #S_PC] // load ELR, SPSR 369 370#ifdef CONFIG_ARM64_SW_TTBR0_PAN 371alternative_if_not ARM64_HAS_PAN 372 bl __swpan_exit_el\el 373alternative_else_nop_endif 374#endif 375 376 .if \el == 0 377 ldr x23, [sp, #S_SP] // load return stack pointer 378 msr sp_el0, x23 379 tst x22, #PSR_MODE32_BIT // native task? 380 b.eq 3f 381 382#ifdef CONFIG_ARM64_ERRATUM_845719 383alternative_if ARM64_WORKAROUND_845719 384#ifdef CONFIG_PID_IN_CONTEXTIDR 385 mrs x29, contextidr_el1 386 msr contextidr_el1, x29 387#else 388 msr contextidr_el1, xzr 389#endif 390alternative_else_nop_endif 391#endif 3923: 393 /* Ignore asynchronous tag check faults in the uaccess routines */ 394 ldr x0, [tsk, THREAD_SCTLR_USER] 395 clear_mte_async_tcf x0 396 397#ifdef CONFIG_ARM64_PTR_AUTH 398alternative_if ARM64_HAS_ADDRESS_AUTH 399 /* 400 * IA was enabled for in-kernel PAC. Disable it now if needed, or 401 * alternatively install the user's IA. All other per-task keys and 402 * SCTLR bits were updated on task switch. 403 * 404 * No kernel C function calls after this. 405 */ 406 tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f 407 __ptrauth_keys_install_user tsk, x0, x1, x2 408 b 2f 4091: 410 mrs x0, sctlr_el1 411 bic x0, x0, SCTLR_ELx_ENIA 412 msr sctlr_el1, x0 4132: 414alternative_else_nop_endif 415#endif 416 417 mte_set_user_gcr tsk, x0, x1 418 419 apply_ssbd 0, x0, x1 420 .endif 421 422 msr elr_el1, x21 // set up the return data 423 msr spsr_el1, x22 424 ldp x0, x1, [sp, #16 * 0] 425 ldp x2, x3, [sp, #16 * 1] 426 ldp x4, x5, [sp, #16 * 2] 427 ldp x6, x7, [sp, #16 * 3] 428 ldp x8, x9, [sp, #16 * 4] 429 ldp x10, x11, [sp, #16 * 5] 430 ldp x12, x13, [sp, #16 * 6] 431 ldp x14, x15, [sp, #16 * 7] 432 ldp x16, x17, [sp, #16 * 8] 433 ldp x18, x19, [sp, #16 * 9] 434 ldp x20, x21, [sp, #16 * 10] 435 ldp x22, x23, [sp, #16 * 11] 436 ldp x24, x25, [sp, #16 * 12] 437 ldp x26, x27, [sp, #16 * 13] 438 ldp x28, x29, [sp, #16 * 14] 439 440 .if \el == 0 441#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 442 alternative_insn "b .L_skip_tramp_exit_\@", nop, ARM64_UNMAP_KERNEL_AT_EL0 443 444 msr far_el1, x29 445 446 ldr_this_cpu x30, this_cpu_vector, x29 447 tramp_alias x29, tramp_exit 448 msr vbar_el1, x30 // install vector table 449 ldr lr, [sp, #S_LR] // restore x30 450 add sp, sp, #PT_REGS_SIZE // restore sp 451 br x29 452 453.L_skip_tramp_exit_\@: 454#endif 455 .endif 456 457 ldr lr, [sp, #S_LR] 458 add sp, sp, #PT_REGS_SIZE // restore sp 459 460 .if \el == 0 461 /* This must be after the last explicit memory access */ 462alternative_if ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD 463 tlbi vale1, xzr 464 dsb nsh 465alternative_else_nop_endif 466 .else 467 /* Ensure any device/NC reads complete */ 468 alternative_insn nop, "dmb sy", ARM64_WORKAROUND_1508412 469 .endif 470 471 eret 472 sb 473 .endm 474 475#ifdef CONFIG_ARM64_SW_TTBR0_PAN 476 /* 477 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from 478 * EL0, there is no need to check the state of TTBR0_EL1 since 479 * accesses are always enabled. 480 * Note that the meaning of this bit differs from the ARMv8.1 PAN 481 * feature as all TTBR0_EL1 accesses are disabled, not just those to 482 * user mappings. 483 */ 484SYM_CODE_START_LOCAL(__swpan_entry_el1) 485 mrs x21, ttbr0_el1 486 tst x21, #TTBRx_EL1_ASID_MASK // Check for the reserved ASID 487 orr x23, x23, #PSR_PAN_BIT // Set the emulated PAN in the saved SPSR 488 b.eq 1f // TTBR0 access already disabled 489 and x23, x23, #~PSR_PAN_BIT // Clear the emulated PAN in the saved SPSR 490SYM_INNER_LABEL(__swpan_entry_el0, SYM_L_LOCAL) 491 __uaccess_ttbr0_disable x21 4921: ret 493SYM_CODE_END(__swpan_entry_el1) 494 495 /* 496 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR 497 * PAN bit checking. 498 */ 499SYM_CODE_START_LOCAL(__swpan_exit_el1) 500 tbnz x22, #22, 1f // Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set 501 __uaccess_ttbr0_enable x0, x1 5021: and x22, x22, #~PSR_PAN_BIT // ARMv8.0 CPUs do not understand this bit 503 ret 504SYM_CODE_END(__swpan_exit_el1) 505 506SYM_CODE_START_LOCAL(__swpan_exit_el0) 507 __uaccess_ttbr0_enable x0, x1 508 /* 509 * Enable errata workarounds only if returning to user. The only 510 * workaround currently required for TTBR0_EL1 changes are for the 511 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache 512 * corruption). 513 */ 514 b post_ttbr_update_workaround 515SYM_CODE_END(__swpan_exit_el0) 516#endif 517 518/* GPRs used by entry code */ 519tsk .req x28 // current thread_info 520 521 .text 522 523/* 524 * Exception vectors. 525 */ 526 .pushsection ".entry.text", "ax" 527 528 .align 11 529SYM_CODE_START(vectors) 530 kernel_ventry 1, t, 64, sync // Synchronous EL1t 531 kernel_ventry 1, t, 64, irq // IRQ EL1t 532 kernel_ventry 1, t, 64, fiq // FIQ EL1t 533 kernel_ventry 1, t, 64, error // Error EL1t 534 535 kernel_ventry 1, h, 64, sync // Synchronous EL1h 536 kernel_ventry 1, h, 64, irq // IRQ EL1h 537 kernel_ventry 1, h, 64, fiq // FIQ EL1h 538 kernel_ventry 1, h, 64, error // Error EL1h 539 540 kernel_ventry 0, t, 64, sync // Synchronous 64-bit EL0 541 kernel_ventry 0, t, 64, irq // IRQ 64-bit EL0 542 kernel_ventry 0, t, 64, fiq // FIQ 64-bit EL0 543 kernel_ventry 0, t, 64, error // Error 64-bit EL0 544 545 kernel_ventry 0, t, 32, sync // Synchronous 32-bit EL0 546 kernel_ventry 0, t, 32, irq // IRQ 32-bit EL0 547 kernel_ventry 0, t, 32, fiq // FIQ 32-bit EL0 548 kernel_ventry 0, t, 32, error // Error 32-bit EL0 549SYM_CODE_END(vectors) 550 551SYM_CODE_START_LOCAL(__bad_stack) 552 /* 553 * We detected an overflow in kernel_ventry, which switched to the 554 * overflow stack. Stash the exception regs, and head to our overflow 555 * handler. 556 */ 557 558 /* Restore the original x0 value */ 559 mrs x0, tpidrro_el0 560 561 /* 562 * Store the original GPRs to the new stack. The orginal SP (minus 563 * PT_REGS_SIZE) was stashed in tpidr_el0 by kernel_ventry. 564 */ 565 sub sp, sp, #PT_REGS_SIZE 566 kernel_entry 1 567 mrs x0, tpidr_el0 568 add x0, x0, #PT_REGS_SIZE 569 str x0, [sp, #S_SP] 570 571 /* Stash the regs for handle_bad_stack */ 572 mov x0, sp 573 574 /* Time to die */ 575 bl handle_bad_stack 576 ASM_BUG() 577SYM_CODE_END(__bad_stack) 578 579 580 .macro entry_handler el:req, ht:req, regsize:req, label:req 581SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label) 582 kernel_entry \el, \regsize 583 mov x0, sp 584 bl el\el\ht\()_\regsize\()_\label\()_handler 585 .if \el == 0 586 b ret_to_user 587 .else 588 b ret_to_kernel 589 .endif 590SYM_CODE_END(el\el\ht\()_\regsize\()_\label) 591 .endm 592 593/* 594 * Early exception handlers 595 */ 596 entry_handler 1, t, 64, sync 597 entry_handler 1, t, 64, irq 598 entry_handler 1, t, 64, fiq 599 entry_handler 1, t, 64, error 600 601 entry_handler 1, h, 64, sync 602 entry_handler 1, h, 64, irq 603 entry_handler 1, h, 64, fiq 604 entry_handler 1, h, 64, error 605 606 entry_handler 0, t, 64, sync 607 entry_handler 0, t, 64, irq 608 entry_handler 0, t, 64, fiq 609 entry_handler 0, t, 64, error 610 611 entry_handler 0, t, 32, sync 612 entry_handler 0, t, 32, irq 613 entry_handler 0, t, 32, fiq 614 entry_handler 0, t, 32, error 615 616SYM_CODE_START_LOCAL(ret_to_kernel) 617 kernel_exit 1 618SYM_CODE_END(ret_to_kernel) 619 620SYM_CODE_START_LOCAL(ret_to_user) 621 ldr x19, [tsk, #TSK_TI_FLAGS] // re-check for single-step 622 enable_step_tsk x19, x2 623#ifdef CONFIG_KSTACK_ERASE 624 bl stackleak_erase_on_task_stack 625#endif 626 kernel_exit 0 627SYM_CODE_END(ret_to_user) 628 629 .popsection // .entry.text 630 631 // Move from tramp_pg_dir to swapper_pg_dir 632 .macro tramp_map_kernel, tmp 633 mrs \tmp, ttbr1_el1 634 add \tmp, \tmp, #TRAMP_SWAPPER_OFFSET 635 bic \tmp, \tmp, #USER_ASID_FLAG 636 msr ttbr1_el1, \tmp 637#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003 638alternative_if ARM64_WORKAROUND_QCOM_FALKOR_E1003 639 /* ASID already in \tmp[63:48] */ 640 movk \tmp, #:abs_g2_nc:(TRAMP_VALIAS >> 12) 641 movk \tmp, #:abs_g1_nc:(TRAMP_VALIAS >> 12) 642 /* 2MB boundary containing the vectors, so we nobble the walk cache */ 643 movk \tmp, #:abs_g0_nc:((TRAMP_VALIAS & ~(SZ_2M - 1)) >> 12) 644 isb 645 tlbi vae1, \tmp 646 dsb nsh 647alternative_else_nop_endif 648#endif /* CONFIG_QCOM_FALKOR_ERRATUM_1003 */ 649 .endm 650 651 // Move from swapper_pg_dir to tramp_pg_dir 652 .macro tramp_unmap_kernel, tmp 653 mrs \tmp, ttbr1_el1 654 sub \tmp, \tmp, #TRAMP_SWAPPER_OFFSET 655 orr \tmp, \tmp, #USER_ASID_FLAG 656 msr ttbr1_el1, \tmp 657 /* 658 * We avoid running the post_ttbr_update_workaround here because 659 * it's only needed by Cavium ThunderX, which requires KPTI to be 660 * disabled. 661 */ 662 .endm 663 664 .macro tramp_data_read_var dst, var 665#ifdef CONFIG_RELOCATABLE 666 ldr \dst, .L__tramp_data_\var 667 .ifndef .L__tramp_data_\var 668 .pushsection ".entry.tramp.rodata", "a", %progbits 669 .align 3 670.L__tramp_data_\var: 671 .quad \var 672 .popsection 673 .endif 674#else 675 /* 676 * As !RELOCATABLE implies !RANDOMIZE_BASE the address is always a 677 * compile time constant (and hence not secret and not worth hiding). 678 * 679 * As statically allocated kernel code and data always live in the top 680 * 47 bits of the address space we can sign-extend bit 47 and avoid an 681 * instruction to load the upper 16 bits (which must be 0xFFFF). 682 */ 683 movz \dst, :abs_g2_s:\var 684 movk \dst, :abs_g1_nc:\var 685 movk \dst, :abs_g0_nc:\var 686#endif 687 .endm 688 689#define BHB_MITIGATION_NONE 0 690#define BHB_MITIGATION_LOOP 1 691#define BHB_MITIGATION_FW 2 692#define BHB_MITIGATION_INSN 3 693 694 .macro tramp_ventry, vector_start, regsize, kpti, bhb 695 .align 7 6961: 697 .if \regsize == 64 698 msr tpidrro_el0, x30 // Restored in kernel_ventry 699 .endif 700 701 .if \bhb == BHB_MITIGATION_LOOP 702 /* 703 * This sequence must appear before the first indirect branch. i.e. the 704 * ret out of tramp_ventry. It appears here because x30 is free. 705 */ 706 __mitigate_spectre_bhb_loop x30 707 .endif // \bhb == BHB_MITIGATION_LOOP 708 709 .if \bhb == BHB_MITIGATION_INSN 710 clearbhb 711 isb 712 .endif // \bhb == BHB_MITIGATION_INSN 713 714 .if \kpti == 1 715 /* 716 * Defend against branch aliasing attacks by pushing a dummy 717 * entry onto the return stack and using a RET instruction to 718 * enter the full-fat kernel vectors. 719 */ 720 bl 2f 721 b . 7222: 723 tramp_map_kernel x30 724alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003 725 tramp_data_read_var x30, vectors 726alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM 727 prfm plil1strm, [x30, #(1b - \vector_start)] 728alternative_else_nop_endif 729 730 msr vbar_el1, x30 731 isb 732 .else 733 adr_l x30, vectors 734 .endif // \kpti == 1 735 736 .if \bhb == BHB_MITIGATION_FW 737 /* 738 * The firmware sequence must appear before the first indirect branch. 739 * i.e. the ret out of tramp_ventry. But it also needs the stack to be 740 * mapped to save/restore the registers the SMC clobbers. 741 */ 742 __mitigate_spectre_bhb_fw 743 .endif // \bhb == BHB_MITIGATION_FW 744 745 add x30, x30, #(1b - \vector_start + 4) 746 ret 747.org 1b + 128 // Did we overflow the ventry slot? 748 .endm 749 750 .macro generate_tramp_vector, kpti, bhb 751.Lvector_start\@: 752 .space 0x400 753 754 .rept 4 755 tramp_ventry .Lvector_start\@, 64, \kpti, \bhb 756 .endr 757 .rept 4 758 tramp_ventry .Lvector_start\@, 32, \kpti, \bhb 759 .endr 760 .endm 761 762#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 763/* 764 * Exception vectors trampoline. 765 * The order must match __bp_harden_el1_vectors and the 766 * arm64_bp_harden_el1_vectors enum. 767 */ 768 .pushsection ".entry.tramp.text", "ax" 769 .align 11 770SYM_CODE_START_LOCAL_NOALIGN(tramp_vectors) 771#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 772 generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP 773 generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW 774 generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_INSN 775#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 776 generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE 777SYM_CODE_END(tramp_vectors) 778 779SYM_CODE_START_LOCAL(tramp_exit) 780 tramp_unmap_kernel x29 781 mrs x29, far_el1 // restore x29 782 eret 783 sb 784SYM_CODE_END(tramp_exit) 785 .popsection // .entry.tramp.text 786#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ 787 788/* 789 * Exception vectors for spectre mitigations on entry from EL1 when 790 * kpti is not in use. 791 */ 792 .macro generate_el1_vector, bhb 793.Lvector_start\@: 794 kernel_ventry 1, t, 64, sync // Synchronous EL1t 795 kernel_ventry 1, t, 64, irq // IRQ EL1t 796 kernel_ventry 1, t, 64, fiq // FIQ EL1h 797 kernel_ventry 1, t, 64, error // Error EL1t 798 799 kernel_ventry 1, h, 64, sync // Synchronous EL1h 800 kernel_ventry 1, h, 64, irq // IRQ EL1h 801 kernel_ventry 1, h, 64, fiq // FIQ EL1h 802 kernel_ventry 1, h, 64, error // Error EL1h 803 804 .rept 4 805 tramp_ventry .Lvector_start\@, 64, 0, \bhb 806 .endr 807 .rept 4 808 tramp_ventry .Lvector_start\@, 32, 0, \bhb 809 .endr 810 .endm 811 812/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */ 813 .pushsection ".entry.text", "ax" 814 .align 11 815SYM_CODE_START(__bp_harden_el1_vectors) 816#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 817 generate_el1_vector bhb=BHB_MITIGATION_LOOP 818 generate_el1_vector bhb=BHB_MITIGATION_FW 819 generate_el1_vector bhb=BHB_MITIGATION_INSN 820#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 821SYM_CODE_END(__bp_harden_el1_vectors) 822 .popsection 823 824 825/* 826 * Register switch for AArch64. The callee-saved registers need to be saved 827 * and restored. On entry: 828 * x0 = previous task_struct (must be preserved across the switch) 829 * x1 = next task_struct 830 * Previous and next are guaranteed not to be the same. 831 * 832 */ 833SYM_FUNC_START(cpu_switch_to) 834 save_and_disable_daif x11 835 mov x10, #THREAD_CPU_CONTEXT 836 add x8, x0, x10 837 mov x9, sp 838 stp x19, x20, [x8], #16 // store callee-saved registers 839 stp x21, x22, [x8], #16 840 stp x23, x24, [x8], #16 841 stp x25, x26, [x8], #16 842 stp x27, x28, [x8], #16 843 stp x29, x9, [x8], #16 844 str lr, [x8] 845 add x8, x1, x10 846 ldp x19, x20, [x8], #16 // restore callee-saved registers 847 ldp x21, x22, [x8], #16 848 ldp x23, x24, [x8], #16 849 ldp x25, x26, [x8], #16 850 ldp x27, x28, [x8], #16 851 ldp x29, x9, [x8], #16 852 ldr lr, [x8] 853 mov sp, x9 854 msr sp_el0, x1 855 ptrauth_keys_install_kernel x1, x8, x9, x10 856 scs_save x0 857 scs_load_current 858 restore_irq x11 859 ret 860SYM_FUNC_END(cpu_switch_to) 861NOKPROBE(cpu_switch_to) 862 863/* 864 * This is how we return from a fork. 865 */ 866SYM_CODE_START(ret_from_fork) 867 bl schedule_tail 868 cbz x19, 1f // not a kernel thread 869 mov x0, x20 870 blr x19 8711: get_current_task tsk 872 mov x0, sp 873 bl asm_exit_to_user_mode 874 b ret_to_user 875SYM_CODE_END(ret_from_fork) 876NOKPROBE(ret_from_fork) 877 878/* 879 * void call_on_irq_stack(struct pt_regs *regs, 880 * void (*func)(struct pt_regs *)); 881 * 882 * Calls func(regs) using this CPU's irq stack and shadow irq stack. 883 */ 884SYM_FUNC_START(call_on_irq_stack) 885 save_and_disable_daif x9 886#ifdef CONFIG_SHADOW_CALL_STACK 887 get_current_task x16 888 scs_save x16 889 ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x17 890#endif 891 892 /* Create a frame record to save our LR and SP (implicit in FP) */ 893 stp x29, x30, [sp, #-16]! 894 mov x29, sp 895 896 ldr_this_cpu x16, irq_stack_ptr, x17 897 898 /* Move to the new stack and call the function there */ 899 add sp, x16, #IRQ_STACK_SIZE 900 restore_irq x9 901 blr x1 902 903 save_and_disable_daif x9 904 /* 905 * Restore the SP from the FP, and restore the FP and LR from the frame 906 * record. 907 */ 908 mov sp, x29 909 ldp x29, x30, [sp], #16 910 scs_load_current 911 restore_irq x9 912 ret 913SYM_FUNC_END(call_on_irq_stack) 914NOKPROBE(call_on_irq_stack) 915 916#ifdef CONFIG_ARM_SDE_INTERFACE 917 918#include <asm/sdei.h> 919#include <uapi/linux/arm_sdei.h> 920 921.macro sdei_handler_exit exit_mode 922 /* On success, this call never returns... */ 923 cmp \exit_mode, #SDEI_EXIT_SMC 924 b.ne 99f 925 smc #0 926 b . 92799: hvc #0 928 b . 929.endm 930 931#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 932/* 933 * The regular SDEI entry point may have been unmapped along with the rest of 934 * the kernel. This trampoline restores the kernel mapping to make the x1 memory 935 * argument accessible. 936 * 937 * This clobbers x4, __sdei_handler() will restore this from firmware's 938 * copy. 939 */ 940.pushsection ".entry.tramp.text", "ax" 941SYM_CODE_START(__sdei_asm_entry_trampoline) 942 mrs x4, ttbr1_el1 943 tbz x4, #USER_ASID_BIT, 1f 944 945 tramp_map_kernel tmp=x4 946 isb 947 mov x4, xzr 948 949 /* 950 * Remember whether to unmap the kernel on exit. 951 */ 9521: str x4, [x1, #(SDEI_EVENT_INTREGS + S_SDEI_TTBR1)] 953 tramp_data_read_var x4, __sdei_asm_handler 954 br x4 955SYM_CODE_END(__sdei_asm_entry_trampoline) 956NOKPROBE(__sdei_asm_entry_trampoline) 957 958/* 959 * Make the exit call and restore the original ttbr1_el1 960 * 961 * x0 & x1: setup for the exit API call 962 * x2: exit_mode 963 * x4: struct sdei_registered_event argument from registration time. 964 */ 965SYM_CODE_START(__sdei_asm_exit_trampoline) 966 ldr x4, [x4, #(SDEI_EVENT_INTREGS + S_SDEI_TTBR1)] 967 cbnz x4, 1f 968 969 tramp_unmap_kernel tmp=x4 970 9711: sdei_handler_exit exit_mode=x2 972SYM_CODE_END(__sdei_asm_exit_trampoline) 973NOKPROBE(__sdei_asm_exit_trampoline) 974.popsection // .entry.tramp.text 975#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ 976 977/* 978 * Software Delegated Exception entry point. 979 * 980 * x0: Event number 981 * x1: struct sdei_registered_event argument from registration time. 982 * x2: interrupted PC 983 * x3: interrupted PSTATE 984 * x4: maybe clobbered by the trampoline 985 * 986 * Firmware has preserved x0->x17 for us, we must save/restore the rest to 987 * follow SMC-CC. We save (or retrieve) all the registers as the handler may 988 * want them. 989 */ 990SYM_CODE_START(__sdei_asm_handler) 991 stp x2, x3, [x1, #SDEI_EVENT_INTREGS + S_PC] 992 stp x4, x5, [x1, #SDEI_EVENT_INTREGS + 16 * 2] 993 stp x6, x7, [x1, #SDEI_EVENT_INTREGS + 16 * 3] 994 stp x8, x9, [x1, #SDEI_EVENT_INTREGS + 16 * 4] 995 stp x10, x11, [x1, #SDEI_EVENT_INTREGS + 16 * 5] 996 stp x12, x13, [x1, #SDEI_EVENT_INTREGS + 16 * 6] 997 stp x14, x15, [x1, #SDEI_EVENT_INTREGS + 16 * 7] 998 stp x16, x17, [x1, #SDEI_EVENT_INTREGS + 16 * 8] 999 stp x18, x19, [x1, #SDEI_EVENT_INTREGS + 16 * 9] 1000 stp x20, x21, [x1, #SDEI_EVENT_INTREGS + 16 * 10] 1001 stp x22, x23, [x1, #SDEI_EVENT_INTREGS + 16 * 11] 1002 stp x24, x25, [x1, #SDEI_EVENT_INTREGS + 16 * 12] 1003 stp x26, x27, [x1, #SDEI_EVENT_INTREGS + 16 * 13] 1004 stp x28, x29, [x1, #SDEI_EVENT_INTREGS + 16 * 14] 1005 mov x4, sp 1006 stp lr, x4, [x1, #SDEI_EVENT_INTREGS + S_LR] 1007 1008 mov x19, x1 1009 1010 /* Store the registered-event for crash_smp_send_stop() */ 1011 ldrb w4, [x19, #SDEI_EVENT_PRIORITY] 1012 cbnz w4, 1f 1013 adr_this_cpu dst=x5, sym=sdei_active_normal_event, tmp=x6 1014 b 2f 10151: adr_this_cpu dst=x5, sym=sdei_active_critical_event, tmp=x6 10162: str x19, [x5] 1017 1018 /* 1019 * entry.S may have been using sp as a scratch register, find whether 1020 * this is a normal or critical event and switch to the appropriate 1021 * stack for this CPU. 1022 */ 1023 cbnz w4, 1f 1024 ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6 1025 b 2f 10261: ldr_this_cpu dst=x5, sym=sdei_stack_critical_ptr, tmp=x6 10272: mov x6, #SDEI_STACK_SIZE 1028 add x5, x5, x6 1029 mov sp, x5 1030 1031#ifdef CONFIG_SHADOW_CALL_STACK 1032 /* Use a separate shadow call stack for normal and critical events */ 1033 cbnz w4, 3f 1034 ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6 1035 b 4f 10363: ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6 10374: 1038#endif 1039 1040 /* 1041 * We may have interrupted userspace, or a guest, or exit-from or 1042 * return-to either of these. Preserve the interrupted sp_el0, and 1043 * initialize sp_el0 to the current task. 1044 */ 1045 mrs x20, sp_el0 1046 ldr_this_cpu dst=tsk, sym=__entry_task, tmp=x1 1047 msr sp_el0, tsk 1048 1049 /* If we interrupted the kernel point to the previous stack/frame. */ 1050 and x0, x3, #0xc 1051 mrs x1, CurrentEL 1052 cmp x0, x1 1053 csel x29, x29, xzr, eq // fp, or zero 1054 csel x4, x2, xzr, eq // elr, or zero 1055 1056 stp x29, x4, [sp, #-16]! 1057 mov x29, sp 1058 1059 add x16, x19, #SDEI_EVENT_INTREGS 1060 pcpu_gprs_entry tsk, x16, x17 1061 1062 add x0, x19, #SDEI_EVENT_INTREGS 1063 mov x1, x19 1064 bl __sdei_handler 1065 1066 add x16, x19, #SDEI_EVENT_INTREGS 1067 pcpu_gprs_exit tsk, x16, x17 1068 1069 msr sp_el0, x20 1070 /* restore regs >x17 that firmware won't restore */ 1071 mov x4, x19 // keep x4 for __sdei_asm_exit_trampoline 1072 ldp x18, x19, [x4, #SDEI_EVENT_INTREGS + 16 * 9] 1073 ldp x20, x21, [x4, #SDEI_EVENT_INTREGS + 16 * 10] 1074 ldp x22, x23, [x4, #SDEI_EVENT_INTREGS + 16 * 11] 1075 ldp x24, x25, [x4, #SDEI_EVENT_INTREGS + 16 * 12] 1076 ldp x26, x27, [x4, #SDEI_EVENT_INTREGS + 16 * 13] 1077 ldp x28, x29, [x4, #SDEI_EVENT_INTREGS + 16 * 14] 1078 ldp lr, x1, [x4, #SDEI_EVENT_INTREGS + S_LR] 1079 mov sp, x1 1080 1081 mov x1, x0 // address to complete_and_resume 1082 /* x0 = (x0 <= SDEI_EV_FAILED) ? 1083 * EVENT_COMPLETE:EVENT_COMPLETE_AND_RESUME 1084 */ 1085 cmp x0, #SDEI_EV_FAILED 1086 mov_q x2, SDEI_1_0_FN_SDEI_EVENT_COMPLETE 1087 mov_q x3, SDEI_1_0_FN_SDEI_EVENT_COMPLETE_AND_RESUME 1088 csel x0, x2, x3, ls 1089 1090 ldr_l x2, sdei_exit_mode 1091 1092 /* Clear the registered-event seen by crash_smp_send_stop() */ 1093 ldrb w3, [x4, #SDEI_EVENT_PRIORITY] 1094 cbnz w3, 1f 1095 adr_this_cpu dst=x5, sym=sdei_active_normal_event, tmp=x6 1096 b 2f 10971: adr_this_cpu dst=x5, sym=sdei_active_critical_event, tmp=x6 10982: str xzr, [x5] 1099 1100alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 1101 sdei_handler_exit exit_mode=x2 1102alternative_else_nop_endif 1103 1104#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 1105 tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline 1106 br x5 1107#endif 1108SYM_CODE_END(__sdei_asm_handler) 1109NOKPROBE(__sdei_asm_handler) 1110 1111SYM_CODE_START(__sdei_handler_abort) 1112 mov_q x0, SDEI_1_0_FN_SDEI_EVENT_COMPLETE_AND_RESUME 1113 adr x1, 1f 1114 ldr_l x2, sdei_exit_mode 1115 sdei_handler_exit exit_mode=x2 1116 // exit the handler and jump to the next instruction. 1117 // Exit will stomp x0-x17, PSTATE, ELR_ELx, and SPSR_ELx. 11181: ret 1119SYM_CODE_END(__sdei_handler_abort) 1120NOKPROBE(__sdei_handler_abort) 1121#endif /* CONFIG_ARM_SDE_INTERFACE */ 1122