1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 4 * 5 * Copyright (C) 1996-2000 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASSEMBLER__ 9 #error "Only include this from assembly code" 10 #endif 11 12 #ifndef __ASM_ASSEMBLER_H 13 #define __ASM_ASSEMBLER_H 14 15 #include <linux/export.h> 16 17 #include <asm/alternative.h> 18 #include <asm/asm-bug.h> 19 #include <asm/asm-extable.h> 20 #include <asm/asm-offsets.h> 21 #include <asm/cpufeature.h> 22 #include <asm/cputype.h> 23 #include <asm/debug-monitors.h> 24 #include <asm/page.h> 25 #include <asm/pgtable-hwdef.h> 26 #include <asm/ptrace.h> 27 #include <asm/thread_info.h> 28 29 /* 30 * Provide a wxN alias for each wN register so what we can paste a xN 31 * reference after a 'w' to obtain the 32-bit version. 32 */ 33 .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 34 wx\n .req w\n 35 .endr 36 37 /* 38 * Save/restore interrupts. 39 */ 40 .macro save_and_disable_daif, flags 41 mrs \flags, daif 42 msr daifset, #0xf 43 .endm 44 45 .macro save_and_disable_irq, flags 46 mrs \flags, daif 47 msr daifset, #3 48 .endm 49 50 .macro restore_irq, flags 51 msr daif, \flags 52 .endm 53 54 .macro disable_step_tsk, flgs, tmp 55 tbz \flgs, #TIF_SINGLESTEP, 9990f 56 mrs \tmp, mdscr_el1 57 bic \tmp, \tmp, #MDSCR_EL1_SS 58 msr mdscr_el1, \tmp 59 isb // Take effect before a subsequent clear of DAIF.D 60 9990: 61 .endm 62 63 /* call with daif masked */ 64 .macro enable_step_tsk, flgs, tmp 65 tbz \flgs, #TIF_SINGLESTEP, 9990f 66 mrs \tmp, mdscr_el1 67 orr \tmp, \tmp, #MDSCR_EL1_SS 68 msr mdscr_el1, \tmp 69 9990: 70 .endm 71 72 /* 73 * RAS Error Synchronization barrier 74 */ 75 .macro esb 76 #ifdef CONFIG_ARM64_RAS_EXTN 77 hint #16 78 #else 79 nop 80 #endif 81 .endm 82 83 /* 84 * Value prediction barrier 85 */ 86 .macro csdb 87 hint #20 88 .endm 89 90 /* 91 * Clear Branch History instruction 92 */ 93 .macro clearbhb 94 hint #22 95 .endm 96 97 /* 98 * Speculation barrier 99 */ 100 .macro sb 101 alternative_if_not ARM64_HAS_SB 102 dsb nsh 103 isb 104 alternative_else 105 SB_BARRIER_INSN 106 nop 107 alternative_endif 108 .endm 109 110 /* 111 * NOP sequence 112 */ 113 .macro nops, num 114 .rept \num 115 nop 116 .endr 117 .endm 118 119 /* 120 * Register aliases. 121 */ 122 lr .req x30 // link register 123 124 /* 125 * Vector entry 126 */ 127 .macro ventry label 128 .align 7 129 b \label 130 .endm 131 132 /* 133 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 134 * <symbol> is within the range +/- 4 GB of the PC. 135 */ 136 /* 137 * @dst: destination register (64 bit wide) 138 * @sym: name of the symbol 139 */ 140 .macro adr_l, dst, sym 141 adrp \dst, \sym 142 add \dst, \dst, :lo12:\sym 143 .endm 144 145 /* 146 * @dst: destination register (32 or 64 bit wide) 147 * @sym: name of the symbol 148 * @tmp: optional 64-bit scratch register to be used if <dst> is a 149 * 32-bit wide register, in which case it cannot be used to hold 150 * the address 151 */ 152 .macro ldr_l, dst, sym, tmp= 153 .ifb \tmp 154 adrp \dst, \sym 155 ldr \dst, [\dst, :lo12:\sym] 156 .else 157 adrp \tmp, \sym 158 ldr \dst, [\tmp, :lo12:\sym] 159 .endif 160 .endm 161 162 /* 163 * @src: source register (32 or 64 bit wide) 164 * @sym: name of the symbol 165 * @tmp: mandatory 64-bit scratch register to calculate the address 166 * while <src> needs to be preserved. 167 */ 168 .macro str_l, src, sym, tmp 169 adrp \tmp, \sym 170 str \src, [\tmp, :lo12:\sym] 171 .endm 172 173 /* 174 * @dst: destination register 175 */ 176 #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__) 177 .macro get_this_cpu_offset, dst 178 mrs \dst, tpidr_el2 179 .endm 180 #else 181 .macro get_this_cpu_offset, dst 182 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 183 mrs \dst, tpidr_el1 184 alternative_else 185 mrs \dst, tpidr_el2 186 alternative_endif 187 .endm 188 189 .macro set_this_cpu_offset, src 190 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 191 msr tpidr_el1, \src 192 alternative_else 193 msr tpidr_el2, \src 194 alternative_endif 195 .endm 196 #endif 197 198 /* 199 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP) 200 * @sym: The name of the per-cpu variable 201 * @tmp: scratch register 202 */ 203 .macro adr_this_cpu, dst, sym, tmp 204 adrp \tmp, \sym 205 add \dst, \tmp, #:lo12:\sym 206 get_this_cpu_offset \tmp 207 add \dst, \dst, \tmp 208 .endm 209 210 /* 211 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 212 * @sym: The name of the per-cpu variable 213 * @tmp: scratch register 214 */ 215 .macro ldr_this_cpu dst, sym, tmp 216 adr_l \dst, \sym 217 get_this_cpu_offset \tmp 218 ldr \dst, [\dst, \tmp] 219 .endm 220 221 /* 222 * read_ctr - read CTR_EL0. If the system has mismatched register fields, 223 * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val 224 */ 225 .macro read_ctr, reg 226 #ifndef __KVM_NVHE_HYPERVISOR__ 227 alternative_if_not ARM64_MISMATCHED_CACHE_TYPE 228 mrs \reg, ctr_el0 // read CTR 229 nop 230 alternative_else 231 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 232 alternative_endif 233 #else 234 alternative_if_not ARM64_KVM_PROTECTED_MODE 235 ASM_BUG() 236 alternative_else_nop_endif 237 alternative_cb ARM64_ALWAYS_SYSTEM, kvm_compute_final_ctr_el0 238 movz \reg, #0 239 movk \reg, #0, lsl #16 240 movk \reg, #0, lsl #32 241 movk \reg, #0, lsl #48 242 alternative_cb_end 243 #endif 244 .endm 245 246 247 /* 248 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 249 * from the CTR register. 250 */ 251 .macro raw_dcache_line_size, reg, tmp 252 mrs \tmp, ctr_el0 // read CTR 253 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 254 mov \reg, #4 // bytes per word 255 lsl \reg, \reg, \tmp // actual cache line size 256 .endm 257 258 /* 259 * dcache_line_size - get the safe D-cache line size across all CPUs 260 */ 261 .macro dcache_line_size, reg, tmp 262 read_ctr \tmp 263 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 264 mov \reg, #4 // bytes per word 265 lsl \reg, \reg, \tmp // actual cache line size 266 .endm 267 268 /* 269 * raw_icache_line_size - get the minimum I-cache line size on this CPU 270 * from the CTR register. 271 */ 272 .macro raw_icache_line_size, reg, tmp 273 mrs \tmp, ctr_el0 // read CTR 274 and \tmp, \tmp, #0xf // cache line size encoding 275 mov \reg, #4 // bytes per word 276 lsl \reg, \reg, \tmp // actual cache line size 277 .endm 278 279 /* 280 * icache_line_size - get the safe I-cache line size across all CPUs 281 */ 282 .macro icache_line_size, reg, tmp 283 read_ctr \tmp 284 and \tmp, \tmp, #0xf // cache line size encoding 285 mov \reg, #4 // bytes per word 286 lsl \reg, \reg, \tmp // actual cache line size 287 .endm 288 289 /* 290 * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map 291 */ 292 .macro tcr_set_t0sz, valreg, t0sz 293 bfi \valreg, \t0sz, #TCR_EL1_T0SZ_SHIFT, #TCR_EL1_T0SZ_WIDTH 294 .endm 295 296 /* 297 * tcr_set_t1sz - update TCR.T1SZ 298 */ 299 .macro tcr_set_t1sz, valreg, t1sz 300 bfi \valreg, \t1sz, #TCR_EL1_T1SZ_SHIFT, #TCR_EL1_T1SZ_WIDTH 301 .endm 302 303 /* 304 * tcr_compute_pa_size - set TCR.(I)PS to the highest supported 305 * ID_AA64MMFR0_EL1.PARange value 306 * 307 * tcr: register with the TCR_ELx value to be updated 308 * pos: IPS or PS bitfield position 309 * tmp{0,1}: temporary registers 310 */ 311 .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1 312 mrs \tmp0, ID_AA64MMFR0_EL1 313 // Narrow PARange to fit the PS field in TCR_ELx 314 ubfx \tmp0, \tmp0, #ID_AA64MMFR0_EL1_PARANGE_SHIFT, #3 315 mov \tmp1, #ID_AA64MMFR0_EL1_PARANGE_MAX 316 #ifdef CONFIG_ARM64_LPA2 317 alternative_if_not ARM64_HAS_VA52 318 mov \tmp1, #ID_AA64MMFR0_EL1_PARANGE_48 319 alternative_else_nop_endif 320 #endif 321 cmp \tmp0, \tmp1 322 csel \tmp0, \tmp1, \tmp0, hi 323 bfi \tcr, \tmp0, \pos, #3 324 .endm 325 326 .macro __dcache_op_workaround_clean_cache, op, addr 327 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 328 dc \op, \addr 329 alternative_else 330 dc civac, \addr 331 alternative_endif 332 .endm 333 334 /* 335 * Macro to perform a data cache maintenance for the interval 336 * [start, end) with dcache line size explicitly provided. 337 * 338 * op: operation passed to dc instruction 339 * start: starting virtual address of the region 340 * end: end virtual address of the region 341 * linesz: dcache line size 342 * fixup: optional label to branch to on user fault 343 * Corrupts: start, end, tmp 344 */ 345 .macro dcache_by_myline_op_nosync op, start, end, linesz, tmp, fixup 346 sub \tmp, \linesz, #1 347 bic \start, \start, \tmp 348 alternative_if ARM64_WORKAROUND_4311569 349 mov \tmp, \start 350 alternative_else_nop_endif 351 .Ldcache_op\@: 352 .ifc \op, cvau 353 __dcache_op_workaround_clean_cache \op, \start 354 .else 355 .ifc \op, cvac 356 __dcache_op_workaround_clean_cache \op, \start 357 .else 358 .ifc \op, cvap 359 sys 3, c7, c12, 1, \start // dc cvap 360 .else 361 .ifc \op, cvadp 362 sys 3, c7, c13, 1, \start // dc cvadp 363 .else 364 dc \op, \start 365 .endif 366 .endif 367 .endif 368 .endif 369 add \start, \start, \linesz 370 cmp \start, \end 371 b.lo .Ldcache_op\@ 372 alternative_if ARM64_WORKAROUND_4311569 373 .ifnc \op, cvau 374 mov \start, \tmp 375 mov \tmp, xzr 376 cbnz \start, .Ldcache_op\@ 377 .endif 378 alternative_else_nop_endif 379 380 _cond_uaccess_extable .Ldcache_op\@, \fixup 381 .endm 382 383 /* 384 * Macro to perform a data cache maintenance for the interval 385 * [start, end) without waiting for completion 386 * 387 * op: operation passed to dc instruction 388 * start: starting virtual address of the region 389 * end: end virtual address of the region 390 * fixup: optional label to branch to on user fault 391 * Corrupts: start, end, tmp1, tmp2 392 */ 393 .macro dcache_by_line_op_nosync op, start, end, tmp1, tmp2, fixup 394 dcache_line_size \tmp1, \tmp2 395 dcache_by_myline_op_nosync \op, \start, \end, \tmp1, \tmp2, \fixup 396 .endm 397 398 /* 399 * Macro to perform a data cache maintenance for the interval 400 * [start, end) and wait for completion 401 * 402 * op: operation passed to dc instruction 403 * domain: domain used in dsb instruction 404 * start: starting virtual address of the region 405 * end: end virtual address of the region 406 * fixup: optional label to branch to on user fault 407 * Corrupts: start, end, tmp1, tmp2 408 */ 409 .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup 410 dcache_by_line_op_nosync \op, \start, \end, \tmp1, \tmp2, \fixup 411 dsb \domain 412 .endm 413 414 /* 415 * Macro to perform an instruction cache maintenance for the interval 416 * [start, end) 417 * 418 * start, end: virtual addresses describing the region 419 * fixup: optional label to branch to on user fault 420 * Corrupts: tmp1, tmp2 421 */ 422 .macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup 423 alternative_if ARM64_WORKAROUND_NXP_ERR050104 424 ic ialluis 425 b .Licache_done\@ 426 alternative_else_nop_endif 427 icache_line_size \tmp1, \tmp2 428 sub \tmp2, \tmp1, #1 429 bic \tmp2, \start, \tmp2 430 .Licache_op\@: 431 ic ivau, \tmp2 // invalidate I line PoU 432 add \tmp2, \tmp2, \tmp1 433 cmp \tmp2, \end 434 b.lo .Licache_op\@ 435 .Licache_done\@: 436 dsb ish 437 isb 438 439 _cond_uaccess_extable .Licache_op\@, \fixup 440 .endm 441 442 /* 443 * load_ttbr1 - install @pgtbl as a TTBR1 page table 444 * pgtbl preserved 445 * tmp1/tmp2 clobbered, either may overlap with pgtbl 446 */ 447 .macro load_ttbr1, pgtbl, tmp1, tmp2 448 phys_to_ttbr \tmp1, \pgtbl 449 offset_ttbr1 \tmp1, \tmp2 450 msr ttbr1_el1, \tmp1 451 isb 452 .endm 453 454 /* 455 * To prevent the possibility of old and new partial table walks being visible 456 * in the tlb, switch the ttbr to a zero page when we invalidate the old 457 * records. D4.7.1 'General TLB maintenance requirements' in ARM DDI 0487A.i 458 * Even switching to our copied tables will cause a changed output address at 459 * each stage of the walk. 460 */ 461 .macro break_before_make_ttbr_switch zero_page, page_table, tmp, tmp2 462 phys_to_ttbr \tmp, \zero_page 463 msr ttbr1_el1, \tmp 464 isb 465 tlbi vmalle1 466 dsb nsh 467 load_ttbr1 \page_table, \tmp, \tmp2 468 .endm 469 470 /* 471 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 472 */ 473 .macro reset_pmuserenr_el0, tmpreg 474 mrs \tmpreg, id_aa64dfr0_el1 475 ubfx \tmpreg, \tmpreg, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4 476 cmp \tmpreg, #ID_AA64DFR0_EL1_PMUVer_NI 477 ccmp \tmpreg, #ID_AA64DFR0_EL1_PMUVer_IMP_DEF, #4, ne 478 b.eq 9000f // Skip if no PMU present or IMP_DEF 479 msr pmuserenr_el0, xzr // Disable PMU access from EL0 480 9000: 481 .endm 482 483 /* 484 * reset_amuserenr_el0 - reset AMUSERENR_EL0 if AMUv1 present 485 */ 486 .macro reset_amuserenr_el0, tmpreg 487 mrs \tmpreg, id_aa64pfr0_el1 // Check ID_AA64PFR0_EL1 488 ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_EL1_AMU_SHIFT, #4 489 cbz \tmpreg, .Lskip_\@ // Skip if no AMU present 490 msr_s SYS_AMUSERENR_EL0, xzr // Disable AMU access from EL0 491 .Lskip_\@: 492 .endm 493 /* 494 * copy_page - copy src to dest using temp registers t1-t8 495 */ 496 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 497 9998: ldp \t1, \t2, [\src] 498 ldp \t3, \t4, [\src, #16] 499 ldp \t5, \t6, [\src, #32] 500 ldp \t7, \t8, [\src, #48] 501 add \src, \src, #64 502 stnp \t1, \t2, [\dest] 503 stnp \t3, \t4, [\dest, #16] 504 stnp \t5, \t6, [\dest, #32] 505 stnp \t7, \t8, [\dest, #48] 506 add \dest, \dest, #64 507 tst \src, #(PAGE_SIZE - 1) 508 b.ne 9998b 509 .endm 510 511 /* 512 * Annotate a function as being unsuitable for kprobes. 513 */ 514 #ifdef CONFIG_KPROBES 515 #define NOKPROBE(x) \ 516 .pushsection "_kprobe_blacklist", "aw"; \ 517 .quad x; \ 518 .popsection; 519 #else 520 #define NOKPROBE(x) 521 #endif 522 523 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) 524 #define EXPORT_SYMBOL_NOKASAN(name) 525 #else 526 #define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name) 527 #endif 528 529 /* 530 * Emit a 64-bit absolute little endian symbol reference in a way that 531 * ensures that it will be resolved at build time, even when building a 532 * PIE binary. This requires cooperation from the linker script, which 533 * must emit the lo32/hi32 halves individually. 534 */ 535 .macro le64sym, sym 536 .long \sym\()_lo32 537 .long \sym\()_hi32 538 .endm 539 540 /* 541 * mov_q - move an immediate constant into a 64-bit register using 542 * between 2 and 4 movz/movk instructions (depending on the 543 * magnitude and sign of the operand) 544 */ 545 .macro mov_q, reg, val 546 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 547 movz \reg, :abs_g1_s:\val 548 .else 549 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 550 movz \reg, :abs_g2_s:\val 551 .else 552 movz \reg, :abs_g3:\val 553 movk \reg, :abs_g2_nc:\val 554 .endif 555 movk \reg, :abs_g1_nc:\val 556 .endif 557 movk \reg, :abs_g0_nc:\val 558 .endm 559 560 /* 561 * Return the current task_struct. 562 */ 563 .macro get_current_task, rd 564 mrs \rd, sp_el0 565 .endm 566 567 /* 568 * If the kernel is built for 52-bit virtual addressing but the hardware only 569 * supports 48 bits, we cannot program the pgdir address into TTBR1 directly, 570 * but we have to add an offset so that the TTBR1 address corresponds with the 571 * pgdir entry that covers the lowest 48-bit addressable VA. 572 * 573 * Note that this trick is only used for LVA/64k pages - LPA2/4k pages uses an 574 * additional paging level, and on LPA2/16k pages, we would end up with a root 575 * level table with only 2 entries, which is suboptimal in terms of TLB 576 * utilization, so there we fall back to 47 bits of translation if LPA2 is not 577 * supported. 578 * 579 * orr is used as it can cover the immediate value (and is idempotent). 580 * ttbr: Value of ttbr to set, modified. 581 */ 582 .macro offset_ttbr1, ttbr, tmp 583 #if defined(CONFIG_ARM64_VA_BITS_52) && !defined(CONFIG_ARM64_LPA2) 584 mrs \tmp, tcr_el1 585 and \tmp, \tmp, #TCR_EL1_T1SZ_MASK 586 cmp \tmp, #TCR_T1SZ(VA_BITS_MIN) 587 orr \tmp, \ttbr, #TTBR1_BADDR_4852_OFFSET 588 csel \ttbr, \tmp, \ttbr, eq 589 #endif 590 .endm 591 592 /* 593 * Arrange a physical address in a TTBR register, taking care of 52-bit 594 * addresses. 595 * 596 * phys: physical address, preserved 597 * ttbr: returns the TTBR value 598 */ 599 .macro phys_to_ttbr, ttbr, phys 600 #ifdef CONFIG_ARM64_PA_BITS_52 601 orr \ttbr, \phys, \phys, lsr #46 602 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52 603 #else 604 mov \ttbr, \phys 605 #endif 606 .endm 607 608 .macro phys_to_pte, pte, phys 609 #ifdef CONFIG_ARM64_PA_BITS_52 610 orr \pte, \phys, \phys, lsr #PTE_ADDR_HIGH_SHIFT 611 and \pte, \pte, #PHYS_TO_PTE_ADDR_MASK 612 #else 613 mov \pte, \phys 614 #endif 615 .endm 616 617 /* 618 * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU. 619 */ 620 .macro tcr_clear_errata_bits, tcr, tmp1, tmp2 621 #ifdef CONFIG_FUJITSU_ERRATUM_010001 622 mrs \tmp1, midr_el1 623 624 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK 625 and \tmp1, \tmp1, \tmp2 626 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001 627 cmp \tmp1, \tmp2 628 b.ne 10f 629 630 mov_q \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001 631 bic \tcr, \tcr, \tmp2 632 10: 633 #endif /* CONFIG_FUJITSU_ERRATUM_010001 */ 634 .endm 635 636 /** 637 * Errata workaround prior to disable MMU. Insert an ISB immediately prior 638 * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0. 639 */ 640 .macro pre_disable_mmu_workaround 641 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041 642 isb 643 #endif 644 .endm 645 646 /* 647 * frame_push - Push @regcount callee saved registers to the stack, 648 * starting at x19, as well as x29/x30, and set x29 to 649 * the new value of sp. Add @extra bytes of stack space 650 * for locals. 651 */ 652 .macro frame_push, regcount:req, extra 653 __frame st, \regcount, \extra 654 .endm 655 656 /* 657 * frame_pop - Pop the callee saved registers from the stack that were 658 * pushed in the most recent call to frame_push, as well 659 * as x29/x30 and any extra stack space that may have been 660 * allocated. 661 */ 662 .macro frame_pop 663 __frame ld 664 .endm 665 666 .macro __frame_regs, reg1, reg2, op, num 667 .if .Lframe_regcount == \num 668 \op\()r \reg1, [sp, #(\num + 1) * 8] 669 .elseif .Lframe_regcount > \num 670 \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8] 671 .endif 672 .endm 673 674 .macro __frame, op, regcount, extra=0 675 .ifc \op, st 676 .if (\regcount) < 0 || (\regcount) > 10 677 .error "regcount should be in the range [0 ... 10]" 678 .endif 679 .if ((\extra) % 16) != 0 680 .error "extra should be a multiple of 16 bytes" 681 .endif 682 .ifdef .Lframe_regcount 683 .if .Lframe_regcount != -1 684 .error "frame_push/frame_pop may not be nested" 685 .endif 686 .endif 687 .set .Lframe_regcount, \regcount 688 .set .Lframe_extra, \extra 689 .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16 690 stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]! 691 mov x29, sp 692 .endif 693 694 __frame_regs x19, x20, \op, 1 695 __frame_regs x21, x22, \op, 3 696 __frame_regs x23, x24, \op, 5 697 __frame_regs x25, x26, \op, 7 698 __frame_regs x27, x28, \op, 9 699 700 .ifc \op, ld 701 .if .Lframe_regcount == -1 702 .error "frame_push/frame_pop may not be nested" 703 .endif 704 ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra 705 .set .Lframe_regcount, -1 706 .endif 707 .endm 708 709 /* 710 * Set SCTLR_ELx to the @reg value, and invalidate the local icache 711 * in the process. This is called when setting the MMU on. 712 */ 713 .macro set_sctlr, sreg, reg 714 msr \sreg, \reg 715 isb 716 /* 717 * Invalidate the local I-cache so that any instructions fetched 718 * speculatively from the PoC are discarded, since they may have 719 * been dynamically patched at the PoU. 720 */ 721 ic iallu 722 dsb nsh 723 isb 724 .endm 725 726 .macro set_sctlr_el1, reg 727 set_sctlr sctlr_el1, \reg 728 .endm 729 730 .macro set_sctlr_el2, reg 731 set_sctlr sctlr_el2, \reg 732 .endm 733 734 /* 735 * Branch Target Identifier (BTI) 736 */ 737 .macro bti, targets 738 .equ .L__bti_targets_c, 34 739 .equ .L__bti_targets_j, 36 740 .equ .L__bti_targets_jc,38 741 hint #.L__bti_targets_\targets 742 .endm 743 744 /* 745 * This macro emits a program property note section identifying 746 * architecture features which require special handling, mainly for 747 * use in assembly files included in the VDSO. 748 */ 749 750 #define NT_GNU_PROPERTY_TYPE_0 5 751 #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000 752 753 #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0) 754 #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1) 755 756 #ifdef CONFIG_ARM64_BTI_KERNEL 757 #define GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT \ 758 ((GNU_PROPERTY_AARCH64_FEATURE_1_BTI | \ 759 GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) 760 #endif 761 762 #ifdef GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 763 .macro emit_aarch64_feature_1_and, feat=GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 764 .pushsection .note.gnu.property, "a" 765 .align 3 766 .long 2f - 1f 767 .long 6f - 3f 768 .long NT_GNU_PROPERTY_TYPE_0 769 1: .string "GNU" 770 2: 771 .align 3 772 3: .long GNU_PROPERTY_AARCH64_FEATURE_1_AND 773 .long 5f - 4f 774 4: 775 /* 776 * This is described with an array of char in the Linux API 777 * spec but the text and all other usage (including binutils, 778 * clang and GCC) treat this as a 32 bit value so no swizzling 779 * is required for big endian. 780 */ 781 .long \feat 782 5: 783 .align 3 784 6: 785 .popsection 786 .endm 787 788 #else 789 .macro emit_aarch64_feature_1_and, feat=0 790 .endm 791 792 #endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */ 793 794 .macro __mitigate_spectre_bhb_loop tmp 795 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 796 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_iter 797 mov \tmp, #32 // Patched to correct the immediate 798 alternative_cb_end 799 .Lspectre_bhb_loop\@: 800 b . + 4 801 subs \tmp, \tmp, #1 802 b.ne .Lspectre_bhb_loop\@ 803 sb 804 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 805 .endm 806 807 .macro mitigate_spectre_bhb_loop tmp 808 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 809 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_mitigation_enable 810 b .L_spectre_bhb_loop_done\@ // Patched to NOP 811 alternative_cb_end 812 __mitigate_spectre_bhb_loop \tmp 813 .L_spectre_bhb_loop_done\@: 814 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 815 .endm 816 817 /* Save/restores x0-x3 to the stack */ 818 .macro __mitigate_spectre_bhb_fw 819 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 820 stp x0, x1, [sp, #-16]! 821 stp x2, x3, [sp, #-16]! 822 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 823 alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit 824 nop // Patched to SMC/HVC #0 825 alternative_cb_end 826 ldp x2, x3, [sp], #16 827 ldp x0, x1, [sp], #16 828 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 829 .endm 830 831 .macro mitigate_spectre_bhb_clear_insn 832 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 833 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_clearbhb 834 /* Patched to NOP when not supported */ 835 clearbhb 836 isb 837 alternative_cb_end 838 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 839 .endm 840 #endif /* __ASM_ASSEMBLER_H */ 841