1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 4 * 5 * Copyright (C) 1996-2000 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASSEMBLER__ 9 #error "Only include this from assembly code" 10 #endif 11 12 #ifndef __ASM_ASSEMBLER_H 13 #define __ASM_ASSEMBLER_H 14 15 #include <linux/export.h> 16 17 #include <asm/alternative.h> 18 #include <asm/asm-bug.h> 19 #include <asm/asm-extable.h> 20 #include <asm/asm-offsets.h> 21 #include <asm/cpufeature.h> 22 #include <asm/cputype.h> 23 #include <asm/debug-monitors.h> 24 #include <asm/page.h> 25 #include <asm/pgtable-hwdef.h> 26 #include <asm/ptrace.h> 27 #include <asm/thread_info.h> 28 29 /* 30 * Provide a wxN alias for each wN register so what we can paste a xN 31 * reference after a 'w' to obtain the 32-bit version. 32 */ 33 .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 34 wx\n .req w\n 35 .endr 36 37 /* 38 * Save/restore interrupts. 39 */ 40 .macro save_and_disable_daif, flags 41 mrs \flags, daif 42 msr daifset, #0xf 43 .endm 44 45 .macro save_and_disable_irq, flags 46 mrs \flags, daif 47 msr daifset, #3 48 .endm 49 50 .macro restore_irq, flags 51 msr daif, \flags 52 .endm 53 54 .macro disable_step_tsk, flgs, tmp 55 tbz \flgs, #TIF_SINGLESTEP, 9990f 56 mrs \tmp, mdscr_el1 57 bic \tmp, \tmp, #MDSCR_EL1_SS 58 msr mdscr_el1, \tmp 59 isb // Take effect before a subsequent clear of DAIF.D 60 9990: 61 .endm 62 63 /* call with daif masked */ 64 .macro enable_step_tsk, flgs, tmp 65 tbz \flgs, #TIF_SINGLESTEP, 9990f 66 mrs \tmp, mdscr_el1 67 orr \tmp, \tmp, #MDSCR_EL1_SS 68 msr mdscr_el1, \tmp 69 9990: 70 .endm 71 72 /* 73 * RAS Error Synchronization barrier 74 */ 75 .macro esb 76 #ifdef CONFIG_ARM64_RAS_EXTN 77 hint #16 78 #else 79 nop 80 #endif 81 .endm 82 83 /* 84 * Value prediction barrier 85 */ 86 .macro csdb 87 hint #20 88 .endm 89 90 /* 91 * Clear Branch History instruction 92 */ 93 .macro clearbhb 94 hint #22 95 .endm 96 97 /* 98 * Speculation barrier 99 */ 100 .macro sb 101 alternative_if_not ARM64_HAS_SB 102 dsb nsh 103 isb 104 alternative_else 105 SB_BARRIER_INSN 106 nop 107 alternative_endif 108 .endm 109 110 /* 111 * NOP sequence 112 */ 113 .macro nops, num 114 .rept \num 115 nop 116 .endr 117 .endm 118 119 /* 120 * Register aliases. 121 */ 122 lr .req x30 // link register 123 124 /* 125 * Vector entry 126 */ 127 .macro ventry label 128 .align 7 129 b \label 130 .endm 131 132 /* 133 * Select code when configured for BE. 134 */ 135 #ifdef CONFIG_CPU_BIG_ENDIAN 136 #define CPU_BE(code...) code 137 #else 138 #define CPU_BE(code...) 139 #endif 140 141 /* 142 * Select code when configured for LE. 143 */ 144 #ifdef CONFIG_CPU_BIG_ENDIAN 145 #define CPU_LE(code...) 146 #else 147 #define CPU_LE(code...) code 148 #endif 149 150 /* 151 * Define a macro that constructs a 64-bit value by concatenating two 152 * 32-bit registers. Note that on big endian systems the order of the 153 * registers is swapped. 154 */ 155 #ifndef CONFIG_CPU_BIG_ENDIAN 156 .macro regs_to_64, rd, lbits, hbits 157 #else 158 .macro regs_to_64, rd, hbits, lbits 159 #endif 160 orr \rd, \lbits, \hbits, lsl #32 161 .endm 162 163 /* 164 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 165 * <symbol> is within the range +/- 4 GB of the PC. 166 */ 167 /* 168 * @dst: destination register (64 bit wide) 169 * @sym: name of the symbol 170 */ 171 .macro adr_l, dst, sym 172 adrp \dst, \sym 173 add \dst, \dst, :lo12:\sym 174 .endm 175 176 /* 177 * @dst: destination register (32 or 64 bit wide) 178 * @sym: name of the symbol 179 * @tmp: optional 64-bit scratch register to be used if <dst> is a 180 * 32-bit wide register, in which case it cannot be used to hold 181 * the address 182 */ 183 .macro ldr_l, dst, sym, tmp= 184 .ifb \tmp 185 adrp \dst, \sym 186 ldr \dst, [\dst, :lo12:\sym] 187 .else 188 adrp \tmp, \sym 189 ldr \dst, [\tmp, :lo12:\sym] 190 .endif 191 .endm 192 193 /* 194 * @src: source register (32 or 64 bit wide) 195 * @sym: name of the symbol 196 * @tmp: mandatory 64-bit scratch register to calculate the address 197 * while <src> needs to be preserved. 198 */ 199 .macro str_l, src, sym, tmp 200 adrp \tmp, \sym 201 str \src, [\tmp, :lo12:\sym] 202 .endm 203 204 /* 205 * @dst: destination register 206 */ 207 #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__) 208 .macro get_this_cpu_offset, dst 209 mrs \dst, tpidr_el2 210 .endm 211 #else 212 .macro get_this_cpu_offset, dst 213 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 214 mrs \dst, tpidr_el1 215 alternative_else 216 mrs \dst, tpidr_el2 217 alternative_endif 218 .endm 219 220 .macro set_this_cpu_offset, src 221 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 222 msr tpidr_el1, \src 223 alternative_else 224 msr tpidr_el2, \src 225 alternative_endif 226 .endm 227 #endif 228 229 /* 230 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP) 231 * @sym: The name of the per-cpu variable 232 * @tmp: scratch register 233 */ 234 .macro adr_this_cpu, dst, sym, tmp 235 adrp \tmp, \sym 236 add \dst, \tmp, #:lo12:\sym 237 get_this_cpu_offset \tmp 238 add \dst, \dst, \tmp 239 .endm 240 241 /* 242 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 243 * @sym: The name of the per-cpu variable 244 * @tmp: scratch register 245 */ 246 .macro ldr_this_cpu dst, sym, tmp 247 adr_l \dst, \sym 248 get_this_cpu_offset \tmp 249 ldr \dst, [\dst, \tmp] 250 .endm 251 252 /* 253 * read_ctr - read CTR_EL0. If the system has mismatched register fields, 254 * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val 255 */ 256 .macro read_ctr, reg 257 #ifndef __KVM_NVHE_HYPERVISOR__ 258 alternative_if_not ARM64_MISMATCHED_CACHE_TYPE 259 mrs \reg, ctr_el0 // read CTR 260 nop 261 alternative_else 262 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 263 alternative_endif 264 #else 265 alternative_if_not ARM64_KVM_PROTECTED_MODE 266 ASM_BUG() 267 alternative_else_nop_endif 268 alternative_cb ARM64_ALWAYS_SYSTEM, kvm_compute_final_ctr_el0 269 movz \reg, #0 270 movk \reg, #0, lsl #16 271 movk \reg, #0, lsl #32 272 movk \reg, #0, lsl #48 273 alternative_cb_end 274 #endif 275 .endm 276 277 278 /* 279 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 280 * from the CTR register. 281 */ 282 .macro raw_dcache_line_size, reg, tmp 283 mrs \tmp, ctr_el0 // read CTR 284 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 285 mov \reg, #4 // bytes per word 286 lsl \reg, \reg, \tmp // actual cache line size 287 .endm 288 289 /* 290 * dcache_line_size - get the safe D-cache line size across all CPUs 291 */ 292 .macro dcache_line_size, reg, tmp 293 read_ctr \tmp 294 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 295 mov \reg, #4 // bytes per word 296 lsl \reg, \reg, \tmp // actual cache line size 297 .endm 298 299 /* 300 * raw_icache_line_size - get the minimum I-cache line size on this CPU 301 * from the CTR register. 302 */ 303 .macro raw_icache_line_size, reg, tmp 304 mrs \tmp, ctr_el0 // read CTR 305 and \tmp, \tmp, #0xf // cache line size encoding 306 mov \reg, #4 // bytes per word 307 lsl \reg, \reg, \tmp // actual cache line size 308 .endm 309 310 /* 311 * icache_line_size - get the safe I-cache line size across all CPUs 312 */ 313 .macro icache_line_size, reg, tmp 314 read_ctr \tmp 315 and \tmp, \tmp, #0xf // cache line size encoding 316 mov \reg, #4 // bytes per word 317 lsl \reg, \reg, \tmp // actual cache line size 318 .endm 319 320 /* 321 * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map 322 */ 323 .macro tcr_set_t0sz, valreg, t0sz 324 bfi \valreg, \t0sz, #TCR_EL1_T0SZ_SHIFT, #TCR_EL1_T0SZ_WIDTH 325 .endm 326 327 /* 328 * tcr_set_t1sz - update TCR.T1SZ 329 */ 330 .macro tcr_set_t1sz, valreg, t1sz 331 bfi \valreg, \t1sz, #TCR_EL1_T1SZ_SHIFT, #TCR_EL1_T1SZ_WIDTH 332 .endm 333 334 /* 335 * tcr_compute_pa_size - set TCR.(I)PS to the highest supported 336 * ID_AA64MMFR0_EL1.PARange value 337 * 338 * tcr: register with the TCR_ELx value to be updated 339 * pos: IPS or PS bitfield position 340 * tmp{0,1}: temporary registers 341 */ 342 .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1 343 mrs \tmp0, ID_AA64MMFR0_EL1 344 // Narrow PARange to fit the PS field in TCR_ELx 345 ubfx \tmp0, \tmp0, #ID_AA64MMFR0_EL1_PARANGE_SHIFT, #3 346 mov \tmp1, #ID_AA64MMFR0_EL1_PARANGE_MAX 347 #ifdef CONFIG_ARM64_LPA2 348 alternative_if_not ARM64_HAS_VA52 349 mov \tmp1, #ID_AA64MMFR0_EL1_PARANGE_48 350 alternative_else_nop_endif 351 #endif 352 cmp \tmp0, \tmp1 353 csel \tmp0, \tmp1, \tmp0, hi 354 bfi \tcr, \tmp0, \pos, #3 355 .endm 356 357 .macro __dcache_op_workaround_clean_cache, op, addr 358 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 359 dc \op, \addr 360 alternative_else 361 dc civac, \addr 362 alternative_endif 363 .endm 364 365 /* 366 * Macro to perform a data cache maintenance for the interval 367 * [start, end) with dcache line size explicitly provided. 368 * 369 * op: operation passed to dc instruction 370 * start: starting virtual address of the region 371 * end: end virtual address of the region 372 * linesz: dcache line size 373 * fixup: optional label to branch to on user fault 374 * Corrupts: start, end, tmp 375 */ 376 .macro dcache_by_myline_op_nosync op, start, end, linesz, tmp, fixup 377 sub \tmp, \linesz, #1 378 bic \start, \start, \tmp 379 alternative_if ARM64_WORKAROUND_4311569 380 mov \tmp, \start 381 alternative_else_nop_endif 382 .Ldcache_op\@: 383 .ifc \op, cvau 384 __dcache_op_workaround_clean_cache \op, \start 385 .else 386 .ifc \op, cvac 387 __dcache_op_workaround_clean_cache \op, \start 388 .else 389 .ifc \op, cvap 390 sys 3, c7, c12, 1, \start // dc cvap 391 .else 392 .ifc \op, cvadp 393 sys 3, c7, c13, 1, \start // dc cvadp 394 .else 395 dc \op, \start 396 .endif 397 .endif 398 .endif 399 .endif 400 add \start, \start, \linesz 401 cmp \start, \end 402 b.lo .Ldcache_op\@ 403 alternative_if ARM64_WORKAROUND_4311569 404 .ifnc \op, cvau 405 mov \start, \tmp 406 mov \tmp, xzr 407 cbnz \start, .Ldcache_op\@ 408 .endif 409 alternative_else_nop_endif 410 411 _cond_uaccess_extable .Ldcache_op\@, \fixup 412 .endm 413 414 /* 415 * Macro to perform a data cache maintenance for the interval 416 * [start, end) without waiting for completion 417 * 418 * op: operation passed to dc instruction 419 * start: starting virtual address of the region 420 * end: end virtual address of the region 421 * fixup: optional label to branch to on user fault 422 * Corrupts: start, end, tmp1, tmp2 423 */ 424 .macro dcache_by_line_op_nosync op, start, end, tmp1, tmp2, fixup 425 dcache_line_size \tmp1, \tmp2 426 dcache_by_myline_op_nosync \op, \start, \end, \tmp1, \tmp2, \fixup 427 .endm 428 429 /* 430 * Macro to perform a data cache maintenance for the interval 431 * [start, end) and wait for completion 432 * 433 * op: operation passed to dc instruction 434 * domain: domain used in dsb instruction 435 * start: starting virtual address of the region 436 * end: end virtual address of the region 437 * fixup: optional label to branch to on user fault 438 * Corrupts: start, end, tmp1, tmp2 439 */ 440 .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup 441 dcache_by_line_op_nosync \op, \start, \end, \tmp1, \tmp2, \fixup 442 dsb \domain 443 .endm 444 445 /* 446 * Macro to perform an instruction cache maintenance for the interval 447 * [start, end) 448 * 449 * start, end: virtual addresses describing the region 450 * fixup: optional label to branch to on user fault 451 * Corrupts: tmp1, tmp2 452 */ 453 .macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup 454 icache_line_size \tmp1, \tmp2 455 sub \tmp2, \tmp1, #1 456 bic \tmp2, \start, \tmp2 457 .Licache_op\@: 458 ic ivau, \tmp2 // invalidate I line PoU 459 add \tmp2, \tmp2, \tmp1 460 cmp \tmp2, \end 461 b.lo .Licache_op\@ 462 dsb ish 463 isb 464 465 _cond_uaccess_extable .Licache_op\@, \fixup 466 .endm 467 468 /* 469 * load_ttbr1 - install @pgtbl as a TTBR1 page table 470 * pgtbl preserved 471 * tmp1/tmp2 clobbered, either may overlap with pgtbl 472 */ 473 .macro load_ttbr1, pgtbl, tmp1, tmp2 474 phys_to_ttbr \tmp1, \pgtbl 475 offset_ttbr1 \tmp1, \tmp2 476 msr ttbr1_el1, \tmp1 477 isb 478 .endm 479 480 /* 481 * To prevent the possibility of old and new partial table walks being visible 482 * in the tlb, switch the ttbr to a zero page when we invalidate the old 483 * records. D4.7.1 'General TLB maintenance requirements' in ARM DDI 0487A.i 484 * Even switching to our copied tables will cause a changed output address at 485 * each stage of the walk. 486 */ 487 .macro break_before_make_ttbr_switch zero_page, page_table, tmp, tmp2 488 phys_to_ttbr \tmp, \zero_page 489 msr ttbr1_el1, \tmp 490 isb 491 tlbi vmalle1 492 dsb nsh 493 load_ttbr1 \page_table, \tmp, \tmp2 494 .endm 495 496 /* 497 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 498 */ 499 .macro reset_pmuserenr_el0, tmpreg 500 mrs \tmpreg, id_aa64dfr0_el1 501 ubfx \tmpreg, \tmpreg, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4 502 cmp \tmpreg, #ID_AA64DFR0_EL1_PMUVer_NI 503 ccmp \tmpreg, #ID_AA64DFR0_EL1_PMUVer_IMP_DEF, #4, ne 504 b.eq 9000f // Skip if no PMU present or IMP_DEF 505 msr pmuserenr_el0, xzr // Disable PMU access from EL0 506 9000: 507 .endm 508 509 /* 510 * reset_amuserenr_el0 - reset AMUSERENR_EL0 if AMUv1 present 511 */ 512 .macro reset_amuserenr_el0, tmpreg 513 mrs \tmpreg, id_aa64pfr0_el1 // Check ID_AA64PFR0_EL1 514 ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_EL1_AMU_SHIFT, #4 515 cbz \tmpreg, .Lskip_\@ // Skip if no AMU present 516 msr_s SYS_AMUSERENR_EL0, xzr // Disable AMU access from EL0 517 .Lskip_\@: 518 .endm 519 /* 520 * copy_page - copy src to dest using temp registers t1-t8 521 */ 522 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 523 9998: ldp \t1, \t2, [\src] 524 ldp \t3, \t4, [\src, #16] 525 ldp \t5, \t6, [\src, #32] 526 ldp \t7, \t8, [\src, #48] 527 add \src, \src, #64 528 stnp \t1, \t2, [\dest] 529 stnp \t3, \t4, [\dest, #16] 530 stnp \t5, \t6, [\dest, #32] 531 stnp \t7, \t8, [\dest, #48] 532 add \dest, \dest, #64 533 tst \src, #(PAGE_SIZE - 1) 534 b.ne 9998b 535 .endm 536 537 /* 538 * Annotate a function as being unsuitable for kprobes. 539 */ 540 #ifdef CONFIG_KPROBES 541 #define NOKPROBE(x) \ 542 .pushsection "_kprobe_blacklist", "aw"; \ 543 .quad x; \ 544 .popsection; 545 #else 546 #define NOKPROBE(x) 547 #endif 548 549 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) 550 #define EXPORT_SYMBOL_NOKASAN(name) 551 #else 552 #define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name) 553 #endif 554 555 /* 556 * Emit a 64-bit absolute little endian symbol reference in a way that 557 * ensures that it will be resolved at build time, even when building a 558 * PIE binary. This requires cooperation from the linker script, which 559 * must emit the lo32/hi32 halves individually. 560 */ 561 .macro le64sym, sym 562 .long \sym\()_lo32 563 .long \sym\()_hi32 564 .endm 565 566 /* 567 * mov_q - move an immediate constant into a 64-bit register using 568 * between 2 and 4 movz/movk instructions (depending on the 569 * magnitude and sign of the operand) 570 */ 571 .macro mov_q, reg, val 572 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 573 movz \reg, :abs_g1_s:\val 574 .else 575 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 576 movz \reg, :abs_g2_s:\val 577 .else 578 movz \reg, :abs_g3:\val 579 movk \reg, :abs_g2_nc:\val 580 .endif 581 movk \reg, :abs_g1_nc:\val 582 .endif 583 movk \reg, :abs_g0_nc:\val 584 .endm 585 586 /* 587 * Return the current task_struct. 588 */ 589 .macro get_current_task, rd 590 mrs \rd, sp_el0 591 .endm 592 593 /* 594 * If the kernel is built for 52-bit virtual addressing but the hardware only 595 * supports 48 bits, we cannot program the pgdir address into TTBR1 directly, 596 * but we have to add an offset so that the TTBR1 address corresponds with the 597 * pgdir entry that covers the lowest 48-bit addressable VA. 598 * 599 * Note that this trick is only used for LVA/64k pages - LPA2/4k pages uses an 600 * additional paging level, and on LPA2/16k pages, we would end up with a root 601 * level table with only 2 entries, which is suboptimal in terms of TLB 602 * utilization, so there we fall back to 47 bits of translation if LPA2 is not 603 * supported. 604 * 605 * orr is used as it can cover the immediate value (and is idempotent). 606 * ttbr: Value of ttbr to set, modified. 607 */ 608 .macro offset_ttbr1, ttbr, tmp 609 #if defined(CONFIG_ARM64_VA_BITS_52) && !defined(CONFIG_ARM64_LPA2) 610 mrs \tmp, tcr_el1 611 and \tmp, \tmp, #TCR_EL1_T1SZ_MASK 612 cmp \tmp, #TCR_T1SZ(VA_BITS_MIN) 613 orr \tmp, \ttbr, #TTBR1_BADDR_4852_OFFSET 614 csel \ttbr, \tmp, \ttbr, eq 615 #endif 616 .endm 617 618 /* 619 * Arrange a physical address in a TTBR register, taking care of 52-bit 620 * addresses. 621 * 622 * phys: physical address, preserved 623 * ttbr: returns the TTBR value 624 */ 625 .macro phys_to_ttbr, ttbr, phys 626 #ifdef CONFIG_ARM64_PA_BITS_52 627 orr \ttbr, \phys, \phys, lsr #46 628 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52 629 #else 630 mov \ttbr, \phys 631 #endif 632 .endm 633 634 .macro phys_to_pte, pte, phys 635 #ifdef CONFIG_ARM64_PA_BITS_52 636 orr \pte, \phys, \phys, lsr #PTE_ADDR_HIGH_SHIFT 637 and \pte, \pte, #PHYS_TO_PTE_ADDR_MASK 638 #else 639 mov \pte, \phys 640 #endif 641 .endm 642 643 /* 644 * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU. 645 */ 646 .macro tcr_clear_errata_bits, tcr, tmp1, tmp2 647 #ifdef CONFIG_FUJITSU_ERRATUM_010001 648 mrs \tmp1, midr_el1 649 650 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK 651 and \tmp1, \tmp1, \tmp2 652 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001 653 cmp \tmp1, \tmp2 654 b.ne 10f 655 656 mov_q \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001 657 bic \tcr, \tcr, \tmp2 658 10: 659 #endif /* CONFIG_FUJITSU_ERRATUM_010001 */ 660 .endm 661 662 /** 663 * Errata workaround prior to disable MMU. Insert an ISB immediately prior 664 * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0. 665 */ 666 .macro pre_disable_mmu_workaround 667 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041 668 isb 669 #endif 670 .endm 671 672 /* 673 * frame_push - Push @regcount callee saved registers to the stack, 674 * starting at x19, as well as x29/x30, and set x29 to 675 * the new value of sp. Add @extra bytes of stack space 676 * for locals. 677 */ 678 .macro frame_push, regcount:req, extra 679 __frame st, \regcount, \extra 680 .endm 681 682 /* 683 * frame_pop - Pop the callee saved registers from the stack that were 684 * pushed in the most recent call to frame_push, as well 685 * as x29/x30 and any extra stack space that may have been 686 * allocated. 687 */ 688 .macro frame_pop 689 __frame ld 690 .endm 691 692 .macro __frame_regs, reg1, reg2, op, num 693 .if .Lframe_regcount == \num 694 \op\()r \reg1, [sp, #(\num + 1) * 8] 695 .elseif .Lframe_regcount > \num 696 \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8] 697 .endif 698 .endm 699 700 .macro __frame, op, regcount, extra=0 701 .ifc \op, st 702 .if (\regcount) < 0 || (\regcount) > 10 703 .error "regcount should be in the range [0 ... 10]" 704 .endif 705 .if ((\extra) % 16) != 0 706 .error "extra should be a multiple of 16 bytes" 707 .endif 708 .ifdef .Lframe_regcount 709 .if .Lframe_regcount != -1 710 .error "frame_push/frame_pop may not be nested" 711 .endif 712 .endif 713 .set .Lframe_regcount, \regcount 714 .set .Lframe_extra, \extra 715 .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16 716 stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]! 717 mov x29, sp 718 .endif 719 720 __frame_regs x19, x20, \op, 1 721 __frame_regs x21, x22, \op, 3 722 __frame_regs x23, x24, \op, 5 723 __frame_regs x25, x26, \op, 7 724 __frame_regs x27, x28, \op, 9 725 726 .ifc \op, ld 727 .if .Lframe_regcount == -1 728 .error "frame_push/frame_pop may not be nested" 729 .endif 730 ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra 731 .set .Lframe_regcount, -1 732 .endif 733 .endm 734 735 /* 736 * Set SCTLR_ELx to the @reg value, and invalidate the local icache 737 * in the process. This is called when setting the MMU on. 738 */ 739 .macro set_sctlr, sreg, reg 740 msr \sreg, \reg 741 isb 742 /* 743 * Invalidate the local I-cache so that any instructions fetched 744 * speculatively from the PoC are discarded, since they may have 745 * been dynamically patched at the PoU. 746 */ 747 ic iallu 748 dsb nsh 749 isb 750 .endm 751 752 .macro set_sctlr_el1, reg 753 set_sctlr sctlr_el1, \reg 754 .endm 755 756 .macro set_sctlr_el2, reg 757 set_sctlr sctlr_el2, \reg 758 .endm 759 760 /* 761 * Branch Target Identifier (BTI) 762 */ 763 .macro bti, targets 764 .equ .L__bti_targets_c, 34 765 .equ .L__bti_targets_j, 36 766 .equ .L__bti_targets_jc,38 767 hint #.L__bti_targets_\targets 768 .endm 769 770 /* 771 * This macro emits a program property note section identifying 772 * architecture features which require special handling, mainly for 773 * use in assembly files included in the VDSO. 774 */ 775 776 #define NT_GNU_PROPERTY_TYPE_0 5 777 #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000 778 779 #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0) 780 #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1) 781 782 #ifdef CONFIG_ARM64_BTI_KERNEL 783 #define GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT \ 784 ((GNU_PROPERTY_AARCH64_FEATURE_1_BTI | \ 785 GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) 786 #endif 787 788 #ifdef GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 789 .macro emit_aarch64_feature_1_and, feat=GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 790 .pushsection .note.gnu.property, "a" 791 .align 3 792 .long 2f - 1f 793 .long 6f - 3f 794 .long NT_GNU_PROPERTY_TYPE_0 795 1: .string "GNU" 796 2: 797 .align 3 798 3: .long GNU_PROPERTY_AARCH64_FEATURE_1_AND 799 .long 5f - 4f 800 4: 801 /* 802 * This is described with an array of char in the Linux API 803 * spec but the text and all other usage (including binutils, 804 * clang and GCC) treat this as a 32 bit value so no swizzling 805 * is required for big endian. 806 */ 807 .long \feat 808 5: 809 .align 3 810 6: 811 .popsection 812 .endm 813 814 #else 815 .macro emit_aarch64_feature_1_and, feat=0 816 .endm 817 818 #endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */ 819 820 .macro __mitigate_spectre_bhb_loop tmp 821 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 822 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_iter 823 mov \tmp, #32 // Patched to correct the immediate 824 alternative_cb_end 825 .Lspectre_bhb_loop\@: 826 b . + 4 827 subs \tmp, \tmp, #1 828 b.ne .Lspectre_bhb_loop\@ 829 sb 830 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 831 .endm 832 833 .macro mitigate_spectre_bhb_loop tmp 834 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 835 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_mitigation_enable 836 b .L_spectre_bhb_loop_done\@ // Patched to NOP 837 alternative_cb_end 838 __mitigate_spectre_bhb_loop \tmp 839 .L_spectre_bhb_loop_done\@: 840 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 841 .endm 842 843 /* Save/restores x0-x3 to the stack */ 844 .macro __mitigate_spectre_bhb_fw 845 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 846 stp x0, x1, [sp, #-16]! 847 stp x2, x3, [sp, #-16]! 848 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 849 alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit 850 nop // Patched to SMC/HVC #0 851 alternative_cb_end 852 ldp x2, x3, [sp], #16 853 ldp x0, x1, [sp], #16 854 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 855 .endm 856 857 .macro mitigate_spectre_bhb_clear_insn 858 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 859 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_clearbhb 860 /* Patched to NOP when not supported */ 861 clearbhb 862 isb 863 alternative_cb_end 864 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 865 .endm 866 #endif /* __ASM_ASSEMBLER_H */ 867