1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 4 * 5 * Copyright (C) 1996-2000 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASSEMBLY__ 9 #error "Only include this from assembly code" 10 #endif 11 12 #ifndef __ASM_ASSEMBLER_H 13 #define __ASM_ASSEMBLER_H 14 15 #include <asm-generic/export.h> 16 17 #include <asm/alternative.h> 18 #include <asm/asm-bug.h> 19 #include <asm/asm-extable.h> 20 #include <asm/asm-offsets.h> 21 #include <asm/cpufeature.h> 22 #include <asm/cputype.h> 23 #include <asm/debug-monitors.h> 24 #include <asm/page.h> 25 #include <asm/pgtable-hwdef.h> 26 #include <asm/ptrace.h> 27 #include <asm/thread_info.h> 28 29 /* 30 * Provide a wxN alias for each wN register so what we can paste a xN 31 * reference after a 'w' to obtain the 32-bit version. 32 */ 33 .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 34 wx\n .req w\n 35 .endr 36 37 .macro save_and_disable_daif, flags 38 mrs \flags, daif 39 msr daifset, #0xf 40 .endm 41 42 .macro disable_daif 43 msr daifset, #0xf 44 .endm 45 46 .macro enable_daif 47 msr daifclr, #0xf 48 .endm 49 50 .macro restore_daif, flags:req 51 msr daif, \flags 52 .endm 53 54 /* IRQ/FIQ are the lowest priority flags, unconditionally unmask the rest. */ 55 .macro enable_da 56 msr daifclr, #(8 | 4) 57 .endm 58 59 /* 60 * Save/restore interrupts. 61 */ 62 .macro save_and_disable_irq, flags 63 mrs \flags, daif 64 msr daifset, #3 65 .endm 66 67 .macro restore_irq, flags 68 msr daif, \flags 69 .endm 70 71 .macro enable_dbg 72 msr daifclr, #8 73 .endm 74 75 .macro disable_step_tsk, flgs, tmp 76 tbz \flgs, #TIF_SINGLESTEP, 9990f 77 mrs \tmp, mdscr_el1 78 bic \tmp, \tmp, #DBG_MDSCR_SS 79 msr mdscr_el1, \tmp 80 isb // Synchronise with enable_dbg 81 9990: 82 .endm 83 84 /* call with daif masked */ 85 .macro enable_step_tsk, flgs, tmp 86 tbz \flgs, #TIF_SINGLESTEP, 9990f 87 mrs \tmp, mdscr_el1 88 orr \tmp, \tmp, #DBG_MDSCR_SS 89 msr mdscr_el1, \tmp 90 9990: 91 .endm 92 93 /* 94 * RAS Error Synchronization barrier 95 */ 96 .macro esb 97 #ifdef CONFIG_ARM64_RAS_EXTN 98 hint #16 99 #else 100 nop 101 #endif 102 .endm 103 104 /* 105 * Value prediction barrier 106 */ 107 .macro csdb 108 hint #20 109 .endm 110 111 /* 112 * Clear Branch History instruction 113 */ 114 .macro clearbhb 115 hint #22 116 .endm 117 118 /* 119 * Speculation barrier 120 */ 121 .macro sb 122 alternative_if_not ARM64_HAS_SB 123 dsb nsh 124 isb 125 alternative_else 126 SB_BARRIER_INSN 127 nop 128 alternative_endif 129 .endm 130 131 /* 132 * NOP sequence 133 */ 134 .macro nops, num 135 .rept \num 136 nop 137 .endr 138 .endm 139 140 /* 141 * Register aliases. 142 */ 143 lr .req x30 // link register 144 145 /* 146 * Vector entry 147 */ 148 .macro ventry label 149 .align 7 150 b \label 151 .endm 152 153 /* 154 * Select code when configured for BE. 155 */ 156 #ifdef CONFIG_CPU_BIG_ENDIAN 157 #define CPU_BE(code...) code 158 #else 159 #define CPU_BE(code...) 160 #endif 161 162 /* 163 * Select code when configured for LE. 164 */ 165 #ifdef CONFIG_CPU_BIG_ENDIAN 166 #define CPU_LE(code...) 167 #else 168 #define CPU_LE(code...) code 169 #endif 170 171 /* 172 * Define a macro that constructs a 64-bit value by concatenating two 173 * 32-bit registers. Note that on big endian systems the order of the 174 * registers is swapped. 175 */ 176 #ifndef CONFIG_CPU_BIG_ENDIAN 177 .macro regs_to_64, rd, lbits, hbits 178 #else 179 .macro regs_to_64, rd, hbits, lbits 180 #endif 181 orr \rd, \lbits, \hbits, lsl #32 182 .endm 183 184 /* 185 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 186 * <symbol> is within the range +/- 4 GB of the PC. 187 */ 188 /* 189 * @dst: destination register (64 bit wide) 190 * @sym: name of the symbol 191 */ 192 .macro adr_l, dst, sym 193 adrp \dst, \sym 194 add \dst, \dst, :lo12:\sym 195 .endm 196 197 /* 198 * @dst: destination register (32 or 64 bit wide) 199 * @sym: name of the symbol 200 * @tmp: optional 64-bit scratch register to be used if <dst> is a 201 * 32-bit wide register, in which case it cannot be used to hold 202 * the address 203 */ 204 .macro ldr_l, dst, sym, tmp= 205 .ifb \tmp 206 adrp \dst, \sym 207 ldr \dst, [\dst, :lo12:\sym] 208 .else 209 adrp \tmp, \sym 210 ldr \dst, [\tmp, :lo12:\sym] 211 .endif 212 .endm 213 214 /* 215 * @src: source register (32 or 64 bit wide) 216 * @sym: name of the symbol 217 * @tmp: mandatory 64-bit scratch register to calculate the address 218 * while <src> needs to be preserved. 219 */ 220 .macro str_l, src, sym, tmp 221 adrp \tmp, \sym 222 str \src, [\tmp, :lo12:\sym] 223 .endm 224 225 /* 226 * @dst: destination register 227 */ 228 #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__) 229 .macro get_this_cpu_offset, dst 230 mrs \dst, tpidr_el2 231 .endm 232 #else 233 .macro get_this_cpu_offset, dst 234 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 235 mrs \dst, tpidr_el1 236 alternative_else 237 mrs \dst, tpidr_el2 238 alternative_endif 239 .endm 240 241 .macro set_this_cpu_offset, src 242 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 243 msr tpidr_el1, \src 244 alternative_else 245 msr tpidr_el2, \src 246 alternative_endif 247 .endm 248 #endif 249 250 /* 251 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP) 252 * @sym: The name of the per-cpu variable 253 * @tmp: scratch register 254 */ 255 .macro adr_this_cpu, dst, sym, tmp 256 adrp \tmp, \sym 257 add \dst, \tmp, #:lo12:\sym 258 get_this_cpu_offset \tmp 259 add \dst, \dst, \tmp 260 .endm 261 262 /* 263 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 264 * @sym: The name of the per-cpu variable 265 * @tmp: scratch register 266 */ 267 .macro ldr_this_cpu dst, sym, tmp 268 adr_l \dst, \sym 269 get_this_cpu_offset \tmp 270 ldr \dst, [\dst, \tmp] 271 .endm 272 273 /* 274 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) 275 */ 276 .macro vma_vm_mm, rd, rn 277 ldr \rd, [\rn, #VMA_VM_MM] 278 .endm 279 280 /* 281 * read_ctr - read CTR_EL0. If the system has mismatched register fields, 282 * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val 283 */ 284 .macro read_ctr, reg 285 #ifndef __KVM_NVHE_HYPERVISOR__ 286 alternative_if_not ARM64_MISMATCHED_CACHE_TYPE 287 mrs \reg, ctr_el0 // read CTR 288 nop 289 alternative_else 290 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 291 alternative_endif 292 #else 293 alternative_if_not ARM64_KVM_PROTECTED_MODE 294 ASM_BUG() 295 alternative_else_nop_endif 296 alternative_cb kvm_compute_final_ctr_el0 297 movz \reg, #0 298 movk \reg, #0, lsl #16 299 movk \reg, #0, lsl #32 300 movk \reg, #0, lsl #48 301 alternative_cb_end 302 #endif 303 .endm 304 305 306 /* 307 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 308 * from the CTR register. 309 */ 310 .macro raw_dcache_line_size, reg, tmp 311 mrs \tmp, ctr_el0 // read CTR 312 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 313 mov \reg, #4 // bytes per word 314 lsl \reg, \reg, \tmp // actual cache line size 315 .endm 316 317 /* 318 * dcache_line_size - get the safe D-cache line size across all CPUs 319 */ 320 .macro dcache_line_size, reg, tmp 321 read_ctr \tmp 322 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 323 mov \reg, #4 // bytes per word 324 lsl \reg, \reg, \tmp // actual cache line size 325 .endm 326 327 /* 328 * raw_icache_line_size - get the minimum I-cache line size on this CPU 329 * from the CTR register. 330 */ 331 .macro raw_icache_line_size, reg, tmp 332 mrs \tmp, ctr_el0 // read CTR 333 and \tmp, \tmp, #0xf // cache line size encoding 334 mov \reg, #4 // bytes per word 335 lsl \reg, \reg, \tmp // actual cache line size 336 .endm 337 338 /* 339 * icache_line_size - get the safe I-cache line size across all CPUs 340 */ 341 .macro icache_line_size, reg, tmp 342 read_ctr \tmp 343 and \tmp, \tmp, #0xf // cache line size encoding 344 mov \reg, #4 // bytes per word 345 lsl \reg, \reg, \tmp // actual cache line size 346 .endm 347 348 /* 349 * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map 350 */ 351 .macro tcr_set_t0sz, valreg, t0sz 352 bfi \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH 353 .endm 354 355 /* 356 * tcr_set_t1sz - update TCR.T1SZ 357 */ 358 .macro tcr_set_t1sz, valreg, t1sz 359 bfi \valreg, \t1sz, #TCR_T1SZ_OFFSET, #TCR_TxSZ_WIDTH 360 .endm 361 362 /* 363 * idmap_get_t0sz - get the T0SZ value needed to cover the ID map 364 * 365 * Calculate the maximum allowed value for TCR_EL1.T0SZ so that the 366 * entire ID map region can be mapped. As T0SZ == (64 - #bits used), 367 * this number conveniently equals the number of leading zeroes in 368 * the physical address of _end. 369 */ 370 .macro idmap_get_t0sz, reg 371 adrp \reg, _end 372 orr \reg, \reg, #(1 << VA_BITS_MIN) - 1 373 clz \reg, \reg 374 .endm 375 376 /* 377 * tcr_compute_pa_size - set TCR.(I)PS to the highest supported 378 * ID_AA64MMFR0_EL1.PARange value 379 * 380 * tcr: register with the TCR_ELx value to be updated 381 * pos: IPS or PS bitfield position 382 * tmp{0,1}: temporary registers 383 */ 384 .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1 385 mrs \tmp0, ID_AA64MMFR0_EL1 386 // Narrow PARange to fit the PS field in TCR_ELx 387 ubfx \tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3 388 mov \tmp1, #ID_AA64MMFR0_PARANGE_MAX 389 cmp \tmp0, \tmp1 390 csel \tmp0, \tmp1, \tmp0, hi 391 bfi \tcr, \tmp0, \pos, #3 392 .endm 393 394 .macro __dcache_op_workaround_clean_cache, op, addr 395 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 396 dc \op, \addr 397 alternative_else 398 dc civac, \addr 399 alternative_endif 400 .endm 401 402 /* 403 * Macro to perform a data cache maintenance for the interval 404 * [start, end) with dcache line size explicitly provided. 405 * 406 * op: operation passed to dc instruction 407 * domain: domain used in dsb instruciton 408 * start: starting virtual address of the region 409 * end: end virtual address of the region 410 * linesz: dcache line size 411 * fixup: optional label to branch to on user fault 412 * Corrupts: start, end, tmp 413 */ 414 .macro dcache_by_myline_op op, domain, start, end, linesz, tmp, fixup 415 sub \tmp, \linesz, #1 416 bic \start, \start, \tmp 417 .Ldcache_op\@: 418 .ifc \op, cvau 419 __dcache_op_workaround_clean_cache \op, \start 420 .else 421 .ifc \op, cvac 422 __dcache_op_workaround_clean_cache \op, \start 423 .else 424 .ifc \op, cvap 425 sys 3, c7, c12, 1, \start // dc cvap 426 .else 427 .ifc \op, cvadp 428 sys 3, c7, c13, 1, \start // dc cvadp 429 .else 430 dc \op, \start 431 .endif 432 .endif 433 .endif 434 .endif 435 add \start, \start, \linesz 436 cmp \start, \end 437 b.lo .Ldcache_op\@ 438 dsb \domain 439 440 _cond_uaccess_extable .Ldcache_op\@, \fixup 441 .endm 442 443 /* 444 * Macro to perform a data cache maintenance for the interval 445 * [start, end) 446 * 447 * op: operation passed to dc instruction 448 * domain: domain used in dsb instruciton 449 * start: starting virtual address of the region 450 * end: end virtual address of the region 451 * fixup: optional label to branch to on user fault 452 * Corrupts: start, end, tmp1, tmp2 453 */ 454 .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup 455 dcache_line_size \tmp1, \tmp2 456 dcache_by_myline_op \op, \domain, \start, \end, \tmp1, \tmp2, \fixup 457 .endm 458 459 /* 460 * Macro to perform an instruction cache maintenance for the interval 461 * [start, end) 462 * 463 * start, end: virtual addresses describing the region 464 * fixup: optional label to branch to on user fault 465 * Corrupts: tmp1, tmp2 466 */ 467 .macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup 468 icache_line_size \tmp1, \tmp2 469 sub \tmp2, \tmp1, #1 470 bic \tmp2, \start, \tmp2 471 .Licache_op\@: 472 ic ivau, \tmp2 // invalidate I line PoU 473 add \tmp2, \tmp2, \tmp1 474 cmp \tmp2, \end 475 b.lo .Licache_op\@ 476 dsb ish 477 isb 478 479 _cond_uaccess_extable .Licache_op\@, \fixup 480 .endm 481 482 /* 483 * load_ttbr1 - install @pgtbl as a TTBR1 page table 484 * pgtbl preserved 485 * tmp1/tmp2 clobbered, either may overlap with pgtbl 486 */ 487 .macro load_ttbr1, pgtbl, tmp1, tmp2 488 phys_to_ttbr \tmp1, \pgtbl 489 offset_ttbr1 \tmp1, \tmp2 490 msr ttbr1_el1, \tmp1 491 isb 492 .endm 493 494 /* 495 * To prevent the possibility of old and new partial table walks being visible 496 * in the tlb, switch the ttbr to a zero page when we invalidate the old 497 * records. D4.7.1 'General TLB maintenance requirements' in ARM DDI 0487A.i 498 * Even switching to our copied tables will cause a changed output address at 499 * each stage of the walk. 500 */ 501 .macro break_before_make_ttbr_switch zero_page, page_table, tmp, tmp2 502 phys_to_ttbr \tmp, \zero_page 503 msr ttbr1_el1, \tmp 504 isb 505 tlbi vmalle1 506 dsb nsh 507 load_ttbr1 \page_table, \tmp, \tmp2 508 .endm 509 510 /* 511 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 512 */ 513 .macro reset_pmuserenr_el0, tmpreg 514 mrs \tmpreg, id_aa64dfr0_el1 515 sbfx \tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4 516 cmp \tmpreg, #1 // Skip if no PMU present 517 b.lt 9000f 518 msr pmuserenr_el0, xzr // Disable PMU access from EL0 519 9000: 520 .endm 521 522 /* 523 * reset_amuserenr_el0 - reset AMUSERENR_EL0 if AMUv1 present 524 */ 525 .macro reset_amuserenr_el0, tmpreg 526 mrs \tmpreg, id_aa64pfr0_el1 // Check ID_AA64PFR0_EL1 527 ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_AMU_SHIFT, #4 528 cbz \tmpreg, .Lskip_\@ // Skip if no AMU present 529 msr_s SYS_AMUSERENR_EL0, xzr // Disable AMU access from EL0 530 .Lskip_\@: 531 .endm 532 /* 533 * copy_page - copy src to dest using temp registers t1-t8 534 */ 535 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 536 9998: ldp \t1, \t2, [\src] 537 ldp \t3, \t4, [\src, #16] 538 ldp \t5, \t6, [\src, #32] 539 ldp \t7, \t8, [\src, #48] 540 add \src, \src, #64 541 stnp \t1, \t2, [\dest] 542 stnp \t3, \t4, [\dest, #16] 543 stnp \t5, \t6, [\dest, #32] 544 stnp \t7, \t8, [\dest, #48] 545 add \dest, \dest, #64 546 tst \src, #(PAGE_SIZE - 1) 547 b.ne 9998b 548 .endm 549 550 /* 551 * Annotate a function as being unsuitable for kprobes. 552 */ 553 #ifdef CONFIG_KPROBES 554 #define NOKPROBE(x) \ 555 .pushsection "_kprobe_blacklist", "aw"; \ 556 .quad x; \ 557 .popsection; 558 #else 559 #define NOKPROBE(x) 560 #endif 561 562 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) 563 #define EXPORT_SYMBOL_NOKASAN(name) 564 #else 565 #define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name) 566 #endif 567 568 /* 569 * Emit a 64-bit absolute little endian symbol reference in a way that 570 * ensures that it will be resolved at build time, even when building a 571 * PIE binary. This requires cooperation from the linker script, which 572 * must emit the lo32/hi32 halves individually. 573 */ 574 .macro le64sym, sym 575 .long \sym\()_lo32 576 .long \sym\()_hi32 577 .endm 578 579 /* 580 * mov_q - move an immediate constant into a 64-bit register using 581 * between 2 and 4 movz/movk instructions (depending on the 582 * magnitude and sign of the operand) 583 */ 584 .macro mov_q, reg, val 585 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 586 movz \reg, :abs_g1_s:\val 587 .else 588 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 589 movz \reg, :abs_g2_s:\val 590 .else 591 movz \reg, :abs_g3:\val 592 movk \reg, :abs_g2_nc:\val 593 .endif 594 movk \reg, :abs_g1_nc:\val 595 .endif 596 movk \reg, :abs_g0_nc:\val 597 .endm 598 599 /* 600 * Return the current task_struct. 601 */ 602 .macro get_current_task, rd 603 mrs \rd, sp_el0 604 .endm 605 606 /* 607 * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD. 608 * orr is used as it can cover the immediate value (and is idempotent). 609 * In future this may be nop'ed out when dealing with 52-bit kernel VAs. 610 * ttbr: Value of ttbr to set, modified. 611 */ 612 .macro offset_ttbr1, ttbr, tmp 613 #ifdef CONFIG_ARM64_VA_BITS_52 614 mrs_s \tmp, SYS_ID_AA64MMFR2_EL1 615 and \tmp, \tmp, #(0xf << ID_AA64MMFR2_LVA_SHIFT) 616 cbnz \tmp, .Lskipoffs_\@ 617 orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET 618 .Lskipoffs_\@ : 619 #endif 620 .endm 621 622 /* 623 * Perform the reverse of offset_ttbr1. 624 * bic is used as it can cover the immediate value and, in future, won't need 625 * to be nop'ed out when dealing with 52-bit kernel VAs. 626 */ 627 .macro restore_ttbr1, ttbr 628 #ifdef CONFIG_ARM64_VA_BITS_52 629 bic \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET 630 #endif 631 .endm 632 633 /* 634 * Arrange a physical address in a TTBR register, taking care of 52-bit 635 * addresses. 636 * 637 * phys: physical address, preserved 638 * ttbr: returns the TTBR value 639 */ 640 .macro phys_to_ttbr, ttbr, phys 641 #ifdef CONFIG_ARM64_PA_BITS_52 642 orr \ttbr, \phys, \phys, lsr #46 643 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52 644 #else 645 mov \ttbr, \phys 646 #endif 647 .endm 648 649 .macro phys_to_pte, pte, phys 650 #ifdef CONFIG_ARM64_PA_BITS_52 651 /* 652 * We assume \phys is 64K aligned and this is guaranteed by only 653 * supporting this configuration with 64K pages. 654 */ 655 orr \pte, \phys, \phys, lsr #36 656 and \pte, \pte, #PTE_ADDR_MASK 657 #else 658 mov \pte, \phys 659 #endif 660 .endm 661 662 .macro pte_to_phys, phys, pte 663 #ifdef CONFIG_ARM64_PA_BITS_52 664 ubfiz \phys, \pte, #(48 - 16 - 12), #16 665 bfxil \phys, \pte, #16, #32 666 lsl \phys, \phys, #16 667 #else 668 and \phys, \pte, #PTE_ADDR_MASK 669 #endif 670 .endm 671 672 /* 673 * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU. 674 */ 675 .macro tcr_clear_errata_bits, tcr, tmp1, tmp2 676 #ifdef CONFIG_FUJITSU_ERRATUM_010001 677 mrs \tmp1, midr_el1 678 679 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK 680 and \tmp1, \tmp1, \tmp2 681 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001 682 cmp \tmp1, \tmp2 683 b.ne 10f 684 685 mov_q \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001 686 bic \tcr, \tcr, \tmp2 687 10: 688 #endif /* CONFIG_FUJITSU_ERRATUM_010001 */ 689 .endm 690 691 /** 692 * Errata workaround prior to disable MMU. Insert an ISB immediately prior 693 * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0. 694 */ 695 .macro pre_disable_mmu_workaround 696 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041 697 isb 698 #endif 699 .endm 700 701 /* 702 * frame_push - Push @regcount callee saved registers to the stack, 703 * starting at x19, as well as x29/x30, and set x29 to 704 * the new value of sp. Add @extra bytes of stack space 705 * for locals. 706 */ 707 .macro frame_push, regcount:req, extra 708 __frame st, \regcount, \extra 709 .endm 710 711 /* 712 * frame_pop - Pop the callee saved registers from the stack that were 713 * pushed in the most recent call to frame_push, as well 714 * as x29/x30 and any extra stack space that may have been 715 * allocated. 716 */ 717 .macro frame_pop 718 __frame ld 719 .endm 720 721 .macro __frame_regs, reg1, reg2, op, num 722 .if .Lframe_regcount == \num 723 \op\()r \reg1, [sp, #(\num + 1) * 8] 724 .elseif .Lframe_regcount > \num 725 \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8] 726 .endif 727 .endm 728 729 .macro __frame, op, regcount, extra=0 730 .ifc \op, st 731 .if (\regcount) < 0 || (\regcount) > 10 732 .error "regcount should be in the range [0 ... 10]" 733 .endif 734 .if ((\extra) % 16) != 0 735 .error "extra should be a multiple of 16 bytes" 736 .endif 737 .ifdef .Lframe_regcount 738 .if .Lframe_regcount != -1 739 .error "frame_push/frame_pop may not be nested" 740 .endif 741 .endif 742 .set .Lframe_regcount, \regcount 743 .set .Lframe_extra, \extra 744 .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16 745 stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]! 746 mov x29, sp 747 .endif 748 749 __frame_regs x19, x20, \op, 1 750 __frame_regs x21, x22, \op, 3 751 __frame_regs x23, x24, \op, 5 752 __frame_regs x25, x26, \op, 7 753 __frame_regs x27, x28, \op, 9 754 755 .ifc \op, ld 756 .if .Lframe_regcount == -1 757 .error "frame_push/frame_pop may not be nested" 758 .endif 759 ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra 760 .set .Lframe_regcount, -1 761 .endif 762 .endm 763 764 /* 765 * Set SCTLR_ELx to the @reg value, and invalidate the local icache 766 * in the process. This is called when setting the MMU on. 767 */ 768 .macro set_sctlr, sreg, reg 769 msr \sreg, \reg 770 isb 771 /* 772 * Invalidate the local I-cache so that any instructions fetched 773 * speculatively from the PoC are discarded, since they may have 774 * been dynamically patched at the PoU. 775 */ 776 ic iallu 777 dsb nsh 778 isb 779 .endm 780 781 .macro set_sctlr_el1, reg 782 set_sctlr sctlr_el1, \reg 783 .endm 784 785 .macro set_sctlr_el2, reg 786 set_sctlr sctlr_el2, \reg 787 .endm 788 789 /* 790 * Check whether preempt/bh-disabled asm code should yield as soon as 791 * it is able. This is the case if we are currently running in task 792 * context, and either a softirq is pending, or the TIF_NEED_RESCHED 793 * flag is set and re-enabling preemption a single time would result in 794 * a preempt count of zero. (Note that the TIF_NEED_RESCHED flag is 795 * stored negated in the top word of the thread_info::preempt_count 796 * field) 797 */ 798 .macro cond_yield, lbl:req, tmp:req, tmp2:req 799 get_current_task \tmp 800 ldr \tmp, [\tmp, #TSK_TI_PREEMPT] 801 /* 802 * If we are serving a softirq, there is no point in yielding: the 803 * softirq will not be preempted no matter what we do, so we should 804 * run to completion as quickly as we can. 805 */ 806 tbnz \tmp, #SOFTIRQ_SHIFT, .Lnoyield_\@ 807 #ifdef CONFIG_PREEMPTION 808 sub \tmp, \tmp, #PREEMPT_DISABLE_OFFSET 809 cbz \tmp, \lbl 810 #endif 811 adr_l \tmp, irq_stat + IRQ_CPUSTAT_SOFTIRQ_PENDING 812 get_this_cpu_offset \tmp2 813 ldr w\tmp, [\tmp, \tmp2] 814 cbnz w\tmp, \lbl // yield on pending softirq in task context 815 .Lnoyield_\@: 816 .endm 817 818 /* 819 * Branch Target Identifier (BTI) 820 */ 821 .macro bti, targets 822 .equ .L__bti_targets_c, 34 823 .equ .L__bti_targets_j, 36 824 .equ .L__bti_targets_jc,38 825 hint #.L__bti_targets_\targets 826 .endm 827 828 /* 829 * This macro emits a program property note section identifying 830 * architecture features which require special handling, mainly for 831 * use in assembly files included in the VDSO. 832 */ 833 834 #define NT_GNU_PROPERTY_TYPE_0 5 835 #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000 836 837 #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0) 838 #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1) 839 840 #ifdef CONFIG_ARM64_BTI_KERNEL 841 #define GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT \ 842 ((GNU_PROPERTY_AARCH64_FEATURE_1_BTI | \ 843 GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) 844 #endif 845 846 #ifdef GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 847 .macro emit_aarch64_feature_1_and, feat=GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 848 .pushsection .note.gnu.property, "a" 849 .align 3 850 .long 2f - 1f 851 .long 6f - 3f 852 .long NT_GNU_PROPERTY_TYPE_0 853 1: .string "GNU" 854 2: 855 .align 3 856 3: .long GNU_PROPERTY_AARCH64_FEATURE_1_AND 857 .long 5f - 4f 858 4: 859 /* 860 * This is described with an array of char in the Linux API 861 * spec but the text and all other usage (including binutils, 862 * clang and GCC) treat this as a 32 bit value so no swizzling 863 * is required for big endian. 864 */ 865 .long \feat 866 5: 867 .align 3 868 6: 869 .popsection 870 .endm 871 872 #else 873 .macro emit_aarch64_feature_1_and, feat=0 874 .endm 875 876 #endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */ 877 878 .macro __mitigate_spectre_bhb_loop tmp 879 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 880 alternative_cb spectre_bhb_patch_loop_iter 881 mov \tmp, #32 // Patched to correct the immediate 882 alternative_cb_end 883 .Lspectre_bhb_loop\@: 884 b . + 4 885 subs \tmp, \tmp, #1 886 b.ne .Lspectre_bhb_loop\@ 887 sb 888 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 889 .endm 890 891 .macro mitigate_spectre_bhb_loop tmp 892 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 893 alternative_cb spectre_bhb_patch_loop_mitigation_enable 894 b .L_spectre_bhb_loop_done\@ // Patched to NOP 895 alternative_cb_end 896 __mitigate_spectre_bhb_loop \tmp 897 .L_spectre_bhb_loop_done\@: 898 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 899 .endm 900 901 /* Save/restores x0-x3 to the stack */ 902 .macro __mitigate_spectre_bhb_fw 903 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 904 stp x0, x1, [sp, #-16]! 905 stp x2, x3, [sp, #-16]! 906 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 907 alternative_cb smccc_patch_fw_mitigation_conduit 908 nop // Patched to SMC/HVC #0 909 alternative_cb_end 910 ldp x2, x3, [sp], #16 911 ldp x0, x1, [sp], #16 912 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 913 .endm 914 915 .macro mitigate_spectre_bhb_clear_insn 916 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 917 alternative_cb spectre_bhb_patch_clearbhb 918 /* Patched to NOP when not supported */ 919 clearbhb 920 isb 921 alternative_cb_end 922 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 923 .endm 924 #endif /* __ASM_ASSEMBLER_H */ 925