1 /* 2 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 3 * 4 * Copyright (C) 1996-2000 Russell King 5 * Copyright (C) 2012 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifndef __ASSEMBLY__ 20 #error "Only include this from assembly code" 21 #endif 22 23 #ifndef __ASM_ASSEMBLER_H 24 #define __ASM_ASSEMBLER_H 25 26 #include <asm/asm-offsets.h> 27 #include <asm/cpufeature.h> 28 #include <asm/mmu_context.h> 29 #include <asm/page.h> 30 #include <asm/pgtable-hwdef.h> 31 #include <asm/ptrace.h> 32 #include <asm/thread_info.h> 33 34 /* 35 * Enable and disable interrupts. 36 */ 37 .macro disable_irq 38 msr daifset, #2 39 .endm 40 41 .macro enable_irq 42 msr daifclr, #2 43 .endm 44 45 .macro save_and_disable_irq, flags 46 mrs \flags, daif 47 msr daifset, #2 48 .endm 49 50 .macro restore_irq, flags 51 msr daif, \flags 52 .endm 53 54 /* 55 * Enable and disable debug exceptions. 56 */ 57 .macro disable_dbg 58 msr daifset, #8 59 .endm 60 61 .macro enable_dbg 62 msr daifclr, #8 63 .endm 64 65 .macro disable_step_tsk, flgs, tmp 66 tbz \flgs, #TIF_SINGLESTEP, 9990f 67 mrs \tmp, mdscr_el1 68 bic \tmp, \tmp, #1 69 msr mdscr_el1, \tmp 70 isb // Synchronise with enable_dbg 71 9990: 72 .endm 73 74 .macro enable_step_tsk, flgs, tmp 75 tbz \flgs, #TIF_SINGLESTEP, 9990f 76 disable_dbg 77 mrs \tmp, mdscr_el1 78 orr \tmp, \tmp, #1 79 msr mdscr_el1, \tmp 80 9990: 81 .endm 82 83 /* 84 * Enable both debug exceptions and interrupts. This is likely to be 85 * faster than two daifclr operations, since writes to this register 86 * are self-synchronising. 87 */ 88 .macro enable_dbg_and_irq 89 msr daifclr, #(8 | 2) 90 .endm 91 92 /* 93 * SMP data memory barrier 94 */ 95 .macro smp_dmb, opt 96 dmb \opt 97 .endm 98 99 /* 100 * NOP sequence 101 */ 102 .macro nops, num 103 .rept \num 104 nop 105 .endr 106 .endm 107 108 /* 109 * Emit an entry into the exception table 110 */ 111 .macro _asm_extable, from, to 112 .pushsection __ex_table, "a" 113 .align 3 114 .long (\from - .), (\to - .) 115 .popsection 116 .endm 117 118 #define USER(l, x...) \ 119 9999: x; \ 120 _asm_extable 9999b, l 121 122 /* 123 * Register aliases. 124 */ 125 lr .req x30 // link register 126 127 /* 128 * Vector entry 129 */ 130 .macro ventry label 131 .align 7 132 b \label 133 .endm 134 135 /* 136 * Select code when configured for BE. 137 */ 138 #ifdef CONFIG_CPU_BIG_ENDIAN 139 #define CPU_BE(code...) code 140 #else 141 #define CPU_BE(code...) 142 #endif 143 144 /* 145 * Select code when configured for LE. 146 */ 147 #ifdef CONFIG_CPU_BIG_ENDIAN 148 #define CPU_LE(code...) 149 #else 150 #define CPU_LE(code...) code 151 #endif 152 153 /* 154 * Define a macro that constructs a 64-bit value by concatenating two 155 * 32-bit registers. Note that on big endian systems the order of the 156 * registers is swapped. 157 */ 158 #ifndef CONFIG_CPU_BIG_ENDIAN 159 .macro regs_to_64, rd, lbits, hbits 160 #else 161 .macro regs_to_64, rd, hbits, lbits 162 #endif 163 orr \rd, \lbits, \hbits, lsl #32 164 .endm 165 166 /* 167 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 168 * <symbol> is within the range +/- 4 GB of the PC when running 169 * in core kernel context. In module context, a movz/movk sequence 170 * is used, since modules may be loaded far away from the kernel 171 * when KASLR is in effect. 172 */ 173 /* 174 * @dst: destination register (64 bit wide) 175 * @sym: name of the symbol 176 */ 177 .macro adr_l, dst, sym 178 #ifndef MODULE 179 adrp \dst, \sym 180 add \dst, \dst, :lo12:\sym 181 #else 182 movz \dst, #:abs_g3:\sym 183 movk \dst, #:abs_g2_nc:\sym 184 movk \dst, #:abs_g1_nc:\sym 185 movk \dst, #:abs_g0_nc:\sym 186 #endif 187 .endm 188 189 /* 190 * @dst: destination register (32 or 64 bit wide) 191 * @sym: name of the symbol 192 * @tmp: optional 64-bit scratch register to be used if <dst> is a 193 * 32-bit wide register, in which case it cannot be used to hold 194 * the address 195 */ 196 .macro ldr_l, dst, sym, tmp= 197 #ifndef MODULE 198 .ifb \tmp 199 adrp \dst, \sym 200 ldr \dst, [\dst, :lo12:\sym] 201 .else 202 adrp \tmp, \sym 203 ldr \dst, [\tmp, :lo12:\sym] 204 .endif 205 #else 206 .ifb \tmp 207 adr_l \dst, \sym 208 ldr \dst, [\dst] 209 .else 210 adr_l \tmp, \sym 211 ldr \dst, [\tmp] 212 .endif 213 #endif 214 .endm 215 216 /* 217 * @src: source register (32 or 64 bit wide) 218 * @sym: name of the symbol 219 * @tmp: mandatory 64-bit scratch register to calculate the address 220 * while <src> needs to be preserved. 221 */ 222 .macro str_l, src, sym, tmp 223 #ifndef MODULE 224 adrp \tmp, \sym 225 str \src, [\tmp, :lo12:\sym] 226 #else 227 adr_l \tmp, \sym 228 str \src, [\tmp] 229 #endif 230 .endm 231 232 /* 233 * @dst: Result of per_cpu(sym, smp_processor_id()) 234 * @sym: The name of the per-cpu variable 235 * @tmp: scratch register 236 */ 237 .macro adr_this_cpu, dst, sym, tmp 238 adr_l \dst, \sym 239 mrs \tmp, tpidr_el1 240 add \dst, \dst, \tmp 241 .endm 242 243 /* 244 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 245 * @sym: The name of the per-cpu variable 246 * @tmp: scratch register 247 */ 248 .macro ldr_this_cpu dst, sym, tmp 249 adr_l \dst, \sym 250 mrs \tmp, tpidr_el1 251 ldr \dst, [\dst, \tmp] 252 .endm 253 254 /* 255 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) 256 */ 257 .macro vma_vm_mm, rd, rn 258 ldr \rd, [\rn, #VMA_VM_MM] 259 .endm 260 261 /* 262 * mmid - get context id from mm pointer (mm->context.id) 263 */ 264 .macro mmid, rd, rn 265 ldr \rd, [\rn, #MM_CONTEXT_ID] 266 .endm 267 /* 268 * read_ctr - read CTR_EL0. If the system has mismatched 269 * cache line sizes, provide the system wide safe value 270 * from arm64_ftr_reg_ctrel0.sys_val 271 */ 272 .macro read_ctr, reg 273 alternative_if_not ARM64_MISMATCHED_CACHE_LINE_SIZE 274 mrs \reg, ctr_el0 // read CTR 275 nop 276 alternative_else 277 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 278 alternative_endif 279 .endm 280 281 282 /* 283 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 284 * from the CTR register. 285 */ 286 .macro raw_dcache_line_size, reg, tmp 287 mrs \tmp, ctr_el0 // read CTR 288 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 289 mov \reg, #4 // bytes per word 290 lsl \reg, \reg, \tmp // actual cache line size 291 .endm 292 293 /* 294 * dcache_line_size - get the safe D-cache line size across all CPUs 295 */ 296 .macro dcache_line_size, reg, tmp 297 read_ctr \tmp 298 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 299 mov \reg, #4 // bytes per word 300 lsl \reg, \reg, \tmp // actual cache line size 301 .endm 302 303 /* 304 * raw_icache_line_size - get the minimum I-cache line size on this CPU 305 * from the CTR register. 306 */ 307 .macro raw_icache_line_size, reg, tmp 308 mrs \tmp, ctr_el0 // read CTR 309 and \tmp, \tmp, #0xf // cache line size encoding 310 mov \reg, #4 // bytes per word 311 lsl \reg, \reg, \tmp // actual cache line size 312 .endm 313 314 /* 315 * icache_line_size - get the safe I-cache line size across all CPUs 316 */ 317 .macro icache_line_size, reg, tmp 318 read_ctr \tmp 319 and \tmp, \tmp, #0xf // cache line size encoding 320 mov \reg, #4 // bytes per word 321 lsl \reg, \reg, \tmp // actual cache line size 322 .endm 323 324 /* 325 * tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map 326 */ 327 .macro tcr_set_idmap_t0sz, valreg, tmpreg 328 #ifndef CONFIG_ARM64_VA_BITS_48 329 ldr_l \tmpreg, idmap_t0sz 330 bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH 331 #endif 332 .endm 333 334 /* 335 * Macro to perform a data cache maintenance for the interval 336 * [kaddr, kaddr + size) 337 * 338 * op: operation passed to dc instruction 339 * domain: domain used in dsb instruciton 340 * kaddr: starting virtual address of the region 341 * size: size of the region 342 * Corrupts: kaddr, size, tmp1, tmp2 343 */ 344 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 345 dcache_line_size \tmp1, \tmp2 346 add \size, \kaddr, \size 347 sub \tmp2, \tmp1, #1 348 bic \kaddr, \kaddr, \tmp2 349 9998: 350 .if (\op == cvau || \op == cvac) 351 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 352 dc \op, \kaddr 353 alternative_else 354 dc civac, \kaddr 355 alternative_endif 356 .else 357 dc \op, \kaddr 358 .endif 359 add \kaddr, \kaddr, \tmp1 360 cmp \kaddr, \size 361 b.lo 9998b 362 dsb \domain 363 .endm 364 365 /* 366 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 367 */ 368 .macro reset_pmuserenr_el0, tmpreg 369 mrs \tmpreg, id_aa64dfr0_el1 // Check ID_AA64DFR0_EL1 PMUVer 370 sbfx \tmpreg, \tmpreg, #8, #4 371 cmp \tmpreg, #1 // Skip if no PMU present 372 b.lt 9000f 373 msr pmuserenr_el0, xzr // Disable PMU access from EL0 374 9000: 375 .endm 376 377 /* 378 * copy_page - copy src to dest using temp registers t1-t8 379 */ 380 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 381 9998: ldp \t1, \t2, [\src] 382 ldp \t3, \t4, [\src, #16] 383 ldp \t5, \t6, [\src, #32] 384 ldp \t7, \t8, [\src, #48] 385 add \src, \src, #64 386 stnp \t1, \t2, [\dest] 387 stnp \t3, \t4, [\dest, #16] 388 stnp \t5, \t6, [\dest, #32] 389 stnp \t7, \t8, [\dest, #48] 390 add \dest, \dest, #64 391 tst \src, #(PAGE_SIZE - 1) 392 b.ne 9998b 393 .endm 394 395 /* 396 * Annotate a function as position independent, i.e., safe to be called before 397 * the kernel virtual mapping is activated. 398 */ 399 #define ENDPIPROC(x) \ 400 .globl __pi_##x; \ 401 .type __pi_##x, %function; \ 402 .set __pi_##x, x; \ 403 .size __pi_##x, . - x; \ 404 ENDPROC(x) 405 406 /* 407 * Emit a 64-bit absolute little endian symbol reference in a way that 408 * ensures that it will be resolved at build time, even when building a 409 * PIE binary. This requires cooperation from the linker script, which 410 * must emit the lo32/hi32 halves individually. 411 */ 412 .macro le64sym, sym 413 .long \sym\()_lo32 414 .long \sym\()_hi32 415 .endm 416 417 /* 418 * mov_q - move an immediate constant into a 64-bit register using 419 * between 2 and 4 movz/movk instructions (depending on the 420 * magnitude and sign of the operand) 421 */ 422 .macro mov_q, reg, val 423 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 424 movz \reg, :abs_g1_s:\val 425 .else 426 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 427 movz \reg, :abs_g2_s:\val 428 .else 429 movz \reg, :abs_g3:\val 430 movk \reg, :abs_g2_nc:\val 431 .endif 432 movk \reg, :abs_g1_nc:\val 433 .endif 434 movk \reg, :abs_g0_nc:\val 435 .endm 436 437 /* 438 * Return the current thread_info. 439 */ 440 .macro get_thread_info, rd 441 mrs \rd, sp_el0 442 .endm 443 444 /* 445 * Errata workaround prior to TTBR0_EL1 update 446 * 447 * val: TTBR value with new BADDR, preserved 448 * tmp0: temporary register, clobbered 449 * tmp1: other temporary register, clobbered 450 */ 451 .macro pre_ttbr0_update_workaround, val, tmp0, tmp1 452 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003 453 alternative_if ARM64_WORKAROUND_QCOM_FALKOR_E1003 454 mrs \tmp0, ttbr0_el1 455 mov \tmp1, #FALKOR_RESERVED_ASID 456 bfi \tmp0, \tmp1, #48, #16 // reserved ASID + old BADDR 457 msr ttbr0_el1, \tmp0 458 isb 459 bfi \tmp0, \val, #0, #48 // reserved ASID + new BADDR 460 msr ttbr0_el1, \tmp0 461 isb 462 alternative_else_nop_endif 463 #endif 464 .endm 465 466 /* 467 * Errata workaround post TTBR0_EL1 update. 468 */ 469 .macro post_ttbr0_update_workaround 470 #ifdef CONFIG_CAVIUM_ERRATUM_27456 471 alternative_if ARM64_WORKAROUND_CAVIUM_27456 472 ic iallu 473 dsb nsh 474 isb 475 alternative_else_nop_endif 476 #endif 477 .endm 478 479 #endif /* __ASM_ASSEMBLER_H */ 480