1 /* 2 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 3 * 4 * Copyright (C) 1996-2000 Russell King 5 * Copyright (C) 2012 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifndef __ASSEMBLY__ 20 #error "Only include this from assembly code" 21 #endif 22 23 #ifndef __ASM_ASSEMBLER_H 24 #define __ASM_ASSEMBLER_H 25 26 #include <asm/asm-offsets.h> 27 #include <asm/cpufeature.h> 28 #include <asm/page.h> 29 #include <asm/pgtable-hwdef.h> 30 #include <asm/ptrace.h> 31 #include <asm/thread_info.h> 32 33 /* 34 * Enable and disable interrupts. 35 */ 36 .macro disable_irq 37 msr daifset, #2 38 .endm 39 40 .macro enable_irq 41 msr daifclr, #2 42 .endm 43 44 .macro save_and_disable_irq, flags 45 mrs \flags, daif 46 msr daifset, #2 47 .endm 48 49 .macro restore_irq, flags 50 msr daif, \flags 51 .endm 52 53 /* 54 * Enable and disable debug exceptions. 55 */ 56 .macro disable_dbg 57 msr daifset, #8 58 .endm 59 60 .macro enable_dbg 61 msr daifclr, #8 62 .endm 63 64 .macro disable_step_tsk, flgs, tmp 65 tbz \flgs, #TIF_SINGLESTEP, 9990f 66 mrs \tmp, mdscr_el1 67 bic \tmp, \tmp, #1 68 msr mdscr_el1, \tmp 69 isb // Synchronise with enable_dbg 70 9990: 71 .endm 72 73 .macro enable_step_tsk, flgs, tmp 74 tbz \flgs, #TIF_SINGLESTEP, 9990f 75 disable_dbg 76 mrs \tmp, mdscr_el1 77 orr \tmp, \tmp, #1 78 msr mdscr_el1, \tmp 79 9990: 80 .endm 81 82 /* 83 * Enable both debug exceptions and interrupts. This is likely to be 84 * faster than two daifclr operations, since writes to this register 85 * are self-synchronising. 86 */ 87 .macro enable_dbg_and_irq 88 msr daifclr, #(8 | 2) 89 .endm 90 91 /* 92 * SMP data memory barrier 93 */ 94 .macro smp_dmb, opt 95 dmb \opt 96 .endm 97 98 /* 99 * NOP sequence 100 */ 101 .macro nops, num 102 .rept \num 103 nop 104 .endr 105 .endm 106 107 /* 108 * Emit an entry into the exception table 109 */ 110 .macro _asm_extable, from, to 111 .pushsection __ex_table, "a" 112 .align 3 113 .long (\from - .), (\to - .) 114 .popsection 115 .endm 116 117 #define USER(l, x...) \ 118 9999: x; \ 119 _asm_extable 9999b, l 120 121 /* 122 * Register aliases. 123 */ 124 lr .req x30 // link register 125 126 /* 127 * Vector entry 128 */ 129 .macro ventry label 130 .align 7 131 b \label 132 .endm 133 134 /* 135 * Select code when configured for BE. 136 */ 137 #ifdef CONFIG_CPU_BIG_ENDIAN 138 #define CPU_BE(code...) code 139 #else 140 #define CPU_BE(code...) 141 #endif 142 143 /* 144 * Select code when configured for LE. 145 */ 146 #ifdef CONFIG_CPU_BIG_ENDIAN 147 #define CPU_LE(code...) 148 #else 149 #define CPU_LE(code...) code 150 #endif 151 152 /* 153 * Define a macro that constructs a 64-bit value by concatenating two 154 * 32-bit registers. Note that on big endian systems the order of the 155 * registers is swapped. 156 */ 157 #ifndef CONFIG_CPU_BIG_ENDIAN 158 .macro regs_to_64, rd, lbits, hbits 159 #else 160 .macro regs_to_64, rd, hbits, lbits 161 #endif 162 orr \rd, \lbits, \hbits, lsl #32 163 .endm 164 165 /* 166 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 167 * <symbol> is within the range +/- 4 GB of the PC when running 168 * in core kernel context. In module context, a movz/movk sequence 169 * is used, since modules may be loaded far away from the kernel 170 * when KASLR is in effect. 171 */ 172 /* 173 * @dst: destination register (64 bit wide) 174 * @sym: name of the symbol 175 */ 176 .macro adr_l, dst, sym 177 #ifndef MODULE 178 adrp \dst, \sym 179 add \dst, \dst, :lo12:\sym 180 #else 181 movz \dst, #:abs_g3:\sym 182 movk \dst, #:abs_g2_nc:\sym 183 movk \dst, #:abs_g1_nc:\sym 184 movk \dst, #:abs_g0_nc:\sym 185 #endif 186 .endm 187 188 /* 189 * @dst: destination register (32 or 64 bit wide) 190 * @sym: name of the symbol 191 * @tmp: optional 64-bit scratch register to be used if <dst> is a 192 * 32-bit wide register, in which case it cannot be used to hold 193 * the address 194 */ 195 .macro ldr_l, dst, sym, tmp= 196 #ifndef MODULE 197 .ifb \tmp 198 adrp \dst, \sym 199 ldr \dst, [\dst, :lo12:\sym] 200 .else 201 adrp \tmp, \sym 202 ldr \dst, [\tmp, :lo12:\sym] 203 .endif 204 #else 205 .ifb \tmp 206 adr_l \dst, \sym 207 ldr \dst, [\dst] 208 .else 209 adr_l \tmp, \sym 210 ldr \dst, [\tmp] 211 .endif 212 #endif 213 .endm 214 215 /* 216 * @src: source register (32 or 64 bit wide) 217 * @sym: name of the symbol 218 * @tmp: mandatory 64-bit scratch register to calculate the address 219 * while <src> needs to be preserved. 220 */ 221 .macro str_l, src, sym, tmp 222 #ifndef MODULE 223 adrp \tmp, \sym 224 str \src, [\tmp, :lo12:\sym] 225 #else 226 adr_l \tmp, \sym 227 str \src, [\tmp] 228 #endif 229 .endm 230 231 /* 232 * @dst: Result of per_cpu(sym, smp_processor_id()) 233 * @sym: The name of the per-cpu variable 234 * @tmp: scratch register 235 */ 236 .macro adr_this_cpu, dst, sym, tmp 237 adr_l \dst, \sym 238 mrs \tmp, tpidr_el1 239 add \dst, \dst, \tmp 240 .endm 241 242 /* 243 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 244 * @sym: The name of the per-cpu variable 245 * @tmp: scratch register 246 */ 247 .macro ldr_this_cpu dst, sym, tmp 248 adr_l \dst, \sym 249 mrs \tmp, tpidr_el1 250 ldr \dst, [\dst, \tmp] 251 .endm 252 253 /* 254 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) 255 */ 256 .macro vma_vm_mm, rd, rn 257 ldr \rd, [\rn, #VMA_VM_MM] 258 .endm 259 260 /* 261 * mmid - get context id from mm pointer (mm->context.id) 262 */ 263 .macro mmid, rd, rn 264 ldr \rd, [\rn, #MM_CONTEXT_ID] 265 .endm 266 /* 267 * read_ctr - read CTR_EL0. If the system has mismatched 268 * cache line sizes, provide the system wide safe value 269 * from arm64_ftr_reg_ctrel0.sys_val 270 */ 271 .macro read_ctr, reg 272 alternative_if_not ARM64_MISMATCHED_CACHE_LINE_SIZE 273 mrs \reg, ctr_el0 // read CTR 274 nop 275 alternative_else 276 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 277 alternative_endif 278 .endm 279 280 281 /* 282 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 283 * from the CTR register. 284 */ 285 .macro raw_dcache_line_size, reg, tmp 286 mrs \tmp, ctr_el0 // read CTR 287 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 288 mov \reg, #4 // bytes per word 289 lsl \reg, \reg, \tmp // actual cache line size 290 .endm 291 292 /* 293 * dcache_line_size - get the safe D-cache line size across all CPUs 294 */ 295 .macro dcache_line_size, reg, tmp 296 read_ctr \tmp 297 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 298 mov \reg, #4 // bytes per word 299 lsl \reg, \reg, \tmp // actual cache line size 300 .endm 301 302 /* 303 * raw_icache_line_size - get the minimum I-cache line size on this CPU 304 * from the CTR register. 305 */ 306 .macro raw_icache_line_size, reg, tmp 307 mrs \tmp, ctr_el0 // read CTR 308 and \tmp, \tmp, #0xf // cache line size encoding 309 mov \reg, #4 // bytes per word 310 lsl \reg, \reg, \tmp // actual cache line size 311 .endm 312 313 /* 314 * icache_line_size - get the safe I-cache line size across all CPUs 315 */ 316 .macro icache_line_size, reg, tmp 317 read_ctr \tmp 318 and \tmp, \tmp, #0xf // cache line size encoding 319 mov \reg, #4 // bytes per word 320 lsl \reg, \reg, \tmp // actual cache line size 321 .endm 322 323 /* 324 * tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map 325 */ 326 .macro tcr_set_idmap_t0sz, valreg, tmpreg 327 #ifndef CONFIG_ARM64_VA_BITS_48 328 ldr_l \tmpreg, idmap_t0sz 329 bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH 330 #endif 331 .endm 332 333 /* 334 * Macro to perform a data cache maintenance for the interval 335 * [kaddr, kaddr + size) 336 * 337 * op: operation passed to dc instruction 338 * domain: domain used in dsb instruciton 339 * kaddr: starting virtual address of the region 340 * size: size of the region 341 * Corrupts: kaddr, size, tmp1, tmp2 342 */ 343 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 344 dcache_line_size \tmp1, \tmp2 345 add \size, \kaddr, \size 346 sub \tmp2, \tmp1, #1 347 bic \kaddr, \kaddr, \tmp2 348 9998: 349 .if (\op == cvau || \op == cvac) 350 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 351 dc \op, \kaddr 352 alternative_else 353 dc civac, \kaddr 354 alternative_endif 355 .else 356 dc \op, \kaddr 357 .endif 358 add \kaddr, \kaddr, \tmp1 359 cmp \kaddr, \size 360 b.lo 9998b 361 dsb \domain 362 .endm 363 364 /* 365 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 366 */ 367 .macro reset_pmuserenr_el0, tmpreg 368 mrs \tmpreg, id_aa64dfr0_el1 // Check ID_AA64DFR0_EL1 PMUVer 369 sbfx \tmpreg, \tmpreg, #8, #4 370 cmp \tmpreg, #1 // Skip if no PMU present 371 b.lt 9000f 372 msr pmuserenr_el0, xzr // Disable PMU access from EL0 373 9000: 374 .endm 375 376 /* 377 * copy_page - copy src to dest using temp registers t1-t8 378 */ 379 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 380 9998: ldp \t1, \t2, [\src] 381 ldp \t3, \t4, [\src, #16] 382 ldp \t5, \t6, [\src, #32] 383 ldp \t7, \t8, [\src, #48] 384 add \src, \src, #64 385 stnp \t1, \t2, [\dest] 386 stnp \t3, \t4, [\dest, #16] 387 stnp \t5, \t6, [\dest, #32] 388 stnp \t7, \t8, [\dest, #48] 389 add \dest, \dest, #64 390 tst \src, #(PAGE_SIZE - 1) 391 b.ne 9998b 392 .endm 393 394 /* 395 * Annotate a function as position independent, i.e., safe to be called before 396 * the kernel virtual mapping is activated. 397 */ 398 #define ENDPIPROC(x) \ 399 .globl __pi_##x; \ 400 .type __pi_##x, %function; \ 401 .set __pi_##x, x; \ 402 .size __pi_##x, . - x; \ 403 ENDPROC(x) 404 405 /* 406 * Emit a 64-bit absolute little endian symbol reference in a way that 407 * ensures that it will be resolved at build time, even when building a 408 * PIE binary. This requires cooperation from the linker script, which 409 * must emit the lo32/hi32 halves individually. 410 */ 411 .macro le64sym, sym 412 .long \sym\()_lo32 413 .long \sym\()_hi32 414 .endm 415 416 /* 417 * mov_q - move an immediate constant into a 64-bit register using 418 * between 2 and 4 movz/movk instructions (depending on the 419 * magnitude and sign of the operand) 420 */ 421 .macro mov_q, reg, val 422 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 423 movz \reg, :abs_g1_s:\val 424 .else 425 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 426 movz \reg, :abs_g2_s:\val 427 .else 428 movz \reg, :abs_g3:\val 429 movk \reg, :abs_g2_nc:\val 430 .endif 431 movk \reg, :abs_g1_nc:\val 432 .endif 433 movk \reg, :abs_g0_nc:\val 434 .endm 435 436 /* 437 * Return the current thread_info. 438 */ 439 .macro get_thread_info, rd 440 mrs \rd, sp_el0 441 .endm 442 443 /* 444 * Errata workaround post TTBR0_EL1 update. 445 */ 446 .macro post_ttbr0_update_workaround 447 #ifdef CONFIG_CAVIUM_ERRATUM_27456 448 alternative_if ARM64_WORKAROUND_CAVIUM_27456 449 ic iallu 450 dsb nsh 451 isb 452 alternative_else_nop_endif 453 #endif 454 .endm 455 456 #endif /* __ASM_ASSEMBLER_H */ 457