1 /* 2 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 3 * 4 * Copyright (C) 1996-2000 Russell King 5 * Copyright (C) 2012 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifndef __ASSEMBLY__ 20 #error "Only include this from assembly code" 21 #endif 22 23 #ifndef __ASM_ASSEMBLER_H 24 #define __ASM_ASSEMBLER_H 25 26 #include <asm/asm-offsets.h> 27 #include <asm/cpufeature.h> 28 #include <asm/page.h> 29 #include <asm/pgtable-hwdef.h> 30 #include <asm/ptrace.h> 31 #include <asm/thread_info.h> 32 33 /* 34 * Enable and disable interrupts. 35 */ 36 .macro disable_irq 37 msr daifset, #2 38 .endm 39 40 .macro enable_irq 41 msr daifclr, #2 42 .endm 43 44 .macro save_and_disable_irq, flags 45 mrs \flags, daif 46 msr daifset, #2 47 .endm 48 49 .macro restore_irq, flags 50 msr daif, \flags 51 .endm 52 53 /* 54 * Enable and disable debug exceptions. 55 */ 56 .macro disable_dbg 57 msr daifset, #8 58 .endm 59 60 .macro enable_dbg 61 msr daifclr, #8 62 .endm 63 64 .macro disable_step_tsk, flgs, tmp 65 tbz \flgs, #TIF_SINGLESTEP, 9990f 66 mrs \tmp, mdscr_el1 67 bic \tmp, \tmp, #1 68 msr mdscr_el1, \tmp 69 isb // Synchronise with enable_dbg 70 9990: 71 .endm 72 73 .macro enable_step_tsk, flgs, tmp 74 tbz \flgs, #TIF_SINGLESTEP, 9990f 75 disable_dbg 76 mrs \tmp, mdscr_el1 77 orr \tmp, \tmp, #1 78 msr mdscr_el1, \tmp 79 9990: 80 .endm 81 82 /* 83 * Enable both debug exceptions and interrupts. This is likely to be 84 * faster than two daifclr operations, since writes to this register 85 * are self-synchronising. 86 */ 87 .macro enable_dbg_and_irq 88 msr daifclr, #(8 | 2) 89 .endm 90 91 /* 92 * SMP data memory barrier 93 */ 94 .macro smp_dmb, opt 95 dmb \opt 96 .endm 97 98 /* 99 * NOP sequence 100 */ 101 .macro nops, num 102 .rept \num 103 nop 104 .endr 105 .endm 106 107 /* 108 * Emit an entry into the exception table 109 */ 110 .macro _asm_extable, from, to 111 .pushsection __ex_table, "a" 112 .align 3 113 .long (\from - .), (\to - .) 114 .popsection 115 .endm 116 117 #define USER(l, x...) \ 118 9999: x; \ 119 _asm_extable 9999b, l 120 121 /* 122 * Register aliases. 123 */ 124 lr .req x30 // link register 125 126 /* 127 * Vector entry 128 */ 129 .macro ventry label 130 .align 7 131 b \label 132 .endm 133 134 /* 135 * Select code when configured for BE. 136 */ 137 #ifdef CONFIG_CPU_BIG_ENDIAN 138 #define CPU_BE(code...) code 139 #else 140 #define CPU_BE(code...) 141 #endif 142 143 /* 144 * Select code when configured for LE. 145 */ 146 #ifdef CONFIG_CPU_BIG_ENDIAN 147 #define CPU_LE(code...) 148 #else 149 #define CPU_LE(code...) code 150 #endif 151 152 /* 153 * Define a macro that constructs a 64-bit value by concatenating two 154 * 32-bit registers. Note that on big endian systems the order of the 155 * registers is swapped. 156 */ 157 #ifndef CONFIG_CPU_BIG_ENDIAN 158 .macro regs_to_64, rd, lbits, hbits 159 #else 160 .macro regs_to_64, rd, hbits, lbits 161 #endif 162 orr \rd, \lbits, \hbits, lsl #32 163 .endm 164 165 /* 166 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 167 * <symbol> is within the range +/- 4 GB of the PC. 168 */ 169 /* 170 * @dst: destination register (64 bit wide) 171 * @sym: name of the symbol 172 * @tmp: optional scratch register to be used if <dst> == sp, which 173 * is not allowed in an adrp instruction 174 */ 175 .macro adr_l, dst, sym, tmp= 176 .ifb \tmp 177 adrp \dst, \sym 178 add \dst, \dst, :lo12:\sym 179 .else 180 adrp \tmp, \sym 181 add \dst, \tmp, :lo12:\sym 182 .endif 183 .endm 184 185 /* 186 * @dst: destination register (32 or 64 bit wide) 187 * @sym: name of the symbol 188 * @tmp: optional 64-bit scratch register to be used if <dst> is a 189 * 32-bit wide register, in which case it cannot be used to hold 190 * the address 191 */ 192 .macro ldr_l, dst, sym, tmp= 193 .ifb \tmp 194 adrp \dst, \sym 195 ldr \dst, [\dst, :lo12:\sym] 196 .else 197 adrp \tmp, \sym 198 ldr \dst, [\tmp, :lo12:\sym] 199 .endif 200 .endm 201 202 /* 203 * @src: source register (32 or 64 bit wide) 204 * @sym: name of the symbol 205 * @tmp: mandatory 64-bit scratch register to calculate the address 206 * while <src> needs to be preserved. 207 */ 208 .macro str_l, src, sym, tmp 209 adrp \tmp, \sym 210 str \src, [\tmp, :lo12:\sym] 211 .endm 212 213 /* 214 * @dst: Result of per_cpu(sym, smp_processor_id()) 215 * @sym: The name of the per-cpu variable 216 * @tmp: scratch register 217 */ 218 .macro adr_this_cpu, dst, sym, tmp 219 adr_l \dst, \sym 220 mrs \tmp, tpidr_el1 221 add \dst, \dst, \tmp 222 .endm 223 224 /* 225 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 226 * @sym: The name of the per-cpu variable 227 * @tmp: scratch register 228 */ 229 .macro ldr_this_cpu dst, sym, tmp 230 adr_l \dst, \sym 231 mrs \tmp, tpidr_el1 232 ldr \dst, [\dst, \tmp] 233 .endm 234 235 /* 236 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) 237 */ 238 .macro vma_vm_mm, rd, rn 239 ldr \rd, [\rn, #VMA_VM_MM] 240 .endm 241 242 /* 243 * mmid - get context id from mm pointer (mm->context.id) 244 */ 245 .macro mmid, rd, rn 246 ldr \rd, [\rn, #MM_CONTEXT_ID] 247 .endm 248 /* 249 * read_ctr - read CTR_EL0. If the system has mismatched 250 * cache line sizes, provide the system wide safe value 251 * from arm64_ftr_reg_ctrel0.sys_val 252 */ 253 .macro read_ctr, reg 254 alternative_if_not ARM64_MISMATCHED_CACHE_LINE_SIZE 255 mrs \reg, ctr_el0 // read CTR 256 nop 257 alternative_else 258 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 259 alternative_endif 260 .endm 261 262 263 /* 264 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 265 * from the CTR register. 266 */ 267 .macro raw_dcache_line_size, reg, tmp 268 mrs \tmp, ctr_el0 // read CTR 269 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 270 mov \reg, #4 // bytes per word 271 lsl \reg, \reg, \tmp // actual cache line size 272 .endm 273 274 /* 275 * dcache_line_size - get the safe D-cache line size across all CPUs 276 */ 277 .macro dcache_line_size, reg, tmp 278 read_ctr \tmp 279 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 280 mov \reg, #4 // bytes per word 281 lsl \reg, \reg, \tmp // actual cache line size 282 .endm 283 284 /* 285 * raw_icache_line_size - get the minimum I-cache line size on this CPU 286 * from the CTR register. 287 */ 288 .macro raw_icache_line_size, reg, tmp 289 mrs \tmp, ctr_el0 // read CTR 290 and \tmp, \tmp, #0xf // cache line size encoding 291 mov \reg, #4 // bytes per word 292 lsl \reg, \reg, \tmp // actual cache line size 293 .endm 294 295 /* 296 * icache_line_size - get the safe I-cache line size across all CPUs 297 */ 298 .macro icache_line_size, reg, tmp 299 read_ctr \tmp 300 and \tmp, \tmp, #0xf // cache line size encoding 301 mov \reg, #4 // bytes per word 302 lsl \reg, \reg, \tmp // actual cache line size 303 .endm 304 305 /* 306 * tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map 307 */ 308 .macro tcr_set_idmap_t0sz, valreg, tmpreg 309 #ifndef CONFIG_ARM64_VA_BITS_48 310 ldr_l \tmpreg, idmap_t0sz 311 bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH 312 #endif 313 .endm 314 315 /* 316 * Macro to perform a data cache maintenance for the interval 317 * [kaddr, kaddr + size) 318 * 319 * op: operation passed to dc instruction 320 * domain: domain used in dsb instruciton 321 * kaddr: starting virtual address of the region 322 * size: size of the region 323 * Corrupts: kaddr, size, tmp1, tmp2 324 */ 325 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 326 dcache_line_size \tmp1, \tmp2 327 add \size, \kaddr, \size 328 sub \tmp2, \tmp1, #1 329 bic \kaddr, \kaddr, \tmp2 330 9998: 331 .if (\op == cvau || \op == cvac) 332 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 333 dc \op, \kaddr 334 alternative_else 335 dc civac, \kaddr 336 alternative_endif 337 .else 338 dc \op, \kaddr 339 .endif 340 add \kaddr, \kaddr, \tmp1 341 cmp \kaddr, \size 342 b.lo 9998b 343 dsb \domain 344 .endm 345 346 /* 347 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 348 */ 349 .macro reset_pmuserenr_el0, tmpreg 350 mrs \tmpreg, id_aa64dfr0_el1 // Check ID_AA64DFR0_EL1 PMUVer 351 sbfx \tmpreg, \tmpreg, #8, #4 352 cmp \tmpreg, #1 // Skip if no PMU present 353 b.lt 9000f 354 msr pmuserenr_el0, xzr // Disable PMU access from EL0 355 9000: 356 .endm 357 358 /* 359 * copy_page - copy src to dest using temp registers t1-t8 360 */ 361 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 362 9998: ldp \t1, \t2, [\src] 363 ldp \t3, \t4, [\src, #16] 364 ldp \t5, \t6, [\src, #32] 365 ldp \t7, \t8, [\src, #48] 366 add \src, \src, #64 367 stnp \t1, \t2, [\dest] 368 stnp \t3, \t4, [\dest, #16] 369 stnp \t5, \t6, [\dest, #32] 370 stnp \t7, \t8, [\dest, #48] 371 add \dest, \dest, #64 372 tst \src, #(PAGE_SIZE - 1) 373 b.ne 9998b 374 .endm 375 376 /* 377 * Annotate a function as position independent, i.e., safe to be called before 378 * the kernel virtual mapping is activated. 379 */ 380 #define ENDPIPROC(x) \ 381 .globl __pi_##x; \ 382 .type __pi_##x, %function; \ 383 .set __pi_##x, x; \ 384 .size __pi_##x, . - x; \ 385 ENDPROC(x) 386 387 /* 388 * Emit a 64-bit absolute little endian symbol reference in a way that 389 * ensures that it will be resolved at build time, even when building a 390 * PIE binary. This requires cooperation from the linker script, which 391 * must emit the lo32/hi32 halves individually. 392 */ 393 .macro le64sym, sym 394 .long \sym\()_lo32 395 .long \sym\()_hi32 396 .endm 397 398 /* 399 * mov_q - move an immediate constant into a 64-bit register using 400 * between 2 and 4 movz/movk instructions (depending on the 401 * magnitude and sign of the operand) 402 */ 403 .macro mov_q, reg, val 404 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 405 movz \reg, :abs_g1_s:\val 406 .else 407 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 408 movz \reg, :abs_g2_s:\val 409 .else 410 movz \reg, :abs_g3:\val 411 movk \reg, :abs_g2_nc:\val 412 .endif 413 movk \reg, :abs_g1_nc:\val 414 .endif 415 movk \reg, :abs_g0_nc:\val 416 .endm 417 418 /* 419 * Return the current thread_info. 420 */ 421 .macro get_thread_info, rd 422 mrs \rd, sp_el0 423 .endm 424 425 /* 426 * Errata workaround post TTBR0_EL1 update. 427 */ 428 .macro post_ttbr0_update_workaround 429 #ifdef CONFIG_CAVIUM_ERRATUM_27456 430 alternative_if ARM64_WORKAROUND_CAVIUM_27456 431 ic iallu 432 dsb nsh 433 isb 434 alternative_else_nop_endif 435 #endif 436 .endm 437 438 #endif /* __ASM_ASSEMBLER_H */ 439