1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Based on arch/arm/mm/proc.S 4 * 5 * Copyright (C) 2001 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 * Author: Catalin Marinas <catalin.marinas@arm.com> 8 */ 9 10#include <linux/init.h> 11#include <linux/linkage.h> 12#include <linux/pgtable.h> 13#include <linux/cfi_types.h> 14#include <asm/assembler.h> 15#include <asm/asm-offsets.h> 16#include <asm/asm_pointer_auth.h> 17#include <asm/hwcap.h> 18#include <asm/kernel-pgtable.h> 19#include <asm/pgtable-hwdef.h> 20#include <asm/cpufeature.h> 21#include <asm/alternative.h> 22#include <asm/smp.h> 23#include <asm/sysreg.h> 24 25#ifdef CONFIG_ARM64_64K_PAGES 26#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K 27#elif defined(CONFIG_ARM64_16K_PAGES) 28#define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K 29#else /* CONFIG_ARM64_4K_PAGES */ 30#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K 31#endif 32 33#ifdef CONFIG_RANDOMIZE_BASE 34#define TCR_KASLR_FLAGS TCR_NFD1 35#else 36#define TCR_KASLR_FLAGS 0 37#endif 38 39#define TCR_SMP_FLAGS TCR_SHARED 40 41/* PTWs cacheable, inner/outer WBWA */ 42#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA 43 44#ifdef CONFIG_KASAN_SW_TAGS 45#define TCR_KASAN_SW_FLAGS TCR_TBI1 | TCR_TBID1 46#else 47#define TCR_KASAN_SW_FLAGS 0 48#endif 49 50#ifdef CONFIG_KASAN_HW_TAGS 51#define TCR_MTE_FLAGS TCR_TCMA1 | TCR_TBI1 | TCR_TBID1 52#elif defined(CONFIG_ARM64_MTE) 53/* 54 * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on 55 * TBI being enabled at EL1. 56 */ 57#define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1 58#else 59#define TCR_MTE_FLAGS 0 60#endif 61 62/* 63 * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and 64 * changed during mte_cpu_setup to Normal Tagged if the system supports MTE. 65 */ 66#define MAIR_EL1_SET \ 67 (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \ 68 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \ 69 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \ 70 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \ 71 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED)) 72 73#ifdef CONFIG_CPU_PM 74/** 75 * cpu_do_suspend - save CPU registers context 76 * 77 * x0: virtual address of context pointer 78 * 79 * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>. 80 */ 81SYM_FUNC_START(cpu_do_suspend) 82 mrs x2, tpidr_el0 83 mrs x3, tpidrro_el0 84 mrs x4, contextidr_el1 85 mrs x5, osdlr_el1 86 mrs x6, cpacr_el1 87 mrs x7, tcr_el1 88 mrs x8, vbar_el1 89 mrs x9, mdscr_el1 90 mrs x10, oslsr_el1 91 mrs x11, sctlr_el1 92 get_this_cpu_offset x12 93 mrs x13, sp_el0 94 stp x2, x3, [x0] 95 stp x4, x5, [x0, #16] 96 stp x6, x7, [x0, #32] 97 stp x8, x9, [x0, #48] 98 stp x10, x11, [x0, #64] 99 stp x12, x13, [x0, #80] 100 /* 101 * Save x18 as it may be used as a platform register, e.g. by shadow 102 * call stack. 103 */ 104 str x18, [x0, #96] 105 ret 106SYM_FUNC_END(cpu_do_suspend) 107 108/** 109 * cpu_do_resume - restore CPU register context 110 * 111 * x0: Address of context pointer 112 */ 113SYM_FUNC_START(cpu_do_resume) 114 ldp x2, x3, [x0] 115 ldp x4, x5, [x0, #16] 116 ldp x6, x8, [x0, #32] 117 ldp x9, x10, [x0, #48] 118 ldp x11, x12, [x0, #64] 119 ldp x13, x14, [x0, #80] 120 /* 121 * Restore x18, as it may be used as a platform register, and clear 122 * the buffer to minimize the risk of exposure when used for shadow 123 * call stack. 124 */ 125 ldr x18, [x0, #96] 126 str xzr, [x0, #96] 127 msr tpidr_el0, x2 128 msr tpidrro_el0, x3 129 msr contextidr_el1, x4 130 msr cpacr_el1, x6 131 132 /* Don't change t0sz here, mask those bits when restoring */ 133 mrs x7, tcr_el1 134 bfi x8, x7, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH 135 136 msr tcr_el1, x8 137 msr vbar_el1, x9 138 msr mdscr_el1, x10 139 140 msr sctlr_el1, x12 141 set_this_cpu_offset x13 142 msr sp_el0, x14 143 /* 144 * Restore oslsr_el1 by writing oslar_el1 145 */ 146 msr osdlr_el1, x5 147 ubfx x11, x11, #1, #1 148 msr oslar_el1, x11 149 reset_pmuserenr_el0 x0 // Disable PMU access from EL0 150 reset_amuserenr_el0 x0 // Disable AMU access from EL0 151 152alternative_if ARM64_HAS_RAS_EXTN 153 msr_s SYS_DISR_EL1, xzr 154alternative_else_nop_endif 155 156 ptrauth_keys_install_kernel_nosync x14, x1, x2, x3 157 isb 158 ret 159SYM_FUNC_END(cpu_do_resume) 160#endif 161 162 .pushsection ".idmap.text", "a" 163 164.macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2 165 adrp \tmp1, reserved_pg_dir 166 phys_to_ttbr \tmp2, \tmp1 167 offset_ttbr1 \tmp2, \tmp1 168 msr ttbr1_el1, \tmp2 169 isb 170 tlbi vmalle1 171 dsb nsh 172 isb 173.endm 174 175/* 176 * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1) 177 * 178 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be 179 * called by anything else. It can only be executed from a TTBR0 mapping. 180 */ 181SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1) 182 __idmap_cpu_set_reserved_ttbr1 x1, x3 183 184 offset_ttbr1 x0, x3 185 msr ttbr1_el1, x0 186 isb 187 188 ret 189SYM_FUNC_END(idmap_cpu_replace_ttbr1) 190SYM_FUNC_ALIAS(__pi_idmap_cpu_replace_ttbr1, idmap_cpu_replace_ttbr1) 191 .popsection 192 193#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 194 195#define KPTI_NG_PTE_FLAGS (PTE_ATTRINDX(MT_NORMAL) | PTE_TYPE_PAGE | \ 196 PTE_AF | PTE_SHARED | PTE_UXN | PTE_WRITE) 197 198 .pushsection ".idmap.text", "a" 199 200 .macro pte_to_phys, phys, pte 201 and \phys, \pte, #PTE_ADDR_LOW 202#ifdef CONFIG_ARM64_PA_BITS_52 203 and \pte, \pte, #PTE_ADDR_HIGH 204 orr \phys, \phys, \pte, lsl #PTE_ADDR_HIGH_SHIFT 205#endif 206 .endm 207 208 .macro kpti_mk_tbl_ng, type, num_entries 209 add end_\type\()p, cur_\type\()p, #\num_entries * 8 210.Ldo_\type: 211 ldr \type, [cur_\type\()p], #8 // Load the entry and advance 212 tbz \type, #0, .Lnext_\type // Skip invalid and 213 tbnz \type, #11, .Lnext_\type // non-global entries 214 orr \type, \type, #PTE_NG // Same bit for blocks and pages 215 str \type, [cur_\type\()p, #-8] // Update the entry 216 .ifnc \type, pte 217 tbnz \type, #1, .Lderef_\type 218 .endif 219.Lnext_\type: 220 cmp cur_\type\()p, end_\type\()p 221 b.ne .Ldo_\type 222 .endm 223 224 /* 225 * Dereference the current table entry and map it into the temporary 226 * fixmap slot associated with the current level. 227 */ 228 .macro kpti_map_pgtbl, type, level 229 str xzr, [temp_pte, #8 * (\level + 2)] // break before make 230 dsb nshst 231 add pte, temp_pte, #PAGE_SIZE * (\level + 2) 232 lsr pte, pte, #12 233 tlbi vaae1, pte 234 dsb nsh 235 isb 236 237 phys_to_pte pte, cur_\type\()p 238 add cur_\type\()p, temp_pte, #PAGE_SIZE * (\level + 2) 239 orr pte, pte, pte_flags 240 str pte, [temp_pte, #8 * (\level + 2)] 241 dsb nshst 242 .endm 243 244/* 245 * void __kpti_install_ng_mappings(int cpu, int num_secondaries, phys_addr_t temp_pgd, 246 * unsigned long temp_pte_va) 247 * 248 * Called exactly once from stop_machine context by each CPU found during boot. 249 */ 250 .pushsection ".data", "aw", %progbits 251SYM_DATA(__idmap_kpti_flag, .long 1) 252 .popsection 253 254SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) 255 cpu .req w0 256 temp_pte .req x0 257 num_cpus .req w1 258 pte_flags .req x1 259 temp_pgd_phys .req x2 260 swapper_ttb .req x3 261 flag_ptr .req x4 262 cur_pgdp .req x5 263 end_pgdp .req x6 264 pgd .req x7 265 cur_pudp .req x8 266 end_pudp .req x9 267 cur_pmdp .req x11 268 end_pmdp .req x12 269 cur_ptep .req x14 270 end_ptep .req x15 271 pte .req x16 272 valid .req x17 273 cur_p4dp .req x19 274 end_p4dp .req x20 275 276 mov x5, x3 // preserve temp_pte arg 277 mrs swapper_ttb, ttbr1_el1 278 adr_l flag_ptr, __idmap_kpti_flag 279 280 cbnz cpu, __idmap_kpti_secondary 281 282#if CONFIG_PGTABLE_LEVELS > 4 283 stp x29, x30, [sp, #-32]! 284 mov x29, sp 285 stp x19, x20, [sp, #16] 286#endif 287 288 /* We're the boot CPU. Wait for the others to catch up */ 289 sevl 2901: wfe 291 ldaxr w17, [flag_ptr] 292 eor w17, w17, num_cpus 293 cbnz w17, 1b 294 295 /* Switch to the temporary page tables on this CPU only */ 296 __idmap_cpu_set_reserved_ttbr1 x8, x9 297 offset_ttbr1 temp_pgd_phys, x8 298 msr ttbr1_el1, temp_pgd_phys 299 isb 300 301 mov temp_pte, x5 302 mov_q pte_flags, KPTI_NG_PTE_FLAGS 303 304 /* Everybody is enjoying the idmap, so we can rewrite swapper. */ 305 306#ifdef CONFIG_ARM64_LPA2 307 /* 308 * If LPA2 support is configured, but 52-bit virtual addressing is not 309 * enabled at runtime, we will fall back to one level of paging less, 310 * and so we have to walk swapper_pg_dir as if we dereferenced its 311 * address from a PGD level entry, and terminate the PGD level loop 312 * right after. 313 */ 314 adrp pgd, swapper_pg_dir // walk &swapper_pg_dir at the next level 315 mov cur_pgdp, end_pgdp // must be equal to terminate the PGD loop 316alternative_if_not ARM64_HAS_VA52 317 b .Lderef_pgd // skip to the next level 318alternative_else_nop_endif 319 /* 320 * LPA2 based 52-bit virtual addressing requires 52-bit physical 321 * addressing to be enabled as well. In this case, the shareability 322 * bits are repurposed as physical address bits, and should not be 323 * set in pte_flags. 324 */ 325 bic pte_flags, pte_flags, #PTE_SHARED 326#endif 327 328 /* PGD */ 329 adrp cur_pgdp, swapper_pg_dir 330 kpti_map_pgtbl pgd, -1 331 kpti_mk_tbl_ng pgd, PTRS_PER_PGD 332 333 /* Ensure all the updated entries are visible to secondary CPUs */ 334 dsb ishst 335 336 /* We're done: fire up swapper_pg_dir again */ 337 __idmap_cpu_set_reserved_ttbr1 x8, x9 338 msr ttbr1_el1, swapper_ttb 339 isb 340 341 /* Set the flag to zero to indicate that we're all done */ 342 str wzr, [flag_ptr] 343#if CONFIG_PGTABLE_LEVELS > 4 344 ldp x19, x20, [sp, #16] 345 ldp x29, x30, [sp], #32 346#endif 347 ret 348 349.Lderef_pgd: 350 /* P4D */ 351 .if CONFIG_PGTABLE_LEVELS > 4 352 p4d .req x30 353 pte_to_phys cur_p4dp, pgd 354 kpti_map_pgtbl p4d, 0 355 kpti_mk_tbl_ng p4d, PTRS_PER_P4D 356 b .Lnext_pgd 357 .else /* CONFIG_PGTABLE_LEVELS <= 4 */ 358 p4d .req pgd 359 .set .Lnext_p4d, .Lnext_pgd 360 .endif 361 362.Lderef_p4d: 363 /* PUD */ 364 .if CONFIG_PGTABLE_LEVELS > 3 365 pud .req x10 366 pte_to_phys cur_pudp, p4d 367 kpti_map_pgtbl pud, 1 368 kpti_mk_tbl_ng pud, PTRS_PER_PUD 369 b .Lnext_p4d 370 .else /* CONFIG_PGTABLE_LEVELS <= 3 */ 371 pud .req pgd 372 .set .Lnext_pud, .Lnext_pgd 373 .endif 374 375.Lderef_pud: 376 /* PMD */ 377 .if CONFIG_PGTABLE_LEVELS > 2 378 pmd .req x13 379 pte_to_phys cur_pmdp, pud 380 kpti_map_pgtbl pmd, 2 381 kpti_mk_tbl_ng pmd, PTRS_PER_PMD 382 b .Lnext_pud 383 .else /* CONFIG_PGTABLE_LEVELS <= 2 */ 384 pmd .req pgd 385 .set .Lnext_pmd, .Lnext_pgd 386 .endif 387 388.Lderef_pmd: 389 /* PTE */ 390 pte_to_phys cur_ptep, pmd 391 kpti_map_pgtbl pte, 3 392 kpti_mk_tbl_ng pte, PTRS_PER_PTE 393 b .Lnext_pmd 394 395 .unreq cpu 396 .unreq temp_pte 397 .unreq num_cpus 398 .unreq pte_flags 399 .unreq temp_pgd_phys 400 .unreq cur_pgdp 401 .unreq end_pgdp 402 .unreq pgd 403 .unreq cur_pudp 404 .unreq end_pudp 405 .unreq pud 406 .unreq cur_pmdp 407 .unreq end_pmdp 408 .unreq pmd 409 .unreq cur_ptep 410 .unreq end_ptep 411 .unreq pte 412 .unreq valid 413 .unreq cur_p4dp 414 .unreq end_p4dp 415 .unreq p4d 416 417 /* Secondary CPUs end up here */ 418__idmap_kpti_secondary: 419 /* Uninstall swapper before surgery begins */ 420 __idmap_cpu_set_reserved_ttbr1 x16, x17 421 422 /* Increment the flag to let the boot CPU we're ready */ 4231: ldxr w16, [flag_ptr] 424 add w16, w16, #1 425 stxr w17, w16, [flag_ptr] 426 cbnz w17, 1b 427 428 /* Wait for the boot CPU to finish messing around with swapper */ 429 sevl 4301: wfe 431 ldxr w16, [flag_ptr] 432 cbnz w16, 1b 433 434 /* All done, act like nothing happened */ 435 msr ttbr1_el1, swapper_ttb 436 isb 437 ret 438 439 .unreq swapper_ttb 440 .unreq flag_ptr 441SYM_FUNC_END(idmap_kpti_install_ng_mappings) 442 .popsection 443#endif 444 445/* 446 * __cpu_setup 447 * 448 * Initialise the processor for turning the MMU on. 449 * 450 * Output: 451 * Return in x0 the value of the SCTLR_EL1 register. 452 */ 453 .pushsection ".idmap.text", "a" 454SYM_FUNC_START(__cpu_setup) 455 tlbi vmalle1 // Invalidate local TLB 456 dsb nsh 457 458 msr cpacr_el1, xzr // Reset cpacr_el1 459 mov x1, #1 << 12 // Reset mdscr_el1 and disable 460 msr mdscr_el1, x1 // access to the DCC from EL0 461 reset_pmuserenr_el0 x1 // Disable PMU access from EL0 462 reset_amuserenr_el0 x1 // Disable AMU access from EL0 463 464 /* 465 * Default values for VMSA control registers. These will be adjusted 466 * below depending on detected CPU features. 467 */ 468 mair .req x17 469 tcr .req x16 470 mov_q mair, MAIR_EL1_SET 471 mov_q tcr, TCR_T0SZ(IDMAP_VA_BITS) | TCR_T1SZ(VA_BITS_MIN) | TCR_CACHE_FLAGS | \ 472 TCR_SMP_FLAGS | TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ 473 TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS 474 475 tcr_clear_errata_bits tcr, x9, x5 476 477#ifdef CONFIG_ARM64_VA_BITS_52 478 mov x9, #64 - VA_BITS 479alternative_if ARM64_HAS_VA52 480 tcr_set_t1sz tcr, x9 481#ifdef CONFIG_ARM64_LPA2 482 orr tcr, tcr, #TCR_DS 483#endif 484alternative_else_nop_endif 485#endif 486 487 /* 488 * Set the IPS bits in TCR_EL1. 489 */ 490 tcr_compute_pa_size tcr, #TCR_IPS_SHIFT, x5, x6 491#ifdef CONFIG_ARM64_HW_AFDBM 492 /* 493 * Enable hardware update of the Access Flags bit. 494 * Hardware dirty bit management is enabled later, 495 * via capabilities. 496 */ 497 mrs x9, ID_AA64MMFR1_EL1 498 and x9, x9, ID_AA64MMFR1_EL1_HAFDBS_MASK 499 cbz x9, 1f 500 orr tcr, tcr, #TCR_HA // hardware Access flag update 5011: 502#endif /* CONFIG_ARM64_HW_AFDBM */ 503 msr mair_el1, mair 504 msr tcr_el1, tcr 505 506 mrs_s x1, SYS_ID_AA64MMFR3_EL1 507 ubfx x1, x1, #ID_AA64MMFR3_EL1_S1PIE_SHIFT, #4 508 cbz x1, .Lskip_indirection 509 510 /* 511 * The PROT_* macros describing the various memory types may resolve to 512 * C expressions if they include the PTE_MAYBE_* macros, and so they 513 * can only be used from C code. The PIE_E* constants below are also 514 * defined in terms of those macros, but will mask out those 515 * PTE_MAYBE_* constants, whether they are set or not. So #define them 516 * as 0x0 here so we can evaluate the PIE_E* constants in asm context. 517 */ 518 519#define PTE_MAYBE_NG 0 520#define PTE_MAYBE_SHARED 0 521 522 mov_q x0, PIE_E0 523 msr REG_PIRE0_EL1, x0 524 mov_q x0, PIE_E1 525 msr REG_PIR_EL1, x0 526 527#undef PTE_MAYBE_NG 528#undef PTE_MAYBE_SHARED 529 530 mov x0, TCR2_EL1x_PIE 531 msr REG_TCR2_EL1, x0 532 533.Lskip_indirection: 534 535 /* 536 * Prepare SCTLR 537 */ 538 mov_q x0, INIT_SCTLR_EL1_MMU_ON 539 ret // return to head.S 540 541 .unreq mair 542 .unreq tcr 543SYM_FUNC_END(__cpu_setup) 544