1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Based on arch/arm/mm/proc.S 4 * 5 * Copyright (C) 2001 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 * Author: Catalin Marinas <catalin.marinas@arm.com> 8 */ 9 10#include <linux/init.h> 11#include <linux/linkage.h> 12#include <linux/pgtable.h> 13#include <linux/cfi_types.h> 14#include <asm/assembler.h> 15#include <asm/asm-offsets.h> 16#include <asm/asm_pointer_auth.h> 17#include <asm/hwcap.h> 18#include <asm/kernel-pgtable.h> 19#include <asm/pgtable-hwdef.h> 20#include <asm/cpufeature.h> 21#include <asm/alternative.h> 22#include <asm/smp.h> 23#include <asm/sysreg.h> 24 25#ifdef CONFIG_ARM64_64K_PAGES 26#define TCR_TG_FLAGS ((TCR_EL1_TG0_64K << TCR_EL1_TG0_SHIFT) |\ 27 (TCR_EL1_TG1_64K << TCR_EL1_TG1_SHIFT)) 28#elif defined(CONFIG_ARM64_16K_PAGES) 29#define TCR_TG_FLAGS ((TCR_EL1_TG0_16K << TCR_EL1_TG0_SHIFT) |\ 30 (TCR_EL1_TG1_16K << TCR_EL1_TG1_SHIFT)) 31#else /* CONFIG_ARM64_4K_PAGES */ 32#define TCR_TG_FLAGS ((TCR_EL1_TG0_4K << TCR_EL1_TG0_SHIFT) |\ 33 (TCR_EL1_TG1_4K << TCR_EL1_TG1_SHIFT)) 34#endif 35 36#ifdef CONFIG_RANDOMIZE_BASE 37#define TCR_KASLR_FLAGS TCR_EL1_NFD1 38#else 39#define TCR_KASLR_FLAGS 0 40#endif 41 42/* PTWs cacheable, inner/outer WBWA */ 43#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA 44 45#ifdef CONFIG_KASAN_SW_TAGS 46#define TCR_KASAN_SW_FLAGS TCR_EL1_TBI1 | TCR_EL1_TBID1 47#else 48#define TCR_KASAN_SW_FLAGS 0 49#endif 50 51#ifdef CONFIG_KASAN_HW_TAGS 52#define TCR_MTE_FLAGS TCR_EL1_TCMA1 | TCR_EL1_TBI1 | TCR_EL1_TBID1 53#elif defined(CONFIG_ARM64_MTE) 54/* 55 * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on 56 * TBI being enabled at EL1. 57 */ 58#define TCR_MTE_FLAGS TCR_EL1_TBI1 | TCR_EL1_TBID1 59#else 60#define TCR_MTE_FLAGS 0 61#endif 62 63#define TCR_IRGN_WBWA ((TCR_EL1_IRGN0_WBWA << TCR_EL1_IRGN0_SHIFT) |\ 64 (TCR_EL1_IRGN1_WBWA << TCR_EL1_IRGN1_SHIFT)) 65#define TCR_ORGN_WBWA ((TCR_EL1_ORGN0_WBWA << TCR_EL1_ORGN0_SHIFT) |\ 66 (TCR_EL1_ORGN1_WBWA << TCR_EL1_ORGN1_SHIFT)) 67#define TCR_SHARED ((TCR_EL1_SH0_INNER << TCR_EL1_SH0_SHIFT) |\ 68 (TCR_EL1_SH1_INNER << TCR_EL1_SH1_SHIFT)) 69 70/* 71 * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and 72 * changed during mte_cpu_setup to Normal Tagged if the system supports MTE. 73 */ 74#define MAIR_EL1_SET \ 75 (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \ 76 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \ 77 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \ 78 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \ 79 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED)) 80 81#ifdef CONFIG_CPU_PM 82/** 83 * cpu_do_suspend - save CPU registers context 84 * 85 * x0: virtual address of context pointer 86 * 87 * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>. 88 */ 89SYM_FUNC_START(cpu_do_suspend) 90 mrs x2, tpidr_el0 91 mrs x3, tpidrro_el0 92 mrs x4, contextidr_el1 93 mrs x5, osdlr_el1 94 mrs x6, cpacr_el1 95 mrs x7, tcr_el1 96 mrs x8, vbar_el1 97 mrs x9, mdscr_el1 98 mrs x10, oslsr_el1 99 mrs x11, sctlr_el1 100 get_this_cpu_offset x12 101 mrs x13, sp_el0 102 stp x2, x3, [x0] 103 stp x4, x5, [x0, #16] 104 stp x6, x7, [x0, #32] 105 stp x8, x9, [x0, #48] 106 stp x10, x11, [x0, #64] 107 stp x12, x13, [x0, #80] 108 /* 109 * Save x18 as it may be used as a platform register, e.g. by shadow 110 * call stack. 111 */ 112 str x18, [x0, #96] 113 ret 114SYM_FUNC_END(cpu_do_suspend) 115 116/** 117 * cpu_do_resume - restore CPU register context 118 * 119 * x0: Address of context pointer 120 */ 121SYM_FUNC_START(cpu_do_resume) 122 ldp x2, x3, [x0] 123 ldp x4, x5, [x0, #16] 124 ldp x6, x8, [x0, #32] 125 ldp x9, x10, [x0, #48] 126 ldp x11, x12, [x0, #64] 127 ldp x13, x14, [x0, #80] 128 /* 129 * Restore x18, as it may be used as a platform register, and clear 130 * the buffer to minimize the risk of exposure when used for shadow 131 * call stack. 132 */ 133 ldr x18, [x0, #96] 134 str xzr, [x0, #96] 135 msr tpidr_el0, x2 136 msr tpidrro_el0, x3 137 msr contextidr_el1, x4 138 msr cpacr_el1, x6 139 140 /* Don't change t0sz here, mask those bits when restoring */ 141 mrs x7, tcr_el1 142 bfi x8, x7, TCR_EL1_T0SZ_SHIFT, TCR_EL1_T0SZ_WIDTH 143 144 msr tcr_el1, x8 145 msr vbar_el1, x9 146 msr mdscr_el1, x10 147 148 msr sctlr_el1, x12 149 set_this_cpu_offset x13 150 msr sp_el0, x14 151 /* 152 * Restore oslsr_el1 by writing oslar_el1 153 */ 154 msr osdlr_el1, x5 155 ubfx x11, x11, #1, #1 156 msr oslar_el1, x11 157 reset_pmuserenr_el0 x0 // Disable PMU access from EL0 158 reset_amuserenr_el0 x0 // Disable AMU access from EL0 159 160alternative_if ARM64_HAS_RAS_EXTN 161 msr_s SYS_DISR_EL1, xzr 162alternative_else_nop_endif 163 164 ptrauth_keys_install_kernel_nosync x14, x1, x2, x3 165 isb 166 ret 167SYM_FUNC_END(cpu_do_resume) 168#endif 169 170 .pushsection ".idmap.text", "a" 171 172.macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2 173 adrp \tmp1, reserved_pg_dir 174 phys_to_ttbr \tmp2, \tmp1 175 offset_ttbr1 \tmp2, \tmp1 176 msr ttbr1_el1, \tmp2 177 isb 178 tlbi vmalle1 179 dsb nsh 180 isb 181.endm 182 183/* 184 * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1) 185 * 186 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be 187 * called by anything else. It can only be executed from a TTBR0 mapping. 188 */ 189SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1) 190 __idmap_cpu_set_reserved_ttbr1 x1, x3 191 192 offset_ttbr1 x0, x3 193 msr ttbr1_el1, x0 194 isb 195 196 ret 197SYM_FUNC_END(idmap_cpu_replace_ttbr1) 198SYM_FUNC_ALIAS(__pi_idmap_cpu_replace_ttbr1, idmap_cpu_replace_ttbr1) 199 .popsection 200 201#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 202 203#define KPTI_NG_PTE_FLAGS (PTE_ATTRINDX(MT_NORMAL) | PTE_TYPE_PAGE | \ 204 PTE_AF | PTE_SHARED | PTE_UXN | PTE_WRITE) 205 206 .pushsection ".idmap.text", "a" 207 208 .macro pte_to_phys, phys, pte 209 and \phys, \pte, #PTE_ADDR_LOW 210#ifdef CONFIG_ARM64_PA_BITS_52 211 and \pte, \pte, #PTE_ADDR_HIGH 212 orr \phys, \phys, \pte, lsl #PTE_ADDR_HIGH_SHIFT 213#endif 214 .endm 215 216 .macro kpti_mk_tbl_ng, type, num_entries 217 add end_\type\()p, cur_\type\()p, #\num_entries * 8 218.Ldo_\type: 219 ldr \type, [cur_\type\()p], #8 // Load the entry and advance 220 tbz \type, #0, .Lnext_\type // Skip invalid and 221 tbnz \type, #11, .Lnext_\type // non-global entries 222 orr \type, \type, #PTE_NG // Same bit for blocks and pages 223 str \type, [cur_\type\()p, #-8] // Update the entry 224 .ifnc \type, pte 225 tbnz \type, #1, .Lderef_\type 226 .endif 227.Lnext_\type: 228 cmp cur_\type\()p, end_\type\()p 229 b.ne .Ldo_\type 230 .endm 231 232 /* 233 * Dereference the current table entry and map it into the temporary 234 * fixmap slot associated with the current level. 235 */ 236 .macro kpti_map_pgtbl, type, level 237 str xzr, [temp_pte, #8 * (\level + 2)] // break before make 238 dsb nshst 239 add pte, temp_pte, #PAGE_SIZE * (\level + 2) 240 lsr pte, pte, #12 241 tlbi vaae1, pte 242 dsb nsh 243 isb 244 245 phys_to_pte pte, cur_\type\()p 246 add cur_\type\()p, temp_pte, #PAGE_SIZE * (\level + 2) 247 orr pte, pte, pte_flags 248 str pte, [temp_pte, #8 * (\level + 2)] 249 dsb nshst 250 .endm 251 252/* 253 * void __kpti_install_ng_mappings(int cpu, int num_secondaries, phys_addr_t temp_pgd, 254 * unsigned long temp_pte_va) 255 * 256 * Called exactly once from stop_machine context by each CPU found during boot. 257 */ 258SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) 259 cpu .req w0 260 temp_pte .req x0 261 num_cpus .req w1 262 pte_flags .req x1 263 temp_pgd_phys .req x2 264 swapper_ttb .req x3 265 flag_ptr .req x4 266 cur_pgdp .req x5 267 end_pgdp .req x6 268 pgd .req x7 269 cur_pudp .req x8 270 end_pudp .req x9 271 cur_pmdp .req x11 272 end_pmdp .req x12 273 cur_ptep .req x14 274 end_ptep .req x15 275 pte .req x16 276 valid .req x17 277 cur_p4dp .req x19 278 end_p4dp .req x20 279 280 mov x5, x3 // preserve temp_pte arg 281 mrs swapper_ttb, ttbr1_el1 282 adr_l flag_ptr, idmap_kpti_bbml2_flag 283 284 cbnz cpu, __idmap_kpti_secondary 285 286#if CONFIG_PGTABLE_LEVELS > 4 287 stp x29, x30, [sp, #-32]! 288 mov x29, sp 289 stp x19, x20, [sp, #16] 290#endif 291 292 /* We're the boot CPU. Wait for the others to catch up */ 293 sevl 2941: wfe 295 ldaxr w17, [flag_ptr] 296 eor w17, w17, num_cpus 297 cbnz w17, 1b 298 299 /* Switch to the temporary page tables on this CPU only */ 300 __idmap_cpu_set_reserved_ttbr1 x8, x9 301 offset_ttbr1 temp_pgd_phys, x8 302 msr ttbr1_el1, temp_pgd_phys 303 isb 304 305 mov temp_pte, x5 306 mov_q pte_flags, KPTI_NG_PTE_FLAGS 307 308 /* Everybody is enjoying the idmap, so we can rewrite swapper. */ 309 310#ifdef CONFIG_ARM64_LPA2 311 /* 312 * If LPA2 support is configured, but 52-bit virtual addressing is not 313 * enabled at runtime, we will fall back to one level of paging less, 314 * and so we have to walk swapper_pg_dir as if we dereferenced its 315 * address from a PGD level entry, and terminate the PGD level loop 316 * right after. 317 */ 318 adrp pgd, swapper_pg_dir // walk &swapper_pg_dir at the next level 319 mov cur_pgdp, end_pgdp // must be equal to terminate the PGD loop 320alternative_if_not ARM64_HAS_VA52 321 b .Lderef_pgd // skip to the next level 322alternative_else_nop_endif 323 /* 324 * LPA2 based 52-bit virtual addressing requires 52-bit physical 325 * addressing to be enabled as well. In this case, the shareability 326 * bits are repurposed as physical address bits, and should not be 327 * set in pte_flags. 328 */ 329 bic pte_flags, pte_flags, #PTE_SHARED 330#endif 331 332 /* PGD */ 333 adrp cur_pgdp, swapper_pg_dir 334 kpti_map_pgtbl pgd, -1 335 kpti_mk_tbl_ng pgd, PTRS_PER_PGD 336 337 /* Ensure all the updated entries are visible to secondary CPUs */ 338 dsb ishst 339 340 /* We're done: fire up swapper_pg_dir again */ 341 __idmap_cpu_set_reserved_ttbr1 x8, x9 342 msr ttbr1_el1, swapper_ttb 343 isb 344 345 /* Set the flag to zero to indicate that we're all done */ 346 str wzr, [flag_ptr] 347#if CONFIG_PGTABLE_LEVELS > 4 348 ldp x19, x20, [sp, #16] 349 ldp x29, x30, [sp], #32 350#endif 351 ret 352 353.Lderef_pgd: 354 /* P4D */ 355 .if CONFIG_PGTABLE_LEVELS > 4 356 p4d .req x30 357 pte_to_phys cur_p4dp, pgd 358 kpti_map_pgtbl p4d, 0 359 kpti_mk_tbl_ng p4d, PTRS_PER_P4D 360 b .Lnext_pgd 361 .else /* CONFIG_PGTABLE_LEVELS <= 4 */ 362 p4d .req pgd 363 .set .Lnext_p4d, .Lnext_pgd 364 .endif 365 366.Lderef_p4d: 367 /* PUD */ 368 .if CONFIG_PGTABLE_LEVELS > 3 369 pud .req x10 370 pte_to_phys cur_pudp, p4d 371 kpti_map_pgtbl pud, 1 372 kpti_mk_tbl_ng pud, PTRS_PER_PUD 373 b .Lnext_p4d 374 .else /* CONFIG_PGTABLE_LEVELS <= 3 */ 375 pud .req pgd 376 .set .Lnext_pud, .Lnext_pgd 377 .endif 378 379.Lderef_pud: 380 /* PMD */ 381 .if CONFIG_PGTABLE_LEVELS > 2 382 pmd .req x13 383 pte_to_phys cur_pmdp, pud 384 kpti_map_pgtbl pmd, 2 385 kpti_mk_tbl_ng pmd, PTRS_PER_PMD 386 b .Lnext_pud 387 .else /* CONFIG_PGTABLE_LEVELS <= 2 */ 388 pmd .req pgd 389 .set .Lnext_pmd, .Lnext_pgd 390 .endif 391 392.Lderef_pmd: 393 /* PTE */ 394 pte_to_phys cur_ptep, pmd 395 kpti_map_pgtbl pte, 3 396 kpti_mk_tbl_ng pte, PTRS_PER_PTE 397 b .Lnext_pmd 398 399 .unreq cpu 400 .unreq temp_pte 401 .unreq num_cpus 402 .unreq pte_flags 403 .unreq temp_pgd_phys 404 .unreq cur_pgdp 405 .unreq end_pgdp 406 .unreq pgd 407 .unreq cur_pudp 408 .unreq end_pudp 409 .unreq pud 410 .unreq cur_pmdp 411 .unreq end_pmdp 412 .unreq pmd 413 .unreq cur_ptep 414 .unreq end_ptep 415 .unreq pte 416 .unreq valid 417 .unreq cur_p4dp 418 .unreq end_p4dp 419 .unreq p4d 420 421 /* Secondary CPUs end up here */ 422__idmap_kpti_secondary: 423 /* Uninstall swapper before surgery begins */ 424 __idmap_cpu_set_reserved_ttbr1 x16, x17 425 b scondary_cpu_wait 426 427 .unreq swapper_ttb 428 .unreq flag_ptr 429SYM_FUNC_END(idmap_kpti_install_ng_mappings) 430 .popsection 431#endif 432 433 .pushsection ".idmap.text", "a" 434SYM_TYPED_FUNC_START(wait_linear_map_split_to_ptes) 435 /* Must be same registers as in idmap_kpti_install_ng_mappings */ 436 swapper_ttb .req x3 437 flag_ptr .req x4 438 439 mrs swapper_ttb, ttbr1_el1 440 adr_l flag_ptr, idmap_kpti_bbml2_flag 441 __idmap_cpu_set_reserved_ttbr1 x16, x17 442 443scondary_cpu_wait: 444 /* Increment the flag to let the boot CPU we're ready */ 4451: ldxr w16, [flag_ptr] 446 add w16, w16, #1 447 stxr w17, w16, [flag_ptr] 448 cbnz w17, 1b 449 450 /* Wait for the boot CPU to finish messing around with swapper */ 451 sevl 4521: wfe 453 ldxr w16, [flag_ptr] 454 cbnz w16, 1b 455 456 /* All done, act like nothing happened */ 457 msr ttbr1_el1, swapper_ttb 458 isb 459 ret 460 461 .unreq swapper_ttb 462 .unreq flag_ptr 463SYM_FUNC_END(wait_linear_map_split_to_ptes) 464 .popsection 465 466/* 467 * __cpu_setup 468 * 469 * Initialise the processor for turning the MMU on. 470 * 471 * Output: 472 * Return in x0 the value of the SCTLR_EL1 register. 473 */ 474 .pushsection ".idmap.text", "a" 475SYM_FUNC_START(__cpu_setup) 476 tlbi vmalle1 // Invalidate local TLB 477 dsb nsh 478 479 msr cpacr_el1, xzr // Reset cpacr_el1 480 mov x1, MDSCR_EL1_TDCC // Reset mdscr_el1 and disable 481 msr mdscr_el1, x1 // access to the DCC from EL0 482 reset_pmuserenr_el0 x1 // Disable PMU access from EL0 483 reset_amuserenr_el0 x1 // Disable AMU access from EL0 484 485 /* 486 * Default values for VMSA control registers. These will be adjusted 487 * below depending on detected CPU features. 488 */ 489 mair .req x17 490 tcr .req x16 491 tcr2 .req x15 492 mov_q mair, MAIR_EL1_SET 493 mov_q tcr, TCR_T0SZ(IDMAP_VA_BITS) | TCR_T1SZ(VA_BITS_MIN) | TCR_CACHE_FLAGS | \ 494 TCR_SHARED | TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_EL1_AS | \ 495 TCR_EL1_TBI0 | TCR_EL1_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS 496 mov tcr2, xzr 497 498 tcr_clear_errata_bits tcr, x9, x5 499 500#ifdef CONFIG_ARM64_VA_BITS_52 501 mov x9, #64 - VA_BITS 502alternative_if ARM64_HAS_VA52 503 tcr_set_t1sz tcr, x9 504#ifdef CONFIG_ARM64_LPA2 505 orr tcr, tcr, #TCR_EL1_DS 506#endif 507alternative_else_nop_endif 508#endif 509 510 /* 511 * Set the IPS bits in TCR_EL1. 512 */ 513 tcr_compute_pa_size tcr, #TCR_EL1_IPS_SHIFT, x5, x6 514#ifdef CONFIG_ARM64_HW_AFDBM 515 /* 516 * Enable hardware update of the Access Flags bit. 517 * Hardware dirty bit management is enabled later, 518 * via capabilities. 519 */ 520 mrs x9, ID_AA64MMFR1_EL1 521 ubfx x9, x9, ID_AA64MMFR1_EL1_HAFDBS_SHIFT, #4 522 cbz x9, 1f 523 orr tcr, tcr, #TCR_EL1_HA // hardware Access flag update 524#ifdef CONFIG_ARM64_HAFT 525 cmp x9, ID_AA64MMFR1_EL1_HAFDBS_HAFT 526 b.lt 1f 527 orr tcr2, tcr2, TCR2_EL1_HAFT 528#endif /* CONFIG_ARM64_HAFT */ 5291: 530#endif /* CONFIG_ARM64_HW_AFDBM */ 531 msr mair_el1, mair 532 msr tcr_el1, tcr 533 534 mrs_s x1, SYS_ID_AA64MMFR3_EL1 535 ubfx x1, x1, #ID_AA64MMFR3_EL1_S1PIE_SHIFT, #4 536 cbz x1, .Lskip_indirection 537 538 mov_q x0, PIE_E0_ASM 539 msr REG_PIRE0_EL1, x0 540 mov_q x0, PIE_E1_ASM 541 msr REG_PIR_EL1, x0 542 543 orr tcr2, tcr2, TCR2_EL1_PIE 544 545.Lskip_indirection: 546 547 mrs_s x1, SYS_ID_AA64MMFR3_EL1 548 ubfx x1, x1, #ID_AA64MMFR3_EL1_TCRX_SHIFT, #4 549 cbz x1, 1f 550 msr REG_TCR2_EL1, tcr2 5511: 552 553 /* 554 * Prepare SCTLR 555 */ 556 mov_q x0, INIT_SCTLR_EL1_MMU_ON 557 ret // return to head.S 558 559 .unreq mair 560 .unreq tcr 561 .unreq tcr2 562SYM_FUNC_END(__cpu_setup) 563