1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Based on arch/arm/mm/proc.S 4 * 5 * Copyright (C) 2001 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 * Author: Catalin Marinas <catalin.marinas@arm.com> 8 */ 9 10#include <linux/init.h> 11#include <linux/linkage.h> 12#include <linux/pgtable.h> 13#include <linux/cfi_types.h> 14#include <asm/assembler.h> 15#include <asm/asm-offsets.h> 16#include <asm/asm_pointer_auth.h> 17#include <asm/hwcap.h> 18#include <asm/kernel-pgtable.h> 19#include <asm/pgtable-hwdef.h> 20#include <asm/cpufeature.h> 21#include <asm/alternative.h> 22#include <asm/smp.h> 23#include <asm/sysreg.h> 24 25#ifdef CONFIG_ARM64_64K_PAGES 26#define TCR_TG_FLAGS ((TCR_EL1_TG0_64K << TCR_EL1_TG0_SHIFT) |\ 27 (TCR_EL1_TG1_64K << TCR_EL1_TG1_SHIFT)) 28#elif defined(CONFIG_ARM64_16K_PAGES) 29#define TCR_TG_FLAGS ((TCR_EL1_TG0_16K << TCR_EL1_TG0_SHIFT) |\ 30 (TCR_EL1_TG1_16K << TCR_EL1_TG1_SHIFT)) 31#else /* CONFIG_ARM64_4K_PAGES */ 32#define TCR_TG_FLAGS ((TCR_EL1_TG0_4K << TCR_EL1_TG0_SHIFT) |\ 33 (TCR_EL1_TG1_4K << TCR_EL1_TG1_SHIFT)) 34#endif 35 36#ifdef CONFIG_RANDOMIZE_BASE 37#define TCR_KASLR_FLAGS TCR_EL1_NFD1 38#else 39#define TCR_KASLR_FLAGS 0 40#endif 41 42/* PTWs cacheable, inner/outer WBWA */ 43#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA 44 45#ifdef CONFIG_KASAN_SW_TAGS 46#define TCR_KASAN_SW_FLAGS TCR_EL1_TBI1 | TCR_EL1_TBID1 47#else 48#define TCR_KASAN_SW_FLAGS 0 49#endif 50 51#ifdef CONFIG_ARM64_MTE 52/* 53 * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on 54 * TBI being enabled at EL1. TCMA1 is needed to treat accesses with the 55 * match-all tag (0xF) as Tag Unchecked, irrespective of the SCTLR_EL1.TCF 56 * setting. 57 */ 58#define TCR_MTE_FLAGS TCR_EL1_TCMA1 | TCR_EL1_TBI1 | TCR_EL1_TBID1 59#else 60#define TCR_MTE_FLAGS 0 61#endif 62 63#define TCR_IRGN_WBWA ((TCR_EL1_IRGN0_WBWA << TCR_EL1_IRGN0_SHIFT) |\ 64 (TCR_EL1_IRGN1_WBWA << TCR_EL1_IRGN1_SHIFT)) 65#define TCR_ORGN_WBWA ((TCR_EL1_ORGN0_WBWA << TCR_EL1_ORGN0_SHIFT) |\ 66 (TCR_EL1_ORGN1_WBWA << TCR_EL1_ORGN1_SHIFT)) 67#define TCR_SHARED ((TCR_EL1_SH0_INNER << TCR_EL1_SH0_SHIFT) |\ 68 (TCR_EL1_SH1_INNER << TCR_EL1_SH1_SHIFT)) 69 70/* 71 * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and 72 * changed during mte_cpu_setup to Normal Tagged if the system supports MTE. 73 */ 74#define MAIR_EL1_SET \ 75 (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \ 76 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \ 77 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \ 78 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \ 79 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED)) 80 81#ifdef CONFIG_CPU_PM 82/** 83 * cpu_do_suspend - save CPU registers context 84 * 85 * x0: virtual address of context pointer 86 * 87 * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>. 88 */ 89SYM_FUNC_START(cpu_do_suspend) 90 mrs x2, tpidr_el0 91 mrs x3, tpidrro_el0 92 mrs x4, contextidr_el1 93 mrs x5, osdlr_el1 94 mrs x6, cpacr_el1 95 mrs x7, tcr_el1 96 mrs x8, vbar_el1 97 mrs x9, mdscr_el1 98 mrs x10, oslsr_el1 99 mrs x11, sctlr_el1 100 get_this_cpu_offset x12 101 mrs x13, sp_el0 102 stp x2, x3, [x0] 103 stp x4, x5, [x0, #16] 104 stp x6, x7, [x0, #32] 105 stp x8, x9, [x0, #48] 106 stp x10, x11, [x0, #64] 107 stp x12, x13, [x0, #80] 108 /* 109 * Save x18 as it may be used as a platform register, e.g. by shadow 110 * call stack. 111 */ 112 str x18, [x0, #96] 113alternative_if ARM64_HAS_TCR2 114 mrs x2, REG_TCR2_EL1 115 str x2, [x0, #104] 116alternative_else_nop_endif 117 ret 118SYM_FUNC_END(cpu_do_suspend) 119 120/** 121 * cpu_do_resume - restore CPU register context 122 * 123 * x0: Address of context pointer 124 */ 125SYM_FUNC_START(cpu_do_resume) 126 ldp x2, x3, [x0] 127 ldp x4, x5, [x0, #16] 128 ldp x6, x8, [x0, #32] 129 ldp x9, x10, [x0, #48] 130 ldp x11, x12, [x0, #64] 131 ldp x13, x14, [x0, #80] 132 /* 133 * Restore x18, as it may be used as a platform register, and clear 134 * the buffer to minimize the risk of exposure when used for shadow 135 * call stack. 136 */ 137 ldr x18, [x0, #96] 138 str xzr, [x0, #96] 139 msr tpidr_el0, x2 140 msr tpidrro_el0, x3 141 msr contextidr_el1, x4 142 msr cpacr_el1, x6 143 144 /* Don't change t0sz here, mask those bits when restoring */ 145 mrs x7, tcr_el1 146 bfi x8, x7, TCR_EL1_T0SZ_SHIFT, TCR_EL1_T0SZ_WIDTH 147 148 msr tcr_el1, x8 149 msr vbar_el1, x9 150 msr mdscr_el1, x10 151alternative_if ARM64_HAS_TCR2 152 ldr x2, [x0, #104] 153 msr REG_TCR2_EL1, x2 154alternative_else_nop_endif 155 156 msr sctlr_el1, x12 157 set_this_cpu_offset x13 158 msr sp_el0, x14 159 /* 160 * Restore oslsr_el1 by writing oslar_el1 161 */ 162 msr osdlr_el1, x5 163 ubfx x11, x11, #1, #1 164 msr oslar_el1, x11 165 reset_pmuserenr_el0 x0 // Disable PMU access from EL0 166 reset_amuserenr_el0 x0 // Disable AMU access from EL0 167 168alternative_if ARM64_HAS_RAS_EXTN 169 msr_s SYS_DISR_EL1, xzr 170alternative_else_nop_endif 171 172#ifdef CONFIG_ARM64_PSEUDO_NMI 173alternative_if ARM64_HAS_GIC_PRIO_MASKING 174 mov x1, #GIC_PRIO_IRQON 175 msr_s SYS_ICC_PMR_EL1, x1 176alternative_else_nop_endif 177#endif 178 179 ptrauth_keys_install_kernel_nosync x14, x1, x2, x3 180 isb 181 ret 182SYM_FUNC_END(cpu_do_resume) 183#endif 184 185 .pushsection ".idmap.text", "a" 186 187.macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2 188 adrp \tmp1, reserved_pg_dir 189 phys_to_ttbr \tmp2, \tmp1 190 offset_ttbr1 \tmp2, \tmp1 191 msr ttbr1_el1, \tmp2 192 isb 193 tlbi vmalle1 194 dsb nsh 195 isb 196.endm 197 198/* 199 * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1) 200 * 201 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be 202 * called by anything else. It can only be executed from a TTBR0 mapping. 203 */ 204SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1) 205 __idmap_cpu_set_reserved_ttbr1 x1, x3 206 207 offset_ttbr1 x0, x3 208 msr ttbr1_el1, x0 209 isb 210 211 ret 212SYM_FUNC_END(idmap_cpu_replace_ttbr1) 213SYM_FUNC_ALIAS(__pi_idmap_cpu_replace_ttbr1, idmap_cpu_replace_ttbr1) 214 .popsection 215 216#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 217 218#define KPTI_NG_PTE_FLAGS (PTE_ATTRINDX(MT_NORMAL) | PTE_TYPE_PAGE | \ 219 PTE_AF | PTE_SHARED | PTE_UXN | PTE_WRITE) 220 221 .pushsection ".idmap.text", "a" 222 223 .macro pte_to_phys, phys, pte 224 and \phys, \pte, #PTE_ADDR_LOW 225#ifdef CONFIG_ARM64_PA_BITS_52 226 and \pte, \pte, #PTE_ADDR_HIGH 227 orr \phys, \phys, \pte, lsl #PTE_ADDR_HIGH_SHIFT 228#endif 229 .endm 230 231 .macro kpti_mk_tbl_ng, type, num_entries 232 add end_\type\()p, cur_\type\()p, #\num_entries * 8 233.Ldo_\type: 234 ldr \type, [cur_\type\()p], #8 // Load the entry and advance 235 tbz \type, #0, .Lnext_\type // Skip invalid and 236 tbnz \type, #11, .Lnext_\type // non-global entries 237 orr \type, \type, #PTE_NG // Same bit for blocks and pages 238 str \type, [cur_\type\()p, #-8] // Update the entry 239 .ifnc \type, pte 240 tbnz \type, #1, .Lderef_\type 241 .endif 242.Lnext_\type: 243 cmp cur_\type\()p, end_\type\()p 244 b.ne .Ldo_\type 245 .endm 246 247 /* 248 * Dereference the current table entry and map it into the temporary 249 * fixmap slot associated with the current level. 250 */ 251 .macro kpti_map_pgtbl, type, level 252 str xzr, [temp_pte, #8 * (\level + 2)] // break before make 253 dsb nshst 254 add pte, temp_pte, #PAGE_SIZE * (\level + 2) 255 lsr pte, pte, #12 256 tlbi vaae1, pte 257 dsb nsh 258 isb 259 260 phys_to_pte pte, cur_\type\()p 261 add cur_\type\()p, temp_pte, #PAGE_SIZE * (\level + 2) 262 orr pte, pte, pte_flags 263 str pte, [temp_pte, #8 * (\level + 2)] 264 dsb nshst 265 .endm 266 267/* 268 * void __kpti_install_ng_mappings(int cpu, int num_secondaries, phys_addr_t temp_pgd, 269 * unsigned long temp_pte_va) 270 * 271 * Called exactly once from stop_machine context by each CPU found during boot. 272 */ 273SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) 274 cpu .req w0 275 temp_pte .req x0 276 num_cpus .req w1 277 pte_flags .req x1 278 temp_pgd_phys .req x2 279 swapper_ttb .req x3 280 flag_ptr .req x4 281 cur_pgdp .req x5 282 end_pgdp .req x6 283 pgd .req x7 284 cur_pudp .req x8 285 end_pudp .req x9 286 cur_pmdp .req x11 287 end_pmdp .req x12 288 cur_ptep .req x14 289 end_ptep .req x15 290 pte .req x16 291 valid .req x17 292 cur_p4dp .req x19 293 end_p4dp .req x20 294 295 mov x5, x3 // preserve temp_pte arg 296 mrs swapper_ttb, ttbr1_el1 297 adr_l flag_ptr, idmap_kpti_bbml3_flag 298 299 cbnz cpu, __idmap_kpti_secondary 300 301#if CONFIG_PGTABLE_LEVELS > 4 302 stp x29, x30, [sp, #-32]! 303 mov x29, sp 304 stp x19, x20, [sp, #16] 305#endif 306 307 /* We're the boot CPU. Wait for the others to catch up */ 308 sevl 3091: wfe 310 ldaxr w17, [flag_ptr] 311 eor w17, w17, num_cpus 312 cbnz w17, 1b 313 314 /* Switch to the temporary page tables on this CPU only */ 315 __idmap_cpu_set_reserved_ttbr1 x8, x9 316 offset_ttbr1 temp_pgd_phys, x8 317 msr ttbr1_el1, temp_pgd_phys 318 isb 319 320 mov temp_pte, x5 321 mov_q pte_flags, KPTI_NG_PTE_FLAGS 322 323 /* Everybody is enjoying the idmap, so we can rewrite swapper. */ 324 325#ifdef CONFIG_ARM64_LPA2 326 /* 327 * If LPA2 support is configured, but 52-bit virtual addressing is not 328 * enabled at runtime, we will fall back to one level of paging less, 329 * and so we have to walk swapper_pg_dir as if we dereferenced its 330 * address from a PGD level entry, and terminate the PGD level loop 331 * right after. 332 */ 333 adrp pgd, swapper_pg_dir // walk &swapper_pg_dir at the next level 334 mov cur_pgdp, end_pgdp // must be equal to terminate the PGD loop 335alternative_if_not ARM64_HAS_VA52 336 b .Lderef_pgd // skip to the next level 337alternative_else_nop_endif 338 /* 339 * LPA2 based 52-bit virtual addressing requires 52-bit physical 340 * addressing to be enabled as well. In this case, the shareability 341 * bits are repurposed as physical address bits, and should not be 342 * set in pte_flags. 343 */ 344 bic pte_flags, pte_flags, #PTE_SHARED 345#endif 346 347 /* PGD */ 348 adrp cur_pgdp, swapper_pg_dir 349 kpti_map_pgtbl pgd, -1 350 kpti_mk_tbl_ng pgd, PTRS_PER_PGD 351 352 /* Ensure all the updated entries are visible to secondary CPUs */ 353 dsb ishst 354 355 /* We're done: fire up swapper_pg_dir again */ 356 __idmap_cpu_set_reserved_ttbr1 x8, x9 357 msr ttbr1_el1, swapper_ttb 358 isb 359 360 /* Set the flag to zero to indicate that we're all done */ 361 str wzr, [flag_ptr] 362#if CONFIG_PGTABLE_LEVELS > 4 363 ldp x19, x20, [sp, #16] 364 ldp x29, x30, [sp], #32 365#endif 366 ret 367 368.Lderef_pgd: 369 /* P4D */ 370 .if CONFIG_PGTABLE_LEVELS > 4 371 p4d .req x30 372 pte_to_phys cur_p4dp, pgd 373 kpti_map_pgtbl p4d, 0 374 kpti_mk_tbl_ng p4d, PTRS_PER_P4D 375 b .Lnext_pgd 376 .else /* CONFIG_PGTABLE_LEVELS <= 4 */ 377 p4d .req pgd 378 .set .Lnext_p4d, .Lnext_pgd 379 .endif 380 381.Lderef_p4d: 382 /* PUD */ 383 .if CONFIG_PGTABLE_LEVELS > 3 384 pud .req x10 385 pte_to_phys cur_pudp, p4d 386 kpti_map_pgtbl pud, 1 387 kpti_mk_tbl_ng pud, PTRS_PER_PUD 388 b .Lnext_p4d 389 .else /* CONFIG_PGTABLE_LEVELS <= 3 */ 390 pud .req pgd 391 .set .Lnext_pud, .Lnext_pgd 392 .endif 393 394.Lderef_pud: 395 /* PMD */ 396 .if CONFIG_PGTABLE_LEVELS > 2 397 pmd .req x13 398 pte_to_phys cur_pmdp, pud 399 kpti_map_pgtbl pmd, 2 400 kpti_mk_tbl_ng pmd, PTRS_PER_PMD 401 b .Lnext_pud 402 .else /* CONFIG_PGTABLE_LEVELS <= 2 */ 403 pmd .req pgd 404 .set .Lnext_pmd, .Lnext_pgd 405 .endif 406 407.Lderef_pmd: 408 /* PTE */ 409 pte_to_phys cur_ptep, pmd 410 kpti_map_pgtbl pte, 3 411 kpti_mk_tbl_ng pte, PTRS_PER_PTE 412 b .Lnext_pmd 413 414 .unreq cpu 415 .unreq temp_pte 416 .unreq num_cpus 417 .unreq pte_flags 418 .unreq temp_pgd_phys 419 .unreq cur_pgdp 420 .unreq end_pgdp 421 .unreq pgd 422 .unreq cur_pudp 423 .unreq end_pudp 424 .unreq pud 425 .unreq cur_pmdp 426 .unreq end_pmdp 427 .unreq pmd 428 .unreq cur_ptep 429 .unreq end_ptep 430 .unreq pte 431 .unreq valid 432 .unreq cur_p4dp 433 .unreq end_p4dp 434 .unreq p4d 435 436 /* Secondary CPUs end up here */ 437__idmap_kpti_secondary: 438 /* Uninstall swapper before surgery begins */ 439 __idmap_cpu_set_reserved_ttbr1 x16, x17 440 b scondary_cpu_wait 441 442 .unreq swapper_ttb 443 .unreq flag_ptr 444SYM_FUNC_END(idmap_kpti_install_ng_mappings) 445 .popsection 446#endif 447 448 .pushsection ".idmap.text", "a" 449SYM_TYPED_FUNC_START(wait_linear_map_split_to_ptes) 450 /* Must be same registers as in idmap_kpti_install_ng_mappings */ 451 swapper_ttb .req x3 452 flag_ptr .req x4 453 454 mrs swapper_ttb, ttbr1_el1 455 adr_l flag_ptr, idmap_kpti_bbml3_flag 456 __idmap_cpu_set_reserved_ttbr1 x16, x17 457 458scondary_cpu_wait: 459 /* Increment the flag to let the boot CPU we're ready */ 4601: ldxr w16, [flag_ptr] 461 add w16, w16, #1 462 stxr w17, w16, [flag_ptr] 463 cbnz w17, 1b 464 465 /* Wait for the boot CPU to finish messing around with swapper */ 466 sevl 4671: wfe 468 ldxr w16, [flag_ptr] 469 cbnz w16, 1b 470 471 /* All done, act like nothing happened */ 472 msr ttbr1_el1, swapper_ttb 473 isb 474 ret 475 476 .unreq swapper_ttb 477 .unreq flag_ptr 478SYM_FUNC_END(wait_linear_map_split_to_ptes) 479 .popsection 480 481/* 482 * __cpu_setup 483 * 484 * Initialise the processor for turning the MMU on. 485 * 486 * Output: 487 * Return in x0 the value of the SCTLR_EL1 register. 488 */ 489 .pushsection ".idmap.text", "a" 490SYM_FUNC_START(__cpu_setup) 491 tlbi vmalle1 // Invalidate local TLB 492 dsb nsh 493 494 msr cpacr_el1, xzr // Reset cpacr_el1 495 mov x1, MDSCR_EL1_TDCC // Reset mdscr_el1 and disable 496 msr mdscr_el1, x1 // access to the DCC from EL0 497 reset_pmuserenr_el0 x1 // Disable PMU access from EL0 498 reset_amuserenr_el0 x1 // Disable AMU access from EL0 499 500 /* 501 * Default values for VMSA control registers. These will be adjusted 502 * below depending on detected CPU features. 503 */ 504 mair .req x17 505 tcr .req x16 506 tcr2 .req x15 507 mov_q mair, MAIR_EL1_SET 508 mov_q tcr, TCR_T0SZ(IDMAP_VA_BITS) | TCR_T1SZ(VA_BITS_MIN) | TCR_CACHE_FLAGS | \ 509 TCR_SHARED | TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_EL1_AS | \ 510 TCR_EL1_TBI0 | TCR_EL1_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS 511 mov tcr2, xzr 512 513 tcr_clear_errata_bits tcr, x9, x5 514 515#ifdef CONFIG_ARM64_VA_BITS_52 516 mov x9, #64 - VA_BITS 517alternative_if ARM64_HAS_VA52 518 tcr_set_t1sz tcr, x9 519#ifdef CONFIG_ARM64_LPA2 520 orr tcr, tcr, #TCR_EL1_DS 521#endif 522alternative_else_nop_endif 523#endif 524 525 /* 526 * Set the IPS bits in TCR_EL1. 527 */ 528 tcr_compute_pa_size tcr, #TCR_EL1_IPS_SHIFT, x5, x6 529#ifdef CONFIG_ARM64_HW_AFDBM 530 /* 531 * Enable hardware update of the Access Flags bit. 532 * Hardware dirty bit management is enabled later, 533 * via capabilities. 534 */ 535 mrs x9, ID_AA64MMFR1_EL1 536 ubfx x9, x9, ID_AA64MMFR1_EL1_HAFDBS_SHIFT, #4 537 cbz x9, 1f 538 orr tcr, tcr, #TCR_EL1_HA // hardware Access flag update 539#ifdef CONFIG_ARM64_HAFT 540 cmp x9, ID_AA64MMFR1_EL1_HAFDBS_HAFT 541 b.lt 1f 542 orr tcr2, tcr2, TCR2_EL1_HAFT 543#endif /* CONFIG_ARM64_HAFT */ 5441: 545#endif /* CONFIG_ARM64_HW_AFDBM */ 546 msr mair_el1, mair 547 msr tcr_el1, tcr 548 549 mrs_s x1, SYS_ID_AA64MMFR3_EL1 550 ubfx x1, x1, #ID_AA64MMFR3_EL1_S1PIE_SHIFT, #4 551 cbz x1, .Lskip_indirection 552 553 mov_q x0, PIE_E0_ASM 554 msr REG_PIRE0_EL1, x0 555 mov_q x0, PIE_E1_ASM 556 msr REG_PIR_EL1, x0 557 558 orr tcr2, tcr2, TCR2_EL1_PIE 559 560.Lskip_indirection: 561 562 mrs_s x1, SYS_ID_AA64MMFR3_EL1 563 ubfx x1, x1, #ID_AA64MMFR3_EL1_TCRX_SHIFT, #4 564 cbz x1, 1f 565 msr REG_TCR2_EL1, tcr2 5661: 567 568 /* 569 * Prepare SCTLR 570 */ 571 mov_q x0, INIT_SCTLR_EL1_MMU_ON 572 ret // return to head.S 573 574 .unreq mair 575 .unreq tcr 576 .unreq tcr2 577SYM_FUNC_END(__cpu_setup) 578