1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Based on arch/arm/mm/proc.S 4 * 5 * Copyright (C) 2001 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 * Author: Catalin Marinas <catalin.marinas@arm.com> 8 */ 9 10#include <linux/init.h> 11#include <linux/linkage.h> 12#include <linux/pgtable.h> 13#include <linux/cfi_types.h> 14#include <asm/assembler.h> 15#include <asm/asm-offsets.h> 16#include <asm/asm_pointer_auth.h> 17#include <asm/hwcap.h> 18#include <asm/kernel-pgtable.h> 19#include <asm/pgtable-hwdef.h> 20#include <asm/cpufeature.h> 21#include <asm/alternative.h> 22#include <asm/smp.h> 23#include <asm/sysreg.h> 24 25#ifdef CONFIG_ARM64_64K_PAGES 26#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K 27#elif defined(CONFIG_ARM64_16K_PAGES) 28#define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K 29#else /* CONFIG_ARM64_4K_PAGES */ 30#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K 31#endif 32 33#ifdef CONFIG_RANDOMIZE_BASE 34#define TCR_KASLR_FLAGS TCR_NFD1 35#else 36#define TCR_KASLR_FLAGS 0 37#endif 38 39/* PTWs cacheable, inner/outer WBWA */ 40#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA 41 42#ifdef CONFIG_KASAN_SW_TAGS 43#define TCR_KASAN_SW_FLAGS TCR_TBI1 | TCR_TBID1 44#else 45#define TCR_KASAN_SW_FLAGS 0 46#endif 47 48#ifdef CONFIG_KASAN_HW_TAGS 49#define TCR_MTE_FLAGS TCR_TCMA1 | TCR_TBI1 | TCR_TBID1 50#elif defined(CONFIG_ARM64_MTE) 51/* 52 * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on 53 * TBI being enabled at EL1. 54 */ 55#define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1 56#else 57#define TCR_MTE_FLAGS 0 58#endif 59 60/* 61 * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and 62 * changed during mte_cpu_setup to Normal Tagged if the system supports MTE. 63 */ 64#define MAIR_EL1_SET \ 65 (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \ 66 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \ 67 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \ 68 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \ 69 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED)) 70 71#ifdef CONFIG_CPU_PM 72/** 73 * cpu_do_suspend - save CPU registers context 74 * 75 * x0: virtual address of context pointer 76 * 77 * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>. 78 */ 79SYM_FUNC_START(cpu_do_suspend) 80 mrs x2, tpidr_el0 81 mrs x3, tpidrro_el0 82 mrs x4, contextidr_el1 83 mrs x5, osdlr_el1 84 mrs x6, cpacr_el1 85 mrs x7, tcr_el1 86 mrs x8, vbar_el1 87 mrs x9, mdscr_el1 88 mrs x10, oslsr_el1 89 mrs x11, sctlr_el1 90 get_this_cpu_offset x12 91 mrs x13, sp_el0 92 stp x2, x3, [x0] 93 stp x4, x5, [x0, #16] 94 stp x6, x7, [x0, #32] 95 stp x8, x9, [x0, #48] 96 stp x10, x11, [x0, #64] 97 stp x12, x13, [x0, #80] 98 /* 99 * Save x18 as it may be used as a platform register, e.g. by shadow 100 * call stack. 101 */ 102 str x18, [x0, #96] 103 ret 104SYM_FUNC_END(cpu_do_suspend) 105 106/** 107 * cpu_do_resume - restore CPU register context 108 * 109 * x0: Address of context pointer 110 */ 111SYM_FUNC_START(cpu_do_resume) 112 ldp x2, x3, [x0] 113 ldp x4, x5, [x0, #16] 114 ldp x6, x8, [x0, #32] 115 ldp x9, x10, [x0, #48] 116 ldp x11, x12, [x0, #64] 117 ldp x13, x14, [x0, #80] 118 /* 119 * Restore x18, as it may be used as a platform register, and clear 120 * the buffer to minimize the risk of exposure when used for shadow 121 * call stack. 122 */ 123 ldr x18, [x0, #96] 124 str xzr, [x0, #96] 125 msr tpidr_el0, x2 126 msr tpidrro_el0, x3 127 msr contextidr_el1, x4 128 msr cpacr_el1, x6 129 130 /* Don't change t0sz here, mask those bits when restoring */ 131 mrs x7, tcr_el1 132 bfi x8, x7, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH 133 134 msr tcr_el1, x8 135 msr vbar_el1, x9 136 msr mdscr_el1, x10 137 138 msr sctlr_el1, x12 139 set_this_cpu_offset x13 140 msr sp_el0, x14 141 /* 142 * Restore oslsr_el1 by writing oslar_el1 143 */ 144 msr osdlr_el1, x5 145 ubfx x11, x11, #1, #1 146 msr oslar_el1, x11 147 reset_pmuserenr_el0 x0 // Disable PMU access from EL0 148 reset_amuserenr_el0 x0 // Disable AMU access from EL0 149 150alternative_if ARM64_HAS_RAS_EXTN 151 msr_s SYS_DISR_EL1, xzr 152alternative_else_nop_endif 153 154 ptrauth_keys_install_kernel_nosync x14, x1, x2, x3 155 isb 156 ret 157SYM_FUNC_END(cpu_do_resume) 158#endif 159 160 .pushsection ".idmap.text", "a" 161 162.macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2 163 adrp \tmp1, reserved_pg_dir 164 phys_to_ttbr \tmp2, \tmp1 165 offset_ttbr1 \tmp2, \tmp1 166 msr ttbr1_el1, \tmp2 167 isb 168 tlbi vmalle1 169 dsb nsh 170 isb 171.endm 172 173/* 174 * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1) 175 * 176 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be 177 * called by anything else. It can only be executed from a TTBR0 mapping. 178 */ 179SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1) 180 __idmap_cpu_set_reserved_ttbr1 x1, x3 181 182 offset_ttbr1 x0, x3 183 msr ttbr1_el1, x0 184 isb 185 186 ret 187SYM_FUNC_END(idmap_cpu_replace_ttbr1) 188SYM_FUNC_ALIAS(__pi_idmap_cpu_replace_ttbr1, idmap_cpu_replace_ttbr1) 189 .popsection 190 191#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 192 193#define KPTI_NG_PTE_FLAGS (PTE_ATTRINDX(MT_NORMAL) | PTE_TYPE_PAGE | \ 194 PTE_AF | PTE_SHARED | PTE_UXN | PTE_WRITE) 195 196 .pushsection ".idmap.text", "a" 197 198 .macro pte_to_phys, phys, pte 199 and \phys, \pte, #PTE_ADDR_LOW 200#ifdef CONFIG_ARM64_PA_BITS_52 201 and \pte, \pte, #PTE_ADDR_HIGH 202 orr \phys, \phys, \pte, lsl #PTE_ADDR_HIGH_SHIFT 203#endif 204 .endm 205 206 .macro kpti_mk_tbl_ng, type, num_entries 207 add end_\type\()p, cur_\type\()p, #\num_entries * 8 208.Ldo_\type: 209 ldr \type, [cur_\type\()p], #8 // Load the entry and advance 210 tbz \type, #0, .Lnext_\type // Skip invalid and 211 tbnz \type, #11, .Lnext_\type // non-global entries 212 orr \type, \type, #PTE_NG // Same bit for blocks and pages 213 str \type, [cur_\type\()p, #-8] // Update the entry 214 .ifnc \type, pte 215 tbnz \type, #1, .Lderef_\type 216 .endif 217.Lnext_\type: 218 cmp cur_\type\()p, end_\type\()p 219 b.ne .Ldo_\type 220 .endm 221 222 /* 223 * Dereference the current table entry and map it into the temporary 224 * fixmap slot associated with the current level. 225 */ 226 .macro kpti_map_pgtbl, type, level 227 str xzr, [temp_pte, #8 * (\level + 2)] // break before make 228 dsb nshst 229 add pte, temp_pte, #PAGE_SIZE * (\level + 2) 230 lsr pte, pte, #12 231 tlbi vaae1, pte 232 dsb nsh 233 isb 234 235 phys_to_pte pte, cur_\type\()p 236 add cur_\type\()p, temp_pte, #PAGE_SIZE * (\level + 2) 237 orr pte, pte, pte_flags 238 str pte, [temp_pte, #8 * (\level + 2)] 239 dsb nshst 240 .endm 241 242/* 243 * void __kpti_install_ng_mappings(int cpu, int num_secondaries, phys_addr_t temp_pgd, 244 * unsigned long temp_pte_va) 245 * 246 * Called exactly once from stop_machine context by each CPU found during boot. 247 */ 248SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) 249 cpu .req w0 250 temp_pte .req x0 251 num_cpus .req w1 252 pte_flags .req x1 253 temp_pgd_phys .req x2 254 swapper_ttb .req x3 255 flag_ptr .req x4 256 cur_pgdp .req x5 257 end_pgdp .req x6 258 pgd .req x7 259 cur_pudp .req x8 260 end_pudp .req x9 261 cur_pmdp .req x11 262 end_pmdp .req x12 263 cur_ptep .req x14 264 end_ptep .req x15 265 pte .req x16 266 valid .req x17 267 cur_p4dp .req x19 268 end_p4dp .req x20 269 270 mov x5, x3 // preserve temp_pte arg 271 mrs swapper_ttb, ttbr1_el1 272 adr_l flag_ptr, idmap_kpti_bbml2_flag 273 274 cbnz cpu, __idmap_kpti_secondary 275 276#if CONFIG_PGTABLE_LEVELS > 4 277 stp x29, x30, [sp, #-32]! 278 mov x29, sp 279 stp x19, x20, [sp, #16] 280#endif 281 282 /* We're the boot CPU. Wait for the others to catch up */ 283 sevl 2841: wfe 285 ldaxr w17, [flag_ptr] 286 eor w17, w17, num_cpus 287 cbnz w17, 1b 288 289 /* Switch to the temporary page tables on this CPU only */ 290 __idmap_cpu_set_reserved_ttbr1 x8, x9 291 offset_ttbr1 temp_pgd_phys, x8 292 msr ttbr1_el1, temp_pgd_phys 293 isb 294 295 mov temp_pte, x5 296 mov_q pte_flags, KPTI_NG_PTE_FLAGS 297 298 /* Everybody is enjoying the idmap, so we can rewrite swapper. */ 299 300#ifdef CONFIG_ARM64_LPA2 301 /* 302 * If LPA2 support is configured, but 52-bit virtual addressing is not 303 * enabled at runtime, we will fall back to one level of paging less, 304 * and so we have to walk swapper_pg_dir as if we dereferenced its 305 * address from a PGD level entry, and terminate the PGD level loop 306 * right after. 307 */ 308 adrp pgd, swapper_pg_dir // walk &swapper_pg_dir at the next level 309 mov cur_pgdp, end_pgdp // must be equal to terminate the PGD loop 310alternative_if_not ARM64_HAS_VA52 311 b .Lderef_pgd // skip to the next level 312alternative_else_nop_endif 313 /* 314 * LPA2 based 52-bit virtual addressing requires 52-bit physical 315 * addressing to be enabled as well. In this case, the shareability 316 * bits are repurposed as physical address bits, and should not be 317 * set in pte_flags. 318 */ 319 bic pte_flags, pte_flags, #PTE_SHARED 320#endif 321 322 /* PGD */ 323 adrp cur_pgdp, swapper_pg_dir 324 kpti_map_pgtbl pgd, -1 325 kpti_mk_tbl_ng pgd, PTRS_PER_PGD 326 327 /* Ensure all the updated entries are visible to secondary CPUs */ 328 dsb ishst 329 330 /* We're done: fire up swapper_pg_dir again */ 331 __idmap_cpu_set_reserved_ttbr1 x8, x9 332 msr ttbr1_el1, swapper_ttb 333 isb 334 335 /* Set the flag to zero to indicate that we're all done */ 336 str wzr, [flag_ptr] 337#if CONFIG_PGTABLE_LEVELS > 4 338 ldp x19, x20, [sp, #16] 339 ldp x29, x30, [sp], #32 340#endif 341 ret 342 343.Lderef_pgd: 344 /* P4D */ 345 .if CONFIG_PGTABLE_LEVELS > 4 346 p4d .req x30 347 pte_to_phys cur_p4dp, pgd 348 kpti_map_pgtbl p4d, 0 349 kpti_mk_tbl_ng p4d, PTRS_PER_P4D 350 b .Lnext_pgd 351 .else /* CONFIG_PGTABLE_LEVELS <= 4 */ 352 p4d .req pgd 353 .set .Lnext_p4d, .Lnext_pgd 354 .endif 355 356.Lderef_p4d: 357 /* PUD */ 358 .if CONFIG_PGTABLE_LEVELS > 3 359 pud .req x10 360 pte_to_phys cur_pudp, p4d 361 kpti_map_pgtbl pud, 1 362 kpti_mk_tbl_ng pud, PTRS_PER_PUD 363 b .Lnext_p4d 364 .else /* CONFIG_PGTABLE_LEVELS <= 3 */ 365 pud .req pgd 366 .set .Lnext_pud, .Lnext_pgd 367 .endif 368 369.Lderef_pud: 370 /* PMD */ 371 .if CONFIG_PGTABLE_LEVELS > 2 372 pmd .req x13 373 pte_to_phys cur_pmdp, pud 374 kpti_map_pgtbl pmd, 2 375 kpti_mk_tbl_ng pmd, PTRS_PER_PMD 376 b .Lnext_pud 377 .else /* CONFIG_PGTABLE_LEVELS <= 2 */ 378 pmd .req pgd 379 .set .Lnext_pmd, .Lnext_pgd 380 .endif 381 382.Lderef_pmd: 383 /* PTE */ 384 pte_to_phys cur_ptep, pmd 385 kpti_map_pgtbl pte, 3 386 kpti_mk_tbl_ng pte, PTRS_PER_PTE 387 b .Lnext_pmd 388 389 .unreq cpu 390 .unreq temp_pte 391 .unreq num_cpus 392 .unreq pte_flags 393 .unreq temp_pgd_phys 394 .unreq cur_pgdp 395 .unreq end_pgdp 396 .unreq pgd 397 .unreq cur_pudp 398 .unreq end_pudp 399 .unreq pud 400 .unreq cur_pmdp 401 .unreq end_pmdp 402 .unreq pmd 403 .unreq cur_ptep 404 .unreq end_ptep 405 .unreq pte 406 .unreq valid 407 .unreq cur_p4dp 408 .unreq end_p4dp 409 .unreq p4d 410 411 /* Secondary CPUs end up here */ 412__idmap_kpti_secondary: 413 /* Uninstall swapper before surgery begins */ 414 __idmap_cpu_set_reserved_ttbr1 x16, x17 415 b scondary_cpu_wait 416 417 .unreq swapper_ttb 418 .unreq flag_ptr 419SYM_FUNC_END(idmap_kpti_install_ng_mappings) 420 .popsection 421#endif 422 423 .pushsection ".idmap.text", "a" 424SYM_TYPED_FUNC_START(wait_linear_map_split_to_ptes) 425 /* Must be same registers as in idmap_kpti_install_ng_mappings */ 426 swapper_ttb .req x3 427 flag_ptr .req x4 428 429 mrs swapper_ttb, ttbr1_el1 430 adr_l flag_ptr, idmap_kpti_bbml2_flag 431 __idmap_cpu_set_reserved_ttbr1 x16, x17 432 433scondary_cpu_wait: 434 /* Increment the flag to let the boot CPU we're ready */ 4351: ldxr w16, [flag_ptr] 436 add w16, w16, #1 437 stxr w17, w16, [flag_ptr] 438 cbnz w17, 1b 439 440 /* Wait for the boot CPU to finish messing around with swapper */ 441 sevl 4421: wfe 443 ldxr w16, [flag_ptr] 444 cbnz w16, 1b 445 446 /* All done, act like nothing happened */ 447 msr ttbr1_el1, swapper_ttb 448 isb 449 ret 450 451 .unreq swapper_ttb 452 .unreq flag_ptr 453SYM_FUNC_END(wait_linear_map_split_to_ptes) 454 .popsection 455 456/* 457 * __cpu_setup 458 * 459 * Initialise the processor for turning the MMU on. 460 * 461 * Output: 462 * Return in x0 the value of the SCTLR_EL1 register. 463 */ 464 .pushsection ".idmap.text", "a" 465SYM_FUNC_START(__cpu_setup) 466 tlbi vmalle1 // Invalidate local TLB 467 dsb nsh 468 469 msr cpacr_el1, xzr // Reset cpacr_el1 470 mov x1, MDSCR_EL1_TDCC // Reset mdscr_el1 and disable 471 msr mdscr_el1, x1 // access to the DCC from EL0 472 reset_pmuserenr_el0 x1 // Disable PMU access from EL0 473 reset_amuserenr_el0 x1 // Disable AMU access from EL0 474 475 /* 476 * Default values for VMSA control registers. These will be adjusted 477 * below depending on detected CPU features. 478 */ 479 mair .req x17 480 tcr .req x16 481 tcr2 .req x15 482 mov_q mair, MAIR_EL1_SET 483 mov_q tcr, TCR_T0SZ(IDMAP_VA_BITS) | TCR_T1SZ(VA_BITS_MIN) | TCR_CACHE_FLAGS | \ 484 TCR_SHARED | TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ 485 TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS 486 mov tcr2, xzr 487 488 tcr_clear_errata_bits tcr, x9, x5 489 490#ifdef CONFIG_ARM64_VA_BITS_52 491 mov x9, #64 - VA_BITS 492alternative_if ARM64_HAS_VA52 493 tcr_set_t1sz tcr, x9 494#ifdef CONFIG_ARM64_LPA2 495 orr tcr, tcr, #TCR_DS 496#endif 497alternative_else_nop_endif 498#endif 499 500 /* 501 * Set the IPS bits in TCR_EL1. 502 */ 503 tcr_compute_pa_size tcr, #TCR_IPS_SHIFT, x5, x6 504#ifdef CONFIG_ARM64_HW_AFDBM 505 /* 506 * Enable hardware update of the Access Flags bit. 507 * Hardware dirty bit management is enabled later, 508 * via capabilities. 509 */ 510 mrs x9, ID_AA64MMFR1_EL1 511 ubfx x9, x9, ID_AA64MMFR1_EL1_HAFDBS_SHIFT, #4 512 cbz x9, 1f 513 orr tcr, tcr, #TCR_HA // hardware Access flag update 514#ifdef CONFIG_ARM64_HAFT 515 cmp x9, ID_AA64MMFR1_EL1_HAFDBS_HAFT 516 b.lt 1f 517 orr tcr2, tcr2, TCR2_EL1_HAFT 518#endif /* CONFIG_ARM64_HAFT */ 5191: 520#endif /* CONFIG_ARM64_HW_AFDBM */ 521 msr mair_el1, mair 522 msr tcr_el1, tcr 523 524 mrs_s x1, SYS_ID_AA64MMFR3_EL1 525 ubfx x1, x1, #ID_AA64MMFR3_EL1_S1PIE_SHIFT, #4 526 cbz x1, .Lskip_indirection 527 528 mov_q x0, PIE_E0_ASM 529 msr REG_PIRE0_EL1, x0 530 mov_q x0, PIE_E1_ASM 531 msr REG_PIR_EL1, x0 532 533 orr tcr2, tcr2, TCR2_EL1_PIE 534 535.Lskip_indirection: 536 537 mrs_s x1, SYS_ID_AA64MMFR3_EL1 538 ubfx x1, x1, #ID_AA64MMFR3_EL1_TCRX_SHIFT, #4 539 cbz x1, 1f 540 msr REG_TCR2_EL1, tcr2 5411: 542 543 /* 544 * Prepare SCTLR 545 */ 546 mov_q x0, INIT_SCTLR_EL1_MMU_ON 547 ret // return to head.S 548 549 .unreq mair 550 .unreq tcr 551 .unreq tcr2 552SYM_FUNC_END(__cpu_setup) 553