1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/mmu_context.h 4 * 5 * Copyright (C) 1996 Russell King. 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASM_MMU_CONTEXT_H 9 #define __ASM_MMU_CONTEXT_H 10 11 #ifndef __ASSEMBLER__ 12 13 #include <linux/compiler.h> 14 #include <linux/sched.h> 15 #include <linux/sched/hotplug.h> 16 #include <linux/mm_types.h> 17 #include <linux/pgtable.h> 18 #include <linux/pkeys.h> 19 20 #include <asm/cacheflush.h> 21 #include <asm/cpufeature.h> 22 #include <asm/daifflags.h> 23 #include <asm/gcs.h> 24 #include <asm/proc-fns.h> 25 #include <asm/cputype.h> 26 #include <asm/sysreg.h> 27 #include <asm/tlbflush.h> 28 29 extern bool rodata_full; 30 31 static inline void contextidr_thread_switch(struct task_struct *next) 32 { 33 if (!IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR)) 34 return; 35 36 write_sysreg(task_pid_nr(next), contextidr_el1); 37 isb(); 38 } 39 40 /* 41 * Set TTBR0 to reserved_pg_dir. No translations will be possible via TTBR0. 42 */ 43 static inline void cpu_set_reserved_ttbr0_nosync(void) 44 { 45 unsigned long ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); 46 47 write_sysreg(ttbr, ttbr0_el1); 48 } 49 50 static inline void cpu_set_reserved_ttbr0(void) 51 { 52 cpu_set_reserved_ttbr0_nosync(); 53 isb(); 54 } 55 56 void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); 57 58 static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) 59 { 60 BUG_ON(pgd == swapper_pg_dir); 61 cpu_do_switch_mm(virt_to_phys(pgd),mm); 62 } 63 64 /* 65 * Ensure TCR.T0SZ is set to the provided value. 66 */ 67 static inline void __cpu_set_tcr_t0sz(unsigned long t0sz) 68 { 69 unsigned long tcr = read_sysreg(tcr_el1); 70 71 if ((tcr & TCR_EL1_T0SZ_MASK) == t0sz) 72 return; 73 74 tcr &= ~TCR_EL1_T0SZ_MASK; 75 tcr |= t0sz; 76 write_sysreg(tcr, tcr_el1); 77 isb(); 78 } 79 80 /* 81 * Remove the idmap from TTBR0_EL1 and install the pgd of the active mm. 82 * 83 * The idmap lives in the same VA range as userspace, but uses global entries 84 * and may use a different TCR_EL1.T0SZ. To avoid issues resulting from 85 * speculative TLB fetches, we must temporarily install the reserved page 86 * tables while we invalidate the TLBs and set up the correct TCR_EL1.T0SZ. 87 * 88 * If current is a not a user task, the mm covers the TTBR1_EL1 page tables, 89 * which should not be installed in TTBR0_EL1. In this case we can leave the 90 * reserved page tables in place. 91 */ 92 static inline void cpu_uninstall_idmap(void) 93 { 94 struct mm_struct *mm = current->active_mm; 95 96 cpu_set_reserved_ttbr0(); 97 local_flush_tlb_all(); 98 __cpu_set_tcr_t0sz(TCR_T0SZ(vabits_actual)); 99 100 if (mm != &init_mm && !system_uses_ttbr0_pan()) 101 cpu_switch_mm(mm->pgd, mm); 102 } 103 104 static inline void cpu_install_idmap(void) 105 { 106 cpu_set_reserved_ttbr0(); 107 local_flush_tlb_all(); 108 __cpu_set_tcr_t0sz(TCR_T0SZ(IDMAP_VA_BITS)); 109 110 cpu_switch_mm(lm_alias(idmap_pg_dir), &init_mm); 111 } 112 113 /* 114 * Load our new page tables. A strict BBM approach requires that we ensure that 115 * TLBs are free of any entries that may overlap with the global mappings we are 116 * about to install. 117 * 118 * For a real hibernate/resume/kexec cycle TTBR0 currently points to a zero 119 * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI runtime 120 * services), while for a userspace-driven test_resume cycle it points to 121 * userspace page tables (and we must point it at a zero page ourselves). 122 * 123 * We change T0SZ as part of installing the idmap. This is undone by 124 * cpu_uninstall_idmap() in __cpu_suspend_exit(). 125 */ 126 static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz) 127 { 128 cpu_set_reserved_ttbr0(); 129 local_flush_tlb_all(); 130 __cpu_set_tcr_t0sz(t0sz); 131 132 /* avoid cpu_switch_mm() and its SW-PAN and CNP interactions */ 133 write_sysreg(ttbr0, ttbr0_el1); 134 isb(); 135 } 136 137 void __cpu_replace_ttbr1(pgd_t *pgdp, bool cnp); 138 139 static inline void cpu_enable_swapper_cnp(void) 140 { 141 __cpu_replace_ttbr1(lm_alias(swapper_pg_dir), true); 142 } 143 144 static inline void cpu_replace_ttbr1(pgd_t *pgdp) 145 { 146 /* 147 * Only for early TTBR1 replacement before cpucaps are finalized and 148 * before we've decided whether to use CNP. 149 */ 150 WARN_ON(system_capabilities_finalized()); 151 __cpu_replace_ttbr1(pgdp, false); 152 } 153 154 /* 155 * It would be nice to return ASIDs back to the allocator, but unfortunately 156 * that introduces a race with a generation rollover where we could erroneously 157 * free an ASID allocated in a future generation. We could workaround this by 158 * freeing the ASID from the context of the dying mm (e.g. in arch_exit_mmap), 159 * but we'd then need to make sure that we didn't dirty any TLBs afterwards. 160 * Setting a reserved TTBR0 or EPD0 would work, but it all gets ugly when you 161 * take CPU migration into account. 162 */ 163 void check_and_switch_context(struct mm_struct *mm); 164 165 #define init_new_context(tsk, mm) init_new_context(tsk, mm) 166 static inline int 167 init_new_context(struct task_struct *tsk, struct mm_struct *mm) 168 { 169 atomic64_set(&mm->context.id, 0); 170 refcount_set(&mm->context.pinned, 0); 171 172 /* pkey 0 is the default, so always reserve it. */ 173 mm->context.pkey_allocation_map = BIT(0); 174 175 return 0; 176 } 177 178 static inline void arch_dup_pkeys(struct mm_struct *oldmm, 179 struct mm_struct *mm) 180 { 181 /* Duplicate the oldmm pkey state in mm: */ 182 mm->context.pkey_allocation_map = oldmm->context.pkey_allocation_map; 183 } 184 185 static inline int arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm) 186 { 187 arch_dup_pkeys(oldmm, mm); 188 189 return 0; 190 } 191 192 static inline void arch_exit_mmap(struct mm_struct *mm) 193 { 194 } 195 196 static inline void arch_unmap(struct mm_struct *mm, 197 unsigned long start, unsigned long end) 198 { 199 } 200 201 #ifdef CONFIG_ARM64_SW_TTBR0_PAN 202 static inline void update_saved_ttbr0(struct task_struct *tsk, 203 struct mm_struct *mm) 204 { 205 u64 ttbr; 206 207 if (!system_uses_ttbr0_pan()) 208 return; 209 210 if (mm == &init_mm) 211 ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); 212 else 213 ttbr = phys_to_ttbr(virt_to_phys(mm->pgd)) | 214 FIELD_PREP(TTBRx_EL1_ASID_MASK, ASID(mm)); 215 216 WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr); 217 } 218 #else 219 static inline void update_saved_ttbr0(struct task_struct *tsk, 220 struct mm_struct *mm) 221 { 222 } 223 #endif 224 225 #define enter_lazy_tlb enter_lazy_tlb 226 static inline void 227 enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) 228 { 229 /* 230 * We don't actually care about the ttbr0 mapping, so point it at the 231 * zero page. 232 */ 233 update_saved_ttbr0(tsk, &init_mm); 234 } 235 236 static inline void __switch_mm(struct mm_struct *next) 237 { 238 /* 239 * init_mm.pgd does not contain any user mappings and it is always 240 * active for kernel addresses in TTBR1. Just set the reserved TTBR0. 241 */ 242 if (next == &init_mm) { 243 cpu_set_reserved_ttbr0(); 244 return; 245 } 246 247 check_and_switch_context(next); 248 } 249 250 static inline void 251 switch_mm(struct mm_struct *prev, struct mm_struct *next, 252 struct task_struct *tsk) 253 { 254 if (prev != next) 255 __switch_mm(next); 256 257 /* 258 * Update the saved TTBR0_EL1 of the scheduled-in task as the previous 259 * value may have not been initialised yet (activate_mm caller) or the 260 * ASID has changed since the last run (following the context switch 261 * of another thread of the same process). 262 */ 263 update_saved_ttbr0(tsk, next); 264 } 265 266 static inline const struct cpumask * 267 __task_cpu_possible_mask(struct task_struct *p, const struct cpumask *mask) 268 { 269 if (!static_branch_unlikely(&arm64_mismatched_32bit_el0)) 270 return mask; 271 272 if (!is_compat_thread(task_thread_info(p))) 273 return mask; 274 275 return system_32bit_el0_cpumask(); 276 } 277 278 static inline const struct cpumask * 279 task_cpu_possible_mask(struct task_struct *p) 280 { 281 return __task_cpu_possible_mask(p, cpu_possible_mask); 282 } 283 #define task_cpu_possible_mask task_cpu_possible_mask 284 285 const struct cpumask *task_cpu_fallback_mask(struct task_struct *p); 286 287 void verify_cpu_asid_bits(void); 288 void post_ttbr_update_workaround(void); 289 290 unsigned long arm64_mm_context_get(struct mm_struct *mm); 291 void arm64_mm_context_put(struct mm_struct *mm); 292 293 #define mm_untag_mask mm_untag_mask 294 static inline unsigned long mm_untag_mask(struct mm_struct *mm) 295 { 296 return -1UL >> 8; 297 } 298 299 /* 300 * Only enforce protection keys on the current process, because there is no 301 * user context to access POR_EL0 for another address space. 302 */ 303 static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, 304 bool write, bool execute, bool foreign) 305 { 306 if (!system_supports_poe()) 307 return true; 308 309 /* allow access if the VMA is not one from this process */ 310 if (foreign || vma_is_foreign(vma)) 311 return true; 312 313 return por_el0_allows_pkey(vma_pkey(vma), write, execute); 314 } 315 316 #define deactivate_mm deactivate_mm 317 static inline void deactivate_mm(struct task_struct *tsk, 318 struct mm_struct *mm) 319 { 320 gcs_free(tsk); 321 } 322 323 324 #include <asm-generic/mmu_context.h> 325 326 #endif /* !__ASSEMBLER__ */ 327 328 #endif /* !__ASM_MMU_CONTEXT_H */ 329