1 #ifndef _ASM_X86_TLBFLUSH_H 2 #define _ASM_X86_TLBFLUSH_H 3 4 #include <linux/mm.h> 5 #include <linux/sched.h> 6 7 #include <asm/processor.h> 8 #include <asm/cpufeature.h> 9 #include <asm/special_insns.h> 10 #include <asm/smp.h> 11 12 static inline void __invpcid(unsigned long pcid, unsigned long addr, 13 unsigned long type) 14 { 15 struct { u64 d[2]; } desc = { { pcid, addr } }; 16 17 /* 18 * The memory clobber is because the whole point is to invalidate 19 * stale TLB entries and, especially if we're flushing global 20 * mappings, we don't want the compiler to reorder any subsequent 21 * memory accesses before the TLB flush. 22 * 23 * The hex opcode is invpcid (%ecx), %eax in 32-bit mode and 24 * invpcid (%rcx), %rax in long mode. 25 */ 26 asm volatile (".byte 0x66, 0x0f, 0x38, 0x82, 0x01" 27 : : "m" (desc), "a" (type), "c" (&desc) : "memory"); 28 } 29 30 #define INVPCID_TYPE_INDIV_ADDR 0 31 #define INVPCID_TYPE_SINGLE_CTXT 1 32 #define INVPCID_TYPE_ALL_INCL_GLOBAL 2 33 #define INVPCID_TYPE_ALL_NON_GLOBAL 3 34 35 /* Flush all mappings for a given pcid and addr, not including globals. */ 36 static inline void invpcid_flush_one(unsigned long pcid, 37 unsigned long addr) 38 { 39 __invpcid(pcid, addr, INVPCID_TYPE_INDIV_ADDR); 40 } 41 42 /* Flush all mappings for a given PCID, not including globals. */ 43 static inline void invpcid_flush_single_context(unsigned long pcid) 44 { 45 __invpcid(pcid, 0, INVPCID_TYPE_SINGLE_CTXT); 46 } 47 48 /* Flush all mappings, including globals, for all PCIDs. */ 49 static inline void invpcid_flush_all(void) 50 { 51 __invpcid(0, 0, INVPCID_TYPE_ALL_INCL_GLOBAL); 52 } 53 54 /* Flush all mappings for all PCIDs except globals. */ 55 static inline void invpcid_flush_all_nonglobals(void) 56 { 57 __invpcid(0, 0, INVPCID_TYPE_ALL_NON_GLOBAL); 58 } 59 60 static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) 61 { 62 u64 new_tlb_gen; 63 64 /* 65 * Bump the generation count. This also serves as a full barrier 66 * that synchronizes with switch_mm(): callers are required to order 67 * their read of mm_cpumask after their writes to the paging 68 * structures. 69 */ 70 smp_mb__before_atomic(); 71 new_tlb_gen = atomic64_inc_return(&mm->context.tlb_gen); 72 smp_mb__after_atomic(); 73 74 return new_tlb_gen; 75 } 76 77 #ifdef CONFIG_PARAVIRT 78 #include <asm/paravirt.h> 79 #else 80 #define __flush_tlb() __native_flush_tlb() 81 #define __flush_tlb_global() __native_flush_tlb_global() 82 #define __flush_tlb_single(addr) __native_flush_tlb_single(addr) 83 #endif 84 85 /* 86 * If tlb_use_lazy_mode is true, then we try to avoid switching CR3 to point 87 * to init_mm when we switch to a kernel thread (e.g. the idle thread). If 88 * it's false, then we immediately switch CR3 when entering a kernel thread. 89 */ 90 DECLARE_STATIC_KEY_TRUE(tlb_use_lazy_mode); 91 92 /* 93 * 6 because 6 should be plenty and struct tlb_state will fit in 94 * two cache lines. 95 */ 96 #define TLB_NR_DYN_ASIDS 6 97 98 struct tlb_context { 99 u64 ctx_id; 100 u64 tlb_gen; 101 }; 102 103 struct tlb_state { 104 /* 105 * cpu_tlbstate.loaded_mm should match CR3 whenever interrupts 106 * are on. This means that it may not match current->active_mm, 107 * which will contain the previous user mm when we're in lazy TLB 108 * mode even if we've already switched back to swapper_pg_dir. 109 */ 110 struct mm_struct *loaded_mm; 111 u16 loaded_mm_asid; 112 u16 next_asid; 113 114 /* 115 * We can be in one of several states: 116 * 117 * - Actively using an mm. Our CPU's bit will be set in 118 * mm_cpumask(loaded_mm) and is_lazy == false; 119 * 120 * - Not using a real mm. loaded_mm == &init_mm. Our CPU's bit 121 * will not be set in mm_cpumask(&init_mm) and is_lazy == false. 122 * 123 * - Lazily using a real mm. loaded_mm != &init_mm, our bit 124 * is set in mm_cpumask(loaded_mm), but is_lazy == true. 125 * We're heuristically guessing that the CR3 load we 126 * skipped more than makes up for the overhead added by 127 * lazy mode. 128 */ 129 bool is_lazy; 130 131 /* 132 * Access to this CR4 shadow and to H/W CR4 is protected by 133 * disabling interrupts when modifying either one. 134 */ 135 unsigned long cr4; 136 137 /* 138 * This is a list of all contexts that might exist in the TLB. 139 * There is one per ASID that we use, and the ASID (what the 140 * CPU calls PCID) is the index into ctxts. 141 * 142 * For each context, ctx_id indicates which mm the TLB's user 143 * entries came from. As an invariant, the TLB will never 144 * contain entries that are out-of-date as when that mm reached 145 * the tlb_gen in the list. 146 * 147 * To be clear, this means that it's legal for the TLB code to 148 * flush the TLB without updating tlb_gen. This can happen 149 * (for now, at least) due to paravirt remote flushes. 150 * 151 * NB: context 0 is a bit special, since it's also used by 152 * various bits of init code. This is fine -- code that 153 * isn't aware of PCID will end up harmlessly flushing 154 * context 0. 155 */ 156 struct tlb_context ctxs[TLB_NR_DYN_ASIDS]; 157 }; 158 DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate); 159 160 /* Initialize cr4 shadow for this CPU. */ 161 static inline void cr4_init_shadow(void) 162 { 163 this_cpu_write(cpu_tlbstate.cr4, __read_cr4()); 164 } 165 166 /* Set in this cpu's CR4. */ 167 static inline void cr4_set_bits(unsigned long mask) 168 { 169 unsigned long cr4; 170 171 cr4 = this_cpu_read(cpu_tlbstate.cr4); 172 if ((cr4 | mask) != cr4) { 173 cr4 |= mask; 174 this_cpu_write(cpu_tlbstate.cr4, cr4); 175 __write_cr4(cr4); 176 } 177 } 178 179 /* Clear in this cpu's CR4. */ 180 static inline void cr4_clear_bits(unsigned long mask) 181 { 182 unsigned long cr4; 183 184 cr4 = this_cpu_read(cpu_tlbstate.cr4); 185 if ((cr4 & ~mask) != cr4) { 186 cr4 &= ~mask; 187 this_cpu_write(cpu_tlbstate.cr4, cr4); 188 __write_cr4(cr4); 189 } 190 } 191 192 static inline void cr4_toggle_bits(unsigned long mask) 193 { 194 unsigned long cr4; 195 196 cr4 = this_cpu_read(cpu_tlbstate.cr4); 197 cr4 ^= mask; 198 this_cpu_write(cpu_tlbstate.cr4, cr4); 199 __write_cr4(cr4); 200 } 201 202 /* Read the CR4 shadow. */ 203 static inline unsigned long cr4_read_shadow(void) 204 { 205 return this_cpu_read(cpu_tlbstate.cr4); 206 } 207 208 /* 209 * Save some of cr4 feature set we're using (e.g. Pentium 4MB 210 * enable and PPro Global page enable), so that any CPU's that boot 211 * up after us can get the correct flags. This should only be used 212 * during boot on the boot cpu. 213 */ 214 extern unsigned long mmu_cr4_features; 215 extern u32 *trampoline_cr4_features; 216 217 static inline void cr4_set_bits_and_update_boot(unsigned long mask) 218 { 219 mmu_cr4_features |= mask; 220 if (trampoline_cr4_features) 221 *trampoline_cr4_features = mmu_cr4_features; 222 cr4_set_bits(mask); 223 } 224 225 extern void initialize_tlbstate_and_flush(void); 226 227 static inline void __native_flush_tlb(void) 228 { 229 /* 230 * If current->mm == NULL then we borrow a mm which may change during a 231 * task switch and therefore we must not be preempted while we write CR3 232 * back: 233 */ 234 preempt_disable(); 235 native_write_cr3(__native_read_cr3()); 236 preempt_enable(); 237 } 238 239 static inline void __native_flush_tlb_global_irq_disabled(void) 240 { 241 unsigned long cr4; 242 243 cr4 = this_cpu_read(cpu_tlbstate.cr4); 244 /* clear PGE */ 245 native_write_cr4(cr4 & ~X86_CR4_PGE); 246 /* write old PGE again and flush TLBs */ 247 native_write_cr4(cr4); 248 } 249 250 static inline void __native_flush_tlb_global(void) 251 { 252 unsigned long flags; 253 254 if (static_cpu_has(X86_FEATURE_INVPCID)) { 255 /* 256 * Using INVPCID is considerably faster than a pair of writes 257 * to CR4 sandwiched inside an IRQ flag save/restore. 258 */ 259 invpcid_flush_all(); 260 return; 261 } 262 263 /* 264 * Read-modify-write to CR4 - protect it from preemption and 265 * from interrupts. (Use the raw variant because this code can 266 * be called from deep inside debugging code.) 267 */ 268 raw_local_irq_save(flags); 269 270 __native_flush_tlb_global_irq_disabled(); 271 272 raw_local_irq_restore(flags); 273 } 274 275 static inline void __native_flush_tlb_single(unsigned long addr) 276 { 277 asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); 278 } 279 280 static inline void __flush_tlb_all(void) 281 { 282 if (boot_cpu_has(X86_FEATURE_PGE)) 283 __flush_tlb_global(); 284 else 285 __flush_tlb(); 286 287 /* 288 * Note: if we somehow had PCID but not PGE, then this wouldn't work -- 289 * we'd end up flushing kernel translations for the current ASID but 290 * we might fail to flush kernel translations for other cached ASIDs. 291 * 292 * To avoid this issue, we force PCID off if PGE is off. 293 */ 294 } 295 296 static inline void __flush_tlb_one(unsigned long addr) 297 { 298 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE); 299 __flush_tlb_single(addr); 300 } 301 302 #define TLB_FLUSH_ALL -1UL 303 304 /* 305 * TLB flushing: 306 * 307 * - flush_tlb_all() flushes all processes TLBs 308 * - flush_tlb_mm(mm) flushes the specified mm context TLB's 309 * - flush_tlb_page(vma, vmaddr) flushes one page 310 * - flush_tlb_range(vma, start, end) flushes a range of pages 311 * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages 312 * - flush_tlb_others(cpumask, info) flushes TLBs on other cpus 313 * 314 * ..but the i386 has somewhat limited tlb flushing capabilities, 315 * and page-granular flushes are available only on i486 and up. 316 */ 317 struct flush_tlb_info { 318 /* 319 * We support several kinds of flushes. 320 * 321 * - Fully flush a single mm. .mm will be set, .end will be 322 * TLB_FLUSH_ALL, and .new_tlb_gen will be the tlb_gen to 323 * which the IPI sender is trying to catch us up. 324 * 325 * - Partially flush a single mm. .mm will be set, .start and 326 * .end will indicate the range, and .new_tlb_gen will be set 327 * such that the changes between generation .new_tlb_gen-1 and 328 * .new_tlb_gen are entirely contained in the indicated range. 329 * 330 * - Fully flush all mms whose tlb_gens have been updated. .mm 331 * will be NULL, .end will be TLB_FLUSH_ALL, and .new_tlb_gen 332 * will be zero. 333 */ 334 struct mm_struct *mm; 335 unsigned long start; 336 unsigned long end; 337 u64 new_tlb_gen; 338 }; 339 340 #define local_flush_tlb() __flush_tlb() 341 342 #define flush_tlb_mm(mm) flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL) 343 344 #define flush_tlb_range(vma, start, end) \ 345 flush_tlb_mm_range(vma->vm_mm, start, end, vma->vm_flags) 346 347 extern void flush_tlb_all(void); 348 extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, 349 unsigned long end, unsigned long vmflag); 350 extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); 351 352 static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long a) 353 { 354 flush_tlb_mm_range(vma->vm_mm, a, a + PAGE_SIZE, VM_NONE); 355 } 356 357 void native_flush_tlb_others(const struct cpumask *cpumask, 358 const struct flush_tlb_info *info); 359 360 static inline void arch_tlbbatch_add_mm(struct arch_tlbflush_unmap_batch *batch, 361 struct mm_struct *mm) 362 { 363 inc_mm_tlb_gen(mm); 364 cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm)); 365 } 366 367 extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch); 368 369 #ifndef CONFIG_PARAVIRT 370 #define flush_tlb_others(mask, info) \ 371 native_flush_tlb_others(mask, info) 372 #endif 373 374 #endif /* _ASM_X86_TLBFLUSH_H */ 375