1 #ifndef _ASM_X86_TLBFLUSH_H 2 #define _ASM_X86_TLBFLUSH_H 3 4 #include <linux/mm.h> 5 #include <linux/sched.h> 6 7 #include <asm/processor.h> 8 #include <asm/cpufeature.h> 9 #include <asm/special_insns.h> 10 11 static inline void __invpcid(unsigned long pcid, unsigned long addr, 12 unsigned long type) 13 { 14 struct { u64 d[2]; } desc = { { pcid, addr } }; 15 16 /* 17 * The memory clobber is because the whole point is to invalidate 18 * stale TLB entries and, especially if we're flushing global 19 * mappings, we don't want the compiler to reorder any subsequent 20 * memory accesses before the TLB flush. 21 * 22 * The hex opcode is invpcid (%ecx), %eax in 32-bit mode and 23 * invpcid (%rcx), %rax in long mode. 24 */ 25 asm volatile (".byte 0x66, 0x0f, 0x38, 0x82, 0x01" 26 : : "m" (desc), "a" (type), "c" (&desc) : "memory"); 27 } 28 29 #define INVPCID_TYPE_INDIV_ADDR 0 30 #define INVPCID_TYPE_SINGLE_CTXT 1 31 #define INVPCID_TYPE_ALL_INCL_GLOBAL 2 32 #define INVPCID_TYPE_ALL_NON_GLOBAL 3 33 34 /* Flush all mappings for a given pcid and addr, not including globals. */ 35 static inline void invpcid_flush_one(unsigned long pcid, 36 unsigned long addr) 37 { 38 __invpcid(pcid, addr, INVPCID_TYPE_INDIV_ADDR); 39 } 40 41 /* Flush all mappings for a given PCID, not including globals. */ 42 static inline void invpcid_flush_single_context(unsigned long pcid) 43 { 44 __invpcid(pcid, 0, INVPCID_TYPE_SINGLE_CTXT); 45 } 46 47 /* Flush all mappings, including globals, for all PCIDs. */ 48 static inline void invpcid_flush_all(void) 49 { 50 __invpcid(0, 0, INVPCID_TYPE_ALL_INCL_GLOBAL); 51 } 52 53 /* Flush all mappings for all PCIDs except globals. */ 54 static inline void invpcid_flush_all_nonglobals(void) 55 { 56 __invpcid(0, 0, INVPCID_TYPE_ALL_NON_GLOBAL); 57 } 58 59 #ifdef CONFIG_PARAVIRT 60 #include <asm/paravirt.h> 61 #else 62 #define __flush_tlb() __native_flush_tlb() 63 #define __flush_tlb_global() __native_flush_tlb_global() 64 #define __flush_tlb_single(addr) __native_flush_tlb_single(addr) 65 #endif 66 67 struct tlb_state { 68 #ifdef CONFIG_SMP 69 struct mm_struct *active_mm; 70 int state; 71 #endif 72 73 /* 74 * Access to this CR4 shadow and to H/W CR4 is protected by 75 * disabling interrupts when modifying either one. 76 */ 77 unsigned long cr4; 78 }; 79 DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate); 80 81 /* Initialize cr4 shadow for this CPU. */ 82 static inline void cr4_init_shadow(void) 83 { 84 this_cpu_write(cpu_tlbstate.cr4, __read_cr4()); 85 } 86 87 /* Set in this cpu's CR4. */ 88 static inline void cr4_set_bits(unsigned long mask) 89 { 90 unsigned long cr4; 91 92 cr4 = this_cpu_read(cpu_tlbstate.cr4); 93 if ((cr4 | mask) != cr4) { 94 cr4 |= mask; 95 this_cpu_write(cpu_tlbstate.cr4, cr4); 96 __write_cr4(cr4); 97 } 98 } 99 100 /* Clear in this cpu's CR4. */ 101 static inline void cr4_clear_bits(unsigned long mask) 102 { 103 unsigned long cr4; 104 105 cr4 = this_cpu_read(cpu_tlbstate.cr4); 106 if ((cr4 & ~mask) != cr4) { 107 cr4 &= ~mask; 108 this_cpu_write(cpu_tlbstate.cr4, cr4); 109 __write_cr4(cr4); 110 } 111 } 112 113 /* Read the CR4 shadow. */ 114 static inline unsigned long cr4_read_shadow(void) 115 { 116 return this_cpu_read(cpu_tlbstate.cr4); 117 } 118 119 /* 120 * Save some of cr4 feature set we're using (e.g. Pentium 4MB 121 * enable and PPro Global page enable), so that any CPU's that boot 122 * up after us can get the correct flags. This should only be used 123 * during boot on the boot cpu. 124 */ 125 extern unsigned long mmu_cr4_features; 126 extern u32 *trampoline_cr4_features; 127 128 static inline void cr4_set_bits_and_update_boot(unsigned long mask) 129 { 130 mmu_cr4_features |= mask; 131 if (trampoline_cr4_features) 132 *trampoline_cr4_features = mmu_cr4_features; 133 cr4_set_bits(mask); 134 } 135 136 static inline void __native_flush_tlb(void) 137 { 138 native_write_cr3(native_read_cr3()); 139 } 140 141 static inline void __native_flush_tlb_global_irq_disabled(void) 142 { 143 unsigned long cr4; 144 145 cr4 = this_cpu_read(cpu_tlbstate.cr4); 146 /* clear PGE */ 147 native_write_cr4(cr4 & ~X86_CR4_PGE); 148 /* write old PGE again and flush TLBs */ 149 native_write_cr4(cr4); 150 } 151 152 static inline void __native_flush_tlb_global(void) 153 { 154 unsigned long flags; 155 156 if (static_cpu_has(X86_FEATURE_INVPCID)) { 157 /* 158 * Using INVPCID is considerably faster than a pair of writes 159 * to CR4 sandwiched inside an IRQ flag save/restore. 160 */ 161 invpcid_flush_all(); 162 return; 163 } 164 165 /* 166 * Read-modify-write to CR4 - protect it from preemption and 167 * from interrupts. (Use the raw variant because this code can 168 * be called from deep inside debugging code.) 169 */ 170 raw_local_irq_save(flags); 171 172 __native_flush_tlb_global_irq_disabled(); 173 174 raw_local_irq_restore(flags); 175 } 176 177 static inline void __native_flush_tlb_single(unsigned long addr) 178 { 179 asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); 180 } 181 182 static inline void __flush_tlb_all(void) 183 { 184 if (static_cpu_has(X86_FEATURE_PGE)) 185 __flush_tlb_global(); 186 else 187 __flush_tlb(); 188 } 189 190 static inline void __flush_tlb_one(unsigned long addr) 191 { 192 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE); 193 __flush_tlb_single(addr); 194 } 195 196 #define TLB_FLUSH_ALL -1UL 197 198 /* 199 * TLB flushing: 200 * 201 * - flush_tlb() flushes the current mm struct TLBs 202 * - flush_tlb_all() flushes all processes TLBs 203 * - flush_tlb_mm(mm) flushes the specified mm context TLB's 204 * - flush_tlb_page(vma, vmaddr) flushes one page 205 * - flush_tlb_range(vma, start, end) flushes a range of pages 206 * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages 207 * - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus 208 * 209 * ..but the i386 has somewhat limited tlb flushing capabilities, 210 * and page-granular flushes are available only on i486 and up. 211 */ 212 213 #ifndef CONFIG_SMP 214 215 /* "_up" is for UniProcessor. 216 * 217 * This is a helper for other header functions. *Not* intended to be called 218 * directly. All global TLB flushes need to either call this, or to bump the 219 * vm statistics themselves. 220 */ 221 static inline void __flush_tlb_up(void) 222 { 223 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); 224 __flush_tlb(); 225 } 226 227 static inline void flush_tlb_all(void) 228 { 229 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); 230 __flush_tlb_all(); 231 } 232 233 static inline void flush_tlb(void) 234 { 235 __flush_tlb_up(); 236 } 237 238 static inline void local_flush_tlb(void) 239 { 240 __flush_tlb_up(); 241 } 242 243 static inline void flush_tlb_mm(struct mm_struct *mm) 244 { 245 if (mm == current->active_mm) 246 __flush_tlb_up(); 247 } 248 249 static inline void flush_tlb_page(struct vm_area_struct *vma, 250 unsigned long addr) 251 { 252 if (vma->vm_mm == current->active_mm) 253 __flush_tlb_one(addr); 254 } 255 256 static inline void flush_tlb_range(struct vm_area_struct *vma, 257 unsigned long start, unsigned long end) 258 { 259 if (vma->vm_mm == current->active_mm) 260 __flush_tlb_up(); 261 } 262 263 static inline void flush_tlb_mm_range(struct mm_struct *mm, 264 unsigned long start, unsigned long end, unsigned long vmflag) 265 { 266 if (mm == current->active_mm) 267 __flush_tlb_up(); 268 } 269 270 static inline void native_flush_tlb_others(const struct cpumask *cpumask, 271 struct mm_struct *mm, 272 unsigned long start, 273 unsigned long end) 274 { 275 } 276 277 static inline void reset_lazy_tlbstate(void) 278 { 279 } 280 281 static inline void flush_tlb_kernel_range(unsigned long start, 282 unsigned long end) 283 { 284 flush_tlb_all(); 285 } 286 287 #else /* SMP */ 288 289 #include <asm/smp.h> 290 291 #define local_flush_tlb() __flush_tlb() 292 293 #define flush_tlb_mm(mm) flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL) 294 295 #define flush_tlb_range(vma, start, end) \ 296 flush_tlb_mm_range(vma->vm_mm, start, end, vma->vm_flags) 297 298 extern void flush_tlb_all(void); 299 extern void flush_tlb_current_task(void); 300 extern void flush_tlb_page(struct vm_area_struct *, unsigned long); 301 extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, 302 unsigned long end, unsigned long vmflag); 303 extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); 304 305 #define flush_tlb() flush_tlb_current_task() 306 307 void native_flush_tlb_others(const struct cpumask *cpumask, 308 struct mm_struct *mm, 309 unsigned long start, unsigned long end); 310 311 #define TLBSTATE_OK 1 312 #define TLBSTATE_LAZY 2 313 314 static inline void reset_lazy_tlbstate(void) 315 { 316 this_cpu_write(cpu_tlbstate.state, 0); 317 this_cpu_write(cpu_tlbstate.active_mm, &init_mm); 318 } 319 320 #endif /* SMP */ 321 322 #ifndef CONFIG_PARAVIRT 323 #define flush_tlb_others(mask, mm, start, end) \ 324 native_flush_tlb_others(mask, mm, start, end) 325 #endif 326 327 #endif /* _ASM_X86_TLBFLUSH_H */ 328