1 #ifndef _ASM_X86_SPECIAL_INSNS_H 2 #define _ASM_X86_SPECIAL_INSNS_H 3 4 5 #ifdef __KERNEL__ 6 7 #include <asm/nops.h> 8 9 static inline void native_clts(void) 10 { 11 asm volatile("clts"); 12 } 13 14 /* 15 * Volatile isn't enough to prevent the compiler from reordering the 16 * read/write functions for the control registers and messing everything up. 17 * A memory clobber would solve the problem, but would prevent reordering of 18 * all loads stores around it, which can hurt performance. Solution is to 19 * use a variable and mimic reads and writes to it to enforce serialization 20 */ 21 extern unsigned long __force_order; 22 23 static inline unsigned long native_read_cr0(void) 24 { 25 unsigned long val; 26 asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order)); 27 return val; 28 } 29 30 static inline void native_write_cr0(unsigned long val) 31 { 32 asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order)); 33 } 34 35 static inline unsigned long native_read_cr2(void) 36 { 37 unsigned long val; 38 asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); 39 return val; 40 } 41 42 static inline void native_write_cr2(unsigned long val) 43 { 44 asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); 45 } 46 47 static inline unsigned long native_read_cr3(void) 48 { 49 unsigned long val; 50 asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order)); 51 return val; 52 } 53 54 static inline void native_write_cr3(unsigned long val) 55 { 56 asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order)); 57 } 58 59 static inline unsigned long native_read_cr4(void) 60 { 61 unsigned long val; 62 asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order)); 63 return val; 64 } 65 66 static inline unsigned long native_read_cr4_safe(void) 67 { 68 unsigned long val; 69 /* This could fault if %cr4 does not exist. In x86_64, a cr4 always 70 * exists, so it will never fail. */ 71 #ifdef CONFIG_X86_32 72 asm volatile("1: mov %%cr4, %0\n" 73 "2:\n" 74 _ASM_EXTABLE(1b, 2b) 75 : "=r" (val), "=m" (__force_order) : "0" (0)); 76 #else 77 val = native_read_cr4(); 78 #endif 79 return val; 80 } 81 82 static inline void native_write_cr4(unsigned long val) 83 { 84 asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order)); 85 } 86 87 #ifdef CONFIG_X86_64 88 static inline unsigned long native_read_cr8(void) 89 { 90 unsigned long cr8; 91 asm volatile("movq %%cr8,%0" : "=r" (cr8)); 92 return cr8; 93 } 94 95 static inline void native_write_cr8(unsigned long val) 96 { 97 asm volatile("movq %0,%%cr8" :: "r" (val) : "memory"); 98 } 99 #endif 100 101 static inline void native_wbinvd(void) 102 { 103 asm volatile("wbinvd": : :"memory"); 104 } 105 106 extern asmlinkage void native_load_gs_index(unsigned); 107 108 #ifdef CONFIG_PARAVIRT 109 #include <asm/paravirt.h> 110 #else 111 112 static inline unsigned long read_cr0(void) 113 { 114 return native_read_cr0(); 115 } 116 117 static inline void write_cr0(unsigned long x) 118 { 119 native_write_cr0(x); 120 } 121 122 static inline unsigned long read_cr2(void) 123 { 124 return native_read_cr2(); 125 } 126 127 static inline void write_cr2(unsigned long x) 128 { 129 native_write_cr2(x); 130 } 131 132 static inline unsigned long read_cr3(void) 133 { 134 return native_read_cr3(); 135 } 136 137 static inline void write_cr3(unsigned long x) 138 { 139 native_write_cr3(x); 140 } 141 142 static inline unsigned long __read_cr4(void) 143 { 144 return native_read_cr4(); 145 } 146 147 static inline unsigned long __read_cr4_safe(void) 148 { 149 return native_read_cr4_safe(); 150 } 151 152 static inline void __write_cr4(unsigned long x) 153 { 154 native_write_cr4(x); 155 } 156 157 static inline void wbinvd(void) 158 { 159 native_wbinvd(); 160 } 161 162 #ifdef CONFIG_X86_64 163 164 static inline unsigned long read_cr8(void) 165 { 166 return native_read_cr8(); 167 } 168 169 static inline void write_cr8(unsigned long x) 170 { 171 native_write_cr8(x); 172 } 173 174 static inline void load_gs_index(unsigned selector) 175 { 176 native_load_gs_index(selector); 177 } 178 179 #endif 180 181 /* Clear the 'TS' bit */ 182 static inline void clts(void) 183 { 184 native_clts(); 185 } 186 187 #endif/* CONFIG_PARAVIRT */ 188 189 #define stts() write_cr0(read_cr0() | X86_CR0_TS) 190 191 static inline void clflush(volatile void *__p) 192 { 193 asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p)); 194 } 195 196 static inline void clflushopt(volatile void *__p) 197 { 198 alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0", 199 ".byte 0x66; clflush %P0", 200 X86_FEATURE_CLFLUSHOPT, 201 "+m" (*(volatile char __force *)__p)); 202 } 203 204 static inline void clwb(volatile void *__p) 205 { 206 volatile struct { char x[64]; } *p = __p; 207 208 asm volatile(ALTERNATIVE_2( 209 ".byte " __stringify(NOP_DS_PREFIX) "; clflush (%[pax])", 210 ".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */ 211 X86_FEATURE_CLFLUSHOPT, 212 ".byte 0x66, 0x0f, 0xae, 0x30", /* clwb (%%rax) */ 213 X86_FEATURE_CLWB) 214 : [p] "+m" (*p) 215 : [pax] "a" (p)); 216 } 217 218 /** 219 * pcommit_sfence() - persistent commit and fence 220 * 221 * The PCOMMIT instruction ensures that data that has been flushed from the 222 * processor's cache hierarchy with CLWB, CLFLUSHOPT or CLFLUSH is accepted to 223 * memory and is durable on the DIMM. The primary use case for this is 224 * persistent memory. 225 * 226 * This function shows how to properly use CLWB/CLFLUSHOPT/CLFLUSH and PCOMMIT 227 * with appropriate fencing. 228 * 229 * Example: 230 * void flush_and_commit_buffer(void *vaddr, unsigned int size) 231 * { 232 * unsigned long clflush_mask = boot_cpu_data.x86_clflush_size - 1; 233 * void *vend = vaddr + size; 234 * void *p; 235 * 236 * for (p = (void *)((unsigned long)vaddr & ~clflush_mask); 237 * p < vend; p += boot_cpu_data.x86_clflush_size) 238 * clwb(p); 239 * 240 * // SFENCE to order CLWB/CLFLUSHOPT/CLFLUSH cache flushes 241 * // MFENCE via mb() also works 242 * wmb(); 243 * 244 * // PCOMMIT and the required SFENCE for ordering 245 * pcommit_sfence(); 246 * } 247 * 248 * After this function completes the data pointed to by 'vaddr' has been 249 * accepted to memory and will be durable if the 'vaddr' points to persistent 250 * memory. 251 * 252 * PCOMMIT must always be ordered by an MFENCE or SFENCE, so to help simplify 253 * things we include both the PCOMMIT and the required SFENCE in the 254 * alternatives generated by pcommit_sfence(). 255 */ 256 static inline void pcommit_sfence(void) 257 { 258 alternative(ASM_NOP7, 259 ".byte 0x66, 0x0f, 0xae, 0xf8\n\t" /* pcommit */ 260 "sfence", 261 X86_FEATURE_PCOMMIT); 262 } 263 264 #define nop() asm volatile ("nop") 265 266 267 #endif /* __KERNEL__ */ 268 269 #endif /* _ASM_X86_SPECIAL_INSNS_H */ 270