1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_MSR_H 3 #define _ASM_X86_MSR_H 4 5 #include "msr-index.h" 6 7 #ifndef __ASSEMBLY__ 8 9 #include <asm/asm.h> 10 #include <asm/errno.h> 11 #include <asm/cpumask.h> 12 #include <uapi/asm/msr.h> 13 #include <asm/shared/msr.h> 14 15 #include <linux/percpu.h> 16 17 struct msr_info { 18 u32 msr_no; 19 struct msr reg; 20 struct msr __percpu *msrs; 21 int err; 22 }; 23 24 struct msr_regs_info { 25 u32 *regs; 26 int err; 27 }; 28 29 struct saved_msr { 30 bool valid; 31 struct msr_info info; 32 }; 33 34 struct saved_msrs { 35 unsigned int num; 36 struct saved_msr *array; 37 }; 38 39 /* 40 * both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A" 41 * constraint has different meanings. For i386, "A" means exactly 42 * edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead, 43 * it means rax *or* rdx. 44 */ 45 #ifdef CONFIG_X86_64 46 /* Using 64-bit values saves one instruction clearing the high half of low */ 47 #define DECLARE_ARGS(val, low, high) unsigned long low, high 48 #define EAX_EDX_VAL(val, low, high) ((low) | (high) << 32) 49 #define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) 50 #else 51 #define DECLARE_ARGS(val, low, high) unsigned long long val 52 #define EAX_EDX_VAL(val, low, high) (val) 53 #define EAX_EDX_RET(val, low, high) "=A" (val) 54 #endif 55 56 /* 57 * Be very careful with includes. This header is prone to include loops. 58 */ 59 #include <asm/atomic.h> 60 #include <linux/tracepoint-defs.h> 61 62 #ifdef CONFIG_TRACEPOINTS 63 DECLARE_TRACEPOINT(read_msr); 64 DECLARE_TRACEPOINT(write_msr); 65 DECLARE_TRACEPOINT(rdpmc); 66 extern void do_trace_write_msr(unsigned int msr, u64 val, int failed); 67 extern void do_trace_read_msr(unsigned int msr, u64 val, int failed); 68 extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed); 69 #else 70 static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {} 71 static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {} 72 static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {} 73 #endif 74 75 /* 76 * __rdmsr() and __wrmsr() are the two primitives which are the bare minimum MSR 77 * accessors and should not have any tracing or other functionality piggybacking 78 * on them - those are *purely* for accessing MSRs and nothing more. So don't even 79 * think of extending them - you will be slapped with a stinking trout or a frozen 80 * shark will reach you, wherever you are! You've been warned. 81 */ 82 static __always_inline unsigned long long __rdmsr(unsigned int msr) 83 { 84 DECLARE_ARGS(val, low, high); 85 86 asm volatile("1: rdmsr\n" 87 "2:\n" 88 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR) 89 : EAX_EDX_RET(val, low, high) : "c" (msr)); 90 91 return EAX_EDX_VAL(val, low, high); 92 } 93 94 static __always_inline void __wrmsr(unsigned int msr, u32 low, u32 high) 95 { 96 asm volatile("1: wrmsr\n" 97 "2:\n" 98 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR) 99 : : "c" (msr), "a"(low), "d" (high) : "memory"); 100 } 101 102 #define native_rdmsr(msr, val1, val2) \ 103 do { \ 104 u64 __val = __rdmsr((msr)); \ 105 (void)((val1) = (u32)__val); \ 106 (void)((val2) = (u32)(__val >> 32)); \ 107 } while (0) 108 109 #define native_wrmsr(msr, low, high) \ 110 __wrmsr(msr, low, high) 111 112 #define native_wrmsrl(msr, val) \ 113 __wrmsr((msr), (u32)((u64)(val)), \ 114 (u32)((u64)(val) >> 32)) 115 116 static inline unsigned long long native_read_msr(unsigned int msr) 117 { 118 unsigned long long val; 119 120 val = __rdmsr(msr); 121 122 if (tracepoint_enabled(read_msr)) 123 do_trace_read_msr(msr, val, 0); 124 125 return val; 126 } 127 128 static inline unsigned long long native_read_msr_safe(unsigned int msr, 129 int *err) 130 { 131 DECLARE_ARGS(val, low, high); 132 133 asm volatile("1: rdmsr ; xor %[err],%[err]\n" 134 "2:\n\t" 135 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_RDMSR_SAFE, %[err]) 136 : [err] "=r" (*err), EAX_EDX_RET(val, low, high) 137 : "c" (msr)); 138 if (tracepoint_enabled(read_msr)) 139 do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err); 140 return EAX_EDX_VAL(val, low, high); 141 } 142 143 /* Can be uninlined because referenced by paravirt */ 144 static inline void notrace 145 native_write_msr(unsigned int msr, u32 low, u32 high) 146 { 147 __wrmsr(msr, low, high); 148 149 if (tracepoint_enabled(write_msr)) 150 do_trace_write_msr(msr, ((u64)high << 32 | low), 0); 151 } 152 153 /* Can be uninlined because referenced by paravirt */ 154 static inline int notrace 155 native_write_msr_safe(unsigned int msr, u32 low, u32 high) 156 { 157 int err; 158 159 asm volatile("1: wrmsr ; xor %[err],%[err]\n" 160 "2:\n\t" 161 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_WRMSR_SAFE, %[err]) 162 : [err] "=a" (err) 163 : "c" (msr), "0" (low), "d" (high) 164 : "memory"); 165 if (tracepoint_enabled(write_msr)) 166 do_trace_write_msr(msr, ((u64)high << 32 | low), err); 167 return err; 168 } 169 170 extern int rdmsr_safe_regs(u32 regs[8]); 171 extern int wrmsr_safe_regs(u32 regs[8]); 172 173 /** 174 * rdtsc() - returns the current TSC without ordering constraints 175 * 176 * rdtsc() returns the result of RDTSC as a 64-bit integer. The 177 * only ordering constraint it supplies is the ordering implied by 178 * "asm volatile": it will put the RDTSC in the place you expect. The 179 * CPU can and will speculatively execute that RDTSC, though, so the 180 * results can be non-monotonic if compared on different CPUs. 181 */ 182 static __always_inline unsigned long long rdtsc(void) 183 { 184 DECLARE_ARGS(val, low, high); 185 186 asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); 187 188 return EAX_EDX_VAL(val, low, high); 189 } 190 191 /** 192 * rdtsc_ordered() - read the current TSC in program order 193 * 194 * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer. 195 * It is ordered like a load to a global in-memory counter. It should 196 * be impossible to observe non-monotonic rdtsc_unordered() behavior 197 * across multiple CPUs as long as the TSC is synced. 198 */ 199 static __always_inline unsigned long long rdtsc_ordered(void) 200 { 201 DECLARE_ARGS(val, low, high); 202 203 /* 204 * The RDTSC instruction is not ordered relative to memory 205 * access. The Intel SDM and the AMD APM are both vague on this 206 * point, but empirically an RDTSC instruction can be 207 * speculatively executed before prior loads. An RDTSC 208 * immediately after an appropriate barrier appears to be 209 * ordered as a normal load, that is, it provides the same 210 * ordering guarantees as reading from a global memory location 211 * that some other imaginary CPU is updating continuously with a 212 * time stamp. 213 * 214 * Thus, use the preferred barrier on the respective CPU, aiming for 215 * RDTSCP as the default. 216 */ 217 asm volatile(ALTERNATIVE_2("rdtsc", 218 "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC, 219 "rdtscp", X86_FEATURE_RDTSCP) 220 : EAX_EDX_RET(val, low, high) 221 /* RDTSCP clobbers ECX with MSR_TSC_AUX. */ 222 :: "ecx"); 223 224 return EAX_EDX_VAL(val, low, high); 225 } 226 227 static inline unsigned long long native_read_pmc(int counter) 228 { 229 DECLARE_ARGS(val, low, high); 230 231 asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter)); 232 if (tracepoint_enabled(rdpmc)) 233 do_trace_rdpmc(counter, EAX_EDX_VAL(val, low, high), 0); 234 return EAX_EDX_VAL(val, low, high); 235 } 236 237 #ifdef CONFIG_PARAVIRT_XXL 238 #include <asm/paravirt.h> 239 #else 240 #include <linux/errno.h> 241 /* 242 * Access to machine-specific registers (available on 586 and better only) 243 * Note: the rd* operations modify the parameters directly (without using 244 * pointer indirection), this allows gcc to optimize better 245 */ 246 247 #define rdmsr(msr, low, high) \ 248 do { \ 249 u64 __val = native_read_msr((msr)); \ 250 (void)((low) = (u32)__val); \ 251 (void)((high) = (u32)(__val >> 32)); \ 252 } while (0) 253 254 static inline void wrmsr(unsigned int msr, u32 low, u32 high) 255 { 256 native_write_msr(msr, low, high); 257 } 258 259 #define rdmsrl(msr, val) \ 260 ((val) = native_read_msr((msr))) 261 262 static inline void wrmsrl(unsigned int msr, u64 val) 263 { 264 native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32)); 265 } 266 267 /* wrmsr with exception handling */ 268 static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high) 269 { 270 return native_write_msr_safe(msr, low, high); 271 } 272 273 /* rdmsr with exception handling */ 274 #define rdmsr_safe(msr, low, high) \ 275 ({ \ 276 int __err; \ 277 u64 __val = native_read_msr_safe((msr), &__err); \ 278 (*low) = (u32)__val; \ 279 (*high) = (u32)(__val >> 32); \ 280 __err; \ 281 }) 282 283 static inline int rdmsrl_safe(unsigned int msr, unsigned long long *p) 284 { 285 int err; 286 287 *p = native_read_msr_safe(msr, &err); 288 return err; 289 } 290 291 #define rdpmc(counter, low, high) \ 292 do { \ 293 u64 _l = native_read_pmc((counter)); \ 294 (low) = (u32)_l; \ 295 (high) = (u32)(_l >> 32); \ 296 } while (0) 297 298 #define rdpmcl(counter, val) ((val) = native_read_pmc(counter)) 299 300 #endif /* !CONFIG_PARAVIRT_XXL */ 301 302 /* Instruction opcode for WRMSRNS supported in binutils >= 2.40 */ 303 #define WRMSRNS _ASM_BYTES(0x0f,0x01,0xc6) 304 305 /* Non-serializing WRMSR, when available. Falls back to a serializing WRMSR. */ 306 static __always_inline void wrmsrns(u32 msr, u64 val) 307 { 308 /* 309 * WRMSR is 2 bytes. WRMSRNS is 3 bytes. Pad WRMSR with a redundant 310 * DS prefix to avoid a trailing NOP. 311 */ 312 asm volatile("1: " ALTERNATIVE("ds wrmsr", WRMSRNS, X86_FEATURE_WRMSRNS) 313 "2: " _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR) 314 : : "c" (msr), "a" ((u32)val), "d" ((u32)(val >> 32))); 315 } 316 317 /* 318 * 64-bit version of wrmsr_safe(): 319 */ 320 static inline int wrmsrl_safe(u32 msr, u64 val) 321 { 322 return wrmsr_safe(msr, (u32)val, (u32)(val >> 32)); 323 } 324 325 struct msr __percpu *msrs_alloc(void); 326 void msrs_free(struct msr __percpu *msrs); 327 int msr_set_bit(u32 msr, u8 bit); 328 int msr_clear_bit(u32 msr, u8 bit); 329 330 #ifdef CONFIG_SMP 331 int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); 332 int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); 333 int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); 334 int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q); 335 void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs); 336 void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs); 337 int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); 338 int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); 339 int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); 340 int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q); 341 int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); 342 int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); 343 #else /* CONFIG_SMP */ 344 static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) 345 { 346 rdmsr(msr_no, *l, *h); 347 return 0; 348 } 349 static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) 350 { 351 wrmsr(msr_no, l, h); 352 return 0; 353 } 354 static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) 355 { 356 rdmsrl(msr_no, *q); 357 return 0; 358 } 359 static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q) 360 { 361 wrmsrl(msr_no, q); 362 return 0; 363 } 364 static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no, 365 struct msr __percpu *msrs) 366 { 367 rdmsr_on_cpu(0, msr_no, raw_cpu_ptr(&msrs->l), raw_cpu_ptr(&msrs->h)); 368 } 369 static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no, 370 struct msr __percpu *msrs) 371 { 372 wrmsr_on_cpu(0, msr_no, raw_cpu_read(msrs->l), raw_cpu_read(msrs->h)); 373 } 374 static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, 375 u32 *l, u32 *h) 376 { 377 return rdmsr_safe(msr_no, l, h); 378 } 379 static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) 380 { 381 return wrmsr_safe(msr_no, l, h); 382 } 383 static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) 384 { 385 return rdmsrl_safe(msr_no, q); 386 } 387 static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) 388 { 389 return wrmsrl_safe(msr_no, q); 390 } 391 static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) 392 { 393 return rdmsr_safe_regs(regs); 394 } 395 static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) 396 { 397 return wrmsr_safe_regs(regs); 398 } 399 #endif /* CONFIG_SMP */ 400 #endif /* __ASSEMBLY__ */ 401 #endif /* _ASM_X86_MSR_H */ 402