1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_MSR_H 3 #define _ASM_X86_MSR_H 4 5 #include "msr-index.h" 6 7 #ifndef __ASSEMBLER__ 8 9 #include <asm/asm.h> 10 #include <asm/errno.h> 11 #include <asm/cpumask.h> 12 #include <uapi/asm/msr.h> 13 #include <asm/shared/msr.h> 14 15 #include <linux/percpu.h> 16 17 struct msr_info { 18 u32 msr_no; 19 struct msr reg; 20 struct msr __percpu *msrs; 21 int err; 22 }; 23 24 struct msr_regs_info { 25 u32 *regs; 26 int err; 27 }; 28 29 struct saved_msr { 30 bool valid; 31 struct msr_info info; 32 }; 33 34 struct saved_msrs { 35 unsigned int num; 36 struct saved_msr *array; 37 }; 38 39 /* 40 * both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A" 41 * constraint has different meanings. For i386, "A" means exactly 42 * edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead, 43 * it means rax *or* rdx. 44 */ 45 #ifdef CONFIG_X86_64 46 /* Using 64-bit values saves one instruction clearing the high half of low */ 47 #define DECLARE_ARGS(val, low, high) unsigned long low, high 48 #define EAX_EDX_VAL(val, low, high) ((low) | (high) << 32) 49 #define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) 50 #else 51 #define DECLARE_ARGS(val, low, high) u64 val 52 #define EAX_EDX_VAL(val, low, high) (val) 53 #define EAX_EDX_RET(val, low, high) "=A" (val) 54 #endif 55 56 /* 57 * Be very careful with includes. This header is prone to include loops. 58 */ 59 #include <asm/atomic.h> 60 #include <linux/tracepoint-defs.h> 61 62 #ifdef CONFIG_TRACEPOINTS 63 DECLARE_TRACEPOINT(read_msr); 64 DECLARE_TRACEPOINT(write_msr); 65 DECLARE_TRACEPOINT(rdpmc); 66 extern void do_trace_write_msr(u32 msr, u64 val, int failed); 67 extern void do_trace_read_msr(u32 msr, u64 val, int failed); 68 extern void do_trace_rdpmc(u32 msr, u64 val, int failed); 69 #else 70 static inline void do_trace_write_msr(u32 msr, u64 val, int failed) {} 71 static inline void do_trace_read_msr(u32 msr, u64 val, int failed) {} 72 static inline void do_trace_rdpmc(u32 msr, u64 val, int failed) {} 73 #endif 74 75 /* 76 * __rdmsr() and __wrmsr() are the two primitives which are the bare minimum MSR 77 * accessors and should not have any tracing or other functionality piggybacking 78 * on them - those are *purely* for accessing MSRs and nothing more. So don't even 79 * think of extending them - you will be slapped with a stinking trout or a frozen 80 * shark will reach you, wherever you are! You've been warned. 81 */ 82 static __always_inline u64 __rdmsr(u32 msr) 83 { 84 DECLARE_ARGS(val, low, high); 85 86 asm volatile("1: rdmsr\n" 87 "2:\n" 88 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR) 89 : EAX_EDX_RET(val, low, high) : "c" (msr)); 90 91 return EAX_EDX_VAL(val, low, high); 92 } 93 94 static __always_inline void __wrmsr(u32 msr, u32 low, u32 high) 95 { 96 asm volatile("1: wrmsr\n" 97 "2:\n" 98 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR) 99 : : "c" (msr), "a"(low), "d" (high) : "memory"); 100 } 101 102 #define native_rdmsr(msr, val1, val2) \ 103 do { \ 104 u64 __val = __rdmsr((msr)); \ 105 (void)((val1) = (u32)__val); \ 106 (void)((val2) = (u32)(__val >> 32)); \ 107 } while (0) 108 109 #define native_wrmsr(msr, low, high) \ 110 __wrmsr(msr, low, high) 111 112 #define native_wrmsrl(msr, val) \ 113 __wrmsr((msr), (u32)((u64)(val)), \ 114 (u32)((u64)(val) >> 32)) 115 116 static inline u64 native_read_msr(u32 msr) 117 { 118 u64 val; 119 120 val = __rdmsr(msr); 121 122 if (tracepoint_enabled(read_msr)) 123 do_trace_read_msr(msr, val, 0); 124 125 return val; 126 } 127 128 static inline u64 native_read_msr_safe(u32 msr, int *err) 129 { 130 DECLARE_ARGS(val, low, high); 131 132 asm volatile("1: rdmsr ; xor %[err],%[err]\n" 133 "2:\n\t" 134 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_RDMSR_SAFE, %[err]) 135 : [err] "=r" (*err), EAX_EDX_RET(val, low, high) 136 : "c" (msr)); 137 if (tracepoint_enabled(read_msr)) 138 do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err); 139 return EAX_EDX_VAL(val, low, high); 140 } 141 142 /* Can be uninlined because referenced by paravirt */ 143 static inline void notrace 144 native_write_msr(u32 msr, u32 low, u32 high) 145 { 146 __wrmsr(msr, low, high); 147 148 if (tracepoint_enabled(write_msr)) 149 do_trace_write_msr(msr, ((u64)high << 32 | low), 0); 150 } 151 152 /* Can be uninlined because referenced by paravirt */ 153 static inline int notrace 154 native_write_msr_safe(u32 msr, u32 low, u32 high) 155 { 156 int err; 157 158 asm volatile("1: wrmsr ; xor %[err],%[err]\n" 159 "2:\n\t" 160 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_WRMSR_SAFE, %[err]) 161 : [err] "=a" (err) 162 : "c" (msr), "0" (low), "d" (high) 163 : "memory"); 164 if (tracepoint_enabled(write_msr)) 165 do_trace_write_msr(msr, ((u64)high << 32 | low), err); 166 return err; 167 } 168 169 extern int rdmsr_safe_regs(u32 regs[8]); 170 extern int wrmsr_safe_regs(u32 regs[8]); 171 172 /** 173 * rdtsc() - returns the current TSC without ordering constraints 174 * 175 * rdtsc() returns the result of RDTSC as a 64-bit integer. The 176 * only ordering constraint it supplies is the ordering implied by 177 * "asm volatile": it will put the RDTSC in the place you expect. The 178 * CPU can and will speculatively execute that RDTSC, though, so the 179 * results can be non-monotonic if compared on different CPUs. 180 */ 181 static __always_inline u64 rdtsc(void) 182 { 183 DECLARE_ARGS(val, low, high); 184 185 asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); 186 187 return EAX_EDX_VAL(val, low, high); 188 } 189 190 /** 191 * rdtsc_ordered() - read the current TSC in program order 192 * 193 * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer. 194 * It is ordered like a load to a global in-memory counter. It should 195 * be impossible to observe non-monotonic rdtsc_unordered() behavior 196 * across multiple CPUs as long as the TSC is synced. 197 */ 198 static __always_inline u64 rdtsc_ordered(void) 199 { 200 DECLARE_ARGS(val, low, high); 201 202 /* 203 * The RDTSC instruction is not ordered relative to memory 204 * access. The Intel SDM and the AMD APM are both vague on this 205 * point, but empirically an RDTSC instruction can be 206 * speculatively executed before prior loads. An RDTSC 207 * immediately after an appropriate barrier appears to be 208 * ordered as a normal load, that is, it provides the same 209 * ordering guarantees as reading from a global memory location 210 * that some other imaginary CPU is updating continuously with a 211 * time stamp. 212 * 213 * Thus, use the preferred barrier on the respective CPU, aiming for 214 * RDTSCP as the default. 215 */ 216 asm volatile(ALTERNATIVE_2("rdtsc", 217 "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC, 218 "rdtscp", X86_FEATURE_RDTSCP) 219 : EAX_EDX_RET(val, low, high) 220 /* RDTSCP clobbers ECX with MSR_TSC_AUX. */ 221 :: "ecx"); 222 223 return EAX_EDX_VAL(val, low, high); 224 } 225 226 static inline u64 native_read_pmc(int counter) 227 { 228 DECLARE_ARGS(val, low, high); 229 230 asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter)); 231 if (tracepoint_enabled(rdpmc)) 232 do_trace_rdpmc(counter, EAX_EDX_VAL(val, low, high), 0); 233 return EAX_EDX_VAL(val, low, high); 234 } 235 236 #ifdef CONFIG_PARAVIRT_XXL 237 #include <asm/paravirt.h> 238 #else 239 #include <linux/errno.h> 240 /* 241 * Access to machine-specific registers (available on 586 and better only) 242 * Note: the rd* operations modify the parameters directly (without using 243 * pointer indirection), this allows gcc to optimize better 244 */ 245 246 #define rdmsr(msr, low, high) \ 247 do { \ 248 u64 __val = native_read_msr((msr)); \ 249 (void)((low) = (u32)__val); \ 250 (void)((high) = (u32)(__val >> 32)); \ 251 } while (0) 252 253 static inline void wrmsr(u32 msr, u32 low, u32 high) 254 { 255 native_write_msr(msr, low, high); 256 } 257 258 #define rdmsrq(msr, val) \ 259 ((val) = native_read_msr((msr))) 260 261 static inline void wrmsrq(u32 msr, u64 val) 262 { 263 native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32)); 264 } 265 266 /* wrmsr with exception handling */ 267 static inline int wrmsr_safe(u32 msr, u32 low, u32 high) 268 { 269 return native_write_msr_safe(msr, low, high); 270 } 271 272 /* rdmsr with exception handling */ 273 #define rdmsr_safe(msr, low, high) \ 274 ({ \ 275 int __err; \ 276 u64 __val = native_read_msr_safe((msr), &__err); \ 277 (*low) = (u32)__val; \ 278 (*high) = (u32)(__val >> 32); \ 279 __err; \ 280 }) 281 282 static inline int rdmsrq_safe(u32 msr, u64 *p) 283 { 284 int err; 285 286 *p = native_read_msr_safe(msr, &err); 287 return err; 288 } 289 290 #define rdpmc(counter, low, high) \ 291 do { \ 292 u64 _l = native_read_pmc((counter)); \ 293 (low) = (u32)_l; \ 294 (high) = (u32)(_l >> 32); \ 295 } while (0) 296 297 #define rdpmcl(counter, val) ((val) = native_read_pmc(counter)) 298 299 #endif /* !CONFIG_PARAVIRT_XXL */ 300 301 /* Instruction opcode for WRMSRNS supported in binutils >= 2.40 */ 302 #define WRMSRNS _ASM_BYTES(0x0f,0x01,0xc6) 303 304 /* Non-serializing WRMSR, when available. Falls back to a serializing WRMSR. */ 305 static __always_inline void wrmsrns(u32 msr, u64 val) 306 { 307 /* 308 * WRMSR is 2 bytes. WRMSRNS is 3 bytes. Pad WRMSR with a redundant 309 * DS prefix to avoid a trailing NOP. 310 */ 311 asm volatile("1: " ALTERNATIVE("ds wrmsr", WRMSRNS, X86_FEATURE_WRMSRNS) 312 "2: " _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR) 313 : : "c" (msr), "a" ((u32)val), "d" ((u32)(val >> 32))); 314 } 315 316 /* 317 * 64-bit version of wrmsr_safe(): 318 */ 319 static inline int wrmsrq_safe(u32 msr, u64 val) 320 { 321 return wrmsr_safe(msr, (u32)val, (u32)(val >> 32)); 322 } 323 324 struct msr __percpu *msrs_alloc(void); 325 void msrs_free(struct msr __percpu *msrs); 326 int msr_set_bit(u32 msr, u8 bit); 327 int msr_clear_bit(u32 msr, u8 bit); 328 329 #ifdef CONFIG_SMP 330 int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); 331 int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); 332 int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); 333 int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q); 334 void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs); 335 void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs); 336 int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); 337 int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); 338 int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); 339 int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q); 340 int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); 341 int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); 342 #else /* CONFIG_SMP */ 343 static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) 344 { 345 rdmsr(msr_no, *l, *h); 346 return 0; 347 } 348 static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) 349 { 350 wrmsr(msr_no, l, h); 351 return 0; 352 } 353 static inline int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) 354 { 355 rdmsrq(msr_no, *q); 356 return 0; 357 } 358 static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q) 359 { 360 wrmsrq(msr_no, q); 361 return 0; 362 } 363 static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no, 364 struct msr __percpu *msrs) 365 { 366 rdmsr_on_cpu(0, msr_no, raw_cpu_ptr(&msrs->l), raw_cpu_ptr(&msrs->h)); 367 } 368 static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no, 369 struct msr __percpu *msrs) 370 { 371 wrmsr_on_cpu(0, msr_no, raw_cpu_read(msrs->l), raw_cpu_read(msrs->h)); 372 } 373 static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, 374 u32 *l, u32 *h) 375 { 376 return rdmsr_safe(msr_no, l, h); 377 } 378 static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) 379 { 380 return wrmsr_safe(msr_no, l, h); 381 } 382 static inline int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) 383 { 384 return rdmsrq_safe(msr_no, q); 385 } 386 static inline int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) 387 { 388 return wrmsrq_safe(msr_no, q); 389 } 390 static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) 391 { 392 return rdmsr_safe_regs(regs); 393 } 394 static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) 395 { 396 return wrmsr_safe_regs(regs); 397 } 398 #endif /* CONFIG_SMP */ 399 #endif /* __ASSEMBLER__ */ 400 #endif /* _ASM_X86_MSR_H */ 401