1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/export.h> 3 #include <linux/preempt.h> 4 #include <linux/smp.h> 5 #include <linux/completion.h> 6 #include <asm/msr.h> 7 8 static void __rdmsr_on_cpu(void *info) 9 { 10 struct msr_info *rv = info; 11 struct msr *reg; 12 13 if (rv->msrs) 14 reg = this_cpu_ptr(rv->msrs); 15 else 16 reg = &rv->reg; 17 18 rdmsrq(rv->msr_no, reg->q); 19 } 20 21 static void __wrmsr_on_cpu(void *info) 22 { 23 struct msr_info *rv = info; 24 struct msr *reg; 25 26 if (rv->msrs) 27 reg = this_cpu_ptr(rv->msrs); 28 else 29 reg = &rv->reg; 30 31 wrmsrq(rv->msr_no, reg->q); 32 } 33 34 int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) 35 { 36 int err; 37 struct msr_info rv; 38 39 memset(&rv, 0, sizeof(rv)); 40 41 rv.msr_no = msr_no; 42 err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1); 43 *q = rv.reg.q; 44 45 return err; 46 } 47 EXPORT_SYMBOL(rdmsrq_on_cpu); 48 49 int wrmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 q) 50 { 51 int err; 52 struct msr_info rv; 53 54 memset(&rv, 0, sizeof(rv)); 55 56 rv.msr_no = msr_no; 57 rv.reg.q = q; 58 59 err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1); 60 61 return err; 62 } 63 EXPORT_SYMBOL(wrmsrq_on_cpu); 64 65 static void __rwmsr_on_cpus(const struct cpumask *mask, u32 msr_no, 66 struct msr __percpu *msrs, 67 void (*msr_func) (void *info)) 68 { 69 struct msr_info rv; 70 int this_cpu; 71 72 memset(&rv, 0, sizeof(rv)); 73 74 rv.msrs = msrs; 75 rv.msr_no = msr_no; 76 77 this_cpu = get_cpu(); 78 79 if (cpumask_test_cpu(this_cpu, mask)) 80 msr_func(&rv); 81 82 smp_call_function_many(mask, msr_func, &rv, 1); 83 put_cpu(); 84 } 85 86 /* rdmsr on a bunch of CPUs 87 * 88 * @mask: which CPUs 89 * @msr_no: which MSR 90 * @msrs: array of MSR values 91 * 92 */ 93 void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs) 94 { 95 __rwmsr_on_cpus(mask, msr_no, msrs, __rdmsr_on_cpu); 96 } 97 EXPORT_SYMBOL(rdmsr_on_cpus); 98 99 /* 100 * wrmsr on a bunch of CPUs 101 * 102 * @mask: which CPUs 103 * @msr_no: which MSR 104 * @msrs: array of MSR values 105 * 106 */ 107 void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs) 108 { 109 __rwmsr_on_cpus(mask, msr_no, msrs, __wrmsr_on_cpu); 110 } 111 EXPORT_SYMBOL(wrmsr_on_cpus); 112 113 struct msr_info_completion { 114 struct msr_info msr; 115 struct completion done; 116 }; 117 118 /* These "safe" variants are slower and should be used when the target MSR 119 may not actually exist. */ 120 static void __rdmsr_safe_on_cpu(void *info) 121 { 122 struct msr_info_completion *rv = info; 123 124 rv->msr.err = rdmsrq_safe(rv->msr.msr_no, &rv->msr.reg.q); 125 complete(&rv->done); 126 } 127 128 static void __wrmsr_safe_on_cpu(void *info) 129 { 130 struct msr_info *rv = info; 131 132 rv->err = wrmsrq_safe(rv->msr_no, rv->reg.q); 133 } 134 135 int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) 136 { 137 int err; 138 struct msr_info rv; 139 140 memset(&rv, 0, sizeof(rv)); 141 142 rv.msr_no = msr_no; 143 rv.reg.q = q; 144 145 err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1); 146 147 return err ? err : rv.err; 148 } 149 EXPORT_SYMBOL(wrmsrq_safe_on_cpu); 150 151 int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) 152 { 153 struct msr_info_completion rv; 154 call_single_data_t csd; 155 int err; 156 157 INIT_CSD(&csd, __rdmsr_safe_on_cpu, &rv); 158 159 memset(&rv, 0, sizeof(rv)); 160 init_completion(&rv.done); 161 rv.msr.msr_no = msr_no; 162 163 err = smp_call_function_single_async(cpu, &csd); 164 if (!err) { 165 wait_for_completion(&rv.done); 166 err = rv.msr.err; 167 } 168 *q = rv.msr.reg.q; 169 170 return err; 171 } 172 EXPORT_SYMBOL(rdmsrq_safe_on_cpu); 173 174 /* 175 * These variants are significantly slower, but allows control over 176 * the entire 32-bit GPR set. 177 */ 178 static void __rdmsr_safe_regs_on_cpu(void *info) 179 { 180 struct msr_regs_info *rv = info; 181 182 rv->err = rdmsr_safe_regs(rv->regs); 183 } 184 185 static void __wrmsr_safe_regs_on_cpu(void *info) 186 { 187 struct msr_regs_info *rv = info; 188 189 rv->err = wrmsr_safe_regs(rv->regs); 190 } 191 192 int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) 193 { 194 int err; 195 struct msr_regs_info rv; 196 197 rv.regs = regs; 198 rv.err = -EIO; 199 err = smp_call_function_single(cpu, __rdmsr_safe_regs_on_cpu, &rv, 1); 200 201 return err ? err : rv.err; 202 } 203 EXPORT_SYMBOL(rdmsr_safe_regs_on_cpu); 204 205 int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) 206 { 207 int err; 208 struct msr_regs_info rv; 209 210 rv.regs = regs; 211 rv.err = -EIO; 212 err = smp_call_function_single(cpu, __wrmsr_safe_regs_on_cpu, &rv, 1); 213 214 return err ? err : rv.err; 215 } 216 EXPORT_SYMBOL(wrmsr_safe_regs_on_cpu); 217