1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Intel Transactional Synchronization Extensions (TSX) control. 4 * 5 * Copyright (C) 2019-2021 Intel Corporation 6 * 7 * Author: 8 * Pawan Gupta <pawan.kumar.gupta@linux.intel.com> 9 */ 10 11 #include <linux/cpufeature.h> 12 13 #include <asm/cmdline.h> 14 #include <asm/cpu.h> 15 #include <asm/msr.h> 16 17 #include "cpu.h" 18 19 #undef pr_fmt 20 #define pr_fmt(fmt) "tsx: " fmt 21 22 enum tsx_ctrl_states { 23 TSX_CTRL_AUTO, 24 TSX_CTRL_ENABLE, 25 TSX_CTRL_DISABLE, 26 TSX_CTRL_RTM_ALWAYS_ABORT, 27 TSX_CTRL_NOT_SUPPORTED, 28 }; 29 30 static enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = 31 IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO) ? TSX_CTRL_AUTO : 32 IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF) ? TSX_CTRL_DISABLE : TSX_CTRL_ENABLE; 33 34 static void tsx_disable(void) 35 { 36 u64 tsx; 37 38 rdmsrq(MSR_IA32_TSX_CTRL, tsx); 39 40 /* Force all transactions to immediately abort */ 41 tsx |= TSX_CTRL_RTM_DISABLE; 42 43 /* 44 * Ensure TSX support is not enumerated in CPUID. 45 * This is visible to userspace and will ensure they 46 * do not waste resources trying TSX transactions that 47 * will always abort. 48 */ 49 tsx |= TSX_CTRL_CPUID_CLEAR; 50 51 wrmsrq(MSR_IA32_TSX_CTRL, tsx); 52 } 53 54 static void tsx_enable(void) 55 { 56 u64 tsx; 57 58 rdmsrq(MSR_IA32_TSX_CTRL, tsx); 59 60 /* Enable the RTM feature in the cpu */ 61 tsx &= ~TSX_CTRL_RTM_DISABLE; 62 63 /* 64 * Ensure TSX support is enumerated in CPUID. 65 * This is visible to userspace and will ensure they 66 * can enumerate and use the TSX feature. 67 */ 68 tsx &= ~TSX_CTRL_CPUID_CLEAR; 69 70 wrmsrq(MSR_IA32_TSX_CTRL, tsx); 71 } 72 73 static enum tsx_ctrl_states x86_get_tsx_auto_mode(void) 74 { 75 if (boot_cpu_has_bug(X86_BUG_TAA)) 76 return TSX_CTRL_DISABLE; 77 78 return TSX_CTRL_ENABLE; 79 } 80 81 /* 82 * Disabling TSX is not a trivial business. 83 * 84 * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT 85 * which says that TSX is practically disabled (all transactions are 86 * aborted by default). When that bit is set, the kernel unconditionally 87 * disables TSX. 88 * 89 * In order to do that, however, it needs to dance a bit: 90 * 91 * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and 92 * the MSR is present only when *two* CPUID bits are set: 93 * 94 * - X86_FEATURE_RTM_ALWAYS_ABORT 95 * - X86_FEATURE_TSX_FORCE_ABORT 96 * 97 * 2. The second method is for CPUs which do not have the above-mentioned 98 * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX 99 * through that one. Those CPUs can also have the initially mentioned 100 * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy 101 * applies: TSX gets disabled unconditionally. 102 * 103 * When either of the two methods are present, the kernel disables TSX and 104 * clears the respective RTM and HLE feature flags. 105 * 106 * An additional twist in the whole thing presents late microcode loading 107 * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID 108 * bit to be set after the update. 109 * 110 * A subsequent hotplug operation on any logical CPU except the BSP will 111 * cause for the supported CPUID feature bits to get re-detected and, if 112 * RTM and HLE get cleared all of a sudden, but, userspace did consult 113 * them before the update, then funny explosions will happen. Long story 114 * short: the kernel doesn't modify CPUID feature bits after booting. 115 * 116 * That's why, this function's call in init_intel() doesn't clear the 117 * feature flags. 118 */ 119 static void tsx_clear_cpuid(void) 120 { 121 u64 msr; 122 123 /* 124 * MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID 125 * bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present. 126 */ 127 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) && 128 boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) { 129 rdmsrq(MSR_TSX_FORCE_ABORT, msr); 130 msr |= MSR_TFA_TSX_CPUID_CLEAR; 131 wrmsrq(MSR_TSX_FORCE_ABORT, msr); 132 } else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) { 133 rdmsrq(MSR_IA32_TSX_CTRL, msr); 134 msr |= TSX_CTRL_CPUID_CLEAR; 135 wrmsrq(MSR_IA32_TSX_CTRL, msr); 136 } 137 } 138 139 /* 140 * Disable TSX development mode 141 * 142 * When the microcode released in Feb 2022 is applied, TSX will be disabled by 143 * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123 144 * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is 145 * not recommended for production deployments. In particular, applying MD_CLEAR 146 * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient 147 * execution attack may not be effective on these processors when Intel TSX is 148 * enabled with updated microcode. 149 */ 150 static void tsx_dev_mode_disable(void) 151 { 152 u64 mcu_opt_ctrl; 153 154 /* Check if RTM_ALLOW exists */ 155 if (!boot_cpu_has_bug(X86_BUG_TAA) || 156 !cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL) || 157 !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL)) 158 return; 159 160 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); 161 162 if (mcu_opt_ctrl & RTM_ALLOW) { 163 mcu_opt_ctrl &= ~RTM_ALLOW; 164 wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); 165 setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT); 166 } 167 } 168 169 static int __init tsx_parse_cmdline(char *str) 170 { 171 if (!str) 172 return -EINVAL; 173 174 if (!strcmp(str, "on")) { 175 tsx_ctrl_state = TSX_CTRL_ENABLE; 176 } else if (!strcmp(str, "off")) { 177 tsx_ctrl_state = TSX_CTRL_DISABLE; 178 } else if (!strcmp(str, "auto")) { 179 tsx_ctrl_state = TSX_CTRL_AUTO; 180 } else { 181 tsx_ctrl_state = TSX_CTRL_DISABLE; 182 pr_err("invalid option, defaulting to off\n"); 183 } 184 185 return 0; 186 } 187 early_param("tsx", tsx_parse_cmdline); 188 189 void __init tsx_init(void) 190 { 191 tsx_dev_mode_disable(); 192 193 /* 194 * Hardware will always abort a TSX transaction when the CPUID bit 195 * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate 196 * CPUID.RTM and CPUID.HLE bits. Clear them here. 197 */ 198 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) { 199 tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT; 200 tsx_clear_cpuid(); 201 setup_clear_cpu_cap(X86_FEATURE_RTM); 202 setup_clear_cpu_cap(X86_FEATURE_HLE); 203 return; 204 } 205 206 /* 207 * TSX is controlled via MSR_IA32_TSX_CTRL. However, support for this 208 * MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES. 209 * 210 * TSX control (aka MSR_IA32_TSX_CTRL) is only available after a 211 * microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES 212 * bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get 213 * MSR_IA32_TSX_CTRL support even after a microcode update. Thus, 214 * tsx= cmdline requests will do nothing on CPUs without 215 * MSR_IA32_TSX_CTRL support. 216 */ 217 if (x86_read_arch_cap_msr() & ARCH_CAP_TSX_CTRL_MSR) { 218 setup_force_cpu_cap(X86_FEATURE_MSR_TSX_CTRL); 219 } else { 220 tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED; 221 return; 222 } 223 224 if (tsx_ctrl_state == TSX_CTRL_AUTO) 225 tsx_ctrl_state = x86_get_tsx_auto_mode(); 226 227 if (tsx_ctrl_state == TSX_CTRL_DISABLE) { 228 tsx_disable(); 229 230 /* 231 * tsx_disable() will change the state of the RTM and HLE CPUID 232 * bits. Clear them here since they are now expected to be not 233 * set. 234 */ 235 setup_clear_cpu_cap(X86_FEATURE_RTM); 236 setup_clear_cpu_cap(X86_FEATURE_HLE); 237 } else if (tsx_ctrl_state == TSX_CTRL_ENABLE) { 238 239 /* 240 * HW defaults TSX to be enabled at bootup. 241 * We may still need the TSX enable support 242 * during init for special cases like 243 * kexec after TSX is disabled. 244 */ 245 tsx_enable(); 246 247 /* 248 * tsx_enable() will change the state of the RTM and HLE CPUID 249 * bits. Force them here since they are now expected to be set. 250 */ 251 setup_force_cpu_cap(X86_FEATURE_RTM); 252 setup_force_cpu_cap(X86_FEATURE_HLE); 253 } 254 } 255 256 void tsx_ap_init(void) 257 { 258 tsx_dev_mode_disable(); 259 260 if (tsx_ctrl_state == TSX_CTRL_ENABLE) 261 tsx_enable(); 262 else if (tsx_ctrl_state == TSX_CTRL_DISABLE) 263 tsx_disable(); 264 else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT) 265 /* See comment over that function for more details. */ 266 tsx_clear_cpuid(); 267 } 268