1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/export.h> 3 #include <linux/bitops.h> 4 #include <linux/dmi.h> 5 #include <linux/elf.h> 6 #include <linux/mm.h> 7 #include <linux/kvm_types.h> 8 #include <linux/io.h> 9 #include <linux/sched.h> 10 #include <linux/sched/clock.h> 11 #include <linux/random.h> 12 #include <linux/topology.h> 13 #include <linux/platform_data/x86/amd-fch.h> 14 #include <asm/processor.h> 15 #include <asm/apic.h> 16 #include <asm/cacheinfo.h> 17 #include <asm/cpu.h> 18 #include <asm/cpu_device_id.h> 19 #include <asm/cpuid/api.h> 20 #include <asm/spec-ctrl.h> 21 #include <asm/smp.h> 22 #include <asm/numa.h> 23 #include <asm/pci-direct.h> 24 #include <asm/delay.h> 25 #include <asm/debugreg.h> 26 #include <asm/resctrl.h> 27 #include <asm/msr.h> 28 #include <asm/sev.h> 29 30 #ifdef CONFIG_X86_64 31 # include <asm/mmconfig.h> 32 #endif 33 34 #include "cpu.h" 35 36 u16 invlpgb_count_max __ro_after_init = 1; 37 38 static inline int rdmsrq_amd_safe(unsigned msr, u64 *p) 39 { 40 u32 gprs[8] = { 0 }; 41 int err; 42 43 WARN_ONCE((boot_cpu_data.x86 != 0xf), 44 "%s should only be used on K8!\n", __func__); 45 46 gprs[1] = msr; 47 gprs[7] = 0x9c5a203a; 48 49 err = rdmsr_safe_regs(gprs); 50 51 *p = gprs[0] | ((u64)gprs[2] << 32); 52 53 return err; 54 } 55 56 static inline int wrmsrq_amd_safe(unsigned msr, u64 val) 57 { 58 u32 gprs[8] = { 0 }; 59 60 WARN_ONCE((boot_cpu_data.x86 != 0xf), 61 "%s should only be used on K8!\n", __func__); 62 63 gprs[0] = (u32)val; 64 gprs[1] = msr; 65 gprs[2] = val >> 32; 66 gprs[7] = 0x9c5a203a; 67 68 return wrmsr_safe_regs(gprs); 69 } 70 71 /* 72 * B step AMD K6 before B 9730xxxx have hardware bugs that can cause 73 * misexecution of code under Linux. Owners of such processors should 74 * contact AMD for precise details and a CPU swap. 75 * 76 * See http://www.multimania.com/poulot/k6bug.html 77 * and section 2.6.2 of "AMD-K6 Processor Revision Guide - Model 6" 78 * (Publication # 21266 Issue Date: August 1998) 79 * 80 * The following test is erm.. interesting. AMD neglected to up 81 * the chip setting when fixing the bug but they also tweaked some 82 * performance at the same time.. 83 */ 84 85 #ifdef CONFIG_X86_32 86 extern __visible void vide(void); 87 __asm__(".text\n" 88 ".globl vide\n" 89 ".type vide, @function\n" 90 ".align 4\n" 91 "vide: ret\n"); 92 #endif 93 94 static void init_amd_k5(struct cpuinfo_x86 *c) 95 { 96 #ifdef CONFIG_X86_32 97 /* 98 * General Systems BIOSen alias the cpu frequency registers 99 * of the Elan at 0x000df000. Unfortunately, one of the Linux 100 * drivers subsequently pokes it, and changes the CPU speed. 101 * Workaround : Remove the unneeded alias. 102 */ 103 #define CBAR (0xfffc) /* Configuration Base Address (32-bit) */ 104 #define CBAR_ENB (0x80000000) 105 #define CBAR_KEY (0X000000CB) 106 if (c->x86_model == 9 || c->x86_model == 10) { 107 if (inl(CBAR) & CBAR_ENB) 108 outl(0 | CBAR_KEY, CBAR); 109 } 110 #endif 111 } 112 113 static void init_amd_k6(struct cpuinfo_x86 *c) 114 { 115 #ifdef CONFIG_X86_32 116 u32 l, h; 117 int mbytes = get_num_physpages() >> (20-PAGE_SHIFT); 118 119 if (c->x86_model < 6) { 120 /* Based on AMD doc 20734R - June 2000 */ 121 if (c->x86_model == 0) { 122 clear_cpu_cap(c, X86_FEATURE_APIC); 123 set_cpu_cap(c, X86_FEATURE_PGE); 124 } 125 return; 126 } 127 128 if (c->x86_model == 6 && c->x86_stepping == 1) { 129 const int K6_BUG_LOOP = 1000000; 130 int n; 131 void (*f_vide)(void); 132 u64 d, d2; 133 134 pr_info("AMD K6 stepping B detected - "); 135 136 /* 137 * It looks like AMD fixed the 2.6.2 bug and improved indirect 138 * calls at the same time. 139 */ 140 141 n = K6_BUG_LOOP; 142 f_vide = vide; 143 OPTIMIZER_HIDE_VAR(f_vide); 144 d = rdtsc(); 145 while (n--) 146 f_vide(); 147 d2 = rdtsc(); 148 d = d2-d; 149 150 if (d > 20*K6_BUG_LOOP) 151 pr_cont("system stability may be impaired when more than 32 MB are used.\n"); 152 else 153 pr_cont("probably OK (after B9730xxxx).\n"); 154 } 155 156 /* K6 with old style WHCR */ 157 if (c->x86_model < 8 || 158 (c->x86_model == 8 && c->x86_stepping < 8)) { 159 /* We can only write allocate on the low 508Mb */ 160 if (mbytes > 508) 161 mbytes = 508; 162 163 rdmsr(MSR_K6_WHCR, l, h); 164 if ((l&0x0000FFFF) == 0) { 165 unsigned long flags; 166 l = (1<<0)|((mbytes/4)<<1); 167 local_irq_save(flags); 168 wbinvd(); 169 wrmsr(MSR_K6_WHCR, l, h); 170 local_irq_restore(flags); 171 pr_info("Enabling old style K6 write allocation for %d Mb\n", 172 mbytes); 173 } 174 return; 175 } 176 177 if ((c->x86_model == 8 && c->x86_stepping > 7) || 178 c->x86_model == 9 || c->x86_model == 13) { 179 /* The more serious chips .. */ 180 181 if (mbytes > 4092) 182 mbytes = 4092; 183 184 rdmsr(MSR_K6_WHCR, l, h); 185 if ((l&0xFFFF0000) == 0) { 186 unsigned long flags; 187 l = ((mbytes>>2)<<22)|(1<<16); 188 local_irq_save(flags); 189 wbinvd(); 190 wrmsr(MSR_K6_WHCR, l, h); 191 local_irq_restore(flags); 192 pr_info("Enabling new style K6 write allocation for %d Mb\n", 193 mbytes); 194 } 195 196 return; 197 } 198 199 if (c->x86_model == 10) { 200 /* AMD Geode LX is model 10 */ 201 /* placeholder for any needed mods */ 202 return; 203 } 204 #endif 205 } 206 207 static void init_amd_k7(struct cpuinfo_x86 *c) 208 { 209 #ifdef CONFIG_X86_32 210 u32 l, h; 211 212 /* 213 * Bit 15 of Athlon specific MSR 15, needs to be 0 214 * to enable SSE on Palomino/Morgan/Barton CPU's. 215 * If the BIOS didn't enable it already, enable it here. 216 */ 217 if (c->x86_model >= 6 && c->x86_model <= 10) { 218 if (!cpu_has(c, X86_FEATURE_XMM)) { 219 pr_info("Enabling disabled K7/SSE Support.\n"); 220 msr_clear_bit(MSR_K7_HWCR, 15); 221 set_cpu_cap(c, X86_FEATURE_XMM); 222 } 223 } 224 225 /* 226 * It's been determined by AMD that Athlons since model 8 stepping 1 227 * are more robust with CLK_CTL set to 200xxxxx instead of 600xxxxx 228 * As per AMD technical note 27212 0.2 229 */ 230 if ((c->x86_model == 8 && c->x86_stepping >= 1) || (c->x86_model > 8)) { 231 rdmsr(MSR_K7_CLK_CTL, l, h); 232 if ((l & 0xfff00000) != 0x20000000) { 233 pr_info("CPU: CLK_CTL MSR was %x. Reprogramming to %x\n", 234 l, ((l & 0x000fffff)|0x20000000)); 235 wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h); 236 } 237 } 238 239 /* calling is from identify_secondary_cpu() ? */ 240 if (!c->cpu_index) 241 return; 242 243 /* 244 * Certain Athlons might work (for various values of 'work') in SMP 245 * but they are not certified as MP capable. 246 */ 247 /* Athlon 660/661 is valid. */ 248 if ((c->x86_model == 6) && ((c->x86_stepping == 0) || 249 (c->x86_stepping == 1))) 250 return; 251 252 /* Duron 670 is valid */ 253 if ((c->x86_model == 7) && (c->x86_stepping == 0)) 254 return; 255 256 /* 257 * Athlon 662, Duron 671, and Athlon >model 7 have capability 258 * bit. It's worth noting that the A5 stepping (662) of some 259 * Athlon XP's have the MP bit set. 260 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for 261 * more. 262 */ 263 if (((c->x86_model == 6) && (c->x86_stepping >= 2)) || 264 ((c->x86_model == 7) && (c->x86_stepping >= 1)) || 265 (c->x86_model > 7)) 266 if (cpu_has(c, X86_FEATURE_MP)) 267 return; 268 269 /* If we get here, not a certified SMP capable AMD system. */ 270 271 /* 272 * Don't taint if we are running SMP kernel on a single non-MP 273 * approved Athlon 274 */ 275 WARN_ONCE(1, "WARNING: This combination of AMD" 276 " processors is not suitable for SMP.\n"); 277 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_NOW_UNRELIABLE); 278 #endif 279 } 280 281 #ifdef CONFIG_NUMA 282 /* 283 * To workaround broken NUMA config. Read the comment in 284 * srat_detect_node(). 285 */ 286 static int nearby_node(int apicid) 287 { 288 int i, node; 289 290 for (i = apicid - 1; i >= 0; i--) { 291 node = __apicid_to_node[i]; 292 if (node != NUMA_NO_NODE && node_online(node)) 293 return node; 294 } 295 for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) { 296 node = __apicid_to_node[i]; 297 if (node != NUMA_NO_NODE && node_online(node)) 298 return node; 299 } 300 return first_node(node_online_map); /* Shouldn't happen */ 301 } 302 #endif 303 304 static void srat_detect_node(struct cpuinfo_x86 *c) 305 { 306 #ifdef CONFIG_NUMA 307 int cpu = smp_processor_id(); 308 int node; 309 unsigned apicid = c->topo.apicid; 310 311 node = numa_cpu_node(cpu); 312 if (node == NUMA_NO_NODE) 313 node = per_cpu_llc_id(cpu); 314 315 /* 316 * On multi-fabric platform (e.g. Numascale NumaChip) a 317 * platform-specific handler needs to be called to fixup some 318 * IDs of the CPU. 319 */ 320 if (x86_cpuinit.fixup_cpu_id) 321 x86_cpuinit.fixup_cpu_id(c, node); 322 323 if (!node_online(node)) { 324 /* 325 * Two possibilities here: 326 * 327 * - The CPU is missing memory and no node was created. In 328 * that case try picking one from a nearby CPU. 329 * 330 * - The APIC IDs differ from the HyperTransport node IDs 331 * which the K8 northbridge parsing fills in. Assume 332 * they are all increased by a constant offset, but in 333 * the same order as the HT nodeids. If that doesn't 334 * result in a usable node fall back to the path for the 335 * previous case. 336 * 337 * This workaround operates directly on the mapping between 338 * APIC ID and NUMA node, assuming certain relationship 339 * between APIC ID, HT node ID and NUMA topology. As going 340 * through CPU mapping may alter the outcome, directly 341 * access __apicid_to_node[]. 342 */ 343 int ht_nodeid = c->topo.initial_apicid; 344 345 if (__apicid_to_node[ht_nodeid] != NUMA_NO_NODE) 346 node = __apicid_to_node[ht_nodeid]; 347 /* Pick a nearby node */ 348 if (!node_online(node)) 349 node = nearby_node(apicid); 350 } 351 numa_set_node(cpu, node); 352 #endif 353 } 354 355 static void bsp_determine_snp(struct cpuinfo_x86 *c) 356 { 357 #ifdef CONFIG_ARCH_HAS_CC_PLATFORM 358 cc_vendor = CC_VENDOR_AMD; 359 360 if (cpu_has(c, X86_FEATURE_SEV_SNP)) { 361 /* 362 * RMP table entry format is not architectural and is defined by the 363 * per-processor PPR. Restrict SNP support on the known CPU models 364 * for which the RMP table entry format is currently defined or for 365 * processors which support the architecturally defined RMPREAD 366 * instruction. 367 */ 368 if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && 369 (cpu_feature_enabled(X86_FEATURE_ZEN3) || 370 cpu_feature_enabled(X86_FEATURE_ZEN4) || 371 cpu_feature_enabled(X86_FEATURE_RMPREAD)) && 372 snp_probe_rmptable_info()) { 373 cc_platform_set(CC_ATTR_HOST_SEV_SNP); 374 } else { 375 setup_clear_cpu_cap(X86_FEATURE_SEV_SNP); 376 cc_platform_clear(CC_ATTR_HOST_SEV_SNP); 377 } 378 } 379 #endif 380 } 381 382 #define ZEN_MODEL_STEP_UCODE(fam, model, step, ucode) \ 383 X86_MATCH_VFM_STEPS(VFM_MAKE(X86_VENDOR_AMD, fam, model), \ 384 step, step, ucode) 385 386 static const struct x86_cpu_id amd_tsa_microcode[] = { 387 ZEN_MODEL_STEP_UCODE(0x19, 0x01, 0x1, 0x0a0011d7), 388 ZEN_MODEL_STEP_UCODE(0x19, 0x01, 0x2, 0x0a00123b), 389 ZEN_MODEL_STEP_UCODE(0x19, 0x08, 0x2, 0x0a00820d), 390 ZEN_MODEL_STEP_UCODE(0x19, 0x11, 0x1, 0x0a10114c), 391 ZEN_MODEL_STEP_UCODE(0x19, 0x11, 0x2, 0x0a10124c), 392 ZEN_MODEL_STEP_UCODE(0x19, 0x18, 0x1, 0x0a108109), 393 ZEN_MODEL_STEP_UCODE(0x19, 0x21, 0x0, 0x0a20102e), 394 ZEN_MODEL_STEP_UCODE(0x19, 0x21, 0x2, 0x0a201211), 395 ZEN_MODEL_STEP_UCODE(0x19, 0x44, 0x1, 0x0a404108), 396 ZEN_MODEL_STEP_UCODE(0x19, 0x50, 0x0, 0x0a500012), 397 ZEN_MODEL_STEP_UCODE(0x19, 0x61, 0x2, 0x0a60120a), 398 ZEN_MODEL_STEP_UCODE(0x19, 0x74, 0x1, 0x0a704108), 399 ZEN_MODEL_STEP_UCODE(0x19, 0x75, 0x2, 0x0a705208), 400 ZEN_MODEL_STEP_UCODE(0x19, 0x78, 0x0, 0x0a708008), 401 ZEN_MODEL_STEP_UCODE(0x19, 0x7c, 0x0, 0x0a70c008), 402 ZEN_MODEL_STEP_UCODE(0x19, 0xa0, 0x2, 0x0aa00216), 403 {}, 404 }; 405 406 static void tsa_init(struct cpuinfo_x86 *c) 407 { 408 if (cpu_has(c, X86_FEATURE_HYPERVISOR)) 409 return; 410 411 if (cpu_has(c, X86_FEATURE_ZEN3) || 412 cpu_has(c, X86_FEATURE_ZEN4)) { 413 if (x86_match_min_microcode_rev(amd_tsa_microcode)) 414 setup_force_cpu_cap(X86_FEATURE_VERW_CLEAR); 415 else 416 pr_debug("%s: current revision: 0x%x\n", __func__, c->microcode); 417 } else { 418 setup_force_cpu_cap(X86_FEATURE_TSA_SQ_NO); 419 setup_force_cpu_cap(X86_FEATURE_TSA_L1_NO); 420 } 421 } 422 423 static void bsp_init_amd(struct cpuinfo_x86 *c) 424 { 425 if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) { 426 427 if (c->x86 > 0x10 || 428 (c->x86 == 0x10 && c->x86_model >= 0x2)) { 429 u64 val; 430 431 rdmsrq(MSR_K7_HWCR, val); 432 if (!(val & BIT(24))) 433 pr_warn(FW_BUG "TSC doesn't count with P0 frequency!\n"); 434 } 435 } 436 437 if (c->x86 == 0x15) { 438 unsigned long upperbit; 439 u32 cpuid, assoc; 440 441 cpuid = cpuid_edx(0x80000005); 442 assoc = cpuid >> 16 & 0xff; 443 upperbit = ((cpuid >> 24) << 10) / assoc; 444 445 va_align.mask = (upperbit - 1) & PAGE_MASK; 446 va_align.flags = ALIGN_VA_32 | ALIGN_VA_64; 447 448 /* A random value per boot for bit slice [12:upper_bit) */ 449 va_align.bits = get_random_u32() & va_align.mask; 450 } 451 452 if (cpu_has(c, X86_FEATURE_MWAITX)) 453 use_mwaitx_delay(); 454 455 if (!boot_cpu_has(X86_FEATURE_AMD_SSBD) && 456 !boot_cpu_has(X86_FEATURE_VIRT_SSBD) && 457 c->x86 >= 0x15 && c->x86 <= 0x17) { 458 unsigned int bit; 459 460 switch (c->x86) { 461 case 0x15: bit = 54; break; 462 case 0x16: bit = 33; break; 463 case 0x17: bit = 10; break; 464 default: return; 465 } 466 /* 467 * Try to cache the base value so further operations can 468 * avoid RMW. If that faults, do not enable SSBD. 469 */ 470 if (!rdmsrq_safe(MSR_AMD64_LS_CFG, &x86_amd_ls_cfg_base)) { 471 setup_force_cpu_cap(X86_FEATURE_LS_CFG_SSBD); 472 setup_force_cpu_cap(X86_FEATURE_SSBD); 473 x86_amd_ls_cfg_ssbd_mask = 1ULL << bit; 474 } 475 } 476 477 resctrl_cpu_detect(c); 478 479 /* Figure out Zen generations: */ 480 switch (c->x86) { 481 case 0x17: 482 switch (c->x86_model) { 483 case 0x00 ... 0x2f: 484 case 0x50 ... 0x5f: 485 setup_force_cpu_cap(X86_FEATURE_ZEN1); 486 break; 487 case 0x30 ... 0x4f: 488 case 0x60 ... 0x7f: 489 case 0x90 ... 0x91: 490 case 0xa0 ... 0xaf: 491 setup_force_cpu_cap(X86_FEATURE_ZEN2); 492 break; 493 default: 494 goto warn; 495 } 496 break; 497 498 case 0x19: 499 switch (c->x86_model) { 500 case 0x00 ... 0x0f: 501 case 0x20 ... 0x5f: 502 setup_force_cpu_cap(X86_FEATURE_ZEN3); 503 break; 504 case 0x10 ... 0x1f: 505 case 0x60 ... 0xaf: 506 setup_force_cpu_cap(X86_FEATURE_ZEN4); 507 break; 508 default: 509 goto warn; 510 } 511 break; 512 513 case 0x1a: 514 switch (c->x86_model) { 515 case 0x00 ... 0x2f: 516 case 0x40 ... 0x4f: 517 case 0x60 ... 0x7f: 518 setup_force_cpu_cap(X86_FEATURE_ZEN5); 519 break; 520 case 0x50 ... 0x5f: 521 case 0x80 ... 0xaf: 522 case 0xc0 ... 0xef: 523 setup_force_cpu_cap(X86_FEATURE_ZEN6); 524 break; 525 default: 526 goto warn; 527 } 528 break; 529 530 default: 531 break; 532 } 533 534 bsp_determine_snp(c); 535 tsa_init(c); 536 537 if (cpu_has(c, X86_FEATURE_GP_ON_USER_CPUID)) 538 setup_force_cpu_cap(X86_FEATURE_CPUID_FAULT); 539 540 return; 541 542 warn: 543 WARN_ONCE(1, "Family 0x%x, model: 0x%x??\n", c->x86, c->x86_model); 544 } 545 546 static void early_detect_mem_encrypt(struct cpuinfo_x86 *c) 547 { 548 u64 msr; 549 550 /* 551 * Mark using WBINVD is needed during kexec on processors that 552 * support SME. This provides support for performing a successful 553 * kexec when going from SME inactive to SME active (or vice-versa). 554 * 555 * The cache must be cleared so that if there are entries with the 556 * same physical address, both with and without the encryption bit, 557 * they don't race each other when flushed and potentially end up 558 * with the wrong entry being committed to memory. 559 * 560 * Test the CPUID bit directly because with mem_encrypt=off the 561 * BSP will clear the X86_FEATURE_SME bit and the APs will not 562 * see it set after that. 563 */ 564 if (c->extended_cpuid_level >= 0x8000001f && (cpuid_eax(0x8000001f) & BIT(0))) 565 __this_cpu_write(cache_state_incoherent, true); 566 567 /* 568 * BIOS support is required for SME and SEV. 569 * For SME: If BIOS has enabled SME then adjust x86_phys_bits by 570 * the SME physical address space reduction value. 571 * If BIOS has not enabled SME then don't advertise the 572 * SME feature (set in scattered.c). 573 * If the kernel has not enabled SME via any means then 574 * don't advertise the SME feature. 575 * For SEV: If BIOS has not enabled SEV then don't advertise SEV and 576 * any additional functionality based on it. 577 * 578 * In all cases, since support for SME and SEV requires long mode, 579 * don't advertise the feature under CONFIG_X86_32. 580 */ 581 if (cpu_has(c, X86_FEATURE_SME) || cpu_has(c, X86_FEATURE_SEV)) { 582 /* Check if memory encryption is enabled */ 583 rdmsrq(MSR_AMD64_SYSCFG, msr); 584 if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) 585 goto clear_all; 586 587 /* 588 * Always adjust physical address bits. Even though this 589 * will be a value above 32-bits this is still done for 590 * CONFIG_X86_32 so that accurate values are reported. 591 */ 592 c->x86_phys_bits -= (cpuid_ebx(0x8000001f) >> 6) & 0x3f; 593 594 if (IS_ENABLED(CONFIG_X86_32)) 595 goto clear_all; 596 597 if (!sme_me_mask) 598 setup_clear_cpu_cap(X86_FEATURE_SME); 599 600 rdmsrq(MSR_K7_HWCR, msr); 601 if (!(msr & MSR_K7_HWCR_SMMLOCK)) 602 goto clear_sev; 603 604 return; 605 606 clear_all: 607 setup_clear_cpu_cap(X86_FEATURE_SME); 608 clear_sev: 609 setup_clear_cpu_cap(X86_FEATURE_SEV); 610 setup_clear_cpu_cap(X86_FEATURE_SEV_ES); 611 setup_clear_cpu_cap(X86_FEATURE_SEV_SNP); 612 } 613 } 614 615 static void early_init_amd(struct cpuinfo_x86 *c) 616 { 617 u32 dummy; 618 619 if (c->x86 >= 0xf) 620 set_cpu_cap(c, X86_FEATURE_K8); 621 622 rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy); 623 624 /* 625 * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate 626 * with P/T states and does not stop in deep C-states 627 */ 628 if (c->x86_power & (1 << 8)) { 629 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); 630 set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC); 631 } 632 633 /* Bit 12 of 8000_0007 edx is accumulated power mechanism. */ 634 if (c->x86_power & BIT(12)) 635 set_cpu_cap(c, X86_FEATURE_ACC_POWER); 636 637 /* Bit 14 indicates the Runtime Average Power Limit interface. */ 638 if (c->x86_power & BIT(14)) 639 set_cpu_cap(c, X86_FEATURE_RAPL); 640 641 #ifdef CONFIG_X86_64 642 set_cpu_cap(c, X86_FEATURE_SYSCALL32); 643 #else 644 /* Set MTRR capability flag if appropriate */ 645 if (c->x86 == 5) 646 if (c->x86_model == 13 || c->x86_model == 9 || 647 (c->x86_model == 8 && c->x86_stepping >= 8)) 648 set_cpu_cap(c, X86_FEATURE_K6_MTRR); 649 #endif 650 #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI) 651 /* 652 * ApicID can always be treated as an 8-bit value for AMD APIC versions 653 * >= 0x10, but even old K8s came out of reset with version 0x10. So, we 654 * can safely set X86_FEATURE_EXTD_APICID unconditionally for families 655 * after 16h. 656 */ 657 if (boot_cpu_has(X86_FEATURE_APIC)) { 658 if (c->x86 > 0x16) 659 set_cpu_cap(c, X86_FEATURE_EXTD_APICID); 660 else if (c->x86 >= 0xf) { 661 /* check CPU config space for extended APIC ID */ 662 unsigned int val; 663 664 val = read_pci_config(0, 24, 0, 0x68); 665 if ((val >> 17 & 0x3) == 0x3) 666 set_cpu_cap(c, X86_FEATURE_EXTD_APICID); 667 } 668 } 669 #endif 670 671 /* 672 * This is only needed to tell the kernel whether to use VMCALL 673 * and VMMCALL. VMMCALL is never executed except under virt, so 674 * we can set it unconditionally. 675 */ 676 set_cpu_cap(c, X86_FEATURE_VMMCALL); 677 678 /* F16h erratum 793, CVE-2013-6885 */ 679 if (c->x86 == 0x16 && c->x86_model <= 0xf) 680 msr_set_bit(MSR_AMD64_LS_CFG, 15); 681 682 early_detect_mem_encrypt(c); 683 684 if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && !cpu_has(c, X86_FEATURE_IBPB_BRTYPE)) { 685 if (c->x86 == 0x17 && boot_cpu_has(X86_FEATURE_AMD_IBPB)) 686 setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE); 687 else if (c->x86 >= 0x19 && !wrmsrq_safe(MSR_IA32_PRED_CMD, PRED_CMD_SBPB)) { 688 setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE); 689 setup_force_cpu_cap(X86_FEATURE_SBPB); 690 } 691 } 692 } 693 694 static void init_amd_k8(struct cpuinfo_x86 *c) 695 { 696 u32 level; 697 u64 value; 698 699 /* On C+ stepping K8 rep microcode works well for copy/memset */ 700 level = cpuid_eax(1); 701 if ((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58) 702 set_cpu_cap(c, X86_FEATURE_REP_GOOD); 703 704 /* 705 * Some BIOSes incorrectly force this feature, but only K8 revision D 706 * (model = 0x14) and later actually support it. 707 * (AMD Erratum #110, docId: 25759). 708 */ 709 if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM) && !cpu_has(c, X86_FEATURE_HYPERVISOR)) { 710 clear_cpu_cap(c, X86_FEATURE_LAHF_LM); 711 if (!rdmsrq_amd_safe(0xc001100d, &value)) { 712 value &= ~BIT_64(32); 713 wrmsrq_amd_safe(0xc001100d, value); 714 } 715 } 716 717 if (!c->x86_model_id[0]) 718 strscpy(c->x86_model_id, "Hammer"); 719 720 #ifdef CONFIG_SMP 721 /* 722 * Disable TLB flush filter by setting HWCR.FFDIS on K8 723 * bit 6 of msr C001_0015 724 * 725 * Errata 63 for SH-B3 steppings 726 * Errata 122 for all steppings (F+ have it disabled by default) 727 */ 728 msr_set_bit(MSR_K7_HWCR, 6); 729 #endif 730 set_cpu_bug(c, X86_BUG_SWAPGS_FENCE); 731 732 /* 733 * Check models and steppings affected by erratum 400. This is 734 * used to select the proper idle routine and to enable the 735 * check whether the machine is affected in arch_post_acpi_subsys_init() 736 * which sets the X86_BUG_AMD_APIC_C1E bug depending on the MSR check. 737 */ 738 if (c->x86_model > 0x41 || 739 (c->x86_model == 0x41 && c->x86_stepping >= 0x2)) 740 setup_force_cpu_bug(X86_BUG_AMD_E400); 741 } 742 743 static void init_amd_gh(struct cpuinfo_x86 *c) 744 { 745 #ifdef CONFIG_MMCONF_FAM10H 746 /* do this for boot cpu */ 747 if (c == &boot_cpu_data) 748 check_enable_amd_mmconf_dmi(); 749 750 fam10h_check_enable_mmcfg(); 751 #endif 752 753 /* 754 * Disable GART TLB Walk Errors on Fam10h. We do this here because this 755 * is always needed when GART is enabled, even in a kernel which has no 756 * MCE support built in. BIOS should disable GartTlbWlk Errors already. 757 * If it doesn't, we do it here as suggested by the BKDG. 758 * 759 * Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=33012 760 */ 761 msr_set_bit(MSR_AMD64_MCx_MASK(4), 10); 762 763 /* 764 * On family 10h BIOS may not have properly enabled WC+ support, causing 765 * it to be converted to CD memtype. This may result in performance 766 * degradation for certain nested-paging guests. Prevent this conversion 767 * by clearing bit 24 in MSR_AMD64_BU_CFG2. 768 * 769 * NOTE: we want to use the _safe accessors so as not to #GP kvm 770 * guests on older kvm hosts. 771 */ 772 msr_clear_bit(MSR_AMD64_BU_CFG2, 24); 773 774 set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH); 775 776 /* 777 * Check models and steppings affected by erratum 400. This is 778 * used to select the proper idle routine and to enable the 779 * check whether the machine is affected in arch_post_acpi_subsys_init() 780 * which sets the X86_BUG_AMD_APIC_C1E bug depending on the MSR check. 781 */ 782 if (c->x86_model > 0x2 || 783 (c->x86_model == 0x2 && c->x86_stepping >= 0x1)) 784 setup_force_cpu_bug(X86_BUG_AMD_E400); 785 } 786 787 static void init_amd_ln(struct cpuinfo_x86 *c) 788 { 789 /* 790 * Apply erratum 665 fix unconditionally so machines without a BIOS 791 * fix work. 792 */ 793 msr_set_bit(MSR_AMD64_DE_CFG, 31); 794 } 795 796 static bool rdrand_force; 797 798 static int __init rdrand_cmdline(char *str) 799 { 800 if (!str) 801 return -EINVAL; 802 803 if (!strcmp(str, "force")) 804 rdrand_force = true; 805 else 806 return -EINVAL; 807 808 return 0; 809 } 810 early_param("rdrand", rdrand_cmdline); 811 812 static void clear_rdrand_cpuid_bit(struct cpuinfo_x86 *c) 813 { 814 /* 815 * Saving of the MSR used to hide the RDRAND support during 816 * suspend/resume is done by arch/x86/power/cpu.c, which is 817 * dependent on CONFIG_PM_SLEEP. 818 */ 819 if (!IS_ENABLED(CONFIG_PM_SLEEP)) 820 return; 821 822 /* 823 * The self-test can clear X86_FEATURE_RDRAND, so check for 824 * RDRAND support using the CPUID function directly. 825 */ 826 if (!(cpuid_ecx(1) & BIT(30)) || rdrand_force) 827 return; 828 829 msr_clear_bit(MSR_AMD64_CPUID_FN_1, 62); 830 831 /* 832 * Verify that the CPUID change has occurred in case the kernel is 833 * running virtualized and the hypervisor doesn't support the MSR. 834 */ 835 if (cpuid_ecx(1) & BIT(30)) { 836 pr_info_once("BIOS may not properly restore RDRAND after suspend, but hypervisor does not support hiding RDRAND via CPUID.\n"); 837 return; 838 } 839 840 clear_cpu_cap(c, X86_FEATURE_RDRAND); 841 pr_info_once("BIOS may not properly restore RDRAND after suspend, hiding RDRAND via CPUID. Use rdrand=force to reenable.\n"); 842 } 843 844 static void init_amd_jg(struct cpuinfo_x86 *c) 845 { 846 /* 847 * Some BIOS implementations do not restore proper RDRAND support 848 * across suspend and resume. Check on whether to hide the RDRAND 849 * instruction support via CPUID. 850 */ 851 clear_rdrand_cpuid_bit(c); 852 } 853 854 static void init_amd_bd(struct cpuinfo_x86 *c) 855 { 856 u64 value; 857 858 /* 859 * The way access filter has a performance penalty on some workloads. 860 * Disable it on the affected CPUs. 861 */ 862 if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) { 863 if (!rdmsrq_safe(MSR_F15H_IC_CFG, &value) && !(value & 0x1E)) { 864 value |= 0x1E; 865 wrmsrq_safe(MSR_F15H_IC_CFG, value); 866 } 867 } 868 869 /* 870 * Some BIOS implementations do not restore proper RDRAND support 871 * across suspend and resume. Check on whether to hide the RDRAND 872 * instruction support via CPUID. 873 */ 874 clear_rdrand_cpuid_bit(c); 875 } 876 877 static const struct x86_cpu_id erratum_1386_microcode[] = { 878 ZEN_MODEL_STEP_UCODE(0x17, 0x01, 0x2, 0x0800126e), 879 ZEN_MODEL_STEP_UCODE(0x17, 0x31, 0x0, 0x08301052), 880 {} 881 }; 882 883 static void fix_erratum_1386(struct cpuinfo_x86 *c) 884 { 885 /* 886 * Work around Erratum 1386. The XSAVES instruction malfunctions in 887 * certain circumstances on Zen1/2 uarch, and not all parts have had 888 * updated microcode at the time of writing (March 2023). 889 * 890 * Affected parts all have no supervisor XSAVE states, meaning that 891 * the XSAVEC instruction (which works fine) is equivalent. 892 * 893 * Clear the feature flag only on microcode revisions which 894 * don't have the fix. 895 */ 896 if (x86_match_min_microcode_rev(erratum_1386_microcode)) 897 return; 898 899 clear_cpu_cap(c, X86_FEATURE_XSAVES); 900 } 901 902 void init_spectral_chicken(struct cpuinfo_x86 *c) 903 { 904 #ifdef CONFIG_MITIGATION_UNRET_ENTRY 905 /* 906 * On Zen2 we offer this chicken (bit) on the altar of Speculation. 907 * 908 * This suppresses speculation from the middle of a basic block, i.e. it 909 * suppresses non-branch predictions. 910 */ 911 if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) 912 msr_set_bit(MSR_ZEN2_SPECTRAL_CHICKEN, MSR_ZEN2_SPECTRAL_CHICKEN_BIT); 913 #endif 914 } 915 916 static void init_amd_zen_common(void) 917 { 918 setup_force_cpu_cap(X86_FEATURE_ZEN); 919 #ifdef CONFIG_NUMA 920 node_reclaim_distance = 32; 921 #endif 922 } 923 924 static void init_amd_zen1(struct cpuinfo_x86 *c) 925 { 926 fix_erratum_1386(c); 927 928 /* Fix up CPUID bits, but only if not virtualised. */ 929 if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) { 930 931 /* Erratum 1076: CPB feature bit not being set in CPUID. */ 932 if (!cpu_has(c, X86_FEATURE_CPB)) 933 set_cpu_cap(c, X86_FEATURE_CPB); 934 } 935 936 pr_notice_once("AMD Zen1 DIV0 bug detected. Disable SMT for full protection.\n"); 937 setup_force_cpu_bug(X86_BUG_DIV0); 938 939 /* 940 * Turn off the Instructions Retired free counter on machines that are 941 * susceptible to erratum #1054 "Instructions Retired Performance 942 * Counter May Be Inaccurate". 943 */ 944 if (c->x86_model < 0x30) { 945 msr_clear_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT); 946 clear_cpu_cap(c, X86_FEATURE_IRPERF); 947 } 948 949 pr_notice_once("AMD Zen1 FPDSS bug detected, enabling mitigation.\n"); 950 msr_set_bit(MSR_AMD64_FP_CFG, MSR_AMD64_FP_CFG_ZEN1_DENORM_FIX_BIT); 951 } 952 953 static const struct x86_cpu_id amd_zenbleed_microcode[] = { 954 ZEN_MODEL_STEP_UCODE(0x17, 0x31, 0x0, 0x0830107b), 955 ZEN_MODEL_STEP_UCODE(0x17, 0x60, 0x1, 0x0860010c), 956 ZEN_MODEL_STEP_UCODE(0x17, 0x68, 0x1, 0x08608107), 957 ZEN_MODEL_STEP_UCODE(0x17, 0x71, 0x0, 0x08701033), 958 ZEN_MODEL_STEP_UCODE(0x17, 0xa0, 0x0, 0x08a00009), 959 {} 960 }; 961 962 static void zen2_zenbleed_check(struct cpuinfo_x86 *c) 963 { 964 if (cpu_has(c, X86_FEATURE_HYPERVISOR)) 965 return; 966 967 if (!cpu_has(c, X86_FEATURE_AVX)) 968 return; 969 970 if (!x86_match_min_microcode_rev(amd_zenbleed_microcode)) { 971 pr_notice_once("Zenbleed: please update your microcode for the most optimal fix\n"); 972 msr_set_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT); 973 } else { 974 msr_clear_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT); 975 } 976 } 977 978 static void init_amd_zen2(struct cpuinfo_x86 *c) 979 { 980 init_spectral_chicken(c); 981 fix_erratum_1386(c); 982 zen2_zenbleed_check(c); 983 984 /* Disable RDSEED on AMD Cyan Skillfish because of an error. */ 985 if (c->x86_model == 0x47 && c->x86_stepping == 0x0) { 986 clear_cpu_cap(c, X86_FEATURE_RDSEED); 987 msr_clear_bit(MSR_AMD64_CPUID_FN_7, 18); 988 pr_emerg("RDSEED is not reliable on this platform; disabling.\n"); 989 } 990 991 /* Correct misconfigured CPUID on some clients. */ 992 clear_cpu_cap(c, X86_FEATURE_INVLPGB); 993 994 if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) 995 msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN2_BP_CFG_BUG_FIX_BIT); 996 } 997 998 static void init_amd_zen3(struct cpuinfo_x86 *c) 999 { 1000 if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) { 1001 /* 1002 * Zen3 (Fam19 model < 0x10) parts are not susceptible to 1003 * Branch Type Confusion, but predate the allocation of the 1004 * BTC_NO bit. 1005 */ 1006 if (!cpu_has(c, X86_FEATURE_BTC_NO)) 1007 set_cpu_cap(c, X86_FEATURE_BTC_NO); 1008 } 1009 } 1010 1011 static void init_amd_zen4(struct cpuinfo_x86 *c) 1012 { 1013 if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) 1014 msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT); 1015 1016 /* 1017 * These Zen4 SoCs advertise support for virtualized VMLOAD/VMSAVE 1018 * in some BIOS versions but they can lead to random host reboots. 1019 */ 1020 switch (c->x86_model) { 1021 case 0x18 ... 0x1f: 1022 case 0x60 ... 0x7f: 1023 clear_cpu_cap(c, X86_FEATURE_V_VMSAVE_VMLOAD); 1024 break; 1025 } 1026 } 1027 1028 static const struct x86_cpu_id zen5_rdseed_microcode[] = { 1029 ZEN_MODEL_STEP_UCODE(0x1a, 0x02, 0x1, 0x0b00215a), 1030 ZEN_MODEL_STEP_UCODE(0x1a, 0x08, 0x1, 0x0b008121), 1031 ZEN_MODEL_STEP_UCODE(0x1a, 0x11, 0x0, 0x0b101054), 1032 ZEN_MODEL_STEP_UCODE(0x1a, 0x24, 0x0, 0x0b204037), 1033 ZEN_MODEL_STEP_UCODE(0x1a, 0x44, 0x0, 0x0b404035), 1034 ZEN_MODEL_STEP_UCODE(0x1a, 0x44, 0x1, 0x0b404108), 1035 ZEN_MODEL_STEP_UCODE(0x1a, 0x60, 0x0, 0x0b600037), 1036 ZEN_MODEL_STEP_UCODE(0x1a, 0x68, 0x0, 0x0b608038), 1037 ZEN_MODEL_STEP_UCODE(0x1a, 0x70, 0x0, 0x0b700037), 1038 {}, 1039 }; 1040 1041 static void init_amd_zen5(struct cpuinfo_x86 *c) 1042 { 1043 if (!x86_match_min_microcode_rev(zen5_rdseed_microcode)) { 1044 clear_cpu_cap(c, X86_FEATURE_RDSEED); 1045 msr_clear_bit(MSR_AMD64_CPUID_FN_7, 18); 1046 pr_emerg_once("RDSEED32 is broken. Disabling the corresponding CPUID bit.\n"); 1047 } 1048 } 1049 1050 static void init_amd(struct cpuinfo_x86 *c) 1051 { 1052 u64 vm_cr; 1053 1054 early_init_amd(c); 1055 1056 if (c->x86 >= 0x10) 1057 set_cpu_cap(c, X86_FEATURE_REP_GOOD); 1058 1059 /* AMD FSRM also implies FSRS */ 1060 if (cpu_has(c, X86_FEATURE_FSRM)) 1061 set_cpu_cap(c, X86_FEATURE_FSRS); 1062 1063 /* K6s reports MCEs but don't actually have all the MSRs */ 1064 if (c->x86 < 6) 1065 clear_cpu_cap(c, X86_FEATURE_MCE); 1066 1067 switch (c->x86) { 1068 case 4: init_amd_k5(c); break; 1069 case 5: init_amd_k6(c); break; 1070 case 6: init_amd_k7(c); break; 1071 case 0xf: init_amd_k8(c); break; 1072 case 0x10: init_amd_gh(c); break; 1073 case 0x12: init_amd_ln(c); break; 1074 case 0x15: init_amd_bd(c); break; 1075 case 0x16: init_amd_jg(c); break; 1076 } 1077 1078 /* 1079 * Save up on some future enablement work and do common Zen 1080 * settings. 1081 */ 1082 if (c->x86 >= 0x17) 1083 init_amd_zen_common(); 1084 1085 if (boot_cpu_has(X86_FEATURE_ZEN1)) 1086 init_amd_zen1(c); 1087 else if (boot_cpu_has(X86_FEATURE_ZEN2)) 1088 init_amd_zen2(c); 1089 else if (boot_cpu_has(X86_FEATURE_ZEN3)) 1090 init_amd_zen3(c); 1091 else if (boot_cpu_has(X86_FEATURE_ZEN4)) 1092 init_amd_zen4(c); 1093 else if (boot_cpu_has(X86_FEATURE_ZEN5)) 1094 init_amd_zen5(c); 1095 1096 /* 1097 * Enable workaround for FXSAVE leak on CPUs 1098 * without a XSaveErPtr feature 1099 */ 1100 if ((c->x86 >= 6) && (!cpu_has(c, X86_FEATURE_XSAVEERPTR))) 1101 set_cpu_bug(c, X86_BUG_FXSAVE_LEAK); 1102 1103 cpu_detect_cache_sizes(c); 1104 1105 srat_detect_node(c); 1106 1107 init_amd_cacheinfo(c); 1108 1109 if (cpu_has(c, X86_FEATURE_SVM)) { 1110 rdmsrq(MSR_VM_CR, vm_cr); 1111 if (vm_cr & SVM_VM_CR_SVM_DIS_MASK) { 1112 pr_notice_once("SVM disabled (by BIOS) in MSR_VM_CR\n"); 1113 clear_cpu_cap(c, X86_FEATURE_SVM); 1114 } 1115 } 1116 1117 if (!cpu_has(c, X86_FEATURE_LFENCE_RDTSC) && cpu_has(c, X86_FEATURE_XMM2)) { 1118 /* 1119 * Use LFENCE for execution serialization. On families which 1120 * don't have that MSR, LFENCE is already serializing. 1121 * msr_set_bit() uses the safe accessors, too, even if the MSR 1122 * is not present. 1123 */ 1124 msr_set_bit(MSR_AMD64_DE_CFG, 1125 MSR_AMD64_DE_CFG_LFENCE_SERIALIZE_BIT); 1126 1127 /* A serializing LFENCE stops RDTSC speculation */ 1128 set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC); 1129 } 1130 1131 /* 1132 * Family 0x12 and above processors have APIC timer 1133 * running in deep C states. 1134 */ 1135 if (c->x86 > 0x11) 1136 set_cpu_cap(c, X86_FEATURE_ARAT); 1137 1138 /* 3DNow or LM implies PREFETCHW */ 1139 if (!cpu_has(c, X86_FEATURE_3DNOWPREFETCH)) 1140 if (cpu_has(c, X86_FEATURE_3DNOW) || cpu_has(c, X86_FEATURE_LM)) 1141 set_cpu_cap(c, X86_FEATURE_3DNOWPREFETCH); 1142 1143 /* AMD CPUs don't reset SS attributes on SYSRET, Xen does. */ 1144 if (!cpu_feature_enabled(X86_FEATURE_XENPV)) 1145 set_cpu_bug(c, X86_BUG_SYSRET_SS_ATTRS); 1146 1147 /* Enable the Instructions Retired free counter */ 1148 if (cpu_has(c, X86_FEATURE_IRPERF)) 1149 msr_set_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT); 1150 1151 check_null_seg_clears_base(c); 1152 1153 /* 1154 * Make sure EFER[AIBRSE - Automatic IBRS Enable] is set. The APs are brought up 1155 * using the trampoline code and as part of it, MSR_EFER gets prepared there in 1156 * order to be replicated onto them. Regardless, set it here again, if not set, 1157 * to protect against any future refactoring/code reorganization which might 1158 * miss setting this important bit. 1159 */ 1160 if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 1161 cpu_has(c, X86_FEATURE_AUTOIBRS)) 1162 WARN_ON_ONCE(msr_set_bit(MSR_EFER, _EFER_AUTOIBRS) < 0); 1163 1164 /* AMD CPUs don't need fencing after x2APIC/TSC_DEADLINE MSR writes. */ 1165 clear_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE); 1166 1167 /* Enable Translation Cache Extension */ 1168 if (cpu_has(c, X86_FEATURE_TCE)) 1169 msr_set_bit(MSR_EFER, _EFER_TCE); 1170 } 1171 1172 #ifdef CONFIG_X86_32 1173 static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size) 1174 { 1175 /* AMD errata T13 (order #21922) */ 1176 if (c->x86 == 6) { 1177 /* Duron Rev A0 */ 1178 if (c->x86_model == 3 && c->x86_stepping == 0) 1179 size = 64; 1180 /* Tbird rev A1/A2 */ 1181 if (c->x86_model == 4 && 1182 (c->x86_stepping == 0 || c->x86_stepping == 1)) 1183 size = 256; 1184 } 1185 return size; 1186 } 1187 #endif 1188 1189 static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c) 1190 { 1191 u32 ebx, eax, ecx, edx; 1192 u16 mask = 0xfff; 1193 1194 if (c->x86 < 0xf) 1195 return; 1196 1197 if (c->extended_cpuid_level < 0x80000006) 1198 return; 1199 1200 cpuid(0x80000006, &eax, &ebx, &ecx, &edx); 1201 1202 tlb_lld_4k = (ebx >> 16) & mask; 1203 tlb_lli_4k = ebx & mask; 1204 1205 /* 1206 * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB 1207 * characteristics from the CPUID function 0x80000005 instead. 1208 */ 1209 if (c->x86 == 0xf) { 1210 cpuid(0x80000005, &eax, &ebx, &ecx, &edx); 1211 mask = 0xff; 1212 } 1213 1214 /* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */ 1215 if (!((eax >> 16) & mask)) 1216 tlb_lld_2m = (cpuid_eax(0x80000005) >> 16) & 0xff; 1217 else 1218 tlb_lld_2m = (eax >> 16) & mask; 1219 1220 /* a 4M entry uses two 2M entries */ 1221 tlb_lld_4m = tlb_lld_2m >> 1; 1222 1223 /* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */ 1224 if (!(eax & mask)) { 1225 /* Erratum 658 */ 1226 if (c->x86 == 0x15 && c->x86_model <= 0x1f) { 1227 tlb_lli_2m = 1024; 1228 } else { 1229 cpuid(0x80000005, &eax, &ebx, &ecx, &edx); 1230 tlb_lli_2m = eax & 0xff; 1231 } 1232 } else 1233 tlb_lli_2m = eax & mask; 1234 1235 tlb_lli_4m = tlb_lli_2m >> 1; 1236 1237 /* Max number of pages INVLPGB can invalidate in one shot */ 1238 if (cpu_has(c, X86_FEATURE_INVLPGB)) 1239 invlpgb_count_max = (cpuid_edx(0x80000008) & 0xffff) + 1; 1240 } 1241 1242 static const struct cpu_dev amd_cpu_dev = { 1243 .c_vendor = "AMD", 1244 .c_ident = { "AuthenticAMD" }, 1245 #ifdef CONFIG_X86_32 1246 .legacy_models = { 1247 { .family = 4, .model_names = 1248 { 1249 [3] = "486 DX/2", 1250 [7] = "486 DX/2-WB", 1251 [8] = "486 DX/4", 1252 [9] = "486 DX/4-WB", 1253 [14] = "Am5x86-WT", 1254 [15] = "Am5x86-WB" 1255 } 1256 }, 1257 }, 1258 .legacy_cache_size = amd_size_cache, 1259 #endif 1260 .c_early_init = early_init_amd, 1261 .c_detect_tlb = cpu_detect_tlb_amd, 1262 .c_bsp_init = bsp_init_amd, 1263 .c_init = init_amd, 1264 .c_x86_vendor = X86_VENDOR_AMD, 1265 }; 1266 1267 cpu_dev_register(amd_cpu_dev); 1268 1269 static DEFINE_PER_CPU_READ_MOSTLY(unsigned long[4], amd_dr_addr_mask); 1270 1271 static unsigned int amd_msr_dr_addr_masks[] = { 1272 MSR_F16H_DR0_ADDR_MASK, 1273 MSR_F16H_DR1_ADDR_MASK, 1274 MSR_F16H_DR1_ADDR_MASK + 1, 1275 MSR_F16H_DR1_ADDR_MASK + 2 1276 }; 1277 1278 void amd_set_dr_addr_mask(unsigned long mask, unsigned int dr) 1279 { 1280 int cpu = smp_processor_id(); 1281 1282 if (!cpu_feature_enabled(X86_FEATURE_BPEXT)) 1283 return; 1284 1285 if (WARN_ON_ONCE(dr >= ARRAY_SIZE(amd_msr_dr_addr_masks))) 1286 return; 1287 1288 if (per_cpu(amd_dr_addr_mask, cpu)[dr] == mask) 1289 return; 1290 1291 wrmsrq(amd_msr_dr_addr_masks[dr], mask); 1292 per_cpu(amd_dr_addr_mask, cpu)[dr] = mask; 1293 } 1294 1295 unsigned long amd_get_dr_addr_mask(unsigned int dr) 1296 { 1297 if (!cpu_feature_enabled(X86_FEATURE_BPEXT)) 1298 return 0; 1299 1300 if (WARN_ON_ONCE(dr >= ARRAY_SIZE(amd_msr_dr_addr_masks))) 1301 return 0; 1302 1303 return per_cpu(amd_dr_addr_mask[dr], smp_processor_id()); 1304 } 1305 EXPORT_SYMBOL_FOR_KVM(amd_get_dr_addr_mask); 1306 1307 static void zenbleed_check_cpu(void *unused) 1308 { 1309 struct cpuinfo_x86 *c = &cpu_data(smp_processor_id()); 1310 1311 zen2_zenbleed_check(c); 1312 } 1313 1314 void amd_check_microcode(void) 1315 { 1316 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 1317 return; 1318 1319 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) 1320 on_each_cpu(zenbleed_check_cpu, NULL, 1); 1321 } 1322 1323 static const char * const s5_reset_reason_txt[] = { 1324 [0] = "thermal pin BP_THERMTRIP_L was tripped", 1325 [1] = "power button was pressed for 4 seconds", 1326 [2] = "shutdown pin was tripped", 1327 [4] = "remote ASF power off command was received", 1328 [9] = "internal CPU thermal limit was tripped", 1329 [16] = "system reset pin BP_SYS_RST_L was tripped", 1330 [17] = "software issued PCI reset", 1331 [18] = "software wrote 0x4 to reset control register 0xCF9", 1332 [19] = "software wrote 0x6 to reset control register 0xCF9", 1333 [20] = "software wrote 0xE to reset control register 0xCF9", 1334 [21] = "ACPI power state transition occurred", 1335 [22] = "keyboard reset pin KB_RST_L was tripped", 1336 [23] = "internal CPU shutdown event occurred", 1337 [24] = "system failed to boot before failed boot timer expired", 1338 [25] = "hardware watchdog timer expired", 1339 [26] = "remote ASF reset command was received", 1340 [27] = "an uncorrected error caused a data fabric sync flood event", 1341 [29] = "FCH and MP1 failed warm reset handshake", 1342 [30] = "a parity error occurred", 1343 [31] = "a software sync flood event occurred", 1344 }; 1345 1346 static __init int print_s5_reset_status_mmio(void) 1347 { 1348 void __iomem *addr; 1349 u32 value; 1350 int i; 1351 1352 if (!cpu_feature_enabled(X86_FEATURE_ZEN)) 1353 return 0; 1354 1355 addr = ioremap(FCH_PM_BASE + FCH_PM_S5_RESET_STATUS, sizeof(value)); 1356 if (!addr) 1357 return 0; 1358 1359 value = ioread32(addr); 1360 1361 /* Value with "all bits set" is an error response and should be ignored. */ 1362 if (value == U32_MAX) { 1363 iounmap(addr); 1364 return 0; 1365 } 1366 1367 /* 1368 * Clear all reason bits so they won't be retained if the next reset 1369 * does not update the register. Besides, some bits are never cleared by 1370 * hardware so it's software's responsibility to clear them. 1371 * 1372 * Writing the value back effectively clears all reason bits as they are 1373 * write-1-to-clear. 1374 */ 1375 iowrite32(value, addr); 1376 iounmap(addr); 1377 1378 for (i = 0; i < ARRAY_SIZE(s5_reset_reason_txt); i++) { 1379 if (!(value & BIT(i))) 1380 continue; 1381 1382 if (s5_reset_reason_txt[i]) { 1383 pr_info("x86/amd: Previous system reset reason [0x%08x]: %s\n", 1384 value, s5_reset_reason_txt[i]); 1385 } 1386 } 1387 1388 return 0; 1389 } 1390 late_initcall(print_s5_reset_status_mmio); 1391 1392 static void __init dmi_scan_additional(const struct dmi_header *d, void *p) 1393 { 1394 struct dmi_a_info *info = (struct dmi_a_info *)d; 1395 void *next, *end; 1396 1397 if (!IS_ENABLED(CONFIG_DMI)) 1398 return; 1399 1400 if (info->header.type != DMI_ENTRY_ADDITIONAL || 1401 info->header.length < DMI_A_INFO_MIN_SIZE || 1402 info->count < 1) 1403 return; 1404 1405 next = (void *)(info + 1); 1406 end = (void *)info + info->header.length; 1407 1408 do { 1409 struct dmi_a_info_entry *entry; 1410 const char *string_ptr; 1411 1412 entry = (struct dmi_a_info_entry *)next; 1413 1414 /* 1415 * Not much can be done to validate data. At least the entry 1416 * length shouldn't be 0. 1417 */ 1418 if (!entry->length) 1419 return; 1420 1421 string_ptr = dmi_string_nosave(&info->header, entry->str_num); 1422 1423 /* Sample string: AGESA!V9 StrixKrackanPI-FP8 1.1.0.0c */ 1424 if (!strncmp(string_ptr, "AGESA", 5)) { 1425 pr_info("AGESA: %s\n", string_ptr); 1426 break; 1427 } 1428 1429 next += entry->length; 1430 } while (end - next >= DMI_A_INFO_ENT_MIN_SIZE); 1431 } 1432 1433 static __init int print_dmi_agesa(void) 1434 { 1435 dmi_walk(dmi_scan_additional, NULL); 1436 return 0; 1437 } 1438 late_initcall(print_dmi_agesa); 1439