1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2017, Nicholas Piggin, IBM Corporation 4 */ 5 6 #define pr_fmt(fmt) "dt-cpu-ftrs: " fmt 7 8 #include <linux/export.h> 9 #include <linux/init.h> 10 #include <linux/jump_label.h> 11 #include <linux/libfdt.h> 12 #include <linux/memblock.h> 13 #include <linux/of_fdt.h> 14 #include <linux/printk.h> 15 #include <linux/sched.h> 16 #include <linux/string.h> 17 #include <linux/threads.h> 18 19 #include <asm/cputable.h> 20 #include <asm/dt_cpu_ftrs.h> 21 #include <asm/mce.h> 22 #include <asm/mmu.h> 23 #include <asm/setup.h> 24 25 26 /* Device-tree visible constants follow */ 27 #define ISA_V3_0B 3000 28 #define ISA_V3_1 3100 29 #define ISA_V3_2 3200 30 31 #define USABLE_PR (1U << 0) 32 #define USABLE_OS (1U << 1) 33 #define USABLE_HV (1U << 2) 34 35 #define HV_SUPPORT_HFSCR (1U << 0) 36 #define OS_SUPPORT_FSCR (1U << 0) 37 38 /* For parsing, we define all bits set as "NONE" case */ 39 #define HV_SUPPORT_NONE 0xffffffffU 40 #define OS_SUPPORT_NONE 0xffffffffU 41 42 struct dt_cpu_feature { 43 const char *name; 44 uint32_t isa; 45 uint32_t usable_privilege; 46 uint32_t hv_support; 47 uint32_t os_support; 48 uint32_t hfscr_bit_nr; 49 uint32_t fscr_bit_nr; 50 uint32_t hwcap_bit_nr; 51 /* fdt parsing */ 52 unsigned long node; 53 int enabled; 54 int disabled; 55 }; 56 57 #define MMU_FTRS_HASH_BASE (MMU_FTRS_POWER8) 58 59 #define COMMON_USER_BASE (PPC_FEATURE_32 | PPC_FEATURE_64 | \ 60 PPC_FEATURE_ARCH_2_06 |\ 61 PPC_FEATURE_ICACHE_SNOOP) 62 #define COMMON_USER2_BASE (PPC_FEATURE2_ARCH_2_07 | \ 63 PPC_FEATURE2_ISEL) 64 /* 65 * Set up the base CPU 66 */ 67 68 static int hv_mode; 69 70 static struct { 71 u64 lpcr; 72 u64 hfscr; 73 u64 fscr; 74 u64 pcr; 75 } system_registers; 76 77 static void (*init_pmu_registers)(void); 78 79 static void __restore_cpu_cpufeatures(void) 80 { 81 mtspr(SPRN_LPCR, system_registers.lpcr); 82 if (hv_mode) { 83 mtspr(SPRN_LPID, 0); 84 mtspr(SPRN_AMOR, ~0); 85 mtspr(SPRN_HFSCR, system_registers.hfscr); 86 mtspr(SPRN_PCR, system_registers.pcr); 87 } 88 mtspr(SPRN_FSCR, system_registers.fscr); 89 90 if (init_pmu_registers) 91 init_pmu_registers(); 92 } 93 94 static struct cpu_spec __initdata base_cpu_spec = { 95 .cpu_name = NULL, 96 .cpu_features = CPU_FTRS_DT_CPU_BASE, 97 .cpu_user_features = COMMON_USER_BASE, 98 .cpu_user_features2 = COMMON_USER2_BASE, 99 .mmu_features = 0, 100 .icache_bsize = 32, /* minimum block size, fixed by */ 101 .dcache_bsize = 32, /* cache info init. */ 102 .num_pmcs = 0, 103 .pmc_type = PPC_PMC_DEFAULT, 104 .cpu_setup = NULL, 105 .cpu_restore = __restore_cpu_cpufeatures, 106 .machine_check_early = NULL, 107 .platform = NULL, 108 }; 109 110 static void __init cpufeatures_setup_cpu(void) 111 { 112 set_cur_cpu_spec(&base_cpu_spec); 113 114 cur_cpu_spec->pvr_mask = -1; 115 cur_cpu_spec->pvr_value = mfspr(SPRN_PVR); 116 117 /* Initialize the base environment -- clear FSCR/HFSCR. */ 118 hv_mode = !!(mfmsr() & MSR_HV); 119 if (hv_mode) { 120 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 121 mtspr(SPRN_HFSCR, 0); 122 } 123 mtspr(SPRN_FSCR, 0); 124 mtspr(SPRN_PCR, PCR_MASK); 125 126 /* 127 * LPCR does not get cleared, to match behaviour with secondaries 128 * in __restore_cpu_cpufeatures. Once the idle code is fixed, this 129 * could clear LPCR too. 130 */ 131 } 132 133 static int __init feat_try_enable_unknown(struct dt_cpu_feature *f) 134 { 135 if (f->hv_support == HV_SUPPORT_NONE) { 136 } else if (f->hv_support & HV_SUPPORT_HFSCR) { 137 u64 hfscr = mfspr(SPRN_HFSCR); 138 hfscr |= 1UL << f->hfscr_bit_nr; 139 mtspr(SPRN_HFSCR, hfscr); 140 } else { 141 /* Does not have a known recipe */ 142 return 0; 143 } 144 145 if (f->os_support == OS_SUPPORT_NONE) { 146 } else if (f->os_support & OS_SUPPORT_FSCR) { 147 u64 fscr = mfspr(SPRN_FSCR); 148 fscr |= 1UL << f->fscr_bit_nr; 149 mtspr(SPRN_FSCR, fscr); 150 } else { 151 /* Does not have a known recipe */ 152 return 0; 153 } 154 155 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 156 uint32_t word = f->hwcap_bit_nr / 32; 157 uint32_t bit = f->hwcap_bit_nr % 32; 158 159 if (word == 0) 160 cur_cpu_spec->cpu_user_features |= 1U << bit; 161 else if (word == 1) 162 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 163 else 164 pr_err("%s could not advertise to user (no hwcap bits)\n", f->name); 165 } 166 167 return 1; 168 } 169 170 static int __init feat_enable(struct dt_cpu_feature *f) 171 { 172 if (f->hv_support != HV_SUPPORT_NONE) { 173 if (f->hfscr_bit_nr != -1) { 174 u64 hfscr = mfspr(SPRN_HFSCR); 175 hfscr |= 1UL << f->hfscr_bit_nr; 176 mtspr(SPRN_HFSCR, hfscr); 177 } 178 } 179 180 if (f->os_support != OS_SUPPORT_NONE) { 181 if (f->fscr_bit_nr != -1) { 182 u64 fscr = mfspr(SPRN_FSCR); 183 fscr |= 1UL << f->fscr_bit_nr; 184 mtspr(SPRN_FSCR, fscr); 185 } 186 } 187 188 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 189 uint32_t word = f->hwcap_bit_nr / 32; 190 uint32_t bit = f->hwcap_bit_nr % 32; 191 192 if (word == 0) 193 cur_cpu_spec->cpu_user_features |= 1U << bit; 194 else if (word == 1) 195 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 196 else 197 pr_err("CPU feature: %s could not advertise to user (no hwcap bits)\n", f->name); 198 } 199 200 return 1; 201 } 202 203 static int __init feat_disable(struct dt_cpu_feature *f) 204 { 205 return 0; 206 } 207 208 static int __init feat_enable_hv(struct dt_cpu_feature *f) 209 { 210 u64 lpcr; 211 212 if (!hv_mode) { 213 pr_err("CPU feature hypervisor present in device tree but HV mode not enabled in the CPU. Ignoring.\n"); 214 return 0; 215 } 216 217 mtspr(SPRN_LPID, 0); 218 mtspr(SPRN_AMOR, ~0); 219 220 lpcr = mfspr(SPRN_LPCR); 221 lpcr &= ~LPCR_LPES0; /* HV external interrupts */ 222 mtspr(SPRN_LPCR, lpcr); 223 224 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 225 226 return 1; 227 } 228 229 static int __init feat_enable_le(struct dt_cpu_feature *f) 230 { 231 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_TRUE_LE; 232 return 1; 233 } 234 235 static int __init feat_enable_smt(struct dt_cpu_feature *f) 236 { 237 cur_cpu_spec->cpu_features |= CPU_FTR_SMT; 238 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_SMT; 239 return 1; 240 } 241 242 static int __init feat_enable_idle_nap(struct dt_cpu_feature *f) 243 { 244 u64 lpcr; 245 246 /* Set PECE wakeup modes for ISA 207 */ 247 lpcr = mfspr(SPRN_LPCR); 248 lpcr |= LPCR_PECE0; 249 lpcr |= LPCR_PECE1; 250 lpcr |= LPCR_PECE2; 251 mtspr(SPRN_LPCR, lpcr); 252 253 return 1; 254 } 255 256 static int __init feat_enable_idle_stop(struct dt_cpu_feature *f) 257 { 258 u64 lpcr; 259 260 /* Set PECE wakeup modes for ISAv3.0B */ 261 lpcr = mfspr(SPRN_LPCR); 262 lpcr |= LPCR_PECE0; 263 lpcr |= LPCR_PECE1; 264 lpcr |= LPCR_PECE2; 265 mtspr(SPRN_LPCR, lpcr); 266 267 return 1; 268 } 269 270 static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) 271 { 272 u64 lpcr; 273 274 if (!IS_ENABLED(CONFIG_PPC_64S_HASH_MMU)) 275 return 0; 276 277 lpcr = mfspr(SPRN_LPCR); 278 lpcr &= ~LPCR_ISL; 279 280 /* VRMASD */ 281 lpcr |= LPCR_VPM0; 282 lpcr &= ~LPCR_VPM1; 283 lpcr |= 0x10UL << LPCR_VRMASD_SH; /* L=1 LP=00 */ 284 mtspr(SPRN_LPCR, lpcr); 285 286 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 287 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 288 289 return 1; 290 } 291 292 static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) 293 { 294 u64 lpcr; 295 296 if (!IS_ENABLED(CONFIG_PPC_64S_HASH_MMU)) 297 return 0; 298 299 lpcr = mfspr(SPRN_LPCR); 300 lpcr &= ~(LPCR_ISL | LPCR_UPRT | LPCR_HR); 301 mtspr(SPRN_LPCR, lpcr); 302 303 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 304 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 305 306 return 1; 307 } 308 309 310 static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) 311 { 312 if (!IS_ENABLED(CONFIG_PPC_RADIX_MMU)) 313 return 0; 314 315 cur_cpu_spec->mmu_features |= MMU_FTR_KERNEL_RO; 316 cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; 317 cur_cpu_spec->mmu_features |= MMU_FTR_GTSE; 318 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 319 320 return 1; 321 } 322 323 static int __init feat_enable_dscr(struct dt_cpu_feature *f) 324 { 325 u64 lpcr; 326 327 /* 328 * Linux relies on FSCR[DSCR] being clear, so that we can take the 329 * facility unavailable interrupt and track the task's usage of DSCR. 330 * See facility_unavailable_exception(). 331 * Clear the bit here so that feat_enable() doesn't set it. 332 */ 333 f->fscr_bit_nr = -1; 334 335 feat_enable(f); 336 337 lpcr = mfspr(SPRN_LPCR); 338 lpcr &= ~LPCR_DPFD; 339 lpcr |= (4UL << LPCR_DPFD_SH); 340 mtspr(SPRN_LPCR, lpcr); 341 342 return 1; 343 } 344 345 static void __init hfscr_pmu_enable(void) 346 { 347 u64 hfscr = mfspr(SPRN_HFSCR); 348 hfscr |= PPC_BIT(60); 349 mtspr(SPRN_HFSCR, hfscr); 350 } 351 352 static void init_pmu_power8(void) 353 { 354 if (hv_mode) { 355 mtspr(SPRN_MMCRC, 0); 356 mtspr(SPRN_MMCRH, 0); 357 } 358 359 mtspr(SPRN_MMCRA, 0); 360 mtspr(SPRN_MMCR0, MMCR0_FC); 361 mtspr(SPRN_MMCR1, 0); 362 mtspr(SPRN_MMCR2, 0); 363 mtspr(SPRN_MMCRS, 0); 364 } 365 366 static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) 367 { 368 cur_cpu_spec->platform = "power8"; 369 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; 370 371 return 1; 372 } 373 374 static int __init feat_enable_pmu_power8(struct dt_cpu_feature *f) 375 { 376 hfscr_pmu_enable(); 377 378 init_pmu_power8(); 379 init_pmu_registers = init_pmu_power8; 380 381 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 382 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 383 if (pvr_version_is(PVR_POWER8E)) 384 cur_cpu_spec->cpu_features |= CPU_FTR_PMAO_BUG; 385 386 cur_cpu_spec->num_pmcs = 6; 387 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 388 389 return 1; 390 } 391 392 static void init_pmu_power9(void) 393 { 394 if (hv_mode) 395 mtspr(SPRN_MMCRC, 0); 396 397 mtspr(SPRN_MMCRA, 0); 398 mtspr(SPRN_MMCR0, MMCR0_FC); 399 mtspr(SPRN_MMCR1, 0); 400 mtspr(SPRN_MMCR2, 0); 401 } 402 403 static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) 404 { 405 cur_cpu_spec->platform = "power9"; 406 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; 407 408 return 1; 409 } 410 411 static int __init feat_enable_pmu_power9(struct dt_cpu_feature *f) 412 { 413 hfscr_pmu_enable(); 414 415 init_pmu_power9(); 416 init_pmu_registers = init_pmu_power9; 417 418 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 419 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 420 421 cur_cpu_spec->num_pmcs = 6; 422 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 423 424 return 1; 425 } 426 427 static void init_pmu_power10(void) 428 { 429 init_pmu_power9(); 430 431 mtspr(SPRN_MMCR3, 0); 432 mtspr(SPRN_MMCRA, MMCRA_BHRB_DISABLE); 433 mtspr(SPRN_MMCR0, MMCR0_FC | MMCR0_PMCCEXT); 434 } 435 436 static int __init feat_enable_pmu_power10(struct dt_cpu_feature *f) 437 { 438 hfscr_pmu_enable(); 439 440 init_pmu_power10(); 441 init_pmu_registers = init_pmu_power10; 442 443 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 444 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 445 446 cur_cpu_spec->num_pmcs = 6; 447 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 448 449 return 1; 450 } 451 452 static int __init feat_enable_mce_power10(struct dt_cpu_feature *f) 453 { 454 cur_cpu_spec->platform = "power10"; 455 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; 456 457 return 1; 458 } 459 460 static int __init feat_enable_mce_power11(struct dt_cpu_feature *f) 461 { 462 cur_cpu_spec->platform = "power11"; 463 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; 464 465 return 1; 466 } 467 468 static void init_pmu_power12(void) 469 { 470 init_pmu_power10(); 471 } 472 473 static int __init feat_enable_pmu_power12(struct dt_cpu_feature *f) 474 { 475 hfscr_pmu_enable(); 476 477 init_pmu_power12(); 478 init_pmu_registers = init_pmu_power12; 479 480 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 481 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 482 483 cur_cpu_spec->num_pmcs = 6; 484 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 485 486 return 1; 487 } 488 489 static int __init feat_enable_mce_power12(struct dt_cpu_feature *f) 490 { 491 cur_cpu_spec->platform = "power12"; 492 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; 493 494 return 1; 495 } 496 497 static int __init feat_enable_tm(struct dt_cpu_feature *f) 498 { 499 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 500 feat_enable(f); 501 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_HTM_NOSC; 502 return 1; 503 #endif 504 return 0; 505 } 506 507 static int __init feat_enable_fp(struct dt_cpu_feature *f) 508 { 509 feat_enable(f); 510 cur_cpu_spec->cpu_features &= ~CPU_FTR_FPU_UNAVAILABLE; 511 512 return 1; 513 } 514 515 static int __init feat_enable_vector(struct dt_cpu_feature *f) 516 { 517 #ifdef CONFIG_ALTIVEC 518 feat_enable(f); 519 cur_cpu_spec->cpu_features |= CPU_FTR_ALTIVEC; 520 cur_cpu_spec->cpu_features |= CPU_FTR_VMX_COPY; 521 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_ALTIVEC; 522 523 return 1; 524 #endif 525 return 0; 526 } 527 528 static int __init feat_enable_vsx(struct dt_cpu_feature *f) 529 { 530 #ifdef CONFIG_VSX 531 feat_enable(f); 532 cur_cpu_spec->cpu_features |= CPU_FTR_VSX; 533 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_VSX; 534 535 return 1; 536 #endif 537 return 0; 538 } 539 540 static int __init feat_enable_purr(struct dt_cpu_feature *f) 541 { 542 cur_cpu_spec->cpu_features |= CPU_FTR_PURR | CPU_FTR_SPURR; 543 544 return 1; 545 } 546 547 static int __init feat_enable_ebb(struct dt_cpu_feature *f) 548 { 549 /* 550 * PPC_FEATURE2_EBB is enabled in PMU init code because it has 551 * historically been related to the PMU facility. This may have 552 * to be decoupled if EBB becomes more generic. For now, follow 553 * existing convention. 554 */ 555 f->hwcap_bit_nr = -1; 556 feat_enable(f); 557 558 return 1; 559 } 560 561 static int __init feat_enable_dbell(struct dt_cpu_feature *f) 562 { 563 u64 lpcr; 564 565 /* P9 has an HFSCR for privileged state */ 566 feat_enable(f); 567 568 cur_cpu_spec->cpu_features |= CPU_FTR_DBELL; 569 570 lpcr = mfspr(SPRN_LPCR); 571 lpcr |= LPCR_PECEDH; /* hyp doorbell wakeup */ 572 mtspr(SPRN_LPCR, lpcr); 573 574 return 1; 575 } 576 577 static int __init feat_enable_hvi(struct dt_cpu_feature *f) 578 { 579 u64 lpcr; 580 581 /* 582 * POWER9 XIVE interrupts including in OPAL XICS compatibility 583 * are always delivered as hypervisor virtualization interrupts (HVI) 584 * rather than EE. 585 * 586 * However LPES0 is not set here, in the chance that an EE does get 587 * delivered to the host somehow, the EE handler would not expect it 588 * to be delivered in LPES0 mode (e.g., using SRR[01]). This could 589 * happen if there is a bug in interrupt controller code, or IC is 590 * misconfigured in systemsim. 591 */ 592 593 lpcr = mfspr(SPRN_LPCR); 594 lpcr |= LPCR_HVICE; /* enable hvi interrupts */ 595 lpcr |= LPCR_HEIC; /* disable ee interrupts when MSR_HV */ 596 lpcr |= LPCR_PECE_HVEE; /* hvi can wake from stop */ 597 mtspr(SPRN_LPCR, lpcr); 598 599 return 1; 600 } 601 602 static int __init feat_enable_large_ci(struct dt_cpu_feature *f) 603 { 604 cur_cpu_spec->mmu_features |= MMU_FTR_CI_LARGE_PAGE; 605 606 return 1; 607 } 608 609 static int __init feat_enable_mma(struct dt_cpu_feature *f) 610 { 611 u64 pcr; 612 613 feat_enable(f); 614 pcr = mfspr(SPRN_PCR); 615 pcr &= ~PCR_MMA_DIS; 616 mtspr(SPRN_PCR, pcr); 617 618 return 1; 619 } 620 621 struct dt_cpu_feature_match { 622 const char *name; 623 int (*enable)(struct dt_cpu_feature *f); 624 u64 cpu_ftr_bit_mask; 625 }; 626 627 static struct dt_cpu_feature_match __initdata 628 dt_cpu_feature_match_table[] = { 629 {"hypervisor", feat_enable_hv, 0}, 630 {"big-endian", feat_enable, 0}, 631 {"little-endian", feat_enable_le, CPU_FTR_REAL_LE}, 632 {"smt", feat_enable_smt, 0}, 633 {"interrupt-facilities", feat_enable, 0}, 634 {"system-call-vectored", feat_enable, 0}, 635 {"timer-facilities", feat_enable, 0}, 636 {"timer-facilities-v3", feat_enable, 0}, 637 {"debug-facilities", feat_enable, 0}, 638 {"come-from-address-register", feat_enable, CPU_FTR_CFAR}, 639 {"branch-tracing", feat_enable, 0}, 640 {"floating-point", feat_enable_fp, 0}, 641 {"vector", feat_enable_vector, 0}, 642 {"vector-scalar", feat_enable_vsx, 0}, 643 {"vector-scalar-v3", feat_enable, 0}, 644 {"decimal-floating-point", feat_enable, 0}, 645 {"decimal-integer", feat_enable, 0}, 646 {"quadword-load-store", feat_enable, 0}, 647 {"vector-crypto", feat_enable, 0}, 648 {"mmu-hash", feat_enable_mmu_hash, 0}, 649 {"mmu-radix", feat_enable_mmu_radix, 0}, 650 {"mmu-hash-v3", feat_enable_mmu_hash_v3, 0}, 651 {"virtual-page-class-key-protection", feat_enable, 0}, 652 {"transactional-memory", feat_enable_tm, CPU_FTR_TM}, 653 {"transactional-memory-v3", feat_enable_tm, 0}, 654 {"tm-suspend-hypervisor-assist", feat_enable, CPU_FTR_P9_TM_HV_ASSIST}, 655 {"tm-suspend-xer-so-bug", feat_enable, CPU_FTR_P9_TM_XER_SO_BUG}, 656 {"idle-nap", feat_enable_idle_nap, 0}, 657 /* alignment-interrupt-dsisr ignored */ 658 {"idle-stop", feat_enable_idle_stop, 0}, 659 {"machine-check-power8", feat_enable_mce_power8, 0}, 660 {"performance-monitor-power8", feat_enable_pmu_power8, 0}, 661 {"data-stream-control-register", feat_enable_dscr, CPU_FTR_DSCR}, 662 {"event-based-branch", feat_enable_ebb, 0}, 663 {"target-address-register", feat_enable, 0}, 664 {"branch-history-rolling-buffer", feat_enable, 0}, 665 {"control-register", feat_enable, CPU_FTR_CTRL}, 666 {"processor-control-facility", feat_enable_dbell, CPU_FTR_DBELL}, 667 {"processor-control-facility-v3", feat_enable_dbell, CPU_FTR_DBELL}, 668 {"processor-utilization-of-resources-register", feat_enable_purr, 0}, 669 {"no-execute", feat_enable, 0}, 670 {"strong-access-ordering", feat_enable, CPU_FTR_SAO}, 671 {"cache-inhibited-large-page", feat_enable_large_ci, 0}, 672 {"coprocessor-icswx", feat_enable, 0}, 673 {"hypervisor-virtualization-interrupt", feat_enable_hvi, 0}, 674 {"program-priority-register", feat_enable, CPU_FTR_HAS_PPR}, 675 {"wait", feat_enable, 0}, 676 {"atomic-memory-operations", feat_enable, 0}, 677 {"branch-v3", feat_enable, 0}, 678 {"copy-paste", feat_enable, 0}, 679 {"decimal-floating-point-v3", feat_enable, 0}, 680 {"decimal-integer-v3", feat_enable, 0}, 681 {"fixed-point-v3", feat_enable, 0}, 682 {"floating-point-v3", feat_enable, 0}, 683 {"group-start-register", feat_enable, 0}, 684 {"pc-relative-addressing", feat_enable, 0}, 685 {"machine-check-power9", feat_enable_mce_power9, 0}, 686 {"machine-check-power10", feat_enable_mce_power10, 0}, 687 {"machine-check-power11", feat_enable_mce_power11, 0}, 688 {"machine-check-power12", feat_enable_mce_power12, 0}, 689 {"performance-monitor-power9", feat_enable_pmu_power9, 0}, 690 {"performance-monitor-power10", feat_enable_pmu_power10, 0}, 691 {"performance-monitor-power11", feat_enable_pmu_power10, 0}, 692 {"performance-monitor-power12", feat_enable_pmu_power12, 0}, 693 {"event-based-branch-v3", feat_enable, 0}, 694 {"random-number-generator", feat_enable, 0}, 695 {"system-call-vectored", feat_disable, 0}, 696 {"trace-interrupt-v3", feat_enable, 0}, 697 {"vector-v3", feat_enable, 0}, 698 {"vector-binary128", feat_enable, 0}, 699 {"vector-binary16", feat_enable, 0}, 700 {"wait-v3", feat_enable, 0}, 701 {"prefix-instructions", feat_enable, 0}, 702 {"matrix-multiply-assist", feat_enable_mma, 0}, 703 {"debug-facilities-v31", feat_enable, CPU_FTR_DAWR1}, 704 }; 705 706 static bool __initdata using_dt_cpu_ftrs; 707 static bool __initdata enable_unknown = true; 708 709 static int __init dt_cpu_ftrs_parse(char *str) 710 { 711 if (!str) 712 return 0; 713 714 if (!strcmp(str, "off")) 715 using_dt_cpu_ftrs = false; 716 else if (!strcmp(str, "known")) 717 enable_unknown = false; 718 else 719 return 1; 720 721 return 0; 722 } 723 early_param("dt_cpu_ftrs", dt_cpu_ftrs_parse); 724 725 static void __init cpufeatures_setup_start(u32 isa) 726 { 727 pr_info("setup for ISA %d\n", isa); 728 729 if (isa >= ISA_V3_0B) { 730 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300; 731 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00; 732 } 733 734 if (isa >= ISA_V3_1) { 735 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31; 736 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1; 737 738 /* 739 * CPU_FTR_P11_PVR is a kernel-internal flag to identify 740 * Power11 and later processors. While ISA v3.1 is supported 741 * by Power10+, this flag specifically indicates Power11+ 742 * for code that needs to distinguish between P10 and P11. 743 */ 744 if (PVR_VER(mfspr(SPRN_PVR)) >= PVR_POWER11) 745 cur_cpu_spec->cpu_features |= CPU_FTR_P11_PVR; 746 } 747 748 if (isa >= ISA_V3_2) { 749 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_32; 750 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_2; 751 } 752 } 753 754 static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f) 755 { 756 const struct dt_cpu_feature_match *m; 757 bool known = false; 758 int i; 759 760 for (i = 0; i < ARRAY_SIZE(dt_cpu_feature_match_table); i++) { 761 m = &dt_cpu_feature_match_table[i]; 762 if (!strcmp(f->name, m->name)) { 763 known = true; 764 if (m->enable(f)) { 765 cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask; 766 break; 767 } 768 769 pr_info("not enabling: %s (disabled or unsupported by kernel)\n", 770 f->name); 771 return false; 772 } 773 } 774 775 if (!known && (!enable_unknown || !feat_try_enable_unknown(f))) { 776 pr_info("not enabling: %s (unknown and unsupported by kernel)\n", 777 f->name); 778 return false; 779 } 780 781 if (known) 782 pr_debug("enabling: %s\n", f->name); 783 else 784 pr_debug("enabling: %s (unknown)\n", f->name); 785 786 return true; 787 } 788 789 /* 790 * Handle POWER9 broadcast tlbie invalidation issue using 791 * cpu feature flag. 792 */ 793 static __init void update_tlbie_feature_flag(unsigned long pvr) 794 { 795 if (PVR_VER(pvr) == PVR_POWER9) { 796 /* 797 * Set the tlbie feature flag for anything below 798 * Nimbus DD 2.3 and Cumulus DD 1.3 799 */ 800 if ((pvr & 0xe000) == 0) { 801 /* Nimbus */ 802 if ((pvr & 0xfff) < 0x203) 803 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 804 } else if ((pvr & 0xc000) == 0) { 805 /* Cumulus */ 806 if ((pvr & 0xfff) < 0x103) 807 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 808 } else { 809 WARN_ONCE(1, "Unknown PVR"); 810 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 811 } 812 813 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_ERAT_BUG; 814 } 815 } 816 817 static __init void cpufeatures_cpu_quirks(void) 818 { 819 unsigned long version = mfspr(SPRN_PVR); 820 821 /* 822 * Not all quirks can be derived from the cpufeatures device tree. 823 */ 824 if ((version & 0xffffefff) == 0x004e0200) { 825 /* DD2.0 has no feature flag */ 826 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 827 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 828 } else if ((version & 0xffffefff) == 0x004e0201) { 829 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 830 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 831 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 832 } else if ((version & 0xffffefff) == 0x004e0202) { 833 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; 834 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; 835 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 836 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 837 } else if ((version & 0xffffefff) == 0x004e0203) { 838 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; 839 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; 840 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 841 } else if ((version & 0xffff0000) == 0x004e0000) { 842 /* DD2.1 and up have DD2_1 */ 843 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 844 } 845 846 if ((version & 0xffff0000) == 0x004e0000) { 847 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TIDR; 848 } 849 850 update_tlbie_feature_flag(version); 851 } 852 853 static void __init cpufeatures_setup_finished(void) 854 { 855 cpufeatures_cpu_quirks(); 856 857 if (hv_mode && !(cur_cpu_spec->cpu_features & CPU_FTR_HVMODE)) { 858 pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n"); 859 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 860 } 861 862 /* Make sure powerpc_base_platform is non-NULL */ 863 powerpc_base_platform = cur_cpu_spec->platform; 864 865 system_registers.lpcr = mfspr(SPRN_LPCR); 866 system_registers.hfscr = mfspr(SPRN_HFSCR); 867 system_registers.fscr = mfspr(SPRN_FSCR); 868 system_registers.pcr = mfspr(SPRN_PCR); 869 870 pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n", 871 cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); 872 } 873 874 static int __init disabled_on_cmdline(void) 875 { 876 unsigned long root, chosen; 877 const char *p; 878 879 root = of_get_flat_dt_root(); 880 chosen = of_get_flat_dt_subnode_by_name(root, "chosen"); 881 if (chosen == -FDT_ERR_NOTFOUND) 882 return false; 883 884 p = of_get_flat_dt_prop(chosen, "bootargs", NULL); 885 if (!p) 886 return false; 887 888 if (strstr(p, "dt_cpu_ftrs=off")) 889 return true; 890 891 return false; 892 } 893 894 static int __init fdt_find_cpu_features(unsigned long node, const char *uname, 895 int depth, void *data) 896 { 897 if (of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features") 898 && of_get_flat_dt_prop(node, "isa", NULL)) 899 return 1; 900 901 return 0; 902 } 903 904 bool __init dt_cpu_ftrs_in_use(void) 905 { 906 return using_dt_cpu_ftrs; 907 } 908 909 bool __init dt_cpu_ftrs_init(void *fdt) 910 { 911 using_dt_cpu_ftrs = false; 912 913 /* Setup and verify the FDT, if it fails we just bail */ 914 if (!early_init_dt_verify(fdt, __pa(fdt))) 915 return false; 916 917 if (!of_scan_flat_dt(fdt_find_cpu_features, NULL)) 918 return false; 919 920 if (disabled_on_cmdline()) 921 return false; 922 923 cpufeatures_setup_cpu(); 924 925 using_dt_cpu_ftrs = true; 926 return true; 927 } 928 929 static int nr_dt_cpu_features; 930 static struct dt_cpu_feature *dt_cpu_features; 931 932 static int __init process_cpufeatures_node(unsigned long node, 933 const char *uname, int i) 934 { 935 const __be32 *prop; 936 struct dt_cpu_feature *f; 937 int len; 938 939 f = &dt_cpu_features[i]; 940 941 f->node = node; 942 943 f->name = uname; 944 945 prop = of_get_flat_dt_prop(node, "isa", &len); 946 if (!prop) { 947 pr_warn("%s: missing isa property\n", uname); 948 return 0; 949 } 950 f->isa = be32_to_cpup(prop); 951 952 prop = of_get_flat_dt_prop(node, "usable-privilege", &len); 953 if (!prop) { 954 pr_warn("%s: missing usable-privilege property", uname); 955 return 0; 956 } 957 f->usable_privilege = be32_to_cpup(prop); 958 959 prop = of_get_flat_dt_prop(node, "hv-support", &len); 960 if (prop) 961 f->hv_support = be32_to_cpup(prop); 962 else 963 f->hv_support = HV_SUPPORT_NONE; 964 965 prop = of_get_flat_dt_prop(node, "os-support", &len); 966 if (prop) 967 f->os_support = be32_to_cpup(prop); 968 else 969 f->os_support = OS_SUPPORT_NONE; 970 971 prop = of_get_flat_dt_prop(node, "hfscr-bit-nr", &len); 972 if (prop) 973 f->hfscr_bit_nr = be32_to_cpup(prop); 974 else 975 f->hfscr_bit_nr = -1; 976 prop = of_get_flat_dt_prop(node, "fscr-bit-nr", &len); 977 if (prop) 978 f->fscr_bit_nr = be32_to_cpup(prop); 979 else 980 f->fscr_bit_nr = -1; 981 prop = of_get_flat_dt_prop(node, "hwcap-bit-nr", &len); 982 if (prop) 983 f->hwcap_bit_nr = be32_to_cpup(prop); 984 else 985 f->hwcap_bit_nr = -1; 986 987 if (f->usable_privilege & USABLE_HV) { 988 if (!(mfmsr() & MSR_HV)) { 989 pr_warn("%s: HV feature passed to guest\n", uname); 990 return 0; 991 } 992 993 if (f->hv_support == HV_SUPPORT_NONE && f->hfscr_bit_nr != -1) { 994 pr_warn("%s: unwanted hfscr_bit_nr\n", uname); 995 return 0; 996 } 997 998 if (f->hv_support == HV_SUPPORT_HFSCR) { 999 if (f->hfscr_bit_nr == -1) { 1000 pr_warn("%s: missing hfscr_bit_nr\n", uname); 1001 return 0; 1002 } 1003 } 1004 } else { 1005 if (f->hv_support != HV_SUPPORT_NONE || f->hfscr_bit_nr != -1) { 1006 pr_warn("%s: unwanted hv_support/hfscr_bit_nr\n", uname); 1007 return 0; 1008 } 1009 } 1010 1011 if (f->usable_privilege & USABLE_OS) { 1012 if (f->os_support == OS_SUPPORT_NONE && f->fscr_bit_nr != -1) { 1013 pr_warn("%s: unwanted fscr_bit_nr\n", uname); 1014 return 0; 1015 } 1016 1017 if (f->os_support == OS_SUPPORT_FSCR) { 1018 if (f->fscr_bit_nr == -1) { 1019 pr_warn("%s: missing fscr_bit_nr\n", uname); 1020 return 0; 1021 } 1022 } 1023 } else { 1024 if (f->os_support != OS_SUPPORT_NONE || f->fscr_bit_nr != -1) { 1025 pr_warn("%s: unwanted os_support/fscr_bit_nr\n", uname); 1026 return 0; 1027 } 1028 } 1029 1030 if (!(f->usable_privilege & USABLE_PR)) { 1031 if (f->hwcap_bit_nr != -1) { 1032 pr_warn("%s: unwanted hwcap_bit_nr\n", uname); 1033 return 0; 1034 } 1035 } 1036 1037 /* Do all the independent features in the first pass */ 1038 if (!of_get_flat_dt_prop(node, "dependencies", &len)) { 1039 if (cpufeatures_process_feature(f)) 1040 f->enabled = 1; 1041 else 1042 f->disabled = 1; 1043 } 1044 1045 return 0; 1046 } 1047 1048 static void __init cpufeatures_deps_enable(struct dt_cpu_feature *f) 1049 { 1050 const __be32 *prop; 1051 int len; 1052 int nr_deps; 1053 int i; 1054 1055 if (f->enabled || f->disabled) 1056 return; 1057 1058 prop = of_get_flat_dt_prop(f->node, "dependencies", &len); 1059 if (!prop) { 1060 pr_warn("%s: missing dependencies property", f->name); 1061 return; 1062 } 1063 1064 nr_deps = len / sizeof(int); 1065 1066 for (i = 0; i < nr_deps; i++) { 1067 unsigned long phandle = be32_to_cpu(prop[i]); 1068 int j; 1069 1070 for (j = 0; j < nr_dt_cpu_features; j++) { 1071 struct dt_cpu_feature *d = &dt_cpu_features[j]; 1072 1073 if (of_get_flat_dt_phandle(d->node) == phandle) { 1074 cpufeatures_deps_enable(d); 1075 if (d->disabled) { 1076 f->disabled = 1; 1077 return; 1078 } 1079 } 1080 } 1081 } 1082 1083 if (cpufeatures_process_feature(f)) 1084 f->enabled = 1; 1085 else 1086 f->disabled = 1; 1087 } 1088 1089 static int __init scan_cpufeatures_subnodes(unsigned long node, 1090 const char *uname, 1091 void *data) 1092 { 1093 int *count = data; 1094 1095 process_cpufeatures_node(node, uname, *count); 1096 1097 (*count)++; 1098 1099 return 0; 1100 } 1101 1102 static int __init count_cpufeatures_subnodes(unsigned long node, 1103 const char *uname, 1104 void *data) 1105 { 1106 int *count = data; 1107 1108 (*count)++; 1109 1110 return 0; 1111 } 1112 1113 static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char 1114 *uname, int depth, void *data) 1115 { 1116 static char dt_cpu_name[64]; 1117 const __be32 *prop; 1118 int count, i; 1119 u32 isa; 1120 1121 /* We are scanning "ibm,powerpc-cpu-features" nodes only */ 1122 if (!of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features")) 1123 return 0; 1124 1125 prop = of_get_flat_dt_prop(node, "isa", NULL); 1126 if (!prop) 1127 /* We checked before, "can't happen" */ 1128 return 0; 1129 1130 isa = be32_to_cpup(prop); 1131 1132 /* Count and allocate space for cpu features */ 1133 of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, 1134 &nr_dt_cpu_features); 1135 dt_cpu_features = 1136 memblock_alloc_or_panic( 1137 sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, 1138 PAGE_SIZE); 1139 1140 cpufeatures_setup_start(isa); 1141 1142 /* Scan nodes into dt_cpu_features and enable those without deps */ 1143 count = 0; 1144 of_scan_flat_dt_subnodes(node, scan_cpufeatures_subnodes, &count); 1145 1146 /* Recursive enable remaining features with dependencies */ 1147 for (i = 0; i < nr_dt_cpu_features; i++) { 1148 struct dt_cpu_feature *f = &dt_cpu_features[i]; 1149 1150 cpufeatures_deps_enable(f); 1151 } 1152 1153 prop = of_get_flat_dt_prop(node, "display-name", NULL); 1154 if (prop && *(char *)prop != 0) { 1155 strscpy(dt_cpu_name, (char *)prop); 1156 cur_cpu_spec->cpu_name = dt_cpu_name; 1157 } 1158 1159 cpufeatures_setup_finished(); 1160 1161 memblock_free(dt_cpu_features, 1162 sizeof(struct dt_cpu_feature) * nr_dt_cpu_features); 1163 1164 return 0; 1165 } 1166 1167 void __init dt_cpu_ftrs_scan(void) 1168 { 1169 if (!using_dt_cpu_ftrs) 1170 return; 1171 1172 of_scan_flat_dt(dt_cpu_ftrs_scan_callback, NULL); 1173 } 1174