1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2017, Nicholas Piggin, IBM Corporation 4 */ 5 6 #define pr_fmt(fmt) "dt-cpu-ftrs: " fmt 7 8 #include <linux/export.h> 9 #include <linux/init.h> 10 #include <linux/jump_label.h> 11 #include <linux/libfdt.h> 12 #include <linux/memblock.h> 13 #include <linux/of_fdt.h> 14 #include <linux/printk.h> 15 #include <linux/sched.h> 16 #include <linux/string.h> 17 #include <linux/threads.h> 18 19 #include <asm/cputable.h> 20 #include <asm/dt_cpu_ftrs.h> 21 #include <asm/mce.h> 22 #include <asm/mmu.h> 23 #include <asm/setup.h> 24 25 26 /* Device-tree visible constants follow */ 27 #define ISA_V3_0B 3000 28 #define ISA_V3_1 3100 29 30 #define USABLE_PR (1U << 0) 31 #define USABLE_OS (1U << 1) 32 #define USABLE_HV (1U << 2) 33 34 #define HV_SUPPORT_HFSCR (1U << 0) 35 #define OS_SUPPORT_FSCR (1U << 0) 36 37 /* For parsing, we define all bits set as "NONE" case */ 38 #define HV_SUPPORT_NONE 0xffffffffU 39 #define OS_SUPPORT_NONE 0xffffffffU 40 41 struct dt_cpu_feature { 42 const char *name; 43 uint32_t isa; 44 uint32_t usable_privilege; 45 uint32_t hv_support; 46 uint32_t os_support; 47 uint32_t hfscr_bit_nr; 48 uint32_t fscr_bit_nr; 49 uint32_t hwcap_bit_nr; 50 /* fdt parsing */ 51 unsigned long node; 52 int enabled; 53 int disabled; 54 }; 55 56 #define MMU_FTRS_HASH_BASE (MMU_FTRS_POWER8) 57 58 #define COMMON_USER_BASE (PPC_FEATURE_32 | PPC_FEATURE_64 | \ 59 PPC_FEATURE_ARCH_2_06 |\ 60 PPC_FEATURE_ICACHE_SNOOP) 61 #define COMMON_USER2_BASE (PPC_FEATURE2_ARCH_2_07 | \ 62 PPC_FEATURE2_ISEL) 63 /* 64 * Set up the base CPU 65 */ 66 67 static int hv_mode; 68 69 static struct { 70 u64 lpcr; 71 u64 hfscr; 72 u64 fscr; 73 u64 pcr; 74 } system_registers; 75 76 static void (*init_pmu_registers)(void); 77 78 static void __restore_cpu_cpufeatures(void) 79 { 80 mtspr(SPRN_LPCR, system_registers.lpcr); 81 if (hv_mode) { 82 mtspr(SPRN_LPID, 0); 83 mtspr(SPRN_AMOR, ~0); 84 mtspr(SPRN_HFSCR, system_registers.hfscr); 85 mtspr(SPRN_PCR, system_registers.pcr); 86 } 87 mtspr(SPRN_FSCR, system_registers.fscr); 88 89 if (init_pmu_registers) 90 init_pmu_registers(); 91 } 92 93 static char dt_cpu_name[64]; 94 95 static struct cpu_spec __initdata base_cpu_spec = { 96 .cpu_name = NULL, 97 .cpu_features = CPU_FTRS_DT_CPU_BASE, 98 .cpu_user_features = COMMON_USER_BASE, 99 .cpu_user_features2 = COMMON_USER2_BASE, 100 .mmu_features = 0, 101 .icache_bsize = 32, /* minimum block size, fixed by */ 102 .dcache_bsize = 32, /* cache info init. */ 103 .num_pmcs = 0, 104 .pmc_type = PPC_PMC_DEFAULT, 105 .cpu_setup = NULL, 106 .cpu_restore = __restore_cpu_cpufeatures, 107 .machine_check_early = NULL, 108 .platform = NULL, 109 }; 110 111 static void __init cpufeatures_setup_cpu(void) 112 { 113 set_cur_cpu_spec(&base_cpu_spec); 114 115 cur_cpu_spec->pvr_mask = -1; 116 cur_cpu_spec->pvr_value = mfspr(SPRN_PVR); 117 118 /* Initialize the base environment -- clear FSCR/HFSCR. */ 119 hv_mode = !!(mfmsr() & MSR_HV); 120 if (hv_mode) { 121 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 122 mtspr(SPRN_HFSCR, 0); 123 } 124 mtspr(SPRN_FSCR, 0); 125 mtspr(SPRN_PCR, PCR_MASK); 126 127 /* 128 * LPCR does not get cleared, to match behaviour with secondaries 129 * in __restore_cpu_cpufeatures. Once the idle code is fixed, this 130 * could clear LPCR too. 131 */ 132 } 133 134 static int __init feat_try_enable_unknown(struct dt_cpu_feature *f) 135 { 136 if (f->hv_support == HV_SUPPORT_NONE) { 137 } else if (f->hv_support & HV_SUPPORT_HFSCR) { 138 u64 hfscr = mfspr(SPRN_HFSCR); 139 hfscr |= 1UL << f->hfscr_bit_nr; 140 mtspr(SPRN_HFSCR, hfscr); 141 } else { 142 /* Does not have a known recipe */ 143 return 0; 144 } 145 146 if (f->os_support == OS_SUPPORT_NONE) { 147 } else if (f->os_support & OS_SUPPORT_FSCR) { 148 u64 fscr = mfspr(SPRN_FSCR); 149 fscr |= 1UL << f->fscr_bit_nr; 150 mtspr(SPRN_FSCR, fscr); 151 } else { 152 /* Does not have a known recipe */ 153 return 0; 154 } 155 156 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 157 uint32_t word = f->hwcap_bit_nr / 32; 158 uint32_t bit = f->hwcap_bit_nr % 32; 159 160 if (word == 0) 161 cur_cpu_spec->cpu_user_features |= 1U << bit; 162 else if (word == 1) 163 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 164 else 165 pr_err("%s could not advertise to user (no hwcap bits)\n", f->name); 166 } 167 168 return 1; 169 } 170 171 static int __init feat_enable(struct dt_cpu_feature *f) 172 { 173 if (f->hv_support != HV_SUPPORT_NONE) { 174 if (f->hfscr_bit_nr != -1) { 175 u64 hfscr = mfspr(SPRN_HFSCR); 176 hfscr |= 1UL << f->hfscr_bit_nr; 177 mtspr(SPRN_HFSCR, hfscr); 178 } 179 } 180 181 if (f->os_support != OS_SUPPORT_NONE) { 182 if (f->fscr_bit_nr != -1) { 183 u64 fscr = mfspr(SPRN_FSCR); 184 fscr |= 1UL << f->fscr_bit_nr; 185 mtspr(SPRN_FSCR, fscr); 186 } 187 } 188 189 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 190 uint32_t word = f->hwcap_bit_nr / 32; 191 uint32_t bit = f->hwcap_bit_nr % 32; 192 193 if (word == 0) 194 cur_cpu_spec->cpu_user_features |= 1U << bit; 195 else if (word == 1) 196 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 197 else 198 pr_err("CPU feature: %s could not advertise to user (no hwcap bits)\n", f->name); 199 } 200 201 return 1; 202 } 203 204 static int __init feat_disable(struct dt_cpu_feature *f) 205 { 206 return 0; 207 } 208 209 static int __init feat_enable_hv(struct dt_cpu_feature *f) 210 { 211 u64 lpcr; 212 213 if (!hv_mode) { 214 pr_err("CPU feature hypervisor present in device tree but HV mode not enabled in the CPU. Ignoring.\n"); 215 return 0; 216 } 217 218 mtspr(SPRN_LPID, 0); 219 mtspr(SPRN_AMOR, ~0); 220 221 lpcr = mfspr(SPRN_LPCR); 222 lpcr &= ~LPCR_LPES0; /* HV external interrupts */ 223 mtspr(SPRN_LPCR, lpcr); 224 225 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 226 227 return 1; 228 } 229 230 static int __init feat_enable_le(struct dt_cpu_feature *f) 231 { 232 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_TRUE_LE; 233 return 1; 234 } 235 236 static int __init feat_enable_smt(struct dt_cpu_feature *f) 237 { 238 cur_cpu_spec->cpu_features |= CPU_FTR_SMT; 239 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_SMT; 240 return 1; 241 } 242 243 static int __init feat_enable_idle_nap(struct dt_cpu_feature *f) 244 { 245 u64 lpcr; 246 247 /* Set PECE wakeup modes for ISA 207 */ 248 lpcr = mfspr(SPRN_LPCR); 249 lpcr |= LPCR_PECE0; 250 lpcr |= LPCR_PECE1; 251 lpcr |= LPCR_PECE2; 252 mtspr(SPRN_LPCR, lpcr); 253 254 return 1; 255 } 256 257 static int __init feat_enable_idle_stop(struct dt_cpu_feature *f) 258 { 259 u64 lpcr; 260 261 /* Set PECE wakeup modes for ISAv3.0B */ 262 lpcr = mfspr(SPRN_LPCR); 263 lpcr |= LPCR_PECE0; 264 lpcr |= LPCR_PECE1; 265 lpcr |= LPCR_PECE2; 266 mtspr(SPRN_LPCR, lpcr); 267 268 return 1; 269 } 270 271 static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) 272 { 273 u64 lpcr; 274 275 if (!IS_ENABLED(CONFIG_PPC_64S_HASH_MMU)) 276 return 0; 277 278 lpcr = mfspr(SPRN_LPCR); 279 lpcr &= ~LPCR_ISL; 280 281 /* VRMASD */ 282 lpcr |= LPCR_VPM0; 283 lpcr &= ~LPCR_VPM1; 284 lpcr |= 0x10UL << LPCR_VRMASD_SH; /* L=1 LP=00 */ 285 mtspr(SPRN_LPCR, lpcr); 286 287 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 288 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 289 290 return 1; 291 } 292 293 static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) 294 { 295 u64 lpcr; 296 297 if (!IS_ENABLED(CONFIG_PPC_64S_HASH_MMU)) 298 return 0; 299 300 lpcr = mfspr(SPRN_LPCR); 301 lpcr &= ~(LPCR_ISL | LPCR_UPRT | LPCR_HR); 302 mtspr(SPRN_LPCR, lpcr); 303 304 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 305 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 306 307 return 1; 308 } 309 310 311 static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) 312 { 313 if (!IS_ENABLED(CONFIG_PPC_RADIX_MMU)) 314 return 0; 315 316 cur_cpu_spec->mmu_features |= MMU_FTR_KERNEL_RO; 317 cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; 318 cur_cpu_spec->mmu_features |= MMU_FTR_GTSE; 319 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 320 321 return 1; 322 } 323 324 static int __init feat_enable_dscr(struct dt_cpu_feature *f) 325 { 326 u64 lpcr; 327 328 /* 329 * Linux relies on FSCR[DSCR] being clear, so that we can take the 330 * facility unavailable interrupt and track the task's usage of DSCR. 331 * See facility_unavailable_exception(). 332 * Clear the bit here so that feat_enable() doesn't set it. 333 */ 334 f->fscr_bit_nr = -1; 335 336 feat_enable(f); 337 338 lpcr = mfspr(SPRN_LPCR); 339 lpcr &= ~LPCR_DPFD; 340 lpcr |= (4UL << LPCR_DPFD_SH); 341 mtspr(SPRN_LPCR, lpcr); 342 343 return 1; 344 } 345 346 static void __init hfscr_pmu_enable(void) 347 { 348 u64 hfscr = mfspr(SPRN_HFSCR); 349 hfscr |= PPC_BIT(60); 350 mtspr(SPRN_HFSCR, hfscr); 351 } 352 353 static void init_pmu_power8(void) 354 { 355 if (hv_mode) { 356 mtspr(SPRN_MMCRC, 0); 357 mtspr(SPRN_MMCRH, 0); 358 } 359 360 mtspr(SPRN_MMCRA, 0); 361 mtspr(SPRN_MMCR0, MMCR0_FC); 362 mtspr(SPRN_MMCR1, 0); 363 mtspr(SPRN_MMCR2, 0); 364 mtspr(SPRN_MMCRS, 0); 365 } 366 367 static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) 368 { 369 cur_cpu_spec->platform = "power8"; 370 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; 371 372 return 1; 373 } 374 375 static int __init feat_enable_pmu_power8(struct dt_cpu_feature *f) 376 { 377 hfscr_pmu_enable(); 378 379 init_pmu_power8(); 380 init_pmu_registers = init_pmu_power8; 381 382 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 383 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 384 if (pvr_version_is(PVR_POWER8E)) 385 cur_cpu_spec->cpu_features |= CPU_FTR_PMAO_BUG; 386 387 cur_cpu_spec->num_pmcs = 6; 388 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 389 390 return 1; 391 } 392 393 static void init_pmu_power9(void) 394 { 395 if (hv_mode) 396 mtspr(SPRN_MMCRC, 0); 397 398 mtspr(SPRN_MMCRA, 0); 399 mtspr(SPRN_MMCR0, MMCR0_FC); 400 mtspr(SPRN_MMCR1, 0); 401 mtspr(SPRN_MMCR2, 0); 402 } 403 404 static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) 405 { 406 cur_cpu_spec->platform = "power9"; 407 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; 408 409 return 1; 410 } 411 412 static int __init feat_enable_pmu_power9(struct dt_cpu_feature *f) 413 { 414 hfscr_pmu_enable(); 415 416 init_pmu_power9(); 417 init_pmu_registers = init_pmu_power9; 418 419 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 420 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 421 422 cur_cpu_spec->num_pmcs = 6; 423 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 424 425 return 1; 426 } 427 428 static void init_pmu_power10(void) 429 { 430 init_pmu_power9(); 431 432 mtspr(SPRN_MMCR3, 0); 433 mtspr(SPRN_MMCRA, MMCRA_BHRB_DISABLE); 434 mtspr(SPRN_MMCR0, MMCR0_FC | MMCR0_PMCCEXT); 435 } 436 437 static int __init feat_enable_pmu_power10(struct dt_cpu_feature *f) 438 { 439 hfscr_pmu_enable(); 440 441 init_pmu_power10(); 442 init_pmu_registers = init_pmu_power10; 443 444 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 445 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 446 447 cur_cpu_spec->num_pmcs = 6; 448 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 449 450 return 1; 451 } 452 453 static int __init feat_enable_mce_power10(struct dt_cpu_feature *f) 454 { 455 cur_cpu_spec->platform = "power10"; 456 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; 457 458 return 1; 459 } 460 461 static int __init feat_enable_mce_power11(struct dt_cpu_feature *f) 462 { 463 cur_cpu_spec->platform = "power11"; 464 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; 465 466 return 1; 467 } 468 469 static int __init feat_enable_tm(struct dt_cpu_feature *f) 470 { 471 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 472 feat_enable(f); 473 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_HTM_NOSC; 474 return 1; 475 #endif 476 return 0; 477 } 478 479 static int __init feat_enable_fp(struct dt_cpu_feature *f) 480 { 481 feat_enable(f); 482 cur_cpu_spec->cpu_features &= ~CPU_FTR_FPU_UNAVAILABLE; 483 484 return 1; 485 } 486 487 static int __init feat_enable_vector(struct dt_cpu_feature *f) 488 { 489 #ifdef CONFIG_ALTIVEC 490 feat_enable(f); 491 cur_cpu_spec->cpu_features |= CPU_FTR_ALTIVEC; 492 cur_cpu_spec->cpu_features |= CPU_FTR_VMX_COPY; 493 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_ALTIVEC; 494 495 return 1; 496 #endif 497 return 0; 498 } 499 500 static int __init feat_enable_vsx(struct dt_cpu_feature *f) 501 { 502 #ifdef CONFIG_VSX 503 feat_enable(f); 504 cur_cpu_spec->cpu_features |= CPU_FTR_VSX; 505 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_VSX; 506 507 return 1; 508 #endif 509 return 0; 510 } 511 512 static int __init feat_enable_purr(struct dt_cpu_feature *f) 513 { 514 cur_cpu_spec->cpu_features |= CPU_FTR_PURR | CPU_FTR_SPURR; 515 516 return 1; 517 } 518 519 static int __init feat_enable_ebb(struct dt_cpu_feature *f) 520 { 521 /* 522 * PPC_FEATURE2_EBB is enabled in PMU init code because it has 523 * historically been related to the PMU facility. This may have 524 * to be decoupled if EBB becomes more generic. For now, follow 525 * existing convention. 526 */ 527 f->hwcap_bit_nr = -1; 528 feat_enable(f); 529 530 return 1; 531 } 532 533 static int __init feat_enable_dbell(struct dt_cpu_feature *f) 534 { 535 u64 lpcr; 536 537 /* P9 has an HFSCR for privileged state */ 538 feat_enable(f); 539 540 cur_cpu_spec->cpu_features |= CPU_FTR_DBELL; 541 542 lpcr = mfspr(SPRN_LPCR); 543 lpcr |= LPCR_PECEDH; /* hyp doorbell wakeup */ 544 mtspr(SPRN_LPCR, lpcr); 545 546 return 1; 547 } 548 549 static int __init feat_enable_hvi(struct dt_cpu_feature *f) 550 { 551 u64 lpcr; 552 553 /* 554 * POWER9 XIVE interrupts including in OPAL XICS compatibility 555 * are always delivered as hypervisor virtualization interrupts (HVI) 556 * rather than EE. 557 * 558 * However LPES0 is not set here, in the chance that an EE does get 559 * delivered to the host somehow, the EE handler would not expect it 560 * to be delivered in LPES0 mode (e.g., using SRR[01]). This could 561 * happen if there is a bug in interrupt controller code, or IC is 562 * misconfigured in systemsim. 563 */ 564 565 lpcr = mfspr(SPRN_LPCR); 566 lpcr |= LPCR_HVICE; /* enable hvi interrupts */ 567 lpcr |= LPCR_HEIC; /* disable ee interrupts when MSR_HV */ 568 lpcr |= LPCR_PECE_HVEE; /* hvi can wake from stop */ 569 mtspr(SPRN_LPCR, lpcr); 570 571 return 1; 572 } 573 574 static int __init feat_enable_large_ci(struct dt_cpu_feature *f) 575 { 576 cur_cpu_spec->mmu_features |= MMU_FTR_CI_LARGE_PAGE; 577 578 return 1; 579 } 580 581 static int __init feat_enable_mma(struct dt_cpu_feature *f) 582 { 583 u64 pcr; 584 585 feat_enable(f); 586 pcr = mfspr(SPRN_PCR); 587 pcr &= ~PCR_MMA_DIS; 588 mtspr(SPRN_PCR, pcr); 589 590 return 1; 591 } 592 593 struct dt_cpu_feature_match { 594 const char *name; 595 int (*enable)(struct dt_cpu_feature *f); 596 u64 cpu_ftr_bit_mask; 597 }; 598 599 static struct dt_cpu_feature_match __initdata 600 dt_cpu_feature_match_table[] = { 601 {"hypervisor", feat_enable_hv, 0}, 602 {"big-endian", feat_enable, 0}, 603 {"little-endian", feat_enable_le, CPU_FTR_REAL_LE}, 604 {"smt", feat_enable_smt, 0}, 605 {"interrupt-facilities", feat_enable, 0}, 606 {"system-call-vectored", feat_enable, 0}, 607 {"timer-facilities", feat_enable, 0}, 608 {"timer-facilities-v3", feat_enable, 0}, 609 {"debug-facilities", feat_enable, 0}, 610 {"come-from-address-register", feat_enable, CPU_FTR_CFAR}, 611 {"branch-tracing", feat_enable, 0}, 612 {"floating-point", feat_enable_fp, 0}, 613 {"vector", feat_enable_vector, 0}, 614 {"vector-scalar", feat_enable_vsx, 0}, 615 {"vector-scalar-v3", feat_enable, 0}, 616 {"decimal-floating-point", feat_enable, 0}, 617 {"decimal-integer", feat_enable, 0}, 618 {"quadword-load-store", feat_enable, 0}, 619 {"vector-crypto", feat_enable, 0}, 620 {"mmu-hash", feat_enable_mmu_hash, 0}, 621 {"mmu-radix", feat_enable_mmu_radix, 0}, 622 {"mmu-hash-v3", feat_enable_mmu_hash_v3, 0}, 623 {"virtual-page-class-key-protection", feat_enable, 0}, 624 {"transactional-memory", feat_enable_tm, CPU_FTR_TM}, 625 {"transactional-memory-v3", feat_enable_tm, 0}, 626 {"tm-suspend-hypervisor-assist", feat_enable, CPU_FTR_P9_TM_HV_ASSIST}, 627 {"tm-suspend-xer-so-bug", feat_enable, CPU_FTR_P9_TM_XER_SO_BUG}, 628 {"idle-nap", feat_enable_idle_nap, 0}, 629 /* alignment-interrupt-dsisr ignored */ 630 {"idle-stop", feat_enable_idle_stop, 0}, 631 {"machine-check-power8", feat_enable_mce_power8, 0}, 632 {"performance-monitor-power8", feat_enable_pmu_power8, 0}, 633 {"data-stream-control-register", feat_enable_dscr, CPU_FTR_DSCR}, 634 {"event-based-branch", feat_enable_ebb, 0}, 635 {"target-address-register", feat_enable, 0}, 636 {"branch-history-rolling-buffer", feat_enable, 0}, 637 {"control-register", feat_enable, CPU_FTR_CTRL}, 638 {"processor-control-facility", feat_enable_dbell, CPU_FTR_DBELL}, 639 {"processor-control-facility-v3", feat_enable_dbell, CPU_FTR_DBELL}, 640 {"processor-utilization-of-resources-register", feat_enable_purr, 0}, 641 {"no-execute", feat_enable, 0}, 642 {"strong-access-ordering", feat_enable, CPU_FTR_SAO}, 643 {"cache-inhibited-large-page", feat_enable_large_ci, 0}, 644 {"coprocessor-icswx", feat_enable, 0}, 645 {"hypervisor-virtualization-interrupt", feat_enable_hvi, 0}, 646 {"program-priority-register", feat_enable, CPU_FTR_HAS_PPR}, 647 {"wait", feat_enable, 0}, 648 {"atomic-memory-operations", feat_enable, 0}, 649 {"branch-v3", feat_enable, 0}, 650 {"copy-paste", feat_enable, 0}, 651 {"decimal-floating-point-v3", feat_enable, 0}, 652 {"decimal-integer-v3", feat_enable, 0}, 653 {"fixed-point-v3", feat_enable, 0}, 654 {"floating-point-v3", feat_enable, 0}, 655 {"group-start-register", feat_enable, 0}, 656 {"pc-relative-addressing", feat_enable, 0}, 657 {"machine-check-power9", feat_enable_mce_power9, 0}, 658 {"machine-check-power10", feat_enable_mce_power10, 0}, 659 {"machine-check-power11", feat_enable_mce_power11, 0}, 660 {"performance-monitor-power9", feat_enable_pmu_power9, 0}, 661 {"performance-monitor-power10", feat_enable_pmu_power10, 0}, 662 {"performance-monitor-power11", feat_enable_pmu_power10, 0}, 663 {"event-based-branch-v3", feat_enable, 0}, 664 {"random-number-generator", feat_enable, 0}, 665 {"system-call-vectored", feat_disable, 0}, 666 {"trace-interrupt-v3", feat_enable, 0}, 667 {"vector-v3", feat_enable, 0}, 668 {"vector-binary128", feat_enable, 0}, 669 {"vector-binary16", feat_enable, 0}, 670 {"wait-v3", feat_enable, 0}, 671 {"prefix-instructions", feat_enable, 0}, 672 {"matrix-multiply-assist", feat_enable_mma, 0}, 673 {"debug-facilities-v31", feat_enable, CPU_FTR_DAWR1}, 674 }; 675 676 static bool __initdata using_dt_cpu_ftrs; 677 static bool __initdata enable_unknown = true; 678 679 static int __init dt_cpu_ftrs_parse(char *str) 680 { 681 if (!str) 682 return 0; 683 684 if (!strcmp(str, "off")) 685 using_dt_cpu_ftrs = false; 686 else if (!strcmp(str, "known")) 687 enable_unknown = false; 688 else 689 return 1; 690 691 return 0; 692 } 693 early_param("dt_cpu_ftrs", dt_cpu_ftrs_parse); 694 695 static void __init cpufeatures_setup_start(u32 isa) 696 { 697 pr_info("setup for ISA %d\n", isa); 698 699 if (isa >= ISA_V3_0B) { 700 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300; 701 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00; 702 } 703 704 if (isa >= ISA_V3_1) { 705 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31; 706 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1; 707 } 708 } 709 710 static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f) 711 { 712 const struct dt_cpu_feature_match *m; 713 bool known = false; 714 int i; 715 716 for (i = 0; i < ARRAY_SIZE(dt_cpu_feature_match_table); i++) { 717 m = &dt_cpu_feature_match_table[i]; 718 if (!strcmp(f->name, m->name)) { 719 known = true; 720 if (m->enable(f)) { 721 cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask; 722 break; 723 } 724 725 pr_info("not enabling: %s (disabled or unsupported by kernel)\n", 726 f->name); 727 return false; 728 } 729 } 730 731 if (!known && (!enable_unknown || !feat_try_enable_unknown(f))) { 732 pr_info("not enabling: %s (unknown and unsupported by kernel)\n", 733 f->name); 734 return false; 735 } 736 737 if (known) 738 pr_debug("enabling: %s\n", f->name); 739 else 740 pr_debug("enabling: %s (unknown)\n", f->name); 741 742 return true; 743 } 744 745 /* 746 * Handle POWER9 broadcast tlbie invalidation issue using 747 * cpu feature flag. 748 */ 749 static __init void update_tlbie_feature_flag(unsigned long pvr) 750 { 751 if (PVR_VER(pvr) == PVR_POWER9) { 752 /* 753 * Set the tlbie feature flag for anything below 754 * Nimbus DD 2.3 and Cumulus DD 1.3 755 */ 756 if ((pvr & 0xe000) == 0) { 757 /* Nimbus */ 758 if ((pvr & 0xfff) < 0x203) 759 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 760 } else if ((pvr & 0xc000) == 0) { 761 /* Cumulus */ 762 if ((pvr & 0xfff) < 0x103) 763 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 764 } else { 765 WARN_ONCE(1, "Unknown PVR"); 766 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 767 } 768 769 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_ERAT_BUG; 770 } 771 } 772 773 static __init void cpufeatures_cpu_quirks(void) 774 { 775 unsigned long version = mfspr(SPRN_PVR); 776 777 /* 778 * Not all quirks can be derived from the cpufeatures device tree. 779 */ 780 if ((version & 0xffffefff) == 0x004e0200) { 781 /* DD2.0 has no feature flag */ 782 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 783 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 784 } else if ((version & 0xffffefff) == 0x004e0201) { 785 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 786 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 787 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 788 } else if ((version & 0xffffefff) == 0x004e0202) { 789 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; 790 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; 791 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 792 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 793 } else if ((version & 0xffffefff) == 0x004e0203) { 794 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; 795 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; 796 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 797 } else if ((version & 0xffff0000) == 0x004e0000) { 798 /* DD2.1 and up have DD2_1 */ 799 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 800 } 801 802 if ((version & 0xffff0000) == 0x004e0000) { 803 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TIDR; 804 } 805 806 update_tlbie_feature_flag(version); 807 } 808 809 static void __init cpufeatures_setup_finished(void) 810 { 811 cpufeatures_cpu_quirks(); 812 813 if (hv_mode && !(cur_cpu_spec->cpu_features & CPU_FTR_HVMODE)) { 814 pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n"); 815 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 816 } 817 818 /* Make sure powerpc_base_platform is non-NULL */ 819 powerpc_base_platform = cur_cpu_spec->platform; 820 821 system_registers.lpcr = mfspr(SPRN_LPCR); 822 system_registers.hfscr = mfspr(SPRN_HFSCR); 823 system_registers.fscr = mfspr(SPRN_FSCR); 824 system_registers.pcr = mfspr(SPRN_PCR); 825 826 pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n", 827 cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); 828 } 829 830 static int __init disabled_on_cmdline(void) 831 { 832 unsigned long root, chosen; 833 const char *p; 834 835 root = of_get_flat_dt_root(); 836 chosen = of_get_flat_dt_subnode_by_name(root, "chosen"); 837 if (chosen == -FDT_ERR_NOTFOUND) 838 return false; 839 840 p = of_get_flat_dt_prop(chosen, "bootargs", NULL); 841 if (!p) 842 return false; 843 844 if (strstr(p, "dt_cpu_ftrs=off")) 845 return true; 846 847 return false; 848 } 849 850 static int __init fdt_find_cpu_features(unsigned long node, const char *uname, 851 int depth, void *data) 852 { 853 if (of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features") 854 && of_get_flat_dt_prop(node, "isa", NULL)) 855 return 1; 856 857 return 0; 858 } 859 860 bool __init dt_cpu_ftrs_in_use(void) 861 { 862 return using_dt_cpu_ftrs; 863 } 864 865 bool __init dt_cpu_ftrs_init(void *fdt) 866 { 867 using_dt_cpu_ftrs = false; 868 869 /* Setup and verify the FDT, if it fails we just bail */ 870 if (!early_init_dt_verify(fdt, __pa(fdt))) 871 return false; 872 873 if (!of_scan_flat_dt(fdt_find_cpu_features, NULL)) 874 return false; 875 876 if (disabled_on_cmdline()) 877 return false; 878 879 cpufeatures_setup_cpu(); 880 881 using_dt_cpu_ftrs = true; 882 return true; 883 } 884 885 static int nr_dt_cpu_features; 886 static struct dt_cpu_feature *dt_cpu_features; 887 888 static int __init process_cpufeatures_node(unsigned long node, 889 const char *uname, int i) 890 { 891 const __be32 *prop; 892 struct dt_cpu_feature *f; 893 int len; 894 895 f = &dt_cpu_features[i]; 896 897 f->node = node; 898 899 f->name = uname; 900 901 prop = of_get_flat_dt_prop(node, "isa", &len); 902 if (!prop) { 903 pr_warn("%s: missing isa property\n", uname); 904 return 0; 905 } 906 f->isa = be32_to_cpup(prop); 907 908 prop = of_get_flat_dt_prop(node, "usable-privilege", &len); 909 if (!prop) { 910 pr_warn("%s: missing usable-privilege property", uname); 911 return 0; 912 } 913 f->usable_privilege = be32_to_cpup(prop); 914 915 prop = of_get_flat_dt_prop(node, "hv-support", &len); 916 if (prop) 917 f->hv_support = be32_to_cpup(prop); 918 else 919 f->hv_support = HV_SUPPORT_NONE; 920 921 prop = of_get_flat_dt_prop(node, "os-support", &len); 922 if (prop) 923 f->os_support = be32_to_cpup(prop); 924 else 925 f->os_support = OS_SUPPORT_NONE; 926 927 prop = of_get_flat_dt_prop(node, "hfscr-bit-nr", &len); 928 if (prop) 929 f->hfscr_bit_nr = be32_to_cpup(prop); 930 else 931 f->hfscr_bit_nr = -1; 932 prop = of_get_flat_dt_prop(node, "fscr-bit-nr", &len); 933 if (prop) 934 f->fscr_bit_nr = be32_to_cpup(prop); 935 else 936 f->fscr_bit_nr = -1; 937 prop = of_get_flat_dt_prop(node, "hwcap-bit-nr", &len); 938 if (prop) 939 f->hwcap_bit_nr = be32_to_cpup(prop); 940 else 941 f->hwcap_bit_nr = -1; 942 943 if (f->usable_privilege & USABLE_HV) { 944 if (!(mfmsr() & MSR_HV)) { 945 pr_warn("%s: HV feature passed to guest\n", uname); 946 return 0; 947 } 948 949 if (f->hv_support == HV_SUPPORT_NONE && f->hfscr_bit_nr != -1) { 950 pr_warn("%s: unwanted hfscr_bit_nr\n", uname); 951 return 0; 952 } 953 954 if (f->hv_support == HV_SUPPORT_HFSCR) { 955 if (f->hfscr_bit_nr == -1) { 956 pr_warn("%s: missing hfscr_bit_nr\n", uname); 957 return 0; 958 } 959 } 960 } else { 961 if (f->hv_support != HV_SUPPORT_NONE || f->hfscr_bit_nr != -1) { 962 pr_warn("%s: unwanted hv_support/hfscr_bit_nr\n", uname); 963 return 0; 964 } 965 } 966 967 if (f->usable_privilege & USABLE_OS) { 968 if (f->os_support == OS_SUPPORT_NONE && f->fscr_bit_nr != -1) { 969 pr_warn("%s: unwanted fscr_bit_nr\n", uname); 970 return 0; 971 } 972 973 if (f->os_support == OS_SUPPORT_FSCR) { 974 if (f->fscr_bit_nr == -1) { 975 pr_warn("%s: missing fscr_bit_nr\n", uname); 976 return 0; 977 } 978 } 979 } else { 980 if (f->os_support != OS_SUPPORT_NONE || f->fscr_bit_nr != -1) { 981 pr_warn("%s: unwanted os_support/fscr_bit_nr\n", uname); 982 return 0; 983 } 984 } 985 986 if (!(f->usable_privilege & USABLE_PR)) { 987 if (f->hwcap_bit_nr != -1) { 988 pr_warn("%s: unwanted hwcap_bit_nr\n", uname); 989 return 0; 990 } 991 } 992 993 /* Do all the independent features in the first pass */ 994 if (!of_get_flat_dt_prop(node, "dependencies", &len)) { 995 if (cpufeatures_process_feature(f)) 996 f->enabled = 1; 997 else 998 f->disabled = 1; 999 } 1000 1001 return 0; 1002 } 1003 1004 static void __init cpufeatures_deps_enable(struct dt_cpu_feature *f) 1005 { 1006 const __be32 *prop; 1007 int len; 1008 int nr_deps; 1009 int i; 1010 1011 if (f->enabled || f->disabled) 1012 return; 1013 1014 prop = of_get_flat_dt_prop(f->node, "dependencies", &len); 1015 if (!prop) { 1016 pr_warn("%s: missing dependencies property", f->name); 1017 return; 1018 } 1019 1020 nr_deps = len / sizeof(int); 1021 1022 for (i = 0; i < nr_deps; i++) { 1023 unsigned long phandle = be32_to_cpu(prop[i]); 1024 int j; 1025 1026 for (j = 0; j < nr_dt_cpu_features; j++) { 1027 struct dt_cpu_feature *d = &dt_cpu_features[j]; 1028 1029 if (of_get_flat_dt_phandle(d->node) == phandle) { 1030 cpufeatures_deps_enable(d); 1031 if (d->disabled) { 1032 f->disabled = 1; 1033 return; 1034 } 1035 } 1036 } 1037 } 1038 1039 if (cpufeatures_process_feature(f)) 1040 f->enabled = 1; 1041 else 1042 f->disabled = 1; 1043 } 1044 1045 static int __init scan_cpufeatures_subnodes(unsigned long node, 1046 const char *uname, 1047 void *data) 1048 { 1049 int *count = data; 1050 1051 process_cpufeatures_node(node, uname, *count); 1052 1053 (*count)++; 1054 1055 return 0; 1056 } 1057 1058 static int __init count_cpufeatures_subnodes(unsigned long node, 1059 const char *uname, 1060 void *data) 1061 { 1062 int *count = data; 1063 1064 (*count)++; 1065 1066 return 0; 1067 } 1068 1069 static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char 1070 *uname, int depth, void *data) 1071 { 1072 const __be32 *prop; 1073 int count, i; 1074 u32 isa; 1075 1076 /* We are scanning "ibm,powerpc-cpu-features" nodes only */ 1077 if (!of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features")) 1078 return 0; 1079 1080 prop = of_get_flat_dt_prop(node, "isa", NULL); 1081 if (!prop) 1082 /* We checked before, "can't happen" */ 1083 return 0; 1084 1085 isa = be32_to_cpup(prop); 1086 1087 /* Count and allocate space for cpu features */ 1088 of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, 1089 &nr_dt_cpu_features); 1090 dt_cpu_features = memblock_alloc(sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE); 1091 if (!dt_cpu_features) 1092 panic("%s: Failed to allocate %zu bytes align=0x%lx\n", 1093 __func__, 1094 sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, 1095 PAGE_SIZE); 1096 1097 cpufeatures_setup_start(isa); 1098 1099 /* Scan nodes into dt_cpu_features and enable those without deps */ 1100 count = 0; 1101 of_scan_flat_dt_subnodes(node, scan_cpufeatures_subnodes, &count); 1102 1103 /* Recursive enable remaining features with dependencies */ 1104 for (i = 0; i < nr_dt_cpu_features; i++) { 1105 struct dt_cpu_feature *f = &dt_cpu_features[i]; 1106 1107 cpufeatures_deps_enable(f); 1108 } 1109 1110 prop = of_get_flat_dt_prop(node, "display-name", NULL); 1111 if (prop && strlen((char *)prop) != 0) { 1112 strscpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); 1113 cur_cpu_spec->cpu_name = dt_cpu_name; 1114 } 1115 1116 cpufeatures_setup_finished(); 1117 1118 memblock_free(dt_cpu_features, 1119 sizeof(struct dt_cpu_feature) * nr_dt_cpu_features); 1120 1121 return 0; 1122 } 1123 1124 void __init dt_cpu_ftrs_scan(void) 1125 { 1126 if (!using_dt_cpu_ftrs) 1127 return; 1128 1129 of_scan_flat_dt(dt_cpu_ftrs_scan_callback, NULL); 1130 } 1131