1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2017, Nicholas Piggin, IBM Corporation 4 */ 5 6 #define pr_fmt(fmt) "dt-cpu-ftrs: " fmt 7 8 #include <linux/export.h> 9 #include <linux/init.h> 10 #include <linux/jump_label.h> 11 #include <linux/libfdt.h> 12 #include <linux/memblock.h> 13 #include <linux/printk.h> 14 #include <linux/sched.h> 15 #include <linux/string.h> 16 #include <linux/threads.h> 17 18 #include <asm/cputable.h> 19 #include <asm/dt_cpu_ftrs.h> 20 #include <asm/mce.h> 21 #include <asm/mmu.h> 22 #include <asm/oprofile_impl.h> 23 #include <asm/prom.h> 24 #include <asm/setup.h> 25 26 27 /* Device-tree visible constants follow */ 28 #define ISA_V3_0B 3000 29 #define ISA_V3_1 3100 30 31 #define USABLE_PR (1U << 0) 32 #define USABLE_OS (1U << 1) 33 #define USABLE_HV (1U << 2) 34 35 #define HV_SUPPORT_HFSCR (1U << 0) 36 #define OS_SUPPORT_FSCR (1U << 0) 37 38 /* For parsing, we define all bits set as "NONE" case */ 39 #define HV_SUPPORT_NONE 0xffffffffU 40 #define OS_SUPPORT_NONE 0xffffffffU 41 42 struct dt_cpu_feature { 43 const char *name; 44 uint32_t isa; 45 uint32_t usable_privilege; 46 uint32_t hv_support; 47 uint32_t os_support; 48 uint32_t hfscr_bit_nr; 49 uint32_t fscr_bit_nr; 50 uint32_t hwcap_bit_nr; 51 /* fdt parsing */ 52 unsigned long node; 53 int enabled; 54 int disabled; 55 }; 56 57 #define MMU_FTRS_HASH_BASE (MMU_FTRS_POWER8) 58 59 #define COMMON_USER_BASE (PPC_FEATURE_32 | PPC_FEATURE_64 | \ 60 PPC_FEATURE_ARCH_2_06 |\ 61 PPC_FEATURE_ICACHE_SNOOP) 62 #define COMMON_USER2_BASE (PPC_FEATURE2_ARCH_2_07 | \ 63 PPC_FEATURE2_ISEL) 64 /* 65 * Set up the base CPU 66 */ 67 68 static int hv_mode; 69 70 static struct { 71 u64 lpcr; 72 u64 hfscr; 73 u64 fscr; 74 u64 pcr; 75 } system_registers; 76 77 static void (*init_pmu_registers)(void); 78 79 static void __restore_cpu_cpufeatures(void) 80 { 81 mtspr(SPRN_LPCR, system_registers.lpcr); 82 if (hv_mode) { 83 mtspr(SPRN_LPID, 0); 84 mtspr(SPRN_HFSCR, system_registers.hfscr); 85 mtspr(SPRN_PCR, system_registers.pcr); 86 } 87 mtspr(SPRN_FSCR, system_registers.fscr); 88 89 if (init_pmu_registers) 90 init_pmu_registers(); 91 } 92 93 static char dt_cpu_name[64]; 94 95 static struct cpu_spec __initdata base_cpu_spec = { 96 .cpu_name = NULL, 97 .cpu_features = CPU_FTRS_DT_CPU_BASE, 98 .cpu_user_features = COMMON_USER_BASE, 99 .cpu_user_features2 = COMMON_USER2_BASE, 100 .mmu_features = 0, 101 .icache_bsize = 32, /* minimum block size, fixed by */ 102 .dcache_bsize = 32, /* cache info init. */ 103 .num_pmcs = 0, 104 .pmc_type = PPC_PMC_DEFAULT, 105 .oprofile_cpu_type = NULL, 106 .oprofile_type = PPC_OPROFILE_INVALID, 107 .cpu_setup = NULL, 108 .cpu_restore = __restore_cpu_cpufeatures, 109 .machine_check_early = NULL, 110 .platform = NULL, 111 }; 112 113 static void __init cpufeatures_setup_cpu(void) 114 { 115 set_cur_cpu_spec(&base_cpu_spec); 116 117 cur_cpu_spec->pvr_mask = -1; 118 cur_cpu_spec->pvr_value = mfspr(SPRN_PVR); 119 120 /* Initialize the base environment -- clear FSCR/HFSCR. */ 121 hv_mode = !!(mfmsr() & MSR_HV); 122 if (hv_mode) { 123 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 124 mtspr(SPRN_HFSCR, 0); 125 } 126 mtspr(SPRN_FSCR, 0); 127 mtspr(SPRN_PCR, PCR_MASK); 128 129 /* 130 * LPCR does not get cleared, to match behaviour with secondaries 131 * in __restore_cpu_cpufeatures. Once the idle code is fixed, this 132 * could clear LPCR too. 133 */ 134 } 135 136 static int __init feat_try_enable_unknown(struct dt_cpu_feature *f) 137 { 138 if (f->hv_support == HV_SUPPORT_NONE) { 139 } else if (f->hv_support & HV_SUPPORT_HFSCR) { 140 u64 hfscr = mfspr(SPRN_HFSCR); 141 hfscr |= 1UL << f->hfscr_bit_nr; 142 mtspr(SPRN_HFSCR, hfscr); 143 } else { 144 /* Does not have a known recipe */ 145 return 0; 146 } 147 148 if (f->os_support == OS_SUPPORT_NONE) { 149 } else if (f->os_support & OS_SUPPORT_FSCR) { 150 u64 fscr = mfspr(SPRN_FSCR); 151 fscr |= 1UL << f->fscr_bit_nr; 152 mtspr(SPRN_FSCR, fscr); 153 } else { 154 /* Does not have a known recipe */ 155 return 0; 156 } 157 158 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 159 uint32_t word = f->hwcap_bit_nr / 32; 160 uint32_t bit = f->hwcap_bit_nr % 32; 161 162 if (word == 0) 163 cur_cpu_spec->cpu_user_features |= 1U << bit; 164 else if (word == 1) 165 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 166 else 167 pr_err("%s could not advertise to user (no hwcap bits)\n", f->name); 168 } 169 170 return 1; 171 } 172 173 static int __init feat_enable(struct dt_cpu_feature *f) 174 { 175 if (f->hv_support != HV_SUPPORT_NONE) { 176 if (f->hfscr_bit_nr != -1) { 177 u64 hfscr = mfspr(SPRN_HFSCR); 178 hfscr |= 1UL << f->hfscr_bit_nr; 179 mtspr(SPRN_HFSCR, hfscr); 180 } 181 } 182 183 if (f->os_support != OS_SUPPORT_NONE) { 184 if (f->fscr_bit_nr != -1) { 185 u64 fscr = mfspr(SPRN_FSCR); 186 fscr |= 1UL << f->fscr_bit_nr; 187 mtspr(SPRN_FSCR, fscr); 188 } 189 } 190 191 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 192 uint32_t word = f->hwcap_bit_nr / 32; 193 uint32_t bit = f->hwcap_bit_nr % 32; 194 195 if (word == 0) 196 cur_cpu_spec->cpu_user_features |= 1U << bit; 197 else if (word == 1) 198 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 199 else 200 pr_err("CPU feature: %s could not advertise to user (no hwcap bits)\n", f->name); 201 } 202 203 return 1; 204 } 205 206 static int __init feat_disable(struct dt_cpu_feature *f) 207 { 208 return 0; 209 } 210 211 static int __init feat_enable_hv(struct dt_cpu_feature *f) 212 { 213 u64 lpcr; 214 215 if (!hv_mode) { 216 pr_err("CPU feature hypervisor present in device tree but HV mode not enabled in the CPU. Ignoring.\n"); 217 return 0; 218 } 219 220 mtspr(SPRN_LPID, 0); 221 222 lpcr = mfspr(SPRN_LPCR); 223 lpcr &= ~LPCR_LPES0; /* HV external interrupts */ 224 mtspr(SPRN_LPCR, lpcr); 225 226 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 227 228 return 1; 229 } 230 231 static int __init feat_enable_le(struct dt_cpu_feature *f) 232 { 233 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_TRUE_LE; 234 return 1; 235 } 236 237 static int __init feat_enable_smt(struct dt_cpu_feature *f) 238 { 239 cur_cpu_spec->cpu_features |= CPU_FTR_SMT; 240 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_SMT; 241 return 1; 242 } 243 244 static int __init feat_enable_idle_nap(struct dt_cpu_feature *f) 245 { 246 u64 lpcr; 247 248 /* Set PECE wakeup modes for ISA 207 */ 249 lpcr = mfspr(SPRN_LPCR); 250 lpcr |= LPCR_PECE0; 251 lpcr |= LPCR_PECE1; 252 lpcr |= LPCR_PECE2; 253 mtspr(SPRN_LPCR, lpcr); 254 255 return 1; 256 } 257 258 static int __init feat_enable_idle_stop(struct dt_cpu_feature *f) 259 { 260 u64 lpcr; 261 262 /* Set PECE wakeup modes for ISAv3.0B */ 263 lpcr = mfspr(SPRN_LPCR); 264 lpcr |= LPCR_PECE0; 265 lpcr |= LPCR_PECE1; 266 lpcr |= LPCR_PECE2; 267 mtspr(SPRN_LPCR, lpcr); 268 269 return 1; 270 } 271 272 static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) 273 { 274 u64 lpcr; 275 276 lpcr = mfspr(SPRN_LPCR); 277 lpcr &= ~LPCR_ISL; 278 279 /* VRMASD */ 280 lpcr |= LPCR_VPM0; 281 lpcr &= ~LPCR_VPM1; 282 lpcr |= 0x10UL << LPCR_VRMASD_SH; /* L=1 LP=00 */ 283 mtspr(SPRN_LPCR, lpcr); 284 285 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 286 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 287 288 return 1; 289 } 290 291 static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) 292 { 293 u64 lpcr; 294 295 lpcr = mfspr(SPRN_LPCR); 296 lpcr &= ~(LPCR_ISL | LPCR_UPRT | LPCR_HR); 297 mtspr(SPRN_LPCR, lpcr); 298 299 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 300 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 301 302 return 1; 303 } 304 305 306 static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) 307 { 308 #ifdef CONFIG_PPC_RADIX_MMU 309 cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; 310 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 311 cur_cpu_spec->mmu_features |= MMU_FTR_GTSE; 312 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 313 314 return 1; 315 #endif 316 return 0; 317 } 318 319 static int __init feat_enable_dscr(struct dt_cpu_feature *f) 320 { 321 u64 lpcr; 322 323 /* 324 * Linux relies on FSCR[DSCR] being clear, so that we can take the 325 * facility unavailable interrupt and track the task's usage of DSCR. 326 * See facility_unavailable_exception(). 327 * Clear the bit here so that feat_enable() doesn't set it. 328 */ 329 f->fscr_bit_nr = -1; 330 331 feat_enable(f); 332 333 lpcr = mfspr(SPRN_LPCR); 334 lpcr &= ~LPCR_DPFD; 335 lpcr |= (4UL << LPCR_DPFD_SH); 336 mtspr(SPRN_LPCR, lpcr); 337 338 return 1; 339 } 340 341 static void hfscr_pmu_enable(void) 342 { 343 u64 hfscr = mfspr(SPRN_HFSCR); 344 hfscr |= PPC_BIT(60); 345 mtspr(SPRN_HFSCR, hfscr); 346 } 347 348 static void init_pmu_power8(void) 349 { 350 if (hv_mode) { 351 mtspr(SPRN_MMCRC, 0); 352 mtspr(SPRN_MMCRH, 0); 353 } 354 355 mtspr(SPRN_MMCRA, 0); 356 mtspr(SPRN_MMCR0, 0); 357 mtspr(SPRN_MMCR1, 0); 358 mtspr(SPRN_MMCR2, 0); 359 mtspr(SPRN_MMCRS, 0); 360 } 361 362 static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) 363 { 364 cur_cpu_spec->platform = "power8"; 365 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; 366 367 return 1; 368 } 369 370 static int __init feat_enable_pmu_power8(struct dt_cpu_feature *f) 371 { 372 hfscr_pmu_enable(); 373 374 init_pmu_power8(); 375 init_pmu_registers = init_pmu_power8; 376 377 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 378 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 379 if (pvr_version_is(PVR_POWER8E)) 380 cur_cpu_spec->cpu_features |= CPU_FTR_PMAO_BUG; 381 382 cur_cpu_spec->num_pmcs = 6; 383 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 384 cur_cpu_spec->oprofile_cpu_type = "ppc64/power8"; 385 386 return 1; 387 } 388 389 static void init_pmu_power9(void) 390 { 391 if (hv_mode) 392 mtspr(SPRN_MMCRC, 0); 393 394 mtspr(SPRN_MMCRA, 0); 395 mtspr(SPRN_MMCR0, 0); 396 mtspr(SPRN_MMCR1, 0); 397 mtspr(SPRN_MMCR2, 0); 398 } 399 400 static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) 401 { 402 cur_cpu_spec->platform = "power9"; 403 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; 404 405 return 1; 406 } 407 408 static int __init feat_enable_pmu_power9(struct dt_cpu_feature *f) 409 { 410 hfscr_pmu_enable(); 411 412 init_pmu_power9(); 413 init_pmu_registers = init_pmu_power9; 414 415 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 416 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 417 418 cur_cpu_spec->num_pmcs = 6; 419 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 420 cur_cpu_spec->oprofile_cpu_type = "ppc64/power9"; 421 422 return 1; 423 } 424 425 static void init_pmu_power10(void) 426 { 427 init_pmu_power9(); 428 429 mtspr(SPRN_MMCR3, 0); 430 mtspr(SPRN_MMCRA, MMCRA_BHRB_DISABLE); 431 mtspr(SPRN_MMCR0, MMCR0_PMCCEXT); 432 } 433 434 static int __init feat_enable_pmu_power10(struct dt_cpu_feature *f) 435 { 436 hfscr_pmu_enable(); 437 438 init_pmu_power10(); 439 init_pmu_registers = init_pmu_power10; 440 441 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 442 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 443 444 cur_cpu_spec->num_pmcs = 6; 445 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 446 cur_cpu_spec->oprofile_cpu_type = "ppc64/power10"; 447 448 return 1; 449 } 450 451 static int __init feat_enable_mce_power10(struct dt_cpu_feature *f) 452 { 453 cur_cpu_spec->platform = "power10"; 454 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; 455 456 return 1; 457 } 458 459 static int __init feat_enable_tm(struct dt_cpu_feature *f) 460 { 461 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 462 feat_enable(f); 463 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_HTM_NOSC; 464 return 1; 465 #endif 466 return 0; 467 } 468 469 static int __init feat_enable_fp(struct dt_cpu_feature *f) 470 { 471 feat_enable(f); 472 cur_cpu_spec->cpu_features &= ~CPU_FTR_FPU_UNAVAILABLE; 473 474 return 1; 475 } 476 477 static int __init feat_enable_vector(struct dt_cpu_feature *f) 478 { 479 #ifdef CONFIG_ALTIVEC 480 feat_enable(f); 481 cur_cpu_spec->cpu_features |= CPU_FTR_ALTIVEC; 482 cur_cpu_spec->cpu_features |= CPU_FTR_VMX_COPY; 483 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_ALTIVEC; 484 485 return 1; 486 #endif 487 return 0; 488 } 489 490 static int __init feat_enable_vsx(struct dt_cpu_feature *f) 491 { 492 #ifdef CONFIG_VSX 493 feat_enable(f); 494 cur_cpu_spec->cpu_features |= CPU_FTR_VSX; 495 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_VSX; 496 497 return 1; 498 #endif 499 return 0; 500 } 501 502 static int __init feat_enable_purr(struct dt_cpu_feature *f) 503 { 504 cur_cpu_spec->cpu_features |= CPU_FTR_PURR | CPU_FTR_SPURR; 505 506 return 1; 507 } 508 509 static int __init feat_enable_ebb(struct dt_cpu_feature *f) 510 { 511 /* 512 * PPC_FEATURE2_EBB is enabled in PMU init code because it has 513 * historically been related to the PMU facility. This may have 514 * to be decoupled if EBB becomes more generic. For now, follow 515 * existing convention. 516 */ 517 f->hwcap_bit_nr = -1; 518 feat_enable(f); 519 520 return 1; 521 } 522 523 static int __init feat_enable_dbell(struct dt_cpu_feature *f) 524 { 525 u64 lpcr; 526 527 /* P9 has an HFSCR for privileged state */ 528 feat_enable(f); 529 530 cur_cpu_spec->cpu_features |= CPU_FTR_DBELL; 531 532 lpcr = mfspr(SPRN_LPCR); 533 lpcr |= LPCR_PECEDH; /* hyp doorbell wakeup */ 534 mtspr(SPRN_LPCR, lpcr); 535 536 return 1; 537 } 538 539 static int __init feat_enable_hvi(struct dt_cpu_feature *f) 540 { 541 u64 lpcr; 542 543 /* 544 * POWER9 XIVE interrupts including in OPAL XICS compatibility 545 * are always delivered as hypervisor virtualization interrupts (HVI) 546 * rather than EE. 547 * 548 * However LPES0 is not set here, in the chance that an EE does get 549 * delivered to the host somehow, the EE handler would not expect it 550 * to be delivered in LPES0 mode (e.g., using SRR[01]). This could 551 * happen if there is a bug in interrupt controller code, or IC is 552 * misconfigured in systemsim. 553 */ 554 555 lpcr = mfspr(SPRN_LPCR); 556 lpcr |= LPCR_HVICE; /* enable hvi interrupts */ 557 lpcr |= LPCR_HEIC; /* disable ee interrupts when MSR_HV */ 558 lpcr |= LPCR_PECE_HVEE; /* hvi can wake from stop */ 559 mtspr(SPRN_LPCR, lpcr); 560 561 return 1; 562 } 563 564 static int __init feat_enable_large_ci(struct dt_cpu_feature *f) 565 { 566 cur_cpu_spec->mmu_features |= MMU_FTR_CI_LARGE_PAGE; 567 568 return 1; 569 } 570 571 static int __init feat_enable_mma(struct dt_cpu_feature *f) 572 { 573 u64 pcr; 574 575 feat_enable(f); 576 pcr = mfspr(SPRN_PCR); 577 pcr &= ~PCR_MMA_DIS; 578 mtspr(SPRN_PCR, pcr); 579 580 return 1; 581 } 582 583 struct dt_cpu_feature_match { 584 const char *name; 585 int (*enable)(struct dt_cpu_feature *f); 586 u64 cpu_ftr_bit_mask; 587 }; 588 589 static struct dt_cpu_feature_match __initdata 590 dt_cpu_feature_match_table[] = { 591 {"hypervisor", feat_enable_hv, 0}, 592 {"big-endian", feat_enable, 0}, 593 {"little-endian", feat_enable_le, CPU_FTR_REAL_LE}, 594 {"smt", feat_enable_smt, 0}, 595 {"interrupt-facilities", feat_enable, 0}, 596 {"system-call-vectored", feat_enable, 0}, 597 {"timer-facilities", feat_enable, 0}, 598 {"timer-facilities-v3", feat_enable, 0}, 599 {"debug-facilities", feat_enable, 0}, 600 {"come-from-address-register", feat_enable, CPU_FTR_CFAR}, 601 {"branch-tracing", feat_enable, 0}, 602 {"floating-point", feat_enable_fp, 0}, 603 {"vector", feat_enable_vector, 0}, 604 {"vector-scalar", feat_enable_vsx, 0}, 605 {"vector-scalar-v3", feat_enable, 0}, 606 {"decimal-floating-point", feat_enable, 0}, 607 {"decimal-integer", feat_enable, 0}, 608 {"quadword-load-store", feat_enable, 0}, 609 {"vector-crypto", feat_enable, 0}, 610 {"mmu-hash", feat_enable_mmu_hash, 0}, 611 {"mmu-radix", feat_enable_mmu_radix, 0}, 612 {"mmu-hash-v3", feat_enable_mmu_hash_v3, 0}, 613 {"virtual-page-class-key-protection", feat_enable, 0}, 614 {"transactional-memory", feat_enable_tm, CPU_FTR_TM}, 615 {"transactional-memory-v3", feat_enable_tm, 0}, 616 {"tm-suspend-hypervisor-assist", feat_enable, CPU_FTR_P9_TM_HV_ASSIST}, 617 {"tm-suspend-xer-so-bug", feat_enable, CPU_FTR_P9_TM_XER_SO_BUG}, 618 {"idle-nap", feat_enable_idle_nap, 0}, 619 /* alignment-interrupt-dsisr ignored */ 620 {"idle-stop", feat_enable_idle_stop, 0}, 621 {"machine-check-power8", feat_enable_mce_power8, 0}, 622 {"performance-monitor-power8", feat_enable_pmu_power8, 0}, 623 {"data-stream-control-register", feat_enable_dscr, CPU_FTR_DSCR}, 624 {"event-based-branch", feat_enable_ebb, 0}, 625 {"target-address-register", feat_enable, 0}, 626 {"branch-history-rolling-buffer", feat_enable, 0}, 627 {"control-register", feat_enable, CPU_FTR_CTRL}, 628 {"processor-control-facility", feat_enable_dbell, CPU_FTR_DBELL}, 629 {"processor-control-facility-v3", feat_enable_dbell, CPU_FTR_DBELL}, 630 {"processor-utilization-of-resources-register", feat_enable_purr, 0}, 631 {"no-execute", feat_enable, 0}, 632 {"strong-access-ordering", feat_enable, CPU_FTR_SAO}, 633 {"cache-inhibited-large-page", feat_enable_large_ci, 0}, 634 {"coprocessor-icswx", feat_enable, 0}, 635 {"hypervisor-virtualization-interrupt", feat_enable_hvi, 0}, 636 {"program-priority-register", feat_enable, CPU_FTR_HAS_PPR}, 637 {"wait", feat_enable, 0}, 638 {"atomic-memory-operations", feat_enable, 0}, 639 {"branch-v3", feat_enable, 0}, 640 {"copy-paste", feat_enable, 0}, 641 {"decimal-floating-point-v3", feat_enable, 0}, 642 {"decimal-integer-v3", feat_enable, 0}, 643 {"fixed-point-v3", feat_enable, 0}, 644 {"floating-point-v3", feat_enable, 0}, 645 {"group-start-register", feat_enable, 0}, 646 {"pc-relative-addressing", feat_enable, 0}, 647 {"machine-check-power9", feat_enable_mce_power9, 0}, 648 {"machine-check-power10", feat_enable_mce_power10, 0}, 649 {"performance-monitor-power9", feat_enable_pmu_power9, 0}, 650 {"performance-monitor-power10", feat_enable_pmu_power10, 0}, 651 {"event-based-branch-v3", feat_enable, 0}, 652 {"random-number-generator", feat_enable, 0}, 653 {"system-call-vectored", feat_disable, 0}, 654 {"trace-interrupt-v3", feat_enable, 0}, 655 {"vector-v3", feat_enable, 0}, 656 {"vector-binary128", feat_enable, 0}, 657 {"vector-binary16", feat_enable, 0}, 658 {"wait-v3", feat_enable, 0}, 659 {"prefix-instructions", feat_enable, 0}, 660 {"matrix-multiply-assist", feat_enable_mma, 0}, 661 {"debug-facilities-v31", feat_enable, CPU_FTR_DAWR1}, 662 }; 663 664 static bool __initdata using_dt_cpu_ftrs; 665 static bool __initdata enable_unknown = true; 666 667 static int __init dt_cpu_ftrs_parse(char *str) 668 { 669 if (!str) 670 return 0; 671 672 if (!strcmp(str, "off")) 673 using_dt_cpu_ftrs = false; 674 else if (!strcmp(str, "known")) 675 enable_unknown = false; 676 else 677 return 1; 678 679 return 0; 680 } 681 early_param("dt_cpu_ftrs", dt_cpu_ftrs_parse); 682 683 static void __init cpufeatures_setup_start(u32 isa) 684 { 685 pr_info("setup for ISA %d\n", isa); 686 687 if (isa >= ISA_V3_0B) { 688 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300; 689 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00; 690 } 691 692 if (isa >= ISA_V3_1) { 693 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31; 694 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1; 695 } 696 } 697 698 static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f) 699 { 700 const struct dt_cpu_feature_match *m; 701 bool known = false; 702 int i; 703 704 for (i = 0; i < ARRAY_SIZE(dt_cpu_feature_match_table); i++) { 705 m = &dt_cpu_feature_match_table[i]; 706 if (!strcmp(f->name, m->name)) { 707 known = true; 708 if (m->enable(f)) { 709 cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask; 710 break; 711 } 712 713 pr_info("not enabling: %s (disabled or unsupported by kernel)\n", 714 f->name); 715 return false; 716 } 717 } 718 719 if (!known && (!enable_unknown || !feat_try_enable_unknown(f))) { 720 pr_info("not enabling: %s (unknown and unsupported by kernel)\n", 721 f->name); 722 return false; 723 } 724 725 if (known) 726 pr_debug("enabling: %s\n", f->name); 727 else 728 pr_debug("enabling: %s (unknown)\n", f->name); 729 730 return true; 731 } 732 733 /* 734 * Handle POWER9 broadcast tlbie invalidation issue using 735 * cpu feature flag. 736 */ 737 static __init void update_tlbie_feature_flag(unsigned long pvr) 738 { 739 if (PVR_VER(pvr) == PVR_POWER9) { 740 /* 741 * Set the tlbie feature flag for anything below 742 * Nimbus DD 2.3 and Cumulus DD 1.3 743 */ 744 if ((pvr & 0xe000) == 0) { 745 /* Nimbus */ 746 if ((pvr & 0xfff) < 0x203) 747 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 748 } else if ((pvr & 0xc000) == 0) { 749 /* Cumulus */ 750 if ((pvr & 0xfff) < 0x103) 751 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 752 } else { 753 WARN_ONCE(1, "Unknown PVR"); 754 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 755 } 756 757 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_ERAT_BUG; 758 } 759 } 760 761 static __init void cpufeatures_cpu_quirks(void) 762 { 763 unsigned long version = mfspr(SPRN_PVR); 764 765 /* 766 * Not all quirks can be derived from the cpufeatures device tree. 767 */ 768 if ((version & 0xffffefff) == 0x004e0200) { 769 /* DD2.0 has no feature flag */ 770 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 771 } else if ((version & 0xffffefff) == 0x004e0201) { 772 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 773 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 774 } else if ((version & 0xffffefff) == 0x004e0202) { 775 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; 776 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; 777 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 778 } else if ((version & 0xffff0000) == 0x004e0000) { 779 /* DD2.1 and up have DD2_1 */ 780 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 781 } 782 783 if ((version & 0xffff0000) == 0x004e0000) { 784 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 785 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TIDR; 786 } 787 788 update_tlbie_feature_flag(version); 789 } 790 791 static void __init cpufeatures_setup_finished(void) 792 { 793 cpufeatures_cpu_quirks(); 794 795 if (hv_mode && !(cur_cpu_spec->cpu_features & CPU_FTR_HVMODE)) { 796 pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n"); 797 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 798 } 799 800 /* Make sure powerpc_base_platform is non-NULL */ 801 powerpc_base_platform = cur_cpu_spec->platform; 802 803 system_registers.lpcr = mfspr(SPRN_LPCR); 804 system_registers.hfscr = mfspr(SPRN_HFSCR); 805 system_registers.fscr = mfspr(SPRN_FSCR); 806 system_registers.pcr = mfspr(SPRN_PCR); 807 808 pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n", 809 cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); 810 } 811 812 static int __init disabled_on_cmdline(void) 813 { 814 unsigned long root, chosen; 815 const char *p; 816 817 root = of_get_flat_dt_root(); 818 chosen = of_get_flat_dt_subnode_by_name(root, "chosen"); 819 if (chosen == -FDT_ERR_NOTFOUND) 820 return false; 821 822 p = of_get_flat_dt_prop(chosen, "bootargs", NULL); 823 if (!p) 824 return false; 825 826 if (strstr(p, "dt_cpu_ftrs=off")) 827 return true; 828 829 return false; 830 } 831 832 static int __init fdt_find_cpu_features(unsigned long node, const char *uname, 833 int depth, void *data) 834 { 835 if (of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features") 836 && of_get_flat_dt_prop(node, "isa", NULL)) 837 return 1; 838 839 return 0; 840 } 841 842 bool __init dt_cpu_ftrs_in_use(void) 843 { 844 return using_dt_cpu_ftrs; 845 } 846 847 bool __init dt_cpu_ftrs_init(void *fdt) 848 { 849 using_dt_cpu_ftrs = false; 850 851 /* Setup and verify the FDT, if it fails we just bail */ 852 if (!early_init_dt_verify(fdt)) 853 return false; 854 855 if (!of_scan_flat_dt(fdt_find_cpu_features, NULL)) 856 return false; 857 858 if (disabled_on_cmdline()) 859 return false; 860 861 cpufeatures_setup_cpu(); 862 863 using_dt_cpu_ftrs = true; 864 return true; 865 } 866 867 static int nr_dt_cpu_features; 868 static struct dt_cpu_feature *dt_cpu_features; 869 870 static int __init process_cpufeatures_node(unsigned long node, 871 const char *uname, int i) 872 { 873 const __be32 *prop; 874 struct dt_cpu_feature *f; 875 int len; 876 877 f = &dt_cpu_features[i]; 878 879 f->node = node; 880 881 f->name = uname; 882 883 prop = of_get_flat_dt_prop(node, "isa", &len); 884 if (!prop) { 885 pr_warn("%s: missing isa property\n", uname); 886 return 0; 887 } 888 f->isa = be32_to_cpup(prop); 889 890 prop = of_get_flat_dt_prop(node, "usable-privilege", &len); 891 if (!prop) { 892 pr_warn("%s: missing usable-privilege property", uname); 893 return 0; 894 } 895 f->usable_privilege = be32_to_cpup(prop); 896 897 prop = of_get_flat_dt_prop(node, "hv-support", &len); 898 if (prop) 899 f->hv_support = be32_to_cpup(prop); 900 else 901 f->hv_support = HV_SUPPORT_NONE; 902 903 prop = of_get_flat_dt_prop(node, "os-support", &len); 904 if (prop) 905 f->os_support = be32_to_cpup(prop); 906 else 907 f->os_support = OS_SUPPORT_NONE; 908 909 prop = of_get_flat_dt_prop(node, "hfscr-bit-nr", &len); 910 if (prop) 911 f->hfscr_bit_nr = be32_to_cpup(prop); 912 else 913 f->hfscr_bit_nr = -1; 914 prop = of_get_flat_dt_prop(node, "fscr-bit-nr", &len); 915 if (prop) 916 f->fscr_bit_nr = be32_to_cpup(prop); 917 else 918 f->fscr_bit_nr = -1; 919 prop = of_get_flat_dt_prop(node, "hwcap-bit-nr", &len); 920 if (prop) 921 f->hwcap_bit_nr = be32_to_cpup(prop); 922 else 923 f->hwcap_bit_nr = -1; 924 925 if (f->usable_privilege & USABLE_HV) { 926 if (!(mfmsr() & MSR_HV)) { 927 pr_warn("%s: HV feature passed to guest\n", uname); 928 return 0; 929 } 930 931 if (f->hv_support == HV_SUPPORT_NONE && f->hfscr_bit_nr != -1) { 932 pr_warn("%s: unwanted hfscr_bit_nr\n", uname); 933 return 0; 934 } 935 936 if (f->hv_support == HV_SUPPORT_HFSCR) { 937 if (f->hfscr_bit_nr == -1) { 938 pr_warn("%s: missing hfscr_bit_nr\n", uname); 939 return 0; 940 } 941 } 942 } else { 943 if (f->hv_support != HV_SUPPORT_NONE || f->hfscr_bit_nr != -1) { 944 pr_warn("%s: unwanted hv_support/hfscr_bit_nr\n", uname); 945 return 0; 946 } 947 } 948 949 if (f->usable_privilege & USABLE_OS) { 950 if (f->os_support == OS_SUPPORT_NONE && f->fscr_bit_nr != -1) { 951 pr_warn("%s: unwanted fscr_bit_nr\n", uname); 952 return 0; 953 } 954 955 if (f->os_support == OS_SUPPORT_FSCR) { 956 if (f->fscr_bit_nr == -1) { 957 pr_warn("%s: missing fscr_bit_nr\n", uname); 958 return 0; 959 } 960 } 961 } else { 962 if (f->os_support != OS_SUPPORT_NONE || f->fscr_bit_nr != -1) { 963 pr_warn("%s: unwanted os_support/fscr_bit_nr\n", uname); 964 return 0; 965 } 966 } 967 968 if (!(f->usable_privilege & USABLE_PR)) { 969 if (f->hwcap_bit_nr != -1) { 970 pr_warn("%s: unwanted hwcap_bit_nr\n", uname); 971 return 0; 972 } 973 } 974 975 /* Do all the independent features in the first pass */ 976 if (!of_get_flat_dt_prop(node, "dependencies", &len)) { 977 if (cpufeatures_process_feature(f)) 978 f->enabled = 1; 979 else 980 f->disabled = 1; 981 } 982 983 return 0; 984 } 985 986 static void __init cpufeatures_deps_enable(struct dt_cpu_feature *f) 987 { 988 const __be32 *prop; 989 int len; 990 int nr_deps; 991 int i; 992 993 if (f->enabled || f->disabled) 994 return; 995 996 prop = of_get_flat_dt_prop(f->node, "dependencies", &len); 997 if (!prop) { 998 pr_warn("%s: missing dependencies property", f->name); 999 return; 1000 } 1001 1002 nr_deps = len / sizeof(int); 1003 1004 for (i = 0; i < nr_deps; i++) { 1005 unsigned long phandle = be32_to_cpu(prop[i]); 1006 int j; 1007 1008 for (j = 0; j < nr_dt_cpu_features; j++) { 1009 struct dt_cpu_feature *d = &dt_cpu_features[j]; 1010 1011 if (of_get_flat_dt_phandle(d->node) == phandle) { 1012 cpufeatures_deps_enable(d); 1013 if (d->disabled) { 1014 f->disabled = 1; 1015 return; 1016 } 1017 } 1018 } 1019 } 1020 1021 if (cpufeatures_process_feature(f)) 1022 f->enabled = 1; 1023 else 1024 f->disabled = 1; 1025 } 1026 1027 static int __init scan_cpufeatures_subnodes(unsigned long node, 1028 const char *uname, 1029 void *data) 1030 { 1031 int *count = data; 1032 1033 process_cpufeatures_node(node, uname, *count); 1034 1035 (*count)++; 1036 1037 return 0; 1038 } 1039 1040 static int __init count_cpufeatures_subnodes(unsigned long node, 1041 const char *uname, 1042 void *data) 1043 { 1044 int *count = data; 1045 1046 (*count)++; 1047 1048 return 0; 1049 } 1050 1051 static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char 1052 *uname, int depth, void *data) 1053 { 1054 const __be32 *prop; 1055 int count, i; 1056 u32 isa; 1057 1058 /* We are scanning "ibm,powerpc-cpu-features" nodes only */ 1059 if (!of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features")) 1060 return 0; 1061 1062 prop = of_get_flat_dt_prop(node, "isa", NULL); 1063 if (!prop) 1064 /* We checked before, "can't happen" */ 1065 return 0; 1066 1067 isa = be32_to_cpup(prop); 1068 1069 /* Count and allocate space for cpu features */ 1070 of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, 1071 &nr_dt_cpu_features); 1072 dt_cpu_features = memblock_alloc(sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE); 1073 if (!dt_cpu_features) 1074 panic("%s: Failed to allocate %zu bytes align=0x%lx\n", 1075 __func__, 1076 sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, 1077 PAGE_SIZE); 1078 1079 cpufeatures_setup_start(isa); 1080 1081 /* Scan nodes into dt_cpu_features and enable those without deps */ 1082 count = 0; 1083 of_scan_flat_dt_subnodes(node, scan_cpufeatures_subnodes, &count); 1084 1085 /* Recursive enable remaining features with dependencies */ 1086 for (i = 0; i < nr_dt_cpu_features; i++) { 1087 struct dt_cpu_feature *f = &dt_cpu_features[i]; 1088 1089 cpufeatures_deps_enable(f); 1090 } 1091 1092 prop = of_get_flat_dt_prop(node, "display-name", NULL); 1093 if (prop && strlen((char *)prop) != 0) { 1094 strlcpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); 1095 cur_cpu_spec->cpu_name = dt_cpu_name; 1096 } 1097 1098 cpufeatures_setup_finished(); 1099 1100 memblock_free(__pa(dt_cpu_features), 1101 sizeof(struct dt_cpu_feature)*nr_dt_cpu_features); 1102 1103 return 0; 1104 } 1105 1106 void __init dt_cpu_ftrs_scan(void) 1107 { 1108 if (!using_dt_cpu_ftrs) 1109 return; 1110 1111 of_scan_flat_dt(dt_cpu_ftrs_scan_callback, NULL); 1112 } 1113