1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copied from arch/arm64/kernel/cpufeature.c 4 * 5 * Copyright (C) 2015 ARM Ltd. 6 * Copyright (C) 2017 SiFive 7 */ 8 9 #include <linux/acpi.h> 10 #include <linux/bitmap.h> 11 #include <linux/cpu.h> 12 #include <linux/cpuhotplug.h> 13 #include <linux/ctype.h> 14 #include <linux/log2.h> 15 #include <linux/memory.h> 16 #include <linux/module.h> 17 #include <linux/of.h> 18 #include <asm/acpi.h> 19 #include <asm/alternative.h> 20 #include <asm/bugs.h> 21 #include <asm/cacheflush.h> 22 #include <asm/cpufeature.h> 23 #include <asm/hwcap.h> 24 #include <asm/text-patching.h> 25 #include <asm/hwprobe.h> 26 #include <asm/processor.h> 27 #include <asm/sbi.h> 28 #include <asm/vector.h> 29 #include <asm/vendor_extensions.h> 30 #include <asm/vendor_extensions/thead.h> 31 #include <asm/usercfi.h> 32 33 #define NUM_ALPHA_EXTS ('z' - 'a' + 1) 34 35 static bool any_cpu_has_zicboz; 36 static bool any_cpu_has_zicbop; 37 static bool any_cpu_has_zicbom; 38 39 unsigned long elf_hwcap __read_mostly; 40 41 /* Host ISA bitmap */ 42 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; 43 44 /* Per-cpu ISA extensions. */ 45 struct riscv_isainfo hart_isa[NR_CPUS]; 46 47 u32 thead_vlenb_of; 48 49 /** 50 * riscv_isa_extension_base() - Get base extension word 51 * 52 * @isa_bitmap: ISA bitmap to use 53 * Return: base extension word as unsigned long value 54 * 55 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 56 */ 57 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) 58 { 59 return !isa_bitmap ? riscv_isa[0] : isa_bitmap[0]; 60 } 61 EXPORT_SYMBOL_GPL(riscv_isa_extension_base); 62 63 /** 64 * __riscv_isa_extension_available() - Check whether given extension 65 * is available or not 66 * 67 * @isa_bitmap: ISA bitmap to use 68 * @bit: bit position of the desired extension 69 * Return: true or false 70 * 71 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 72 */ 73 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit) 74 { 75 const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; 76 77 if (bit >= RISCV_ISA_EXT_MAX) 78 return false; 79 80 return test_bit(bit, bmap); 81 } 82 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); 83 84 static int riscv_ext_f_depends(const struct riscv_isa_ext_data *data, 85 const unsigned long *isa_bitmap) 86 { 87 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F)) 88 return 0; 89 90 return -EPROBE_DEFER; 91 } 92 93 static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data, 94 const unsigned long *isa_bitmap) 95 { 96 if (!riscv_cbom_block_size) { 97 pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n"); 98 return -EINVAL; 99 } 100 if (!is_power_of_2(riscv_cbom_block_size)) { 101 pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); 102 return -EINVAL; 103 } 104 105 any_cpu_has_zicbom = true; 106 return 0; 107 } 108 109 static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data, 110 const unsigned long *isa_bitmap) 111 { 112 if (!riscv_cboz_block_size) { 113 pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n"); 114 return -EINVAL; 115 } 116 if (!is_power_of_2(riscv_cboz_block_size)) { 117 pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n"); 118 return -EINVAL; 119 } 120 any_cpu_has_zicboz = true; 121 return 0; 122 } 123 124 static int riscv_ext_zicbop_validate(const struct riscv_isa_ext_data *data, 125 const unsigned long *isa_bitmap) 126 { 127 if (!riscv_cbop_block_size) { 128 pr_err("Zicbop detected in ISA string, disabling as no cbop-block-size found\n"); 129 return -EINVAL; 130 } 131 if (!is_power_of_2(riscv_cbop_block_size)) { 132 pr_err("Zicbop disabled as cbop-block-size present, but is not a power-of-2\n"); 133 return -EINVAL; 134 } 135 any_cpu_has_zicbop = true; 136 return 0; 137 } 138 139 static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data, 140 const unsigned long *isa_bitmap) 141 { 142 if (!IS_ENABLED(CONFIG_FPU)) 143 return -EINVAL; 144 145 /* 146 * Due to extension ordering, d is checked before f, so no deferral 147 * is required. 148 */ 149 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) { 150 pr_warn_once("This kernel does not support systems with F but not D\n"); 151 return -EINVAL; 152 } 153 154 return 0; 155 } 156 157 static int riscv_ext_d_validate(const struct riscv_isa_ext_data *data, 158 const unsigned long *isa_bitmap) 159 { 160 if (!IS_ENABLED(CONFIG_FPU)) 161 return -EINVAL; 162 163 return 0; 164 } 165 166 static int riscv_ext_vector_x_validate(const struct riscv_isa_ext_data *data, 167 const unsigned long *isa_bitmap) 168 { 169 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 170 return -EINVAL; 171 172 return 0; 173 } 174 175 static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data, 176 const unsigned long *isa_bitmap) 177 { 178 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 179 return -EINVAL; 180 181 if (!IS_ENABLED(CONFIG_FPU)) 182 return -EINVAL; 183 184 /* 185 * The kernel doesn't support systems that don't implement both of 186 * F and D, so if any of the vector extensions that do floating point 187 * are to be usable, both floating point extensions need to be usable. 188 * 189 * Since this function validates vector only, and v/Zve* are probed 190 * after f/d, there's no need for a deferral here. 191 */ 192 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) 193 return -EINVAL; 194 195 return 0; 196 } 197 198 static int riscv_ext_vector_crypto_validate(const struct riscv_isa_ext_data *data, 199 const unsigned long *isa_bitmap) 200 { 201 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 202 return -EINVAL; 203 204 /* 205 * It isn't the kernel's job to check that the binding is correct, so 206 * it should be enough to check that any of the vector extensions are 207 * enabled, which in-turn means that vector is usable in this kernel 208 */ 209 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32X)) 210 return -EPROBE_DEFER; 211 212 return 0; 213 } 214 215 static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data, 216 const unsigned long *isa_bitmap) 217 { 218 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA)) 219 return 0; 220 221 return -EPROBE_DEFER; 222 } 223 static int riscv_ext_zcd_validate(const struct riscv_isa_ext_data *data, 224 const unsigned long *isa_bitmap) 225 { 226 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && 227 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) 228 return 0; 229 230 return -EPROBE_DEFER; 231 } 232 233 static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data, 234 const unsigned long *isa_bitmap) 235 { 236 if (IS_ENABLED(CONFIG_64BIT)) 237 return -EINVAL; 238 239 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && 240 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F)) 241 return 0; 242 243 return -EPROBE_DEFER; 244 } 245 246 static int riscv_ext_zilsd_validate(const struct riscv_isa_ext_data *data, 247 const unsigned long *isa_bitmap) 248 { 249 if (IS_ENABLED(CONFIG_64BIT)) 250 return -EINVAL; 251 252 return 0; 253 } 254 255 static int riscv_ext_zclsd_validate(const struct riscv_isa_ext_data *data, 256 const unsigned long *isa_bitmap) 257 { 258 if (IS_ENABLED(CONFIG_64BIT)) 259 return -EINVAL; 260 261 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZILSD) && 262 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA)) 263 return 0; 264 265 return -EPROBE_DEFER; 266 } 267 268 static int riscv_vector_f_validate(const struct riscv_isa_ext_data *data, 269 const unsigned long *isa_bitmap) 270 { 271 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 272 return -EINVAL; 273 274 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32F)) 275 return 0; 276 277 return -EPROBE_DEFER; 278 } 279 280 static int riscv_ext_zvfbfwma_validate(const struct riscv_isa_ext_data *data, 281 const unsigned long *isa_bitmap) 282 { 283 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZFBFMIN) && 284 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVFBFMIN)) 285 return 0; 286 287 return -EPROBE_DEFER; 288 } 289 290 static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data, 291 const unsigned long *isa_bitmap) 292 { 293 /* SVADE has already been detected, use SVADE only */ 294 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SVADE)) 295 return -EOPNOTSUPP; 296 297 return 0; 298 } 299 300 static int riscv_cfilp_validate(const struct riscv_isa_ext_data *data, 301 const unsigned long *isa_bitmap) 302 { 303 if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) || 304 (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI)) 305 return -EINVAL; 306 307 return 0; 308 } 309 310 static int riscv_cfiss_validate(const struct riscv_isa_ext_data *data, 311 const unsigned long *isa_bitmap) 312 { 313 if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) || 314 (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI)) 315 return -EINVAL; 316 317 return 0; 318 } 319 320 static const unsigned int riscv_a_exts[] = { 321 RISCV_ISA_EXT_ZAAMO, 322 RISCV_ISA_EXT_ZALRSC, 323 }; 324 325 #define RISCV_ISA_EXT_ZKN \ 326 RISCV_ISA_EXT_ZBKB, \ 327 RISCV_ISA_EXT_ZBKC, \ 328 RISCV_ISA_EXT_ZBKX, \ 329 RISCV_ISA_EXT_ZKND, \ 330 RISCV_ISA_EXT_ZKNE, \ 331 RISCV_ISA_EXT_ZKNH 332 333 static int riscv_ext_smcdeleg_validate(const struct riscv_isa_ext_data *data, 334 const unsigned long *isa_bitmap) 335 { 336 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SSCSRIND) && 337 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZIHPM) && 338 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZICNTR)) 339 return 0; 340 341 return -EPROBE_DEFER; 342 } 343 344 static int riscv_ext_ssccfg_validate(const struct riscv_isa_ext_data *data, 345 const unsigned long *isa_bitmap) 346 { 347 if (!riscv_ext_smcdeleg_validate(data, isa_bitmap) && 348 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SMCDELEG)) 349 return 0; 350 351 return -EPROBE_DEFER; 352 } 353 354 static const unsigned int riscv_zk_bundled_exts[] = { 355 RISCV_ISA_EXT_ZKN, 356 RISCV_ISA_EXT_ZKR, 357 RISCV_ISA_EXT_ZKT 358 }; 359 360 static const unsigned int riscv_zkn_bundled_exts[] = { 361 RISCV_ISA_EXT_ZKN 362 }; 363 364 static const unsigned int riscv_zks_bundled_exts[] = { 365 RISCV_ISA_EXT_ZBKB, 366 RISCV_ISA_EXT_ZBKC, 367 RISCV_ISA_EXT_ZKSED, 368 RISCV_ISA_EXT_ZKSH 369 }; 370 371 #define RISCV_ISA_EXT_ZVKN \ 372 RISCV_ISA_EXT_ZVKNED, \ 373 RISCV_ISA_EXT_ZVKNHB, \ 374 RISCV_ISA_EXT_ZVKB, \ 375 RISCV_ISA_EXT_ZVKT 376 377 static const unsigned int riscv_zvkn_bundled_exts[] = { 378 RISCV_ISA_EXT_ZVKN 379 }; 380 381 static const unsigned int riscv_zvknc_bundled_exts[] = { 382 RISCV_ISA_EXT_ZVKN, 383 RISCV_ISA_EXT_ZVBC 384 }; 385 386 static const unsigned int riscv_zvkng_bundled_exts[] = { 387 RISCV_ISA_EXT_ZVKN, 388 RISCV_ISA_EXT_ZVKG 389 }; 390 391 #define RISCV_ISA_EXT_ZVKS \ 392 RISCV_ISA_EXT_ZVKSED, \ 393 RISCV_ISA_EXT_ZVKSH, \ 394 RISCV_ISA_EXT_ZVKB, \ 395 RISCV_ISA_EXT_ZVKT 396 397 static const unsigned int riscv_zvks_bundled_exts[] = { 398 RISCV_ISA_EXT_ZVKS 399 }; 400 401 static const unsigned int riscv_zvksc_bundled_exts[] = { 402 RISCV_ISA_EXT_ZVKS, 403 RISCV_ISA_EXT_ZVBC 404 }; 405 406 static const unsigned int riscv_zvksg_bundled_exts[] = { 407 RISCV_ISA_EXT_ZVKS, 408 RISCV_ISA_EXT_ZVKG 409 }; 410 411 static const unsigned int riscv_zvbb_exts[] = { 412 RISCV_ISA_EXT_ZVKB 413 }; 414 415 /* 416 * The RISC-V ISA manual specifies that Zfh implies Zfhmin and Zvfh implies 417 * Zvfhmin. Report the implied subset extensions whenever the supersets are 418 * detected (see https://github.com/riscv/riscv-isa-manual/pull/3070). 419 */ 420 static const unsigned int riscv_zfh_exts[] = { 421 RISCV_ISA_EXT_ZFHMIN 422 }; 423 424 static const unsigned int riscv_zvfh_exts[] = { 425 RISCV_ISA_EXT_ZVFHMIN 426 }; 427 428 #define RISCV_ISA_EXT_ZVE64F_IMPLY_LIST \ 429 RISCV_ISA_EXT_ZVE64X, \ 430 RISCV_ISA_EXT_ZVE32F, \ 431 RISCV_ISA_EXT_ZVE32X 432 433 #define RISCV_ISA_EXT_ZVE64D_IMPLY_LIST \ 434 RISCV_ISA_EXT_ZVE64F, \ 435 RISCV_ISA_EXT_ZVE64F_IMPLY_LIST 436 437 #define RISCV_ISA_EXT_V_IMPLY_LIST \ 438 RISCV_ISA_EXT_ZVE64D, \ 439 RISCV_ISA_EXT_ZVE64D_IMPLY_LIST 440 441 static const unsigned int riscv_zve32f_exts[] = { 442 RISCV_ISA_EXT_ZVE32X 443 }; 444 445 static const unsigned int riscv_zve64f_exts[] = { 446 RISCV_ISA_EXT_ZVE64F_IMPLY_LIST 447 }; 448 449 static const unsigned int riscv_zve64d_exts[] = { 450 RISCV_ISA_EXT_ZVE64D_IMPLY_LIST 451 }; 452 453 static const unsigned int riscv_v_exts[] = { 454 RISCV_ISA_EXT_V_IMPLY_LIST 455 }; 456 457 static const unsigned int riscv_zve64x_exts[] = { 458 RISCV_ISA_EXT_ZVE32X, 459 RISCV_ISA_EXT_ZVE64X 460 }; 461 462 /* 463 * While the [ms]envcfg CSRs were not defined until version 1.12 of the RISC-V 464 * privileged ISA, the existence of the CSRs is implied by any extension which 465 * specifies [ms]envcfg bit(s). Hence, we define a custom ISA extension for the 466 * existence of the CSR, and treat it as a subset of those other extensions. 467 */ 468 static const unsigned int riscv_xlinuxenvcfg_exts[] = { 469 RISCV_ISA_EXT_XLINUXENVCFG 470 }; 471 472 /* 473 * Zc* spec states that: 474 * - C always implies Zca 475 * - C+F implies Zcf (RV32 only) 476 * - C+D implies Zcd 477 * 478 * These extensions will be enabled and then validated depending on the 479 * availability of F/D RV32. 480 */ 481 static const unsigned int riscv_c_exts[] = { 482 RISCV_ISA_EXT_ZCA, 483 RISCV_ISA_EXT_ZCF, 484 RISCV_ISA_EXT_ZCD, 485 }; 486 487 /* 488 * The canonical order of ISA extension names in the ISA string is defined in 489 * Chapter 27 of the RISC-V Instruction Set Manual Volume I Unprivileged ISA 490 * (Document Version 20191213). 491 * 492 * Ordinarily, for in-kernel data structures, this order is unimportant but 493 * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. 494 * 495 * The specification uses vague wording, such as should, when it comes to 496 * ordering, so for our purposes the following rules apply: 497 * 498 * 1. All multi-letter extensions must be separated from other extensions by an 499 * underscore. 500 * 501 * 2. Additional standard extensions (starting with 'Z') must be sorted after 502 * single-letter extensions and before any higher-privileged extensions. 503 * 504 * 3. The first letter following the 'Z' conventionally indicates the most 505 * closely related alphabetical extension category, IMAFDQLCBKJTPVH. 506 * If multiple 'Z' extensions are named, they must be ordered first by 507 * category, then alphabetically within a category. 508 * 509 * 3. Standard supervisor-level extensions (starting with 'S') must be listed 510 * after standard unprivileged extensions. If multiple supervisor-level 511 * extensions are listed, they must be ordered alphabetically. 512 * 513 * 4. Standard machine-level extensions (starting with 'Zxm') must be listed 514 * after any lower-privileged, standard extensions. If multiple 515 * machine-level extensions are listed, they must be ordered 516 * alphabetically. 517 * 518 * 5. Non-standard extensions (starting with 'X') must be listed after all 519 * standard extensions. If multiple non-standard extensions are listed, they 520 * must be ordered alphabetically. 521 * 522 * An example string following the order is: 523 * rv64imadc_zifoo_zigoo_zafoo_sbar_scar_zxmbaz_xqux_xrux 524 * 525 * New entries to this struct should follow the ordering rules described above. 526 */ 527 const struct riscv_isa_ext_data riscv_isa_ext[] = { 528 __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_I), 529 __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_M), 530 __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_A, riscv_a_exts), 531 __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_F, riscv_ext_f_validate), 532 __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_D, riscv_ext_d_validate), 533 __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_Q), 534 __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_C, riscv_c_exts), 535 __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_V, riscv_v_exts, 536 riscv_ext_vector_float_validate), 537 __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_H), 538 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate), 539 __RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate), 540 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate), 541 __RISCV_ISA_EXT_DATA(ziccamoa, RISCV_ISA_EXT_ZICCAMOA), 542 __RISCV_ISA_EXT_DATA(ziccif, RISCV_ISA_EXT_ZICCIF), 543 __RISCV_ISA_EXT_DATA(zicclsm, RISCV_ISA_EXT_ZICCLSM), 544 __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE), 545 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfilp, RISCV_ISA_EXT_ZICFILP, riscv_xlinuxenvcfg_exts, 546 riscv_cfilp_validate), 547 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfiss, RISCV_ISA_EXT_ZICFISS, riscv_xlinuxenvcfg_exts, 548 riscv_cfiss_validate), 549 __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), 550 __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), 551 __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), 552 __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI), 553 __RISCV_ISA_EXT_DATA(zihintntl, RISCV_ISA_EXT_ZIHINTNTL), 554 __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), 555 __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), 556 __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP), 557 __RISCV_ISA_EXT_DATA(za64rs, RISCV_ISA_EXT_ZA64RS), 558 __RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO), 559 __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA), 560 __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), 561 __RISCV_ISA_EXT_DATA(zalasr, RISCV_ISA_EXT_ZALASR), 562 __RISCV_ISA_EXT_DATA(zalrsc, RISCV_ISA_EXT_ZALRSC), 563 __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), 564 __RISCV_ISA_EXT_DATA_VALIDATE(zfa, RISCV_ISA_EXT_ZFA, riscv_ext_f_depends), 565 __RISCV_ISA_EXT_DATA_VALIDATE(zfbfmin, RISCV_ISA_EXT_ZFBFMIN, riscv_ext_f_depends), 566 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zfh, RISCV_ISA_EXT_ZFH, 567 riscv_zfh_exts, riscv_ext_f_depends), 568 __RISCV_ISA_EXT_DATA_VALIDATE(zfhmin, RISCV_ISA_EXT_ZFHMIN, riscv_ext_f_depends), 569 __RISCV_ISA_EXT_DATA(zca, RISCV_ISA_EXT_ZCA), 570 __RISCV_ISA_EXT_DATA_VALIDATE(zcb, RISCV_ISA_EXT_ZCB, riscv_ext_zca_depends), 571 __RISCV_ISA_EXT_DATA_VALIDATE(zcd, RISCV_ISA_EXT_ZCD, riscv_ext_zcd_validate), 572 __RISCV_ISA_EXT_DATA_VALIDATE(zcf, RISCV_ISA_EXT_ZCF, riscv_ext_zcf_validate), 573 __RISCV_ISA_EXT_DATA_VALIDATE(zcmop, RISCV_ISA_EXT_ZCMOP, riscv_ext_zca_depends), 574 __RISCV_ISA_EXT_DATA_VALIDATE(zclsd, RISCV_ISA_EXT_ZCLSD, riscv_ext_zclsd_validate), 575 __RISCV_ISA_EXT_DATA_VALIDATE(zilsd, RISCV_ISA_EXT_ZILSD, riscv_ext_zilsd_validate), 576 __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), 577 __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), 578 __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC), 579 __RISCV_ISA_EXT_DATA(zbkb, RISCV_ISA_EXT_ZBKB), 580 __RISCV_ISA_EXT_DATA(zbkc, RISCV_ISA_EXT_ZBKC), 581 __RISCV_ISA_EXT_DATA(zbkx, RISCV_ISA_EXT_ZBKX), 582 __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS), 583 __RISCV_ISA_EXT_BUNDLE(zk, riscv_zk_bundled_exts), 584 __RISCV_ISA_EXT_BUNDLE(zkn, riscv_zkn_bundled_exts), 585 __RISCV_ISA_EXT_DATA(zknd, RISCV_ISA_EXT_ZKND), 586 __RISCV_ISA_EXT_DATA(zkne, RISCV_ISA_EXT_ZKNE), 587 __RISCV_ISA_EXT_DATA(zknh, RISCV_ISA_EXT_ZKNH), 588 __RISCV_ISA_EXT_DATA(zkr, RISCV_ISA_EXT_ZKR), 589 __RISCV_ISA_EXT_BUNDLE(zks, riscv_zks_bundled_exts), 590 __RISCV_ISA_EXT_DATA(zkt, RISCV_ISA_EXT_ZKT), 591 __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED), 592 __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), 593 __RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO), 594 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts, riscv_ext_vector_crypto_validate), 595 __RISCV_ISA_EXT_DATA_VALIDATE(zvbc, RISCV_ISA_EXT_ZVBC, riscv_ext_vector_crypto_validate), 596 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts, riscv_ext_vector_float_validate), 597 __RISCV_ISA_EXT_DATA_VALIDATE(zve32x, RISCV_ISA_EXT_ZVE32X, riscv_ext_vector_x_validate), 598 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts, riscv_ext_vector_float_validate), 599 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts, riscv_ext_vector_float_validate), 600 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts, riscv_ext_vector_x_validate), 601 __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfmin, RISCV_ISA_EXT_ZVFBFMIN, riscv_vector_f_validate), 602 __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfwma, RISCV_ISA_EXT_ZVFBFWMA, riscv_ext_zvfbfwma_validate), 603 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zvfh, RISCV_ISA_EXT_ZVFH, 604 riscv_zvfh_exts, 605 riscv_ext_vector_float_validate), 606 __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), 607 __RISCV_ISA_EXT_DATA_VALIDATE(zvkb, RISCV_ISA_EXT_ZVKB, riscv_ext_vector_crypto_validate), 608 __RISCV_ISA_EXT_DATA_VALIDATE(zvkg, RISCV_ISA_EXT_ZVKG, riscv_ext_vector_crypto_validate), 609 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkn, riscv_zvkn_bundled_exts, riscv_ext_vector_crypto_validate), 610 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvknc, riscv_zvknc_bundled_exts, riscv_ext_vector_crypto_validate), 611 __RISCV_ISA_EXT_DATA_VALIDATE(zvkned, RISCV_ISA_EXT_ZVKNED, riscv_ext_vector_crypto_validate), 612 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkng, riscv_zvkng_bundled_exts, riscv_ext_vector_crypto_validate), 613 __RISCV_ISA_EXT_DATA_VALIDATE(zvknha, RISCV_ISA_EXT_ZVKNHA, riscv_ext_vector_crypto_validate), 614 __RISCV_ISA_EXT_DATA_VALIDATE(zvknhb, RISCV_ISA_EXT_ZVKNHB, riscv_ext_vector_crypto_validate), 615 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvks, riscv_zvks_bundled_exts, riscv_ext_vector_crypto_validate), 616 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksc, riscv_zvksc_bundled_exts, riscv_ext_vector_crypto_validate), 617 __RISCV_ISA_EXT_DATA_VALIDATE(zvksed, RISCV_ISA_EXT_ZVKSED, riscv_ext_vector_crypto_validate), 618 __RISCV_ISA_EXT_DATA_VALIDATE(zvksh, RISCV_ISA_EXT_ZVKSH, riscv_ext_vector_crypto_validate), 619 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksg, riscv_zvksg_bundled_exts, riscv_ext_vector_crypto_validate), 620 __RISCV_ISA_EXT_DATA_VALIDATE(zvkt, RISCV_ISA_EXT_ZVKT, riscv_ext_vector_crypto_validate), 621 __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), 622 __RISCV_ISA_EXT_DATA_VALIDATE(smcdeleg, RISCV_ISA_EXT_SMCDELEG, 623 riscv_ext_smcdeleg_validate), 624 __RISCV_ISA_EXT_DATA(smcntrpmf, RISCV_ISA_EXT_SMCNTRPMF), 625 __RISCV_ISA_EXT_DATA(smcsrind, RISCV_ISA_EXT_SMCSRIND), 626 __RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM), 627 __RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts), 628 __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), 629 __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), 630 __RISCV_ISA_EXT_DATA_VALIDATE(ssccfg, RISCV_ISA_EXT_SSCCFG, riscv_ext_ssccfg_validate), 631 __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), 632 __RISCV_ISA_EXT_DATA(sscsrind, RISCV_ISA_EXT_SSCSRIND), 633 __RISCV_ISA_EXT_SUPERSET(ssnpm, RISCV_ISA_EXT_SSNPM, riscv_xlinuxenvcfg_exts), 634 __RISCV_ISA_EXT_DATA(ssqosid, RISCV_ISA_EXT_SSQOSID), 635 __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), 636 __RISCV_ISA_EXT_DATA(svade, RISCV_ISA_EXT_SVADE), 637 __RISCV_ISA_EXT_DATA_VALIDATE(svadu, RISCV_ISA_EXT_SVADU, riscv_ext_svadu_validate), 638 __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), 639 __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT), 640 __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), 641 __RISCV_ISA_EXT_DATA(svrsw60t59b, RISCV_ISA_EXT_SVRSW60T59B), 642 __RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC), 643 }; 644 645 const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); 646 647 static void riscv_isa_set_ext(const struct riscv_isa_ext_data *ext, unsigned long *bitmap) 648 { 649 if (ext->id != RISCV_ISA_EXT_INVALID) 650 set_bit(ext->id, bitmap); 651 652 for (int i = 0; i < ext->subset_ext_size; i++) { 653 if (ext->subset_ext_ids[i] != RISCV_ISA_EXT_INVALID) 654 set_bit(ext->subset_ext_ids[i], bitmap); 655 } 656 } 657 658 static const struct riscv_isa_ext_data *riscv_get_isa_ext_data(unsigned int ext_id) 659 { 660 for (int i = 0; i < riscv_isa_ext_count; i++) { 661 if (riscv_isa_ext[i].id == ext_id) 662 return &riscv_isa_ext[i]; 663 } 664 665 return NULL; 666 } 667 668 /* 669 * "Resolve" a source ISA bitmap into one that matches kernel configuration as 670 * well as correct extension dependencies. Some extensions depends on specific 671 * kernel configuration to be usable (V needs CONFIG_RISCV_ISA_V for instance) 672 * and this function will actually validate all the extensions provided in 673 * source_isa into the resolved_isa based on extensions validate() callbacks. 674 */ 675 static void __init riscv_resolve_isa(unsigned long *source_isa, 676 unsigned long *resolved_isa, unsigned long *this_hwcap, 677 unsigned long *isa2hwcap) 678 { 679 bool loop; 680 const struct riscv_isa_ext_data *ext; 681 DECLARE_BITMAP(prev_resolved_isa, RISCV_ISA_EXT_MAX); 682 int max_loop_count = riscv_isa_ext_count, ret; 683 unsigned int bit; 684 685 do { 686 loop = false; 687 if (max_loop_count-- < 0) { 688 pr_err("Failed to reach a stable ISA state\n"); 689 return; 690 } 691 bitmap_copy(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX); 692 for_each_set_bit(bit, source_isa, RISCV_ISA_EXT_MAX) { 693 ext = riscv_get_isa_ext_data(bit); 694 695 if (ext && ext->validate) { 696 ret = ext->validate(ext, resolved_isa); 697 if (ret == -EPROBE_DEFER) { 698 loop = true; 699 continue; 700 } else if (ret) { 701 /* Disable the extension entirely */ 702 clear_bit(bit, source_isa); 703 continue; 704 } 705 } 706 707 set_bit(bit, resolved_isa); 708 /* No need to keep it in source isa now that it is enabled */ 709 clear_bit(bit, source_isa); 710 711 /* Single letter extensions get set in hwcap */ 712 if (bit < RISCV_ISA_EXT_BASE) 713 *this_hwcap |= isa2hwcap[bit]; 714 } 715 } while (loop && !bitmap_equal(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX)); 716 } 717 718 static void __init match_isa_ext(const char *name, const char *name_end, unsigned long *bitmap) 719 { 720 for (int i = 0; i < riscv_isa_ext_count; i++) { 721 const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i]; 722 723 if ((name_end - name == strlen(ext->name)) && 724 !strncasecmp(name, ext->name, name_end - name)) { 725 riscv_isa_set_ext(ext, bitmap); 726 break; 727 } 728 } 729 } 730 731 static void __init riscv_parse_isa_string(const char *isa, unsigned long *bitmap) 732 { 733 /* 734 * For all possible cpus, we have already validated in 735 * the boot process that they at least contain "rv" and 736 * whichever of "32"/"64" this kernel supports, and so this 737 * section can be skipped. 738 */ 739 isa += 4; 740 741 while (*isa) { 742 const char *ext = isa++; 743 const char *ext_end = isa; 744 bool ext_err = false; 745 746 switch (*ext) { 747 case 'x': 748 case 'X': 749 if (acpi_disabled) 750 pr_warn_once("Vendor extensions are ignored in riscv,isa. Use riscv,isa-extensions instead."); 751 /* 752 * To skip an extension, we find its end. 753 * As multi-letter extensions must be split from other multi-letter 754 * extensions with an "_", the end of a multi-letter extension will 755 * either be the null character or the "_" at the start of the next 756 * multi-letter extension. 757 */ 758 for (; *isa && *isa != '_'; ++isa) 759 ; 760 ext_err = true; 761 break; 762 case 's': 763 /* 764 * Workaround for invalid single-letter 's' & 'u' (QEMU). 765 * No need to set the bit in riscv_isa as 's' & 'u' are 766 * not valid ISA extensions. It works unless the first 767 * multi-letter extension in the ISA string begins with 768 * "Su" and is not prefixed with an underscore. 769 */ 770 if (ext[-1] != '_' && ext[1] == 'u') { 771 ++isa; 772 ext_err = true; 773 break; 774 } 775 fallthrough; 776 case 'S': 777 case 'z': 778 case 'Z': 779 /* 780 * Before attempting to parse the extension itself, we find its end. 781 * As multi-letter extensions must be split from other multi-letter 782 * extensions with an "_", the end of a multi-letter extension will 783 * either be the null character or the "_" at the start of the next 784 * multi-letter extension. 785 * 786 * Next, as the extensions version is currently ignored, we 787 * eliminate that portion. This is done by parsing backwards from 788 * the end of the extension, removing any numbers. This may be a 789 * major or minor number however, so the process is repeated if a 790 * minor number was found. 791 * 792 * ext_end is intended to represent the first character *after* the 793 * name portion of an extension, but will be decremented to the last 794 * character itself while eliminating the extensions version number. 795 * A simple re-increment solves this problem. 796 */ 797 for (; *isa && *isa != '_'; ++isa) 798 if (unlikely(!isalnum(*isa))) 799 ext_err = true; 800 801 ext_end = isa; 802 if (unlikely(ext_err)) 803 break; 804 805 if (!isdigit(ext_end[-1])) 806 break; 807 808 while (isdigit(*--ext_end)) 809 ; 810 811 if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) { 812 ++ext_end; 813 break; 814 } 815 816 while (isdigit(*--ext_end)) 817 ; 818 819 ++ext_end; 820 break; 821 default: 822 /* 823 * Things are a little easier for single-letter extensions, as they 824 * are parsed forwards. 825 * 826 * After checking that our starting position is valid, we need to 827 * ensure that, when isa was incremented at the start of the loop, 828 * that it arrived at the start of the next extension. 829 * 830 * If we are already on a non-digit, there is nothing to do. Either 831 * we have a multi-letter extension's _, or the start of an 832 * extension. 833 * 834 * Otherwise we have found the current extension's major version 835 * number. Parse past it, and a subsequent p/minor version number 836 * if present. The `p` extension must not appear immediately after 837 * a number, so there is no fear of missing it. 838 * 839 */ 840 if (unlikely(!isalpha(*ext))) { 841 ext_err = true; 842 break; 843 } 844 845 if (!isdigit(*isa)) 846 break; 847 848 while (isdigit(*++isa)) 849 ; 850 851 if (tolower(*isa) != 'p') 852 break; 853 854 if (!isdigit(*++isa)) { 855 --isa; 856 break; 857 } 858 859 while (isdigit(*++isa)) 860 ; 861 862 break; 863 } 864 865 /* 866 * The parser expects that at the start of an iteration isa points to the 867 * first character of the next extension. As we stop parsing an extension 868 * on meeting a non-alphanumeric character, an extra increment is needed 869 * where the succeeding extension is a multi-letter prefixed with an "_". 870 */ 871 if (*isa == '_') 872 ++isa; 873 874 if (unlikely(ext_err)) 875 continue; 876 877 match_isa_ext(ext, ext_end, bitmap); 878 } 879 } 880 881 static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) 882 { 883 struct device_node *node; 884 const char *isa; 885 int rc; 886 struct acpi_table_header *rhct; 887 acpi_status status; 888 unsigned int cpu; 889 u64 boot_vendorid; 890 u64 boot_archid; 891 892 if (!acpi_disabled) { 893 status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); 894 if (ACPI_FAILURE(status)) 895 return; 896 } 897 898 boot_vendorid = riscv_get_mvendorid(); 899 boot_archid = riscv_get_marchid(); 900 901 for_each_possible_cpu(cpu) { 902 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 903 unsigned long this_hwcap = 0; 904 DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 }; 905 906 if (acpi_disabled) { 907 node = of_cpu_device_node_get(cpu); 908 if (!node) { 909 pr_warn("Unable to find cpu node\n"); 910 continue; 911 } 912 913 rc = of_property_read_string(node, "riscv,isa", &isa); 914 of_node_put(node); 915 if (rc) { 916 pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); 917 continue; 918 } 919 } else { 920 rc = acpi_get_riscv_isa(rhct, cpu, &isa); 921 if (rc < 0) { 922 pr_warn("Unable to get ISA for the hart - %d\n", cpu); 923 continue; 924 } 925 } 926 927 riscv_parse_isa_string(isa, source_isa); 928 929 /* 930 * These ones were as they were part of the base ISA when the 931 * port & dt-bindings were upstreamed, and so can be set 932 * unconditionally where `i` is in riscv,isa on DT systems. 933 */ 934 if (acpi_disabled) { 935 set_bit(RISCV_ISA_EXT_ZICSR, source_isa); 936 set_bit(RISCV_ISA_EXT_ZIFENCEI, source_isa); 937 set_bit(RISCV_ISA_EXT_ZICNTR, source_isa); 938 set_bit(RISCV_ISA_EXT_ZIHPM, source_isa); 939 } 940 941 /* 942 * "V" in ISA strings is ambiguous in practice: it should mean 943 * just the standard V-1.0 but vendors aren't well behaved. 944 * Many vendors with T-Head CPU cores which implement the 0.7.1 945 * version of the vector specification put "v" into their DTs. 946 * CPU cores with the ratified spec will contain non-zero 947 * marchid. 948 */ 949 if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) 950 clear_bit(RISCV_ISA_EXT_V, source_isa); 951 952 riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); 953 954 /* 955 * All "okay" hart should have same isa. Set HWCAP based on 956 * common capabilities of every "okay" hart, in case they don't 957 * have. 958 */ 959 if (elf_hwcap) 960 elf_hwcap &= this_hwcap; 961 else 962 elf_hwcap = this_hwcap; 963 964 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 965 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 966 else 967 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 968 } 969 970 if (!acpi_disabled && rhct) 971 acpi_put_table((struct acpi_table_header *)rhct); 972 } 973 974 static void __init riscv_fill_cpu_vendor_ext(struct device_node *cpu_node, int cpu) 975 { 976 if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) 977 return; 978 979 for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) { 980 struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i]; 981 982 for (int j = 0; j < ext_list->ext_data_count; j++) { 983 const struct riscv_isa_ext_data ext = ext_list->ext_data[j]; 984 struct riscv_isavendorinfo *isavendorinfo = &ext_list->per_hart_isa_bitmap[cpu]; 985 986 if (of_property_match_string(cpu_node, "riscv,isa-extensions", 987 ext.property) < 0) 988 continue; 989 990 /* 991 * Assume that subset extensions are all members of the 992 * same vendor. 993 */ 994 if (ext.subset_ext_size) 995 for (int k = 0; k < ext.subset_ext_size; k++) 996 set_bit(ext.subset_ext_ids[k], isavendorinfo->isa); 997 998 set_bit(ext.id, isavendorinfo->isa); 999 } 1000 } 1001 } 1002 1003 /* 1004 * Populate all_harts_isa_bitmap for each vendor with all of the extensions that 1005 * are shared across CPUs for that vendor. 1006 */ 1007 static void __init riscv_fill_vendor_ext_list(int cpu) 1008 { 1009 if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) 1010 return; 1011 1012 for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) { 1013 struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i]; 1014 1015 if (!ext_list->is_initialized) { 1016 bitmap_copy(ext_list->all_harts_isa_bitmap.isa, 1017 ext_list->per_hart_isa_bitmap[cpu].isa, 1018 RISCV_ISA_VENDOR_EXT_MAX); 1019 ext_list->is_initialized = true; 1020 } else { 1021 bitmap_and(ext_list->all_harts_isa_bitmap.isa, 1022 ext_list->all_harts_isa_bitmap.isa, 1023 ext_list->per_hart_isa_bitmap[cpu].isa, 1024 RISCV_ISA_VENDOR_EXT_MAX); 1025 } 1026 } 1027 } 1028 1029 static int has_thead_homogeneous_vlenb(void) 1030 { 1031 int cpu; 1032 u32 prev_vlenb = 0; 1033 u32 vlenb = 0; 1034 1035 /* Ignore thead,vlenb property if xtheadvector is not enabled in the kernel */ 1036 if (!IS_ENABLED(CONFIG_RISCV_ISA_XTHEADVECTOR)) 1037 return 0; 1038 1039 for_each_possible_cpu(cpu) { 1040 struct device_node *cpu_node; 1041 1042 cpu_node = of_cpu_device_node_get(cpu); 1043 if (!cpu_node) { 1044 pr_warn("Unable to find cpu node\n"); 1045 return -ENOENT; 1046 } 1047 1048 if (of_property_read_u32(cpu_node, "thead,vlenb", &vlenb)) { 1049 of_node_put(cpu_node); 1050 1051 if (prev_vlenb) 1052 return -ENOENT; 1053 continue; 1054 } 1055 1056 if (prev_vlenb && vlenb != prev_vlenb) { 1057 of_node_put(cpu_node); 1058 return -ENOENT; 1059 } 1060 1061 prev_vlenb = vlenb; 1062 of_node_put(cpu_node); 1063 } 1064 1065 thead_vlenb_of = vlenb; 1066 return 0; 1067 } 1068 1069 static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) 1070 { 1071 unsigned int cpu; 1072 bool mitigated; 1073 1074 for_each_possible_cpu(cpu) { 1075 unsigned long this_hwcap = 0; 1076 struct device_node *cpu_node; 1077 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 1078 DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 }; 1079 1080 cpu_node = of_cpu_device_node_get(cpu); 1081 if (!cpu_node) { 1082 pr_warn("Unable to find cpu node\n"); 1083 continue; 1084 } 1085 1086 if (!of_property_present(cpu_node, "riscv,isa-extensions")) { 1087 of_node_put(cpu_node); 1088 continue; 1089 } 1090 1091 for (int i = 0; i < riscv_isa_ext_count; i++) { 1092 const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i]; 1093 1094 if (of_property_match_string(cpu_node, "riscv,isa-extensions", 1095 ext->property) < 0) 1096 continue; 1097 1098 riscv_isa_set_ext(ext, source_isa); 1099 } 1100 1101 riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); 1102 riscv_fill_cpu_vendor_ext(cpu_node, cpu); 1103 1104 of_node_put(cpu_node); 1105 1106 /* 1107 * All "okay" harts should have same isa. Set HWCAP based on 1108 * common capabilities of every "okay" hart, in case they don't. 1109 */ 1110 if (elf_hwcap) 1111 elf_hwcap &= this_hwcap; 1112 else 1113 elf_hwcap = this_hwcap; 1114 1115 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 1116 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 1117 else 1118 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 1119 1120 riscv_fill_vendor_ext_list(cpu); 1121 } 1122 1123 /* 1124 * Execute ghostwrite mitigation immediately after detecting extensions 1125 * to disable xtheadvector if necessary. 1126 */ 1127 mitigated = ghostwrite_enable_mitigation(); 1128 1129 if (!mitigated && has_xtheadvector_no_alternatives() && has_thead_homogeneous_vlenb() < 0) { 1130 pr_warn("Unsupported heterogeneous vlenb detected, vector extension disabled.\n"); 1131 disable_xtheadvector(); 1132 } 1133 1134 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 1135 return -ENOENT; 1136 1137 return 0; 1138 } 1139 1140 #ifdef CONFIG_RISCV_ISA_FALLBACK 1141 bool __initdata riscv_isa_fallback = true; 1142 #else 1143 bool __initdata riscv_isa_fallback; 1144 static int __init riscv_isa_fallback_setup(char *__unused) 1145 { 1146 riscv_isa_fallback = true; 1147 return 1; 1148 } 1149 early_param("riscv_isa_fallback", riscv_isa_fallback_setup); 1150 #endif 1151 1152 void __init riscv_fill_hwcap(void) 1153 { 1154 char print_str[NUM_ALPHA_EXTS + 1]; 1155 unsigned long isa2hwcap[RISCV_ISA_EXT_BASE] = {0}; 1156 int i, j; 1157 1158 isa2hwcap[RISCV_ISA_EXT_I] = COMPAT_HWCAP_ISA_I; 1159 isa2hwcap[RISCV_ISA_EXT_M] = COMPAT_HWCAP_ISA_M; 1160 isa2hwcap[RISCV_ISA_EXT_A] = COMPAT_HWCAP_ISA_A; 1161 isa2hwcap[RISCV_ISA_EXT_F] = COMPAT_HWCAP_ISA_F; 1162 isa2hwcap[RISCV_ISA_EXT_D] = COMPAT_HWCAP_ISA_D; 1163 isa2hwcap[RISCV_ISA_EXT_C] = COMPAT_HWCAP_ISA_C; 1164 isa2hwcap[RISCV_ISA_EXT_V] = COMPAT_HWCAP_ISA_V; 1165 1166 if (!acpi_disabled) { 1167 riscv_fill_hwcap_from_isa_string(isa2hwcap); 1168 } else { 1169 int ret = riscv_fill_hwcap_from_ext_list(isa2hwcap); 1170 1171 if (ret && riscv_isa_fallback) { 1172 pr_info("Falling back to deprecated \"riscv,isa\"\n"); 1173 riscv_fill_hwcap_from_isa_string(isa2hwcap); 1174 } 1175 } 1176 1177 /* 1178 * We don't support systems with F but without D, so mask those out 1179 * here. 1180 */ 1181 if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { 1182 pr_info("This kernel does not support systems with F but not D\n"); 1183 elf_hwcap &= ~COMPAT_HWCAP_ISA_F; 1184 } 1185 1186 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_ZVE32X) || 1187 has_xtheadvector_no_alternatives()) { 1188 /* 1189 * This cannot fail when called on the boot hart 1190 */ 1191 riscv_v_setup_vsize(); 1192 } 1193 1194 memset(print_str, 0, sizeof(print_str)); 1195 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 1196 if (riscv_isa[0] & BIT_MASK(i)) 1197 print_str[j++] = (char)('a' + i); 1198 pr_info("riscv: base ISA extensions %s\n", print_str); 1199 1200 memset(print_str, 0, sizeof(print_str)); 1201 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 1202 if (elf_hwcap & BIT_MASK(i)) 1203 print_str[j++] = (char)('a' + i); 1204 pr_info("riscv: ELF capabilities %s\n", print_str); 1205 } 1206 1207 unsigned long riscv_get_elf_hwcap(void) 1208 { 1209 unsigned long hwcap; 1210 1211 hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1)); 1212 1213 if (!riscv_v_vstate_ctrl_user_allowed()) 1214 hwcap &= ~COMPAT_HWCAP_ISA_V; 1215 1216 return hwcap; 1217 } 1218 1219 void __init riscv_user_isa_enable(void) 1220 { 1221 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOZ)) 1222 current->thread.envcfg |= ENVCFG_CBZE; 1223 else if (any_cpu_has_zicboz) 1224 pr_warn("Zicboz disabled as it is unavailable on some harts\n"); 1225 1226 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOM)) 1227 current->thread.envcfg |= ENVCFG_CBCFE; 1228 else if (any_cpu_has_zicbom) 1229 pr_warn("Zicbom disabled as it is unavailable on some harts\n"); 1230 1231 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOP) && 1232 any_cpu_has_zicbop) 1233 pr_warn("Zicbop disabled as it is unavailable on some harts\n"); 1234 } 1235 1236 #ifdef CONFIG_RISCV_ALTERNATIVE 1237 /* 1238 * Alternative patch sites consider 48 bits when determining when to patch 1239 * the old instruction sequence with the new. These bits are broken into a 1240 * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the 1241 * patch site is for an erratum, identified by the 32-bit patch ID. When 1242 * the vendor ID is zero, the patch site is for a cpufeature. cpufeatures 1243 * further break down patch ID into two 16-bit numbers. The lower 16 bits 1244 * are the cpufeature ID and the upper 16 bits are used for a value specific 1245 * to the cpufeature and patch site. If the upper 16 bits are zero, then it 1246 * implies no specific value is specified. cpufeatures that want to control 1247 * patching on a per-site basis will provide non-zero values and implement 1248 * checks here. The checks return true when patching should be done, and 1249 * false otherwise. 1250 */ 1251 static bool riscv_cpufeature_patch_check(u16 id, u16 value) 1252 { 1253 if (!value) 1254 return true; 1255 1256 switch (id) { 1257 case RISCV_ISA_EXT_ZICBOZ: 1258 /* 1259 * Zicboz alternative applications provide the maximum 1260 * supported block size order, or zero when it doesn't 1261 * matter. If the current block size exceeds the maximum, 1262 * then the alternative cannot be applied. 1263 */ 1264 return riscv_cboz_block_size <= (1U << value); 1265 } 1266 1267 return false; 1268 } 1269 1270 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, 1271 struct alt_entry *end, 1272 unsigned int stage) 1273 { 1274 struct alt_entry *alt; 1275 void *oldptr, *altptr; 1276 u16 id, value, vendor; 1277 1278 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 1279 return; 1280 1281 for (alt = begin; alt < end; alt++) { 1282 id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); 1283 vendor = PATCH_ID_CPUFEATURE_ID(alt->vendor_id); 1284 1285 /* 1286 * Any alternative with a patch_id that is less than 1287 * RISCV_ISA_EXT_MAX is interpreted as a standard extension. 1288 * 1289 * Any alternative with patch_id that is greater than or equal 1290 * to RISCV_VENDOR_EXT_ALTERNATIVES_BASE is interpreted as a 1291 * vendor extension. 1292 */ 1293 if (id < RISCV_ISA_EXT_MAX) { 1294 /* 1295 * This patch should be treated as errata so skip 1296 * processing here. 1297 */ 1298 if (alt->vendor_id != 0) 1299 continue; 1300 1301 if (!__riscv_isa_extension_available(NULL, id)) 1302 continue; 1303 1304 value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id); 1305 if (!riscv_cpufeature_patch_check(id, value)) 1306 continue; 1307 } else if (id >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE) { 1308 if (!__riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor, 1309 id - RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) 1310 continue; 1311 } else { 1312 WARN(1, "This extension id:%d is not in ISA extension list", id); 1313 continue; 1314 } 1315 1316 oldptr = ALT_OLD_PTR(alt); 1317 altptr = ALT_ALT_PTR(alt); 1318 1319 mutex_lock(&text_mutex); 1320 patch_text_nosync(oldptr, altptr, alt->alt_len); 1321 riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); 1322 mutex_unlock(&text_mutex); 1323 } 1324 } 1325 #endif 1326