1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copied from arch/arm64/kernel/cpufeature.c 4 * 5 * Copyright (C) 2015 ARM Ltd. 6 * Copyright (C) 2017 SiFive 7 */ 8 9 #include <linux/acpi.h> 10 #include <linux/bitmap.h> 11 #include <linux/cpu.h> 12 #include <linux/cpuhotplug.h> 13 #include <linux/ctype.h> 14 #include <linux/log2.h> 15 #include <linux/memory.h> 16 #include <linux/module.h> 17 #include <linux/of.h> 18 #include <asm/acpi.h> 19 #include <asm/alternative.h> 20 #include <asm/bugs.h> 21 #include <asm/cacheflush.h> 22 #include <asm/cpufeature.h> 23 #include <asm/hwcap.h> 24 #include <asm/text-patching.h> 25 #include <asm/hwprobe.h> 26 #include <asm/processor.h> 27 #include <asm/sbi.h> 28 #include <asm/vector.h> 29 #include <asm/vendor_extensions.h> 30 #include <asm/vendor_extensions/thead.h> 31 #include <asm/usercfi.h> 32 33 #define NUM_ALPHA_EXTS ('z' - 'a' + 1) 34 35 static bool any_cpu_has_zicboz; 36 static bool any_cpu_has_zicbop; 37 static bool any_cpu_has_zicbom; 38 39 unsigned long elf_hwcap __read_mostly; 40 41 /* Host ISA bitmap */ 42 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; 43 44 /* Per-cpu ISA extensions. */ 45 struct riscv_isainfo hart_isa[NR_CPUS]; 46 47 u32 thead_vlenb_of; 48 49 /** 50 * riscv_isa_extension_base() - Get base extension word 51 * 52 * @isa_bitmap: ISA bitmap to use 53 * Return: base extension word as unsigned long value 54 * 55 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 56 */ 57 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) 58 { 59 return !isa_bitmap ? riscv_isa[0] : isa_bitmap[0]; 60 } 61 EXPORT_SYMBOL_GPL(riscv_isa_extension_base); 62 63 /** 64 * __riscv_isa_extension_available() - Check whether given extension 65 * is available or not 66 * 67 * @isa_bitmap: ISA bitmap to use 68 * @bit: bit position of the desired extension 69 * Return: true or false 70 * 71 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 72 */ 73 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit) 74 { 75 const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; 76 77 if (bit >= RISCV_ISA_EXT_MAX) 78 return false; 79 80 return test_bit(bit, bmap); 81 } 82 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); 83 84 static int riscv_ext_f_depends(const struct riscv_isa_ext_data *data, 85 const unsigned long *isa_bitmap) 86 { 87 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F)) 88 return 0; 89 90 return -EPROBE_DEFER; 91 } 92 93 static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data, 94 const unsigned long *isa_bitmap) 95 { 96 if (!riscv_cbom_block_size) { 97 pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n"); 98 return -EINVAL; 99 } 100 if (!is_power_of_2(riscv_cbom_block_size)) { 101 pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); 102 return -EINVAL; 103 } 104 105 any_cpu_has_zicbom = true; 106 return 0; 107 } 108 109 static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data, 110 const unsigned long *isa_bitmap) 111 { 112 if (!riscv_cboz_block_size) { 113 pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n"); 114 return -EINVAL; 115 } 116 if (!is_power_of_2(riscv_cboz_block_size)) { 117 pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n"); 118 return -EINVAL; 119 } 120 any_cpu_has_zicboz = true; 121 return 0; 122 } 123 124 static int riscv_ext_zicbop_validate(const struct riscv_isa_ext_data *data, 125 const unsigned long *isa_bitmap) 126 { 127 if (!riscv_cbop_block_size) { 128 pr_err("Zicbop detected in ISA string, disabling as no cbop-block-size found\n"); 129 return -EINVAL; 130 } 131 if (!is_power_of_2(riscv_cbop_block_size)) { 132 pr_err("Zicbop disabled as cbop-block-size present, but is not a power-of-2\n"); 133 return -EINVAL; 134 } 135 any_cpu_has_zicbop = true; 136 return 0; 137 } 138 139 static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data, 140 const unsigned long *isa_bitmap) 141 { 142 if (!IS_ENABLED(CONFIG_FPU)) 143 return -EINVAL; 144 145 /* 146 * Due to extension ordering, d is checked before f, so no deferral 147 * is required. 148 */ 149 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) { 150 pr_warn_once("This kernel does not support systems with F but not D\n"); 151 return -EINVAL; 152 } 153 154 return 0; 155 } 156 157 static int riscv_ext_d_validate(const struct riscv_isa_ext_data *data, 158 const unsigned long *isa_bitmap) 159 { 160 if (!IS_ENABLED(CONFIG_FPU)) 161 return -EINVAL; 162 163 return 0; 164 } 165 166 static int riscv_ext_vector_x_validate(const struct riscv_isa_ext_data *data, 167 const unsigned long *isa_bitmap) 168 { 169 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 170 return -EINVAL; 171 172 return 0; 173 } 174 175 static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data, 176 const unsigned long *isa_bitmap) 177 { 178 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 179 return -EINVAL; 180 181 if (!IS_ENABLED(CONFIG_FPU)) 182 return -EINVAL; 183 184 /* 185 * The kernel doesn't support systems that don't implement both of 186 * F and D, so if any of the vector extensions that do floating point 187 * are to be usable, both floating point extensions need to be usable. 188 * 189 * Since this function validates vector only, and v/Zve* are probed 190 * after f/d, there's no need for a deferral here. 191 */ 192 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) 193 return -EINVAL; 194 195 return 0; 196 } 197 198 static int riscv_ext_vector_crypto_validate(const struct riscv_isa_ext_data *data, 199 const unsigned long *isa_bitmap) 200 { 201 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 202 return -EINVAL; 203 204 /* 205 * It isn't the kernel's job to check that the binding is correct, so 206 * it should be enough to check that any of the vector extensions are 207 * enabled, which in-turn means that vector is usable in this kernel 208 */ 209 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32X)) 210 return -EPROBE_DEFER; 211 212 return 0; 213 } 214 215 static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data, 216 const unsigned long *isa_bitmap) 217 { 218 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA)) 219 return 0; 220 221 return -EPROBE_DEFER; 222 } 223 static int riscv_ext_zcd_validate(const struct riscv_isa_ext_data *data, 224 const unsigned long *isa_bitmap) 225 { 226 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && 227 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) 228 return 0; 229 230 return -EPROBE_DEFER; 231 } 232 233 static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data, 234 const unsigned long *isa_bitmap) 235 { 236 if (IS_ENABLED(CONFIG_64BIT)) 237 return -EINVAL; 238 239 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && 240 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F)) 241 return 0; 242 243 return -EPROBE_DEFER; 244 } 245 246 static int riscv_ext_zilsd_validate(const struct riscv_isa_ext_data *data, 247 const unsigned long *isa_bitmap) 248 { 249 if (IS_ENABLED(CONFIG_64BIT)) 250 return -EINVAL; 251 252 return 0; 253 } 254 255 static int riscv_ext_zclsd_validate(const struct riscv_isa_ext_data *data, 256 const unsigned long *isa_bitmap) 257 { 258 if (IS_ENABLED(CONFIG_64BIT)) 259 return -EINVAL; 260 261 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZILSD) && 262 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA)) 263 return 0; 264 265 return -EPROBE_DEFER; 266 } 267 268 static int riscv_vector_f_validate(const struct riscv_isa_ext_data *data, 269 const unsigned long *isa_bitmap) 270 { 271 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 272 return -EINVAL; 273 274 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32F)) 275 return 0; 276 277 return -EPROBE_DEFER; 278 } 279 280 static int riscv_ext_zvfbfwma_validate(const struct riscv_isa_ext_data *data, 281 const unsigned long *isa_bitmap) 282 { 283 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZFBFMIN) && 284 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVFBFMIN)) 285 return 0; 286 287 return -EPROBE_DEFER; 288 } 289 290 static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data, 291 const unsigned long *isa_bitmap) 292 { 293 /* SVADE has already been detected, use SVADE only */ 294 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SVADE)) 295 return -EOPNOTSUPP; 296 297 return 0; 298 } 299 300 static int riscv_cfilp_validate(const struct riscv_isa_ext_data *data, 301 const unsigned long *isa_bitmap) 302 { 303 if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) || 304 (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI)) 305 return -EINVAL; 306 307 return 0; 308 } 309 310 static int riscv_cfiss_validate(const struct riscv_isa_ext_data *data, 311 const unsigned long *isa_bitmap) 312 { 313 if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) || 314 (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI)) 315 return -EINVAL; 316 317 return 0; 318 } 319 320 static const unsigned int riscv_a_exts[] = { 321 RISCV_ISA_EXT_ZAAMO, 322 RISCV_ISA_EXT_ZALRSC, 323 }; 324 325 #define RISCV_ISA_EXT_ZKN \ 326 RISCV_ISA_EXT_ZBKB, \ 327 RISCV_ISA_EXT_ZBKC, \ 328 RISCV_ISA_EXT_ZBKX, \ 329 RISCV_ISA_EXT_ZKND, \ 330 RISCV_ISA_EXT_ZKNE, \ 331 RISCV_ISA_EXT_ZKNH 332 333 static int riscv_ext_smcdeleg_validate(const struct riscv_isa_ext_data *data, 334 const unsigned long *isa_bitmap) 335 { 336 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SSCSRIND) && 337 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZIHPM) && 338 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZICNTR)) 339 return 0; 340 341 return -EPROBE_DEFER; 342 } 343 344 static int riscv_ext_ssccfg_validate(const struct riscv_isa_ext_data *data, 345 const unsigned long *isa_bitmap) 346 { 347 if (!riscv_ext_smcdeleg_validate(data, isa_bitmap) && 348 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SMCDELEG)) 349 return 0; 350 351 return -EPROBE_DEFER; 352 } 353 354 static const unsigned int riscv_zk_bundled_exts[] = { 355 RISCV_ISA_EXT_ZKN, 356 RISCV_ISA_EXT_ZKR, 357 RISCV_ISA_EXT_ZKT 358 }; 359 360 static const unsigned int riscv_zkn_bundled_exts[] = { 361 RISCV_ISA_EXT_ZKN 362 }; 363 364 static const unsigned int riscv_zks_bundled_exts[] = { 365 RISCV_ISA_EXT_ZBKB, 366 RISCV_ISA_EXT_ZBKC, 367 RISCV_ISA_EXT_ZKSED, 368 RISCV_ISA_EXT_ZKSH 369 }; 370 371 #define RISCV_ISA_EXT_ZVKN \ 372 RISCV_ISA_EXT_ZVKNED, \ 373 RISCV_ISA_EXT_ZVKNHB, \ 374 RISCV_ISA_EXT_ZVKB, \ 375 RISCV_ISA_EXT_ZVKT 376 377 static const unsigned int riscv_zvkn_bundled_exts[] = { 378 RISCV_ISA_EXT_ZVKN 379 }; 380 381 static const unsigned int riscv_zvknc_bundled_exts[] = { 382 RISCV_ISA_EXT_ZVKN, 383 RISCV_ISA_EXT_ZVBC 384 }; 385 386 static const unsigned int riscv_zvkng_bundled_exts[] = { 387 RISCV_ISA_EXT_ZVKN, 388 RISCV_ISA_EXT_ZVKG 389 }; 390 391 #define RISCV_ISA_EXT_ZVKS \ 392 RISCV_ISA_EXT_ZVKSED, \ 393 RISCV_ISA_EXT_ZVKSH, \ 394 RISCV_ISA_EXT_ZVKB, \ 395 RISCV_ISA_EXT_ZVKT 396 397 static const unsigned int riscv_zvks_bundled_exts[] = { 398 RISCV_ISA_EXT_ZVKS 399 }; 400 401 static const unsigned int riscv_zvksc_bundled_exts[] = { 402 RISCV_ISA_EXT_ZVKS, 403 RISCV_ISA_EXT_ZVBC 404 }; 405 406 static const unsigned int riscv_zvksg_bundled_exts[] = { 407 RISCV_ISA_EXT_ZVKS, 408 RISCV_ISA_EXT_ZVKG 409 }; 410 411 static const unsigned int riscv_zvbb_exts[] = { 412 RISCV_ISA_EXT_ZVKB 413 }; 414 415 #define RISCV_ISA_EXT_ZVE64F_IMPLY_LIST \ 416 RISCV_ISA_EXT_ZVE64X, \ 417 RISCV_ISA_EXT_ZVE32F, \ 418 RISCV_ISA_EXT_ZVE32X 419 420 #define RISCV_ISA_EXT_ZVE64D_IMPLY_LIST \ 421 RISCV_ISA_EXT_ZVE64F, \ 422 RISCV_ISA_EXT_ZVE64F_IMPLY_LIST 423 424 #define RISCV_ISA_EXT_V_IMPLY_LIST \ 425 RISCV_ISA_EXT_ZVE64D, \ 426 RISCV_ISA_EXT_ZVE64D_IMPLY_LIST 427 428 static const unsigned int riscv_zve32f_exts[] = { 429 RISCV_ISA_EXT_ZVE32X 430 }; 431 432 static const unsigned int riscv_zve64f_exts[] = { 433 RISCV_ISA_EXT_ZVE64F_IMPLY_LIST 434 }; 435 436 static const unsigned int riscv_zve64d_exts[] = { 437 RISCV_ISA_EXT_ZVE64D_IMPLY_LIST 438 }; 439 440 static const unsigned int riscv_v_exts[] = { 441 RISCV_ISA_EXT_V_IMPLY_LIST 442 }; 443 444 static const unsigned int riscv_zve64x_exts[] = { 445 RISCV_ISA_EXT_ZVE32X, 446 RISCV_ISA_EXT_ZVE64X 447 }; 448 449 /* 450 * While the [ms]envcfg CSRs were not defined until version 1.12 of the RISC-V 451 * privileged ISA, the existence of the CSRs is implied by any extension which 452 * specifies [ms]envcfg bit(s). Hence, we define a custom ISA extension for the 453 * existence of the CSR, and treat it as a subset of those other extensions. 454 */ 455 static const unsigned int riscv_xlinuxenvcfg_exts[] = { 456 RISCV_ISA_EXT_XLINUXENVCFG 457 }; 458 459 /* 460 * Zc* spec states that: 461 * - C always implies Zca 462 * - C+F implies Zcf (RV32 only) 463 * - C+D implies Zcd 464 * 465 * These extensions will be enabled and then validated depending on the 466 * availability of F/D RV32. 467 */ 468 static const unsigned int riscv_c_exts[] = { 469 RISCV_ISA_EXT_ZCA, 470 RISCV_ISA_EXT_ZCF, 471 RISCV_ISA_EXT_ZCD, 472 }; 473 474 /* 475 * The canonical order of ISA extension names in the ISA string is defined in 476 * Chapter 27 of the RISC-V Instruction Set Manual Volume I Unprivileged ISA 477 * (Document Version 20191213). 478 * 479 * Ordinarily, for in-kernel data structures, this order is unimportant but 480 * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. 481 * 482 * The specification uses vague wording, such as should, when it comes to 483 * ordering, so for our purposes the following rules apply: 484 * 485 * 1. All multi-letter extensions must be separated from other extensions by an 486 * underscore. 487 * 488 * 2. Additional standard extensions (starting with 'Z') must be sorted after 489 * single-letter extensions and before any higher-privileged extensions. 490 * 491 * 3. The first letter following the 'Z' conventionally indicates the most 492 * closely related alphabetical extension category, IMAFDQLCBKJTPVH. 493 * If multiple 'Z' extensions are named, they must be ordered first by 494 * category, then alphabetically within a category. 495 * 496 * 3. Standard supervisor-level extensions (starting with 'S') must be listed 497 * after standard unprivileged extensions. If multiple supervisor-level 498 * extensions are listed, they must be ordered alphabetically. 499 * 500 * 4. Standard machine-level extensions (starting with 'Zxm') must be listed 501 * after any lower-privileged, standard extensions. If multiple 502 * machine-level extensions are listed, they must be ordered 503 * alphabetically. 504 * 505 * 5. Non-standard extensions (starting with 'X') must be listed after all 506 * standard extensions. If multiple non-standard extensions are listed, they 507 * must be ordered alphabetically. 508 * 509 * An example string following the order is: 510 * rv64imadc_zifoo_zigoo_zafoo_sbar_scar_zxmbaz_xqux_xrux 511 * 512 * New entries to this struct should follow the ordering rules described above. 513 */ 514 const struct riscv_isa_ext_data riscv_isa_ext[] = { 515 __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_I), 516 __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_M), 517 __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_A, riscv_a_exts), 518 __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_F, riscv_ext_f_validate), 519 __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_D, riscv_ext_d_validate), 520 __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_Q), 521 __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_C, riscv_c_exts), 522 __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_V, riscv_v_exts, 523 riscv_ext_vector_float_validate), 524 __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_H), 525 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate), 526 __RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate), 527 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate), 528 __RISCV_ISA_EXT_DATA(ziccamoa, RISCV_ISA_EXT_ZICCAMOA), 529 __RISCV_ISA_EXT_DATA(ziccif, RISCV_ISA_EXT_ZICCIF), 530 __RISCV_ISA_EXT_DATA(zicclsm, RISCV_ISA_EXT_ZICCLSM), 531 __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE), 532 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfilp, RISCV_ISA_EXT_ZICFILP, riscv_xlinuxenvcfg_exts, 533 riscv_cfilp_validate), 534 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfiss, RISCV_ISA_EXT_ZICFISS, riscv_xlinuxenvcfg_exts, 535 riscv_cfiss_validate), 536 __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), 537 __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), 538 __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), 539 __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI), 540 __RISCV_ISA_EXT_DATA(zihintntl, RISCV_ISA_EXT_ZIHINTNTL), 541 __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), 542 __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), 543 __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP), 544 __RISCV_ISA_EXT_DATA(za64rs, RISCV_ISA_EXT_ZA64RS), 545 __RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO), 546 __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA), 547 __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), 548 __RISCV_ISA_EXT_DATA(zalasr, RISCV_ISA_EXT_ZALASR), 549 __RISCV_ISA_EXT_DATA(zalrsc, RISCV_ISA_EXT_ZALRSC), 550 __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), 551 __RISCV_ISA_EXT_DATA_VALIDATE(zfa, RISCV_ISA_EXT_ZFA, riscv_ext_f_depends), 552 __RISCV_ISA_EXT_DATA_VALIDATE(zfbfmin, RISCV_ISA_EXT_ZFBFMIN, riscv_ext_f_depends), 553 __RISCV_ISA_EXT_DATA_VALIDATE(zfh, RISCV_ISA_EXT_ZFH, riscv_ext_f_depends), 554 __RISCV_ISA_EXT_DATA_VALIDATE(zfhmin, RISCV_ISA_EXT_ZFHMIN, riscv_ext_f_depends), 555 __RISCV_ISA_EXT_DATA(zca, RISCV_ISA_EXT_ZCA), 556 __RISCV_ISA_EXT_DATA_VALIDATE(zcb, RISCV_ISA_EXT_ZCB, riscv_ext_zca_depends), 557 __RISCV_ISA_EXT_DATA_VALIDATE(zcd, RISCV_ISA_EXT_ZCD, riscv_ext_zcd_validate), 558 __RISCV_ISA_EXT_DATA_VALIDATE(zcf, RISCV_ISA_EXT_ZCF, riscv_ext_zcf_validate), 559 __RISCV_ISA_EXT_DATA_VALIDATE(zcmop, RISCV_ISA_EXT_ZCMOP, riscv_ext_zca_depends), 560 __RISCV_ISA_EXT_DATA_VALIDATE(zclsd, RISCV_ISA_EXT_ZCLSD, riscv_ext_zclsd_validate), 561 __RISCV_ISA_EXT_DATA_VALIDATE(zilsd, RISCV_ISA_EXT_ZILSD, riscv_ext_zilsd_validate), 562 __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), 563 __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), 564 __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC), 565 __RISCV_ISA_EXT_DATA(zbkb, RISCV_ISA_EXT_ZBKB), 566 __RISCV_ISA_EXT_DATA(zbkc, RISCV_ISA_EXT_ZBKC), 567 __RISCV_ISA_EXT_DATA(zbkx, RISCV_ISA_EXT_ZBKX), 568 __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS), 569 __RISCV_ISA_EXT_BUNDLE(zk, riscv_zk_bundled_exts), 570 __RISCV_ISA_EXT_BUNDLE(zkn, riscv_zkn_bundled_exts), 571 __RISCV_ISA_EXT_DATA(zknd, RISCV_ISA_EXT_ZKND), 572 __RISCV_ISA_EXT_DATA(zkne, RISCV_ISA_EXT_ZKNE), 573 __RISCV_ISA_EXT_DATA(zknh, RISCV_ISA_EXT_ZKNH), 574 __RISCV_ISA_EXT_DATA(zkr, RISCV_ISA_EXT_ZKR), 575 __RISCV_ISA_EXT_BUNDLE(zks, riscv_zks_bundled_exts), 576 __RISCV_ISA_EXT_DATA(zkt, RISCV_ISA_EXT_ZKT), 577 __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED), 578 __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), 579 __RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO), 580 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts, riscv_ext_vector_crypto_validate), 581 __RISCV_ISA_EXT_DATA_VALIDATE(zvbc, RISCV_ISA_EXT_ZVBC, riscv_ext_vector_crypto_validate), 582 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts, riscv_ext_vector_float_validate), 583 __RISCV_ISA_EXT_DATA_VALIDATE(zve32x, RISCV_ISA_EXT_ZVE32X, riscv_ext_vector_x_validate), 584 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts, riscv_ext_vector_float_validate), 585 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts, riscv_ext_vector_float_validate), 586 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts, riscv_ext_vector_x_validate), 587 __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfmin, RISCV_ISA_EXT_ZVFBFMIN, riscv_vector_f_validate), 588 __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfwma, RISCV_ISA_EXT_ZVFBFWMA, riscv_ext_zvfbfwma_validate), 589 __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), 590 __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), 591 __RISCV_ISA_EXT_DATA_VALIDATE(zvkb, RISCV_ISA_EXT_ZVKB, riscv_ext_vector_crypto_validate), 592 __RISCV_ISA_EXT_DATA_VALIDATE(zvkg, RISCV_ISA_EXT_ZVKG, riscv_ext_vector_crypto_validate), 593 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkn, riscv_zvkn_bundled_exts, riscv_ext_vector_crypto_validate), 594 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvknc, riscv_zvknc_bundled_exts, riscv_ext_vector_crypto_validate), 595 __RISCV_ISA_EXT_DATA_VALIDATE(zvkned, RISCV_ISA_EXT_ZVKNED, riscv_ext_vector_crypto_validate), 596 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkng, riscv_zvkng_bundled_exts, riscv_ext_vector_crypto_validate), 597 __RISCV_ISA_EXT_DATA_VALIDATE(zvknha, RISCV_ISA_EXT_ZVKNHA, riscv_ext_vector_crypto_validate), 598 __RISCV_ISA_EXT_DATA_VALIDATE(zvknhb, RISCV_ISA_EXT_ZVKNHB, riscv_ext_vector_crypto_validate), 599 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvks, riscv_zvks_bundled_exts, riscv_ext_vector_crypto_validate), 600 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksc, riscv_zvksc_bundled_exts, riscv_ext_vector_crypto_validate), 601 __RISCV_ISA_EXT_DATA_VALIDATE(zvksed, RISCV_ISA_EXT_ZVKSED, riscv_ext_vector_crypto_validate), 602 __RISCV_ISA_EXT_DATA_VALIDATE(zvksh, RISCV_ISA_EXT_ZVKSH, riscv_ext_vector_crypto_validate), 603 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksg, riscv_zvksg_bundled_exts, riscv_ext_vector_crypto_validate), 604 __RISCV_ISA_EXT_DATA_VALIDATE(zvkt, RISCV_ISA_EXT_ZVKT, riscv_ext_vector_crypto_validate), 605 __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), 606 __RISCV_ISA_EXT_DATA_VALIDATE(smcdeleg, RISCV_ISA_EXT_SMCDELEG, 607 riscv_ext_smcdeleg_validate), 608 __RISCV_ISA_EXT_DATA(smcntrpmf, RISCV_ISA_EXT_SMCNTRPMF), 609 __RISCV_ISA_EXT_DATA(smcsrind, RISCV_ISA_EXT_SMCSRIND), 610 __RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM), 611 __RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts), 612 __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), 613 __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), 614 __RISCV_ISA_EXT_DATA_VALIDATE(ssccfg, RISCV_ISA_EXT_SSCCFG, riscv_ext_ssccfg_validate), 615 __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), 616 __RISCV_ISA_EXT_DATA(sscsrind, RISCV_ISA_EXT_SSCSRIND), 617 __RISCV_ISA_EXT_SUPERSET(ssnpm, RISCV_ISA_EXT_SSNPM, riscv_xlinuxenvcfg_exts), 618 __RISCV_ISA_EXT_DATA(ssqosid, RISCV_ISA_EXT_SSQOSID), 619 __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), 620 __RISCV_ISA_EXT_DATA(svade, RISCV_ISA_EXT_SVADE), 621 __RISCV_ISA_EXT_DATA_VALIDATE(svadu, RISCV_ISA_EXT_SVADU, riscv_ext_svadu_validate), 622 __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), 623 __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT), 624 __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), 625 __RISCV_ISA_EXT_DATA(svrsw60t59b, RISCV_ISA_EXT_SVRSW60T59B), 626 __RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC), 627 }; 628 629 const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); 630 631 static void riscv_isa_set_ext(const struct riscv_isa_ext_data *ext, unsigned long *bitmap) 632 { 633 if (ext->id != RISCV_ISA_EXT_INVALID) 634 set_bit(ext->id, bitmap); 635 636 for (int i = 0; i < ext->subset_ext_size; i++) { 637 if (ext->subset_ext_ids[i] != RISCV_ISA_EXT_INVALID) 638 set_bit(ext->subset_ext_ids[i], bitmap); 639 } 640 } 641 642 static const struct riscv_isa_ext_data *riscv_get_isa_ext_data(unsigned int ext_id) 643 { 644 for (int i = 0; i < riscv_isa_ext_count; i++) { 645 if (riscv_isa_ext[i].id == ext_id) 646 return &riscv_isa_ext[i]; 647 } 648 649 return NULL; 650 } 651 652 /* 653 * "Resolve" a source ISA bitmap into one that matches kernel configuration as 654 * well as correct extension dependencies. Some extensions depends on specific 655 * kernel configuration to be usable (V needs CONFIG_RISCV_ISA_V for instance) 656 * and this function will actually validate all the extensions provided in 657 * source_isa into the resolved_isa based on extensions validate() callbacks. 658 */ 659 static void __init riscv_resolve_isa(unsigned long *source_isa, 660 unsigned long *resolved_isa, unsigned long *this_hwcap, 661 unsigned long *isa2hwcap) 662 { 663 bool loop; 664 const struct riscv_isa_ext_data *ext; 665 DECLARE_BITMAP(prev_resolved_isa, RISCV_ISA_EXT_MAX); 666 int max_loop_count = riscv_isa_ext_count, ret; 667 unsigned int bit; 668 669 do { 670 loop = false; 671 if (max_loop_count-- < 0) { 672 pr_err("Failed to reach a stable ISA state\n"); 673 return; 674 } 675 bitmap_copy(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX); 676 for_each_set_bit(bit, source_isa, RISCV_ISA_EXT_MAX) { 677 ext = riscv_get_isa_ext_data(bit); 678 679 if (ext && ext->validate) { 680 ret = ext->validate(ext, resolved_isa); 681 if (ret == -EPROBE_DEFER) { 682 loop = true; 683 continue; 684 } else if (ret) { 685 /* Disable the extension entirely */ 686 clear_bit(bit, source_isa); 687 continue; 688 } 689 } 690 691 set_bit(bit, resolved_isa); 692 /* No need to keep it in source isa now that it is enabled */ 693 clear_bit(bit, source_isa); 694 695 /* Single letter extensions get set in hwcap */ 696 if (bit < RISCV_ISA_EXT_BASE) 697 *this_hwcap |= isa2hwcap[bit]; 698 } 699 } while (loop && !bitmap_equal(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX)); 700 } 701 702 static void __init match_isa_ext(const char *name, const char *name_end, unsigned long *bitmap) 703 { 704 for (int i = 0; i < riscv_isa_ext_count; i++) { 705 const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i]; 706 707 if ((name_end - name == strlen(ext->name)) && 708 !strncasecmp(name, ext->name, name_end - name)) { 709 riscv_isa_set_ext(ext, bitmap); 710 break; 711 } 712 } 713 } 714 715 static void __init riscv_parse_isa_string(const char *isa, unsigned long *bitmap) 716 { 717 /* 718 * For all possible cpus, we have already validated in 719 * the boot process that they at least contain "rv" and 720 * whichever of "32"/"64" this kernel supports, and so this 721 * section can be skipped. 722 */ 723 isa += 4; 724 725 while (*isa) { 726 const char *ext = isa++; 727 const char *ext_end = isa; 728 bool ext_err = false; 729 730 switch (*ext) { 731 case 'x': 732 case 'X': 733 if (acpi_disabled) 734 pr_warn_once("Vendor extensions are ignored in riscv,isa. Use riscv,isa-extensions instead."); 735 /* 736 * To skip an extension, we find its end. 737 * As multi-letter extensions must be split from other multi-letter 738 * extensions with an "_", the end of a multi-letter extension will 739 * either be the null character or the "_" at the start of the next 740 * multi-letter extension. 741 */ 742 for (; *isa && *isa != '_'; ++isa) 743 ; 744 ext_err = true; 745 break; 746 case 's': 747 /* 748 * Workaround for invalid single-letter 's' & 'u' (QEMU). 749 * No need to set the bit in riscv_isa as 's' & 'u' are 750 * not valid ISA extensions. It works unless the first 751 * multi-letter extension in the ISA string begins with 752 * "Su" and is not prefixed with an underscore. 753 */ 754 if (ext[-1] != '_' && ext[1] == 'u') { 755 ++isa; 756 ext_err = true; 757 break; 758 } 759 fallthrough; 760 case 'S': 761 case 'z': 762 case 'Z': 763 /* 764 * Before attempting to parse the extension itself, we find its end. 765 * As multi-letter extensions must be split from other multi-letter 766 * extensions with an "_", the end of a multi-letter extension will 767 * either be the null character or the "_" at the start of the next 768 * multi-letter extension. 769 * 770 * Next, as the extensions version is currently ignored, we 771 * eliminate that portion. This is done by parsing backwards from 772 * the end of the extension, removing any numbers. This may be a 773 * major or minor number however, so the process is repeated if a 774 * minor number was found. 775 * 776 * ext_end is intended to represent the first character *after* the 777 * name portion of an extension, but will be decremented to the last 778 * character itself while eliminating the extensions version number. 779 * A simple re-increment solves this problem. 780 */ 781 for (; *isa && *isa != '_'; ++isa) 782 if (unlikely(!isalnum(*isa))) 783 ext_err = true; 784 785 ext_end = isa; 786 if (unlikely(ext_err)) 787 break; 788 789 if (!isdigit(ext_end[-1])) 790 break; 791 792 while (isdigit(*--ext_end)) 793 ; 794 795 if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) { 796 ++ext_end; 797 break; 798 } 799 800 while (isdigit(*--ext_end)) 801 ; 802 803 ++ext_end; 804 break; 805 default: 806 /* 807 * Things are a little easier for single-letter extensions, as they 808 * are parsed forwards. 809 * 810 * After checking that our starting position is valid, we need to 811 * ensure that, when isa was incremented at the start of the loop, 812 * that it arrived at the start of the next extension. 813 * 814 * If we are already on a non-digit, there is nothing to do. Either 815 * we have a multi-letter extension's _, or the start of an 816 * extension. 817 * 818 * Otherwise we have found the current extension's major version 819 * number. Parse past it, and a subsequent p/minor version number 820 * if present. The `p` extension must not appear immediately after 821 * a number, so there is no fear of missing it. 822 * 823 */ 824 if (unlikely(!isalpha(*ext))) { 825 ext_err = true; 826 break; 827 } 828 829 if (!isdigit(*isa)) 830 break; 831 832 while (isdigit(*++isa)) 833 ; 834 835 if (tolower(*isa) != 'p') 836 break; 837 838 if (!isdigit(*++isa)) { 839 --isa; 840 break; 841 } 842 843 while (isdigit(*++isa)) 844 ; 845 846 break; 847 } 848 849 /* 850 * The parser expects that at the start of an iteration isa points to the 851 * first character of the next extension. As we stop parsing an extension 852 * on meeting a non-alphanumeric character, an extra increment is needed 853 * where the succeeding extension is a multi-letter prefixed with an "_". 854 */ 855 if (*isa == '_') 856 ++isa; 857 858 if (unlikely(ext_err)) 859 continue; 860 861 match_isa_ext(ext, ext_end, bitmap); 862 } 863 } 864 865 static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) 866 { 867 struct device_node *node; 868 const char *isa; 869 int rc; 870 struct acpi_table_header *rhct; 871 acpi_status status; 872 unsigned int cpu; 873 u64 boot_vendorid; 874 u64 boot_archid; 875 876 if (!acpi_disabled) { 877 status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); 878 if (ACPI_FAILURE(status)) 879 return; 880 } 881 882 boot_vendorid = riscv_get_mvendorid(); 883 boot_archid = riscv_get_marchid(); 884 885 for_each_possible_cpu(cpu) { 886 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 887 unsigned long this_hwcap = 0; 888 DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 }; 889 890 if (acpi_disabled) { 891 node = of_cpu_device_node_get(cpu); 892 if (!node) { 893 pr_warn("Unable to find cpu node\n"); 894 continue; 895 } 896 897 rc = of_property_read_string(node, "riscv,isa", &isa); 898 of_node_put(node); 899 if (rc) { 900 pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); 901 continue; 902 } 903 } else { 904 rc = acpi_get_riscv_isa(rhct, cpu, &isa); 905 if (rc < 0) { 906 pr_warn("Unable to get ISA for the hart - %d\n", cpu); 907 continue; 908 } 909 } 910 911 riscv_parse_isa_string(isa, source_isa); 912 913 /* 914 * These ones were as they were part of the base ISA when the 915 * port & dt-bindings were upstreamed, and so can be set 916 * unconditionally where `i` is in riscv,isa on DT systems. 917 */ 918 if (acpi_disabled) { 919 set_bit(RISCV_ISA_EXT_ZICSR, source_isa); 920 set_bit(RISCV_ISA_EXT_ZIFENCEI, source_isa); 921 set_bit(RISCV_ISA_EXT_ZICNTR, source_isa); 922 set_bit(RISCV_ISA_EXT_ZIHPM, source_isa); 923 } 924 925 /* 926 * "V" in ISA strings is ambiguous in practice: it should mean 927 * just the standard V-1.0 but vendors aren't well behaved. 928 * Many vendors with T-Head CPU cores which implement the 0.7.1 929 * version of the vector specification put "v" into their DTs. 930 * CPU cores with the ratified spec will contain non-zero 931 * marchid. 932 */ 933 if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) 934 clear_bit(RISCV_ISA_EXT_V, source_isa); 935 936 riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); 937 938 /* 939 * All "okay" hart should have same isa. Set HWCAP based on 940 * common capabilities of every "okay" hart, in case they don't 941 * have. 942 */ 943 if (elf_hwcap) 944 elf_hwcap &= this_hwcap; 945 else 946 elf_hwcap = this_hwcap; 947 948 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 949 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 950 else 951 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 952 } 953 954 if (!acpi_disabled && rhct) 955 acpi_put_table((struct acpi_table_header *)rhct); 956 } 957 958 static void __init riscv_fill_cpu_vendor_ext(struct device_node *cpu_node, int cpu) 959 { 960 if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) 961 return; 962 963 for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) { 964 struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i]; 965 966 for (int j = 0; j < ext_list->ext_data_count; j++) { 967 const struct riscv_isa_ext_data ext = ext_list->ext_data[j]; 968 struct riscv_isavendorinfo *isavendorinfo = &ext_list->per_hart_isa_bitmap[cpu]; 969 970 if (of_property_match_string(cpu_node, "riscv,isa-extensions", 971 ext.property) < 0) 972 continue; 973 974 /* 975 * Assume that subset extensions are all members of the 976 * same vendor. 977 */ 978 if (ext.subset_ext_size) 979 for (int k = 0; k < ext.subset_ext_size; k++) 980 set_bit(ext.subset_ext_ids[k], isavendorinfo->isa); 981 982 set_bit(ext.id, isavendorinfo->isa); 983 } 984 } 985 } 986 987 /* 988 * Populate all_harts_isa_bitmap for each vendor with all of the extensions that 989 * are shared across CPUs for that vendor. 990 */ 991 static void __init riscv_fill_vendor_ext_list(int cpu) 992 { 993 if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) 994 return; 995 996 for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) { 997 struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i]; 998 999 if (!ext_list->is_initialized) { 1000 bitmap_copy(ext_list->all_harts_isa_bitmap.isa, 1001 ext_list->per_hart_isa_bitmap[cpu].isa, 1002 RISCV_ISA_VENDOR_EXT_MAX); 1003 ext_list->is_initialized = true; 1004 } else { 1005 bitmap_and(ext_list->all_harts_isa_bitmap.isa, 1006 ext_list->all_harts_isa_bitmap.isa, 1007 ext_list->per_hart_isa_bitmap[cpu].isa, 1008 RISCV_ISA_VENDOR_EXT_MAX); 1009 } 1010 } 1011 } 1012 1013 static int has_thead_homogeneous_vlenb(void) 1014 { 1015 int cpu; 1016 u32 prev_vlenb = 0; 1017 u32 vlenb = 0; 1018 1019 /* Ignore thead,vlenb property if xtheadvector is not enabled in the kernel */ 1020 if (!IS_ENABLED(CONFIG_RISCV_ISA_XTHEADVECTOR)) 1021 return 0; 1022 1023 for_each_possible_cpu(cpu) { 1024 struct device_node *cpu_node; 1025 1026 cpu_node = of_cpu_device_node_get(cpu); 1027 if (!cpu_node) { 1028 pr_warn("Unable to find cpu node\n"); 1029 return -ENOENT; 1030 } 1031 1032 if (of_property_read_u32(cpu_node, "thead,vlenb", &vlenb)) { 1033 of_node_put(cpu_node); 1034 1035 if (prev_vlenb) 1036 return -ENOENT; 1037 continue; 1038 } 1039 1040 if (prev_vlenb && vlenb != prev_vlenb) { 1041 of_node_put(cpu_node); 1042 return -ENOENT; 1043 } 1044 1045 prev_vlenb = vlenb; 1046 of_node_put(cpu_node); 1047 } 1048 1049 thead_vlenb_of = vlenb; 1050 return 0; 1051 } 1052 1053 static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) 1054 { 1055 unsigned int cpu; 1056 bool mitigated; 1057 1058 for_each_possible_cpu(cpu) { 1059 unsigned long this_hwcap = 0; 1060 struct device_node *cpu_node; 1061 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 1062 DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 }; 1063 1064 cpu_node = of_cpu_device_node_get(cpu); 1065 if (!cpu_node) { 1066 pr_warn("Unable to find cpu node\n"); 1067 continue; 1068 } 1069 1070 if (!of_property_present(cpu_node, "riscv,isa-extensions")) { 1071 of_node_put(cpu_node); 1072 continue; 1073 } 1074 1075 for (int i = 0; i < riscv_isa_ext_count; i++) { 1076 const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i]; 1077 1078 if (of_property_match_string(cpu_node, "riscv,isa-extensions", 1079 ext->property) < 0) 1080 continue; 1081 1082 riscv_isa_set_ext(ext, source_isa); 1083 } 1084 1085 riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); 1086 riscv_fill_cpu_vendor_ext(cpu_node, cpu); 1087 1088 of_node_put(cpu_node); 1089 1090 /* 1091 * All "okay" harts should have same isa. Set HWCAP based on 1092 * common capabilities of every "okay" hart, in case they don't. 1093 */ 1094 if (elf_hwcap) 1095 elf_hwcap &= this_hwcap; 1096 else 1097 elf_hwcap = this_hwcap; 1098 1099 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 1100 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 1101 else 1102 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 1103 1104 riscv_fill_vendor_ext_list(cpu); 1105 } 1106 1107 /* 1108 * Execute ghostwrite mitigation immediately after detecting extensions 1109 * to disable xtheadvector if necessary. 1110 */ 1111 mitigated = ghostwrite_enable_mitigation(); 1112 1113 if (!mitigated && has_xtheadvector_no_alternatives() && has_thead_homogeneous_vlenb() < 0) { 1114 pr_warn("Unsupported heterogeneous vlenb detected, vector extension disabled.\n"); 1115 disable_xtheadvector(); 1116 } 1117 1118 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 1119 return -ENOENT; 1120 1121 return 0; 1122 } 1123 1124 #ifdef CONFIG_RISCV_ISA_FALLBACK 1125 bool __initdata riscv_isa_fallback = true; 1126 #else 1127 bool __initdata riscv_isa_fallback; 1128 static int __init riscv_isa_fallback_setup(char *__unused) 1129 { 1130 riscv_isa_fallback = true; 1131 return 1; 1132 } 1133 early_param("riscv_isa_fallback", riscv_isa_fallback_setup); 1134 #endif 1135 1136 void __init riscv_fill_hwcap(void) 1137 { 1138 char print_str[NUM_ALPHA_EXTS + 1]; 1139 unsigned long isa2hwcap[RISCV_ISA_EXT_BASE] = {0}; 1140 int i, j; 1141 1142 isa2hwcap[RISCV_ISA_EXT_I] = COMPAT_HWCAP_ISA_I; 1143 isa2hwcap[RISCV_ISA_EXT_M] = COMPAT_HWCAP_ISA_M; 1144 isa2hwcap[RISCV_ISA_EXT_A] = COMPAT_HWCAP_ISA_A; 1145 isa2hwcap[RISCV_ISA_EXT_F] = COMPAT_HWCAP_ISA_F; 1146 isa2hwcap[RISCV_ISA_EXT_D] = COMPAT_HWCAP_ISA_D; 1147 isa2hwcap[RISCV_ISA_EXT_C] = COMPAT_HWCAP_ISA_C; 1148 isa2hwcap[RISCV_ISA_EXT_V] = COMPAT_HWCAP_ISA_V; 1149 1150 if (!acpi_disabled) { 1151 riscv_fill_hwcap_from_isa_string(isa2hwcap); 1152 } else { 1153 int ret = riscv_fill_hwcap_from_ext_list(isa2hwcap); 1154 1155 if (ret && riscv_isa_fallback) { 1156 pr_info("Falling back to deprecated \"riscv,isa\"\n"); 1157 riscv_fill_hwcap_from_isa_string(isa2hwcap); 1158 } 1159 } 1160 1161 /* 1162 * We don't support systems with F but without D, so mask those out 1163 * here. 1164 */ 1165 if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { 1166 pr_info("This kernel does not support systems with F but not D\n"); 1167 elf_hwcap &= ~COMPAT_HWCAP_ISA_F; 1168 } 1169 1170 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_ZVE32X) || 1171 has_xtheadvector_no_alternatives()) { 1172 /* 1173 * This cannot fail when called on the boot hart 1174 */ 1175 riscv_v_setup_vsize(); 1176 } 1177 1178 memset(print_str, 0, sizeof(print_str)); 1179 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 1180 if (riscv_isa[0] & BIT_MASK(i)) 1181 print_str[j++] = (char)('a' + i); 1182 pr_info("riscv: base ISA extensions %s\n", print_str); 1183 1184 memset(print_str, 0, sizeof(print_str)); 1185 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 1186 if (elf_hwcap & BIT_MASK(i)) 1187 print_str[j++] = (char)('a' + i); 1188 pr_info("riscv: ELF capabilities %s\n", print_str); 1189 } 1190 1191 unsigned long riscv_get_elf_hwcap(void) 1192 { 1193 unsigned long hwcap; 1194 1195 hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1)); 1196 1197 if (!riscv_v_vstate_ctrl_user_allowed()) 1198 hwcap &= ~COMPAT_HWCAP_ISA_V; 1199 1200 return hwcap; 1201 } 1202 1203 void __init riscv_user_isa_enable(void) 1204 { 1205 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOZ)) 1206 current->thread.envcfg |= ENVCFG_CBZE; 1207 else if (any_cpu_has_zicboz) 1208 pr_warn("Zicboz disabled as it is unavailable on some harts\n"); 1209 1210 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOM)) 1211 current->thread.envcfg |= ENVCFG_CBCFE; 1212 else if (any_cpu_has_zicbom) 1213 pr_warn("Zicbom disabled as it is unavailable on some harts\n"); 1214 1215 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOP) && 1216 any_cpu_has_zicbop) 1217 pr_warn("Zicbop disabled as it is unavailable on some harts\n"); 1218 } 1219 1220 #ifdef CONFIG_RISCV_ALTERNATIVE 1221 /* 1222 * Alternative patch sites consider 48 bits when determining when to patch 1223 * the old instruction sequence with the new. These bits are broken into a 1224 * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the 1225 * patch site is for an erratum, identified by the 32-bit patch ID. When 1226 * the vendor ID is zero, the patch site is for a cpufeature. cpufeatures 1227 * further break down patch ID into two 16-bit numbers. The lower 16 bits 1228 * are the cpufeature ID and the upper 16 bits are used for a value specific 1229 * to the cpufeature and patch site. If the upper 16 bits are zero, then it 1230 * implies no specific value is specified. cpufeatures that want to control 1231 * patching on a per-site basis will provide non-zero values and implement 1232 * checks here. The checks return true when patching should be done, and 1233 * false otherwise. 1234 */ 1235 static bool riscv_cpufeature_patch_check(u16 id, u16 value) 1236 { 1237 if (!value) 1238 return true; 1239 1240 switch (id) { 1241 case RISCV_ISA_EXT_ZICBOZ: 1242 /* 1243 * Zicboz alternative applications provide the maximum 1244 * supported block size order, or zero when it doesn't 1245 * matter. If the current block size exceeds the maximum, 1246 * then the alternative cannot be applied. 1247 */ 1248 return riscv_cboz_block_size <= (1U << value); 1249 } 1250 1251 return false; 1252 } 1253 1254 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, 1255 struct alt_entry *end, 1256 unsigned int stage) 1257 { 1258 struct alt_entry *alt; 1259 void *oldptr, *altptr; 1260 u16 id, value, vendor; 1261 1262 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 1263 return; 1264 1265 for (alt = begin; alt < end; alt++) { 1266 id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); 1267 vendor = PATCH_ID_CPUFEATURE_ID(alt->vendor_id); 1268 1269 /* 1270 * Any alternative with a patch_id that is less than 1271 * RISCV_ISA_EXT_MAX is interpreted as a standard extension. 1272 * 1273 * Any alternative with patch_id that is greater than or equal 1274 * to RISCV_VENDOR_EXT_ALTERNATIVES_BASE is interpreted as a 1275 * vendor extension. 1276 */ 1277 if (id < RISCV_ISA_EXT_MAX) { 1278 /* 1279 * This patch should be treated as errata so skip 1280 * processing here. 1281 */ 1282 if (alt->vendor_id != 0) 1283 continue; 1284 1285 if (!__riscv_isa_extension_available(NULL, id)) 1286 continue; 1287 1288 value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id); 1289 if (!riscv_cpufeature_patch_check(id, value)) 1290 continue; 1291 } else if (id >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE) { 1292 if (!__riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor, 1293 id - RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) 1294 continue; 1295 } else { 1296 WARN(1, "This extension id:%d is not in ISA extension list", id); 1297 continue; 1298 } 1299 1300 oldptr = ALT_OLD_PTR(alt); 1301 altptr = ALT_ALT_PTR(alt); 1302 1303 mutex_lock(&text_mutex); 1304 patch_text_nosync(oldptr, altptr, alt->alt_len); 1305 riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); 1306 mutex_unlock(&text_mutex); 1307 } 1308 } 1309 #endif 1310