1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copied from arch/arm64/kernel/cpufeature.c 4 * 5 * Copyright (C) 2015 ARM Ltd. 6 * Copyright (C) 2017 SiFive 7 */ 8 9 #include <linux/acpi.h> 10 #include <linux/bitmap.h> 11 #include <linux/cpu.h> 12 #include <linux/cpuhotplug.h> 13 #include <linux/ctype.h> 14 #include <linux/log2.h> 15 #include <linux/memory.h> 16 #include <linux/module.h> 17 #include <linux/of.h> 18 #include <asm/acpi.h> 19 #include <asm/alternative.h> 20 #include <asm/bugs.h> 21 #include <asm/cacheflush.h> 22 #include <asm/cpufeature.h> 23 #include <asm/hwcap.h> 24 #include <asm/text-patching.h> 25 #include <asm/hwprobe.h> 26 #include <asm/processor.h> 27 #include <asm/sbi.h> 28 #include <asm/vector.h> 29 #include <asm/vendor_extensions.h> 30 #include <asm/vendor_extensions/thead.h> 31 #include <asm/usercfi.h> 32 33 #define NUM_ALPHA_EXTS ('z' - 'a' + 1) 34 35 static bool any_cpu_has_zicboz; 36 static bool any_cpu_has_zicbop; 37 static bool any_cpu_has_zicbom; 38 39 unsigned long elf_hwcap __read_mostly; 40 41 /* Host ISA bitmap */ 42 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; 43 44 /* Per-cpu ISA extensions. */ 45 struct riscv_isainfo hart_isa[NR_CPUS]; 46 47 u32 thead_vlenb_of; 48 49 /** 50 * riscv_isa_extension_base() - Get base extension word 51 * 52 * @isa_bitmap: ISA bitmap to use 53 * Return: base extension word as unsigned long value 54 * 55 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 56 */ 57 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) 58 { 59 return !isa_bitmap ? riscv_isa[0] : isa_bitmap[0]; 60 } 61 EXPORT_SYMBOL_GPL(riscv_isa_extension_base); 62 63 /** 64 * __riscv_isa_extension_available() - Check whether given extension 65 * is available or not 66 * 67 * @isa_bitmap: ISA bitmap to use 68 * @bit: bit position of the desired extension 69 * Return: true or false 70 * 71 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 72 */ 73 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit) 74 { 75 const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; 76 77 if (bit >= RISCV_ISA_EXT_MAX) 78 return false; 79 80 return test_bit(bit, bmap); 81 } 82 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); 83 84 static int riscv_ext_f_depends(const struct riscv_isa_ext_data *data, 85 const unsigned long *isa_bitmap) 86 { 87 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f)) 88 return 0; 89 90 return -EPROBE_DEFER; 91 } 92 93 static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data, 94 const unsigned long *isa_bitmap) 95 { 96 if (!riscv_cbom_block_size) { 97 pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n"); 98 return -EINVAL; 99 } 100 if (!is_power_of_2(riscv_cbom_block_size)) { 101 pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); 102 return -EINVAL; 103 } 104 105 any_cpu_has_zicbom = true; 106 return 0; 107 } 108 109 static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data, 110 const unsigned long *isa_bitmap) 111 { 112 if (!riscv_cboz_block_size) { 113 pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n"); 114 return -EINVAL; 115 } 116 if (!is_power_of_2(riscv_cboz_block_size)) { 117 pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n"); 118 return -EINVAL; 119 } 120 any_cpu_has_zicboz = true; 121 return 0; 122 } 123 124 static int riscv_ext_zicbop_validate(const struct riscv_isa_ext_data *data, 125 const unsigned long *isa_bitmap) 126 { 127 if (!riscv_cbop_block_size) { 128 pr_err("Zicbop detected in ISA string, disabling as no cbop-block-size found\n"); 129 return -EINVAL; 130 } 131 if (!is_power_of_2(riscv_cbop_block_size)) { 132 pr_err("Zicbop disabled as cbop-block-size present, but is not a power-of-2\n"); 133 return -EINVAL; 134 } 135 any_cpu_has_zicbop = true; 136 return 0; 137 } 138 139 static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data, 140 const unsigned long *isa_bitmap) 141 { 142 if (!IS_ENABLED(CONFIG_FPU)) 143 return -EINVAL; 144 145 /* 146 * Due to extension ordering, d is checked before f, so no deferral 147 * is required. 148 */ 149 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) { 150 pr_warn_once("This kernel does not support systems with F but not D\n"); 151 return -EINVAL; 152 } 153 154 return 0; 155 } 156 157 static int riscv_ext_d_validate(const struct riscv_isa_ext_data *data, 158 const unsigned long *isa_bitmap) 159 { 160 if (!IS_ENABLED(CONFIG_FPU)) 161 return -EINVAL; 162 163 return 0; 164 } 165 166 static int riscv_ext_vector_x_validate(const struct riscv_isa_ext_data *data, 167 const unsigned long *isa_bitmap) 168 { 169 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 170 return -EINVAL; 171 172 return 0; 173 } 174 175 static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data, 176 const unsigned long *isa_bitmap) 177 { 178 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 179 return -EINVAL; 180 181 if (!IS_ENABLED(CONFIG_FPU)) 182 return -EINVAL; 183 184 /* 185 * The kernel doesn't support systems that don't implement both of 186 * F and D, so if any of the vector extensions that do floating point 187 * are to be usable, both floating point extensions need to be usable. 188 * 189 * Since this function validates vector only, and v/Zve* are probed 190 * after f/d, there's no need for a deferral here. 191 */ 192 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) 193 return -EINVAL; 194 195 return 0; 196 } 197 198 static int riscv_ext_vector_crypto_validate(const struct riscv_isa_ext_data *data, 199 const unsigned long *isa_bitmap) 200 { 201 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 202 return -EINVAL; 203 204 /* 205 * It isn't the kernel's job to check that the binding is correct, so 206 * it should be enough to check that any of the vector extensions are 207 * enabled, which in-turn means that vector is usable in this kernel 208 */ 209 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32X)) 210 return -EPROBE_DEFER; 211 212 return 0; 213 } 214 215 static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data, 216 const unsigned long *isa_bitmap) 217 { 218 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA)) 219 return 0; 220 221 return -EPROBE_DEFER; 222 } 223 static int riscv_ext_zcd_validate(const struct riscv_isa_ext_data *data, 224 const unsigned long *isa_bitmap) 225 { 226 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && 227 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) 228 return 0; 229 230 return -EPROBE_DEFER; 231 } 232 233 static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data, 234 const unsigned long *isa_bitmap) 235 { 236 if (IS_ENABLED(CONFIG_64BIT)) 237 return -EINVAL; 238 239 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && 240 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f)) 241 return 0; 242 243 return -EPROBE_DEFER; 244 } 245 246 static int riscv_ext_zilsd_validate(const struct riscv_isa_ext_data *data, 247 const unsigned long *isa_bitmap) 248 { 249 if (IS_ENABLED(CONFIG_64BIT)) 250 return -EINVAL; 251 252 return 0; 253 } 254 255 static int riscv_ext_zclsd_validate(const struct riscv_isa_ext_data *data, 256 const unsigned long *isa_bitmap) 257 { 258 if (IS_ENABLED(CONFIG_64BIT)) 259 return -EINVAL; 260 261 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZILSD) && 262 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA)) 263 return 0; 264 265 return -EPROBE_DEFER; 266 } 267 268 static int riscv_vector_f_validate(const struct riscv_isa_ext_data *data, 269 const unsigned long *isa_bitmap) 270 { 271 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 272 return -EINVAL; 273 274 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32F)) 275 return 0; 276 277 return -EPROBE_DEFER; 278 } 279 280 static int riscv_ext_zvfbfwma_validate(const struct riscv_isa_ext_data *data, 281 const unsigned long *isa_bitmap) 282 { 283 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZFBFMIN) && 284 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVFBFMIN)) 285 return 0; 286 287 return -EPROBE_DEFER; 288 } 289 290 static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data, 291 const unsigned long *isa_bitmap) 292 { 293 /* SVADE has already been detected, use SVADE only */ 294 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SVADE)) 295 return -EOPNOTSUPP; 296 297 return 0; 298 } 299 300 static int riscv_cfilp_validate(const struct riscv_isa_ext_data *data, 301 const unsigned long *isa_bitmap) 302 { 303 if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) || 304 (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI)) 305 return -EINVAL; 306 307 return 0; 308 } 309 310 static int riscv_cfiss_validate(const struct riscv_isa_ext_data *data, 311 const unsigned long *isa_bitmap) 312 { 313 if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) || 314 (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI)) 315 return -EINVAL; 316 317 return 0; 318 } 319 320 static const unsigned int riscv_a_exts[] = { 321 RISCV_ISA_EXT_ZAAMO, 322 RISCV_ISA_EXT_ZALRSC, 323 }; 324 325 #define RISCV_ISA_EXT_ZKN \ 326 RISCV_ISA_EXT_ZBKB, \ 327 RISCV_ISA_EXT_ZBKC, \ 328 RISCV_ISA_EXT_ZBKX, \ 329 RISCV_ISA_EXT_ZKND, \ 330 RISCV_ISA_EXT_ZKNE, \ 331 RISCV_ISA_EXT_ZKNH 332 333 static const unsigned int riscv_zk_bundled_exts[] = { 334 RISCV_ISA_EXT_ZKN, 335 RISCV_ISA_EXT_ZKR, 336 RISCV_ISA_EXT_ZKT 337 }; 338 339 static const unsigned int riscv_zkn_bundled_exts[] = { 340 RISCV_ISA_EXT_ZKN 341 }; 342 343 static const unsigned int riscv_zks_bundled_exts[] = { 344 RISCV_ISA_EXT_ZBKB, 345 RISCV_ISA_EXT_ZBKC, 346 RISCV_ISA_EXT_ZKSED, 347 RISCV_ISA_EXT_ZKSH 348 }; 349 350 #define RISCV_ISA_EXT_ZVKN \ 351 RISCV_ISA_EXT_ZVKNED, \ 352 RISCV_ISA_EXT_ZVKNHB, \ 353 RISCV_ISA_EXT_ZVKB, \ 354 RISCV_ISA_EXT_ZVKT 355 356 static const unsigned int riscv_zvkn_bundled_exts[] = { 357 RISCV_ISA_EXT_ZVKN 358 }; 359 360 static const unsigned int riscv_zvknc_bundled_exts[] = { 361 RISCV_ISA_EXT_ZVKN, 362 RISCV_ISA_EXT_ZVBC 363 }; 364 365 static const unsigned int riscv_zvkng_bundled_exts[] = { 366 RISCV_ISA_EXT_ZVKN, 367 RISCV_ISA_EXT_ZVKG 368 }; 369 370 #define RISCV_ISA_EXT_ZVKS \ 371 RISCV_ISA_EXT_ZVKSED, \ 372 RISCV_ISA_EXT_ZVKSH, \ 373 RISCV_ISA_EXT_ZVKB, \ 374 RISCV_ISA_EXT_ZVKT 375 376 static const unsigned int riscv_zvks_bundled_exts[] = { 377 RISCV_ISA_EXT_ZVKS 378 }; 379 380 static const unsigned int riscv_zvksc_bundled_exts[] = { 381 RISCV_ISA_EXT_ZVKS, 382 RISCV_ISA_EXT_ZVBC 383 }; 384 385 static const unsigned int riscv_zvksg_bundled_exts[] = { 386 RISCV_ISA_EXT_ZVKS, 387 RISCV_ISA_EXT_ZVKG 388 }; 389 390 static const unsigned int riscv_zvbb_exts[] = { 391 RISCV_ISA_EXT_ZVKB 392 }; 393 394 #define RISCV_ISA_EXT_ZVE64F_IMPLY_LIST \ 395 RISCV_ISA_EXT_ZVE64X, \ 396 RISCV_ISA_EXT_ZVE32F, \ 397 RISCV_ISA_EXT_ZVE32X 398 399 #define RISCV_ISA_EXT_ZVE64D_IMPLY_LIST \ 400 RISCV_ISA_EXT_ZVE64F, \ 401 RISCV_ISA_EXT_ZVE64F_IMPLY_LIST 402 403 #define RISCV_ISA_EXT_V_IMPLY_LIST \ 404 RISCV_ISA_EXT_ZVE64D, \ 405 RISCV_ISA_EXT_ZVE64D_IMPLY_LIST 406 407 static const unsigned int riscv_zve32f_exts[] = { 408 RISCV_ISA_EXT_ZVE32X 409 }; 410 411 static const unsigned int riscv_zve64f_exts[] = { 412 RISCV_ISA_EXT_ZVE64F_IMPLY_LIST 413 }; 414 415 static const unsigned int riscv_zve64d_exts[] = { 416 RISCV_ISA_EXT_ZVE64D_IMPLY_LIST 417 }; 418 419 static const unsigned int riscv_v_exts[] = { 420 RISCV_ISA_EXT_V_IMPLY_LIST 421 }; 422 423 static const unsigned int riscv_zve64x_exts[] = { 424 RISCV_ISA_EXT_ZVE32X, 425 RISCV_ISA_EXT_ZVE64X 426 }; 427 428 /* 429 * While the [ms]envcfg CSRs were not defined until version 1.12 of the RISC-V 430 * privileged ISA, the existence of the CSRs is implied by any extension which 431 * specifies [ms]envcfg bit(s). Hence, we define a custom ISA extension for the 432 * existence of the CSR, and treat it as a subset of those other extensions. 433 */ 434 static const unsigned int riscv_xlinuxenvcfg_exts[] = { 435 RISCV_ISA_EXT_XLINUXENVCFG 436 }; 437 438 /* 439 * Zc* spec states that: 440 * - C always implies Zca 441 * - C+F implies Zcf (RV32 only) 442 * - C+D implies Zcd 443 * 444 * These extensions will be enabled and then validated depending on the 445 * availability of F/D RV32. 446 */ 447 static const unsigned int riscv_c_exts[] = { 448 RISCV_ISA_EXT_ZCA, 449 RISCV_ISA_EXT_ZCF, 450 RISCV_ISA_EXT_ZCD, 451 }; 452 453 /* 454 * The canonical order of ISA extension names in the ISA string is defined in 455 * chapter 27 of the unprivileged specification. 456 * 457 * Ordinarily, for in-kernel data structures, this order is unimportant but 458 * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. 459 * 460 * The specification uses vague wording, such as should, when it comes to 461 * ordering, so for our purposes the following rules apply: 462 * 463 * 1. All multi-letter extensions must be separated from other extensions by an 464 * underscore. 465 * 466 * 2. Additional standard extensions (starting with 'Z') must be sorted after 467 * single-letter extensions and before any higher-privileged extensions. 468 * 469 * 3. The first letter following the 'Z' conventionally indicates the most 470 * closely related alphabetical extension category, IMAFDQLCBKJTPVH. 471 * If multiple 'Z' extensions are named, they must be ordered first by 472 * category, then alphabetically within a category. 473 * 474 * 3. Standard supervisor-level extensions (starting with 'S') must be listed 475 * after standard unprivileged extensions. If multiple supervisor-level 476 * extensions are listed, they must be ordered alphabetically. 477 * 478 * 4. Standard machine-level extensions (starting with 'Zxm') must be listed 479 * after any lower-privileged, standard extensions. If multiple 480 * machine-level extensions are listed, they must be ordered 481 * alphabetically. 482 * 483 * 5. Non-standard extensions (starting with 'X') must be listed after all 484 * standard extensions. If multiple non-standard extensions are listed, they 485 * must be ordered alphabetically. 486 * 487 * An example string following the order is: 488 * rv64imadc_zifoo_zigoo_zafoo_sbar_scar_zxmbaz_xqux_xrux 489 * 490 * New entries to this struct should follow the ordering rules described above. 491 */ 492 const struct riscv_isa_ext_data riscv_isa_ext[] = { 493 __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i), 494 __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m), 495 __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_a, riscv_a_exts), 496 __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_f, riscv_ext_f_validate), 497 __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_d, riscv_ext_d_validate), 498 __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), 499 __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_c, riscv_c_exts), 500 __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_v, riscv_v_exts, riscv_ext_vector_float_validate), 501 __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), 502 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate), 503 __RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate), 504 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate), 505 __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE), 506 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfilp, RISCV_ISA_EXT_ZICFILP, riscv_xlinuxenvcfg_exts, 507 riscv_cfilp_validate), 508 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfiss, RISCV_ISA_EXT_ZICFISS, riscv_xlinuxenvcfg_exts, 509 riscv_cfiss_validate), 510 __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), 511 __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), 512 __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), 513 __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI), 514 __RISCV_ISA_EXT_DATA(zihintntl, RISCV_ISA_EXT_ZIHINTNTL), 515 __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), 516 __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), 517 __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP), 518 __RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO), 519 __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA), 520 __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), 521 __RISCV_ISA_EXT_DATA(zalasr, RISCV_ISA_EXT_ZALASR), 522 __RISCV_ISA_EXT_DATA(zalrsc, RISCV_ISA_EXT_ZALRSC), 523 __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), 524 __RISCV_ISA_EXT_DATA_VALIDATE(zfa, RISCV_ISA_EXT_ZFA, riscv_ext_f_depends), 525 __RISCV_ISA_EXT_DATA_VALIDATE(zfbfmin, RISCV_ISA_EXT_ZFBFMIN, riscv_ext_f_depends), 526 __RISCV_ISA_EXT_DATA_VALIDATE(zfh, RISCV_ISA_EXT_ZFH, riscv_ext_f_depends), 527 __RISCV_ISA_EXT_DATA_VALIDATE(zfhmin, RISCV_ISA_EXT_ZFHMIN, riscv_ext_f_depends), 528 __RISCV_ISA_EXT_DATA(zca, RISCV_ISA_EXT_ZCA), 529 __RISCV_ISA_EXT_DATA_VALIDATE(zcb, RISCV_ISA_EXT_ZCB, riscv_ext_zca_depends), 530 __RISCV_ISA_EXT_DATA_VALIDATE(zcd, RISCV_ISA_EXT_ZCD, riscv_ext_zcd_validate), 531 __RISCV_ISA_EXT_DATA_VALIDATE(zcf, RISCV_ISA_EXT_ZCF, riscv_ext_zcf_validate), 532 __RISCV_ISA_EXT_DATA_VALIDATE(zcmop, RISCV_ISA_EXT_ZCMOP, riscv_ext_zca_depends), 533 __RISCV_ISA_EXT_DATA_VALIDATE(zclsd, RISCV_ISA_EXT_ZCLSD, riscv_ext_zclsd_validate), 534 __RISCV_ISA_EXT_DATA_VALIDATE(zilsd, RISCV_ISA_EXT_ZILSD, riscv_ext_zilsd_validate), 535 __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), 536 __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), 537 __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC), 538 __RISCV_ISA_EXT_DATA(zbkb, RISCV_ISA_EXT_ZBKB), 539 __RISCV_ISA_EXT_DATA(zbkc, RISCV_ISA_EXT_ZBKC), 540 __RISCV_ISA_EXT_DATA(zbkx, RISCV_ISA_EXT_ZBKX), 541 __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS), 542 __RISCV_ISA_EXT_BUNDLE(zk, riscv_zk_bundled_exts), 543 __RISCV_ISA_EXT_BUNDLE(zkn, riscv_zkn_bundled_exts), 544 __RISCV_ISA_EXT_DATA(zknd, RISCV_ISA_EXT_ZKND), 545 __RISCV_ISA_EXT_DATA(zkne, RISCV_ISA_EXT_ZKNE), 546 __RISCV_ISA_EXT_DATA(zknh, RISCV_ISA_EXT_ZKNH), 547 __RISCV_ISA_EXT_DATA(zkr, RISCV_ISA_EXT_ZKR), 548 __RISCV_ISA_EXT_BUNDLE(zks, riscv_zks_bundled_exts), 549 __RISCV_ISA_EXT_DATA(zkt, RISCV_ISA_EXT_ZKT), 550 __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED), 551 __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), 552 __RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO), 553 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts, riscv_ext_vector_crypto_validate), 554 __RISCV_ISA_EXT_DATA_VALIDATE(zvbc, RISCV_ISA_EXT_ZVBC, riscv_ext_vector_crypto_validate), 555 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts, riscv_ext_vector_float_validate), 556 __RISCV_ISA_EXT_DATA_VALIDATE(zve32x, RISCV_ISA_EXT_ZVE32X, riscv_ext_vector_x_validate), 557 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts, riscv_ext_vector_float_validate), 558 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts, riscv_ext_vector_float_validate), 559 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts, riscv_ext_vector_x_validate), 560 __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfmin, RISCV_ISA_EXT_ZVFBFMIN, riscv_vector_f_validate), 561 __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfwma, RISCV_ISA_EXT_ZVFBFWMA, riscv_ext_zvfbfwma_validate), 562 __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), 563 __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), 564 __RISCV_ISA_EXT_DATA_VALIDATE(zvkb, RISCV_ISA_EXT_ZVKB, riscv_ext_vector_crypto_validate), 565 __RISCV_ISA_EXT_DATA_VALIDATE(zvkg, RISCV_ISA_EXT_ZVKG, riscv_ext_vector_crypto_validate), 566 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkn, riscv_zvkn_bundled_exts, riscv_ext_vector_crypto_validate), 567 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvknc, riscv_zvknc_bundled_exts, riscv_ext_vector_crypto_validate), 568 __RISCV_ISA_EXT_DATA_VALIDATE(zvkned, RISCV_ISA_EXT_ZVKNED, riscv_ext_vector_crypto_validate), 569 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkng, riscv_zvkng_bundled_exts, riscv_ext_vector_crypto_validate), 570 __RISCV_ISA_EXT_DATA_VALIDATE(zvknha, RISCV_ISA_EXT_ZVKNHA, riscv_ext_vector_crypto_validate), 571 __RISCV_ISA_EXT_DATA_VALIDATE(zvknhb, RISCV_ISA_EXT_ZVKNHB, riscv_ext_vector_crypto_validate), 572 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvks, riscv_zvks_bundled_exts, riscv_ext_vector_crypto_validate), 573 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksc, riscv_zvksc_bundled_exts, riscv_ext_vector_crypto_validate), 574 __RISCV_ISA_EXT_DATA_VALIDATE(zvksed, RISCV_ISA_EXT_ZVKSED, riscv_ext_vector_crypto_validate), 575 __RISCV_ISA_EXT_DATA_VALIDATE(zvksh, RISCV_ISA_EXT_ZVKSH, riscv_ext_vector_crypto_validate), 576 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksg, riscv_zvksg_bundled_exts, riscv_ext_vector_crypto_validate), 577 __RISCV_ISA_EXT_DATA_VALIDATE(zvkt, RISCV_ISA_EXT_ZVKT, riscv_ext_vector_crypto_validate), 578 __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), 579 __RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM), 580 __RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts), 581 __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), 582 __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), 583 __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), 584 __RISCV_ISA_EXT_SUPERSET(ssnpm, RISCV_ISA_EXT_SSNPM, riscv_xlinuxenvcfg_exts), 585 __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), 586 __RISCV_ISA_EXT_DATA(svade, RISCV_ISA_EXT_SVADE), 587 __RISCV_ISA_EXT_DATA_VALIDATE(svadu, RISCV_ISA_EXT_SVADU, riscv_ext_svadu_validate), 588 __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), 589 __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT), 590 __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), 591 __RISCV_ISA_EXT_DATA(svrsw60t59b, RISCV_ISA_EXT_SVRSW60T59B), 592 __RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC), 593 }; 594 595 const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); 596 597 static void riscv_isa_set_ext(const struct riscv_isa_ext_data *ext, unsigned long *bitmap) 598 { 599 if (ext->id != RISCV_ISA_EXT_INVALID) 600 set_bit(ext->id, bitmap); 601 602 for (int i = 0; i < ext->subset_ext_size; i++) { 603 if (ext->subset_ext_ids[i] != RISCV_ISA_EXT_INVALID) 604 set_bit(ext->subset_ext_ids[i], bitmap); 605 } 606 } 607 608 static const struct riscv_isa_ext_data *riscv_get_isa_ext_data(unsigned int ext_id) 609 { 610 for (int i = 0; i < riscv_isa_ext_count; i++) { 611 if (riscv_isa_ext[i].id == ext_id) 612 return &riscv_isa_ext[i]; 613 } 614 615 return NULL; 616 } 617 618 /* 619 * "Resolve" a source ISA bitmap into one that matches kernel configuration as 620 * well as correct extension dependencies. Some extensions depends on specific 621 * kernel configuration to be usable (V needs CONFIG_RISCV_ISA_V for instance) 622 * and this function will actually validate all the extensions provided in 623 * source_isa into the resolved_isa based on extensions validate() callbacks. 624 */ 625 static void __init riscv_resolve_isa(unsigned long *source_isa, 626 unsigned long *resolved_isa, unsigned long *this_hwcap, 627 unsigned long *isa2hwcap) 628 { 629 bool loop; 630 const struct riscv_isa_ext_data *ext; 631 DECLARE_BITMAP(prev_resolved_isa, RISCV_ISA_EXT_MAX); 632 int max_loop_count = riscv_isa_ext_count, ret; 633 unsigned int bit; 634 635 do { 636 loop = false; 637 if (max_loop_count-- < 0) { 638 pr_err("Failed to reach a stable ISA state\n"); 639 return; 640 } 641 bitmap_copy(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX); 642 for_each_set_bit(bit, source_isa, RISCV_ISA_EXT_MAX) { 643 ext = riscv_get_isa_ext_data(bit); 644 645 if (ext && ext->validate) { 646 ret = ext->validate(ext, resolved_isa); 647 if (ret == -EPROBE_DEFER) { 648 loop = true; 649 continue; 650 } else if (ret) { 651 /* Disable the extension entirely */ 652 clear_bit(bit, source_isa); 653 continue; 654 } 655 } 656 657 set_bit(bit, resolved_isa); 658 /* No need to keep it in source isa now that it is enabled */ 659 clear_bit(bit, source_isa); 660 661 /* Single letter extensions get set in hwcap */ 662 if (bit < RISCV_ISA_EXT_BASE) 663 *this_hwcap |= isa2hwcap[bit]; 664 } 665 } while (loop && !bitmap_equal(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX)); 666 } 667 668 static void __init match_isa_ext(const char *name, const char *name_end, unsigned long *bitmap) 669 { 670 for (int i = 0; i < riscv_isa_ext_count; i++) { 671 const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i]; 672 673 if ((name_end - name == strlen(ext->name)) && 674 !strncasecmp(name, ext->name, name_end - name)) { 675 riscv_isa_set_ext(ext, bitmap); 676 break; 677 } 678 } 679 } 680 681 static void __init riscv_parse_isa_string(const char *isa, unsigned long *bitmap) 682 { 683 /* 684 * For all possible cpus, we have already validated in 685 * the boot process that they at least contain "rv" and 686 * whichever of "32"/"64" this kernel supports, and so this 687 * section can be skipped. 688 */ 689 isa += 4; 690 691 while (*isa) { 692 const char *ext = isa++; 693 const char *ext_end = isa; 694 bool ext_err = false; 695 696 switch (*ext) { 697 case 'x': 698 case 'X': 699 if (acpi_disabled) 700 pr_warn_once("Vendor extensions are ignored in riscv,isa. Use riscv,isa-extensions instead."); 701 /* 702 * To skip an extension, we find its end. 703 * As multi-letter extensions must be split from other multi-letter 704 * extensions with an "_", the end of a multi-letter extension will 705 * either be the null character or the "_" at the start of the next 706 * multi-letter extension. 707 */ 708 for (; *isa && *isa != '_'; ++isa) 709 ; 710 ext_err = true; 711 break; 712 case 's': 713 /* 714 * Workaround for invalid single-letter 's' & 'u' (QEMU). 715 * No need to set the bit in riscv_isa as 's' & 'u' are 716 * not valid ISA extensions. It works unless the first 717 * multi-letter extension in the ISA string begins with 718 * "Su" and is not prefixed with an underscore. 719 */ 720 if (ext[-1] != '_' && ext[1] == 'u') { 721 ++isa; 722 ext_err = true; 723 break; 724 } 725 fallthrough; 726 case 'S': 727 case 'z': 728 case 'Z': 729 /* 730 * Before attempting to parse the extension itself, we find its end. 731 * As multi-letter extensions must be split from other multi-letter 732 * extensions with an "_", the end of a multi-letter extension will 733 * either be the null character or the "_" at the start of the next 734 * multi-letter extension. 735 * 736 * Next, as the extensions version is currently ignored, we 737 * eliminate that portion. This is done by parsing backwards from 738 * the end of the extension, removing any numbers. This may be a 739 * major or minor number however, so the process is repeated if a 740 * minor number was found. 741 * 742 * ext_end is intended to represent the first character *after* the 743 * name portion of an extension, but will be decremented to the last 744 * character itself while eliminating the extensions version number. 745 * A simple re-increment solves this problem. 746 */ 747 for (; *isa && *isa != '_'; ++isa) 748 if (unlikely(!isalnum(*isa))) 749 ext_err = true; 750 751 ext_end = isa; 752 if (unlikely(ext_err)) 753 break; 754 755 if (!isdigit(ext_end[-1])) 756 break; 757 758 while (isdigit(*--ext_end)) 759 ; 760 761 if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) { 762 ++ext_end; 763 break; 764 } 765 766 while (isdigit(*--ext_end)) 767 ; 768 769 ++ext_end; 770 break; 771 default: 772 /* 773 * Things are a little easier for single-letter extensions, as they 774 * are parsed forwards. 775 * 776 * After checking that our starting position is valid, we need to 777 * ensure that, when isa was incremented at the start of the loop, 778 * that it arrived at the start of the next extension. 779 * 780 * If we are already on a non-digit, there is nothing to do. Either 781 * we have a multi-letter extension's _, or the start of an 782 * extension. 783 * 784 * Otherwise we have found the current extension's major version 785 * number. Parse past it, and a subsequent p/minor version number 786 * if present. The `p` extension must not appear immediately after 787 * a number, so there is no fear of missing it. 788 * 789 */ 790 if (unlikely(!isalpha(*ext))) { 791 ext_err = true; 792 break; 793 } 794 795 if (!isdigit(*isa)) 796 break; 797 798 while (isdigit(*++isa)) 799 ; 800 801 if (tolower(*isa) != 'p') 802 break; 803 804 if (!isdigit(*++isa)) { 805 --isa; 806 break; 807 } 808 809 while (isdigit(*++isa)) 810 ; 811 812 break; 813 } 814 815 /* 816 * The parser expects that at the start of an iteration isa points to the 817 * first character of the next extension. As we stop parsing an extension 818 * on meeting a non-alphanumeric character, an extra increment is needed 819 * where the succeeding extension is a multi-letter prefixed with an "_". 820 */ 821 if (*isa == '_') 822 ++isa; 823 824 if (unlikely(ext_err)) 825 continue; 826 827 match_isa_ext(ext, ext_end, bitmap); 828 } 829 } 830 831 static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) 832 { 833 struct device_node *node; 834 const char *isa; 835 int rc; 836 struct acpi_table_header *rhct; 837 acpi_status status; 838 unsigned int cpu; 839 u64 boot_vendorid; 840 u64 boot_archid; 841 842 if (!acpi_disabled) { 843 status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); 844 if (ACPI_FAILURE(status)) 845 return; 846 } 847 848 boot_vendorid = riscv_get_mvendorid(); 849 boot_archid = riscv_get_marchid(); 850 851 for_each_possible_cpu(cpu) { 852 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 853 unsigned long this_hwcap = 0; 854 DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 }; 855 856 if (acpi_disabled) { 857 node = of_cpu_device_node_get(cpu); 858 if (!node) { 859 pr_warn("Unable to find cpu node\n"); 860 continue; 861 } 862 863 rc = of_property_read_string(node, "riscv,isa", &isa); 864 of_node_put(node); 865 if (rc) { 866 pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); 867 continue; 868 } 869 } else { 870 rc = acpi_get_riscv_isa(rhct, cpu, &isa); 871 if (rc < 0) { 872 pr_warn("Unable to get ISA for the hart - %d\n", cpu); 873 continue; 874 } 875 } 876 877 riscv_parse_isa_string(isa, source_isa); 878 879 /* 880 * These ones were as they were part of the base ISA when the 881 * port & dt-bindings were upstreamed, and so can be set 882 * unconditionally where `i` is in riscv,isa on DT systems. 883 */ 884 if (acpi_disabled) { 885 set_bit(RISCV_ISA_EXT_ZICSR, source_isa); 886 set_bit(RISCV_ISA_EXT_ZIFENCEI, source_isa); 887 set_bit(RISCV_ISA_EXT_ZICNTR, source_isa); 888 set_bit(RISCV_ISA_EXT_ZIHPM, source_isa); 889 } 890 891 /* 892 * "V" in ISA strings is ambiguous in practice: it should mean 893 * just the standard V-1.0 but vendors aren't well behaved. 894 * Many vendors with T-Head CPU cores which implement the 0.7.1 895 * version of the vector specification put "v" into their DTs. 896 * CPU cores with the ratified spec will contain non-zero 897 * marchid. 898 */ 899 if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) { 900 this_hwcap &= ~isa2hwcap[RISCV_ISA_EXT_v]; 901 clear_bit(RISCV_ISA_EXT_v, source_isa); 902 } 903 904 riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); 905 906 /* 907 * All "okay" hart should have same isa. Set HWCAP based on 908 * common capabilities of every "okay" hart, in case they don't 909 * have. 910 */ 911 if (elf_hwcap) 912 elf_hwcap &= this_hwcap; 913 else 914 elf_hwcap = this_hwcap; 915 916 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 917 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 918 else 919 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 920 } 921 922 if (!acpi_disabled && rhct) 923 acpi_put_table((struct acpi_table_header *)rhct); 924 } 925 926 static void __init riscv_fill_cpu_vendor_ext(struct device_node *cpu_node, int cpu) 927 { 928 if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) 929 return; 930 931 for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) { 932 struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i]; 933 934 for (int j = 0; j < ext_list->ext_data_count; j++) { 935 const struct riscv_isa_ext_data ext = ext_list->ext_data[j]; 936 struct riscv_isavendorinfo *isavendorinfo = &ext_list->per_hart_isa_bitmap[cpu]; 937 938 if (of_property_match_string(cpu_node, "riscv,isa-extensions", 939 ext.property) < 0) 940 continue; 941 942 /* 943 * Assume that subset extensions are all members of the 944 * same vendor. 945 */ 946 if (ext.subset_ext_size) 947 for (int k = 0; k < ext.subset_ext_size; k++) 948 set_bit(ext.subset_ext_ids[k], isavendorinfo->isa); 949 950 set_bit(ext.id, isavendorinfo->isa); 951 } 952 } 953 } 954 955 /* 956 * Populate all_harts_isa_bitmap for each vendor with all of the extensions that 957 * are shared across CPUs for that vendor. 958 */ 959 static void __init riscv_fill_vendor_ext_list(int cpu) 960 { 961 if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) 962 return; 963 964 for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) { 965 struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i]; 966 967 if (!ext_list->is_initialized) { 968 bitmap_copy(ext_list->all_harts_isa_bitmap.isa, 969 ext_list->per_hart_isa_bitmap[cpu].isa, 970 RISCV_ISA_VENDOR_EXT_MAX); 971 ext_list->is_initialized = true; 972 } else { 973 bitmap_and(ext_list->all_harts_isa_bitmap.isa, 974 ext_list->all_harts_isa_bitmap.isa, 975 ext_list->per_hart_isa_bitmap[cpu].isa, 976 RISCV_ISA_VENDOR_EXT_MAX); 977 } 978 } 979 } 980 981 static int has_thead_homogeneous_vlenb(void) 982 { 983 int cpu; 984 u32 prev_vlenb = 0; 985 u32 vlenb = 0; 986 987 /* Ignore thead,vlenb property if xtheadvector is not enabled in the kernel */ 988 if (!IS_ENABLED(CONFIG_RISCV_ISA_XTHEADVECTOR)) 989 return 0; 990 991 for_each_possible_cpu(cpu) { 992 struct device_node *cpu_node; 993 994 cpu_node = of_cpu_device_node_get(cpu); 995 if (!cpu_node) { 996 pr_warn("Unable to find cpu node\n"); 997 return -ENOENT; 998 } 999 1000 if (of_property_read_u32(cpu_node, "thead,vlenb", &vlenb)) { 1001 of_node_put(cpu_node); 1002 1003 if (prev_vlenb) 1004 return -ENOENT; 1005 continue; 1006 } 1007 1008 if (prev_vlenb && vlenb != prev_vlenb) { 1009 of_node_put(cpu_node); 1010 return -ENOENT; 1011 } 1012 1013 prev_vlenb = vlenb; 1014 of_node_put(cpu_node); 1015 } 1016 1017 thead_vlenb_of = vlenb; 1018 return 0; 1019 } 1020 1021 static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) 1022 { 1023 unsigned int cpu; 1024 bool mitigated; 1025 1026 for_each_possible_cpu(cpu) { 1027 unsigned long this_hwcap = 0; 1028 struct device_node *cpu_node; 1029 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 1030 DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 }; 1031 1032 cpu_node = of_cpu_device_node_get(cpu); 1033 if (!cpu_node) { 1034 pr_warn("Unable to find cpu node\n"); 1035 continue; 1036 } 1037 1038 if (!of_property_present(cpu_node, "riscv,isa-extensions")) { 1039 of_node_put(cpu_node); 1040 continue; 1041 } 1042 1043 for (int i = 0; i < riscv_isa_ext_count; i++) { 1044 const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i]; 1045 1046 if (of_property_match_string(cpu_node, "riscv,isa-extensions", 1047 ext->property) < 0) 1048 continue; 1049 1050 riscv_isa_set_ext(ext, source_isa); 1051 } 1052 1053 riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); 1054 riscv_fill_cpu_vendor_ext(cpu_node, cpu); 1055 1056 of_node_put(cpu_node); 1057 1058 /* 1059 * All "okay" harts should have same isa. Set HWCAP based on 1060 * common capabilities of every "okay" hart, in case they don't. 1061 */ 1062 if (elf_hwcap) 1063 elf_hwcap &= this_hwcap; 1064 else 1065 elf_hwcap = this_hwcap; 1066 1067 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 1068 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 1069 else 1070 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 1071 1072 riscv_fill_vendor_ext_list(cpu); 1073 } 1074 1075 /* 1076 * Execute ghostwrite mitigation immediately after detecting extensions 1077 * to disable xtheadvector if necessary. 1078 */ 1079 mitigated = ghostwrite_enable_mitigation(); 1080 1081 if (!mitigated && has_xtheadvector_no_alternatives() && has_thead_homogeneous_vlenb() < 0) { 1082 pr_warn("Unsupported heterogeneous vlenb detected, vector extension disabled.\n"); 1083 disable_xtheadvector(); 1084 } 1085 1086 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 1087 return -ENOENT; 1088 1089 return 0; 1090 } 1091 1092 #ifdef CONFIG_RISCV_ISA_FALLBACK 1093 bool __initdata riscv_isa_fallback = true; 1094 #else 1095 bool __initdata riscv_isa_fallback; 1096 static int __init riscv_isa_fallback_setup(char *__unused) 1097 { 1098 riscv_isa_fallback = true; 1099 return 1; 1100 } 1101 early_param("riscv_isa_fallback", riscv_isa_fallback_setup); 1102 #endif 1103 1104 void __init riscv_fill_hwcap(void) 1105 { 1106 char print_str[NUM_ALPHA_EXTS + 1]; 1107 unsigned long isa2hwcap[26] = {0}; 1108 int i, j; 1109 1110 isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I; 1111 isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M; 1112 isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A; 1113 isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F; 1114 isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D; 1115 isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C; 1116 isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V; 1117 1118 if (!acpi_disabled) { 1119 riscv_fill_hwcap_from_isa_string(isa2hwcap); 1120 } else { 1121 int ret = riscv_fill_hwcap_from_ext_list(isa2hwcap); 1122 1123 if (ret && riscv_isa_fallback) { 1124 pr_info("Falling back to deprecated \"riscv,isa\"\n"); 1125 riscv_fill_hwcap_from_isa_string(isa2hwcap); 1126 } 1127 } 1128 1129 /* 1130 * We don't support systems with F but without D, so mask those out 1131 * here. 1132 */ 1133 if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { 1134 pr_info("This kernel does not support systems with F but not D\n"); 1135 elf_hwcap &= ~COMPAT_HWCAP_ISA_F; 1136 } 1137 1138 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_ZVE32X) || 1139 has_xtheadvector_no_alternatives()) { 1140 /* 1141 * This cannot fail when called on the boot hart 1142 */ 1143 riscv_v_setup_vsize(); 1144 } 1145 1146 memset(print_str, 0, sizeof(print_str)); 1147 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 1148 if (riscv_isa[0] & BIT_MASK(i)) 1149 print_str[j++] = (char)('a' + i); 1150 pr_info("riscv: base ISA extensions %s\n", print_str); 1151 1152 memset(print_str, 0, sizeof(print_str)); 1153 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 1154 if (elf_hwcap & BIT_MASK(i)) 1155 print_str[j++] = (char)('a' + i); 1156 pr_info("riscv: ELF capabilities %s\n", print_str); 1157 } 1158 1159 unsigned long riscv_get_elf_hwcap(void) 1160 { 1161 unsigned long hwcap; 1162 1163 hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1)); 1164 1165 if (!riscv_v_vstate_ctrl_user_allowed()) 1166 hwcap &= ~COMPAT_HWCAP_ISA_V; 1167 1168 return hwcap; 1169 } 1170 1171 void __init riscv_user_isa_enable(void) 1172 { 1173 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOZ)) 1174 current->thread.envcfg |= ENVCFG_CBZE; 1175 else if (any_cpu_has_zicboz) 1176 pr_warn("Zicboz disabled as it is unavailable on some harts\n"); 1177 1178 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOM)) 1179 current->thread.envcfg |= ENVCFG_CBCFE; 1180 else if (any_cpu_has_zicbom) 1181 pr_warn("Zicbom disabled as it is unavailable on some harts\n"); 1182 1183 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOP) && 1184 any_cpu_has_zicbop) 1185 pr_warn("Zicbop disabled as it is unavailable on some harts\n"); 1186 } 1187 1188 #ifdef CONFIG_RISCV_ALTERNATIVE 1189 /* 1190 * Alternative patch sites consider 48 bits when determining when to patch 1191 * the old instruction sequence with the new. These bits are broken into a 1192 * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the 1193 * patch site is for an erratum, identified by the 32-bit patch ID. When 1194 * the vendor ID is zero, the patch site is for a cpufeature. cpufeatures 1195 * further break down patch ID into two 16-bit numbers. The lower 16 bits 1196 * are the cpufeature ID and the upper 16 bits are used for a value specific 1197 * to the cpufeature and patch site. If the upper 16 bits are zero, then it 1198 * implies no specific value is specified. cpufeatures that want to control 1199 * patching on a per-site basis will provide non-zero values and implement 1200 * checks here. The checks return true when patching should be done, and 1201 * false otherwise. 1202 */ 1203 static bool riscv_cpufeature_patch_check(u16 id, u16 value) 1204 { 1205 if (!value) 1206 return true; 1207 1208 switch (id) { 1209 case RISCV_ISA_EXT_ZICBOZ: 1210 /* 1211 * Zicboz alternative applications provide the maximum 1212 * supported block size order, or zero when it doesn't 1213 * matter. If the current block size exceeds the maximum, 1214 * then the alternative cannot be applied. 1215 */ 1216 return riscv_cboz_block_size <= (1U << value); 1217 } 1218 1219 return false; 1220 } 1221 1222 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, 1223 struct alt_entry *end, 1224 unsigned int stage) 1225 { 1226 struct alt_entry *alt; 1227 void *oldptr, *altptr; 1228 u16 id, value, vendor; 1229 1230 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 1231 return; 1232 1233 for (alt = begin; alt < end; alt++) { 1234 id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); 1235 vendor = PATCH_ID_CPUFEATURE_ID(alt->vendor_id); 1236 1237 /* 1238 * Any alternative with a patch_id that is less than 1239 * RISCV_ISA_EXT_MAX is interpreted as a standard extension. 1240 * 1241 * Any alternative with patch_id that is greater than or equal 1242 * to RISCV_VENDOR_EXT_ALTERNATIVES_BASE is interpreted as a 1243 * vendor extension. 1244 */ 1245 if (id < RISCV_ISA_EXT_MAX) { 1246 /* 1247 * This patch should be treated as errata so skip 1248 * processing here. 1249 */ 1250 if (alt->vendor_id != 0) 1251 continue; 1252 1253 if (!__riscv_isa_extension_available(NULL, id)) 1254 continue; 1255 1256 value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id); 1257 if (!riscv_cpufeature_patch_check(id, value)) 1258 continue; 1259 } else if (id >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE) { 1260 if (!__riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor, 1261 id - RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) 1262 continue; 1263 } else { 1264 WARN(1, "This extension id:%d is not in ISA extension list", id); 1265 continue; 1266 } 1267 1268 oldptr = ALT_OLD_PTR(alt); 1269 altptr = ALT_ALT_PTR(alt); 1270 1271 mutex_lock(&text_mutex); 1272 patch_text_nosync(oldptr, altptr, alt->alt_len); 1273 riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); 1274 mutex_unlock(&text_mutex); 1275 } 1276 } 1277 #endif 1278