1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copied from arch/arm64/kernel/cpufeature.c 4 * 5 * Copyright (C) 2015 ARM Ltd. 6 * Copyright (C) 2017 SiFive 7 */ 8 9 #include <linux/acpi.h> 10 #include <linux/bitmap.h> 11 #include <linux/cpu.h> 12 #include <linux/cpuhotplug.h> 13 #include <linux/ctype.h> 14 #include <linux/log2.h> 15 #include <linux/memory.h> 16 #include <linux/module.h> 17 #include <linux/of.h> 18 #include <asm/acpi.h> 19 #include <asm/alternative.h> 20 #include <asm/cacheflush.h> 21 #include <asm/cpufeature.h> 22 #include <asm/hwcap.h> 23 #include <asm/patch.h> 24 #include <asm/processor.h> 25 #include <asm/sbi.h> 26 #include <asm/vector.h> 27 28 #define NUM_ALPHA_EXTS ('z' - 'a' + 1) 29 30 unsigned long elf_hwcap __read_mostly; 31 32 /* Host ISA bitmap */ 33 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; 34 35 /* Per-cpu ISA extensions. */ 36 struct riscv_isainfo hart_isa[NR_CPUS]; 37 38 /** 39 * riscv_isa_extension_base() - Get base extension word 40 * 41 * @isa_bitmap: ISA bitmap to use 42 * Return: base extension word as unsigned long value 43 * 44 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 45 */ 46 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) 47 { 48 if (!isa_bitmap) 49 return riscv_isa[0]; 50 return isa_bitmap[0]; 51 } 52 EXPORT_SYMBOL_GPL(riscv_isa_extension_base); 53 54 /** 55 * __riscv_isa_extension_available() - Check whether given extension 56 * is available or not 57 * 58 * @isa_bitmap: ISA bitmap to use 59 * @bit: bit position of the desired extension 60 * Return: true or false 61 * 62 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 63 */ 64 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit) 65 { 66 const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; 67 68 if (bit >= RISCV_ISA_EXT_MAX) 69 return false; 70 71 return test_bit(bit, bmap) ? true : false; 72 } 73 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); 74 75 static bool riscv_isa_extension_check(int id) 76 { 77 switch (id) { 78 case RISCV_ISA_EXT_ZICBOM: 79 if (!riscv_cbom_block_size) { 80 pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n"); 81 return false; 82 } else if (!is_power_of_2(riscv_cbom_block_size)) { 83 pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); 84 return false; 85 } 86 return true; 87 case RISCV_ISA_EXT_ZICBOZ: 88 if (!riscv_cboz_block_size) { 89 pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n"); 90 return false; 91 } else if (!is_power_of_2(riscv_cboz_block_size)) { 92 pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n"); 93 return false; 94 } 95 return true; 96 case RISCV_ISA_EXT_INVALID: 97 return false; 98 } 99 100 return true; 101 } 102 103 #define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size) { \ 104 .name = #_name, \ 105 .property = #_name, \ 106 .id = _id, \ 107 .subset_ext_ids = _subset_exts, \ 108 .subset_ext_size = _subset_exts_size \ 109 } 110 111 #define __RISCV_ISA_EXT_DATA(_name, _id) _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0) 112 113 /* Used to declare pure "lasso" extension (Zk for instance) */ 114 #define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \ 115 _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, ARRAY_SIZE(_bundled_exts)) 116 117 /* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */ 118 #define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \ 119 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts)) 120 121 static const unsigned int riscv_zk_bundled_exts[] = { 122 RISCV_ISA_EXT_ZBKB, 123 RISCV_ISA_EXT_ZBKC, 124 RISCV_ISA_EXT_ZBKX, 125 RISCV_ISA_EXT_ZKND, 126 RISCV_ISA_EXT_ZKNE, 127 RISCV_ISA_EXT_ZKR, 128 RISCV_ISA_EXT_ZKT, 129 }; 130 131 static const unsigned int riscv_zkn_bundled_exts[] = { 132 RISCV_ISA_EXT_ZBKB, 133 RISCV_ISA_EXT_ZBKC, 134 RISCV_ISA_EXT_ZBKX, 135 RISCV_ISA_EXT_ZKND, 136 RISCV_ISA_EXT_ZKNE, 137 RISCV_ISA_EXT_ZKNH, 138 }; 139 140 static const unsigned int riscv_zks_bundled_exts[] = { 141 RISCV_ISA_EXT_ZBKB, 142 RISCV_ISA_EXT_ZBKC, 143 RISCV_ISA_EXT_ZKSED, 144 RISCV_ISA_EXT_ZKSH 145 }; 146 147 #define RISCV_ISA_EXT_ZVKN \ 148 RISCV_ISA_EXT_ZVKNED, \ 149 RISCV_ISA_EXT_ZVKNHB, \ 150 RISCV_ISA_EXT_ZVKB, \ 151 RISCV_ISA_EXT_ZVKT 152 153 static const unsigned int riscv_zvkn_bundled_exts[] = { 154 RISCV_ISA_EXT_ZVKN 155 }; 156 157 static const unsigned int riscv_zvknc_bundled_exts[] = { 158 RISCV_ISA_EXT_ZVKN, 159 RISCV_ISA_EXT_ZVBC 160 }; 161 162 static const unsigned int riscv_zvkng_bundled_exts[] = { 163 RISCV_ISA_EXT_ZVKN, 164 RISCV_ISA_EXT_ZVKG 165 }; 166 167 #define RISCV_ISA_EXT_ZVKS \ 168 RISCV_ISA_EXT_ZVKSED, \ 169 RISCV_ISA_EXT_ZVKSH, \ 170 RISCV_ISA_EXT_ZVKB, \ 171 RISCV_ISA_EXT_ZVKT 172 173 static const unsigned int riscv_zvks_bundled_exts[] = { 174 RISCV_ISA_EXT_ZVKS 175 }; 176 177 static const unsigned int riscv_zvksc_bundled_exts[] = { 178 RISCV_ISA_EXT_ZVKS, 179 RISCV_ISA_EXT_ZVBC 180 }; 181 182 static const unsigned int riscv_zvksg_bundled_exts[] = { 183 RISCV_ISA_EXT_ZVKS, 184 RISCV_ISA_EXT_ZVKG 185 }; 186 187 static const unsigned int riscv_zvbb_exts[] = { 188 RISCV_ISA_EXT_ZVKB 189 }; 190 191 /* 192 * While the [ms]envcfg CSRs were not defined until version 1.12 of the RISC-V 193 * privileged ISA, the existence of the CSRs is implied by any extension which 194 * specifies [ms]envcfg bit(s). Hence, we define a custom ISA extension for the 195 * existence of the CSR, and treat it as a subset of those other extensions. 196 */ 197 static const unsigned int riscv_xlinuxenvcfg_exts[] = { 198 RISCV_ISA_EXT_XLINUXENVCFG 199 }; 200 201 /* 202 * The canonical order of ISA extension names in the ISA string is defined in 203 * chapter 27 of the unprivileged specification. 204 * 205 * Ordinarily, for in-kernel data structures, this order is unimportant but 206 * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. 207 * 208 * The specification uses vague wording, such as should, when it comes to 209 * ordering, so for our purposes the following rules apply: 210 * 211 * 1. All multi-letter extensions must be separated from other extensions by an 212 * underscore. 213 * 214 * 2. Additional standard extensions (starting with 'Z') must be sorted after 215 * single-letter extensions and before any higher-privileged extensions. 216 * 217 * 3. The first letter following the 'Z' conventionally indicates the most 218 * closely related alphabetical extension category, IMAFDQLCBKJTPVH. 219 * If multiple 'Z' extensions are named, they must be ordered first by 220 * category, then alphabetically within a category. 221 * 222 * 3. Standard supervisor-level extensions (starting with 'S') must be listed 223 * after standard unprivileged extensions. If multiple supervisor-level 224 * extensions are listed, they must be ordered alphabetically. 225 * 226 * 4. Standard machine-level extensions (starting with 'Zxm') must be listed 227 * after any lower-privileged, standard extensions. If multiple 228 * machine-level extensions are listed, they must be ordered 229 * alphabetically. 230 * 231 * 5. Non-standard extensions (starting with 'X') must be listed after all 232 * standard extensions. If multiple non-standard extensions are listed, they 233 * must be ordered alphabetically. 234 * 235 * An example string following the order is: 236 * rv64imadc_zifoo_zigoo_zafoo_sbar_scar_zxmbaz_xqux_xrux 237 * 238 * New entries to this struct should follow the ordering rules described above. 239 */ 240 const struct riscv_isa_ext_data riscv_isa_ext[] = { 241 __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i), 242 __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m), 243 __RISCV_ISA_EXT_DATA(a, RISCV_ISA_EXT_a), 244 __RISCV_ISA_EXT_DATA(f, RISCV_ISA_EXT_f), 245 __RISCV_ISA_EXT_DATA(d, RISCV_ISA_EXT_d), 246 __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), 247 __RISCV_ISA_EXT_DATA(c, RISCV_ISA_EXT_c), 248 __RISCV_ISA_EXT_DATA(v, RISCV_ISA_EXT_v), 249 __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), 250 __RISCV_ISA_EXT_SUPERSET(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts), 251 __RISCV_ISA_EXT_SUPERSET(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts), 252 __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), 253 __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), 254 __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), 255 __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI), 256 __RISCV_ISA_EXT_DATA(zihintntl, RISCV_ISA_EXT_ZIHINTNTL), 257 __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), 258 __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), 259 __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), 260 __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA), 261 __RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH), 262 __RISCV_ISA_EXT_DATA(zfhmin, RISCV_ISA_EXT_ZFHMIN), 263 __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), 264 __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), 265 __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC), 266 __RISCV_ISA_EXT_DATA(zbkb, RISCV_ISA_EXT_ZBKB), 267 __RISCV_ISA_EXT_DATA(zbkc, RISCV_ISA_EXT_ZBKC), 268 __RISCV_ISA_EXT_DATA(zbkx, RISCV_ISA_EXT_ZBKX), 269 __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS), 270 __RISCV_ISA_EXT_BUNDLE(zk, riscv_zk_bundled_exts), 271 __RISCV_ISA_EXT_BUNDLE(zkn, riscv_zkn_bundled_exts), 272 __RISCV_ISA_EXT_DATA(zknd, RISCV_ISA_EXT_ZKND), 273 __RISCV_ISA_EXT_DATA(zkne, RISCV_ISA_EXT_ZKNE), 274 __RISCV_ISA_EXT_DATA(zknh, RISCV_ISA_EXT_ZKNH), 275 __RISCV_ISA_EXT_DATA(zkr, RISCV_ISA_EXT_ZKR), 276 __RISCV_ISA_EXT_BUNDLE(zks, riscv_zks_bundled_exts), 277 __RISCV_ISA_EXT_DATA(zkt, RISCV_ISA_EXT_ZKT), 278 __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED), 279 __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), 280 __RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO), 281 __RISCV_ISA_EXT_SUPERSET(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts), 282 __RISCV_ISA_EXT_DATA(zvbc, RISCV_ISA_EXT_ZVBC), 283 __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), 284 __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), 285 __RISCV_ISA_EXT_DATA(zvkb, RISCV_ISA_EXT_ZVKB), 286 __RISCV_ISA_EXT_DATA(zvkg, RISCV_ISA_EXT_ZVKG), 287 __RISCV_ISA_EXT_BUNDLE(zvkn, riscv_zvkn_bundled_exts), 288 __RISCV_ISA_EXT_BUNDLE(zvknc, riscv_zvknc_bundled_exts), 289 __RISCV_ISA_EXT_DATA(zvkned, RISCV_ISA_EXT_ZVKNED), 290 __RISCV_ISA_EXT_BUNDLE(zvkng, riscv_zvkng_bundled_exts), 291 __RISCV_ISA_EXT_DATA(zvknha, RISCV_ISA_EXT_ZVKNHA), 292 __RISCV_ISA_EXT_DATA(zvknhb, RISCV_ISA_EXT_ZVKNHB), 293 __RISCV_ISA_EXT_BUNDLE(zvks, riscv_zvks_bundled_exts), 294 __RISCV_ISA_EXT_BUNDLE(zvksc, riscv_zvksc_bundled_exts), 295 __RISCV_ISA_EXT_DATA(zvksed, RISCV_ISA_EXT_ZVKSED), 296 __RISCV_ISA_EXT_DATA(zvksh, RISCV_ISA_EXT_ZVKSH), 297 __RISCV_ISA_EXT_BUNDLE(zvksg, riscv_zvksg_bundled_exts), 298 __RISCV_ISA_EXT_DATA(zvkt, RISCV_ISA_EXT_ZVKT), 299 __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), 300 __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), 301 __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), 302 __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), 303 __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), 304 __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), 305 __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT), 306 __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), 307 __RISCV_ISA_EXT_DATA(xandespmu, RISCV_ISA_EXT_XANDESPMU), 308 }; 309 310 const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); 311 312 static void __init match_isa_ext(const struct riscv_isa_ext_data *ext, const char *name, 313 const char *name_end, struct riscv_isainfo *isainfo) 314 { 315 if ((name_end - name == strlen(ext->name)) && 316 !strncasecmp(name, ext->name, name_end - name)) { 317 /* 318 * If this is a bundle, enable all the ISA extensions that 319 * comprise the bundle. 320 */ 321 if (ext->subset_ext_size) { 322 for (int i = 0; i < ext->subset_ext_size; i++) { 323 if (riscv_isa_extension_check(ext->subset_ext_ids[i])) 324 set_bit(ext->subset_ext_ids[i], isainfo->isa); 325 } 326 } 327 328 /* 329 * This is valid even for bundle extensions which uses the RISCV_ISA_EXT_INVALID id 330 * (rejected by riscv_isa_extension_check()). 331 */ 332 if (riscv_isa_extension_check(ext->id)) 333 set_bit(ext->id, isainfo->isa); 334 } 335 } 336 337 static void __init riscv_parse_isa_string(unsigned long *this_hwcap, struct riscv_isainfo *isainfo, 338 unsigned long *isa2hwcap, const char *isa) 339 { 340 /* 341 * For all possible cpus, we have already validated in 342 * the boot process that they at least contain "rv" and 343 * whichever of "32"/"64" this kernel supports, and so this 344 * section can be skipped. 345 */ 346 isa += 4; 347 348 while (*isa) { 349 const char *ext = isa++; 350 const char *ext_end = isa; 351 bool ext_long = false, ext_err = false; 352 353 switch (*ext) { 354 case 's': 355 /* 356 * Workaround for invalid single-letter 's' & 'u' (QEMU). 357 * No need to set the bit in riscv_isa as 's' & 'u' are 358 * not valid ISA extensions. It works unless the first 359 * multi-letter extension in the ISA string begins with 360 * "Su" and is not prefixed with an underscore. 361 */ 362 if (ext[-1] != '_' && ext[1] == 'u') { 363 ++isa; 364 ext_err = true; 365 break; 366 } 367 fallthrough; 368 case 'S': 369 case 'x': 370 case 'X': 371 case 'z': 372 case 'Z': 373 /* 374 * Before attempting to parse the extension itself, we find its end. 375 * As multi-letter extensions must be split from other multi-letter 376 * extensions with an "_", the end of a multi-letter extension will 377 * either be the null character or the "_" at the start of the next 378 * multi-letter extension. 379 * 380 * Next, as the extensions version is currently ignored, we 381 * eliminate that portion. This is done by parsing backwards from 382 * the end of the extension, removing any numbers. This may be a 383 * major or minor number however, so the process is repeated if a 384 * minor number was found. 385 * 386 * ext_end is intended to represent the first character *after* the 387 * name portion of an extension, but will be decremented to the last 388 * character itself while eliminating the extensions version number. 389 * A simple re-increment solves this problem. 390 */ 391 ext_long = true; 392 for (; *isa && *isa != '_'; ++isa) 393 if (unlikely(!isalnum(*isa))) 394 ext_err = true; 395 396 ext_end = isa; 397 if (unlikely(ext_err)) 398 break; 399 400 if (!isdigit(ext_end[-1])) 401 break; 402 403 while (isdigit(*--ext_end)) 404 ; 405 406 if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) { 407 ++ext_end; 408 break; 409 } 410 411 while (isdigit(*--ext_end)) 412 ; 413 414 ++ext_end; 415 break; 416 default: 417 /* 418 * Things are a little easier for single-letter extensions, as they 419 * are parsed forwards. 420 * 421 * After checking that our starting position is valid, we need to 422 * ensure that, when isa was incremented at the start of the loop, 423 * that it arrived at the start of the next extension. 424 * 425 * If we are already on a non-digit, there is nothing to do. Either 426 * we have a multi-letter extension's _, or the start of an 427 * extension. 428 * 429 * Otherwise we have found the current extension's major version 430 * number. Parse past it, and a subsequent p/minor version number 431 * if present. The `p` extension must not appear immediately after 432 * a number, so there is no fear of missing it. 433 * 434 */ 435 if (unlikely(!isalpha(*ext))) { 436 ext_err = true; 437 break; 438 } 439 440 if (!isdigit(*isa)) 441 break; 442 443 while (isdigit(*++isa)) 444 ; 445 446 if (tolower(*isa) != 'p') 447 break; 448 449 if (!isdigit(*++isa)) { 450 --isa; 451 break; 452 } 453 454 while (isdigit(*++isa)) 455 ; 456 457 break; 458 } 459 460 /* 461 * The parser expects that at the start of an iteration isa points to the 462 * first character of the next extension. As we stop parsing an extension 463 * on meeting a non-alphanumeric character, an extra increment is needed 464 * where the succeeding extension is a multi-letter prefixed with an "_". 465 */ 466 if (*isa == '_') 467 ++isa; 468 469 if (unlikely(ext_err)) 470 continue; 471 if (!ext_long) { 472 int nr = tolower(*ext) - 'a'; 473 474 if (riscv_isa_extension_check(nr)) { 475 *this_hwcap |= isa2hwcap[nr]; 476 set_bit(nr, isainfo->isa); 477 } 478 } else { 479 for (int i = 0; i < riscv_isa_ext_count; i++) 480 match_isa_ext(&riscv_isa_ext[i], ext, ext_end, isainfo); 481 } 482 } 483 } 484 485 static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) 486 { 487 struct device_node *node; 488 const char *isa; 489 int rc; 490 struct acpi_table_header *rhct; 491 acpi_status status; 492 unsigned int cpu; 493 494 if (!acpi_disabled) { 495 status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); 496 if (ACPI_FAILURE(status)) 497 return; 498 } 499 500 for_each_possible_cpu(cpu) { 501 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 502 unsigned long this_hwcap = 0; 503 504 if (acpi_disabled) { 505 node = of_cpu_device_node_get(cpu); 506 if (!node) { 507 pr_warn("Unable to find cpu node\n"); 508 continue; 509 } 510 511 rc = of_property_read_string(node, "riscv,isa", &isa); 512 of_node_put(node); 513 if (rc) { 514 pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); 515 continue; 516 } 517 } else { 518 rc = acpi_get_riscv_isa(rhct, cpu, &isa); 519 if (rc < 0) { 520 pr_warn("Unable to get ISA for the hart - %d\n", cpu); 521 continue; 522 } 523 } 524 525 riscv_parse_isa_string(&this_hwcap, isainfo, isa2hwcap, isa); 526 527 /* 528 * These ones were as they were part of the base ISA when the 529 * port & dt-bindings were upstreamed, and so can be set 530 * unconditionally where `i` is in riscv,isa on DT systems. 531 */ 532 if (acpi_disabled) { 533 set_bit(RISCV_ISA_EXT_ZICSR, isainfo->isa); 534 set_bit(RISCV_ISA_EXT_ZIFENCEI, isainfo->isa); 535 set_bit(RISCV_ISA_EXT_ZICNTR, isainfo->isa); 536 set_bit(RISCV_ISA_EXT_ZIHPM, isainfo->isa); 537 } 538 539 /* 540 * "V" in ISA strings is ambiguous in practice: it should mean 541 * just the standard V-1.0 but vendors aren't well behaved. 542 * Many vendors with T-Head CPU cores which implement the 0.7.1 543 * version of the vector specification put "v" into their DTs. 544 * CPU cores with the ratified spec will contain non-zero 545 * marchid. 546 */ 547 if (acpi_disabled && riscv_cached_mvendorid(cpu) == THEAD_VENDOR_ID && 548 riscv_cached_marchid(cpu) == 0x0) { 549 this_hwcap &= ~isa2hwcap[RISCV_ISA_EXT_v]; 550 clear_bit(RISCV_ISA_EXT_v, isainfo->isa); 551 } 552 553 /* 554 * All "okay" hart should have same isa. Set HWCAP based on 555 * common capabilities of every "okay" hart, in case they don't 556 * have. 557 */ 558 if (elf_hwcap) 559 elf_hwcap &= this_hwcap; 560 else 561 elf_hwcap = this_hwcap; 562 563 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 564 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 565 else 566 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 567 } 568 569 if (!acpi_disabled && rhct) 570 acpi_put_table((struct acpi_table_header *)rhct); 571 } 572 573 static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) 574 { 575 unsigned int cpu; 576 577 for_each_possible_cpu(cpu) { 578 unsigned long this_hwcap = 0; 579 struct device_node *cpu_node; 580 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 581 582 cpu_node = of_cpu_device_node_get(cpu); 583 if (!cpu_node) { 584 pr_warn("Unable to find cpu node\n"); 585 continue; 586 } 587 588 if (!of_property_present(cpu_node, "riscv,isa-extensions")) { 589 of_node_put(cpu_node); 590 continue; 591 } 592 593 for (int i = 0; i < riscv_isa_ext_count; i++) { 594 const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i]; 595 596 if (of_property_match_string(cpu_node, "riscv,isa-extensions", 597 ext->property) < 0) 598 continue; 599 600 if (ext->subset_ext_size) { 601 for (int j = 0; j < ext->subset_ext_size; j++) { 602 if (riscv_isa_extension_check(ext->subset_ext_ids[i])) 603 set_bit(ext->subset_ext_ids[j], isainfo->isa); 604 } 605 } 606 607 if (riscv_isa_extension_check(ext->id)) { 608 set_bit(ext->id, isainfo->isa); 609 610 /* Only single letter extensions get set in hwcap */ 611 if (strnlen(riscv_isa_ext[i].name, 2) == 1) 612 this_hwcap |= isa2hwcap[riscv_isa_ext[i].id]; 613 } 614 } 615 616 of_node_put(cpu_node); 617 618 /* 619 * All "okay" harts should have same isa. Set HWCAP based on 620 * common capabilities of every "okay" hart, in case they don't. 621 */ 622 if (elf_hwcap) 623 elf_hwcap &= this_hwcap; 624 else 625 elf_hwcap = this_hwcap; 626 627 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 628 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 629 else 630 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); 631 } 632 633 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 634 return -ENOENT; 635 636 return 0; 637 } 638 639 #ifdef CONFIG_RISCV_ISA_FALLBACK 640 bool __initdata riscv_isa_fallback = true; 641 #else 642 bool __initdata riscv_isa_fallback; 643 static int __init riscv_isa_fallback_setup(char *__unused) 644 { 645 riscv_isa_fallback = true; 646 return 1; 647 } 648 early_param("riscv_isa_fallback", riscv_isa_fallback_setup); 649 #endif 650 651 void __init riscv_fill_hwcap(void) 652 { 653 char print_str[NUM_ALPHA_EXTS + 1]; 654 unsigned long isa2hwcap[26] = {0}; 655 int i, j; 656 657 isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I; 658 isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M; 659 isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A; 660 isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F; 661 isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D; 662 isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C; 663 isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V; 664 665 if (!acpi_disabled) { 666 riscv_fill_hwcap_from_isa_string(isa2hwcap); 667 } else { 668 int ret = riscv_fill_hwcap_from_ext_list(isa2hwcap); 669 670 if (ret && riscv_isa_fallback) { 671 pr_info("Falling back to deprecated \"riscv,isa\"\n"); 672 riscv_fill_hwcap_from_isa_string(isa2hwcap); 673 } 674 } 675 676 /* 677 * We don't support systems with F but without D, so mask those out 678 * here. 679 */ 680 if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { 681 pr_info("This kernel does not support systems with F but not D\n"); 682 elf_hwcap &= ~COMPAT_HWCAP_ISA_F; 683 } 684 685 if (elf_hwcap & COMPAT_HWCAP_ISA_V) { 686 riscv_v_setup_vsize(); 687 /* 688 * ISA string in device tree might have 'v' flag, but 689 * CONFIG_RISCV_ISA_V is disabled in kernel. 690 * Clear V flag in elf_hwcap if CONFIG_RISCV_ISA_V is disabled. 691 */ 692 if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) 693 elf_hwcap &= ~COMPAT_HWCAP_ISA_V; 694 } 695 696 memset(print_str, 0, sizeof(print_str)); 697 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 698 if (riscv_isa[0] & BIT_MASK(i)) 699 print_str[j++] = (char)('a' + i); 700 pr_info("riscv: base ISA extensions %s\n", print_str); 701 702 memset(print_str, 0, sizeof(print_str)); 703 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 704 if (elf_hwcap & BIT_MASK(i)) 705 print_str[j++] = (char)('a' + i); 706 pr_info("riscv: ELF capabilities %s\n", print_str); 707 } 708 709 unsigned long riscv_get_elf_hwcap(void) 710 { 711 unsigned long hwcap; 712 713 hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1)); 714 715 if (!riscv_v_vstate_ctrl_user_allowed()) 716 hwcap &= ~COMPAT_HWCAP_ISA_V; 717 718 return hwcap; 719 } 720 721 void riscv_user_isa_enable(void) 722 { 723 if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_ZICBOZ)) 724 csr_set(CSR_ENVCFG, ENVCFG_CBZE); 725 } 726 727 #ifdef CONFIG_RISCV_ALTERNATIVE 728 /* 729 * Alternative patch sites consider 48 bits when determining when to patch 730 * the old instruction sequence with the new. These bits are broken into a 731 * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the 732 * patch site is for an erratum, identified by the 32-bit patch ID. When 733 * the vendor ID is zero, the patch site is for a cpufeature. cpufeatures 734 * further break down patch ID into two 16-bit numbers. The lower 16 bits 735 * are the cpufeature ID and the upper 16 bits are used for a value specific 736 * to the cpufeature and patch site. If the upper 16 bits are zero, then it 737 * implies no specific value is specified. cpufeatures that want to control 738 * patching on a per-site basis will provide non-zero values and implement 739 * checks here. The checks return true when patching should be done, and 740 * false otherwise. 741 */ 742 static bool riscv_cpufeature_patch_check(u16 id, u16 value) 743 { 744 if (!value) 745 return true; 746 747 switch (id) { 748 case RISCV_ISA_EXT_ZICBOZ: 749 /* 750 * Zicboz alternative applications provide the maximum 751 * supported block size order, or zero when it doesn't 752 * matter. If the current block size exceeds the maximum, 753 * then the alternative cannot be applied. 754 */ 755 return riscv_cboz_block_size <= (1U << value); 756 } 757 758 return false; 759 } 760 761 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, 762 struct alt_entry *end, 763 unsigned int stage) 764 { 765 struct alt_entry *alt; 766 void *oldptr, *altptr; 767 u16 id, value; 768 769 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 770 return; 771 772 for (alt = begin; alt < end; alt++) { 773 if (alt->vendor_id != 0) 774 continue; 775 776 id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); 777 778 if (id >= RISCV_ISA_EXT_MAX) { 779 WARN(1, "This extension id:%d is not in ISA extension list", id); 780 continue; 781 } 782 783 if (!__riscv_isa_extension_available(NULL, id)) 784 continue; 785 786 value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id); 787 if (!riscv_cpufeature_patch_check(id, value)) 788 continue; 789 790 oldptr = ALT_OLD_PTR(alt); 791 altptr = ALT_ALT_PTR(alt); 792 793 mutex_lock(&text_mutex); 794 patch_text_nosync(oldptr, altptr, alt->alt_len); 795 riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); 796 mutex_unlock(&text_mutex); 797 } 798 } 799 #endif 800