1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * The hwprobe interface, for allowing userspace to probe to see which features 4 * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for 5 * more details. 6 */ 7 #include <linux/syscalls.h> 8 #include <linux/completion.h> 9 #include <linux/atomic.h> 10 #include <linux/once.h> 11 #include <asm/cacheflush.h> 12 #include <asm/cpufeature.h> 13 #include <asm/hwprobe.h> 14 #include <asm/processor.h> 15 #include <asm/delay.h> 16 #include <asm/sbi.h> 17 #include <asm/switch_to.h> 18 #include <asm/uaccess.h> 19 #include <asm/unistd.h> 20 #include <asm/vector.h> 21 #include <asm/vendor_extensions/mips_hwprobe.h> 22 #include <asm/vendor_extensions/sifive_hwprobe.h> 23 #include <asm/vendor_extensions/thead_hwprobe.h> 24 #include <vdso/vsyscall.h> 25 26 27 #define EXT_KEY(isa_arg, ext, pv, missing) \ 28 do { \ 29 if (__riscv_isa_extension_available(isa_arg, RISCV_ISA_EXT_##ext)) \ 30 pv |= RISCV_HWPROBE_EXT_##ext; \ 31 else \ 32 missing |= RISCV_HWPROBE_EXT_##ext; \ 33 } while (false) 34 35 static void hwprobe_arch_id(struct riscv_hwprobe *pair, 36 const struct cpumask *cpus) 37 { 38 u64 id = -1ULL; 39 bool first = true; 40 int cpu; 41 42 if (pair->key != RISCV_HWPROBE_KEY_MVENDORID && 43 pair->key != RISCV_HWPROBE_KEY_MIMPID && 44 pair->key != RISCV_HWPROBE_KEY_MARCHID) 45 goto out; 46 47 for_each_cpu(cpu, cpus) { 48 u64 cpu_id; 49 50 switch (pair->key) { 51 case RISCV_HWPROBE_KEY_MVENDORID: 52 cpu_id = riscv_cached_mvendorid(cpu); 53 break; 54 case RISCV_HWPROBE_KEY_MIMPID: 55 cpu_id = riscv_cached_mimpid(cpu); 56 break; 57 case RISCV_HWPROBE_KEY_MARCHID: 58 cpu_id = riscv_cached_marchid(cpu); 59 break; 60 } 61 62 if (first) { 63 id = cpu_id; 64 first = false; 65 } 66 67 /* 68 * If there's a mismatch for the given set, return -1 in the 69 * value. 70 */ 71 if (id != cpu_id) { 72 id = -1ULL; 73 break; 74 } 75 } 76 77 out: 78 pair->value = id; 79 } 80 81 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, 82 const struct cpumask *cpus) 83 { 84 int cpu; 85 u64 missing = 0; 86 87 pair->value = 0; 88 if (has_fpu()) 89 pair->value |= RISCV_HWPROBE_IMA_FD; 90 91 if (riscv_isa_extension_available(NULL, c)) 92 pair->value |= RISCV_HWPROBE_IMA_C; 93 94 if (has_vector() && riscv_isa_extension_available(NULL, v)) 95 pair->value |= RISCV_HWPROBE_IMA_V; 96 97 /* 98 * Loop through and record extensions that 1) anyone has, and 2) anyone 99 * doesn't have. 100 */ 101 for_each_cpu(cpu, cpus) { 102 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 103 104 /* 105 * Only use EXT_KEY() for extensions which can be exposed to userspace, 106 * regardless of the kernel's configuration, as no other checks, besides 107 * presence in the hart_isa bitmap, are made. 108 */ 109 EXT_KEY(isainfo->isa, ZAAMO, pair->value, missing); 110 EXT_KEY(isainfo->isa, ZABHA, pair->value, missing); 111 EXT_KEY(isainfo->isa, ZACAS, pair->value, missing); 112 EXT_KEY(isainfo->isa, ZALASR, pair->value, missing); 113 EXT_KEY(isainfo->isa, ZALRSC, pair->value, missing); 114 EXT_KEY(isainfo->isa, ZAWRS, pair->value, missing); 115 EXT_KEY(isainfo->isa, ZBA, pair->value, missing); 116 EXT_KEY(isainfo->isa, ZBB, pair->value, missing); 117 EXT_KEY(isainfo->isa, ZBC, pair->value, missing); 118 EXT_KEY(isainfo->isa, ZBKB, pair->value, missing); 119 EXT_KEY(isainfo->isa, ZBKC, pair->value, missing); 120 EXT_KEY(isainfo->isa, ZBKX, pair->value, missing); 121 EXT_KEY(isainfo->isa, ZBS, pair->value, missing); 122 EXT_KEY(isainfo->isa, ZCA, pair->value, missing); 123 EXT_KEY(isainfo->isa, ZCB, pair->value, missing); 124 EXT_KEY(isainfo->isa, ZCLSD, pair->value, missing); 125 EXT_KEY(isainfo->isa, ZCMOP, pair->value, missing); 126 EXT_KEY(isainfo->isa, ZICBOM, pair->value, missing); 127 EXT_KEY(isainfo->isa, ZICBOP, pair->value, missing); 128 EXT_KEY(isainfo->isa, ZICBOZ, pair->value, missing); 129 EXT_KEY(isainfo->isa, ZICFILP, pair->value, missing); 130 EXT_KEY(isainfo->isa, ZICNTR, pair->value, missing); 131 EXT_KEY(isainfo->isa, ZICOND, pair->value, missing); 132 EXT_KEY(isainfo->isa, ZIHINTNTL, pair->value, missing); 133 EXT_KEY(isainfo->isa, ZIHINTPAUSE, pair->value, missing); 134 EXT_KEY(isainfo->isa, ZIHPM, pair->value, missing); 135 EXT_KEY(isainfo->isa, ZILSD, pair->value, missing); 136 EXT_KEY(isainfo->isa, ZIMOP, pair->value, missing); 137 EXT_KEY(isainfo->isa, ZKND, pair->value, missing); 138 EXT_KEY(isainfo->isa, ZKNE, pair->value, missing); 139 EXT_KEY(isainfo->isa, ZKNH, pair->value, missing); 140 EXT_KEY(isainfo->isa, ZKSED, pair->value, missing); 141 EXT_KEY(isainfo->isa, ZKSH, pair->value, missing); 142 EXT_KEY(isainfo->isa, ZKT, pair->value, missing); 143 EXT_KEY(isainfo->isa, ZTSO, pair->value, missing); 144 145 /* 146 * All the following extensions must depend on the kernel 147 * support of V. 148 */ 149 if (has_vector()) { 150 EXT_KEY(isainfo->isa, ZVBB, pair->value, missing); 151 EXT_KEY(isainfo->isa, ZVBC, pair->value, missing); 152 EXT_KEY(isainfo->isa, ZVE32F, pair->value, missing); 153 EXT_KEY(isainfo->isa, ZVE32X, pair->value, missing); 154 EXT_KEY(isainfo->isa, ZVE64D, pair->value, missing); 155 EXT_KEY(isainfo->isa, ZVE64F, pair->value, missing); 156 EXT_KEY(isainfo->isa, ZVE64X, pair->value, missing); 157 EXT_KEY(isainfo->isa, ZVFBFMIN, pair->value, missing); 158 EXT_KEY(isainfo->isa, ZVFBFWMA, pair->value, missing); 159 EXT_KEY(isainfo->isa, ZVFH, pair->value, missing); 160 EXT_KEY(isainfo->isa, ZVFHMIN, pair->value, missing); 161 EXT_KEY(isainfo->isa, ZVKB, pair->value, missing); 162 EXT_KEY(isainfo->isa, ZVKG, pair->value, missing); 163 EXT_KEY(isainfo->isa, ZVKNED, pair->value, missing); 164 EXT_KEY(isainfo->isa, ZVKNHA, pair->value, missing); 165 EXT_KEY(isainfo->isa, ZVKNHB, pair->value, missing); 166 EXT_KEY(isainfo->isa, ZVKSED, pair->value, missing); 167 EXT_KEY(isainfo->isa, ZVKSH, pair->value, missing); 168 EXT_KEY(isainfo->isa, ZVKT, pair->value, missing); 169 } 170 171 EXT_KEY(isainfo->isa, ZCD, pair->value, missing); 172 EXT_KEY(isainfo->isa, ZCF, pair->value, missing); 173 EXT_KEY(isainfo->isa, ZFA, pair->value, missing); 174 EXT_KEY(isainfo->isa, ZFBFMIN, pair->value, missing); 175 EXT_KEY(isainfo->isa, ZFH, pair->value, missing); 176 EXT_KEY(isainfo->isa, ZFHMIN, pair->value, missing); 177 178 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM)) 179 EXT_KEY(isainfo->isa, SUPM, pair->value, missing); 180 } 181 182 /* Now turn off reporting features if any CPU is missing it. */ 183 pair->value &= ~missing; 184 } 185 186 static void hwprobe_isa_ext1(struct riscv_hwprobe *pair, 187 const struct cpumask *cpus) 188 { 189 int cpu; 190 u64 missing = 0; 191 192 pair->value = 0; 193 194 /* 195 * Loop through and record extensions that 1) anyone has, and 2) anyone 196 * doesn't have. 197 */ 198 for_each_cpu(cpu, cpus) { 199 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 200 201 /* 202 * Only use EXT_KEY() for extensions which can be 203 * exposed to userspace, regardless of the kernel's 204 * configuration, as no other checks, besides presence 205 * in the hart_isa bitmap, are made. 206 */ 207 EXT_KEY(isainfo->isa, ZICFISS, pair->value, missing); 208 } 209 210 /* Now turn off reporting features if any CPU is missing it. */ 211 pair->value &= ~missing; 212 } 213 214 static bool hwprobe_ext0_has(const struct cpumask *cpus, u64 ext) 215 { 216 struct riscv_hwprobe pair; 217 218 hwprobe_isa_ext0(&pair, cpus); 219 return (pair.value & ext); 220 } 221 222 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) 223 static u64 hwprobe_misaligned(const struct cpumask *cpus) 224 { 225 int cpu; 226 u64 perf = -1ULL; 227 228 for_each_cpu(cpu, cpus) { 229 int this_perf = per_cpu(misaligned_access_speed, cpu); 230 231 if (perf == -1ULL) 232 perf = this_perf; 233 234 if (perf != this_perf) { 235 perf = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 236 break; 237 } 238 } 239 240 if (perf == -1ULL) 241 return RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 242 243 return perf; 244 } 245 #else 246 static u64 hwprobe_misaligned(const struct cpumask *cpus) 247 { 248 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS)) 249 return RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; 250 251 if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available()) 252 return RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED; 253 254 return RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; 255 } 256 #endif 257 258 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 259 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) 260 { 261 int cpu; 262 u64 perf = -1ULL; 263 264 /* Return if supported or not even if speed wasn't probed */ 265 for_each_cpu(cpu, cpus) { 266 int this_perf = per_cpu(vector_misaligned_access, cpu); 267 268 if (perf == -1ULL) 269 perf = this_perf; 270 271 if (perf != this_perf) { 272 perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 273 break; 274 } 275 } 276 277 if (perf == -1ULL) 278 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 279 280 return perf; 281 } 282 #else 283 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) 284 { 285 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS)) 286 return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; 287 288 if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS)) 289 return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; 290 291 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 292 } 293 #endif 294 295 static void hwprobe_one_pair(struct riscv_hwprobe *pair, 296 const struct cpumask *cpus) 297 { 298 switch (pair->key) { 299 case RISCV_HWPROBE_KEY_MVENDORID: 300 case RISCV_HWPROBE_KEY_MARCHID: 301 case RISCV_HWPROBE_KEY_MIMPID: 302 hwprobe_arch_id(pair, cpus); 303 break; 304 /* 305 * The kernel already assumes that the base single-letter ISA 306 * extensions are supported on all harts, and only supports the 307 * IMA base, so just cheat a bit here and tell that to 308 * userspace. 309 */ 310 case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: 311 pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA; 312 break; 313 314 case RISCV_HWPROBE_KEY_IMA_EXT_0: 315 hwprobe_isa_ext0(pair, cpus); 316 break; 317 318 case RISCV_HWPROBE_KEY_IMA_EXT_1: 319 hwprobe_isa_ext1(pair, cpus); 320 break; 321 322 case RISCV_HWPROBE_KEY_CPUPERF_0: 323 case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF: 324 pair->value = hwprobe_misaligned(cpus); 325 break; 326 327 case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF: 328 pair->value = hwprobe_vec_misaligned(cpus); 329 break; 330 331 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: 332 pair->value = 0; 333 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) 334 pair->value = riscv_cboz_block_size; 335 break; 336 case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE: 337 pair->value = 0; 338 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM)) 339 pair->value = riscv_cbom_block_size; 340 break; 341 case RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE: 342 pair->value = 0; 343 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOP)) 344 pair->value = riscv_cbop_block_size; 345 break; 346 case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS: 347 pair->value = user_max_virt_addr(); 348 break; 349 350 case RISCV_HWPROBE_KEY_TIME_CSR_FREQ: 351 pair->value = riscv_timebase; 352 break; 353 354 case RISCV_HWPROBE_KEY_VENDOR_EXT_SIFIVE_0: 355 hwprobe_isa_vendor_ext_sifive_0(pair, cpus); 356 break; 357 358 case RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0: 359 hwprobe_isa_vendor_ext_thead_0(pair, cpus); 360 break; 361 case RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0: 362 hwprobe_isa_vendor_ext_mips_0(pair, cpus); 363 break; 364 365 /* 366 * For forward compatibility, unknown keys don't fail the whole 367 * call, but get their element key set to -1 and value set to 0 368 * indicating they're unrecognized. 369 */ 370 default: 371 pair->key = -1; 372 pair->value = 0; 373 break; 374 } 375 } 376 377 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs, 378 size_t pair_count, size_t cpusetsize, 379 unsigned long __user *cpus_user, 380 unsigned int flags) 381 { 382 size_t out; 383 int ret; 384 cpumask_t cpus; 385 386 /* Check the reserved flags. */ 387 if (flags != 0) 388 return -EINVAL; 389 390 /* 391 * The interface supports taking in a CPU mask, and returns values that 392 * are consistent across that mask. Allow userspace to specify NULL and 393 * 0 as a shortcut to all online CPUs. 394 */ 395 cpumask_clear(&cpus); 396 if (!cpusetsize && !cpus_user) { 397 cpumask_copy(&cpus, cpu_online_mask); 398 } else { 399 if (cpusetsize > cpumask_size()) 400 cpusetsize = cpumask_size(); 401 402 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 403 if (ret) 404 return -EFAULT; 405 406 /* 407 * Userspace must provide at least one online CPU, without that 408 * there's no way to define what is supported. 409 */ 410 cpumask_and(&cpus, &cpus, cpu_online_mask); 411 if (cpumask_empty(&cpus)) 412 return -EINVAL; 413 } 414 415 for (out = 0; out < pair_count; out++, pairs++) { 416 struct riscv_hwprobe pair; 417 418 if (get_user(pair.key, &pairs->key)) 419 return -EFAULT; 420 421 pair.value = 0; 422 hwprobe_one_pair(&pair, &cpus); 423 ret = put_user(pair.key, &pairs->key); 424 if (ret == 0) 425 ret = put_user(pair.value, &pairs->value); 426 427 if (ret) 428 return -EFAULT; 429 } 430 431 return 0; 432 } 433 434 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, 435 size_t pair_count, size_t cpusetsize, 436 unsigned long __user *cpus_user, 437 unsigned int flags) 438 { 439 cpumask_t cpus, one_cpu; 440 bool clear_all = false; 441 size_t i; 442 int ret; 443 444 if (flags != RISCV_HWPROBE_WHICH_CPUS) 445 return -EINVAL; 446 447 if (!cpusetsize || !cpus_user) 448 return -EINVAL; 449 450 if (cpusetsize > cpumask_size()) 451 cpusetsize = cpumask_size(); 452 453 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 454 if (ret) 455 return -EFAULT; 456 457 if (cpumask_empty(&cpus)) 458 cpumask_copy(&cpus, cpu_online_mask); 459 460 cpumask_and(&cpus, &cpus, cpu_online_mask); 461 462 cpumask_clear(&one_cpu); 463 464 for (i = 0; i < pair_count; i++) { 465 struct riscv_hwprobe pair, tmp; 466 int cpu; 467 468 ret = copy_from_user(&pair, &pairs[i], sizeof(pair)); 469 if (ret) 470 return -EFAULT; 471 472 if (!riscv_hwprobe_key_is_valid(pair.key)) { 473 clear_all = true; 474 pair = (struct riscv_hwprobe){ .key = -1, }; 475 ret = copy_to_user(&pairs[i], &pair, sizeof(pair)); 476 if (ret) 477 return -EFAULT; 478 } 479 480 if (clear_all) 481 continue; 482 483 tmp = (struct riscv_hwprobe){ .key = pair.key, }; 484 485 for_each_cpu(cpu, &cpus) { 486 cpumask_set_cpu(cpu, &one_cpu); 487 488 hwprobe_one_pair(&tmp, &one_cpu); 489 490 if (!riscv_hwprobe_pair_cmp(&tmp, &pair)) 491 cpumask_clear_cpu(cpu, &cpus); 492 493 cpumask_clear_cpu(cpu, &one_cpu); 494 } 495 } 496 497 if (clear_all) 498 cpumask_clear(&cpus); 499 500 ret = copy_to_user(cpus_user, &cpus, cpusetsize); 501 if (ret) 502 return -EFAULT; 503 504 return 0; 505 } 506 507 #ifdef CONFIG_MMU 508 509 static DECLARE_COMPLETION(boot_probes_done); 510 static atomic_t pending_boot_probes = ATOMIC_INIT(1); 511 512 void riscv_hwprobe_register_async_probe(void) 513 { 514 atomic_inc(&pending_boot_probes); 515 } 516 517 void riscv_hwprobe_complete_async_probe(void) 518 { 519 if (atomic_dec_and_test(&pending_boot_probes)) 520 complete(&boot_probes_done); 521 } 522 523 static int complete_hwprobe_vdso_data(void) 524 { 525 struct vdso_arch_data *avd = vdso_k_arch_data; 526 u64 id_bitsmash = 0; 527 struct riscv_hwprobe pair; 528 int key; 529 530 if (unlikely(!atomic_dec_and_test(&pending_boot_probes))) 531 wait_for_completion(&boot_probes_done); 532 533 /* 534 * Initialize vDSO data with the answers for the "all CPUs" case, to 535 * save a syscall in the common case. 536 */ 537 for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) { 538 pair.key = key; 539 hwprobe_one_pair(&pair, cpu_online_mask); 540 541 WARN_ON_ONCE(pair.key < 0); 542 543 avd->all_cpu_hwprobe_values[key] = pair.value; 544 /* 545 * Smash together the vendor, arch, and impl IDs to see if 546 * they're all 0 or any negative. 547 */ 548 if (key <= RISCV_HWPROBE_KEY_MIMPID) 549 id_bitsmash |= pair.value; 550 } 551 552 /* 553 * If the arch, vendor, and implementation ID are all the same across 554 * all harts, then assume all CPUs are the same, and allow the vDSO to 555 * answer queries for arbitrary masks. However if all values are 0 (not 556 * populated) or any value returns -1 (varies across CPUs), then the 557 * vDSO should defer to the kernel for exotic cpu masks. 558 */ 559 avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; 560 561 /* 562 * Make sure all the VDSO values are visible before we look at them. 563 * This pairs with the implicit "no speculativly visible accesses" 564 * barrier in the VDSO hwprobe code. 565 */ 566 smp_wmb(); 567 avd->ready = true; 568 return 0; 569 } 570 571 static int __init init_hwprobe_vdso_data(void) 572 { 573 struct vdso_arch_data *avd = vdso_k_arch_data; 574 575 /* 576 * Prevent the vDSO cached values from being used, as they're not ready 577 * yet. 578 */ 579 avd->ready = false; 580 return 0; 581 } 582 583 arch_initcall_sync(init_hwprobe_vdso_data); 584 585 #else 586 587 static int complete_hwprobe_vdso_data(void) { return 0; } 588 589 #endif /* CONFIG_MMU */ 590 591 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, 592 size_t pair_count, size_t cpusetsize, 593 unsigned long __user *cpus_user, 594 unsigned int flags) 595 { 596 DO_ONCE_SLEEPABLE(complete_hwprobe_vdso_data); 597 598 if (flags & RISCV_HWPROBE_WHICH_CPUS) 599 return hwprobe_get_cpus(pairs, pair_count, cpusetsize, 600 cpus_user, flags); 601 602 return hwprobe_get_values(pairs, pair_count, cpusetsize, 603 cpus_user, flags); 604 } 605 606 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, 607 size_t, pair_count, size_t, cpusetsize, unsigned long __user *, 608 cpus, unsigned int, flags) 609 { 610 return do_riscv_hwprobe(pairs, pair_count, cpusetsize, 611 cpus, flags); 612 } 613