1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * The hwprobe interface, for allowing userspace to probe to see which features 4 * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for 5 * more details. 6 */ 7 #include <linux/syscalls.h> 8 #include <asm/cacheflush.h> 9 #include <asm/cpufeature.h> 10 #include <asm/hwprobe.h> 11 #include <asm/processor.h> 12 #include <asm/delay.h> 13 #include <asm/sbi.h> 14 #include <asm/switch_to.h> 15 #include <asm/uaccess.h> 16 #include <asm/unistd.h> 17 #include <asm/vector.h> 18 #include <asm/vendor_extensions/mips_hwprobe.h> 19 #include <asm/vendor_extensions/sifive_hwprobe.h> 20 #include <asm/vendor_extensions/thead_hwprobe.h> 21 #include <vdso/vsyscall.h> 22 23 24 #define EXT_KEY(isa_arg, ext, pv, missing) \ 25 do { \ 26 if (__riscv_isa_extension_available(isa_arg, RISCV_ISA_EXT_##ext)) \ 27 pv |= RISCV_HWPROBE_EXT_##ext; \ 28 else \ 29 missing |= RISCV_HWPROBE_EXT_##ext; \ 30 } while (false) 31 32 static void hwprobe_arch_id(struct riscv_hwprobe *pair, 33 const struct cpumask *cpus) 34 { 35 u64 id = -1ULL; 36 bool first = true; 37 int cpu; 38 39 if (pair->key != RISCV_HWPROBE_KEY_MVENDORID && 40 pair->key != RISCV_HWPROBE_KEY_MIMPID && 41 pair->key != RISCV_HWPROBE_KEY_MARCHID) 42 goto out; 43 44 for_each_cpu(cpu, cpus) { 45 u64 cpu_id; 46 47 switch (pair->key) { 48 case RISCV_HWPROBE_KEY_MVENDORID: 49 cpu_id = riscv_cached_mvendorid(cpu); 50 break; 51 case RISCV_HWPROBE_KEY_MIMPID: 52 cpu_id = riscv_cached_mimpid(cpu); 53 break; 54 case RISCV_HWPROBE_KEY_MARCHID: 55 cpu_id = riscv_cached_marchid(cpu); 56 break; 57 } 58 59 if (first) { 60 id = cpu_id; 61 first = false; 62 } 63 64 /* 65 * If there's a mismatch for the given set, return -1 in the 66 * value. 67 */ 68 if (id != cpu_id) { 69 id = -1ULL; 70 break; 71 } 72 } 73 74 out: 75 pair->value = id; 76 } 77 78 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, 79 const struct cpumask *cpus) 80 { 81 int cpu; 82 u64 missing = 0; 83 84 pair->value = 0; 85 if (has_fpu()) 86 pair->value |= RISCV_HWPROBE_IMA_FD; 87 88 if (riscv_isa_extension_available(NULL, C)) 89 pair->value |= RISCV_HWPROBE_IMA_C; 90 91 if (has_vector() && riscv_isa_extension_available(NULL, V)) 92 pair->value |= RISCV_HWPROBE_IMA_V; 93 94 /* 95 * Loop through and record extensions that 1) anyone has, and 2) anyone 96 * doesn't have. 97 */ 98 for_each_cpu(cpu, cpus) { 99 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 100 101 /* 102 * Only use EXT_KEY() for extensions which can be exposed to userspace, 103 * regardless of the kernel's configuration, as no other checks, besides 104 * presence in the hart_isa bitmap, are made. 105 */ 106 EXT_KEY(isainfo->isa, ZAAMO, pair->value, missing); 107 EXT_KEY(isainfo->isa, ZABHA, pair->value, missing); 108 EXT_KEY(isainfo->isa, ZACAS, pair->value, missing); 109 EXT_KEY(isainfo->isa, ZALASR, pair->value, missing); 110 EXT_KEY(isainfo->isa, ZALRSC, pair->value, missing); 111 EXT_KEY(isainfo->isa, ZAWRS, pair->value, missing); 112 EXT_KEY(isainfo->isa, ZBA, pair->value, missing); 113 EXT_KEY(isainfo->isa, ZBB, pair->value, missing); 114 EXT_KEY(isainfo->isa, ZBC, pair->value, missing); 115 EXT_KEY(isainfo->isa, ZBKB, pair->value, missing); 116 EXT_KEY(isainfo->isa, ZBKC, pair->value, missing); 117 EXT_KEY(isainfo->isa, ZBKX, pair->value, missing); 118 EXT_KEY(isainfo->isa, ZBS, pair->value, missing); 119 EXT_KEY(isainfo->isa, ZCA, pair->value, missing); 120 EXT_KEY(isainfo->isa, ZCB, pair->value, missing); 121 EXT_KEY(isainfo->isa, ZCLSD, pair->value, missing); 122 EXT_KEY(isainfo->isa, ZCMOP, pair->value, missing); 123 EXT_KEY(isainfo->isa, ZICBOM, pair->value, missing); 124 EXT_KEY(isainfo->isa, ZICBOP, pair->value, missing); 125 EXT_KEY(isainfo->isa, ZICBOZ, pair->value, missing); 126 EXT_KEY(isainfo->isa, ZICFILP, pair->value, missing); 127 EXT_KEY(isainfo->isa, ZICNTR, pair->value, missing); 128 EXT_KEY(isainfo->isa, ZICOND, pair->value, missing); 129 EXT_KEY(isainfo->isa, ZIHINTNTL, pair->value, missing); 130 EXT_KEY(isainfo->isa, ZIHINTPAUSE, pair->value, missing); 131 EXT_KEY(isainfo->isa, ZIHPM, pair->value, missing); 132 EXT_KEY(isainfo->isa, ZILSD, pair->value, missing); 133 EXT_KEY(isainfo->isa, ZIMOP, pair->value, missing); 134 EXT_KEY(isainfo->isa, ZKND, pair->value, missing); 135 EXT_KEY(isainfo->isa, ZKNE, pair->value, missing); 136 EXT_KEY(isainfo->isa, ZKNH, pair->value, missing); 137 EXT_KEY(isainfo->isa, ZKSED, pair->value, missing); 138 EXT_KEY(isainfo->isa, ZKSH, pair->value, missing); 139 EXT_KEY(isainfo->isa, ZKT, pair->value, missing); 140 EXT_KEY(isainfo->isa, ZTSO, pair->value, missing); 141 142 /* 143 * All the following extensions must depend on the kernel 144 * support of V. 145 */ 146 if (has_vector()) { 147 EXT_KEY(isainfo->isa, ZVBB, pair->value, missing); 148 EXT_KEY(isainfo->isa, ZVBC, pair->value, missing); 149 EXT_KEY(isainfo->isa, ZVE32F, pair->value, missing); 150 EXT_KEY(isainfo->isa, ZVE32X, pair->value, missing); 151 EXT_KEY(isainfo->isa, ZVE64D, pair->value, missing); 152 EXT_KEY(isainfo->isa, ZVE64F, pair->value, missing); 153 EXT_KEY(isainfo->isa, ZVE64X, pair->value, missing); 154 EXT_KEY(isainfo->isa, ZVFBFMIN, pair->value, missing); 155 EXT_KEY(isainfo->isa, ZVFBFWMA, pair->value, missing); 156 EXT_KEY(isainfo->isa, ZVFH, pair->value, missing); 157 EXT_KEY(isainfo->isa, ZVFHMIN, pair->value, missing); 158 EXT_KEY(isainfo->isa, ZVKB, pair->value, missing); 159 EXT_KEY(isainfo->isa, ZVKG, pair->value, missing); 160 EXT_KEY(isainfo->isa, ZVKNED, pair->value, missing); 161 EXT_KEY(isainfo->isa, ZVKNHA, pair->value, missing); 162 EXT_KEY(isainfo->isa, ZVKNHB, pair->value, missing); 163 EXT_KEY(isainfo->isa, ZVKSED, pair->value, missing); 164 EXT_KEY(isainfo->isa, ZVKSH, pair->value, missing); 165 EXT_KEY(isainfo->isa, ZVKT, pair->value, missing); 166 } 167 168 EXT_KEY(isainfo->isa, ZCD, pair->value, missing); 169 EXT_KEY(isainfo->isa, ZCF, pair->value, missing); 170 EXT_KEY(isainfo->isa, ZFA, pair->value, missing); 171 EXT_KEY(isainfo->isa, ZFBFMIN, pair->value, missing); 172 EXT_KEY(isainfo->isa, ZFH, pair->value, missing); 173 EXT_KEY(isainfo->isa, ZFHMIN, pair->value, missing); 174 175 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM)) 176 EXT_KEY(isainfo->isa, SUPM, pair->value, missing); 177 } 178 179 /* Now turn off reporting features if any CPU is missing it. */ 180 pair->value &= ~missing; 181 } 182 183 static void hwprobe_isa_ext1(struct riscv_hwprobe *pair, 184 const struct cpumask *cpus) 185 { 186 int cpu; 187 u64 missing = 0; 188 189 pair->value = 0; 190 191 /* 192 * Loop through and record extensions that 1) anyone has, and 2) anyone 193 * doesn't have. 194 */ 195 for_each_cpu(cpu, cpus) { 196 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 197 198 /* 199 * Only use EXT_KEY() for extensions which can be 200 * exposed to userspace, regardless of the kernel's 201 * configuration, as no other checks, besides presence 202 * in the hart_isa bitmap, are made. 203 */ 204 EXT_KEY(isainfo->isa, ZICFISS, pair->value, missing); 205 EXT_KEY(isainfo->isa, ZICCLSM, pair->value, missing); 206 EXT_KEY(isainfo->isa, ZICCAMOA, pair->value, missing); 207 EXT_KEY(isainfo->isa, ZICCIF, pair->value, missing); 208 EXT_KEY(isainfo->isa, ZICCRSE, pair->value, missing); 209 EXT_KEY(isainfo->isa, ZA64RS, pair->value, missing); 210 } 211 212 /* Now turn off reporting features if any CPU is missing it. */ 213 pair->value &= ~missing; 214 } 215 216 static bool hwprobe_ext0_has(const struct cpumask *cpus, u64 ext) 217 { 218 struct riscv_hwprobe pair; 219 220 hwprobe_isa_ext0(&pair, cpus); 221 return (pair.value & ext); 222 } 223 224 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) 225 static u64 hwprobe_misaligned(const struct cpumask *cpus) 226 { 227 int cpu; 228 u64 perf = -1ULL; 229 230 for_each_cpu(cpu, cpus) { 231 int this_perf = per_cpu(misaligned_access_speed, cpu); 232 233 if (perf == -1ULL) 234 perf = this_perf; 235 236 if (perf != this_perf) { 237 perf = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 238 break; 239 } 240 } 241 242 if (perf == -1ULL) 243 return RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 244 245 return perf; 246 } 247 #else 248 static u64 hwprobe_misaligned(const struct cpumask *cpus) 249 { 250 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS)) 251 return RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; 252 253 if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available()) 254 return RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED; 255 256 return RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; 257 } 258 #endif 259 260 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 261 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) 262 { 263 int cpu; 264 u64 perf = -1ULL; 265 266 /* Return if supported or not even if speed wasn't probed */ 267 for_each_cpu(cpu, cpus) { 268 int this_perf = per_cpu(vector_misaligned_access, cpu); 269 270 if (perf == -1ULL) 271 perf = this_perf; 272 273 if (perf != this_perf) { 274 perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 275 break; 276 } 277 } 278 279 if (perf == -1ULL) 280 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 281 282 return perf; 283 } 284 #else 285 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) 286 { 287 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS)) 288 return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; 289 290 if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS)) 291 return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; 292 293 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 294 } 295 #endif 296 297 static void hwprobe_one_pair(struct riscv_hwprobe *pair, 298 const struct cpumask *cpus) 299 { 300 switch (pair->key) { 301 case RISCV_HWPROBE_KEY_MVENDORID: 302 case RISCV_HWPROBE_KEY_MARCHID: 303 case RISCV_HWPROBE_KEY_MIMPID: 304 hwprobe_arch_id(pair, cpus); 305 break; 306 /* 307 * The kernel already assumes that the base single-letter ISA 308 * extensions are supported on all harts, and only supports the 309 * IMA base, so just cheat a bit here and tell that to 310 * userspace. 311 */ 312 case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: 313 pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA; 314 break; 315 316 case RISCV_HWPROBE_KEY_IMA_EXT_0: 317 hwprobe_isa_ext0(pair, cpus); 318 break; 319 320 case RISCV_HWPROBE_KEY_IMA_EXT_1: 321 hwprobe_isa_ext1(pair, cpus); 322 break; 323 324 case RISCV_HWPROBE_KEY_CPUPERF_0: 325 case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF: 326 pair->value = hwprobe_misaligned(cpus); 327 break; 328 329 case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF: 330 pair->value = hwprobe_vec_misaligned(cpus); 331 break; 332 333 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: 334 pair->value = 0; 335 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) 336 pair->value = riscv_cboz_block_size; 337 break; 338 case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE: 339 pair->value = 0; 340 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM)) 341 pair->value = riscv_cbom_block_size; 342 break; 343 case RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE: 344 pair->value = 0; 345 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOP)) 346 pair->value = riscv_cbop_block_size; 347 break; 348 case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS: 349 pair->value = user_max_virt_addr(); 350 break; 351 352 case RISCV_HWPROBE_KEY_TIME_CSR_FREQ: 353 pair->value = riscv_timebase; 354 break; 355 356 case RISCV_HWPROBE_KEY_VENDOR_EXT_SIFIVE_0: 357 hwprobe_isa_vendor_ext_sifive_0(pair, cpus); 358 break; 359 360 case RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0: 361 hwprobe_isa_vendor_ext_thead_0(pair, cpus); 362 break; 363 case RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0: 364 hwprobe_isa_vendor_ext_mips_0(pair, cpus); 365 break; 366 367 /* 368 * For forward compatibility, unknown keys don't fail the whole 369 * call, but get their element key set to -1 and value set to 0 370 * indicating they're unrecognized. 371 */ 372 default: 373 pair->key = -1; 374 pair->value = 0; 375 break; 376 } 377 } 378 379 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs, 380 size_t pair_count, size_t cpusetsize, 381 unsigned long __user *cpus_user, 382 unsigned int flags) 383 { 384 size_t out; 385 int ret; 386 cpumask_t cpus; 387 388 /* Check the reserved flags. */ 389 if (flags != 0) 390 return -EINVAL; 391 392 /* 393 * The interface supports taking in a CPU mask, and returns values that 394 * are consistent across that mask. Allow userspace to specify NULL and 395 * 0 as a shortcut to all online CPUs. 396 */ 397 cpumask_clear(&cpus); 398 if (!cpusetsize && !cpus_user) { 399 cpumask_copy(&cpus, cpu_online_mask); 400 } else { 401 if (cpusetsize > cpumask_size()) 402 cpusetsize = cpumask_size(); 403 404 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 405 if (ret) 406 return -EFAULT; 407 408 /* 409 * Userspace must provide at least one online CPU, without that 410 * there's no way to define what is supported. 411 */ 412 cpumask_and(&cpus, &cpus, cpu_online_mask); 413 if (cpumask_empty(&cpus)) 414 return -EINVAL; 415 } 416 417 for (out = 0; out < pair_count; out++, pairs++) { 418 struct riscv_hwprobe pair; 419 420 if (get_user(pair.key, &pairs->key)) 421 return -EFAULT; 422 423 pair.value = 0; 424 hwprobe_one_pair(&pair, &cpus); 425 ret = put_user(pair.key, &pairs->key); 426 if (ret == 0) 427 ret = put_user(pair.value, &pairs->value); 428 429 if (ret) 430 return -EFAULT; 431 } 432 433 return 0; 434 } 435 436 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, 437 size_t pair_count, size_t cpusetsize, 438 unsigned long __user *cpus_user, 439 unsigned int flags) 440 { 441 cpumask_t cpus, one_cpu; 442 bool clear_all = false; 443 size_t i; 444 int ret; 445 446 if (flags != RISCV_HWPROBE_WHICH_CPUS) 447 return -EINVAL; 448 449 if (!cpusetsize || !cpus_user) 450 return -EINVAL; 451 452 if (cpusetsize > cpumask_size()) 453 cpusetsize = cpumask_size(); 454 455 cpumask_clear(&cpus); 456 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 457 if (ret) 458 return -EFAULT; 459 460 if (cpumask_empty(&cpus)) 461 cpumask_copy(&cpus, cpu_online_mask); 462 463 cpumask_and(&cpus, &cpus, cpu_online_mask); 464 465 cpumask_clear(&one_cpu); 466 467 for (i = 0; i < pair_count; i++) { 468 struct riscv_hwprobe pair, tmp; 469 int cpu; 470 471 ret = copy_from_user(&pair, &pairs[i], sizeof(pair)); 472 if (ret) 473 return -EFAULT; 474 475 if (!riscv_hwprobe_key_is_valid(pair.key)) { 476 clear_all = true; 477 pair = (struct riscv_hwprobe){ .key = -1, }; 478 ret = copy_to_user(&pairs[i], &pair, sizeof(pair)); 479 if (ret) 480 return -EFAULT; 481 } 482 483 if (clear_all) 484 continue; 485 486 tmp = (struct riscv_hwprobe){ .key = pair.key, }; 487 488 for_each_cpu(cpu, &cpus) { 489 cpumask_set_cpu(cpu, &one_cpu); 490 491 hwprobe_one_pair(&tmp, &one_cpu); 492 493 if (!riscv_hwprobe_pair_cmp(&tmp, &pair)) 494 cpumask_clear_cpu(cpu, &cpus); 495 496 cpumask_clear_cpu(cpu, &one_cpu); 497 } 498 } 499 500 if (clear_all) 501 cpumask_clear(&cpus); 502 503 ret = copy_to_user(cpus_user, &cpus, cpusetsize); 504 if (ret) 505 return -EFAULT; 506 507 return 0; 508 } 509 510 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, 511 size_t pair_count, size_t cpusetsize, 512 unsigned long __user *cpus_user, 513 unsigned int flags) 514 { 515 if (flags & RISCV_HWPROBE_WHICH_CPUS) 516 return hwprobe_get_cpus(pairs, pair_count, cpusetsize, 517 cpus_user, flags); 518 519 return hwprobe_get_values(pairs, pair_count, cpusetsize, 520 cpus_user, flags); 521 } 522 523 #ifdef CONFIG_MMU 524 525 static int __init init_hwprobe_vdso_data(void) 526 { 527 struct vdso_arch_data *avd = vdso_k_arch_data; 528 u64 id_bitsmash = 0; 529 struct riscv_hwprobe pair; 530 int key; 531 532 /* 533 * Initialize vDSO data with the answers for the "all CPUs" case, to 534 * save a syscall in the common case. 535 */ 536 for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) { 537 pair.key = key; 538 hwprobe_one_pair(&pair, cpu_online_mask); 539 540 WARN_ON_ONCE(pair.key < 0); 541 542 avd->all_cpu_hwprobe_values[key] = pair.value; 543 /* 544 * Smash together the vendor, arch, and impl IDs to see if 545 * they're all 0 or any negative. 546 */ 547 if (key <= RISCV_HWPROBE_KEY_MIMPID) 548 id_bitsmash |= pair.value; 549 } 550 551 /* 552 * If the arch, vendor, and implementation ID are all the same across 553 * all harts, then assume all CPUs are the same, and allow the vDSO to 554 * answer queries for arbitrary masks. However if all values are 0 (not 555 * populated) or any value returns -1 (varies across CPUs), then the 556 * vDSO should defer to the kernel for exotic cpu masks. 557 */ 558 avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; 559 return 0; 560 } 561 562 arch_initcall_sync(init_hwprobe_vdso_data); 563 564 #endif /* CONFIG_MMU */ 565 566 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, 567 size_t, pair_count, size_t, cpusetsize, unsigned long __user *, 568 cpus, unsigned int, flags) 569 { 570 return do_riscv_hwprobe(pairs, pair_count, cpusetsize, 571 cpus, flags); 572 } 573