1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * The hwprobe interface, for allowing userspace to probe to see which features 4 * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for 5 * more details. 6 */ 7 #include <linux/syscalls.h> 8 #include <asm/cacheflush.h> 9 #include <asm/cpufeature.h> 10 #include <asm/hwprobe.h> 11 #include <asm/processor.h> 12 #include <asm/delay.h> 13 #include <asm/sbi.h> 14 #include <asm/switch_to.h> 15 #include <asm/uaccess.h> 16 #include <asm/unistd.h> 17 #include <asm/vector.h> 18 #include <asm/vendor_extensions/mips_hwprobe.h> 19 #include <asm/vendor_extensions/sifive_hwprobe.h> 20 #include <asm/vendor_extensions/thead_hwprobe.h> 21 #include <vdso/vsyscall.h> 22 23 24 static void hwprobe_arch_id(struct riscv_hwprobe *pair, 25 const struct cpumask *cpus) 26 { 27 u64 id = -1ULL; 28 bool first = true; 29 int cpu; 30 31 for_each_cpu(cpu, cpus) { 32 u64 cpu_id; 33 34 switch (pair->key) { 35 case RISCV_HWPROBE_KEY_MVENDORID: 36 cpu_id = riscv_cached_mvendorid(cpu); 37 break; 38 case RISCV_HWPROBE_KEY_MIMPID: 39 cpu_id = riscv_cached_mimpid(cpu); 40 break; 41 case RISCV_HWPROBE_KEY_MARCHID: 42 cpu_id = riscv_cached_marchid(cpu); 43 break; 44 } 45 46 if (first) { 47 id = cpu_id; 48 first = false; 49 } 50 51 /* 52 * If there's a mismatch for the given set, return -1 in the 53 * value. 54 */ 55 if (id != cpu_id) { 56 id = -1ULL; 57 break; 58 } 59 } 60 61 pair->value = id; 62 } 63 64 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, 65 const struct cpumask *cpus) 66 { 67 int cpu; 68 u64 missing = 0; 69 70 pair->value = 0; 71 if (has_fpu()) 72 pair->value |= RISCV_HWPROBE_IMA_FD; 73 74 if (riscv_isa_extension_available(NULL, c)) 75 pair->value |= RISCV_HWPROBE_IMA_C; 76 77 if (has_vector() && riscv_isa_extension_available(NULL, v)) 78 pair->value |= RISCV_HWPROBE_IMA_V; 79 80 /* 81 * Loop through and record extensions that 1) anyone has, and 2) anyone 82 * doesn't have. 83 */ 84 for_each_cpu(cpu, cpus) { 85 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 86 87 #define EXT_KEY(ext) \ 88 do { \ 89 if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \ 90 pair->value |= RISCV_HWPROBE_EXT_##ext; \ 91 else \ 92 missing |= RISCV_HWPROBE_EXT_##ext; \ 93 } while (false) 94 95 /* 96 * Only use EXT_KEY() for extensions which can be exposed to userspace, 97 * regardless of the kernel's configuration, as no other checks, besides 98 * presence in the hart_isa bitmap, are made. 99 */ 100 EXT_KEY(ZAAMO); 101 EXT_KEY(ZABHA); 102 EXT_KEY(ZACAS); 103 EXT_KEY(ZALRSC); 104 EXT_KEY(ZAWRS); 105 EXT_KEY(ZBA); 106 EXT_KEY(ZBB); 107 EXT_KEY(ZBC); 108 EXT_KEY(ZBKB); 109 EXT_KEY(ZBKC); 110 EXT_KEY(ZBKX); 111 EXT_KEY(ZBS); 112 EXT_KEY(ZCA); 113 EXT_KEY(ZCB); 114 EXT_KEY(ZCMOP); 115 EXT_KEY(ZICBOM); 116 EXT_KEY(ZICBOZ); 117 EXT_KEY(ZICNTR); 118 EXT_KEY(ZICOND); 119 EXT_KEY(ZIHINTNTL); 120 EXT_KEY(ZIHINTPAUSE); 121 EXT_KEY(ZIHPM); 122 EXT_KEY(ZIMOP); 123 EXT_KEY(ZKND); 124 EXT_KEY(ZKNE); 125 EXT_KEY(ZKNH); 126 EXT_KEY(ZKSED); 127 EXT_KEY(ZKSH); 128 EXT_KEY(ZKT); 129 EXT_KEY(ZTSO); 130 131 /* 132 * All the following extensions must depend on the kernel 133 * support of V. 134 */ 135 if (has_vector()) { 136 EXT_KEY(ZVBB); 137 EXT_KEY(ZVBC); 138 EXT_KEY(ZVE32F); 139 EXT_KEY(ZVE32X); 140 EXT_KEY(ZVE64D); 141 EXT_KEY(ZVE64F); 142 EXT_KEY(ZVE64X); 143 EXT_KEY(ZVFBFMIN); 144 EXT_KEY(ZVFBFWMA); 145 EXT_KEY(ZVFH); 146 EXT_KEY(ZVFHMIN); 147 EXT_KEY(ZVKB); 148 EXT_KEY(ZVKG); 149 EXT_KEY(ZVKNED); 150 EXT_KEY(ZVKNHA); 151 EXT_KEY(ZVKNHB); 152 EXT_KEY(ZVKSED); 153 EXT_KEY(ZVKSH); 154 EXT_KEY(ZVKT); 155 } 156 157 EXT_KEY(ZCD); 158 EXT_KEY(ZCF); 159 EXT_KEY(ZFA); 160 EXT_KEY(ZFBFMIN); 161 EXT_KEY(ZFH); 162 EXT_KEY(ZFHMIN); 163 164 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM)) 165 EXT_KEY(SUPM); 166 #undef EXT_KEY 167 } 168 169 /* Now turn off reporting features if any CPU is missing it. */ 170 pair->value &= ~missing; 171 } 172 173 static bool hwprobe_ext0_has(const struct cpumask *cpus, u64 ext) 174 { 175 struct riscv_hwprobe pair; 176 177 hwprobe_isa_ext0(&pair, cpus); 178 return (pair.value & ext); 179 } 180 181 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) 182 static u64 hwprobe_misaligned(const struct cpumask *cpus) 183 { 184 int cpu; 185 u64 perf = -1ULL; 186 187 for_each_cpu(cpu, cpus) { 188 int this_perf = per_cpu(misaligned_access_speed, cpu); 189 190 if (perf == -1ULL) 191 perf = this_perf; 192 193 if (perf != this_perf) { 194 perf = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 195 break; 196 } 197 } 198 199 if (perf == -1ULL) 200 return RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 201 202 return perf; 203 } 204 #else 205 static u64 hwprobe_misaligned(const struct cpumask *cpus) 206 { 207 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS)) 208 return RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; 209 210 if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available()) 211 return RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED; 212 213 return RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; 214 } 215 #endif 216 217 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 218 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) 219 { 220 int cpu; 221 u64 perf = -1ULL; 222 223 /* Return if supported or not even if speed wasn't probed */ 224 for_each_cpu(cpu, cpus) { 225 int this_perf = per_cpu(vector_misaligned_access, cpu); 226 227 if (perf == -1ULL) 228 perf = this_perf; 229 230 if (perf != this_perf) { 231 perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 232 break; 233 } 234 } 235 236 if (perf == -1ULL) 237 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 238 239 return perf; 240 } 241 #else 242 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) 243 { 244 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS)) 245 return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; 246 247 if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS)) 248 return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; 249 250 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 251 } 252 #endif 253 254 static void hwprobe_one_pair(struct riscv_hwprobe *pair, 255 const struct cpumask *cpus) 256 { 257 switch (pair->key) { 258 case RISCV_HWPROBE_KEY_MVENDORID: 259 case RISCV_HWPROBE_KEY_MARCHID: 260 case RISCV_HWPROBE_KEY_MIMPID: 261 hwprobe_arch_id(pair, cpus); 262 break; 263 /* 264 * The kernel already assumes that the base single-letter ISA 265 * extensions are supported on all harts, and only supports the 266 * IMA base, so just cheat a bit here and tell that to 267 * userspace. 268 */ 269 case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: 270 pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA; 271 break; 272 273 case RISCV_HWPROBE_KEY_IMA_EXT_0: 274 hwprobe_isa_ext0(pair, cpus); 275 break; 276 277 case RISCV_HWPROBE_KEY_CPUPERF_0: 278 case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF: 279 pair->value = hwprobe_misaligned(cpus); 280 break; 281 282 case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF: 283 pair->value = hwprobe_vec_misaligned(cpus); 284 break; 285 286 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: 287 pair->value = 0; 288 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) 289 pair->value = riscv_cboz_block_size; 290 break; 291 case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE: 292 pair->value = 0; 293 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM)) 294 pair->value = riscv_cbom_block_size; 295 break; 296 case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS: 297 pair->value = user_max_virt_addr(); 298 break; 299 300 case RISCV_HWPROBE_KEY_TIME_CSR_FREQ: 301 pair->value = riscv_timebase; 302 break; 303 304 case RISCV_HWPROBE_KEY_VENDOR_EXT_SIFIVE_0: 305 hwprobe_isa_vendor_ext_sifive_0(pair, cpus); 306 break; 307 308 case RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0: 309 hwprobe_isa_vendor_ext_thead_0(pair, cpus); 310 break; 311 case RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0: 312 hwprobe_isa_vendor_ext_mips_0(pair, cpus); 313 break; 314 315 /* 316 * For forward compatibility, unknown keys don't fail the whole 317 * call, but get their element key set to -1 and value set to 0 318 * indicating they're unrecognized. 319 */ 320 default: 321 pair->key = -1; 322 pair->value = 0; 323 break; 324 } 325 } 326 327 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs, 328 size_t pair_count, size_t cpusetsize, 329 unsigned long __user *cpus_user, 330 unsigned int flags) 331 { 332 size_t out; 333 int ret; 334 cpumask_t cpus; 335 336 /* Check the reserved flags. */ 337 if (flags != 0) 338 return -EINVAL; 339 340 /* 341 * The interface supports taking in a CPU mask, and returns values that 342 * are consistent across that mask. Allow userspace to specify NULL and 343 * 0 as a shortcut to all online CPUs. 344 */ 345 cpumask_clear(&cpus); 346 if (!cpusetsize && !cpus_user) { 347 cpumask_copy(&cpus, cpu_online_mask); 348 } else { 349 if (cpusetsize > cpumask_size()) 350 cpusetsize = cpumask_size(); 351 352 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 353 if (ret) 354 return -EFAULT; 355 356 /* 357 * Userspace must provide at least one online CPU, without that 358 * there's no way to define what is supported. 359 */ 360 cpumask_and(&cpus, &cpus, cpu_online_mask); 361 if (cpumask_empty(&cpus)) 362 return -EINVAL; 363 } 364 365 for (out = 0; out < pair_count; out++, pairs++) { 366 struct riscv_hwprobe pair; 367 368 if (get_user(pair.key, &pairs->key)) 369 return -EFAULT; 370 371 pair.value = 0; 372 hwprobe_one_pair(&pair, &cpus); 373 ret = put_user(pair.key, &pairs->key); 374 if (ret == 0) 375 ret = put_user(pair.value, &pairs->value); 376 377 if (ret) 378 return -EFAULT; 379 } 380 381 return 0; 382 } 383 384 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, 385 size_t pair_count, size_t cpusetsize, 386 unsigned long __user *cpus_user, 387 unsigned int flags) 388 { 389 cpumask_t cpus, one_cpu; 390 bool clear_all = false; 391 size_t i; 392 int ret; 393 394 if (flags != RISCV_HWPROBE_WHICH_CPUS) 395 return -EINVAL; 396 397 if (!cpusetsize || !cpus_user) 398 return -EINVAL; 399 400 if (cpusetsize > cpumask_size()) 401 cpusetsize = cpumask_size(); 402 403 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 404 if (ret) 405 return -EFAULT; 406 407 if (cpumask_empty(&cpus)) 408 cpumask_copy(&cpus, cpu_online_mask); 409 410 cpumask_and(&cpus, &cpus, cpu_online_mask); 411 412 cpumask_clear(&one_cpu); 413 414 for (i = 0; i < pair_count; i++) { 415 struct riscv_hwprobe pair, tmp; 416 int cpu; 417 418 ret = copy_from_user(&pair, &pairs[i], sizeof(pair)); 419 if (ret) 420 return -EFAULT; 421 422 if (!riscv_hwprobe_key_is_valid(pair.key)) { 423 clear_all = true; 424 pair = (struct riscv_hwprobe){ .key = -1, }; 425 ret = copy_to_user(&pairs[i], &pair, sizeof(pair)); 426 if (ret) 427 return -EFAULT; 428 } 429 430 if (clear_all) 431 continue; 432 433 tmp = (struct riscv_hwprobe){ .key = pair.key, }; 434 435 for_each_cpu(cpu, &cpus) { 436 cpumask_set_cpu(cpu, &one_cpu); 437 438 hwprobe_one_pair(&tmp, &one_cpu); 439 440 if (!riscv_hwprobe_pair_cmp(&tmp, &pair)) 441 cpumask_clear_cpu(cpu, &cpus); 442 443 cpumask_clear_cpu(cpu, &one_cpu); 444 } 445 } 446 447 if (clear_all) 448 cpumask_clear(&cpus); 449 450 ret = copy_to_user(cpus_user, &cpus, cpusetsize); 451 if (ret) 452 return -EFAULT; 453 454 return 0; 455 } 456 457 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, 458 size_t pair_count, size_t cpusetsize, 459 unsigned long __user *cpus_user, 460 unsigned int flags) 461 { 462 if (flags & RISCV_HWPROBE_WHICH_CPUS) 463 return hwprobe_get_cpus(pairs, pair_count, cpusetsize, 464 cpus_user, flags); 465 466 return hwprobe_get_values(pairs, pair_count, cpusetsize, 467 cpus_user, flags); 468 } 469 470 #ifdef CONFIG_MMU 471 472 static int __init init_hwprobe_vdso_data(void) 473 { 474 struct vdso_arch_data *avd = vdso_k_arch_data; 475 u64 id_bitsmash = 0; 476 struct riscv_hwprobe pair; 477 int key; 478 479 /* 480 * Initialize vDSO data with the answers for the "all CPUs" case, to 481 * save a syscall in the common case. 482 */ 483 for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) { 484 pair.key = key; 485 hwprobe_one_pair(&pair, cpu_online_mask); 486 487 WARN_ON_ONCE(pair.key < 0); 488 489 avd->all_cpu_hwprobe_values[key] = pair.value; 490 /* 491 * Smash together the vendor, arch, and impl IDs to see if 492 * they're all 0 or any negative. 493 */ 494 if (key <= RISCV_HWPROBE_KEY_MIMPID) 495 id_bitsmash |= pair.value; 496 } 497 498 /* 499 * If the arch, vendor, and implementation ID are all the same across 500 * all harts, then assume all CPUs are the same, and allow the vDSO to 501 * answer queries for arbitrary masks. However if all values are 0 (not 502 * populated) or any value returns -1 (varies across CPUs), then the 503 * vDSO should defer to the kernel for exotic cpu masks. 504 */ 505 avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; 506 return 0; 507 } 508 509 arch_initcall_sync(init_hwprobe_vdso_data); 510 511 #endif /* CONFIG_MMU */ 512 513 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, 514 size_t, pair_count, size_t, cpusetsize, unsigned long __user *, 515 cpus, unsigned int, flags) 516 { 517 return do_riscv_hwprobe(pairs, pair_count, cpusetsize, 518 cpus, flags); 519 } 520