1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * The hwprobe interface, for allowing userspace to probe to see which features 4 * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for 5 * more details. 6 */ 7 #include <linux/syscalls.h> 8 #include <asm/cacheflush.h> 9 #include <asm/cpufeature.h> 10 #include <asm/hwprobe.h> 11 #include <asm/processor.h> 12 #include <asm/delay.h> 13 #include <asm/sbi.h> 14 #include <asm/switch_to.h> 15 #include <asm/uaccess.h> 16 #include <asm/unistd.h> 17 #include <asm/vector.h> 18 #include <asm/vendor_extensions/thead_hwprobe.h> 19 #include <vdso/vsyscall.h> 20 21 22 static void hwprobe_arch_id(struct riscv_hwprobe *pair, 23 const struct cpumask *cpus) 24 { 25 u64 id = -1ULL; 26 bool first = true; 27 int cpu; 28 29 for_each_cpu(cpu, cpus) { 30 u64 cpu_id; 31 32 switch (pair->key) { 33 case RISCV_HWPROBE_KEY_MVENDORID: 34 cpu_id = riscv_cached_mvendorid(cpu); 35 break; 36 case RISCV_HWPROBE_KEY_MIMPID: 37 cpu_id = riscv_cached_mimpid(cpu); 38 break; 39 case RISCV_HWPROBE_KEY_MARCHID: 40 cpu_id = riscv_cached_marchid(cpu); 41 break; 42 } 43 44 if (first) { 45 id = cpu_id; 46 first = false; 47 } 48 49 /* 50 * If there's a mismatch for the given set, return -1 in the 51 * value. 52 */ 53 if (id != cpu_id) { 54 id = -1ULL; 55 break; 56 } 57 } 58 59 pair->value = id; 60 } 61 62 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, 63 const struct cpumask *cpus) 64 { 65 int cpu; 66 u64 missing = 0; 67 68 pair->value = 0; 69 if (has_fpu()) 70 pair->value |= RISCV_HWPROBE_IMA_FD; 71 72 if (riscv_isa_extension_available(NULL, c)) 73 pair->value |= RISCV_HWPROBE_IMA_C; 74 75 if (has_vector() && riscv_isa_extension_available(NULL, v)) 76 pair->value |= RISCV_HWPROBE_IMA_V; 77 78 /* 79 * Loop through and record extensions that 1) anyone has, and 2) anyone 80 * doesn't have. 81 */ 82 for_each_cpu(cpu, cpus) { 83 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 84 85 #define EXT_KEY(ext) \ 86 do { \ 87 if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \ 88 pair->value |= RISCV_HWPROBE_EXT_##ext; \ 89 else \ 90 missing |= RISCV_HWPROBE_EXT_##ext; \ 91 } while (false) 92 93 /* 94 * Only use EXT_KEY() for extensions which can be exposed to userspace, 95 * regardless of the kernel's configuration, as no other checks, besides 96 * presence in the hart_isa bitmap, are made. 97 */ 98 EXT_KEY(ZACAS); 99 EXT_KEY(ZAWRS); 100 EXT_KEY(ZBA); 101 EXT_KEY(ZBB); 102 EXT_KEY(ZBC); 103 EXT_KEY(ZBKB); 104 EXT_KEY(ZBKC); 105 EXT_KEY(ZBKX); 106 EXT_KEY(ZBS); 107 EXT_KEY(ZCA); 108 EXT_KEY(ZCB); 109 EXT_KEY(ZCMOP); 110 EXT_KEY(ZICBOM); 111 EXT_KEY(ZICBOZ); 112 EXT_KEY(ZICNTR); 113 EXT_KEY(ZICOND); 114 EXT_KEY(ZIHINTNTL); 115 EXT_KEY(ZIHINTPAUSE); 116 EXT_KEY(ZIHPM); 117 EXT_KEY(ZIMOP); 118 EXT_KEY(ZKND); 119 EXT_KEY(ZKNE); 120 EXT_KEY(ZKNH); 121 EXT_KEY(ZKSED); 122 EXT_KEY(ZKSH); 123 EXT_KEY(ZKT); 124 EXT_KEY(ZTSO); 125 126 /* 127 * All the following extensions must depend on the kernel 128 * support of V. 129 */ 130 if (has_vector()) { 131 EXT_KEY(ZVBB); 132 EXT_KEY(ZVBC); 133 EXT_KEY(ZVE32F); 134 EXT_KEY(ZVE32X); 135 EXT_KEY(ZVE64D); 136 EXT_KEY(ZVE64F); 137 EXT_KEY(ZVE64X); 138 EXT_KEY(ZVFBFMIN); 139 EXT_KEY(ZVFBFWMA); 140 EXT_KEY(ZVFH); 141 EXT_KEY(ZVFHMIN); 142 EXT_KEY(ZVKB); 143 EXT_KEY(ZVKG); 144 EXT_KEY(ZVKNED); 145 EXT_KEY(ZVKNHA); 146 EXT_KEY(ZVKNHB); 147 EXT_KEY(ZVKSED); 148 EXT_KEY(ZVKSH); 149 EXT_KEY(ZVKT); 150 } 151 152 if (has_fpu()) { 153 EXT_KEY(ZCD); 154 EXT_KEY(ZCF); 155 EXT_KEY(ZFA); 156 EXT_KEY(ZFBFMIN); 157 EXT_KEY(ZFH); 158 EXT_KEY(ZFHMIN); 159 } 160 161 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM)) 162 EXT_KEY(SUPM); 163 #undef EXT_KEY 164 } 165 166 /* Now turn off reporting features if any CPU is missing it. */ 167 pair->value &= ~missing; 168 } 169 170 static bool hwprobe_ext0_has(const struct cpumask *cpus, u64 ext) 171 { 172 struct riscv_hwprobe pair; 173 174 hwprobe_isa_ext0(&pair, cpus); 175 return (pair.value & ext); 176 } 177 178 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) 179 static u64 hwprobe_misaligned(const struct cpumask *cpus) 180 { 181 int cpu; 182 u64 perf = -1ULL; 183 184 for_each_cpu(cpu, cpus) { 185 int this_perf = per_cpu(misaligned_access_speed, cpu); 186 187 if (perf == -1ULL) 188 perf = this_perf; 189 190 if (perf != this_perf) { 191 perf = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 192 break; 193 } 194 } 195 196 if (perf == -1ULL) 197 return RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 198 199 return perf; 200 } 201 #else 202 static u64 hwprobe_misaligned(const struct cpumask *cpus) 203 { 204 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS)) 205 return RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; 206 207 if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available()) 208 return RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED; 209 210 return RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; 211 } 212 #endif 213 214 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 215 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) 216 { 217 int cpu; 218 u64 perf = -1ULL; 219 220 /* Return if supported or not even if speed wasn't probed */ 221 for_each_cpu(cpu, cpus) { 222 int this_perf = per_cpu(vector_misaligned_access, cpu); 223 224 if (perf == -1ULL) 225 perf = this_perf; 226 227 if (perf != this_perf) { 228 perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 229 break; 230 } 231 } 232 233 if (perf == -1ULL) 234 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 235 236 return perf; 237 } 238 #else 239 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) 240 { 241 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS)) 242 return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; 243 244 if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS)) 245 return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; 246 247 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 248 } 249 #endif 250 251 static void hwprobe_one_pair(struct riscv_hwprobe *pair, 252 const struct cpumask *cpus) 253 { 254 switch (pair->key) { 255 case RISCV_HWPROBE_KEY_MVENDORID: 256 case RISCV_HWPROBE_KEY_MARCHID: 257 case RISCV_HWPROBE_KEY_MIMPID: 258 hwprobe_arch_id(pair, cpus); 259 break; 260 /* 261 * The kernel already assumes that the base single-letter ISA 262 * extensions are supported on all harts, and only supports the 263 * IMA base, so just cheat a bit here and tell that to 264 * userspace. 265 */ 266 case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: 267 pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA; 268 break; 269 270 case RISCV_HWPROBE_KEY_IMA_EXT_0: 271 hwprobe_isa_ext0(pair, cpus); 272 break; 273 274 case RISCV_HWPROBE_KEY_CPUPERF_0: 275 case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF: 276 pair->value = hwprobe_misaligned(cpus); 277 break; 278 279 case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF: 280 pair->value = hwprobe_vec_misaligned(cpus); 281 break; 282 283 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: 284 pair->value = 0; 285 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) 286 pair->value = riscv_cboz_block_size; 287 break; 288 case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE: 289 pair->value = 0; 290 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM)) 291 pair->value = riscv_cbom_block_size; 292 break; 293 case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS: 294 pair->value = user_max_virt_addr(); 295 break; 296 297 case RISCV_HWPROBE_KEY_TIME_CSR_FREQ: 298 pair->value = riscv_timebase; 299 break; 300 301 case RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0: 302 hwprobe_isa_vendor_ext_thead_0(pair, cpus); 303 break; 304 305 /* 306 * For forward compatibility, unknown keys don't fail the whole 307 * call, but get their element key set to -1 and value set to 0 308 * indicating they're unrecognized. 309 */ 310 default: 311 pair->key = -1; 312 pair->value = 0; 313 break; 314 } 315 } 316 317 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs, 318 size_t pair_count, size_t cpusetsize, 319 unsigned long __user *cpus_user, 320 unsigned int flags) 321 { 322 size_t out; 323 int ret; 324 cpumask_t cpus; 325 326 /* Check the reserved flags. */ 327 if (flags != 0) 328 return -EINVAL; 329 330 /* 331 * The interface supports taking in a CPU mask, and returns values that 332 * are consistent across that mask. Allow userspace to specify NULL and 333 * 0 as a shortcut to all online CPUs. 334 */ 335 cpumask_clear(&cpus); 336 if (!cpusetsize && !cpus_user) { 337 cpumask_copy(&cpus, cpu_online_mask); 338 } else { 339 if (cpusetsize > cpumask_size()) 340 cpusetsize = cpumask_size(); 341 342 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 343 if (ret) 344 return -EFAULT; 345 346 /* 347 * Userspace must provide at least one online CPU, without that 348 * there's no way to define what is supported. 349 */ 350 cpumask_and(&cpus, &cpus, cpu_online_mask); 351 if (cpumask_empty(&cpus)) 352 return -EINVAL; 353 } 354 355 for (out = 0; out < pair_count; out++, pairs++) { 356 struct riscv_hwprobe pair; 357 358 if (get_user(pair.key, &pairs->key)) 359 return -EFAULT; 360 361 pair.value = 0; 362 hwprobe_one_pair(&pair, &cpus); 363 ret = put_user(pair.key, &pairs->key); 364 if (ret == 0) 365 ret = put_user(pair.value, &pairs->value); 366 367 if (ret) 368 return -EFAULT; 369 } 370 371 return 0; 372 } 373 374 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, 375 size_t pair_count, size_t cpusetsize, 376 unsigned long __user *cpus_user, 377 unsigned int flags) 378 { 379 cpumask_t cpus, one_cpu; 380 bool clear_all = false; 381 size_t i; 382 int ret; 383 384 if (flags != RISCV_HWPROBE_WHICH_CPUS) 385 return -EINVAL; 386 387 if (!cpusetsize || !cpus_user) 388 return -EINVAL; 389 390 if (cpusetsize > cpumask_size()) 391 cpusetsize = cpumask_size(); 392 393 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 394 if (ret) 395 return -EFAULT; 396 397 if (cpumask_empty(&cpus)) 398 cpumask_copy(&cpus, cpu_online_mask); 399 400 cpumask_and(&cpus, &cpus, cpu_online_mask); 401 402 cpumask_clear(&one_cpu); 403 404 for (i = 0; i < pair_count; i++) { 405 struct riscv_hwprobe pair, tmp; 406 int cpu; 407 408 ret = copy_from_user(&pair, &pairs[i], sizeof(pair)); 409 if (ret) 410 return -EFAULT; 411 412 if (!riscv_hwprobe_key_is_valid(pair.key)) { 413 clear_all = true; 414 pair = (struct riscv_hwprobe){ .key = -1, }; 415 ret = copy_to_user(&pairs[i], &pair, sizeof(pair)); 416 if (ret) 417 return -EFAULT; 418 } 419 420 if (clear_all) 421 continue; 422 423 tmp = (struct riscv_hwprobe){ .key = pair.key, }; 424 425 for_each_cpu(cpu, &cpus) { 426 cpumask_set_cpu(cpu, &one_cpu); 427 428 hwprobe_one_pair(&tmp, &one_cpu); 429 430 if (!riscv_hwprobe_pair_cmp(&tmp, &pair)) 431 cpumask_clear_cpu(cpu, &cpus); 432 433 cpumask_clear_cpu(cpu, &one_cpu); 434 } 435 } 436 437 if (clear_all) 438 cpumask_clear(&cpus); 439 440 ret = copy_to_user(cpus_user, &cpus, cpusetsize); 441 if (ret) 442 return -EFAULT; 443 444 return 0; 445 } 446 447 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, 448 size_t pair_count, size_t cpusetsize, 449 unsigned long __user *cpus_user, 450 unsigned int flags) 451 { 452 if (flags & RISCV_HWPROBE_WHICH_CPUS) 453 return hwprobe_get_cpus(pairs, pair_count, cpusetsize, 454 cpus_user, flags); 455 456 return hwprobe_get_values(pairs, pair_count, cpusetsize, 457 cpus_user, flags); 458 } 459 460 #ifdef CONFIG_MMU 461 462 static int __init init_hwprobe_vdso_data(void) 463 { 464 struct vdso_data *vd = __arch_get_k_vdso_data(); 465 struct arch_vdso_time_data *avd = &vd->arch_data; 466 u64 id_bitsmash = 0; 467 struct riscv_hwprobe pair; 468 int key; 469 470 /* 471 * Initialize vDSO data with the answers for the "all CPUs" case, to 472 * save a syscall in the common case. 473 */ 474 for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) { 475 pair.key = key; 476 hwprobe_one_pair(&pair, cpu_online_mask); 477 478 WARN_ON_ONCE(pair.key < 0); 479 480 avd->all_cpu_hwprobe_values[key] = pair.value; 481 /* 482 * Smash together the vendor, arch, and impl IDs to see if 483 * they're all 0 or any negative. 484 */ 485 if (key <= RISCV_HWPROBE_KEY_MIMPID) 486 id_bitsmash |= pair.value; 487 } 488 489 /* 490 * If the arch, vendor, and implementation ID are all the same across 491 * all harts, then assume all CPUs are the same, and allow the vDSO to 492 * answer queries for arbitrary masks. However if all values are 0 (not 493 * populated) or any value returns -1 (varies across CPUs), then the 494 * vDSO should defer to the kernel for exotic cpu masks. 495 */ 496 avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; 497 return 0; 498 } 499 500 arch_initcall_sync(init_hwprobe_vdso_data); 501 502 #endif /* CONFIG_MMU */ 503 504 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, 505 size_t, pair_count, size_t, cpusetsize, unsigned long __user *, 506 cpus, unsigned int, flags) 507 { 508 return do_riscv_hwprobe(pairs, pair_count, cpusetsize, 509 cpus, flags); 510 } 511