1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * The hwprobe interface, for allowing userspace to probe to see which features 4 * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for 5 * more details. 6 */ 7 #include <linux/syscalls.h> 8 #include <asm/cacheflush.h> 9 #include <asm/cpufeature.h> 10 #include <asm/hwprobe.h> 11 #include <asm/processor.h> 12 #include <asm/delay.h> 13 #include <asm/sbi.h> 14 #include <asm/switch_to.h> 15 #include <asm/uaccess.h> 16 #include <asm/unistd.h> 17 #include <asm/vector.h> 18 #include <vdso/vsyscall.h> 19 20 21 static void hwprobe_arch_id(struct riscv_hwprobe *pair, 22 const struct cpumask *cpus) 23 { 24 u64 id = -1ULL; 25 bool first = true; 26 int cpu; 27 28 for_each_cpu(cpu, cpus) { 29 u64 cpu_id; 30 31 switch (pair->key) { 32 case RISCV_HWPROBE_KEY_MVENDORID: 33 cpu_id = riscv_cached_mvendorid(cpu); 34 break; 35 case RISCV_HWPROBE_KEY_MIMPID: 36 cpu_id = riscv_cached_mimpid(cpu); 37 break; 38 case RISCV_HWPROBE_KEY_MARCHID: 39 cpu_id = riscv_cached_marchid(cpu); 40 break; 41 } 42 43 if (first) { 44 id = cpu_id; 45 first = false; 46 } 47 48 /* 49 * If there's a mismatch for the given set, return -1 in the 50 * value. 51 */ 52 if (id != cpu_id) { 53 id = -1ULL; 54 break; 55 } 56 } 57 58 pair->value = id; 59 } 60 61 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, 62 const struct cpumask *cpus) 63 { 64 int cpu; 65 u64 missing = 0; 66 67 pair->value = 0; 68 if (has_fpu()) 69 pair->value |= RISCV_HWPROBE_IMA_FD; 70 71 if (riscv_isa_extension_available(NULL, c)) 72 pair->value |= RISCV_HWPROBE_IMA_C; 73 74 if (has_vector() && riscv_isa_extension_available(NULL, v)) 75 pair->value |= RISCV_HWPROBE_IMA_V; 76 77 /* 78 * Loop through and record extensions that 1) anyone has, and 2) anyone 79 * doesn't have. 80 */ 81 for_each_cpu(cpu, cpus) { 82 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 83 84 #define EXT_KEY(ext) \ 85 do { \ 86 if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \ 87 pair->value |= RISCV_HWPROBE_EXT_##ext; \ 88 else \ 89 missing |= RISCV_HWPROBE_EXT_##ext; \ 90 } while (false) 91 92 /* 93 * Only use EXT_KEY() for extensions which can be exposed to userspace, 94 * regardless of the kernel's configuration, as no other checks, besides 95 * presence in the hart_isa bitmap, are made. 96 */ 97 EXT_KEY(ZACAS); 98 EXT_KEY(ZAWRS); 99 EXT_KEY(ZBA); 100 EXT_KEY(ZBB); 101 EXT_KEY(ZBC); 102 EXT_KEY(ZBKB); 103 EXT_KEY(ZBKC); 104 EXT_KEY(ZBKX); 105 EXT_KEY(ZBS); 106 EXT_KEY(ZCA); 107 EXT_KEY(ZCB); 108 EXT_KEY(ZCMOP); 109 EXT_KEY(ZICBOZ); 110 EXT_KEY(ZICOND); 111 EXT_KEY(ZIHINTNTL); 112 EXT_KEY(ZIHINTPAUSE); 113 EXT_KEY(ZIMOP); 114 EXT_KEY(ZKND); 115 EXT_KEY(ZKNE); 116 EXT_KEY(ZKNH); 117 EXT_KEY(ZKSED); 118 EXT_KEY(ZKSH); 119 EXT_KEY(ZKT); 120 EXT_KEY(ZTSO); 121 122 /* 123 * All the following extensions must depend on the kernel 124 * support of V. 125 */ 126 if (has_vector()) { 127 EXT_KEY(ZVBB); 128 EXT_KEY(ZVBC); 129 EXT_KEY(ZVE32F); 130 EXT_KEY(ZVE32X); 131 EXT_KEY(ZVE64D); 132 EXT_KEY(ZVE64F); 133 EXT_KEY(ZVE64X); 134 EXT_KEY(ZVFH); 135 EXT_KEY(ZVFHMIN); 136 EXT_KEY(ZVKB); 137 EXT_KEY(ZVKG); 138 EXT_KEY(ZVKNED); 139 EXT_KEY(ZVKNHA); 140 EXT_KEY(ZVKNHB); 141 EXT_KEY(ZVKSED); 142 EXT_KEY(ZVKSH); 143 EXT_KEY(ZVKT); 144 } 145 146 if (has_fpu()) { 147 EXT_KEY(ZCD); 148 EXT_KEY(ZCF); 149 EXT_KEY(ZFA); 150 EXT_KEY(ZFH); 151 EXT_KEY(ZFHMIN); 152 } 153 #undef EXT_KEY 154 } 155 156 /* Now turn off reporting features if any CPU is missing it. */ 157 pair->value &= ~missing; 158 } 159 160 static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext) 161 { 162 struct riscv_hwprobe pair; 163 164 hwprobe_isa_ext0(&pair, cpus); 165 return (pair.value & ext); 166 } 167 168 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) 169 static u64 hwprobe_misaligned(const struct cpumask *cpus) 170 { 171 int cpu; 172 u64 perf = -1ULL; 173 174 for_each_cpu(cpu, cpus) { 175 int this_perf = per_cpu(misaligned_access_speed, cpu); 176 177 if (perf == -1ULL) 178 perf = this_perf; 179 180 if (perf != this_perf) { 181 perf = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 182 break; 183 } 184 } 185 186 if (perf == -1ULL) 187 return RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 188 189 return perf; 190 } 191 #else 192 static u64 hwprobe_misaligned(const struct cpumask *cpus) 193 { 194 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS)) 195 return RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; 196 197 if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available()) 198 return RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED; 199 200 return RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; 201 } 202 #endif 203 204 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 205 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) 206 { 207 int cpu; 208 u64 perf = -1ULL; 209 210 /* Return if supported or not even if speed wasn't probed */ 211 for_each_cpu(cpu, cpus) { 212 int this_perf = per_cpu(vector_misaligned_access, cpu); 213 214 if (perf == -1ULL) 215 perf = this_perf; 216 217 if (perf != this_perf) { 218 perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 219 break; 220 } 221 } 222 223 if (perf == -1ULL) 224 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 225 226 return perf; 227 } 228 #else 229 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) 230 { 231 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS)) 232 return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; 233 234 if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS)) 235 return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; 236 237 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 238 } 239 #endif 240 241 static void hwprobe_one_pair(struct riscv_hwprobe *pair, 242 const struct cpumask *cpus) 243 { 244 switch (pair->key) { 245 case RISCV_HWPROBE_KEY_MVENDORID: 246 case RISCV_HWPROBE_KEY_MARCHID: 247 case RISCV_HWPROBE_KEY_MIMPID: 248 hwprobe_arch_id(pair, cpus); 249 break; 250 /* 251 * The kernel already assumes that the base single-letter ISA 252 * extensions are supported on all harts, and only supports the 253 * IMA base, so just cheat a bit here and tell that to 254 * userspace. 255 */ 256 case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: 257 pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA; 258 break; 259 260 case RISCV_HWPROBE_KEY_IMA_EXT_0: 261 hwprobe_isa_ext0(pair, cpus); 262 break; 263 264 case RISCV_HWPROBE_KEY_CPUPERF_0: 265 case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF: 266 pair->value = hwprobe_misaligned(cpus); 267 break; 268 269 case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF: 270 pair->value = hwprobe_vec_misaligned(cpus); 271 break; 272 273 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: 274 pair->value = 0; 275 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) 276 pair->value = riscv_cboz_block_size; 277 break; 278 case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS: 279 pair->value = user_max_virt_addr(); 280 break; 281 282 case RISCV_HWPROBE_KEY_TIME_CSR_FREQ: 283 pair->value = riscv_timebase; 284 break; 285 286 /* 287 * For forward compatibility, unknown keys don't fail the whole 288 * call, but get their element key set to -1 and value set to 0 289 * indicating they're unrecognized. 290 */ 291 default: 292 pair->key = -1; 293 pair->value = 0; 294 break; 295 } 296 } 297 298 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs, 299 size_t pair_count, size_t cpusetsize, 300 unsigned long __user *cpus_user, 301 unsigned int flags) 302 { 303 size_t out; 304 int ret; 305 cpumask_t cpus; 306 307 /* Check the reserved flags. */ 308 if (flags != 0) 309 return -EINVAL; 310 311 /* 312 * The interface supports taking in a CPU mask, and returns values that 313 * are consistent across that mask. Allow userspace to specify NULL and 314 * 0 as a shortcut to all online CPUs. 315 */ 316 cpumask_clear(&cpus); 317 if (!cpusetsize && !cpus_user) { 318 cpumask_copy(&cpus, cpu_online_mask); 319 } else { 320 if (cpusetsize > cpumask_size()) 321 cpusetsize = cpumask_size(); 322 323 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 324 if (ret) 325 return -EFAULT; 326 327 /* 328 * Userspace must provide at least one online CPU, without that 329 * there's no way to define what is supported. 330 */ 331 cpumask_and(&cpus, &cpus, cpu_online_mask); 332 if (cpumask_empty(&cpus)) 333 return -EINVAL; 334 } 335 336 for (out = 0; out < pair_count; out++, pairs++) { 337 struct riscv_hwprobe pair; 338 339 if (get_user(pair.key, &pairs->key)) 340 return -EFAULT; 341 342 pair.value = 0; 343 hwprobe_one_pair(&pair, &cpus); 344 ret = put_user(pair.key, &pairs->key); 345 if (ret == 0) 346 ret = put_user(pair.value, &pairs->value); 347 348 if (ret) 349 return -EFAULT; 350 } 351 352 return 0; 353 } 354 355 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, 356 size_t pair_count, size_t cpusetsize, 357 unsigned long __user *cpus_user, 358 unsigned int flags) 359 { 360 cpumask_t cpus, one_cpu; 361 bool clear_all = false; 362 size_t i; 363 int ret; 364 365 if (flags != RISCV_HWPROBE_WHICH_CPUS) 366 return -EINVAL; 367 368 if (!cpusetsize || !cpus_user) 369 return -EINVAL; 370 371 if (cpusetsize > cpumask_size()) 372 cpusetsize = cpumask_size(); 373 374 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 375 if (ret) 376 return -EFAULT; 377 378 if (cpumask_empty(&cpus)) 379 cpumask_copy(&cpus, cpu_online_mask); 380 381 cpumask_and(&cpus, &cpus, cpu_online_mask); 382 383 cpumask_clear(&one_cpu); 384 385 for (i = 0; i < pair_count; i++) { 386 struct riscv_hwprobe pair, tmp; 387 int cpu; 388 389 ret = copy_from_user(&pair, &pairs[i], sizeof(pair)); 390 if (ret) 391 return -EFAULT; 392 393 if (!riscv_hwprobe_key_is_valid(pair.key)) { 394 clear_all = true; 395 pair = (struct riscv_hwprobe){ .key = -1, }; 396 ret = copy_to_user(&pairs[i], &pair, sizeof(pair)); 397 if (ret) 398 return -EFAULT; 399 } 400 401 if (clear_all) 402 continue; 403 404 tmp = (struct riscv_hwprobe){ .key = pair.key, }; 405 406 for_each_cpu(cpu, &cpus) { 407 cpumask_set_cpu(cpu, &one_cpu); 408 409 hwprobe_one_pair(&tmp, &one_cpu); 410 411 if (!riscv_hwprobe_pair_cmp(&tmp, &pair)) 412 cpumask_clear_cpu(cpu, &cpus); 413 414 cpumask_clear_cpu(cpu, &one_cpu); 415 } 416 } 417 418 if (clear_all) 419 cpumask_clear(&cpus); 420 421 ret = copy_to_user(cpus_user, &cpus, cpusetsize); 422 if (ret) 423 return -EFAULT; 424 425 return 0; 426 } 427 428 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, 429 size_t pair_count, size_t cpusetsize, 430 unsigned long __user *cpus_user, 431 unsigned int flags) 432 { 433 if (flags & RISCV_HWPROBE_WHICH_CPUS) 434 return hwprobe_get_cpus(pairs, pair_count, cpusetsize, 435 cpus_user, flags); 436 437 return hwprobe_get_values(pairs, pair_count, cpusetsize, 438 cpus_user, flags); 439 } 440 441 #ifdef CONFIG_MMU 442 443 static int __init init_hwprobe_vdso_data(void) 444 { 445 struct vdso_data *vd = __arch_get_k_vdso_data(); 446 struct arch_vdso_data *avd = &vd->arch_data; 447 u64 id_bitsmash = 0; 448 struct riscv_hwprobe pair; 449 int key; 450 451 /* 452 * Initialize vDSO data with the answers for the "all CPUs" case, to 453 * save a syscall in the common case. 454 */ 455 for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) { 456 pair.key = key; 457 hwprobe_one_pair(&pair, cpu_online_mask); 458 459 WARN_ON_ONCE(pair.key < 0); 460 461 avd->all_cpu_hwprobe_values[key] = pair.value; 462 /* 463 * Smash together the vendor, arch, and impl IDs to see if 464 * they're all 0 or any negative. 465 */ 466 if (key <= RISCV_HWPROBE_KEY_MIMPID) 467 id_bitsmash |= pair.value; 468 } 469 470 /* 471 * If the arch, vendor, and implementation ID are all the same across 472 * all harts, then assume all CPUs are the same, and allow the vDSO to 473 * answer queries for arbitrary masks. However if all values are 0 (not 474 * populated) or any value returns -1 (varies across CPUs), then the 475 * vDSO should defer to the kernel for exotic cpu masks. 476 */ 477 avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; 478 return 0; 479 } 480 481 arch_initcall_sync(init_hwprobe_vdso_data); 482 483 #endif /* CONFIG_MMU */ 484 485 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, 486 size_t, pair_count, size_t, cpusetsize, unsigned long __user *, 487 cpus, unsigned int, flags) 488 { 489 return do_riscv_hwprobe(pairs, pair_count, cpusetsize, 490 cpus, flags); 491 } 492