1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * The hwprobe interface, for allowing userspace to probe to see which features 4 * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for 5 * more details. 6 */ 7 #include <linux/syscalls.h> 8 #include <asm/cacheflush.h> 9 #include <asm/cpufeature.h> 10 #include <asm/hwprobe.h> 11 #include <asm/processor.h> 12 #include <asm/sbi.h> 13 #include <asm/switch_to.h> 14 #include <asm/uaccess.h> 15 #include <asm/unistd.h> 16 #include <asm/vector.h> 17 #include <vdso/vsyscall.h> 18 19 20 static void hwprobe_arch_id(struct riscv_hwprobe *pair, 21 const struct cpumask *cpus) 22 { 23 u64 id = -1ULL; 24 bool first = true; 25 int cpu; 26 27 for_each_cpu(cpu, cpus) { 28 u64 cpu_id; 29 30 switch (pair->key) { 31 case RISCV_HWPROBE_KEY_MVENDORID: 32 cpu_id = riscv_cached_mvendorid(cpu); 33 break; 34 case RISCV_HWPROBE_KEY_MIMPID: 35 cpu_id = riscv_cached_mimpid(cpu); 36 break; 37 case RISCV_HWPROBE_KEY_MARCHID: 38 cpu_id = riscv_cached_marchid(cpu); 39 break; 40 } 41 42 if (first) { 43 id = cpu_id; 44 first = false; 45 } 46 47 /* 48 * If there's a mismatch for the given set, return -1 in the 49 * value. 50 */ 51 if (id != cpu_id) { 52 id = -1ULL; 53 break; 54 } 55 } 56 57 pair->value = id; 58 } 59 60 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, 61 const struct cpumask *cpus) 62 { 63 int cpu; 64 u64 missing = 0; 65 66 pair->value = 0; 67 if (has_fpu()) 68 pair->value |= RISCV_HWPROBE_IMA_FD; 69 70 if (riscv_isa_extension_available(NULL, c)) 71 pair->value |= RISCV_HWPROBE_IMA_C; 72 73 if (has_vector() && riscv_isa_extension_available(NULL, v)) 74 pair->value |= RISCV_HWPROBE_IMA_V; 75 76 /* 77 * Loop through and record extensions that 1) anyone has, and 2) anyone 78 * doesn't have. 79 */ 80 for_each_cpu(cpu, cpus) { 81 struct riscv_isainfo *isainfo = &hart_isa[cpu]; 82 83 #define EXT_KEY(ext) \ 84 do { \ 85 if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \ 86 pair->value |= RISCV_HWPROBE_EXT_##ext; \ 87 else \ 88 missing |= RISCV_HWPROBE_EXT_##ext; \ 89 } while (false) 90 91 /* 92 * Only use EXT_KEY() for extensions which can be exposed to userspace, 93 * regardless of the kernel's configuration, as no other checks, besides 94 * presence in the hart_isa bitmap, are made. 95 */ 96 EXT_KEY(ZBA); 97 EXT_KEY(ZBB); 98 EXT_KEY(ZBS); 99 EXT_KEY(ZICBOZ); 100 EXT_KEY(ZBC); 101 102 EXT_KEY(ZBKB); 103 EXT_KEY(ZBKC); 104 EXT_KEY(ZBKX); 105 EXT_KEY(ZKND); 106 EXT_KEY(ZKNE); 107 EXT_KEY(ZKNH); 108 EXT_KEY(ZKSED); 109 EXT_KEY(ZKSH); 110 EXT_KEY(ZKT); 111 EXT_KEY(ZIHINTNTL); 112 EXT_KEY(ZTSO); 113 EXT_KEY(ZACAS); 114 EXT_KEY(ZICOND); 115 EXT_KEY(ZIHINTPAUSE); 116 EXT_KEY(ZIMOP); 117 EXT_KEY(ZCA); 118 EXT_KEY(ZCB); 119 EXT_KEY(ZCMOP); 120 EXT_KEY(ZAWRS); 121 122 /* 123 * All the following extensions must depend on the kernel 124 * support of V. 125 */ 126 if (has_vector()) { 127 EXT_KEY(ZVE32X); 128 EXT_KEY(ZVE32F); 129 EXT_KEY(ZVE64X); 130 EXT_KEY(ZVE64F); 131 EXT_KEY(ZVE64D); 132 EXT_KEY(ZVBB); 133 EXT_KEY(ZVBC); 134 EXT_KEY(ZVKB); 135 EXT_KEY(ZVKG); 136 EXT_KEY(ZVKNED); 137 EXT_KEY(ZVKNHA); 138 EXT_KEY(ZVKNHB); 139 EXT_KEY(ZVKSED); 140 EXT_KEY(ZVKSH); 141 EXT_KEY(ZVKT); 142 EXT_KEY(ZVFH); 143 EXT_KEY(ZVFHMIN); 144 } 145 146 if (has_fpu()) { 147 EXT_KEY(ZFH); 148 EXT_KEY(ZFHMIN); 149 EXT_KEY(ZFA); 150 EXT_KEY(ZCD); 151 EXT_KEY(ZCF); 152 } 153 #undef EXT_KEY 154 } 155 156 /* Now turn off reporting features if any CPU is missing it. */ 157 pair->value &= ~missing; 158 } 159 160 static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext) 161 { 162 struct riscv_hwprobe pair; 163 164 hwprobe_isa_ext0(&pair, cpus); 165 return (pair.value & ext); 166 } 167 168 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) 169 static u64 hwprobe_misaligned(const struct cpumask *cpus) 170 { 171 int cpu; 172 u64 perf = -1ULL; 173 174 for_each_cpu(cpu, cpus) { 175 int this_perf = per_cpu(misaligned_access_speed, cpu); 176 177 if (perf == -1ULL) 178 perf = this_perf; 179 180 if (perf != this_perf) { 181 perf = RISCV_HWPROBE_MISALIGNED_UNKNOWN; 182 break; 183 } 184 } 185 186 if (perf == -1ULL) 187 return RISCV_HWPROBE_MISALIGNED_UNKNOWN; 188 189 return perf; 190 } 191 #else 192 static u64 hwprobe_misaligned(const struct cpumask *cpus) 193 { 194 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS)) 195 return RISCV_HWPROBE_MISALIGNED_FAST; 196 197 if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available()) 198 return RISCV_HWPROBE_MISALIGNED_EMULATED; 199 200 return RISCV_HWPROBE_MISALIGNED_SLOW; 201 } 202 #endif 203 204 static void hwprobe_one_pair(struct riscv_hwprobe *pair, 205 const struct cpumask *cpus) 206 { 207 switch (pair->key) { 208 case RISCV_HWPROBE_KEY_MVENDORID: 209 case RISCV_HWPROBE_KEY_MARCHID: 210 case RISCV_HWPROBE_KEY_MIMPID: 211 hwprobe_arch_id(pair, cpus); 212 break; 213 /* 214 * The kernel already assumes that the base single-letter ISA 215 * extensions are supported on all harts, and only supports the 216 * IMA base, so just cheat a bit here and tell that to 217 * userspace. 218 */ 219 case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: 220 pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA; 221 break; 222 223 case RISCV_HWPROBE_KEY_IMA_EXT_0: 224 hwprobe_isa_ext0(pair, cpus); 225 break; 226 227 case RISCV_HWPROBE_KEY_CPUPERF_0: 228 pair->value = hwprobe_misaligned(cpus); 229 break; 230 231 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: 232 pair->value = 0; 233 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) 234 pair->value = riscv_cboz_block_size; 235 break; 236 case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS: 237 pair->value = user_max_virt_addr(); 238 break; 239 240 /* 241 * For forward compatibility, unknown keys don't fail the whole 242 * call, but get their element key set to -1 and value set to 0 243 * indicating they're unrecognized. 244 */ 245 default: 246 pair->key = -1; 247 pair->value = 0; 248 break; 249 } 250 } 251 252 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs, 253 size_t pair_count, size_t cpusetsize, 254 unsigned long __user *cpus_user, 255 unsigned int flags) 256 { 257 size_t out; 258 int ret; 259 cpumask_t cpus; 260 261 /* Check the reserved flags. */ 262 if (flags != 0) 263 return -EINVAL; 264 265 /* 266 * The interface supports taking in a CPU mask, and returns values that 267 * are consistent across that mask. Allow userspace to specify NULL and 268 * 0 as a shortcut to all online CPUs. 269 */ 270 cpumask_clear(&cpus); 271 if (!cpusetsize && !cpus_user) { 272 cpumask_copy(&cpus, cpu_online_mask); 273 } else { 274 if (cpusetsize > cpumask_size()) 275 cpusetsize = cpumask_size(); 276 277 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 278 if (ret) 279 return -EFAULT; 280 281 /* 282 * Userspace must provide at least one online CPU, without that 283 * there's no way to define what is supported. 284 */ 285 cpumask_and(&cpus, &cpus, cpu_online_mask); 286 if (cpumask_empty(&cpus)) 287 return -EINVAL; 288 } 289 290 for (out = 0; out < pair_count; out++, pairs++) { 291 struct riscv_hwprobe pair; 292 293 if (get_user(pair.key, &pairs->key)) 294 return -EFAULT; 295 296 pair.value = 0; 297 hwprobe_one_pair(&pair, &cpus); 298 ret = put_user(pair.key, &pairs->key); 299 if (ret == 0) 300 ret = put_user(pair.value, &pairs->value); 301 302 if (ret) 303 return -EFAULT; 304 } 305 306 return 0; 307 } 308 309 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, 310 size_t pair_count, size_t cpusetsize, 311 unsigned long __user *cpus_user, 312 unsigned int flags) 313 { 314 cpumask_t cpus, one_cpu; 315 bool clear_all = false; 316 size_t i; 317 int ret; 318 319 if (flags != RISCV_HWPROBE_WHICH_CPUS) 320 return -EINVAL; 321 322 if (!cpusetsize || !cpus_user) 323 return -EINVAL; 324 325 if (cpusetsize > cpumask_size()) 326 cpusetsize = cpumask_size(); 327 328 ret = copy_from_user(&cpus, cpus_user, cpusetsize); 329 if (ret) 330 return -EFAULT; 331 332 if (cpumask_empty(&cpus)) 333 cpumask_copy(&cpus, cpu_online_mask); 334 335 cpumask_and(&cpus, &cpus, cpu_online_mask); 336 337 cpumask_clear(&one_cpu); 338 339 for (i = 0; i < pair_count; i++) { 340 struct riscv_hwprobe pair, tmp; 341 int cpu; 342 343 ret = copy_from_user(&pair, &pairs[i], sizeof(pair)); 344 if (ret) 345 return -EFAULT; 346 347 if (!riscv_hwprobe_key_is_valid(pair.key)) { 348 clear_all = true; 349 pair = (struct riscv_hwprobe){ .key = -1, }; 350 ret = copy_to_user(&pairs[i], &pair, sizeof(pair)); 351 if (ret) 352 return -EFAULT; 353 } 354 355 if (clear_all) 356 continue; 357 358 tmp = (struct riscv_hwprobe){ .key = pair.key, }; 359 360 for_each_cpu(cpu, &cpus) { 361 cpumask_set_cpu(cpu, &one_cpu); 362 363 hwprobe_one_pair(&tmp, &one_cpu); 364 365 if (!riscv_hwprobe_pair_cmp(&tmp, &pair)) 366 cpumask_clear_cpu(cpu, &cpus); 367 368 cpumask_clear_cpu(cpu, &one_cpu); 369 } 370 } 371 372 if (clear_all) 373 cpumask_clear(&cpus); 374 375 ret = copy_to_user(cpus_user, &cpus, cpusetsize); 376 if (ret) 377 return -EFAULT; 378 379 return 0; 380 } 381 382 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, 383 size_t pair_count, size_t cpusetsize, 384 unsigned long __user *cpus_user, 385 unsigned int flags) 386 { 387 if (flags & RISCV_HWPROBE_WHICH_CPUS) 388 return hwprobe_get_cpus(pairs, pair_count, cpusetsize, 389 cpus_user, flags); 390 391 return hwprobe_get_values(pairs, pair_count, cpusetsize, 392 cpus_user, flags); 393 } 394 395 #ifdef CONFIG_MMU 396 397 static int __init init_hwprobe_vdso_data(void) 398 { 399 struct vdso_data *vd = __arch_get_k_vdso_data(); 400 struct arch_vdso_data *avd = &vd->arch_data; 401 u64 id_bitsmash = 0; 402 struct riscv_hwprobe pair; 403 int key; 404 405 /* 406 * Initialize vDSO data with the answers for the "all CPUs" case, to 407 * save a syscall in the common case. 408 */ 409 for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) { 410 pair.key = key; 411 hwprobe_one_pair(&pair, cpu_online_mask); 412 413 WARN_ON_ONCE(pair.key < 0); 414 415 avd->all_cpu_hwprobe_values[key] = pair.value; 416 /* 417 * Smash together the vendor, arch, and impl IDs to see if 418 * they're all 0 or any negative. 419 */ 420 if (key <= RISCV_HWPROBE_KEY_MIMPID) 421 id_bitsmash |= pair.value; 422 } 423 424 /* 425 * If the arch, vendor, and implementation ID are all the same across 426 * all harts, then assume all CPUs are the same, and allow the vDSO to 427 * answer queries for arbitrary masks. However if all values are 0 (not 428 * populated) or any value returns -1 (varies across CPUs), then the 429 * vDSO should defer to the kernel for exotic cpu masks. 430 */ 431 avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; 432 return 0; 433 } 434 435 arch_initcall_sync(init_hwprobe_vdso_data); 436 437 #endif /* CONFIG_MMU */ 438 439 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, 440 size_t, pair_count, size_t, cpusetsize, unsigned long __user *, 441 cpus, unsigned int, flags) 442 { 443 return do_riscv_hwprobe(pairs, pair_count, cpusetsize, 444 cpus, flags); 445 } 446