1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * The hwprobe interface, for allowing userspace to probe to see which features
4 * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for
5 * more details.
6 */
7 #include <linux/syscalls.h>
8 #include <asm/cacheflush.h>
9 #include <asm/cpufeature.h>
10 #include <asm/hwprobe.h>
11 #include <asm/processor.h>
12 #include <asm/delay.h>
13 #include <asm/sbi.h>
14 #include <asm/switch_to.h>
15 #include <asm/uaccess.h>
16 #include <asm/unistd.h>
17 #include <asm/vector.h>
18 #include <asm/vendor_extensions/mips_hwprobe.h>
19 #include <asm/vendor_extensions/sifive_hwprobe.h>
20 #include <asm/vendor_extensions/thead_hwprobe.h>
21 #include <vdso/vsyscall.h>
22
23
24 #define EXT_KEY(isa_arg, ext, pv, missing) \
25 do { \
26 if (__riscv_isa_extension_available(isa_arg, RISCV_ISA_EXT_##ext)) \
27 pv |= RISCV_HWPROBE_EXT_##ext; \
28 else \
29 missing |= RISCV_HWPROBE_EXT_##ext; \
30 } while (false)
31
hwprobe_arch_id(struct riscv_hwprobe * pair,const struct cpumask * cpus)32 static void hwprobe_arch_id(struct riscv_hwprobe *pair,
33 const struct cpumask *cpus)
34 {
35 u64 id = -1ULL;
36 bool first = true;
37 int cpu;
38
39 if (pair->key != RISCV_HWPROBE_KEY_MVENDORID &&
40 pair->key != RISCV_HWPROBE_KEY_MIMPID &&
41 pair->key != RISCV_HWPROBE_KEY_MARCHID)
42 goto out;
43
44 for_each_cpu(cpu, cpus) {
45 u64 cpu_id;
46
47 switch (pair->key) {
48 case RISCV_HWPROBE_KEY_MVENDORID:
49 cpu_id = riscv_cached_mvendorid(cpu);
50 break;
51 case RISCV_HWPROBE_KEY_MIMPID:
52 cpu_id = riscv_cached_mimpid(cpu);
53 break;
54 case RISCV_HWPROBE_KEY_MARCHID:
55 cpu_id = riscv_cached_marchid(cpu);
56 break;
57 }
58
59 if (first) {
60 id = cpu_id;
61 first = false;
62 }
63
64 /*
65 * If there's a mismatch for the given set, return -1 in the
66 * value.
67 */
68 if (id != cpu_id) {
69 id = -1ULL;
70 break;
71 }
72 }
73
74 out:
75 pair->value = id;
76 }
77
hwprobe_isa_ext0(struct riscv_hwprobe * pair,const struct cpumask * cpus)78 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
79 const struct cpumask *cpus)
80 {
81 int cpu;
82 u64 missing = 0;
83
84 pair->value = 0;
85 if (has_fpu())
86 pair->value |= RISCV_HWPROBE_IMA_FD;
87
88 if (riscv_isa_extension_available(NULL, C))
89 pair->value |= RISCV_HWPROBE_IMA_C;
90
91 if (has_vector() && riscv_isa_extension_available(NULL, V))
92 pair->value |= RISCV_HWPROBE_IMA_V;
93
94 /*
95 * Loop through and record extensions that 1) anyone has, and 2) anyone
96 * doesn't have.
97 */
98 for_each_cpu(cpu, cpus) {
99 struct riscv_isainfo *isainfo = &hart_isa[cpu];
100
101 /*
102 * Only use EXT_KEY() for extensions which can be exposed to userspace,
103 * regardless of the kernel's configuration, as no other checks, besides
104 * presence in the hart_isa bitmap, are made.
105 */
106 EXT_KEY(isainfo->isa, ZAAMO, pair->value, missing);
107 EXT_KEY(isainfo->isa, ZABHA, pair->value, missing);
108 EXT_KEY(isainfo->isa, ZACAS, pair->value, missing);
109 EXT_KEY(isainfo->isa, ZALASR, pair->value, missing);
110 EXT_KEY(isainfo->isa, ZALRSC, pair->value, missing);
111 EXT_KEY(isainfo->isa, ZAWRS, pair->value, missing);
112 EXT_KEY(isainfo->isa, ZBA, pair->value, missing);
113 EXT_KEY(isainfo->isa, ZBB, pair->value, missing);
114 EXT_KEY(isainfo->isa, ZBC, pair->value, missing);
115 EXT_KEY(isainfo->isa, ZBKB, pair->value, missing);
116 EXT_KEY(isainfo->isa, ZBKC, pair->value, missing);
117 EXT_KEY(isainfo->isa, ZBKX, pair->value, missing);
118 EXT_KEY(isainfo->isa, ZBS, pair->value, missing);
119 EXT_KEY(isainfo->isa, ZCA, pair->value, missing);
120 EXT_KEY(isainfo->isa, ZCB, pair->value, missing);
121 EXT_KEY(isainfo->isa, ZCLSD, pair->value, missing);
122 EXT_KEY(isainfo->isa, ZCMOP, pair->value, missing);
123 EXT_KEY(isainfo->isa, ZICBOM, pair->value, missing);
124 EXT_KEY(isainfo->isa, ZICBOP, pair->value, missing);
125 EXT_KEY(isainfo->isa, ZICBOZ, pair->value, missing);
126 EXT_KEY(isainfo->isa, ZICFILP, pair->value, missing);
127 EXT_KEY(isainfo->isa, ZICNTR, pair->value, missing);
128 EXT_KEY(isainfo->isa, ZICOND, pair->value, missing);
129 EXT_KEY(isainfo->isa, ZIHINTNTL, pair->value, missing);
130 EXT_KEY(isainfo->isa, ZIHINTPAUSE, pair->value, missing);
131 EXT_KEY(isainfo->isa, ZIHPM, pair->value, missing);
132 EXT_KEY(isainfo->isa, ZILSD, pair->value, missing);
133 EXT_KEY(isainfo->isa, ZIMOP, pair->value, missing);
134 EXT_KEY(isainfo->isa, ZKND, pair->value, missing);
135 EXT_KEY(isainfo->isa, ZKNE, pair->value, missing);
136 EXT_KEY(isainfo->isa, ZKNH, pair->value, missing);
137 EXT_KEY(isainfo->isa, ZKSED, pair->value, missing);
138 EXT_KEY(isainfo->isa, ZKSH, pair->value, missing);
139 EXT_KEY(isainfo->isa, ZKT, pair->value, missing);
140 EXT_KEY(isainfo->isa, ZTSO, pair->value, missing);
141
142 /*
143 * All the following extensions must depend on the kernel
144 * support of V.
145 */
146 if (has_vector()) {
147 EXT_KEY(isainfo->isa, ZVBB, pair->value, missing);
148 EXT_KEY(isainfo->isa, ZVBC, pair->value, missing);
149 EXT_KEY(isainfo->isa, ZVE32F, pair->value, missing);
150 EXT_KEY(isainfo->isa, ZVE32X, pair->value, missing);
151 EXT_KEY(isainfo->isa, ZVE64D, pair->value, missing);
152 EXT_KEY(isainfo->isa, ZVE64F, pair->value, missing);
153 EXT_KEY(isainfo->isa, ZVE64X, pair->value, missing);
154 EXT_KEY(isainfo->isa, ZVFBFMIN, pair->value, missing);
155 EXT_KEY(isainfo->isa, ZVFBFWMA, pair->value, missing);
156 EXT_KEY(isainfo->isa, ZVFH, pair->value, missing);
157 EXT_KEY(isainfo->isa, ZVFHMIN, pair->value, missing);
158 EXT_KEY(isainfo->isa, ZVKB, pair->value, missing);
159 EXT_KEY(isainfo->isa, ZVKG, pair->value, missing);
160 EXT_KEY(isainfo->isa, ZVKNED, pair->value, missing);
161 EXT_KEY(isainfo->isa, ZVKNHA, pair->value, missing);
162 EXT_KEY(isainfo->isa, ZVKNHB, pair->value, missing);
163 EXT_KEY(isainfo->isa, ZVKSED, pair->value, missing);
164 EXT_KEY(isainfo->isa, ZVKSH, pair->value, missing);
165 EXT_KEY(isainfo->isa, ZVKT, pair->value, missing);
166 }
167
168 EXT_KEY(isainfo->isa, ZCD, pair->value, missing);
169 EXT_KEY(isainfo->isa, ZCF, pair->value, missing);
170 EXT_KEY(isainfo->isa, ZFA, pair->value, missing);
171 EXT_KEY(isainfo->isa, ZFBFMIN, pair->value, missing);
172 EXT_KEY(isainfo->isa, ZFH, pair->value, missing);
173 EXT_KEY(isainfo->isa, ZFHMIN, pair->value, missing);
174
175 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM))
176 EXT_KEY(isainfo->isa, SUPM, pair->value, missing);
177 }
178
179 /* Now turn off reporting features if any CPU is missing it. */
180 pair->value &= ~missing;
181 }
182
hwprobe_isa_ext1(struct riscv_hwprobe * pair,const struct cpumask * cpus)183 static void hwprobe_isa_ext1(struct riscv_hwprobe *pair,
184 const struct cpumask *cpus)
185 {
186 int cpu;
187 u64 missing = 0;
188
189 pair->value = 0;
190
191 /*
192 * Loop through and record extensions that 1) anyone has, and 2) anyone
193 * doesn't have.
194 */
195 for_each_cpu(cpu, cpus) {
196 struct riscv_isainfo *isainfo = &hart_isa[cpu];
197
198 /*
199 * Only use EXT_KEY() for extensions which can be
200 * exposed to userspace, regardless of the kernel's
201 * configuration, as no other checks, besides presence
202 * in the hart_isa bitmap, are made.
203 */
204 EXT_KEY(isainfo->isa, ZICFISS, pair->value, missing);
205 EXT_KEY(isainfo->isa, ZICCLSM, pair->value, missing);
206 EXT_KEY(isainfo->isa, ZICCAMOA, pair->value, missing);
207 EXT_KEY(isainfo->isa, ZICCIF, pair->value, missing);
208 EXT_KEY(isainfo->isa, ZICCRSE, pair->value, missing);
209 EXT_KEY(isainfo->isa, ZA64RS, pair->value, missing);
210 }
211
212 /* Now turn off reporting features if any CPU is missing it. */
213 pair->value &= ~missing;
214 }
215
hwprobe_ext0_has(const struct cpumask * cpus,u64 ext)216 static bool hwprobe_ext0_has(const struct cpumask *cpus, u64 ext)
217 {
218 struct riscv_hwprobe pair;
219
220 hwprobe_isa_ext0(&pair, cpus);
221 return (pair.value & ext);
222 }
223
224 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
hwprobe_misaligned(const struct cpumask * cpus)225 static u64 hwprobe_misaligned(const struct cpumask *cpus)
226 {
227 int cpu;
228 u64 perf = -1ULL;
229
230 for_each_cpu(cpu, cpus) {
231 int this_perf = per_cpu(misaligned_access_speed, cpu);
232
233 if (perf == -1ULL)
234 perf = this_perf;
235
236 if (perf != this_perf) {
237 perf = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
238 break;
239 }
240 }
241
242 if (perf == -1ULL)
243 return RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
244
245 return perf;
246 }
247 #else
hwprobe_misaligned(const struct cpumask * cpus)248 static u64 hwprobe_misaligned(const struct cpumask *cpus)
249 {
250 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS))
251 return RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
252
253 if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available())
254 return RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
255
256 return RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
257 }
258 #endif
259
260 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
hwprobe_vec_misaligned(const struct cpumask * cpus)261 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
262 {
263 int cpu;
264 u64 perf = -1ULL;
265
266 /* Return if supported or not even if speed wasn't probed */
267 for_each_cpu(cpu, cpus) {
268 int this_perf = per_cpu(vector_misaligned_access, cpu);
269
270 if (perf == -1ULL)
271 perf = this_perf;
272
273 if (perf != this_perf) {
274 perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
275 break;
276 }
277 }
278
279 if (perf == -1ULL)
280 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
281
282 return perf;
283 }
284 #else
hwprobe_vec_misaligned(const struct cpumask * cpus)285 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
286 {
287 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS))
288 return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
289
290 if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS))
291 return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
292
293 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
294 }
295 #endif
296
hwprobe_one_pair(struct riscv_hwprobe * pair,const struct cpumask * cpus)297 static void hwprobe_one_pair(struct riscv_hwprobe *pair,
298 const struct cpumask *cpus)
299 {
300 switch (pair->key) {
301 case RISCV_HWPROBE_KEY_MVENDORID:
302 case RISCV_HWPROBE_KEY_MARCHID:
303 case RISCV_HWPROBE_KEY_MIMPID:
304 hwprobe_arch_id(pair, cpus);
305 break;
306 /*
307 * The kernel already assumes that the base single-letter ISA
308 * extensions are supported on all harts, and only supports the
309 * IMA base, so just cheat a bit here and tell that to
310 * userspace.
311 */
312 case RISCV_HWPROBE_KEY_BASE_BEHAVIOR:
313 pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA;
314 break;
315
316 case RISCV_HWPROBE_KEY_IMA_EXT_0:
317 hwprobe_isa_ext0(pair, cpus);
318 break;
319
320 case RISCV_HWPROBE_KEY_IMA_EXT_1:
321 hwprobe_isa_ext1(pair, cpus);
322 break;
323
324 case RISCV_HWPROBE_KEY_CPUPERF_0:
325 case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF:
326 pair->value = hwprobe_misaligned(cpus);
327 break;
328
329 case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF:
330 pair->value = hwprobe_vec_misaligned(cpus);
331 break;
332
333 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE:
334 pair->value = 0;
335 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))
336 pair->value = riscv_cboz_block_size;
337 break;
338 case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE:
339 pair->value = 0;
340 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM))
341 pair->value = riscv_cbom_block_size;
342 break;
343 case RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE:
344 pair->value = 0;
345 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOP))
346 pair->value = riscv_cbop_block_size;
347 break;
348 case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS:
349 pair->value = user_max_virt_addr();
350 break;
351
352 case RISCV_HWPROBE_KEY_TIME_CSR_FREQ:
353 pair->value = riscv_timebase;
354 break;
355
356 case RISCV_HWPROBE_KEY_VENDOR_EXT_SIFIVE_0:
357 hwprobe_isa_vendor_ext_sifive_0(pair, cpus);
358 break;
359
360 case RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0:
361 hwprobe_isa_vendor_ext_thead_0(pair, cpus);
362 break;
363 case RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0:
364 hwprobe_isa_vendor_ext_mips_0(pair, cpus);
365 break;
366
367 /*
368 * For forward compatibility, unknown keys don't fail the whole
369 * call, but get their element key set to -1 and value set to 0
370 * indicating they're unrecognized.
371 */
372 default:
373 pair->key = -1;
374 pair->value = 0;
375 break;
376 }
377 }
378
hwprobe_get_values(struct riscv_hwprobe __user * pairs,size_t pair_count,size_t cpusetsize,unsigned long __user * cpus_user,unsigned int flags)379 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs,
380 size_t pair_count, size_t cpusetsize,
381 unsigned long __user *cpus_user,
382 unsigned int flags)
383 {
384 size_t out;
385 int ret;
386 cpumask_t cpus;
387
388 /* Check the reserved flags. */
389 if (flags != 0)
390 return -EINVAL;
391
392 /*
393 * The interface supports taking in a CPU mask, and returns values that
394 * are consistent across that mask. Allow userspace to specify NULL and
395 * 0 as a shortcut to all online CPUs.
396 */
397 cpumask_clear(&cpus);
398 if (!cpusetsize && !cpus_user) {
399 cpumask_copy(&cpus, cpu_online_mask);
400 } else {
401 if (cpusetsize > cpumask_size())
402 cpusetsize = cpumask_size();
403
404 ret = copy_from_user(&cpus, cpus_user, cpusetsize);
405 if (ret)
406 return -EFAULT;
407
408 /*
409 * Userspace must provide at least one online CPU, without that
410 * there's no way to define what is supported.
411 */
412 cpumask_and(&cpus, &cpus, cpu_online_mask);
413 if (cpumask_empty(&cpus))
414 return -EINVAL;
415 }
416
417 for (out = 0; out < pair_count; out++, pairs++) {
418 struct riscv_hwprobe pair;
419
420 if (get_user(pair.key, &pairs->key))
421 return -EFAULT;
422
423 pair.value = 0;
424 hwprobe_one_pair(&pair, &cpus);
425 ret = put_user(pair.key, &pairs->key);
426 if (ret == 0)
427 ret = put_user(pair.value, &pairs->value);
428
429 if (ret)
430 return -EFAULT;
431 }
432
433 return 0;
434 }
435
hwprobe_get_cpus(struct riscv_hwprobe __user * pairs,size_t pair_count,size_t cpusetsize,unsigned long __user * cpus_user,unsigned int flags)436 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs,
437 size_t pair_count, size_t cpusetsize,
438 unsigned long __user *cpus_user,
439 unsigned int flags)
440 {
441 cpumask_t cpus, one_cpu;
442 bool clear_all = false;
443 size_t i;
444 int ret;
445
446 if (flags != RISCV_HWPROBE_WHICH_CPUS)
447 return -EINVAL;
448
449 if (!cpusetsize || !cpus_user)
450 return -EINVAL;
451
452 if (cpusetsize > cpumask_size())
453 cpusetsize = cpumask_size();
454
455 cpumask_clear(&cpus);
456 ret = copy_from_user(&cpus, cpus_user, cpusetsize);
457 if (ret)
458 return -EFAULT;
459
460 if (cpumask_empty(&cpus))
461 cpumask_copy(&cpus, cpu_online_mask);
462
463 cpumask_and(&cpus, &cpus, cpu_online_mask);
464
465 cpumask_clear(&one_cpu);
466
467 for (i = 0; i < pair_count; i++) {
468 struct riscv_hwprobe pair, tmp;
469 int cpu;
470
471 ret = copy_from_user(&pair, &pairs[i], sizeof(pair));
472 if (ret)
473 return -EFAULT;
474
475 if (!riscv_hwprobe_key_is_valid(pair.key)) {
476 clear_all = true;
477 pair = (struct riscv_hwprobe){ .key = -1, };
478 ret = copy_to_user(&pairs[i], &pair, sizeof(pair));
479 if (ret)
480 return -EFAULT;
481 }
482
483 if (clear_all)
484 continue;
485
486 tmp = (struct riscv_hwprobe){ .key = pair.key, };
487
488 for_each_cpu(cpu, &cpus) {
489 cpumask_set_cpu(cpu, &one_cpu);
490
491 hwprobe_one_pair(&tmp, &one_cpu);
492
493 if (!riscv_hwprobe_pair_cmp(&tmp, &pair))
494 cpumask_clear_cpu(cpu, &cpus);
495
496 cpumask_clear_cpu(cpu, &one_cpu);
497 }
498 }
499
500 if (clear_all)
501 cpumask_clear(&cpus);
502
503 ret = copy_to_user(cpus_user, &cpus, cpusetsize);
504 if (ret)
505 return -EFAULT;
506
507 return 0;
508 }
509
do_riscv_hwprobe(struct riscv_hwprobe __user * pairs,size_t pair_count,size_t cpusetsize,unsigned long __user * cpus_user,unsigned int flags)510 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs,
511 size_t pair_count, size_t cpusetsize,
512 unsigned long __user *cpus_user,
513 unsigned int flags)
514 {
515 if (flags & RISCV_HWPROBE_WHICH_CPUS)
516 return hwprobe_get_cpus(pairs, pair_count, cpusetsize,
517 cpus_user, flags);
518
519 return hwprobe_get_values(pairs, pair_count, cpusetsize,
520 cpus_user, flags);
521 }
522
523 #ifdef CONFIG_MMU
524
init_hwprobe_vdso_data(void)525 static int __init init_hwprobe_vdso_data(void)
526 {
527 struct vdso_arch_data *avd = vdso_k_arch_data;
528 u64 id_bitsmash = 0;
529 struct riscv_hwprobe pair;
530 int key;
531
532 /*
533 * Initialize vDSO data with the answers for the "all CPUs" case, to
534 * save a syscall in the common case.
535 */
536 for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) {
537 pair.key = key;
538 hwprobe_one_pair(&pair, cpu_online_mask);
539
540 WARN_ON_ONCE(pair.key < 0);
541
542 avd->all_cpu_hwprobe_values[key] = pair.value;
543 /*
544 * Smash together the vendor, arch, and impl IDs to see if
545 * they're all 0 or any negative.
546 */
547 if (key <= RISCV_HWPROBE_KEY_MIMPID)
548 id_bitsmash |= pair.value;
549 }
550
551 /*
552 * If the arch, vendor, and implementation ID are all the same across
553 * all harts, then assume all CPUs are the same, and allow the vDSO to
554 * answer queries for arbitrary masks. However if all values are 0 (not
555 * populated) or any value returns -1 (varies across CPUs), then the
556 * vDSO should defer to the kernel for exotic cpu masks.
557 */
558 avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1;
559 return 0;
560 }
561
562 arch_initcall_sync(init_hwprobe_vdso_data);
563
564 #endif /* CONFIG_MMU */
565
SYSCALL_DEFINE5(riscv_hwprobe,struct riscv_hwprobe __user *,pairs,size_t,pair_count,size_t,cpusetsize,unsigned long __user *,cpus,unsigned int,flags)566 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs,
567 size_t, pair_count, size_t, cpusetsize, unsigned long __user *,
568 cpus, unsigned int, flags)
569 {
570 return do_riscv_hwprobe(pairs, pair_count, cpusetsize,
571 cpus, flags);
572 }
573