1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * The hwprobe interface, for allowing userspace to probe to see which features
4 * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for
5 * more details.
6 */
7 #include <linux/syscalls.h>
8 #include <asm/cacheflush.h>
9 #include <asm/cpufeature.h>
10 #include <asm/hwprobe.h>
11 #include <asm/processor.h>
12 #include <asm/delay.h>
13 #include <asm/sbi.h>
14 #include <asm/switch_to.h>
15 #include <asm/uaccess.h>
16 #include <asm/unistd.h>
17 #include <asm/vector.h>
18 #include <asm/vendor_extensions/mips_hwprobe.h>
19 #include <asm/vendor_extensions/sifive_hwprobe.h>
20 #include <asm/vendor_extensions/thead_hwprobe.h>
21 #include <vdso/vsyscall.h>
22
23
hwprobe_arch_id(struct riscv_hwprobe * pair,const struct cpumask * cpus)24 static void hwprobe_arch_id(struct riscv_hwprobe *pair,
25 const struct cpumask *cpus)
26 {
27 u64 id = -1ULL;
28 bool first = true;
29 int cpu;
30
31 for_each_cpu(cpu, cpus) {
32 u64 cpu_id;
33
34 switch (pair->key) {
35 case RISCV_HWPROBE_KEY_MVENDORID:
36 cpu_id = riscv_cached_mvendorid(cpu);
37 break;
38 case RISCV_HWPROBE_KEY_MIMPID:
39 cpu_id = riscv_cached_mimpid(cpu);
40 break;
41 case RISCV_HWPROBE_KEY_MARCHID:
42 cpu_id = riscv_cached_marchid(cpu);
43 break;
44 }
45
46 if (first) {
47 id = cpu_id;
48 first = false;
49 }
50
51 /*
52 * If there's a mismatch for the given set, return -1 in the
53 * value.
54 */
55 if (id != cpu_id) {
56 id = -1ULL;
57 break;
58 }
59 }
60
61 pair->value = id;
62 }
63
hwprobe_isa_ext0(struct riscv_hwprobe * pair,const struct cpumask * cpus)64 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
65 const struct cpumask *cpus)
66 {
67 int cpu;
68 u64 missing = 0;
69
70 pair->value = 0;
71 if (has_fpu())
72 pair->value |= RISCV_HWPROBE_IMA_FD;
73
74 if (riscv_isa_extension_available(NULL, c))
75 pair->value |= RISCV_HWPROBE_IMA_C;
76
77 if (has_vector() && riscv_isa_extension_available(NULL, v))
78 pair->value |= RISCV_HWPROBE_IMA_V;
79
80 /*
81 * Loop through and record extensions that 1) anyone has, and 2) anyone
82 * doesn't have.
83 */
84 for_each_cpu(cpu, cpus) {
85 struct riscv_isainfo *isainfo = &hart_isa[cpu];
86
87 #define EXT_KEY(ext) \
88 do { \
89 if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \
90 pair->value |= RISCV_HWPROBE_EXT_##ext; \
91 else \
92 missing |= RISCV_HWPROBE_EXT_##ext; \
93 } while (false)
94
95 /*
96 * Only use EXT_KEY() for extensions which can be exposed to userspace,
97 * regardless of the kernel's configuration, as no other checks, besides
98 * presence in the hart_isa bitmap, are made.
99 */
100 EXT_KEY(ZAAMO);
101 EXT_KEY(ZABHA);
102 EXT_KEY(ZACAS);
103 EXT_KEY(ZALRSC);
104 EXT_KEY(ZAWRS);
105 EXT_KEY(ZBA);
106 EXT_KEY(ZBB);
107 EXT_KEY(ZBC);
108 EXT_KEY(ZBKB);
109 EXT_KEY(ZBKC);
110 EXT_KEY(ZBKX);
111 EXT_KEY(ZBS);
112 EXT_KEY(ZCA);
113 EXT_KEY(ZCB);
114 EXT_KEY(ZCMOP);
115 EXT_KEY(ZICBOM);
116 EXT_KEY(ZICBOZ);
117 EXT_KEY(ZICNTR);
118 EXT_KEY(ZICOND);
119 EXT_KEY(ZIHINTNTL);
120 EXT_KEY(ZIHINTPAUSE);
121 EXT_KEY(ZIHPM);
122 EXT_KEY(ZIMOP);
123 EXT_KEY(ZKND);
124 EXT_KEY(ZKNE);
125 EXT_KEY(ZKNH);
126 EXT_KEY(ZKSED);
127 EXT_KEY(ZKSH);
128 EXT_KEY(ZKT);
129 EXT_KEY(ZTSO);
130
131 /*
132 * All the following extensions must depend on the kernel
133 * support of V.
134 */
135 if (has_vector()) {
136 EXT_KEY(ZVBB);
137 EXT_KEY(ZVBC);
138 EXT_KEY(ZVE32F);
139 EXT_KEY(ZVE32X);
140 EXT_KEY(ZVE64D);
141 EXT_KEY(ZVE64F);
142 EXT_KEY(ZVE64X);
143 EXT_KEY(ZVFBFMIN);
144 EXT_KEY(ZVFBFWMA);
145 EXT_KEY(ZVFH);
146 EXT_KEY(ZVFHMIN);
147 EXT_KEY(ZVKB);
148 EXT_KEY(ZVKG);
149 EXT_KEY(ZVKNED);
150 EXT_KEY(ZVKNHA);
151 EXT_KEY(ZVKNHB);
152 EXT_KEY(ZVKSED);
153 EXT_KEY(ZVKSH);
154 EXT_KEY(ZVKT);
155 }
156
157 EXT_KEY(ZCD);
158 EXT_KEY(ZCF);
159 EXT_KEY(ZFA);
160 EXT_KEY(ZFBFMIN);
161 EXT_KEY(ZFH);
162 EXT_KEY(ZFHMIN);
163
164 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM))
165 EXT_KEY(SUPM);
166 #undef EXT_KEY
167 }
168
169 /* Now turn off reporting features if any CPU is missing it. */
170 pair->value &= ~missing;
171 }
172
hwprobe_ext0_has(const struct cpumask * cpus,u64 ext)173 static bool hwprobe_ext0_has(const struct cpumask *cpus, u64 ext)
174 {
175 struct riscv_hwprobe pair;
176
177 hwprobe_isa_ext0(&pair, cpus);
178 return (pair.value & ext);
179 }
180
181 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
hwprobe_misaligned(const struct cpumask * cpus)182 static u64 hwprobe_misaligned(const struct cpumask *cpus)
183 {
184 int cpu;
185 u64 perf = -1ULL;
186
187 for_each_cpu(cpu, cpus) {
188 int this_perf = per_cpu(misaligned_access_speed, cpu);
189
190 if (perf == -1ULL)
191 perf = this_perf;
192
193 if (perf != this_perf) {
194 perf = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
195 break;
196 }
197 }
198
199 if (perf == -1ULL)
200 return RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
201
202 return perf;
203 }
204 #else
hwprobe_misaligned(const struct cpumask * cpus)205 static u64 hwprobe_misaligned(const struct cpumask *cpus)
206 {
207 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS))
208 return RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
209
210 if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available())
211 return RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
212
213 return RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
214 }
215 #endif
216
217 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
hwprobe_vec_misaligned(const struct cpumask * cpus)218 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
219 {
220 int cpu;
221 u64 perf = -1ULL;
222
223 /* Return if supported or not even if speed wasn't probed */
224 for_each_cpu(cpu, cpus) {
225 int this_perf = per_cpu(vector_misaligned_access, cpu);
226
227 if (perf == -1ULL)
228 perf = this_perf;
229
230 if (perf != this_perf) {
231 perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
232 break;
233 }
234 }
235
236 if (perf == -1ULL)
237 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
238
239 return perf;
240 }
241 #else
hwprobe_vec_misaligned(const struct cpumask * cpus)242 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
243 {
244 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS))
245 return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
246
247 if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS))
248 return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
249
250 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
251 }
252 #endif
253
hwprobe_one_pair(struct riscv_hwprobe * pair,const struct cpumask * cpus)254 static void hwprobe_one_pair(struct riscv_hwprobe *pair,
255 const struct cpumask *cpus)
256 {
257 switch (pair->key) {
258 case RISCV_HWPROBE_KEY_MVENDORID:
259 case RISCV_HWPROBE_KEY_MARCHID:
260 case RISCV_HWPROBE_KEY_MIMPID:
261 hwprobe_arch_id(pair, cpus);
262 break;
263 /*
264 * The kernel already assumes that the base single-letter ISA
265 * extensions are supported on all harts, and only supports the
266 * IMA base, so just cheat a bit here and tell that to
267 * userspace.
268 */
269 case RISCV_HWPROBE_KEY_BASE_BEHAVIOR:
270 pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA;
271 break;
272
273 case RISCV_HWPROBE_KEY_IMA_EXT_0:
274 hwprobe_isa_ext0(pair, cpus);
275 break;
276
277 case RISCV_HWPROBE_KEY_CPUPERF_0:
278 case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF:
279 pair->value = hwprobe_misaligned(cpus);
280 break;
281
282 case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF:
283 pair->value = hwprobe_vec_misaligned(cpus);
284 break;
285
286 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE:
287 pair->value = 0;
288 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))
289 pair->value = riscv_cboz_block_size;
290 break;
291 case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE:
292 pair->value = 0;
293 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM))
294 pair->value = riscv_cbom_block_size;
295 break;
296 case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS:
297 pair->value = user_max_virt_addr();
298 break;
299
300 case RISCV_HWPROBE_KEY_TIME_CSR_FREQ:
301 pair->value = riscv_timebase;
302 break;
303
304 case RISCV_HWPROBE_KEY_VENDOR_EXT_SIFIVE_0:
305 hwprobe_isa_vendor_ext_sifive_0(pair, cpus);
306 break;
307
308 case RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0:
309 hwprobe_isa_vendor_ext_thead_0(pair, cpus);
310 break;
311 case RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0:
312 hwprobe_isa_vendor_ext_mips_0(pair, cpus);
313 break;
314
315 /*
316 * For forward compatibility, unknown keys don't fail the whole
317 * call, but get their element key set to -1 and value set to 0
318 * indicating they're unrecognized.
319 */
320 default:
321 pair->key = -1;
322 pair->value = 0;
323 break;
324 }
325 }
326
hwprobe_get_values(struct riscv_hwprobe __user * pairs,size_t pair_count,size_t cpusetsize,unsigned long __user * cpus_user,unsigned int flags)327 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs,
328 size_t pair_count, size_t cpusetsize,
329 unsigned long __user *cpus_user,
330 unsigned int flags)
331 {
332 size_t out;
333 int ret;
334 cpumask_t cpus;
335
336 /* Check the reserved flags. */
337 if (flags != 0)
338 return -EINVAL;
339
340 /*
341 * The interface supports taking in a CPU mask, and returns values that
342 * are consistent across that mask. Allow userspace to specify NULL and
343 * 0 as a shortcut to all online CPUs.
344 */
345 cpumask_clear(&cpus);
346 if (!cpusetsize && !cpus_user) {
347 cpumask_copy(&cpus, cpu_online_mask);
348 } else {
349 if (cpusetsize > cpumask_size())
350 cpusetsize = cpumask_size();
351
352 ret = copy_from_user(&cpus, cpus_user, cpusetsize);
353 if (ret)
354 return -EFAULT;
355
356 /*
357 * Userspace must provide at least one online CPU, without that
358 * there's no way to define what is supported.
359 */
360 cpumask_and(&cpus, &cpus, cpu_online_mask);
361 if (cpumask_empty(&cpus))
362 return -EINVAL;
363 }
364
365 for (out = 0; out < pair_count; out++, pairs++) {
366 struct riscv_hwprobe pair;
367
368 if (get_user(pair.key, &pairs->key))
369 return -EFAULT;
370
371 pair.value = 0;
372 hwprobe_one_pair(&pair, &cpus);
373 ret = put_user(pair.key, &pairs->key);
374 if (ret == 0)
375 ret = put_user(pair.value, &pairs->value);
376
377 if (ret)
378 return -EFAULT;
379 }
380
381 return 0;
382 }
383
hwprobe_get_cpus(struct riscv_hwprobe __user * pairs,size_t pair_count,size_t cpusetsize,unsigned long __user * cpus_user,unsigned int flags)384 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs,
385 size_t pair_count, size_t cpusetsize,
386 unsigned long __user *cpus_user,
387 unsigned int flags)
388 {
389 cpumask_t cpus, one_cpu;
390 bool clear_all = false;
391 size_t i;
392 int ret;
393
394 if (flags != RISCV_HWPROBE_WHICH_CPUS)
395 return -EINVAL;
396
397 if (!cpusetsize || !cpus_user)
398 return -EINVAL;
399
400 if (cpusetsize > cpumask_size())
401 cpusetsize = cpumask_size();
402
403 ret = copy_from_user(&cpus, cpus_user, cpusetsize);
404 if (ret)
405 return -EFAULT;
406
407 if (cpumask_empty(&cpus))
408 cpumask_copy(&cpus, cpu_online_mask);
409
410 cpumask_and(&cpus, &cpus, cpu_online_mask);
411
412 cpumask_clear(&one_cpu);
413
414 for (i = 0; i < pair_count; i++) {
415 struct riscv_hwprobe pair, tmp;
416 int cpu;
417
418 ret = copy_from_user(&pair, &pairs[i], sizeof(pair));
419 if (ret)
420 return -EFAULT;
421
422 if (!riscv_hwprobe_key_is_valid(pair.key)) {
423 clear_all = true;
424 pair = (struct riscv_hwprobe){ .key = -1, };
425 ret = copy_to_user(&pairs[i], &pair, sizeof(pair));
426 if (ret)
427 return -EFAULT;
428 }
429
430 if (clear_all)
431 continue;
432
433 tmp = (struct riscv_hwprobe){ .key = pair.key, };
434
435 for_each_cpu(cpu, &cpus) {
436 cpumask_set_cpu(cpu, &one_cpu);
437
438 hwprobe_one_pair(&tmp, &one_cpu);
439
440 if (!riscv_hwprobe_pair_cmp(&tmp, &pair))
441 cpumask_clear_cpu(cpu, &cpus);
442
443 cpumask_clear_cpu(cpu, &one_cpu);
444 }
445 }
446
447 if (clear_all)
448 cpumask_clear(&cpus);
449
450 ret = copy_to_user(cpus_user, &cpus, cpusetsize);
451 if (ret)
452 return -EFAULT;
453
454 return 0;
455 }
456
do_riscv_hwprobe(struct riscv_hwprobe __user * pairs,size_t pair_count,size_t cpusetsize,unsigned long __user * cpus_user,unsigned int flags)457 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs,
458 size_t pair_count, size_t cpusetsize,
459 unsigned long __user *cpus_user,
460 unsigned int flags)
461 {
462 if (flags & RISCV_HWPROBE_WHICH_CPUS)
463 return hwprobe_get_cpus(pairs, pair_count, cpusetsize,
464 cpus_user, flags);
465
466 return hwprobe_get_values(pairs, pair_count, cpusetsize,
467 cpus_user, flags);
468 }
469
470 #ifdef CONFIG_MMU
471
init_hwprobe_vdso_data(void)472 static int __init init_hwprobe_vdso_data(void)
473 {
474 struct vdso_arch_data *avd = vdso_k_arch_data;
475 u64 id_bitsmash = 0;
476 struct riscv_hwprobe pair;
477 int key;
478
479 /*
480 * Initialize vDSO data with the answers for the "all CPUs" case, to
481 * save a syscall in the common case.
482 */
483 for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) {
484 pair.key = key;
485 hwprobe_one_pair(&pair, cpu_online_mask);
486
487 WARN_ON_ONCE(pair.key < 0);
488
489 avd->all_cpu_hwprobe_values[key] = pair.value;
490 /*
491 * Smash together the vendor, arch, and impl IDs to see if
492 * they're all 0 or any negative.
493 */
494 if (key <= RISCV_HWPROBE_KEY_MIMPID)
495 id_bitsmash |= pair.value;
496 }
497
498 /*
499 * If the arch, vendor, and implementation ID are all the same across
500 * all harts, then assume all CPUs are the same, and allow the vDSO to
501 * answer queries for arbitrary masks. However if all values are 0 (not
502 * populated) or any value returns -1 (varies across CPUs), then the
503 * vDSO should defer to the kernel for exotic cpu masks.
504 */
505 avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1;
506 return 0;
507 }
508
509 arch_initcall_sync(init_hwprobe_vdso_data);
510
511 #endif /* CONFIG_MMU */
512
SYSCALL_DEFINE5(riscv_hwprobe,struct riscv_hwprobe __user *,pairs,size_t,pair_count,size_t,cpusetsize,unsigned long __user *,cpus,unsigned int,flags)513 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs,
514 size_t, pair_count, size_t, cpusetsize, unsigned long __user *,
515 cpus, unsigned int, flags)
516 {
517 return do_riscv_hwprobe(pairs, pair_count, cpusetsize,
518 cpus, flags);
519 }
520