xref: /linux/arch/riscv/kernel/sys_hwprobe.c (revision c1668520c9aa4019738f27097b187a5460646cbc)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * The hwprobe interface, for allowing userspace to probe to see which features
4  * are supported by the hardware.  See Documentation/arch/riscv/hwprobe.rst for
5  * more details.
6  */
7 #include <linux/syscalls.h>
8 #include <asm/cacheflush.h>
9 #include <asm/cpufeature.h>
10 #include <asm/hwprobe.h>
11 #include <asm/processor.h>
12 #include <asm/delay.h>
13 #include <asm/sbi.h>
14 #include <asm/switch_to.h>
15 #include <asm/uaccess.h>
16 #include <asm/unistd.h>
17 #include <asm/vector.h>
18 #include <vdso/vsyscall.h>
19 
20 
hwprobe_arch_id(struct riscv_hwprobe * pair,const struct cpumask * cpus)21 static void hwprobe_arch_id(struct riscv_hwprobe *pair,
22 			    const struct cpumask *cpus)
23 {
24 	u64 id = -1ULL;
25 	bool first = true;
26 	int cpu;
27 
28 	for_each_cpu(cpu, cpus) {
29 		u64 cpu_id;
30 
31 		switch (pair->key) {
32 		case RISCV_HWPROBE_KEY_MVENDORID:
33 			cpu_id = riscv_cached_mvendorid(cpu);
34 			break;
35 		case RISCV_HWPROBE_KEY_MIMPID:
36 			cpu_id = riscv_cached_mimpid(cpu);
37 			break;
38 		case RISCV_HWPROBE_KEY_MARCHID:
39 			cpu_id = riscv_cached_marchid(cpu);
40 			break;
41 		}
42 
43 		if (first) {
44 			id = cpu_id;
45 			first = false;
46 		}
47 
48 		/*
49 		 * If there's a mismatch for the given set, return -1 in the
50 		 * value.
51 		 */
52 		if (id != cpu_id) {
53 			id = -1ULL;
54 			break;
55 		}
56 	}
57 
58 	pair->value = id;
59 }
60 
hwprobe_isa_ext0(struct riscv_hwprobe * pair,const struct cpumask * cpus)61 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
62 			     const struct cpumask *cpus)
63 {
64 	int cpu;
65 	u64 missing = 0;
66 
67 	pair->value = 0;
68 	if (has_fpu())
69 		pair->value |= RISCV_HWPROBE_IMA_FD;
70 
71 	if (riscv_isa_extension_available(NULL, c))
72 		pair->value |= RISCV_HWPROBE_IMA_C;
73 
74 	if (has_vector() && riscv_isa_extension_available(NULL, v))
75 		pair->value |= RISCV_HWPROBE_IMA_V;
76 
77 	/*
78 	 * Loop through and record extensions that 1) anyone has, and 2) anyone
79 	 * doesn't have.
80 	 */
81 	for_each_cpu(cpu, cpus) {
82 		struct riscv_isainfo *isainfo = &hart_isa[cpu];
83 
84 #define EXT_KEY(ext)									\
85 	do {										\
86 		if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext))	\
87 			pair->value |= RISCV_HWPROBE_EXT_##ext;				\
88 		else									\
89 			missing |= RISCV_HWPROBE_EXT_##ext;				\
90 	} while (false)
91 
92 		/*
93 		 * Only use EXT_KEY() for extensions which can be exposed to userspace,
94 		 * regardless of the kernel's configuration, as no other checks, besides
95 		 * presence in the hart_isa bitmap, are made.
96 		 */
97 		EXT_KEY(ZACAS);
98 		EXT_KEY(ZAWRS);
99 		EXT_KEY(ZBA);
100 		EXT_KEY(ZBB);
101 		EXT_KEY(ZBC);
102 		EXT_KEY(ZBKB);
103 		EXT_KEY(ZBKC);
104 		EXT_KEY(ZBKX);
105 		EXT_KEY(ZBS);
106 		EXT_KEY(ZCA);
107 		EXT_KEY(ZCB);
108 		EXT_KEY(ZCMOP);
109 		EXT_KEY(ZICBOZ);
110 		EXT_KEY(ZICOND);
111 		EXT_KEY(ZIHINTNTL);
112 		EXT_KEY(ZIHINTPAUSE);
113 		EXT_KEY(ZIMOP);
114 		EXT_KEY(ZKND);
115 		EXT_KEY(ZKNE);
116 		EXT_KEY(ZKNH);
117 		EXT_KEY(ZKSED);
118 		EXT_KEY(ZKSH);
119 		EXT_KEY(ZKT);
120 		EXT_KEY(ZTSO);
121 
122 		/*
123 		 * All the following extensions must depend on the kernel
124 		 * support of V.
125 		 */
126 		if (has_vector()) {
127 			EXT_KEY(ZVBB);
128 			EXT_KEY(ZVBC);
129 			EXT_KEY(ZVE32F);
130 			EXT_KEY(ZVE32X);
131 			EXT_KEY(ZVE64D);
132 			EXT_KEY(ZVE64F);
133 			EXT_KEY(ZVE64X);
134 			EXT_KEY(ZVFH);
135 			EXT_KEY(ZVFHMIN);
136 			EXT_KEY(ZVKB);
137 			EXT_KEY(ZVKG);
138 			EXT_KEY(ZVKNED);
139 			EXT_KEY(ZVKNHA);
140 			EXT_KEY(ZVKNHB);
141 			EXT_KEY(ZVKSED);
142 			EXT_KEY(ZVKSH);
143 			EXT_KEY(ZVKT);
144 		}
145 
146 		if (has_fpu()) {
147 			EXT_KEY(ZCD);
148 			EXT_KEY(ZCF);
149 			EXT_KEY(ZFA);
150 			EXT_KEY(ZFH);
151 			EXT_KEY(ZFHMIN);
152 		}
153 
154 		if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM))
155 			EXT_KEY(SUPM);
156 #undef EXT_KEY
157 	}
158 
159 	/* Now turn off reporting features if any CPU is missing it. */
160 	pair->value &= ~missing;
161 }
162 
hwprobe_ext0_has(const struct cpumask * cpus,unsigned long ext)163 static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext)
164 {
165 	struct riscv_hwprobe pair;
166 
167 	hwprobe_isa_ext0(&pair, cpus);
168 	return (pair.value & ext);
169 }
170 
171 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
hwprobe_misaligned(const struct cpumask * cpus)172 static u64 hwprobe_misaligned(const struct cpumask *cpus)
173 {
174 	int cpu;
175 	u64 perf = -1ULL;
176 
177 	for_each_cpu(cpu, cpus) {
178 		int this_perf = per_cpu(misaligned_access_speed, cpu);
179 
180 		if (perf == -1ULL)
181 			perf = this_perf;
182 
183 		if (perf != this_perf) {
184 			perf = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
185 			break;
186 		}
187 	}
188 
189 	if (perf == -1ULL)
190 		return RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
191 
192 	return perf;
193 }
194 #else
hwprobe_misaligned(const struct cpumask * cpus)195 static u64 hwprobe_misaligned(const struct cpumask *cpus)
196 {
197 	if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS))
198 		return RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
199 
200 	if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available())
201 		return RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
202 
203 	return RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
204 }
205 #endif
206 
207 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
hwprobe_vec_misaligned(const struct cpumask * cpus)208 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
209 {
210 	int cpu;
211 	u64 perf = -1ULL;
212 
213 	/* Return if supported or not even if speed wasn't probed */
214 	for_each_cpu(cpu, cpus) {
215 		int this_perf = per_cpu(vector_misaligned_access, cpu);
216 
217 		if (perf == -1ULL)
218 			perf = this_perf;
219 
220 		if (perf != this_perf) {
221 			perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
222 			break;
223 		}
224 	}
225 
226 	if (perf == -1ULL)
227 		return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
228 
229 	return perf;
230 }
231 #else
hwprobe_vec_misaligned(const struct cpumask * cpus)232 static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
233 {
234 	if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS))
235 		return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
236 
237 	if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS))
238 		return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
239 
240 	return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
241 }
242 #endif
243 
hwprobe_one_pair(struct riscv_hwprobe * pair,const struct cpumask * cpus)244 static void hwprobe_one_pair(struct riscv_hwprobe *pair,
245 			     const struct cpumask *cpus)
246 {
247 	switch (pair->key) {
248 	case RISCV_HWPROBE_KEY_MVENDORID:
249 	case RISCV_HWPROBE_KEY_MARCHID:
250 	case RISCV_HWPROBE_KEY_MIMPID:
251 		hwprobe_arch_id(pair, cpus);
252 		break;
253 	/*
254 	 * The kernel already assumes that the base single-letter ISA
255 	 * extensions are supported on all harts, and only supports the
256 	 * IMA base, so just cheat a bit here and tell that to
257 	 * userspace.
258 	 */
259 	case RISCV_HWPROBE_KEY_BASE_BEHAVIOR:
260 		pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA;
261 		break;
262 
263 	case RISCV_HWPROBE_KEY_IMA_EXT_0:
264 		hwprobe_isa_ext0(pair, cpus);
265 		break;
266 
267 	case RISCV_HWPROBE_KEY_CPUPERF_0:
268 	case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF:
269 		pair->value = hwprobe_misaligned(cpus);
270 		break;
271 
272 	case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF:
273 		pair->value = hwprobe_vec_misaligned(cpus);
274 		break;
275 
276 	case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE:
277 		pair->value = 0;
278 		if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))
279 			pair->value = riscv_cboz_block_size;
280 		break;
281 	case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS:
282 		pair->value = user_max_virt_addr();
283 		break;
284 
285 	case RISCV_HWPROBE_KEY_TIME_CSR_FREQ:
286 		pair->value = riscv_timebase;
287 		break;
288 
289 	/*
290 	 * For forward compatibility, unknown keys don't fail the whole
291 	 * call, but get their element key set to -1 and value set to 0
292 	 * indicating they're unrecognized.
293 	 */
294 	default:
295 		pair->key = -1;
296 		pair->value = 0;
297 		break;
298 	}
299 }
300 
hwprobe_get_values(struct riscv_hwprobe __user * pairs,size_t pair_count,size_t cpusetsize,unsigned long __user * cpus_user,unsigned int flags)301 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs,
302 			      size_t pair_count, size_t cpusetsize,
303 			      unsigned long __user *cpus_user,
304 			      unsigned int flags)
305 {
306 	size_t out;
307 	int ret;
308 	cpumask_t cpus;
309 
310 	/* Check the reserved flags. */
311 	if (flags != 0)
312 		return -EINVAL;
313 
314 	/*
315 	 * The interface supports taking in a CPU mask, and returns values that
316 	 * are consistent across that mask. Allow userspace to specify NULL and
317 	 * 0 as a shortcut to all online CPUs.
318 	 */
319 	cpumask_clear(&cpus);
320 	if (!cpusetsize && !cpus_user) {
321 		cpumask_copy(&cpus, cpu_online_mask);
322 	} else {
323 		if (cpusetsize > cpumask_size())
324 			cpusetsize = cpumask_size();
325 
326 		ret = copy_from_user(&cpus, cpus_user, cpusetsize);
327 		if (ret)
328 			return -EFAULT;
329 
330 		/*
331 		 * Userspace must provide at least one online CPU, without that
332 		 * there's no way to define what is supported.
333 		 */
334 		cpumask_and(&cpus, &cpus, cpu_online_mask);
335 		if (cpumask_empty(&cpus))
336 			return -EINVAL;
337 	}
338 
339 	for (out = 0; out < pair_count; out++, pairs++) {
340 		struct riscv_hwprobe pair;
341 
342 		if (get_user(pair.key, &pairs->key))
343 			return -EFAULT;
344 
345 		pair.value = 0;
346 		hwprobe_one_pair(&pair, &cpus);
347 		ret = put_user(pair.key, &pairs->key);
348 		if (ret == 0)
349 			ret = put_user(pair.value, &pairs->value);
350 
351 		if (ret)
352 			return -EFAULT;
353 	}
354 
355 	return 0;
356 }
357 
hwprobe_get_cpus(struct riscv_hwprobe __user * pairs,size_t pair_count,size_t cpusetsize,unsigned long __user * cpus_user,unsigned int flags)358 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs,
359 			    size_t pair_count, size_t cpusetsize,
360 			    unsigned long __user *cpus_user,
361 			    unsigned int flags)
362 {
363 	cpumask_t cpus, one_cpu;
364 	bool clear_all = false;
365 	size_t i;
366 	int ret;
367 
368 	if (flags != RISCV_HWPROBE_WHICH_CPUS)
369 		return -EINVAL;
370 
371 	if (!cpusetsize || !cpus_user)
372 		return -EINVAL;
373 
374 	if (cpusetsize > cpumask_size())
375 		cpusetsize = cpumask_size();
376 
377 	ret = copy_from_user(&cpus, cpus_user, cpusetsize);
378 	if (ret)
379 		return -EFAULT;
380 
381 	if (cpumask_empty(&cpus))
382 		cpumask_copy(&cpus, cpu_online_mask);
383 
384 	cpumask_and(&cpus, &cpus, cpu_online_mask);
385 
386 	cpumask_clear(&one_cpu);
387 
388 	for (i = 0; i < pair_count; i++) {
389 		struct riscv_hwprobe pair, tmp;
390 		int cpu;
391 
392 		ret = copy_from_user(&pair, &pairs[i], sizeof(pair));
393 		if (ret)
394 			return -EFAULT;
395 
396 		if (!riscv_hwprobe_key_is_valid(pair.key)) {
397 			clear_all = true;
398 			pair = (struct riscv_hwprobe){ .key = -1, };
399 			ret = copy_to_user(&pairs[i], &pair, sizeof(pair));
400 			if (ret)
401 				return -EFAULT;
402 		}
403 
404 		if (clear_all)
405 			continue;
406 
407 		tmp = (struct riscv_hwprobe){ .key = pair.key, };
408 
409 		for_each_cpu(cpu, &cpus) {
410 			cpumask_set_cpu(cpu, &one_cpu);
411 
412 			hwprobe_one_pair(&tmp, &one_cpu);
413 
414 			if (!riscv_hwprobe_pair_cmp(&tmp, &pair))
415 				cpumask_clear_cpu(cpu, &cpus);
416 
417 			cpumask_clear_cpu(cpu, &one_cpu);
418 		}
419 	}
420 
421 	if (clear_all)
422 		cpumask_clear(&cpus);
423 
424 	ret = copy_to_user(cpus_user, &cpus, cpusetsize);
425 	if (ret)
426 		return -EFAULT;
427 
428 	return 0;
429 }
430 
do_riscv_hwprobe(struct riscv_hwprobe __user * pairs,size_t pair_count,size_t cpusetsize,unsigned long __user * cpus_user,unsigned int flags)431 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs,
432 			    size_t pair_count, size_t cpusetsize,
433 			    unsigned long __user *cpus_user,
434 			    unsigned int flags)
435 {
436 	if (flags & RISCV_HWPROBE_WHICH_CPUS)
437 		return hwprobe_get_cpus(pairs, pair_count, cpusetsize,
438 					cpus_user, flags);
439 
440 	return hwprobe_get_values(pairs, pair_count, cpusetsize,
441 				  cpus_user, flags);
442 }
443 
444 #ifdef CONFIG_MMU
445 
init_hwprobe_vdso_data(void)446 static int __init init_hwprobe_vdso_data(void)
447 {
448 	struct vdso_data *vd = __arch_get_k_vdso_data();
449 	struct arch_vdso_time_data *avd = &vd->arch_data;
450 	u64 id_bitsmash = 0;
451 	struct riscv_hwprobe pair;
452 	int key;
453 
454 	/*
455 	 * Initialize vDSO data with the answers for the "all CPUs" case, to
456 	 * save a syscall in the common case.
457 	 */
458 	for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) {
459 		pair.key = key;
460 		hwprobe_one_pair(&pair, cpu_online_mask);
461 
462 		WARN_ON_ONCE(pair.key < 0);
463 
464 		avd->all_cpu_hwprobe_values[key] = pair.value;
465 		/*
466 		 * Smash together the vendor, arch, and impl IDs to see if
467 		 * they're all 0 or any negative.
468 		 */
469 		if (key <= RISCV_HWPROBE_KEY_MIMPID)
470 			id_bitsmash |= pair.value;
471 	}
472 
473 	/*
474 	 * If the arch, vendor, and implementation ID are all the same across
475 	 * all harts, then assume all CPUs are the same, and allow the vDSO to
476 	 * answer queries for arbitrary masks. However if all values are 0 (not
477 	 * populated) or any value returns -1 (varies across CPUs), then the
478 	 * vDSO should defer to the kernel for exotic cpu masks.
479 	 */
480 	avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1;
481 	return 0;
482 }
483 
484 arch_initcall_sync(init_hwprobe_vdso_data);
485 
486 #endif /* CONFIG_MMU */
487 
SYSCALL_DEFINE5(riscv_hwprobe,struct riscv_hwprobe __user *,pairs,size_t,pair_count,size_t,cpusetsize,unsigned long __user *,cpus,unsigned int,flags)488 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs,
489 		size_t, pair_count, size_t, cpusetsize, unsigned long __user *,
490 		cpus, unsigned int, flags)
491 {
492 	return do_riscv_hwprobe(pairs, pair_count, cpusetsize,
493 				cpus, flags);
494 }
495