xref: /linux/arch/riscv/kernel/sys_hwprobe.c (revision 3c2e0aff7b4f03fbc11b7d63c8db5b94a48978cf) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * The hwprobe interface, for allowing userspace to probe to see which features
4  * are supported by the hardware.  See Documentation/arch/riscv/hwprobe.rst for
5  * more details.
6  */
7 #include <linux/syscalls.h>
8 #include <asm/cacheflush.h>
9 #include <asm/cpufeature.h>
10 #include <asm/hwprobe.h>
11 #include <asm/processor.h>
12 #include <asm/delay.h>
13 #include <asm/sbi.h>
14 #include <asm/switch_to.h>
15 #include <asm/uaccess.h>
16 #include <asm/unistd.h>
17 #include <asm/vector.h>
18 #include <vdso/vsyscall.h>
19 
20 
21 static void hwprobe_arch_id(struct riscv_hwprobe *pair,
22 			    const struct cpumask *cpus)
23 {
24 	u64 id = -1ULL;
25 	bool first = true;
26 	int cpu;
27 
28 	for_each_cpu(cpu, cpus) {
29 		u64 cpu_id;
30 
31 		switch (pair->key) {
32 		case RISCV_HWPROBE_KEY_MVENDORID:
33 			cpu_id = riscv_cached_mvendorid(cpu);
34 			break;
35 		case RISCV_HWPROBE_KEY_MIMPID:
36 			cpu_id = riscv_cached_mimpid(cpu);
37 			break;
38 		case RISCV_HWPROBE_KEY_MARCHID:
39 			cpu_id = riscv_cached_marchid(cpu);
40 			break;
41 		}
42 
43 		if (first) {
44 			id = cpu_id;
45 			first = false;
46 		}
47 
48 		/*
49 		 * If there's a mismatch for the given set, return -1 in the
50 		 * value.
51 		 */
52 		if (id != cpu_id) {
53 			id = -1ULL;
54 			break;
55 		}
56 	}
57 
58 	pair->value = id;
59 }
60 
61 static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
62 			     const struct cpumask *cpus)
63 {
64 	int cpu;
65 	u64 missing = 0;
66 
67 	pair->value = 0;
68 	if (has_fpu())
69 		pair->value |= RISCV_HWPROBE_IMA_FD;
70 
71 	if (riscv_isa_extension_available(NULL, c))
72 		pair->value |= RISCV_HWPROBE_IMA_C;
73 
74 	if (has_vector() && riscv_isa_extension_available(NULL, v))
75 		pair->value |= RISCV_HWPROBE_IMA_V;
76 
77 	/*
78 	 * Loop through and record extensions that 1) anyone has, and 2) anyone
79 	 * doesn't have.
80 	 */
81 	for_each_cpu(cpu, cpus) {
82 		struct riscv_isainfo *isainfo = &hart_isa[cpu];
83 
84 #define EXT_KEY(ext)									\
85 	do {										\
86 		if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext))	\
87 			pair->value |= RISCV_HWPROBE_EXT_##ext;				\
88 		else									\
89 			missing |= RISCV_HWPROBE_EXT_##ext;				\
90 	} while (false)
91 
92 		/*
93 		 * Only use EXT_KEY() for extensions which can be exposed to userspace,
94 		 * regardless of the kernel's configuration, as no other checks, besides
95 		 * presence in the hart_isa bitmap, are made.
96 		 */
97 		EXT_KEY(ZACAS);
98 		EXT_KEY(ZAWRS);
99 		EXT_KEY(ZBA);
100 		EXT_KEY(ZBB);
101 		EXT_KEY(ZBC);
102 		EXT_KEY(ZBKB);
103 		EXT_KEY(ZBKC);
104 		EXT_KEY(ZBKX);
105 		EXT_KEY(ZBS);
106 		EXT_KEY(ZCA);
107 		EXT_KEY(ZCB);
108 		EXT_KEY(ZCMOP);
109 		EXT_KEY(ZICBOZ);
110 		EXT_KEY(ZICOND);
111 		EXT_KEY(ZIHINTNTL);
112 		EXT_KEY(ZIHINTPAUSE);
113 		EXT_KEY(ZIMOP);
114 		EXT_KEY(ZKND);
115 		EXT_KEY(ZKNE);
116 		EXT_KEY(ZKNH);
117 		EXT_KEY(ZKSED);
118 		EXT_KEY(ZKSH);
119 		EXT_KEY(ZKT);
120 		EXT_KEY(ZTSO);
121 
122 		/*
123 		 * All the following extensions must depend on the kernel
124 		 * support of V.
125 		 */
126 		if (has_vector()) {
127 			EXT_KEY(ZVBB);
128 			EXT_KEY(ZVBC);
129 			EXT_KEY(ZVE32F);
130 			EXT_KEY(ZVE32X);
131 			EXT_KEY(ZVE64D);
132 			EXT_KEY(ZVE64F);
133 			EXT_KEY(ZVE64X);
134 			EXT_KEY(ZVFH);
135 			EXT_KEY(ZVFHMIN);
136 			EXT_KEY(ZVKB);
137 			EXT_KEY(ZVKG);
138 			EXT_KEY(ZVKNED);
139 			EXT_KEY(ZVKNHA);
140 			EXT_KEY(ZVKNHB);
141 			EXT_KEY(ZVKSED);
142 			EXT_KEY(ZVKSH);
143 			EXT_KEY(ZVKT);
144 		}
145 
146 		if (has_fpu()) {
147 			EXT_KEY(ZCD);
148 			EXT_KEY(ZCF);
149 			EXT_KEY(ZFA);
150 			EXT_KEY(ZFH);
151 			EXT_KEY(ZFHMIN);
152 		}
153 
154 		if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM))
155 			EXT_KEY(SUPM);
156 #undef EXT_KEY
157 	}
158 
159 	/* Now turn off reporting features if any CPU is missing it. */
160 	pair->value &= ~missing;
161 }
162 
163 static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext)
164 {
165 	struct riscv_hwprobe pair;
166 
167 	hwprobe_isa_ext0(&pair, cpus);
168 	return (pair.value & ext);
169 }
170 
171 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
172 static u64 hwprobe_misaligned(const struct cpumask *cpus)
173 {
174 	int cpu;
175 	u64 perf = -1ULL;
176 
177 	for_each_cpu(cpu, cpus) {
178 		int this_perf = per_cpu(misaligned_access_speed, cpu);
179 
180 		if (perf == -1ULL)
181 			perf = this_perf;
182 
183 		if (perf != this_perf) {
184 			perf = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
185 			break;
186 		}
187 	}
188 
189 	if (perf == -1ULL)
190 		return RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
191 
192 	return perf;
193 }
194 #else
195 static u64 hwprobe_misaligned(const struct cpumask *cpus)
196 {
197 	if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS))
198 		return RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
199 
200 	if (IS_ENABLED(CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS) && unaligned_ctl_available())
201 		return RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
202 
203 	return RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
204 }
205 #endif
206 
207 static void hwprobe_one_pair(struct riscv_hwprobe *pair,
208 			     const struct cpumask *cpus)
209 {
210 	switch (pair->key) {
211 	case RISCV_HWPROBE_KEY_MVENDORID:
212 	case RISCV_HWPROBE_KEY_MARCHID:
213 	case RISCV_HWPROBE_KEY_MIMPID:
214 		hwprobe_arch_id(pair, cpus);
215 		break;
216 	/*
217 	 * The kernel already assumes that the base single-letter ISA
218 	 * extensions are supported on all harts, and only supports the
219 	 * IMA base, so just cheat a bit here and tell that to
220 	 * userspace.
221 	 */
222 	case RISCV_HWPROBE_KEY_BASE_BEHAVIOR:
223 		pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA;
224 		break;
225 
226 	case RISCV_HWPROBE_KEY_IMA_EXT_0:
227 		hwprobe_isa_ext0(pair, cpus);
228 		break;
229 
230 	case RISCV_HWPROBE_KEY_CPUPERF_0:
231 	case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF:
232 		pair->value = hwprobe_misaligned(cpus);
233 		break;
234 
235 	case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE:
236 		pair->value = 0;
237 		if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))
238 			pair->value = riscv_cboz_block_size;
239 		break;
240 	case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS:
241 		pair->value = user_max_virt_addr();
242 		break;
243 
244 	case RISCV_HWPROBE_KEY_TIME_CSR_FREQ:
245 		pair->value = riscv_timebase;
246 		break;
247 
248 	/*
249 	 * For forward compatibility, unknown keys don't fail the whole
250 	 * call, but get their element key set to -1 and value set to 0
251 	 * indicating they're unrecognized.
252 	 */
253 	default:
254 		pair->key = -1;
255 		pair->value = 0;
256 		break;
257 	}
258 }
259 
260 static int hwprobe_get_values(struct riscv_hwprobe __user *pairs,
261 			      size_t pair_count, size_t cpusetsize,
262 			      unsigned long __user *cpus_user,
263 			      unsigned int flags)
264 {
265 	size_t out;
266 	int ret;
267 	cpumask_t cpus;
268 
269 	/* Check the reserved flags. */
270 	if (flags != 0)
271 		return -EINVAL;
272 
273 	/*
274 	 * The interface supports taking in a CPU mask, and returns values that
275 	 * are consistent across that mask. Allow userspace to specify NULL and
276 	 * 0 as a shortcut to all online CPUs.
277 	 */
278 	cpumask_clear(&cpus);
279 	if (!cpusetsize && !cpus_user) {
280 		cpumask_copy(&cpus, cpu_online_mask);
281 	} else {
282 		if (cpusetsize > cpumask_size())
283 			cpusetsize = cpumask_size();
284 
285 		ret = copy_from_user(&cpus, cpus_user, cpusetsize);
286 		if (ret)
287 			return -EFAULT;
288 
289 		/*
290 		 * Userspace must provide at least one online CPU, without that
291 		 * there's no way to define what is supported.
292 		 */
293 		cpumask_and(&cpus, &cpus, cpu_online_mask);
294 		if (cpumask_empty(&cpus))
295 			return -EINVAL;
296 	}
297 
298 	for (out = 0; out < pair_count; out++, pairs++) {
299 		struct riscv_hwprobe pair;
300 
301 		if (get_user(pair.key, &pairs->key))
302 			return -EFAULT;
303 
304 		pair.value = 0;
305 		hwprobe_one_pair(&pair, &cpus);
306 		ret = put_user(pair.key, &pairs->key);
307 		if (ret == 0)
308 			ret = put_user(pair.value, &pairs->value);
309 
310 		if (ret)
311 			return -EFAULT;
312 	}
313 
314 	return 0;
315 }
316 
317 static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs,
318 			    size_t pair_count, size_t cpusetsize,
319 			    unsigned long __user *cpus_user,
320 			    unsigned int flags)
321 {
322 	cpumask_t cpus, one_cpu;
323 	bool clear_all = false;
324 	size_t i;
325 	int ret;
326 
327 	if (flags != RISCV_HWPROBE_WHICH_CPUS)
328 		return -EINVAL;
329 
330 	if (!cpusetsize || !cpus_user)
331 		return -EINVAL;
332 
333 	if (cpusetsize > cpumask_size())
334 		cpusetsize = cpumask_size();
335 
336 	ret = copy_from_user(&cpus, cpus_user, cpusetsize);
337 	if (ret)
338 		return -EFAULT;
339 
340 	if (cpumask_empty(&cpus))
341 		cpumask_copy(&cpus, cpu_online_mask);
342 
343 	cpumask_and(&cpus, &cpus, cpu_online_mask);
344 
345 	cpumask_clear(&one_cpu);
346 
347 	for (i = 0; i < pair_count; i++) {
348 		struct riscv_hwprobe pair, tmp;
349 		int cpu;
350 
351 		ret = copy_from_user(&pair, &pairs[i], sizeof(pair));
352 		if (ret)
353 			return -EFAULT;
354 
355 		if (!riscv_hwprobe_key_is_valid(pair.key)) {
356 			clear_all = true;
357 			pair = (struct riscv_hwprobe){ .key = -1, };
358 			ret = copy_to_user(&pairs[i], &pair, sizeof(pair));
359 			if (ret)
360 				return -EFAULT;
361 		}
362 
363 		if (clear_all)
364 			continue;
365 
366 		tmp = (struct riscv_hwprobe){ .key = pair.key, };
367 
368 		for_each_cpu(cpu, &cpus) {
369 			cpumask_set_cpu(cpu, &one_cpu);
370 
371 			hwprobe_one_pair(&tmp, &one_cpu);
372 
373 			if (!riscv_hwprobe_pair_cmp(&tmp, &pair))
374 				cpumask_clear_cpu(cpu, &cpus);
375 
376 			cpumask_clear_cpu(cpu, &one_cpu);
377 		}
378 	}
379 
380 	if (clear_all)
381 		cpumask_clear(&cpus);
382 
383 	ret = copy_to_user(cpus_user, &cpus, cpusetsize);
384 	if (ret)
385 		return -EFAULT;
386 
387 	return 0;
388 }
389 
390 static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs,
391 			    size_t pair_count, size_t cpusetsize,
392 			    unsigned long __user *cpus_user,
393 			    unsigned int flags)
394 {
395 	if (flags & RISCV_HWPROBE_WHICH_CPUS)
396 		return hwprobe_get_cpus(pairs, pair_count, cpusetsize,
397 					cpus_user, flags);
398 
399 	return hwprobe_get_values(pairs, pair_count, cpusetsize,
400 				  cpus_user, flags);
401 }
402 
403 #ifdef CONFIG_MMU
404 
405 static int __init init_hwprobe_vdso_data(void)
406 {
407 	struct vdso_data *vd = __arch_get_k_vdso_data();
408 	struct arch_vdso_data *avd = &vd->arch_data;
409 	u64 id_bitsmash = 0;
410 	struct riscv_hwprobe pair;
411 	int key;
412 
413 	/*
414 	 * Initialize vDSO data with the answers for the "all CPUs" case, to
415 	 * save a syscall in the common case.
416 	 */
417 	for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) {
418 		pair.key = key;
419 		hwprobe_one_pair(&pair, cpu_online_mask);
420 
421 		WARN_ON_ONCE(pair.key < 0);
422 
423 		avd->all_cpu_hwprobe_values[key] = pair.value;
424 		/*
425 		 * Smash together the vendor, arch, and impl IDs to see if
426 		 * they're all 0 or any negative.
427 		 */
428 		if (key <= RISCV_HWPROBE_KEY_MIMPID)
429 			id_bitsmash |= pair.value;
430 	}
431 
432 	/*
433 	 * If the arch, vendor, and implementation ID are all the same across
434 	 * all harts, then assume all CPUs are the same, and allow the vDSO to
435 	 * answer queries for arbitrary masks. However if all values are 0 (not
436 	 * populated) or any value returns -1 (varies across CPUs), then the
437 	 * vDSO should defer to the kernel for exotic cpu masks.
438 	 */
439 	avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1;
440 	return 0;
441 }
442 
443 arch_initcall_sync(init_hwprobe_vdso_data);
444 
445 #endif /* CONFIG_MMU */
446 
447 SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs,
448 		size_t, pair_count, size_t, cpusetsize, unsigned long __user *,
449 		cpus, unsigned int, flags)
450 {
451 	return do_riscv_hwprobe(pairs, pair_count, cpusetsize,
452 				cpus, flags);
453 }
454