1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2023 Ventana Micro Systems Inc.
4 *
5 * Run with 'taskset -c <cpu-list> cbo' to only execute hwprobe on a
6 * subset of cpus, as well as only executing the tests on those cpus.
7 */
8 #define _GNU_SOURCE
9 #include <stdbool.h>
10 #include <stdint.h>
11 #include <string.h>
12 #include <sched.h>
13 #include <signal.h>
14 #include <assert.h>
15 #include <linux/compiler.h>
16 #include <linux/kernel.h>
17 #include <asm/ucontext.h>
18 #include <getopt.h>
19
20 #include "hwprobe.h"
21 #include "kselftest.h"
22
23 #define MK_CBO(fn) le32_bswap((uint32_t)(fn) << 20 | 10 << 15 | 2 << 12 | 0 << 7 | 15)
24 #define MK_PREFETCH(fn) \
25 le32_bswap(0 << 25 | (uint32_t)(fn) << 20 | 10 << 15 | 6 << 12 | 0 << 7 | 19)
26
27 static char mem[4096] __aligned(4096) = { [0 ... 4095] = 0xa5 };
28
29 static bool got_fault;
30
fault_handler(int sig,siginfo_t * info,void * context)31 static void fault_handler(int sig, siginfo_t *info, void *context)
32 {
33 unsigned long *regs = (unsigned long *)&((ucontext_t *)context)->uc_mcontext;
34 uint32_t insn = *(uint32_t *)regs[0];
35
36 if (sig == SIGILL)
37 assert(insn == MK_CBO(regs[11]));
38
39 if (sig == SIGSEGV || sig == SIGBUS)
40 assert(insn == MK_PREFETCH(regs[11]));
41
42 got_fault = true;
43 regs[0] += 4;
44 }
45
46 #define cbo_insn(base, fn) \
47 ({ \
48 asm volatile( \
49 "mv a0, %0\n" \
50 "li a1, %1\n" \
51 ".4byte %2\n" \
52 : : "r" (base), "i" (fn), "i" (MK_CBO(fn)) : "a0", "a1", "memory"); \
53 })
54
55 #define prefetch_insn(base, fn) \
56 ({ \
57 asm volatile( \
58 "mv a0, %0\n" \
59 "li a1, %1\n" \
60 ".4byte %2\n" \
61 : : "r" (base), "i" (fn), "i" (MK_PREFETCH(fn)) : "a0", "a1"); \
62 })
63
cbo_inval(char * base)64 static void cbo_inval(char *base) { cbo_insn(base, 0); }
cbo_clean(char * base)65 static void cbo_clean(char *base) { cbo_insn(base, 1); }
cbo_flush(char * base)66 static void cbo_flush(char *base) { cbo_insn(base, 2); }
cbo_zero(char * base)67 static void cbo_zero(char *base) { cbo_insn(base, 4); }
prefetch_i(char * base)68 static void prefetch_i(char *base) { prefetch_insn(base, 0); }
prefetch_r(char * base)69 static void prefetch_r(char *base) { prefetch_insn(base, 1); }
prefetch_w(char * base)70 static void prefetch_w(char *base) { prefetch_insn(base, 3); }
71
test_no_cbo_inval(void * arg)72 static void test_no_cbo_inval(void *arg)
73 {
74 ksft_print_msg("Testing cbo.inval instruction remain privileged\n");
75 got_fault = false;
76 cbo_inval(&mem[0]);
77 ksft_test_result(got_fault, "No cbo.inval\n");
78 }
79
test_no_zicbom(void * arg)80 static void test_no_zicbom(void *arg)
81 {
82 ksft_print_msg("Testing Zicbom instructions remain privileged\n");
83
84 got_fault = false;
85 cbo_clean(&mem[0]);
86 ksft_test_result(got_fault, "No cbo.clean\n");
87
88 got_fault = false;
89 cbo_flush(&mem[0]);
90 ksft_test_result(got_fault, "No cbo.flush\n");
91 }
92
test_no_zicboz(void * arg)93 static void test_no_zicboz(void *arg)
94 {
95 ksft_print_msg("No Zicboz, testing cbo.zero remains privileged\n");
96
97 got_fault = false;
98 cbo_zero(&mem[0]);
99 ksft_test_result(got_fault, "No cbo.zero\n");
100 }
101
is_power_of_2(__u64 n)102 static bool is_power_of_2(__u64 n)
103 {
104 return n != 0 && (n & (n - 1)) == 0;
105 }
106
test_zicbop(void * arg)107 static void test_zicbop(void *arg)
108 {
109 struct riscv_hwprobe pair = {
110 .key = RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE,
111 };
112 struct sigaction act = {
113 .sa_sigaction = &fault_handler,
114 .sa_flags = SA_SIGINFO
115 };
116 struct sigaction dfl = {
117 .sa_handler = SIG_DFL
118 };
119 cpu_set_t *cpus = (cpu_set_t *)arg;
120 __u64 block_size;
121 long rc;
122
123 rc = sigaction(SIGSEGV, &act, NULL);
124 assert(rc == 0);
125 rc = sigaction(SIGBUS, &act, NULL);
126 assert(rc == 0);
127
128 rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0);
129 block_size = pair.value;
130 ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE &&
131 is_power_of_2(block_size), "Zicbop block size\n");
132 ksft_print_msg("Zicbop block size: %llu\n", block_size);
133
134 got_fault = false;
135 prefetch_i(&mem[0]);
136 prefetch_r(&mem[0]);
137 prefetch_w(&mem[0]);
138 ksft_test_result(!got_fault, "Zicbop prefetch.* on valid address\n");
139
140 got_fault = false;
141 prefetch_i(NULL);
142 prefetch_r(NULL);
143 prefetch_w(NULL);
144 ksft_test_result(!got_fault, "Zicbop prefetch.* on NULL\n");
145
146 rc = sigaction(SIGBUS, &dfl, NULL);
147 assert(rc == 0);
148 rc = sigaction(SIGSEGV, &dfl, NULL);
149 assert(rc == 0);
150 }
151
test_zicbom(void * arg)152 static void test_zicbom(void *arg)
153 {
154 struct riscv_hwprobe pair = {
155 .key = RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE,
156 };
157 cpu_set_t *cpus = (cpu_set_t *)arg;
158 __u64 block_size;
159 long rc;
160
161 rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0);
162 block_size = pair.value;
163 ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE &&
164 is_power_of_2(block_size), "Zicbom block size\n");
165 ksft_print_msg("Zicbom block size: %llu\n", block_size);
166
167 got_fault = false;
168 cbo_clean(&mem[block_size]);
169 ksft_test_result(!got_fault, "cbo.clean\n");
170
171 got_fault = false;
172 cbo_flush(&mem[block_size]);
173 ksft_test_result(!got_fault, "cbo.flush\n");
174 }
175
test_zicboz(void * arg)176 static void test_zicboz(void *arg)
177 {
178 struct riscv_hwprobe pair = {
179 .key = RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE,
180 };
181 cpu_set_t *cpus = (cpu_set_t *)arg;
182 __u64 block_size;
183 int i, j;
184 long rc;
185
186 rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0);
187 block_size = pair.value;
188 ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE &&
189 is_power_of_2(block_size), "Zicboz block size\n");
190 ksft_print_msg("Zicboz block size: %llu\n", block_size);
191
192 got_fault = false;
193 cbo_zero(&mem[block_size]);
194 ksft_test_result(!got_fault, "cbo.zero\n");
195
196 if (got_fault || !is_power_of_2(block_size)) {
197 ksft_test_result_skip("cbo.zero check\n");
198 return;
199 }
200
201 assert(block_size <= 1024);
202
203 for (i = 0; i < 4096 / block_size; ++i) {
204 if (i % 2)
205 cbo_zero(&mem[i * block_size]);
206 }
207
208 for (i = 0; i < 4096 / block_size; ++i) {
209 char expected = i % 2 ? 0x0 : 0xa5;
210
211 for (j = 0; j < block_size; ++j) {
212 if (mem[i * block_size + j] != expected) {
213 ksft_test_result_fail("cbo.zero check\n");
214 ksft_print_msg("cbo.zero check: mem[%llu] != 0x%x\n",
215 i * block_size + j, expected);
216 return;
217 }
218 }
219 }
220
221 ksft_test_result_pass("cbo.zero check\n");
222 }
223
check_no_zicbo_cpus(cpu_set_t * cpus,__u64 cbo)224 static void check_no_zicbo_cpus(cpu_set_t *cpus, __u64 cbo)
225 {
226 struct riscv_hwprobe pair = {
227 .key = RISCV_HWPROBE_KEY_IMA_EXT_0,
228 };
229 cpu_set_t one_cpu;
230 int i = 0, c = 0;
231 long rc;
232 char *cbostr;
233
234 while (i++ < CPU_COUNT(cpus)) {
235 while (!CPU_ISSET(c, cpus))
236 ++c;
237
238 CPU_ZERO(&one_cpu);
239 CPU_SET(c, &one_cpu);
240
241 rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)&one_cpu, 0);
242 assert(rc == 0 && pair.key == RISCV_HWPROBE_KEY_IMA_EXT_0);
243
244 switch (cbo) {
245 case RISCV_HWPROBE_EXT_ZICBOZ:
246 cbostr = "Zicboz";
247 break;
248 case RISCV_HWPROBE_EXT_ZICBOM:
249 cbostr = "Zicbom";
250 break;
251 case RISCV_HWPROBE_EXT_ZICBOP:
252 cbostr = "Zicbop";
253 break;
254 default:
255 ksft_exit_fail_msg("Internal error: invalid cbo %llu\n", cbo);
256 }
257
258 if (pair.value & cbo)
259 ksft_exit_fail_msg("%s is only present on a subset of harts.\n"
260 "Use taskset to select a set of harts where %s\n"
261 "presence (present or not) is consistent for each hart\n",
262 cbostr, cbostr);
263 ++c;
264 }
265 }
266
267 enum {
268 TEST_ZICBOZ,
269 TEST_NO_ZICBOZ,
270 TEST_ZICBOM,
271 TEST_NO_ZICBOM,
272 TEST_NO_CBO_INVAL,
273 TEST_ZICBOP,
274 };
275
276 static struct test_info {
277 bool enabled;
278 unsigned int nr_tests;
279 void (*test_fn)(void *arg);
280 } tests[] = {
281 [TEST_ZICBOZ] = { .nr_tests = 3, test_zicboz },
282 [TEST_NO_ZICBOZ] = { .nr_tests = 1, test_no_zicboz },
283 [TEST_ZICBOM] = { .nr_tests = 3, test_zicbom },
284 [TEST_NO_ZICBOM] = { .nr_tests = 2, test_no_zicbom },
285 [TEST_NO_CBO_INVAL] = { .nr_tests = 1, test_no_cbo_inval },
286 [TEST_ZICBOP] = { .nr_tests = 3, test_zicbop },
287 };
288
289 static const struct option long_opts[] = {
290 {"zicbom-raises-sigill", no_argument, 0, 'm'},
291 {"zicboz-raises-sigill", no_argument, 0, 'z'},
292 {0, 0, 0, 0}
293 };
294
main(int argc,char ** argv)295 int main(int argc, char **argv)
296 {
297 struct sigaction act = {
298 .sa_sigaction = &fault_handler,
299 .sa_flags = SA_SIGINFO,
300 };
301 struct riscv_hwprobe pair;
302 unsigned int plan = 0;
303 cpu_set_t cpus;
304 long rc;
305 int i, opt, long_index;
306
307 long_index = 0;
308
309 while ((opt = getopt_long(argc, argv, "mz", long_opts, &long_index)) != -1) {
310 switch (opt) {
311 case 'm':
312 tests[TEST_NO_ZICBOM].enabled = true;
313 tests[TEST_NO_CBO_INVAL].enabled = true;
314 rc = sigaction(SIGILL, &act, NULL);
315 assert(rc == 0);
316 break;
317 case 'z':
318 tests[TEST_NO_ZICBOZ].enabled = true;
319 tests[TEST_NO_CBO_INVAL].enabled = true;
320 rc = sigaction(SIGILL, &act, NULL);
321 assert(rc == 0);
322 break;
323 case '?':
324 fprintf(stderr,
325 "Usage: %s [--zicbom-raises-sigill|-m] [--zicboz-raises-sigill|-z]\n",
326 argv[0]);
327 exit(1);
328 default:
329 break;
330 }
331 }
332
333 rc = sched_getaffinity(0, sizeof(cpu_set_t), &cpus);
334 assert(rc == 0);
335
336 ksft_print_header();
337
338 pair.key = RISCV_HWPROBE_KEY_IMA_EXT_0;
339 rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)&cpus, 0);
340 if (rc < 0)
341 ksft_exit_fail_msg("hwprobe() failed with %ld\n", rc);
342 assert(rc == 0 && pair.key == RISCV_HWPROBE_KEY_IMA_EXT_0);
343
344 if (pair.value & RISCV_HWPROBE_EXT_ZICBOZ) {
345 tests[TEST_ZICBOZ].enabled = true;
346 tests[TEST_NO_ZICBOZ].enabled = false;
347 } else {
348 check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOZ);
349 }
350
351 if (pair.value & RISCV_HWPROBE_EXT_ZICBOM) {
352 tests[TEST_ZICBOM].enabled = true;
353 tests[TEST_NO_ZICBOM].enabled = false;
354 } else {
355 check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOM);
356 }
357
358 if (pair.value & RISCV_HWPROBE_EXT_ZICBOP)
359 tests[TEST_ZICBOP].enabled = true;
360 else
361 check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOP);
362
363 for (i = 0; i < ARRAY_SIZE(tests); ++i)
364 plan += tests[i].enabled ? tests[i].nr_tests : 0;
365
366 if (plan == 0)
367 ksft_print_msg("No tests enabled.\n");
368 else
369 ksft_set_plan(plan);
370
371 for (i = 0; i < ARRAY_SIZE(tests); ++i) {
372 if (tests[i].enabled)
373 tests[i].test_fn(&cpus);
374 }
375
376 ksft_finished();
377 }
378