1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2023 Ventana Micro Systems Inc. 4 * 5 * Run with 'taskset -c <cpu-list> cbo' to only execute hwprobe on a 6 * subset of cpus, as well as only executing the tests on those cpus. 7 */ 8 #define _GNU_SOURCE 9 #include <stdbool.h> 10 #include <stdint.h> 11 #include <string.h> 12 #include <sched.h> 13 #include <signal.h> 14 #include <assert.h> 15 #include <linux/compiler.h> 16 #include <linux/kernel.h> 17 #include <asm/ucontext.h> 18 #include <getopt.h> 19 20 #include "hwprobe.h" 21 #include "kselftest.h" 22 23 #define MK_CBO(fn) le32_bswap((uint32_t)(fn) << 20 | 10 << 15 | 2 << 12 | 0 << 7 | 15) 24 #define MK_PREFETCH(fn) \ 25 le32_bswap(0 << 25 | (uint32_t)(fn) << 20 | 10 << 15 | 6 << 12 | 0 << 7 | 19) 26 27 static char mem[4096] __aligned(4096) = { [0 ... 4095] = 0xa5 }; 28 29 static bool got_fault; 30 31 static void fault_handler(int sig, siginfo_t *info, void *context) 32 { 33 unsigned long *regs = (unsigned long *)&((ucontext_t *)context)->uc_mcontext; 34 uint32_t insn = *(uint32_t *)regs[0]; 35 36 if (sig == SIGILL) 37 assert(insn == MK_CBO(regs[11])); 38 39 if (sig == SIGSEGV || sig == SIGBUS) 40 assert(insn == MK_PREFETCH(regs[11])); 41 42 got_fault = true; 43 regs[0] += 4; 44 } 45 46 #define cbo_insn(base, fn) \ 47 ({ \ 48 asm volatile( \ 49 "mv a0, %0\n" \ 50 "li a1, %1\n" \ 51 ".4byte %2\n" \ 52 : : "r" (base), "i" (fn), "i" (MK_CBO(fn)) : "a0", "a1", "memory"); \ 53 }) 54 55 #define prefetch_insn(base, fn) \ 56 ({ \ 57 asm volatile( \ 58 "mv a0, %0\n" \ 59 "li a1, %1\n" \ 60 ".4byte %2\n" \ 61 : : "r" (base), "i" (fn), "i" (MK_PREFETCH(fn)) : "a0", "a1"); \ 62 }) 63 64 static void cbo_inval(char *base) { cbo_insn(base, 0); } 65 static void cbo_clean(char *base) { cbo_insn(base, 1); } 66 static void cbo_flush(char *base) { cbo_insn(base, 2); } 67 static void cbo_zero(char *base) { cbo_insn(base, 4); } 68 static void prefetch_i(char *base) { prefetch_insn(base, 0); } 69 static void prefetch_r(char *base) { prefetch_insn(base, 1); } 70 static void prefetch_w(char *base) { prefetch_insn(base, 3); } 71 72 static void test_no_cbo_inval(void *arg) 73 { 74 ksft_print_msg("Testing cbo.inval instruction remain privileged\n"); 75 got_fault = false; 76 cbo_inval(&mem[0]); 77 ksft_test_result(got_fault, "No cbo.inval\n"); 78 } 79 80 static void test_no_zicbom(void *arg) 81 { 82 ksft_print_msg("Testing Zicbom instructions remain privileged\n"); 83 84 got_fault = false; 85 cbo_clean(&mem[0]); 86 ksft_test_result(got_fault, "No cbo.clean\n"); 87 88 got_fault = false; 89 cbo_flush(&mem[0]); 90 ksft_test_result(got_fault, "No cbo.flush\n"); 91 } 92 93 static void test_no_zicboz(void *arg) 94 { 95 ksft_print_msg("No Zicboz, testing cbo.zero remains privileged\n"); 96 97 got_fault = false; 98 cbo_zero(&mem[0]); 99 ksft_test_result(got_fault, "No cbo.zero\n"); 100 } 101 102 static bool is_power_of_2(__u64 n) 103 { 104 return n != 0 && (n & (n - 1)) == 0; 105 } 106 107 static void test_zicbop(void *arg) 108 { 109 struct riscv_hwprobe pair = { 110 .key = RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE, 111 }; 112 struct sigaction act = { 113 .sa_sigaction = &fault_handler, 114 .sa_flags = SA_SIGINFO 115 }; 116 struct sigaction dfl = { 117 .sa_handler = SIG_DFL 118 }; 119 cpu_set_t *cpus = (cpu_set_t *)arg; 120 __u64 block_size; 121 long rc; 122 123 rc = sigaction(SIGSEGV, &act, NULL); 124 assert(rc == 0); 125 rc = sigaction(SIGBUS, &act, NULL); 126 assert(rc == 0); 127 128 rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0); 129 block_size = pair.value; 130 ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE && 131 is_power_of_2(block_size), "Zicbop block size\n"); 132 ksft_print_msg("Zicbop block size: %llu\n", block_size); 133 134 got_fault = false; 135 prefetch_i(&mem[0]); 136 prefetch_r(&mem[0]); 137 prefetch_w(&mem[0]); 138 ksft_test_result(!got_fault, "Zicbop prefetch.* on valid address\n"); 139 140 got_fault = false; 141 prefetch_i(NULL); 142 prefetch_r(NULL); 143 prefetch_w(NULL); 144 ksft_test_result(!got_fault, "Zicbop prefetch.* on NULL\n"); 145 146 rc = sigaction(SIGBUS, &dfl, NULL); 147 assert(rc == 0); 148 rc = sigaction(SIGSEGV, &dfl, NULL); 149 assert(rc == 0); 150 } 151 152 static void test_zicbom(void *arg) 153 { 154 struct riscv_hwprobe pair = { 155 .key = RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE, 156 }; 157 cpu_set_t *cpus = (cpu_set_t *)arg; 158 __u64 block_size; 159 long rc; 160 161 rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0); 162 block_size = pair.value; 163 ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE && 164 is_power_of_2(block_size), "Zicbom block size\n"); 165 ksft_print_msg("Zicbom block size: %llu\n", block_size); 166 167 got_fault = false; 168 cbo_clean(&mem[block_size]); 169 ksft_test_result(!got_fault, "cbo.clean\n"); 170 171 got_fault = false; 172 cbo_flush(&mem[block_size]); 173 ksft_test_result(!got_fault, "cbo.flush\n"); 174 } 175 176 static void test_zicboz(void *arg) 177 { 178 struct riscv_hwprobe pair = { 179 .key = RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE, 180 }; 181 cpu_set_t *cpus = (cpu_set_t *)arg; 182 __u64 block_size; 183 int i, j; 184 long rc; 185 186 rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0); 187 block_size = pair.value; 188 ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE && 189 is_power_of_2(block_size), "Zicboz block size\n"); 190 ksft_print_msg("Zicboz block size: %llu\n", block_size); 191 192 got_fault = false; 193 cbo_zero(&mem[block_size]); 194 ksft_test_result(!got_fault, "cbo.zero\n"); 195 196 if (got_fault || !is_power_of_2(block_size)) { 197 ksft_test_result_skip("cbo.zero check\n"); 198 return; 199 } 200 201 assert(block_size <= 1024); 202 203 for (i = 0; i < 4096 / block_size; ++i) { 204 if (i % 2) 205 cbo_zero(&mem[i * block_size]); 206 } 207 208 for (i = 0; i < 4096 / block_size; ++i) { 209 char expected = i % 2 ? 0x0 : 0xa5; 210 211 for (j = 0; j < block_size; ++j) { 212 if (mem[i * block_size + j] != expected) { 213 ksft_test_result_fail("cbo.zero check\n"); 214 ksft_print_msg("cbo.zero check: mem[%llu] != 0x%x\n", 215 i * block_size + j, expected); 216 return; 217 } 218 } 219 } 220 221 ksft_test_result_pass("cbo.zero check\n"); 222 } 223 224 static void check_no_zicbo_cpus(cpu_set_t *cpus, __u64 cbo) 225 { 226 struct riscv_hwprobe pair = { 227 .key = RISCV_HWPROBE_KEY_IMA_EXT_0, 228 }; 229 cpu_set_t one_cpu; 230 int i = 0, c = 0; 231 long rc; 232 char *cbostr; 233 234 while (i++ < CPU_COUNT(cpus)) { 235 while (!CPU_ISSET(c, cpus)) 236 ++c; 237 238 CPU_ZERO(&one_cpu); 239 CPU_SET(c, &one_cpu); 240 241 rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)&one_cpu, 0); 242 assert(rc == 0 && pair.key == RISCV_HWPROBE_KEY_IMA_EXT_0); 243 244 switch (cbo) { 245 case RISCV_HWPROBE_EXT_ZICBOZ: 246 cbostr = "Zicboz"; 247 break; 248 case RISCV_HWPROBE_EXT_ZICBOM: 249 cbostr = "Zicbom"; 250 break; 251 case RISCV_HWPROBE_EXT_ZICBOP: 252 cbostr = "Zicbop"; 253 break; 254 default: 255 ksft_exit_fail_msg("Internal error: invalid cbo %llu\n", cbo); 256 } 257 258 if (pair.value & cbo) 259 ksft_exit_fail_msg("%s is only present on a subset of harts.\n" 260 "Use taskset to select a set of harts where %s\n" 261 "presence (present or not) is consistent for each hart\n", 262 cbostr, cbostr); 263 ++c; 264 } 265 } 266 267 enum { 268 TEST_ZICBOZ, 269 TEST_NO_ZICBOZ, 270 TEST_ZICBOM, 271 TEST_NO_ZICBOM, 272 TEST_NO_CBO_INVAL, 273 TEST_ZICBOP, 274 }; 275 276 static struct test_info { 277 bool enabled; 278 unsigned int nr_tests; 279 void (*test_fn)(void *arg); 280 } tests[] = { 281 [TEST_ZICBOZ] = { .nr_tests = 3, test_zicboz }, 282 [TEST_NO_ZICBOZ] = { .nr_tests = 1, test_no_zicboz }, 283 [TEST_ZICBOM] = { .nr_tests = 3, test_zicbom }, 284 [TEST_NO_ZICBOM] = { .nr_tests = 2, test_no_zicbom }, 285 [TEST_NO_CBO_INVAL] = { .nr_tests = 1, test_no_cbo_inval }, 286 [TEST_ZICBOP] = { .nr_tests = 3, test_zicbop }, 287 }; 288 289 static const struct option long_opts[] = { 290 {"zicbom-raises-sigill", no_argument, 0, 'm'}, 291 {"zicboz-raises-sigill", no_argument, 0, 'z'}, 292 {0, 0, 0, 0} 293 }; 294 295 int main(int argc, char **argv) 296 { 297 struct sigaction act = { 298 .sa_sigaction = &fault_handler, 299 .sa_flags = SA_SIGINFO, 300 }; 301 struct riscv_hwprobe pair; 302 unsigned int plan = 0; 303 cpu_set_t cpus; 304 long rc; 305 int i, opt, long_index; 306 307 long_index = 0; 308 309 while ((opt = getopt_long(argc, argv, "mz", long_opts, &long_index)) != -1) { 310 switch (opt) { 311 case 'm': 312 tests[TEST_NO_ZICBOM].enabled = true; 313 tests[TEST_NO_CBO_INVAL].enabled = true; 314 rc = sigaction(SIGILL, &act, NULL); 315 assert(rc == 0); 316 break; 317 case 'z': 318 tests[TEST_NO_ZICBOZ].enabled = true; 319 tests[TEST_NO_CBO_INVAL].enabled = true; 320 rc = sigaction(SIGILL, &act, NULL); 321 assert(rc == 0); 322 break; 323 case '?': 324 fprintf(stderr, 325 "Usage: %s [--zicbom-raises-sigill|-m] [--zicboz-raises-sigill|-z]\n", 326 argv[0]); 327 exit(1); 328 default: 329 break; 330 } 331 } 332 333 rc = sched_getaffinity(0, sizeof(cpu_set_t), &cpus); 334 assert(rc == 0); 335 336 ksft_print_header(); 337 338 pair.key = RISCV_HWPROBE_KEY_IMA_EXT_0; 339 rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)&cpus, 0); 340 if (rc < 0) 341 ksft_exit_fail_msg("hwprobe() failed with %ld\n", rc); 342 assert(rc == 0 && pair.key == RISCV_HWPROBE_KEY_IMA_EXT_0); 343 344 if (pair.value & RISCV_HWPROBE_EXT_ZICBOZ) { 345 tests[TEST_ZICBOZ].enabled = true; 346 tests[TEST_NO_ZICBOZ].enabled = false; 347 } else { 348 check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOZ); 349 } 350 351 if (pair.value & RISCV_HWPROBE_EXT_ZICBOM) { 352 tests[TEST_ZICBOM].enabled = true; 353 tests[TEST_NO_ZICBOM].enabled = false; 354 } else { 355 check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOM); 356 } 357 358 if (pair.value & RISCV_HWPROBE_EXT_ZICBOP) 359 tests[TEST_ZICBOP].enabled = true; 360 else 361 check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOP); 362 363 for (i = 0; i < ARRAY_SIZE(tests); ++i) 364 plan += tests[i].enabled ? tests[i].nr_tests : 0; 365 366 if (plan == 0) 367 ksft_print_msg("No tests enabled.\n"); 368 else 369 ksft_set_plan(plan); 370 371 for (i = 0; i < ARRAY_SIZE(tests); ++i) { 372 if (tests[i].enabled) 373 tests[i].test_fn(&cpus); 374 } 375 376 ksft_finished(); 377 } 378