1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 */ 5 6 #include <assert.h> 7 #include <pthread.h> 8 #include <stdint.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <time.h> 13 #include <unistd.h> 14 #include <sched.h> 15 #include <signal.h> 16 #include <sys/auxv.h> 17 #include <sys/mman.h> 18 #include <sys/random.h> 19 #include <sys/syscall.h> 20 #include <sys/ptrace.h> 21 #include <sys/wait.h> 22 #include <sys/types.h> 23 #include <linux/random.h> 24 #include <linux/compiler.h> 25 #include <linux/ptrace.h> 26 27 #include "../kselftest.h" 28 #include "parse_vdso.h" 29 #include "vdso_config.h" 30 #include "vdso_call.h" 31 32 #ifndef timespecsub 33 #define timespecsub(tsp, usp, vsp) \ 34 do { \ 35 (vsp)->tv_sec = (tsp)->tv_sec - (usp)->tv_sec; \ 36 (vsp)->tv_nsec = (tsp)->tv_nsec - (usp)->tv_nsec; \ 37 if ((vsp)->tv_nsec < 0) { \ 38 (vsp)->tv_sec--; \ 39 (vsp)->tv_nsec += 1000000000L; \ 40 } \ 41 } while (0) 42 #endif 43 44 #define ksft_assert(condition) \ 45 do { if (!(condition)) ksft_exit_fail_msg("Assertion failed: %s\n", #condition); } while (0) 46 47 static struct { 48 pthread_mutex_t lock; 49 void **states; 50 size_t len, cap; 51 ssize_t(*fn)(void *, size_t, unsigned long, void *, size_t); 52 struct vgetrandom_opaque_params params; 53 } vgrnd = { 54 .lock = PTHREAD_MUTEX_INITIALIZER 55 }; 56 57 static void *vgetrandom_get_state(void) 58 { 59 void *state = NULL; 60 61 pthread_mutex_lock(&vgrnd.lock); 62 if (!vgrnd.len) { 63 size_t page_size = getpagesize(); 64 size_t new_cap; 65 size_t alloc_size, num = sysconf(_SC_NPROCESSORS_ONLN); /* Just a decent heuristic. */ 66 size_t state_size_aligned, cache_line_size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE) ?: 1; 67 void *new_block, *new_states; 68 69 state_size_aligned = (vgrnd.params.size_of_opaque_state + cache_line_size - 1) & (~(cache_line_size - 1)); 70 alloc_size = (num * state_size_aligned + page_size - 1) & (~(page_size - 1)); 71 num = (page_size / state_size_aligned) * (alloc_size / page_size); 72 new_block = mmap(0, alloc_size, vgrnd.params.mmap_prot, vgrnd.params.mmap_flags, -1, 0); 73 if (new_block == MAP_FAILED) 74 goto out; 75 76 new_cap = vgrnd.cap + num; 77 new_states = reallocarray(vgrnd.states, new_cap, sizeof(*vgrnd.states)); 78 if (!new_states) 79 goto unmap; 80 vgrnd.cap = new_cap; 81 vgrnd.states = new_states; 82 83 for (size_t i = 0; i < num; ++i) { 84 if (((uintptr_t)new_block & (page_size - 1)) + vgrnd.params.size_of_opaque_state > page_size) 85 new_block = (void *)(((uintptr_t)new_block + page_size - 1) & (~(page_size - 1))); 86 vgrnd.states[i] = new_block; 87 new_block += state_size_aligned; 88 } 89 vgrnd.len = num; 90 goto success; 91 92 unmap: 93 munmap(new_block, alloc_size); 94 goto out; 95 } 96 success: 97 state = vgrnd.states[--vgrnd.len]; 98 99 out: 100 pthread_mutex_unlock(&vgrnd.lock); 101 return state; 102 } 103 104 static void vgetrandom_put_state(void *state) 105 { 106 if (!state) 107 return; 108 pthread_mutex_lock(&vgrnd.lock); 109 vgrnd.states[vgrnd.len++] = state; 110 pthread_mutex_unlock(&vgrnd.lock); 111 } 112 113 static void vgetrandom_init(void) 114 { 115 const char *version = versions[VDSO_VERSION]; 116 const char *name = names[VDSO_NAMES][6]; 117 unsigned long sysinfo_ehdr = getauxval(AT_SYSINFO_EHDR); 118 ssize_t ret; 119 120 if (!sysinfo_ehdr) 121 ksft_exit_skip("AT_SYSINFO_EHDR is not present\n"); 122 vdso_init_from_sysinfo_ehdr(sysinfo_ehdr); 123 vgrnd.fn = (__typeof__(vgrnd.fn))vdso_sym(version, name); 124 if (!vgrnd.fn) 125 ksft_exit_skip("%s@%s symbol is missing from vDSO\n", name, version); 126 ret = VDSO_CALL(vgrnd.fn, 5, NULL, 0, 0, &vgrnd.params, ~0UL); 127 if (ret == -ENOSYS) 128 ksft_exit_skip("CPU does not have runtime support\n"); 129 else if (ret) 130 ksft_exit_fail_msg("Failed to fetch vgetrandom params: %zd\n", ret); 131 } 132 133 static ssize_t vgetrandom(void *buf, size_t len, unsigned long flags) 134 { 135 static __thread void *state; 136 137 if (!state) { 138 state = vgetrandom_get_state(); 139 ksft_assert(state); 140 } 141 return VDSO_CALL(vgrnd.fn, 5, buf, len, flags, state, vgrnd.params.size_of_opaque_state); 142 } 143 144 enum { TRIALS = 25000000, THREADS = 256 }; 145 146 static void *test_vdso_getrandom(void *ctx) 147 { 148 for (size_t i = 0; i < TRIALS; ++i) { 149 unsigned int val; 150 ssize_t ret = vgetrandom(&val, sizeof(val), 0); 151 ksft_assert(ret == sizeof(val)); 152 } 153 return NULL; 154 } 155 156 static void *test_libc_getrandom(void *ctx) 157 { 158 for (size_t i = 0; i < TRIALS; ++i) { 159 unsigned int val; 160 ssize_t ret = getrandom(&val, sizeof(val), 0); 161 ksft_assert(ret == sizeof(val)); 162 } 163 return NULL; 164 } 165 166 static void *test_syscall_getrandom(void *ctx) 167 { 168 for (size_t i = 0; i < TRIALS; ++i) { 169 unsigned int val; 170 ssize_t ret = syscall(__NR_getrandom, &val, sizeof(val), 0); 171 ksft_assert(ret == sizeof(val)); 172 } 173 return NULL; 174 } 175 176 static void bench_single(void) 177 { 178 struct timespec start, end, diff; 179 180 clock_gettime(CLOCK_MONOTONIC, &start); 181 test_vdso_getrandom(NULL); 182 clock_gettime(CLOCK_MONOTONIC, &end); 183 timespecsub(&end, &start, &diff); 184 printf(" vdso: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec); 185 186 clock_gettime(CLOCK_MONOTONIC, &start); 187 test_libc_getrandom(NULL); 188 clock_gettime(CLOCK_MONOTONIC, &end); 189 timespecsub(&end, &start, &diff); 190 printf(" libc: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec); 191 192 clock_gettime(CLOCK_MONOTONIC, &start); 193 test_syscall_getrandom(NULL); 194 clock_gettime(CLOCK_MONOTONIC, &end); 195 timespecsub(&end, &start, &diff); 196 printf("syscall: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec); 197 } 198 199 static void bench_multi(void) 200 { 201 struct timespec start, end, diff; 202 pthread_t threads[THREADS]; 203 204 clock_gettime(CLOCK_MONOTONIC, &start); 205 for (size_t i = 0; i < THREADS; ++i) 206 ksft_assert(pthread_create(&threads[i], NULL, test_vdso_getrandom, NULL) == 0); 207 for (size_t i = 0; i < THREADS; ++i) 208 pthread_join(threads[i], NULL); 209 clock_gettime(CLOCK_MONOTONIC, &end); 210 timespecsub(&end, &start, &diff); 211 printf(" vdso: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec); 212 213 clock_gettime(CLOCK_MONOTONIC, &start); 214 for (size_t i = 0; i < THREADS; ++i) 215 ksft_assert(pthread_create(&threads[i], NULL, test_libc_getrandom, NULL) == 0); 216 for (size_t i = 0; i < THREADS; ++i) 217 pthread_join(threads[i], NULL); 218 clock_gettime(CLOCK_MONOTONIC, &end); 219 timespecsub(&end, &start, &diff); 220 printf(" libc: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec); 221 222 clock_gettime(CLOCK_MONOTONIC, &start); 223 for (size_t i = 0; i < THREADS; ++i) 224 ksft_assert(pthread_create(&threads[i], NULL, test_syscall_getrandom, NULL) == 0); 225 for (size_t i = 0; i < THREADS; ++i) 226 pthread_join(threads[i], NULL); 227 clock_gettime(CLOCK_MONOTONIC, &end); 228 timespecsub(&end, &start, &diff); 229 printf(" syscall: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec); 230 } 231 232 static void fill(void) 233 { 234 uint8_t weird_size[323929]; 235 for (;;) 236 vgetrandom(weird_size, sizeof(weird_size), 0); 237 } 238 239 static void kselftest(void) 240 { 241 uint8_t weird_size[1263]; 242 pid_t child; 243 244 ksft_print_header(); 245 ksft_set_plan(2); 246 247 for (size_t i = 0; i < 1000; ++i) { 248 ssize_t ret = vgetrandom(weird_size, sizeof(weird_size), 0); 249 ksft_assert(ret == sizeof(weird_size)); 250 } 251 252 ksft_test_result_pass("getrandom: PASS\n"); 253 254 unshare(CLONE_NEWUSER); 255 ksft_assert(unshare(CLONE_NEWTIME) == 0); 256 child = fork(); 257 ksft_assert(child >= 0); 258 if (!child) { 259 vgetrandom_init(); 260 child = getpid(); 261 ksft_assert(ptrace(PTRACE_TRACEME, 0, NULL, NULL) == 0); 262 ksft_assert(kill(child, SIGSTOP) == 0); 263 ksft_assert(vgetrandom(weird_size, sizeof(weird_size), 0) == sizeof(weird_size)); 264 _exit(0); 265 } 266 for (;;) { 267 struct ptrace_syscall_info info = { 0 }; 268 int status, ret; 269 ksft_assert(waitpid(child, &status, 0) >= 0); 270 if (WIFEXITED(status)) { 271 ksft_assert(WEXITSTATUS(status) == 0); 272 break; 273 } 274 ksft_assert(WIFSTOPPED(status)); 275 if (WSTOPSIG(status) == SIGSTOP) 276 ksft_assert(ptrace(PTRACE_SETOPTIONS, child, 0, PTRACE_O_TRACESYSGOOD) == 0); 277 else if (WSTOPSIG(status) == (SIGTRAP | 0x80)) { 278 ksft_assert(ptrace(PTRACE_GET_SYSCALL_INFO, child, sizeof(info), &info) > 0); 279 if (info.op == PTRACE_SYSCALL_INFO_ENTRY && info.entry.nr == __NR_getrandom && 280 info.entry.args[0] == (uintptr_t)weird_size && info.entry.args[1] == sizeof(weird_size)) 281 ksft_exit_fail_msg("vgetrandom passed buffer to syscall getrandom unexpectedly\n"); 282 } 283 ksft_assert(ptrace(PTRACE_SYSCALL, child, 0, 0) == 0); 284 } 285 286 ksft_test_result_pass("getrandom timens: PASS\n"); 287 288 ksft_exit_pass(); 289 } 290 291 static void usage(const char *argv0) 292 { 293 fprintf(stderr, "Usage: %s [bench-single|bench-multi|fill]\n", argv0); 294 } 295 296 int main(int argc, char *argv[]) 297 { 298 vgetrandom_init(); 299 300 if (argc == 1) { 301 kselftest(); 302 return 0; 303 } 304 305 if (argc != 2) { 306 usage(argv[0]); 307 return 1; 308 } 309 if (!strcmp(argv[1], "bench-single")) 310 bench_single(); 311 else if (!strcmp(argv[1], "bench-multi")) 312 bench_multi(); 313 else if (!strcmp(argv[1], "fill")) 314 fill(); 315 else { 316 usage(argv[0]); 317 return 1; 318 } 319 return 0; 320 } 321