1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 */ 5 6 #include <assert.h> 7 #include <pthread.h> 8 #include <stdint.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <time.h> 13 #include <unistd.h> 14 #include <signal.h> 15 #include <sys/auxv.h> 16 #include <sys/mman.h> 17 #include <sys/random.h> 18 #include <sys/syscall.h> 19 #include <sys/ptrace.h> 20 #include <sys/wait.h> 21 #include <sys/types.h> 22 #include <linux/random.h> 23 #include <linux/compiler.h> 24 #include <linux/ptrace.h> 25 26 #include "../kselftest.h" 27 #include "parse_vdso.h" 28 #include "vdso_config.h" 29 #include "vdso_call.h" 30 31 #ifndef timespecsub 32 #define timespecsub(tsp, usp, vsp) \ 33 do { \ 34 (vsp)->tv_sec = (tsp)->tv_sec - (usp)->tv_sec; \ 35 (vsp)->tv_nsec = (tsp)->tv_nsec - (usp)->tv_nsec; \ 36 if ((vsp)->tv_nsec < 0) { \ 37 (vsp)->tv_sec--; \ 38 (vsp)->tv_nsec += 1000000000L; \ 39 } \ 40 } while (0) 41 #endif 42 43 static struct { 44 pthread_mutex_t lock; 45 void **states; 46 size_t len, cap; 47 ssize_t(*fn)(void *, size_t, unsigned long, void *, size_t); 48 struct vgetrandom_opaque_params params; 49 } vgrnd = { 50 .lock = PTHREAD_MUTEX_INITIALIZER 51 }; 52 53 static void *vgetrandom_get_state(void) 54 { 55 void *state = NULL; 56 57 pthread_mutex_lock(&vgrnd.lock); 58 if (!vgrnd.len) { 59 size_t page_size = getpagesize(); 60 size_t new_cap; 61 size_t alloc_size, num = sysconf(_SC_NPROCESSORS_ONLN); /* Just a decent heuristic. */ 62 size_t state_size_aligned, cache_line_size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE) ?: 1; 63 void *new_block, *new_states; 64 65 state_size_aligned = (vgrnd.params.size_of_opaque_state + cache_line_size - 1) & (~(cache_line_size - 1)); 66 alloc_size = (num * state_size_aligned + page_size - 1) & (~(page_size - 1)); 67 num = (page_size / state_size_aligned) * (alloc_size / page_size); 68 new_block = mmap(0, alloc_size, vgrnd.params.mmap_prot, vgrnd.params.mmap_flags, -1, 0); 69 if (new_block == MAP_FAILED) 70 goto out; 71 72 new_cap = vgrnd.cap + num; 73 new_states = reallocarray(vgrnd.states, new_cap, sizeof(*vgrnd.states)); 74 if (!new_states) 75 goto unmap; 76 vgrnd.cap = new_cap; 77 vgrnd.states = new_states; 78 79 for (size_t i = 0; i < num; ++i) { 80 if (((uintptr_t)new_block & (page_size - 1)) + vgrnd.params.size_of_opaque_state > page_size) 81 new_block = (void *)(((uintptr_t)new_block + page_size - 1) & (~(page_size - 1))); 82 vgrnd.states[i] = new_block; 83 new_block += state_size_aligned; 84 } 85 vgrnd.len = num; 86 goto success; 87 88 unmap: 89 munmap(new_block, alloc_size); 90 goto out; 91 } 92 success: 93 state = vgrnd.states[--vgrnd.len]; 94 95 out: 96 pthread_mutex_unlock(&vgrnd.lock); 97 return state; 98 } 99 100 static void vgetrandom_put_state(void *state) 101 { 102 if (!state) 103 return; 104 pthread_mutex_lock(&vgrnd.lock); 105 vgrnd.states[vgrnd.len++] = state; 106 pthread_mutex_unlock(&vgrnd.lock); 107 } 108 109 static void vgetrandom_init(void) 110 { 111 const char *version = versions[VDSO_VERSION]; 112 const char *name = names[VDSO_NAMES][6]; 113 unsigned long sysinfo_ehdr = getauxval(AT_SYSINFO_EHDR); 114 size_t ret; 115 116 if (!sysinfo_ehdr) { 117 printf("AT_SYSINFO_EHDR is not present!\n"); 118 exit(KSFT_SKIP); 119 } 120 vdso_init_from_sysinfo_ehdr(sysinfo_ehdr); 121 vgrnd.fn = (__typeof__(vgrnd.fn))vdso_sym(version, name); 122 if (!vgrnd.fn) { 123 printf("%s is missing!\n", name); 124 exit(KSFT_FAIL); 125 } 126 ret = VDSO_CALL(vgrnd.fn, 5, NULL, 0, 0, &vgrnd.params, ~0UL); 127 if (ret == -ENOSYS) { 128 printf("unsupported architecture\n"); 129 exit(KSFT_SKIP); 130 } else if (ret) { 131 printf("failed to fetch vgetrandom params!\n"); 132 exit(KSFT_FAIL); 133 } 134 } 135 136 static ssize_t vgetrandom(void *buf, size_t len, unsigned long flags) 137 { 138 static __thread void *state; 139 140 if (!state) { 141 state = vgetrandom_get_state(); 142 if (!state) { 143 printf("vgetrandom_get_state failed!\n"); 144 exit(KSFT_FAIL); 145 } 146 } 147 return VDSO_CALL(vgrnd.fn, 5, buf, len, flags, state, vgrnd.params.size_of_opaque_state); 148 } 149 150 enum { TRIALS = 25000000, THREADS = 256 }; 151 152 static void *test_vdso_getrandom(void *ctx) 153 { 154 for (size_t i = 0; i < TRIALS; ++i) { 155 unsigned int val; 156 ssize_t ret = vgetrandom(&val, sizeof(val), 0); 157 assert(ret == sizeof(val)); 158 } 159 return NULL; 160 } 161 162 static void *test_libc_getrandom(void *ctx) 163 { 164 for (size_t i = 0; i < TRIALS; ++i) { 165 unsigned int val; 166 ssize_t ret = getrandom(&val, sizeof(val), 0); 167 assert(ret == sizeof(val)); 168 } 169 return NULL; 170 } 171 172 static void *test_syscall_getrandom(void *ctx) 173 { 174 for (size_t i = 0; i < TRIALS; ++i) { 175 unsigned int val; 176 ssize_t ret = syscall(__NR_getrandom, &val, sizeof(val), 0); 177 assert(ret == sizeof(val)); 178 } 179 return NULL; 180 } 181 182 static void bench_single(void) 183 { 184 struct timespec start, end, diff; 185 186 clock_gettime(CLOCK_MONOTONIC, &start); 187 test_vdso_getrandom(NULL); 188 clock_gettime(CLOCK_MONOTONIC, &end); 189 timespecsub(&end, &start, &diff); 190 printf(" vdso: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec); 191 192 clock_gettime(CLOCK_MONOTONIC, &start); 193 test_libc_getrandom(NULL); 194 clock_gettime(CLOCK_MONOTONIC, &end); 195 timespecsub(&end, &start, &diff); 196 printf(" libc: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec); 197 198 clock_gettime(CLOCK_MONOTONIC, &start); 199 test_syscall_getrandom(NULL); 200 clock_gettime(CLOCK_MONOTONIC, &end); 201 timespecsub(&end, &start, &diff); 202 printf("syscall: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec); 203 } 204 205 static void bench_multi(void) 206 { 207 struct timespec start, end, diff; 208 pthread_t threads[THREADS]; 209 210 clock_gettime(CLOCK_MONOTONIC, &start); 211 for (size_t i = 0; i < THREADS; ++i) 212 assert(pthread_create(&threads[i], NULL, test_vdso_getrandom, NULL) == 0); 213 for (size_t i = 0; i < THREADS; ++i) 214 pthread_join(threads[i], NULL); 215 clock_gettime(CLOCK_MONOTONIC, &end); 216 timespecsub(&end, &start, &diff); 217 printf(" vdso: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec); 218 219 clock_gettime(CLOCK_MONOTONIC, &start); 220 for (size_t i = 0; i < THREADS; ++i) 221 assert(pthread_create(&threads[i], NULL, test_libc_getrandom, NULL) == 0); 222 for (size_t i = 0; i < THREADS; ++i) 223 pthread_join(threads[i], NULL); 224 clock_gettime(CLOCK_MONOTONIC, &end); 225 timespecsub(&end, &start, &diff); 226 printf(" libc: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec); 227 228 clock_gettime(CLOCK_MONOTONIC, &start); 229 for (size_t i = 0; i < THREADS; ++i) 230 assert(pthread_create(&threads[i], NULL, test_syscall_getrandom, NULL) == 0); 231 for (size_t i = 0; i < THREADS; ++i) 232 pthread_join(threads[i], NULL); 233 clock_gettime(CLOCK_MONOTONIC, &end); 234 timespecsub(&end, &start, &diff); 235 printf(" syscall: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec); 236 } 237 238 static void fill(void) 239 { 240 uint8_t weird_size[323929]; 241 for (;;) 242 vgetrandom(weird_size, sizeof(weird_size), 0); 243 } 244 245 static void kselftest(void) 246 { 247 uint8_t weird_size[1263]; 248 pid_t child; 249 250 ksft_print_header(); 251 ksft_set_plan(2); 252 253 for (size_t i = 0; i < 1000; ++i) { 254 ssize_t ret = vgetrandom(weird_size, sizeof(weird_size), 0); 255 if (ret != sizeof(weird_size)) 256 exit(KSFT_FAIL); 257 } 258 259 ksft_test_result_pass("getrandom: PASS\n"); 260 261 unshare(CLONE_NEWUSER); 262 assert(unshare(CLONE_NEWTIME) == 0); 263 child = fork(); 264 assert(child >= 0); 265 if (!child) { 266 vgetrandom_init(); 267 child = getpid(); 268 assert(ptrace(PTRACE_TRACEME, 0, NULL, NULL) == 0); 269 assert(kill(child, SIGSTOP) == 0); 270 assert(vgetrandom(weird_size, sizeof(weird_size), 0) == sizeof(weird_size)); 271 _exit(0); 272 } 273 for (;;) { 274 struct ptrace_syscall_info info = { 0 }; 275 int status, ret; 276 assert(waitpid(child, &status, 0) >= 0); 277 if (WIFEXITED(status)) { 278 if (WEXITSTATUS(status) != 0) 279 exit(KSFT_FAIL); 280 break; 281 } 282 assert(WIFSTOPPED(status)); 283 if (WSTOPSIG(status) == SIGSTOP) 284 assert(ptrace(PTRACE_SETOPTIONS, child, 0, PTRACE_O_TRACESYSGOOD) == 0); 285 else if (WSTOPSIG(status) == (SIGTRAP | 0x80)) { 286 assert(ptrace(PTRACE_GET_SYSCALL_INFO, child, sizeof(info), &info) > 0); 287 if (info.op == PTRACE_SYSCALL_INFO_ENTRY && info.entry.nr == __NR_getrandom && 288 info.entry.args[0] == (uintptr_t)weird_size && info.entry.args[1] == sizeof(weird_size)) 289 exit(KSFT_FAIL); 290 } 291 assert(ptrace(PTRACE_SYSCALL, child, 0, 0) == 0); 292 } 293 294 ksft_test_result_pass("getrandom timens: PASS\n"); 295 296 exit(KSFT_PASS); 297 } 298 299 static void usage(const char *argv0) 300 { 301 fprintf(stderr, "Usage: %s [bench-single|bench-multi|fill]\n", argv0); 302 } 303 304 int main(int argc, char *argv[]) 305 { 306 vgetrandom_init(); 307 308 if (argc == 1) { 309 kselftest(); 310 return 0; 311 } 312 313 if (argc != 2) { 314 usage(argv[0]); 315 return 1; 316 } 317 if (!strcmp(argv[1], "bench-single")) 318 bench_single(); 319 else if (!strcmp(argv[1], "bench-multi")) 320 bench_multi(); 321 else if (!strcmp(argv[1], "fill")) 322 fill(); 323 else { 324 usage(argv[0]); 325 return 1; 326 } 327 return 0; 328 } 329