1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Tests Memory Protection Keys (see Documentation/core-api/protection-keys.rst) 4 * 5 * There are examples in here of: 6 * * how to set protection keys on memory 7 * * how to set/clear bits in pkey registers (the rights register) 8 * * how to handle SEGV_PKUERR signals and extract pkey-relevant 9 * information from the siginfo 10 * 11 * Things to add: 12 * make sure KSM and KSM COW breaking works 13 * prefault pages in at malloc, or not 14 * protect MPX bounds tables with protection keys? 15 * make sure VMA splitting/merging is working correctly 16 * OOMs can destroy mm->mmap (see exit_mmap()), so make sure it is immune to pkeys 17 * look for pkey "leaks" where it is still set on a VMA but "freed" back to the kernel 18 * do a plain mprotect() to a mprotect_pkey() area and make sure the pkey sticks 19 * 20 * Compile like this: 21 * gcc -mxsave -o protection_keys -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm 22 * gcc -mxsave -m32 -o protection_keys_32 -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm 23 */ 24 #define _GNU_SOURCE 25 #define __SANE_USERSPACE_TYPES__ 26 #include <errno.h> 27 #include <linux/elf.h> 28 #include <linux/futex.h> 29 #include <time.h> 30 #include <sys/time.h> 31 #include <sys/syscall.h> 32 #include <string.h> 33 #include <stdio.h> 34 #include <stdint.h> 35 #include <stdbool.h> 36 #include <signal.h> 37 #include <assert.h> 38 #include <stdlib.h> 39 #include <ucontext.h> 40 #include <sys/mman.h> 41 #include <sys/types.h> 42 #include <sys/wait.h> 43 #include <sys/stat.h> 44 #include <fcntl.h> 45 #include <asm-generic/unistd.h> 46 #include <sys/ptrace.h> 47 #include <setjmp.h> 48 49 #include "pkey-helpers.h" 50 51 int iteration_nr = 1; 52 int test_nr; 53 54 u64 shadow_pkey_reg; 55 int dprint_in_signal; 56 57 noinline int read_ptr(int *ptr) 58 { 59 /* Keep GCC from optimizing this away somehow */ 60 barrier(); 61 return *ptr; 62 } 63 64 static void cat_into_file(char *str, char *file) 65 { 66 int fd = open(file, O_RDWR); 67 int ret; 68 69 dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file); 70 /* 71 * these need to be raw because they are called under 72 * pkey_assert() 73 */ 74 if (fd < 0) { 75 fprintf(stderr, "error opening '%s'\n", str); 76 perror("error: "); 77 exit(__LINE__); 78 } 79 80 ret = write(fd, str, strlen(str)); 81 if (ret != strlen(str)) { 82 perror("write to file failed"); 83 fprintf(stderr, "filename: '%s' str: '%s'\n", file, str); 84 exit(__LINE__); 85 } 86 close(fd); 87 } 88 89 #if CONTROL_TRACING > 0 90 static int warned_tracing; 91 static int tracing_root_ok(void) 92 { 93 if (geteuid() != 0) { 94 if (!warned_tracing) 95 fprintf(stderr, "WARNING: not run as root, " 96 "can not do tracing control\n"); 97 warned_tracing = 1; 98 return 0; 99 } 100 return 1; 101 } 102 #endif 103 104 static void tracing_on(void) 105 { 106 #if CONTROL_TRACING > 0 107 #define TRACEDIR "/sys/kernel/tracing" 108 char pidstr[32]; 109 110 if (!tracing_root_ok()) 111 return; 112 113 sprintf(pidstr, "%d", getpid()); 114 cat_into_file("0", TRACEDIR "/tracing_on"); 115 cat_into_file("\n", TRACEDIR "/trace"); 116 if (1) { 117 cat_into_file("function_graph", TRACEDIR "/current_tracer"); 118 cat_into_file("1", TRACEDIR "/options/funcgraph-proc"); 119 } else { 120 cat_into_file("nop", TRACEDIR "/current_tracer"); 121 } 122 cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid"); 123 cat_into_file("1", TRACEDIR "/tracing_on"); 124 dprintf1("enabled tracing\n"); 125 #endif 126 } 127 128 static void tracing_off(void) 129 { 130 #if CONTROL_TRACING > 0 131 if (!tracing_root_ok()) 132 return; 133 cat_into_file("0", "/sys/kernel/tracing/tracing_on"); 134 #endif 135 } 136 137 void abort_hooks(void) 138 { 139 fprintf(stderr, "running %s()...\n", __func__); 140 tracing_off(); 141 #ifdef SLEEP_ON_ABORT 142 sleep(SLEEP_ON_ABORT); 143 #endif 144 } 145 146 /* 147 * This attempts to have roughly a page of instructions followed by a few 148 * instructions that do a write, and another page of instructions. That 149 * way, we are pretty sure that the write is in the second page of 150 * instructions and has at least a page of padding behind it. 151 * 152 * *That* lets us be sure to madvise() away the write instruction, which 153 * will then fault, which makes sure that the fault code handles 154 * execute-only memory properly. 155 */ 156 #if defined(__powerpc64__) || defined(__aarch64__) 157 /* This way, both 4K and 64K alignment are maintained */ 158 __attribute__((__aligned__(65536))) 159 #else 160 __attribute__((__aligned__(PAGE_SIZE))) 161 #endif 162 static void lots_o_noops_around_write(int *write_to_me) 163 { 164 dprintf3("running %s()\n", __func__); 165 __page_o_noops(); 166 /* Assume this happens in the second page of instructions: */ 167 *write_to_me = __LINE__; 168 /* pad out by another page: */ 169 __page_o_noops(); 170 dprintf3("%s() done\n", __func__); 171 } 172 173 static void dump_mem(void *dumpme, int len_bytes) 174 { 175 char *c = (void *)dumpme; 176 int i; 177 178 for (i = 0; i < len_bytes; i += sizeof(u64)) { 179 u64 *ptr = (u64 *)(c + i); 180 dprintf1("dump[%03d][@%p]: %016llx\n", i, ptr, *ptr); 181 } 182 } 183 184 static u32 hw_pkey_get(int pkey, unsigned long flags) 185 { 186 u64 pkey_reg = __read_pkey_reg(); 187 188 dprintf1("%s(pkey=%d, flags=%lx) = %x / %d\n", 189 __func__, pkey, flags, 0, 0); 190 dprintf2("%s() raw pkey_reg: %016llx\n", __func__, pkey_reg); 191 192 return (u32) get_pkey_bits(pkey_reg, pkey); 193 } 194 195 static int hw_pkey_set(int pkey, unsigned long rights, unsigned long flags) 196 { 197 u32 mask = (PKEY_DISABLE_ACCESS|PKEY_DISABLE_WRITE); 198 u64 old_pkey_reg = __read_pkey_reg(); 199 u64 new_pkey_reg; 200 201 /* make sure that 'rights' only contains the bits we expect: */ 202 assert(!(rights & ~mask)); 203 204 /* modify bits accordingly in old pkey_reg and assign it */ 205 new_pkey_reg = set_pkey_bits(old_pkey_reg, pkey, rights); 206 207 __write_pkey_reg(new_pkey_reg); 208 209 dprintf3("%s(pkey=%d, rights=%lx, flags=%lx) = %x" 210 " pkey_reg now: %016llx old_pkey_reg: %016llx\n", 211 __func__, pkey, rights, flags, 0, __read_pkey_reg(), 212 old_pkey_reg); 213 return 0; 214 } 215 216 static void pkey_disable_set(int pkey, int flags) 217 { 218 unsigned long syscall_flags = 0; 219 int ret; 220 int pkey_rights; 221 222 dprintf1("START->%s(%d, 0x%x)\n", __func__, 223 pkey, flags); 224 pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE)); 225 226 pkey_rights = hw_pkey_get(pkey, syscall_flags); 227 228 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__, 229 pkey, pkey, pkey_rights); 230 231 pkey_assert(pkey_rights >= 0); 232 233 pkey_rights |= flags; 234 235 ret = hw_pkey_set(pkey, pkey_rights, syscall_flags); 236 assert(!ret); 237 /* pkey_reg and flags have the same format */ 238 shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, pkey, pkey_rights); 239 dprintf1("%s(%d) shadow: 0x%016llx\n", 240 __func__, pkey, shadow_pkey_reg); 241 242 pkey_assert(ret >= 0); 243 244 pkey_rights = hw_pkey_get(pkey, syscall_flags); 245 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__, 246 pkey, pkey, pkey_rights); 247 248 dprintf1("%s(%d) pkey_reg: 0x%016llx\n", 249 __func__, pkey, read_pkey_reg()); 250 dprintf1("END<---%s(%d, 0x%x)\n", __func__, 251 pkey, flags); 252 } 253 254 static void pkey_disable_clear(int pkey, int flags) 255 { 256 unsigned long syscall_flags = 0; 257 int ret; 258 int pkey_rights = hw_pkey_get(pkey, syscall_flags); 259 260 pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE)); 261 262 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__, 263 pkey, pkey, pkey_rights); 264 pkey_assert(pkey_rights >= 0); 265 266 pkey_rights &= ~flags; 267 268 ret = hw_pkey_set(pkey, pkey_rights, 0); 269 shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, pkey, pkey_rights); 270 pkey_assert(ret >= 0); 271 272 pkey_rights = hw_pkey_get(pkey, syscall_flags); 273 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__, 274 pkey, pkey, pkey_rights); 275 276 dprintf1("%s(%d) pkey_reg: 0x%016llx\n", __func__, 277 pkey, read_pkey_reg()); 278 } 279 280 __maybe_unused static void pkey_write_allow(int pkey) 281 { 282 pkey_disable_clear(pkey, PKEY_DISABLE_WRITE); 283 } 284 __maybe_unused static void pkey_write_deny(int pkey) 285 { 286 pkey_disable_set(pkey, PKEY_DISABLE_WRITE); 287 } 288 __maybe_unused static void pkey_access_allow(int pkey) 289 { 290 pkey_disable_clear(pkey, PKEY_DISABLE_ACCESS); 291 } 292 __maybe_unused static void pkey_access_deny(int pkey) 293 { 294 pkey_disable_set(pkey, PKEY_DISABLE_ACCESS); 295 } 296 297 static char *si_code_str(int si_code) 298 { 299 if (si_code == SEGV_MAPERR) 300 return "SEGV_MAPERR"; 301 if (si_code == SEGV_ACCERR) 302 return "SEGV_ACCERR"; 303 if (si_code == SEGV_BNDERR) 304 return "SEGV_BNDERR"; 305 if (si_code == SEGV_PKUERR) 306 return "SEGV_PKUERR"; 307 return "UNKNOWN"; 308 } 309 310 static int pkey_faults; 311 static int last_si_pkey = -1; 312 static void signal_handler(int signum, siginfo_t *si, void *vucontext) 313 { 314 ucontext_t *uctxt = vucontext; 315 int trapno; 316 unsigned long ip; 317 #ifdef MCONTEXT_FPREGS 318 char *fpregs; 319 #endif 320 #if defined(__i386__) || defined(__x86_64__) /* arch */ 321 u32 *pkey_reg_ptr; 322 int pkey_reg_offset; 323 #endif /* arch */ 324 u64 siginfo_pkey; 325 u32 *si_pkey_ptr; 326 327 dprint_in_signal = 1; 328 dprintf1(">>>>===============SIGSEGV============================\n"); 329 dprintf1("%s()::%d, pkey_reg: 0x%016llx shadow: %016llx\n", 330 __func__, __LINE__, 331 __read_pkey_reg(), shadow_pkey_reg); 332 333 trapno = MCONTEXT_TRAPNO(uctxt->uc_mcontext); 334 ip = MCONTEXT_IP(uctxt->uc_mcontext); 335 #ifdef MCONTEXT_FPREGS 336 fpregs = (char *) uctxt->uc_mcontext.fpregs; 337 #endif 338 339 dprintf2("%s() trapno: %d ip: 0x%016lx info->si_code: %s/%d\n", 340 __func__, trapno, ip, si_code_str(si->si_code), 341 si->si_code); 342 343 #if defined(__i386__) || defined(__x86_64__) /* arch */ 344 #ifdef __i386__ 345 /* 346 * 32-bit has some extra padding so that userspace can tell whether 347 * the XSTATE header is present in addition to the "legacy" FPU 348 * state. We just assume that it is here. 349 */ 350 fpregs += 0x70; 351 #endif /* i386 */ 352 pkey_reg_offset = pkey_reg_xstate_offset(); 353 pkey_reg_ptr = (void *)(&fpregs[pkey_reg_offset]); 354 355 /* 356 * If we got a PKEY fault, we *HAVE* to have at least one bit set in 357 * here. 358 */ 359 dprintf1("pkey_reg_xstate_offset: %d\n", pkey_reg_xstate_offset()); 360 if (DEBUG_LEVEL > 4) 361 dump_mem(pkey_reg_ptr - 128, 256); 362 pkey_assert(*pkey_reg_ptr); 363 #endif /* arch */ 364 365 dprintf1("siginfo: %p\n", si); 366 #ifdef MCONTEXT_FPREGS 367 dprintf1(" fpregs: %p\n", fpregs); 368 #endif 369 370 if ((si->si_code == SEGV_MAPERR) || 371 (si->si_code == SEGV_ACCERR) || 372 (si->si_code == SEGV_BNDERR)) { 373 printf("non-PK si_code, exiting...\n"); 374 exit(4); 375 } 376 377 si_pkey_ptr = siginfo_get_pkey_ptr(si); 378 dprintf1("si_pkey_ptr: %p\n", si_pkey_ptr); 379 dump_mem((u8 *)si_pkey_ptr - 8, 24); 380 siginfo_pkey = *si_pkey_ptr; 381 pkey_assert(siginfo_pkey < NR_PKEYS); 382 last_si_pkey = siginfo_pkey; 383 384 /* 385 * need __read_pkey_reg() version so we do not do shadow_pkey_reg 386 * checking 387 */ 388 dprintf1("signal pkey_reg from pkey_reg: %016llx\n", 389 __read_pkey_reg()); 390 dprintf1("pkey from siginfo: %016llx\n", siginfo_pkey); 391 #if defined(__i386__) || defined(__x86_64__) /* arch */ 392 dprintf1("signal pkey_reg from xsave: %08x\n", *pkey_reg_ptr); 393 *(u64 *)pkey_reg_ptr = 0x00000000; 394 dprintf1("WARNING: set PKEY_REG=0 to allow faulting instruction to continue\n"); 395 #elif defined(__powerpc64__) /* arch */ 396 /* restore access and let the faulting instruction continue */ 397 pkey_access_allow(siginfo_pkey); 398 #elif defined(__aarch64__) 399 aarch64_write_signal_pkey(uctxt, PKEY_REG_ALLOW_ALL); 400 #endif /* arch */ 401 pkey_faults++; 402 dprintf1("<<<<==================================================\n"); 403 dprint_in_signal = 0; 404 } 405 406 static void sig_chld(int x) 407 { 408 dprint_in_signal = 1; 409 dprintf2("[%d] SIGCHLD: %d\n", getpid(), x); 410 dprint_in_signal = 0; 411 } 412 413 static void setup_sigsegv_handler(void) 414 { 415 int r, rs; 416 struct sigaction newact; 417 struct sigaction oldact; 418 419 /* #PF is mapped to sigsegv */ 420 int signum = SIGSEGV; 421 422 newact.sa_handler = 0; 423 newact.sa_sigaction = signal_handler; 424 425 /*sigset_t - signals to block while in the handler */ 426 /* get the old signal mask. */ 427 rs = sigprocmask(SIG_SETMASK, 0, &newact.sa_mask); 428 pkey_assert(rs == 0); 429 430 /* call sa_sigaction, not sa_handler*/ 431 newact.sa_flags = SA_SIGINFO; 432 433 newact.sa_restorer = 0; /* void(*)(), obsolete */ 434 r = sigaction(signum, &newact, &oldact); 435 r = sigaction(SIGALRM, &newact, &oldact); 436 pkey_assert(r == 0); 437 } 438 439 static void setup_handlers(void) 440 { 441 signal(SIGCHLD, &sig_chld); 442 setup_sigsegv_handler(); 443 } 444 445 static pid_t fork_lazy_child(void) 446 { 447 pid_t forkret; 448 449 forkret = fork(); 450 pkey_assert(forkret >= 0); 451 dprintf3("[%d] fork() ret: %d\n", getpid(), forkret); 452 453 if (!forkret) { 454 /* in the child */ 455 while (1) { 456 dprintf1("child sleeping...\n"); 457 sleep(30); 458 } 459 } 460 return forkret; 461 } 462 463 static int alloc_pkey(void) 464 { 465 int ret; 466 unsigned long init_val = 0x0; 467 468 dprintf1("%s()::%d, pkey_reg: 0x%016llx shadow: %016llx\n", 469 __func__, __LINE__, __read_pkey_reg(), shadow_pkey_reg); 470 ret = sys_pkey_alloc(0, init_val); 471 /* 472 * pkey_alloc() sets PKEY register, so we need to reflect it in 473 * shadow_pkey_reg: 474 */ 475 dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx" 476 " shadow: 0x%016llx\n", 477 __func__, __LINE__, ret, __read_pkey_reg(), 478 shadow_pkey_reg); 479 if (ret > 0) { 480 /* clear both the bits: */ 481 shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, ret, 482 ~PKEY_MASK); 483 dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx" 484 " shadow: 0x%016llx\n", 485 __func__, 486 __LINE__, ret, __read_pkey_reg(), 487 shadow_pkey_reg); 488 /* 489 * move the new state in from init_val 490 * (remember, we cheated and init_val == pkey_reg format) 491 */ 492 shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, ret, 493 init_val); 494 } 495 dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx" 496 " shadow: 0x%016llx\n", 497 __func__, __LINE__, ret, __read_pkey_reg(), 498 shadow_pkey_reg); 499 dprintf1("%s()::%d errno: %d\n", __func__, __LINE__, errno); 500 /* for shadow checking: */ 501 read_pkey_reg(); 502 dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx" 503 " shadow: 0x%016llx\n", 504 __func__, __LINE__, ret, __read_pkey_reg(), 505 shadow_pkey_reg); 506 return ret; 507 } 508 509 /* 510 * I had a bug where pkey bits could be set by mprotect() but 511 * not cleared. This ensures we get lots of random bit sets 512 * and clears on the vma and pte pkey bits. 513 */ 514 static int alloc_random_pkey(void) 515 { 516 int max_nr_pkey_allocs; 517 int ret; 518 int i; 519 int alloced_pkeys[NR_PKEYS]; 520 int nr_alloced = 0; 521 int random_index; 522 memset(alloced_pkeys, 0, sizeof(alloced_pkeys)); 523 524 /* allocate every possible key and make a note of which ones we got */ 525 max_nr_pkey_allocs = NR_PKEYS; 526 for (i = 0; i < max_nr_pkey_allocs; i++) { 527 int new_pkey = alloc_pkey(); 528 if (new_pkey < 0) 529 break; 530 alloced_pkeys[nr_alloced++] = new_pkey; 531 } 532 533 pkey_assert(nr_alloced > 0); 534 /* select a random one out of the allocated ones */ 535 random_index = rand() % nr_alloced; 536 ret = alloced_pkeys[random_index]; 537 /* now zero it out so we don't free it next */ 538 alloced_pkeys[random_index] = 0; 539 540 /* go through the allocated ones that we did not want and free them */ 541 for (i = 0; i < nr_alloced; i++) { 542 int free_ret; 543 if (!alloced_pkeys[i]) 544 continue; 545 free_ret = sys_pkey_free(alloced_pkeys[i]); 546 pkey_assert(!free_ret); 547 } 548 dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx" 549 " shadow: 0x%016llx\n", __func__, 550 __LINE__, ret, __read_pkey_reg(), shadow_pkey_reg); 551 return ret; 552 } 553 554 int mprotect_pkey(void *ptr, size_t size, unsigned long orig_prot, 555 unsigned long pkey) 556 { 557 int nr_iterations = random() % 100; 558 int ret; 559 560 while (0) { 561 int rpkey = alloc_random_pkey(); 562 ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey); 563 dprintf1("sys_mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n", 564 ptr, size, orig_prot, pkey, ret); 565 if (nr_iterations-- < 0) 566 break; 567 568 dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx" 569 " shadow: 0x%016llx\n", 570 __func__, __LINE__, ret, __read_pkey_reg(), 571 shadow_pkey_reg); 572 sys_pkey_free(rpkey); 573 dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx" 574 " shadow: 0x%016llx\n", 575 __func__, __LINE__, ret, __read_pkey_reg(), 576 shadow_pkey_reg); 577 } 578 pkey_assert(pkey < NR_PKEYS); 579 580 ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey); 581 dprintf1("mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n", 582 ptr, size, orig_prot, pkey, ret); 583 pkey_assert(!ret); 584 dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx" 585 " shadow: 0x%016llx\n", __func__, 586 __LINE__, ret, __read_pkey_reg(), shadow_pkey_reg); 587 return ret; 588 } 589 590 struct pkey_malloc_record { 591 void *ptr; 592 long size; 593 int prot; 594 }; 595 struct pkey_malloc_record *pkey_malloc_records; 596 struct pkey_malloc_record *pkey_last_malloc_record; 597 static long nr_pkey_malloc_records; 598 void record_pkey_malloc(void *ptr, long size, int prot) 599 { 600 long i; 601 struct pkey_malloc_record *rec = NULL; 602 603 for (i = 0; i < nr_pkey_malloc_records; i++) { 604 rec = &pkey_malloc_records[i]; 605 /* find a free record */ 606 if (rec) 607 break; 608 } 609 if (!rec) { 610 /* every record is full */ 611 size_t old_nr_records = nr_pkey_malloc_records; 612 size_t new_nr_records = (nr_pkey_malloc_records * 2 + 1); 613 size_t new_size = new_nr_records * sizeof(struct pkey_malloc_record); 614 dprintf2("new_nr_records: %zd\n", new_nr_records); 615 dprintf2("new_size: %zd\n", new_size); 616 pkey_malloc_records = realloc(pkey_malloc_records, new_size); 617 pkey_assert(pkey_malloc_records != NULL); 618 rec = &pkey_malloc_records[nr_pkey_malloc_records]; 619 /* 620 * realloc() does not initialize memory, so zero it from 621 * the first new record all the way to the end. 622 */ 623 for (i = 0; i < new_nr_records - old_nr_records; i++) 624 memset(rec + i, 0, sizeof(*rec)); 625 } 626 dprintf3("filling malloc record[%d/%p]: {%p, %ld}\n", 627 (int)(rec - pkey_malloc_records), rec, ptr, size); 628 rec->ptr = ptr; 629 rec->size = size; 630 rec->prot = prot; 631 pkey_last_malloc_record = rec; 632 nr_pkey_malloc_records++; 633 } 634 635 static void free_pkey_malloc(void *ptr) 636 { 637 long i; 638 int ret; 639 dprintf3("%s(%p)\n", __func__, ptr); 640 for (i = 0; i < nr_pkey_malloc_records; i++) { 641 struct pkey_malloc_record *rec = &pkey_malloc_records[i]; 642 dprintf4("looking for ptr %p at record[%ld/%p]: {%p, %ld}\n", 643 ptr, i, rec, rec->ptr, rec->size); 644 if ((ptr < rec->ptr) || 645 (ptr >= rec->ptr + rec->size)) 646 continue; 647 648 dprintf3("found ptr %p at record[%ld/%p]: {%p, %ld}\n", 649 ptr, i, rec, rec->ptr, rec->size); 650 nr_pkey_malloc_records--; 651 ret = munmap(rec->ptr, rec->size); 652 dprintf3("munmap ret: %d\n", ret); 653 pkey_assert(!ret); 654 dprintf3("clearing rec->ptr, rec: %p\n", rec); 655 rec->ptr = NULL; 656 dprintf3("done clearing rec->ptr, rec: %p\n", rec); 657 return; 658 } 659 pkey_assert(false); 660 } 661 662 static void *malloc_pkey_with_mprotect(long size, int prot, u16 pkey) 663 { 664 void *ptr; 665 int ret; 666 667 read_pkey_reg(); 668 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__, 669 size, prot, pkey); 670 pkey_assert(pkey < NR_PKEYS); 671 ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); 672 pkey_assert(ptr != (void *)-1); 673 ret = mprotect_pkey((void *)ptr, PAGE_SIZE, prot, pkey); 674 pkey_assert(!ret); 675 record_pkey_malloc(ptr, size, prot); 676 read_pkey_reg(); 677 678 dprintf1("%s() for pkey %d @ %p\n", __func__, pkey, ptr); 679 return ptr; 680 } 681 682 static void *malloc_pkey_anon_huge(long size, int prot, u16 pkey) 683 { 684 int ret; 685 void *ptr; 686 687 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__, 688 size, prot, pkey); 689 /* 690 * Guarantee we can fit at least one huge page in the resulting 691 * allocation by allocating space for 2: 692 */ 693 size = ALIGN_UP(size, HPAGE_SIZE * 2); 694 ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); 695 pkey_assert(ptr != (void *)-1); 696 record_pkey_malloc(ptr, size, prot); 697 mprotect_pkey(ptr, size, prot, pkey); 698 699 dprintf1("unaligned ptr: %p\n", ptr); 700 ptr = ALIGN_PTR_UP(ptr, HPAGE_SIZE); 701 dprintf1(" aligned ptr: %p\n", ptr); 702 ret = madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE); 703 dprintf1("MADV_HUGEPAGE ret: %d\n", ret); 704 ret = madvise(ptr, HPAGE_SIZE, MADV_WILLNEED); 705 dprintf1("MADV_WILLNEED ret: %d\n", ret); 706 memset(ptr, 0, HPAGE_SIZE); 707 708 dprintf1("mmap()'d thp for pkey %d @ %p\n", pkey, ptr); 709 return ptr; 710 } 711 712 static int hugetlb_setup_ok; 713 #define SYSFS_FMT_NR_HUGE_PAGES "/sys/kernel/mm/hugepages/hugepages-%ldkB/nr_hugepages" 714 #define GET_NR_HUGE_PAGES 10 715 static void setup_hugetlbfs(void) 716 { 717 int err; 718 int fd; 719 char buf[256]; 720 long hpagesz_kb; 721 long hpagesz_mb; 722 723 if (geteuid() != 0) { 724 fprintf(stderr, "WARNING: not run as root, can not do hugetlb test\n"); 725 return; 726 } 727 728 cat_into_file(__stringify(GET_NR_HUGE_PAGES), "/proc/sys/vm/nr_hugepages"); 729 730 /* 731 * Now go make sure that we got the pages and that they 732 * are PMD-level pages. Someone might have made PUD-level 733 * pages the default. 734 */ 735 hpagesz_kb = HPAGE_SIZE / 1024; 736 hpagesz_mb = hpagesz_kb / 1024; 737 sprintf(buf, SYSFS_FMT_NR_HUGE_PAGES, hpagesz_kb); 738 fd = open(buf, O_RDONLY); 739 if (fd < 0) { 740 fprintf(stderr, "opening sysfs %ldM hugetlb config: %s\n", 741 hpagesz_mb, strerror(errno)); 742 return; 743 } 744 745 /* -1 to guarantee leaving the trailing \0 */ 746 err = read(fd, buf, sizeof(buf)-1); 747 close(fd); 748 if (err <= 0) { 749 fprintf(stderr, "reading sysfs %ldM hugetlb config: %s\n", 750 hpagesz_mb, strerror(errno)); 751 return; 752 } 753 754 if (atoi(buf) != GET_NR_HUGE_PAGES) { 755 fprintf(stderr, "could not confirm %ldM pages, got: '%s' expected %d\n", 756 hpagesz_mb, buf, GET_NR_HUGE_PAGES); 757 return; 758 } 759 760 hugetlb_setup_ok = 1; 761 } 762 763 static void *malloc_pkey_hugetlb(long size, int prot, u16 pkey) 764 { 765 void *ptr; 766 int flags = MAP_ANONYMOUS|MAP_PRIVATE|MAP_HUGETLB; 767 768 if (!hugetlb_setup_ok) 769 return PTR_ERR_ENOTSUP; 770 771 dprintf1("doing %s(%ld, %x, %x)\n", __func__, size, prot, pkey); 772 size = ALIGN_UP(size, HPAGE_SIZE * 2); 773 pkey_assert(pkey < NR_PKEYS); 774 ptr = mmap(NULL, size, PROT_NONE, flags, -1, 0); 775 pkey_assert(ptr != (void *)-1); 776 mprotect_pkey(ptr, size, prot, pkey); 777 778 record_pkey_malloc(ptr, size, prot); 779 780 dprintf1("mmap()'d hugetlbfs for pkey %d @ %p\n", pkey, ptr); 781 return ptr; 782 } 783 784 static void *(*pkey_malloc[])(long size, int prot, u16 pkey) = { 785 786 malloc_pkey_with_mprotect, 787 malloc_pkey_with_mprotect_subpage, 788 malloc_pkey_anon_huge, 789 malloc_pkey_hugetlb 790 }; 791 792 static void *malloc_pkey(long size, int prot, u16 pkey) 793 { 794 void *ret; 795 static int malloc_type; 796 int nr_malloc_types = ARRAY_SIZE(pkey_malloc); 797 798 pkey_assert(pkey < NR_PKEYS); 799 800 while (1) { 801 pkey_assert(malloc_type < nr_malloc_types); 802 803 ret = pkey_malloc[malloc_type](size, prot, pkey); 804 pkey_assert(ret != (void *)-1); 805 806 malloc_type++; 807 if (malloc_type >= nr_malloc_types) 808 malloc_type = (random()%nr_malloc_types); 809 810 /* try again if the malloc_type we tried is unsupported */ 811 if (ret == PTR_ERR_ENOTSUP) 812 continue; 813 814 break; 815 } 816 817 dprintf3("%s(%ld, prot=%x, pkey=%x) returning: %p\n", __func__, 818 size, prot, pkey, ret); 819 return ret; 820 } 821 822 static int last_pkey_faults; 823 #define UNKNOWN_PKEY -2 824 void expected_pkey_fault(int pkey) 825 { 826 dprintf2("%s(): last_pkey_faults: %d pkey_faults: %d\n", 827 __func__, last_pkey_faults, pkey_faults); 828 dprintf2("%s(%d): last_si_pkey: %d\n", __func__, pkey, last_si_pkey); 829 pkey_assert(last_pkey_faults + 1 == pkey_faults); 830 831 /* 832 * For exec-only memory, we do not know the pkey in 833 * advance, so skip this check. 834 */ 835 if (pkey != UNKNOWN_PKEY) 836 pkey_assert(last_si_pkey == pkey); 837 838 #if defined(__i386__) || defined(__x86_64__) /* arch */ 839 /* 840 * The signal handler shold have cleared out PKEY register to let the 841 * test program continue. We now have to restore it. 842 */ 843 if (__read_pkey_reg() != 0) 844 #elif defined(__aarch64__) 845 if (__read_pkey_reg() != PKEY_REG_ALLOW_ALL) 846 #else 847 if (__read_pkey_reg() != shadow_pkey_reg) 848 #endif /* arch */ 849 pkey_assert(0); 850 851 __write_pkey_reg(shadow_pkey_reg); 852 dprintf1("%s() set pkey_reg=%016llx to restore state after signal " 853 "nuked it\n", __func__, shadow_pkey_reg); 854 last_pkey_faults = pkey_faults; 855 last_si_pkey = -1; 856 } 857 858 #define do_not_expect_pkey_fault(msg) do { \ 859 if (last_pkey_faults != pkey_faults) \ 860 dprintf0("unexpected PKey fault: %s\n", msg); \ 861 pkey_assert(last_pkey_faults == pkey_faults); \ 862 } while (0) 863 864 static int test_fds[10] = { -1 }; 865 static int nr_test_fds; 866 static void __save_test_fd(int fd) 867 { 868 pkey_assert(fd >= 0); 869 pkey_assert(nr_test_fds < ARRAY_SIZE(test_fds)); 870 test_fds[nr_test_fds] = fd; 871 nr_test_fds++; 872 } 873 874 static int get_test_read_fd(void) 875 { 876 int test_fd = open("/etc/passwd", O_RDONLY); 877 __save_test_fd(test_fd); 878 return test_fd; 879 } 880 881 static void close_test_fds(void) 882 { 883 int i; 884 885 for (i = 0; i < nr_test_fds; i++) { 886 if (test_fds[i] < 0) 887 continue; 888 close(test_fds[i]); 889 test_fds[i] = -1; 890 } 891 nr_test_fds = 0; 892 } 893 894 static void test_pkey_alloc_free_attach_pkey0(int *ptr, u16 pkey) 895 { 896 int i, err; 897 int max_nr_pkey_allocs; 898 int alloced_pkeys[NR_PKEYS]; 899 int nr_alloced = 0; 900 long size; 901 902 pkey_assert(pkey_last_malloc_record); 903 size = pkey_last_malloc_record->size; 904 /* 905 * This is a bit of a hack. But mprotect() requires 906 * huge-page-aligned sizes when operating on hugetlbfs. 907 * So, make sure that we use something that's a multiple 908 * of a huge page when we can. 909 */ 910 if (size >= HPAGE_SIZE) 911 size = HPAGE_SIZE; 912 913 /* allocate every possible key and make sure key-0 never got allocated */ 914 max_nr_pkey_allocs = NR_PKEYS; 915 for (i = 0; i < max_nr_pkey_allocs; i++) { 916 int new_pkey = alloc_pkey(); 917 pkey_assert(new_pkey != 0); 918 919 if (new_pkey < 0) 920 break; 921 alloced_pkeys[nr_alloced++] = new_pkey; 922 } 923 /* free all the allocated keys */ 924 for (i = 0; i < nr_alloced; i++) { 925 int free_ret; 926 927 if (!alloced_pkeys[i]) 928 continue; 929 free_ret = sys_pkey_free(alloced_pkeys[i]); 930 pkey_assert(!free_ret); 931 } 932 933 /* attach key-0 in various modes */ 934 err = sys_mprotect_pkey(ptr, size, PROT_READ, 0); 935 pkey_assert(!err); 936 err = sys_mprotect_pkey(ptr, size, PROT_WRITE, 0); 937 pkey_assert(!err); 938 err = sys_mprotect_pkey(ptr, size, PROT_EXEC, 0); 939 pkey_assert(!err); 940 err = sys_mprotect_pkey(ptr, size, PROT_READ|PROT_WRITE, 0); 941 pkey_assert(!err); 942 err = sys_mprotect_pkey(ptr, size, PROT_READ|PROT_WRITE|PROT_EXEC, 0); 943 pkey_assert(!err); 944 } 945 946 static void test_read_of_write_disabled_region(int *ptr, u16 pkey) 947 { 948 int ptr_contents; 949 950 dprintf1("disabling write access to PKEY[1], doing read\n"); 951 pkey_write_deny(pkey); 952 ptr_contents = read_ptr(ptr); 953 dprintf1("*ptr: %d\n", ptr_contents); 954 dprintf1("\n"); 955 } 956 static void test_read_of_access_disabled_region(int *ptr, u16 pkey) 957 { 958 int ptr_contents; 959 960 dprintf1("disabling access to PKEY[%02d], doing read @ %p\n", pkey, ptr); 961 read_pkey_reg(); 962 pkey_access_deny(pkey); 963 ptr_contents = read_ptr(ptr); 964 dprintf1("*ptr: %d\n", ptr_contents); 965 expected_pkey_fault(pkey); 966 } 967 968 static void test_read_of_access_disabled_region_with_page_already_mapped(int *ptr, 969 u16 pkey) 970 { 971 int ptr_contents; 972 973 dprintf1("disabling access to PKEY[%02d], doing read @ %p\n", 974 pkey, ptr); 975 ptr_contents = read_ptr(ptr); 976 dprintf1("reading ptr before disabling the read : %d\n", 977 ptr_contents); 978 read_pkey_reg(); 979 pkey_access_deny(pkey); 980 ptr_contents = read_ptr(ptr); 981 dprintf1("*ptr: %d\n", ptr_contents); 982 expected_pkey_fault(pkey); 983 } 984 985 static void test_write_of_write_disabled_region_with_page_already_mapped(int *ptr, 986 u16 pkey) 987 { 988 *ptr = __LINE__; 989 dprintf1("disabling write access; after accessing the page, " 990 "to PKEY[%02d], doing write\n", pkey); 991 pkey_write_deny(pkey); 992 *ptr = __LINE__; 993 expected_pkey_fault(pkey); 994 } 995 996 static void test_write_of_write_disabled_region(int *ptr, u16 pkey) 997 { 998 dprintf1("disabling write access to PKEY[%02d], doing write\n", pkey); 999 pkey_write_deny(pkey); 1000 *ptr = __LINE__; 1001 expected_pkey_fault(pkey); 1002 } 1003 static void test_write_of_access_disabled_region(int *ptr, u16 pkey) 1004 { 1005 dprintf1("disabling access to PKEY[%02d], doing write\n", pkey); 1006 pkey_access_deny(pkey); 1007 *ptr = __LINE__; 1008 expected_pkey_fault(pkey); 1009 } 1010 1011 static void test_write_of_access_disabled_region_with_page_already_mapped(int *ptr, 1012 u16 pkey) 1013 { 1014 *ptr = __LINE__; 1015 dprintf1("disabling access; after accessing the page, " 1016 " to PKEY[%02d], doing write\n", pkey); 1017 pkey_access_deny(pkey); 1018 *ptr = __LINE__; 1019 expected_pkey_fault(pkey); 1020 } 1021 1022 static void test_kernel_write_of_access_disabled_region(int *ptr, u16 pkey) 1023 { 1024 int ret; 1025 int test_fd = get_test_read_fd(); 1026 1027 dprintf1("disabling access to PKEY[%02d], " 1028 "having kernel read() to buffer\n", pkey); 1029 pkey_access_deny(pkey); 1030 ret = read(test_fd, ptr, 1); 1031 dprintf1("read ret: %d\n", ret); 1032 pkey_assert(ret); 1033 } 1034 1035 static void test_kernel_write_of_write_disabled_region(int *ptr, u16 pkey) 1036 { 1037 int ret; 1038 int test_fd = get_test_read_fd(); 1039 1040 pkey_write_deny(pkey); 1041 ret = read(test_fd, ptr, 100); 1042 dprintf1("read ret: %d\n", ret); 1043 if (ret < 0 && (DEBUG_LEVEL > 0)) 1044 perror("verbose read result (OK for this to be bad)"); 1045 pkey_assert(ret); 1046 } 1047 1048 static void test_kernel_gup_of_access_disabled_region(int *ptr, u16 pkey) 1049 { 1050 int pipe_ret, vmsplice_ret; 1051 struct iovec iov; 1052 int pipe_fds[2]; 1053 1054 pipe_ret = pipe(pipe_fds); 1055 1056 pkey_assert(pipe_ret == 0); 1057 dprintf1("disabling access to PKEY[%02d], " 1058 "having kernel vmsplice from buffer\n", pkey); 1059 pkey_access_deny(pkey); 1060 iov.iov_base = ptr; 1061 iov.iov_len = PAGE_SIZE; 1062 vmsplice_ret = vmsplice(pipe_fds[1], &iov, 1, SPLICE_F_GIFT); 1063 dprintf1("vmsplice() ret: %d\n", vmsplice_ret); 1064 pkey_assert(vmsplice_ret == -1); 1065 1066 close(pipe_fds[0]); 1067 close(pipe_fds[1]); 1068 } 1069 1070 static void test_kernel_gup_write_to_write_disabled_region(int *ptr, u16 pkey) 1071 { 1072 int ignored = 0xdada; 1073 int futex_ret; 1074 int some_int = __LINE__; 1075 1076 dprintf1("disabling write to PKEY[%02d], " 1077 "doing futex gunk in buffer\n", pkey); 1078 *ptr = some_int; 1079 pkey_write_deny(pkey); 1080 futex_ret = syscall(SYS_futex, ptr, FUTEX_WAIT, some_int-1, NULL, 1081 &ignored, ignored); 1082 if (DEBUG_LEVEL > 0) 1083 perror("futex"); 1084 dprintf1("futex() ret: %d\n", futex_ret); 1085 } 1086 1087 /* Assumes that all pkeys other than 'pkey' are unallocated */ 1088 static void test_pkey_syscalls_on_non_allocated_pkey(int *ptr, u16 pkey) 1089 { 1090 int err; 1091 int i; 1092 1093 /* Note: 0 is the default pkey, so don't mess with it */ 1094 for (i = 1; i < NR_PKEYS; i++) { 1095 if (pkey == i) 1096 continue; 1097 1098 dprintf1("trying get/set/free to non-allocated pkey: %2d\n", i); 1099 err = sys_pkey_free(i); 1100 pkey_assert(err); 1101 1102 err = sys_pkey_free(i); 1103 pkey_assert(err); 1104 1105 err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, i); 1106 pkey_assert(err); 1107 } 1108 } 1109 1110 /* Assumes that all pkeys other than 'pkey' are unallocated */ 1111 static void test_pkey_syscalls_bad_args(int *ptr, u16 pkey) 1112 { 1113 int err; 1114 int bad_pkey = NR_PKEYS+99; 1115 1116 /* pass a known-invalid pkey in: */ 1117 err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, bad_pkey); 1118 pkey_assert(err); 1119 } 1120 1121 static void become_child(void) 1122 { 1123 pid_t forkret; 1124 1125 forkret = fork(); 1126 pkey_assert(forkret >= 0); 1127 dprintf3("[%d] fork() ret: %d\n", getpid(), forkret); 1128 1129 if (!forkret) { 1130 /* in the child */ 1131 return; 1132 } 1133 exit(0); 1134 } 1135 1136 /* Assumes that all pkeys other than 'pkey' are unallocated */ 1137 static void test_pkey_alloc_exhaust(int *ptr, u16 pkey) 1138 { 1139 int err; 1140 int allocated_pkeys[NR_PKEYS] = {0}; 1141 int nr_allocated_pkeys = 0; 1142 int i; 1143 1144 for (i = 0; i < NR_PKEYS*3; i++) { 1145 int new_pkey; 1146 dprintf1("%s() alloc loop: %d\n", __func__, i); 1147 new_pkey = alloc_pkey(); 1148 dprintf4("%s()::%d, err: %d pkey_reg: 0x%016llx" 1149 " shadow: 0x%016llx\n", 1150 __func__, __LINE__, err, __read_pkey_reg(), 1151 shadow_pkey_reg); 1152 read_pkey_reg(); /* for shadow checking */ 1153 dprintf2("%s() errno: %d ENOSPC: %d\n", __func__, errno, ENOSPC); 1154 if ((new_pkey == -1) && (errno == ENOSPC)) { 1155 dprintf2("%s() failed to allocate pkey after %d tries\n", 1156 __func__, nr_allocated_pkeys); 1157 } else { 1158 /* 1159 * Ensure the number of successes never 1160 * exceeds the number of keys supported 1161 * in the hardware. 1162 */ 1163 pkey_assert(nr_allocated_pkeys < NR_PKEYS); 1164 allocated_pkeys[nr_allocated_pkeys++] = new_pkey; 1165 } 1166 1167 /* 1168 * Make sure that allocation state is properly 1169 * preserved across fork(). 1170 */ 1171 if (i == NR_PKEYS*2) 1172 become_child(); 1173 } 1174 1175 dprintf3("%s()::%d\n", __func__, __LINE__); 1176 1177 /* 1178 * On x86: 1179 * There are 16 pkeys supported in hardware. Three are 1180 * allocated by the time we get here: 1181 * 1. The default key (0) 1182 * 2. One possibly consumed by an execute-only mapping. 1183 * 3. One allocated by the test code and passed in via 1184 * 'pkey' to this function. 1185 * Ensure that we can allocate at least another 13 (16-3). 1186 * 1187 * On powerpc: 1188 * There are either 5, 28, 29 or 32 pkeys supported in 1189 * hardware depending on the page size (4K or 64K) and 1190 * platform (powernv or powervm). Four are allocated by 1191 * the time we get here. These include pkey-0, pkey-1, 1192 * exec-only pkey and the one allocated by the test code. 1193 * Ensure that we can allocate the remaining. 1194 */ 1195 pkey_assert(i >= (NR_PKEYS - get_arch_reserved_keys() - 1)); 1196 1197 for (i = 0; i < nr_allocated_pkeys; i++) { 1198 err = sys_pkey_free(allocated_pkeys[i]); 1199 pkey_assert(!err); 1200 read_pkey_reg(); /* for shadow checking */ 1201 } 1202 } 1203 1204 static void arch_force_pkey_reg_init(void) 1205 { 1206 #if defined(__i386__) || defined(__x86_64__) /* arch */ 1207 u64 *buf; 1208 1209 /* 1210 * All keys should be allocated and set to allow reads and 1211 * writes, so the register should be all 0. If not, just 1212 * skip the test. 1213 */ 1214 if (read_pkey_reg()) 1215 return; 1216 1217 /* 1218 * Just allocate an absurd about of memory rather than 1219 * doing the XSAVE size enumeration dance. 1220 */ 1221 buf = mmap(NULL, 1*MB, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); 1222 1223 /* These __builtins require compiling with -mxsave */ 1224 1225 /* XSAVE to build a valid buffer: */ 1226 __builtin_ia32_xsave(buf, XSTATE_PKEY); 1227 /* Clear XSTATE_BV[PKRU]: */ 1228 buf[XSTATE_BV_OFFSET/sizeof(u64)] &= ~XSTATE_PKEY; 1229 /* XRSTOR will likely get PKRU back to the init state: */ 1230 __builtin_ia32_xrstor(buf, XSTATE_PKEY); 1231 1232 munmap(buf, 1*MB); 1233 #endif 1234 } 1235 1236 1237 /* 1238 * This is mostly useless on ppc for now. But it will not 1239 * hurt anything and should give some better coverage as 1240 * a long-running test that continually checks the pkey 1241 * register. 1242 */ 1243 static void test_pkey_init_state(int *ptr, u16 pkey) 1244 { 1245 int err; 1246 int allocated_pkeys[NR_PKEYS] = {0}; 1247 int nr_allocated_pkeys = 0; 1248 int i; 1249 1250 for (i = 0; i < NR_PKEYS; i++) { 1251 int new_pkey = alloc_pkey(); 1252 1253 if (new_pkey < 0) 1254 continue; 1255 allocated_pkeys[nr_allocated_pkeys++] = new_pkey; 1256 } 1257 1258 dprintf3("%s()::%d\n", __func__, __LINE__); 1259 1260 arch_force_pkey_reg_init(); 1261 1262 /* 1263 * Loop for a bit, hoping to get exercise the kernel 1264 * context switch code. 1265 */ 1266 for (i = 0; i < 1000000; i++) 1267 read_pkey_reg(); 1268 1269 for (i = 0; i < nr_allocated_pkeys; i++) { 1270 err = sys_pkey_free(allocated_pkeys[i]); 1271 pkey_assert(!err); 1272 read_pkey_reg(); /* for shadow checking */ 1273 } 1274 } 1275 1276 /* 1277 * pkey 0 is special. It is allocated by default, so you do not 1278 * have to call pkey_alloc() to use it first. Make sure that it 1279 * is usable. 1280 */ 1281 static void test_mprotect_with_pkey_0(int *ptr, u16 pkey) 1282 { 1283 long size; 1284 int prot; 1285 1286 assert(pkey_last_malloc_record); 1287 size = pkey_last_malloc_record->size; 1288 /* 1289 * This is a bit of a hack. But mprotect() requires 1290 * huge-page-aligned sizes when operating on hugetlbfs. 1291 * So, make sure that we use something that's a multiple 1292 * of a huge page when we can. 1293 */ 1294 if (size >= HPAGE_SIZE) 1295 size = HPAGE_SIZE; 1296 prot = pkey_last_malloc_record->prot; 1297 1298 /* Use pkey 0 */ 1299 mprotect_pkey(ptr, size, prot, 0); 1300 1301 /* Make sure that we can set it back to the original pkey. */ 1302 mprotect_pkey(ptr, size, prot, pkey); 1303 } 1304 1305 static void test_ptrace_of_child(int *ptr, u16 pkey) 1306 { 1307 __attribute__((__unused__)) int peek_result; 1308 pid_t child_pid; 1309 void *ignored = 0; 1310 long ret; 1311 int status; 1312 /* 1313 * This is the "control" for our little expermient. Make sure 1314 * we can always access it when ptracing. 1315 */ 1316 int *plain_ptr_unaligned = malloc(HPAGE_SIZE); 1317 int *plain_ptr = ALIGN_PTR_UP(plain_ptr_unaligned, PAGE_SIZE); 1318 1319 /* 1320 * Fork a child which is an exact copy of this process, of course. 1321 * That means we can do all of our tests via ptrace() and then plain 1322 * memory access and ensure they work differently. 1323 */ 1324 child_pid = fork_lazy_child(); 1325 dprintf1("[%d] child pid: %d\n", getpid(), child_pid); 1326 1327 ret = ptrace(PTRACE_ATTACH, child_pid, ignored, ignored); 1328 if (ret) 1329 perror("attach"); 1330 dprintf1("[%d] attach ret: %ld %d\n", getpid(), ret, __LINE__); 1331 pkey_assert(ret != -1); 1332 ret = waitpid(child_pid, &status, WUNTRACED); 1333 if ((ret != child_pid) || !(WIFSTOPPED(status))) { 1334 fprintf(stderr, "weird waitpid result %ld stat %x\n", 1335 ret, status); 1336 pkey_assert(0); 1337 } 1338 dprintf2("waitpid ret: %ld\n", ret); 1339 dprintf2("waitpid status: %d\n", status); 1340 1341 pkey_access_deny(pkey); 1342 pkey_write_deny(pkey); 1343 1344 /* Write access, untested for now: 1345 ret = ptrace(PTRACE_POKEDATA, child_pid, peek_at, data); 1346 pkey_assert(ret != -1); 1347 dprintf1("poke at %p: %ld\n", peek_at, ret); 1348 */ 1349 1350 /* 1351 * Try to access the pkey-protected "ptr" via ptrace: 1352 */ 1353 ret = ptrace(PTRACE_PEEKDATA, child_pid, ptr, ignored); 1354 /* expect it to work, without an error: */ 1355 pkey_assert(ret != -1); 1356 /* Now access from the current task, and expect an exception: */ 1357 peek_result = read_ptr(ptr); 1358 expected_pkey_fault(pkey); 1359 1360 /* 1361 * Try to access the NON-pkey-protected "plain_ptr" via ptrace: 1362 */ 1363 ret = ptrace(PTRACE_PEEKDATA, child_pid, plain_ptr, ignored); 1364 /* expect it to work, without an error: */ 1365 pkey_assert(ret != -1); 1366 /* Now access from the current task, and expect NO exception: */ 1367 peek_result = read_ptr(plain_ptr); 1368 do_not_expect_pkey_fault("read plain pointer after ptrace"); 1369 1370 ret = ptrace(PTRACE_DETACH, child_pid, ignored, 0); 1371 pkey_assert(ret != -1); 1372 1373 ret = kill(child_pid, SIGKILL); 1374 pkey_assert(ret != -1); 1375 1376 wait(&status); 1377 1378 free(plain_ptr_unaligned); 1379 } 1380 1381 static void *get_pointer_to_instructions(void) 1382 { 1383 void *p1; 1384 1385 p1 = ALIGN_PTR_UP(&lots_o_noops_around_write, PAGE_SIZE); 1386 dprintf3("&lots_o_noops: %p\n", &lots_o_noops_around_write); 1387 /* lots_o_noops_around_write should be page-aligned already */ 1388 assert(p1 == &lots_o_noops_around_write); 1389 1390 /* Point 'p1' at the *second* page of the function: */ 1391 p1 += PAGE_SIZE; 1392 1393 /* 1394 * Try to ensure we fault this in on next touch to ensure 1395 * we get an instruction fault as opposed to a data one 1396 */ 1397 madvise(p1, PAGE_SIZE, MADV_DONTNEED); 1398 1399 return p1; 1400 } 1401 1402 static void test_executing_on_unreadable_memory(int *ptr, u16 pkey) 1403 { 1404 void *p1; 1405 int scratch; 1406 int ptr_contents; 1407 int ret; 1408 1409 p1 = get_pointer_to_instructions(); 1410 lots_o_noops_around_write(&scratch); 1411 ptr_contents = read_ptr(p1); 1412 dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents); 1413 1414 ret = mprotect_pkey(p1, PAGE_SIZE, PROT_EXEC, (u64)pkey); 1415 pkey_assert(!ret); 1416 pkey_access_deny(pkey); 1417 1418 dprintf2("pkey_reg: %016llx\n", read_pkey_reg()); 1419 1420 /* 1421 * Make sure this is an *instruction* fault 1422 */ 1423 madvise(p1, PAGE_SIZE, MADV_DONTNEED); 1424 lots_o_noops_around_write(&scratch); 1425 do_not_expect_pkey_fault("executing on PROT_EXEC memory"); 1426 expect_fault_on_read_execonly_key(p1, pkey); 1427 1428 // Reset back to PROT_EXEC | PROT_READ for architectures that support 1429 // non-PKEY execute-only permissions. 1430 ret = mprotect_pkey(p1, PAGE_SIZE, PROT_EXEC | PROT_READ, (u64)pkey); 1431 pkey_assert(!ret); 1432 } 1433 1434 static void test_implicit_mprotect_exec_only_memory(int *ptr, u16 pkey) 1435 { 1436 void *p1; 1437 int scratch; 1438 int ptr_contents; 1439 int ret; 1440 1441 dprintf1("%s() start\n", __func__); 1442 1443 p1 = get_pointer_to_instructions(); 1444 lots_o_noops_around_write(&scratch); 1445 ptr_contents = read_ptr(p1); 1446 dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents); 1447 1448 /* Use a *normal* mprotect(), not mprotect_pkey(): */ 1449 ret = mprotect(p1, PAGE_SIZE, PROT_EXEC); 1450 pkey_assert(!ret); 1451 1452 /* 1453 * Reset the shadow, assuming that the above mprotect() 1454 * correctly changed PKRU, but to an unknown value since 1455 * the actual allocated pkey is unknown. 1456 */ 1457 shadow_pkey_reg = __read_pkey_reg(); 1458 1459 dprintf2("pkey_reg: %016llx\n", read_pkey_reg()); 1460 1461 /* Make sure this is an *instruction* fault */ 1462 madvise(p1, PAGE_SIZE, MADV_DONTNEED); 1463 lots_o_noops_around_write(&scratch); 1464 do_not_expect_pkey_fault("executing on PROT_EXEC memory"); 1465 expect_fault_on_read_execonly_key(p1, UNKNOWN_PKEY); 1466 1467 /* 1468 * Put the memory back to non-PROT_EXEC. Should clear the 1469 * exec-only pkey off the VMA and allow it to be readable 1470 * again. Go to PROT_NONE first to check for a kernel bug 1471 * that did not clear the pkey when doing PROT_NONE. 1472 */ 1473 ret = mprotect(p1, PAGE_SIZE, PROT_NONE); 1474 pkey_assert(!ret); 1475 1476 ret = mprotect(p1, PAGE_SIZE, PROT_READ|PROT_EXEC); 1477 pkey_assert(!ret); 1478 ptr_contents = read_ptr(p1); 1479 do_not_expect_pkey_fault("plain read on recently PROT_EXEC area"); 1480 } 1481 1482 #if defined(__i386__) || defined(__x86_64__) 1483 static void test_ptrace_modifies_pkru(int *ptr, u16 pkey) 1484 { 1485 u32 new_pkru; 1486 pid_t child; 1487 int status, ret; 1488 int pkey_offset = pkey_reg_xstate_offset(); 1489 size_t xsave_size = cpu_max_xsave_size(); 1490 void *xsave; 1491 u32 *pkey_register; 1492 u64 *xstate_bv; 1493 struct iovec iov; 1494 1495 new_pkru = ~read_pkey_reg(); 1496 /* Don't make PROT_EXEC mappings inaccessible */ 1497 new_pkru &= ~3; 1498 1499 child = fork(); 1500 pkey_assert(child >= 0); 1501 dprintf3("[%d] fork() ret: %d\n", getpid(), child); 1502 if (!child) { 1503 ptrace(PTRACE_TRACEME, 0, 0, 0); 1504 /* Stop and allow the tracer to modify PKRU directly */ 1505 raise(SIGSTOP); 1506 1507 /* 1508 * need __read_pkey_reg() version so we do not do shadow_pkey_reg 1509 * checking 1510 */ 1511 if (__read_pkey_reg() != new_pkru) 1512 exit(1); 1513 1514 /* Stop and allow the tracer to clear XSTATE_BV for PKRU */ 1515 raise(SIGSTOP); 1516 1517 if (__read_pkey_reg() != 0) 1518 exit(1); 1519 1520 /* Stop and allow the tracer to examine PKRU */ 1521 raise(SIGSTOP); 1522 1523 exit(0); 1524 } 1525 1526 pkey_assert(child == waitpid(child, &status, 0)); 1527 dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status); 1528 pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP); 1529 1530 xsave = (void *)malloc(xsave_size); 1531 pkey_assert(xsave > 0); 1532 1533 /* Modify the PKRU register directly */ 1534 iov.iov_base = xsave; 1535 iov.iov_len = xsave_size; 1536 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov); 1537 pkey_assert(ret == 0); 1538 1539 pkey_register = (u32 *)(xsave + pkey_offset); 1540 pkey_assert(*pkey_register == read_pkey_reg()); 1541 1542 *pkey_register = new_pkru; 1543 1544 ret = ptrace(PTRACE_SETREGSET, child, (void *)NT_X86_XSTATE, &iov); 1545 pkey_assert(ret == 0); 1546 1547 /* Test that the modification is visible in ptrace before any execution */ 1548 memset(xsave, 0xCC, xsave_size); 1549 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov); 1550 pkey_assert(ret == 0); 1551 pkey_assert(*pkey_register == new_pkru); 1552 1553 /* Execute the tracee */ 1554 ret = ptrace(PTRACE_CONT, child, 0, 0); 1555 pkey_assert(ret == 0); 1556 1557 /* Test that the tracee saw the PKRU value change */ 1558 pkey_assert(child == waitpid(child, &status, 0)); 1559 dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status); 1560 pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP); 1561 1562 /* Test that the modification is visible in ptrace after execution */ 1563 memset(xsave, 0xCC, xsave_size); 1564 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov); 1565 pkey_assert(ret == 0); 1566 pkey_assert(*pkey_register == new_pkru); 1567 1568 /* Clear the PKRU bit from XSTATE_BV */ 1569 xstate_bv = (u64 *)(xsave + 512); 1570 *xstate_bv &= ~(1 << 9); 1571 1572 ret = ptrace(PTRACE_SETREGSET, child, (void *)NT_X86_XSTATE, &iov); 1573 pkey_assert(ret == 0); 1574 1575 /* Test that the modification is visible in ptrace before any execution */ 1576 memset(xsave, 0xCC, xsave_size); 1577 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov); 1578 pkey_assert(ret == 0); 1579 pkey_assert(*pkey_register == 0); 1580 1581 ret = ptrace(PTRACE_CONT, child, 0, 0); 1582 pkey_assert(ret == 0); 1583 1584 /* Test that the tracee saw the PKRU value go to 0 */ 1585 pkey_assert(child == waitpid(child, &status, 0)); 1586 dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status); 1587 pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP); 1588 1589 /* Test that the modification is visible in ptrace after execution */ 1590 memset(xsave, 0xCC, xsave_size); 1591 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov); 1592 pkey_assert(ret == 0); 1593 pkey_assert(*pkey_register == 0); 1594 1595 ret = ptrace(PTRACE_CONT, child, 0, 0); 1596 pkey_assert(ret == 0); 1597 pkey_assert(child == waitpid(child, &status, 0)); 1598 dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status); 1599 pkey_assert(WIFEXITED(status)); 1600 pkey_assert(WEXITSTATUS(status) == 0); 1601 free(xsave); 1602 } 1603 #endif 1604 1605 #if defined(__aarch64__) 1606 static void test_ptrace_modifies_pkru(int *ptr, u16 pkey) 1607 { 1608 pid_t child; 1609 int status, ret; 1610 struct iovec iov; 1611 u64 trace_pkey; 1612 /* Just a random pkey value.. */ 1613 u64 new_pkey = (POE_X << PKEY_BITS_PER_PKEY * 2) | 1614 (POE_NONE << PKEY_BITS_PER_PKEY) | 1615 POE_RWX; 1616 1617 child = fork(); 1618 pkey_assert(child >= 0); 1619 dprintf3("[%d] fork() ret: %d\n", getpid(), child); 1620 if (!child) { 1621 ptrace(PTRACE_TRACEME, 0, 0, 0); 1622 1623 /* Stop and allow the tracer to modify PKRU directly */ 1624 raise(SIGSTOP); 1625 1626 /* 1627 * need __read_pkey_reg() version so we do not do shadow_pkey_reg 1628 * checking 1629 */ 1630 if (__read_pkey_reg() != new_pkey) 1631 exit(1); 1632 1633 raise(SIGSTOP); 1634 1635 exit(0); 1636 } 1637 1638 pkey_assert(child == waitpid(child, &status, 0)); 1639 dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status); 1640 pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP); 1641 1642 iov.iov_base = &trace_pkey; 1643 iov.iov_len = 8; 1644 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_ARM_POE, &iov); 1645 pkey_assert(ret == 0); 1646 pkey_assert(trace_pkey == read_pkey_reg()); 1647 1648 trace_pkey = new_pkey; 1649 1650 ret = ptrace(PTRACE_SETREGSET, child, (void *)NT_ARM_POE, &iov); 1651 pkey_assert(ret == 0); 1652 1653 /* Test that the modification is visible in ptrace before any execution */ 1654 memset(&trace_pkey, 0, sizeof(trace_pkey)); 1655 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_ARM_POE, &iov); 1656 pkey_assert(ret == 0); 1657 pkey_assert(trace_pkey == new_pkey); 1658 1659 /* Execute the tracee */ 1660 ret = ptrace(PTRACE_CONT, child, 0, 0); 1661 pkey_assert(ret == 0); 1662 1663 /* Test that the tracee saw the PKRU value change */ 1664 pkey_assert(child == waitpid(child, &status, 0)); 1665 dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status); 1666 pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP); 1667 1668 /* Test that the modification is visible in ptrace after execution */ 1669 memset(&trace_pkey, 0, sizeof(trace_pkey)); 1670 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_ARM_POE, &iov); 1671 pkey_assert(ret == 0); 1672 pkey_assert(trace_pkey == new_pkey); 1673 1674 ret = ptrace(PTRACE_CONT, child, 0, 0); 1675 pkey_assert(ret == 0); 1676 pkey_assert(child == waitpid(child, &status, 0)); 1677 dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status); 1678 pkey_assert(WIFEXITED(status)); 1679 pkey_assert(WEXITSTATUS(status) == 0); 1680 } 1681 #endif 1682 1683 static void test_mprotect_pkey_on_unsupported_cpu(int *ptr, u16 pkey) 1684 { 1685 int size = PAGE_SIZE; 1686 int sret; 1687 1688 if (cpu_has_pkeys()) { 1689 dprintf1("SKIP: %s: no CPU support\n", __func__); 1690 return; 1691 } 1692 1693 sret = syscall(__NR_pkey_mprotect, ptr, size, PROT_READ, pkey); 1694 pkey_assert(sret < 0); 1695 } 1696 1697 static void (*pkey_tests[])(int *ptr, u16 pkey) = { 1698 test_read_of_write_disabled_region, 1699 test_read_of_access_disabled_region, 1700 test_read_of_access_disabled_region_with_page_already_mapped, 1701 test_write_of_write_disabled_region, 1702 test_write_of_write_disabled_region_with_page_already_mapped, 1703 test_write_of_access_disabled_region, 1704 test_write_of_access_disabled_region_with_page_already_mapped, 1705 test_kernel_write_of_access_disabled_region, 1706 test_kernel_write_of_write_disabled_region, 1707 test_kernel_gup_of_access_disabled_region, 1708 test_kernel_gup_write_to_write_disabled_region, 1709 test_executing_on_unreadable_memory, 1710 test_implicit_mprotect_exec_only_memory, 1711 test_mprotect_with_pkey_0, 1712 test_ptrace_of_child, 1713 test_pkey_init_state, 1714 test_pkey_syscalls_on_non_allocated_pkey, 1715 test_pkey_syscalls_bad_args, 1716 test_pkey_alloc_exhaust, 1717 test_pkey_alloc_free_attach_pkey0, 1718 #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) 1719 test_ptrace_modifies_pkru, 1720 #endif 1721 }; 1722 1723 static void run_tests_once(void) 1724 { 1725 int *ptr; 1726 int prot = PROT_READ|PROT_WRITE; 1727 1728 for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) { 1729 int pkey; 1730 int orig_pkey_faults = pkey_faults; 1731 1732 dprintf1("======================\n"); 1733 dprintf1("test %d preparing...\n", test_nr); 1734 1735 tracing_on(); 1736 pkey = alloc_random_pkey(); 1737 dprintf1("test %d starting with pkey: %d\n", test_nr, pkey); 1738 ptr = malloc_pkey(PAGE_SIZE, prot, pkey); 1739 dprintf1("test %d starting...\n", test_nr); 1740 pkey_tests[test_nr](ptr, pkey); 1741 dprintf1("freeing test memory: %p\n", ptr); 1742 free_pkey_malloc(ptr); 1743 sys_pkey_free(pkey); 1744 1745 dprintf1("pkey_faults: %d\n", pkey_faults); 1746 dprintf1("orig_pkey_faults: %d\n", orig_pkey_faults); 1747 1748 tracing_off(); 1749 close_test_fds(); 1750 1751 printf("test %2d PASSED (iteration %d)\n", test_nr, iteration_nr); 1752 dprintf1("======================\n\n"); 1753 } 1754 iteration_nr++; 1755 } 1756 1757 static void pkey_setup_shadow(void) 1758 { 1759 shadow_pkey_reg = __read_pkey_reg(); 1760 } 1761 1762 int main(void) 1763 { 1764 int nr_iterations = 22; 1765 int pkeys_supported = is_pkeys_supported(); 1766 1767 srand((unsigned int)time(NULL)); 1768 1769 setup_handlers(); 1770 1771 printf("has pkeys: %d\n", pkeys_supported); 1772 1773 if (!pkeys_supported) { 1774 int size = PAGE_SIZE; 1775 int *ptr; 1776 1777 printf("running PKEY tests for unsupported CPU/OS\n"); 1778 1779 ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); 1780 assert(ptr != (void *)-1); 1781 test_mprotect_pkey_on_unsupported_cpu(ptr, 1); 1782 exit(0); 1783 } 1784 1785 pkey_setup_shadow(); 1786 printf("startup pkey_reg: %016llx\n", read_pkey_reg()); 1787 setup_hugetlbfs(); 1788 1789 while (nr_iterations-- > 0) 1790 run_tests_once(); 1791 1792 printf("done (all tests OK)\n"); 1793 return 0; 1794 } 1795