1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Race collapse against faults, GUP pins, fork, mremap and MADV_DONTNEED 4 * over the same ranges. A racing page must read as its pattern or as 5 * zero, never anything else; the kernel's own assertions in dmesg are the 6 * other half of the check. 7 */ 8 #define _GNU_SOURCE 9 #include <errno.h> 10 #include <fcntl.h> 11 #include <pthread.h> 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <string.h> 15 #include <sys/ioctl.h> 16 #include <sys/mman.h> 17 #include <sys/time.h> 18 #include <sys/wait.h> 19 #include <unistd.h> 20 21 #include "kselftest.h" 22 #include "vm_util.h" 23 #include <mm/hugepage_settings.h> 24 #include "../../../../mm/gup_test.h" 25 26 #ifndef FOLL_WRITE 27 #define FOLL_WRITE 0x01 28 #endif 29 30 #define BASE_ADDR ((void *)(1UL << 30)) 31 #define PASS_TIMEOUT_S 30 32 33 /* 34 * PMD-sized areas the racing threads share, plus one for the mremap 35 * thread. -a shrinks it where a PMD is 512M. 36 */ 37 #define DEFAULT_SHARED_AREAS 3 38 static int nr_shared_areas; 39 static int nr_areas; 40 41 static unsigned long hpage_pmd_size; 42 static unsigned long page_size; 43 /* nr_areas PMD-sized areas; the last one belongs to the mremap thread */ 44 static char *region; 45 static char *mremap_area; 46 static char *mremap_scratch; 47 static char *pageout_area; 48 static size_t pageout_size; 49 static int gup_fd = -1; 50 static volatile int stop; 51 static volatile int corrupted; 52 53 static unsigned int pattern(unsigned long page_idx) 54 { 55 unsigned int val = (unsigned int)page_idx * 2654435761U; 56 57 return val ? val : 1; /* never collides with the zero-fill */ 58 } 59 60 /* Zero means never written; anything else must be this page's pattern */ 61 static bool page_is_corrupt(unsigned long page_idx, unsigned int *val) 62 { 63 *val = *(unsigned int *)(region + page_idx * page_size); 64 65 return *val && *val != pattern(page_idx); 66 } 67 68 static void check_page(unsigned long page_idx) 69 { 70 unsigned int val; 71 72 if (page_is_corrupt(page_idx, &val)) { 73 corrupted = 1; 74 ksft_print_msg("Corruption at page %lu: %#x != %#x\n", 75 page_idx, val, pattern(page_idx)); 76 } 77 } 78 79 static unsigned long shared_pages(void) 80 { 81 return nr_shared_areas * hpage_pmd_size / page_size; 82 } 83 84 static unsigned long rand_page(unsigned int *seed) 85 { 86 return (unsigned long)rand_r(seed) % shared_pages(); 87 } 88 89 /* Clamp so a range never reaches the mremap thread's area */ 90 static unsigned long room_from(unsigned long page_idx, unsigned long want) 91 { 92 unsigned long left = shared_pages() - page_idx; 93 94 return want < left ? want : left; 95 } 96 97 static void *faulter_fn(void *arg) 98 { 99 unsigned int seed = (unsigned long)arg; 100 101 while (!stop) { 102 unsigned long page_idx = rand_page(&seed); 103 104 if (rand_r(&seed) & 1) 105 *(unsigned int *)(region + page_idx * page_size) = 106 pattern(page_idx); 107 else 108 check_page(page_idx); 109 } 110 return NULL; 111 } 112 113 static void *dontneed_fn(void *arg) 114 { 115 unsigned int seed = (unsigned long)arg; 116 117 while (!stop) { 118 unsigned long page_idx = rand_page(&seed); 119 unsigned long nr = 1UL << (rand_r(&seed) % 6); /* 1..32 pages */ 120 121 /* 122 * Now and then zap a whole PMD-aligned area: only a zap that 123 * covers the full table frees the table itself (CONFIG_PT_RECLAIM). 124 */ 125 if (!(rand_r(&seed) % 64)) { 126 unsigned long area = page_idx / 127 (hpage_pmd_size / page_size); 128 129 madvise(region + area * hpage_pmd_size, 130 hpage_pmd_size, MADV_DONTNEED); 131 } else { 132 madvise(region + page_idx * page_size, 133 room_from(page_idx, nr) * page_size, 134 MADV_DONTNEED); 135 } 136 usleep(rand_r(&seed) % 500); 137 } 138 return NULL; 139 } 140 141 static void *pinner_fn(void *arg) 142 { 143 unsigned int seed = (unsigned long)arg; 144 145 while (!stop) { 146 struct gup_test gup = {}; 147 unsigned long page_idx = rand_page(&seed); 148 unsigned long nr = room_from(page_idx, 16); 149 150 gup.addr = (unsigned long)(region + page_idx * page_size); 151 gup.size = nr * page_size; 152 gup.nr_pages_per_call = nr; 153 gup.gup_flags = FOLL_WRITE; 154 /* Racing MADV_DONTNEED makes transient failures expected */ 155 ioctl(gup_fd, PIN_FAST_BENCHMARK, &gup); 156 usleep(rand_r(&seed) % 200); 157 } 158 return NULL; 159 } 160 161 static void *forker_fn(void *arg) 162 { 163 unsigned int seed = (unsigned long)arg; 164 165 while (!stop) { 166 pid_t pid = fork(); 167 168 if (pid == 0) { 169 unsigned int val; 170 int bad = 0; 171 172 /* 173 * No stdio in the child: a thread may hold stdout's 174 * lock across the fork, and printing under it hangs. 175 */ 176 for (int i = 0; i < 16; i++) 177 bad |= page_is_corrupt(rand_page(&seed), &val); 178 _exit(bad); 179 } 180 if (pid > 0) { 181 int wstatus; 182 183 if (waitpid(pid, &wstatus, 0) < 0) 184 ksft_exit_fail_perror("waitpid()"); 185 /* A child killed on the read counts too */ 186 if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus)) 187 corrupted = 1; 188 } 189 usleep(rand_r(&seed) % 2000); 190 } 191 return NULL; 192 } 193 194 static void *mremapper_fn(void *arg) 195 { 196 unsigned int seed = (unsigned long)arg; 197 198 while (!stop) { 199 void *p; 200 201 p = mremap(mremap_area, hpage_pmd_size, hpage_pmd_size, 202 MREMAP_MAYMOVE | MREMAP_FIXED, mremap_scratch); 203 if (p == MAP_FAILED) 204 ksft_exit_fail_perror("mremap() away"); 205 for (int i = 0; i < 8; i++) 206 mremap_scratch[(rand_r(&seed) % 207 (hpage_pmd_size / page_size)) * page_size] = 1; 208 p = mremap(mremap_scratch, hpage_pmd_size, hpage_pmd_size, 209 MREMAP_MAYMOVE | MREMAP_FIXED, mremap_area); 210 if (p == MAP_FAILED) 211 ksft_exit_fail_perror("mremap() back"); 212 /* The move back unmapped the scratch address: claim it again */ 213 if (mmap(mremap_scratch, hpage_pmd_size, PROT_NONE, 214 MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 215 -1, 0) != (void *)mremap_scratch) 216 ksft_exit_fail_perror("mmap() mremap scratch"); 217 usleep(rand_r(&seed) % 2000); 218 } 219 return NULL; 220 } 221 222 /* 223 * Swap traffic and LRU churn on a region nothing else writes, so a page's 224 * pattern must survive the trip through swap exactly. 225 */ 226 static void *pageout_fn(void *arg) 227 { 228 unsigned int seed = (unsigned long)arg; 229 unsigned long nr = pageout_size / page_size; 230 unsigned long i; 231 232 for (i = 0; i < nr; i++) 233 *(unsigned int *)(pageout_area + i * page_size) = pattern(i); 234 235 while (!stop) { 236 madvise(pageout_area, pageout_size, MADV_PAGEOUT); 237 for (i = 0; i < nr && !stop; i++) { 238 unsigned int val = *(unsigned int *)(pageout_area + 239 i * page_size); 240 241 if (val != pattern(i)) { 242 corrupted = 1; 243 ksft_print_msg("Pageout corruption at page %lu: %#x != %#x\n", 244 i, val, pattern(i)); 245 } 246 } 247 usleep(rand_r(&seed) % 2000); 248 } 249 return NULL; 250 } 251 252 /* Compaction migrates the collapse sources while they are being gathered */ 253 static void *compactor_fn(void *arg) 254 { 255 unsigned int seed = (unsigned long)arg; 256 int fd = open("/proc/sys/vm/compact_memory", O_WRONLY); 257 258 if (fd < 0) { 259 ksft_print_msg("No compact_memory; compactor idle\n"); 260 return NULL; 261 } 262 while (!stop) { 263 if (write(fd, "1", 1) < 0) 264 break; 265 usleep(10000 + rand_r(&seed) % 100000); 266 } 267 close(fd); 268 return NULL; 269 } 270 271 static bool swap_available(void) 272 { 273 char line[256]; 274 int lines = 0; 275 FILE *fp = fopen("/proc/swaps", "r"); 276 277 if (!fp) 278 return false; 279 while (fgets(line, sizeof(line), fp)) 280 lines++; 281 fclose(fp); 282 return lines > 1; 283 } 284 285 static unsigned long now_ms(void) 286 { 287 struct timeval tv; 288 289 gettimeofday(&tv, NULL); 290 return tv.tv_sec * 1000UL + tv.tv_usec / 1000; 291 } 292 293 static void usage(void) 294 { 295 fprintf(stderr, 296 "Usage: khugepaged_race [-d seconds] [-m stepped|free|madvise] [-a areas] [-t mask]\n" 297 "\tWithout -m, every mode runs in turn.\n" 298 "\t-d: seconds per mode (default 5)\n" 299 "\t-a: number of shared PMD-sized playground areas (default 3)\n" 300 "\t-t: bitmask of racing threads to start, for bisecting a failure\n"); 301 exit(1); 302 } 303 304 int main(int argc, char **argv) 305 { 306 static const char * const thread_names[] = { 307 "faulter", "faulter2", "dontneed", "pinner", "forker", 308 "mremapper", "pageout", "compactor", 309 }; 310 void *(*const thread_fns[])(void *) = { 311 faulter_fn, faulter_fn, dontneed_fn, pinner_fn, forker_fn, 312 mremapper_fn, pageout_fn, compactor_fn, 313 }; 314 enum { T_FAULTER, T_FAULTER2, T_DONTNEED, T_PINNER, T_FORKER, 315 T_MREMAPPER, T_PAGEOUT, T_COMPACTOR }; 316 const unsigned long pageout_bit = 1UL << T_PAGEOUT; 317 const unsigned long compactor_bit = 1UL << T_COMPACTOR; 318 const int nr_threads = ARRAY_SIZE(thread_names); 319 pthread_t threads[ARRAY_SIZE(thread_names)]; 320 static const char * const all_modes[] = { "stepped", "free", "madvise" }; 321 static const bool occupancies[] = { false, true }; /* strict, holes */ 322 static const bool pressures[] = { false, true }; /* quiet, under pressure */ 323 const int nr_occupancies = ARRAY_SIZE(occupancies); 324 const int nr_pressures = ARRAY_SIZE(pressures); 325 const char *one_mode[1]; 326 const char * const *modes = all_modes; 327 int nr_modes = ARRAY_SIZE(all_modes); 328 const char *mode_arg = NULL; 329 struct thp_settings settings; 330 unsigned long end_ms; 331 int duration_s = 5; 332 unsigned long thread_mask = ~0UL; 333 unsigned long base_mask; 334 bool have_swap; 335 char label[64]; 336 int nr_areas_arg = 0; 337 unsigned long i; 338 int steps = 0; 339 int opt; 340 341 while ((opt = getopt(argc, argv, "a:d:m:t:h")) != -1) { 342 switch (opt) { 343 case 'a': 344 nr_areas_arg = atoi(optarg); 345 break; 346 case 'd': 347 duration_s = atoi(optarg); 348 break; 349 case 'm': 350 mode_arg = optarg; 351 break; 352 case 't': 353 thread_mask = strtoul(optarg, NULL, 0); 354 break; 355 default: 356 usage(); 357 } 358 } 359 360 if (mode_arg) { 361 if (strcmp(mode_arg, "stepped") && strcmp(mode_arg, "free") && 362 strcmp(mode_arg, "madvise")) 363 usage(); 364 one_mode[0] = mode_arg; 365 modes = one_mode; 366 nr_modes = 1; 367 } 368 369 ksft_print_header(); 370 if (!thp_available()) 371 ksft_exit_skip("Transparent Hugepages not available\n"); 372 373 page_size = getpagesize(); 374 hpage_pmd_size = read_pmd_pagesize(); 375 if (!hpage_pmd_size) 376 ksft_exit_fail_msg("Reading PMD pagesize failed\n"); 377 378 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); 379 if (gup_fd < 0) 380 ksft_exit_skip("/sys/kernel/debug/gup_test requires CONFIG_GUP_TEST and root\n"); 381 382 nr_shared_areas = nr_areas_arg > 0 ? nr_areas_arg : DEFAULT_SHARED_AREAS; 383 nr_areas = nr_shared_areas + 1; 384 385 /* 386 * MREMAP_FIXED unmaps whatever is in the way without saying so, so 387 * claim the mremap thread's scratch address up front. 388 */ 389 mremap_scratch = (char *)BASE_ADDR + 2 * nr_areas * hpage_pmd_size; 390 if (mmap(mremap_scratch, hpage_pmd_size, PROT_NONE, 391 MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 392 -1, 0) != (void *)mremap_scratch) 393 ksft_exit_fail_perror("mmap() mremap scratch"); 394 395 base_mask = thread_mask; 396 have_swap = swap_available(); 397 if (!have_swap) 398 /* No swap, no anon reclaim: compaction-only pressure */ 399 ksft_print_msg("no swap: the pageout thread is not started\n"); 400 401 ksft_set_plan(nr_modes * nr_occupancies * nr_pressures); 402 403 thp_save_settings(); 404 thp_read_settings(&settings); 405 406 /* Base of the settings stack; the bottom entry is never popped */ 407 thp_push_settings(&settings); 408 409 for (int run = 0; run < nr_modes * nr_occupancies * nr_pressures; run++) { 410 int rem = run % (nr_occupancies * nr_pressures); 411 const char *mode = modes[run / (nr_occupancies * nr_pressures)]; 412 bool holes = occupancies[rem / nr_pressures]; 413 bool pressure = pressures[rem % nr_pressures]; 414 415 snprintf(label, sizeof(label), "%s/%s%s", mode, 416 holes ? "holes" : "strict", pressure ? "/pressure" : ""); 417 if (corrupted) { 418 /* Memory is suspect; the rest would prove nothing */ 419 ksft_test_result_skip("%s: skipped after corruption\n", label); 420 continue; 421 } 422 423 thread_mask = base_mask; 424 if (!pressure) 425 thread_mask &= ~(pageout_bit | compactor_bit); 426 else if (!have_swap) 427 thread_mask &= ~pageout_bit; 428 429 thp_read_settings(&settings); 430 settings.thp_enabled = THP_MADVISE; 431 settings.thp_defrag = THP_DEFRAG_ALWAYS; 432 settings.shmem_enabled = SHMEM_NEVER; 433 settings.khugepaged.defrag = 1; 434 settings.khugepaged.scan_sleep_millisecs = 435 strcmp(mode, "free") ? 1000 : 0; 436 settings.khugepaged.alloc_sleep_millisecs = 10; 437 438 /* 439 * mTHP collapse honours only 0 or HPAGE_PMD_NR - 1 here. The two 440 * ends race different paths: a strict window has every PTE 441 * present, a hole-heavy one is mostly zero-filled. 442 */ 443 settings.khugepaged.max_ptes_none = holes ? 444 (hpage_pmd_size / page_size) - 1 : 0; 445 /* One wake, one pass: the playground plus the forked children's copies */ 446 settings.khugepaged.pages_to_scan = 447 nr_areas * (hpage_pmd_size / page_size) * 8; 448 for (i = 0; i < NR_ORDERS; i++) { 449 if (thp_supported_orders() & (1UL << i)) 450 settings.hugepages[i].enabled = THP_INHERIT; 451 } 452 thp_push_settings(&settings); 453 454 region = mmap(BASE_ADDR, nr_areas * hpage_pmd_size, 455 PROT_READ | PROT_WRITE, MAP_ANONYMOUS | 456 MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0); 457 if (region != BASE_ADDR) 458 ksft_exit_fail_perror("mmap() playground"); 459 mremap_area = region + nr_shared_areas * hpage_pmd_size; 460 461 if (thread_mask & pageout_bit) { 462 /* Enough to drive real reclaim without swamping a small guest */ 463 pageout_size = 4 * hpage_pmd_size; 464 if (pageout_size < 16UL << 20) 465 pageout_size = 16UL << 20; 466 if (pageout_size > 64UL << 20) 467 pageout_size = 64UL << 20; 468 pageout_area = mmap(NULL, pageout_size, 469 PROT_READ | PROT_WRITE, 470 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 471 if (pageout_area == MAP_FAILED) 472 ksft_exit_fail_perror("mmap() pageout area"); 473 } 474 475 /* Populate so the first pass has something to collapse */ 476 for (i = 0; i < nr_shared_areas * hpage_pmd_size / page_size; i++) 477 *(unsigned int *)(region + i * page_size) = pattern(i); 478 memset(mremap_area, 1, hpage_pmd_size); 479 if (madvise(region, nr_areas * hpage_pmd_size, MADV_HUGEPAGE)) 480 ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); 481 482 for (i = 0; i < nr_threads; i++) { 483 if (!(thread_mask & (1UL << i))) { 484 threads[i] = 0; 485 continue; 486 } 487 if (pthread_create(&threads[i], NULL, thread_fns[i], 488 (void *)(i + 1))) 489 ksft_exit_fail_perror(thread_names[i]); 490 } 491 492 end_ms = now_ms() + duration_s * 1000UL; 493 if (!strcmp(mode, "stepped")) { 494 while (now_ms() < end_ms && !corrupted) { 495 if (!khugepaged_full_pass(PASS_TIMEOUT_S)) 496 ksft_exit_fail_msg("khugepaged pass timed out\n"); 497 steps++; 498 } 499 } else if (!strcmp(mode, "free")) { 500 while (now_ms() < end_ms && !corrupted) 501 usleep(100 * 1000); 502 } else { /* madvise */ 503 while (now_ms() < end_ms && !corrupted) { 504 for (i = 0; i < nr_shared_areas; i++) { 505 madvise(region + i * hpage_pmd_size, 506 hpage_pmd_size, MADV_COLLAPSE); 507 } 508 madvise(region, nr_shared_areas * hpage_pmd_size, 509 MADV_DONTNEED); 510 steps++; 511 } 512 } 513 514 stop = 1; 515 for (i = 0; i < nr_threads; i++) { 516 if (threads[i]) 517 pthread_join(threads[i], NULL); 518 } 519 520 for (i = 0; i < nr_shared_areas * hpage_pmd_size / page_size; i++) 521 check_page(i); 522 523 ksft_test_result(!corrupted, "%s: %ds, %d steps, no corruption\n", 524 label, duration_s, steps); 525 526 /* The next mode maps the same fixed address with its own settings */ 527 munmap(region, nr_areas * hpage_pmd_size); 528 if (pageout_area) { 529 munmap(pageout_area, pageout_size); 530 pageout_area = NULL; 531 } 532 thp_pop_settings(); 533 stop = 0; 534 steps = 0; 535 } 536 537 ksft_finished(); 538 } 539