1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * The test creates shmem PMD huge pages, fills all pages with known patterns, 4 * then continuously verifies non-punched pages with 16 threads. Meanwhile, the 5 * main thread punches holes via MADV_REMOVE on the shmem. 6 * 7 * It tests the race condition between folio_split() and filemap_get_entry(), 8 * where the hole punches on shmem lead to folio_split() and reading the shmem 9 * lead to filemap_get_entry(). 10 */ 11 12 #define _GNU_SOURCE 13 #include <errno.h> 14 #include <inttypes.h> 15 #include <linux/mman.h> 16 #include <pthread.h> 17 #include <stdatomic.h> 18 #include <stdbool.h> 19 #include <stdint.h> 20 #include <stdio.h> 21 #include <stdlib.h> 22 #include <string.h> 23 #include <sys/mman.h> 24 #include <signal.h> 25 #include <unistd.h> 26 #include "vm_util.h" 27 #include "kselftest.h" 28 29 uint64_t page_size; 30 uint64_t pmd_pagesize; 31 #define NR_PMD_PAGE 5 32 #define FILE_SIZE (pmd_pagesize * NR_PMD_PAGE) 33 #define TOTAL_PAGES (FILE_SIZE / page_size) 34 35 /* Every N-th to N+M-th pages are punched; not aligned with huge page boundaries. */ 36 #define PUNCH_INTERVAL 50 /* N */ 37 #define PUNCH_SIZE_FACTOR 3 /* M */ 38 39 #define NUM_READER_THREADS 16 40 #define FILL_BYTE 0xAF 41 #define NUM_ITERATIONS 100 42 43 /* Shared control block: control reading threads and record stats */ 44 struct shared_ctl { 45 atomic_uint_fast32_t stop; 46 atomic_uint_fast64_t reader_failures; 47 atomic_uint_fast64_t reader_verified; 48 pthread_barrier_t barrier; 49 }; 50 51 static void fill_page(unsigned char *base, size_t page_idx) 52 { 53 unsigned char *page_ptr = base + page_idx * page_size; 54 uint64_t idx = (uint64_t)page_idx; 55 56 memset(page_ptr, FILL_BYTE, page_size); 57 memcpy(page_ptr, &idx, sizeof(idx)); 58 } 59 60 /* Returns true if valid, false if corrupted. */ 61 static bool check_page(unsigned char *base, uint64_t page_idx) 62 { 63 unsigned char *page_ptr = base + page_idx * page_size; 64 uint64_t expected_idx = (uint64_t)page_idx; 65 uint64_t got_idx; 66 67 memcpy(&got_idx, page_ptr, 8); 68 69 if (got_idx != expected_idx) { 70 uint64_t off; 71 int all_zero = 1; 72 73 for (off = 0; off < page_size; off++) { 74 if (page_ptr[off] != 0) { 75 all_zero = 0; 76 break; 77 } 78 } 79 if (all_zero) { 80 ksft_print_msg("CORRUPTED: page %" PRIu64 81 " (huge page %" PRIu64 82 ") is ALL ZEROS\n", 83 page_idx, 84 (page_idx * page_size) / pmd_pagesize); 85 } else { 86 ksft_print_msg("CORRUPTED: page %" PRIu64 87 " (huge page %" PRIu64 88 "): expected idx %" PRIu64 89 ", got %" PRIu64 "\n", 90 page_idx, 91 (page_idx * page_size) / pmd_pagesize, 92 page_idx, got_idx); 93 } 94 return false; 95 } 96 return true; 97 } 98 99 struct reader_arg { 100 unsigned char *base; 101 struct shared_ctl *ctl; 102 int tid; 103 atomic_uint_fast64_t *failures; 104 atomic_uint_fast64_t *verified; 105 }; 106 107 static void *reader_thread(void *arg) 108 { 109 struct reader_arg *ra = (struct reader_arg *)arg; 110 unsigned char *base = ra->base; 111 struct shared_ctl *ctl = ra->ctl; 112 int tid = ra->tid; 113 atomic_uint_fast64_t *failures = ra->failures; 114 atomic_uint_fast64_t *verified = ra->verified; 115 uint64_t page_idx; 116 117 pthread_barrier_wait(&ctl->barrier); 118 119 while (atomic_load_explicit(&ctl->stop, memory_order_acquire) == 0) { 120 for (page_idx = (size_t)tid; page_idx < TOTAL_PAGES; 121 page_idx += NUM_READER_THREADS) { 122 /* 123 * page_idx % PUNCH_INTERVAL is in [0, PUNCH_INTERVAL), 124 * skip [0, PUNCH_SIZE_FACTOR) 125 */ 126 if (page_idx % PUNCH_INTERVAL < PUNCH_SIZE_FACTOR) 127 continue; 128 if (check_page(base, page_idx)) 129 atomic_fetch_add_explicit(verified, 1, 130 memory_order_relaxed); 131 else 132 atomic_fetch_add_explicit(failures, 1, 133 memory_order_relaxed); 134 } 135 if (atomic_load_explicit(failures, memory_order_relaxed) > 0) 136 break; 137 } 138 139 return NULL; 140 } 141 142 static void create_readers(pthread_t *threads, struct reader_arg *args, 143 unsigned char *base, struct shared_ctl *ctl) 144 { 145 int i; 146 147 for (i = 0; i < NUM_READER_THREADS; i++) { 148 args[i].base = base; 149 args[i].ctl = ctl; 150 args[i].tid = i; 151 args[i].failures = &ctl->reader_failures; 152 args[i].verified = &ctl->reader_verified; 153 if (pthread_create(&threads[i], NULL, reader_thread, 154 &args[i]) != 0) 155 ksft_exit_fail_msg("pthread_create failed\n"); 156 } 157 } 158 159 /* Run a single iteration. Returns total number of corrupted pages. */ 160 static uint64_t run_iteration(void) 161 { 162 uint64_t reader_failures, reader_verified; 163 struct reader_arg args[NUM_READER_THREADS]; 164 pthread_t threads[NUM_READER_THREADS]; 165 unsigned char *mmap_base; 166 struct shared_ctl ctl; 167 uint64_t i; 168 169 memset(&ctl, 0, sizeof(struct shared_ctl)); 170 171 mmap_base = mmap(NULL, FILE_SIZE, PROT_READ | PROT_WRITE, 172 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 173 174 if (mmap_base == MAP_FAILED) 175 ksft_exit_fail_msg("mmap failed: %d\n", errno); 176 177 if (madvise(mmap_base, FILE_SIZE, MADV_HUGEPAGE) != 0) 178 ksft_exit_fail_msg("madvise(MADV_HUGEPAGE) failed: %d\n", 179 errno); 180 181 for (i = 0; i < TOTAL_PAGES; i++) 182 fill_page(mmap_base, i); 183 184 if (!check_huge_shmem(mmap_base, FILE_SIZE, NR_PMD_PAGE, pmd_pagesize)) 185 ksft_exit_fail_msg("No shmem THP is allocated\n"); 186 187 if (pthread_barrier_init(&ctl.barrier, NULL, NUM_READER_THREADS + 1) != 0) 188 ksft_exit_fail_msg("pthread_barrier_init failed\n"); 189 190 create_readers(threads, args, mmap_base, &ctl); 191 192 /* Wait for all reader threads to be ready before punching holes. */ 193 pthread_barrier_wait(&ctl.barrier); 194 195 for (i = 0; i < TOTAL_PAGES; i++) { 196 if (i % PUNCH_INTERVAL != 0) 197 continue; 198 if (madvise(mmap_base + i * page_size, 199 PUNCH_SIZE_FACTOR * page_size, MADV_REMOVE) != 0) { 200 ksft_exit_fail_msg( 201 "madvise(MADV_REMOVE) failed on page %" PRIu64 ": %d\n", 202 i, errno); 203 } 204 205 i += PUNCH_SIZE_FACTOR - 1; 206 } 207 208 atomic_store_explicit(&ctl.stop, 1, memory_order_release); 209 210 for (i = 0; i < NUM_READER_THREADS; i++) 211 pthread_join(threads[i], NULL); 212 213 pthread_barrier_destroy(&ctl.barrier); 214 215 reader_failures = atomic_load_explicit(&ctl.reader_failures, 216 memory_order_acquire); 217 reader_verified = atomic_load_explicit(&ctl.reader_verified, 218 memory_order_acquire); 219 if (reader_failures) 220 ksft_print_msg("Child: %" PRIu64 " pages verified, %" PRIu64 " failures\n", 221 reader_verified, reader_failures); 222 223 munmap(mmap_base, FILE_SIZE); 224 225 return reader_failures; 226 } 227 228 int main(void) 229 { 230 struct thp_settings current_settings; 231 uint64_t corrupted_pages; 232 uint64_t iter; 233 234 ksft_print_header(); 235 236 page_size = getpagesize(); 237 pmd_pagesize = read_pmd_pagesize(); 238 239 if (!thp_available() || !pmd_pagesize) 240 ksft_exit_skip("Transparent Hugepages not available\n"); 241 242 if (geteuid() != 0) 243 ksft_exit_skip("Please run the test as root\n"); 244 245 thp_save_settings(); 246 247 thp_read_settings(¤t_settings); 248 current_settings.shmem_enabled = SHMEM_ADVISE; 249 thp_write_settings(¤t_settings); 250 251 ksft_set_plan(1); 252 253 ksft_print_msg("folio split race test\n"); 254 255 for (iter = 0; iter < NUM_ITERATIONS; iter++) { 256 corrupted_pages = run_iteration(); 257 if (corrupted_pages > 0) 258 break; 259 } 260 261 if (iter < NUM_ITERATIONS) 262 ksft_test_result_fail("FAILED on iteration %" PRIu64 263 ": %" PRIu64 264 " pages corrupted by MADV_REMOVE!\n", 265 iter, corrupted_pages); 266 else 267 ksft_test_result_pass("All %d iterations passed\n", 268 NUM_ITERATIONS); 269 270 ksft_exit(iter == NUM_ITERATIONS); 271 } 272