1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * The main purpose of the tests here is to exercise the migration entry code 4 * paths in the kernel. 5 */ 6 7 #include "kselftest_harness.h" 8 #include "hugepage_settings.h" 9 10 #include <string.h> 11 #include <pthread.h> 12 #include <numa.h> 13 #include <numaif.h> 14 #include <sys/mman.h> 15 #include <sys/prctl.h> 16 #include <sys/types.h> 17 #include <signal.h> 18 #include <time.h> 19 #include "vm_util.h" 20 21 #define TWOMEG (2<<20) 22 #define RUNTIME (20) 23 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) 24 25 HUGETLB_SETUP_DEFAULT_PAGES(1) 26 27 FIXTURE(migration) 28 { 29 pthread_t *threads; 30 pid_t *pids; 31 int nthreads; 32 int n1; 33 int n2; 34 }; 35 36 static void reset_signals(void) 37 { 38 struct sigaction sa = { .sa_handler = SIG_DFL }; 39 40 sigemptyset(&sa.sa_mask); 41 sigaction(SIGTERM, &sa, NULL); 42 sigaction(SIGHUP, &sa, NULL); 43 sigaction(SIGINT, &sa, NULL); 44 sigaction(SIGQUIT, &sa, NULL); 45 } 46 47 FIXTURE_SETUP(migration) 48 { 49 int n; 50 51 reset_signals(); 52 53 if (numa_available() < 0) 54 SKIP(return, "NUMA not available"); 55 self->nthreads = numa_num_task_cpus() - 2; 56 self->n1 = -1; 57 self->n2 = -1; 58 59 for (n = 0; n < numa_max_possible_node(); n++) 60 if (numa_bitmask_isbitset(numa_all_nodes_ptr, n)) { 61 if (self->n1 == -1) { 62 self->n1 = n; 63 } else { 64 self->n2 = n; 65 break; 66 } 67 } 68 69 if (self->nthreads < 1 || self->n1 < 0 || self->n2 < 0) 70 SKIP(return, "Not enough threads or NUMA nodes available"); 71 72 self->threads = malloc(self->nthreads * sizeof(*self->threads)); 73 ASSERT_NE(self->threads, NULL); 74 self->pids = malloc(self->nthreads * sizeof(*self->pids)); 75 ASSERT_NE(self->pids, NULL); 76 }; 77 78 FIXTURE_TEARDOWN(migration) 79 { 80 free(self->threads); 81 free(self->pids); 82 } 83 84 static bool kill_children(FIXTURE_DATA(migration) * self) 85 { 86 bool err = false; 87 pid_t pid; 88 int i; 89 90 for (i = 0; i < self->nthreads; i++) { 91 int status = 0; 92 93 pid = self->pids[i]; 94 if (pid < 0) 95 continue; 96 if (kill(pid, SIGTERM)) 97 err = true; 98 if (pid != waitpid(pid, &status, 0)) 99 err = true; 100 if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGTERM) 101 err = true; 102 } 103 104 return !err; 105 } 106 107 int migrate(uint64_t *ptr, int n1, int n2) 108 { 109 int ret, tmp; 110 int status = 0; 111 struct timespec ts1, ts2; 112 int success = 0; 113 114 if (clock_gettime(CLOCK_MONOTONIC, &ts1)) 115 return -1; 116 117 while (1) { 118 if (clock_gettime(CLOCK_MONOTONIC, &ts2)) 119 return -1; 120 121 if (ts2.tv_sec - ts1.tv_sec >= RUNTIME) { 122 /* Reaching both targets verifies a cross-node move. */ 123 if (success >= 2) 124 return 0; 125 else 126 return -2; 127 } 128 129 ret = move_pages(0, 1, (void **) &ptr, &n2, &status, 130 MPOL_MF_MOVE_ALL); 131 if (ret < 0) { 132 perror("Couldn't migrate pages"); 133 return ret; 134 } 135 /* Migration is best effort. Try again */ 136 if (ret > 0 || status < 0) 137 continue; 138 if (status != n2) { 139 printf("Page is on node %d instead of target node %d\n", 140 status, n2); 141 return -2; 142 } 143 success++; 144 tmp = n2; 145 n2 = n1; 146 n1 = tmp; 147 } 148 } 149 150 void *access_mem(void *ptr) 151 { 152 while (1) { 153 pthread_testcancel(); 154 /* Force a read from the memory pointed to by ptr. This ensures 155 * the memory access actually happens and prevents the compiler 156 * from optimizing away this entire loop. 157 */ 158 FORCE_READ(*(uint64_t *)ptr); 159 } 160 161 return NULL; 162 } 163 164 /* 165 * Basic migration entry testing. One thread will move pages back and forth 166 * between nodes whilst other threads try and access them triggering the 167 * migration entry wait paths in the kernel. 168 */ 169 TEST_F_TIMEOUT(migration, private_anon, 2*RUNTIME) 170 { 171 uint64_t *ptr; 172 int i; 173 174 ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, 175 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 176 ASSERT_NE(ptr, MAP_FAILED); 177 178 memset(ptr, 0xde, TWOMEG); 179 for (i = 0; i < self->nthreads; i++) 180 if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) 181 perror("Couldn't create thread"); 182 183 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); 184 for (i = 0; i < self->nthreads; i++) 185 ASSERT_EQ(pthread_cancel(self->threads[i]), 0); 186 } 187 188 /* 189 * Same as the previous test but with shared memory. 190 */ 191 TEST_F_TIMEOUT(migration, shared_anon, 2*RUNTIME) 192 { 193 pid_t pid; 194 uint64_t *ptr; 195 int i, err; 196 197 ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, 198 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 199 ASSERT_NE(ptr, MAP_FAILED); 200 201 memset(ptr, 0xde, TWOMEG); 202 for (i = 0; i < self->nthreads; i++) { 203 pid = fork(); 204 if (!pid) { 205 prctl(PR_SET_PDEATHSIG, SIGHUP); 206 /* Parent may have died before prctl so check now. */ 207 if (getppid() == 1) 208 kill(getpid(), SIGHUP); 209 access_mem(ptr); 210 } else { 211 self->pids[i] = pid; 212 } 213 } 214 215 err = migrate(ptr, self->n1, self->n2); 216 ASSERT_EQ(kill_children(self), true); 217 ASSERT_EQ(err, 0); 218 } 219 220 /* 221 * Tests the pmd migration entry paths. 222 */ 223 TEST_F_TIMEOUT(migration, private_anon_thp, 2*RUNTIME) 224 { 225 uint64_t pmdsize; 226 uint64_t *ptr; 227 int i; 228 229 if (!thp_is_enabled()) 230 SKIP(return, "Transparent Hugepages not available"); 231 232 pmdsize = read_pmd_pagesize(); 233 if (!pmdsize) 234 SKIP(return, "Reading PMD pagesize failed"); 235 236 ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE, 237 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 238 ASSERT_NE(ptr, MAP_FAILED); 239 240 ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize); 241 ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0); 242 memset(ptr, 0xde, pmdsize); 243 for (i = 0; i < self->nthreads; i++) 244 if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) 245 perror("Couldn't create thread"); 246 247 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); 248 for (i = 0; i < self->nthreads; i++) 249 ASSERT_EQ(pthread_cancel(self->threads[i]), 0); 250 } 251 252 /* 253 * migration test with shared anon THP page 254 */ 255 256 TEST_F_TIMEOUT(migration, shared_anon_thp, 2*RUNTIME) 257 { 258 uint64_t pmdsize; 259 pid_t pid; 260 uint64_t *ptr; 261 int i, err; 262 263 if (!thp_is_enabled()) 264 SKIP(return, "Transparent Hugepages not available"); 265 266 pmdsize = read_pmd_pagesize(); 267 if (!pmdsize) 268 SKIP(return, "Reading PMD pagesize failed"); 269 270 ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE, 271 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 272 ASSERT_NE(ptr, MAP_FAILED); 273 274 ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize); 275 ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0); 276 277 memset(ptr, 0xde, pmdsize); 278 for (i = 0; i < self->nthreads; i++) { 279 pid = fork(); 280 if (!pid) { 281 prctl(PR_SET_PDEATHSIG, SIGHUP); 282 /* Parent may have died before prctl so check now. */ 283 if (getppid() == 1) 284 kill(getpid(), SIGHUP); 285 access_mem(ptr); 286 } else { 287 self->pids[i] = pid; 288 } 289 } 290 291 err = migrate(ptr, self->n1, self->n2); 292 ASSERT_EQ(kill_children(self), true); 293 ASSERT_EQ(err, 0); 294 } 295 296 /* 297 * migration test with private anon hugetlb page 298 */ 299 TEST_F_TIMEOUT(migration, private_anon_htlb, 2*RUNTIME) 300 { 301 unsigned long hugepage_size; 302 uint64_t *ptr; 303 int i; 304 305 hugepage_size = default_huge_page_size(); 306 if (!hugepage_size) 307 SKIP(return, "Reading HugeTLB pagesize failed"); 308 309 if (hugetlb_free_default_pages() < 1) 310 SKIP(return, "Not enough huge pages"); 311 312 ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE, 313 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); 314 ASSERT_NE(ptr, MAP_FAILED); 315 316 memset(ptr, 0xde, hugepage_size); 317 for (i = 0; i < self->nthreads; i++) 318 if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) 319 perror("Couldn't create thread"); 320 321 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); 322 for (i = 0; i < self->nthreads; i++) 323 ASSERT_EQ(pthread_cancel(self->threads[i]), 0); 324 } 325 326 /* 327 * migration test with shared anon hugetlb page 328 */ 329 TEST_F_TIMEOUT(migration, shared_anon_htlb, 2*RUNTIME) 330 { 331 unsigned long hugepage_size; 332 pid_t pid; 333 uint64_t *ptr; 334 int i, err; 335 336 hugepage_size = default_huge_page_size(); 337 if (!hugepage_size) 338 SKIP(return, "Reading HugeTLB pagesize failed"); 339 340 if (hugetlb_free_default_pages() < 1) 341 SKIP(return, "Not enough huge pages"); 342 343 ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE, 344 MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); 345 ASSERT_NE(ptr, MAP_FAILED); 346 347 memset(ptr, 0xde, hugepage_size); 348 for (i = 0; i < self->nthreads; i++) { 349 pid = fork(); 350 if (!pid) { 351 prctl(PR_SET_PDEATHSIG, SIGHUP); 352 /* Parent may have died before prctl so check now. */ 353 if (getppid() == 1) 354 kill(getpid(), SIGHUP); 355 access_mem(ptr); 356 } else { 357 self->pids[i] = pid; 358 } 359 } 360 361 err = migrate(ptr, self->n1, self->n2); 362 ASSERT_EQ(kill_children(self), true); 363 ASSERT_EQ(err, 0); 364 } 365 366 TEST_HARNESS_MAIN 367