1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * The main purpose of the tests here is to exercise the migration entry code
4 * paths in the kernel.
5 */
6
7 #include "kselftest_harness.h"
8 #include "hugepage_settings.h"
9
10 #include <string.h>
11 #include <pthread.h>
12 #include <numa.h>
13 #include <numaif.h>
14 #include <sys/mman.h>
15 #include <sys/prctl.h>
16 #include <sys/types.h>
17 #include <signal.h>
18 #include <time.h>
19 #include "vm_util.h"
20
21 #define TWOMEG (2<<20)
22 #define RUNTIME (20)
23 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
24
25 HUGETLB_SETUP_DEFAULT_PAGES(1)
26
FIXTURE(migration)27 FIXTURE(migration)
28 {
29 pthread_t *threads;
30 pid_t *pids;
31 int nthreads;
32 int n1;
33 int n2;
34 };
35
reset_signals(void)36 static void reset_signals(void)
37 {
38 struct sigaction sa = { .sa_handler = SIG_DFL };
39
40 sigemptyset(&sa.sa_mask);
41 sigaction(SIGTERM, &sa, NULL);
42 sigaction(SIGHUP, &sa, NULL);
43 sigaction(SIGINT, &sa, NULL);
44 sigaction(SIGQUIT, &sa, NULL);
45 }
46
FIXTURE_SETUP(migration)47 FIXTURE_SETUP(migration)
48 {
49 int n;
50
51 reset_signals();
52
53 if (numa_available() < 0)
54 SKIP(return, "NUMA not available");
55 self->nthreads = numa_num_task_cpus() - 2;
56 self->n1 = -1;
57 self->n2 = -1;
58
59 for (n = 0; n < numa_max_possible_node(); n++)
60 if (numa_bitmask_isbitset(numa_all_nodes_ptr, n)) {
61 if (self->n1 == -1) {
62 self->n1 = n;
63 } else {
64 self->n2 = n;
65 break;
66 }
67 }
68
69 if (self->nthreads < 1 || self->n1 < 0 || self->n2 < 0)
70 SKIP(return, "Not enough threads or NUMA nodes available");
71
72 self->threads = malloc(self->nthreads * sizeof(*self->threads));
73 ASSERT_NE(self->threads, NULL);
74 self->pids = malloc(self->nthreads * sizeof(*self->pids));
75 ASSERT_NE(self->pids, NULL);
76 };
77
FIXTURE_TEARDOWN(migration)78 FIXTURE_TEARDOWN(migration)
79 {
80 free(self->threads);
81 free(self->pids);
82 }
83
kill_children(FIXTURE_DATA (migration)* self)84 static bool kill_children(FIXTURE_DATA(migration) * self)
85 {
86 bool err = false;
87 pid_t pid;
88 int i;
89
90 for (i = 0; i < self->nthreads; i++) {
91 int status = 0;
92
93 pid = self->pids[i];
94 if (pid < 0)
95 continue;
96 if (kill(pid, SIGTERM))
97 err = true;
98 if (pid != waitpid(pid, &status, 0))
99 err = true;
100 if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGTERM)
101 err = true;
102 }
103
104 return !err;
105 }
106
migrate(uint64_t * ptr,int n1,int n2)107 int migrate(uint64_t *ptr, int n1, int n2)
108 {
109 int ret, tmp;
110 int status = 0;
111 struct timespec ts1, ts2;
112 int success = 0;
113
114 if (clock_gettime(CLOCK_MONOTONIC, &ts1))
115 return -1;
116
117 while (1) {
118 if (clock_gettime(CLOCK_MONOTONIC, &ts2))
119 return -1;
120
121 if (ts2.tv_sec - ts1.tv_sec >= RUNTIME) {
122 /* Reaching both targets verifies a cross-node move. */
123 if (success >= 2)
124 return 0;
125 else
126 return -2;
127 }
128
129 ret = move_pages(0, 1, (void **) &ptr, &n2, &status,
130 MPOL_MF_MOVE_ALL);
131 if (ret < 0) {
132 perror("Couldn't migrate pages");
133 return ret;
134 }
135 /* Migration is best effort. Try again */
136 if (ret > 0 || status < 0)
137 continue;
138 if (status != n2) {
139 printf("Page is on node %d instead of target node %d\n",
140 status, n2);
141 return -2;
142 }
143 success++;
144 tmp = n2;
145 n2 = n1;
146 n1 = tmp;
147 }
148 }
149
access_mem(void * ptr)150 void *access_mem(void *ptr)
151 {
152 while (1) {
153 pthread_testcancel();
154 /* Force a read from the memory pointed to by ptr. This ensures
155 * the memory access actually happens and prevents the compiler
156 * from optimizing away this entire loop.
157 */
158 FORCE_READ(*(uint64_t *)ptr);
159 }
160
161 return NULL;
162 }
163
164 /*
165 * Basic migration entry testing. One thread will move pages back and forth
166 * between nodes whilst other threads try and access them triggering the
167 * migration entry wait paths in the kernel.
168 */
169 TEST_F_TIMEOUT(migration, private_anon, 2*RUNTIME)
170 {
171 uint64_t *ptr;
172 int i;
173
174 ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE,
175 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
176 ASSERT_NE(ptr, MAP_FAILED);
177
178 memset(ptr, 0xde, TWOMEG);
179 for (i = 0; i < self->nthreads; i++)
180 if (pthread_create(&self->threads[i], NULL, access_mem, ptr))
181 perror("Couldn't create thread");
182
183 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
184 for (i = 0; i < self->nthreads; i++)
185 ASSERT_EQ(pthread_cancel(self->threads[i]), 0);
186 }
187
188 /*
189 * Same as the previous test but with shared memory.
190 */
191 TEST_F_TIMEOUT(migration, shared_anon, 2*RUNTIME)
192 {
193 pid_t pid;
194 uint64_t *ptr;
195 int i, err;
196
197 ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE,
198 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
199 ASSERT_NE(ptr, MAP_FAILED);
200
201 memset(ptr, 0xde, TWOMEG);
202 for (i = 0; i < self->nthreads; i++) {
203 pid = fork();
204 if (!pid) {
205 prctl(PR_SET_PDEATHSIG, SIGHUP);
206 /* Parent may have died before prctl so check now. */
207 if (getppid() == 1)
208 kill(getpid(), SIGHUP);
209 access_mem(ptr);
210 } else {
211 self->pids[i] = pid;
212 }
213 }
214
215 err = migrate(ptr, self->n1, self->n2);
216 ASSERT_EQ(kill_children(self), true);
217 ASSERT_EQ(err, 0);
218 }
219
220 /*
221 * Tests the pmd migration entry paths.
222 */
223 TEST_F_TIMEOUT(migration, private_anon_thp, 2*RUNTIME)
224 {
225 uint64_t pmdsize;
226 uint64_t *ptr;
227 int i;
228
229 if (!thp_is_enabled())
230 SKIP(return, "Transparent Hugepages not available");
231
232 pmdsize = read_pmd_pagesize();
233 if (!pmdsize)
234 SKIP(return, "Reading PMD pagesize failed");
235
236 ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE,
237 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
238 ASSERT_NE(ptr, MAP_FAILED);
239
240 ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize);
241 ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0);
242 memset(ptr, 0xde, pmdsize);
243 for (i = 0; i < self->nthreads; i++)
244 if (pthread_create(&self->threads[i], NULL, access_mem, ptr))
245 perror("Couldn't create thread");
246
247 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
248 for (i = 0; i < self->nthreads; i++)
249 ASSERT_EQ(pthread_cancel(self->threads[i]), 0);
250 }
251
252 /*
253 * migration test with shared anon THP page
254 */
255
256 TEST_F_TIMEOUT(migration, shared_anon_thp, 2*RUNTIME)
257 {
258 uint64_t pmdsize;
259 pid_t pid;
260 uint64_t *ptr;
261 int i, err;
262
263 if (!thp_is_enabled())
264 SKIP(return, "Transparent Hugepages not available");
265
266 pmdsize = read_pmd_pagesize();
267 if (!pmdsize)
268 SKIP(return, "Reading PMD pagesize failed");
269
270 ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE,
271 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
272 ASSERT_NE(ptr, MAP_FAILED);
273
274 ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize);
275 ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0);
276
277 memset(ptr, 0xde, pmdsize);
278 for (i = 0; i < self->nthreads; i++) {
279 pid = fork();
280 if (!pid) {
281 prctl(PR_SET_PDEATHSIG, SIGHUP);
282 /* Parent may have died before prctl so check now. */
283 if (getppid() == 1)
284 kill(getpid(), SIGHUP);
285 access_mem(ptr);
286 } else {
287 self->pids[i] = pid;
288 }
289 }
290
291 err = migrate(ptr, self->n1, self->n2);
292 ASSERT_EQ(kill_children(self), true);
293 ASSERT_EQ(err, 0);
294 }
295
296 /*
297 * migration test with private anon hugetlb page
298 */
299 TEST_F_TIMEOUT(migration, private_anon_htlb, 2*RUNTIME)
300 {
301 unsigned long hugepage_size;
302 uint64_t *ptr;
303 int i;
304
305 hugepage_size = default_huge_page_size();
306 if (!hugepage_size)
307 SKIP(return, "Reading HugeTLB pagesize failed");
308
309 if (hugetlb_free_default_pages() < 1)
310 SKIP(return, "Not enough huge pages");
311
312 ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE,
313 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
314 ASSERT_NE(ptr, MAP_FAILED);
315
316 memset(ptr, 0xde, hugepage_size);
317 for (i = 0; i < self->nthreads; i++)
318 if (pthread_create(&self->threads[i], NULL, access_mem, ptr))
319 perror("Couldn't create thread");
320
321 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
322 for (i = 0; i < self->nthreads; i++)
323 ASSERT_EQ(pthread_cancel(self->threads[i]), 0);
324 }
325
326 /*
327 * migration test with shared anon hugetlb page
328 */
329 TEST_F_TIMEOUT(migration, shared_anon_htlb, 2*RUNTIME)
330 {
331 unsigned long hugepage_size;
332 pid_t pid;
333 uint64_t *ptr;
334 int i, err;
335
336 hugepage_size = default_huge_page_size();
337 if (!hugepage_size)
338 SKIP(return, "Reading HugeTLB pagesize failed");
339
340 if (hugetlb_free_default_pages() < 1)
341 SKIP(return, "Not enough huge pages");
342
343 ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE,
344 MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
345 ASSERT_NE(ptr, MAP_FAILED);
346
347 memset(ptr, 0xde, hugepage_size);
348 for (i = 0; i < self->nthreads; i++) {
349 pid = fork();
350 if (!pid) {
351 prctl(PR_SET_PDEATHSIG, SIGHUP);
352 /* Parent may have died before prctl so check now. */
353 if (getppid() == 1)
354 kill(getpid(), SIGHUP);
355 access_mem(ptr);
356 } else {
357 self->pids[i] = pid;
358 }
359 }
360
361 err = migrate(ptr, self->n1, self->n2);
362 ASSERT_EQ(kill_children(self), true);
363 ASSERT_EQ(err, 0);
364 }
365
366 TEST_HARNESS_MAIN
367