1 // SPDX-License-Identifier: GPL-2.0
2
3 #define _GNU_SOURCE
4 #include <stdio.h>
5 #include <fcntl.h>
6 #include <string.h>
7 #include <sys/mman.h>
8 #include <errno.h>
9 #include <malloc.h>
10 #include <linux/types.h>
11 #include <linux/mman.h>
12 #include <linux/memfd.h>
13 #include <linux/userfaultfd.h>
14 #include <linux/fs.h>
15 #include <sys/ioctl.h>
16 #include <sys/stat.h>
17 #include <math.h>
18 #include <asm/unistd.h>
19 #include <pthread.h>
20 #include <sys/resource.h>
21 #include <assert.h>
22 #include <sys/ipc.h>
23 #include <sys/shm.h>
24
25 #include "vm_util.h"
26 #include "kselftest.h"
27 #include "hugepage_settings.h"
28
29 #define PAGEMAP_BITS_ALL (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \
30 PAGE_IS_FILE | PAGE_IS_PRESENT | \
31 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \
32 PAGE_IS_HUGE)
33 #define PAGEMAP_NON_WRITTEN_BITS (PAGE_IS_WPALLOWED | PAGE_IS_FILE | \
34 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | \
35 PAGE_IS_PFNZERO | PAGE_IS_HUGE)
36
37 #define TEST_ITERATIONS 100
38 #define PAGEMAP "/proc/self/pagemap"
39 int pagemap_fd;
40 int uffd;
41 size_t page_size;
42 size_t hpage_size;
43 const char *progname;
44
45 #define LEN(region) ((region.end - region.start)/page_size)
46
pagemap_ioctl(void * start,int len,void * vec,int vec_len,int flag,int max_pages,long required_mask,long anyof_mask,long excluded_mask,long return_mask)47 static long pagemap_ioctl(void *start, int len, void *vec, int vec_len, int flag,
48 int max_pages, long required_mask, long anyof_mask, long excluded_mask,
49 long return_mask)
50 {
51 struct pm_scan_arg arg;
52
53 arg.start = (uintptr_t)start;
54 arg.end = (uintptr_t)(start + len);
55 arg.vec = (uintptr_t)vec;
56 arg.vec_len = vec_len;
57 arg.flags = flag;
58 arg.size = sizeof(struct pm_scan_arg);
59 arg.max_pages = max_pages;
60 arg.category_mask = required_mask;
61 arg.category_anyof_mask = anyof_mask;
62 arg.category_inverted = excluded_mask;
63 arg.return_mask = return_mask;
64
65 return ioctl(pagemap_fd, PAGEMAP_SCAN, &arg);
66 }
67
pagemap_ioc(void * start,int len,void * vec,int vec_len,int flag,int max_pages,long required_mask,long anyof_mask,long excluded_mask,long return_mask,long * walk_end)68 static long pagemap_ioc(void *start, int len, void *vec, int vec_len, int flag,
69 int max_pages, long required_mask, long anyof_mask, long excluded_mask,
70 long return_mask, long *walk_end)
71 {
72 struct pm_scan_arg arg;
73 int ret;
74
75 arg.start = (uintptr_t)start;
76 arg.end = (uintptr_t)(start + len);
77 arg.vec = (uintptr_t)vec;
78 arg.vec_len = vec_len;
79 arg.flags = flag;
80 arg.size = sizeof(struct pm_scan_arg);
81 arg.max_pages = max_pages;
82 arg.category_mask = required_mask;
83 arg.category_anyof_mask = anyof_mask;
84 arg.category_inverted = excluded_mask;
85 arg.return_mask = return_mask;
86
87 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg);
88
89 if (walk_end)
90 *walk_end = arg.walk_end;
91
92 return ret;
93 }
94
95
init_uffd(void)96 int init_uffd(void)
97 {
98 struct uffdio_api uffdio_api;
99
100 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY);
101 if (uffd == -1)
102 return uffd;
103
104 uffdio_api.api = UFFD_API;
105 uffdio_api.features = UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC |
106 UFFD_FEATURE_WP_HUGETLBFS_SHMEM;
107 if (ioctl(uffd, UFFDIO_API, &uffdio_api))
108 return -1;
109
110 if (!(uffdio_api.api & UFFDIO_REGISTER_MODE_WP) ||
111 !(uffdio_api.features & UFFD_FEATURE_WP_UNPOPULATED) ||
112 !(uffdio_api.features & UFFD_FEATURE_WP_ASYNC) ||
113 !(uffdio_api.features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM))
114 return -1;
115
116 return 0;
117 }
118
wp_init(void * addr,long size)119 int wp_init(void *addr, long size)
120 {
121 struct uffdio_register uffdio_register;
122 struct uffdio_writeprotect wp;
123
124 uffdio_register.range.start = (unsigned long)addr;
125 uffdio_register.range.len = size;
126 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP;
127 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
128 ksft_exit_fail_msg("ioctl(UFFDIO_REGISTER) %d %s\n", errno, strerror(errno));
129
130 if (!(uffdio_register.ioctls & UFFDIO_WRITEPROTECT))
131 ksft_exit_fail_msg("ioctl set is incorrect\n");
132
133 wp.range.start = (unsigned long)addr;
134 wp.range.len = size;
135 wp.mode = UFFDIO_WRITEPROTECT_MODE_WP;
136
137 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &wp))
138 ksft_exit_fail_msg("ioctl(UFFDIO_WRITEPROTECT)\n");
139
140 return 0;
141 }
142
wp_free(void * addr,long size)143 int wp_free(void *addr, long size)
144 {
145 struct uffdio_register uffdio_register;
146
147 uffdio_register.range.start = (unsigned long)addr;
148 uffdio_register.range.len = size;
149 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP;
150 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range))
151 ksft_exit_fail_msg("ioctl unregister failure\n");
152 return 0;
153 }
154
wp_addr_range(void * addr,int size)155 int wp_addr_range(void *addr, int size)
156 {
157 if (pagemap_ioctl(addr, size, NULL, 0,
158 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
159 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0)
160 ksft_exit_fail_msg("error %d %d %s\n", 1, errno, strerror(errno));
161
162 return 0;
163 }
164
gethugetlb_mem(int size,int * shmid)165 void *gethugetlb_mem(int size, int *shmid)
166 {
167 char *mem;
168
169 if (shmid) {
170 *shmid = shmget(2, size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W);
171 if (*shmid < 0)
172 return NULL;
173
174 mem = shmat(*shmid, 0, 0);
175 if (mem == (char *)-1) {
176 shmctl(*shmid, IPC_RMID, NULL);
177 ksft_exit_fail_msg("Shared memory attach failure\n");
178 }
179 } else {
180 mem = mmap(NULL, size, PROT_READ | PROT_WRITE,
181 MAP_ANONYMOUS | MAP_HUGETLB | MAP_PRIVATE, -1, 0);
182 if (mem == MAP_FAILED)
183 return NULL;
184 }
185
186 return mem;
187 }
188
userfaultfd_tests(void)189 int userfaultfd_tests(void)
190 {
191 long mem_size, vec_size, written, num_pages = 16;
192 char *mem, *vec;
193
194 mem_size = num_pages * page_size;
195 mem = mmap(NULL, mem_size, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0);
196 if (mem == MAP_FAILED)
197 ksft_exit_fail_msg("error nomem\n");
198
199 wp_init(mem, mem_size);
200
201 /* Change protection of pages differently */
202 mprotect(mem, mem_size/8, PROT_READ|PROT_WRITE);
203 mprotect(mem + 1 * mem_size/8, mem_size/8, PROT_READ);
204 mprotect(mem + 2 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE);
205 mprotect(mem + 3 * mem_size/8, mem_size/8, PROT_READ);
206 mprotect(mem + 4 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE);
207 mprotect(mem + 5 * mem_size/8, mem_size/8, PROT_NONE);
208 mprotect(mem + 6 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE);
209 mprotect(mem + 7 * mem_size/8, mem_size/8, PROT_READ);
210
211 wp_addr_range(mem + (mem_size/16), mem_size - 2 * (mem_size/8));
212 wp_addr_range(mem, mem_size);
213
214 vec_size = mem_size/page_size;
215 vec = calloc(vec_size, sizeof(struct page_region));
216 if (!vec)
217 ksft_exit_fail_msg("error nomem\n");
218
219 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
220 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
221 if (written < 0)
222 ksft_exit_fail_msg("error %ld %d %s\n", written, errno, strerror(errno));
223
224 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", __func__);
225
226 wp_free(mem, mem_size);
227 munmap(mem, mem_size);
228 free(vec);
229 return 0;
230 }
231
get_reads(struct page_region * vec,int vec_size)232 int get_reads(struct page_region *vec, int vec_size)
233 {
234 int i, sum = 0;
235
236 for (i = 0; i < vec_size; i++)
237 sum += LEN(vec[i]);
238
239 return sum;
240 }
241
sanity_tests_sd(void)242 int sanity_tests_sd(void)
243 {
244 unsigned long long mem_size, vec_size, i, total_pages = 0;
245 long ret, ret2, ret3;
246 int num_pages = 1000;
247 int total_writes, total_reads, reads, count;
248 struct page_region *vec, *vec2;
249 char *mem, *m[2];
250 long walk_end;
251
252 vec_size = num_pages/2;
253 mem_size = num_pages * page_size;
254
255 vec = calloc(vec_size, sizeof(struct page_region));
256 if (!vec)
257 ksft_exit_fail_msg("error nomem\n");
258
259 vec2 = calloc(vec_size, sizeof(struct page_region));
260 if (!vec2)
261 ksft_exit_fail_msg("error nomem\n");
262
263 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
264 if (mem == MAP_FAILED)
265 ksft_exit_fail_msg("error nomem\n");
266
267 wp_init(mem, mem_size);
268 wp_addr_range(mem, mem_size);
269
270 /* 1. wrong operation */
271 ksft_test_result(pagemap_ioctl(mem, 0, vec, vec_size, 0,
272 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0,
273 "%s Zero range size is valid\n", __func__);
274
275 ksft_test_result(pagemap_ioctl(mem, mem_size, NULL, vec_size, 0,
276 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) < 0,
277 "%s output buffer must be specified with size\n", __func__);
278
279 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, 0, 0,
280 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0,
281 "%s output buffer can be 0\n", __func__);
282
283 ksft_test_result(pagemap_ioctl(mem, mem_size, 0, 0, 0,
284 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0,
285 "%s output buffer can be 0\n", __func__);
286
287 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, -1,
288 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0,
289 "%s wrong flag specified\n", __func__);
290
291 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,
292 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC | 0xFF,
293 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0,
294 "%s flag has extra bits specified\n", __func__);
295
296 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0,
297 0, 0, 0, 0, PAGE_IS_WRITTEN) >= 0,
298 "%s no selection mask is specified\n", __func__);
299
300 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0,
301 0, PAGE_IS_WRITTEN, PAGE_IS_WRITTEN, 0, 0) == 0,
302 "%s no return mask is specified\n", __func__);
303
304 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0,
305 0, PAGE_IS_WRITTEN, 0, 0, 0x1000) < 0,
306 "%s wrong return mask specified\n", __func__);
307
308 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,
309 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
310 0, 0xFFF, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN) < 0,
311 "%s mixture of correct and wrong flag\n", __func__);
312
313 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,
314 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
315 0, 0, 0, PAGEMAP_BITS_ALL, PAGE_IS_WRITTEN) >= 0,
316 "%s PAGEMAP_BITS_ALL can be specified with PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n",
317 __func__);
318
319 /* 2. Clear area with larger vec size */
320 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
321 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0,
322 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
323 ksft_test_result(ret >= 0, "%s Clear area with larger vec size\n", __func__);
324
325 /* 3. Repeated pattern of written and non-written pages */
326 for (i = 0; i < mem_size; i += 2 * page_size)
327 mem[i]++;
328
329 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 0,
330 0, PAGE_IS_WRITTEN);
331 if (ret < 0)
332 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
333
334 ksft_test_result((unsigned long long)ret == mem_size/(page_size * 2),
335 "%s Repeated pattern of written and non-written pages\n", __func__);
336
337 /* 4. Repeated pattern of written and non-written pages in parts */
338 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
339 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
340 num_pages/2 - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
341 if (ret < 0)
342 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
343
344 ret2 = pagemap_ioctl(mem, mem_size, vec, 2, 0, 0, PAGE_IS_WRITTEN, 0, 0,
345 PAGE_IS_WRITTEN);
346 if (ret2 < 0)
347 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno));
348
349 ret3 = pagemap_ioctl(mem, mem_size, vec, vec_size,
350 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
351 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
352 if (ret3 < 0)
353 ksft_exit_fail_msg("error %ld %d %s\n", ret3, errno, strerror(errno));
354
355 ksft_test_result((ret + ret3) == num_pages/2 && ret2 == 2,
356 "%s Repeated pattern of written and non-written pages in parts %ld %ld %ld\n",
357 __func__, ret, ret3, ret2);
358
359 /* 5. Repeated pattern of written and non-written pages max_pages */
360 for (i = 0; i < mem_size; i += 2 * page_size)
361 mem[i]++;
362 mem[(mem_size/page_size - 1) * page_size]++;
363
364 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
365 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
366 num_pages/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
367 if (ret < 0)
368 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
369
370 ret2 = pagemap_ioctl(mem, mem_size, vec, vec_size,
371 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
372 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
373 if (ret2 < 0)
374 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno));
375
376 ksft_test_result(ret == num_pages/2 && ret2 == 1,
377 "%s Repeated pattern of written and non-written pages max_pages\n",
378 __func__);
379
380 /* 6. only get 2 dirty pages and clear them as well */
381 vec_size = mem_size/page_size;
382 memset(mem, -1, mem_size);
383
384 /* get and clear second and third pages */
385 ret = pagemap_ioctl(mem + page_size, 2 * page_size, vec, 1,
386 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
387 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
388 if (ret < 0)
389 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
390
391 ret2 = pagemap_ioctl(mem, mem_size, vec2, vec_size, 0, 0,
392 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
393 if (ret2 < 0)
394 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno));
395
396 ksft_test_result(ret == 1 && LEN(vec[0]) == 2 &&
397 vec[0].start == (uintptr_t)(mem + page_size) &&
398 ret2 == 2 && LEN(vec2[0]) == 1 && vec2[0].start == (uintptr_t)mem &&
399 LEN(vec2[1]) == vec_size - 3 &&
400 vec2[1].start == (uintptr_t)(mem + 3 * page_size),
401 "%s only get 2 written pages and clear them as well\n", __func__);
402
403 wp_free(mem, mem_size);
404 munmap(mem, mem_size);
405
406 /* 7. Two regions */
407 m[0] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
408 if (m[0] == MAP_FAILED)
409 ksft_exit_fail_msg("error nomem\n");
410 m[1] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
411 if (m[1] == MAP_FAILED)
412 ksft_exit_fail_msg("error nomem\n");
413
414 wp_init(m[0], mem_size);
415 wp_init(m[1], mem_size);
416 wp_addr_range(m[0], mem_size);
417 wp_addr_range(m[1], mem_size);
418
419 memset(m[0], 'a', mem_size);
420 memset(m[1], 'b', mem_size);
421
422 wp_addr_range(m[0], mem_size);
423
424 ret = pagemap_ioctl(m[1], mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0,
425 PAGE_IS_WRITTEN);
426 if (ret < 0)
427 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
428
429 ksft_test_result(ret == 1 && LEN(vec[0]) == mem_size/page_size,
430 "%s Two regions\n", __func__);
431
432 wp_free(m[0], mem_size);
433 wp_free(m[1], mem_size);
434 munmap(m[0], mem_size);
435 munmap(m[1], mem_size);
436
437 free(vec);
438 free(vec2);
439
440 /* 8. Smaller vec */
441 mem_size = 1050 * page_size;
442 vec_size = mem_size/(page_size*2);
443
444 vec = calloc(vec_size, sizeof(struct page_region));
445 if (!vec)
446 ksft_exit_fail_msg("error nomem\n");
447
448 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
449 if (mem == MAP_FAILED)
450 ksft_exit_fail_msg("error nomem\n");
451
452 wp_init(mem, mem_size);
453 wp_addr_range(mem, mem_size);
454
455 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
456 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0,
457 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
458 if (ret < 0)
459 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
460
461 for (i = 0; i < mem_size/page_size; i += 2)
462 mem[i * page_size]++;
463
464 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
465 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
466 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
467 if (ret < 0)
468 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
469
470 total_pages += ret;
471
472 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
473 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
474 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
475 if (ret < 0)
476 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
477
478 total_pages += ret;
479
480 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
481 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
482 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
483 if (ret < 0)
484 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
485
486 total_pages += ret;
487
488 ksft_test_result(total_pages == mem_size/(page_size*2), "%s Smaller max_pages\n", __func__);
489
490 free(vec);
491 wp_free(mem, mem_size);
492 munmap(mem, mem_size);
493 total_pages = 0;
494
495 /* 9. Smaller vec */
496 mem_size = 10000 * page_size;
497 vec_size = 50;
498
499 vec = calloc(vec_size, sizeof(struct page_region));
500 if (!vec)
501 ksft_exit_fail_msg("error nomem\n");
502
503 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
504 if (mem == MAP_FAILED)
505 ksft_exit_fail_msg("error nomem\n");
506
507 wp_init(mem, mem_size);
508 wp_addr_range(mem, mem_size);
509
510 for (count = 0; count < TEST_ITERATIONS; count++) {
511 total_writes = total_reads = 0;
512 walk_end = (long)mem;
513
514 for (i = 0; i < mem_size; i += page_size) {
515 if (rand() % 2) {
516 mem[i]++;
517 total_writes++;
518 }
519 }
520
521 while (total_reads < total_writes) {
522 ret = pagemap_ioc((void *)walk_end, mem_size-(walk_end - (long)mem), vec,
523 vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
524 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
525 if (ret < 0)
526 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
527
528 if ((unsigned long)ret > vec_size)
529 break;
530
531 reads = get_reads(vec, ret);
532 total_reads += reads;
533 }
534
535 if (total_reads != total_writes)
536 break;
537 }
538
539 ksft_test_result(count == TEST_ITERATIONS, "Smaller vec\n");
540
541 free(vec);
542 wp_free(mem, mem_size);
543 munmap(mem, mem_size);
544
545 /* 10. Walk_end tester */
546 vec_size = 1000;
547 mem_size = vec_size * page_size;
548
549 vec = calloc(vec_size, sizeof(struct page_region));
550 if (!vec)
551 ksft_exit_fail_msg("error nomem\n");
552
553 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
554 if (mem == MAP_FAILED)
555 ksft_exit_fail_msg("error nomem\n");
556
557 wp_init(mem, mem_size);
558 wp_addr_range(mem, mem_size);
559
560 memset(mem, 0, mem_size);
561
562 ret = pagemap_ioc(mem, 0, vec, vec_size, 0,
563 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
564 if (ret < 0)
565 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
566 ksft_test_result(ret == 0 && walk_end == (long)mem,
567 "Walk_end: Same start and end address\n");
568
569 ret = pagemap_ioc(mem, 0, vec, vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
570 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
571 if (ret < 0)
572 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
573 ksft_test_result(ret == 0 && walk_end == (long)mem,
574 "Walk_end: Same start and end with WP\n");
575
576 ret = pagemap_ioc(mem, 0, vec, 0, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
577 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
578 if (ret < 0)
579 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
580 ksft_test_result(ret == 0 && walk_end == (long)mem,
581 "Walk_end: Same start and end with 0 output buffer\n");
582
583 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
584 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
585 if (ret < 0)
586 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
587 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),
588 "Walk_end: Big vec\n");
589
590 ret = pagemap_ioc(mem, mem_size, vec, 1, 0,
591 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
592 if (ret < 0)
593 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
594 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),
595 "Walk_end: vec of minimum length\n");
596
597 ret = pagemap_ioc(mem, mem_size, vec, 1, 0,
598 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
599 if (ret < 0)
600 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
601 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),
602 "Walk_end: Max pages specified\n");
603
604 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
605 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
606 if (ret < 0)
607 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
608 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size/2),
609 "Walk_end: Half max pages\n");
610
611 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
612 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
613 if (ret < 0)
614 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
615 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size),
616 "Walk_end: 1 max page\n");
617
618 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
619 -1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
620 if (ret < 0)
621 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
622 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),
623 "Walk_end: max pages\n");
624
625 wp_addr_range(mem, mem_size);
626 for (i = 0; i < mem_size; i += 2 * page_size)
627 mem[i]++;
628
629 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
630 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
631 if (ret < 0)
632 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
633 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),
634 "Walk_end sparse: Big vec\n");
635
636 ret = pagemap_ioc(mem, mem_size, vec, 1, 0,
637 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
638 if (ret < 0)
639 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
640 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2),
641 "Walk_end sparse: vec of minimum length\n");
642
643 ret = pagemap_ioc(mem, mem_size, vec, 1, 0,
644 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
645 if (ret < 0)
646 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
647 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2),
648 "Walk_end sparse: Max pages specified\n");
649
650 ret = pagemap_ioc(mem, mem_size, vec, vec_size/2, 0,
651 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
652 if (ret < 0)
653 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
654 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),
655 "Walk_end sparse: Max pages specified\n");
656
657 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
658 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
659 if (ret < 0)
660 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
661 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),
662 "Walk_end sparse: Max pages specified\n");
663
664 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
665 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
666 if (ret < 0)
667 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
668 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),
669 "Walk_endsparse : Half max pages\n");
670
671 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
672 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
673 if (ret < 0)
674 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
675 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2),
676 "Walk_end: 1 max page\n");
677
678 free(vec);
679 wp_free(mem, mem_size);
680 munmap(mem, mem_size);
681
682 return 0;
683 }
684
base_tests(char * prefix,char * mem,unsigned long long mem_size,int skip)685 int base_tests(char *prefix, char *mem, unsigned long long mem_size, int skip)
686 {
687 unsigned long long vec_size;
688 int written;
689 struct page_region *vec, *vec2;
690
691 if (skip) {
692 ksft_test_result_skip("%s all new pages must not be written (dirty)\n", prefix);
693 ksft_test_result_skip("%s all pages must be written (dirty)\n", prefix);
694 ksft_test_result_skip("%s all pages dirty other than first and the last one\n",
695 prefix);
696 ksft_test_result_skip("%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix);
697 ksft_test_result_skip("%s only middle page dirty\n", prefix);
698 ksft_test_result_skip("%s only two middle pages dirty\n", prefix);
699 return 0;
700 }
701
702 vec_size = mem_size/page_size;
703 vec = calloc(vec_size, sizeof(struct page_region));
704 vec2 = calloc(vec_size, sizeof(struct page_region));
705 if (!vec || !vec2)
706 ksft_exit_fail_msg("error nomem\n");
707
708 /* 1. all new pages must be not be written (dirty) */
709 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
710 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
711 if (written < 0)
712 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
713
714 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", prefix);
715
716 /* 2. all pages must be written */
717 memset(mem, -1, mem_size);
718
719 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0,
720 PAGE_IS_WRITTEN);
721 if (written < 0)
722 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
723
724 ksft_test_result(written == 1 && LEN(vec[0]) == mem_size/page_size,
725 "%s all pages must be written (dirty)\n", prefix);
726
727 /* 3. all pages dirty other than first and the last one */
728 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
729 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
730 if (written < 0)
731 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
732
733 memset(mem + page_size, 0, mem_size - (2 * page_size));
734
735 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
736 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
737 if (written < 0)
738 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
739
740 ksft_test_result(written == 1 && LEN(vec[0]) >= vec_size - 2 && LEN(vec[0]) <= vec_size,
741 "%s all pages dirty other than first and the last one\n", prefix);
742
743 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0,
744 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
745 if (written < 0)
746 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
747
748 ksft_test_result(written == 0,
749 "%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix);
750
751 /* 4. only middle page dirty */
752 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
753 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
754 if (written < 0)
755 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
756
757 mem[vec_size/2 * page_size]++;
758
759 written = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN,
760 0, 0, PAGE_IS_WRITTEN);
761 if (written < 0)
762 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
763
764 ksft_test_result(written == 1 && LEN(vec[0]) >= 1,
765 "%s only middle page dirty\n", prefix);
766
767 /* 5. only two middle pages dirty and walk over only middle pages */
768 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
769 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE);
770 if (written < 0)
771 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
772
773 mem[vec_size/2 * page_size]++;
774 mem[(vec_size/2 + 1) * page_size]++;
775
776 written = pagemap_ioctl(&mem[vec_size/2 * page_size], 2 * page_size, vec, 1, 0,
777 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE);
778 if (written < 0)
779 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
780
781 ksft_test_result(written == 1 && vec[0].start == (uintptr_t)(&mem[vec_size/2 * page_size])
782 && LEN(vec[0]) == 2,
783 "%s only two middle pages dirty\n", prefix);
784
785 free(vec);
786 free(vec2);
787 return 0;
788 }
789
gethugepage(int map_size)790 void *gethugepage(int map_size)
791 {
792 int ret;
793 char *map;
794
795 map = memalign(hpage_size, map_size);
796 if (!map)
797 ksft_exit_fail_msg("memalign failed %d %s\n", errno, strerror(errno));
798
799 ret = madvise(map, map_size, MADV_HUGEPAGE);
800 if (ret)
801 return NULL;
802
803 memset(map, 0, map_size);
804
805 return map;
806 }
807
hpage_unit_tests(void)808 int hpage_unit_tests(void)
809 {
810 char *map;
811 int ret, ret2;
812 size_t num_pages = 10;
813 unsigned long long map_size = hpage_size * num_pages;
814 unsigned long long vec_size = map_size/page_size;
815 struct page_region *vec, *vec2;
816
817 vec = calloc(vec_size, sizeof(struct page_region));
818 vec2 = calloc(vec_size, sizeof(struct page_region));
819 if (!vec || !vec2)
820 ksft_exit_fail_msg("malloc failed\n");
821
822 map = gethugepage(map_size);
823 if (map) {
824 wp_init(map, map_size);
825 wp_addr_range(map, map_size);
826
827 /* 1. all new huge page must not be written (dirty) */
828 ret = pagemap_ioctl(map, map_size, vec, vec_size,
829 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0,
830 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
831 if (ret < 0)
832 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
833
834 ksft_test_result(ret == 0, "%s all new huge page must not be written (dirty)\n",
835 __func__);
836
837 /* 2. all the huge page must not be written */
838 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
839 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
840 if (ret < 0)
841 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
842
843 ksft_test_result(ret == 0, "%s all the huge page must not be written\n", __func__);
844
845 /* 3. all the huge page must be written and clear dirty as well */
846 memset(map, -1, map_size);
847 ret = pagemap_ioctl(map, map_size, vec, vec_size,
848 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
849 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
850 if (ret < 0)
851 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
852
853 ksft_test_result(ret == 1 && vec[0].start == (uintptr_t)map &&
854 LEN(vec[0]) == vec_size && vec[0].categories == PAGE_IS_WRITTEN,
855 "%s all the huge page must be written and clear\n", __func__);
856
857 /* 4. only middle page written */
858 wp_free(map, map_size);
859 free(map);
860 map = gethugepage(map_size);
861 wp_init(map, map_size);
862 wp_addr_range(map, map_size);
863 map[vec_size/2 * page_size]++;
864
865 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
866 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
867 if (ret < 0)
868 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
869
870 ksft_test_result(ret == 1 && LEN(vec[0]) > 0,
871 "%s only middle page written\n", __func__);
872
873 wp_free(map, map_size);
874 free(map);
875 } else {
876 ksft_test_result_skip("%s all new huge page must be written\n", __func__);
877 ksft_test_result_skip("%s all the huge page must not be written\n", __func__);
878 ksft_test_result_skip("%s all the huge page must be written and clear\n", __func__);
879 ksft_test_result_skip("%s only middle page written\n", __func__);
880 }
881
882 /* 5. clear first half of huge page */
883 map = gethugepage(map_size);
884 if (map) {
885 wp_init(map, map_size);
886 wp_addr_range(map, map_size);
887
888 memset(map, 0, map_size);
889
890 wp_addr_range(map, map_size/2);
891
892 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
893 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
894 if (ret < 0)
895 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
896
897 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 &&
898 vec[0].start == (uintptr_t)(map + map_size/2),
899 "%s clear first half of huge page\n", __func__);
900 wp_free(map, map_size);
901 free(map);
902 } else {
903 ksft_test_result_skip("%s clear first half of huge page\n", __func__);
904 }
905
906 /* 6. clear first half of huge page with limited buffer */
907 map = gethugepage(map_size);
908 if (map) {
909 wp_init(map, map_size);
910 wp_addr_range(map, map_size);
911
912 memset(map, 0, map_size);
913
914 ret = pagemap_ioctl(map, map_size, vec, vec_size,
915 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
916 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
917 if (ret < 0)
918 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
919
920 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
921 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
922 if (ret < 0)
923 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
924
925 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 &&
926 vec[0].start == (uintptr_t)(map + map_size/2),
927 "%s clear first half of huge page with limited buffer\n",
928 __func__);
929 wp_free(map, map_size);
930 free(map);
931 } else {
932 ksft_test_result_skip("%s clear first half of huge page with limited buffer\n",
933 __func__);
934 }
935
936 /* 7. clear second half of huge page */
937 map = gethugepage(map_size);
938 if (map) {
939 wp_init(map, map_size);
940 wp_addr_range(map, map_size);
941
942 memset(map, -1, map_size);
943
944 ret = pagemap_ioctl(map + map_size/2, map_size/2, vec, vec_size,
945 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, vec_size/2,
946 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
947 if (ret < 0)
948 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
949
950 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
951 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
952 if (ret < 0)
953 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
954
955 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2,
956 "%s clear second half huge page\n", __func__);
957 wp_free(map, map_size);
958 free(map);
959 } else {
960 ksft_test_result_skip("%s clear second half huge page\n", __func__);
961 }
962
963 /* 8. get half huge page */
964 map = gethugepage(map_size);
965 if (map) {
966 wp_init(map, map_size);
967 wp_addr_range(map, map_size);
968
969 memset(map, -1, map_size);
970 usleep(100);
971
972 ret = pagemap_ioctl(map, map_size, vec, 1,
973 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
974 hpage_size/(2*page_size), PAGE_IS_WRITTEN, 0, 0,
975 PAGE_IS_WRITTEN);
976 if (ret < 0)
977 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
978
979 ksft_test_result(ret == 1 && LEN(vec[0]) == hpage_size/(2*page_size),
980 "%s get half huge page\n", __func__);
981
982 ret2 = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
983 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
984 if (ret2 < 0)
985 ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno));
986
987 ksft_test_result(ret2 == 1 && LEN(vec[0]) == (map_size - hpage_size/2)/page_size,
988 "%s get half huge page\n", __func__);
989
990 wp_free(map, map_size);
991 free(map);
992 } else {
993 ksft_test_result_skip("%s get half huge page\n", __func__);
994 ksft_test_result_skip("%s get half huge page\n", __func__);
995 }
996
997 free(vec);
998 free(vec2);
999 return 0;
1000 }
1001
unmapped_region_tests(void)1002 int unmapped_region_tests(void)
1003 {
1004 void *start = (void *)0x10000000;
1005 int written, len = 0x00040000;
1006 long vec_size = len / page_size;
1007 struct page_region *vec = calloc(vec_size, sizeof(struct page_region));
1008 if (!vec)
1009 ksft_exit_fail_msg("error nomem\n");
1010
1011 /* 1. Get written pages */
1012 written = pagemap_ioctl(start, len, vec, vec_size, 0, 0,
1013 PAGEMAP_NON_WRITTEN_BITS, 0, 0, PAGEMAP_NON_WRITTEN_BITS);
1014 if (written < 0)
1015 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
1016
1017 ksft_test_result(written >= 0, "%s Get status of pages\n", __func__);
1018
1019 free(vec);
1020 return 0;
1021 }
1022
test_simple(void)1023 static void test_simple(void)
1024 {
1025 int i;
1026 char *map;
1027 struct page_region vec;
1028
1029 map = aligned_alloc(page_size, page_size);
1030 if (!map)
1031 ksft_exit_fail_msg("aligned_alloc failed\n");
1032
1033 wp_init(map, page_size);
1034 wp_addr_range(map, page_size);
1035
1036 for (i = 0 ; i < TEST_ITERATIONS; i++) {
1037 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0,
1038 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 1) {
1039 ksft_print_msg("written bit was 1, but should be 0 (i=%d)\n", i);
1040 break;
1041 }
1042
1043 wp_addr_range(map, page_size);
1044 /* Write something to the page to get the written bit enabled on the page */
1045 map[0]++;
1046
1047 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0,
1048 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0) {
1049 ksft_print_msg("written bit was 0, but should be 1 (i=%d)\n", i);
1050 break;
1051 }
1052
1053 wp_addr_range(map, page_size);
1054 }
1055 wp_free(map, page_size);
1056 free(map);
1057
1058 ksft_test_result(i == TEST_ITERATIONS, "Test %s\n", __func__);
1059 }
1060
1061 /*
1062 * A range that was populated and then MADV_DONTNEED'd is genuine pte_none
1063 * with no uffd-wp marker. Such a pte must read the same regardless of which
1064 * PAGEMAP_SCAN path serves the request: both the PAGE_IS_WRITTEN fast path and
1065 * the generic path (reached e.g. via category_anyof_mask) must report every
1066 * page written.
1067 */
1068 /*
1069 * Populate @mem (optionally collapsing it into a THP first), drop it with
1070 * MADV_DONTNEED, then check PAGEMAP_SCAN reports the whole range written via
1071 * both the fast and generic query paths. A dropped THP leaves a pmd_none hole
1072 * with no page table, exercising pagemap_scan_pte_hole(); a base-page range
1073 * leaves pte_none entries.
1074 */
unpopulated_written_test(const char * name,char * mem,long size,bool use_thp)1075 static void unpopulated_written_test(const char *name, char *mem, long size,
1076 bool use_thp)
1077 {
1078 long npages = size / page_size, fast = 0, slow = 0, ret;
1079 struct page_region regions[16];
1080 int i;
1081
1082 wp_init(mem, size);
1083
1084 /* Populate, optionally collapse to a THP, then drop it. */
1085 memset(mem, 1, size);
1086 if (use_thp &&
1087 (madvise(mem, size, MADV_COLLAPSE) ||
1088 !check_huge_anon(mem, size, size / hpage_size, hpage_size))) {
1089 ksft_test_result_skip("%s could not form a THP\n", name);
1090 goto out;
1091 }
1092 if (madvise(mem, size, MADV_DONTNEED)) {
1093 ksft_test_result_fail("%s MADV_DONTNEED failed\n", name);
1094 goto out;
1095 }
1096
1097 /* Fast path: category_mask == return_mask == PAGE_IS_WRITTEN. */
1098 ret = pagemap_ioctl(mem, size, regions, ARRAY_SIZE(regions), 0, 0,
1099 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
1100 for (i = 0; ret > 0 && i < ret; i++)
1101 fast += LEN(regions[i]);
1102
1103 /* Generic path: same query expressed via category_anyof_mask. */
1104 ret = pagemap_ioctl(mem, size, regions, ARRAY_SIZE(regions), 0, 0,
1105 0, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN);
1106 for (i = 0; ret > 0 && i < ret; i++)
1107 slow += LEN(regions[i]);
1108
1109 ksft_test_result(fast == npages && slow == npages,
1110 "%s unpopulated range reported written by both paths (%ld, %ld of %ld)\n",
1111 name, fast, slow, npages);
1112 out:
1113 wp_free(mem, size);
1114 }
1115
unpopulated_scan_test(void)1116 static void unpopulated_scan_test(void)
1117 {
1118 long mem_size = 16 * page_size;
1119 char *mem;
1120
1121 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE,
1122 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1123 if (mem == MAP_FAILED) {
1124 ksft_test_result_skip("%s mmap failed\n", __func__);
1125 return;
1126 }
1127
1128 unpopulated_written_test(__func__, mem, mem_size, false);
1129 munmap(mem, mem_size);
1130 }
1131
1132 /*
1133 * Same as unpopulated_scan_test(), but the range is a THP: a full-PMD
1134 * MADV_DONTNEED leaves a pmd_none hole with no page table.
1135 */
unpopulated_thp_scan_test(void)1136 static void unpopulated_thp_scan_test(void)
1137 {
1138 char *area, *mem;
1139
1140 if (!hpage_size) {
1141 ksft_test_result_skip("%s THP not supported\n", __func__);
1142 return;
1143 }
1144
1145 /* Over-allocate so a PMD-aligned, THP-sized range fits inside. */
1146 area = mmap(NULL, 2 * hpage_size, PROT_READ | PROT_WRITE,
1147 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1148 if (area == MAP_FAILED) {
1149 ksft_test_result_skip("%s mmap failed\n", __func__);
1150 return;
1151 }
1152 mem = (char *)(((unsigned long)area + hpage_size - 1) & ~(hpage_size - 1));
1153
1154 unpopulated_written_test(__func__, mem, hpage_size, true);
1155 munmap(area, 2 * hpage_size);
1156 }
1157
sanity_tests(void)1158 int sanity_tests(void)
1159 {
1160 unsigned long long mem_size, vec_size;
1161 long ret, fd, i, buf_size, nr_pages;
1162 struct page_region *vec;
1163 char *mem, *fmem;
1164 struct stat sbuf;
1165
1166 /* 1. wrong operation */
1167 mem_size = 10 * page_size;
1168 vec_size = mem_size / page_size;
1169
1170 vec = calloc(vec_size, sizeof(struct page_region));
1171 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1172 if (mem == MAP_FAILED || !vec)
1173 ksft_exit_fail_msg("error nomem\n");
1174
1175 wp_init(mem, mem_size);
1176 wp_addr_range(mem, mem_size);
1177
1178 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,
1179 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
1180 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0,
1181 "%s WP op can be specified with !PAGE_IS_WRITTEN\n", __func__);
1182 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1183 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0,
1184 "%s required_mask specified\n", __func__);
1185 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1186 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL) >= 0,
1187 "%s anyof_mask specified\n", __func__);
1188 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1189 0, 0, PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL) >= 0,
1190 "%s excluded_mask specified\n", __func__);
1191 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1192 PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL, 0,
1193 PAGEMAP_BITS_ALL) >= 0,
1194 "%s required_mask and anyof_mask specified\n", __func__);
1195 wp_free(mem, mem_size);
1196 munmap(mem, mem_size);
1197
1198 /* 2. Get sd and present pages with anyof_mask */
1199 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1200 if (mem == MAP_FAILED)
1201 ksft_exit_fail_msg("error nomem\n");
1202 wp_init(mem, mem_size);
1203 wp_addr_range(mem, mem_size);
1204
1205 memset(mem, 0, mem_size);
1206
1207 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1208 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL);
1209 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&
1210 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) ==
1211 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT),
1212 "%s Get sd and present pages with anyof_mask\n", __func__);
1213
1214 /* 3. Get sd and present pages with required_mask */
1215 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1216 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL);
1217 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&
1218 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) ==
1219 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT),
1220 "%s Get all the pages with required_mask\n", __func__);
1221
1222 /* 4. Get sd and present pages with required_mask and anyof_mask */
1223 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1224 PAGE_IS_WRITTEN, PAGE_IS_PRESENT, 0, PAGEMAP_BITS_ALL);
1225 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&
1226 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) ==
1227 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT),
1228 "%s Get sd and present pages with required_mask and anyof_mask\n",
1229 __func__);
1230
1231 /* 5. Don't get sd pages */
1232 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1233 PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN, PAGEMAP_BITS_ALL);
1234 ksft_test_result(ret == 0, "%s Don't get sd pages\n", __func__);
1235
1236 /* 6. Don't get present pages */
1237 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1238 PAGE_IS_PRESENT, 0, PAGE_IS_PRESENT, PAGEMAP_BITS_ALL);
1239 ksft_test_result(ret == 0, "%s Don't get present pages\n", __func__);
1240
1241 wp_free(mem, mem_size);
1242 munmap(mem, mem_size);
1243
1244 /* 8. Find written present pages with return mask */
1245 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1246 if (mem == MAP_FAILED)
1247 ksft_exit_fail_msg("error nomem\n");
1248 wp_init(mem, mem_size);
1249 wp_addr_range(mem, mem_size);
1250
1251 memset(mem, 0, mem_size);
1252
1253 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
1254 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0,
1255 0, PAGEMAP_BITS_ALL, 0, PAGE_IS_WRITTEN);
1256 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&
1257 vec[0].categories == PAGE_IS_WRITTEN,
1258 "%s Find written present pages with return mask\n", __func__);
1259 wp_free(mem, mem_size);
1260 munmap(mem, mem_size);
1261
1262 /* 9. Memory mapped file */
1263 fd = open(progname, O_RDONLY);
1264 if (fd < 0)
1265 ksft_exit_fail_msg("%s Memory mapped file\n", __func__);
1266
1267 ret = stat(progname, &sbuf);
1268 if (ret < 0)
1269 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
1270
1271 fmem = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
1272 if (fmem == MAP_FAILED)
1273 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno));
1274
1275 nr_pages = (sbuf.st_size + page_size - 1) / page_size;
1276 force_read_pages(fmem, nr_pages, page_size);
1277
1278 ret = pagemap_ioctl(fmem, sbuf.st_size, vec, vec_size, 0, 0,
1279 0, PAGEMAP_NON_WRITTEN_BITS, 0, PAGEMAP_NON_WRITTEN_BITS);
1280
1281 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem &&
1282 LEN(vec[0]) == nr_pages &&
1283 (vec[0].categories & PAGE_IS_FILE),
1284 "%s Memory mapped file\n", __func__);
1285
1286 munmap(fmem, sbuf.st_size);
1287 close(fd);
1288
1289 /* 10. Create and read/write to a memory mapped file */
1290 buf_size = page_size * 10;
1291
1292 fd = open(__FILE__".tmp2", O_RDWR | O_CREAT, 0666);
1293 if (fd < 0)
1294 ksft_exit_fail_msg("Read/write to memory: %s\n",
1295 strerror(errno));
1296
1297 for (i = 0; i < buf_size; i++)
1298 if (write(fd, "c", 1) < 0)
1299 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n");
1300
1301 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1302 if (fmem == MAP_FAILED)
1303 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno));
1304
1305 wp_init(fmem, buf_size);
1306 wp_addr_range(fmem, buf_size);
1307
1308 for (i = 0; i < buf_size; i++)
1309 fmem[i] = 'z';
1310
1311 msync(fmem, buf_size, MS_SYNC);
1312
1313 ret = pagemap_ioctl(fmem, buf_size, vec, vec_size, 0, 0,
1314 PAGE_IS_WRITTEN, PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_FILE, 0,
1315 PAGEMAP_BITS_ALL);
1316
1317 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem &&
1318 LEN(vec[0]) == (buf_size/page_size) &&
1319 (vec[0].categories & PAGE_IS_WRITTEN),
1320 "%s Read/write to memory\n", __func__);
1321
1322 wp_free(fmem, buf_size);
1323 munmap(fmem, buf_size);
1324 close(fd);
1325
1326 free(vec);
1327 return 0;
1328 }
1329
mprotect_tests(void)1330 int mprotect_tests(void)
1331 {
1332 int ret;
1333 char *mem, *mem2;
1334 struct page_region vec;
1335
1336 /* 1. Map two pages */
1337 mem = mmap(0, 2 * page_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1338 if (mem == MAP_FAILED)
1339 ksft_exit_fail_msg("error nomem\n");
1340 wp_init(mem, 2 * page_size);
1341 wp_addr_range(mem, 2 * page_size);
1342
1343 /* Populate both pages. */
1344 memset(mem, 1, 2 * page_size);
1345
1346 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN,
1347 0, 0, PAGE_IS_WRITTEN);
1348 if (ret < 0)
1349 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1350
1351 ksft_test_result(ret == 1 && LEN(vec) == 2, "%s Both pages written\n", __func__);
1352
1353 /* 2. Start tracking */
1354 wp_addr_range(mem, 2 * page_size);
1355
1356 ksft_test_result(pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0,
1357 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0,
1358 "%s Both pages are not written (dirty)\n", __func__);
1359
1360 /* 3. Remap the second page */
1361 mem2 = mmap(mem + page_size, page_size, PROT_READ|PROT_WRITE,
1362 MAP_PRIVATE|MAP_ANON|MAP_FIXED, -1, 0);
1363 if (mem2 == MAP_FAILED)
1364 ksft_exit_fail_msg("error nomem\n");
1365 wp_init(mem2, page_size);
1366 wp_addr_range(mem2, page_size);
1367
1368 /* Protect + unprotect. */
1369 mprotect(mem, page_size, PROT_NONE);
1370 mprotect(mem, 2 * page_size, PROT_READ);
1371 mprotect(mem, 2 * page_size, PROT_READ|PROT_WRITE);
1372
1373 /* Modify both pages. */
1374 memset(mem, 2, 2 * page_size);
1375
1376 /* Protect + unprotect. */
1377 mprotect(mem, page_size, PROT_NONE);
1378 mprotect(mem, page_size, PROT_READ);
1379 mprotect(mem, page_size, PROT_READ|PROT_WRITE);
1380
1381 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN,
1382 0, 0, PAGE_IS_WRITTEN);
1383 if (ret < 0)
1384 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1385
1386 ksft_test_result(ret == 1 && LEN(vec) == 2,
1387 "%s Both pages written after remap and mprotect\n", __func__);
1388
1389 /* 4. Clear and make the pages written */
1390 wp_addr_range(mem, 2 * page_size);
1391
1392 memset(mem, 'A', 2 * page_size);
1393
1394 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN,
1395 0, 0, PAGE_IS_WRITTEN);
1396 if (ret < 0)
1397 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1398
1399 ksft_test_result(ret == 1 && LEN(vec) == 2,
1400 "%s Clear and make the pages written\n", __func__);
1401
1402 wp_free(mem, 2 * page_size);
1403 munmap(mem, 2 * page_size);
1404 return 0;
1405 }
1406
1407 /* transact test */
1408 static const unsigned int nthreads = 6, pages_per_thread = 32, access_per_thread = 8;
1409 static pthread_barrier_t start_barrier, end_barrier;
1410 static unsigned int extra_thread_faults;
1411 static unsigned int iter_count = 1000;
1412 static volatile int finish;
1413
get_dirty_pages_reset(char * mem,unsigned int count,int reset,int page_size)1414 static ssize_t get_dirty_pages_reset(char *mem, unsigned int count,
1415 int reset, int page_size)
1416 {
1417 struct pm_scan_arg arg = {0};
1418 struct page_region rgns[256];
1419 unsigned long long i, j;
1420 long ret;
1421 int cnt;
1422
1423 arg.size = sizeof(struct pm_scan_arg);
1424 arg.start = (uintptr_t)mem;
1425 arg.max_pages = count;
1426 arg.end = (uintptr_t)(mem + count * page_size);
1427 arg.vec = (uintptr_t)rgns;
1428 arg.vec_len = sizeof(rgns) / sizeof(*rgns);
1429 if (reset)
1430 arg.flags |= PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC;
1431 arg.category_mask = PAGE_IS_WRITTEN;
1432 arg.return_mask = PAGE_IS_WRITTEN;
1433
1434 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg);
1435 if (ret < 0)
1436 ksft_exit_fail_msg("ioctl failed\n");
1437
1438 cnt = 0;
1439 for (i = 0; i < (unsigned long)ret; ++i) {
1440 if (rgns[i].categories != PAGE_IS_WRITTEN)
1441 ksft_exit_fail_msg("wrong flags\n");
1442
1443 for (j = 0; j < LEN(rgns[i]); ++j)
1444 cnt++;
1445 }
1446
1447 return cnt;
1448 }
1449
thread_proc(void * mem)1450 void *thread_proc(void *mem)
1451 {
1452 int *m = mem;
1453 long curr_faults, faults;
1454 struct rusage r;
1455 unsigned int i;
1456 int ret;
1457
1458 if (getrusage(RUSAGE_THREAD, &r))
1459 ksft_exit_fail_msg("getrusage\n");
1460
1461 curr_faults = r.ru_minflt;
1462
1463 while (!finish) {
1464 ret = pthread_barrier_wait(&start_barrier);
1465 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD)
1466 ksft_exit_fail_msg("pthread_barrier_wait\n");
1467
1468 for (i = 0; i < access_per_thread; ++i)
1469 __atomic_add_fetch(m + i * (page_size / sizeof(*m)), 1, __ATOMIC_SEQ_CST);
1470
1471 ret = pthread_barrier_wait(&end_barrier);
1472 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD)
1473 ksft_exit_fail_msg("pthread_barrier_wait\n");
1474
1475 if (getrusage(RUSAGE_THREAD, &r))
1476 ksft_exit_fail_msg("getrusage\n");
1477
1478 faults = r.ru_minflt - curr_faults;
1479 if (faults < access_per_thread)
1480 ksft_exit_fail_msg("faults < access_per_thread");
1481
1482 __atomic_add_fetch(&extra_thread_faults, faults - access_per_thread,
1483 __ATOMIC_SEQ_CST);
1484 curr_faults = r.ru_minflt;
1485 }
1486
1487 return NULL;
1488 }
1489
transact_test(int page_size)1490 static void transact_test(int page_size)
1491 {
1492 unsigned int i, count, extra_pages;
1493 unsigned int c;
1494 pthread_t th;
1495 char *mem;
1496 int ret;
1497
1498 if (pthread_barrier_init(&start_barrier, NULL, nthreads + 1))
1499 ksft_exit_fail_msg("pthread_barrier_init\n");
1500
1501 if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1))
1502 ksft_exit_fail_msg("pthread_barrier_init\n");
1503
1504 mem = mmap(NULL, page_size * nthreads * pages_per_thread, PROT_READ | PROT_WRITE,
1505 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
1506 if (mem == MAP_FAILED)
1507 ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno));
1508
1509 wp_init(mem, page_size * nthreads * pages_per_thread);
1510 wp_addr_range(mem, page_size * nthreads * pages_per_thread);
1511
1512 memset(mem, 0, page_size * nthreads * pages_per_thread);
1513
1514 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size);
1515 ksft_test_result(count > 0, "%s count %u\n", __func__, count);
1516 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size);
1517 ksft_test_result(count == 0, "%s count %u\n", __func__, count);
1518
1519 finish = 0;
1520 for (i = 0; i < nthreads; ++i)
1521 pthread_create(&th, NULL, thread_proc, mem + page_size * i * pages_per_thread);
1522
1523 extra_pages = 0;
1524 for (i = 0; i < iter_count; ++i) {
1525 count = 0;
1526
1527 ret = pthread_barrier_wait(&start_barrier);
1528 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD)
1529 ksft_exit_fail_msg("pthread_barrier_wait\n");
1530
1531 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1,
1532 page_size);
1533
1534 ret = pthread_barrier_wait(&end_barrier);
1535 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD)
1536 ksft_exit_fail_msg("pthread_barrier_wait\n");
1537
1538 if (count > nthreads * access_per_thread)
1539 ksft_exit_fail_msg("Too big count %u expected %u, iter %u\n",
1540 count, nthreads * access_per_thread, i);
1541
1542 c = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size);
1543 count += c;
1544
1545 if (c > nthreads * access_per_thread) {
1546 ksft_test_result_fail(" %s count > nthreads\n", __func__);
1547 return;
1548 }
1549
1550 if (count != nthreads * access_per_thread) {
1551 /*
1552 * The purpose of the test is to make sure that no page updates are lost
1553 * when the page updates and read-resetting soft dirty flags are performed
1554 * in parallel. However, it is possible that the application will get the
1555 * soft dirty flags twice on the two consecutive read-resets. This seems
1556 * unavoidable as soft dirty flag is handled in software through page faults
1557 * in kernel. While the updating the flags is supposed to be synchronized
1558 * between page fault handling and read-reset, it is possible that
1559 * read-reset happens after page fault PTE update but before the application
1560 * re-executes write instruction. So read-reset gets the flag, clears write
1561 * access and application gets page fault again for the same write.
1562 */
1563 if (count < nthreads * access_per_thread) {
1564 ksft_test_result_fail("Lost update, iter %u, %u vs %u.\n", i, count,
1565 nthreads * access_per_thread);
1566 return;
1567 }
1568
1569 extra_pages += count - nthreads * access_per_thread;
1570 }
1571 }
1572
1573 pthread_barrier_wait(&start_barrier);
1574 finish = 1;
1575 pthread_barrier_wait(&end_barrier);
1576
1577 ksft_test_result_pass("%s Extra pages %u (%.1lf%%), extra thread faults %u.\n", __func__,
1578 extra_pages,
1579 100.0 * extra_pages / (iter_count * nthreads * access_per_thread),
1580 extra_thread_faults);
1581 }
1582
zeropfn_tests(void)1583 void zeropfn_tests(void)
1584 {
1585 unsigned long long mem_size;
1586 struct page_region vec;
1587 int i, ret;
1588 char *mmap_mem, *mem;
1589
1590 /* Test with normal memory */
1591 mem_size = 10 * page_size;
1592 mem = mmap(NULL, mem_size, PROT_READ, MAP_PRIVATE | MAP_ANON, -1, 0);
1593 if (mem == MAP_FAILED)
1594 ksft_exit_fail_msg("error nomem\n");
1595
1596 /* Touch each page to ensure it's mapped */
1597 for (i = 0; i < mem_size; i += page_size)
1598 (void)((volatile char *)mem)[i];
1599
1600 ret = pagemap_ioctl(mem, mem_size, &vec, 1, 0,
1601 (mem_size / page_size), PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO);
1602 if (ret < 0)
1603 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1604
1605 ksft_test_result(ret == 1 && LEN(vec) == (mem_size / page_size),
1606 "%s all pages must have PFNZERO set\n", __func__);
1607
1608 munmap(mem, mem_size);
1609
1610 /* Test with huge page if user_zero_page is set to 1 */
1611 if (!detect_huge_zeropage()) {
1612 ksft_test_result_skip("%s use_zero_page not supported or set to 1\n", __func__);
1613 return;
1614 }
1615
1616 mem_size = 2 * hpage_size;
1617 mmap_mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE,
1618 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1619 if (mmap_mem == MAP_FAILED)
1620 ksft_exit_fail_msg("error nomem\n");
1621
1622 /* We need a THP-aligned memory area. */
1623 mem = (char *)(((uintptr_t)mmap_mem + hpage_size) & ~(hpage_size - 1));
1624
1625 ret = madvise(mem, hpage_size, MADV_HUGEPAGE);
1626 if (!ret) {
1627 FORCE_READ(*mem);
1628
1629 ret = pagemap_ioctl(mem, hpage_size, &vec, 1, 0,
1630 0, PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO);
1631 if (ret < 0)
1632 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1633
1634 ksft_test_result(ret == 1 && LEN(vec) == (hpage_size / page_size),
1635 "%s all huge pages must have PFNZERO set\n", __func__);
1636 } else {
1637 ksft_test_result_skip("%s huge page not supported\n", __func__);
1638 }
1639
1640 munmap(mmap_mem, mem_size);
1641 }
1642
main(int argc,char * argv[])1643 int main(int __attribute__((unused)) argc, char *argv[])
1644 {
1645 int shmid, buf_size, fd, i, ret;
1646 unsigned long long mem_size;
1647 char *mem, *map, *fmem;
1648 struct stat sbuf;
1649
1650 progname = argv[0];
1651
1652 ksft_print_header();
1653
1654 if (init_uffd())
1655 ksft_exit_skip("Failed to initialize userfaultfd\n");
1656
1657 if (!hugetlb_setup_default(4))
1658 ksft_print_msg("HugeTLB test will be skipped\n");
1659
1660 ksft_set_plan(119);
1661
1662 page_size = getpagesize();
1663 hpage_size = read_pmd_pagesize();
1664
1665 pagemap_fd = open(PAGEMAP, O_RDONLY);
1666 if (pagemap_fd < 0)
1667 ksft_exit_fail_msg("Failed to open " PAGEMAP "\n");
1668
1669 /* 1. Sanity testing */
1670 sanity_tests_sd();
1671
1672 /* 2. Normal page testing */
1673 mem_size = 10 * page_size;
1674 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1675 if (mem == MAP_FAILED)
1676 ksft_exit_fail_msg("error nomem\n");
1677 wp_init(mem, mem_size);
1678 wp_addr_range(mem, mem_size);
1679
1680 base_tests("Page testing:", mem, mem_size, 0);
1681
1682 wp_free(mem, mem_size);
1683 munmap(mem, mem_size);
1684
1685 /* 3. Large page testing */
1686 mem_size = 512 * 10 * page_size;
1687 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1688 if (mem == MAP_FAILED)
1689 ksft_exit_fail_msg("error nomem\n");
1690 wp_init(mem, mem_size);
1691 wp_addr_range(mem, mem_size);
1692
1693 base_tests("Large Page testing:", mem, mem_size, 0);
1694
1695 wp_free(mem, mem_size);
1696 munmap(mem, mem_size);
1697
1698 /* 4. Huge page testing */
1699 map = gethugepage(hpage_size);
1700 if (map) {
1701 wp_init(map, hpage_size);
1702 wp_addr_range(map, hpage_size);
1703 base_tests("Huge page testing:", map, hpage_size, 0);
1704 wp_free(map, hpage_size);
1705 free(map);
1706 } else {
1707 base_tests("Huge page testing:", NULL, 0, 1);
1708 }
1709
1710 /* 5. SHM Hugetlb page testing */
1711 mem_size = default_huge_page_size();
1712 mem = gethugetlb_mem(mem_size, &shmid);
1713 if (mem) {
1714 wp_init(mem, mem_size);
1715 wp_addr_range(mem, mem_size);
1716
1717 base_tests("Hugetlb shmem testing:", mem, mem_size, 0);
1718
1719 wp_free(mem, mem_size);
1720 shmctl(shmid, IPC_RMID, NULL);
1721 } else {
1722 base_tests("Hugetlb shmem testing:", NULL, 0, 1);
1723 }
1724
1725 /* 6. Hugetlb page testing */
1726 mem = gethugetlb_mem(mem_size, NULL);
1727 if (mem) {
1728 wp_init(mem, mem_size);
1729 wp_addr_range(mem, mem_size);
1730
1731 base_tests("Hugetlb mem testing:", mem, mem_size, 0);
1732
1733 wp_free(mem, mem_size);
1734 } else {
1735 base_tests("Hugetlb mem testing:", NULL, 0, 1);
1736 }
1737
1738 /* 7. File Hugetlb testing */
1739 mem_size = default_huge_page_size();
1740 fd = memfd_create("uffd-test", MFD_HUGETLB | MFD_NOEXEC_SEAL);
1741 if (fd < 0)
1742 ksft_exit_fail_msg("uffd-test creation failed %d %s\n", errno, strerror(errno));
1743 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1744 if (mem != MAP_FAILED) {
1745 wp_init(mem, mem_size);
1746 wp_addr_range(mem, mem_size);
1747
1748 base_tests("Hugetlb shmem testing:", mem, mem_size, 0);
1749
1750 wp_free(mem, mem_size);
1751 shmctl(shmid, IPC_RMID, NULL);
1752 } else {
1753 base_tests("Hugetlb shmem testing:", NULL, 0, 1);
1754 }
1755 close(fd);
1756
1757 /* 8. File memory testing */
1758 buf_size = page_size * 10;
1759
1760 fd = open(__FILE__".tmp0", O_RDWR | O_CREAT, 0777);
1761 if (fd < 0)
1762 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n",
1763 strerror(errno));
1764
1765 for (i = 0; i < buf_size; i++)
1766 if (write(fd, "c", 1) < 0)
1767 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n");
1768
1769 ret = stat(__FILE__".tmp0", &sbuf);
1770 if (ret < 0)
1771 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1772
1773 fmem = mmap(NULL, sbuf.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1774 if (fmem == MAP_FAILED)
1775 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno));
1776
1777 wp_init(fmem, sbuf.st_size);
1778 wp_addr_range(fmem, sbuf.st_size);
1779
1780 base_tests("File memory testing:", fmem, sbuf.st_size, 0);
1781
1782 wp_free(fmem, sbuf.st_size);
1783 munmap(fmem, sbuf.st_size);
1784 close(fd);
1785
1786 /* 9. File memory testing */
1787 buf_size = page_size * 10;
1788
1789 fd = memfd_create(__FILE__".tmp00", MFD_NOEXEC_SEAL);
1790 if (fd < 0)
1791 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n",
1792 strerror(errno));
1793
1794 if (ftruncate(fd, buf_size))
1795 ksft_exit_fail_msg("Error ftruncate\n");
1796
1797 for (i = 0; i < buf_size; i++)
1798 if (write(fd, "c", 1) < 0)
1799 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n");
1800
1801 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1802 if (fmem == MAP_FAILED)
1803 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno));
1804
1805 wp_init(fmem, buf_size);
1806 wp_addr_range(fmem, buf_size);
1807
1808 base_tests("File anonymous memory testing:", fmem, buf_size, 0);
1809
1810 wp_free(fmem, buf_size);
1811 munmap(fmem, buf_size);
1812 close(fd);
1813
1814 /* 10. Huge page tests */
1815 hpage_unit_tests();
1816
1817 /* 11. Iterative test */
1818 test_simple();
1819
1820 /* 12. Mprotect test */
1821 mprotect_tests();
1822
1823 /* 13. Transact test */
1824 transact_test(page_size);
1825
1826 /* 14. Sanity testing */
1827 sanity_tests();
1828
1829 /*15. Unmapped address test */
1830 unmapped_region_tests();
1831
1832 /* 16. Userfaultfd tests */
1833 userfaultfd_tests();
1834
1835 /* 17. ZEROPFN tests */
1836 zeropfn_tests();
1837
1838 /* 18. Unpopulated pte scan-path consistency */
1839 unpopulated_scan_test();
1840 unpopulated_thp_scan_test();
1841
1842 close(pagemap_fd);
1843 ksft_finished();
1844 }
1845