1 // SPDX-License-Identifier: GPL-2.0 2 3 #define _GNU_SOURCE 4 #include <stdio.h> 5 #include <fcntl.h> 6 #include <string.h> 7 #include <sys/mman.h> 8 #include <errno.h> 9 #include <malloc.h> 10 #include <linux/types.h> 11 #include <linux/memfd.h> 12 #include <linux/userfaultfd.h> 13 #include <linux/fs.h> 14 #include <sys/ioctl.h> 15 #include <sys/stat.h> 16 #include <math.h> 17 #include <asm/unistd.h> 18 #include <pthread.h> 19 #include <sys/resource.h> 20 #include <assert.h> 21 #include <sys/ipc.h> 22 #include <sys/shm.h> 23 24 #include "vm_util.h" 25 #include "kselftest.h" 26 #include "hugepage_settings.h" 27 28 #define PAGEMAP_BITS_ALL (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ 29 PAGE_IS_FILE | PAGE_IS_PRESENT | \ 30 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ 31 PAGE_IS_HUGE) 32 #define PAGEMAP_NON_WRITTEN_BITS (PAGE_IS_WPALLOWED | PAGE_IS_FILE | \ 33 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | \ 34 PAGE_IS_PFNZERO | PAGE_IS_HUGE) 35 36 #define TEST_ITERATIONS 100 37 #define PAGEMAP "/proc/self/pagemap" 38 int pagemap_fd; 39 int uffd; 40 size_t page_size; 41 size_t hpage_size; 42 const char *progname; 43 44 #define LEN(region) ((region.end - region.start)/page_size) 45 46 static long pagemap_ioctl(void *start, int len, void *vec, int vec_len, int flag, 47 int max_pages, long required_mask, long anyof_mask, long excluded_mask, 48 long return_mask) 49 { 50 struct pm_scan_arg arg; 51 52 arg.start = (uintptr_t)start; 53 arg.end = (uintptr_t)(start + len); 54 arg.vec = (uintptr_t)vec; 55 arg.vec_len = vec_len; 56 arg.flags = flag; 57 arg.size = sizeof(struct pm_scan_arg); 58 arg.max_pages = max_pages; 59 arg.category_mask = required_mask; 60 arg.category_anyof_mask = anyof_mask; 61 arg.category_inverted = excluded_mask; 62 arg.return_mask = return_mask; 63 64 return ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 65 } 66 67 static long pagemap_ioc(void *start, int len, void *vec, int vec_len, int flag, 68 int max_pages, long required_mask, long anyof_mask, long excluded_mask, 69 long return_mask, long *walk_end) 70 { 71 struct pm_scan_arg arg; 72 int ret; 73 74 arg.start = (uintptr_t)start; 75 arg.end = (uintptr_t)(start + len); 76 arg.vec = (uintptr_t)vec; 77 arg.vec_len = vec_len; 78 arg.flags = flag; 79 arg.size = sizeof(struct pm_scan_arg); 80 arg.max_pages = max_pages; 81 arg.category_mask = required_mask; 82 arg.category_anyof_mask = anyof_mask; 83 arg.category_inverted = excluded_mask; 84 arg.return_mask = return_mask; 85 86 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 87 88 if (walk_end) 89 *walk_end = arg.walk_end; 90 91 return ret; 92 } 93 94 95 int init_uffd(void) 96 { 97 struct uffdio_api uffdio_api; 98 99 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); 100 if (uffd == -1) 101 return uffd; 102 103 uffdio_api.api = UFFD_API; 104 uffdio_api.features = UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC | 105 UFFD_FEATURE_WP_HUGETLBFS_SHMEM; 106 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) 107 return -1; 108 109 if (!(uffdio_api.api & UFFDIO_REGISTER_MODE_WP) || 110 !(uffdio_api.features & UFFD_FEATURE_WP_UNPOPULATED) || 111 !(uffdio_api.features & UFFD_FEATURE_WP_ASYNC) || 112 !(uffdio_api.features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM)) 113 return -1; 114 115 return 0; 116 } 117 118 int wp_init(void *addr, long size) 119 { 120 struct uffdio_register uffdio_register; 121 struct uffdio_writeprotect wp; 122 123 uffdio_register.range.start = (unsigned long)addr; 124 uffdio_register.range.len = size; 125 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; 126 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) 127 ksft_exit_fail_msg("ioctl(UFFDIO_REGISTER) %d %s\n", errno, strerror(errno)); 128 129 if (!(uffdio_register.ioctls & UFFDIO_WRITEPROTECT)) 130 ksft_exit_fail_msg("ioctl set is incorrect\n"); 131 132 wp.range.start = (unsigned long)addr; 133 wp.range.len = size; 134 wp.mode = UFFDIO_WRITEPROTECT_MODE_WP; 135 136 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &wp)) 137 ksft_exit_fail_msg("ioctl(UFFDIO_WRITEPROTECT)\n"); 138 139 return 0; 140 } 141 142 int wp_free(void *addr, long size) 143 { 144 struct uffdio_register uffdio_register; 145 146 uffdio_register.range.start = (unsigned long)addr; 147 uffdio_register.range.len = size; 148 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; 149 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range)) 150 ksft_exit_fail_msg("ioctl unregister failure\n"); 151 return 0; 152 } 153 154 int wp_addr_range(void *addr, int size) 155 { 156 if (pagemap_ioctl(addr, size, NULL, 0, 157 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 158 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0) 159 ksft_exit_fail_msg("error %d %d %s\n", 1, errno, strerror(errno)); 160 161 return 0; 162 } 163 164 void *gethugetlb_mem(int size, int *shmid) 165 { 166 char *mem; 167 168 if (shmid) { 169 *shmid = shmget(2, size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); 170 if (*shmid < 0) 171 return NULL; 172 173 mem = shmat(*shmid, 0, 0); 174 if (mem == (char *)-1) { 175 shmctl(*shmid, IPC_RMID, NULL); 176 ksft_exit_fail_msg("Shared memory attach failure\n"); 177 } 178 } else { 179 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, 180 MAP_ANONYMOUS | MAP_HUGETLB | MAP_PRIVATE, -1, 0); 181 if (mem == MAP_FAILED) 182 return NULL; 183 } 184 185 return mem; 186 } 187 188 int userfaultfd_tests(void) 189 { 190 long mem_size, vec_size, written, num_pages = 16; 191 char *mem, *vec; 192 193 mem_size = num_pages * page_size; 194 mem = mmap(NULL, mem_size, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0); 195 if (mem == MAP_FAILED) 196 ksft_exit_fail_msg("error nomem\n"); 197 198 wp_init(mem, mem_size); 199 200 /* Change protection of pages differently */ 201 mprotect(mem, mem_size/8, PROT_READ|PROT_WRITE); 202 mprotect(mem + 1 * mem_size/8, mem_size/8, PROT_READ); 203 mprotect(mem + 2 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 204 mprotect(mem + 3 * mem_size/8, mem_size/8, PROT_READ); 205 mprotect(mem + 4 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 206 mprotect(mem + 5 * mem_size/8, mem_size/8, PROT_NONE); 207 mprotect(mem + 6 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 208 mprotect(mem + 7 * mem_size/8, mem_size/8, PROT_READ); 209 210 wp_addr_range(mem + (mem_size/16), mem_size - 2 * (mem_size/8)); 211 wp_addr_range(mem, mem_size); 212 213 vec_size = mem_size/page_size; 214 vec = calloc(vec_size, sizeof(struct page_region)); 215 216 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 217 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 218 if (written < 0) 219 ksft_exit_fail_msg("error %ld %d %s\n", written, errno, strerror(errno)); 220 221 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", __func__); 222 223 wp_free(mem, mem_size); 224 munmap(mem, mem_size); 225 free(vec); 226 return 0; 227 } 228 229 int get_reads(struct page_region *vec, int vec_size) 230 { 231 int i, sum = 0; 232 233 for (i = 0; i < vec_size; i++) 234 sum += LEN(vec[i]); 235 236 return sum; 237 } 238 239 int sanity_tests_sd(void) 240 { 241 unsigned long long mem_size, vec_size, i, total_pages = 0; 242 long ret, ret2, ret3; 243 int num_pages = 1000; 244 int total_writes, total_reads, reads, count; 245 struct page_region *vec, *vec2; 246 char *mem, *m[2]; 247 long walk_end; 248 249 vec_size = num_pages/2; 250 mem_size = num_pages * page_size; 251 252 vec = calloc(vec_size, sizeof(struct page_region)); 253 if (!vec) 254 ksft_exit_fail_msg("error nomem\n"); 255 256 vec2 = calloc(vec_size, sizeof(struct page_region)); 257 if (!vec2) 258 ksft_exit_fail_msg("error nomem\n"); 259 260 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 261 if (mem == MAP_FAILED) 262 ksft_exit_fail_msg("error nomem\n"); 263 264 wp_init(mem, mem_size); 265 wp_addr_range(mem, mem_size); 266 267 /* 1. wrong operation */ 268 ksft_test_result(pagemap_ioctl(mem, 0, vec, vec_size, 0, 269 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 270 "%s Zero range size is valid\n", __func__); 271 272 ksft_test_result(pagemap_ioctl(mem, mem_size, NULL, vec_size, 0, 273 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) < 0, 274 "%s output buffer must be specified with size\n", __func__); 275 276 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, 0, 0, 277 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 278 "%s output buffer can be 0\n", __func__); 279 280 ksft_test_result(pagemap_ioctl(mem, mem_size, 0, 0, 0, 281 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 282 "%s output buffer can be 0\n", __func__); 283 284 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, -1, 285 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0, 286 "%s wrong flag specified\n", __func__); 287 288 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 289 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC | 0xFF, 290 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0, 291 "%s flag has extra bits specified\n", __func__); 292 293 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 294 0, 0, 0, 0, PAGE_IS_WRITTEN) >= 0, 295 "%s no selection mask is specified\n", __func__); 296 297 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 298 0, PAGE_IS_WRITTEN, PAGE_IS_WRITTEN, 0, 0) == 0, 299 "%s no return mask is specified\n", __func__); 300 301 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 302 0, PAGE_IS_WRITTEN, 0, 0, 0x1000) < 0, 303 "%s wrong return mask specified\n", __func__); 304 305 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 306 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 307 0, 0xFFF, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN) < 0, 308 "%s mixture of correct and wrong flag\n", __func__); 309 310 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 311 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 312 0, 0, 0, PAGEMAP_BITS_ALL, PAGE_IS_WRITTEN) >= 0, 313 "%s PAGEMAP_BITS_ALL can be specified with PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", 314 __func__); 315 316 /* 2. Clear area with larger vec size */ 317 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 318 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 319 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 320 ksft_test_result(ret >= 0, "%s Clear area with larger vec size\n", __func__); 321 322 /* 3. Repeated pattern of written and non-written pages */ 323 for (i = 0; i < mem_size; i += 2 * page_size) 324 mem[i]++; 325 326 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 0, 327 0, PAGE_IS_WRITTEN); 328 if (ret < 0) 329 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 330 331 ksft_test_result((unsigned long long)ret == mem_size/(page_size * 2), 332 "%s Repeated pattern of written and non-written pages\n", __func__); 333 334 /* 4. Repeated pattern of written and non-written pages in parts */ 335 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 336 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 337 num_pages/2 - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 338 if (ret < 0) 339 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 340 341 ret2 = pagemap_ioctl(mem, mem_size, vec, 2, 0, 0, PAGE_IS_WRITTEN, 0, 0, 342 PAGE_IS_WRITTEN); 343 if (ret2 < 0) 344 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno)); 345 346 ret3 = pagemap_ioctl(mem, mem_size, vec, vec_size, 347 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 348 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 349 if (ret3 < 0) 350 ksft_exit_fail_msg("error %ld %d %s\n", ret3, errno, strerror(errno)); 351 352 ksft_test_result((ret + ret3) == num_pages/2 && ret2 == 2, 353 "%s Repeated pattern of written and non-written pages in parts %ld %ld %ld\n", 354 __func__, ret, ret3, ret2); 355 356 /* 5. Repeated pattern of written and non-written pages max_pages */ 357 for (i = 0; i < mem_size; i += 2 * page_size) 358 mem[i]++; 359 mem[(mem_size/page_size - 1) * page_size]++; 360 361 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 362 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 363 num_pages/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 364 if (ret < 0) 365 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 366 367 ret2 = pagemap_ioctl(mem, mem_size, vec, vec_size, 368 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 369 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 370 if (ret2 < 0) 371 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno)); 372 373 ksft_test_result(ret == num_pages/2 && ret2 == 1, 374 "%s Repeated pattern of written and non-written pages max_pages\n", 375 __func__); 376 377 /* 6. only get 2 dirty pages and clear them as well */ 378 vec_size = mem_size/page_size; 379 memset(mem, -1, mem_size); 380 381 /* get and clear second and third pages */ 382 ret = pagemap_ioctl(mem + page_size, 2 * page_size, vec, 1, 383 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 384 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 385 if (ret < 0) 386 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 387 388 ret2 = pagemap_ioctl(mem, mem_size, vec2, vec_size, 0, 0, 389 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 390 if (ret2 < 0) 391 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno)); 392 393 ksft_test_result(ret == 1 && LEN(vec[0]) == 2 && 394 vec[0].start == (uintptr_t)(mem + page_size) && 395 ret2 == 2 && LEN(vec2[0]) == 1 && vec2[0].start == (uintptr_t)mem && 396 LEN(vec2[1]) == vec_size - 3 && 397 vec2[1].start == (uintptr_t)(mem + 3 * page_size), 398 "%s only get 2 written pages and clear them as well\n", __func__); 399 400 wp_free(mem, mem_size); 401 munmap(mem, mem_size); 402 403 /* 7. Two regions */ 404 m[0] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 405 if (m[0] == MAP_FAILED) 406 ksft_exit_fail_msg("error nomem\n"); 407 m[1] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 408 if (m[1] == MAP_FAILED) 409 ksft_exit_fail_msg("error nomem\n"); 410 411 wp_init(m[0], mem_size); 412 wp_init(m[1], mem_size); 413 wp_addr_range(m[0], mem_size); 414 wp_addr_range(m[1], mem_size); 415 416 memset(m[0], 'a', mem_size); 417 memset(m[1], 'b', mem_size); 418 419 wp_addr_range(m[0], mem_size); 420 421 ret = pagemap_ioctl(m[1], mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0, 422 PAGE_IS_WRITTEN); 423 if (ret < 0) 424 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 425 426 ksft_test_result(ret == 1 && LEN(vec[0]) == mem_size/page_size, 427 "%s Two regions\n", __func__); 428 429 wp_free(m[0], mem_size); 430 wp_free(m[1], mem_size); 431 munmap(m[0], mem_size); 432 munmap(m[1], mem_size); 433 434 free(vec); 435 free(vec2); 436 437 /* 8. Smaller vec */ 438 mem_size = 1050 * page_size; 439 vec_size = mem_size/(page_size*2); 440 441 vec = calloc(vec_size, sizeof(struct page_region)); 442 if (!vec) 443 ksft_exit_fail_msg("error nomem\n"); 444 445 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 446 if (mem == MAP_FAILED) 447 ksft_exit_fail_msg("error nomem\n"); 448 449 wp_init(mem, mem_size); 450 wp_addr_range(mem, mem_size); 451 452 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 453 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 454 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 455 if (ret < 0) 456 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 457 458 for (i = 0; i < mem_size/page_size; i += 2) 459 mem[i * page_size]++; 460 461 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 462 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 463 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 464 if (ret < 0) 465 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 466 467 total_pages += ret; 468 469 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 470 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 471 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 472 if (ret < 0) 473 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 474 475 total_pages += ret; 476 477 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 478 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 479 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 480 if (ret < 0) 481 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 482 483 total_pages += ret; 484 485 ksft_test_result(total_pages == mem_size/(page_size*2), "%s Smaller max_pages\n", __func__); 486 487 free(vec); 488 wp_free(mem, mem_size); 489 munmap(mem, mem_size); 490 total_pages = 0; 491 492 /* 9. Smaller vec */ 493 mem_size = 10000 * page_size; 494 vec_size = 50; 495 496 vec = calloc(vec_size, sizeof(struct page_region)); 497 if (!vec) 498 ksft_exit_fail_msg("error nomem\n"); 499 500 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 501 if (mem == MAP_FAILED) 502 ksft_exit_fail_msg("error nomem\n"); 503 504 wp_init(mem, mem_size); 505 wp_addr_range(mem, mem_size); 506 507 for (count = 0; count < TEST_ITERATIONS; count++) { 508 total_writes = total_reads = 0; 509 walk_end = (long)mem; 510 511 for (i = 0; i < mem_size; i += page_size) { 512 if (rand() % 2) { 513 mem[i]++; 514 total_writes++; 515 } 516 } 517 518 while (total_reads < total_writes) { 519 ret = pagemap_ioc((void *)walk_end, mem_size-(walk_end - (long)mem), vec, 520 vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 521 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 522 if (ret < 0) 523 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 524 525 if ((unsigned long)ret > vec_size) 526 break; 527 528 reads = get_reads(vec, ret); 529 total_reads += reads; 530 } 531 532 if (total_reads != total_writes) 533 break; 534 } 535 536 ksft_test_result(count == TEST_ITERATIONS, "Smaller vec\n"); 537 538 free(vec); 539 wp_free(mem, mem_size); 540 munmap(mem, mem_size); 541 542 /* 10. Walk_end tester */ 543 vec_size = 1000; 544 mem_size = vec_size * page_size; 545 546 vec = calloc(vec_size, sizeof(struct page_region)); 547 if (!vec) 548 ksft_exit_fail_msg("error nomem\n"); 549 550 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 551 if (mem == MAP_FAILED) 552 ksft_exit_fail_msg("error nomem\n"); 553 554 wp_init(mem, mem_size); 555 wp_addr_range(mem, mem_size); 556 557 memset(mem, 0, mem_size); 558 559 ret = pagemap_ioc(mem, 0, vec, vec_size, 0, 560 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 561 if (ret < 0) 562 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 563 ksft_test_result(ret == 0 && walk_end == (long)mem, 564 "Walk_end: Same start and end address\n"); 565 566 ret = pagemap_ioc(mem, 0, vec, vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 567 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 568 if (ret < 0) 569 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 570 ksft_test_result(ret == 0 && walk_end == (long)mem, 571 "Walk_end: Same start and end with WP\n"); 572 573 ret = pagemap_ioc(mem, 0, vec, 0, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 574 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 575 if (ret < 0) 576 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 577 ksft_test_result(ret == 0 && walk_end == (long)mem, 578 "Walk_end: Same start and end with 0 output buffer\n"); 579 580 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 581 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 582 if (ret < 0) 583 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 584 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 585 "Walk_end: Big vec\n"); 586 587 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 588 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 589 if (ret < 0) 590 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 591 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 592 "Walk_end: vec of minimum length\n"); 593 594 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 595 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 596 if (ret < 0) 597 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 598 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 599 "Walk_end: Max pages specified\n"); 600 601 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 602 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 603 if (ret < 0) 604 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 605 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size/2), 606 "Walk_end: Half max pages\n"); 607 608 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 609 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 610 if (ret < 0) 611 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 612 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size), 613 "Walk_end: 1 max page\n"); 614 615 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 616 -1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 617 if (ret < 0) 618 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 619 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 620 "Walk_end: max pages\n"); 621 622 wp_addr_range(mem, mem_size); 623 for (i = 0; i < mem_size; i += 2 * page_size) 624 mem[i]++; 625 626 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 627 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 628 if (ret < 0) 629 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 630 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 631 "Walk_end sparse: Big vec\n"); 632 633 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 634 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 635 if (ret < 0) 636 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 637 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 638 "Walk_end sparse: vec of minimum length\n"); 639 640 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 641 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 642 if (ret < 0) 643 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 644 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 645 "Walk_end sparse: Max pages specified\n"); 646 647 ret = pagemap_ioc(mem, mem_size, vec, vec_size/2, 0, 648 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 649 if (ret < 0) 650 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 651 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 652 "Walk_end sparse: Max pages specified\n"); 653 654 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 655 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 656 if (ret < 0) 657 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 658 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 659 "Walk_end sparse: Max pages specified\n"); 660 661 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 662 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 663 if (ret < 0) 664 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 665 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 666 "Walk_endsparse : Half max pages\n"); 667 668 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 669 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 670 if (ret < 0) 671 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 672 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 673 "Walk_end: 1 max page\n"); 674 675 free(vec); 676 wp_free(mem, mem_size); 677 munmap(mem, mem_size); 678 679 return 0; 680 } 681 682 int base_tests(char *prefix, char *mem, unsigned long long mem_size, int skip) 683 { 684 unsigned long long vec_size; 685 int written; 686 struct page_region *vec, *vec2; 687 688 if (skip) { 689 ksft_test_result_skip("%s all new pages must not be written (dirty)\n", prefix); 690 ksft_test_result_skip("%s all pages must be written (dirty)\n", prefix); 691 ksft_test_result_skip("%s all pages dirty other than first and the last one\n", 692 prefix); 693 ksft_test_result_skip("%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix); 694 ksft_test_result_skip("%s only middle page dirty\n", prefix); 695 ksft_test_result_skip("%s only two middle pages dirty\n", prefix); 696 return 0; 697 } 698 699 vec_size = mem_size/page_size; 700 vec = calloc(vec_size, sizeof(struct page_region)); 701 vec2 = calloc(vec_size, sizeof(struct page_region)); 702 703 /* 1. all new pages must be not be written (dirty) */ 704 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 705 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 706 if (written < 0) 707 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 708 709 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", prefix); 710 711 /* 2. all pages must be written */ 712 memset(mem, -1, mem_size); 713 714 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0, 715 PAGE_IS_WRITTEN); 716 if (written < 0) 717 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 718 719 ksft_test_result(written == 1 && LEN(vec[0]) == mem_size/page_size, 720 "%s all pages must be written (dirty)\n", prefix); 721 722 /* 3. all pages dirty other than first and the last one */ 723 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 724 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 725 if (written < 0) 726 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 727 728 memset(mem + page_size, 0, mem_size - (2 * page_size)); 729 730 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 731 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 732 if (written < 0) 733 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 734 735 ksft_test_result(written == 1 && LEN(vec[0]) >= vec_size - 2 && LEN(vec[0]) <= vec_size, 736 "%s all pages dirty other than first and the last one\n", prefix); 737 738 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, 739 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 740 if (written < 0) 741 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 742 743 ksft_test_result(written == 0, 744 "%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix); 745 746 /* 4. only middle page dirty */ 747 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 748 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 749 if (written < 0) 750 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 751 752 mem[vec_size/2 * page_size]++; 753 754 written = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 755 0, 0, PAGE_IS_WRITTEN); 756 if (written < 0) 757 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 758 759 ksft_test_result(written == 1 && LEN(vec[0]) >= 1, 760 "%s only middle page dirty\n", prefix); 761 762 /* 5. only two middle pages dirty and walk over only middle pages */ 763 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 764 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE); 765 if (written < 0) 766 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 767 768 mem[vec_size/2 * page_size]++; 769 mem[(vec_size/2 + 1) * page_size]++; 770 771 written = pagemap_ioctl(&mem[vec_size/2 * page_size], 2 * page_size, vec, 1, 0, 772 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE); 773 if (written < 0) 774 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 775 776 ksft_test_result(written == 1 && vec[0].start == (uintptr_t)(&mem[vec_size/2 * page_size]) 777 && LEN(vec[0]) == 2, 778 "%s only two middle pages dirty\n", prefix); 779 780 free(vec); 781 free(vec2); 782 return 0; 783 } 784 785 void *gethugepage(int map_size) 786 { 787 int ret; 788 char *map; 789 790 map = memalign(hpage_size, map_size); 791 if (!map) 792 ksft_exit_fail_msg("memalign failed %d %s\n", errno, strerror(errno)); 793 794 ret = madvise(map, map_size, MADV_HUGEPAGE); 795 if (ret) 796 return NULL; 797 798 memset(map, 0, map_size); 799 800 return map; 801 } 802 803 int hpage_unit_tests(void) 804 { 805 char *map; 806 int ret, ret2; 807 size_t num_pages = 10; 808 unsigned long long map_size = hpage_size * num_pages; 809 unsigned long long vec_size = map_size/page_size; 810 struct page_region *vec, *vec2; 811 812 vec = calloc(vec_size, sizeof(struct page_region)); 813 vec2 = calloc(vec_size, sizeof(struct page_region)); 814 if (!vec || !vec2) 815 ksft_exit_fail_msg("malloc failed\n"); 816 817 map = gethugepage(map_size); 818 if (map) { 819 wp_init(map, map_size); 820 wp_addr_range(map, map_size); 821 822 /* 1. all new huge page must not be written (dirty) */ 823 ret = pagemap_ioctl(map, map_size, vec, vec_size, 824 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 825 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 826 if (ret < 0) 827 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 828 829 ksft_test_result(ret == 0, "%s all new huge page must not be written (dirty)\n", 830 __func__); 831 832 /* 2. all the huge page must not be written */ 833 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 834 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 835 if (ret < 0) 836 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 837 838 ksft_test_result(ret == 0, "%s all the huge page must not be written\n", __func__); 839 840 /* 3. all the huge page must be written and clear dirty as well */ 841 memset(map, -1, map_size); 842 ret = pagemap_ioctl(map, map_size, vec, vec_size, 843 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 844 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 845 if (ret < 0) 846 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 847 848 ksft_test_result(ret == 1 && vec[0].start == (uintptr_t)map && 849 LEN(vec[0]) == vec_size && vec[0].categories == PAGE_IS_WRITTEN, 850 "%s all the huge page must be written and clear\n", __func__); 851 852 /* 4. only middle page written */ 853 wp_free(map, map_size); 854 free(map); 855 map = gethugepage(map_size); 856 wp_init(map, map_size); 857 wp_addr_range(map, map_size); 858 map[vec_size/2 * page_size]++; 859 860 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 861 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 862 if (ret < 0) 863 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 864 865 ksft_test_result(ret == 1 && LEN(vec[0]) > 0, 866 "%s only middle page written\n", __func__); 867 868 wp_free(map, map_size); 869 free(map); 870 } else { 871 ksft_test_result_skip("%s all new huge page must be written\n", __func__); 872 ksft_test_result_skip("%s all the huge page must not be written\n", __func__); 873 ksft_test_result_skip("%s all the huge page must be written and clear\n", __func__); 874 ksft_test_result_skip("%s only middle page written\n", __func__); 875 } 876 877 /* 5. clear first half of huge page */ 878 map = gethugepage(map_size); 879 if (map) { 880 wp_init(map, map_size); 881 wp_addr_range(map, map_size); 882 883 memset(map, 0, map_size); 884 885 wp_addr_range(map, map_size/2); 886 887 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 888 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 889 if (ret < 0) 890 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 891 892 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 && 893 vec[0].start == (uintptr_t)(map + map_size/2), 894 "%s clear first half of huge page\n", __func__); 895 wp_free(map, map_size); 896 free(map); 897 } else { 898 ksft_test_result_skip("%s clear first half of huge page\n", __func__); 899 } 900 901 /* 6. clear first half of huge page with limited buffer */ 902 map = gethugepage(map_size); 903 if (map) { 904 wp_init(map, map_size); 905 wp_addr_range(map, map_size); 906 907 memset(map, 0, map_size); 908 909 ret = pagemap_ioctl(map, map_size, vec, vec_size, 910 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 911 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 912 if (ret < 0) 913 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 914 915 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 916 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 917 if (ret < 0) 918 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 919 920 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 && 921 vec[0].start == (uintptr_t)(map + map_size/2), 922 "%s clear first half of huge page with limited buffer\n", 923 __func__); 924 wp_free(map, map_size); 925 free(map); 926 } else { 927 ksft_test_result_skip("%s clear first half of huge page with limited buffer\n", 928 __func__); 929 } 930 931 /* 7. clear second half of huge page */ 932 map = gethugepage(map_size); 933 if (map) { 934 wp_init(map, map_size); 935 wp_addr_range(map, map_size); 936 937 memset(map, -1, map_size); 938 939 ret = pagemap_ioctl(map + map_size/2, map_size/2, vec, vec_size, 940 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, vec_size/2, 941 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 942 if (ret < 0) 943 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 944 945 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 946 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 947 if (ret < 0) 948 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 949 950 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2, 951 "%s clear second half huge page\n", __func__); 952 wp_free(map, map_size); 953 free(map); 954 } else { 955 ksft_test_result_skip("%s clear second half huge page\n", __func__); 956 } 957 958 /* 8. get half huge page */ 959 map = gethugepage(map_size); 960 if (map) { 961 wp_init(map, map_size); 962 wp_addr_range(map, map_size); 963 964 memset(map, -1, map_size); 965 usleep(100); 966 967 ret = pagemap_ioctl(map, map_size, vec, 1, 968 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 969 hpage_size/(2*page_size), PAGE_IS_WRITTEN, 0, 0, 970 PAGE_IS_WRITTEN); 971 if (ret < 0) 972 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 973 974 ksft_test_result(ret == 1 && LEN(vec[0]) == hpage_size/(2*page_size), 975 "%s get half huge page\n", __func__); 976 977 ret2 = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 978 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 979 if (ret2 < 0) 980 ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno)); 981 982 ksft_test_result(ret2 == 1 && LEN(vec[0]) == (map_size - hpage_size/2)/page_size, 983 "%s get half huge page\n", __func__); 984 985 wp_free(map, map_size); 986 free(map); 987 } else { 988 ksft_test_result_skip("%s get half huge page\n", __func__); 989 ksft_test_result_skip("%s get half huge page\n", __func__); 990 } 991 992 free(vec); 993 free(vec2); 994 return 0; 995 } 996 997 int unmapped_region_tests(void) 998 { 999 void *start = (void *)0x10000000; 1000 int written, len = 0x00040000; 1001 long vec_size = len / page_size; 1002 struct page_region *vec = calloc(vec_size, sizeof(struct page_region)); 1003 1004 /* 1. Get written pages */ 1005 written = pagemap_ioctl(start, len, vec, vec_size, 0, 0, 1006 PAGEMAP_NON_WRITTEN_BITS, 0, 0, PAGEMAP_NON_WRITTEN_BITS); 1007 if (written < 0) 1008 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 1009 1010 ksft_test_result(written >= 0, "%s Get status of pages\n", __func__); 1011 1012 free(vec); 1013 return 0; 1014 } 1015 1016 static void test_simple(void) 1017 { 1018 int i; 1019 char *map; 1020 struct page_region vec; 1021 1022 map = aligned_alloc(page_size, page_size); 1023 if (!map) 1024 ksft_exit_fail_msg("aligned_alloc failed\n"); 1025 1026 wp_init(map, page_size); 1027 wp_addr_range(map, page_size); 1028 1029 for (i = 0 ; i < TEST_ITERATIONS; i++) { 1030 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0, 1031 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 1) { 1032 ksft_print_msg("written bit was 1, but should be 0 (i=%d)\n", i); 1033 break; 1034 } 1035 1036 wp_addr_range(map, page_size); 1037 /* Write something to the page to get the written bit enabled on the page */ 1038 map[0]++; 1039 1040 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0, 1041 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0) { 1042 ksft_print_msg("written bit was 0, but should be 1 (i=%d)\n", i); 1043 break; 1044 } 1045 1046 wp_addr_range(map, page_size); 1047 } 1048 wp_free(map, page_size); 1049 free(map); 1050 1051 ksft_test_result(i == TEST_ITERATIONS, "Test %s\n", __func__); 1052 } 1053 1054 /* 1055 * A range that was populated and then MADV_DONTNEED'd is genuine pte_none 1056 * with no uffd-wp marker. Such a pte must read the same regardless of which 1057 * PAGEMAP_SCAN path serves the request: both the PAGE_IS_WRITTEN fast path and 1058 * the generic path (reached e.g. via category_anyof_mask) must report every 1059 * page written. 1060 */ 1061 static void unpopulated_scan_test(void) 1062 { 1063 int npages = 16, i; 1064 long mem_size = npages * page_size; 1065 struct page_region regions[16]; 1066 long fast = 0, slow = 0, ret; 1067 char *mem; 1068 1069 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, 1070 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1071 if (mem == MAP_FAILED) 1072 ksft_exit_fail_msg("%s mmap failed\n", __func__); 1073 1074 wp_init(mem, mem_size); 1075 1076 /* Populate, then drop: the ptes become pte_none without a marker. */ 1077 memset(mem, 1, mem_size); 1078 if (madvise(mem, mem_size, MADV_DONTNEED)) 1079 ksft_exit_fail_msg("%s MADV_DONTNEED failed\n", __func__); 1080 1081 /* Fast path: category_mask == return_mask == PAGE_IS_WRITTEN. */ 1082 ret = pagemap_ioctl(mem, mem_size, regions, npages, 0, 0, 1083 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 1084 if (ret < 0) 1085 ksft_exit_fail_msg("%s fast scan failed\n", __func__); 1086 for (i = 0; i < ret; i++) 1087 fast += LEN(regions[i]); 1088 1089 /* Generic path: same query expressed via category_anyof_mask. */ 1090 ret = pagemap_ioctl(mem, mem_size, regions, npages, 0, 0, 1091 0, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN); 1092 if (ret < 0) 1093 ksft_exit_fail_msg("%s generic scan failed\n", __func__); 1094 for (i = 0; i < ret; i++) 1095 slow += LEN(regions[i]); 1096 1097 ksft_test_result(fast == npages && slow == npages, 1098 "%s unpopulated ptes reported written by both paths (%ld, %ld of %d)\n", 1099 __func__, fast, slow, npages); 1100 1101 wp_free(mem, mem_size); 1102 munmap(mem, mem_size); 1103 } 1104 1105 int sanity_tests(void) 1106 { 1107 unsigned long long mem_size, vec_size; 1108 long ret, fd, i, buf_size, nr_pages; 1109 struct page_region *vec; 1110 char *mem, *fmem; 1111 struct stat sbuf; 1112 1113 /* 1. wrong operation */ 1114 mem_size = 10 * page_size; 1115 vec_size = mem_size / page_size; 1116 1117 vec = calloc(vec_size, sizeof(struct page_region)); 1118 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1119 if (mem == MAP_FAILED || vec == MAP_FAILED) 1120 ksft_exit_fail_msg("error nomem\n"); 1121 1122 wp_init(mem, mem_size); 1123 wp_addr_range(mem, mem_size); 1124 1125 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 1126 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 1127 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0, 1128 "%s WP op can be specified with !PAGE_IS_WRITTEN\n", __func__); 1129 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1130 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0, 1131 "%s required_mask specified\n", __func__); 1132 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1133 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL) >= 0, 1134 "%s anyof_mask specified\n", __func__); 1135 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1136 0, 0, PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL) >= 0, 1137 "%s excluded_mask specified\n", __func__); 1138 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1139 PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL, 0, 1140 PAGEMAP_BITS_ALL) >= 0, 1141 "%s required_mask and anyof_mask specified\n", __func__); 1142 wp_free(mem, mem_size); 1143 munmap(mem, mem_size); 1144 1145 /* 2. Get sd and present pages with anyof_mask */ 1146 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1147 if (mem == MAP_FAILED) 1148 ksft_exit_fail_msg("error nomem\n"); 1149 wp_init(mem, mem_size); 1150 wp_addr_range(mem, mem_size); 1151 1152 memset(mem, 0, mem_size); 1153 1154 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1155 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL); 1156 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1157 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1158 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1159 "%s Get sd and present pages with anyof_mask\n", __func__); 1160 1161 /* 3. Get sd and present pages with required_mask */ 1162 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1163 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL); 1164 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1165 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1166 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1167 "%s Get all the pages with required_mask\n", __func__); 1168 1169 /* 4. Get sd and present pages with required_mask and anyof_mask */ 1170 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1171 PAGE_IS_WRITTEN, PAGE_IS_PRESENT, 0, PAGEMAP_BITS_ALL); 1172 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1173 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1174 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1175 "%s Get sd and present pages with required_mask and anyof_mask\n", 1176 __func__); 1177 1178 /* 5. Don't get sd pages */ 1179 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1180 PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN, PAGEMAP_BITS_ALL); 1181 ksft_test_result(ret == 0, "%s Don't get sd pages\n", __func__); 1182 1183 /* 6. Don't get present pages */ 1184 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1185 PAGE_IS_PRESENT, 0, PAGE_IS_PRESENT, PAGEMAP_BITS_ALL); 1186 ksft_test_result(ret == 0, "%s Don't get present pages\n", __func__); 1187 1188 wp_free(mem, mem_size); 1189 munmap(mem, mem_size); 1190 1191 /* 8. Find written present pages with return mask */ 1192 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1193 if (mem == MAP_FAILED) 1194 ksft_exit_fail_msg("error nomem\n"); 1195 wp_init(mem, mem_size); 1196 wp_addr_range(mem, mem_size); 1197 1198 memset(mem, 0, mem_size); 1199 1200 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 1201 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 1202 0, PAGEMAP_BITS_ALL, 0, PAGE_IS_WRITTEN); 1203 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1204 vec[0].categories == PAGE_IS_WRITTEN, 1205 "%s Find written present pages with return mask\n", __func__); 1206 wp_free(mem, mem_size); 1207 munmap(mem, mem_size); 1208 1209 /* 9. Memory mapped file */ 1210 fd = open(progname, O_RDONLY); 1211 if (fd < 0) 1212 ksft_exit_fail_msg("%s Memory mapped file\n", __func__); 1213 1214 ret = stat(progname, &sbuf); 1215 if (ret < 0) 1216 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 1217 1218 fmem = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0); 1219 if (fmem == MAP_FAILED) 1220 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1221 1222 nr_pages = (sbuf.st_size + page_size - 1) / page_size; 1223 force_read_pages(fmem, nr_pages, page_size); 1224 1225 ret = pagemap_ioctl(fmem, sbuf.st_size, vec, vec_size, 0, 0, 1226 0, PAGEMAP_NON_WRITTEN_BITS, 0, PAGEMAP_NON_WRITTEN_BITS); 1227 1228 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem && 1229 LEN(vec[0]) == nr_pages && 1230 (vec[0].categories & PAGE_IS_FILE), 1231 "%s Memory mapped file\n", __func__); 1232 1233 munmap(fmem, sbuf.st_size); 1234 close(fd); 1235 1236 /* 10. Create and read/write to a memory mapped file */ 1237 buf_size = page_size * 10; 1238 1239 fd = open(__FILE__".tmp2", O_RDWR | O_CREAT, 0666); 1240 if (fd < 0) 1241 ksft_exit_fail_msg("Read/write to memory: %s\n", 1242 strerror(errno)); 1243 1244 for (i = 0; i < buf_size; i++) 1245 if (write(fd, "c", 1) < 0) 1246 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1247 1248 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1249 if (fmem == MAP_FAILED) 1250 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1251 1252 wp_init(fmem, buf_size); 1253 wp_addr_range(fmem, buf_size); 1254 1255 for (i = 0; i < buf_size; i++) 1256 fmem[i] = 'z'; 1257 1258 msync(fmem, buf_size, MS_SYNC); 1259 1260 ret = pagemap_ioctl(fmem, buf_size, vec, vec_size, 0, 0, 1261 PAGE_IS_WRITTEN, PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_FILE, 0, 1262 PAGEMAP_BITS_ALL); 1263 1264 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem && 1265 LEN(vec[0]) == (buf_size/page_size) && 1266 (vec[0].categories & PAGE_IS_WRITTEN), 1267 "%s Read/write to memory\n", __func__); 1268 1269 wp_free(fmem, buf_size); 1270 munmap(fmem, buf_size); 1271 close(fd); 1272 1273 free(vec); 1274 return 0; 1275 } 1276 1277 int mprotect_tests(void) 1278 { 1279 int ret; 1280 char *mem, *mem2; 1281 struct page_region vec; 1282 int pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 1283 1284 if (pagemap_fd < 0) { 1285 fprintf(stderr, "open() failed\n"); 1286 exit(1); 1287 } 1288 1289 /* 1. Map two pages */ 1290 mem = mmap(0, 2 * page_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1291 if (mem == MAP_FAILED) 1292 ksft_exit_fail_msg("error nomem\n"); 1293 wp_init(mem, 2 * page_size); 1294 wp_addr_range(mem, 2 * page_size); 1295 1296 /* Populate both pages. */ 1297 memset(mem, 1, 2 * page_size); 1298 1299 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1300 0, 0, PAGE_IS_WRITTEN); 1301 if (ret < 0) 1302 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1303 1304 ksft_test_result(ret == 1 && LEN(vec) == 2, "%s Both pages written\n", __func__); 1305 1306 /* 2. Start tracking */ 1307 wp_addr_range(mem, 2 * page_size); 1308 1309 ksft_test_result(pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, 1310 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0, 1311 "%s Both pages are not written (dirty)\n", __func__); 1312 1313 /* 3. Remap the second page */ 1314 mem2 = mmap(mem + page_size, page_size, PROT_READ|PROT_WRITE, 1315 MAP_PRIVATE|MAP_ANON|MAP_FIXED, -1, 0); 1316 if (mem2 == MAP_FAILED) 1317 ksft_exit_fail_msg("error nomem\n"); 1318 wp_init(mem2, page_size); 1319 wp_addr_range(mem2, page_size); 1320 1321 /* Protect + unprotect. */ 1322 mprotect(mem, page_size, PROT_NONE); 1323 mprotect(mem, 2 * page_size, PROT_READ); 1324 mprotect(mem, 2 * page_size, PROT_READ|PROT_WRITE); 1325 1326 /* Modify both pages. */ 1327 memset(mem, 2, 2 * page_size); 1328 1329 /* Protect + unprotect. */ 1330 mprotect(mem, page_size, PROT_NONE); 1331 mprotect(mem, page_size, PROT_READ); 1332 mprotect(mem, page_size, PROT_READ|PROT_WRITE); 1333 1334 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1335 0, 0, PAGE_IS_WRITTEN); 1336 if (ret < 0) 1337 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1338 1339 ksft_test_result(ret == 1 && LEN(vec) == 2, 1340 "%s Both pages written after remap and mprotect\n", __func__); 1341 1342 /* 4. Clear and make the pages written */ 1343 wp_addr_range(mem, 2 * page_size); 1344 1345 memset(mem, 'A', 2 * page_size); 1346 1347 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1348 0, 0, PAGE_IS_WRITTEN); 1349 if (ret < 0) 1350 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1351 1352 ksft_test_result(ret == 1 && LEN(vec) == 2, 1353 "%s Clear and make the pages written\n", __func__); 1354 1355 wp_free(mem, 2 * page_size); 1356 munmap(mem, 2 * page_size); 1357 return 0; 1358 } 1359 1360 /* transact test */ 1361 static const unsigned int nthreads = 6, pages_per_thread = 32, access_per_thread = 8; 1362 static pthread_barrier_t start_barrier, end_barrier; 1363 static unsigned int extra_thread_faults; 1364 static unsigned int iter_count = 1000; 1365 static volatile int finish; 1366 1367 static ssize_t get_dirty_pages_reset(char *mem, unsigned int count, 1368 int reset, int page_size) 1369 { 1370 struct pm_scan_arg arg = {0}; 1371 struct page_region rgns[256]; 1372 unsigned long long i, j; 1373 long ret; 1374 int cnt; 1375 1376 arg.size = sizeof(struct pm_scan_arg); 1377 arg.start = (uintptr_t)mem; 1378 arg.max_pages = count; 1379 arg.end = (uintptr_t)(mem + count * page_size); 1380 arg.vec = (uintptr_t)rgns; 1381 arg.vec_len = sizeof(rgns) / sizeof(*rgns); 1382 if (reset) 1383 arg.flags |= PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC; 1384 arg.category_mask = PAGE_IS_WRITTEN; 1385 arg.return_mask = PAGE_IS_WRITTEN; 1386 1387 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 1388 if (ret < 0) 1389 ksft_exit_fail_msg("ioctl failed\n"); 1390 1391 cnt = 0; 1392 for (i = 0; i < (unsigned long)ret; ++i) { 1393 if (rgns[i].categories != PAGE_IS_WRITTEN) 1394 ksft_exit_fail_msg("wrong flags\n"); 1395 1396 for (j = 0; j < LEN(rgns[i]); ++j) 1397 cnt++; 1398 } 1399 1400 return cnt; 1401 } 1402 1403 void *thread_proc(void *mem) 1404 { 1405 int *m = mem; 1406 long curr_faults, faults; 1407 struct rusage r; 1408 unsigned int i; 1409 int ret; 1410 1411 if (getrusage(RUSAGE_THREAD, &r)) 1412 ksft_exit_fail_msg("getrusage\n"); 1413 1414 curr_faults = r.ru_minflt; 1415 1416 while (!finish) { 1417 ret = pthread_barrier_wait(&start_barrier); 1418 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1419 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1420 1421 for (i = 0; i < access_per_thread; ++i) 1422 __atomic_add_fetch(m + i * (page_size / sizeof(*m)), 1, __ATOMIC_SEQ_CST); 1423 1424 ret = pthread_barrier_wait(&end_barrier); 1425 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1426 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1427 1428 if (getrusage(RUSAGE_THREAD, &r)) 1429 ksft_exit_fail_msg("getrusage\n"); 1430 1431 faults = r.ru_minflt - curr_faults; 1432 if (faults < access_per_thread) 1433 ksft_exit_fail_msg("faults < access_per_thread"); 1434 1435 __atomic_add_fetch(&extra_thread_faults, faults - access_per_thread, 1436 __ATOMIC_SEQ_CST); 1437 curr_faults = r.ru_minflt; 1438 } 1439 1440 return NULL; 1441 } 1442 1443 static void transact_test(int page_size) 1444 { 1445 unsigned int i, count, extra_pages; 1446 unsigned int c; 1447 pthread_t th; 1448 char *mem; 1449 int ret; 1450 1451 if (pthread_barrier_init(&start_barrier, NULL, nthreads + 1)) 1452 ksft_exit_fail_msg("pthread_barrier_init\n"); 1453 1454 if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1)) 1455 ksft_exit_fail_msg("pthread_barrier_init\n"); 1456 1457 mem = mmap(NULL, page_size * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, 1458 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 1459 if (mem == MAP_FAILED) 1460 ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno)); 1461 1462 wp_init(mem, page_size * nthreads * pages_per_thread); 1463 wp_addr_range(mem, page_size * nthreads * pages_per_thread); 1464 1465 memset(mem, 0, page_size * nthreads * pages_per_thread); 1466 1467 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1468 ksft_test_result(count > 0, "%s count %u\n", __func__, count); 1469 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1470 ksft_test_result(count == 0, "%s count %u\n", __func__, count); 1471 1472 finish = 0; 1473 for (i = 0; i < nthreads; ++i) 1474 pthread_create(&th, NULL, thread_proc, mem + page_size * i * pages_per_thread); 1475 1476 extra_pages = 0; 1477 for (i = 0; i < iter_count; ++i) { 1478 count = 0; 1479 1480 ret = pthread_barrier_wait(&start_barrier); 1481 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1482 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1483 1484 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, 1485 page_size); 1486 1487 ret = pthread_barrier_wait(&end_barrier); 1488 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1489 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1490 1491 if (count > nthreads * access_per_thread) 1492 ksft_exit_fail_msg("Too big count %u expected %u, iter %u\n", 1493 count, nthreads * access_per_thread, i); 1494 1495 c = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1496 count += c; 1497 1498 if (c > nthreads * access_per_thread) { 1499 ksft_test_result_fail(" %s count > nthreads\n", __func__); 1500 return; 1501 } 1502 1503 if (count != nthreads * access_per_thread) { 1504 /* 1505 * The purpose of the test is to make sure that no page updates are lost 1506 * when the page updates and read-resetting soft dirty flags are performed 1507 * in parallel. However, it is possible that the application will get the 1508 * soft dirty flags twice on the two consecutive read-resets. This seems 1509 * unavoidable as soft dirty flag is handled in software through page faults 1510 * in kernel. While the updating the flags is supposed to be synchronized 1511 * between page fault handling and read-reset, it is possible that 1512 * read-reset happens after page fault PTE update but before the application 1513 * re-executes write instruction. So read-reset gets the flag, clears write 1514 * access and application gets page fault again for the same write. 1515 */ 1516 if (count < nthreads * access_per_thread) { 1517 ksft_test_result_fail("Lost update, iter %u, %u vs %u.\n", i, count, 1518 nthreads * access_per_thread); 1519 return; 1520 } 1521 1522 extra_pages += count - nthreads * access_per_thread; 1523 } 1524 } 1525 1526 pthread_barrier_wait(&start_barrier); 1527 finish = 1; 1528 pthread_barrier_wait(&end_barrier); 1529 1530 ksft_test_result_pass("%s Extra pages %u (%.1lf%%), extra thread faults %u.\n", __func__, 1531 extra_pages, 1532 100.0 * extra_pages / (iter_count * nthreads * access_per_thread), 1533 extra_thread_faults); 1534 } 1535 1536 void zeropfn_tests(void) 1537 { 1538 unsigned long long mem_size; 1539 struct page_region vec; 1540 int i, ret; 1541 char *mmap_mem, *mem; 1542 1543 /* Test with normal memory */ 1544 mem_size = 10 * page_size; 1545 mem = mmap(NULL, mem_size, PROT_READ, MAP_PRIVATE | MAP_ANON, -1, 0); 1546 if (mem == MAP_FAILED) 1547 ksft_exit_fail_msg("error nomem\n"); 1548 1549 /* Touch each page to ensure it's mapped */ 1550 for (i = 0; i < mem_size; i += page_size) 1551 (void)((volatile char *)mem)[i]; 1552 1553 ret = pagemap_ioctl(mem, mem_size, &vec, 1, 0, 1554 (mem_size / page_size), PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO); 1555 if (ret < 0) 1556 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1557 1558 ksft_test_result(ret == 1 && LEN(vec) == (mem_size / page_size), 1559 "%s all pages must have PFNZERO set\n", __func__); 1560 1561 munmap(mem, mem_size); 1562 1563 /* Test with huge page if user_zero_page is set to 1 */ 1564 if (!detect_huge_zeropage()) { 1565 ksft_test_result_skip("%s use_zero_page not supported or set to 1\n", __func__); 1566 return; 1567 } 1568 1569 mem_size = 2 * hpage_size; 1570 mmap_mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, 1571 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1572 if (mmap_mem == MAP_FAILED) 1573 ksft_exit_fail_msg("error nomem\n"); 1574 1575 /* We need a THP-aligned memory area. */ 1576 mem = (char *)(((uintptr_t)mmap_mem + hpage_size) & ~(hpage_size - 1)); 1577 1578 ret = madvise(mem, hpage_size, MADV_HUGEPAGE); 1579 if (!ret) { 1580 FORCE_READ(*mem); 1581 1582 ret = pagemap_ioctl(mem, hpage_size, &vec, 1, 0, 1583 0, PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO); 1584 if (ret < 0) 1585 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1586 1587 ksft_test_result(ret == 1 && LEN(vec) == (hpage_size / page_size), 1588 "%s all huge pages must have PFNZERO set\n", __func__); 1589 } else { 1590 ksft_test_result_skip("%s huge page not supported\n", __func__); 1591 } 1592 1593 munmap(mmap_mem, mem_size); 1594 } 1595 1596 int main(int __attribute__((unused)) argc, char *argv[]) 1597 { 1598 int shmid, buf_size, fd, i, ret; 1599 unsigned long long mem_size; 1600 char *mem, *map, *fmem; 1601 struct stat sbuf; 1602 1603 progname = argv[0]; 1604 1605 ksft_print_header(); 1606 1607 if (init_uffd()) 1608 ksft_exit_skip("Failed to initialize userfaultfd\n"); 1609 1610 if (!hugetlb_setup_default(4)) 1611 ksft_print_msg("HugeTLB test will be skipped\n"); 1612 1613 ksft_set_plan(118); 1614 1615 page_size = getpagesize(); 1616 hpage_size = read_pmd_pagesize(); 1617 1618 pagemap_fd = open(PAGEMAP, O_RDONLY); 1619 if (pagemap_fd < 0) 1620 ksft_exit_fail_msg("Failed to open " PAGEMAP "\n"); 1621 1622 /* 1. Sanity testing */ 1623 sanity_tests_sd(); 1624 1625 /* 2. Normal page testing */ 1626 mem_size = 10 * page_size; 1627 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1628 if (mem == MAP_FAILED) 1629 ksft_exit_fail_msg("error nomem\n"); 1630 wp_init(mem, mem_size); 1631 wp_addr_range(mem, mem_size); 1632 1633 base_tests("Page testing:", mem, mem_size, 0); 1634 1635 wp_free(mem, mem_size); 1636 munmap(mem, mem_size); 1637 1638 /* 3. Large page testing */ 1639 mem_size = 512 * 10 * page_size; 1640 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1641 if (mem == MAP_FAILED) 1642 ksft_exit_fail_msg("error nomem\n"); 1643 wp_init(mem, mem_size); 1644 wp_addr_range(mem, mem_size); 1645 1646 base_tests("Large Page testing:", mem, mem_size, 0); 1647 1648 wp_free(mem, mem_size); 1649 munmap(mem, mem_size); 1650 1651 /* 4. Huge page testing */ 1652 map = gethugepage(hpage_size); 1653 if (map) { 1654 wp_init(map, hpage_size); 1655 wp_addr_range(map, hpage_size); 1656 base_tests("Huge page testing:", map, hpage_size, 0); 1657 wp_free(map, hpage_size); 1658 free(map); 1659 } else { 1660 base_tests("Huge page testing:", NULL, 0, 1); 1661 } 1662 1663 /* 5. SHM Hugetlb page testing */ 1664 mem_size = default_huge_page_size(); 1665 mem = gethugetlb_mem(mem_size, &shmid); 1666 if (mem) { 1667 wp_init(mem, mem_size); 1668 wp_addr_range(mem, mem_size); 1669 1670 base_tests("Hugetlb shmem testing:", mem, mem_size, 0); 1671 1672 wp_free(mem, mem_size); 1673 shmctl(shmid, IPC_RMID, NULL); 1674 } else { 1675 base_tests("Hugetlb shmem testing:", NULL, 0, 1); 1676 } 1677 1678 /* 6. Hugetlb page testing */ 1679 mem = gethugetlb_mem(mem_size, NULL); 1680 if (mem) { 1681 wp_init(mem, mem_size); 1682 wp_addr_range(mem, mem_size); 1683 1684 base_tests("Hugetlb mem testing:", mem, mem_size, 0); 1685 1686 wp_free(mem, mem_size); 1687 } else { 1688 base_tests("Hugetlb mem testing:", NULL, 0, 1); 1689 } 1690 1691 /* 7. File Hugetlb testing */ 1692 mem_size = default_huge_page_size(); 1693 fd = memfd_create("uffd-test", MFD_HUGETLB | MFD_NOEXEC_SEAL); 1694 if (fd < 0) 1695 ksft_exit_fail_msg("uffd-test creation failed %d %s\n", errno, strerror(errno)); 1696 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 1697 if (mem != MAP_FAILED) { 1698 wp_init(mem, mem_size); 1699 wp_addr_range(mem, mem_size); 1700 1701 base_tests("Hugetlb shmem testing:", mem, mem_size, 0); 1702 1703 wp_free(mem, mem_size); 1704 shmctl(shmid, IPC_RMID, NULL); 1705 } else { 1706 base_tests("Hugetlb shmem testing:", NULL, 0, 1); 1707 } 1708 close(fd); 1709 1710 /* 8. File memory testing */ 1711 buf_size = page_size * 10; 1712 1713 fd = open(__FILE__".tmp0", O_RDWR | O_CREAT, 0777); 1714 if (fd < 0) 1715 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n", 1716 strerror(errno)); 1717 1718 for (i = 0; i < buf_size; i++) 1719 if (write(fd, "c", 1) < 0) 1720 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1721 1722 ret = stat(__FILE__".tmp0", &sbuf); 1723 if (ret < 0) 1724 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1725 1726 fmem = mmap(NULL, sbuf.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1727 if (fmem == MAP_FAILED) 1728 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1729 1730 wp_init(fmem, sbuf.st_size); 1731 wp_addr_range(fmem, sbuf.st_size); 1732 1733 base_tests("File memory testing:", fmem, sbuf.st_size, 0); 1734 1735 wp_free(fmem, sbuf.st_size); 1736 munmap(fmem, sbuf.st_size); 1737 close(fd); 1738 1739 /* 9. File memory testing */ 1740 buf_size = page_size * 10; 1741 1742 fd = memfd_create(__FILE__".tmp00", MFD_NOEXEC_SEAL); 1743 if (fd < 0) 1744 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n", 1745 strerror(errno)); 1746 1747 if (ftruncate(fd, buf_size)) 1748 ksft_exit_fail_msg("Error ftruncate\n"); 1749 1750 for (i = 0; i < buf_size; i++) 1751 if (write(fd, "c", 1) < 0) 1752 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1753 1754 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1755 if (fmem == MAP_FAILED) 1756 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1757 1758 wp_init(fmem, buf_size); 1759 wp_addr_range(fmem, buf_size); 1760 1761 base_tests("File anonymous memory testing:", fmem, buf_size, 0); 1762 1763 wp_free(fmem, buf_size); 1764 munmap(fmem, buf_size); 1765 close(fd); 1766 1767 /* 10. Huge page tests */ 1768 hpage_unit_tests(); 1769 1770 /* 11. Iterative test */ 1771 test_simple(); 1772 1773 /* 12. Mprotect test */ 1774 mprotect_tests(); 1775 1776 /* 13. Transact test */ 1777 transact_test(page_size); 1778 1779 /* 14. Sanity testing */ 1780 sanity_tests(); 1781 1782 /*15. Unmapped address test */ 1783 unmapped_region_tests(); 1784 1785 /* 16. Userfaultfd tests */ 1786 userfaultfd_tests(); 1787 1788 /* 17. ZEROPFN tests */ 1789 zeropfn_tests(); 1790 1791 /* 18. Unpopulated pte scan-path consistency */ 1792 unpopulated_scan_test(); 1793 1794 close(pagemap_fd); 1795 ksft_finished(); 1796 } 1797