1 // SPDX-License-Identifier: GPL-2.0 2 3 #define _GNU_SOURCE 4 #include <stdio.h> 5 #include <fcntl.h> 6 #include <string.h> 7 #include <sys/mman.h> 8 #include <errno.h> 9 #include <malloc.h> 10 #include <linux/types.h> 11 #include <linux/mman.h> 12 #include <linux/memfd.h> 13 #include <linux/userfaultfd.h> 14 #include <linux/fs.h> 15 #include <sys/ioctl.h> 16 #include <sys/stat.h> 17 #include <math.h> 18 #include <asm/unistd.h> 19 #include <pthread.h> 20 #include <sys/resource.h> 21 #include <assert.h> 22 #include <sys/ipc.h> 23 #include <sys/shm.h> 24 25 #include "vm_util.h" 26 #include "kselftest.h" 27 #include "hugepage_settings.h" 28 29 #define PAGEMAP_BITS_ALL (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ 30 PAGE_IS_FILE | PAGE_IS_PRESENT | \ 31 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ 32 PAGE_IS_HUGE) 33 #define PAGEMAP_NON_WRITTEN_BITS (PAGE_IS_WPALLOWED | PAGE_IS_FILE | \ 34 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | \ 35 PAGE_IS_PFNZERO | PAGE_IS_HUGE) 36 37 #define TEST_ITERATIONS 100 38 #define PAGEMAP "/proc/self/pagemap" 39 int pagemap_fd; 40 int uffd; 41 size_t page_size; 42 size_t hpage_size; 43 const char *progname; 44 45 #define LEN(region) ((region.end - region.start)/page_size) 46 47 static long pagemap_ioctl(void *start, int len, void *vec, int vec_len, int flag, 48 int max_pages, long required_mask, long anyof_mask, long excluded_mask, 49 long return_mask) 50 { 51 struct pm_scan_arg arg; 52 53 arg.start = (uintptr_t)start; 54 arg.end = (uintptr_t)(start + len); 55 arg.vec = (uintptr_t)vec; 56 arg.vec_len = vec_len; 57 arg.flags = flag; 58 arg.size = sizeof(struct pm_scan_arg); 59 arg.max_pages = max_pages; 60 arg.category_mask = required_mask; 61 arg.category_anyof_mask = anyof_mask; 62 arg.category_inverted = excluded_mask; 63 arg.return_mask = return_mask; 64 65 return ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 66 } 67 68 static long pagemap_ioc(void *start, int len, void *vec, int vec_len, int flag, 69 int max_pages, long required_mask, long anyof_mask, long excluded_mask, 70 long return_mask, long *walk_end) 71 { 72 struct pm_scan_arg arg; 73 int ret; 74 75 arg.start = (uintptr_t)start; 76 arg.end = (uintptr_t)(start + len); 77 arg.vec = (uintptr_t)vec; 78 arg.vec_len = vec_len; 79 arg.flags = flag; 80 arg.size = sizeof(struct pm_scan_arg); 81 arg.max_pages = max_pages; 82 arg.category_mask = required_mask; 83 arg.category_anyof_mask = anyof_mask; 84 arg.category_inverted = excluded_mask; 85 arg.return_mask = return_mask; 86 87 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 88 89 if (walk_end) 90 *walk_end = arg.walk_end; 91 92 return ret; 93 } 94 95 96 int init_uffd(void) 97 { 98 struct uffdio_api uffdio_api; 99 100 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); 101 if (uffd == -1) 102 return uffd; 103 104 uffdio_api.api = UFFD_API; 105 uffdio_api.features = UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC | 106 UFFD_FEATURE_WP_HUGETLBFS_SHMEM; 107 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) 108 return -1; 109 110 if (!(uffdio_api.api & UFFDIO_REGISTER_MODE_WP) || 111 !(uffdio_api.features & UFFD_FEATURE_WP_UNPOPULATED) || 112 !(uffdio_api.features & UFFD_FEATURE_WP_ASYNC) || 113 !(uffdio_api.features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM)) 114 return -1; 115 116 return 0; 117 } 118 119 int wp_init(void *addr, long size) 120 { 121 struct uffdio_register uffdio_register; 122 struct uffdio_writeprotect wp; 123 124 uffdio_register.range.start = (unsigned long)addr; 125 uffdio_register.range.len = size; 126 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; 127 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) 128 ksft_exit_fail_msg("ioctl(UFFDIO_REGISTER) %d %s\n", errno, strerror(errno)); 129 130 if (!(uffdio_register.ioctls & UFFDIO_WRITEPROTECT)) 131 ksft_exit_fail_msg("ioctl set is incorrect\n"); 132 133 wp.range.start = (unsigned long)addr; 134 wp.range.len = size; 135 wp.mode = UFFDIO_WRITEPROTECT_MODE_WP; 136 137 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &wp)) 138 ksft_exit_fail_msg("ioctl(UFFDIO_WRITEPROTECT)\n"); 139 140 return 0; 141 } 142 143 int wp_free(void *addr, long size) 144 { 145 struct uffdio_register uffdio_register; 146 147 uffdio_register.range.start = (unsigned long)addr; 148 uffdio_register.range.len = size; 149 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; 150 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range)) 151 ksft_exit_fail_msg("ioctl unregister failure\n"); 152 return 0; 153 } 154 155 int wp_addr_range(void *addr, int size) 156 { 157 if (pagemap_ioctl(addr, size, NULL, 0, 158 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 159 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0) 160 ksft_exit_fail_msg("error %d %d %s\n", 1, errno, strerror(errno)); 161 162 return 0; 163 } 164 165 void *gethugetlb_mem(int size, int *shmid) 166 { 167 char *mem; 168 169 if (shmid) { 170 *shmid = shmget(2, size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); 171 if (*shmid < 0) 172 return NULL; 173 174 mem = shmat(*shmid, 0, 0); 175 if (mem == (char *)-1) { 176 shmctl(*shmid, IPC_RMID, NULL); 177 ksft_exit_fail_msg("Shared memory attach failure\n"); 178 } 179 } else { 180 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, 181 MAP_ANONYMOUS | MAP_HUGETLB | MAP_PRIVATE, -1, 0); 182 if (mem == MAP_FAILED) 183 return NULL; 184 } 185 186 return mem; 187 } 188 189 int userfaultfd_tests(void) 190 { 191 long mem_size, vec_size, written, num_pages = 16; 192 char *mem, *vec; 193 194 mem_size = num_pages * page_size; 195 mem = mmap(NULL, mem_size, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0); 196 if (mem == MAP_FAILED) 197 ksft_exit_fail_msg("error nomem\n"); 198 199 wp_init(mem, mem_size); 200 201 /* Change protection of pages differently */ 202 mprotect(mem, mem_size/8, PROT_READ|PROT_WRITE); 203 mprotect(mem + 1 * mem_size/8, mem_size/8, PROT_READ); 204 mprotect(mem + 2 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 205 mprotect(mem + 3 * mem_size/8, mem_size/8, PROT_READ); 206 mprotect(mem + 4 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 207 mprotect(mem + 5 * mem_size/8, mem_size/8, PROT_NONE); 208 mprotect(mem + 6 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 209 mprotect(mem + 7 * mem_size/8, mem_size/8, PROT_READ); 210 211 wp_addr_range(mem + (mem_size/16), mem_size - 2 * (mem_size/8)); 212 wp_addr_range(mem, mem_size); 213 214 vec_size = mem_size/page_size; 215 vec = calloc(vec_size, sizeof(struct page_region)); 216 if (!vec) 217 ksft_exit_fail_msg("error nomem\n"); 218 219 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 220 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 221 if (written < 0) 222 ksft_exit_fail_msg("error %ld %d %s\n", written, errno, strerror(errno)); 223 224 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", __func__); 225 226 wp_free(mem, mem_size); 227 munmap(mem, mem_size); 228 free(vec); 229 return 0; 230 } 231 232 int get_reads(struct page_region *vec, int vec_size) 233 { 234 int i, sum = 0; 235 236 for (i = 0; i < vec_size; i++) 237 sum += LEN(vec[i]); 238 239 return sum; 240 } 241 242 int sanity_tests_sd(void) 243 { 244 unsigned long long mem_size, vec_size, i, total_pages = 0; 245 long ret, ret2, ret3; 246 int num_pages = 1000; 247 int total_writes, total_reads, reads, count; 248 struct page_region *vec, *vec2; 249 char *mem, *m[2]; 250 long walk_end; 251 252 vec_size = num_pages/2; 253 mem_size = num_pages * page_size; 254 255 vec = calloc(vec_size, sizeof(struct page_region)); 256 if (!vec) 257 ksft_exit_fail_msg("error nomem\n"); 258 259 vec2 = calloc(vec_size, sizeof(struct page_region)); 260 if (!vec2) 261 ksft_exit_fail_msg("error nomem\n"); 262 263 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 264 if (mem == MAP_FAILED) 265 ksft_exit_fail_msg("error nomem\n"); 266 267 wp_init(mem, mem_size); 268 wp_addr_range(mem, mem_size); 269 270 /* 1. wrong operation */ 271 ksft_test_result(pagemap_ioctl(mem, 0, vec, vec_size, 0, 272 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 273 "%s Zero range size is valid\n", __func__); 274 275 ksft_test_result(pagemap_ioctl(mem, mem_size, NULL, vec_size, 0, 276 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) < 0, 277 "%s output buffer must be specified with size\n", __func__); 278 279 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, 0, 0, 280 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 281 "%s output buffer can be 0\n", __func__); 282 283 ksft_test_result(pagemap_ioctl(mem, mem_size, 0, 0, 0, 284 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 285 "%s output buffer can be 0\n", __func__); 286 287 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, -1, 288 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0, 289 "%s wrong flag specified\n", __func__); 290 291 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 292 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC | 0xFF, 293 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0, 294 "%s flag has extra bits specified\n", __func__); 295 296 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 297 0, 0, 0, 0, PAGE_IS_WRITTEN) >= 0, 298 "%s no selection mask is specified\n", __func__); 299 300 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 301 0, PAGE_IS_WRITTEN, PAGE_IS_WRITTEN, 0, 0) == 0, 302 "%s no return mask is specified\n", __func__); 303 304 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 305 0, PAGE_IS_WRITTEN, 0, 0, 0x1000) < 0, 306 "%s wrong return mask specified\n", __func__); 307 308 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 309 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 310 0, 0xFFF, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN) < 0, 311 "%s mixture of correct and wrong flag\n", __func__); 312 313 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 314 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 315 0, 0, 0, PAGEMAP_BITS_ALL, PAGE_IS_WRITTEN) >= 0, 316 "%s PAGEMAP_BITS_ALL can be specified with PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", 317 __func__); 318 319 /* 2. Clear area with larger vec size */ 320 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 321 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 322 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 323 ksft_test_result(ret >= 0, "%s Clear area with larger vec size\n", __func__); 324 325 /* 3. Repeated pattern of written and non-written pages */ 326 for (i = 0; i < mem_size; i += 2 * page_size) 327 mem[i]++; 328 329 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 0, 330 0, PAGE_IS_WRITTEN); 331 if (ret < 0) 332 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 333 334 ksft_test_result((unsigned long long)ret == mem_size/(page_size * 2), 335 "%s Repeated pattern of written and non-written pages\n", __func__); 336 337 /* 4. Repeated pattern of written and non-written pages in parts */ 338 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 339 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 340 num_pages/2 - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 341 if (ret < 0) 342 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 343 344 ret2 = pagemap_ioctl(mem, mem_size, vec, 2, 0, 0, PAGE_IS_WRITTEN, 0, 0, 345 PAGE_IS_WRITTEN); 346 if (ret2 < 0) 347 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno)); 348 349 ret3 = pagemap_ioctl(mem, mem_size, vec, vec_size, 350 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 351 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 352 if (ret3 < 0) 353 ksft_exit_fail_msg("error %ld %d %s\n", ret3, errno, strerror(errno)); 354 355 ksft_test_result((ret + ret3) == num_pages/2 && ret2 == 2, 356 "%s Repeated pattern of written and non-written pages in parts %ld %ld %ld\n", 357 __func__, ret, ret3, ret2); 358 359 /* 5. Repeated pattern of written and non-written pages max_pages */ 360 for (i = 0; i < mem_size; i += 2 * page_size) 361 mem[i]++; 362 mem[(mem_size/page_size - 1) * page_size]++; 363 364 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 365 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 366 num_pages/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 367 if (ret < 0) 368 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 369 370 ret2 = pagemap_ioctl(mem, mem_size, vec, vec_size, 371 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 372 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 373 if (ret2 < 0) 374 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno)); 375 376 ksft_test_result(ret == num_pages/2 && ret2 == 1, 377 "%s Repeated pattern of written and non-written pages max_pages\n", 378 __func__); 379 380 /* 6. only get 2 dirty pages and clear them as well */ 381 vec_size = mem_size/page_size; 382 memset(mem, -1, mem_size); 383 384 /* get and clear second and third pages */ 385 ret = pagemap_ioctl(mem + page_size, 2 * page_size, vec, 1, 386 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 387 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 388 if (ret < 0) 389 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 390 391 ret2 = pagemap_ioctl(mem, mem_size, vec2, vec_size, 0, 0, 392 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 393 if (ret2 < 0) 394 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno)); 395 396 ksft_test_result(ret == 1 && LEN(vec[0]) == 2 && 397 vec[0].start == (uintptr_t)(mem + page_size) && 398 ret2 == 2 && LEN(vec2[0]) == 1 && vec2[0].start == (uintptr_t)mem && 399 LEN(vec2[1]) == vec_size - 3 && 400 vec2[1].start == (uintptr_t)(mem + 3 * page_size), 401 "%s only get 2 written pages and clear them as well\n", __func__); 402 403 wp_free(mem, mem_size); 404 munmap(mem, mem_size); 405 406 /* 7. Two regions */ 407 m[0] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 408 if (m[0] == MAP_FAILED) 409 ksft_exit_fail_msg("error nomem\n"); 410 m[1] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 411 if (m[1] == MAP_FAILED) 412 ksft_exit_fail_msg("error nomem\n"); 413 414 wp_init(m[0], mem_size); 415 wp_init(m[1], mem_size); 416 wp_addr_range(m[0], mem_size); 417 wp_addr_range(m[1], mem_size); 418 419 memset(m[0], 'a', mem_size); 420 memset(m[1], 'b', mem_size); 421 422 wp_addr_range(m[0], mem_size); 423 424 ret = pagemap_ioctl(m[1], mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0, 425 PAGE_IS_WRITTEN); 426 if (ret < 0) 427 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 428 429 ksft_test_result(ret == 1 && LEN(vec[0]) == mem_size/page_size, 430 "%s Two regions\n", __func__); 431 432 wp_free(m[0], mem_size); 433 wp_free(m[1], mem_size); 434 munmap(m[0], mem_size); 435 munmap(m[1], mem_size); 436 437 free(vec); 438 free(vec2); 439 440 /* 8. Smaller vec */ 441 mem_size = 1050 * page_size; 442 vec_size = mem_size/(page_size*2); 443 444 vec = calloc(vec_size, sizeof(struct page_region)); 445 if (!vec) 446 ksft_exit_fail_msg("error nomem\n"); 447 448 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 449 if (mem == MAP_FAILED) 450 ksft_exit_fail_msg("error nomem\n"); 451 452 wp_init(mem, mem_size); 453 wp_addr_range(mem, mem_size); 454 455 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 456 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 457 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 458 if (ret < 0) 459 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 460 461 for (i = 0; i < mem_size/page_size; i += 2) 462 mem[i * page_size]++; 463 464 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 465 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 466 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 467 if (ret < 0) 468 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 469 470 total_pages += ret; 471 472 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 473 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 474 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 475 if (ret < 0) 476 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 477 478 total_pages += ret; 479 480 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 481 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 482 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 483 if (ret < 0) 484 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 485 486 total_pages += ret; 487 488 ksft_test_result(total_pages == mem_size/(page_size*2), "%s Smaller max_pages\n", __func__); 489 490 free(vec); 491 wp_free(mem, mem_size); 492 munmap(mem, mem_size); 493 total_pages = 0; 494 495 /* 9. Smaller vec */ 496 mem_size = 10000 * page_size; 497 vec_size = 50; 498 499 vec = calloc(vec_size, sizeof(struct page_region)); 500 if (!vec) 501 ksft_exit_fail_msg("error nomem\n"); 502 503 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 504 if (mem == MAP_FAILED) 505 ksft_exit_fail_msg("error nomem\n"); 506 507 wp_init(mem, mem_size); 508 wp_addr_range(mem, mem_size); 509 510 for (count = 0; count < TEST_ITERATIONS; count++) { 511 total_writes = total_reads = 0; 512 walk_end = (long)mem; 513 514 for (i = 0; i < mem_size; i += page_size) { 515 if (rand() % 2) { 516 mem[i]++; 517 total_writes++; 518 } 519 } 520 521 while (total_reads < total_writes) { 522 ret = pagemap_ioc((void *)walk_end, mem_size-(walk_end - (long)mem), vec, 523 vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 524 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 525 if (ret < 0) 526 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 527 528 if ((unsigned long)ret > vec_size) 529 break; 530 531 reads = get_reads(vec, ret); 532 total_reads += reads; 533 } 534 535 if (total_reads != total_writes) 536 break; 537 } 538 539 ksft_test_result(count == TEST_ITERATIONS, "Smaller vec\n"); 540 541 free(vec); 542 wp_free(mem, mem_size); 543 munmap(mem, mem_size); 544 545 /* 10. Walk_end tester */ 546 vec_size = 1000; 547 mem_size = vec_size * page_size; 548 549 vec = calloc(vec_size, sizeof(struct page_region)); 550 if (!vec) 551 ksft_exit_fail_msg("error nomem\n"); 552 553 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 554 if (mem == MAP_FAILED) 555 ksft_exit_fail_msg("error nomem\n"); 556 557 wp_init(mem, mem_size); 558 wp_addr_range(mem, mem_size); 559 560 memset(mem, 0, mem_size); 561 562 ret = pagemap_ioc(mem, 0, vec, vec_size, 0, 563 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 564 if (ret < 0) 565 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 566 ksft_test_result(ret == 0 && walk_end == (long)mem, 567 "Walk_end: Same start and end address\n"); 568 569 ret = pagemap_ioc(mem, 0, vec, vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 570 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 571 if (ret < 0) 572 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 573 ksft_test_result(ret == 0 && walk_end == (long)mem, 574 "Walk_end: Same start and end with WP\n"); 575 576 ret = pagemap_ioc(mem, 0, vec, 0, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 577 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 578 if (ret < 0) 579 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 580 ksft_test_result(ret == 0 && walk_end == (long)mem, 581 "Walk_end: Same start and end with 0 output buffer\n"); 582 583 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 584 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 585 if (ret < 0) 586 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 587 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 588 "Walk_end: Big vec\n"); 589 590 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 591 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 592 if (ret < 0) 593 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 594 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 595 "Walk_end: vec of minimum length\n"); 596 597 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 598 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 599 if (ret < 0) 600 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 601 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 602 "Walk_end: Max pages specified\n"); 603 604 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 605 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 606 if (ret < 0) 607 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 608 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size/2), 609 "Walk_end: Half max pages\n"); 610 611 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 612 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 613 if (ret < 0) 614 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 615 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size), 616 "Walk_end: 1 max page\n"); 617 618 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 619 -1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 620 if (ret < 0) 621 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 622 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 623 "Walk_end: max pages\n"); 624 625 wp_addr_range(mem, mem_size); 626 for (i = 0; i < mem_size; i += 2 * page_size) 627 mem[i]++; 628 629 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 630 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 631 if (ret < 0) 632 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 633 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 634 "Walk_end sparse: Big vec\n"); 635 636 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 637 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 638 if (ret < 0) 639 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 640 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 641 "Walk_end sparse: vec of minimum length\n"); 642 643 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 644 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 645 if (ret < 0) 646 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 647 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 648 "Walk_end sparse: Max pages specified\n"); 649 650 ret = pagemap_ioc(mem, mem_size, vec, vec_size/2, 0, 651 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 652 if (ret < 0) 653 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 654 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 655 "Walk_end sparse: Max pages specified\n"); 656 657 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 658 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 659 if (ret < 0) 660 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 661 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 662 "Walk_end sparse: Max pages specified\n"); 663 664 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 665 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 666 if (ret < 0) 667 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 668 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 669 "Walk_endsparse : Half max pages\n"); 670 671 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 672 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 673 if (ret < 0) 674 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 675 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 676 "Walk_end: 1 max page\n"); 677 678 free(vec); 679 wp_free(mem, mem_size); 680 munmap(mem, mem_size); 681 682 return 0; 683 } 684 685 int base_tests(char *prefix, char *mem, unsigned long long mem_size, int skip) 686 { 687 unsigned long long vec_size; 688 int written; 689 struct page_region *vec, *vec2; 690 691 if (skip) { 692 ksft_test_result_skip("%s all new pages must not be written (dirty)\n", prefix); 693 ksft_test_result_skip("%s all pages must be written (dirty)\n", prefix); 694 ksft_test_result_skip("%s all pages dirty other than first and the last one\n", 695 prefix); 696 ksft_test_result_skip("%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix); 697 ksft_test_result_skip("%s only middle page dirty\n", prefix); 698 ksft_test_result_skip("%s only two middle pages dirty\n", prefix); 699 return 0; 700 } 701 702 vec_size = mem_size/page_size; 703 vec = calloc(vec_size, sizeof(struct page_region)); 704 vec2 = calloc(vec_size, sizeof(struct page_region)); 705 if (!vec || !vec2) 706 ksft_exit_fail_msg("error nomem\n"); 707 708 /* 1. all new pages must be not be written (dirty) */ 709 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 710 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 711 if (written < 0) 712 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 713 714 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", prefix); 715 716 /* 2. all pages must be written */ 717 memset(mem, -1, mem_size); 718 719 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0, 720 PAGE_IS_WRITTEN); 721 if (written < 0) 722 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 723 724 ksft_test_result(written == 1 && LEN(vec[0]) == mem_size/page_size, 725 "%s all pages must be written (dirty)\n", prefix); 726 727 /* 3. all pages dirty other than first and the last one */ 728 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 729 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 730 if (written < 0) 731 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 732 733 memset(mem + page_size, 0, mem_size - (2 * page_size)); 734 735 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 736 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 737 if (written < 0) 738 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 739 740 ksft_test_result(written == 1 && LEN(vec[0]) >= vec_size - 2 && LEN(vec[0]) <= vec_size, 741 "%s all pages dirty other than first and the last one\n", prefix); 742 743 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, 744 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 745 if (written < 0) 746 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 747 748 ksft_test_result(written == 0, 749 "%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix); 750 751 /* 4. only middle page dirty */ 752 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 753 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 754 if (written < 0) 755 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 756 757 mem[vec_size/2 * page_size]++; 758 759 written = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 760 0, 0, PAGE_IS_WRITTEN); 761 if (written < 0) 762 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 763 764 ksft_test_result(written == 1 && LEN(vec[0]) >= 1, 765 "%s only middle page dirty\n", prefix); 766 767 /* 5. only two middle pages dirty and walk over only middle pages */ 768 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 769 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE); 770 if (written < 0) 771 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 772 773 mem[vec_size/2 * page_size]++; 774 mem[(vec_size/2 + 1) * page_size]++; 775 776 written = pagemap_ioctl(&mem[vec_size/2 * page_size], 2 * page_size, vec, 1, 0, 777 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE); 778 if (written < 0) 779 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 780 781 ksft_test_result(written == 1 && vec[0].start == (uintptr_t)(&mem[vec_size/2 * page_size]) 782 && LEN(vec[0]) == 2, 783 "%s only two middle pages dirty\n", prefix); 784 785 free(vec); 786 free(vec2); 787 return 0; 788 } 789 790 void *gethugepage(int map_size) 791 { 792 int ret; 793 char *map; 794 795 map = memalign(hpage_size, map_size); 796 if (!map) 797 ksft_exit_fail_msg("memalign failed %d %s\n", errno, strerror(errno)); 798 799 ret = madvise(map, map_size, MADV_HUGEPAGE); 800 if (ret) 801 return NULL; 802 803 memset(map, 0, map_size); 804 805 return map; 806 } 807 808 int hpage_unit_tests(void) 809 { 810 char *map; 811 int ret, ret2; 812 size_t num_pages = 10; 813 unsigned long long map_size = hpage_size * num_pages; 814 unsigned long long vec_size = map_size/page_size; 815 struct page_region *vec, *vec2; 816 817 vec = calloc(vec_size, sizeof(struct page_region)); 818 vec2 = calloc(vec_size, sizeof(struct page_region)); 819 if (!vec || !vec2) 820 ksft_exit_fail_msg("malloc failed\n"); 821 822 map = gethugepage(map_size); 823 if (map) { 824 wp_init(map, map_size); 825 wp_addr_range(map, map_size); 826 827 /* 1. all new huge page must not be written (dirty) */ 828 ret = pagemap_ioctl(map, map_size, vec, vec_size, 829 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 830 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 831 if (ret < 0) 832 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 833 834 ksft_test_result(ret == 0, "%s all new huge page must not be written (dirty)\n", 835 __func__); 836 837 /* 2. all the huge page must not be written */ 838 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 839 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 840 if (ret < 0) 841 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 842 843 ksft_test_result(ret == 0, "%s all the huge page must not be written\n", __func__); 844 845 /* 3. all the huge page must be written and clear dirty as well */ 846 memset(map, -1, map_size); 847 ret = pagemap_ioctl(map, map_size, vec, vec_size, 848 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 849 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 850 if (ret < 0) 851 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 852 853 ksft_test_result(ret == 1 && vec[0].start == (uintptr_t)map && 854 LEN(vec[0]) == vec_size && vec[0].categories == PAGE_IS_WRITTEN, 855 "%s all the huge page must be written and clear\n", __func__); 856 857 /* 4. only middle page written */ 858 wp_free(map, map_size); 859 free(map); 860 map = gethugepage(map_size); 861 wp_init(map, map_size); 862 wp_addr_range(map, map_size); 863 map[vec_size/2 * page_size]++; 864 865 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 866 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 867 if (ret < 0) 868 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 869 870 ksft_test_result(ret == 1 && LEN(vec[0]) > 0, 871 "%s only middle page written\n", __func__); 872 873 wp_free(map, map_size); 874 free(map); 875 } else { 876 ksft_test_result_skip("%s all new huge page must be written\n", __func__); 877 ksft_test_result_skip("%s all the huge page must not be written\n", __func__); 878 ksft_test_result_skip("%s all the huge page must be written and clear\n", __func__); 879 ksft_test_result_skip("%s only middle page written\n", __func__); 880 } 881 882 /* 5. clear first half of huge page */ 883 map = gethugepage(map_size); 884 if (map) { 885 wp_init(map, map_size); 886 wp_addr_range(map, map_size); 887 888 memset(map, 0, map_size); 889 890 wp_addr_range(map, map_size/2); 891 892 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 893 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 894 if (ret < 0) 895 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 896 897 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 && 898 vec[0].start == (uintptr_t)(map + map_size/2), 899 "%s clear first half of huge page\n", __func__); 900 wp_free(map, map_size); 901 free(map); 902 } else { 903 ksft_test_result_skip("%s clear first half of huge page\n", __func__); 904 } 905 906 /* 6. clear first half of huge page with limited buffer */ 907 map = gethugepage(map_size); 908 if (map) { 909 wp_init(map, map_size); 910 wp_addr_range(map, map_size); 911 912 memset(map, 0, map_size); 913 914 ret = pagemap_ioctl(map, map_size, vec, vec_size, 915 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 916 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 917 if (ret < 0) 918 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 919 920 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 921 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 922 if (ret < 0) 923 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 924 925 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 && 926 vec[0].start == (uintptr_t)(map + map_size/2), 927 "%s clear first half of huge page with limited buffer\n", 928 __func__); 929 wp_free(map, map_size); 930 free(map); 931 } else { 932 ksft_test_result_skip("%s clear first half of huge page with limited buffer\n", 933 __func__); 934 } 935 936 /* 7. clear second half of huge page */ 937 map = gethugepage(map_size); 938 if (map) { 939 wp_init(map, map_size); 940 wp_addr_range(map, map_size); 941 942 memset(map, -1, map_size); 943 944 ret = pagemap_ioctl(map + map_size/2, map_size/2, vec, vec_size, 945 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, vec_size/2, 946 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 947 if (ret < 0) 948 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 949 950 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 951 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 952 if (ret < 0) 953 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 954 955 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2, 956 "%s clear second half huge page\n", __func__); 957 wp_free(map, map_size); 958 free(map); 959 } else { 960 ksft_test_result_skip("%s clear second half huge page\n", __func__); 961 } 962 963 /* 8. get half huge page */ 964 map = gethugepage(map_size); 965 if (map) { 966 wp_init(map, map_size); 967 wp_addr_range(map, map_size); 968 969 memset(map, -1, map_size); 970 usleep(100); 971 972 ret = pagemap_ioctl(map, map_size, vec, 1, 973 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 974 hpage_size/(2*page_size), PAGE_IS_WRITTEN, 0, 0, 975 PAGE_IS_WRITTEN); 976 if (ret < 0) 977 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 978 979 ksft_test_result(ret == 1 && LEN(vec[0]) == hpage_size/(2*page_size), 980 "%s get half huge page\n", __func__); 981 982 ret2 = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 983 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 984 if (ret2 < 0) 985 ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno)); 986 987 ksft_test_result(ret2 == 1 && LEN(vec[0]) == (map_size - hpage_size/2)/page_size, 988 "%s get half huge page\n", __func__); 989 990 wp_free(map, map_size); 991 free(map); 992 } else { 993 ksft_test_result_skip("%s get half huge page\n", __func__); 994 ksft_test_result_skip("%s get half huge page\n", __func__); 995 } 996 997 free(vec); 998 free(vec2); 999 return 0; 1000 } 1001 1002 int unmapped_region_tests(void) 1003 { 1004 void *start = (void *)0x10000000; 1005 int written, len = 0x00040000; 1006 long vec_size = len / page_size; 1007 struct page_region *vec = calloc(vec_size, sizeof(struct page_region)); 1008 if (!vec) 1009 ksft_exit_fail_msg("error nomem\n"); 1010 1011 /* 1. Get written pages */ 1012 written = pagemap_ioctl(start, len, vec, vec_size, 0, 0, 1013 PAGEMAP_NON_WRITTEN_BITS, 0, 0, PAGEMAP_NON_WRITTEN_BITS); 1014 if (written < 0) 1015 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 1016 1017 ksft_test_result(written >= 0, "%s Get status of pages\n", __func__); 1018 1019 free(vec); 1020 return 0; 1021 } 1022 1023 static void test_simple(void) 1024 { 1025 int i; 1026 char *map; 1027 struct page_region vec; 1028 1029 map = aligned_alloc(page_size, page_size); 1030 if (!map) 1031 ksft_exit_fail_msg("aligned_alloc failed\n"); 1032 1033 wp_init(map, page_size); 1034 wp_addr_range(map, page_size); 1035 1036 for (i = 0 ; i < TEST_ITERATIONS; i++) { 1037 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0, 1038 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 1) { 1039 ksft_print_msg("written bit was 1, but should be 0 (i=%d)\n", i); 1040 break; 1041 } 1042 1043 wp_addr_range(map, page_size); 1044 /* Write something to the page to get the written bit enabled on the page */ 1045 map[0]++; 1046 1047 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0, 1048 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0) { 1049 ksft_print_msg("written bit was 0, but should be 1 (i=%d)\n", i); 1050 break; 1051 } 1052 1053 wp_addr_range(map, page_size); 1054 } 1055 wp_free(map, page_size); 1056 free(map); 1057 1058 ksft_test_result(i == TEST_ITERATIONS, "Test %s\n", __func__); 1059 } 1060 1061 /* 1062 * A range that was populated and then MADV_DONTNEED'd is genuine pte_none 1063 * with no uffd-wp marker. Such a pte must read the same regardless of which 1064 * PAGEMAP_SCAN path serves the request: both the PAGE_IS_WRITTEN fast path and 1065 * the generic path (reached e.g. via category_anyof_mask) must report every 1066 * page written. 1067 */ 1068 /* 1069 * Populate @mem (optionally collapsing it into a THP first), drop it with 1070 * MADV_DONTNEED, then check PAGEMAP_SCAN reports the whole range written via 1071 * both the fast and generic query paths. A dropped THP leaves a pmd_none hole 1072 * with no page table, exercising pagemap_scan_pte_hole(); a base-page range 1073 * leaves pte_none entries. 1074 */ 1075 static void unpopulated_written_test(const char *name, char *mem, long size, 1076 bool use_thp) 1077 { 1078 long npages = size / page_size, fast = 0, slow = 0, ret; 1079 struct page_region regions[16]; 1080 int i; 1081 1082 wp_init(mem, size); 1083 1084 /* Populate, optionally collapse to a THP, then drop it. */ 1085 memset(mem, 1, size); 1086 if (use_thp && 1087 (madvise(mem, size, MADV_COLLAPSE) || 1088 !check_huge_anon(mem, size, size / hpage_size, hpage_size))) { 1089 ksft_test_result_skip("%s could not form a THP\n", name); 1090 goto out; 1091 } 1092 if (madvise(mem, size, MADV_DONTNEED)) { 1093 ksft_test_result_fail("%s MADV_DONTNEED failed\n", name); 1094 goto out; 1095 } 1096 1097 /* Fast path: category_mask == return_mask == PAGE_IS_WRITTEN. */ 1098 ret = pagemap_ioctl(mem, size, regions, ARRAY_SIZE(regions), 0, 0, 1099 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 1100 for (i = 0; ret > 0 && i < ret; i++) 1101 fast += LEN(regions[i]); 1102 1103 /* Generic path: same query expressed via category_anyof_mask. */ 1104 ret = pagemap_ioctl(mem, size, regions, ARRAY_SIZE(regions), 0, 0, 1105 0, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN); 1106 for (i = 0; ret > 0 && i < ret; i++) 1107 slow += LEN(regions[i]); 1108 1109 ksft_test_result(fast == npages && slow == npages, 1110 "%s unpopulated range reported written by both paths (%ld, %ld of %ld)\n", 1111 name, fast, slow, npages); 1112 out: 1113 wp_free(mem, size); 1114 } 1115 1116 static void unpopulated_scan_test(void) 1117 { 1118 long mem_size = 16 * page_size; 1119 char *mem; 1120 1121 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, 1122 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1123 if (mem == MAP_FAILED) { 1124 ksft_test_result_skip("%s mmap failed\n", __func__); 1125 return; 1126 } 1127 1128 unpopulated_written_test(__func__, mem, mem_size, false); 1129 munmap(mem, mem_size); 1130 } 1131 1132 /* 1133 * Same as unpopulated_scan_test(), but the range is a THP: a full-PMD 1134 * MADV_DONTNEED leaves a pmd_none hole with no page table. 1135 */ 1136 static void unpopulated_thp_scan_test(void) 1137 { 1138 char *area, *mem; 1139 1140 if (!hpage_size) { 1141 ksft_test_result_skip("%s THP not supported\n", __func__); 1142 return; 1143 } 1144 1145 /* Over-allocate so a PMD-aligned, THP-sized range fits inside. */ 1146 area = mmap(NULL, 2 * hpage_size, PROT_READ | PROT_WRITE, 1147 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1148 if (area == MAP_FAILED) { 1149 ksft_test_result_skip("%s mmap failed\n", __func__); 1150 return; 1151 } 1152 mem = (char *)(((unsigned long)area + hpage_size - 1) & ~(hpage_size - 1)); 1153 1154 unpopulated_written_test(__func__, mem, hpage_size, true); 1155 munmap(area, 2 * hpage_size); 1156 } 1157 1158 int sanity_tests(void) 1159 { 1160 unsigned long long mem_size, vec_size; 1161 long ret, fd, i, buf_size, nr_pages; 1162 struct page_region *vec; 1163 char *mem, *fmem; 1164 struct stat sbuf; 1165 1166 /* 1. wrong operation */ 1167 mem_size = 10 * page_size; 1168 vec_size = mem_size / page_size; 1169 1170 vec = calloc(vec_size, sizeof(struct page_region)); 1171 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1172 if (mem == MAP_FAILED || !vec) 1173 ksft_exit_fail_msg("error nomem\n"); 1174 1175 wp_init(mem, mem_size); 1176 wp_addr_range(mem, mem_size); 1177 1178 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 1179 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 1180 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0, 1181 "%s WP op can be specified with !PAGE_IS_WRITTEN\n", __func__); 1182 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1183 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0, 1184 "%s required_mask specified\n", __func__); 1185 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1186 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL) >= 0, 1187 "%s anyof_mask specified\n", __func__); 1188 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1189 0, 0, PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL) >= 0, 1190 "%s excluded_mask specified\n", __func__); 1191 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1192 PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL, 0, 1193 PAGEMAP_BITS_ALL) >= 0, 1194 "%s required_mask and anyof_mask specified\n", __func__); 1195 wp_free(mem, mem_size); 1196 munmap(mem, mem_size); 1197 1198 /* 2. Get sd and present pages with anyof_mask */ 1199 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1200 if (mem == MAP_FAILED) 1201 ksft_exit_fail_msg("error nomem\n"); 1202 wp_init(mem, mem_size); 1203 wp_addr_range(mem, mem_size); 1204 1205 memset(mem, 0, mem_size); 1206 1207 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1208 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL); 1209 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1210 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1211 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1212 "%s Get sd and present pages with anyof_mask\n", __func__); 1213 1214 /* 3. Get sd and present pages with required_mask */ 1215 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1216 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL); 1217 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1218 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1219 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1220 "%s Get all the pages with required_mask\n", __func__); 1221 1222 /* 4. Get sd and present pages with required_mask and anyof_mask */ 1223 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1224 PAGE_IS_WRITTEN, PAGE_IS_PRESENT, 0, PAGEMAP_BITS_ALL); 1225 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1226 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1227 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1228 "%s Get sd and present pages with required_mask and anyof_mask\n", 1229 __func__); 1230 1231 /* 5. Don't get sd pages */ 1232 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1233 PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN, PAGEMAP_BITS_ALL); 1234 ksft_test_result(ret == 0, "%s Don't get sd pages\n", __func__); 1235 1236 /* 6. Don't get present pages */ 1237 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1238 PAGE_IS_PRESENT, 0, PAGE_IS_PRESENT, PAGEMAP_BITS_ALL); 1239 ksft_test_result(ret == 0, "%s Don't get present pages\n", __func__); 1240 1241 wp_free(mem, mem_size); 1242 munmap(mem, mem_size); 1243 1244 /* 8. Find written present pages with return mask */ 1245 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1246 if (mem == MAP_FAILED) 1247 ksft_exit_fail_msg("error nomem\n"); 1248 wp_init(mem, mem_size); 1249 wp_addr_range(mem, mem_size); 1250 1251 memset(mem, 0, mem_size); 1252 1253 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 1254 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 1255 0, PAGEMAP_BITS_ALL, 0, PAGE_IS_WRITTEN); 1256 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1257 vec[0].categories == PAGE_IS_WRITTEN, 1258 "%s Find written present pages with return mask\n", __func__); 1259 wp_free(mem, mem_size); 1260 munmap(mem, mem_size); 1261 1262 /* 9. Memory mapped file */ 1263 fd = open(progname, O_RDONLY); 1264 if (fd < 0) 1265 ksft_exit_fail_msg("%s Memory mapped file\n", __func__); 1266 1267 ret = stat(progname, &sbuf); 1268 if (ret < 0) 1269 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 1270 1271 fmem = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0); 1272 if (fmem == MAP_FAILED) 1273 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1274 1275 nr_pages = (sbuf.st_size + page_size - 1) / page_size; 1276 force_read_pages(fmem, nr_pages, page_size); 1277 1278 ret = pagemap_ioctl(fmem, sbuf.st_size, vec, vec_size, 0, 0, 1279 0, PAGEMAP_NON_WRITTEN_BITS, 0, PAGEMAP_NON_WRITTEN_BITS); 1280 1281 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem && 1282 LEN(vec[0]) == nr_pages && 1283 (vec[0].categories & PAGE_IS_FILE), 1284 "%s Memory mapped file\n", __func__); 1285 1286 munmap(fmem, sbuf.st_size); 1287 close(fd); 1288 1289 /* 10. Create and read/write to a memory mapped file */ 1290 buf_size = page_size * 10; 1291 1292 fd = open(__FILE__".tmp2", O_RDWR | O_CREAT, 0666); 1293 if (fd < 0) 1294 ksft_exit_fail_msg("Read/write to memory: %s\n", 1295 strerror(errno)); 1296 1297 for (i = 0; i < buf_size; i++) 1298 if (write(fd, "c", 1) < 0) 1299 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1300 1301 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1302 if (fmem == MAP_FAILED) 1303 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1304 1305 wp_init(fmem, buf_size); 1306 wp_addr_range(fmem, buf_size); 1307 1308 for (i = 0; i < buf_size; i++) 1309 fmem[i] = 'z'; 1310 1311 msync(fmem, buf_size, MS_SYNC); 1312 1313 ret = pagemap_ioctl(fmem, buf_size, vec, vec_size, 0, 0, 1314 PAGE_IS_WRITTEN, PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_FILE, 0, 1315 PAGEMAP_BITS_ALL); 1316 1317 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem && 1318 LEN(vec[0]) == (buf_size/page_size) && 1319 (vec[0].categories & PAGE_IS_WRITTEN), 1320 "%s Read/write to memory\n", __func__); 1321 1322 wp_free(fmem, buf_size); 1323 munmap(fmem, buf_size); 1324 close(fd); 1325 1326 free(vec); 1327 return 0; 1328 } 1329 1330 int mprotect_tests(void) 1331 { 1332 int ret; 1333 char *mem, *mem2; 1334 struct page_region vec; 1335 1336 /* 1. Map two pages */ 1337 mem = mmap(0, 2 * page_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1338 if (mem == MAP_FAILED) 1339 ksft_exit_fail_msg("error nomem\n"); 1340 wp_init(mem, 2 * page_size); 1341 wp_addr_range(mem, 2 * page_size); 1342 1343 /* Populate both pages. */ 1344 memset(mem, 1, 2 * page_size); 1345 1346 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1347 0, 0, PAGE_IS_WRITTEN); 1348 if (ret < 0) 1349 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1350 1351 ksft_test_result(ret == 1 && LEN(vec) == 2, "%s Both pages written\n", __func__); 1352 1353 /* 2. Start tracking */ 1354 wp_addr_range(mem, 2 * page_size); 1355 1356 ksft_test_result(pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, 1357 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0, 1358 "%s Both pages are not written (dirty)\n", __func__); 1359 1360 /* 3. Remap the second page */ 1361 mem2 = mmap(mem + page_size, page_size, PROT_READ|PROT_WRITE, 1362 MAP_PRIVATE|MAP_ANON|MAP_FIXED, -1, 0); 1363 if (mem2 == MAP_FAILED) 1364 ksft_exit_fail_msg("error nomem\n"); 1365 wp_init(mem2, page_size); 1366 wp_addr_range(mem2, page_size); 1367 1368 /* Protect + unprotect. */ 1369 mprotect(mem, page_size, PROT_NONE); 1370 mprotect(mem, 2 * page_size, PROT_READ); 1371 mprotect(mem, 2 * page_size, PROT_READ|PROT_WRITE); 1372 1373 /* Modify both pages. */ 1374 memset(mem, 2, 2 * page_size); 1375 1376 /* Protect + unprotect. */ 1377 mprotect(mem, page_size, PROT_NONE); 1378 mprotect(mem, page_size, PROT_READ); 1379 mprotect(mem, page_size, PROT_READ|PROT_WRITE); 1380 1381 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1382 0, 0, PAGE_IS_WRITTEN); 1383 if (ret < 0) 1384 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1385 1386 ksft_test_result(ret == 1 && LEN(vec) == 2, 1387 "%s Both pages written after remap and mprotect\n", __func__); 1388 1389 /* 4. Clear and make the pages written */ 1390 wp_addr_range(mem, 2 * page_size); 1391 1392 memset(mem, 'A', 2 * page_size); 1393 1394 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1395 0, 0, PAGE_IS_WRITTEN); 1396 if (ret < 0) 1397 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1398 1399 ksft_test_result(ret == 1 && LEN(vec) == 2, 1400 "%s Clear and make the pages written\n", __func__); 1401 1402 wp_free(mem, 2 * page_size); 1403 munmap(mem, 2 * page_size); 1404 return 0; 1405 } 1406 1407 /* transact test */ 1408 static const unsigned int nthreads = 6, pages_per_thread = 32, access_per_thread = 8; 1409 static pthread_barrier_t start_barrier, end_barrier; 1410 static unsigned int extra_thread_faults; 1411 static unsigned int iter_count = 1000; 1412 static volatile int finish; 1413 1414 static ssize_t get_dirty_pages_reset(char *mem, unsigned int count, 1415 int reset, int page_size) 1416 { 1417 struct pm_scan_arg arg = {0}; 1418 struct page_region rgns[256]; 1419 unsigned long long i, j; 1420 long ret; 1421 int cnt; 1422 1423 arg.size = sizeof(struct pm_scan_arg); 1424 arg.start = (uintptr_t)mem; 1425 arg.max_pages = count; 1426 arg.end = (uintptr_t)(mem + count * page_size); 1427 arg.vec = (uintptr_t)rgns; 1428 arg.vec_len = sizeof(rgns) / sizeof(*rgns); 1429 if (reset) 1430 arg.flags |= PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC; 1431 arg.category_mask = PAGE_IS_WRITTEN; 1432 arg.return_mask = PAGE_IS_WRITTEN; 1433 1434 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 1435 if (ret < 0) 1436 ksft_exit_fail_msg("ioctl failed\n"); 1437 1438 cnt = 0; 1439 for (i = 0; i < (unsigned long)ret; ++i) { 1440 if (rgns[i].categories != PAGE_IS_WRITTEN) 1441 ksft_exit_fail_msg("wrong flags\n"); 1442 1443 for (j = 0; j < LEN(rgns[i]); ++j) 1444 cnt++; 1445 } 1446 1447 return cnt; 1448 } 1449 1450 void *thread_proc(void *mem) 1451 { 1452 int *m = mem; 1453 long curr_faults, faults; 1454 struct rusage r; 1455 unsigned int i; 1456 int ret; 1457 1458 if (getrusage(RUSAGE_THREAD, &r)) 1459 ksft_exit_fail_msg("getrusage\n"); 1460 1461 curr_faults = r.ru_minflt; 1462 1463 while (!finish) { 1464 ret = pthread_barrier_wait(&start_barrier); 1465 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1466 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1467 1468 for (i = 0; i < access_per_thread; ++i) 1469 __atomic_add_fetch(m + i * (page_size / sizeof(*m)), 1, __ATOMIC_SEQ_CST); 1470 1471 ret = pthread_barrier_wait(&end_barrier); 1472 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1473 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1474 1475 if (getrusage(RUSAGE_THREAD, &r)) 1476 ksft_exit_fail_msg("getrusage\n"); 1477 1478 faults = r.ru_minflt - curr_faults; 1479 if (faults < access_per_thread) 1480 ksft_exit_fail_msg("faults < access_per_thread"); 1481 1482 __atomic_add_fetch(&extra_thread_faults, faults - access_per_thread, 1483 __ATOMIC_SEQ_CST); 1484 curr_faults = r.ru_minflt; 1485 } 1486 1487 return NULL; 1488 } 1489 1490 static void transact_test(int page_size) 1491 { 1492 unsigned int i, count, extra_pages; 1493 unsigned int c; 1494 pthread_t th; 1495 char *mem; 1496 int ret; 1497 1498 if (pthread_barrier_init(&start_barrier, NULL, nthreads + 1)) 1499 ksft_exit_fail_msg("pthread_barrier_init\n"); 1500 1501 if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1)) 1502 ksft_exit_fail_msg("pthread_barrier_init\n"); 1503 1504 mem = mmap(NULL, page_size * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, 1505 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 1506 if (mem == MAP_FAILED) 1507 ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno)); 1508 1509 wp_init(mem, page_size * nthreads * pages_per_thread); 1510 wp_addr_range(mem, page_size * nthreads * pages_per_thread); 1511 1512 memset(mem, 0, page_size * nthreads * pages_per_thread); 1513 1514 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1515 ksft_test_result(count > 0, "%s count %u\n", __func__, count); 1516 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1517 ksft_test_result(count == 0, "%s count %u\n", __func__, count); 1518 1519 finish = 0; 1520 for (i = 0; i < nthreads; ++i) 1521 pthread_create(&th, NULL, thread_proc, mem + page_size * i * pages_per_thread); 1522 1523 extra_pages = 0; 1524 for (i = 0; i < iter_count; ++i) { 1525 count = 0; 1526 1527 ret = pthread_barrier_wait(&start_barrier); 1528 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1529 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1530 1531 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, 1532 page_size); 1533 1534 ret = pthread_barrier_wait(&end_barrier); 1535 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1536 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1537 1538 if (count > nthreads * access_per_thread) 1539 ksft_exit_fail_msg("Too big count %u expected %u, iter %u\n", 1540 count, nthreads * access_per_thread, i); 1541 1542 c = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1543 count += c; 1544 1545 if (c > nthreads * access_per_thread) { 1546 ksft_test_result_fail(" %s count > nthreads\n", __func__); 1547 return; 1548 } 1549 1550 if (count != nthreads * access_per_thread) { 1551 /* 1552 * The purpose of the test is to make sure that no page updates are lost 1553 * when the page updates and read-resetting soft dirty flags are performed 1554 * in parallel. However, it is possible that the application will get the 1555 * soft dirty flags twice on the two consecutive read-resets. This seems 1556 * unavoidable as soft dirty flag is handled in software through page faults 1557 * in kernel. While the updating the flags is supposed to be synchronized 1558 * between page fault handling and read-reset, it is possible that 1559 * read-reset happens after page fault PTE update but before the application 1560 * re-executes write instruction. So read-reset gets the flag, clears write 1561 * access and application gets page fault again for the same write. 1562 */ 1563 if (count < nthreads * access_per_thread) { 1564 ksft_test_result_fail("Lost update, iter %u, %u vs %u.\n", i, count, 1565 nthreads * access_per_thread); 1566 return; 1567 } 1568 1569 extra_pages += count - nthreads * access_per_thread; 1570 } 1571 } 1572 1573 pthread_barrier_wait(&start_barrier); 1574 finish = 1; 1575 pthread_barrier_wait(&end_barrier); 1576 1577 ksft_test_result_pass("%s Extra pages %u (%.1lf%%), extra thread faults %u.\n", __func__, 1578 extra_pages, 1579 100.0 * extra_pages / (iter_count * nthreads * access_per_thread), 1580 extra_thread_faults); 1581 } 1582 1583 void zeropfn_tests(void) 1584 { 1585 unsigned long long mem_size; 1586 struct page_region vec; 1587 int i, ret; 1588 char *mmap_mem, *mem; 1589 1590 /* Test with normal memory */ 1591 mem_size = 10 * page_size; 1592 mem = mmap(NULL, mem_size, PROT_READ, MAP_PRIVATE | MAP_ANON, -1, 0); 1593 if (mem == MAP_FAILED) 1594 ksft_exit_fail_msg("error nomem\n"); 1595 1596 /* Touch each page to ensure it's mapped */ 1597 for (i = 0; i < mem_size; i += page_size) 1598 (void)((volatile char *)mem)[i]; 1599 1600 ret = pagemap_ioctl(mem, mem_size, &vec, 1, 0, 1601 (mem_size / page_size), PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO); 1602 if (ret < 0) 1603 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1604 1605 ksft_test_result(ret == 1 && LEN(vec) == (mem_size / page_size), 1606 "%s all pages must have PFNZERO set\n", __func__); 1607 1608 munmap(mem, mem_size); 1609 1610 /* Test with huge page if user_zero_page is set to 1 */ 1611 if (!detect_huge_zeropage()) { 1612 ksft_test_result_skip("%s use_zero_page not supported or set to 1\n", __func__); 1613 return; 1614 } 1615 1616 mem_size = 2 * hpage_size; 1617 mmap_mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, 1618 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1619 if (mmap_mem == MAP_FAILED) 1620 ksft_exit_fail_msg("error nomem\n"); 1621 1622 /* We need a THP-aligned memory area. */ 1623 mem = (char *)(((uintptr_t)mmap_mem + hpage_size) & ~(hpage_size - 1)); 1624 1625 ret = madvise(mem, hpage_size, MADV_HUGEPAGE); 1626 if (!ret) { 1627 FORCE_READ(*mem); 1628 1629 ret = pagemap_ioctl(mem, hpage_size, &vec, 1, 0, 1630 0, PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO); 1631 if (ret < 0) 1632 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1633 1634 ksft_test_result(ret == 1 && LEN(vec) == (hpage_size / page_size), 1635 "%s all huge pages must have PFNZERO set\n", __func__); 1636 } else { 1637 ksft_test_result_skip("%s huge page not supported\n", __func__); 1638 } 1639 1640 munmap(mmap_mem, mem_size); 1641 } 1642 1643 int main(int __attribute__((unused)) argc, char *argv[]) 1644 { 1645 int shmid, buf_size, fd, i, ret; 1646 unsigned long long mem_size; 1647 char *mem, *map, *fmem; 1648 struct stat sbuf; 1649 1650 progname = argv[0]; 1651 1652 ksft_print_header(); 1653 1654 if (init_uffd()) 1655 ksft_exit_skip("Failed to initialize userfaultfd\n"); 1656 1657 if (!hugetlb_setup_default(4)) 1658 ksft_print_msg("HugeTLB test will be skipped\n"); 1659 1660 ksft_set_plan(119); 1661 1662 page_size = getpagesize(); 1663 hpage_size = read_pmd_pagesize(); 1664 1665 pagemap_fd = open(PAGEMAP, O_RDONLY); 1666 if (pagemap_fd < 0) 1667 ksft_exit_fail_msg("Failed to open " PAGEMAP "\n"); 1668 1669 /* 1. Sanity testing */ 1670 sanity_tests_sd(); 1671 1672 /* 2. Normal page testing */ 1673 mem_size = 10 * page_size; 1674 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1675 if (mem == MAP_FAILED) 1676 ksft_exit_fail_msg("error nomem\n"); 1677 wp_init(mem, mem_size); 1678 wp_addr_range(mem, mem_size); 1679 1680 base_tests("Page testing:", mem, mem_size, 0); 1681 1682 wp_free(mem, mem_size); 1683 munmap(mem, mem_size); 1684 1685 /* 3. Large page testing */ 1686 mem_size = 512 * 10 * page_size; 1687 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1688 if (mem == MAP_FAILED) 1689 ksft_exit_fail_msg("error nomem\n"); 1690 wp_init(mem, mem_size); 1691 wp_addr_range(mem, mem_size); 1692 1693 base_tests("Large Page testing:", mem, mem_size, 0); 1694 1695 wp_free(mem, mem_size); 1696 munmap(mem, mem_size); 1697 1698 /* 4. Huge page testing */ 1699 map = gethugepage(hpage_size); 1700 if (map) { 1701 wp_init(map, hpage_size); 1702 wp_addr_range(map, hpage_size); 1703 base_tests("Huge page testing:", map, hpage_size, 0); 1704 wp_free(map, hpage_size); 1705 free(map); 1706 } else { 1707 base_tests("Huge page testing:", NULL, 0, 1); 1708 } 1709 1710 /* 5. SHM Hugetlb page testing */ 1711 mem_size = default_huge_page_size(); 1712 mem = gethugetlb_mem(mem_size, &shmid); 1713 if (mem) { 1714 wp_init(mem, mem_size); 1715 wp_addr_range(mem, mem_size); 1716 1717 base_tests("Hugetlb shmem testing:", mem, mem_size, 0); 1718 1719 wp_free(mem, mem_size); 1720 shmctl(shmid, IPC_RMID, NULL); 1721 } else { 1722 base_tests("Hugetlb shmem testing:", NULL, 0, 1); 1723 } 1724 1725 /* 6. Hugetlb page testing */ 1726 mem = gethugetlb_mem(mem_size, NULL); 1727 if (mem) { 1728 wp_init(mem, mem_size); 1729 wp_addr_range(mem, mem_size); 1730 1731 base_tests("Hugetlb mem testing:", mem, mem_size, 0); 1732 1733 wp_free(mem, mem_size); 1734 } else { 1735 base_tests("Hugetlb mem testing:", NULL, 0, 1); 1736 } 1737 1738 /* 7. File Hugetlb testing */ 1739 mem_size = default_huge_page_size(); 1740 fd = memfd_create("uffd-test", MFD_HUGETLB | MFD_NOEXEC_SEAL); 1741 if (fd < 0) 1742 ksft_exit_fail_msg("uffd-test creation failed %d %s\n", errno, strerror(errno)); 1743 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 1744 if (mem != MAP_FAILED) { 1745 wp_init(mem, mem_size); 1746 wp_addr_range(mem, mem_size); 1747 1748 base_tests("Hugetlb shmem testing:", mem, mem_size, 0); 1749 1750 wp_free(mem, mem_size); 1751 shmctl(shmid, IPC_RMID, NULL); 1752 } else { 1753 base_tests("Hugetlb shmem testing:", NULL, 0, 1); 1754 } 1755 close(fd); 1756 1757 /* 8. File memory testing */ 1758 buf_size = page_size * 10; 1759 1760 fd = open(__FILE__".tmp0", O_RDWR | O_CREAT, 0777); 1761 if (fd < 0) 1762 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n", 1763 strerror(errno)); 1764 1765 for (i = 0; i < buf_size; i++) 1766 if (write(fd, "c", 1) < 0) 1767 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1768 1769 ret = stat(__FILE__".tmp0", &sbuf); 1770 if (ret < 0) 1771 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1772 1773 fmem = mmap(NULL, sbuf.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1774 if (fmem == MAP_FAILED) 1775 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1776 1777 wp_init(fmem, sbuf.st_size); 1778 wp_addr_range(fmem, sbuf.st_size); 1779 1780 base_tests("File memory testing:", fmem, sbuf.st_size, 0); 1781 1782 wp_free(fmem, sbuf.st_size); 1783 munmap(fmem, sbuf.st_size); 1784 close(fd); 1785 1786 /* 9. File memory testing */ 1787 buf_size = page_size * 10; 1788 1789 fd = memfd_create(__FILE__".tmp00", MFD_NOEXEC_SEAL); 1790 if (fd < 0) 1791 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n", 1792 strerror(errno)); 1793 1794 if (ftruncate(fd, buf_size)) 1795 ksft_exit_fail_msg("Error ftruncate\n"); 1796 1797 for (i = 0; i < buf_size; i++) 1798 if (write(fd, "c", 1) < 0) 1799 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1800 1801 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1802 if (fmem == MAP_FAILED) 1803 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1804 1805 wp_init(fmem, buf_size); 1806 wp_addr_range(fmem, buf_size); 1807 1808 base_tests("File anonymous memory testing:", fmem, buf_size, 0); 1809 1810 wp_free(fmem, buf_size); 1811 munmap(fmem, buf_size); 1812 close(fd); 1813 1814 /* 10. Huge page tests */ 1815 hpage_unit_tests(); 1816 1817 /* 11. Iterative test */ 1818 test_simple(); 1819 1820 /* 12. Mprotect test */ 1821 mprotect_tests(); 1822 1823 /* 13. Transact test */ 1824 transact_test(page_size); 1825 1826 /* 14. Sanity testing */ 1827 sanity_tests(); 1828 1829 /*15. Unmapped address test */ 1830 unmapped_region_tests(); 1831 1832 /* 16. Userfaultfd tests */ 1833 userfaultfd_tests(); 1834 1835 /* 17. ZEROPFN tests */ 1836 zeropfn_tests(); 1837 1838 /* 18. Unpopulated pte scan-path consistency */ 1839 unpopulated_scan_test(); 1840 unpopulated_thp_scan_test(); 1841 1842 close(pagemap_fd); 1843 ksft_finished(); 1844 } 1845