1 // SPDX-License-Identifier: GPL-2.0 2 3 #define _GNU_SOURCE 4 #include <stdio.h> 5 #include <fcntl.h> 6 #include <string.h> 7 #include <sys/mman.h> 8 #include <errno.h> 9 #include <malloc.h> 10 #include <linux/types.h> 11 #include <linux/memfd.h> 12 #include <linux/userfaultfd.h> 13 #include <linux/fs.h> 14 #include <sys/ioctl.h> 15 #include <sys/stat.h> 16 #include <math.h> 17 #include <asm/unistd.h> 18 #include <pthread.h> 19 #include <sys/resource.h> 20 #include <assert.h> 21 #include <sys/ipc.h> 22 #include <sys/shm.h> 23 24 #include "vm_util.h" 25 #include "kselftest.h" 26 #include "hugepage_settings.h" 27 28 #define PAGEMAP_BITS_ALL (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ 29 PAGE_IS_FILE | PAGE_IS_PRESENT | \ 30 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ 31 PAGE_IS_HUGE) 32 #define PAGEMAP_NON_WRITTEN_BITS (PAGE_IS_WPALLOWED | PAGE_IS_FILE | \ 33 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | \ 34 PAGE_IS_PFNZERO | PAGE_IS_HUGE) 35 36 #define TEST_ITERATIONS 100 37 #define PAGEMAP "/proc/self/pagemap" 38 int pagemap_fd; 39 int uffd; 40 size_t page_size; 41 size_t hpage_size; 42 const char *progname; 43 44 #define LEN(region) ((region.end - region.start)/page_size) 45 46 static long pagemap_ioctl(void *start, int len, void *vec, int vec_len, int flag, 47 int max_pages, long required_mask, long anyof_mask, long excluded_mask, 48 long return_mask) 49 { 50 struct pm_scan_arg arg; 51 52 arg.start = (uintptr_t)start; 53 arg.end = (uintptr_t)(start + len); 54 arg.vec = (uintptr_t)vec; 55 arg.vec_len = vec_len; 56 arg.flags = flag; 57 arg.size = sizeof(struct pm_scan_arg); 58 arg.max_pages = max_pages; 59 arg.category_mask = required_mask; 60 arg.category_anyof_mask = anyof_mask; 61 arg.category_inverted = excluded_mask; 62 arg.return_mask = return_mask; 63 64 return ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 65 } 66 67 static long pagemap_ioc(void *start, int len, void *vec, int vec_len, int flag, 68 int max_pages, long required_mask, long anyof_mask, long excluded_mask, 69 long return_mask, long *walk_end) 70 { 71 struct pm_scan_arg arg; 72 int ret; 73 74 arg.start = (uintptr_t)start; 75 arg.end = (uintptr_t)(start + len); 76 arg.vec = (uintptr_t)vec; 77 arg.vec_len = vec_len; 78 arg.flags = flag; 79 arg.size = sizeof(struct pm_scan_arg); 80 arg.max_pages = max_pages; 81 arg.category_mask = required_mask; 82 arg.category_anyof_mask = anyof_mask; 83 arg.category_inverted = excluded_mask; 84 arg.return_mask = return_mask; 85 86 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 87 88 if (walk_end) 89 *walk_end = arg.walk_end; 90 91 return ret; 92 } 93 94 95 int init_uffd(void) 96 { 97 struct uffdio_api uffdio_api; 98 99 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); 100 if (uffd == -1) 101 return uffd; 102 103 uffdio_api.api = UFFD_API; 104 uffdio_api.features = UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC | 105 UFFD_FEATURE_WP_HUGETLBFS_SHMEM; 106 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) 107 return -1; 108 109 if (!(uffdio_api.api & UFFDIO_REGISTER_MODE_WP) || 110 !(uffdio_api.features & UFFD_FEATURE_WP_UNPOPULATED) || 111 !(uffdio_api.features & UFFD_FEATURE_WP_ASYNC) || 112 !(uffdio_api.features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM)) 113 return -1; 114 115 return 0; 116 } 117 118 int wp_init(void *addr, long size) 119 { 120 struct uffdio_register uffdio_register; 121 struct uffdio_writeprotect wp; 122 123 uffdio_register.range.start = (unsigned long)addr; 124 uffdio_register.range.len = size; 125 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; 126 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) 127 ksft_exit_fail_msg("ioctl(UFFDIO_REGISTER) %d %s\n", errno, strerror(errno)); 128 129 if (!(uffdio_register.ioctls & UFFDIO_WRITEPROTECT)) 130 ksft_exit_fail_msg("ioctl set is incorrect\n"); 131 132 wp.range.start = (unsigned long)addr; 133 wp.range.len = size; 134 wp.mode = UFFDIO_WRITEPROTECT_MODE_WP; 135 136 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &wp)) 137 ksft_exit_fail_msg("ioctl(UFFDIO_WRITEPROTECT)\n"); 138 139 return 0; 140 } 141 142 int wp_free(void *addr, long size) 143 { 144 struct uffdio_register uffdio_register; 145 146 uffdio_register.range.start = (unsigned long)addr; 147 uffdio_register.range.len = size; 148 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; 149 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range)) 150 ksft_exit_fail_msg("ioctl unregister failure\n"); 151 return 0; 152 } 153 154 int wp_addr_range(void *addr, int size) 155 { 156 if (pagemap_ioctl(addr, size, NULL, 0, 157 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 158 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0) 159 ksft_exit_fail_msg("error %d %d %s\n", 1, errno, strerror(errno)); 160 161 return 0; 162 } 163 164 void *gethugetlb_mem(int size, int *shmid) 165 { 166 char *mem; 167 168 if (shmid) { 169 *shmid = shmget(2, size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); 170 if (*shmid < 0) 171 return NULL; 172 173 mem = shmat(*shmid, 0, 0); 174 if (mem == (char *)-1) { 175 shmctl(*shmid, IPC_RMID, NULL); 176 ksft_exit_fail_msg("Shared memory attach failure\n"); 177 } 178 } else { 179 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, 180 MAP_ANONYMOUS | MAP_HUGETLB | MAP_PRIVATE, -1, 0); 181 if (mem == MAP_FAILED) 182 return NULL; 183 } 184 185 return mem; 186 } 187 188 int userfaultfd_tests(void) 189 { 190 long mem_size, vec_size, written, num_pages = 16; 191 char *mem, *vec; 192 193 mem_size = num_pages * page_size; 194 mem = mmap(NULL, mem_size, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0); 195 if (mem == MAP_FAILED) 196 ksft_exit_fail_msg("error nomem\n"); 197 198 wp_init(mem, mem_size); 199 200 /* Change protection of pages differently */ 201 mprotect(mem, mem_size/8, PROT_READ|PROT_WRITE); 202 mprotect(mem + 1 * mem_size/8, mem_size/8, PROT_READ); 203 mprotect(mem + 2 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 204 mprotect(mem + 3 * mem_size/8, mem_size/8, PROT_READ); 205 mprotect(mem + 4 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 206 mprotect(mem + 5 * mem_size/8, mem_size/8, PROT_NONE); 207 mprotect(mem + 6 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 208 mprotect(mem + 7 * mem_size/8, mem_size/8, PROT_READ); 209 210 wp_addr_range(mem + (mem_size/16), mem_size - 2 * (mem_size/8)); 211 wp_addr_range(mem, mem_size); 212 213 vec_size = mem_size/page_size; 214 vec = calloc(vec_size, sizeof(struct page_region)); 215 216 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 217 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 218 if (written < 0) 219 ksft_exit_fail_msg("error %ld %d %s\n", written, errno, strerror(errno)); 220 221 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", __func__); 222 223 wp_free(mem, mem_size); 224 munmap(mem, mem_size); 225 free(vec); 226 return 0; 227 } 228 229 int get_reads(struct page_region *vec, int vec_size) 230 { 231 int i, sum = 0; 232 233 for (i = 0; i < vec_size; i++) 234 sum += LEN(vec[i]); 235 236 return sum; 237 } 238 239 int sanity_tests_sd(void) 240 { 241 unsigned long long mem_size, vec_size, i, total_pages = 0; 242 long ret, ret2, ret3; 243 int num_pages = 1000; 244 int total_writes, total_reads, reads, count; 245 struct page_region *vec, *vec2; 246 char *mem, *m[2]; 247 long walk_end; 248 249 vec_size = num_pages/2; 250 mem_size = num_pages * page_size; 251 252 vec = calloc(vec_size, sizeof(struct page_region)); 253 if (!vec) 254 ksft_exit_fail_msg("error nomem\n"); 255 256 vec2 = calloc(vec_size, sizeof(struct page_region)); 257 if (!vec2) 258 ksft_exit_fail_msg("error nomem\n"); 259 260 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 261 if (mem == MAP_FAILED) 262 ksft_exit_fail_msg("error nomem\n"); 263 264 wp_init(mem, mem_size); 265 wp_addr_range(mem, mem_size); 266 267 /* 1. wrong operation */ 268 ksft_test_result(pagemap_ioctl(mem, 0, vec, vec_size, 0, 269 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 270 "%s Zero range size is valid\n", __func__); 271 272 ksft_test_result(pagemap_ioctl(mem, mem_size, NULL, vec_size, 0, 273 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) < 0, 274 "%s output buffer must be specified with size\n", __func__); 275 276 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, 0, 0, 277 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 278 "%s output buffer can be 0\n", __func__); 279 280 ksft_test_result(pagemap_ioctl(mem, mem_size, 0, 0, 0, 281 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 282 "%s output buffer can be 0\n", __func__); 283 284 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, -1, 285 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0, 286 "%s wrong flag specified\n", __func__); 287 288 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 289 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC | 0xFF, 290 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0, 291 "%s flag has extra bits specified\n", __func__); 292 293 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 294 0, 0, 0, 0, PAGE_IS_WRITTEN) >= 0, 295 "%s no selection mask is specified\n", __func__); 296 297 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 298 0, PAGE_IS_WRITTEN, PAGE_IS_WRITTEN, 0, 0) == 0, 299 "%s no return mask is specified\n", __func__); 300 301 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 302 0, PAGE_IS_WRITTEN, 0, 0, 0x1000) < 0, 303 "%s wrong return mask specified\n", __func__); 304 305 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 306 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 307 0, 0xFFF, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN) < 0, 308 "%s mixture of correct and wrong flag\n", __func__); 309 310 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 311 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 312 0, 0, 0, PAGEMAP_BITS_ALL, PAGE_IS_WRITTEN) >= 0, 313 "%s PAGEMAP_BITS_ALL can be specified with PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", 314 __func__); 315 316 /* 2. Clear area with larger vec size */ 317 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 318 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 319 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 320 ksft_test_result(ret >= 0, "%s Clear area with larger vec size\n", __func__); 321 322 /* 3. Repeated pattern of written and non-written pages */ 323 for (i = 0; i < mem_size; i += 2 * page_size) 324 mem[i]++; 325 326 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 0, 327 0, PAGE_IS_WRITTEN); 328 if (ret < 0) 329 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 330 331 ksft_test_result((unsigned long long)ret == mem_size/(page_size * 2), 332 "%s Repeated pattern of written and non-written pages\n", __func__); 333 334 /* 4. Repeated pattern of written and non-written pages in parts */ 335 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 336 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 337 num_pages/2 - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 338 if (ret < 0) 339 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 340 341 ret2 = pagemap_ioctl(mem, mem_size, vec, 2, 0, 0, PAGE_IS_WRITTEN, 0, 0, 342 PAGE_IS_WRITTEN); 343 if (ret2 < 0) 344 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno)); 345 346 ret3 = pagemap_ioctl(mem, mem_size, vec, vec_size, 347 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 348 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 349 if (ret3 < 0) 350 ksft_exit_fail_msg("error %ld %d %s\n", ret3, errno, strerror(errno)); 351 352 ksft_test_result((ret + ret3) == num_pages/2 && ret2 == 2, 353 "%s Repeated pattern of written and non-written pages in parts %ld %ld %ld\n", 354 __func__, ret, ret3, ret2); 355 356 /* 5. Repeated pattern of written and non-written pages max_pages */ 357 for (i = 0; i < mem_size; i += 2 * page_size) 358 mem[i]++; 359 mem[(mem_size/page_size - 1) * page_size]++; 360 361 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 362 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 363 num_pages/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 364 if (ret < 0) 365 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 366 367 ret2 = pagemap_ioctl(mem, mem_size, vec, vec_size, 368 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 369 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 370 if (ret2 < 0) 371 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno)); 372 373 ksft_test_result(ret == num_pages/2 && ret2 == 1, 374 "%s Repeated pattern of written and non-written pages max_pages\n", 375 __func__); 376 377 /* 6. only get 2 dirty pages and clear them as well */ 378 vec_size = mem_size/page_size; 379 memset(mem, -1, mem_size); 380 381 /* get and clear second and third pages */ 382 ret = pagemap_ioctl(mem + page_size, 2 * page_size, vec, 1, 383 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 384 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 385 if (ret < 0) 386 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 387 388 ret2 = pagemap_ioctl(mem, mem_size, vec2, vec_size, 0, 0, 389 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 390 if (ret2 < 0) 391 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno)); 392 393 ksft_test_result(ret == 1 && LEN(vec[0]) == 2 && 394 vec[0].start == (uintptr_t)(mem + page_size) && 395 ret2 == 2 && LEN(vec2[0]) == 1 && vec2[0].start == (uintptr_t)mem && 396 LEN(vec2[1]) == vec_size - 3 && 397 vec2[1].start == (uintptr_t)(mem + 3 * page_size), 398 "%s only get 2 written pages and clear them as well\n", __func__); 399 400 wp_free(mem, mem_size); 401 munmap(mem, mem_size); 402 403 /* 7. Two regions */ 404 m[0] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 405 if (m[0] == MAP_FAILED) 406 ksft_exit_fail_msg("error nomem\n"); 407 m[1] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 408 if (m[1] == MAP_FAILED) 409 ksft_exit_fail_msg("error nomem\n"); 410 411 wp_init(m[0], mem_size); 412 wp_init(m[1], mem_size); 413 wp_addr_range(m[0], mem_size); 414 wp_addr_range(m[1], mem_size); 415 416 memset(m[0], 'a', mem_size); 417 memset(m[1], 'b', mem_size); 418 419 wp_addr_range(m[0], mem_size); 420 421 ret = pagemap_ioctl(m[1], mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0, 422 PAGE_IS_WRITTEN); 423 if (ret < 0) 424 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 425 426 ksft_test_result(ret == 1 && LEN(vec[0]) == mem_size/page_size, 427 "%s Two regions\n", __func__); 428 429 wp_free(m[0], mem_size); 430 wp_free(m[1], mem_size); 431 munmap(m[0], mem_size); 432 munmap(m[1], mem_size); 433 434 free(vec); 435 free(vec2); 436 437 /* 8. Smaller vec */ 438 mem_size = 1050 * page_size; 439 vec_size = mem_size/(page_size*2); 440 441 vec = calloc(vec_size, sizeof(struct page_region)); 442 if (!vec) 443 ksft_exit_fail_msg("error nomem\n"); 444 445 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 446 if (mem == MAP_FAILED) 447 ksft_exit_fail_msg("error nomem\n"); 448 449 wp_init(mem, mem_size); 450 wp_addr_range(mem, mem_size); 451 452 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 453 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 454 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 455 if (ret < 0) 456 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 457 458 for (i = 0; i < mem_size/page_size; i += 2) 459 mem[i * page_size]++; 460 461 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 462 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 463 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 464 if (ret < 0) 465 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 466 467 total_pages += ret; 468 469 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 470 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 471 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 472 if (ret < 0) 473 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 474 475 total_pages += ret; 476 477 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 478 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 479 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 480 if (ret < 0) 481 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 482 483 total_pages += ret; 484 485 ksft_test_result(total_pages == mem_size/(page_size*2), "%s Smaller max_pages\n", __func__); 486 487 free(vec); 488 wp_free(mem, mem_size); 489 munmap(mem, mem_size); 490 total_pages = 0; 491 492 /* 9. Smaller vec */ 493 mem_size = 10000 * page_size; 494 vec_size = 50; 495 496 vec = calloc(vec_size, sizeof(struct page_region)); 497 if (!vec) 498 ksft_exit_fail_msg("error nomem\n"); 499 500 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 501 if (mem == MAP_FAILED) 502 ksft_exit_fail_msg("error nomem\n"); 503 504 wp_init(mem, mem_size); 505 wp_addr_range(mem, mem_size); 506 507 for (count = 0; count < TEST_ITERATIONS; count++) { 508 total_writes = total_reads = 0; 509 walk_end = (long)mem; 510 511 for (i = 0; i < mem_size; i += page_size) { 512 if (rand() % 2) { 513 mem[i]++; 514 total_writes++; 515 } 516 } 517 518 while (total_reads < total_writes) { 519 ret = pagemap_ioc((void *)walk_end, mem_size-(walk_end - (long)mem), vec, 520 vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 521 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 522 if (ret < 0) 523 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 524 525 if ((unsigned long)ret > vec_size) 526 break; 527 528 reads = get_reads(vec, ret); 529 total_reads += reads; 530 } 531 532 if (total_reads != total_writes) 533 break; 534 } 535 536 ksft_test_result(count == TEST_ITERATIONS, "Smaller vec\n"); 537 538 free(vec); 539 wp_free(mem, mem_size); 540 munmap(mem, mem_size); 541 542 /* 10. Walk_end tester */ 543 vec_size = 1000; 544 mem_size = vec_size * page_size; 545 546 vec = calloc(vec_size, sizeof(struct page_region)); 547 if (!vec) 548 ksft_exit_fail_msg("error nomem\n"); 549 550 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 551 if (mem == MAP_FAILED) 552 ksft_exit_fail_msg("error nomem\n"); 553 554 wp_init(mem, mem_size); 555 wp_addr_range(mem, mem_size); 556 557 memset(mem, 0, mem_size); 558 559 ret = pagemap_ioc(mem, 0, vec, vec_size, 0, 560 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 561 if (ret < 0) 562 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 563 ksft_test_result(ret == 0 && walk_end == (long)mem, 564 "Walk_end: Same start and end address\n"); 565 566 ret = pagemap_ioc(mem, 0, vec, vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 567 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 568 if (ret < 0) 569 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 570 ksft_test_result(ret == 0 && walk_end == (long)mem, 571 "Walk_end: Same start and end with WP\n"); 572 573 ret = pagemap_ioc(mem, 0, vec, 0, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 574 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 575 if (ret < 0) 576 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 577 ksft_test_result(ret == 0 && walk_end == (long)mem, 578 "Walk_end: Same start and end with 0 output buffer\n"); 579 580 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 581 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 582 if (ret < 0) 583 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 584 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 585 "Walk_end: Big vec\n"); 586 587 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 588 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 589 if (ret < 0) 590 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 591 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 592 "Walk_end: vec of minimum length\n"); 593 594 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 595 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 596 if (ret < 0) 597 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 598 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 599 "Walk_end: Max pages specified\n"); 600 601 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 602 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 603 if (ret < 0) 604 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 605 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size/2), 606 "Walk_end: Half max pages\n"); 607 608 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 609 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 610 if (ret < 0) 611 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 612 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size), 613 "Walk_end: 1 max page\n"); 614 615 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 616 -1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 617 if (ret < 0) 618 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 619 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 620 "Walk_end: max pages\n"); 621 622 wp_addr_range(mem, mem_size); 623 for (i = 0; i < mem_size; i += 2 * page_size) 624 mem[i]++; 625 626 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 627 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 628 if (ret < 0) 629 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 630 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 631 "Walk_end sparse: Big vec\n"); 632 633 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 634 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 635 if (ret < 0) 636 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 637 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 638 "Walk_end sparse: vec of minimum length\n"); 639 640 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 641 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 642 if (ret < 0) 643 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 644 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 645 "Walk_end sparse: Max pages specified\n"); 646 647 ret = pagemap_ioc(mem, mem_size, vec, vec_size/2, 0, 648 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 649 if (ret < 0) 650 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 651 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 652 "Walk_end sparse: Max pages specified\n"); 653 654 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 655 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 656 if (ret < 0) 657 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 658 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 659 "Walk_end sparse: Max pages specified\n"); 660 661 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 662 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 663 if (ret < 0) 664 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 665 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size), 666 "Walk_endsparse : Half max pages\n"); 667 668 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 669 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 670 if (ret < 0) 671 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 672 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 673 "Walk_end: 1 max page\n"); 674 675 free(vec); 676 wp_free(mem, mem_size); 677 munmap(mem, mem_size); 678 679 return 0; 680 } 681 682 int base_tests(char *prefix, char *mem, unsigned long long mem_size, int skip) 683 { 684 unsigned long long vec_size; 685 int written; 686 struct page_region *vec, *vec2; 687 688 if (skip) { 689 ksft_test_result_skip("%s all new pages must not be written (dirty)\n", prefix); 690 ksft_test_result_skip("%s all pages must be written (dirty)\n", prefix); 691 ksft_test_result_skip("%s all pages dirty other than first and the last one\n", 692 prefix); 693 ksft_test_result_skip("%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix); 694 ksft_test_result_skip("%s only middle page dirty\n", prefix); 695 ksft_test_result_skip("%s only two middle pages dirty\n", prefix); 696 return 0; 697 } 698 699 vec_size = mem_size/page_size; 700 vec = calloc(vec_size, sizeof(struct page_region)); 701 vec2 = calloc(vec_size, sizeof(struct page_region)); 702 703 /* 1. all new pages must be not be written (dirty) */ 704 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 705 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 706 if (written < 0) 707 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 708 709 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", prefix); 710 711 /* 2. all pages must be written */ 712 memset(mem, -1, mem_size); 713 714 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0, 715 PAGE_IS_WRITTEN); 716 if (written < 0) 717 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 718 719 ksft_test_result(written == 1 && LEN(vec[0]) == mem_size/page_size, 720 "%s all pages must be written (dirty)\n", prefix); 721 722 /* 3. all pages dirty other than first and the last one */ 723 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 724 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 725 if (written < 0) 726 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 727 728 memset(mem + page_size, 0, mem_size - (2 * page_size)); 729 730 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 731 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 732 if (written < 0) 733 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 734 735 ksft_test_result(written == 1 && LEN(vec[0]) >= vec_size - 2 && LEN(vec[0]) <= vec_size, 736 "%s all pages dirty other than first and the last one\n", prefix); 737 738 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, 739 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 740 if (written < 0) 741 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 742 743 ksft_test_result(written == 0, 744 "%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix); 745 746 /* 4. only middle page dirty */ 747 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 748 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 749 if (written < 0) 750 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 751 752 mem[vec_size/2 * page_size]++; 753 754 written = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 755 0, 0, PAGE_IS_WRITTEN); 756 if (written < 0) 757 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 758 759 ksft_test_result(written == 1 && LEN(vec[0]) >= 1, 760 "%s only middle page dirty\n", prefix); 761 762 /* 5. only two middle pages dirty and walk over only middle pages */ 763 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 764 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE); 765 if (written < 0) 766 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 767 768 mem[vec_size/2 * page_size]++; 769 mem[(vec_size/2 + 1) * page_size]++; 770 771 written = pagemap_ioctl(&mem[vec_size/2 * page_size], 2 * page_size, vec, 1, 0, 772 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE); 773 if (written < 0) 774 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 775 776 ksft_test_result(written == 1 && vec[0].start == (uintptr_t)(&mem[vec_size/2 * page_size]) 777 && LEN(vec[0]) == 2, 778 "%s only two middle pages dirty\n", prefix); 779 780 free(vec); 781 free(vec2); 782 return 0; 783 } 784 785 void *gethugepage(int map_size) 786 { 787 int ret; 788 char *map; 789 790 map = memalign(hpage_size, map_size); 791 if (!map) 792 ksft_exit_fail_msg("memalign failed %d %s\n", errno, strerror(errno)); 793 794 ret = madvise(map, map_size, MADV_HUGEPAGE); 795 if (ret) 796 return NULL; 797 798 memset(map, 0, map_size); 799 800 return map; 801 } 802 803 int hpage_unit_tests(void) 804 { 805 char *map; 806 int ret, ret2; 807 size_t num_pages = 10; 808 unsigned long long map_size = hpage_size * num_pages; 809 unsigned long long vec_size = map_size/page_size; 810 struct page_region *vec, *vec2; 811 812 vec = calloc(vec_size, sizeof(struct page_region)); 813 vec2 = calloc(vec_size, sizeof(struct page_region)); 814 if (!vec || !vec2) 815 ksft_exit_fail_msg("malloc failed\n"); 816 817 map = gethugepage(map_size); 818 if (map) { 819 wp_init(map, map_size); 820 wp_addr_range(map, map_size); 821 822 /* 1. all new huge page must not be written (dirty) */ 823 ret = pagemap_ioctl(map, map_size, vec, vec_size, 824 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 825 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 826 if (ret < 0) 827 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 828 829 ksft_test_result(ret == 0, "%s all new huge page must not be written (dirty)\n", 830 __func__); 831 832 /* 2. all the huge page must not be written */ 833 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 834 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 835 if (ret < 0) 836 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 837 838 ksft_test_result(ret == 0, "%s all the huge page must not be written\n", __func__); 839 840 /* 3. all the huge page must be written and clear dirty as well */ 841 memset(map, -1, map_size); 842 ret = pagemap_ioctl(map, map_size, vec, vec_size, 843 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 844 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 845 if (ret < 0) 846 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 847 848 ksft_test_result(ret == 1 && vec[0].start == (uintptr_t)map && 849 LEN(vec[0]) == vec_size && vec[0].categories == PAGE_IS_WRITTEN, 850 "%s all the huge page must be written and clear\n", __func__); 851 852 /* 4. only middle page written */ 853 wp_free(map, map_size); 854 free(map); 855 map = gethugepage(map_size); 856 wp_init(map, map_size); 857 wp_addr_range(map, map_size); 858 map[vec_size/2 * page_size]++; 859 860 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 861 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 862 if (ret < 0) 863 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 864 865 ksft_test_result(ret == 1 && LEN(vec[0]) > 0, 866 "%s only middle page written\n", __func__); 867 868 wp_free(map, map_size); 869 free(map); 870 } else { 871 ksft_test_result_skip("%s all new huge page must be written\n", __func__); 872 ksft_test_result_skip("%s all the huge page must not be written\n", __func__); 873 ksft_test_result_skip("%s all the huge page must be written and clear\n", __func__); 874 ksft_test_result_skip("%s only middle page written\n", __func__); 875 } 876 877 /* 5. clear first half of huge page */ 878 map = gethugepage(map_size); 879 if (map) { 880 wp_init(map, map_size); 881 wp_addr_range(map, map_size); 882 883 memset(map, 0, map_size); 884 885 wp_addr_range(map, map_size/2); 886 887 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 888 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 889 if (ret < 0) 890 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 891 892 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 && 893 vec[0].start == (uintptr_t)(map + map_size/2), 894 "%s clear first half of huge page\n", __func__); 895 wp_free(map, map_size); 896 free(map); 897 } else { 898 ksft_test_result_skip("%s clear first half of huge page\n", __func__); 899 } 900 901 /* 6. clear first half of huge page with limited buffer */ 902 map = gethugepage(map_size); 903 if (map) { 904 wp_init(map, map_size); 905 wp_addr_range(map, map_size); 906 907 memset(map, 0, map_size); 908 909 ret = pagemap_ioctl(map, map_size, vec, vec_size, 910 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 911 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 912 if (ret < 0) 913 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 914 915 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 916 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 917 if (ret < 0) 918 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 919 920 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 && 921 vec[0].start == (uintptr_t)(map + map_size/2), 922 "%s clear first half of huge page with limited buffer\n", 923 __func__); 924 wp_free(map, map_size); 925 free(map); 926 } else { 927 ksft_test_result_skip("%s clear first half of huge page with limited buffer\n", 928 __func__); 929 } 930 931 /* 7. clear second half of huge page */ 932 map = gethugepage(map_size); 933 if (map) { 934 wp_init(map, map_size); 935 wp_addr_range(map, map_size); 936 937 memset(map, -1, map_size); 938 939 ret = pagemap_ioctl(map + map_size/2, map_size/2, vec, vec_size, 940 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, vec_size/2, 941 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 942 if (ret < 0) 943 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 944 945 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 946 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 947 if (ret < 0) 948 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 949 950 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2, 951 "%s clear second half huge page\n", __func__); 952 wp_free(map, map_size); 953 free(map); 954 } else { 955 ksft_test_result_skip("%s clear second half huge page\n", __func__); 956 } 957 958 /* 8. get half huge page */ 959 map = gethugepage(map_size); 960 if (map) { 961 wp_init(map, map_size); 962 wp_addr_range(map, map_size); 963 964 memset(map, -1, map_size); 965 usleep(100); 966 967 ret = pagemap_ioctl(map, map_size, vec, 1, 968 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 969 hpage_size/(2*page_size), PAGE_IS_WRITTEN, 0, 0, 970 PAGE_IS_WRITTEN); 971 if (ret < 0) 972 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 973 974 ksft_test_result(ret == 1 && LEN(vec[0]) == hpage_size/(2*page_size), 975 "%s get half huge page\n", __func__); 976 977 ret2 = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 978 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 979 if (ret2 < 0) 980 ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno)); 981 982 ksft_test_result(ret2 == 1 && LEN(vec[0]) == (map_size - hpage_size/2)/page_size, 983 "%s get half huge page\n", __func__); 984 985 wp_free(map, map_size); 986 free(map); 987 } else { 988 ksft_test_result_skip("%s get half huge page\n", __func__); 989 ksft_test_result_skip("%s get half huge page\n", __func__); 990 } 991 992 free(vec); 993 free(vec2); 994 return 0; 995 } 996 997 int unmapped_region_tests(void) 998 { 999 void *start = (void *)0x10000000; 1000 int written, len = 0x00040000; 1001 long vec_size = len / page_size; 1002 struct page_region *vec = calloc(vec_size, sizeof(struct page_region)); 1003 1004 /* 1. Get written pages */ 1005 written = pagemap_ioctl(start, len, vec, vec_size, 0, 0, 1006 PAGEMAP_NON_WRITTEN_BITS, 0, 0, PAGEMAP_NON_WRITTEN_BITS); 1007 if (written < 0) 1008 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 1009 1010 ksft_test_result(written >= 0, "%s Get status of pages\n", __func__); 1011 1012 free(vec); 1013 return 0; 1014 } 1015 1016 static void test_simple(void) 1017 { 1018 int i; 1019 char *map; 1020 struct page_region vec; 1021 1022 map = aligned_alloc(page_size, page_size); 1023 if (!map) 1024 ksft_exit_fail_msg("aligned_alloc failed\n"); 1025 1026 wp_init(map, page_size); 1027 wp_addr_range(map, page_size); 1028 1029 for (i = 0 ; i < TEST_ITERATIONS; i++) { 1030 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0, 1031 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 1) { 1032 ksft_print_msg("written bit was 1, but should be 0 (i=%d)\n", i); 1033 break; 1034 } 1035 1036 wp_addr_range(map, page_size); 1037 /* Write something to the page to get the written bit enabled on the page */ 1038 map[0]++; 1039 1040 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0, 1041 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0) { 1042 ksft_print_msg("written bit was 0, but should be 1 (i=%d)\n", i); 1043 break; 1044 } 1045 1046 wp_addr_range(map, page_size); 1047 } 1048 wp_free(map, page_size); 1049 free(map); 1050 1051 ksft_test_result(i == TEST_ITERATIONS, "Test %s\n", __func__); 1052 } 1053 1054 int sanity_tests(void) 1055 { 1056 unsigned long long mem_size, vec_size; 1057 long ret, fd, i, buf_size, nr_pages; 1058 struct page_region *vec; 1059 char *mem, *fmem; 1060 struct stat sbuf; 1061 1062 /* 1. wrong operation */ 1063 mem_size = 10 * page_size; 1064 vec_size = mem_size / page_size; 1065 1066 vec = calloc(vec_size, sizeof(struct page_region)); 1067 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1068 if (mem == MAP_FAILED || vec == MAP_FAILED) 1069 ksft_exit_fail_msg("error nomem\n"); 1070 1071 wp_init(mem, mem_size); 1072 wp_addr_range(mem, mem_size); 1073 1074 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 1075 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 1076 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0, 1077 "%s WP op can be specified with !PAGE_IS_WRITTEN\n", __func__); 1078 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1079 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0, 1080 "%s required_mask specified\n", __func__); 1081 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1082 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL) >= 0, 1083 "%s anyof_mask specified\n", __func__); 1084 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1085 0, 0, PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL) >= 0, 1086 "%s excluded_mask specified\n", __func__); 1087 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1088 PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL, 0, 1089 PAGEMAP_BITS_ALL) >= 0, 1090 "%s required_mask and anyof_mask specified\n", __func__); 1091 wp_free(mem, mem_size); 1092 munmap(mem, mem_size); 1093 1094 /* 2. Get sd and present pages with anyof_mask */ 1095 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1096 if (mem == MAP_FAILED) 1097 ksft_exit_fail_msg("error nomem\n"); 1098 wp_init(mem, mem_size); 1099 wp_addr_range(mem, mem_size); 1100 1101 memset(mem, 0, mem_size); 1102 1103 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1104 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL); 1105 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1106 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1107 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1108 "%s Get sd and present pages with anyof_mask\n", __func__); 1109 1110 /* 3. Get sd and present pages with required_mask */ 1111 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1112 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL); 1113 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1114 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1115 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1116 "%s Get all the pages with required_mask\n", __func__); 1117 1118 /* 4. Get sd and present pages with required_mask and anyof_mask */ 1119 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1120 PAGE_IS_WRITTEN, PAGE_IS_PRESENT, 0, PAGEMAP_BITS_ALL); 1121 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1122 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1123 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1124 "%s Get sd and present pages with required_mask and anyof_mask\n", 1125 __func__); 1126 1127 /* 5. Don't get sd pages */ 1128 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1129 PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN, PAGEMAP_BITS_ALL); 1130 ksft_test_result(ret == 0, "%s Don't get sd pages\n", __func__); 1131 1132 /* 6. Don't get present pages */ 1133 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1134 PAGE_IS_PRESENT, 0, PAGE_IS_PRESENT, PAGEMAP_BITS_ALL); 1135 ksft_test_result(ret == 0, "%s Don't get present pages\n", __func__); 1136 1137 wp_free(mem, mem_size); 1138 munmap(mem, mem_size); 1139 1140 /* 8. Find written present pages with return mask */ 1141 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1142 if (mem == MAP_FAILED) 1143 ksft_exit_fail_msg("error nomem\n"); 1144 wp_init(mem, mem_size); 1145 wp_addr_range(mem, mem_size); 1146 1147 memset(mem, 0, mem_size); 1148 1149 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 1150 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 1151 0, PAGEMAP_BITS_ALL, 0, PAGE_IS_WRITTEN); 1152 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1153 vec[0].categories == PAGE_IS_WRITTEN, 1154 "%s Find written present pages with return mask\n", __func__); 1155 wp_free(mem, mem_size); 1156 munmap(mem, mem_size); 1157 1158 /* 9. Memory mapped file */ 1159 fd = open(progname, O_RDONLY); 1160 if (fd < 0) 1161 ksft_exit_fail_msg("%s Memory mapped file\n", __func__); 1162 1163 ret = stat(progname, &sbuf); 1164 if (ret < 0) 1165 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); 1166 1167 fmem = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0); 1168 if (fmem == MAP_FAILED) 1169 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1170 1171 nr_pages = (sbuf.st_size + page_size - 1) / page_size; 1172 force_read_pages(fmem, nr_pages, page_size); 1173 1174 ret = pagemap_ioctl(fmem, sbuf.st_size, vec, vec_size, 0, 0, 1175 0, PAGEMAP_NON_WRITTEN_BITS, 0, PAGEMAP_NON_WRITTEN_BITS); 1176 1177 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem && 1178 LEN(vec[0]) == nr_pages && 1179 (vec[0].categories & PAGE_IS_FILE), 1180 "%s Memory mapped file\n", __func__); 1181 1182 munmap(fmem, sbuf.st_size); 1183 close(fd); 1184 1185 /* 10. Create and read/write to a memory mapped file */ 1186 buf_size = page_size * 10; 1187 1188 fd = open(__FILE__".tmp2", O_RDWR | O_CREAT, 0666); 1189 if (fd < 0) 1190 ksft_exit_fail_msg("Read/write to memory: %s\n", 1191 strerror(errno)); 1192 1193 for (i = 0; i < buf_size; i++) 1194 if (write(fd, "c", 1) < 0) 1195 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1196 1197 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1198 if (fmem == MAP_FAILED) 1199 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1200 1201 wp_init(fmem, buf_size); 1202 wp_addr_range(fmem, buf_size); 1203 1204 for (i = 0; i < buf_size; i++) 1205 fmem[i] = 'z'; 1206 1207 msync(fmem, buf_size, MS_SYNC); 1208 1209 ret = pagemap_ioctl(fmem, buf_size, vec, vec_size, 0, 0, 1210 PAGE_IS_WRITTEN, PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_FILE, 0, 1211 PAGEMAP_BITS_ALL); 1212 1213 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem && 1214 LEN(vec[0]) == (buf_size/page_size) && 1215 (vec[0].categories & PAGE_IS_WRITTEN), 1216 "%s Read/write to memory\n", __func__); 1217 1218 wp_free(fmem, buf_size); 1219 munmap(fmem, buf_size); 1220 close(fd); 1221 1222 free(vec); 1223 return 0; 1224 } 1225 1226 int mprotect_tests(void) 1227 { 1228 int ret; 1229 char *mem, *mem2; 1230 struct page_region vec; 1231 int pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 1232 1233 if (pagemap_fd < 0) { 1234 fprintf(stderr, "open() failed\n"); 1235 exit(1); 1236 } 1237 1238 /* 1. Map two pages */ 1239 mem = mmap(0, 2 * page_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1240 if (mem == MAP_FAILED) 1241 ksft_exit_fail_msg("error nomem\n"); 1242 wp_init(mem, 2 * page_size); 1243 wp_addr_range(mem, 2 * page_size); 1244 1245 /* Populate both pages. */ 1246 memset(mem, 1, 2 * page_size); 1247 1248 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1249 0, 0, PAGE_IS_WRITTEN); 1250 if (ret < 0) 1251 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1252 1253 ksft_test_result(ret == 1 && LEN(vec) == 2, "%s Both pages written\n", __func__); 1254 1255 /* 2. Start tracking */ 1256 wp_addr_range(mem, 2 * page_size); 1257 1258 ksft_test_result(pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, 1259 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0, 1260 "%s Both pages are not written (dirty)\n", __func__); 1261 1262 /* 3. Remap the second page */ 1263 mem2 = mmap(mem + page_size, page_size, PROT_READ|PROT_WRITE, 1264 MAP_PRIVATE|MAP_ANON|MAP_FIXED, -1, 0); 1265 if (mem2 == MAP_FAILED) 1266 ksft_exit_fail_msg("error nomem\n"); 1267 wp_init(mem2, page_size); 1268 wp_addr_range(mem2, page_size); 1269 1270 /* Protect + unprotect. */ 1271 mprotect(mem, page_size, PROT_NONE); 1272 mprotect(mem, 2 * page_size, PROT_READ); 1273 mprotect(mem, 2 * page_size, PROT_READ|PROT_WRITE); 1274 1275 /* Modify both pages. */ 1276 memset(mem, 2, 2 * page_size); 1277 1278 /* Protect + unprotect. */ 1279 mprotect(mem, page_size, PROT_NONE); 1280 mprotect(mem, page_size, PROT_READ); 1281 mprotect(mem, page_size, PROT_READ|PROT_WRITE); 1282 1283 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1284 0, 0, PAGE_IS_WRITTEN); 1285 if (ret < 0) 1286 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1287 1288 ksft_test_result(ret == 1 && LEN(vec) == 2, 1289 "%s Both pages written after remap and mprotect\n", __func__); 1290 1291 /* 4. Clear and make the pages written */ 1292 wp_addr_range(mem, 2 * page_size); 1293 1294 memset(mem, 'A', 2 * page_size); 1295 1296 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1297 0, 0, PAGE_IS_WRITTEN); 1298 if (ret < 0) 1299 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1300 1301 ksft_test_result(ret == 1 && LEN(vec) == 2, 1302 "%s Clear and make the pages written\n", __func__); 1303 1304 wp_free(mem, 2 * page_size); 1305 munmap(mem, 2 * page_size); 1306 return 0; 1307 } 1308 1309 /* transact test */ 1310 static const unsigned int nthreads = 6, pages_per_thread = 32, access_per_thread = 8; 1311 static pthread_barrier_t start_barrier, end_barrier; 1312 static unsigned int extra_thread_faults; 1313 static unsigned int iter_count = 1000; 1314 static volatile int finish; 1315 1316 static ssize_t get_dirty_pages_reset(char *mem, unsigned int count, 1317 int reset, int page_size) 1318 { 1319 struct pm_scan_arg arg = {0}; 1320 struct page_region rgns[256]; 1321 unsigned long long i, j; 1322 long ret; 1323 int cnt; 1324 1325 arg.size = sizeof(struct pm_scan_arg); 1326 arg.start = (uintptr_t)mem; 1327 arg.max_pages = count; 1328 arg.end = (uintptr_t)(mem + count * page_size); 1329 arg.vec = (uintptr_t)rgns; 1330 arg.vec_len = sizeof(rgns) / sizeof(*rgns); 1331 if (reset) 1332 arg.flags |= PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC; 1333 arg.category_mask = PAGE_IS_WRITTEN; 1334 arg.return_mask = PAGE_IS_WRITTEN; 1335 1336 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 1337 if (ret < 0) 1338 ksft_exit_fail_msg("ioctl failed\n"); 1339 1340 cnt = 0; 1341 for (i = 0; i < (unsigned long)ret; ++i) { 1342 if (rgns[i].categories != PAGE_IS_WRITTEN) 1343 ksft_exit_fail_msg("wrong flags\n"); 1344 1345 for (j = 0; j < LEN(rgns[i]); ++j) 1346 cnt++; 1347 } 1348 1349 return cnt; 1350 } 1351 1352 void *thread_proc(void *mem) 1353 { 1354 int *m = mem; 1355 long curr_faults, faults; 1356 struct rusage r; 1357 unsigned int i; 1358 int ret; 1359 1360 if (getrusage(RUSAGE_THREAD, &r)) 1361 ksft_exit_fail_msg("getrusage\n"); 1362 1363 curr_faults = r.ru_minflt; 1364 1365 while (!finish) { 1366 ret = pthread_barrier_wait(&start_barrier); 1367 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1368 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1369 1370 for (i = 0; i < access_per_thread; ++i) 1371 __atomic_add_fetch(m + i * (0x1000 / sizeof(*m)), 1, __ATOMIC_SEQ_CST); 1372 1373 ret = pthread_barrier_wait(&end_barrier); 1374 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1375 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1376 1377 if (getrusage(RUSAGE_THREAD, &r)) 1378 ksft_exit_fail_msg("getrusage\n"); 1379 1380 faults = r.ru_minflt - curr_faults; 1381 if (faults < access_per_thread) 1382 ksft_exit_fail_msg("faults < access_per_thread"); 1383 1384 __atomic_add_fetch(&extra_thread_faults, faults - access_per_thread, 1385 __ATOMIC_SEQ_CST); 1386 curr_faults = r.ru_minflt; 1387 } 1388 1389 return NULL; 1390 } 1391 1392 static void transact_test(int page_size) 1393 { 1394 unsigned int i, count, extra_pages; 1395 unsigned int c; 1396 pthread_t th; 1397 char *mem; 1398 int ret; 1399 1400 if (pthread_barrier_init(&start_barrier, NULL, nthreads + 1)) 1401 ksft_exit_fail_msg("pthread_barrier_init\n"); 1402 1403 if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1)) 1404 ksft_exit_fail_msg("pthread_barrier_init\n"); 1405 1406 mem = mmap(NULL, 0x1000 * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, 1407 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 1408 if (mem == MAP_FAILED) 1409 ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno)); 1410 1411 wp_init(mem, 0x1000 * nthreads * pages_per_thread); 1412 wp_addr_range(mem, 0x1000 * nthreads * pages_per_thread); 1413 1414 memset(mem, 0, 0x1000 * nthreads * pages_per_thread); 1415 1416 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1417 ksft_test_result(count > 0, "%s count %u\n", __func__, count); 1418 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1419 ksft_test_result(count == 0, "%s count %u\n", __func__, count); 1420 1421 finish = 0; 1422 for (i = 0; i < nthreads; ++i) 1423 pthread_create(&th, NULL, thread_proc, mem + 0x1000 * i * pages_per_thread); 1424 1425 extra_pages = 0; 1426 for (i = 0; i < iter_count; ++i) { 1427 count = 0; 1428 1429 ret = pthread_barrier_wait(&start_barrier); 1430 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1431 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1432 1433 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, 1434 page_size); 1435 1436 ret = pthread_barrier_wait(&end_barrier); 1437 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1438 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1439 1440 if (count > nthreads * access_per_thread) 1441 ksft_exit_fail_msg("Too big count %u expected %u, iter %u\n", 1442 count, nthreads * access_per_thread, i); 1443 1444 c = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1445 count += c; 1446 1447 if (c > nthreads * access_per_thread) { 1448 ksft_test_result_fail(" %s count > nthreads\n", __func__); 1449 return; 1450 } 1451 1452 if (count != nthreads * access_per_thread) { 1453 /* 1454 * The purpose of the test is to make sure that no page updates are lost 1455 * when the page updates and read-resetting soft dirty flags are performed 1456 * in parallel. However, it is possible that the application will get the 1457 * soft dirty flags twice on the two consecutive read-resets. This seems 1458 * unavoidable as soft dirty flag is handled in software through page faults 1459 * in kernel. While the updating the flags is supposed to be synchronized 1460 * between page fault handling and read-reset, it is possible that 1461 * read-reset happens after page fault PTE update but before the application 1462 * re-executes write instruction. So read-reset gets the flag, clears write 1463 * access and application gets page fault again for the same write. 1464 */ 1465 if (count < nthreads * access_per_thread) { 1466 ksft_test_result_fail("Lost update, iter %u, %u vs %u.\n", i, count, 1467 nthreads * access_per_thread); 1468 return; 1469 } 1470 1471 extra_pages += count - nthreads * access_per_thread; 1472 } 1473 } 1474 1475 pthread_barrier_wait(&start_barrier); 1476 finish = 1; 1477 pthread_barrier_wait(&end_barrier); 1478 1479 ksft_test_result_pass("%s Extra pages %u (%.1lf%%), extra thread faults %u.\n", __func__, 1480 extra_pages, 1481 100.0 * extra_pages / (iter_count * nthreads * access_per_thread), 1482 extra_thread_faults); 1483 } 1484 1485 void zeropfn_tests(void) 1486 { 1487 unsigned long long mem_size; 1488 struct page_region vec; 1489 int i, ret; 1490 char *mmap_mem, *mem; 1491 1492 /* Test with normal memory */ 1493 mem_size = 10 * page_size; 1494 mem = mmap(NULL, mem_size, PROT_READ, MAP_PRIVATE | MAP_ANON, -1, 0); 1495 if (mem == MAP_FAILED) 1496 ksft_exit_fail_msg("error nomem\n"); 1497 1498 /* Touch each page to ensure it's mapped */ 1499 for (i = 0; i < mem_size; i += page_size) 1500 (void)((volatile char *)mem)[i]; 1501 1502 ret = pagemap_ioctl(mem, mem_size, &vec, 1, 0, 1503 (mem_size / page_size), PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO); 1504 if (ret < 0) 1505 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1506 1507 ksft_test_result(ret == 1 && LEN(vec) == (mem_size / page_size), 1508 "%s all pages must have PFNZERO set\n", __func__); 1509 1510 munmap(mem, mem_size); 1511 1512 /* Test with huge page if user_zero_page is set to 1 */ 1513 if (!detect_huge_zeropage()) { 1514 ksft_test_result_skip("%s use_zero_page not supported or set to 1\n", __func__); 1515 return; 1516 } 1517 1518 mem_size = 2 * hpage_size; 1519 mmap_mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, 1520 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1521 if (mmap_mem == MAP_FAILED) 1522 ksft_exit_fail_msg("error nomem\n"); 1523 1524 /* We need a THP-aligned memory area. */ 1525 mem = (char *)(((uintptr_t)mmap_mem + hpage_size) & ~(hpage_size - 1)); 1526 1527 ret = madvise(mem, hpage_size, MADV_HUGEPAGE); 1528 if (!ret) { 1529 FORCE_READ(*mem); 1530 1531 ret = pagemap_ioctl(mem, hpage_size, &vec, 1, 0, 1532 0, PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO); 1533 if (ret < 0) 1534 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1535 1536 ksft_test_result(ret == 1 && LEN(vec) == (hpage_size / page_size), 1537 "%s all huge pages must have PFNZERO set\n", __func__); 1538 } else { 1539 ksft_test_result_skip("%s huge page not supported\n", __func__); 1540 } 1541 1542 munmap(mmap_mem, mem_size); 1543 } 1544 1545 int main(int __attribute__((unused)) argc, char *argv[]) 1546 { 1547 int shmid, buf_size, fd, i, ret; 1548 unsigned long long mem_size; 1549 char *mem, *map, *fmem; 1550 struct stat sbuf; 1551 1552 progname = argv[0]; 1553 1554 ksft_print_header(); 1555 1556 if (init_uffd()) 1557 ksft_exit_skip("Failed to initialize userfaultfd\n"); 1558 1559 if (!hugetlb_setup_default(4)) 1560 ksft_print_msg("HugeTLB test will be skipped\n"); 1561 1562 ksft_set_plan(117); 1563 1564 page_size = getpagesize(); 1565 hpage_size = read_pmd_pagesize(); 1566 1567 pagemap_fd = open(PAGEMAP, O_RDONLY); 1568 if (pagemap_fd < 0) 1569 ksft_exit_fail_msg("Failed to open " PAGEMAP "\n"); 1570 1571 /* 1. Sanity testing */ 1572 sanity_tests_sd(); 1573 1574 /* 2. Normal page testing */ 1575 mem_size = 10 * page_size; 1576 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1577 if (mem == MAP_FAILED) 1578 ksft_exit_fail_msg("error nomem\n"); 1579 wp_init(mem, mem_size); 1580 wp_addr_range(mem, mem_size); 1581 1582 base_tests("Page testing:", mem, mem_size, 0); 1583 1584 wp_free(mem, mem_size); 1585 munmap(mem, mem_size); 1586 1587 /* 3. Large page testing */ 1588 mem_size = 512 * 10 * page_size; 1589 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1590 if (mem == MAP_FAILED) 1591 ksft_exit_fail_msg("error nomem\n"); 1592 wp_init(mem, mem_size); 1593 wp_addr_range(mem, mem_size); 1594 1595 base_tests("Large Page testing:", mem, mem_size, 0); 1596 1597 wp_free(mem, mem_size); 1598 munmap(mem, mem_size); 1599 1600 /* 4. Huge page testing */ 1601 map = gethugepage(hpage_size); 1602 if (map) { 1603 wp_init(map, hpage_size); 1604 wp_addr_range(map, hpage_size); 1605 base_tests("Huge page testing:", map, hpage_size, 0); 1606 wp_free(map, hpage_size); 1607 free(map); 1608 } else { 1609 base_tests("Huge page testing:", NULL, 0, 1); 1610 } 1611 1612 /* 5. SHM Hugetlb page testing */ 1613 mem_size = default_huge_page_size(); 1614 mem = gethugetlb_mem(mem_size, &shmid); 1615 if (mem) { 1616 wp_init(mem, mem_size); 1617 wp_addr_range(mem, mem_size); 1618 1619 base_tests("Hugetlb shmem testing:", mem, mem_size, 0); 1620 1621 wp_free(mem, mem_size); 1622 shmctl(shmid, IPC_RMID, NULL); 1623 } else { 1624 base_tests("Hugetlb shmem testing:", NULL, 0, 1); 1625 } 1626 1627 /* 6. Hugetlb page testing */ 1628 mem = gethugetlb_mem(mem_size, NULL); 1629 if (mem) { 1630 wp_init(mem, mem_size); 1631 wp_addr_range(mem, mem_size); 1632 1633 base_tests("Hugetlb mem testing:", mem, mem_size, 0); 1634 1635 wp_free(mem, mem_size); 1636 } else { 1637 base_tests("Hugetlb mem testing:", NULL, 0, 1); 1638 } 1639 1640 /* 7. File Hugetlb testing */ 1641 mem_size = default_huge_page_size(); 1642 fd = memfd_create("uffd-test", MFD_HUGETLB | MFD_NOEXEC_SEAL); 1643 if (fd < 0) 1644 ksft_exit_fail_msg("uffd-test creation failed %d %s\n", errno, strerror(errno)); 1645 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 1646 if (mem != MAP_FAILED) { 1647 wp_init(mem, mem_size); 1648 wp_addr_range(mem, mem_size); 1649 1650 base_tests("Hugetlb shmem testing:", mem, mem_size, 0); 1651 1652 wp_free(mem, mem_size); 1653 shmctl(shmid, IPC_RMID, NULL); 1654 } else { 1655 base_tests("Hugetlb shmem testing:", NULL, 0, 1); 1656 } 1657 close(fd); 1658 1659 /* 8. File memory testing */ 1660 buf_size = page_size * 10; 1661 1662 fd = open(__FILE__".tmp0", O_RDWR | O_CREAT, 0777); 1663 if (fd < 0) 1664 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n", 1665 strerror(errno)); 1666 1667 for (i = 0; i < buf_size; i++) 1668 if (write(fd, "c", 1) < 0) 1669 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1670 1671 ret = stat(__FILE__".tmp0", &sbuf); 1672 if (ret < 0) 1673 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1674 1675 fmem = mmap(NULL, sbuf.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1676 if (fmem == MAP_FAILED) 1677 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1678 1679 wp_init(fmem, sbuf.st_size); 1680 wp_addr_range(fmem, sbuf.st_size); 1681 1682 base_tests("File memory testing:", fmem, sbuf.st_size, 0); 1683 1684 wp_free(fmem, sbuf.st_size); 1685 munmap(fmem, sbuf.st_size); 1686 close(fd); 1687 1688 /* 9. File memory testing */ 1689 buf_size = page_size * 10; 1690 1691 fd = memfd_create(__FILE__".tmp00", MFD_NOEXEC_SEAL); 1692 if (fd < 0) 1693 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n", 1694 strerror(errno)); 1695 1696 if (ftruncate(fd, buf_size)) 1697 ksft_exit_fail_msg("Error ftruncate\n"); 1698 1699 for (i = 0; i < buf_size; i++) 1700 if (write(fd, "c", 1) < 0) 1701 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1702 1703 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1704 if (fmem == MAP_FAILED) 1705 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1706 1707 wp_init(fmem, buf_size); 1708 wp_addr_range(fmem, buf_size); 1709 1710 base_tests("File anonymous memory testing:", fmem, buf_size, 0); 1711 1712 wp_free(fmem, buf_size); 1713 munmap(fmem, buf_size); 1714 close(fd); 1715 1716 /* 10. Huge page tests */ 1717 hpage_unit_tests(); 1718 1719 /* 11. Iterative test */ 1720 test_simple(); 1721 1722 /* 12. Mprotect test */ 1723 mprotect_tests(); 1724 1725 /* 13. Transact test */ 1726 transact_test(page_size); 1727 1728 /* 14. Sanity testing */ 1729 sanity_tests(); 1730 1731 /*15. Unmapped address test */ 1732 unmapped_region_tests(); 1733 1734 /* 16. Userfaultfd tests */ 1735 userfaultfd_tests(); 1736 1737 /* 17. ZEROPFN tests */ 1738 zeropfn_tests(); 1739 1740 close(pagemap_fd); 1741 ksft_finished(); 1742 } 1743