1 // SPDX-License-Identifier: GPL-2.0 2 #define _GNU_SOURCE 3 #include <stdio.h> 4 #include <fcntl.h> 5 #include <string.h> 6 #include <sys/mman.h> 7 #include <errno.h> 8 #include <malloc.h> 9 #include "vm_util.h" 10 #include "../kselftest.h" 11 #include <linux/types.h> 12 #include <linux/memfd.h> 13 #include <linux/userfaultfd.h> 14 #include <linux/fs.h> 15 #include <sys/ioctl.h> 16 #include <sys/stat.h> 17 #include <math.h> 18 #include <asm/unistd.h> 19 #include <pthread.h> 20 #include <sys/resource.h> 21 #include <assert.h> 22 #include <sys/ipc.h> 23 #include <sys/shm.h> 24 25 #define PAGEMAP_BITS_ALL (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ 26 PAGE_IS_FILE | PAGE_IS_PRESENT | \ 27 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ 28 PAGE_IS_HUGE) 29 #define PAGEMAP_NON_WRITTEN_BITS (PAGE_IS_WPALLOWED | PAGE_IS_FILE | \ 30 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | \ 31 PAGE_IS_PFNZERO | PAGE_IS_HUGE) 32 33 #define TEST_ITERATIONS 100 34 #define PAGEMAP "/proc/self/pagemap" 35 int pagemap_fd; 36 int uffd; 37 int page_size; 38 int hpage_size; 39 40 #define LEN(region) ((region.end - region.start)/page_size) 41 42 static long pagemap_ioctl(void *start, int len, void *vec, int vec_len, int flag, 43 int max_pages, long required_mask, long anyof_mask, long excluded_mask, 44 long return_mask) 45 { 46 struct pm_scan_arg arg; 47 48 arg.start = (uintptr_t)start; 49 arg.end = (uintptr_t)(start + len); 50 arg.vec = (uintptr_t)vec; 51 arg.vec_len = vec_len; 52 arg.flags = flag; 53 arg.size = sizeof(struct pm_scan_arg); 54 arg.max_pages = max_pages; 55 arg.category_mask = required_mask; 56 arg.category_anyof_mask = anyof_mask; 57 arg.category_inverted = excluded_mask; 58 arg.return_mask = return_mask; 59 60 return ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 61 } 62 63 static long pagemap_ioc(void *start, int len, void *vec, int vec_len, int flag, 64 int max_pages, long required_mask, long anyof_mask, long excluded_mask, 65 long return_mask, long *walk_end) 66 { 67 struct pm_scan_arg arg; 68 int ret; 69 70 arg.start = (uintptr_t)start; 71 arg.end = (uintptr_t)(start + len); 72 arg.vec = (uintptr_t)vec; 73 arg.vec_len = vec_len; 74 arg.flags = flag; 75 arg.size = sizeof(struct pm_scan_arg); 76 arg.max_pages = max_pages; 77 arg.category_mask = required_mask; 78 arg.category_anyof_mask = anyof_mask; 79 arg.category_inverted = excluded_mask; 80 arg.return_mask = return_mask; 81 82 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 83 84 if (walk_end) 85 *walk_end = arg.walk_end; 86 87 return ret; 88 } 89 90 91 int init_uffd(void) 92 { 93 struct uffdio_api uffdio_api; 94 95 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); 96 if (uffd == -1) 97 return uffd; 98 99 uffdio_api.api = UFFD_API; 100 uffdio_api.features = UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC | 101 UFFD_FEATURE_WP_HUGETLBFS_SHMEM; 102 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) 103 return -1; 104 105 if (!(uffdio_api.api & UFFDIO_REGISTER_MODE_WP) || 106 !(uffdio_api.features & UFFD_FEATURE_WP_UNPOPULATED) || 107 !(uffdio_api.features & UFFD_FEATURE_WP_ASYNC) || 108 !(uffdio_api.features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM)) 109 return -1; 110 111 return 0; 112 } 113 114 int wp_init(void *lpBaseAddress, int dwRegionSize) 115 { 116 struct uffdio_register uffdio_register; 117 struct uffdio_writeprotect wp; 118 119 uffdio_register.range.start = (unsigned long)lpBaseAddress; 120 uffdio_register.range.len = dwRegionSize; 121 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; 122 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) 123 ksft_exit_fail_msg("ioctl(UFFDIO_REGISTER) %d %s\n", errno, strerror(errno)); 124 125 if (!(uffdio_register.ioctls & UFFDIO_WRITEPROTECT)) 126 ksft_exit_fail_msg("ioctl set is incorrect\n"); 127 128 wp.range.start = (unsigned long)lpBaseAddress; 129 wp.range.len = dwRegionSize; 130 wp.mode = UFFDIO_WRITEPROTECT_MODE_WP; 131 132 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &wp)) 133 ksft_exit_fail_msg("ioctl(UFFDIO_WRITEPROTECT)\n"); 134 135 return 0; 136 } 137 138 int wp_free(void *lpBaseAddress, int dwRegionSize) 139 { 140 struct uffdio_register uffdio_register; 141 142 uffdio_register.range.start = (unsigned long)lpBaseAddress; 143 uffdio_register.range.len = dwRegionSize; 144 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; 145 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range)) 146 ksft_exit_fail_msg("ioctl unregister failure\n"); 147 return 0; 148 } 149 150 int wp_addr_range(void *lpBaseAddress, int dwRegionSize) 151 { 152 if (pagemap_ioctl(lpBaseAddress, dwRegionSize, NULL, 0, 153 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 154 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0) 155 ksft_exit_fail_msg("error %d %d %s\n", 1, errno, strerror(errno)); 156 157 return 0; 158 } 159 160 void *gethugetlb_mem(int size, int *shmid) 161 { 162 char *mem; 163 164 if (shmid) { 165 *shmid = shmget(2, size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); 166 if (*shmid < 0) 167 return NULL; 168 169 mem = shmat(*shmid, 0, 0); 170 if (mem == (char *)-1) { 171 shmctl(*shmid, IPC_RMID, NULL); 172 ksft_exit_fail_msg("Shared memory attach failure\n"); 173 } 174 } else { 175 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, 176 MAP_ANONYMOUS | MAP_HUGETLB | MAP_PRIVATE, -1, 0); 177 if (mem == MAP_FAILED) 178 return NULL; 179 } 180 181 return mem; 182 } 183 184 int userfaultfd_tests(void) 185 { 186 int mem_size, vec_size, written, num_pages = 16; 187 char *mem, *vec; 188 189 mem_size = num_pages * page_size; 190 mem = mmap(NULL, mem_size, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0); 191 if (mem == MAP_FAILED) 192 ksft_exit_fail_msg("error nomem\n"); 193 194 wp_init(mem, mem_size); 195 196 /* Change protection of pages differently */ 197 mprotect(mem, mem_size/8, PROT_READ|PROT_WRITE); 198 mprotect(mem + 1 * mem_size/8, mem_size/8, PROT_READ); 199 mprotect(mem + 2 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 200 mprotect(mem + 3 * mem_size/8, mem_size/8, PROT_READ); 201 mprotect(mem + 4 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 202 mprotect(mem + 5 * mem_size/8, mem_size/8, PROT_NONE); 203 mprotect(mem + 6 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); 204 mprotect(mem + 7 * mem_size/8, mem_size/8, PROT_READ); 205 206 wp_addr_range(mem + (mem_size/16), mem_size - 2 * (mem_size/8)); 207 wp_addr_range(mem, mem_size); 208 209 vec_size = mem_size/page_size; 210 vec = malloc(sizeof(struct page_region) * vec_size); 211 212 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 213 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 214 if (written < 0) 215 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 216 217 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", __func__); 218 219 wp_free(mem, mem_size); 220 munmap(mem, mem_size); 221 free(vec); 222 return 0; 223 } 224 225 int get_reads(struct page_region *vec, int vec_size) 226 { 227 int i, sum = 0; 228 229 for (i = 0; i < vec_size; i++) 230 sum += LEN(vec[i]); 231 232 return sum; 233 } 234 235 int sanity_tests_sd(void) 236 { 237 int mem_size, vec_size, ret, ret2, ret3, i, num_pages = 1000, total_pages = 0; 238 int total_writes, total_reads, reads, count; 239 struct page_region *vec, *vec2; 240 char *mem, *m[2]; 241 long walk_end; 242 243 vec_size = num_pages/2; 244 mem_size = num_pages * page_size; 245 246 vec = malloc(sizeof(struct page_region) * vec_size); 247 if (!vec) 248 ksft_exit_fail_msg("error nomem\n"); 249 250 vec2 = malloc(sizeof(struct page_region) * vec_size); 251 if (!vec2) 252 ksft_exit_fail_msg("error nomem\n"); 253 254 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 255 if (mem == MAP_FAILED) 256 ksft_exit_fail_msg("error nomem\n"); 257 258 wp_init(mem, mem_size); 259 wp_addr_range(mem, mem_size); 260 261 /* 1. wrong operation */ 262 ksft_test_result(pagemap_ioctl(mem, 0, vec, vec_size, 0, 263 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 264 "%s Zero range size is valid\n", __func__); 265 266 ksft_test_result(pagemap_ioctl(mem, mem_size, NULL, vec_size, 0, 267 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) < 0, 268 "%s output buffer must be specified with size\n", __func__); 269 270 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, 0, 0, 271 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 272 "%s output buffer can be 0\n", __func__); 273 274 ksft_test_result(pagemap_ioctl(mem, mem_size, 0, 0, 0, 275 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, 276 "%s output buffer can be 0\n", __func__); 277 278 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, -1, 279 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0, 280 "%s wrong flag specified\n", __func__); 281 282 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 283 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC | 0xFF, 284 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0, 285 "%s flag has extra bits specified\n", __func__); 286 287 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 288 0, 0, 0, 0, PAGE_IS_WRITTEN) >= 0, 289 "%s no selection mask is specified\n", __func__); 290 291 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 292 0, PAGE_IS_WRITTEN, PAGE_IS_WRITTEN, 0, 0) == 0, 293 "%s no return mask is specified\n", __func__); 294 295 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 296 0, PAGE_IS_WRITTEN, 0, 0, 0x1000) < 0, 297 "%s wrong return mask specified\n", __func__); 298 299 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 300 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 301 0, 0xFFF, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN) < 0, 302 "%s mixture of correct and wrong flag\n", __func__); 303 304 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 305 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 306 0, 0, 0, PAGEMAP_BITS_ALL, PAGE_IS_WRITTEN) >= 0, 307 "%s PAGEMAP_BITS_ALL can be specified with PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", 308 __func__); 309 310 /* 2. Clear area with larger vec size */ 311 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 312 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 313 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 314 ksft_test_result(ret >= 0, "%s Clear area with larger vec size\n", __func__); 315 316 /* 3. Repeated pattern of written and non-written pages */ 317 for (i = 0; i < mem_size; i += 2 * page_size) 318 mem[i]++; 319 320 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 0, 321 0, PAGE_IS_WRITTEN); 322 if (ret < 0) 323 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 324 325 ksft_test_result(ret == mem_size/(page_size * 2), 326 "%s Repeated pattern of written and non-written pages\n", __func__); 327 328 /* 4. Repeated pattern of written and non-written pages in parts */ 329 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 330 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 331 num_pages/2 - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 332 if (ret < 0) 333 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 334 335 ret2 = pagemap_ioctl(mem, mem_size, vec, 2, 0, 0, PAGE_IS_WRITTEN, 0, 0, 336 PAGE_IS_WRITTEN); 337 if (ret2 < 0) 338 ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno)); 339 340 ret3 = pagemap_ioctl(mem, mem_size, vec, vec_size, 341 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 342 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 343 if (ret3 < 0) 344 ksft_exit_fail_msg("error %d %d %s\n", ret3, errno, strerror(errno)); 345 346 ksft_test_result((ret + ret3) == num_pages/2 && ret2 == 2, 347 "%s Repeated pattern of written and non-written pages in parts %d %d %d\n", 348 __func__, ret, ret3, ret2); 349 350 /* 5. Repeated pattern of written and non-written pages max_pages */ 351 for (i = 0; i < mem_size; i += 2 * page_size) 352 mem[i]++; 353 mem[(mem_size/page_size - 1) * page_size]++; 354 355 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 356 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 357 num_pages/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 358 if (ret < 0) 359 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 360 361 ret2 = pagemap_ioctl(mem, mem_size, vec, vec_size, 362 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 363 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 364 if (ret2 < 0) 365 ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno)); 366 367 ksft_test_result(ret == num_pages/2 && ret2 == 1, 368 "%s Repeated pattern of written and non-written pages max_pages\n", 369 __func__); 370 371 /* 6. only get 2 dirty pages and clear them as well */ 372 vec_size = mem_size/page_size; 373 memset(mem, -1, mem_size); 374 375 /* get and clear second and third pages */ 376 ret = pagemap_ioctl(mem + page_size, 2 * page_size, vec, 1, 377 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 378 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 379 if (ret < 0) 380 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 381 382 ret2 = pagemap_ioctl(mem, mem_size, vec2, vec_size, 0, 0, 383 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 384 if (ret2 < 0) 385 ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno)); 386 387 ksft_test_result(ret == 1 && LEN(vec[0]) == 2 && 388 vec[0].start == (uintptr_t)(mem + page_size) && 389 ret2 == 2 && LEN(vec2[0]) == 1 && vec2[0].start == (uintptr_t)mem && 390 LEN(vec2[1]) == vec_size - 3 && 391 vec2[1].start == (uintptr_t)(mem + 3 * page_size), 392 "%s only get 2 written pages and clear them as well\n", __func__); 393 394 wp_free(mem, mem_size); 395 munmap(mem, mem_size); 396 397 /* 7. Two regions */ 398 m[0] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 399 if (m[0] == MAP_FAILED) 400 ksft_exit_fail_msg("error nomem\n"); 401 m[1] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 402 if (m[1] == MAP_FAILED) 403 ksft_exit_fail_msg("error nomem\n"); 404 405 wp_init(m[0], mem_size); 406 wp_init(m[1], mem_size); 407 wp_addr_range(m[0], mem_size); 408 wp_addr_range(m[1], mem_size); 409 410 memset(m[0], 'a', mem_size); 411 memset(m[1], 'b', mem_size); 412 413 wp_addr_range(m[0], mem_size); 414 415 ret = pagemap_ioctl(m[1], mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0, 416 PAGE_IS_WRITTEN); 417 if (ret < 0) 418 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 419 420 ksft_test_result(ret == 1 && LEN(vec[0]) == mem_size/page_size, 421 "%s Two regions\n", __func__); 422 423 wp_free(m[0], mem_size); 424 wp_free(m[1], mem_size); 425 munmap(m[0], mem_size); 426 munmap(m[1], mem_size); 427 428 free(vec); 429 free(vec2); 430 431 /* 8. Smaller vec */ 432 mem_size = 1050 * page_size; 433 vec_size = mem_size/(page_size*2); 434 435 vec = malloc(sizeof(struct page_region) * vec_size); 436 if (!vec) 437 ksft_exit_fail_msg("error nomem\n"); 438 439 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 440 if (mem == MAP_FAILED) 441 ksft_exit_fail_msg("error nomem\n"); 442 443 wp_init(mem, mem_size); 444 wp_addr_range(mem, mem_size); 445 446 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 447 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 448 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 449 if (ret < 0) 450 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 451 452 for (i = 0; i < mem_size/page_size; i += 2) 453 mem[i * page_size]++; 454 455 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 456 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 457 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 458 if (ret < 0) 459 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 460 461 total_pages += ret; 462 463 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 464 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 465 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 466 if (ret < 0) 467 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 468 469 total_pages += ret; 470 471 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 472 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 473 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 474 if (ret < 0) 475 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 476 477 total_pages += ret; 478 479 ksft_test_result(total_pages == mem_size/(page_size*2), "%s Smaller max_pages\n", __func__); 480 481 free(vec); 482 wp_free(mem, mem_size); 483 munmap(mem, mem_size); 484 total_pages = 0; 485 486 /* 9. Smaller vec */ 487 mem_size = 10000 * page_size; 488 vec_size = 50; 489 490 vec = malloc(sizeof(struct page_region) * vec_size); 491 if (!vec) 492 ksft_exit_fail_msg("error nomem\n"); 493 494 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 495 if (mem == MAP_FAILED) 496 ksft_exit_fail_msg("error nomem\n"); 497 498 wp_init(mem, mem_size); 499 wp_addr_range(mem, mem_size); 500 501 for (count = 0; count < TEST_ITERATIONS; count++) { 502 total_writes = total_reads = 0; 503 walk_end = (long)mem; 504 505 for (i = 0; i < mem_size; i += page_size) { 506 if (rand() % 2) { 507 mem[i]++; 508 total_writes++; 509 } 510 } 511 512 while (total_reads < total_writes) { 513 ret = pagemap_ioc((void *)walk_end, mem_size-(walk_end - (long)mem), vec, 514 vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 515 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 516 if (ret < 0) 517 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 518 519 if (ret > vec_size) 520 break; 521 522 reads = get_reads(vec, ret); 523 total_reads += reads; 524 } 525 526 if (total_reads != total_writes) 527 break; 528 } 529 530 ksft_test_result(count == TEST_ITERATIONS, "Smaller vec\n"); 531 532 free(vec); 533 wp_free(mem, mem_size); 534 munmap(mem, mem_size); 535 536 /* 10. Walk_end tester */ 537 vec_size = 1000; 538 mem_size = vec_size * page_size; 539 540 vec = malloc(sizeof(struct page_region) * vec_size); 541 if (!vec) 542 ksft_exit_fail_msg("error nomem\n"); 543 544 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 545 if (mem == MAP_FAILED) 546 ksft_exit_fail_msg("error nomem\n"); 547 548 wp_init(mem, mem_size); 549 wp_addr_range(mem, mem_size); 550 551 memset(mem, 0, mem_size); 552 553 ret = pagemap_ioc(mem, 0, vec, vec_size, 0, 554 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 555 if (ret < 0) 556 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 557 ksft_test_result(ret == 0 && walk_end == (long)mem, 558 "Walk_end: Same start and end address\n"); 559 560 ret = pagemap_ioc(mem, 0, vec, vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 561 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 562 if (ret < 0) 563 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 564 ksft_test_result(ret == 0 && walk_end == (long)mem, 565 "Walk_end: Same start and end with WP\n"); 566 567 ret = pagemap_ioc(mem, 0, vec, 0, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 568 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 569 if (ret < 0) 570 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 571 ksft_test_result(ret == 0 && walk_end == (long)mem, 572 "Walk_end: Same start and end with 0 output buffer\n"); 573 574 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 575 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 576 if (ret < 0) 577 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 578 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 579 "Walk_end: Big vec\n"); 580 581 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 582 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 583 if (ret < 0) 584 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 585 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 586 "Walk_end: vec of minimum length\n"); 587 588 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 589 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 590 if (ret < 0) 591 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 592 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 593 "Walk_end: Max pages specified\n"); 594 595 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 596 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 597 if (ret < 0) 598 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 599 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size/2), 600 "Walk_end: Half max pages\n"); 601 602 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 603 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 604 if (ret < 0) 605 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 606 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size), 607 "Walk_end: 1 max page\n"); 608 609 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 610 -1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 611 if (ret < 0) 612 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 613 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), 614 "Walk_end: max pages\n"); 615 616 wp_addr_range(mem, mem_size); 617 for (i = 0; i < mem_size; i += 2 * page_size) 618 mem[i]++; 619 620 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 621 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 622 if (ret < 0) 623 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 624 ksft_test_result(ret == vec_size/2 && walk_end == (long)(mem + mem_size), 625 "Walk_end sparse: Big vec\n"); 626 627 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 628 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 629 if (ret < 0) 630 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 631 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 632 "Walk_end sparse: vec of minimum length\n"); 633 634 ret = pagemap_ioc(mem, mem_size, vec, 1, 0, 635 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 636 if (ret < 0) 637 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 638 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 639 "Walk_end sparse: Max pages specified\n"); 640 641 ret = pagemap_ioc(mem, mem_size, vec, vec_size/2, 0, 642 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 643 if (ret < 0) 644 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 645 ksft_test_result(ret == vec_size/2 && walk_end == (long)(mem + mem_size), 646 "Walk_end sparse: Max pages specified\n"); 647 648 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 649 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 650 if (ret < 0) 651 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 652 ksft_test_result(ret == vec_size/2 && walk_end == (long)(mem + mem_size), 653 "Walk_end sparse: Max pages specified\n"); 654 655 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 656 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 657 if (ret < 0) 658 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 659 ksft_test_result(ret == vec_size/2 && walk_end == (long)(mem + mem_size), 660 "Walk_endsparse : Half max pages\n"); 661 662 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, 663 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); 664 if (ret < 0) 665 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 666 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), 667 "Walk_end: 1 max page\n"); 668 669 free(vec); 670 wp_free(mem, mem_size); 671 munmap(mem, mem_size); 672 673 return 0; 674 } 675 676 int base_tests(char *prefix, char *mem, int mem_size, int skip) 677 { 678 int vec_size, written; 679 struct page_region *vec, *vec2; 680 681 if (skip) { 682 ksft_test_result_skip("%s all new pages must not be written (dirty)\n", prefix); 683 ksft_test_result_skip("%s all pages must be written (dirty)\n", prefix); 684 ksft_test_result_skip("%s all pages dirty other than first and the last one\n", 685 prefix); 686 ksft_test_result_skip("%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix); 687 ksft_test_result_skip("%s only middle page dirty\n", prefix); 688 ksft_test_result_skip("%s only two middle pages dirty\n", prefix); 689 return 0; 690 } 691 692 vec_size = mem_size/page_size; 693 vec = malloc(sizeof(struct page_region) * vec_size); 694 vec2 = malloc(sizeof(struct page_region) * vec_size); 695 696 /* 1. all new pages must be not be written (dirty) */ 697 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 698 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 699 if (written < 0) 700 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 701 702 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", prefix); 703 704 /* 2. all pages must be written */ 705 memset(mem, -1, mem_size); 706 707 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0, 708 PAGE_IS_WRITTEN); 709 if (written < 0) 710 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 711 712 ksft_test_result(written == 1 && LEN(vec[0]) == mem_size/page_size, 713 "%s all pages must be written (dirty)\n", prefix); 714 715 /* 3. all pages dirty other than first and the last one */ 716 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 717 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 718 if (written < 0) 719 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 720 721 memset(mem + page_size, 0, mem_size - (2 * page_size)); 722 723 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 724 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 725 if (written < 0) 726 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 727 728 ksft_test_result(written == 1 && LEN(vec[0]) >= vec_size - 2 && LEN(vec[0]) <= vec_size, 729 "%s all pages dirty other than first and the last one\n", prefix); 730 731 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, 732 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 733 if (written < 0) 734 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 735 736 ksft_test_result(written == 0, 737 "%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix); 738 739 /* 4. only middle page dirty */ 740 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 741 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 742 if (written < 0) 743 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 744 745 mem[vec_size/2 * page_size]++; 746 747 written = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 748 0, 0, PAGE_IS_WRITTEN); 749 if (written < 0) 750 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 751 752 ksft_test_result(written == 1 && LEN(vec[0]) >= 1, 753 "%s only middle page dirty\n", prefix); 754 755 /* 5. only two middle pages dirty and walk over only middle pages */ 756 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 757 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE); 758 if (written < 0) 759 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 760 761 mem[vec_size/2 * page_size]++; 762 mem[(vec_size/2 + 1) * page_size]++; 763 764 written = pagemap_ioctl(&mem[vec_size/2 * page_size], 2 * page_size, vec, 1, 0, 765 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE); 766 if (written < 0) 767 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 768 769 ksft_test_result(written == 1 && vec[0].start == (uintptr_t)(&mem[vec_size/2 * page_size]) 770 && LEN(vec[0]) == 2, 771 "%s only two middle pages dirty\n", prefix); 772 773 free(vec); 774 free(vec2); 775 return 0; 776 } 777 778 void *gethugepage(int map_size) 779 { 780 int ret; 781 char *map; 782 783 map = memalign(hpage_size, map_size); 784 if (!map) 785 ksft_exit_fail_msg("memalign failed %d %s\n", errno, strerror(errno)); 786 787 ret = madvise(map, map_size, MADV_HUGEPAGE); 788 if (ret) 789 return NULL; 790 791 memset(map, 0, map_size); 792 793 return map; 794 } 795 796 int hpage_unit_tests(void) 797 { 798 char *map; 799 int ret, ret2; 800 size_t num_pages = 10; 801 int map_size = hpage_size * num_pages; 802 int vec_size = map_size/page_size; 803 struct page_region *vec, *vec2; 804 805 vec = malloc(sizeof(struct page_region) * vec_size); 806 vec2 = malloc(sizeof(struct page_region) * vec_size); 807 if (!vec || !vec2) 808 ksft_exit_fail_msg("malloc failed\n"); 809 810 map = gethugepage(map_size); 811 if (map) { 812 wp_init(map, map_size); 813 wp_addr_range(map, map_size); 814 815 /* 1. all new huge page must not be written (dirty) */ 816 ret = pagemap_ioctl(map, map_size, vec, vec_size, 817 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 818 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 819 if (ret < 0) 820 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 821 822 ksft_test_result(ret == 0, "%s all new huge page must not be written (dirty)\n", 823 __func__); 824 825 /* 2. all the huge page must not be written */ 826 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 827 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 828 if (ret < 0) 829 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 830 831 ksft_test_result(ret == 0, "%s all the huge page must not be written\n", __func__); 832 833 /* 3. all the huge page must be written and clear dirty as well */ 834 memset(map, -1, map_size); 835 ret = pagemap_ioctl(map, map_size, vec, vec_size, 836 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 837 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 838 if (ret < 0) 839 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 840 841 ksft_test_result(ret == 1 && vec[0].start == (uintptr_t)map && 842 LEN(vec[0]) == vec_size && vec[0].categories == PAGE_IS_WRITTEN, 843 "%s all the huge page must be written and clear\n", __func__); 844 845 /* 4. only middle page written */ 846 wp_free(map, map_size); 847 free(map); 848 map = gethugepage(map_size); 849 wp_init(map, map_size); 850 wp_addr_range(map, map_size); 851 map[vec_size/2 * page_size]++; 852 853 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 854 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 855 if (ret < 0) 856 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 857 858 ksft_test_result(ret == 1 && LEN(vec[0]) > 0, 859 "%s only middle page written\n", __func__); 860 861 wp_free(map, map_size); 862 free(map); 863 } else { 864 ksft_test_result_skip("%s all new huge page must be written\n", __func__); 865 ksft_test_result_skip("%s all the huge page must not be written\n", __func__); 866 ksft_test_result_skip("%s all the huge page must be written and clear\n", __func__); 867 ksft_test_result_skip("%s only middle page written\n", __func__); 868 } 869 870 /* 5. clear first half of huge page */ 871 map = gethugepage(map_size); 872 if (map) { 873 wp_init(map, map_size); 874 wp_addr_range(map, map_size); 875 876 memset(map, 0, map_size); 877 878 wp_addr_range(map, map_size/2); 879 880 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 881 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 882 if (ret < 0) 883 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 884 885 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 && 886 vec[0].start == (uintptr_t)(map + map_size/2), 887 "%s clear first half of huge page\n", __func__); 888 wp_free(map, map_size); 889 free(map); 890 } else { 891 ksft_test_result_skip("%s clear first half of huge page\n", __func__); 892 } 893 894 /* 6. clear first half of huge page with limited buffer */ 895 map = gethugepage(map_size); 896 if (map) { 897 wp_init(map, map_size); 898 wp_addr_range(map, map_size); 899 900 memset(map, 0, map_size); 901 902 ret = pagemap_ioctl(map, map_size, vec, vec_size, 903 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 904 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 905 if (ret < 0) 906 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 907 908 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 909 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 910 if (ret < 0) 911 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 912 913 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 && 914 vec[0].start == (uintptr_t)(map + map_size/2), 915 "%s clear first half of huge page with limited buffer\n", 916 __func__); 917 wp_free(map, map_size); 918 free(map); 919 } else { 920 ksft_test_result_skip("%s clear first half of huge page with limited buffer\n", 921 __func__); 922 } 923 924 /* 7. clear second half of huge page */ 925 map = gethugepage(map_size); 926 if (map) { 927 wp_init(map, map_size); 928 wp_addr_range(map, map_size); 929 930 memset(map, -1, map_size); 931 932 ret = pagemap_ioctl(map + map_size/2, map_size/2, vec, vec_size, 933 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, vec_size/2, 934 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 935 if (ret < 0) 936 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 937 938 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 939 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 940 if (ret < 0) 941 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 942 943 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2, 944 "%s clear second half huge page\n", __func__); 945 wp_free(map, map_size); 946 free(map); 947 } else { 948 ksft_test_result_skip("%s clear second half huge page\n", __func__); 949 } 950 951 /* 8. get half huge page */ 952 map = gethugepage(map_size); 953 if (map) { 954 wp_init(map, map_size); 955 wp_addr_range(map, map_size); 956 957 memset(map, -1, map_size); 958 usleep(100); 959 960 ret = pagemap_ioctl(map, map_size, vec, 1, 961 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 962 hpage_size/(2*page_size), PAGE_IS_WRITTEN, 0, 0, 963 PAGE_IS_WRITTEN); 964 if (ret < 0) 965 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 966 967 ksft_test_result(ret == 1 && LEN(vec[0]) == hpage_size/(2*page_size), 968 "%s get half huge page\n", __func__); 969 970 ret2 = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, 971 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); 972 if (ret2 < 0) 973 ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno)); 974 975 ksft_test_result(ret2 == 1 && LEN(vec[0]) == (map_size - hpage_size/2)/page_size, 976 "%s get half huge page\n", __func__); 977 978 wp_free(map, map_size); 979 free(map); 980 } else { 981 ksft_test_result_skip("%s get half huge page\n", __func__); 982 ksft_test_result_skip("%s get half huge page\n", __func__); 983 } 984 985 free(vec); 986 free(vec2); 987 return 0; 988 } 989 990 int unmapped_region_tests(void) 991 { 992 void *start = (void *)0x10000000; 993 int written, len = 0x00040000; 994 int vec_size = len / page_size; 995 struct page_region *vec = malloc(sizeof(struct page_region) * vec_size); 996 997 /* 1. Get written pages */ 998 written = pagemap_ioctl(start, len, vec, vec_size, 0, 0, 999 PAGEMAP_NON_WRITTEN_BITS, 0, 0, PAGEMAP_NON_WRITTEN_BITS); 1000 if (written < 0) 1001 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); 1002 1003 ksft_test_result(written >= 0, "%s Get status of pages\n", __func__); 1004 1005 free(vec); 1006 return 0; 1007 } 1008 1009 static void test_simple(void) 1010 { 1011 int i; 1012 char *map; 1013 struct page_region vec; 1014 1015 map = aligned_alloc(page_size, page_size); 1016 if (!map) 1017 ksft_exit_fail_msg("aligned_alloc failed\n"); 1018 1019 wp_init(map, page_size); 1020 wp_addr_range(map, page_size); 1021 1022 for (i = 0 ; i < TEST_ITERATIONS; i++) { 1023 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0, 1024 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 1) { 1025 ksft_print_msg("written bit was 1, but should be 0 (i=%d)\n", i); 1026 break; 1027 } 1028 1029 wp_addr_range(map, page_size); 1030 /* Write something to the page to get the written bit enabled on the page */ 1031 map[0]++; 1032 1033 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0, 1034 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0) { 1035 ksft_print_msg("written bit was 0, but should be 1 (i=%d)\n", i); 1036 break; 1037 } 1038 1039 wp_addr_range(map, page_size); 1040 } 1041 wp_free(map, page_size); 1042 free(map); 1043 1044 ksft_test_result(i == TEST_ITERATIONS, "Test %s\n", __func__); 1045 } 1046 1047 int sanity_tests(void) 1048 { 1049 int mem_size, vec_size, ret, fd, i, buf_size; 1050 struct page_region *vec; 1051 char *mem, *fmem; 1052 struct stat sbuf; 1053 char *tmp_buf; 1054 1055 /* 1. wrong operation */ 1056 mem_size = 10 * page_size; 1057 vec_size = mem_size / page_size; 1058 1059 vec = malloc(sizeof(struct page_region) * vec_size); 1060 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1061 if (mem == MAP_FAILED || vec == MAP_FAILED) 1062 ksft_exit_fail_msg("error nomem\n"); 1063 1064 wp_init(mem, mem_size); 1065 wp_addr_range(mem, mem_size); 1066 1067 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 1068 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 1069 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0, 1070 "%s WP op can be specified with !PAGE_IS_WRITTEN\n", __func__); 1071 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1072 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0, 1073 "%s required_mask specified\n", __func__); 1074 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1075 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL) >= 0, 1076 "%s anyof_mask specified\n", __func__); 1077 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1078 0, 0, PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL) >= 0, 1079 "%s excluded_mask specified\n", __func__); 1080 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1081 PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL, 0, 1082 PAGEMAP_BITS_ALL) >= 0, 1083 "%s required_mask and anyof_mask specified\n", __func__); 1084 wp_free(mem, mem_size); 1085 munmap(mem, mem_size); 1086 1087 /* 2. Get sd and present pages with anyof_mask */ 1088 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1089 if (mem == MAP_FAILED) 1090 ksft_exit_fail_msg("error nomem\n"); 1091 wp_init(mem, mem_size); 1092 wp_addr_range(mem, mem_size); 1093 1094 memset(mem, 0, mem_size); 1095 1096 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1097 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL); 1098 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1099 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1100 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1101 "%s Get sd and present pages with anyof_mask\n", __func__); 1102 1103 /* 3. Get sd and present pages with required_mask */ 1104 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1105 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL); 1106 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1107 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1108 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1109 "%s Get all the pages with required_mask\n", __func__); 1110 1111 /* 4. Get sd and present pages with required_mask and anyof_mask */ 1112 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1113 PAGE_IS_WRITTEN, PAGE_IS_PRESENT, 0, PAGEMAP_BITS_ALL); 1114 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1115 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == 1116 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), 1117 "%s Get sd and present pages with required_mask and anyof_mask\n", 1118 __func__); 1119 1120 /* 5. Don't get sd pages */ 1121 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1122 PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN, PAGEMAP_BITS_ALL); 1123 ksft_test_result(ret == 0, "%s Don't get sd pages\n", __func__); 1124 1125 /* 6. Don't get present pages */ 1126 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, 1127 PAGE_IS_PRESENT, 0, PAGE_IS_PRESENT, PAGEMAP_BITS_ALL); 1128 ksft_test_result(ret == 0, "%s Don't get present pages\n", __func__); 1129 1130 wp_free(mem, mem_size); 1131 munmap(mem, mem_size); 1132 1133 /* 8. Find written present pages with return mask */ 1134 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1135 if (mem == MAP_FAILED) 1136 ksft_exit_fail_msg("error nomem\n"); 1137 wp_init(mem, mem_size); 1138 wp_addr_range(mem, mem_size); 1139 1140 memset(mem, 0, mem_size); 1141 1142 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 1143 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, 1144 0, PAGEMAP_BITS_ALL, 0, PAGE_IS_WRITTEN); 1145 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && 1146 vec[0].categories == PAGE_IS_WRITTEN, 1147 "%s Find written present pages with return mask\n", __func__); 1148 wp_free(mem, mem_size); 1149 munmap(mem, mem_size); 1150 1151 /* 9. Memory mapped file */ 1152 fd = open(__FILE__, O_RDONLY); 1153 if (fd < 0) 1154 ksft_exit_fail_msg("%s Memory mapped file\n", __func__); 1155 1156 ret = stat(__FILE__, &sbuf); 1157 if (ret < 0) 1158 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1159 1160 fmem = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0); 1161 if (fmem == MAP_FAILED) 1162 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1163 1164 tmp_buf = malloc(sbuf.st_size); 1165 memcpy(tmp_buf, fmem, sbuf.st_size); 1166 1167 ret = pagemap_ioctl(fmem, sbuf.st_size, vec, vec_size, 0, 0, 1168 0, PAGEMAP_NON_WRITTEN_BITS, 0, PAGEMAP_NON_WRITTEN_BITS); 1169 1170 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem && 1171 LEN(vec[0]) == ceilf((float)sbuf.st_size/page_size) && 1172 (vec[0].categories & PAGE_IS_FILE), 1173 "%s Memory mapped file\n", __func__); 1174 1175 munmap(fmem, sbuf.st_size); 1176 close(fd); 1177 1178 /* 10. Create and read/write to a memory mapped file */ 1179 buf_size = page_size * 10; 1180 1181 fd = open(__FILE__".tmp2", O_RDWR | O_CREAT, 0666); 1182 if (fd < 0) 1183 ksft_exit_fail_msg("Read/write to memory: %s\n", 1184 strerror(errno)); 1185 1186 for (i = 0; i < buf_size; i++) 1187 if (write(fd, "c", 1) < 0) 1188 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1189 1190 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1191 if (fmem == MAP_FAILED) 1192 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1193 1194 wp_init(fmem, buf_size); 1195 wp_addr_range(fmem, buf_size); 1196 1197 for (i = 0; i < buf_size; i++) 1198 fmem[i] = 'z'; 1199 1200 msync(fmem, buf_size, MS_SYNC); 1201 1202 ret = pagemap_ioctl(fmem, buf_size, vec, vec_size, 0, 0, 1203 PAGE_IS_WRITTEN, PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_FILE, 0, 1204 PAGEMAP_BITS_ALL); 1205 1206 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem && 1207 LEN(vec[0]) == (buf_size/page_size) && 1208 (vec[0].categories & PAGE_IS_WRITTEN), 1209 "%s Read/write to memory\n", __func__); 1210 1211 wp_free(fmem, buf_size); 1212 munmap(fmem, buf_size); 1213 close(fd); 1214 1215 free(vec); 1216 return 0; 1217 } 1218 1219 int mprotect_tests(void) 1220 { 1221 int ret; 1222 char *mem, *mem2; 1223 struct page_region vec; 1224 int pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 1225 1226 if (pagemap_fd < 0) { 1227 fprintf(stderr, "open() failed\n"); 1228 exit(1); 1229 } 1230 1231 /* 1. Map two pages */ 1232 mem = mmap(0, 2 * page_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1233 if (mem == MAP_FAILED) 1234 ksft_exit_fail_msg("error nomem\n"); 1235 wp_init(mem, 2 * page_size); 1236 wp_addr_range(mem, 2 * page_size); 1237 1238 /* Populate both pages. */ 1239 memset(mem, 1, 2 * page_size); 1240 1241 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1242 0, 0, PAGE_IS_WRITTEN); 1243 if (ret < 0) 1244 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1245 1246 ksft_test_result(ret == 1 && LEN(vec) == 2, "%s Both pages written\n", __func__); 1247 1248 /* 2. Start tracking */ 1249 wp_addr_range(mem, 2 * page_size); 1250 1251 ksft_test_result(pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, 1252 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0, 1253 "%s Both pages are not written (dirty)\n", __func__); 1254 1255 /* 3. Remap the second page */ 1256 mem2 = mmap(mem + page_size, page_size, PROT_READ|PROT_WRITE, 1257 MAP_PRIVATE|MAP_ANON|MAP_FIXED, -1, 0); 1258 if (mem2 == MAP_FAILED) 1259 ksft_exit_fail_msg("error nomem\n"); 1260 wp_init(mem2, page_size); 1261 wp_addr_range(mem2, page_size); 1262 1263 /* Protect + unprotect. */ 1264 mprotect(mem, page_size, PROT_NONE); 1265 mprotect(mem, 2 * page_size, PROT_READ); 1266 mprotect(mem, 2 * page_size, PROT_READ|PROT_WRITE); 1267 1268 /* Modify both pages. */ 1269 memset(mem, 2, 2 * page_size); 1270 1271 /* Protect + unprotect. */ 1272 mprotect(mem, page_size, PROT_NONE); 1273 mprotect(mem, page_size, PROT_READ); 1274 mprotect(mem, page_size, PROT_READ|PROT_WRITE); 1275 1276 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1277 0, 0, PAGE_IS_WRITTEN); 1278 if (ret < 0) 1279 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1280 1281 ksft_test_result(ret == 1 && LEN(vec) == 2, 1282 "%s Both pages written after remap and mprotect\n", __func__); 1283 1284 /* 4. Clear and make the pages written */ 1285 wp_addr_range(mem, 2 * page_size); 1286 1287 memset(mem, 'A', 2 * page_size); 1288 1289 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, 1290 0, 0, PAGE_IS_WRITTEN); 1291 if (ret < 0) 1292 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1293 1294 ksft_test_result(ret == 1 && LEN(vec) == 2, 1295 "%s Clear and make the pages written\n", __func__); 1296 1297 wp_free(mem, 2 * page_size); 1298 munmap(mem, 2 * page_size); 1299 return 0; 1300 } 1301 1302 /* transact test */ 1303 static const unsigned int nthreads = 6, pages_per_thread = 32, access_per_thread = 8; 1304 static pthread_barrier_t start_barrier, end_barrier; 1305 static unsigned int extra_thread_faults; 1306 static unsigned int iter_count = 1000; 1307 static volatile int finish; 1308 1309 static ssize_t get_dirty_pages_reset(char *mem, unsigned int count, 1310 int reset, int page_size) 1311 { 1312 struct pm_scan_arg arg = {0}; 1313 struct page_region rgns[256]; 1314 int i, j, cnt, ret; 1315 1316 arg.size = sizeof(struct pm_scan_arg); 1317 arg.start = (uintptr_t)mem; 1318 arg.max_pages = count; 1319 arg.end = (uintptr_t)(mem + count * page_size); 1320 arg.vec = (uintptr_t)rgns; 1321 arg.vec_len = sizeof(rgns) / sizeof(*rgns); 1322 if (reset) 1323 arg.flags |= PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC; 1324 arg.category_mask = PAGE_IS_WRITTEN; 1325 arg.return_mask = PAGE_IS_WRITTEN; 1326 1327 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); 1328 if (ret < 0) 1329 ksft_exit_fail_msg("ioctl failed\n"); 1330 1331 cnt = 0; 1332 for (i = 0; i < ret; ++i) { 1333 if (rgns[i].categories != PAGE_IS_WRITTEN) 1334 ksft_exit_fail_msg("wrong flags\n"); 1335 1336 for (j = 0; j < LEN(rgns[i]); ++j) 1337 cnt++; 1338 } 1339 1340 return cnt; 1341 } 1342 1343 void *thread_proc(void *mem) 1344 { 1345 int *m = mem; 1346 long curr_faults, faults; 1347 struct rusage r; 1348 unsigned int i; 1349 int ret; 1350 1351 if (getrusage(RUSAGE_THREAD, &r)) 1352 ksft_exit_fail_msg("getrusage\n"); 1353 1354 curr_faults = r.ru_minflt; 1355 1356 while (!finish) { 1357 ret = pthread_barrier_wait(&start_barrier); 1358 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1359 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1360 1361 for (i = 0; i < access_per_thread; ++i) 1362 __atomic_add_fetch(m + i * (0x1000 / sizeof(*m)), 1, __ATOMIC_SEQ_CST); 1363 1364 ret = pthread_barrier_wait(&end_barrier); 1365 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1366 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1367 1368 if (getrusage(RUSAGE_THREAD, &r)) 1369 ksft_exit_fail_msg("getrusage\n"); 1370 1371 faults = r.ru_minflt - curr_faults; 1372 if (faults < access_per_thread) 1373 ksft_exit_fail_msg("faults < access_per_thread"); 1374 1375 __atomic_add_fetch(&extra_thread_faults, faults - access_per_thread, 1376 __ATOMIC_SEQ_CST); 1377 curr_faults = r.ru_minflt; 1378 } 1379 1380 return NULL; 1381 } 1382 1383 static void transact_test(int page_size) 1384 { 1385 unsigned int i, count, extra_pages; 1386 pthread_t th; 1387 char *mem; 1388 int ret, c; 1389 1390 if (pthread_barrier_init(&start_barrier, NULL, nthreads + 1)) 1391 ksft_exit_fail_msg("pthread_barrier_init\n"); 1392 1393 if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1)) 1394 ksft_exit_fail_msg("pthread_barrier_init\n"); 1395 1396 mem = mmap(NULL, 0x1000 * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, 1397 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 1398 if (mem == MAP_FAILED) 1399 ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno)); 1400 1401 wp_init(mem, 0x1000 * nthreads * pages_per_thread); 1402 wp_addr_range(mem, 0x1000 * nthreads * pages_per_thread); 1403 1404 memset(mem, 0, 0x1000 * nthreads * pages_per_thread); 1405 1406 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1407 ksft_test_result(count > 0, "%s count %d\n", __func__, count); 1408 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1409 ksft_test_result(count == 0, "%s count %d\n", __func__, count); 1410 1411 finish = 0; 1412 for (i = 0; i < nthreads; ++i) 1413 pthread_create(&th, NULL, thread_proc, mem + 0x1000 * i * pages_per_thread); 1414 1415 extra_pages = 0; 1416 for (i = 0; i < iter_count; ++i) { 1417 count = 0; 1418 1419 ret = pthread_barrier_wait(&start_barrier); 1420 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1421 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1422 1423 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, 1424 page_size); 1425 1426 ret = pthread_barrier_wait(&end_barrier); 1427 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) 1428 ksft_exit_fail_msg("pthread_barrier_wait\n"); 1429 1430 if (count > nthreads * access_per_thread) 1431 ksft_exit_fail_msg("Too big count %d expected %d, iter %d\n", 1432 count, nthreads * access_per_thread, i); 1433 1434 c = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); 1435 count += c; 1436 1437 if (c > nthreads * access_per_thread) { 1438 ksft_test_result_fail(" %s count > nthreads\n", __func__); 1439 return; 1440 } 1441 1442 if (count != nthreads * access_per_thread) { 1443 /* 1444 * The purpose of the test is to make sure that no page updates are lost 1445 * when the page updates and read-resetting soft dirty flags are performed 1446 * in parallel. However, it is possible that the application will get the 1447 * soft dirty flags twice on the two consecutive read-resets. This seems 1448 * unavoidable as soft dirty flag is handled in software through page faults 1449 * in kernel. While the updating the flags is supposed to be synchronized 1450 * between page fault handling and read-reset, it is possible that 1451 * read-reset happens after page fault PTE update but before the application 1452 * re-executes write instruction. So read-reset gets the flag, clears write 1453 * access and application gets page fault again for the same write. 1454 */ 1455 if (count < nthreads * access_per_thread) { 1456 ksft_test_result_fail("Lost update, iter %d, %d vs %d.\n", i, count, 1457 nthreads * access_per_thread); 1458 return; 1459 } 1460 1461 extra_pages += count - nthreads * access_per_thread; 1462 } 1463 } 1464 1465 pthread_barrier_wait(&start_barrier); 1466 finish = 1; 1467 pthread_barrier_wait(&end_barrier); 1468 1469 ksft_test_result_pass("%s Extra pages %u (%.1lf%%), extra thread faults %d.\n", __func__, 1470 extra_pages, 1471 100.0 * extra_pages / (iter_count * nthreads * access_per_thread), 1472 extra_thread_faults); 1473 } 1474 1475 int main(void) 1476 { 1477 int mem_size, shmid, buf_size, fd, i, ret; 1478 char *mem, *map, *fmem; 1479 struct stat sbuf; 1480 1481 ksft_print_header(); 1482 1483 if (init_uffd()) 1484 return ksft_exit_pass(); 1485 1486 ksft_set_plan(115); 1487 1488 page_size = getpagesize(); 1489 hpage_size = read_pmd_pagesize(); 1490 1491 pagemap_fd = open(PAGEMAP, O_RDONLY); 1492 if (pagemap_fd < 0) 1493 return -EINVAL; 1494 1495 /* 1. Sanity testing */ 1496 sanity_tests_sd(); 1497 1498 /* 2. Normal page testing */ 1499 mem_size = 10 * page_size; 1500 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1501 if (mem == MAP_FAILED) 1502 ksft_exit_fail_msg("error nomem\n"); 1503 wp_init(mem, mem_size); 1504 wp_addr_range(mem, mem_size); 1505 1506 base_tests("Page testing:", mem, mem_size, 0); 1507 1508 wp_free(mem, mem_size); 1509 munmap(mem, mem_size); 1510 1511 /* 3. Large page testing */ 1512 mem_size = 512 * 10 * page_size; 1513 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 1514 if (mem == MAP_FAILED) 1515 ksft_exit_fail_msg("error nomem\n"); 1516 wp_init(mem, mem_size); 1517 wp_addr_range(mem, mem_size); 1518 1519 base_tests("Large Page testing:", mem, mem_size, 0); 1520 1521 wp_free(mem, mem_size); 1522 munmap(mem, mem_size); 1523 1524 /* 4. Huge page testing */ 1525 map = gethugepage(hpage_size); 1526 if (map) { 1527 wp_init(map, hpage_size); 1528 wp_addr_range(map, hpage_size); 1529 base_tests("Huge page testing:", map, hpage_size, 0); 1530 wp_free(map, hpage_size); 1531 free(map); 1532 } else { 1533 base_tests("Huge page testing:", NULL, 0, 1); 1534 } 1535 1536 /* 5. SHM Hugetlb page testing */ 1537 mem_size = 2*1024*1024; 1538 mem = gethugetlb_mem(mem_size, &shmid); 1539 if (mem) { 1540 wp_init(mem, mem_size); 1541 wp_addr_range(mem, mem_size); 1542 1543 base_tests("Hugetlb shmem testing:", mem, mem_size, 0); 1544 1545 wp_free(mem, mem_size); 1546 shmctl(shmid, IPC_RMID, NULL); 1547 } else { 1548 base_tests("Hugetlb shmem testing:", NULL, 0, 1); 1549 } 1550 1551 /* 6. Hugetlb page testing */ 1552 mem = gethugetlb_mem(mem_size, NULL); 1553 if (mem) { 1554 wp_init(mem, mem_size); 1555 wp_addr_range(mem, mem_size); 1556 1557 base_tests("Hugetlb mem testing:", mem, mem_size, 0); 1558 1559 wp_free(mem, mem_size); 1560 } else { 1561 base_tests("Hugetlb mem testing:", NULL, 0, 1); 1562 } 1563 1564 /* 7. File Hugetlb testing */ 1565 mem_size = 2*1024*1024; 1566 fd = memfd_create("uffd-test", MFD_HUGETLB | MFD_NOEXEC_SEAL); 1567 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 1568 if (mem) { 1569 wp_init(mem, mem_size); 1570 wp_addr_range(mem, mem_size); 1571 1572 base_tests("Hugetlb shmem testing:", mem, mem_size, 0); 1573 1574 wp_free(mem, mem_size); 1575 shmctl(shmid, IPC_RMID, NULL); 1576 } else { 1577 base_tests("Hugetlb shmem testing:", NULL, 0, 1); 1578 } 1579 close(fd); 1580 1581 /* 8. File memory testing */ 1582 buf_size = page_size * 10; 1583 1584 fd = open(__FILE__".tmp0", O_RDWR | O_CREAT, 0777); 1585 if (fd < 0) 1586 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n", 1587 strerror(errno)); 1588 1589 for (i = 0; i < buf_size; i++) 1590 if (write(fd, "c", 1) < 0) 1591 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1592 1593 ret = stat(__FILE__".tmp0", &sbuf); 1594 if (ret < 0) 1595 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); 1596 1597 fmem = mmap(NULL, sbuf.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1598 if (fmem == MAP_FAILED) 1599 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1600 1601 wp_init(fmem, sbuf.st_size); 1602 wp_addr_range(fmem, sbuf.st_size); 1603 1604 base_tests("File memory testing:", fmem, sbuf.st_size, 0); 1605 1606 wp_free(fmem, sbuf.st_size); 1607 munmap(fmem, sbuf.st_size); 1608 close(fd); 1609 1610 /* 9. File memory testing */ 1611 buf_size = page_size * 10; 1612 1613 fd = memfd_create(__FILE__".tmp00", MFD_NOEXEC_SEAL); 1614 if (fd < 0) 1615 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n", 1616 strerror(errno)); 1617 1618 if (ftruncate(fd, buf_size)) 1619 ksft_exit_fail_msg("Error ftruncate\n"); 1620 1621 for (i = 0; i < buf_size; i++) 1622 if (write(fd, "c", 1) < 0) 1623 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); 1624 1625 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1626 if (fmem == MAP_FAILED) 1627 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); 1628 1629 wp_init(fmem, buf_size); 1630 wp_addr_range(fmem, buf_size); 1631 1632 base_tests("File anonymous memory testing:", fmem, buf_size, 0); 1633 1634 wp_free(fmem, buf_size); 1635 munmap(fmem, buf_size); 1636 close(fd); 1637 1638 /* 10. Huge page tests */ 1639 hpage_unit_tests(); 1640 1641 /* 11. Iterative test */ 1642 test_simple(); 1643 1644 /* 12. Mprotect test */ 1645 mprotect_tests(); 1646 1647 /* 13. Transact test */ 1648 transact_test(page_size); 1649 1650 /* 14. Sanity testing */ 1651 sanity_tests(); 1652 1653 /*15. Unmapped address test */ 1654 unmapped_region_tests(); 1655 1656 /* 16. Userfaultfd tests */ 1657 userfaultfd_tests(); 1658 1659 close(pagemap_fd); 1660 return ksft_exit_pass(); 1661 } 1662