1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * HMM stands for Heterogeneous Memory Management, it is a helper layer inside 4 * the linux kernel to help device drivers mirror a process address space in 5 * the device. This allows the device to use the same address space which 6 * makes communication and data exchange a lot easier. 7 * 8 * This framework's sole purpose is to exercise various code paths inside 9 * the kernel to make sure that HMM performs as expected and to flush out any 10 * bugs. 11 */ 12 13 #include "kselftest_harness.h" 14 #include "hugepage_settings.h" 15 16 #include <errno.h> 17 #include <fcntl.h> 18 #include <stdio.h> 19 #include <stdlib.h> 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <strings.h> 23 #include <time.h> 24 #include <pthread.h> 25 #include <limits.h> 26 #include <linux/mman.h> 27 #include <sys/types.h> 28 #include <sys/stat.h> 29 #include <sys/mman.h> 30 #include <sys/ioctl.h> 31 #include <sys/time.h> 32 33 /* 34 * This is a private UAPI to the kernel test module so it isn't exported 35 * in the usual include/uapi/... directory. 36 */ 37 #include <lib/test_hmm_uapi.h> 38 #include <mm/gup_test.h> 39 #include <mm/vm_util.h> 40 41 struct hmm_buffer { 42 void *ptr; 43 void *mirror; 44 unsigned long size; 45 int fd; 46 uint64_t cpages; 47 uint64_t faults; 48 }; 49 50 enum { 51 HMM_PRIVATE_DEVICE_ONE, 52 HMM_PRIVATE_DEVICE_TWO, 53 HMM_COHERENCE_DEVICE_ONE, 54 HMM_COHERENCE_DEVICE_TWO, 55 }; 56 57 #define ONEKB (1 << 10) 58 #define ONEMEG (1 << 20) 59 #define TWOMEG (1 << 21) 60 #define HMM_BUFFER_SIZE (1024 << 12) 61 #define HMM_PATH_MAX 64 62 #define NTIMES 10 63 64 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) 65 /* Just the flags we need, copied from mm.h: */ 66 67 #ifndef FOLL_WRITE 68 #define FOLL_WRITE 0x01 /* check pte is writable */ 69 #endif 70 71 #ifndef FOLL_LONGTERM 72 #define FOLL_LONGTERM 0x100 /* mapping lifetime is indefinite */ 73 #endif 74 75 HUGETLB_SETUP_DEFAULT_PAGES(1) 76 77 FIXTURE(hmm) 78 { 79 int fd; 80 unsigned int page_size; 81 unsigned int page_shift; 82 }; 83 84 FIXTURE_VARIANT(hmm) 85 { 86 int device_number; 87 }; 88 89 FIXTURE_VARIANT_ADD(hmm, hmm_device_private) 90 { 91 .device_number = HMM_PRIVATE_DEVICE_ONE, 92 }; 93 94 FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent) 95 { 96 .device_number = HMM_COHERENCE_DEVICE_ONE, 97 }; 98 99 FIXTURE(hmm2) 100 { 101 int fd0; 102 int fd1; 103 unsigned int page_size; 104 unsigned int page_shift; 105 }; 106 107 FIXTURE_VARIANT(hmm2) 108 { 109 int device_number0; 110 int device_number1; 111 }; 112 113 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private) 114 { 115 .device_number0 = HMM_PRIVATE_DEVICE_ONE, 116 .device_number1 = HMM_PRIVATE_DEVICE_TWO, 117 }; 118 119 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent) 120 { 121 .device_number0 = HMM_COHERENCE_DEVICE_ONE, 122 .device_number1 = HMM_COHERENCE_DEVICE_TWO, 123 }; 124 125 static int hmm_open(int unit) 126 { 127 char pathname[HMM_PATH_MAX]; 128 int fd; 129 130 snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit); 131 fd = open(pathname, O_RDWR, 0); 132 if (fd < 0) 133 fprintf(stderr, "could not open hmm dmirror driver (%s)\n", 134 pathname); 135 return fd; 136 } 137 138 static bool hmm_is_coherent_type(int dev_num) 139 { 140 return (dev_num >= HMM_COHERENCE_DEVICE_ONE); 141 } 142 143 FIXTURE_SETUP(hmm) 144 { 145 self->page_size = sysconf(_SC_PAGE_SIZE); 146 self->page_shift = ffs(self->page_size) - 1; 147 148 self->fd = hmm_open(variant->device_number); 149 if (self->fd < 0 && hmm_is_coherent_type(variant->device_number)) 150 SKIP(return, "DEVICE_COHERENT not available"); 151 ASSERT_GE(self->fd, 0); 152 } 153 154 FIXTURE_SETUP(hmm2) 155 { 156 self->page_size = sysconf(_SC_PAGE_SIZE); 157 self->page_shift = ffs(self->page_size) - 1; 158 159 self->fd0 = hmm_open(variant->device_number0); 160 if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0)) 161 SKIP(return, "DEVICE_COHERENT not available"); 162 ASSERT_GE(self->fd0, 0); 163 self->fd1 = hmm_open(variant->device_number1); 164 ASSERT_GE(self->fd1, 0); 165 } 166 167 FIXTURE_TEARDOWN(hmm) 168 { 169 int ret = close(self->fd); 170 171 ASSERT_EQ(ret, 0); 172 self->fd = -1; 173 } 174 175 FIXTURE_TEARDOWN(hmm2) 176 { 177 int ret = close(self->fd0); 178 179 ASSERT_EQ(ret, 0); 180 self->fd0 = -1; 181 182 ret = close(self->fd1); 183 ASSERT_EQ(ret, 0); 184 self->fd1 = -1; 185 } 186 187 static int hmm_dmirror_cmd(int fd, 188 unsigned long request, 189 struct hmm_buffer *buffer, 190 unsigned long npages) 191 { 192 struct hmm_dmirror_cmd cmd; 193 int ret; 194 195 /* Simulate a device reading system memory. */ 196 cmd.addr = (__u64)buffer->ptr; 197 cmd.ptr = (__u64)buffer->mirror; 198 cmd.npages = npages; 199 200 for (;;) { 201 ret = ioctl(fd, request, &cmd); 202 if (ret == 0) 203 break; 204 if (errno == EINTR) 205 continue; 206 return -errno; 207 } 208 buffer->cpages = cmd.cpages; 209 buffer->faults = cmd.faults; 210 211 return 0; 212 } 213 214 static void hmm_buffer_free(struct hmm_buffer *buffer) 215 { 216 if (buffer == NULL) 217 return; 218 219 if (buffer->ptr) { 220 munmap(buffer->ptr, buffer->size); 221 buffer->ptr = NULL; 222 } 223 free(buffer->mirror); 224 free(buffer); 225 } 226 227 /* 228 * Create a temporary file that will be deleted on close. 229 */ 230 static int hmm_create_file(unsigned long size) 231 { 232 char path[HMM_PATH_MAX]; 233 int fd; 234 235 strcpy(path, "/tmp"); 236 fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600); 237 if (fd >= 0) { 238 int r; 239 240 do { 241 r = ftruncate(fd, size); 242 } while (r == -1 && errno == EINTR); 243 if (!r) 244 return fd; 245 close(fd); 246 } 247 return -1; 248 } 249 250 /* 251 * Return a random unsigned number. 252 */ 253 static unsigned int hmm_random(void) 254 { 255 static int fd = -1; 256 unsigned int r; 257 258 if (fd < 0) { 259 fd = open("/dev/urandom", O_RDONLY); 260 if (fd < 0) { 261 fprintf(stderr, "%s:%d failed to open /dev/urandom\n", 262 __FILE__, __LINE__); 263 return ~0U; 264 } 265 } 266 read(fd, &r, sizeof(r)); 267 return r; 268 } 269 270 static void hmm_nanosleep(unsigned int n) 271 { 272 struct timespec t; 273 274 t.tv_sec = 0; 275 t.tv_nsec = n; 276 nanosleep(&t, NULL); 277 } 278 279 static int hmm_migrate_sys_to_dev(int fd, 280 struct hmm_buffer *buffer, 281 unsigned long npages) 282 { 283 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages); 284 } 285 286 static int hmm_migrate_dev_to_sys(int fd, 287 struct hmm_buffer *buffer, 288 unsigned long npages) 289 { 290 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages); 291 } 292 293 /* 294 * Simple NULL test of device open/close. 295 */ 296 TEST_F(hmm, open_close) 297 { 298 } 299 300 /* 301 * Read private anonymous memory. 302 */ 303 TEST_F(hmm, anon_read) 304 { 305 struct hmm_buffer *buffer; 306 unsigned long npages; 307 unsigned long size; 308 unsigned long i; 309 int *ptr; 310 int ret; 311 int val; 312 313 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 314 ASSERT_NE(npages, 0); 315 size = npages << self->page_shift; 316 317 buffer = malloc(sizeof(*buffer)); 318 ASSERT_NE(buffer, NULL); 319 320 buffer->fd = -1; 321 buffer->size = size; 322 buffer->mirror = malloc(size); 323 ASSERT_NE(buffer->mirror, NULL); 324 325 buffer->ptr = mmap(NULL, size, 326 PROT_READ | PROT_WRITE, 327 MAP_PRIVATE | MAP_ANONYMOUS, 328 buffer->fd, 0); 329 ASSERT_NE(buffer->ptr, MAP_FAILED); 330 331 /* 332 * Initialize buffer in system memory but leave the first two pages 333 * zero (pte_none and pfn_zero). 334 */ 335 i = 2 * self->page_size / sizeof(*ptr); 336 for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 337 ptr[i] = i; 338 339 /* Set buffer permission to read-only. */ 340 ret = mprotect(buffer->ptr, size, PROT_READ); 341 ASSERT_EQ(ret, 0); 342 343 /* Populate the CPU page table with a special zero page. */ 344 val = *(int *)(buffer->ptr + self->page_size); 345 ASSERT_EQ(val, 0); 346 347 /* Simulate a device reading system memory. */ 348 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 349 ASSERT_EQ(ret, 0); 350 ASSERT_EQ(buffer->cpages, npages); 351 ASSERT_EQ(buffer->faults, 1); 352 353 /* Check what the device read. */ 354 ptr = buffer->mirror; 355 for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i) 356 ASSERT_EQ(ptr[i], 0); 357 for (; i < size / sizeof(*ptr); ++i) 358 ASSERT_EQ(ptr[i], i); 359 360 hmm_buffer_free(buffer); 361 } 362 363 /* 364 * Read private anonymous memory which has been protected with 365 * mprotect() PROT_NONE. 366 */ 367 TEST_F(hmm, anon_read_prot) 368 { 369 struct hmm_buffer *buffer; 370 unsigned long npages; 371 unsigned long size; 372 unsigned long i; 373 int *ptr; 374 int ret; 375 376 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 377 ASSERT_NE(npages, 0); 378 size = npages << self->page_shift; 379 380 buffer = malloc(sizeof(*buffer)); 381 ASSERT_NE(buffer, NULL); 382 383 buffer->fd = -1; 384 buffer->size = size; 385 buffer->mirror = malloc(size); 386 ASSERT_NE(buffer->mirror, NULL); 387 388 buffer->ptr = mmap(NULL, size, 389 PROT_READ | PROT_WRITE, 390 MAP_PRIVATE | MAP_ANONYMOUS, 391 buffer->fd, 0); 392 ASSERT_NE(buffer->ptr, MAP_FAILED); 393 394 /* Initialize buffer in system memory. */ 395 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 396 ptr[i] = i; 397 398 /* Initialize mirror buffer so we can verify it isn't written. */ 399 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 400 ptr[i] = -i; 401 402 /* Protect buffer from reading. */ 403 ret = mprotect(buffer->ptr, size, PROT_NONE); 404 ASSERT_EQ(ret, 0); 405 406 /* Simulate a device reading system memory. */ 407 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 408 ASSERT_EQ(ret, -EFAULT); 409 410 /* Allow CPU to read the buffer so we can check it. */ 411 ret = mprotect(buffer->ptr, size, PROT_READ); 412 ASSERT_EQ(ret, 0); 413 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 414 ASSERT_EQ(ptr[i], i); 415 416 /* Check what the device read. */ 417 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 418 ASSERT_EQ(ptr[i], -i); 419 420 hmm_buffer_free(buffer); 421 } 422 423 /* 424 * Write private anonymous memory. 425 */ 426 TEST_F(hmm, anon_write) 427 { 428 struct hmm_buffer *buffer; 429 unsigned long npages; 430 unsigned long size; 431 unsigned long i; 432 int *ptr; 433 int ret; 434 435 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 436 ASSERT_NE(npages, 0); 437 size = npages << self->page_shift; 438 439 buffer = malloc(sizeof(*buffer)); 440 ASSERT_NE(buffer, NULL); 441 442 buffer->fd = -1; 443 buffer->size = size; 444 buffer->mirror = malloc(size); 445 ASSERT_NE(buffer->mirror, NULL); 446 447 buffer->ptr = mmap(NULL, size, 448 PROT_READ | PROT_WRITE, 449 MAP_PRIVATE | MAP_ANONYMOUS, 450 buffer->fd, 0); 451 ASSERT_NE(buffer->ptr, MAP_FAILED); 452 453 /* Initialize data that the device will write to buffer->ptr. */ 454 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 455 ptr[i] = i; 456 457 /* Simulate a device writing system memory. */ 458 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 459 ASSERT_EQ(ret, 0); 460 ASSERT_EQ(buffer->cpages, npages); 461 ASSERT_EQ(buffer->faults, 1); 462 463 /* Check what the device wrote. */ 464 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 465 ASSERT_EQ(ptr[i], i); 466 467 hmm_buffer_free(buffer); 468 } 469 470 /* 471 * Write private anonymous memory which has been protected with 472 * mprotect() PROT_READ. 473 */ 474 TEST_F(hmm, anon_write_prot) 475 { 476 struct hmm_buffer *buffer; 477 unsigned long npages; 478 unsigned long size; 479 unsigned long i; 480 int *ptr; 481 int ret; 482 483 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 484 ASSERT_NE(npages, 0); 485 size = npages << self->page_shift; 486 487 buffer = malloc(sizeof(*buffer)); 488 ASSERT_NE(buffer, NULL); 489 490 buffer->fd = -1; 491 buffer->size = size; 492 buffer->mirror = malloc(size); 493 ASSERT_NE(buffer->mirror, NULL); 494 495 buffer->ptr = mmap(NULL, size, 496 PROT_READ, 497 MAP_PRIVATE | MAP_ANONYMOUS, 498 buffer->fd, 0); 499 ASSERT_NE(buffer->ptr, MAP_FAILED); 500 501 /* Simulate a device reading a zero page of memory. */ 502 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); 503 ASSERT_EQ(ret, 0); 504 ASSERT_EQ(buffer->cpages, 1); 505 ASSERT_EQ(buffer->faults, 1); 506 507 /* Initialize data that the device will write to buffer->ptr. */ 508 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 509 ptr[i] = i; 510 511 /* Simulate a device writing system memory. */ 512 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 513 ASSERT_EQ(ret, -EPERM); 514 515 /* Check what the device wrote. */ 516 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 517 ASSERT_EQ(ptr[i], 0); 518 519 /* Now allow writing and see that the zero page is replaced. */ 520 ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ); 521 ASSERT_EQ(ret, 0); 522 523 /* Simulate a device writing system memory. */ 524 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 525 ASSERT_EQ(ret, 0); 526 ASSERT_EQ(buffer->cpages, npages); 527 ASSERT_EQ(buffer->faults, 1); 528 529 /* Check what the device wrote. */ 530 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 531 ASSERT_EQ(ptr[i], i); 532 533 hmm_buffer_free(buffer); 534 } 535 536 /* 537 * Check that a device writing an anonymous private mapping 538 * will copy-on-write if a child process inherits the mapping. 539 * 540 * Also verifies after fork() memory the device can be read by child. 541 */ 542 TEST_F(hmm, anon_write_child) 543 { 544 struct hmm_buffer *buffer; 545 unsigned long npages; 546 unsigned long size; 547 unsigned long i; 548 void *old_ptr; 549 void *map; 550 int *ptr; 551 pid_t pid; 552 int child_fd; 553 int ret, use_thp, migrate; 554 555 for (migrate = 0; migrate < 2; ++migrate) { 556 for (use_thp = 0; use_thp < 2; ++use_thp) { 557 npages = ALIGN(use_thp ? read_pmd_pagesize() : HMM_BUFFER_SIZE, 558 self->page_size) >> self->page_shift; 559 ASSERT_NE(npages, 0); 560 size = npages << self->page_shift; 561 562 buffer = malloc(sizeof(*buffer)); 563 ASSERT_NE(buffer, NULL); 564 565 buffer->fd = -1; 566 buffer->size = size * 2; 567 buffer->mirror = malloc(size); 568 ASSERT_NE(buffer->mirror, NULL); 569 570 buffer->ptr = mmap(NULL, size * 2, 571 PROT_READ | PROT_WRITE, 572 MAP_PRIVATE | MAP_ANONYMOUS, 573 buffer->fd, 0); 574 ASSERT_NE(buffer->ptr, MAP_FAILED); 575 576 old_ptr = buffer->ptr; 577 if (use_thp) { 578 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 579 ret = madvise(map, size, MADV_HUGEPAGE); 580 ASSERT_EQ(ret, 0); 581 buffer->ptr = map; 582 } 583 584 /* Initialize buffer->ptr so we can tell if it is written. */ 585 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 586 ptr[i] = i; 587 588 /* Initialize data that the device will write to buffer->ptr. */ 589 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 590 ptr[i] = -i; 591 592 if (migrate) { 593 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 594 ASSERT_EQ(ret, 0); 595 ASSERT_EQ(buffer->cpages, npages); 596 597 } 598 599 pid = fork(); 600 if (pid == -1) 601 ASSERT_EQ(pid, 0); 602 if (pid != 0) { 603 waitpid(pid, &ret, 0); 604 ASSERT_EQ(WIFEXITED(ret), 1); 605 606 /* Check that the parent's buffer did not change. */ 607 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 608 ASSERT_EQ(ptr[i], i); 609 610 buffer->ptr = old_ptr; 611 hmm_buffer_free(buffer); 612 continue; 613 } 614 615 /* Check that we see the parent's values. */ 616 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 617 ASSERT_EQ(ptr[i], i); 618 if (!migrate) { 619 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 620 ASSERT_EQ(ptr[i], -i); 621 } 622 623 /* The child process needs its own mirror to its own mm. */ 624 child_fd = hmm_open(0); 625 ASSERT_GE(child_fd, 0); 626 627 /* Simulate a device writing system memory. */ 628 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); 629 ASSERT_EQ(ret, 0); 630 ASSERT_EQ(buffer->cpages, npages); 631 ASSERT_EQ(buffer->faults, 1); 632 633 /* Check what the device wrote. */ 634 if (!migrate) { 635 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 636 ASSERT_EQ(ptr[i], -i); 637 } 638 639 close(child_fd); 640 _exit(0); 641 } 642 } 643 } 644 645 /* 646 * Check that a device writing an anonymous shared mapping 647 * will not copy-on-write if a child process inherits the mapping. 648 */ 649 TEST_F(hmm, anon_write_child_shared) 650 { 651 struct hmm_buffer *buffer; 652 unsigned long npages; 653 unsigned long size; 654 unsigned long i; 655 int *ptr; 656 pid_t pid; 657 int child_fd; 658 int ret; 659 660 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 661 ASSERT_NE(npages, 0); 662 size = npages << self->page_shift; 663 664 buffer = malloc(sizeof(*buffer)); 665 ASSERT_NE(buffer, NULL); 666 667 buffer->fd = -1; 668 buffer->size = size; 669 buffer->mirror = malloc(size); 670 ASSERT_NE(buffer->mirror, NULL); 671 672 buffer->ptr = mmap(NULL, size, 673 PROT_READ | PROT_WRITE, 674 MAP_SHARED | MAP_ANONYMOUS, 675 buffer->fd, 0); 676 ASSERT_NE(buffer->ptr, MAP_FAILED); 677 678 /* Initialize buffer->ptr so we can tell if it is written. */ 679 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 680 ptr[i] = i; 681 682 /* Initialize data that the device will write to buffer->ptr. */ 683 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 684 ptr[i] = -i; 685 686 pid = fork(); 687 if (pid == -1) 688 ASSERT_EQ(pid, 0); 689 if (pid != 0) { 690 waitpid(pid, &ret, 0); 691 ASSERT_EQ(WIFEXITED(ret), 1); 692 693 /* Check that the parent's buffer did change. */ 694 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 695 ASSERT_EQ(ptr[i], -i); 696 return; 697 } 698 699 /* Check that we see the parent's values. */ 700 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 701 ASSERT_EQ(ptr[i], i); 702 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 703 ASSERT_EQ(ptr[i], -i); 704 705 /* The child process needs its own mirror to its own mm. */ 706 child_fd = hmm_open(0); 707 ASSERT_GE(child_fd, 0); 708 709 /* Simulate a device writing system memory. */ 710 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); 711 ASSERT_EQ(ret, 0); 712 ASSERT_EQ(buffer->cpages, npages); 713 ASSERT_EQ(buffer->faults, 1); 714 715 /* Check what the device wrote. */ 716 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 717 ASSERT_EQ(ptr[i], -i); 718 719 close(child_fd); 720 _exit(0); 721 } 722 723 /* 724 * Write private anonymous huge page. 725 */ 726 TEST_F(hmm, anon_write_huge) 727 { 728 struct hmm_buffer *buffer; 729 unsigned long npages; 730 unsigned long size; 731 unsigned long i; 732 void *old_ptr; 733 void *map; 734 int *ptr; 735 int ret; 736 737 size = 2 * read_pmd_pagesize(); 738 739 buffer = malloc(sizeof(*buffer)); 740 ASSERT_NE(buffer, NULL); 741 742 buffer->fd = -1; 743 buffer->size = size; 744 buffer->mirror = malloc(size); 745 ASSERT_NE(buffer->mirror, NULL); 746 747 buffer->ptr = mmap(NULL, size, 748 PROT_READ | PROT_WRITE, 749 MAP_PRIVATE | MAP_ANONYMOUS, 750 buffer->fd, 0); 751 ASSERT_NE(buffer->ptr, MAP_FAILED); 752 753 size /= 2; 754 npages = size >> self->page_shift; 755 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 756 ret = madvise(map, size, MADV_HUGEPAGE); 757 ASSERT_EQ(ret, 0); 758 old_ptr = buffer->ptr; 759 buffer->ptr = map; 760 761 /* Initialize data that the device will write to buffer->ptr. */ 762 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 763 ptr[i] = i; 764 765 /* Simulate a device writing system memory. */ 766 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 767 ASSERT_EQ(ret, 0); 768 ASSERT_EQ(buffer->cpages, npages); 769 ASSERT_EQ(buffer->faults, 1); 770 771 /* Check what the device wrote. */ 772 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 773 ASSERT_EQ(ptr[i], i); 774 775 buffer->ptr = old_ptr; 776 hmm_buffer_free(buffer); 777 } 778 779 /* 780 * Write huge TLBFS page. 781 */ 782 TEST_F(hmm, anon_write_hugetlbfs) 783 { 784 struct hmm_buffer *buffer; 785 unsigned long npages; 786 unsigned long size; 787 unsigned long default_hsize = default_huge_page_size(); 788 unsigned long i; 789 int *ptr; 790 int ret; 791 792 if (!hugetlb_free_default_pages()) 793 SKIP(return, "Not enough huge pages"); 794 795 size = ALIGN(TWOMEG, default_hsize); 796 npages = size >> self->page_shift; 797 798 buffer = malloc(sizeof(*buffer)); 799 ASSERT_NE(buffer, NULL); 800 801 buffer->ptr = mmap(NULL, size, 802 PROT_READ | PROT_WRITE, 803 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 804 -1, 0); 805 if (buffer->ptr == MAP_FAILED) { 806 free(buffer); 807 SKIP(return, "Huge page could not be allocated"); 808 } 809 810 buffer->fd = -1; 811 buffer->size = size; 812 buffer->mirror = malloc(size); 813 ASSERT_NE(buffer->mirror, NULL); 814 815 /* Initialize data that the device will write to buffer->ptr. */ 816 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 817 ptr[i] = i; 818 819 /* Simulate a device writing system memory. */ 820 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 821 ASSERT_EQ(ret, 0); 822 ASSERT_EQ(buffer->cpages, npages); 823 ASSERT_EQ(buffer->faults, 1); 824 825 /* Check what the device wrote. */ 826 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 827 ASSERT_EQ(ptr[i], i); 828 829 munmap(buffer->ptr, buffer->size); 830 buffer->ptr = NULL; 831 hmm_buffer_free(buffer); 832 } 833 834 /* 835 * Read mmap'ed file memory. 836 */ 837 TEST_F(hmm, file_read) 838 { 839 struct hmm_buffer *buffer; 840 unsigned long npages; 841 unsigned long size; 842 unsigned long i; 843 int *ptr; 844 int ret; 845 int fd; 846 ssize_t len; 847 848 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 849 ASSERT_NE(npages, 0); 850 size = npages << self->page_shift; 851 852 fd = hmm_create_file(size); 853 ASSERT_GE(fd, 0); 854 855 buffer = malloc(sizeof(*buffer)); 856 ASSERT_NE(buffer, NULL); 857 858 buffer->fd = fd; 859 buffer->size = size; 860 buffer->mirror = malloc(size); 861 ASSERT_NE(buffer->mirror, NULL); 862 863 /* Write initial contents of the file. */ 864 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 865 ptr[i] = i; 866 len = pwrite(fd, buffer->mirror, size, 0); 867 ASSERT_EQ(len, size); 868 memset(buffer->mirror, 0, size); 869 870 buffer->ptr = mmap(NULL, size, 871 PROT_READ, 872 MAP_SHARED, 873 buffer->fd, 0); 874 ASSERT_NE(buffer->ptr, MAP_FAILED); 875 876 /* Simulate a device reading system memory. */ 877 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 878 ASSERT_EQ(ret, 0); 879 ASSERT_EQ(buffer->cpages, npages); 880 ASSERT_EQ(buffer->faults, 1); 881 882 /* Check what the device read. */ 883 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 884 ASSERT_EQ(ptr[i], i); 885 886 hmm_buffer_free(buffer); 887 } 888 889 /* 890 * Write mmap'ed file memory. 891 */ 892 TEST_F(hmm, file_write) 893 { 894 struct hmm_buffer *buffer; 895 unsigned long npages; 896 unsigned long size; 897 unsigned long i; 898 int *ptr; 899 int ret; 900 int fd; 901 ssize_t len; 902 903 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 904 ASSERT_NE(npages, 0); 905 size = npages << self->page_shift; 906 907 fd = hmm_create_file(size); 908 ASSERT_GE(fd, 0); 909 910 buffer = malloc(sizeof(*buffer)); 911 ASSERT_NE(buffer, NULL); 912 913 buffer->fd = fd; 914 buffer->size = size; 915 buffer->mirror = malloc(size); 916 ASSERT_NE(buffer->mirror, NULL); 917 918 buffer->ptr = mmap(NULL, size, 919 PROT_READ | PROT_WRITE, 920 MAP_SHARED, 921 buffer->fd, 0); 922 ASSERT_NE(buffer->ptr, MAP_FAILED); 923 924 /* Initialize data that the device will write to buffer->ptr. */ 925 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 926 ptr[i] = i; 927 928 /* Simulate a device writing system memory. */ 929 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 930 ASSERT_EQ(ret, 0); 931 ASSERT_EQ(buffer->cpages, npages); 932 ASSERT_EQ(buffer->faults, 1); 933 934 /* Check what the device wrote. */ 935 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 936 ASSERT_EQ(ptr[i], i); 937 938 /* Check that the device also wrote the file. */ 939 len = pread(fd, buffer->mirror, size, 0); 940 ASSERT_EQ(len, size); 941 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 942 ASSERT_EQ(ptr[i], i); 943 944 hmm_buffer_free(buffer); 945 } 946 947 /* 948 * Migrate anonymous memory to device private memory. 949 */ 950 TEST_F(hmm, migrate) 951 { 952 struct hmm_buffer *buffer; 953 unsigned long npages; 954 unsigned long size; 955 unsigned long i; 956 int *ptr; 957 int ret; 958 959 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 960 ASSERT_NE(npages, 0); 961 size = npages << self->page_shift; 962 963 buffer = malloc(sizeof(*buffer)); 964 ASSERT_NE(buffer, NULL); 965 966 buffer->fd = -1; 967 buffer->size = size; 968 buffer->mirror = malloc(size); 969 ASSERT_NE(buffer->mirror, NULL); 970 971 buffer->ptr = mmap(NULL, size, 972 PROT_READ | PROT_WRITE, 973 MAP_PRIVATE | MAP_ANONYMOUS, 974 buffer->fd, 0); 975 ASSERT_NE(buffer->ptr, MAP_FAILED); 976 977 /* Initialize buffer in system memory. */ 978 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 979 ptr[i] = i; 980 981 /* Migrate memory to device. */ 982 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 983 ASSERT_EQ(ret, 0); 984 ASSERT_EQ(buffer->cpages, npages); 985 986 /* Check what the device read. */ 987 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 988 ASSERT_EQ(ptr[i], i); 989 990 hmm_buffer_free(buffer); 991 } 992 993 /* 994 * Migrate private file memory to device private memory. 995 */ 996 TEST_F(hmm, migrate_file_private) 997 { 998 struct hmm_buffer *buffer; 999 unsigned long npages; 1000 unsigned long size; 1001 unsigned long i; 1002 int *ptr; 1003 int ret; 1004 int fd; 1005 1006 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1007 ASSERT_NE(npages, 0); 1008 size = npages << self->page_shift; 1009 1010 fd = hmm_create_file(size); 1011 ASSERT_GE(fd, 0); 1012 1013 buffer = malloc(sizeof(*buffer)); 1014 ASSERT_NE(buffer, NULL); 1015 1016 buffer->fd = fd; 1017 buffer->size = size; 1018 buffer->mirror = malloc(size); 1019 ASSERT_NE(buffer->mirror, NULL); 1020 1021 buffer->ptr = mmap(NULL, size, 1022 PROT_READ | PROT_WRITE, 1023 MAP_PRIVATE, 1024 buffer->fd, 0); 1025 ASSERT_NE(buffer->ptr, MAP_FAILED); 1026 1027 /* Initialize buffer in system memory. */ 1028 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1029 ptr[i] = i; 1030 1031 /* Migrate memory to device. */ 1032 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1033 ASSERT_EQ(ret, 0); 1034 ASSERT_EQ(buffer->cpages, npages); 1035 1036 /* Check what the device read. */ 1037 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1038 ASSERT_EQ(ptr[i], i); 1039 1040 hmm_buffer_free(buffer); 1041 } 1042 1043 /* 1044 * Migrate anonymous memory to device private memory and fault some of it back 1045 * to system memory, then try migrating the resulting mix of system and device 1046 * private memory to the device. 1047 */ 1048 TEST_F(hmm, migrate_fault) 1049 { 1050 struct hmm_buffer *buffer; 1051 unsigned long npages; 1052 unsigned long size; 1053 unsigned long i; 1054 int *ptr; 1055 int ret; 1056 1057 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1058 ASSERT_NE(npages, 0); 1059 size = npages << self->page_shift; 1060 1061 buffer = malloc(sizeof(*buffer)); 1062 ASSERT_NE(buffer, NULL); 1063 1064 buffer->fd = -1; 1065 buffer->size = size; 1066 buffer->mirror = malloc(size); 1067 ASSERT_NE(buffer->mirror, NULL); 1068 1069 buffer->ptr = mmap(NULL, size, 1070 PROT_READ | PROT_WRITE, 1071 MAP_PRIVATE | MAP_ANONYMOUS, 1072 buffer->fd, 0); 1073 ASSERT_NE(buffer->ptr, MAP_FAILED); 1074 1075 /* Initialize buffer in system memory. */ 1076 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1077 ptr[i] = i; 1078 1079 /* Migrate memory to device. */ 1080 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1081 ASSERT_EQ(ret, 0); 1082 ASSERT_EQ(buffer->cpages, npages); 1083 1084 /* Check what the device read. */ 1085 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1086 ASSERT_EQ(ptr[i], i); 1087 1088 /* Fault half the pages back to system memory and check them. */ 1089 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) 1090 ASSERT_EQ(ptr[i], i); 1091 1092 /* Migrate memory to the device again. */ 1093 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1094 ASSERT_EQ(ret, 0); 1095 ASSERT_EQ(buffer->cpages, npages); 1096 1097 /* Check what the device read. */ 1098 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1099 ASSERT_EQ(ptr[i], i); 1100 1101 hmm_buffer_free(buffer); 1102 } 1103 1104 TEST_F(hmm, migrate_release) 1105 { 1106 struct hmm_buffer *buffer; 1107 unsigned long npages; 1108 unsigned long size; 1109 unsigned long i; 1110 int *ptr; 1111 int ret; 1112 1113 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1114 ASSERT_NE(npages, 0); 1115 size = npages << self->page_shift; 1116 1117 buffer = malloc(sizeof(*buffer)); 1118 ASSERT_NE(buffer, NULL); 1119 1120 buffer->fd = -1; 1121 buffer->size = size; 1122 buffer->mirror = malloc(size); 1123 ASSERT_NE(buffer->mirror, NULL); 1124 1125 buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 1126 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 1127 ASSERT_NE(buffer->ptr, MAP_FAILED); 1128 1129 /* Initialize buffer in system memory. */ 1130 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1131 ptr[i] = i; 1132 1133 /* Migrate memory to device. */ 1134 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1135 ASSERT_EQ(ret, 0); 1136 ASSERT_EQ(buffer->cpages, npages); 1137 1138 /* Check what the device read. */ 1139 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1140 ASSERT_EQ(ptr[i], i); 1141 1142 /* Release device memory. */ 1143 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages); 1144 ASSERT_EQ(ret, 0); 1145 1146 /* Fault pages back to system memory and check them. */ 1147 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) 1148 ASSERT_EQ(ptr[i], i); 1149 1150 hmm_buffer_free(buffer); 1151 } 1152 1153 /* 1154 * Migrate anonymous shared memory to device private memory. 1155 */ 1156 TEST_F(hmm, migrate_shared) 1157 { 1158 struct hmm_buffer *buffer; 1159 unsigned long npages; 1160 unsigned long size; 1161 int ret; 1162 1163 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1164 ASSERT_NE(npages, 0); 1165 size = npages << self->page_shift; 1166 1167 buffer = malloc(sizeof(*buffer)); 1168 ASSERT_NE(buffer, NULL); 1169 1170 buffer->fd = -1; 1171 buffer->size = size; 1172 buffer->mirror = malloc(size); 1173 ASSERT_NE(buffer->mirror, NULL); 1174 1175 buffer->ptr = mmap(NULL, size, 1176 PROT_READ | PROT_WRITE, 1177 MAP_SHARED | MAP_ANONYMOUS, 1178 buffer->fd, 0); 1179 ASSERT_NE(buffer->ptr, MAP_FAILED); 1180 1181 /* Migrate memory to device. */ 1182 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1183 ASSERT_EQ(ret, -ENOENT); 1184 1185 hmm_buffer_free(buffer); 1186 } 1187 1188 /* 1189 * Try to migrate various memory types to device private memory. 1190 */ 1191 TEST_F(hmm2, migrate_mixed) 1192 { 1193 struct hmm_buffer *buffer; 1194 unsigned long npages; 1195 unsigned long size; 1196 int *ptr; 1197 unsigned char *p; 1198 int ret; 1199 int val; 1200 1201 npages = 6; 1202 size = npages << self->page_shift; 1203 1204 buffer = malloc(sizeof(*buffer)); 1205 ASSERT_NE(buffer, NULL); 1206 1207 buffer->fd = -1; 1208 buffer->size = size; 1209 buffer->mirror = malloc(size); 1210 ASSERT_NE(buffer->mirror, NULL); 1211 1212 /* Reserve a range of addresses. */ 1213 buffer->ptr = mmap(NULL, size, 1214 PROT_NONE, 1215 MAP_PRIVATE | MAP_ANONYMOUS, 1216 buffer->fd, 0); 1217 ASSERT_NE(buffer->ptr, MAP_FAILED); 1218 p = buffer->ptr; 1219 1220 /* Migrating a protected area should be an error. */ 1221 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); 1222 ASSERT_EQ(ret, -EINVAL); 1223 1224 /* Punch a hole after the first page address. */ 1225 ret = munmap(buffer->ptr + self->page_size, self->page_size); 1226 ASSERT_EQ(ret, 0); 1227 1228 /* We expect an error if the vma doesn't cover the range. */ 1229 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3); 1230 ASSERT_EQ(ret, -EINVAL); 1231 1232 /* Page 2 will be a read-only zero page. */ 1233 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, 1234 PROT_READ); 1235 ASSERT_EQ(ret, 0); 1236 ptr = (int *)(buffer->ptr + 2 * self->page_size); 1237 val = *ptr + 3; 1238 ASSERT_EQ(val, 3); 1239 1240 /* Page 3 will be read-only. */ 1241 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1242 PROT_READ | PROT_WRITE); 1243 ASSERT_EQ(ret, 0); 1244 ptr = (int *)(buffer->ptr + 3 * self->page_size); 1245 *ptr = val; 1246 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1247 PROT_READ); 1248 ASSERT_EQ(ret, 0); 1249 1250 /* Page 4-5 will be read-write. */ 1251 ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size, 1252 PROT_READ | PROT_WRITE); 1253 ASSERT_EQ(ret, 0); 1254 ptr = (int *)(buffer->ptr + 4 * self->page_size); 1255 *ptr = val; 1256 ptr = (int *)(buffer->ptr + 5 * self->page_size); 1257 *ptr = val; 1258 1259 /* Now try to migrate pages 2-5 to device 1. */ 1260 buffer->ptr = p + 2 * self->page_size; 1261 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4); 1262 ASSERT_EQ(ret, 0); 1263 ASSERT_EQ(buffer->cpages, 4); 1264 1265 /* Page 5 won't be migrated to device 0 because it's on device 1. */ 1266 buffer->ptr = p + 5 * self->page_size; 1267 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); 1268 ASSERT_EQ(ret, -ENOENT); 1269 buffer->ptr = p; 1270 1271 buffer->ptr = p; 1272 hmm_buffer_free(buffer); 1273 } 1274 1275 /* 1276 * Migrate anonymous memory to device memory and back to system memory 1277 * multiple times. In case of private zone configuration, this is done 1278 * through fault pages accessed by CPU. In case of coherent zone configuration, 1279 * the pages from the device should be explicitly migrated back to system memory. 1280 * The reason is Coherent device zone has coherent access by CPU, therefore 1281 * it will not generate any page fault. 1282 */ 1283 TEST_F(hmm, migrate_multiple) 1284 { 1285 struct hmm_buffer *buffer; 1286 unsigned long npages; 1287 unsigned long size; 1288 unsigned long i; 1289 unsigned long c; 1290 int *ptr; 1291 int ret; 1292 1293 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1294 ASSERT_NE(npages, 0); 1295 size = npages << self->page_shift; 1296 1297 for (c = 0; c < NTIMES; c++) { 1298 buffer = malloc(sizeof(*buffer)); 1299 ASSERT_NE(buffer, NULL); 1300 1301 buffer->fd = -1; 1302 buffer->size = size; 1303 buffer->mirror = malloc(size); 1304 ASSERT_NE(buffer->mirror, NULL); 1305 1306 buffer->ptr = mmap(NULL, size, 1307 PROT_READ | PROT_WRITE, 1308 MAP_PRIVATE | MAP_ANONYMOUS, 1309 buffer->fd, 0); 1310 ASSERT_NE(buffer->ptr, MAP_FAILED); 1311 1312 /* Initialize buffer in system memory. */ 1313 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1314 ptr[i] = i; 1315 1316 /* Migrate memory to device. */ 1317 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1318 ASSERT_EQ(ret, 0); 1319 ASSERT_EQ(buffer->cpages, npages); 1320 1321 /* Check what the device read. */ 1322 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1323 ASSERT_EQ(ptr[i], i); 1324 1325 /* Migrate back to system memory and check them. */ 1326 if (hmm_is_coherent_type(variant->device_number)) { 1327 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages); 1328 ASSERT_EQ(ret, 0); 1329 ASSERT_EQ(buffer->cpages, npages); 1330 } 1331 1332 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1333 ASSERT_EQ(ptr[i], i); 1334 1335 hmm_buffer_free(buffer); 1336 } 1337 } 1338 1339 /* 1340 * Read anonymous memory multiple times. 1341 */ 1342 TEST_F(hmm, anon_read_multiple) 1343 { 1344 struct hmm_buffer *buffer; 1345 unsigned long npages; 1346 unsigned long size; 1347 unsigned long i; 1348 unsigned long c; 1349 int *ptr; 1350 int ret; 1351 1352 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1353 ASSERT_NE(npages, 0); 1354 size = npages << self->page_shift; 1355 1356 for (c = 0; c < NTIMES; c++) { 1357 buffer = malloc(sizeof(*buffer)); 1358 ASSERT_NE(buffer, NULL); 1359 1360 buffer->fd = -1; 1361 buffer->size = size; 1362 buffer->mirror = malloc(size); 1363 ASSERT_NE(buffer->mirror, NULL); 1364 1365 buffer->ptr = mmap(NULL, size, 1366 PROT_READ | PROT_WRITE, 1367 MAP_PRIVATE | MAP_ANONYMOUS, 1368 buffer->fd, 0); 1369 ASSERT_NE(buffer->ptr, MAP_FAILED); 1370 1371 /* Initialize buffer in system memory. */ 1372 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1373 ptr[i] = i + c; 1374 1375 /* Simulate a device reading system memory. */ 1376 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1377 npages); 1378 ASSERT_EQ(ret, 0); 1379 ASSERT_EQ(buffer->cpages, npages); 1380 ASSERT_EQ(buffer->faults, 1); 1381 1382 /* Check what the device read. */ 1383 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1384 ASSERT_EQ(ptr[i], i + c); 1385 1386 hmm_buffer_free(buffer); 1387 } 1388 } 1389 1390 void *unmap_buffer(void *p) 1391 { 1392 struct hmm_buffer *buffer = p; 1393 1394 /* Delay for a bit and then unmap buffer while it is being read. */ 1395 hmm_nanosleep(hmm_random() % 32000); 1396 munmap(buffer->ptr + buffer->size / 2, buffer->size / 2); 1397 buffer->ptr = NULL; 1398 1399 return NULL; 1400 } 1401 1402 /* 1403 * Try reading anonymous memory while it is being unmapped. 1404 */ 1405 TEST_F(hmm, anon_teardown) 1406 { 1407 unsigned long npages; 1408 unsigned long size; 1409 unsigned long c; 1410 void *ret; 1411 1412 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1413 ASSERT_NE(npages, 0); 1414 size = npages << self->page_shift; 1415 1416 for (c = 0; c < NTIMES; ++c) { 1417 pthread_t thread; 1418 struct hmm_buffer *buffer; 1419 unsigned long i; 1420 int *ptr; 1421 int rc; 1422 1423 buffer = malloc(sizeof(*buffer)); 1424 ASSERT_NE(buffer, NULL); 1425 1426 buffer->fd = -1; 1427 buffer->size = size; 1428 buffer->mirror = malloc(size); 1429 ASSERT_NE(buffer->mirror, NULL); 1430 1431 buffer->ptr = mmap(NULL, size, 1432 PROT_READ | PROT_WRITE, 1433 MAP_PRIVATE | MAP_ANONYMOUS, 1434 buffer->fd, 0); 1435 ASSERT_NE(buffer->ptr, MAP_FAILED); 1436 1437 /* Initialize buffer in system memory. */ 1438 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1439 ptr[i] = i + c; 1440 1441 rc = pthread_create(&thread, NULL, unmap_buffer, buffer); 1442 ASSERT_EQ(rc, 0); 1443 1444 /* Simulate a device reading system memory. */ 1445 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1446 npages); 1447 if (rc == 0) { 1448 ASSERT_EQ(buffer->cpages, npages); 1449 ASSERT_EQ(buffer->faults, 1); 1450 1451 /* Check what the device read. */ 1452 for (i = 0, ptr = buffer->mirror; 1453 i < size / sizeof(*ptr); 1454 ++i) 1455 ASSERT_EQ(ptr[i], i + c); 1456 } 1457 1458 pthread_join(thread, &ret); 1459 hmm_buffer_free(buffer); 1460 } 1461 } 1462 1463 /* 1464 * Test memory snapshot without faulting in pages accessed by the device. 1465 */ 1466 TEST_F(hmm, mixedmap) 1467 { 1468 struct hmm_buffer *buffer; 1469 unsigned long npages; 1470 unsigned long size; 1471 unsigned char *m; 1472 int ret; 1473 1474 npages = 1; 1475 size = npages << self->page_shift; 1476 1477 buffer = malloc(sizeof(*buffer)); 1478 ASSERT_NE(buffer, NULL); 1479 1480 buffer->fd = -1; 1481 buffer->size = size; 1482 buffer->mirror = malloc(npages); 1483 ASSERT_NE(buffer->mirror, NULL); 1484 1485 1486 /* Reserve a range of addresses. */ 1487 buffer->ptr = mmap(NULL, size, 1488 PROT_READ | PROT_WRITE, 1489 MAP_PRIVATE, 1490 self->fd, 0); 1491 ASSERT_NE(buffer->ptr, MAP_FAILED); 1492 1493 /* Simulate a device snapshotting CPU pagetables. */ 1494 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1495 ASSERT_EQ(ret, 0); 1496 ASSERT_EQ(buffer->cpages, npages); 1497 1498 /* Check what the device saw. */ 1499 m = buffer->mirror; 1500 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ); 1501 1502 hmm_buffer_free(buffer); 1503 } 1504 1505 /* 1506 * Test memory snapshot without faulting in pages accessed by the device. 1507 */ 1508 TEST_F(hmm2, snapshot) 1509 { 1510 struct hmm_buffer *buffer; 1511 unsigned long npages; 1512 unsigned long size; 1513 int *ptr; 1514 unsigned char *p; 1515 unsigned char *m; 1516 int ret; 1517 int val; 1518 1519 npages = 7; 1520 size = npages << self->page_shift; 1521 1522 buffer = malloc(sizeof(*buffer)); 1523 ASSERT_NE(buffer, NULL); 1524 1525 buffer->fd = -1; 1526 buffer->size = size; 1527 buffer->mirror = malloc(npages); 1528 ASSERT_NE(buffer->mirror, NULL); 1529 1530 /* Reserve a range of addresses. */ 1531 buffer->ptr = mmap(NULL, size, 1532 PROT_NONE, 1533 MAP_PRIVATE | MAP_ANONYMOUS, 1534 buffer->fd, 0); 1535 ASSERT_NE(buffer->ptr, MAP_FAILED); 1536 p = buffer->ptr; 1537 1538 /* Punch a hole after the first page address. */ 1539 ret = munmap(buffer->ptr + self->page_size, self->page_size); 1540 ASSERT_EQ(ret, 0); 1541 1542 /* Page 2 will be read-only zero page. */ 1543 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, 1544 PROT_READ); 1545 ASSERT_EQ(ret, 0); 1546 ptr = (int *)(buffer->ptr + 2 * self->page_size); 1547 val = *ptr + 3; 1548 ASSERT_EQ(val, 3); 1549 1550 /* Page 3 will be read-only. */ 1551 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1552 PROT_READ | PROT_WRITE); 1553 ASSERT_EQ(ret, 0); 1554 ptr = (int *)(buffer->ptr + 3 * self->page_size); 1555 *ptr = val; 1556 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1557 PROT_READ); 1558 ASSERT_EQ(ret, 0); 1559 1560 /* Page 4-6 will be read-write. */ 1561 ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size, 1562 PROT_READ | PROT_WRITE); 1563 ASSERT_EQ(ret, 0); 1564 ptr = (int *)(buffer->ptr + 4 * self->page_size); 1565 *ptr = val; 1566 1567 /* Page 5 will be migrated to device 0. */ 1568 buffer->ptr = p + 5 * self->page_size; 1569 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); 1570 ASSERT_EQ(ret, 0); 1571 ASSERT_EQ(buffer->cpages, 1); 1572 1573 /* Page 6 will be migrated to device 1. */ 1574 buffer->ptr = p + 6 * self->page_size; 1575 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1); 1576 ASSERT_EQ(ret, 0); 1577 ASSERT_EQ(buffer->cpages, 1); 1578 1579 /* Simulate a device snapshotting CPU pagetables. */ 1580 buffer->ptr = p; 1581 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1582 ASSERT_EQ(ret, 0); 1583 ASSERT_EQ(buffer->cpages, npages); 1584 1585 /* Check what the device saw. */ 1586 m = buffer->mirror; 1587 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR); 1588 ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR); 1589 ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ); 1590 ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ); 1591 ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE); 1592 if (!hmm_is_coherent_type(variant->device_number0)) { 1593 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL | 1594 HMM_DMIRROR_PROT_WRITE); 1595 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE); 1596 } else { 1597 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | 1598 HMM_DMIRROR_PROT_WRITE); 1599 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE | 1600 HMM_DMIRROR_PROT_WRITE); 1601 } 1602 1603 hmm_buffer_free(buffer); 1604 } 1605 1606 /* 1607 * Test the hmm_range_fault() handling of large pages (PMD or PUD) 1608 * that should be mapped by a large page table entry. 1609 */ 1610 TEST_F(hmm, compound) 1611 { 1612 struct hmm_buffer *buffer; 1613 unsigned long npages; 1614 unsigned long size; 1615 unsigned long default_hsize = default_huge_page_size(); 1616 int *ptr; 1617 unsigned char *m; 1618 unsigned char prot; 1619 int ret; 1620 unsigned long i; 1621 1622 /* Skip test if we can't allocate a hugetlbfs page. */ 1623 if (!hugetlb_free_default_pages()) 1624 SKIP(return, "Not enough huge pages"); 1625 1626 size = ALIGN(TWOMEG, default_hsize); 1627 npages = size >> self->page_shift; 1628 1629 buffer = malloc(sizeof(*buffer)); 1630 ASSERT_NE(buffer, NULL); 1631 1632 buffer->ptr = mmap(NULL, size, 1633 PROT_READ | PROT_WRITE, 1634 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 1635 -1, 0); 1636 if (buffer->ptr == MAP_FAILED) { 1637 free(buffer); 1638 return; 1639 } 1640 1641 buffer->size = size; 1642 buffer->mirror = malloc(npages); 1643 ASSERT_NE(buffer->mirror, NULL); 1644 1645 /* Initialize the pages the device will snapshot in buffer->ptr. */ 1646 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1647 ptr[i] = i; 1648 1649 /* Simulate a device snapshotting CPU pagetables. */ 1650 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1651 ASSERT_EQ(ret, 0); 1652 ASSERT_EQ(buffer->cpages, npages); 1653 1654 /* 1655 * Check what the device saw. The region is backed by a single huge 1656 * page that the device reports either at PMD or at PUD level depending 1657 * on the configured default hugepage size. Determine that level from 1658 * the first page and require every page in the range to match it 1659 * exactly, so that a fragmented mapping mixing levels (or a missing 1660 * large-page bit) is still caught and reported with its actual value. 1661 */ 1662 m = buffer->mirror; 1663 prot = HMM_DMIRROR_PROT_WRITE | 1664 ((m[0] & HMM_DMIRROR_PROT_PUD) ? HMM_DMIRROR_PROT_PUD : 1665 HMM_DMIRROR_PROT_PMD); 1666 for (i = 0; i < npages; ++i) 1667 ASSERT_EQ(m[i], prot); 1668 1669 /* Make the region read-only. */ 1670 ret = mprotect(buffer->ptr, size, PROT_READ); 1671 ASSERT_EQ(ret, 0); 1672 1673 /* Simulate a device snapshotting CPU pagetables. */ 1674 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1675 ASSERT_EQ(ret, 0); 1676 ASSERT_EQ(buffer->cpages, npages); 1677 1678 /* 1679 * Check what the device saw after mprotect(PROT_READ). Same 1680 * approach as above: determine the mapping level from the first 1681 * page and require every page to match it exactly. 1682 */ 1683 m = buffer->mirror; 1684 prot = HMM_DMIRROR_PROT_READ | 1685 ((m[0] & HMM_DMIRROR_PROT_PUD) ? HMM_DMIRROR_PROT_PUD : 1686 HMM_DMIRROR_PROT_PMD); 1687 for (i = 0; i < npages; ++i) 1688 ASSERT_EQ(m[i], prot); 1689 1690 munmap(buffer->ptr, buffer->size); 1691 buffer->ptr = NULL; 1692 hmm_buffer_free(buffer); 1693 } 1694 1695 /* 1696 * Test two devices reading the same memory (double mapped). 1697 */ 1698 TEST_F(hmm2, double_map) 1699 { 1700 struct hmm_buffer *buffer; 1701 unsigned long npages; 1702 unsigned long size; 1703 unsigned long i; 1704 int *ptr; 1705 int ret; 1706 1707 npages = 6; 1708 size = npages << self->page_shift; 1709 1710 buffer = malloc(sizeof(*buffer)); 1711 ASSERT_NE(buffer, NULL); 1712 1713 buffer->fd = -1; 1714 buffer->size = size; 1715 buffer->mirror = malloc(size); 1716 ASSERT_NE(buffer->mirror, NULL); 1717 1718 /* Reserve a range of addresses. */ 1719 buffer->ptr = mmap(NULL, size, 1720 PROT_READ | PROT_WRITE, 1721 MAP_PRIVATE | MAP_ANONYMOUS, 1722 buffer->fd, 0); 1723 ASSERT_NE(buffer->ptr, MAP_FAILED); 1724 1725 /* Initialize buffer in system memory. */ 1726 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1727 ptr[i] = i; 1728 1729 /* Make region read-only. */ 1730 ret = mprotect(buffer->ptr, size, PROT_READ); 1731 ASSERT_EQ(ret, 0); 1732 1733 /* Simulate device 0 reading system memory. */ 1734 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); 1735 ASSERT_EQ(ret, 0); 1736 ASSERT_EQ(buffer->cpages, npages); 1737 ASSERT_EQ(buffer->faults, 1); 1738 1739 /* Check what the device read. */ 1740 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1741 ASSERT_EQ(ptr[i], i); 1742 1743 /* Simulate device 1 reading system memory. */ 1744 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages); 1745 ASSERT_EQ(ret, 0); 1746 ASSERT_EQ(buffer->cpages, npages); 1747 ASSERT_EQ(buffer->faults, 1); 1748 1749 /* Check what the device read. */ 1750 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1751 ASSERT_EQ(ptr[i], i); 1752 1753 /* Migrate pages to device 1 and try to read from device 0. */ 1754 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); 1755 ASSERT_EQ(ret, 0); 1756 ASSERT_EQ(buffer->cpages, npages); 1757 1758 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); 1759 ASSERT_EQ(ret, 0); 1760 ASSERT_EQ(buffer->cpages, npages); 1761 ASSERT_EQ(buffer->faults, 1); 1762 1763 /* Check what device 0 read. */ 1764 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1765 ASSERT_EQ(ptr[i], i); 1766 1767 hmm_buffer_free(buffer); 1768 } 1769 1770 /* 1771 * Basic check of exclusive faulting. 1772 */ 1773 TEST_F(hmm, exclusive) 1774 { 1775 struct hmm_buffer *buffer; 1776 unsigned long npages; 1777 unsigned long size; 1778 unsigned long i; 1779 int *ptr; 1780 int ret; 1781 1782 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1783 ASSERT_NE(npages, 0); 1784 size = npages << self->page_shift; 1785 1786 buffer = malloc(sizeof(*buffer)); 1787 ASSERT_NE(buffer, NULL); 1788 1789 buffer->fd = -1; 1790 buffer->size = size; 1791 buffer->mirror = malloc(size); 1792 ASSERT_NE(buffer->mirror, NULL); 1793 1794 buffer->ptr = mmap(NULL, size, 1795 PROT_READ | PROT_WRITE, 1796 MAP_PRIVATE | MAP_ANONYMOUS, 1797 buffer->fd, 0); 1798 ASSERT_NE(buffer->ptr, MAP_FAILED); 1799 1800 /* Initialize buffer in system memory. */ 1801 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1802 ptr[i] = i; 1803 1804 /* Map memory exclusively for device access. */ 1805 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1806 ASSERT_EQ(ret, 0); 1807 ASSERT_EQ(buffer->cpages, npages); 1808 1809 /* Check what the device read. */ 1810 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1811 ASSERT_EQ(ptr[i], i); 1812 1813 /* Fault pages back to system memory and check them. */ 1814 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1815 ASSERT_EQ(ptr[i]++, i); 1816 1817 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1818 ASSERT_EQ(ptr[i], i+1); 1819 1820 /* Check atomic access revoked */ 1821 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages); 1822 ASSERT_EQ(ret, 0); 1823 1824 hmm_buffer_free(buffer); 1825 } 1826 1827 TEST_F(hmm, exclusive_mprotect) 1828 { 1829 struct hmm_buffer *buffer; 1830 unsigned long npages; 1831 unsigned long size; 1832 unsigned long i; 1833 int *ptr; 1834 int ret; 1835 1836 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1837 ASSERT_NE(npages, 0); 1838 size = npages << self->page_shift; 1839 1840 buffer = malloc(sizeof(*buffer)); 1841 ASSERT_NE(buffer, NULL); 1842 1843 buffer->fd = -1; 1844 buffer->size = size; 1845 buffer->mirror = malloc(size); 1846 ASSERT_NE(buffer->mirror, NULL); 1847 1848 buffer->ptr = mmap(NULL, size, 1849 PROT_READ | PROT_WRITE, 1850 MAP_PRIVATE | MAP_ANONYMOUS, 1851 buffer->fd, 0); 1852 ASSERT_NE(buffer->ptr, MAP_FAILED); 1853 1854 /* Initialize buffer in system memory. */ 1855 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1856 ptr[i] = i; 1857 1858 /* Map memory exclusively for device access. */ 1859 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1860 ASSERT_EQ(ret, 0); 1861 ASSERT_EQ(buffer->cpages, npages); 1862 1863 /* Check what the device read. */ 1864 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1865 ASSERT_EQ(ptr[i], i); 1866 1867 ret = mprotect(buffer->ptr, size, PROT_READ); 1868 ASSERT_EQ(ret, 0); 1869 1870 /* Simulate a device writing system memory. */ 1871 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 1872 ASSERT_EQ(ret, -EPERM); 1873 1874 hmm_buffer_free(buffer); 1875 } 1876 1877 /* 1878 * Check copy-on-write works. 1879 */ 1880 TEST_F(hmm, exclusive_cow) 1881 { 1882 struct hmm_buffer *buffer; 1883 unsigned long npages; 1884 unsigned long size; 1885 unsigned long i; 1886 int *ptr; 1887 int ret; 1888 pid_t pid; 1889 int status; 1890 1891 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1892 ASSERT_NE(npages, 0); 1893 size = npages << self->page_shift; 1894 1895 buffer = malloc(sizeof(*buffer)); 1896 ASSERT_NE(buffer, NULL); 1897 1898 buffer->fd = -1; 1899 buffer->size = size; 1900 buffer->mirror = malloc(size); 1901 ASSERT_NE(buffer->mirror, NULL); 1902 1903 buffer->ptr = mmap(NULL, size, 1904 PROT_READ | PROT_WRITE, 1905 MAP_PRIVATE | MAP_ANONYMOUS, 1906 buffer->fd, 0); 1907 ASSERT_NE(buffer->ptr, MAP_FAILED); 1908 1909 /* Initialize buffer in system memory. */ 1910 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1911 ptr[i] = i; 1912 1913 /* Map memory exclusively for device access. */ 1914 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1915 ASSERT_EQ(ret, 0); 1916 ASSERT_EQ(buffer->cpages, npages); 1917 1918 pid = fork(); 1919 if (pid == -1) 1920 ASSERT_EQ(pid, 0); 1921 1922 if (pid == 0) { 1923 /* 1924 * Child verifies COW independently, then _exit(0)s so it does 1925 * not run the test teardown. A failed ASSERT_* here makes the 1926 * harness abort() the child, so the parent sees 1927 * !WIFEXITED(status) below and fails in turn. 1928 */ 1929 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1930 ASSERT_EQ(ptr[i]++, i); 1931 1932 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1933 ASSERT_EQ(ptr[i], i + 1); 1934 1935 _exit(0); 1936 } 1937 1938 /* Parent: also increment to verify COW works for both processes. */ 1939 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1940 ASSERT_EQ(ptr[i]++, i); 1941 1942 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1943 ASSERT_EQ(ptr[i], i + 1); 1944 1945 /* Parent: wait for child and then free the buffer. */ 1946 ASSERT_EQ(waitpid(pid, &status, 0), pid); 1947 ASSERT_TRUE(WIFEXITED(status)); 1948 ASSERT_EQ(WEXITSTATUS(status), 0); 1949 1950 hmm_buffer_free(buffer); 1951 } 1952 1953 static int gup_test_exec(int gup_fd, unsigned long addr, int cmd, 1954 int npages, int size, int flags) 1955 { 1956 struct gup_test gup = { 1957 .nr_pages_per_call = npages, 1958 .addr = addr, 1959 .gup_flags = FOLL_WRITE | flags, 1960 .size = size, 1961 }; 1962 1963 if (ioctl(gup_fd, cmd, &gup)) { 1964 perror("ioctl on error\n"); 1965 return errno; 1966 } 1967 1968 return 0; 1969 } 1970 1971 /* 1972 * Test get user device pages through gup_test. Setting PIN_LONGTERM flag. 1973 * This should trigger a migration back to system memory for both, private 1974 * and coherent type pages. 1975 * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added 1976 * to your configuration before you run it. 1977 */ 1978 TEST_F(hmm, hmm_gup_test) 1979 { 1980 struct hmm_buffer *buffer; 1981 int gup_fd; 1982 unsigned long npages; 1983 unsigned long size; 1984 unsigned long i; 1985 int *ptr; 1986 int ret; 1987 unsigned char *m; 1988 1989 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); 1990 if (gup_fd == -1) 1991 SKIP(return, "Skipping test, could not find gup_test driver"); 1992 1993 npages = 4; 1994 size = npages << self->page_shift; 1995 1996 buffer = malloc(sizeof(*buffer)); 1997 ASSERT_NE(buffer, NULL); 1998 1999 buffer->fd = -1; 2000 buffer->size = size; 2001 buffer->mirror = malloc(size); 2002 ASSERT_NE(buffer->mirror, NULL); 2003 2004 buffer->ptr = mmap(NULL, size, 2005 PROT_READ | PROT_WRITE, 2006 MAP_PRIVATE | MAP_ANONYMOUS, 2007 buffer->fd, 0); 2008 ASSERT_NE(buffer->ptr, MAP_FAILED); 2009 2010 /* Initialize buffer in system memory. */ 2011 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2012 ptr[i] = i; 2013 2014 /* Migrate memory to device. */ 2015 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2016 ASSERT_EQ(ret, 0); 2017 ASSERT_EQ(buffer->cpages, npages); 2018 /* Check what the device read. */ 2019 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2020 ASSERT_EQ(ptr[i], i); 2021 2022 ASSERT_EQ(gup_test_exec(gup_fd, 2023 (unsigned long)buffer->ptr, 2024 GUP_BASIC_TEST, 1, self->page_size, 0), 0); 2025 ASSERT_EQ(gup_test_exec(gup_fd, 2026 (unsigned long)buffer->ptr + 1 * self->page_size, 2027 GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0); 2028 ASSERT_EQ(gup_test_exec(gup_fd, 2029 (unsigned long)buffer->ptr + 2 * self->page_size, 2030 PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0); 2031 ASSERT_EQ(gup_test_exec(gup_fd, 2032 (unsigned long)buffer->ptr + 3 * self->page_size, 2033 PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0); 2034 2035 /* Take snapshot to CPU pagetables */ 2036 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 2037 ASSERT_EQ(ret, 0); 2038 ASSERT_EQ(buffer->cpages, npages); 2039 m = buffer->mirror; 2040 if (hmm_is_coherent_type(variant->device_number)) { 2041 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]); 2042 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]); 2043 } else { 2044 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]); 2045 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]); 2046 } 2047 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]); 2048 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]); 2049 /* 2050 * Check again the content on the pages. Make sure there's no 2051 * corrupted data. 2052 */ 2053 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2054 ASSERT_EQ(ptr[i], i); 2055 2056 close(gup_fd); 2057 hmm_buffer_free(buffer); 2058 } 2059 2060 /* 2061 * Test copy-on-write in device pages. 2062 * In case of writing to COW private page(s), a page fault will migrate pages 2063 * back to system memory first. Then, these pages will be duplicated. In case 2064 * of COW device coherent type, pages are duplicated directly from device 2065 * memory. 2066 */ 2067 TEST_F(hmm, hmm_cow_in_device) 2068 { 2069 struct hmm_buffer *buffer; 2070 unsigned long npages; 2071 unsigned long size; 2072 unsigned long i; 2073 int *ptr; 2074 int ret; 2075 unsigned char *m; 2076 pid_t pid; 2077 int status; 2078 2079 npages = 4; 2080 size = npages << self->page_shift; 2081 2082 buffer = malloc(sizeof(*buffer)); 2083 ASSERT_NE(buffer, NULL); 2084 2085 buffer->fd = -1; 2086 buffer->size = size; 2087 buffer->mirror = malloc(size); 2088 ASSERT_NE(buffer->mirror, NULL); 2089 2090 buffer->ptr = mmap(NULL, size, 2091 PROT_READ | PROT_WRITE, 2092 MAP_PRIVATE | MAP_ANONYMOUS, 2093 buffer->fd, 0); 2094 ASSERT_NE(buffer->ptr, MAP_FAILED); 2095 2096 /* Initialize buffer in system memory. */ 2097 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2098 ptr[i] = i; 2099 2100 /* Migrate memory to device. */ 2101 2102 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2103 ASSERT_EQ(ret, 0); 2104 ASSERT_EQ(buffer->cpages, npages); 2105 2106 pid = fork(); 2107 if (pid == -1) 2108 ASSERT_EQ(pid, 0); 2109 if (!pid) { 2110 /* Child process waits for SIGKILL from the parent. */ 2111 while (1) { 2112 } 2113 /* Should not reach this */ 2114 } 2115 /* Parent process writes to COW pages(s) and gets a 2116 * new copy in system. In case of device private pages, 2117 * this write causes a migration to system mem first. 2118 */ 2119 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2120 ptr[i] = i; 2121 2122 /* Terminate child and wait */ 2123 EXPECT_EQ(0, kill(pid, SIGKILL)); 2124 EXPECT_EQ(pid, waitpid(pid, &status, 0)); 2125 EXPECT_NE(0, WIFSIGNALED(status)); 2126 EXPECT_EQ(SIGKILL, WTERMSIG(status)); 2127 2128 /* Take snapshot to CPU pagetables */ 2129 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 2130 ASSERT_EQ(ret, 0); 2131 ASSERT_EQ(buffer->cpages, npages); 2132 m = buffer->mirror; 2133 for (i = 0; i < npages; i++) 2134 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]); 2135 2136 hmm_buffer_free(buffer); 2137 } 2138 2139 /* 2140 * Migrate private anonymous huge empty page. 2141 */ 2142 TEST_F(hmm, migrate_anon_huge_empty) 2143 { 2144 struct hmm_buffer *buffer; 2145 unsigned long npages; 2146 unsigned long size; 2147 unsigned long i; 2148 void *old_ptr; 2149 void *map; 2150 int *ptr; 2151 int ret; 2152 2153 size = read_pmd_pagesize(); 2154 2155 buffer = malloc(sizeof(*buffer)); 2156 ASSERT_NE(buffer, NULL); 2157 2158 buffer->fd = -1; 2159 buffer->size = 2 * size; 2160 buffer->mirror = malloc(size); 2161 ASSERT_NE(buffer->mirror, NULL); 2162 memset(buffer->mirror, 0xFF, size); 2163 2164 buffer->ptr = mmap(NULL, 2 * size, 2165 PROT_READ, 2166 MAP_PRIVATE | MAP_ANONYMOUS, 2167 buffer->fd, 0); 2168 ASSERT_NE(buffer->ptr, MAP_FAILED); 2169 2170 npages = size >> self->page_shift; 2171 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2172 ret = madvise(map, size, MADV_HUGEPAGE); 2173 ASSERT_EQ(ret, 0); 2174 old_ptr = buffer->ptr; 2175 buffer->ptr = map; 2176 2177 /* Migrate memory to device. */ 2178 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2179 ASSERT_EQ(ret, 0); 2180 ASSERT_EQ(buffer->cpages, npages); 2181 2182 /* Check what the device read. */ 2183 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2184 ASSERT_EQ(ptr[i], 0); 2185 2186 buffer->ptr = old_ptr; 2187 hmm_buffer_free(buffer); 2188 } 2189 2190 /* 2191 * Migrate private anonymous huge zero page. 2192 */ 2193 TEST_F(hmm, migrate_anon_huge_zero) 2194 { 2195 struct hmm_buffer *buffer; 2196 unsigned long npages; 2197 unsigned long size; 2198 unsigned long i; 2199 void *old_ptr; 2200 void *map; 2201 int *ptr; 2202 int ret; 2203 int val; 2204 2205 size = read_pmd_pagesize(); 2206 2207 buffer = malloc(sizeof(*buffer)); 2208 ASSERT_NE(buffer, NULL); 2209 2210 buffer->fd = -1; 2211 buffer->size = 2 * size; 2212 buffer->mirror = malloc(size); 2213 ASSERT_NE(buffer->mirror, NULL); 2214 memset(buffer->mirror, 0xFF, size); 2215 2216 buffer->ptr = mmap(NULL, 2 * size, 2217 PROT_READ, 2218 MAP_PRIVATE | MAP_ANONYMOUS, 2219 buffer->fd, 0); 2220 ASSERT_NE(buffer->ptr, MAP_FAILED); 2221 2222 npages = size >> self->page_shift; 2223 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2224 ret = madvise(map, size, MADV_HUGEPAGE); 2225 ASSERT_EQ(ret, 0); 2226 old_ptr = buffer->ptr; 2227 buffer->ptr = map; 2228 2229 /* Initialize a read-only zero huge page. */ 2230 val = *(int *)buffer->ptr; 2231 ASSERT_EQ(val, 0); 2232 2233 /* Migrate memory to device. */ 2234 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2235 ASSERT_EQ(ret, 0); 2236 ASSERT_EQ(buffer->cpages, npages); 2237 2238 /* Check what the device read. */ 2239 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2240 ASSERT_EQ(ptr[i], 0); 2241 2242 /* Fault pages back to system memory and check them. */ 2243 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) { 2244 ASSERT_EQ(ptr[i], 0); 2245 /* If it asserts once, it probably will 500,000 times */ 2246 if (ptr[i] != 0) 2247 break; 2248 } 2249 2250 buffer->ptr = old_ptr; 2251 hmm_buffer_free(buffer); 2252 } 2253 2254 /* 2255 * Migrate private anonymous huge page and free. 2256 */ 2257 TEST_F(hmm, migrate_anon_huge_free) 2258 { 2259 struct hmm_buffer *buffer; 2260 unsigned long npages; 2261 unsigned long size; 2262 unsigned long i; 2263 void *old_ptr; 2264 void *map; 2265 int *ptr; 2266 int ret; 2267 2268 size = read_pmd_pagesize(); 2269 2270 buffer = malloc(sizeof(*buffer)); 2271 ASSERT_NE(buffer, NULL); 2272 2273 buffer->fd = -1; 2274 buffer->size = 2 * size; 2275 buffer->mirror = malloc(size); 2276 ASSERT_NE(buffer->mirror, NULL); 2277 memset(buffer->mirror, 0xFF, size); 2278 2279 buffer->ptr = mmap(NULL, 2 * size, 2280 PROT_READ | PROT_WRITE, 2281 MAP_PRIVATE | MAP_ANONYMOUS, 2282 buffer->fd, 0); 2283 ASSERT_NE(buffer->ptr, MAP_FAILED); 2284 2285 npages = size >> self->page_shift; 2286 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2287 ret = madvise(map, size, MADV_HUGEPAGE); 2288 ASSERT_EQ(ret, 0); 2289 old_ptr = buffer->ptr; 2290 buffer->ptr = map; 2291 2292 /* Initialize buffer in system memory. */ 2293 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2294 ptr[i] = i; 2295 2296 /* Migrate memory to device. */ 2297 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2298 ASSERT_EQ(ret, 0); 2299 ASSERT_EQ(buffer->cpages, npages); 2300 2301 /* Check what the device read. */ 2302 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2303 ASSERT_EQ(ptr[i], i); 2304 2305 /* Try freeing it. */ 2306 ret = madvise(map, size, MADV_FREE); 2307 ASSERT_EQ(ret, 0); 2308 2309 buffer->ptr = old_ptr; 2310 hmm_buffer_free(buffer); 2311 } 2312 2313 /* 2314 * Migrate private anonymous huge page and fault back to sysmem. 2315 */ 2316 TEST_F(hmm, migrate_anon_huge_fault) 2317 { 2318 struct hmm_buffer *buffer; 2319 unsigned long npages; 2320 unsigned long size; 2321 unsigned long i; 2322 unsigned char *m; 2323 uint64_t entry; 2324 void *old_ptr; 2325 void *map; 2326 int pagemap_fd; 2327 int *ptr; 2328 int ret; 2329 2330 size = read_pmd_pagesize(); 2331 2332 buffer = malloc(sizeof(*buffer)); 2333 ASSERT_NE(buffer, NULL); 2334 2335 buffer->fd = -1; 2336 buffer->size = 2 * size; 2337 buffer->mirror = malloc(size); 2338 ASSERT_NE(buffer->mirror, NULL); 2339 memset(buffer->mirror, 0xFF, size); 2340 2341 buffer->ptr = mmap(NULL, 2 * size, 2342 PROT_READ | PROT_WRITE, 2343 MAP_PRIVATE | MAP_ANONYMOUS, 2344 buffer->fd, 0); 2345 ASSERT_NE(buffer->ptr, MAP_FAILED); 2346 2347 npages = size >> self->page_shift; 2348 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2349 old_ptr = buffer->ptr; 2350 buffer->ptr = map; 2351 2352 /* Initialize buffer in system memory. */ 2353 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2354 ptr[i] = i; 2355 2356 ret = madvise(map, size, MADV_COLLAPSE); 2357 ASSERT_EQ(ret, 0); 2358 2359 /* Migrate memory to device. */ 2360 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2361 ASSERT_EQ(ret, 0); 2362 ASSERT_EQ(buffer->cpages, npages); 2363 2364 /* Check what the device read. */ 2365 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2366 ASSERT_EQ(ptr[i], i); 2367 2368 if (!hmm_is_coherent_type(variant->device_number)) { 2369 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, 2370 buffer, npages); 2371 ASSERT_EQ(ret, 0); 2372 ASSERT_EQ(buffer->cpages, npages); 2373 2374 m = buffer->mirror; 2375 for (i = 0; i < npages; ++i) 2376 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL | 2377 HMM_DMIRROR_PROT_WRITE | 2378 HMM_DMIRROR_PROT_PMD); 2379 2380 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 2381 ASSERT_GE(pagemap_fd, 0); 2382 2383 for (i = 0; i < npages; ++i) { 2384 entry = pagemap_get_entry(pagemap_fd, 2385 (char *)buffer->ptr + i * self->page_size); 2386 2387 ASSERT_NE(entry & PM_SWAP, 0); 2388 ASSERT_FALSE(PAGEMAP_PRESENT(entry)); 2389 } 2390 2391 close(pagemap_fd); 2392 } 2393 2394 /* Fault pages back to system memory and check them. */ 2395 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2396 ASSERT_EQ(ptr[i], i); 2397 2398 buffer->ptr = old_ptr; 2399 hmm_buffer_free(buffer); 2400 } 2401 2402 /* 2403 * Migrate memory and fault back to sysmem after partially unmapping. 2404 */ 2405 TEST_F(hmm, migrate_partial_unmap_fault) 2406 { 2407 struct hmm_buffer *buffer; 2408 unsigned long npages; 2409 unsigned long size = read_pmd_pagesize(); 2410 unsigned long unmap_size; 2411 unsigned long offsets[3]; 2412 unsigned long i; 2413 void *old_ptr; 2414 void *map; 2415 int *ptr; 2416 int ret, j, use_thp; 2417 2418 if (!size) 2419 size = TWOMEG; 2420 2421 unmap_size = size / 2; 2422 offsets[0] = 0; 2423 offsets[1] = size / 4; 2424 offsets[2] = size / 2; 2425 2426 for (use_thp = 0; use_thp < 2; ++use_thp) { 2427 for (j = 0; j < ARRAY_SIZE(offsets); ++j) { 2428 buffer = malloc(sizeof(*buffer)); 2429 ASSERT_NE(buffer, NULL); 2430 2431 buffer->fd = -1; 2432 buffer->size = 2 * size; 2433 buffer->mirror = malloc(size); 2434 ASSERT_NE(buffer->mirror, NULL); 2435 memset(buffer->mirror, 0xFF, size); 2436 2437 buffer->ptr = mmap(NULL, 2 * size, 2438 PROT_READ | PROT_WRITE, 2439 MAP_PRIVATE | MAP_ANONYMOUS, 2440 buffer->fd, 0); 2441 ASSERT_NE(buffer->ptr, MAP_FAILED); 2442 2443 npages = size >> self->page_shift; 2444 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2445 if (use_thp) 2446 ret = madvise(map, size, MADV_HUGEPAGE); 2447 else 2448 ret = madvise(map, size, MADV_NOHUGEPAGE); 2449 ASSERT_EQ(ret, 0); 2450 old_ptr = buffer->ptr; 2451 buffer->ptr = map; 2452 2453 /* Initialize buffer in system memory. */ 2454 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2455 ptr[i] = i; 2456 2457 /* Migrate memory to device. */ 2458 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2459 ASSERT_EQ(ret, 0); 2460 ASSERT_EQ(buffer->cpages, npages); 2461 2462 /* Check what the device read. */ 2463 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2464 ASSERT_EQ(ptr[i], i); 2465 2466 munmap(buffer->ptr + offsets[j], unmap_size); 2467 2468 /* Fault pages back to system memory and check them. */ 2469 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2470 if (i * sizeof(int) < offsets[j] || 2471 i * sizeof(int) >= offsets[j] + unmap_size) 2472 ASSERT_EQ(ptr[i], i); 2473 2474 buffer->ptr = old_ptr; 2475 hmm_buffer_free(buffer); 2476 } 2477 } 2478 } 2479 2480 TEST_F(hmm, migrate_remap_fault) 2481 { 2482 struct hmm_buffer *buffer; 2483 unsigned long npages; 2484 unsigned long size = read_pmd_pagesize(); 2485 unsigned long offsets[3]; 2486 unsigned long i; 2487 void *old_ptr, *new_ptr = NULL; 2488 void *map; 2489 int *ptr; 2490 int ret, j, use_thp, dont_unmap, before; 2491 2492 if (!size) 2493 size = TWOMEG; 2494 2495 offsets[0] = 0; 2496 offsets[1] = size / 4; 2497 offsets[2] = size / 2; 2498 2499 for (before = 0; before < 2; ++before) { 2500 for (dont_unmap = 0; dont_unmap < 2; ++dont_unmap) { 2501 for (use_thp = 0; use_thp < 2; ++use_thp) { 2502 for (j = 0; j < ARRAY_SIZE(offsets); ++j) { 2503 int flags = MREMAP_MAYMOVE | MREMAP_FIXED; 2504 2505 if (dont_unmap) 2506 flags |= MREMAP_DONTUNMAP; 2507 2508 buffer = malloc(sizeof(*buffer)); 2509 ASSERT_NE(buffer, NULL); 2510 2511 buffer->fd = -1; 2512 buffer->size = 8 * size; 2513 buffer->mirror = malloc(size); 2514 ASSERT_NE(buffer->mirror, NULL); 2515 memset(buffer->mirror, 0xFF, size); 2516 2517 buffer->ptr = mmap(NULL, buffer->size, 2518 PROT_READ | PROT_WRITE, 2519 MAP_PRIVATE | MAP_ANONYMOUS, 2520 buffer->fd, 0); 2521 ASSERT_NE(buffer->ptr, MAP_FAILED); 2522 2523 npages = size >> self->page_shift; 2524 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2525 if (use_thp) 2526 ret = madvise(map, size, MADV_HUGEPAGE); 2527 else 2528 ret = madvise(map, size, MADV_NOHUGEPAGE); 2529 ASSERT_EQ(ret, 0); 2530 old_ptr = buffer->ptr; 2531 munmap(map + size, size * 2); 2532 buffer->ptr = map; 2533 2534 /* Initialize buffer in system memory. */ 2535 for (i = 0, ptr = buffer->ptr; 2536 i < size / sizeof(*ptr); ++i) 2537 ptr[i] = i; 2538 2539 if (before) { 2540 new_ptr = mremap((void *)map, size, size, flags, 2541 map + size + offsets[j]); 2542 ASSERT_NE(new_ptr, MAP_FAILED); 2543 buffer->ptr = new_ptr; 2544 } 2545 2546 /* Migrate memory to device. */ 2547 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2548 ASSERT_EQ(ret, 0); 2549 ASSERT_EQ(buffer->cpages, npages); 2550 2551 /* Check what the device read. */ 2552 for (i = 0, ptr = buffer->mirror; 2553 i < size / sizeof(*ptr); ++i) 2554 ASSERT_EQ(ptr[i], i); 2555 2556 if (!before) { 2557 new_ptr = mremap((void *)map, size, size, flags, 2558 map + size + offsets[j]); 2559 ASSERT_NE(new_ptr, MAP_FAILED); 2560 buffer->ptr = new_ptr; 2561 } 2562 2563 /* Fault pages back to system memory and check them. */ 2564 for (i = 0, ptr = buffer->ptr; 2565 i < size / sizeof(*ptr); ++i) 2566 ASSERT_EQ(ptr[i], i); 2567 2568 munmap(new_ptr, size); 2569 buffer->ptr = old_ptr; 2570 hmm_buffer_free(buffer); 2571 } 2572 } 2573 } 2574 } 2575 } 2576 2577 /* 2578 * Migrate private anonymous huge page with allocation errors. 2579 */ 2580 TEST_F(hmm, migrate_anon_huge_err) 2581 { 2582 struct hmm_buffer *buffer; 2583 unsigned long npages; 2584 unsigned long size; 2585 unsigned long i; 2586 void *old_ptr; 2587 void *map; 2588 int *ptr; 2589 int ret; 2590 2591 size = read_pmd_pagesize(); 2592 2593 buffer = malloc(sizeof(*buffer)); 2594 ASSERT_NE(buffer, NULL); 2595 2596 buffer->fd = -1; 2597 buffer->size = 2 * size; 2598 buffer->mirror = malloc(2 * size); 2599 ASSERT_NE(buffer->mirror, NULL); 2600 memset(buffer->mirror, 0xFF, 2 * size); 2601 2602 old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, 2603 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2604 ASSERT_NE(old_ptr, MAP_FAILED); 2605 2606 npages = size >> self->page_shift; 2607 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2608 ret = madvise(map, size, MADV_HUGEPAGE); 2609 ASSERT_EQ(ret, 0); 2610 buffer->ptr = map; 2611 2612 /* Initialize buffer in system memory. */ 2613 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2614 ptr[i] = i; 2615 2616 /* Migrate memory to device but force a THP allocation error. */ 2617 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2618 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2619 ASSERT_EQ(ret, 0); 2620 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2621 ASSERT_EQ(ret, 0); 2622 ASSERT_EQ(buffer->cpages, npages); 2623 2624 /* Check what the device read. */ 2625 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) { 2626 ASSERT_EQ(ptr[i], i); 2627 if (ptr[i] != i) 2628 break; 2629 } 2630 2631 /* Try faulting back a single (PAGE_SIZE) page. */ 2632 ptr = buffer->ptr; 2633 ASSERT_EQ(ptr[2048], 2048); 2634 2635 /* unmap and remap the region to reset things. */ 2636 ret = munmap(old_ptr, 2 * size); 2637 ASSERT_EQ(ret, 0); 2638 old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, 2639 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2640 ASSERT_NE(old_ptr, MAP_FAILED); 2641 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2642 ret = madvise(map, size, MADV_HUGEPAGE); 2643 ASSERT_EQ(ret, 0); 2644 buffer->ptr = map; 2645 2646 /* Initialize buffer in system memory. */ 2647 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2648 ptr[i] = i; 2649 2650 /* Migrate THP to device. */ 2651 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2652 ASSERT_EQ(ret, 0); 2653 ASSERT_EQ(buffer->cpages, npages); 2654 2655 /* 2656 * Force an allocation error when faulting back a THP resident in the 2657 * device. 2658 */ 2659 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2660 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2661 ASSERT_EQ(ret, 0); 2662 2663 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages); 2664 ASSERT_EQ(ret, 0); 2665 ptr = buffer->ptr; 2666 ASSERT_EQ(ptr[2048], 2048); 2667 2668 buffer->ptr = old_ptr; 2669 hmm_buffer_free(buffer); 2670 } 2671 2672 /* 2673 * Migrate private anonymous huge zero page with allocation errors. 2674 */ 2675 TEST_F(hmm, migrate_anon_huge_zero_err) 2676 { 2677 struct hmm_buffer *buffer; 2678 unsigned long npages; 2679 unsigned long size; 2680 unsigned long i; 2681 void *old_ptr; 2682 void *map; 2683 int *ptr; 2684 int ret; 2685 2686 size = read_pmd_pagesize(); 2687 2688 buffer = malloc(sizeof(*buffer)); 2689 ASSERT_NE(buffer, NULL); 2690 2691 buffer->fd = -1; 2692 buffer->size = 2 * size; 2693 buffer->mirror = malloc(2 * size); 2694 ASSERT_NE(buffer->mirror, NULL); 2695 memset(buffer->mirror, 0xFF, 2 * size); 2696 2697 old_ptr = mmap(NULL, 2 * size, PROT_READ, 2698 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2699 ASSERT_NE(old_ptr, MAP_FAILED); 2700 2701 npages = size >> self->page_shift; 2702 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2703 ret = madvise(map, size, MADV_HUGEPAGE); 2704 ASSERT_EQ(ret, 0); 2705 buffer->ptr = map; 2706 2707 /* Migrate memory to device but force a THP allocation error. */ 2708 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2709 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2710 ASSERT_EQ(ret, 0); 2711 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2712 ASSERT_EQ(ret, 0); 2713 ASSERT_EQ(buffer->cpages, npages); 2714 2715 /* Check what the device read. */ 2716 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2717 ASSERT_EQ(ptr[i], 0); 2718 2719 /* Try faulting back a single (PAGE_SIZE) page. */ 2720 ptr = buffer->ptr; 2721 ASSERT_EQ(ptr[2048], 0); 2722 2723 /* unmap and remap the region to reset things. */ 2724 ret = munmap(old_ptr, 2 * size); 2725 ASSERT_EQ(ret, 0); 2726 old_ptr = mmap(NULL, 2 * size, PROT_READ, 2727 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2728 ASSERT_NE(old_ptr, MAP_FAILED); 2729 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2730 ret = madvise(map, size, MADV_HUGEPAGE); 2731 ASSERT_EQ(ret, 0); 2732 buffer->ptr = map; 2733 2734 /* Initialize buffer in system memory (zero THP page). */ 2735 ret = ptr[0]; 2736 ASSERT_EQ(ret, 0); 2737 2738 /* Migrate memory to device but force a THP allocation error. */ 2739 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2740 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2741 ASSERT_EQ(ret, 0); 2742 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2743 ASSERT_EQ(ret, 0); 2744 ASSERT_EQ(buffer->cpages, npages); 2745 2746 /* Fault the device memory back and check it. */ 2747 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2748 ASSERT_EQ(ptr[i], 0); 2749 2750 buffer->ptr = old_ptr; 2751 hmm_buffer_free(buffer); 2752 } 2753 2754 struct benchmark_results { 2755 double sys_to_dev_time; 2756 double dev_to_sys_time; 2757 double throughput_s2d; 2758 double throughput_d2s; 2759 }; 2760 2761 static double get_time_ms(void) 2762 { 2763 struct timeval tv; 2764 2765 gettimeofday(&tv, NULL); 2766 return (tv.tv_sec * 1000.0) + (tv.tv_usec / 1000.0); 2767 } 2768 2769 static inline struct hmm_buffer *hmm_buffer_alloc(unsigned long size) 2770 { 2771 struct hmm_buffer *buffer; 2772 2773 buffer = malloc(sizeof(*buffer)); 2774 2775 buffer->fd = -1; 2776 buffer->size = size; 2777 buffer->mirror = malloc(size); 2778 memset(buffer->mirror, 0xFF, size); 2779 return buffer; 2780 } 2781 2782 static void print_benchmark_results(const char *test_name, size_t buffer_size, 2783 struct benchmark_results *thp, 2784 struct benchmark_results *regular) 2785 { 2786 double s2d_improvement = ((regular->sys_to_dev_time - thp->sys_to_dev_time) / 2787 regular->sys_to_dev_time) * 100.0; 2788 double d2s_improvement = ((regular->dev_to_sys_time - thp->dev_to_sys_time) / 2789 regular->dev_to_sys_time) * 100.0; 2790 double throughput_s2d_improvement = ((thp->throughput_s2d - regular->throughput_s2d) / 2791 regular->throughput_s2d) * 100.0; 2792 double throughput_d2s_improvement = ((thp->throughput_d2s - regular->throughput_d2s) / 2793 regular->throughput_d2s) * 100.0; 2794 2795 printf("\n=== %s (%.1f MB) ===\n", test_name, buffer_size / (1024.0 * 1024.0)); 2796 printf(" | With THP | Without THP | Improvement\n"); 2797 printf("---------------------------------------------------------------------\n"); 2798 printf("Sys->Dev Migration | %.3f ms | %.3f ms | %.1f%%\n", 2799 thp->sys_to_dev_time, regular->sys_to_dev_time, s2d_improvement); 2800 printf("Dev->Sys Migration | %.3f ms | %.3f ms | %.1f%%\n", 2801 thp->dev_to_sys_time, regular->dev_to_sys_time, d2s_improvement); 2802 printf("S->D Throughput | %.2f GB/s | %.2f GB/s | %.1f%%\n", 2803 thp->throughput_s2d, regular->throughput_s2d, throughput_s2d_improvement); 2804 printf("D->S Throughput | %.2f GB/s | %.2f GB/s | %.1f%%\n", 2805 thp->throughput_d2s, regular->throughput_d2s, throughput_d2s_improvement); 2806 } 2807 2808 /* 2809 * Run a single migration benchmark 2810 * fd: file descriptor for hmm device 2811 * use_thp: whether to use THP 2812 * buffer_size: size of buffer to allocate 2813 * iterations: number of iterations 2814 * results: where to store results 2815 */ 2816 static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_size, 2817 int iterations, struct benchmark_results *results) 2818 { 2819 struct hmm_buffer *buffer; 2820 unsigned long npages = buffer_size / sysconf(_SC_PAGESIZE); 2821 double start, end; 2822 double s2d_total = 0, d2s_total = 0; 2823 int ret, i; 2824 int *ptr; 2825 2826 buffer = hmm_buffer_alloc(buffer_size); 2827 2828 /* Map memory */ 2829 buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE, 2830 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 2831 2832 if (buffer->ptr == MAP_FAILED) 2833 return -1; 2834 2835 /* Apply THP hint if requested */ 2836 if (use_thp) 2837 ret = madvise(buffer->ptr, buffer_size, MADV_HUGEPAGE); 2838 else 2839 ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE); 2840 2841 if (ret) 2842 return ret; 2843 2844 /* Initialize memory to make sure pages are allocated */ 2845 ptr = (int *)buffer->ptr; 2846 for (i = 0; i < buffer_size / sizeof(int); i++) 2847 ptr[i] = i & 0xFF; 2848 2849 /* Warmup iteration */ 2850 ret = hmm_migrate_sys_to_dev(fd, buffer, npages); 2851 if (ret) 2852 return ret; 2853 2854 ret = hmm_migrate_dev_to_sys(fd, buffer, npages); 2855 if (ret) 2856 return ret; 2857 2858 /* Benchmark iterations */ 2859 for (i = 0; i < iterations; i++) { 2860 /* System to device migration */ 2861 start = get_time_ms(); 2862 2863 ret = hmm_migrate_sys_to_dev(fd, buffer, npages); 2864 if (ret) 2865 return ret; 2866 2867 end = get_time_ms(); 2868 s2d_total += (end - start); 2869 2870 /* Device to system migration */ 2871 start = get_time_ms(); 2872 2873 ret = hmm_migrate_dev_to_sys(fd, buffer, npages); 2874 if (ret) 2875 return ret; 2876 2877 end = get_time_ms(); 2878 d2s_total += (end - start); 2879 } 2880 2881 /* Calculate average times and throughput */ 2882 results->sys_to_dev_time = s2d_total / iterations; 2883 results->dev_to_sys_time = d2s_total / iterations; 2884 results->throughput_s2d = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / 2885 (results->sys_to_dev_time / 1000.0); 2886 results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / 2887 (results->dev_to_sys_time / 1000.0); 2888 2889 /* Cleanup */ 2890 hmm_buffer_free(buffer); 2891 return 0; 2892 } 2893 2894 /* 2895 * Benchmark THP migration with different buffer sizes 2896 */ 2897 TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120) 2898 { 2899 struct benchmark_results thp_results, regular_results; 2900 size_t thp_size = read_pmd_pagesize(); 2901 int iterations = 5; 2902 2903 if (!thp_size) 2904 thp_size = TWOMEG; 2905 2906 printf("\nHMM THP Migration Benchmark\n"); 2907 printf("---------------------------\n"); 2908 printf("System page size: %ld bytes\n", sysconf(_SC_PAGESIZE)); 2909 2910 /* Test different buffer sizes */ 2911 size_t test_sizes[] = { 2912 thp_size / 4, /* quarter THP */ 2913 thp_size / 2, /* half THP */ 2914 thp_size, /* single THP */ 2915 thp_size * 2, /* two THPs */ 2916 thp_size * 4, /* four THPs */ 2917 thp_size * 8, /* eight THPs */ 2918 thp_size * 128, /* one twenty eight THPs */ 2919 }; 2920 2921 static const char *const test_names[] = { 2922 "Small Buffer", 2923 "Half THP Size", 2924 "Single THP Size", 2925 "Two THP Size", 2926 "Four THP Size", 2927 "Eight THP Size", 2928 "One twenty eight THP Size" 2929 }; 2930 2931 int num_tests = ARRAY_SIZE(test_sizes); 2932 2933 /* Run all tests */ 2934 for (int i = 0; i < num_tests; i++) { 2935 /* Skip test sizes exceeding INT_MAX to avoid overflow */ 2936 if (test_sizes[i] > INT_MAX) 2937 break; 2938 2939 /* Test with THP */ 2940 ASSERT_EQ(run_migration_benchmark(self->fd, 1, test_sizes[i], 2941 iterations, &thp_results), 0); 2942 2943 /* Test without THP */ 2944 ASSERT_EQ(run_migration_benchmark(self->fd, 0, test_sizes[i], 2945 iterations, ®ular_results), 0); 2946 2947 /* Print results */ 2948 print_benchmark_results(test_names[i], test_sizes[i], 2949 &thp_results, ®ular_results); 2950 } 2951 } 2952 TEST_HARNESS_MAIN 2953