1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * HMM stands for Heterogeneous Memory Management, it is a helper layer inside 4 * the linux kernel to help device drivers mirror a process address space in 5 * the device. This allows the device to use the same address space which 6 * makes communication and data exchange a lot easier. 7 * 8 * This framework's sole purpose is to exercise various code paths inside 9 * the kernel to make sure that HMM performs as expected and to flush out any 10 * bugs. 11 */ 12 13 #include "kselftest_harness.h" 14 #include "hugepage_settings.h" 15 16 #include <errno.h> 17 #include <fcntl.h> 18 #include <stdio.h> 19 #include <stdlib.h> 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <strings.h> 23 #include <time.h> 24 #include <pthread.h> 25 #include <limits.h> 26 #include <linux/mman.h> 27 #include <sys/types.h> 28 #include <sys/stat.h> 29 #include <sys/mman.h> 30 #include <sys/ioctl.h> 31 #include <sys/time.h> 32 #include <sys/syscall.h> 33 #include <sys/eventfd.h> 34 #include <linux/userfaultfd.h> 35 #include <poll.h> 36 37 /* 38 * This is a private UAPI to the kernel test module so it isn't exported 39 * in the usual include/uapi/... directory. 40 */ 41 #include <lib/test_hmm_uapi.h> 42 #include <mm/gup_test.h> 43 #include <mm/vm_util.h> 44 45 struct hmm_buffer { 46 void *ptr; 47 void *mirror; 48 unsigned long size; 49 int fd; 50 uint64_t cpages; 51 uint64_t faults; 52 }; 53 54 enum { 55 HMM_PRIVATE_DEVICE_ONE, 56 HMM_PRIVATE_DEVICE_TWO, 57 HMM_COHERENCE_DEVICE_ONE, 58 HMM_COHERENCE_DEVICE_TWO, 59 }; 60 61 #define ONEKB (1 << 10) 62 #define ONEMEG (1 << 20) 63 #define TWOMEG (1 << 21) 64 #define HMM_BUFFER_SIZE (1024 << 12) 65 #define HMM_PATH_MAX 64 66 #define NTIMES 10 67 68 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) 69 /* Just the flags we need, copied from mm.h: */ 70 71 #ifndef FOLL_WRITE 72 #define FOLL_WRITE 0x01 /* check pte is writable */ 73 #endif 74 75 #ifndef FOLL_LONGTERM 76 #define FOLL_LONGTERM 0x100 /* mapping lifetime is indefinite */ 77 #endif 78 79 HUGETLB_SETUP_DEFAULT_PAGES(1) 80 81 FIXTURE(hmm) 82 { 83 int fd; 84 unsigned int page_size; 85 unsigned int page_shift; 86 }; 87 88 FIXTURE_VARIANT(hmm) 89 { 90 int device_number; 91 }; 92 93 FIXTURE_VARIANT_ADD(hmm, hmm_device_private) 94 { 95 .device_number = HMM_PRIVATE_DEVICE_ONE, 96 }; 97 98 FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent) 99 { 100 .device_number = HMM_COHERENCE_DEVICE_ONE, 101 }; 102 103 FIXTURE(hmm2) 104 { 105 int fd0; 106 int fd1; 107 unsigned int page_size; 108 unsigned int page_shift; 109 }; 110 111 FIXTURE_VARIANT(hmm2) 112 { 113 int device_number0; 114 int device_number1; 115 }; 116 117 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private) 118 { 119 .device_number0 = HMM_PRIVATE_DEVICE_ONE, 120 .device_number1 = HMM_PRIVATE_DEVICE_TWO, 121 }; 122 123 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent) 124 { 125 .device_number0 = HMM_COHERENCE_DEVICE_ONE, 126 .device_number1 = HMM_COHERENCE_DEVICE_TWO, 127 }; 128 129 static int hmm_open(int unit) 130 { 131 char pathname[HMM_PATH_MAX]; 132 int fd; 133 134 snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit); 135 fd = open(pathname, O_RDWR, 0); 136 if (fd < 0) 137 fprintf(stderr, "could not open hmm dmirror driver (%s)\n", 138 pathname); 139 return fd; 140 } 141 142 static bool hmm_is_coherent_type(int dev_num) 143 { 144 return (dev_num >= HMM_COHERENCE_DEVICE_ONE); 145 } 146 147 FIXTURE_SETUP(hmm) 148 { 149 self->page_size = sysconf(_SC_PAGE_SIZE); 150 self->page_shift = ffs(self->page_size) - 1; 151 152 self->fd = hmm_open(variant->device_number); 153 if (self->fd < 0 && hmm_is_coherent_type(variant->device_number)) 154 SKIP(return, "DEVICE_COHERENT not available"); 155 ASSERT_GE(self->fd, 0); 156 } 157 158 FIXTURE_SETUP(hmm2) 159 { 160 self->page_size = sysconf(_SC_PAGE_SIZE); 161 self->page_shift = ffs(self->page_size) - 1; 162 163 self->fd0 = hmm_open(variant->device_number0); 164 if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0)) 165 SKIP(return, "DEVICE_COHERENT not available"); 166 ASSERT_GE(self->fd0, 0); 167 self->fd1 = hmm_open(variant->device_number1); 168 ASSERT_GE(self->fd1, 0); 169 } 170 171 FIXTURE_TEARDOWN(hmm) 172 { 173 int ret = close(self->fd); 174 175 ASSERT_EQ(ret, 0); 176 self->fd = -1; 177 } 178 179 FIXTURE_TEARDOWN(hmm2) 180 { 181 int ret = close(self->fd0); 182 183 ASSERT_EQ(ret, 0); 184 self->fd0 = -1; 185 186 ret = close(self->fd1); 187 ASSERT_EQ(ret, 0); 188 self->fd1 = -1; 189 } 190 191 static int hmm_dmirror_cmd(int fd, 192 unsigned long request, 193 struct hmm_buffer *buffer, 194 unsigned long npages) 195 { 196 struct hmm_dmirror_cmd cmd; 197 int ret; 198 199 /* Simulate a device reading system memory. */ 200 cmd.addr = (__u64)buffer->ptr; 201 cmd.ptr = (__u64)buffer->mirror; 202 cmd.npages = npages; 203 204 for (;;) { 205 ret = ioctl(fd, request, &cmd); 206 if (ret == 0) 207 break; 208 if (errno == EINTR) 209 continue; 210 return -errno; 211 } 212 buffer->cpages = cmd.cpages; 213 buffer->faults = cmd.faults; 214 215 return 0; 216 } 217 218 static void hmm_buffer_free(struct hmm_buffer *buffer) 219 { 220 if (buffer == NULL) 221 return; 222 223 if (buffer->ptr) { 224 munmap(buffer->ptr, buffer->size); 225 buffer->ptr = NULL; 226 } 227 free(buffer->mirror); 228 free(buffer); 229 } 230 231 /* 232 * Allocate a buffer structure with memory mapping and mirror. 233 * 234 * @mmap_size: total size of the mmap region (may differ from @mirror_size 235 * for alignment padding in THP tests). 236 * @mirror_size: size of the mirror data buffer (the actual working set). 237 * @prot: protection flags for the mmap (e.g. PROT_READ | PROT_WRITE). 238 * @flags: flags for the mmap (e.g. MAP_PRIVATE, MAP_SHARED, 239 * MAP_ANONYMOUS, MAP_HUGETLB). 240 * @fd: file descriptor for the mmap; pass -1 for MAP_ANONYMOUS. 241 * 242 * All internal allocations are checked; returns NULL and cleans up on any 243 * failure. Caller must ASSERT_NE or otherwise check the return value. 244 */ 245 static struct hmm_buffer *hmm_buffer_alloc(unsigned long mmap_size, 246 unsigned long mirror_size, 247 int prot, int flags, 248 int fd) 249 { 250 struct hmm_buffer *buffer; 251 252 buffer = malloc(sizeof(*buffer)); 253 if (!buffer) { 254 perror("malloc buffer"); 255 return NULL; 256 } 257 258 buffer->fd = fd; 259 buffer->size = mmap_size; 260 buffer->mirror = malloc(mirror_size); 261 if (!buffer->mirror) { 262 perror("malloc mirror"); 263 free(buffer); 264 return NULL; 265 } 266 267 buffer->ptr = mmap(NULL, mmap_size, prot, flags, fd, 0); 268 if (buffer->ptr == MAP_FAILED) { 269 perror("mmap"); 270 free(buffer->mirror); 271 free(buffer); 272 return NULL; 273 } 274 275 return buffer; 276 } 277 278 /* 279 * Create a temporary file that will be deleted on close. 280 */ 281 static int hmm_create_file(unsigned long size) 282 { 283 char path[HMM_PATH_MAX]; 284 int fd; 285 286 strcpy(path, "/tmp"); 287 fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600); 288 if (fd >= 0) { 289 int r; 290 291 do { 292 r = ftruncate(fd, size); 293 } while (r == -1 && errno == EINTR); 294 if (!r) 295 return fd; 296 close(fd); 297 } 298 return -1; 299 } 300 301 /* 302 * Return a random unsigned number. 303 */ 304 static unsigned int hmm_random(void) 305 { 306 static int fd = -1; 307 unsigned int r; 308 309 if (fd < 0) { 310 fd = open("/dev/urandom", O_RDONLY); 311 if (fd < 0) { 312 fprintf(stderr, "%s:%d failed to open /dev/urandom\n", 313 __FILE__, __LINE__); 314 return ~0U; 315 } 316 } 317 read(fd, &r, sizeof(r)); 318 return r; 319 } 320 321 static void hmm_nanosleep(unsigned int n) 322 { 323 struct timespec t; 324 325 t.tv_sec = 0; 326 t.tv_nsec = n; 327 nanosleep(&t, NULL); 328 } 329 330 static int hmm_migrate_sys_to_dev(int fd, 331 struct hmm_buffer *buffer, 332 unsigned long npages) 333 { 334 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages); 335 } 336 337 static int hmm_migrate_dev_to_sys(int fd, 338 struct hmm_buffer *buffer, 339 unsigned long npages) 340 { 341 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages); 342 } 343 344 /* 345 * Simple NULL test of device open/close. 346 */ 347 TEST_F(hmm, open_close) 348 { 349 } 350 351 /* 352 * Read private anonymous memory. 353 */ 354 TEST_F(hmm, anon_read) 355 { 356 struct hmm_buffer *buffer; 357 unsigned long npages; 358 unsigned long size; 359 unsigned long i; 360 int *ptr; 361 int ret; 362 int val; 363 364 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 365 ASSERT_NE(npages, 0); 366 size = npages << self->page_shift; 367 368 buffer = hmm_buffer_alloc(size, size, 369 PROT_READ | PROT_WRITE, 370 MAP_PRIVATE | MAP_ANONYMOUS, -1); 371 ASSERT_NE(buffer, NULL); 372 373 /* 374 * Initialize buffer in system memory but leave the first two pages 375 * zero (pte_none and pfn_zero). 376 */ 377 i = 2 * self->page_size / sizeof(*ptr); 378 for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 379 ptr[i] = i; 380 381 /* Set buffer permission to read-only. */ 382 ret = mprotect(buffer->ptr, size, PROT_READ); 383 ASSERT_EQ(ret, 0); 384 385 /* Populate the CPU page table with a special zero page. */ 386 val = *(int *)(buffer->ptr + self->page_size); 387 ASSERT_EQ(val, 0); 388 389 /* Simulate a device reading system memory. */ 390 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 391 ASSERT_EQ(ret, 0); 392 ASSERT_EQ(buffer->cpages, npages); 393 ASSERT_EQ(buffer->faults, 1); 394 395 /* Check what the device read. */ 396 ptr = buffer->mirror; 397 for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i) 398 ASSERT_EQ(ptr[i], 0); 399 for (; i < size / sizeof(*ptr); ++i) 400 ASSERT_EQ(ptr[i], i); 401 402 hmm_buffer_free(buffer); 403 } 404 405 /* 406 * Read private anonymous memory which has been protected with 407 * mprotect() PROT_NONE. 408 */ 409 TEST_F(hmm, anon_read_prot) 410 { 411 struct hmm_buffer *buffer; 412 unsigned long npages; 413 unsigned long size; 414 unsigned long i; 415 int *ptr; 416 int ret; 417 418 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 419 ASSERT_NE(npages, 0); 420 size = npages << self->page_shift; 421 422 buffer = hmm_buffer_alloc(size, size, 423 PROT_READ | PROT_WRITE, 424 MAP_PRIVATE | MAP_ANONYMOUS, -1); 425 ASSERT_NE(buffer, NULL); 426 427 /* Initialize buffer in system memory. */ 428 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 429 ptr[i] = i; 430 431 /* Initialize mirror buffer so we can verify it isn't written. */ 432 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 433 ptr[i] = -i; 434 435 /* Protect buffer from reading. */ 436 ret = mprotect(buffer->ptr, size, PROT_NONE); 437 ASSERT_EQ(ret, 0); 438 439 /* Simulate a device reading system memory. */ 440 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 441 ASSERT_EQ(ret, -EFAULT); 442 443 /* Allow CPU to read the buffer so we can check it. */ 444 ret = mprotect(buffer->ptr, size, PROT_READ); 445 ASSERT_EQ(ret, 0); 446 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 447 ASSERT_EQ(ptr[i], i); 448 449 /* Check what the device read. */ 450 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 451 ASSERT_EQ(ptr[i], -i); 452 453 hmm_buffer_free(buffer); 454 } 455 456 /* 457 * Write private anonymous memory. 458 */ 459 TEST_F(hmm, anon_write) 460 { 461 struct hmm_buffer *buffer; 462 unsigned long npages; 463 unsigned long size; 464 unsigned long i; 465 int *ptr; 466 int ret; 467 468 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 469 ASSERT_NE(npages, 0); 470 size = npages << self->page_shift; 471 472 buffer = hmm_buffer_alloc(size, size, 473 PROT_READ | PROT_WRITE, 474 MAP_PRIVATE | MAP_ANONYMOUS, -1); 475 ASSERT_NE(buffer, NULL); 476 477 /* Initialize data that the device will write to buffer->ptr. */ 478 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 479 ptr[i] = i; 480 481 /* Simulate a device writing system memory. */ 482 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 483 ASSERT_EQ(ret, 0); 484 ASSERT_EQ(buffer->cpages, npages); 485 ASSERT_EQ(buffer->faults, 1); 486 487 /* Check what the device wrote. */ 488 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 489 ASSERT_EQ(ptr[i], i); 490 491 hmm_buffer_free(buffer); 492 } 493 494 /* 495 * Write private anonymous memory which has been protected with 496 * mprotect() PROT_READ. 497 */ 498 TEST_F(hmm, anon_write_prot) 499 { 500 struct hmm_buffer *buffer; 501 unsigned long npages; 502 unsigned long size; 503 unsigned long i; 504 int *ptr; 505 int ret; 506 507 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 508 ASSERT_NE(npages, 0); 509 size = npages << self->page_shift; 510 511 buffer = hmm_buffer_alloc(size, size, 512 PROT_READ, 513 MAP_PRIVATE | MAP_ANONYMOUS, -1); 514 ASSERT_NE(buffer, NULL); 515 516 /* Simulate a device reading a zero page of memory. */ 517 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); 518 ASSERT_EQ(ret, 0); 519 ASSERT_EQ(buffer->cpages, 1); 520 ASSERT_EQ(buffer->faults, 1); 521 522 /* Initialize data that the device will write to buffer->ptr. */ 523 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 524 ptr[i] = i; 525 526 /* Simulate a device writing system memory. */ 527 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 528 ASSERT_EQ(ret, -EPERM); 529 530 /* Check what the device wrote. */ 531 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 532 ASSERT_EQ(ptr[i], 0); 533 534 /* Now allow writing and see that the zero page is replaced. */ 535 ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ); 536 ASSERT_EQ(ret, 0); 537 538 /* Simulate a device writing system memory. */ 539 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 540 ASSERT_EQ(ret, 0); 541 ASSERT_EQ(buffer->cpages, npages); 542 ASSERT_EQ(buffer->faults, 1); 543 544 /* Check what the device wrote. */ 545 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 546 ASSERT_EQ(ptr[i], i); 547 548 hmm_buffer_free(buffer); 549 } 550 551 /* 552 * Check that a device writing an anonymous private mapping 553 * will copy-on-write if a child process inherits the mapping. 554 * 555 * Also verifies after fork() memory the device can be read by child. 556 */ 557 TEST_F(hmm, anon_write_child) 558 { 559 struct hmm_buffer *buffer; 560 unsigned long npages; 561 unsigned long size; 562 unsigned long i; 563 void *old_ptr; 564 void *map; 565 int *ptr; 566 pid_t pid; 567 int child_fd; 568 int ret, use_thp, migrate; 569 570 for (migrate = 0; migrate < 2; ++migrate) { 571 for (use_thp = 0; use_thp < 2; ++use_thp) { 572 npages = ALIGN(use_thp ? read_pmd_pagesize() : HMM_BUFFER_SIZE, 573 self->page_size) >> self->page_shift; 574 ASSERT_NE(npages, 0); 575 size = npages << self->page_shift; 576 577 buffer = hmm_buffer_alloc(size * 2, size, 578 PROT_READ | PROT_WRITE, 579 MAP_PRIVATE | MAP_ANONYMOUS, -1); 580 ASSERT_NE(buffer, NULL); 581 old_ptr = buffer->ptr; 582 if (use_thp) { 583 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 584 ret = madvise(map, size, MADV_HUGEPAGE); 585 ASSERT_EQ(ret, 0); 586 buffer->ptr = map; 587 } 588 589 /* Initialize buffer->ptr so we can tell if it is written. */ 590 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 591 ptr[i] = i; 592 593 /* Initialize data that the device will write to buffer->ptr. */ 594 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 595 ptr[i] = -i; 596 597 if (migrate) { 598 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 599 ASSERT_EQ(ret, 0); 600 ASSERT_EQ(buffer->cpages, npages); 601 602 } 603 604 pid = fork(); 605 if (pid == -1) 606 ASSERT_EQ(pid, 0); 607 if (pid != 0) { 608 waitpid(pid, &ret, 0); 609 ASSERT_EQ(WIFEXITED(ret), 1); 610 611 /* Check that the parent's buffer did not change. */ 612 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 613 ASSERT_EQ(ptr[i], i); 614 615 buffer->ptr = old_ptr; 616 hmm_buffer_free(buffer); 617 continue; 618 } 619 620 /* Check that we see the parent's values. */ 621 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 622 ASSERT_EQ(ptr[i], i); 623 if (!migrate) { 624 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 625 ASSERT_EQ(ptr[i], -i); 626 } 627 628 /* The child process needs its own mirror to its own mm. */ 629 child_fd = hmm_open(0); 630 ASSERT_GE(child_fd, 0); 631 632 /* Simulate a device writing system memory. */ 633 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); 634 ASSERT_EQ(ret, 0); 635 ASSERT_EQ(buffer->cpages, npages); 636 ASSERT_EQ(buffer->faults, 1); 637 638 /* Check what the device wrote. */ 639 if (!migrate) { 640 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 641 ASSERT_EQ(ptr[i], -i); 642 } 643 644 close(child_fd); 645 _exit(0); 646 } 647 } 648 } 649 650 /* 651 * Check that a device writing an anonymous shared mapping 652 * will not copy-on-write if a child process inherits the mapping. 653 */ 654 TEST_F(hmm, anon_write_child_shared) 655 { 656 struct hmm_buffer *buffer; 657 unsigned long npages; 658 unsigned long size; 659 unsigned long i; 660 int *ptr; 661 pid_t pid; 662 int child_fd; 663 int ret; 664 665 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 666 ASSERT_NE(npages, 0); 667 size = npages << self->page_shift; 668 669 buffer = hmm_buffer_alloc(size, size, 670 PROT_READ | PROT_WRITE, 671 MAP_SHARED | MAP_ANONYMOUS, -1); 672 ASSERT_NE(buffer, NULL); 673 674 /* Initialize buffer->ptr so we can tell if it is written. */ 675 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 676 ptr[i] = i; 677 678 /* Initialize data that the device will write to buffer->ptr. */ 679 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 680 ptr[i] = -i; 681 682 pid = fork(); 683 if (pid == -1) 684 ASSERT_EQ(pid, 0); 685 if (pid != 0) { 686 waitpid(pid, &ret, 0); 687 ASSERT_EQ(WIFEXITED(ret), 1); 688 689 /* Check that the parent's buffer did change. */ 690 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 691 ASSERT_EQ(ptr[i], -i); 692 return; 693 } 694 695 /* Check that we see the parent's values. */ 696 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 697 ASSERT_EQ(ptr[i], i); 698 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 699 ASSERT_EQ(ptr[i], -i); 700 701 /* The child process needs its own mirror to its own mm. */ 702 child_fd = hmm_open(0); 703 ASSERT_GE(child_fd, 0); 704 705 /* Simulate a device writing system memory. */ 706 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); 707 ASSERT_EQ(ret, 0); 708 ASSERT_EQ(buffer->cpages, npages); 709 ASSERT_EQ(buffer->faults, 1); 710 711 /* Check what the device wrote. */ 712 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 713 ASSERT_EQ(ptr[i], -i); 714 715 close(child_fd); 716 _exit(0); 717 } 718 719 /* 720 * Write private anonymous huge page. 721 */ 722 TEST_F(hmm, anon_write_huge) 723 { 724 struct hmm_buffer *buffer; 725 unsigned long npages; 726 unsigned long size; 727 unsigned long i; 728 void *old_ptr; 729 void *map; 730 int *ptr; 731 int ret; 732 733 size = 2 * read_pmd_pagesize(); 734 735 buffer = hmm_buffer_alloc(size, size, 736 PROT_READ | PROT_WRITE, 737 MAP_PRIVATE | MAP_ANONYMOUS, -1); 738 ASSERT_NE(buffer, NULL); 739 size /= 2; 740 npages = size >> self->page_shift; 741 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 742 ret = madvise(map, size, MADV_HUGEPAGE); 743 ASSERT_EQ(ret, 0); 744 old_ptr = buffer->ptr; 745 buffer->ptr = map; 746 747 /* Initialize data that the device will write to buffer->ptr. */ 748 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 749 ptr[i] = i; 750 751 /* Simulate a device writing system memory. */ 752 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 753 ASSERT_EQ(ret, 0); 754 ASSERT_EQ(buffer->cpages, npages); 755 ASSERT_EQ(buffer->faults, 1); 756 757 /* Check what the device wrote. */ 758 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 759 ASSERT_EQ(ptr[i], i); 760 761 buffer->ptr = old_ptr; 762 hmm_buffer_free(buffer); 763 } 764 765 /* 766 * Write huge TLBFS page. 767 */ 768 TEST_F(hmm, anon_write_hugetlbfs) 769 { 770 struct hmm_buffer *buffer; 771 unsigned long npages; 772 unsigned long size; 773 unsigned long default_hsize = default_huge_page_size(); 774 unsigned long i; 775 int *ptr; 776 int ret; 777 778 if (!hugetlb_free_default_pages()) 779 SKIP(return, "Not enough huge pages"); 780 781 size = ALIGN(TWOMEG, default_hsize); 782 npages = size >> self->page_shift; 783 784 buffer = hmm_buffer_alloc(size, size, 785 PROT_READ | PROT_WRITE, 786 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1); 787 if (!buffer) 788 SKIP(return, "Huge page could not be allocated"); 789 790 /* Initialize data that the device will write to buffer->ptr. */ 791 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 792 ptr[i] = i; 793 794 /* Simulate a device writing system memory. */ 795 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 796 ASSERT_EQ(ret, 0); 797 ASSERT_EQ(buffer->cpages, npages); 798 ASSERT_EQ(buffer->faults, 1); 799 800 /* Check what the device wrote. */ 801 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 802 ASSERT_EQ(ptr[i], i); 803 804 munmap(buffer->ptr, buffer->size); 805 buffer->ptr = NULL; 806 hmm_buffer_free(buffer); 807 } 808 809 /* 810 * Read mmap'ed file memory. 811 */ 812 TEST_F(hmm, file_read) 813 { 814 struct hmm_buffer *buffer; 815 unsigned long npages; 816 unsigned long size; 817 unsigned long i; 818 int *ptr; 819 int ret; 820 int fd; 821 ssize_t len; 822 823 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 824 ASSERT_NE(npages, 0); 825 size = npages << self->page_shift; 826 827 fd = hmm_create_file(size); 828 ASSERT_GE(fd, 0); 829 830 buffer = hmm_buffer_alloc(size, size, 831 PROT_READ, 832 MAP_SHARED, fd); 833 ASSERT_NE(buffer, NULL); 834 835 /* Write initial contents of the file. */ 836 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 837 ptr[i] = i; 838 len = pwrite(fd, buffer->mirror, size, 0); 839 ASSERT_EQ(len, size); 840 memset(buffer->mirror, 0, size); 841 842 /* Simulate a device reading system memory. */ 843 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 844 ASSERT_EQ(ret, 0); 845 ASSERT_EQ(buffer->cpages, npages); 846 ASSERT_EQ(buffer->faults, 1); 847 848 /* Check what the device read. */ 849 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 850 ASSERT_EQ(ptr[i], i); 851 852 hmm_buffer_free(buffer); 853 } 854 855 /* 856 * Write mmap'ed file memory. 857 */ 858 TEST_F(hmm, file_write) 859 { 860 struct hmm_buffer *buffer; 861 unsigned long npages; 862 unsigned long size; 863 unsigned long i; 864 int *ptr; 865 int ret; 866 int fd; 867 ssize_t len; 868 869 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 870 ASSERT_NE(npages, 0); 871 size = npages << self->page_shift; 872 873 fd = hmm_create_file(size); 874 ASSERT_GE(fd, 0); 875 876 buffer = hmm_buffer_alloc(size, size, 877 PROT_READ | PROT_WRITE, 878 MAP_SHARED, fd); 879 ASSERT_NE(buffer, NULL); 880 881 /* Initialize data that the device will write to buffer->ptr. */ 882 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 883 ptr[i] = i; 884 885 /* Simulate a device writing system memory. */ 886 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 887 ASSERT_EQ(ret, 0); 888 ASSERT_EQ(buffer->cpages, npages); 889 ASSERT_EQ(buffer->faults, 1); 890 891 /* Check what the device wrote. */ 892 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 893 ASSERT_EQ(ptr[i], i); 894 895 /* Check that the device also wrote the file. */ 896 len = pread(fd, buffer->mirror, size, 0); 897 ASSERT_EQ(len, size); 898 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 899 ASSERT_EQ(ptr[i], i); 900 901 hmm_buffer_free(buffer); 902 } 903 904 /* 905 * Migrate anonymous memory to device private memory. 906 */ 907 TEST_F(hmm, migrate) 908 { 909 struct hmm_buffer *buffer; 910 unsigned long npages; 911 unsigned long size; 912 unsigned long i; 913 int *ptr; 914 int ret; 915 916 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 917 ASSERT_NE(npages, 0); 918 size = npages << self->page_shift; 919 920 buffer = hmm_buffer_alloc(size, size, 921 PROT_READ | PROT_WRITE, 922 MAP_PRIVATE | MAP_ANONYMOUS, -1); 923 ASSERT_NE(buffer, NULL); 924 925 /* Initialize buffer in system memory. */ 926 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 927 ptr[i] = i; 928 929 /* Migrate memory to device. */ 930 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 931 ASSERT_EQ(ret, 0); 932 ASSERT_EQ(buffer->cpages, npages); 933 934 /* Check what the device read. */ 935 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 936 ASSERT_EQ(ptr[i], i); 937 938 hmm_buffer_free(buffer); 939 } 940 941 /* 942 * Migrate private file memory to device private memory. 943 */ 944 TEST_F(hmm, migrate_file_private) 945 { 946 struct hmm_buffer *buffer; 947 unsigned long npages; 948 unsigned long size; 949 unsigned long i; 950 int *ptr; 951 int ret; 952 int fd; 953 954 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 955 ASSERT_NE(npages, 0); 956 size = npages << self->page_shift; 957 958 fd = hmm_create_file(size); 959 ASSERT_GE(fd, 0); 960 961 buffer = hmm_buffer_alloc(size, size, 962 PROT_READ | PROT_WRITE, 963 MAP_PRIVATE, fd); 964 ASSERT_NE(buffer, NULL); 965 966 /* Initialize buffer in system memory. */ 967 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 968 ptr[i] = i; 969 970 /* Migrate memory to device. */ 971 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 972 ASSERT_EQ(ret, 0); 973 ASSERT_EQ(buffer->cpages, npages); 974 975 /* Check what the device read. */ 976 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 977 ASSERT_EQ(ptr[i], i); 978 979 hmm_buffer_free(buffer); 980 } 981 982 /* 983 * Migrate anonymous memory to device private memory and fault some of it back 984 * to system memory, then try migrating the resulting mix of system and device 985 * private memory to the device. 986 */ 987 TEST_F(hmm, migrate_fault) 988 { 989 struct hmm_buffer *buffer; 990 unsigned long npages; 991 unsigned long size; 992 unsigned long i; 993 int *ptr; 994 int ret; 995 996 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 997 ASSERT_NE(npages, 0); 998 size = npages << self->page_shift; 999 1000 buffer = hmm_buffer_alloc(size, size, 1001 PROT_READ | PROT_WRITE, 1002 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1003 ASSERT_NE(buffer, NULL); 1004 1005 /* Initialize buffer in system memory. */ 1006 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1007 ptr[i] = i; 1008 1009 /* Migrate memory to device. */ 1010 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1011 ASSERT_EQ(ret, 0); 1012 ASSERT_EQ(buffer->cpages, npages); 1013 1014 /* Check what the device read. */ 1015 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1016 ASSERT_EQ(ptr[i], i); 1017 1018 /* Fault half the pages back to system memory and check them. */ 1019 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) 1020 ASSERT_EQ(ptr[i], i); 1021 1022 /* Migrate memory to the device again. */ 1023 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1024 ASSERT_EQ(ret, 0); 1025 ASSERT_EQ(buffer->cpages, npages); 1026 1027 /* Check what the device read. */ 1028 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1029 ASSERT_EQ(ptr[i], i); 1030 1031 hmm_buffer_free(buffer); 1032 } 1033 1034 TEST_F(hmm, migrate_release) 1035 { 1036 struct hmm_buffer *buffer; 1037 unsigned long npages; 1038 unsigned long size; 1039 unsigned long i; 1040 int *ptr; 1041 int ret; 1042 1043 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1044 ASSERT_NE(npages, 0); 1045 size = npages << self->page_shift; 1046 1047 buffer = hmm_buffer_alloc(size, size, 1048 PROT_READ | PROT_WRITE, 1049 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1050 ASSERT_NE(buffer, NULL); 1051 1052 /* Initialize buffer in system memory. */ 1053 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1054 ptr[i] = i; 1055 1056 /* Migrate memory to device. */ 1057 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1058 ASSERT_EQ(ret, 0); 1059 ASSERT_EQ(buffer->cpages, npages); 1060 1061 /* Check what the device read. */ 1062 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1063 ASSERT_EQ(ptr[i], i); 1064 1065 /* Release device memory. */ 1066 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages); 1067 ASSERT_EQ(ret, 0); 1068 1069 /* Fault pages back to system memory and check them. */ 1070 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) 1071 ASSERT_EQ(ptr[i], i); 1072 1073 hmm_buffer_free(buffer); 1074 } 1075 1076 /* 1077 * Migrate anonymous shared memory to device private memory. 1078 */ 1079 TEST_F(hmm, migrate_shared) 1080 { 1081 struct hmm_buffer *buffer; 1082 unsigned long npages; 1083 unsigned long size; 1084 int ret; 1085 1086 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1087 ASSERT_NE(npages, 0); 1088 size = npages << self->page_shift; 1089 1090 buffer = hmm_buffer_alloc(size, size, 1091 PROT_READ | PROT_WRITE, 1092 MAP_SHARED | MAP_ANONYMOUS, -1); 1093 ASSERT_NE(buffer, NULL); 1094 1095 /* Migrate memory to device. */ 1096 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1097 ASSERT_EQ(ret, -ENOENT); 1098 1099 hmm_buffer_free(buffer); 1100 } 1101 1102 /* 1103 * Try to migrate various memory types to device private memory. 1104 */ 1105 TEST_F(hmm2, migrate_mixed) 1106 { 1107 struct hmm_buffer *buffer; 1108 unsigned long npages; 1109 unsigned long size; 1110 int *ptr; 1111 unsigned char *p; 1112 int ret; 1113 int val; 1114 1115 npages = 6; 1116 size = npages << self->page_shift; 1117 1118 buffer = hmm_buffer_alloc(size, size, 1119 PROT_NONE, 1120 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1121 ASSERT_NE(buffer, NULL); 1122 p = buffer->ptr; 1123 1124 /* Migrating a protected area should be an error. */ 1125 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); 1126 ASSERT_EQ(ret, -EINVAL); 1127 1128 /* Punch a hole after the first page address. */ 1129 ret = munmap(buffer->ptr + self->page_size, self->page_size); 1130 ASSERT_EQ(ret, 0); 1131 1132 /* We expect an error if the vma doesn't cover the range. */ 1133 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3); 1134 ASSERT_EQ(ret, -EINVAL); 1135 1136 /* Page 2 will be a read-only zero page. */ 1137 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, 1138 PROT_READ); 1139 ASSERT_EQ(ret, 0); 1140 ptr = (int *)(buffer->ptr + 2 * self->page_size); 1141 val = *ptr + 3; 1142 ASSERT_EQ(val, 3); 1143 1144 /* Page 3 will be read-only. */ 1145 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1146 PROT_READ | PROT_WRITE); 1147 ASSERT_EQ(ret, 0); 1148 ptr = (int *)(buffer->ptr + 3 * self->page_size); 1149 *ptr = val; 1150 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1151 PROT_READ); 1152 ASSERT_EQ(ret, 0); 1153 1154 /* Page 4-5 will be read-write. */ 1155 ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size, 1156 PROT_READ | PROT_WRITE); 1157 ASSERT_EQ(ret, 0); 1158 ptr = (int *)(buffer->ptr + 4 * self->page_size); 1159 *ptr = val; 1160 ptr = (int *)(buffer->ptr + 5 * self->page_size); 1161 *ptr = val; 1162 1163 /* Now try to migrate pages 2-5 to device 1. */ 1164 buffer->ptr = p + 2 * self->page_size; 1165 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4); 1166 ASSERT_EQ(ret, 0); 1167 ASSERT_EQ(buffer->cpages, 4); 1168 1169 /* Page 5 won't be migrated to device 0 because it's on device 1. */ 1170 buffer->ptr = p + 5 * self->page_size; 1171 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); 1172 ASSERT_EQ(ret, -ENOENT); 1173 buffer->ptr = p; 1174 1175 buffer->ptr = p; 1176 hmm_buffer_free(buffer); 1177 } 1178 1179 /* 1180 * Migrate anonymous memory to device memory and back to system memory 1181 * multiple times. In case of private zone configuration, this is done 1182 * through fault pages accessed by CPU. In case of coherent zone configuration, 1183 * the pages from the device should be explicitly migrated back to system memory. 1184 * The reason is Coherent device zone has coherent access by CPU, therefore 1185 * it will not generate any page fault. 1186 */ 1187 TEST_F(hmm, migrate_multiple) 1188 { 1189 struct hmm_buffer *buffer; 1190 unsigned long npages; 1191 unsigned long size; 1192 unsigned long i; 1193 unsigned long c; 1194 int *ptr; 1195 int ret; 1196 1197 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1198 ASSERT_NE(npages, 0); 1199 size = npages << self->page_shift; 1200 1201 for (c = 0; c < NTIMES; c++) { 1202 buffer = hmm_buffer_alloc(size, size, 1203 PROT_READ | PROT_WRITE, 1204 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1205 ASSERT_NE(buffer, NULL); 1206 1207 /* Initialize buffer in system memory. */ 1208 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1209 ptr[i] = i; 1210 1211 /* Migrate memory to device. */ 1212 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1213 ASSERT_EQ(ret, 0); 1214 ASSERT_EQ(buffer->cpages, npages); 1215 1216 /* Check what the device read. */ 1217 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1218 ASSERT_EQ(ptr[i], i); 1219 1220 /* Migrate back to system memory and check them. */ 1221 if (hmm_is_coherent_type(variant->device_number)) { 1222 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages); 1223 ASSERT_EQ(ret, 0); 1224 ASSERT_EQ(buffer->cpages, npages); 1225 } 1226 1227 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1228 ASSERT_EQ(ptr[i], i); 1229 1230 hmm_buffer_free(buffer); 1231 } 1232 } 1233 1234 /* 1235 * Read anonymous memory multiple times. 1236 */ 1237 TEST_F(hmm, anon_read_multiple) 1238 { 1239 struct hmm_buffer *buffer; 1240 unsigned long npages; 1241 unsigned long size; 1242 unsigned long i; 1243 unsigned long c; 1244 int *ptr; 1245 int ret; 1246 1247 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1248 ASSERT_NE(npages, 0); 1249 size = npages << self->page_shift; 1250 1251 for (c = 0; c < NTIMES; c++) { 1252 buffer = hmm_buffer_alloc(size, size, 1253 PROT_READ | PROT_WRITE, 1254 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1255 ASSERT_NE(buffer, NULL); 1256 1257 /* Initialize buffer in system memory. */ 1258 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1259 ptr[i] = i + c; 1260 1261 /* Simulate a device reading system memory. */ 1262 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1263 npages); 1264 ASSERT_EQ(ret, 0); 1265 ASSERT_EQ(buffer->cpages, npages); 1266 ASSERT_EQ(buffer->faults, 1); 1267 1268 /* Check what the device read. */ 1269 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1270 ASSERT_EQ(ptr[i], i + c); 1271 1272 hmm_buffer_free(buffer); 1273 } 1274 } 1275 1276 void *unmap_buffer(void *p) 1277 { 1278 struct hmm_buffer *buffer = p; 1279 1280 /* Delay for a bit and then unmap buffer while it is being read. */ 1281 hmm_nanosleep(hmm_random() % 32000); 1282 munmap(buffer->ptr + buffer->size / 2, buffer->size / 2); 1283 buffer->ptr = NULL; 1284 1285 return NULL; 1286 } 1287 1288 /* 1289 * Try reading anonymous memory while it is being unmapped. 1290 */ 1291 TEST_F(hmm, anon_teardown) 1292 { 1293 unsigned long npages; 1294 unsigned long size; 1295 unsigned long c; 1296 void *ret; 1297 1298 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1299 ASSERT_NE(npages, 0); 1300 size = npages << self->page_shift; 1301 1302 for (c = 0; c < NTIMES; ++c) { 1303 pthread_t thread; 1304 struct hmm_buffer *buffer; 1305 unsigned long i; 1306 int *ptr; 1307 int rc; 1308 1309 buffer = hmm_buffer_alloc(size, size, 1310 PROT_READ | PROT_WRITE, 1311 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1312 ASSERT_NE(buffer, NULL); 1313 1314 /* Initialize buffer in system memory. */ 1315 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1316 ptr[i] = i + c; 1317 1318 rc = pthread_create(&thread, NULL, unmap_buffer, buffer); 1319 ASSERT_EQ(rc, 0); 1320 1321 /* Simulate a device reading system memory. */ 1322 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1323 npages); 1324 if (rc == 0) { 1325 ASSERT_EQ(buffer->cpages, npages); 1326 ASSERT_EQ(buffer->faults, 1); 1327 1328 /* Check what the device read. */ 1329 for (i = 0, ptr = buffer->mirror; 1330 i < size / sizeof(*ptr); 1331 ++i) 1332 ASSERT_EQ(ptr[i], i + c); 1333 } 1334 1335 pthread_join(thread, &ret); 1336 hmm_buffer_free(buffer); 1337 } 1338 } 1339 1340 /* 1341 * Test memory snapshot without faulting in pages accessed by the device. 1342 */ 1343 TEST_F(hmm, mixedmap) 1344 { 1345 struct hmm_buffer *buffer; 1346 unsigned long npages; 1347 unsigned long size; 1348 unsigned char *m; 1349 int ret; 1350 1351 npages = 1; 1352 size = npages << self->page_shift; 1353 1354 buffer = hmm_buffer_alloc(size, npages, 1355 PROT_READ | PROT_WRITE, 1356 MAP_PRIVATE, self->fd); 1357 ASSERT_NE(buffer, NULL); 1358 1359 /* Simulate a device snapshotting CPU pagetables. */ 1360 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1361 ASSERT_EQ(ret, 0); 1362 ASSERT_EQ(buffer->cpages, npages); 1363 1364 /* Check what the device saw. */ 1365 m = buffer->mirror; 1366 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ); 1367 1368 hmm_buffer_free(buffer); 1369 } 1370 1371 /* 1372 * Test memory snapshot without faulting in pages accessed by the device. 1373 */ 1374 TEST_F(hmm2, snapshot) 1375 { 1376 struct hmm_buffer *buffer; 1377 unsigned long npages; 1378 unsigned long size; 1379 int *ptr; 1380 unsigned char *p; 1381 unsigned char *m; 1382 int ret; 1383 int val; 1384 1385 npages = 7; 1386 size = npages << self->page_shift; 1387 1388 buffer = hmm_buffer_alloc(size, npages, 1389 PROT_NONE, 1390 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1391 ASSERT_NE(buffer, NULL); 1392 p = buffer->ptr; 1393 1394 /* Punch a hole after the first page address. */ 1395 ret = munmap(buffer->ptr + self->page_size, self->page_size); 1396 ASSERT_EQ(ret, 0); 1397 1398 /* Page 2 will be read-only zero page. */ 1399 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, 1400 PROT_READ); 1401 ASSERT_EQ(ret, 0); 1402 ptr = (int *)(buffer->ptr + 2 * self->page_size); 1403 val = *ptr + 3; 1404 ASSERT_EQ(val, 3); 1405 1406 /* Page 3 will be read-only. */ 1407 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1408 PROT_READ | PROT_WRITE); 1409 ASSERT_EQ(ret, 0); 1410 ptr = (int *)(buffer->ptr + 3 * self->page_size); 1411 *ptr = val; 1412 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1413 PROT_READ); 1414 ASSERT_EQ(ret, 0); 1415 1416 /* Page 4-6 will be read-write. */ 1417 ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size, 1418 PROT_READ | PROT_WRITE); 1419 ASSERT_EQ(ret, 0); 1420 ptr = (int *)(buffer->ptr + 4 * self->page_size); 1421 *ptr = val; 1422 1423 /* Page 5 will be migrated to device 0. */ 1424 buffer->ptr = p + 5 * self->page_size; 1425 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); 1426 ASSERT_EQ(ret, 0); 1427 ASSERT_EQ(buffer->cpages, 1); 1428 1429 /* Page 6 will be migrated to device 1. */ 1430 buffer->ptr = p + 6 * self->page_size; 1431 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1); 1432 ASSERT_EQ(ret, 0); 1433 ASSERT_EQ(buffer->cpages, 1); 1434 1435 /* Simulate a device snapshotting CPU pagetables. */ 1436 buffer->ptr = p; 1437 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1438 ASSERT_EQ(ret, 0); 1439 ASSERT_EQ(buffer->cpages, npages); 1440 1441 /* Check what the device saw. */ 1442 m = buffer->mirror; 1443 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR); 1444 ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR); 1445 ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ); 1446 ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ); 1447 ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE); 1448 if (!hmm_is_coherent_type(variant->device_number0)) { 1449 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL | 1450 HMM_DMIRROR_PROT_WRITE); 1451 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE); 1452 } else { 1453 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | 1454 HMM_DMIRROR_PROT_WRITE); 1455 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE | 1456 HMM_DMIRROR_PROT_WRITE); 1457 } 1458 1459 hmm_buffer_free(buffer); 1460 } 1461 1462 /* 1463 * Test the hmm_range_fault() handling of large pages (PMD or PUD) 1464 * that should be mapped by a large page table entry. 1465 */ 1466 TEST_F(hmm, compound) 1467 { 1468 struct hmm_buffer *buffer; 1469 unsigned long npages; 1470 unsigned long size; 1471 unsigned long default_hsize = default_huge_page_size(); 1472 int *ptr; 1473 unsigned char *m; 1474 unsigned char prot; 1475 int ret; 1476 unsigned long i; 1477 1478 /* Skip test if we can't allocate a hugetlbfs page. */ 1479 if (!hugetlb_free_default_pages()) 1480 SKIP(return, "Not enough huge pages"); 1481 1482 size = ALIGN(TWOMEG, default_hsize); 1483 npages = size >> self->page_shift; 1484 1485 buffer = hmm_buffer_alloc(size, npages, 1486 PROT_READ | PROT_WRITE, 1487 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1); 1488 if (!buffer) 1489 SKIP(return, "Huge page could not be allocated"); 1490 1491 /* Initialize the pages the device will snapshot in buffer->ptr. */ 1492 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1493 ptr[i] = i; 1494 1495 /* Simulate a device snapshotting CPU pagetables. */ 1496 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1497 ASSERT_EQ(ret, 0); 1498 ASSERT_EQ(buffer->cpages, npages); 1499 1500 /* 1501 * Check what the device saw. The region is backed by a single huge 1502 * page that the device reports either at PMD or at PUD level depending 1503 * on the configured default hugepage size. Determine that level from 1504 * the first page and require every page in the range to match it 1505 * exactly, so that a fragmented mapping mixing levels (or a missing 1506 * large-page bit) is still caught and reported with its actual value. 1507 */ 1508 m = buffer->mirror; 1509 prot = HMM_DMIRROR_PROT_WRITE | 1510 ((m[0] & HMM_DMIRROR_PROT_PUD) ? HMM_DMIRROR_PROT_PUD : 1511 HMM_DMIRROR_PROT_PMD); 1512 for (i = 0; i < npages; ++i) 1513 ASSERT_EQ(m[i], prot); 1514 1515 /* Make the region read-only. */ 1516 ret = mprotect(buffer->ptr, size, PROT_READ); 1517 ASSERT_EQ(ret, 0); 1518 1519 /* Simulate a device snapshotting CPU pagetables. */ 1520 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1521 ASSERT_EQ(ret, 0); 1522 ASSERT_EQ(buffer->cpages, npages); 1523 1524 /* 1525 * Check what the device saw after mprotect(PROT_READ). Same 1526 * approach as above: determine the mapping level from the first 1527 * page and require every page to match it exactly. 1528 */ 1529 m = buffer->mirror; 1530 prot = HMM_DMIRROR_PROT_READ | 1531 ((m[0] & HMM_DMIRROR_PROT_PUD) ? HMM_DMIRROR_PROT_PUD : 1532 HMM_DMIRROR_PROT_PMD); 1533 for (i = 0; i < npages; ++i) 1534 ASSERT_EQ(m[i], prot); 1535 1536 munmap(buffer->ptr, buffer->size); 1537 buffer->ptr = NULL; 1538 hmm_buffer_free(buffer); 1539 } 1540 1541 /* 1542 * Test two devices reading the same memory (double mapped). 1543 */ 1544 TEST_F(hmm2, double_map) 1545 { 1546 struct hmm_buffer *buffer; 1547 unsigned long npages; 1548 unsigned long size; 1549 unsigned long i; 1550 int *ptr; 1551 int ret; 1552 1553 npages = 6; 1554 size = npages << self->page_shift; 1555 1556 buffer = hmm_buffer_alloc(size, size, 1557 PROT_READ | PROT_WRITE, 1558 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1559 ASSERT_NE(buffer, NULL); 1560 1561 /* Initialize buffer in system memory. */ 1562 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1563 ptr[i] = i; 1564 1565 /* Make region read-only. */ 1566 ret = mprotect(buffer->ptr, size, PROT_READ); 1567 ASSERT_EQ(ret, 0); 1568 1569 /* Simulate device 0 reading system memory. */ 1570 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); 1571 ASSERT_EQ(ret, 0); 1572 ASSERT_EQ(buffer->cpages, npages); 1573 ASSERT_EQ(buffer->faults, 1); 1574 1575 /* Check what the device read. */ 1576 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1577 ASSERT_EQ(ptr[i], i); 1578 1579 /* Simulate device 1 reading system memory. */ 1580 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages); 1581 ASSERT_EQ(ret, 0); 1582 ASSERT_EQ(buffer->cpages, npages); 1583 ASSERT_EQ(buffer->faults, 1); 1584 1585 /* Check what the device read. */ 1586 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1587 ASSERT_EQ(ptr[i], i); 1588 1589 /* Migrate pages to device 1 and try to read from device 0. */ 1590 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); 1591 ASSERT_EQ(ret, 0); 1592 ASSERT_EQ(buffer->cpages, npages); 1593 1594 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); 1595 ASSERT_EQ(ret, 0); 1596 ASSERT_EQ(buffer->cpages, npages); 1597 ASSERT_EQ(buffer->faults, 1); 1598 1599 /* Check what device 0 read. */ 1600 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1601 ASSERT_EQ(ptr[i], i); 1602 1603 hmm_buffer_free(buffer); 1604 } 1605 1606 /* 1607 * Basic check of exclusive faulting. 1608 */ 1609 TEST_F(hmm, exclusive) 1610 { 1611 struct hmm_buffer *buffer; 1612 unsigned long npages; 1613 unsigned long size; 1614 unsigned long i; 1615 int *ptr; 1616 int ret; 1617 1618 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1619 ASSERT_NE(npages, 0); 1620 size = npages << self->page_shift; 1621 1622 buffer = hmm_buffer_alloc(size, size, 1623 PROT_READ | PROT_WRITE, 1624 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1625 ASSERT_NE(buffer, NULL); 1626 1627 /* Initialize buffer in system memory. */ 1628 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1629 ptr[i] = i; 1630 1631 /* Map memory exclusively for device access. */ 1632 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1633 ASSERT_EQ(ret, 0); 1634 ASSERT_EQ(buffer->cpages, npages); 1635 1636 /* Check what the device read. */ 1637 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1638 ASSERT_EQ(ptr[i], i); 1639 1640 /* Fault pages back to system memory and check them. */ 1641 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1642 ASSERT_EQ(ptr[i]++, i); 1643 1644 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1645 ASSERT_EQ(ptr[i], i+1); 1646 1647 /* Check atomic access revoked */ 1648 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages); 1649 ASSERT_EQ(ret, 0); 1650 1651 hmm_buffer_free(buffer); 1652 } 1653 1654 TEST_F(hmm, exclusive_mprotect) 1655 { 1656 struct hmm_buffer *buffer; 1657 unsigned long npages; 1658 unsigned long size; 1659 unsigned long i; 1660 int *ptr; 1661 int ret; 1662 1663 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1664 ASSERT_NE(npages, 0); 1665 size = npages << self->page_shift; 1666 1667 buffer = hmm_buffer_alloc(size, size, 1668 PROT_READ | PROT_WRITE, 1669 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1670 ASSERT_NE(buffer, NULL); 1671 1672 /* Initialize buffer in system memory. */ 1673 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1674 ptr[i] = i; 1675 1676 /* Map memory exclusively for device access. */ 1677 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1678 ASSERT_EQ(ret, 0); 1679 ASSERT_EQ(buffer->cpages, npages); 1680 1681 /* Check what the device read. */ 1682 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1683 ASSERT_EQ(ptr[i], i); 1684 1685 ret = mprotect(buffer->ptr, size, PROT_READ); 1686 ASSERT_EQ(ret, 0); 1687 1688 /* Simulate a device writing system memory. */ 1689 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 1690 ASSERT_EQ(ret, -EPERM); 1691 1692 hmm_buffer_free(buffer); 1693 } 1694 1695 /* 1696 * Check copy-on-write works. 1697 */ 1698 TEST_F(hmm, exclusive_cow) 1699 { 1700 struct hmm_buffer *buffer; 1701 unsigned long npages; 1702 unsigned long size; 1703 unsigned long i; 1704 int *ptr; 1705 int ret; 1706 pid_t pid; 1707 int status; 1708 1709 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1710 ASSERT_NE(npages, 0); 1711 size = npages << self->page_shift; 1712 1713 buffer = hmm_buffer_alloc(size, size, 1714 PROT_READ | PROT_WRITE, 1715 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1716 ASSERT_NE(buffer, NULL); 1717 1718 /* Initialize buffer in system memory. */ 1719 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1720 ptr[i] = i; 1721 1722 /* Map memory exclusively for device access. */ 1723 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1724 ASSERT_EQ(ret, 0); 1725 ASSERT_EQ(buffer->cpages, npages); 1726 1727 pid = fork(); 1728 if (pid == -1) 1729 ASSERT_EQ(pid, 0); 1730 1731 if (pid == 0) { 1732 /* 1733 * Child verifies COW independently, then _exit(0)s so it does 1734 * not run the test teardown. A failed ASSERT_* here makes the 1735 * harness abort() the child, so the parent sees 1736 * !WIFEXITED(status) below and fails in turn. 1737 */ 1738 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1739 ASSERT_EQ(ptr[i]++, i); 1740 1741 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1742 ASSERT_EQ(ptr[i], i + 1); 1743 1744 _exit(0); 1745 } 1746 1747 /* Parent: also increment to verify COW works for both processes. */ 1748 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1749 ASSERT_EQ(ptr[i]++, i); 1750 1751 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1752 ASSERT_EQ(ptr[i], i + 1); 1753 1754 /* Parent: wait for child and then free the buffer. */ 1755 ASSERT_EQ(waitpid(pid, &status, 0), pid); 1756 ASSERT_TRUE(WIFEXITED(status)); 1757 ASSERT_EQ(WEXITSTATUS(status), 0); 1758 1759 hmm_buffer_free(buffer); 1760 } 1761 1762 static int gup_test_exec(int gup_fd, unsigned long addr, int cmd, 1763 int npages, int size, int flags) 1764 { 1765 struct gup_test gup = { 1766 .nr_pages_per_call = npages, 1767 .addr = addr, 1768 .gup_flags = FOLL_WRITE | flags, 1769 .size = size, 1770 }; 1771 1772 if (ioctl(gup_fd, cmd, &gup)) { 1773 perror("ioctl on error\n"); 1774 return errno; 1775 } 1776 1777 return 0; 1778 } 1779 1780 /* 1781 * Test get user device pages through gup_test. Setting PIN_LONGTERM flag. 1782 * This should trigger a migration back to system memory for both, private 1783 * and coherent type pages. 1784 * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added 1785 * to your configuration before you run it. 1786 */ 1787 TEST_F(hmm, hmm_gup_test) 1788 { 1789 struct hmm_buffer *buffer; 1790 int gup_fd; 1791 unsigned long npages; 1792 unsigned long size; 1793 unsigned long i; 1794 int *ptr; 1795 int ret; 1796 unsigned char *m; 1797 1798 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); 1799 if (gup_fd == -1) 1800 SKIP(return, "Skipping test, could not find gup_test driver"); 1801 1802 npages = 4; 1803 size = npages << self->page_shift; 1804 1805 buffer = hmm_buffer_alloc(size, size, 1806 PROT_READ | PROT_WRITE, 1807 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1808 ASSERT_NE(buffer, NULL); 1809 1810 /* Initialize buffer in system memory. */ 1811 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1812 ptr[i] = i; 1813 1814 /* Migrate memory to device. */ 1815 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1816 ASSERT_EQ(ret, 0); 1817 ASSERT_EQ(buffer->cpages, npages); 1818 /* Check what the device read. */ 1819 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1820 ASSERT_EQ(ptr[i], i); 1821 1822 ASSERT_EQ(gup_test_exec(gup_fd, 1823 (unsigned long)buffer->ptr, 1824 GUP_BASIC_TEST, 1, self->page_size, 0), 0); 1825 ASSERT_EQ(gup_test_exec(gup_fd, 1826 (unsigned long)buffer->ptr + 1 * self->page_size, 1827 GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0); 1828 ASSERT_EQ(gup_test_exec(gup_fd, 1829 (unsigned long)buffer->ptr + 2 * self->page_size, 1830 PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0); 1831 ASSERT_EQ(gup_test_exec(gup_fd, 1832 (unsigned long)buffer->ptr + 3 * self->page_size, 1833 PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0); 1834 1835 /* Take snapshot to CPU pagetables */ 1836 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1837 ASSERT_EQ(ret, 0); 1838 ASSERT_EQ(buffer->cpages, npages); 1839 m = buffer->mirror; 1840 if (hmm_is_coherent_type(variant->device_number)) { 1841 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]); 1842 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]); 1843 } else { 1844 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]); 1845 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]); 1846 } 1847 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]); 1848 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]); 1849 /* 1850 * Check again the content on the pages. Make sure there's no 1851 * corrupted data. 1852 */ 1853 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1854 ASSERT_EQ(ptr[i], i); 1855 1856 close(gup_fd); 1857 hmm_buffer_free(buffer); 1858 } 1859 1860 /* 1861 * Test copy-on-write in device pages. 1862 * In case of writing to COW private page(s), a page fault will migrate pages 1863 * back to system memory first. Then, these pages will be duplicated. In case 1864 * of COW device coherent type, pages are duplicated directly from device 1865 * memory. 1866 */ 1867 TEST_F(hmm, hmm_cow_in_device) 1868 { 1869 struct hmm_buffer *buffer; 1870 unsigned long npages; 1871 unsigned long size; 1872 unsigned long i; 1873 int *ptr; 1874 int ret; 1875 unsigned char *m; 1876 pid_t pid; 1877 int status; 1878 1879 npages = 4; 1880 size = npages << self->page_shift; 1881 1882 buffer = hmm_buffer_alloc(size, size, 1883 PROT_READ | PROT_WRITE, 1884 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1885 ASSERT_NE(buffer, NULL); 1886 1887 /* Initialize buffer in system memory. */ 1888 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1889 ptr[i] = i; 1890 1891 /* Migrate memory to device. */ 1892 1893 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1894 ASSERT_EQ(ret, 0); 1895 ASSERT_EQ(buffer->cpages, npages); 1896 1897 pid = fork(); 1898 if (pid == -1) 1899 ASSERT_EQ(pid, 0); 1900 if (!pid) { 1901 /* Child process waits for SIGKILL from the parent. */ 1902 while (1) { 1903 } 1904 /* Should not reach this */ 1905 } 1906 /* Parent process writes to COW pages(s) and gets a 1907 * new copy in system. In case of device private pages, 1908 * this write causes a migration to system mem first. 1909 */ 1910 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1911 ptr[i] = i; 1912 1913 /* Terminate child and wait */ 1914 EXPECT_EQ(0, kill(pid, SIGKILL)); 1915 EXPECT_EQ(pid, waitpid(pid, &status, 0)); 1916 EXPECT_NE(0, WIFSIGNALED(status)); 1917 EXPECT_EQ(SIGKILL, WTERMSIG(status)); 1918 1919 /* Take snapshot to CPU pagetables */ 1920 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1921 ASSERT_EQ(ret, 0); 1922 ASSERT_EQ(buffer->cpages, npages); 1923 m = buffer->mirror; 1924 for (i = 0; i < npages; i++) 1925 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]); 1926 1927 hmm_buffer_free(buffer); 1928 } 1929 1930 /* 1931 * Migrate private anonymous huge empty page. 1932 */ 1933 TEST_F(hmm, migrate_anon_huge_empty) 1934 { 1935 struct hmm_buffer *buffer; 1936 unsigned long npages; 1937 unsigned long size; 1938 unsigned long i; 1939 void *old_ptr; 1940 void *map; 1941 int *ptr; 1942 int ret; 1943 1944 size = read_pmd_pagesize(); 1945 1946 buffer = hmm_buffer_alloc(2 * size, size, 1947 PROT_READ, 1948 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1949 ASSERT_NE(buffer, NULL); 1950 memset(buffer->mirror, 0xFF, size); 1951 1952 npages = size >> self->page_shift; 1953 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 1954 ret = madvise(map, size, MADV_HUGEPAGE); 1955 ASSERT_EQ(ret, 0); 1956 old_ptr = buffer->ptr; 1957 buffer->ptr = map; 1958 1959 /* Migrate memory to device. */ 1960 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1961 ASSERT_EQ(ret, 0); 1962 ASSERT_EQ(buffer->cpages, npages); 1963 1964 /* Check what the device read. */ 1965 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1966 ASSERT_EQ(ptr[i], 0); 1967 1968 buffer->ptr = old_ptr; 1969 hmm_buffer_free(buffer); 1970 } 1971 1972 /* 1973 * Migrate private anonymous huge zero page. 1974 */ 1975 TEST_F(hmm, migrate_anon_huge_zero) 1976 { 1977 struct hmm_buffer *buffer; 1978 unsigned long npages; 1979 unsigned long size; 1980 unsigned long i; 1981 void *old_ptr; 1982 void *map; 1983 int *ptr; 1984 int ret; 1985 int val; 1986 1987 size = read_pmd_pagesize(); 1988 1989 buffer = hmm_buffer_alloc(2 * size, size, 1990 PROT_READ, 1991 MAP_PRIVATE | MAP_ANONYMOUS, -1); 1992 ASSERT_NE(buffer, NULL); 1993 memset(buffer->mirror, 0xFF, size); 1994 1995 npages = size >> self->page_shift; 1996 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 1997 ret = madvise(map, size, MADV_HUGEPAGE); 1998 ASSERT_EQ(ret, 0); 1999 old_ptr = buffer->ptr; 2000 buffer->ptr = map; 2001 2002 /* Initialize a read-only zero huge page. */ 2003 val = *(int *)buffer->ptr; 2004 ASSERT_EQ(val, 0); 2005 2006 /* Migrate memory to device. */ 2007 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2008 ASSERT_EQ(ret, 0); 2009 ASSERT_EQ(buffer->cpages, npages); 2010 2011 /* Check what the device read. */ 2012 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2013 ASSERT_EQ(ptr[i], 0); 2014 2015 /* Fault pages back to system memory and check them. */ 2016 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) { 2017 ASSERT_EQ(ptr[i], 0); 2018 /* If it asserts once, it probably will 500,000 times */ 2019 if (ptr[i] != 0) 2020 break; 2021 } 2022 2023 buffer->ptr = old_ptr; 2024 hmm_buffer_free(buffer); 2025 } 2026 2027 /* 2028 * Migrate private anonymous huge page and free. 2029 */ 2030 TEST_F(hmm, migrate_anon_huge_free) 2031 { 2032 struct hmm_buffer *buffer; 2033 unsigned long npages; 2034 unsigned long size; 2035 unsigned long i; 2036 void *old_ptr; 2037 void *map; 2038 int *ptr; 2039 int ret; 2040 2041 size = read_pmd_pagesize(); 2042 2043 buffer = hmm_buffer_alloc(2 * size, size, 2044 PROT_READ | PROT_WRITE, 2045 MAP_PRIVATE | MAP_ANONYMOUS, -1); 2046 ASSERT_NE(buffer, NULL); 2047 memset(buffer->mirror, 0xFF, size); 2048 2049 npages = size >> self->page_shift; 2050 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2051 ret = madvise(map, size, MADV_HUGEPAGE); 2052 ASSERT_EQ(ret, 0); 2053 old_ptr = buffer->ptr; 2054 buffer->ptr = map; 2055 2056 /* Initialize buffer in system memory. */ 2057 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2058 ptr[i] = i; 2059 2060 /* Migrate memory to device. */ 2061 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2062 ASSERT_EQ(ret, 0); 2063 ASSERT_EQ(buffer->cpages, npages); 2064 2065 /* Check what the device read. */ 2066 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2067 ASSERT_EQ(ptr[i], i); 2068 2069 /* Try freeing it. */ 2070 ret = madvise(map, size, MADV_FREE); 2071 ASSERT_EQ(ret, 0); 2072 2073 buffer->ptr = old_ptr; 2074 hmm_buffer_free(buffer); 2075 } 2076 2077 /* 2078 * Migrate private anonymous huge page and fault back to sysmem. 2079 */ 2080 TEST_F(hmm, migrate_anon_huge_fault) 2081 { 2082 struct hmm_buffer *buffer; 2083 unsigned long npages; 2084 unsigned long size; 2085 unsigned long i; 2086 unsigned char *m; 2087 uint64_t entry; 2088 void *old_ptr; 2089 void *map; 2090 int pagemap_fd; 2091 int *ptr; 2092 int ret; 2093 2094 size = read_pmd_pagesize(); 2095 2096 buffer = hmm_buffer_alloc(2 * size, size, 2097 PROT_READ | PROT_WRITE, 2098 MAP_PRIVATE | MAP_ANONYMOUS, -1); 2099 ASSERT_NE(buffer, NULL); 2100 memset(buffer->mirror, 0xFF, size); 2101 2102 npages = size >> self->page_shift; 2103 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2104 old_ptr = buffer->ptr; 2105 buffer->ptr = map; 2106 2107 /* Initialize buffer in system memory. */ 2108 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2109 ptr[i] = i; 2110 2111 ret = madvise(map, size, MADV_COLLAPSE); 2112 ASSERT_EQ(ret, 0); 2113 2114 /* Migrate memory to device. */ 2115 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2116 ASSERT_EQ(ret, 0); 2117 ASSERT_EQ(buffer->cpages, npages); 2118 2119 /* Check what the device read. */ 2120 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2121 ASSERT_EQ(ptr[i], i); 2122 2123 if (!hmm_is_coherent_type(variant->device_number)) { 2124 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, 2125 buffer, npages); 2126 ASSERT_EQ(ret, 0); 2127 ASSERT_EQ(buffer->cpages, npages); 2128 2129 m = buffer->mirror; 2130 for (i = 0; i < npages; ++i) 2131 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL | 2132 HMM_DMIRROR_PROT_WRITE | 2133 HMM_DMIRROR_PROT_PMD); 2134 2135 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 2136 ASSERT_GE(pagemap_fd, 0); 2137 2138 for (i = 0; i < npages; ++i) { 2139 entry = pagemap_get_entry(pagemap_fd, 2140 (char *)buffer->ptr + i * self->page_size); 2141 2142 ASSERT_NE(entry & PM_SWAP, 0); 2143 ASSERT_FALSE(PAGEMAP_PRESENT(entry)); 2144 } 2145 2146 close(pagemap_fd); 2147 } 2148 2149 /* Fault pages back to system memory and check them. */ 2150 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2151 ASSERT_EQ(ptr[i], i); 2152 2153 buffer->ptr = old_ptr; 2154 hmm_buffer_free(buffer); 2155 } 2156 2157 /* 2158 * Migrate memory and fault back to sysmem after partially unmapping. 2159 */ 2160 TEST_F(hmm, migrate_partial_unmap_fault) 2161 { 2162 struct hmm_buffer *buffer; 2163 unsigned long npages; 2164 unsigned long size = read_pmd_pagesize(); 2165 unsigned long unmap_size; 2166 unsigned long offsets[3]; 2167 unsigned long i; 2168 void *old_ptr; 2169 void *map; 2170 int *ptr; 2171 int ret, j, use_thp; 2172 2173 if (!size) 2174 size = TWOMEG; 2175 2176 unmap_size = size / 2; 2177 offsets[0] = 0; 2178 offsets[1] = size / 4; 2179 offsets[2] = size / 2; 2180 2181 for (use_thp = 0; use_thp < 2; ++use_thp) { 2182 for (j = 0; j < ARRAY_SIZE(offsets); ++j) { 2183 buffer = hmm_buffer_alloc(2 * size, size, 2184 PROT_READ | PROT_WRITE, 2185 MAP_PRIVATE | MAP_ANONYMOUS, -1); 2186 ASSERT_NE(buffer, NULL); 2187 memset(buffer->mirror, 0xFF, size); 2188 npages = size >> self->page_shift; 2189 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2190 if (use_thp) 2191 ret = madvise(map, size, MADV_HUGEPAGE); 2192 else 2193 ret = madvise(map, size, MADV_NOHUGEPAGE); 2194 ASSERT_EQ(ret, 0); 2195 old_ptr = buffer->ptr; 2196 buffer->ptr = map; 2197 2198 /* Initialize buffer in system memory. */ 2199 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2200 ptr[i] = i; 2201 2202 /* Migrate memory to device. */ 2203 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2204 ASSERT_EQ(ret, 0); 2205 ASSERT_EQ(buffer->cpages, npages); 2206 2207 /* Check what the device read. */ 2208 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2209 ASSERT_EQ(ptr[i], i); 2210 2211 munmap(buffer->ptr + offsets[j], unmap_size); 2212 2213 /* Fault pages back to system memory and check them. */ 2214 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2215 if (i * sizeof(int) < offsets[j] || 2216 i * sizeof(int) >= offsets[j] + unmap_size) 2217 ASSERT_EQ(ptr[i], i); 2218 2219 buffer->ptr = old_ptr; 2220 hmm_buffer_free(buffer); 2221 } 2222 } 2223 } 2224 2225 TEST_F(hmm, migrate_remap_fault) 2226 { 2227 struct hmm_buffer *buffer; 2228 unsigned long npages; 2229 unsigned long size = read_pmd_pagesize(); 2230 unsigned long offsets[3]; 2231 unsigned long i; 2232 void *old_ptr, *new_ptr = NULL; 2233 void *map; 2234 int *ptr; 2235 int ret, j, use_thp, dont_unmap, before; 2236 2237 if (!size) 2238 size = TWOMEG; 2239 2240 offsets[0] = 0; 2241 offsets[1] = size / 4; 2242 offsets[2] = size / 2; 2243 2244 for (before = 0; before < 2; ++before) { 2245 for (dont_unmap = 0; dont_unmap < 2; ++dont_unmap) { 2246 for (use_thp = 0; use_thp < 2; ++use_thp) { 2247 for (j = 0; j < ARRAY_SIZE(offsets); ++j) { 2248 int flags = MREMAP_MAYMOVE | MREMAP_FIXED; 2249 2250 if (dont_unmap) 2251 flags |= MREMAP_DONTUNMAP; 2252 2253 buffer = hmm_buffer_alloc(8 * size, size, 2254 PROT_READ | PROT_WRITE, 2255 MAP_PRIVATE | MAP_ANONYMOUS, -1); 2256 ASSERT_NE(buffer, NULL); 2257 memset(buffer->mirror, 0xFF, size); 2258 npages = size >> self->page_shift; 2259 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2260 if (use_thp) 2261 ret = madvise(map, size, MADV_HUGEPAGE); 2262 else 2263 ret = madvise(map, size, MADV_NOHUGEPAGE); 2264 ASSERT_EQ(ret, 0); 2265 old_ptr = buffer->ptr; 2266 munmap(map + size, size * 2); 2267 buffer->ptr = map; 2268 2269 /* Initialize buffer in system memory. */ 2270 for (i = 0, ptr = buffer->ptr; 2271 i < size / sizeof(*ptr); ++i) 2272 ptr[i] = i; 2273 2274 if (before) { 2275 new_ptr = mremap((void *)map, size, size, flags, 2276 map + size + offsets[j]); 2277 ASSERT_NE(new_ptr, MAP_FAILED); 2278 buffer->ptr = new_ptr; 2279 } 2280 2281 /* Migrate memory to device. */ 2282 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2283 ASSERT_EQ(ret, 0); 2284 ASSERT_EQ(buffer->cpages, npages); 2285 2286 /* Check what the device read. */ 2287 for (i = 0, ptr = buffer->mirror; 2288 i < size / sizeof(*ptr); ++i) 2289 ASSERT_EQ(ptr[i], i); 2290 2291 if (!before) { 2292 new_ptr = mremap((void *)map, size, size, flags, 2293 map + size + offsets[j]); 2294 ASSERT_NE(new_ptr, MAP_FAILED); 2295 buffer->ptr = new_ptr; 2296 } 2297 2298 /* Fault pages back to system memory and check them. */ 2299 for (i = 0, ptr = buffer->ptr; 2300 i < size / sizeof(*ptr); ++i) 2301 ASSERT_EQ(ptr[i], i); 2302 2303 munmap(new_ptr, size); 2304 buffer->ptr = old_ptr; 2305 hmm_buffer_free(buffer); 2306 } 2307 } 2308 } 2309 } 2310 } 2311 2312 /* 2313 * Migrate private anonymous huge page with allocation errors. 2314 */ 2315 TEST_F(hmm, migrate_anon_huge_err) 2316 { 2317 struct hmm_buffer *buffer; 2318 unsigned long npages; 2319 unsigned long size; 2320 unsigned long i; 2321 void *old_ptr; 2322 void *map; 2323 int *ptr; 2324 int ret; 2325 2326 size = read_pmd_pagesize(); 2327 2328 buffer = hmm_buffer_alloc(2 * size, 2 * size, 2329 PROT_READ | PROT_WRITE, 2330 MAP_PRIVATE | MAP_ANONYMOUS, -1); 2331 ASSERT_NE(buffer, NULL); 2332 memset(buffer->mirror, 0xFF, 2 * size); 2333 2334 old_ptr = buffer->ptr; 2335 2336 npages = size >> self->page_shift; 2337 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2338 ret = madvise(map, size, MADV_HUGEPAGE); 2339 ASSERT_EQ(ret, 0); 2340 buffer->ptr = map; 2341 2342 /* Initialize buffer in system memory. */ 2343 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2344 ptr[i] = i; 2345 2346 /* Migrate memory to device but force a THP allocation error. */ 2347 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2348 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2349 ASSERT_EQ(ret, 0); 2350 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2351 ASSERT_EQ(ret, 0); 2352 ASSERT_EQ(buffer->cpages, npages); 2353 2354 /* Check what the device read. */ 2355 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) { 2356 ASSERT_EQ(ptr[i], i); 2357 if (ptr[i] != i) 2358 break; 2359 } 2360 2361 /* Try faulting back a single (PAGE_SIZE) page. */ 2362 ptr = buffer->ptr; 2363 ASSERT_EQ(ptr[2048], 2048); 2364 2365 /* unmap and remap the region to reset things. */ 2366 ret = munmap(old_ptr, 2 * size); 2367 ASSERT_EQ(ret, 0); 2368 old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, 2369 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2370 ASSERT_NE(old_ptr, MAP_FAILED); 2371 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2372 ret = madvise(map, size, MADV_HUGEPAGE); 2373 ASSERT_EQ(ret, 0); 2374 buffer->ptr = map; 2375 2376 /* Initialize buffer in system memory. */ 2377 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2378 ptr[i] = i; 2379 2380 /* Migrate THP to device. */ 2381 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2382 ASSERT_EQ(ret, 0); 2383 ASSERT_EQ(buffer->cpages, npages); 2384 2385 /* 2386 * Force an allocation error when faulting back a THP resident in the 2387 * device. 2388 */ 2389 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2390 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2391 ASSERT_EQ(ret, 0); 2392 2393 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages); 2394 ASSERT_EQ(ret, 0); 2395 ptr = buffer->ptr; 2396 ASSERT_EQ(ptr[2048], 2048); 2397 2398 buffer->ptr = old_ptr; 2399 hmm_buffer_free(buffer); 2400 } 2401 2402 /* 2403 * Migrate private anonymous huge zero page with allocation errors. 2404 */ 2405 TEST_F(hmm, migrate_anon_huge_zero_err) 2406 { 2407 struct hmm_buffer *buffer; 2408 unsigned long npages; 2409 unsigned long size; 2410 unsigned long i; 2411 void *old_ptr; 2412 void *map; 2413 int *ptr; 2414 int ret; 2415 2416 size = read_pmd_pagesize(); 2417 2418 buffer = hmm_buffer_alloc(2 * size, 2 * size, 2419 PROT_READ, 2420 MAP_PRIVATE | MAP_ANONYMOUS, -1); 2421 ASSERT_NE(buffer, NULL); 2422 memset(buffer->mirror, 0xFF, 2 * size); 2423 2424 old_ptr = buffer->ptr; 2425 2426 npages = size >> self->page_shift; 2427 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2428 ret = madvise(map, size, MADV_HUGEPAGE); 2429 ASSERT_EQ(ret, 0); 2430 buffer->ptr = map; 2431 2432 /* Migrate memory to device but force a THP allocation error. */ 2433 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2434 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2435 ASSERT_EQ(ret, 0); 2436 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2437 ASSERT_EQ(ret, 0); 2438 ASSERT_EQ(buffer->cpages, npages); 2439 2440 /* Check what the device read. */ 2441 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2442 ASSERT_EQ(ptr[i], 0); 2443 2444 /* Try faulting back a single (PAGE_SIZE) page. */ 2445 ptr = buffer->ptr; 2446 ASSERT_EQ(ptr[2048], 0); 2447 2448 /* unmap and remap the region to reset things. */ 2449 ret = munmap(old_ptr, 2 * size); 2450 ASSERT_EQ(ret, 0); 2451 old_ptr = mmap(NULL, 2 * size, PROT_READ, 2452 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2453 ASSERT_NE(old_ptr, MAP_FAILED); 2454 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2455 ret = madvise(map, size, MADV_HUGEPAGE); 2456 ASSERT_EQ(ret, 0); 2457 buffer->ptr = map; 2458 2459 /* Initialize buffer in system memory (zero THP page). */ 2460 ret = ptr[0]; 2461 ASSERT_EQ(ret, 0); 2462 2463 /* Migrate memory to device but force a THP allocation error. */ 2464 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2465 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2466 ASSERT_EQ(ret, 0); 2467 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2468 ASSERT_EQ(ret, 0); 2469 ASSERT_EQ(buffer->cpages, npages); 2470 2471 /* Fault the device memory back and check it. */ 2472 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2473 ASSERT_EQ(ptr[i], 0); 2474 2475 buffer->ptr = old_ptr; 2476 hmm_buffer_free(buffer); 2477 } 2478 2479 struct benchmark_results { 2480 double sys_to_dev_time; 2481 double dev_to_sys_time; 2482 double throughput_s2d; 2483 double throughput_d2s; 2484 }; 2485 2486 static double get_time_ms(void) 2487 { 2488 struct timeval tv; 2489 2490 gettimeofday(&tv, NULL); 2491 return (tv.tv_sec * 1000.0) + (tv.tv_usec / 1000.0); 2492 } 2493 2494 static void print_benchmark_results(const char *test_name, size_t buffer_size, 2495 struct benchmark_results *thp, 2496 struct benchmark_results *regular) 2497 { 2498 double s2d_improvement = ((regular->sys_to_dev_time - thp->sys_to_dev_time) / 2499 regular->sys_to_dev_time) * 100.0; 2500 double d2s_improvement = ((regular->dev_to_sys_time - thp->dev_to_sys_time) / 2501 regular->dev_to_sys_time) * 100.0; 2502 double throughput_s2d_improvement = ((thp->throughput_s2d - regular->throughput_s2d) / 2503 regular->throughput_s2d) * 100.0; 2504 double throughput_d2s_improvement = ((thp->throughput_d2s - regular->throughput_d2s) / 2505 regular->throughput_d2s) * 100.0; 2506 2507 printf("\n=== %s (%.1f MB) ===\n", test_name, buffer_size / (1024.0 * 1024.0)); 2508 printf(" | With THP | Without THP | Improvement\n"); 2509 printf("---------------------------------------------------------------------\n"); 2510 printf("Sys->Dev Migration | %.3f ms | %.3f ms | %.1f%%\n", 2511 thp->sys_to_dev_time, regular->sys_to_dev_time, s2d_improvement); 2512 printf("Dev->Sys Migration | %.3f ms | %.3f ms | %.1f%%\n", 2513 thp->dev_to_sys_time, regular->dev_to_sys_time, d2s_improvement); 2514 printf("S->D Throughput | %.2f GB/s | %.2f GB/s | %.1f%%\n", 2515 thp->throughput_s2d, regular->throughput_s2d, throughput_s2d_improvement); 2516 printf("D->S Throughput | %.2f GB/s | %.2f GB/s | %.1f%%\n", 2517 thp->throughput_d2s, regular->throughput_d2s, throughput_d2s_improvement); 2518 } 2519 2520 /* 2521 * Run a single migration benchmark 2522 * fd: file descriptor for hmm device 2523 * use_thp: whether to use THP 2524 * buffer_size: size of buffer to allocate 2525 * iterations: number of iterations 2526 * results: where to store results 2527 */ 2528 static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_size, 2529 int iterations, struct benchmark_results *results) 2530 { 2531 struct hmm_buffer *buffer; 2532 unsigned long npages = buffer_size / sysconf(_SC_PAGESIZE); 2533 double start, end; 2534 double s2d_total = 0, d2s_total = 0; 2535 int ret, i; 2536 int *ptr; 2537 2538 buffer = hmm_buffer_alloc(buffer_size, buffer_size, 2539 PROT_READ | PROT_WRITE, 2540 MAP_PRIVATE | MAP_ANONYMOUS, -1); 2541 if (!buffer) { 2542 ret = -1; 2543 goto cleanup; 2544 } 2545 memset(buffer->mirror, 0xFF, buffer_size); 2546 2547 /* Apply THP hint if requested */ 2548 if (use_thp) 2549 ret = madvise(buffer->ptr, buffer_size, MADV_HUGEPAGE); 2550 else 2551 ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE); 2552 2553 if (ret) 2554 goto cleanup; 2555 2556 /* Initialize memory to make sure pages are allocated */ 2557 ptr = (int *)buffer->ptr; 2558 for (i = 0; i < buffer_size / sizeof(int); i++) 2559 ptr[i] = i & 0xFF; 2560 2561 /* Warmup iteration */ 2562 ret = hmm_migrate_sys_to_dev(fd, buffer, npages); 2563 if (ret) 2564 goto cleanup; 2565 2566 ret = hmm_migrate_dev_to_sys(fd, buffer, npages); 2567 if (ret) 2568 goto cleanup; 2569 2570 /* Benchmark iterations */ 2571 for (i = 0; i < iterations; i++) { 2572 /* System to device migration */ 2573 start = get_time_ms(); 2574 2575 ret = hmm_migrate_sys_to_dev(fd, buffer, npages); 2576 if (ret) 2577 goto cleanup; 2578 2579 end = get_time_ms(); 2580 s2d_total += (end - start); 2581 2582 /* Device to system migration */ 2583 start = get_time_ms(); 2584 2585 ret = hmm_migrate_dev_to_sys(fd, buffer, npages); 2586 if (ret) 2587 goto cleanup; 2588 2589 end = get_time_ms(); 2590 d2s_total += (end - start); 2591 } 2592 2593 /* Calculate average times and throughput */ 2594 results->sys_to_dev_time = s2d_total / iterations; 2595 results->dev_to_sys_time = d2s_total / iterations; 2596 results->throughput_s2d = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / 2597 (results->sys_to_dev_time / 1000.0); 2598 results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / 2599 (results->dev_to_sys_time / 1000.0); 2600 2601 cleanup: 2602 hmm_buffer_free(buffer); 2603 return ret; 2604 } 2605 2606 /* 2607 * Benchmark THP migration with different buffer sizes 2608 */ 2609 TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120) 2610 { 2611 struct benchmark_results thp_results, regular_results; 2612 size_t thp_size = read_pmd_pagesize(); 2613 int iterations = 5; 2614 2615 if (!thp_size) 2616 thp_size = TWOMEG; 2617 2618 printf("\nHMM THP Migration Benchmark\n"); 2619 printf("---------------------------\n"); 2620 printf("System page size: %ld bytes\n", sysconf(_SC_PAGESIZE)); 2621 2622 /* Test different buffer sizes */ 2623 size_t test_sizes[] = { 2624 thp_size / 4, /* quarter THP */ 2625 thp_size / 2, /* half THP */ 2626 thp_size, /* single THP */ 2627 thp_size * 2, /* two THPs */ 2628 thp_size * 4, /* four THPs */ 2629 thp_size * 8, /* eight THPs */ 2630 thp_size * 128, /* one twenty eight THPs */ 2631 }; 2632 2633 static const char *const test_names[] = { 2634 "Small Buffer", 2635 "Half THP Size", 2636 "Single THP Size", 2637 "Two THP Size", 2638 "Four THP Size", 2639 "Eight THP Size", 2640 "One twenty eight THP Size" 2641 }; 2642 2643 int num_tests = ARRAY_SIZE(test_sizes); 2644 2645 /* Run all tests */ 2646 for (int i = 0; i < num_tests; i++) { 2647 /* Skip test sizes exceeding INT_MAX to avoid overflow */ 2648 if (test_sizes[i] > INT_MAX) 2649 break; 2650 2651 /* Test with THP */ 2652 ASSERT_EQ(run_migration_benchmark(self->fd, 1, test_sizes[i], 2653 iterations, &thp_results), 0); 2654 2655 /* Test without THP */ 2656 ASSERT_EQ(run_migration_benchmark(self->fd, 0, test_sizes[i], 2657 iterations, ®ular_results), 0); 2658 2659 /* Print results */ 2660 print_benchmark_results(test_names[i], test_sizes[i], 2661 &thp_results, ®ular_results); 2662 } 2663 } 2664 /* 2665 * Test that HMM can fault in pages backed by userfaultfd using the 2666 * hmm_range_fault_unlocked_timeout() path with no timeout. This exercises 2667 * the lock-drop retry logic in the HMM framework. 2668 */ 2669 struct uffd_thread_args { 2670 int uffd; 2671 int stop_fd; 2672 void *page_buffer; 2673 unsigned long page_size; 2674 }; 2675 2676 static void *uffd_handler_thread(void *arg) 2677 { 2678 struct uffd_thread_args *args = arg; 2679 struct uffd_msg msg; 2680 struct uffdio_copy copy; 2681 struct pollfd pollfd[2]; 2682 int ret; 2683 2684 pollfd[0].fd = args->uffd; 2685 pollfd[0].events = POLLIN; 2686 pollfd[1].fd = args->stop_fd; 2687 pollfd[1].events = POLLIN; 2688 2689 while (1) { 2690 ret = poll(pollfd, 2, -1); 2691 if (ret <= 0) 2692 break; 2693 if (pollfd[1].revents) 2694 break; 2695 if (!(pollfd[0].revents & POLLIN)) 2696 break; 2697 2698 ret = read(args->uffd, &msg, sizeof(msg)); 2699 if (ret != sizeof(msg)) 2700 break; 2701 2702 if (msg.event != UFFD_EVENT_PAGEFAULT) 2703 break; 2704 2705 /* Fill the page with a known pattern */ 2706 memset(args->page_buffer, 0xAB, args->page_size); 2707 2708 copy.dst = msg.arg.pagefault.address & ~(args->page_size - 1); 2709 copy.src = (unsigned long)args->page_buffer; 2710 copy.len = args->page_size; 2711 copy.mode = 0; 2712 copy.copy = 0; 2713 2714 ret = ioctl(args->uffd, UFFDIO_COPY, ©); 2715 if (ret < 0) 2716 break; 2717 } 2718 2719 return NULL; 2720 } 2721 2722 TEST_F(hmm, userfaultfd_read) 2723 { 2724 struct hmm_buffer *buffer; 2725 struct uffd_thread_args uffd_args; 2726 unsigned long npages; 2727 unsigned long size; 2728 unsigned long i; 2729 unsigned char *ptr; 2730 pthread_t thread; 2731 int uffd; 2732 int stop_fd; 2733 int ret; 2734 struct uffdio_api api; 2735 struct uffdio_register reg; 2736 uint64_t stop = 1; 2737 ssize_t nwrite; 2738 2739 npages = 4; 2740 size = npages << self->page_shift; 2741 2742 /* Create userfaultfd */ 2743 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); 2744 if (uffd < 0) 2745 SKIP(return, "userfaultfd not available"); 2746 2747 api.api = UFFD_API; 2748 api.features = 0; 2749 ret = ioctl(uffd, UFFDIO_API, &api); 2750 ASSERT_EQ(ret, 0); 2751 2752 buffer = malloc(sizeof(*buffer)); 2753 ASSERT_NE(buffer, NULL); 2754 2755 buffer->fd = -1; 2756 buffer->size = size; 2757 buffer->mirror = malloc(size); 2758 ASSERT_NE(buffer->mirror, NULL); 2759 2760 /* Create anonymous mapping */ 2761 buffer->ptr = mmap(NULL, size, 2762 PROT_READ | PROT_WRITE, 2763 MAP_PRIVATE | MAP_ANONYMOUS, 2764 -1, 0); 2765 ASSERT_NE(buffer->ptr, MAP_FAILED); 2766 2767 /* Register the region with userfaultfd */ 2768 reg.range.start = (unsigned long)buffer->ptr; 2769 reg.range.len = size; 2770 reg.mode = UFFDIO_REGISTER_MODE_MISSING; 2771 ret = ioctl(uffd, UFFDIO_REGISTER, ®); 2772 ASSERT_EQ(ret, 0); 2773 2774 /* Set up the handler thread */ 2775 uffd_args.uffd = uffd; 2776 stop_fd = eventfd(0, EFD_CLOEXEC); 2777 ASSERT_GE(stop_fd, 0); 2778 uffd_args.stop_fd = stop_fd; 2779 uffd_args.page_buffer = malloc(self->page_size); 2780 ASSERT_NE(uffd_args.page_buffer, NULL); 2781 uffd_args.page_size = self->page_size; 2782 2783 ret = pthread_create(&thread, NULL, uffd_handler_thread, &uffd_args); 2784 ASSERT_EQ(ret, 0); 2785 2786 /* 2787 * Use the unlocked read path which allows the mmap lock to be 2788 * dropped during the fault, enabling userfaultfd resolution. 2789 */ 2790 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ_UNLOCKED, 2791 buffer, npages); 2792 ASSERT_EQ(ret, 0); 2793 ASSERT_EQ(buffer->cpages, npages); 2794 2795 /* Verify the device read the data filled by the uffd handler */ 2796 ptr = buffer->mirror; 2797 for (i = 0; i < size; ++i) 2798 ASSERT_EQ(ptr[i], (unsigned char)0xAB); 2799 2800 nwrite = write(stop_fd, &stop, sizeof(stop)); 2801 ASSERT_EQ(nwrite, sizeof(stop)); 2802 pthread_join(thread, NULL); 2803 close(stop_fd); 2804 free(uffd_args.page_buffer); 2805 close(uffd); 2806 hmm_buffer_free(buffer); 2807 } 2808 2809 2810 TEST_HARNESS_MAIN 2811