1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * HMM stands for Heterogeneous Memory Management, it is a helper layer inside 4 * the linux kernel to help device drivers mirror a process address space in 5 * the device. This allows the device to use the same address space which 6 * makes communication and data exchange a lot easier. 7 * 8 * This framework's sole purpose is to exercise various code paths inside 9 * the kernel to make sure that HMM performs as expected and to flush out any 10 * bugs. 11 */ 12 13 #include "kselftest_harness.h" 14 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <stdio.h> 18 #include <stdlib.h> 19 #include <stdint.h> 20 #include <unistd.h> 21 #include <strings.h> 22 #include <time.h> 23 #include <pthread.h> 24 #include <sys/types.h> 25 #include <sys/stat.h> 26 #include <sys/mman.h> 27 #include <sys/ioctl.h> 28 #include <sys/time.h> 29 30 31 /* 32 * This is a private UAPI to the kernel test module so it isn't exported 33 * in the usual include/uapi/... directory. 34 */ 35 #include <lib/test_hmm_uapi.h> 36 #include <mm/gup_test.h> 37 #include <mm/vm_util.h> 38 39 struct hmm_buffer { 40 void *ptr; 41 void *mirror; 42 unsigned long size; 43 int fd; 44 uint64_t cpages; 45 uint64_t faults; 46 }; 47 48 enum { 49 HMM_PRIVATE_DEVICE_ONE, 50 HMM_PRIVATE_DEVICE_TWO, 51 HMM_COHERENCE_DEVICE_ONE, 52 HMM_COHERENCE_DEVICE_TWO, 53 }; 54 55 #define ONEKB (1 << 10) 56 #define ONEMEG (1 << 20) 57 #define TWOMEG (1 << 21) 58 #define HMM_BUFFER_SIZE (1024 << 12) 59 #define HMM_PATH_MAX 64 60 #define NTIMES 10 61 62 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) 63 /* Just the flags we need, copied from mm.h: */ 64 65 #ifndef FOLL_WRITE 66 #define FOLL_WRITE 0x01 /* check pte is writable */ 67 #endif 68 69 #ifndef FOLL_LONGTERM 70 #define FOLL_LONGTERM 0x100 /* mapping lifetime is indefinite */ 71 #endif 72 FIXTURE(hmm) 73 { 74 int fd; 75 unsigned int page_size; 76 unsigned int page_shift; 77 }; 78 79 FIXTURE_VARIANT(hmm) 80 { 81 int device_number; 82 }; 83 84 FIXTURE_VARIANT_ADD(hmm, hmm_device_private) 85 { 86 .device_number = HMM_PRIVATE_DEVICE_ONE, 87 }; 88 89 FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent) 90 { 91 .device_number = HMM_COHERENCE_DEVICE_ONE, 92 }; 93 94 FIXTURE(hmm2) 95 { 96 int fd0; 97 int fd1; 98 unsigned int page_size; 99 unsigned int page_shift; 100 }; 101 102 FIXTURE_VARIANT(hmm2) 103 { 104 int device_number0; 105 int device_number1; 106 }; 107 108 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private) 109 { 110 .device_number0 = HMM_PRIVATE_DEVICE_ONE, 111 .device_number1 = HMM_PRIVATE_DEVICE_TWO, 112 }; 113 114 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent) 115 { 116 .device_number0 = HMM_COHERENCE_DEVICE_ONE, 117 .device_number1 = HMM_COHERENCE_DEVICE_TWO, 118 }; 119 120 static int hmm_open(int unit) 121 { 122 char pathname[HMM_PATH_MAX]; 123 int fd; 124 125 snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit); 126 fd = open(pathname, O_RDWR, 0); 127 if (fd < 0) 128 fprintf(stderr, "could not open hmm dmirror driver (%s)\n", 129 pathname); 130 return fd; 131 } 132 133 static bool hmm_is_coherent_type(int dev_num) 134 { 135 return (dev_num >= HMM_COHERENCE_DEVICE_ONE); 136 } 137 138 FIXTURE_SETUP(hmm) 139 { 140 self->page_size = sysconf(_SC_PAGE_SIZE); 141 self->page_shift = ffs(self->page_size) - 1; 142 143 self->fd = hmm_open(variant->device_number); 144 if (self->fd < 0 && hmm_is_coherent_type(variant->device_number)) 145 SKIP(return, "DEVICE_COHERENT not available"); 146 ASSERT_GE(self->fd, 0); 147 } 148 149 FIXTURE_SETUP(hmm2) 150 { 151 self->page_size = sysconf(_SC_PAGE_SIZE); 152 self->page_shift = ffs(self->page_size) - 1; 153 154 self->fd0 = hmm_open(variant->device_number0); 155 if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0)) 156 SKIP(return, "DEVICE_COHERENT not available"); 157 ASSERT_GE(self->fd0, 0); 158 self->fd1 = hmm_open(variant->device_number1); 159 ASSERT_GE(self->fd1, 0); 160 } 161 162 FIXTURE_TEARDOWN(hmm) 163 { 164 int ret = close(self->fd); 165 166 ASSERT_EQ(ret, 0); 167 self->fd = -1; 168 } 169 170 FIXTURE_TEARDOWN(hmm2) 171 { 172 int ret = close(self->fd0); 173 174 ASSERT_EQ(ret, 0); 175 self->fd0 = -1; 176 177 ret = close(self->fd1); 178 ASSERT_EQ(ret, 0); 179 self->fd1 = -1; 180 } 181 182 static int hmm_dmirror_cmd(int fd, 183 unsigned long request, 184 struct hmm_buffer *buffer, 185 unsigned long npages) 186 { 187 struct hmm_dmirror_cmd cmd; 188 int ret; 189 190 /* Simulate a device reading system memory. */ 191 cmd.addr = (__u64)buffer->ptr; 192 cmd.ptr = (__u64)buffer->mirror; 193 cmd.npages = npages; 194 195 for (;;) { 196 ret = ioctl(fd, request, &cmd); 197 if (ret == 0) 198 break; 199 if (errno == EINTR) 200 continue; 201 return -errno; 202 } 203 buffer->cpages = cmd.cpages; 204 buffer->faults = cmd.faults; 205 206 return 0; 207 } 208 209 static void hmm_buffer_free(struct hmm_buffer *buffer) 210 { 211 if (buffer == NULL) 212 return; 213 214 if (buffer->ptr) { 215 munmap(buffer->ptr, buffer->size); 216 buffer->ptr = NULL; 217 } 218 free(buffer->mirror); 219 free(buffer); 220 } 221 222 /* 223 * Create a temporary file that will be deleted on close. 224 */ 225 static int hmm_create_file(unsigned long size) 226 { 227 char path[HMM_PATH_MAX]; 228 int fd; 229 230 strcpy(path, "/tmp"); 231 fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600); 232 if (fd >= 0) { 233 int r; 234 235 do { 236 r = ftruncate(fd, size); 237 } while (r == -1 && errno == EINTR); 238 if (!r) 239 return fd; 240 close(fd); 241 } 242 return -1; 243 } 244 245 /* 246 * Return a random unsigned number. 247 */ 248 static unsigned int hmm_random(void) 249 { 250 static int fd = -1; 251 unsigned int r; 252 253 if (fd < 0) { 254 fd = open("/dev/urandom", O_RDONLY); 255 if (fd < 0) { 256 fprintf(stderr, "%s:%d failed to open /dev/urandom\n", 257 __FILE__, __LINE__); 258 return ~0U; 259 } 260 } 261 read(fd, &r, sizeof(r)); 262 return r; 263 } 264 265 static void hmm_nanosleep(unsigned int n) 266 { 267 struct timespec t; 268 269 t.tv_sec = 0; 270 t.tv_nsec = n; 271 nanosleep(&t, NULL); 272 } 273 274 static int hmm_migrate_sys_to_dev(int fd, 275 struct hmm_buffer *buffer, 276 unsigned long npages) 277 { 278 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages); 279 } 280 281 static int hmm_migrate_dev_to_sys(int fd, 282 struct hmm_buffer *buffer, 283 unsigned long npages) 284 { 285 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages); 286 } 287 288 /* 289 * Simple NULL test of device open/close. 290 */ 291 TEST_F(hmm, open_close) 292 { 293 } 294 295 /* 296 * Read private anonymous memory. 297 */ 298 TEST_F(hmm, anon_read) 299 { 300 struct hmm_buffer *buffer; 301 unsigned long npages; 302 unsigned long size; 303 unsigned long i; 304 int *ptr; 305 int ret; 306 int val; 307 308 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 309 ASSERT_NE(npages, 0); 310 size = npages << self->page_shift; 311 312 buffer = malloc(sizeof(*buffer)); 313 ASSERT_NE(buffer, NULL); 314 315 buffer->fd = -1; 316 buffer->size = size; 317 buffer->mirror = malloc(size); 318 ASSERT_NE(buffer->mirror, NULL); 319 320 buffer->ptr = mmap(NULL, size, 321 PROT_READ | PROT_WRITE, 322 MAP_PRIVATE | MAP_ANONYMOUS, 323 buffer->fd, 0); 324 ASSERT_NE(buffer->ptr, MAP_FAILED); 325 326 /* 327 * Initialize buffer in system memory but leave the first two pages 328 * zero (pte_none and pfn_zero). 329 */ 330 i = 2 * self->page_size / sizeof(*ptr); 331 for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 332 ptr[i] = i; 333 334 /* Set buffer permission to read-only. */ 335 ret = mprotect(buffer->ptr, size, PROT_READ); 336 ASSERT_EQ(ret, 0); 337 338 /* Populate the CPU page table with a special zero page. */ 339 val = *(int *)(buffer->ptr + self->page_size); 340 ASSERT_EQ(val, 0); 341 342 /* Simulate a device reading system memory. */ 343 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 344 ASSERT_EQ(ret, 0); 345 ASSERT_EQ(buffer->cpages, npages); 346 ASSERT_EQ(buffer->faults, 1); 347 348 /* Check what the device read. */ 349 ptr = buffer->mirror; 350 for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i) 351 ASSERT_EQ(ptr[i], 0); 352 for (; i < size / sizeof(*ptr); ++i) 353 ASSERT_EQ(ptr[i], i); 354 355 hmm_buffer_free(buffer); 356 } 357 358 /* 359 * Read private anonymous memory which has been protected with 360 * mprotect() PROT_NONE. 361 */ 362 TEST_F(hmm, anon_read_prot) 363 { 364 struct hmm_buffer *buffer; 365 unsigned long npages; 366 unsigned long size; 367 unsigned long i; 368 int *ptr; 369 int ret; 370 371 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 372 ASSERT_NE(npages, 0); 373 size = npages << self->page_shift; 374 375 buffer = malloc(sizeof(*buffer)); 376 ASSERT_NE(buffer, NULL); 377 378 buffer->fd = -1; 379 buffer->size = size; 380 buffer->mirror = malloc(size); 381 ASSERT_NE(buffer->mirror, NULL); 382 383 buffer->ptr = mmap(NULL, size, 384 PROT_READ | PROT_WRITE, 385 MAP_PRIVATE | MAP_ANONYMOUS, 386 buffer->fd, 0); 387 ASSERT_NE(buffer->ptr, MAP_FAILED); 388 389 /* Initialize buffer in system memory. */ 390 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 391 ptr[i] = i; 392 393 /* Initialize mirror buffer so we can verify it isn't written. */ 394 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 395 ptr[i] = -i; 396 397 /* Protect buffer from reading. */ 398 ret = mprotect(buffer->ptr, size, PROT_NONE); 399 ASSERT_EQ(ret, 0); 400 401 /* Simulate a device reading system memory. */ 402 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 403 ASSERT_EQ(ret, -EFAULT); 404 405 /* Allow CPU to read the buffer so we can check it. */ 406 ret = mprotect(buffer->ptr, size, PROT_READ); 407 ASSERT_EQ(ret, 0); 408 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 409 ASSERT_EQ(ptr[i], i); 410 411 /* Check what the device read. */ 412 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 413 ASSERT_EQ(ptr[i], -i); 414 415 hmm_buffer_free(buffer); 416 } 417 418 /* 419 * Write private anonymous memory. 420 */ 421 TEST_F(hmm, anon_write) 422 { 423 struct hmm_buffer *buffer; 424 unsigned long npages; 425 unsigned long size; 426 unsigned long i; 427 int *ptr; 428 int ret; 429 430 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 431 ASSERT_NE(npages, 0); 432 size = npages << self->page_shift; 433 434 buffer = malloc(sizeof(*buffer)); 435 ASSERT_NE(buffer, NULL); 436 437 buffer->fd = -1; 438 buffer->size = size; 439 buffer->mirror = malloc(size); 440 ASSERT_NE(buffer->mirror, NULL); 441 442 buffer->ptr = mmap(NULL, size, 443 PROT_READ | PROT_WRITE, 444 MAP_PRIVATE | MAP_ANONYMOUS, 445 buffer->fd, 0); 446 ASSERT_NE(buffer->ptr, MAP_FAILED); 447 448 /* Initialize data that the device will write to buffer->ptr. */ 449 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 450 ptr[i] = i; 451 452 /* Simulate a device writing system memory. */ 453 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 454 ASSERT_EQ(ret, 0); 455 ASSERT_EQ(buffer->cpages, npages); 456 ASSERT_EQ(buffer->faults, 1); 457 458 /* Check what the device wrote. */ 459 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 460 ASSERT_EQ(ptr[i], i); 461 462 hmm_buffer_free(buffer); 463 } 464 465 /* 466 * Write private anonymous memory which has been protected with 467 * mprotect() PROT_READ. 468 */ 469 TEST_F(hmm, anon_write_prot) 470 { 471 struct hmm_buffer *buffer; 472 unsigned long npages; 473 unsigned long size; 474 unsigned long i; 475 int *ptr; 476 int ret; 477 478 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 479 ASSERT_NE(npages, 0); 480 size = npages << self->page_shift; 481 482 buffer = malloc(sizeof(*buffer)); 483 ASSERT_NE(buffer, NULL); 484 485 buffer->fd = -1; 486 buffer->size = size; 487 buffer->mirror = malloc(size); 488 ASSERT_NE(buffer->mirror, NULL); 489 490 buffer->ptr = mmap(NULL, size, 491 PROT_READ, 492 MAP_PRIVATE | MAP_ANONYMOUS, 493 buffer->fd, 0); 494 ASSERT_NE(buffer->ptr, MAP_FAILED); 495 496 /* Simulate a device reading a zero page of memory. */ 497 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); 498 ASSERT_EQ(ret, 0); 499 ASSERT_EQ(buffer->cpages, 1); 500 ASSERT_EQ(buffer->faults, 1); 501 502 /* Initialize data that the device will write to buffer->ptr. */ 503 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 504 ptr[i] = i; 505 506 /* Simulate a device writing system memory. */ 507 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 508 ASSERT_EQ(ret, -EPERM); 509 510 /* Check what the device wrote. */ 511 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 512 ASSERT_EQ(ptr[i], 0); 513 514 /* Now allow writing and see that the zero page is replaced. */ 515 ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ); 516 ASSERT_EQ(ret, 0); 517 518 /* Simulate a device writing system memory. */ 519 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 520 ASSERT_EQ(ret, 0); 521 ASSERT_EQ(buffer->cpages, npages); 522 ASSERT_EQ(buffer->faults, 1); 523 524 /* Check what the device wrote. */ 525 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 526 ASSERT_EQ(ptr[i], i); 527 528 hmm_buffer_free(buffer); 529 } 530 531 /* 532 * Check that a device writing an anonymous private mapping 533 * will copy-on-write if a child process inherits the mapping. 534 * 535 * Also verifies after fork() memory the device can be read by child. 536 */ 537 TEST_F(hmm, anon_write_child) 538 { 539 struct hmm_buffer *buffer; 540 unsigned long npages; 541 unsigned long size; 542 unsigned long i; 543 void *old_ptr; 544 void *map; 545 int *ptr; 546 pid_t pid; 547 int child_fd; 548 int ret, use_thp, migrate; 549 550 for (migrate = 0; migrate < 2; ++migrate) { 551 for (use_thp = 0; use_thp < 2; ++use_thp) { 552 npages = ALIGN(use_thp ? read_pmd_pagesize() : HMM_BUFFER_SIZE, 553 self->page_size) >> self->page_shift; 554 ASSERT_NE(npages, 0); 555 size = npages << self->page_shift; 556 557 buffer = malloc(sizeof(*buffer)); 558 ASSERT_NE(buffer, NULL); 559 560 buffer->fd = -1; 561 buffer->size = size * 2; 562 buffer->mirror = malloc(size); 563 ASSERT_NE(buffer->mirror, NULL); 564 565 buffer->ptr = mmap(NULL, size * 2, 566 PROT_READ | PROT_WRITE, 567 MAP_PRIVATE | MAP_ANONYMOUS, 568 buffer->fd, 0); 569 ASSERT_NE(buffer->ptr, MAP_FAILED); 570 571 old_ptr = buffer->ptr; 572 if (use_thp) { 573 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 574 ret = madvise(map, size, MADV_HUGEPAGE); 575 ASSERT_EQ(ret, 0); 576 buffer->ptr = map; 577 } 578 579 /* Initialize buffer->ptr so we can tell if it is written. */ 580 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 581 ptr[i] = i; 582 583 /* Initialize data that the device will write to buffer->ptr. */ 584 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 585 ptr[i] = -i; 586 587 if (migrate) { 588 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 589 ASSERT_EQ(ret, 0); 590 ASSERT_EQ(buffer->cpages, npages); 591 592 } 593 594 pid = fork(); 595 if (pid == -1) 596 ASSERT_EQ(pid, 0); 597 if (pid != 0) { 598 waitpid(pid, &ret, 0); 599 ASSERT_EQ(WIFEXITED(ret), 1); 600 601 /* Check that the parent's buffer did not change. */ 602 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 603 ASSERT_EQ(ptr[i], i); 604 605 buffer->ptr = old_ptr; 606 hmm_buffer_free(buffer); 607 continue; 608 } 609 610 /* Check that we see the parent's values. */ 611 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 612 ASSERT_EQ(ptr[i], i); 613 if (!migrate) { 614 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 615 ASSERT_EQ(ptr[i], -i); 616 } 617 618 /* The child process needs its own mirror to its own mm. */ 619 child_fd = hmm_open(0); 620 ASSERT_GE(child_fd, 0); 621 622 /* Simulate a device writing system memory. */ 623 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); 624 ASSERT_EQ(ret, 0); 625 ASSERT_EQ(buffer->cpages, npages); 626 ASSERT_EQ(buffer->faults, 1); 627 628 /* Check what the device wrote. */ 629 if (!migrate) { 630 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 631 ASSERT_EQ(ptr[i], -i); 632 } 633 634 close(child_fd); 635 exit(0); 636 } 637 } 638 } 639 640 /* 641 * Check that a device writing an anonymous shared mapping 642 * will not copy-on-write if a child process inherits the mapping. 643 */ 644 TEST_F(hmm, anon_write_child_shared) 645 { 646 struct hmm_buffer *buffer; 647 unsigned long npages; 648 unsigned long size; 649 unsigned long i; 650 int *ptr; 651 pid_t pid; 652 int child_fd; 653 int ret; 654 655 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 656 ASSERT_NE(npages, 0); 657 size = npages << self->page_shift; 658 659 buffer = malloc(sizeof(*buffer)); 660 ASSERT_NE(buffer, NULL); 661 662 buffer->fd = -1; 663 buffer->size = size; 664 buffer->mirror = malloc(size); 665 ASSERT_NE(buffer->mirror, NULL); 666 667 buffer->ptr = mmap(NULL, size, 668 PROT_READ | PROT_WRITE, 669 MAP_SHARED | MAP_ANONYMOUS, 670 buffer->fd, 0); 671 ASSERT_NE(buffer->ptr, MAP_FAILED); 672 673 /* Initialize buffer->ptr so we can tell if it is written. */ 674 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 675 ptr[i] = i; 676 677 /* Initialize data that the device will write to buffer->ptr. */ 678 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 679 ptr[i] = -i; 680 681 pid = fork(); 682 if (pid == -1) 683 ASSERT_EQ(pid, 0); 684 if (pid != 0) { 685 waitpid(pid, &ret, 0); 686 ASSERT_EQ(WIFEXITED(ret), 1); 687 688 /* Check that the parent's buffer did change. */ 689 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 690 ASSERT_EQ(ptr[i], -i); 691 return; 692 } 693 694 /* Check that we see the parent's values. */ 695 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 696 ASSERT_EQ(ptr[i], i); 697 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 698 ASSERT_EQ(ptr[i], -i); 699 700 /* The child process needs its own mirror to its own mm. */ 701 child_fd = hmm_open(0); 702 ASSERT_GE(child_fd, 0); 703 704 /* Simulate a device writing system memory. */ 705 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); 706 ASSERT_EQ(ret, 0); 707 ASSERT_EQ(buffer->cpages, npages); 708 ASSERT_EQ(buffer->faults, 1); 709 710 /* Check what the device wrote. */ 711 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 712 ASSERT_EQ(ptr[i], -i); 713 714 close(child_fd); 715 exit(0); 716 } 717 718 /* 719 * Write private anonymous huge page. 720 */ 721 TEST_F(hmm, anon_write_huge) 722 { 723 struct hmm_buffer *buffer; 724 unsigned long npages; 725 unsigned long size; 726 unsigned long i; 727 void *old_ptr; 728 void *map; 729 int *ptr; 730 int ret; 731 732 size = 2 * read_pmd_pagesize(); 733 734 buffer = malloc(sizeof(*buffer)); 735 ASSERT_NE(buffer, NULL); 736 737 buffer->fd = -1; 738 buffer->size = size; 739 buffer->mirror = malloc(size); 740 ASSERT_NE(buffer->mirror, NULL); 741 742 buffer->ptr = mmap(NULL, size, 743 PROT_READ | PROT_WRITE, 744 MAP_PRIVATE | MAP_ANONYMOUS, 745 buffer->fd, 0); 746 ASSERT_NE(buffer->ptr, MAP_FAILED); 747 748 size /= 2; 749 npages = size >> self->page_shift; 750 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 751 ret = madvise(map, size, MADV_HUGEPAGE); 752 ASSERT_EQ(ret, 0); 753 old_ptr = buffer->ptr; 754 buffer->ptr = map; 755 756 /* Initialize data that the device will write to buffer->ptr. */ 757 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 758 ptr[i] = i; 759 760 /* Simulate a device writing system memory. */ 761 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 762 ASSERT_EQ(ret, 0); 763 ASSERT_EQ(buffer->cpages, npages); 764 ASSERT_EQ(buffer->faults, 1); 765 766 /* Check what the device wrote. */ 767 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 768 ASSERT_EQ(ptr[i], i); 769 770 buffer->ptr = old_ptr; 771 hmm_buffer_free(buffer); 772 } 773 774 /* 775 * Write huge TLBFS page. 776 */ 777 TEST_F(hmm, anon_write_hugetlbfs) 778 { 779 struct hmm_buffer *buffer; 780 unsigned long npages; 781 unsigned long size; 782 unsigned long default_hsize = default_huge_page_size(); 783 unsigned long i; 784 int *ptr; 785 int ret; 786 787 if (!default_hsize) 788 SKIP(return, "Huge page size could not be determined"); 789 790 size = ALIGN(TWOMEG, default_hsize); 791 npages = size >> self->page_shift; 792 793 buffer = malloc(sizeof(*buffer)); 794 ASSERT_NE(buffer, NULL); 795 796 buffer->ptr = mmap(NULL, size, 797 PROT_READ | PROT_WRITE, 798 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 799 -1, 0); 800 if (buffer->ptr == MAP_FAILED) { 801 free(buffer); 802 SKIP(return, "Huge page could not be allocated"); 803 } 804 805 buffer->fd = -1; 806 buffer->size = size; 807 buffer->mirror = malloc(size); 808 ASSERT_NE(buffer->mirror, NULL); 809 810 /* Initialize data that the device will write to buffer->ptr. */ 811 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 812 ptr[i] = i; 813 814 /* Simulate a device writing system memory. */ 815 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 816 ASSERT_EQ(ret, 0); 817 ASSERT_EQ(buffer->cpages, npages); 818 ASSERT_EQ(buffer->faults, 1); 819 820 /* Check what the device wrote. */ 821 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 822 ASSERT_EQ(ptr[i], i); 823 824 munmap(buffer->ptr, buffer->size); 825 buffer->ptr = NULL; 826 hmm_buffer_free(buffer); 827 } 828 829 /* 830 * Read mmap'ed file memory. 831 */ 832 TEST_F(hmm, file_read) 833 { 834 struct hmm_buffer *buffer; 835 unsigned long npages; 836 unsigned long size; 837 unsigned long i; 838 int *ptr; 839 int ret; 840 int fd; 841 ssize_t len; 842 843 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 844 ASSERT_NE(npages, 0); 845 size = npages << self->page_shift; 846 847 fd = hmm_create_file(size); 848 ASSERT_GE(fd, 0); 849 850 buffer = malloc(sizeof(*buffer)); 851 ASSERT_NE(buffer, NULL); 852 853 buffer->fd = fd; 854 buffer->size = size; 855 buffer->mirror = malloc(size); 856 ASSERT_NE(buffer->mirror, NULL); 857 858 /* Write initial contents of the file. */ 859 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 860 ptr[i] = i; 861 len = pwrite(fd, buffer->mirror, size, 0); 862 ASSERT_EQ(len, size); 863 memset(buffer->mirror, 0, size); 864 865 buffer->ptr = mmap(NULL, size, 866 PROT_READ, 867 MAP_SHARED, 868 buffer->fd, 0); 869 ASSERT_NE(buffer->ptr, MAP_FAILED); 870 871 /* Simulate a device reading system memory. */ 872 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 873 ASSERT_EQ(ret, 0); 874 ASSERT_EQ(buffer->cpages, npages); 875 ASSERT_EQ(buffer->faults, 1); 876 877 /* Check what the device read. */ 878 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 879 ASSERT_EQ(ptr[i], i); 880 881 hmm_buffer_free(buffer); 882 } 883 884 /* 885 * Write mmap'ed file memory. 886 */ 887 TEST_F(hmm, file_write) 888 { 889 struct hmm_buffer *buffer; 890 unsigned long npages; 891 unsigned long size; 892 unsigned long i; 893 int *ptr; 894 int ret; 895 int fd; 896 ssize_t len; 897 898 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 899 ASSERT_NE(npages, 0); 900 size = npages << self->page_shift; 901 902 fd = hmm_create_file(size); 903 ASSERT_GE(fd, 0); 904 905 buffer = malloc(sizeof(*buffer)); 906 ASSERT_NE(buffer, NULL); 907 908 buffer->fd = fd; 909 buffer->size = size; 910 buffer->mirror = malloc(size); 911 ASSERT_NE(buffer->mirror, NULL); 912 913 buffer->ptr = mmap(NULL, size, 914 PROT_READ | PROT_WRITE, 915 MAP_SHARED, 916 buffer->fd, 0); 917 ASSERT_NE(buffer->ptr, MAP_FAILED); 918 919 /* Initialize data that the device will write to buffer->ptr. */ 920 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 921 ptr[i] = i; 922 923 /* Simulate a device writing system memory. */ 924 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 925 ASSERT_EQ(ret, 0); 926 ASSERT_EQ(buffer->cpages, npages); 927 ASSERT_EQ(buffer->faults, 1); 928 929 /* Check what the device wrote. */ 930 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 931 ASSERT_EQ(ptr[i], i); 932 933 /* Check that the device also wrote the file. */ 934 len = pread(fd, buffer->mirror, size, 0); 935 ASSERT_EQ(len, size); 936 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 937 ASSERT_EQ(ptr[i], i); 938 939 hmm_buffer_free(buffer); 940 } 941 942 /* 943 * Migrate anonymous memory to device private memory. 944 */ 945 TEST_F(hmm, migrate) 946 { 947 struct hmm_buffer *buffer; 948 unsigned long npages; 949 unsigned long size; 950 unsigned long i; 951 int *ptr; 952 int ret; 953 954 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 955 ASSERT_NE(npages, 0); 956 size = npages << self->page_shift; 957 958 buffer = malloc(sizeof(*buffer)); 959 ASSERT_NE(buffer, NULL); 960 961 buffer->fd = -1; 962 buffer->size = size; 963 buffer->mirror = malloc(size); 964 ASSERT_NE(buffer->mirror, NULL); 965 966 buffer->ptr = mmap(NULL, size, 967 PROT_READ | PROT_WRITE, 968 MAP_PRIVATE | MAP_ANONYMOUS, 969 buffer->fd, 0); 970 ASSERT_NE(buffer->ptr, MAP_FAILED); 971 972 /* Initialize buffer in system memory. */ 973 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 974 ptr[i] = i; 975 976 /* Migrate memory to device. */ 977 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 978 ASSERT_EQ(ret, 0); 979 ASSERT_EQ(buffer->cpages, npages); 980 981 /* Check what the device read. */ 982 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 983 ASSERT_EQ(ptr[i], i); 984 985 hmm_buffer_free(buffer); 986 } 987 988 /* 989 * Migrate anonymous memory to device private memory and fault some of it back 990 * to system memory, then try migrating the resulting mix of system and device 991 * private memory to the device. 992 */ 993 TEST_F(hmm, migrate_fault) 994 { 995 struct hmm_buffer *buffer; 996 unsigned long npages; 997 unsigned long size; 998 unsigned long i; 999 int *ptr; 1000 int ret; 1001 1002 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1003 ASSERT_NE(npages, 0); 1004 size = npages << self->page_shift; 1005 1006 buffer = malloc(sizeof(*buffer)); 1007 ASSERT_NE(buffer, NULL); 1008 1009 buffer->fd = -1; 1010 buffer->size = size; 1011 buffer->mirror = malloc(size); 1012 ASSERT_NE(buffer->mirror, NULL); 1013 1014 buffer->ptr = mmap(NULL, size, 1015 PROT_READ | PROT_WRITE, 1016 MAP_PRIVATE | MAP_ANONYMOUS, 1017 buffer->fd, 0); 1018 ASSERT_NE(buffer->ptr, MAP_FAILED); 1019 1020 /* Initialize buffer in system memory. */ 1021 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1022 ptr[i] = i; 1023 1024 /* Migrate memory to device. */ 1025 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1026 ASSERT_EQ(ret, 0); 1027 ASSERT_EQ(buffer->cpages, npages); 1028 1029 /* Check what the device read. */ 1030 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1031 ASSERT_EQ(ptr[i], i); 1032 1033 /* Fault half the pages back to system memory and check them. */ 1034 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) 1035 ASSERT_EQ(ptr[i], i); 1036 1037 /* Migrate memory to the device again. */ 1038 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1039 ASSERT_EQ(ret, 0); 1040 ASSERT_EQ(buffer->cpages, npages); 1041 1042 /* Check what the device read. */ 1043 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1044 ASSERT_EQ(ptr[i], i); 1045 1046 hmm_buffer_free(buffer); 1047 } 1048 1049 TEST_F(hmm, migrate_release) 1050 { 1051 struct hmm_buffer *buffer; 1052 unsigned long npages; 1053 unsigned long size; 1054 unsigned long i; 1055 int *ptr; 1056 int ret; 1057 1058 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1059 ASSERT_NE(npages, 0); 1060 size = npages << self->page_shift; 1061 1062 buffer = malloc(sizeof(*buffer)); 1063 ASSERT_NE(buffer, NULL); 1064 1065 buffer->fd = -1; 1066 buffer->size = size; 1067 buffer->mirror = malloc(size); 1068 ASSERT_NE(buffer->mirror, NULL); 1069 1070 buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 1071 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 1072 ASSERT_NE(buffer->ptr, MAP_FAILED); 1073 1074 /* Initialize buffer in system memory. */ 1075 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1076 ptr[i] = i; 1077 1078 /* Migrate memory to device. */ 1079 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1080 ASSERT_EQ(ret, 0); 1081 ASSERT_EQ(buffer->cpages, npages); 1082 1083 /* Check what the device read. */ 1084 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1085 ASSERT_EQ(ptr[i], i); 1086 1087 /* Release device memory. */ 1088 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages); 1089 ASSERT_EQ(ret, 0); 1090 1091 /* Fault pages back to system memory and check them. */ 1092 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) 1093 ASSERT_EQ(ptr[i], i); 1094 1095 hmm_buffer_free(buffer); 1096 } 1097 1098 /* 1099 * Migrate anonymous shared memory to device private memory. 1100 */ 1101 TEST_F(hmm, migrate_shared) 1102 { 1103 struct hmm_buffer *buffer; 1104 unsigned long npages; 1105 unsigned long size; 1106 int ret; 1107 1108 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1109 ASSERT_NE(npages, 0); 1110 size = npages << self->page_shift; 1111 1112 buffer = malloc(sizeof(*buffer)); 1113 ASSERT_NE(buffer, NULL); 1114 1115 buffer->fd = -1; 1116 buffer->size = size; 1117 buffer->mirror = malloc(size); 1118 ASSERT_NE(buffer->mirror, NULL); 1119 1120 buffer->ptr = mmap(NULL, size, 1121 PROT_READ | PROT_WRITE, 1122 MAP_SHARED | MAP_ANONYMOUS, 1123 buffer->fd, 0); 1124 ASSERT_NE(buffer->ptr, MAP_FAILED); 1125 1126 /* Migrate memory to device. */ 1127 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1128 ASSERT_EQ(ret, -ENOENT); 1129 1130 hmm_buffer_free(buffer); 1131 } 1132 1133 /* 1134 * Try to migrate various memory types to device private memory. 1135 */ 1136 TEST_F(hmm2, migrate_mixed) 1137 { 1138 struct hmm_buffer *buffer; 1139 unsigned long npages; 1140 unsigned long size; 1141 int *ptr; 1142 unsigned char *p; 1143 int ret; 1144 int val; 1145 1146 npages = 6; 1147 size = npages << self->page_shift; 1148 1149 buffer = malloc(sizeof(*buffer)); 1150 ASSERT_NE(buffer, NULL); 1151 1152 buffer->fd = -1; 1153 buffer->size = size; 1154 buffer->mirror = malloc(size); 1155 ASSERT_NE(buffer->mirror, NULL); 1156 1157 /* Reserve a range of addresses. */ 1158 buffer->ptr = mmap(NULL, size, 1159 PROT_NONE, 1160 MAP_PRIVATE | MAP_ANONYMOUS, 1161 buffer->fd, 0); 1162 ASSERT_NE(buffer->ptr, MAP_FAILED); 1163 p = buffer->ptr; 1164 1165 /* Migrating a protected area should be an error. */ 1166 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); 1167 ASSERT_EQ(ret, -EINVAL); 1168 1169 /* Punch a hole after the first page address. */ 1170 ret = munmap(buffer->ptr + self->page_size, self->page_size); 1171 ASSERT_EQ(ret, 0); 1172 1173 /* We expect an error if the vma doesn't cover the range. */ 1174 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3); 1175 ASSERT_EQ(ret, -EINVAL); 1176 1177 /* Page 2 will be a read-only zero page. */ 1178 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, 1179 PROT_READ); 1180 ASSERT_EQ(ret, 0); 1181 ptr = (int *)(buffer->ptr + 2 * self->page_size); 1182 val = *ptr + 3; 1183 ASSERT_EQ(val, 3); 1184 1185 /* Page 3 will be read-only. */ 1186 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1187 PROT_READ | PROT_WRITE); 1188 ASSERT_EQ(ret, 0); 1189 ptr = (int *)(buffer->ptr + 3 * self->page_size); 1190 *ptr = val; 1191 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1192 PROT_READ); 1193 ASSERT_EQ(ret, 0); 1194 1195 /* Page 4-5 will be read-write. */ 1196 ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size, 1197 PROT_READ | PROT_WRITE); 1198 ASSERT_EQ(ret, 0); 1199 ptr = (int *)(buffer->ptr + 4 * self->page_size); 1200 *ptr = val; 1201 ptr = (int *)(buffer->ptr + 5 * self->page_size); 1202 *ptr = val; 1203 1204 /* Now try to migrate pages 2-5 to device 1. */ 1205 buffer->ptr = p + 2 * self->page_size; 1206 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4); 1207 ASSERT_EQ(ret, 0); 1208 ASSERT_EQ(buffer->cpages, 4); 1209 1210 /* Page 5 won't be migrated to device 0 because it's on device 1. */ 1211 buffer->ptr = p + 5 * self->page_size; 1212 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); 1213 ASSERT_EQ(ret, -ENOENT); 1214 buffer->ptr = p; 1215 1216 buffer->ptr = p; 1217 hmm_buffer_free(buffer); 1218 } 1219 1220 /* 1221 * Migrate anonymous memory to device memory and back to system memory 1222 * multiple times. In case of private zone configuration, this is done 1223 * through fault pages accessed by CPU. In case of coherent zone configuration, 1224 * the pages from the device should be explicitly migrated back to system memory. 1225 * The reason is Coherent device zone has coherent access by CPU, therefore 1226 * it will not generate any page fault. 1227 */ 1228 TEST_F(hmm, migrate_multiple) 1229 { 1230 struct hmm_buffer *buffer; 1231 unsigned long npages; 1232 unsigned long size; 1233 unsigned long i; 1234 unsigned long c; 1235 int *ptr; 1236 int ret; 1237 1238 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1239 ASSERT_NE(npages, 0); 1240 size = npages << self->page_shift; 1241 1242 for (c = 0; c < NTIMES; c++) { 1243 buffer = malloc(sizeof(*buffer)); 1244 ASSERT_NE(buffer, NULL); 1245 1246 buffer->fd = -1; 1247 buffer->size = size; 1248 buffer->mirror = malloc(size); 1249 ASSERT_NE(buffer->mirror, NULL); 1250 1251 buffer->ptr = mmap(NULL, size, 1252 PROT_READ | PROT_WRITE, 1253 MAP_PRIVATE | MAP_ANONYMOUS, 1254 buffer->fd, 0); 1255 ASSERT_NE(buffer->ptr, MAP_FAILED); 1256 1257 /* Initialize buffer in system memory. */ 1258 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1259 ptr[i] = i; 1260 1261 /* Migrate memory to device. */ 1262 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1263 ASSERT_EQ(ret, 0); 1264 ASSERT_EQ(buffer->cpages, npages); 1265 1266 /* Check what the device read. */ 1267 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1268 ASSERT_EQ(ptr[i], i); 1269 1270 /* Migrate back to system memory and check them. */ 1271 if (hmm_is_coherent_type(variant->device_number)) { 1272 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages); 1273 ASSERT_EQ(ret, 0); 1274 ASSERT_EQ(buffer->cpages, npages); 1275 } 1276 1277 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1278 ASSERT_EQ(ptr[i], i); 1279 1280 hmm_buffer_free(buffer); 1281 } 1282 } 1283 1284 /* 1285 * Read anonymous memory multiple times. 1286 */ 1287 TEST_F(hmm, anon_read_multiple) 1288 { 1289 struct hmm_buffer *buffer; 1290 unsigned long npages; 1291 unsigned long size; 1292 unsigned long i; 1293 unsigned long c; 1294 int *ptr; 1295 int ret; 1296 1297 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1298 ASSERT_NE(npages, 0); 1299 size = npages << self->page_shift; 1300 1301 for (c = 0; c < NTIMES; c++) { 1302 buffer = malloc(sizeof(*buffer)); 1303 ASSERT_NE(buffer, NULL); 1304 1305 buffer->fd = -1; 1306 buffer->size = size; 1307 buffer->mirror = malloc(size); 1308 ASSERT_NE(buffer->mirror, NULL); 1309 1310 buffer->ptr = mmap(NULL, size, 1311 PROT_READ | PROT_WRITE, 1312 MAP_PRIVATE | MAP_ANONYMOUS, 1313 buffer->fd, 0); 1314 ASSERT_NE(buffer->ptr, MAP_FAILED); 1315 1316 /* Initialize buffer in system memory. */ 1317 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1318 ptr[i] = i + c; 1319 1320 /* Simulate a device reading system memory. */ 1321 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1322 npages); 1323 ASSERT_EQ(ret, 0); 1324 ASSERT_EQ(buffer->cpages, npages); 1325 ASSERT_EQ(buffer->faults, 1); 1326 1327 /* Check what the device read. */ 1328 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1329 ASSERT_EQ(ptr[i], i + c); 1330 1331 hmm_buffer_free(buffer); 1332 } 1333 } 1334 1335 void *unmap_buffer(void *p) 1336 { 1337 struct hmm_buffer *buffer = p; 1338 1339 /* Delay for a bit and then unmap buffer while it is being read. */ 1340 hmm_nanosleep(hmm_random() % 32000); 1341 munmap(buffer->ptr + buffer->size / 2, buffer->size / 2); 1342 buffer->ptr = NULL; 1343 1344 return NULL; 1345 } 1346 1347 /* 1348 * Try reading anonymous memory while it is being unmapped. 1349 */ 1350 TEST_F(hmm, anon_teardown) 1351 { 1352 unsigned long npages; 1353 unsigned long size; 1354 unsigned long c; 1355 void *ret; 1356 1357 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1358 ASSERT_NE(npages, 0); 1359 size = npages << self->page_shift; 1360 1361 for (c = 0; c < NTIMES; ++c) { 1362 pthread_t thread; 1363 struct hmm_buffer *buffer; 1364 unsigned long i; 1365 int *ptr; 1366 int rc; 1367 1368 buffer = malloc(sizeof(*buffer)); 1369 ASSERT_NE(buffer, NULL); 1370 1371 buffer->fd = -1; 1372 buffer->size = size; 1373 buffer->mirror = malloc(size); 1374 ASSERT_NE(buffer->mirror, NULL); 1375 1376 buffer->ptr = mmap(NULL, size, 1377 PROT_READ | PROT_WRITE, 1378 MAP_PRIVATE | MAP_ANONYMOUS, 1379 buffer->fd, 0); 1380 ASSERT_NE(buffer->ptr, MAP_FAILED); 1381 1382 /* Initialize buffer in system memory. */ 1383 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1384 ptr[i] = i + c; 1385 1386 rc = pthread_create(&thread, NULL, unmap_buffer, buffer); 1387 ASSERT_EQ(rc, 0); 1388 1389 /* Simulate a device reading system memory. */ 1390 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1391 npages); 1392 if (rc == 0) { 1393 ASSERT_EQ(buffer->cpages, npages); 1394 ASSERT_EQ(buffer->faults, 1); 1395 1396 /* Check what the device read. */ 1397 for (i = 0, ptr = buffer->mirror; 1398 i < size / sizeof(*ptr); 1399 ++i) 1400 ASSERT_EQ(ptr[i], i + c); 1401 } 1402 1403 pthread_join(thread, &ret); 1404 hmm_buffer_free(buffer); 1405 } 1406 } 1407 1408 /* 1409 * Test memory snapshot without faulting in pages accessed by the device. 1410 */ 1411 TEST_F(hmm, mixedmap) 1412 { 1413 struct hmm_buffer *buffer; 1414 unsigned long npages; 1415 unsigned long size; 1416 unsigned char *m; 1417 int ret; 1418 1419 npages = 1; 1420 size = npages << self->page_shift; 1421 1422 buffer = malloc(sizeof(*buffer)); 1423 ASSERT_NE(buffer, NULL); 1424 1425 buffer->fd = -1; 1426 buffer->size = size; 1427 buffer->mirror = malloc(npages); 1428 ASSERT_NE(buffer->mirror, NULL); 1429 1430 1431 /* Reserve a range of addresses. */ 1432 buffer->ptr = mmap(NULL, size, 1433 PROT_READ | PROT_WRITE, 1434 MAP_PRIVATE, 1435 self->fd, 0); 1436 ASSERT_NE(buffer->ptr, MAP_FAILED); 1437 1438 /* Simulate a device snapshotting CPU pagetables. */ 1439 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1440 ASSERT_EQ(ret, 0); 1441 ASSERT_EQ(buffer->cpages, npages); 1442 1443 /* Check what the device saw. */ 1444 m = buffer->mirror; 1445 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ); 1446 1447 hmm_buffer_free(buffer); 1448 } 1449 1450 /* 1451 * Test memory snapshot without faulting in pages accessed by the device. 1452 */ 1453 TEST_F(hmm2, snapshot) 1454 { 1455 struct hmm_buffer *buffer; 1456 unsigned long npages; 1457 unsigned long size; 1458 int *ptr; 1459 unsigned char *p; 1460 unsigned char *m; 1461 int ret; 1462 int val; 1463 1464 npages = 7; 1465 size = npages << self->page_shift; 1466 1467 buffer = malloc(sizeof(*buffer)); 1468 ASSERT_NE(buffer, NULL); 1469 1470 buffer->fd = -1; 1471 buffer->size = size; 1472 buffer->mirror = malloc(npages); 1473 ASSERT_NE(buffer->mirror, NULL); 1474 1475 /* Reserve a range of addresses. */ 1476 buffer->ptr = mmap(NULL, size, 1477 PROT_NONE, 1478 MAP_PRIVATE | MAP_ANONYMOUS, 1479 buffer->fd, 0); 1480 ASSERT_NE(buffer->ptr, MAP_FAILED); 1481 p = buffer->ptr; 1482 1483 /* Punch a hole after the first page address. */ 1484 ret = munmap(buffer->ptr + self->page_size, self->page_size); 1485 ASSERT_EQ(ret, 0); 1486 1487 /* Page 2 will be read-only zero page. */ 1488 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, 1489 PROT_READ); 1490 ASSERT_EQ(ret, 0); 1491 ptr = (int *)(buffer->ptr + 2 * self->page_size); 1492 val = *ptr + 3; 1493 ASSERT_EQ(val, 3); 1494 1495 /* Page 3 will be read-only. */ 1496 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1497 PROT_READ | PROT_WRITE); 1498 ASSERT_EQ(ret, 0); 1499 ptr = (int *)(buffer->ptr + 3 * self->page_size); 1500 *ptr = val; 1501 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1502 PROT_READ); 1503 ASSERT_EQ(ret, 0); 1504 1505 /* Page 4-6 will be read-write. */ 1506 ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size, 1507 PROT_READ | PROT_WRITE); 1508 ASSERT_EQ(ret, 0); 1509 ptr = (int *)(buffer->ptr + 4 * self->page_size); 1510 *ptr = val; 1511 1512 /* Page 5 will be migrated to device 0. */ 1513 buffer->ptr = p + 5 * self->page_size; 1514 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); 1515 ASSERT_EQ(ret, 0); 1516 ASSERT_EQ(buffer->cpages, 1); 1517 1518 /* Page 6 will be migrated to device 1. */ 1519 buffer->ptr = p + 6 * self->page_size; 1520 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1); 1521 ASSERT_EQ(ret, 0); 1522 ASSERT_EQ(buffer->cpages, 1); 1523 1524 /* Simulate a device snapshotting CPU pagetables. */ 1525 buffer->ptr = p; 1526 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1527 ASSERT_EQ(ret, 0); 1528 ASSERT_EQ(buffer->cpages, npages); 1529 1530 /* Check what the device saw. */ 1531 m = buffer->mirror; 1532 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR); 1533 ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR); 1534 ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ); 1535 ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ); 1536 ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE); 1537 if (!hmm_is_coherent_type(variant->device_number0)) { 1538 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL | 1539 HMM_DMIRROR_PROT_WRITE); 1540 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE); 1541 } else { 1542 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | 1543 HMM_DMIRROR_PROT_WRITE); 1544 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE | 1545 HMM_DMIRROR_PROT_WRITE); 1546 } 1547 1548 hmm_buffer_free(buffer); 1549 } 1550 1551 /* 1552 * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that 1553 * should be mapped by a large page table entry. 1554 */ 1555 TEST_F(hmm, compound) 1556 { 1557 struct hmm_buffer *buffer; 1558 unsigned long npages; 1559 unsigned long size; 1560 unsigned long default_hsize = default_huge_page_size(); 1561 int *ptr; 1562 unsigned char *m; 1563 int ret; 1564 unsigned long i; 1565 1566 /* Skip test if we can't allocate a hugetlbfs page. */ 1567 1568 if (!default_hsize) 1569 SKIP(return, "Huge page size could not be determined"); 1570 1571 size = ALIGN(TWOMEG, default_hsize); 1572 npages = size >> self->page_shift; 1573 1574 buffer = malloc(sizeof(*buffer)); 1575 ASSERT_NE(buffer, NULL); 1576 1577 buffer->ptr = mmap(NULL, size, 1578 PROT_READ | PROT_WRITE, 1579 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 1580 -1, 0); 1581 if (buffer->ptr == MAP_FAILED) { 1582 free(buffer); 1583 return; 1584 } 1585 1586 buffer->size = size; 1587 buffer->mirror = malloc(npages); 1588 ASSERT_NE(buffer->mirror, NULL); 1589 1590 /* Initialize the pages the device will snapshot in buffer->ptr. */ 1591 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1592 ptr[i] = i; 1593 1594 /* Simulate a device snapshotting CPU pagetables. */ 1595 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1596 ASSERT_EQ(ret, 0); 1597 ASSERT_EQ(buffer->cpages, npages); 1598 1599 /* Check what the device saw. */ 1600 m = buffer->mirror; 1601 for (i = 0; i < npages; ++i) 1602 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE | 1603 HMM_DMIRROR_PROT_PMD); 1604 1605 /* Make the region read-only. */ 1606 ret = mprotect(buffer->ptr, size, PROT_READ); 1607 ASSERT_EQ(ret, 0); 1608 1609 /* Simulate a device snapshotting CPU pagetables. */ 1610 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1611 ASSERT_EQ(ret, 0); 1612 ASSERT_EQ(buffer->cpages, npages); 1613 1614 /* Check what the device saw. */ 1615 m = buffer->mirror; 1616 for (i = 0; i < npages; ++i) 1617 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ | 1618 HMM_DMIRROR_PROT_PMD); 1619 1620 munmap(buffer->ptr, buffer->size); 1621 buffer->ptr = NULL; 1622 hmm_buffer_free(buffer); 1623 } 1624 1625 /* 1626 * Test two devices reading the same memory (double mapped). 1627 */ 1628 TEST_F(hmm2, double_map) 1629 { 1630 struct hmm_buffer *buffer; 1631 unsigned long npages; 1632 unsigned long size; 1633 unsigned long i; 1634 int *ptr; 1635 int ret; 1636 1637 npages = 6; 1638 size = npages << self->page_shift; 1639 1640 buffer = malloc(sizeof(*buffer)); 1641 ASSERT_NE(buffer, NULL); 1642 1643 buffer->fd = -1; 1644 buffer->size = size; 1645 buffer->mirror = malloc(size); 1646 ASSERT_NE(buffer->mirror, NULL); 1647 1648 /* Reserve a range of addresses. */ 1649 buffer->ptr = mmap(NULL, size, 1650 PROT_READ | PROT_WRITE, 1651 MAP_PRIVATE | MAP_ANONYMOUS, 1652 buffer->fd, 0); 1653 ASSERT_NE(buffer->ptr, MAP_FAILED); 1654 1655 /* Initialize buffer in system memory. */ 1656 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1657 ptr[i] = i; 1658 1659 /* Make region read-only. */ 1660 ret = mprotect(buffer->ptr, size, PROT_READ); 1661 ASSERT_EQ(ret, 0); 1662 1663 /* Simulate device 0 reading system memory. */ 1664 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); 1665 ASSERT_EQ(ret, 0); 1666 ASSERT_EQ(buffer->cpages, npages); 1667 ASSERT_EQ(buffer->faults, 1); 1668 1669 /* Check what the device read. */ 1670 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1671 ASSERT_EQ(ptr[i], i); 1672 1673 /* Simulate device 1 reading system memory. */ 1674 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages); 1675 ASSERT_EQ(ret, 0); 1676 ASSERT_EQ(buffer->cpages, npages); 1677 ASSERT_EQ(buffer->faults, 1); 1678 1679 /* Check what the device read. */ 1680 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1681 ASSERT_EQ(ptr[i], i); 1682 1683 /* Migrate pages to device 1 and try to read from device 0. */ 1684 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); 1685 ASSERT_EQ(ret, 0); 1686 ASSERT_EQ(buffer->cpages, npages); 1687 1688 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); 1689 ASSERT_EQ(ret, 0); 1690 ASSERT_EQ(buffer->cpages, npages); 1691 ASSERT_EQ(buffer->faults, 1); 1692 1693 /* Check what device 0 read. */ 1694 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1695 ASSERT_EQ(ptr[i], i); 1696 1697 hmm_buffer_free(buffer); 1698 } 1699 1700 /* 1701 * Basic check of exclusive faulting. 1702 */ 1703 TEST_F(hmm, exclusive) 1704 { 1705 struct hmm_buffer *buffer; 1706 unsigned long npages; 1707 unsigned long size; 1708 unsigned long i; 1709 int *ptr; 1710 int ret; 1711 1712 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1713 ASSERT_NE(npages, 0); 1714 size = npages << self->page_shift; 1715 1716 buffer = malloc(sizeof(*buffer)); 1717 ASSERT_NE(buffer, NULL); 1718 1719 buffer->fd = -1; 1720 buffer->size = size; 1721 buffer->mirror = malloc(size); 1722 ASSERT_NE(buffer->mirror, NULL); 1723 1724 buffer->ptr = mmap(NULL, size, 1725 PROT_READ | PROT_WRITE, 1726 MAP_PRIVATE | MAP_ANONYMOUS, 1727 buffer->fd, 0); 1728 ASSERT_NE(buffer->ptr, MAP_FAILED); 1729 1730 /* Initialize buffer in system memory. */ 1731 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1732 ptr[i] = i; 1733 1734 /* Map memory exclusively for device access. */ 1735 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1736 ASSERT_EQ(ret, 0); 1737 ASSERT_EQ(buffer->cpages, npages); 1738 1739 /* Check what the device read. */ 1740 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1741 ASSERT_EQ(ptr[i], i); 1742 1743 /* Fault pages back to system memory and check them. */ 1744 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1745 ASSERT_EQ(ptr[i]++, i); 1746 1747 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1748 ASSERT_EQ(ptr[i], i+1); 1749 1750 /* Check atomic access revoked */ 1751 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages); 1752 ASSERT_EQ(ret, 0); 1753 1754 hmm_buffer_free(buffer); 1755 } 1756 1757 TEST_F(hmm, exclusive_mprotect) 1758 { 1759 struct hmm_buffer *buffer; 1760 unsigned long npages; 1761 unsigned long size; 1762 unsigned long i; 1763 int *ptr; 1764 int ret; 1765 1766 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1767 ASSERT_NE(npages, 0); 1768 size = npages << self->page_shift; 1769 1770 buffer = malloc(sizeof(*buffer)); 1771 ASSERT_NE(buffer, NULL); 1772 1773 buffer->fd = -1; 1774 buffer->size = size; 1775 buffer->mirror = malloc(size); 1776 ASSERT_NE(buffer->mirror, NULL); 1777 1778 buffer->ptr = mmap(NULL, size, 1779 PROT_READ | PROT_WRITE, 1780 MAP_PRIVATE | MAP_ANONYMOUS, 1781 buffer->fd, 0); 1782 ASSERT_NE(buffer->ptr, MAP_FAILED); 1783 1784 /* Initialize buffer in system memory. */ 1785 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1786 ptr[i] = i; 1787 1788 /* Map memory exclusively for device access. */ 1789 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1790 ASSERT_EQ(ret, 0); 1791 ASSERT_EQ(buffer->cpages, npages); 1792 1793 /* Check what the device read. */ 1794 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1795 ASSERT_EQ(ptr[i], i); 1796 1797 ret = mprotect(buffer->ptr, size, PROT_READ); 1798 ASSERT_EQ(ret, 0); 1799 1800 /* Simulate a device writing system memory. */ 1801 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 1802 ASSERT_EQ(ret, -EPERM); 1803 1804 hmm_buffer_free(buffer); 1805 } 1806 1807 /* 1808 * Check copy-on-write works. 1809 */ 1810 TEST_F(hmm, exclusive_cow) 1811 { 1812 struct hmm_buffer *buffer; 1813 unsigned long npages; 1814 unsigned long size; 1815 unsigned long i; 1816 int *ptr; 1817 int ret; 1818 1819 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1820 ASSERT_NE(npages, 0); 1821 size = npages << self->page_shift; 1822 1823 buffer = malloc(sizeof(*buffer)); 1824 ASSERT_NE(buffer, NULL); 1825 1826 buffer->fd = -1; 1827 buffer->size = size; 1828 buffer->mirror = malloc(size); 1829 ASSERT_NE(buffer->mirror, NULL); 1830 1831 buffer->ptr = mmap(NULL, size, 1832 PROT_READ | PROT_WRITE, 1833 MAP_PRIVATE | MAP_ANONYMOUS, 1834 buffer->fd, 0); 1835 ASSERT_NE(buffer->ptr, MAP_FAILED); 1836 1837 /* Initialize buffer in system memory. */ 1838 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1839 ptr[i] = i; 1840 1841 /* Map memory exclusively for device access. */ 1842 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1843 ASSERT_EQ(ret, 0); 1844 ASSERT_EQ(buffer->cpages, npages); 1845 1846 fork(); 1847 1848 /* Fault pages back to system memory and check them. */ 1849 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1850 ASSERT_EQ(ptr[i]++, i); 1851 1852 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1853 ASSERT_EQ(ptr[i], i+1); 1854 1855 hmm_buffer_free(buffer); 1856 } 1857 1858 static int gup_test_exec(int gup_fd, unsigned long addr, int cmd, 1859 int npages, int size, int flags) 1860 { 1861 struct gup_test gup = { 1862 .nr_pages_per_call = npages, 1863 .addr = addr, 1864 .gup_flags = FOLL_WRITE | flags, 1865 .size = size, 1866 }; 1867 1868 if (ioctl(gup_fd, cmd, &gup)) { 1869 perror("ioctl on error\n"); 1870 return errno; 1871 } 1872 1873 return 0; 1874 } 1875 1876 /* 1877 * Test get user device pages through gup_test. Setting PIN_LONGTERM flag. 1878 * This should trigger a migration back to system memory for both, private 1879 * and coherent type pages. 1880 * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added 1881 * to your configuration before you run it. 1882 */ 1883 TEST_F(hmm, hmm_gup_test) 1884 { 1885 struct hmm_buffer *buffer; 1886 int gup_fd; 1887 unsigned long npages; 1888 unsigned long size; 1889 unsigned long i; 1890 int *ptr; 1891 int ret; 1892 unsigned char *m; 1893 1894 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); 1895 if (gup_fd == -1) 1896 SKIP(return, "Skipping test, could not find gup_test driver"); 1897 1898 npages = 4; 1899 size = npages << self->page_shift; 1900 1901 buffer = malloc(sizeof(*buffer)); 1902 ASSERT_NE(buffer, NULL); 1903 1904 buffer->fd = -1; 1905 buffer->size = size; 1906 buffer->mirror = malloc(size); 1907 ASSERT_NE(buffer->mirror, NULL); 1908 1909 buffer->ptr = mmap(NULL, size, 1910 PROT_READ | PROT_WRITE, 1911 MAP_PRIVATE | MAP_ANONYMOUS, 1912 buffer->fd, 0); 1913 ASSERT_NE(buffer->ptr, MAP_FAILED); 1914 1915 /* Initialize buffer in system memory. */ 1916 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1917 ptr[i] = i; 1918 1919 /* Migrate memory to device. */ 1920 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1921 ASSERT_EQ(ret, 0); 1922 ASSERT_EQ(buffer->cpages, npages); 1923 /* Check what the device read. */ 1924 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1925 ASSERT_EQ(ptr[i], i); 1926 1927 ASSERT_EQ(gup_test_exec(gup_fd, 1928 (unsigned long)buffer->ptr, 1929 GUP_BASIC_TEST, 1, self->page_size, 0), 0); 1930 ASSERT_EQ(gup_test_exec(gup_fd, 1931 (unsigned long)buffer->ptr + 1 * self->page_size, 1932 GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0); 1933 ASSERT_EQ(gup_test_exec(gup_fd, 1934 (unsigned long)buffer->ptr + 2 * self->page_size, 1935 PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0); 1936 ASSERT_EQ(gup_test_exec(gup_fd, 1937 (unsigned long)buffer->ptr + 3 * self->page_size, 1938 PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0); 1939 1940 /* Take snapshot to CPU pagetables */ 1941 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1942 ASSERT_EQ(ret, 0); 1943 ASSERT_EQ(buffer->cpages, npages); 1944 m = buffer->mirror; 1945 if (hmm_is_coherent_type(variant->device_number)) { 1946 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]); 1947 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]); 1948 } else { 1949 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]); 1950 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]); 1951 } 1952 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]); 1953 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]); 1954 /* 1955 * Check again the content on the pages. Make sure there's no 1956 * corrupted data. 1957 */ 1958 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1959 ASSERT_EQ(ptr[i], i); 1960 1961 close(gup_fd); 1962 hmm_buffer_free(buffer); 1963 } 1964 1965 /* 1966 * Test copy-on-write in device pages. 1967 * In case of writing to COW private page(s), a page fault will migrate pages 1968 * back to system memory first. Then, these pages will be duplicated. In case 1969 * of COW device coherent type, pages are duplicated directly from device 1970 * memory. 1971 */ 1972 TEST_F(hmm, hmm_cow_in_device) 1973 { 1974 struct hmm_buffer *buffer; 1975 unsigned long npages; 1976 unsigned long size; 1977 unsigned long i; 1978 int *ptr; 1979 int ret; 1980 unsigned char *m; 1981 pid_t pid; 1982 int status; 1983 1984 npages = 4; 1985 size = npages << self->page_shift; 1986 1987 buffer = malloc(sizeof(*buffer)); 1988 ASSERT_NE(buffer, NULL); 1989 1990 buffer->fd = -1; 1991 buffer->size = size; 1992 buffer->mirror = malloc(size); 1993 ASSERT_NE(buffer->mirror, NULL); 1994 1995 buffer->ptr = mmap(NULL, size, 1996 PROT_READ | PROT_WRITE, 1997 MAP_PRIVATE | MAP_ANONYMOUS, 1998 buffer->fd, 0); 1999 ASSERT_NE(buffer->ptr, MAP_FAILED); 2000 2001 /* Initialize buffer in system memory. */ 2002 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2003 ptr[i] = i; 2004 2005 /* Migrate memory to device. */ 2006 2007 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2008 ASSERT_EQ(ret, 0); 2009 ASSERT_EQ(buffer->cpages, npages); 2010 2011 pid = fork(); 2012 if (pid == -1) 2013 ASSERT_EQ(pid, 0); 2014 if (!pid) { 2015 /* Child process waits for SIGTERM from the parent. */ 2016 while (1) { 2017 } 2018 /* Should not reach this */ 2019 } 2020 /* Parent process writes to COW pages(s) and gets a 2021 * new copy in system. In case of device private pages, 2022 * this write causes a migration to system mem first. 2023 */ 2024 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2025 ptr[i] = i; 2026 2027 /* Terminate child and wait */ 2028 EXPECT_EQ(0, kill(pid, SIGTERM)); 2029 EXPECT_EQ(pid, waitpid(pid, &status, 0)); 2030 EXPECT_NE(0, WIFSIGNALED(status)); 2031 EXPECT_EQ(SIGTERM, WTERMSIG(status)); 2032 2033 /* Take snapshot to CPU pagetables */ 2034 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 2035 ASSERT_EQ(ret, 0); 2036 ASSERT_EQ(buffer->cpages, npages); 2037 m = buffer->mirror; 2038 for (i = 0; i < npages; i++) 2039 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]); 2040 2041 hmm_buffer_free(buffer); 2042 } 2043 2044 /* 2045 * Migrate private anonymous huge empty page. 2046 */ 2047 TEST_F(hmm, migrate_anon_huge_empty) 2048 { 2049 struct hmm_buffer *buffer; 2050 unsigned long npages; 2051 unsigned long size; 2052 unsigned long i; 2053 void *old_ptr; 2054 void *map; 2055 int *ptr; 2056 int ret; 2057 2058 size = read_pmd_pagesize(); 2059 2060 buffer = malloc(sizeof(*buffer)); 2061 ASSERT_NE(buffer, NULL); 2062 2063 buffer->fd = -1; 2064 buffer->size = 2 * size; 2065 buffer->mirror = malloc(size); 2066 ASSERT_NE(buffer->mirror, NULL); 2067 memset(buffer->mirror, 0xFF, size); 2068 2069 buffer->ptr = mmap(NULL, 2 * size, 2070 PROT_READ, 2071 MAP_PRIVATE | MAP_ANONYMOUS, 2072 buffer->fd, 0); 2073 ASSERT_NE(buffer->ptr, MAP_FAILED); 2074 2075 npages = size >> self->page_shift; 2076 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2077 ret = madvise(map, size, MADV_HUGEPAGE); 2078 ASSERT_EQ(ret, 0); 2079 old_ptr = buffer->ptr; 2080 buffer->ptr = map; 2081 2082 /* Migrate memory to device. */ 2083 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2084 ASSERT_EQ(ret, 0); 2085 ASSERT_EQ(buffer->cpages, npages); 2086 2087 /* Check what the device read. */ 2088 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2089 ASSERT_EQ(ptr[i], 0); 2090 2091 buffer->ptr = old_ptr; 2092 hmm_buffer_free(buffer); 2093 } 2094 2095 /* 2096 * Migrate private anonymous huge zero page. 2097 */ 2098 TEST_F(hmm, migrate_anon_huge_zero) 2099 { 2100 struct hmm_buffer *buffer; 2101 unsigned long npages; 2102 unsigned long size; 2103 unsigned long i; 2104 void *old_ptr; 2105 void *map; 2106 int *ptr; 2107 int ret; 2108 int val; 2109 2110 size = read_pmd_pagesize(); 2111 2112 buffer = malloc(sizeof(*buffer)); 2113 ASSERT_NE(buffer, NULL); 2114 2115 buffer->fd = -1; 2116 buffer->size = 2 * size; 2117 buffer->mirror = malloc(size); 2118 ASSERT_NE(buffer->mirror, NULL); 2119 memset(buffer->mirror, 0xFF, size); 2120 2121 buffer->ptr = mmap(NULL, 2 * size, 2122 PROT_READ, 2123 MAP_PRIVATE | MAP_ANONYMOUS, 2124 buffer->fd, 0); 2125 ASSERT_NE(buffer->ptr, MAP_FAILED); 2126 2127 npages = size >> self->page_shift; 2128 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2129 ret = madvise(map, size, MADV_HUGEPAGE); 2130 ASSERT_EQ(ret, 0); 2131 old_ptr = buffer->ptr; 2132 buffer->ptr = map; 2133 2134 /* Initialize a read-only zero huge page. */ 2135 val = *(int *)buffer->ptr; 2136 ASSERT_EQ(val, 0); 2137 2138 /* Migrate memory to device. */ 2139 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2140 ASSERT_EQ(ret, 0); 2141 ASSERT_EQ(buffer->cpages, npages); 2142 2143 /* Check what the device read. */ 2144 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2145 ASSERT_EQ(ptr[i], 0); 2146 2147 /* Fault pages back to system memory and check them. */ 2148 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) { 2149 ASSERT_EQ(ptr[i], 0); 2150 /* If it asserts once, it probably will 500,000 times */ 2151 if (ptr[i] != 0) 2152 break; 2153 } 2154 2155 buffer->ptr = old_ptr; 2156 hmm_buffer_free(buffer); 2157 } 2158 2159 /* 2160 * Migrate private anonymous huge page and free. 2161 */ 2162 TEST_F(hmm, migrate_anon_huge_free) 2163 { 2164 struct hmm_buffer *buffer; 2165 unsigned long npages; 2166 unsigned long size; 2167 unsigned long i; 2168 void *old_ptr; 2169 void *map; 2170 int *ptr; 2171 int ret; 2172 2173 size = read_pmd_pagesize(); 2174 2175 buffer = malloc(sizeof(*buffer)); 2176 ASSERT_NE(buffer, NULL); 2177 2178 buffer->fd = -1; 2179 buffer->size = 2 * size; 2180 buffer->mirror = malloc(size); 2181 ASSERT_NE(buffer->mirror, NULL); 2182 memset(buffer->mirror, 0xFF, size); 2183 2184 buffer->ptr = mmap(NULL, 2 * size, 2185 PROT_READ | PROT_WRITE, 2186 MAP_PRIVATE | MAP_ANONYMOUS, 2187 buffer->fd, 0); 2188 ASSERT_NE(buffer->ptr, MAP_FAILED); 2189 2190 npages = size >> self->page_shift; 2191 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2192 ret = madvise(map, size, MADV_HUGEPAGE); 2193 ASSERT_EQ(ret, 0); 2194 old_ptr = buffer->ptr; 2195 buffer->ptr = map; 2196 2197 /* Initialize buffer in system memory. */ 2198 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2199 ptr[i] = i; 2200 2201 /* Migrate memory to device. */ 2202 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2203 ASSERT_EQ(ret, 0); 2204 ASSERT_EQ(buffer->cpages, npages); 2205 2206 /* Check what the device read. */ 2207 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2208 ASSERT_EQ(ptr[i], i); 2209 2210 /* Try freeing it. */ 2211 ret = madvise(map, size, MADV_FREE); 2212 ASSERT_EQ(ret, 0); 2213 2214 buffer->ptr = old_ptr; 2215 hmm_buffer_free(buffer); 2216 } 2217 2218 /* 2219 * Migrate private anonymous huge page and fault back to sysmem. 2220 */ 2221 TEST_F(hmm, migrate_anon_huge_fault) 2222 { 2223 struct hmm_buffer *buffer; 2224 unsigned long npages; 2225 unsigned long size; 2226 unsigned long i; 2227 void *old_ptr; 2228 void *map; 2229 int *ptr; 2230 int ret; 2231 2232 size = read_pmd_pagesize(); 2233 2234 buffer = malloc(sizeof(*buffer)); 2235 ASSERT_NE(buffer, NULL); 2236 2237 buffer->fd = -1; 2238 buffer->size = 2 * size; 2239 buffer->mirror = malloc(size); 2240 ASSERT_NE(buffer->mirror, NULL); 2241 memset(buffer->mirror, 0xFF, size); 2242 2243 buffer->ptr = mmap(NULL, 2 * size, 2244 PROT_READ | PROT_WRITE, 2245 MAP_PRIVATE | MAP_ANONYMOUS, 2246 buffer->fd, 0); 2247 ASSERT_NE(buffer->ptr, MAP_FAILED); 2248 2249 npages = size >> self->page_shift; 2250 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2251 ret = madvise(map, size, MADV_HUGEPAGE); 2252 ASSERT_EQ(ret, 0); 2253 old_ptr = buffer->ptr; 2254 buffer->ptr = map; 2255 2256 /* Initialize buffer in system memory. */ 2257 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2258 ptr[i] = i; 2259 2260 /* Migrate memory to device. */ 2261 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2262 ASSERT_EQ(ret, 0); 2263 ASSERT_EQ(buffer->cpages, npages); 2264 2265 /* Check what the device read. */ 2266 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2267 ASSERT_EQ(ptr[i], i); 2268 2269 /* Fault pages back to system memory and check them. */ 2270 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2271 ASSERT_EQ(ptr[i], i); 2272 2273 buffer->ptr = old_ptr; 2274 hmm_buffer_free(buffer); 2275 } 2276 2277 /* 2278 * Migrate memory and fault back to sysmem after partially unmapping. 2279 */ 2280 TEST_F(hmm, migrate_partial_unmap_fault) 2281 { 2282 struct hmm_buffer *buffer; 2283 unsigned long npages; 2284 unsigned long size = read_pmd_pagesize(); 2285 unsigned long i; 2286 void *old_ptr; 2287 void *map; 2288 int *ptr; 2289 int ret, j, use_thp; 2290 int offsets[] = { 0, 512 * ONEKB, ONEMEG }; 2291 2292 for (use_thp = 0; use_thp < 2; ++use_thp) { 2293 for (j = 0; j < ARRAY_SIZE(offsets); ++j) { 2294 buffer = malloc(sizeof(*buffer)); 2295 ASSERT_NE(buffer, NULL); 2296 2297 buffer->fd = -1; 2298 buffer->size = 2 * size; 2299 buffer->mirror = malloc(size); 2300 ASSERT_NE(buffer->mirror, NULL); 2301 memset(buffer->mirror, 0xFF, size); 2302 2303 buffer->ptr = mmap(NULL, 2 * size, 2304 PROT_READ | PROT_WRITE, 2305 MAP_PRIVATE | MAP_ANONYMOUS, 2306 buffer->fd, 0); 2307 ASSERT_NE(buffer->ptr, MAP_FAILED); 2308 2309 npages = size >> self->page_shift; 2310 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2311 if (use_thp) 2312 ret = madvise(map, size, MADV_HUGEPAGE); 2313 else 2314 ret = madvise(map, size, MADV_NOHUGEPAGE); 2315 ASSERT_EQ(ret, 0); 2316 old_ptr = buffer->ptr; 2317 buffer->ptr = map; 2318 2319 /* Initialize buffer in system memory. */ 2320 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2321 ptr[i] = i; 2322 2323 /* Migrate memory to device. */ 2324 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2325 ASSERT_EQ(ret, 0); 2326 ASSERT_EQ(buffer->cpages, npages); 2327 2328 /* Check what the device read. */ 2329 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2330 ASSERT_EQ(ptr[i], i); 2331 2332 munmap(buffer->ptr + offsets[j], ONEMEG); 2333 2334 /* Fault pages back to system memory and check them. */ 2335 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2336 if (i * sizeof(int) < offsets[j] || 2337 i * sizeof(int) >= offsets[j] + ONEMEG) 2338 ASSERT_EQ(ptr[i], i); 2339 2340 buffer->ptr = old_ptr; 2341 hmm_buffer_free(buffer); 2342 } 2343 } 2344 } 2345 2346 TEST_F(hmm, migrate_remap_fault) 2347 { 2348 struct hmm_buffer *buffer; 2349 unsigned long npages; 2350 unsigned long size = read_pmd_pagesize(); 2351 unsigned long i; 2352 void *old_ptr, *new_ptr = NULL; 2353 void *map; 2354 int *ptr; 2355 int ret, j, use_thp, dont_unmap, before; 2356 int offsets[] = { 0, 512 * ONEKB, ONEMEG }; 2357 2358 for (before = 0; before < 2; ++before) { 2359 for (dont_unmap = 0; dont_unmap < 2; ++dont_unmap) { 2360 for (use_thp = 0; use_thp < 2; ++use_thp) { 2361 for (j = 0; j < ARRAY_SIZE(offsets); ++j) { 2362 int flags = MREMAP_MAYMOVE | MREMAP_FIXED; 2363 2364 if (dont_unmap) 2365 flags |= MREMAP_DONTUNMAP; 2366 2367 buffer = malloc(sizeof(*buffer)); 2368 ASSERT_NE(buffer, NULL); 2369 2370 buffer->fd = -1; 2371 buffer->size = 8 * size; 2372 buffer->mirror = malloc(size); 2373 ASSERT_NE(buffer->mirror, NULL); 2374 memset(buffer->mirror, 0xFF, size); 2375 2376 buffer->ptr = mmap(NULL, buffer->size, 2377 PROT_READ | PROT_WRITE, 2378 MAP_PRIVATE | MAP_ANONYMOUS, 2379 buffer->fd, 0); 2380 ASSERT_NE(buffer->ptr, MAP_FAILED); 2381 2382 npages = size >> self->page_shift; 2383 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2384 if (use_thp) 2385 ret = madvise(map, size, MADV_HUGEPAGE); 2386 else 2387 ret = madvise(map, size, MADV_NOHUGEPAGE); 2388 ASSERT_EQ(ret, 0); 2389 old_ptr = buffer->ptr; 2390 munmap(map + size, size * 2); 2391 buffer->ptr = map; 2392 2393 /* Initialize buffer in system memory. */ 2394 for (i = 0, ptr = buffer->ptr; 2395 i < size / sizeof(*ptr); ++i) 2396 ptr[i] = i; 2397 2398 if (before) { 2399 new_ptr = mremap((void *)map, size, size, flags, 2400 map + size + offsets[j]); 2401 ASSERT_NE(new_ptr, MAP_FAILED); 2402 buffer->ptr = new_ptr; 2403 } 2404 2405 /* Migrate memory to device. */ 2406 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2407 ASSERT_EQ(ret, 0); 2408 ASSERT_EQ(buffer->cpages, npages); 2409 2410 /* Check what the device read. */ 2411 for (i = 0, ptr = buffer->mirror; 2412 i < size / sizeof(*ptr); ++i) 2413 ASSERT_EQ(ptr[i], i); 2414 2415 if (!before) { 2416 new_ptr = mremap((void *)map, size, size, flags, 2417 map + size + offsets[j]); 2418 ASSERT_NE(new_ptr, MAP_FAILED); 2419 buffer->ptr = new_ptr; 2420 } 2421 2422 /* Fault pages back to system memory and check them. */ 2423 for (i = 0, ptr = buffer->ptr; 2424 i < size / sizeof(*ptr); ++i) 2425 ASSERT_EQ(ptr[i], i); 2426 2427 munmap(new_ptr, size); 2428 buffer->ptr = old_ptr; 2429 hmm_buffer_free(buffer); 2430 } 2431 } 2432 } 2433 } 2434 } 2435 2436 /* 2437 * Migrate private anonymous huge page with allocation errors. 2438 */ 2439 TEST_F(hmm, migrate_anon_huge_err) 2440 { 2441 struct hmm_buffer *buffer; 2442 unsigned long npages; 2443 unsigned long size; 2444 unsigned long i; 2445 void *old_ptr; 2446 void *map; 2447 int *ptr; 2448 int ret; 2449 2450 size = read_pmd_pagesize(); 2451 2452 buffer = malloc(sizeof(*buffer)); 2453 ASSERT_NE(buffer, NULL); 2454 2455 buffer->fd = -1; 2456 buffer->size = 2 * size; 2457 buffer->mirror = malloc(2 * size); 2458 ASSERT_NE(buffer->mirror, NULL); 2459 memset(buffer->mirror, 0xFF, 2 * size); 2460 2461 old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, 2462 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2463 ASSERT_NE(old_ptr, MAP_FAILED); 2464 2465 npages = size >> self->page_shift; 2466 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2467 ret = madvise(map, size, MADV_HUGEPAGE); 2468 ASSERT_EQ(ret, 0); 2469 buffer->ptr = map; 2470 2471 /* Initialize buffer in system memory. */ 2472 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2473 ptr[i] = i; 2474 2475 /* Migrate memory to device but force a THP allocation error. */ 2476 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2477 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2478 ASSERT_EQ(ret, 0); 2479 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2480 ASSERT_EQ(ret, 0); 2481 ASSERT_EQ(buffer->cpages, npages); 2482 2483 /* Check what the device read. */ 2484 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) { 2485 ASSERT_EQ(ptr[i], i); 2486 if (ptr[i] != i) 2487 break; 2488 } 2489 2490 /* Try faulting back a single (PAGE_SIZE) page. */ 2491 ptr = buffer->ptr; 2492 ASSERT_EQ(ptr[2048], 2048); 2493 2494 /* unmap and remap the region to reset things. */ 2495 ret = munmap(old_ptr, 2 * size); 2496 ASSERT_EQ(ret, 0); 2497 old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, 2498 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2499 ASSERT_NE(old_ptr, MAP_FAILED); 2500 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2501 ret = madvise(map, size, MADV_HUGEPAGE); 2502 ASSERT_EQ(ret, 0); 2503 buffer->ptr = map; 2504 2505 /* Initialize buffer in system memory. */ 2506 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2507 ptr[i] = i; 2508 2509 /* Migrate THP to device. */ 2510 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2511 ASSERT_EQ(ret, 0); 2512 ASSERT_EQ(buffer->cpages, npages); 2513 2514 /* 2515 * Force an allocation error when faulting back a THP resident in the 2516 * device. 2517 */ 2518 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2519 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2520 ASSERT_EQ(ret, 0); 2521 2522 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages); 2523 ASSERT_EQ(ret, 0); 2524 ptr = buffer->ptr; 2525 ASSERT_EQ(ptr[2048], 2048); 2526 2527 buffer->ptr = old_ptr; 2528 hmm_buffer_free(buffer); 2529 } 2530 2531 /* 2532 * Migrate private anonymous huge zero page with allocation errors. 2533 */ 2534 TEST_F(hmm, migrate_anon_huge_zero_err) 2535 { 2536 struct hmm_buffer *buffer; 2537 unsigned long npages; 2538 unsigned long size; 2539 unsigned long i; 2540 void *old_ptr; 2541 void *map; 2542 int *ptr; 2543 int ret; 2544 2545 size = read_pmd_pagesize(); 2546 2547 buffer = malloc(sizeof(*buffer)); 2548 ASSERT_NE(buffer, NULL); 2549 2550 buffer->fd = -1; 2551 buffer->size = 2 * size; 2552 buffer->mirror = malloc(2 * size); 2553 ASSERT_NE(buffer->mirror, NULL); 2554 memset(buffer->mirror, 0xFF, 2 * size); 2555 2556 old_ptr = mmap(NULL, 2 * size, PROT_READ, 2557 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2558 ASSERT_NE(old_ptr, MAP_FAILED); 2559 2560 npages = size >> self->page_shift; 2561 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2562 ret = madvise(map, size, MADV_HUGEPAGE); 2563 ASSERT_EQ(ret, 0); 2564 buffer->ptr = map; 2565 2566 /* Migrate memory to device but force a THP allocation error. */ 2567 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2568 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2569 ASSERT_EQ(ret, 0); 2570 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2571 ASSERT_EQ(ret, 0); 2572 ASSERT_EQ(buffer->cpages, npages); 2573 2574 /* Check what the device read. */ 2575 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2576 ASSERT_EQ(ptr[i], 0); 2577 2578 /* Try faulting back a single (PAGE_SIZE) page. */ 2579 ptr = buffer->ptr; 2580 ASSERT_EQ(ptr[2048], 0); 2581 2582 /* unmap and remap the region to reset things. */ 2583 ret = munmap(old_ptr, 2 * size); 2584 ASSERT_EQ(ret, 0); 2585 old_ptr = mmap(NULL, 2 * size, PROT_READ, 2586 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2587 ASSERT_NE(old_ptr, MAP_FAILED); 2588 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2589 ret = madvise(map, size, MADV_HUGEPAGE); 2590 ASSERT_EQ(ret, 0); 2591 buffer->ptr = map; 2592 2593 /* Initialize buffer in system memory (zero THP page). */ 2594 ret = ptr[0]; 2595 ASSERT_EQ(ret, 0); 2596 2597 /* Migrate memory to device but force a THP allocation error. */ 2598 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2599 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2600 ASSERT_EQ(ret, 0); 2601 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2602 ASSERT_EQ(ret, 0); 2603 ASSERT_EQ(buffer->cpages, npages); 2604 2605 /* Fault the device memory back and check it. */ 2606 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2607 ASSERT_EQ(ptr[i], 0); 2608 2609 buffer->ptr = old_ptr; 2610 hmm_buffer_free(buffer); 2611 } 2612 2613 struct benchmark_results { 2614 double sys_to_dev_time; 2615 double dev_to_sys_time; 2616 double throughput_s2d; 2617 double throughput_d2s; 2618 }; 2619 2620 static double get_time_ms(void) 2621 { 2622 struct timeval tv; 2623 2624 gettimeofday(&tv, NULL); 2625 return (tv.tv_sec * 1000.0) + (tv.tv_usec / 1000.0); 2626 } 2627 2628 static inline struct hmm_buffer *hmm_buffer_alloc(unsigned long size) 2629 { 2630 struct hmm_buffer *buffer; 2631 2632 buffer = malloc(sizeof(*buffer)); 2633 2634 buffer->fd = -1; 2635 buffer->size = size; 2636 buffer->mirror = malloc(size); 2637 memset(buffer->mirror, 0xFF, size); 2638 return buffer; 2639 } 2640 2641 static void print_benchmark_results(const char *test_name, size_t buffer_size, 2642 struct benchmark_results *thp, 2643 struct benchmark_results *regular) 2644 { 2645 double s2d_improvement = ((regular->sys_to_dev_time - thp->sys_to_dev_time) / 2646 regular->sys_to_dev_time) * 100.0; 2647 double d2s_improvement = ((regular->dev_to_sys_time - thp->dev_to_sys_time) / 2648 regular->dev_to_sys_time) * 100.0; 2649 double throughput_s2d_improvement = ((thp->throughput_s2d - regular->throughput_s2d) / 2650 regular->throughput_s2d) * 100.0; 2651 double throughput_d2s_improvement = ((thp->throughput_d2s - regular->throughput_d2s) / 2652 regular->throughput_d2s) * 100.0; 2653 2654 printf("\n=== %s (%.1f MB) ===\n", test_name, buffer_size / (1024.0 * 1024.0)); 2655 printf(" | With THP | Without THP | Improvement\n"); 2656 printf("---------------------------------------------------------------------\n"); 2657 printf("Sys->Dev Migration | %.3f ms | %.3f ms | %.1f%%\n", 2658 thp->sys_to_dev_time, regular->sys_to_dev_time, s2d_improvement); 2659 printf("Dev->Sys Migration | %.3f ms | %.3f ms | %.1f%%\n", 2660 thp->dev_to_sys_time, regular->dev_to_sys_time, d2s_improvement); 2661 printf("S->D Throughput | %.2f GB/s | %.2f GB/s | %.1f%%\n", 2662 thp->throughput_s2d, regular->throughput_s2d, throughput_s2d_improvement); 2663 printf("D->S Throughput | %.2f GB/s | %.2f GB/s | %.1f%%\n", 2664 thp->throughput_d2s, regular->throughput_d2s, throughput_d2s_improvement); 2665 } 2666 2667 /* 2668 * Run a single migration benchmark 2669 * fd: file descriptor for hmm device 2670 * use_thp: whether to use THP 2671 * buffer_size: size of buffer to allocate 2672 * iterations: number of iterations 2673 * results: where to store results 2674 */ 2675 static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_size, 2676 int iterations, struct benchmark_results *results) 2677 { 2678 struct hmm_buffer *buffer; 2679 unsigned long npages = buffer_size / sysconf(_SC_PAGESIZE); 2680 double start, end; 2681 double s2d_total = 0, d2s_total = 0; 2682 int ret, i; 2683 int *ptr; 2684 2685 buffer = hmm_buffer_alloc(buffer_size); 2686 2687 /* Map memory */ 2688 buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE, 2689 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 2690 2691 if (!buffer->ptr) 2692 return -1; 2693 2694 /* Apply THP hint if requested */ 2695 if (use_thp) 2696 ret = madvise(buffer->ptr, buffer_size, MADV_HUGEPAGE); 2697 else 2698 ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE); 2699 2700 if (ret) 2701 return ret; 2702 2703 /* Initialize memory to make sure pages are allocated */ 2704 ptr = (int *)buffer->ptr; 2705 for (i = 0; i < buffer_size / sizeof(int); i++) 2706 ptr[i] = i & 0xFF; 2707 2708 /* Warmup iteration */ 2709 ret = hmm_migrate_sys_to_dev(fd, buffer, npages); 2710 if (ret) 2711 return ret; 2712 2713 ret = hmm_migrate_dev_to_sys(fd, buffer, npages); 2714 if (ret) 2715 return ret; 2716 2717 /* Benchmark iterations */ 2718 for (i = 0; i < iterations; i++) { 2719 /* System to device migration */ 2720 start = get_time_ms(); 2721 2722 ret = hmm_migrate_sys_to_dev(fd, buffer, npages); 2723 if (ret) 2724 return ret; 2725 2726 end = get_time_ms(); 2727 s2d_total += (end - start); 2728 2729 /* Device to system migration */ 2730 start = get_time_ms(); 2731 2732 ret = hmm_migrate_dev_to_sys(fd, buffer, npages); 2733 if (ret) 2734 return ret; 2735 2736 end = get_time_ms(); 2737 d2s_total += (end - start); 2738 } 2739 2740 /* Calculate average times and throughput */ 2741 results->sys_to_dev_time = s2d_total / iterations; 2742 results->dev_to_sys_time = d2s_total / iterations; 2743 results->throughput_s2d = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / 2744 (results->sys_to_dev_time / 1000.0); 2745 results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / 2746 (results->dev_to_sys_time / 1000.0); 2747 2748 /* Cleanup */ 2749 hmm_buffer_free(buffer); 2750 return 0; 2751 } 2752 2753 /* 2754 * Benchmark THP migration with different buffer sizes 2755 */ 2756 TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120) 2757 { 2758 struct benchmark_results thp_results, regular_results; 2759 size_t thp_size = 2 * 1024 * 1024; /* 2MB - typical THP size */ 2760 int iterations = 5; 2761 2762 printf("\nHMM THP Migration Benchmark\n"); 2763 printf("---------------------------\n"); 2764 printf("System page size: %ld bytes\n", sysconf(_SC_PAGESIZE)); 2765 2766 /* Test different buffer sizes */ 2767 size_t test_sizes[] = { 2768 thp_size / 4, /* 512KB - smaller than THP */ 2769 thp_size / 2, /* 1MB - half THP */ 2770 thp_size, /* 2MB - single THP */ 2771 thp_size * 2, /* 4MB - two THPs */ 2772 thp_size * 4, /* 8MB - four THPs */ 2773 thp_size * 8, /* 16MB - eight THPs */ 2774 thp_size * 128, /* 256MB - one twenty eight THPs */ 2775 }; 2776 2777 static const char *const test_names[] = { 2778 "Small Buffer (512KB)", 2779 "Half THP Size (1MB)", 2780 "Single THP Size (2MB)", 2781 "Two THP Size (4MB)", 2782 "Four THP Size (8MB)", 2783 "Eight THP Size (16MB)", 2784 "One twenty eight THP Size (256MB)" 2785 }; 2786 2787 int num_tests = ARRAY_SIZE(test_sizes); 2788 2789 /* Run all tests */ 2790 for (int i = 0; i < num_tests; i++) { 2791 /* Test with THP */ 2792 ASSERT_EQ(run_migration_benchmark(self->fd, 1, test_sizes[i], 2793 iterations, &thp_results), 0); 2794 2795 /* Test without THP */ 2796 ASSERT_EQ(run_migration_benchmark(self->fd, 0, test_sizes[i], 2797 iterations, ®ular_results), 0); 2798 2799 /* Print results */ 2800 print_benchmark_results(test_names[i], test_sizes[i], 2801 &thp_results, ®ular_results); 2802 } 2803 } 2804 TEST_HARNESS_MAIN 2805