1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * HMM stands for Heterogeneous Memory Management, it is a helper layer inside 4 * the linux kernel to help device drivers mirror a process address space in 5 * the device. This allows the device to use the same address space which 6 * makes communication and data exchange a lot easier. 7 * 8 * This framework's sole purpose is to exercise various code paths inside 9 * the kernel to make sure that HMM performs as expected and to flush out any 10 * bugs. 11 */ 12 13 #include "kselftest_harness.h" 14 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <stdio.h> 18 #include <stdlib.h> 19 #include <stdint.h> 20 #include <unistd.h> 21 #include <strings.h> 22 #include <time.h> 23 #include <pthread.h> 24 #include <sys/types.h> 25 #include <sys/stat.h> 26 #include <sys/mman.h> 27 #include <sys/ioctl.h> 28 #include <sys/time.h> 29 30 31 /* 32 * This is a private UAPI to the kernel test module so it isn't exported 33 * in the usual include/uapi/... directory. 34 */ 35 #include <lib/test_hmm_uapi.h> 36 #include <mm/gup_test.h> 37 #include <mm/vm_util.h> 38 39 struct hmm_buffer { 40 void *ptr; 41 void *mirror; 42 unsigned long size; 43 int fd; 44 uint64_t cpages; 45 uint64_t faults; 46 }; 47 48 enum { 49 HMM_PRIVATE_DEVICE_ONE, 50 HMM_PRIVATE_DEVICE_TWO, 51 HMM_COHERENCE_DEVICE_ONE, 52 HMM_COHERENCE_DEVICE_TWO, 53 }; 54 55 #define ONEKB (1 << 10) 56 #define ONEMEG (1 << 20) 57 #define TWOMEG (1 << 21) 58 #define HMM_BUFFER_SIZE (1024 << 12) 59 #define HMM_PATH_MAX 64 60 #define NTIMES 10 61 62 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) 63 /* Just the flags we need, copied from mm.h: */ 64 65 #ifndef FOLL_WRITE 66 #define FOLL_WRITE 0x01 /* check pte is writable */ 67 #endif 68 69 #ifndef FOLL_LONGTERM 70 #define FOLL_LONGTERM 0x100 /* mapping lifetime is indefinite */ 71 #endif 72 FIXTURE(hmm) 73 { 74 int fd; 75 unsigned int page_size; 76 unsigned int page_shift; 77 }; 78 79 FIXTURE_VARIANT(hmm) 80 { 81 int device_number; 82 }; 83 84 FIXTURE_VARIANT_ADD(hmm, hmm_device_private) 85 { 86 .device_number = HMM_PRIVATE_DEVICE_ONE, 87 }; 88 89 FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent) 90 { 91 .device_number = HMM_COHERENCE_DEVICE_ONE, 92 }; 93 94 FIXTURE(hmm2) 95 { 96 int fd0; 97 int fd1; 98 unsigned int page_size; 99 unsigned int page_shift; 100 }; 101 102 FIXTURE_VARIANT(hmm2) 103 { 104 int device_number0; 105 int device_number1; 106 }; 107 108 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private) 109 { 110 .device_number0 = HMM_PRIVATE_DEVICE_ONE, 111 .device_number1 = HMM_PRIVATE_DEVICE_TWO, 112 }; 113 114 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent) 115 { 116 .device_number0 = HMM_COHERENCE_DEVICE_ONE, 117 .device_number1 = HMM_COHERENCE_DEVICE_TWO, 118 }; 119 120 static int hmm_open(int unit) 121 { 122 char pathname[HMM_PATH_MAX]; 123 int fd; 124 125 snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit); 126 fd = open(pathname, O_RDWR, 0); 127 if (fd < 0) 128 fprintf(stderr, "could not open hmm dmirror driver (%s)\n", 129 pathname); 130 return fd; 131 } 132 133 static bool hmm_is_coherent_type(int dev_num) 134 { 135 return (dev_num >= HMM_COHERENCE_DEVICE_ONE); 136 } 137 138 FIXTURE_SETUP(hmm) 139 { 140 self->page_size = sysconf(_SC_PAGE_SIZE); 141 self->page_shift = ffs(self->page_size) - 1; 142 143 self->fd = hmm_open(variant->device_number); 144 if (self->fd < 0 && hmm_is_coherent_type(variant->device_number)) 145 SKIP(return, "DEVICE_COHERENT not available"); 146 ASSERT_GE(self->fd, 0); 147 } 148 149 FIXTURE_SETUP(hmm2) 150 { 151 self->page_size = sysconf(_SC_PAGE_SIZE); 152 self->page_shift = ffs(self->page_size) - 1; 153 154 self->fd0 = hmm_open(variant->device_number0); 155 if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0)) 156 SKIP(return, "DEVICE_COHERENT not available"); 157 ASSERT_GE(self->fd0, 0); 158 self->fd1 = hmm_open(variant->device_number1); 159 ASSERT_GE(self->fd1, 0); 160 } 161 162 FIXTURE_TEARDOWN(hmm) 163 { 164 int ret = close(self->fd); 165 166 ASSERT_EQ(ret, 0); 167 self->fd = -1; 168 } 169 170 FIXTURE_TEARDOWN(hmm2) 171 { 172 int ret = close(self->fd0); 173 174 ASSERT_EQ(ret, 0); 175 self->fd0 = -1; 176 177 ret = close(self->fd1); 178 ASSERT_EQ(ret, 0); 179 self->fd1 = -1; 180 } 181 182 static int hmm_dmirror_cmd(int fd, 183 unsigned long request, 184 struct hmm_buffer *buffer, 185 unsigned long npages) 186 { 187 struct hmm_dmirror_cmd cmd; 188 int ret; 189 190 /* Simulate a device reading system memory. */ 191 cmd.addr = (__u64)buffer->ptr; 192 cmd.ptr = (__u64)buffer->mirror; 193 cmd.npages = npages; 194 195 for (;;) { 196 ret = ioctl(fd, request, &cmd); 197 if (ret == 0) 198 break; 199 if (errno == EINTR) 200 continue; 201 return -errno; 202 } 203 buffer->cpages = cmd.cpages; 204 buffer->faults = cmd.faults; 205 206 return 0; 207 } 208 209 static void hmm_buffer_free(struct hmm_buffer *buffer) 210 { 211 if (buffer == NULL) 212 return; 213 214 if (buffer->ptr) { 215 munmap(buffer->ptr, buffer->size); 216 buffer->ptr = NULL; 217 } 218 free(buffer->mirror); 219 free(buffer); 220 } 221 222 /* 223 * Create a temporary file that will be deleted on close. 224 */ 225 static int hmm_create_file(unsigned long size) 226 { 227 char path[HMM_PATH_MAX]; 228 int fd; 229 230 strcpy(path, "/tmp"); 231 fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600); 232 if (fd >= 0) { 233 int r; 234 235 do { 236 r = ftruncate(fd, size); 237 } while (r == -1 && errno == EINTR); 238 if (!r) 239 return fd; 240 close(fd); 241 } 242 return -1; 243 } 244 245 /* 246 * Return a random unsigned number. 247 */ 248 static unsigned int hmm_random(void) 249 { 250 static int fd = -1; 251 unsigned int r; 252 253 if (fd < 0) { 254 fd = open("/dev/urandom", O_RDONLY); 255 if (fd < 0) { 256 fprintf(stderr, "%s:%d failed to open /dev/urandom\n", 257 __FILE__, __LINE__); 258 return ~0U; 259 } 260 } 261 read(fd, &r, sizeof(r)); 262 return r; 263 } 264 265 static void hmm_nanosleep(unsigned int n) 266 { 267 struct timespec t; 268 269 t.tv_sec = 0; 270 t.tv_nsec = n; 271 nanosleep(&t, NULL); 272 } 273 274 static int hmm_migrate_sys_to_dev(int fd, 275 struct hmm_buffer *buffer, 276 unsigned long npages) 277 { 278 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages); 279 } 280 281 static int hmm_migrate_dev_to_sys(int fd, 282 struct hmm_buffer *buffer, 283 unsigned long npages) 284 { 285 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages); 286 } 287 288 /* 289 * Simple NULL test of device open/close. 290 */ 291 TEST_F(hmm, open_close) 292 { 293 } 294 295 /* 296 * Read private anonymous memory. 297 */ 298 TEST_F(hmm, anon_read) 299 { 300 struct hmm_buffer *buffer; 301 unsigned long npages; 302 unsigned long size; 303 unsigned long i; 304 int *ptr; 305 int ret; 306 int val; 307 308 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 309 ASSERT_NE(npages, 0); 310 size = npages << self->page_shift; 311 312 buffer = malloc(sizeof(*buffer)); 313 ASSERT_NE(buffer, NULL); 314 315 buffer->fd = -1; 316 buffer->size = size; 317 buffer->mirror = malloc(size); 318 ASSERT_NE(buffer->mirror, NULL); 319 320 buffer->ptr = mmap(NULL, size, 321 PROT_READ | PROT_WRITE, 322 MAP_PRIVATE | MAP_ANONYMOUS, 323 buffer->fd, 0); 324 ASSERT_NE(buffer->ptr, MAP_FAILED); 325 326 /* 327 * Initialize buffer in system memory but leave the first two pages 328 * zero (pte_none and pfn_zero). 329 */ 330 i = 2 * self->page_size / sizeof(*ptr); 331 for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 332 ptr[i] = i; 333 334 /* Set buffer permission to read-only. */ 335 ret = mprotect(buffer->ptr, size, PROT_READ); 336 ASSERT_EQ(ret, 0); 337 338 /* Populate the CPU page table with a special zero page. */ 339 val = *(int *)(buffer->ptr + self->page_size); 340 ASSERT_EQ(val, 0); 341 342 /* Simulate a device reading system memory. */ 343 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 344 ASSERT_EQ(ret, 0); 345 ASSERT_EQ(buffer->cpages, npages); 346 ASSERT_EQ(buffer->faults, 1); 347 348 /* Check what the device read. */ 349 ptr = buffer->mirror; 350 for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i) 351 ASSERT_EQ(ptr[i], 0); 352 for (; i < size / sizeof(*ptr); ++i) 353 ASSERT_EQ(ptr[i], i); 354 355 hmm_buffer_free(buffer); 356 } 357 358 /* 359 * Read private anonymous memory which has been protected with 360 * mprotect() PROT_NONE. 361 */ 362 TEST_F(hmm, anon_read_prot) 363 { 364 struct hmm_buffer *buffer; 365 unsigned long npages; 366 unsigned long size; 367 unsigned long i; 368 int *ptr; 369 int ret; 370 371 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 372 ASSERT_NE(npages, 0); 373 size = npages << self->page_shift; 374 375 buffer = malloc(sizeof(*buffer)); 376 ASSERT_NE(buffer, NULL); 377 378 buffer->fd = -1; 379 buffer->size = size; 380 buffer->mirror = malloc(size); 381 ASSERT_NE(buffer->mirror, NULL); 382 383 buffer->ptr = mmap(NULL, size, 384 PROT_READ | PROT_WRITE, 385 MAP_PRIVATE | MAP_ANONYMOUS, 386 buffer->fd, 0); 387 ASSERT_NE(buffer->ptr, MAP_FAILED); 388 389 /* Initialize buffer in system memory. */ 390 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 391 ptr[i] = i; 392 393 /* Initialize mirror buffer so we can verify it isn't written. */ 394 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 395 ptr[i] = -i; 396 397 /* Protect buffer from reading. */ 398 ret = mprotect(buffer->ptr, size, PROT_NONE); 399 ASSERT_EQ(ret, 0); 400 401 /* Simulate a device reading system memory. */ 402 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 403 ASSERT_EQ(ret, -EFAULT); 404 405 /* Allow CPU to read the buffer so we can check it. */ 406 ret = mprotect(buffer->ptr, size, PROT_READ); 407 ASSERT_EQ(ret, 0); 408 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 409 ASSERT_EQ(ptr[i], i); 410 411 /* Check what the device read. */ 412 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 413 ASSERT_EQ(ptr[i], -i); 414 415 hmm_buffer_free(buffer); 416 } 417 418 /* 419 * Write private anonymous memory. 420 */ 421 TEST_F(hmm, anon_write) 422 { 423 struct hmm_buffer *buffer; 424 unsigned long npages; 425 unsigned long size; 426 unsigned long i; 427 int *ptr; 428 int ret; 429 430 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 431 ASSERT_NE(npages, 0); 432 size = npages << self->page_shift; 433 434 buffer = malloc(sizeof(*buffer)); 435 ASSERT_NE(buffer, NULL); 436 437 buffer->fd = -1; 438 buffer->size = size; 439 buffer->mirror = malloc(size); 440 ASSERT_NE(buffer->mirror, NULL); 441 442 buffer->ptr = mmap(NULL, size, 443 PROT_READ | PROT_WRITE, 444 MAP_PRIVATE | MAP_ANONYMOUS, 445 buffer->fd, 0); 446 ASSERT_NE(buffer->ptr, MAP_FAILED); 447 448 /* Initialize data that the device will write to buffer->ptr. */ 449 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 450 ptr[i] = i; 451 452 /* Simulate a device writing system memory. */ 453 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 454 ASSERT_EQ(ret, 0); 455 ASSERT_EQ(buffer->cpages, npages); 456 ASSERT_EQ(buffer->faults, 1); 457 458 /* Check what the device wrote. */ 459 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 460 ASSERT_EQ(ptr[i], i); 461 462 hmm_buffer_free(buffer); 463 } 464 465 /* 466 * Write private anonymous memory which has been protected with 467 * mprotect() PROT_READ. 468 */ 469 TEST_F(hmm, anon_write_prot) 470 { 471 struct hmm_buffer *buffer; 472 unsigned long npages; 473 unsigned long size; 474 unsigned long i; 475 int *ptr; 476 int ret; 477 478 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 479 ASSERT_NE(npages, 0); 480 size = npages << self->page_shift; 481 482 buffer = malloc(sizeof(*buffer)); 483 ASSERT_NE(buffer, NULL); 484 485 buffer->fd = -1; 486 buffer->size = size; 487 buffer->mirror = malloc(size); 488 ASSERT_NE(buffer->mirror, NULL); 489 490 buffer->ptr = mmap(NULL, size, 491 PROT_READ, 492 MAP_PRIVATE | MAP_ANONYMOUS, 493 buffer->fd, 0); 494 ASSERT_NE(buffer->ptr, MAP_FAILED); 495 496 /* Simulate a device reading a zero page of memory. */ 497 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); 498 ASSERT_EQ(ret, 0); 499 ASSERT_EQ(buffer->cpages, 1); 500 ASSERT_EQ(buffer->faults, 1); 501 502 /* Initialize data that the device will write to buffer->ptr. */ 503 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 504 ptr[i] = i; 505 506 /* Simulate a device writing system memory. */ 507 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 508 ASSERT_EQ(ret, -EPERM); 509 510 /* Check what the device wrote. */ 511 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 512 ASSERT_EQ(ptr[i], 0); 513 514 /* Now allow writing and see that the zero page is replaced. */ 515 ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ); 516 ASSERT_EQ(ret, 0); 517 518 /* Simulate a device writing system memory. */ 519 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 520 ASSERT_EQ(ret, 0); 521 ASSERT_EQ(buffer->cpages, npages); 522 ASSERT_EQ(buffer->faults, 1); 523 524 /* Check what the device wrote. */ 525 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 526 ASSERT_EQ(ptr[i], i); 527 528 hmm_buffer_free(buffer); 529 } 530 531 /* 532 * Check that a device writing an anonymous private mapping 533 * will copy-on-write if a child process inherits the mapping. 534 * 535 * Also verifies after fork() memory the device can be read by child. 536 */ 537 TEST_F(hmm, anon_write_child) 538 { 539 struct hmm_buffer *buffer; 540 unsigned long npages; 541 unsigned long size; 542 unsigned long i; 543 void *old_ptr; 544 void *map; 545 int *ptr; 546 pid_t pid; 547 int child_fd; 548 int ret, use_thp, migrate; 549 550 for (migrate = 0; migrate < 2; ++migrate) { 551 for (use_thp = 0; use_thp < 2; ++use_thp) { 552 npages = ALIGN(use_thp ? read_pmd_pagesize() : HMM_BUFFER_SIZE, 553 self->page_size) >> self->page_shift; 554 ASSERT_NE(npages, 0); 555 size = npages << self->page_shift; 556 557 buffer = malloc(sizeof(*buffer)); 558 ASSERT_NE(buffer, NULL); 559 560 buffer->fd = -1; 561 buffer->size = size * 2; 562 buffer->mirror = malloc(size); 563 ASSERT_NE(buffer->mirror, NULL); 564 565 buffer->ptr = mmap(NULL, size * 2, 566 PROT_READ | PROT_WRITE, 567 MAP_PRIVATE | MAP_ANONYMOUS, 568 buffer->fd, 0); 569 ASSERT_NE(buffer->ptr, MAP_FAILED); 570 571 old_ptr = buffer->ptr; 572 if (use_thp) { 573 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 574 ret = madvise(map, size, MADV_HUGEPAGE); 575 ASSERT_EQ(ret, 0); 576 buffer->ptr = map; 577 } 578 579 /* Initialize buffer->ptr so we can tell if it is written. */ 580 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 581 ptr[i] = i; 582 583 /* Initialize data that the device will write to buffer->ptr. */ 584 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 585 ptr[i] = -i; 586 587 if (migrate) { 588 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 589 ASSERT_EQ(ret, 0); 590 ASSERT_EQ(buffer->cpages, npages); 591 592 } 593 594 pid = fork(); 595 if (pid == -1) 596 ASSERT_EQ(pid, 0); 597 if (pid != 0) { 598 waitpid(pid, &ret, 0); 599 ASSERT_EQ(WIFEXITED(ret), 1); 600 601 /* Check that the parent's buffer did not change. */ 602 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 603 ASSERT_EQ(ptr[i], i); 604 605 buffer->ptr = old_ptr; 606 hmm_buffer_free(buffer); 607 continue; 608 } 609 610 /* Check that we see the parent's values. */ 611 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 612 ASSERT_EQ(ptr[i], i); 613 if (!migrate) { 614 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 615 ASSERT_EQ(ptr[i], -i); 616 } 617 618 /* The child process needs its own mirror to its own mm. */ 619 child_fd = hmm_open(0); 620 ASSERT_GE(child_fd, 0); 621 622 /* Simulate a device writing system memory. */ 623 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); 624 ASSERT_EQ(ret, 0); 625 ASSERT_EQ(buffer->cpages, npages); 626 ASSERT_EQ(buffer->faults, 1); 627 628 /* Check what the device wrote. */ 629 if (!migrate) { 630 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 631 ASSERT_EQ(ptr[i], -i); 632 } 633 634 close(child_fd); 635 exit(0); 636 } 637 } 638 } 639 640 /* 641 * Check that a device writing an anonymous shared mapping 642 * will not copy-on-write if a child process inherits the mapping. 643 */ 644 TEST_F(hmm, anon_write_child_shared) 645 { 646 struct hmm_buffer *buffer; 647 unsigned long npages; 648 unsigned long size; 649 unsigned long i; 650 int *ptr; 651 pid_t pid; 652 int child_fd; 653 int ret; 654 655 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 656 ASSERT_NE(npages, 0); 657 size = npages << self->page_shift; 658 659 buffer = malloc(sizeof(*buffer)); 660 ASSERT_NE(buffer, NULL); 661 662 buffer->fd = -1; 663 buffer->size = size; 664 buffer->mirror = malloc(size); 665 ASSERT_NE(buffer->mirror, NULL); 666 667 buffer->ptr = mmap(NULL, size, 668 PROT_READ | PROT_WRITE, 669 MAP_SHARED | MAP_ANONYMOUS, 670 buffer->fd, 0); 671 ASSERT_NE(buffer->ptr, MAP_FAILED); 672 673 /* Initialize buffer->ptr so we can tell if it is written. */ 674 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 675 ptr[i] = i; 676 677 /* Initialize data that the device will write to buffer->ptr. */ 678 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 679 ptr[i] = -i; 680 681 pid = fork(); 682 if (pid == -1) 683 ASSERT_EQ(pid, 0); 684 if (pid != 0) { 685 waitpid(pid, &ret, 0); 686 ASSERT_EQ(WIFEXITED(ret), 1); 687 688 /* Check that the parent's buffer did change. */ 689 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 690 ASSERT_EQ(ptr[i], -i); 691 return; 692 } 693 694 /* Check that we see the parent's values. */ 695 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 696 ASSERT_EQ(ptr[i], i); 697 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 698 ASSERT_EQ(ptr[i], -i); 699 700 /* The child process needs its own mirror to its own mm. */ 701 child_fd = hmm_open(0); 702 ASSERT_GE(child_fd, 0); 703 704 /* Simulate a device writing system memory. */ 705 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); 706 ASSERT_EQ(ret, 0); 707 ASSERT_EQ(buffer->cpages, npages); 708 ASSERT_EQ(buffer->faults, 1); 709 710 /* Check what the device wrote. */ 711 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 712 ASSERT_EQ(ptr[i], -i); 713 714 close(child_fd); 715 exit(0); 716 } 717 718 /* 719 * Write private anonymous huge page. 720 */ 721 TEST_F(hmm, anon_write_huge) 722 { 723 struct hmm_buffer *buffer; 724 unsigned long npages; 725 unsigned long size; 726 unsigned long i; 727 void *old_ptr; 728 void *map; 729 int *ptr; 730 int ret; 731 732 size = 2 * read_pmd_pagesize(); 733 734 buffer = malloc(sizeof(*buffer)); 735 ASSERT_NE(buffer, NULL); 736 737 buffer->fd = -1; 738 buffer->size = size; 739 buffer->mirror = malloc(size); 740 ASSERT_NE(buffer->mirror, NULL); 741 742 buffer->ptr = mmap(NULL, size, 743 PROT_READ | PROT_WRITE, 744 MAP_PRIVATE | MAP_ANONYMOUS, 745 buffer->fd, 0); 746 ASSERT_NE(buffer->ptr, MAP_FAILED); 747 748 size /= 2; 749 npages = size >> self->page_shift; 750 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 751 ret = madvise(map, size, MADV_HUGEPAGE); 752 ASSERT_EQ(ret, 0); 753 old_ptr = buffer->ptr; 754 buffer->ptr = map; 755 756 /* Initialize data that the device will write to buffer->ptr. */ 757 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 758 ptr[i] = i; 759 760 /* Simulate a device writing system memory. */ 761 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 762 ASSERT_EQ(ret, 0); 763 ASSERT_EQ(buffer->cpages, npages); 764 ASSERT_EQ(buffer->faults, 1); 765 766 /* Check what the device wrote. */ 767 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 768 ASSERT_EQ(ptr[i], i); 769 770 buffer->ptr = old_ptr; 771 hmm_buffer_free(buffer); 772 } 773 774 /* 775 * Write huge TLBFS page. 776 */ 777 TEST_F(hmm, anon_write_hugetlbfs) 778 { 779 struct hmm_buffer *buffer; 780 unsigned long npages; 781 unsigned long size; 782 unsigned long default_hsize = default_huge_page_size(); 783 unsigned long i; 784 int *ptr; 785 int ret; 786 787 if (!default_hsize) 788 SKIP(return, "Huge page size could not be determined"); 789 790 size = ALIGN(TWOMEG, default_hsize); 791 npages = size >> self->page_shift; 792 793 buffer = malloc(sizeof(*buffer)); 794 ASSERT_NE(buffer, NULL); 795 796 buffer->ptr = mmap(NULL, size, 797 PROT_READ | PROT_WRITE, 798 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 799 -1, 0); 800 if (buffer->ptr == MAP_FAILED) { 801 free(buffer); 802 SKIP(return, "Huge page could not be allocated"); 803 } 804 805 buffer->fd = -1; 806 buffer->size = size; 807 buffer->mirror = malloc(size); 808 ASSERT_NE(buffer->mirror, NULL); 809 810 /* Initialize data that the device will write to buffer->ptr. */ 811 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 812 ptr[i] = i; 813 814 /* Simulate a device writing system memory. */ 815 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 816 ASSERT_EQ(ret, 0); 817 ASSERT_EQ(buffer->cpages, npages); 818 ASSERT_EQ(buffer->faults, 1); 819 820 /* Check what the device wrote. */ 821 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 822 ASSERT_EQ(ptr[i], i); 823 824 munmap(buffer->ptr, buffer->size); 825 buffer->ptr = NULL; 826 hmm_buffer_free(buffer); 827 } 828 829 /* 830 * Read mmap'ed file memory. 831 */ 832 TEST_F(hmm, file_read) 833 { 834 struct hmm_buffer *buffer; 835 unsigned long npages; 836 unsigned long size; 837 unsigned long i; 838 int *ptr; 839 int ret; 840 int fd; 841 ssize_t len; 842 843 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 844 ASSERT_NE(npages, 0); 845 size = npages << self->page_shift; 846 847 fd = hmm_create_file(size); 848 ASSERT_GE(fd, 0); 849 850 buffer = malloc(sizeof(*buffer)); 851 ASSERT_NE(buffer, NULL); 852 853 buffer->fd = fd; 854 buffer->size = size; 855 buffer->mirror = malloc(size); 856 ASSERT_NE(buffer->mirror, NULL); 857 858 /* Write initial contents of the file. */ 859 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 860 ptr[i] = i; 861 len = pwrite(fd, buffer->mirror, size, 0); 862 ASSERT_EQ(len, size); 863 memset(buffer->mirror, 0, size); 864 865 buffer->ptr = mmap(NULL, size, 866 PROT_READ, 867 MAP_SHARED, 868 buffer->fd, 0); 869 ASSERT_NE(buffer->ptr, MAP_FAILED); 870 871 /* Simulate a device reading system memory. */ 872 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 873 ASSERT_EQ(ret, 0); 874 ASSERT_EQ(buffer->cpages, npages); 875 ASSERT_EQ(buffer->faults, 1); 876 877 /* Check what the device read. */ 878 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 879 ASSERT_EQ(ptr[i], i); 880 881 hmm_buffer_free(buffer); 882 } 883 884 /* 885 * Write mmap'ed file memory. 886 */ 887 TEST_F(hmm, file_write) 888 { 889 struct hmm_buffer *buffer; 890 unsigned long npages; 891 unsigned long size; 892 unsigned long i; 893 int *ptr; 894 int ret; 895 int fd; 896 ssize_t len; 897 898 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 899 ASSERT_NE(npages, 0); 900 size = npages << self->page_shift; 901 902 fd = hmm_create_file(size); 903 ASSERT_GE(fd, 0); 904 905 buffer = malloc(sizeof(*buffer)); 906 ASSERT_NE(buffer, NULL); 907 908 buffer->fd = fd; 909 buffer->size = size; 910 buffer->mirror = malloc(size); 911 ASSERT_NE(buffer->mirror, NULL); 912 913 buffer->ptr = mmap(NULL, size, 914 PROT_READ | PROT_WRITE, 915 MAP_SHARED, 916 buffer->fd, 0); 917 ASSERT_NE(buffer->ptr, MAP_FAILED); 918 919 /* Initialize data that the device will write to buffer->ptr. */ 920 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 921 ptr[i] = i; 922 923 /* Simulate a device writing system memory. */ 924 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 925 ASSERT_EQ(ret, 0); 926 ASSERT_EQ(buffer->cpages, npages); 927 ASSERT_EQ(buffer->faults, 1); 928 929 /* Check what the device wrote. */ 930 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 931 ASSERT_EQ(ptr[i], i); 932 933 /* Check that the device also wrote the file. */ 934 len = pread(fd, buffer->mirror, size, 0); 935 ASSERT_EQ(len, size); 936 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 937 ASSERT_EQ(ptr[i], i); 938 939 hmm_buffer_free(buffer); 940 } 941 942 /* 943 * Migrate anonymous memory to device private memory. 944 */ 945 TEST_F(hmm, migrate) 946 { 947 struct hmm_buffer *buffer; 948 unsigned long npages; 949 unsigned long size; 950 unsigned long i; 951 int *ptr; 952 int ret; 953 954 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 955 ASSERT_NE(npages, 0); 956 size = npages << self->page_shift; 957 958 buffer = malloc(sizeof(*buffer)); 959 ASSERT_NE(buffer, NULL); 960 961 buffer->fd = -1; 962 buffer->size = size; 963 buffer->mirror = malloc(size); 964 ASSERT_NE(buffer->mirror, NULL); 965 966 buffer->ptr = mmap(NULL, size, 967 PROT_READ | PROT_WRITE, 968 MAP_PRIVATE | MAP_ANONYMOUS, 969 buffer->fd, 0); 970 ASSERT_NE(buffer->ptr, MAP_FAILED); 971 972 /* Initialize buffer in system memory. */ 973 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 974 ptr[i] = i; 975 976 /* Migrate memory to device. */ 977 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 978 ASSERT_EQ(ret, 0); 979 ASSERT_EQ(buffer->cpages, npages); 980 981 /* Check what the device read. */ 982 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 983 ASSERT_EQ(ptr[i], i); 984 985 hmm_buffer_free(buffer); 986 } 987 988 /* 989 * Migrate private file memory to device private memory. 990 */ 991 TEST_F(hmm, migrate_file_private) 992 { 993 struct hmm_buffer *buffer; 994 unsigned long npages; 995 unsigned long size; 996 unsigned long i; 997 int *ptr; 998 int ret; 999 int fd; 1000 1001 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1002 ASSERT_NE(npages, 0); 1003 size = npages << self->page_shift; 1004 1005 fd = hmm_create_file(size); 1006 ASSERT_GE(fd, 0); 1007 1008 buffer = malloc(sizeof(*buffer)); 1009 ASSERT_NE(buffer, NULL); 1010 1011 buffer->fd = fd; 1012 buffer->size = size; 1013 buffer->mirror = malloc(size); 1014 ASSERT_NE(buffer->mirror, NULL); 1015 1016 buffer->ptr = mmap(NULL, size, 1017 PROT_READ | PROT_WRITE, 1018 MAP_PRIVATE, 1019 buffer->fd, 0); 1020 ASSERT_NE(buffer->ptr, MAP_FAILED); 1021 1022 /* Initialize buffer in system memory. */ 1023 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1024 ptr[i] = i; 1025 1026 /* Migrate memory to device. */ 1027 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1028 ASSERT_EQ(ret, 0); 1029 ASSERT_EQ(buffer->cpages, npages); 1030 1031 /* Check what the device read. */ 1032 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1033 ASSERT_EQ(ptr[i], i); 1034 1035 hmm_buffer_free(buffer); 1036 } 1037 1038 /* 1039 * Migrate anonymous memory to device private memory and fault some of it back 1040 * to system memory, then try migrating the resulting mix of system and device 1041 * private memory to the device. 1042 */ 1043 TEST_F(hmm, migrate_fault) 1044 { 1045 struct hmm_buffer *buffer; 1046 unsigned long npages; 1047 unsigned long size; 1048 unsigned long i; 1049 int *ptr; 1050 int ret; 1051 1052 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1053 ASSERT_NE(npages, 0); 1054 size = npages << self->page_shift; 1055 1056 buffer = malloc(sizeof(*buffer)); 1057 ASSERT_NE(buffer, NULL); 1058 1059 buffer->fd = -1; 1060 buffer->size = size; 1061 buffer->mirror = malloc(size); 1062 ASSERT_NE(buffer->mirror, NULL); 1063 1064 buffer->ptr = mmap(NULL, size, 1065 PROT_READ | PROT_WRITE, 1066 MAP_PRIVATE | MAP_ANONYMOUS, 1067 buffer->fd, 0); 1068 ASSERT_NE(buffer->ptr, MAP_FAILED); 1069 1070 /* Initialize buffer in system memory. */ 1071 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1072 ptr[i] = i; 1073 1074 /* Migrate memory to device. */ 1075 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1076 ASSERT_EQ(ret, 0); 1077 ASSERT_EQ(buffer->cpages, npages); 1078 1079 /* Check what the device read. */ 1080 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1081 ASSERT_EQ(ptr[i], i); 1082 1083 /* Fault half the pages back to system memory and check them. */ 1084 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) 1085 ASSERT_EQ(ptr[i], i); 1086 1087 /* Migrate memory to the device again. */ 1088 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1089 ASSERT_EQ(ret, 0); 1090 ASSERT_EQ(buffer->cpages, npages); 1091 1092 /* Check what the device read. */ 1093 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1094 ASSERT_EQ(ptr[i], i); 1095 1096 hmm_buffer_free(buffer); 1097 } 1098 1099 TEST_F(hmm, migrate_release) 1100 { 1101 struct hmm_buffer *buffer; 1102 unsigned long npages; 1103 unsigned long size; 1104 unsigned long i; 1105 int *ptr; 1106 int ret; 1107 1108 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1109 ASSERT_NE(npages, 0); 1110 size = npages << self->page_shift; 1111 1112 buffer = malloc(sizeof(*buffer)); 1113 ASSERT_NE(buffer, NULL); 1114 1115 buffer->fd = -1; 1116 buffer->size = size; 1117 buffer->mirror = malloc(size); 1118 ASSERT_NE(buffer->mirror, NULL); 1119 1120 buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 1121 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 1122 ASSERT_NE(buffer->ptr, MAP_FAILED); 1123 1124 /* Initialize buffer in system memory. */ 1125 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1126 ptr[i] = i; 1127 1128 /* Migrate memory to device. */ 1129 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1130 ASSERT_EQ(ret, 0); 1131 ASSERT_EQ(buffer->cpages, npages); 1132 1133 /* Check what the device read. */ 1134 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1135 ASSERT_EQ(ptr[i], i); 1136 1137 /* Release device memory. */ 1138 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages); 1139 ASSERT_EQ(ret, 0); 1140 1141 /* Fault pages back to system memory and check them. */ 1142 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) 1143 ASSERT_EQ(ptr[i], i); 1144 1145 hmm_buffer_free(buffer); 1146 } 1147 1148 /* 1149 * Migrate anonymous shared memory to device private memory. 1150 */ 1151 TEST_F(hmm, migrate_shared) 1152 { 1153 struct hmm_buffer *buffer; 1154 unsigned long npages; 1155 unsigned long size; 1156 int ret; 1157 1158 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1159 ASSERT_NE(npages, 0); 1160 size = npages << self->page_shift; 1161 1162 buffer = malloc(sizeof(*buffer)); 1163 ASSERT_NE(buffer, NULL); 1164 1165 buffer->fd = -1; 1166 buffer->size = size; 1167 buffer->mirror = malloc(size); 1168 ASSERT_NE(buffer->mirror, NULL); 1169 1170 buffer->ptr = mmap(NULL, size, 1171 PROT_READ | PROT_WRITE, 1172 MAP_SHARED | MAP_ANONYMOUS, 1173 buffer->fd, 0); 1174 ASSERT_NE(buffer->ptr, MAP_FAILED); 1175 1176 /* Migrate memory to device. */ 1177 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1178 ASSERT_EQ(ret, -ENOENT); 1179 1180 hmm_buffer_free(buffer); 1181 } 1182 1183 /* 1184 * Try to migrate various memory types to device private memory. 1185 */ 1186 TEST_F(hmm2, migrate_mixed) 1187 { 1188 struct hmm_buffer *buffer; 1189 unsigned long npages; 1190 unsigned long size; 1191 int *ptr; 1192 unsigned char *p; 1193 int ret; 1194 int val; 1195 1196 npages = 6; 1197 size = npages << self->page_shift; 1198 1199 buffer = malloc(sizeof(*buffer)); 1200 ASSERT_NE(buffer, NULL); 1201 1202 buffer->fd = -1; 1203 buffer->size = size; 1204 buffer->mirror = malloc(size); 1205 ASSERT_NE(buffer->mirror, NULL); 1206 1207 /* Reserve a range of addresses. */ 1208 buffer->ptr = mmap(NULL, size, 1209 PROT_NONE, 1210 MAP_PRIVATE | MAP_ANONYMOUS, 1211 buffer->fd, 0); 1212 ASSERT_NE(buffer->ptr, MAP_FAILED); 1213 p = buffer->ptr; 1214 1215 /* Migrating a protected area should be an error. */ 1216 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); 1217 ASSERT_EQ(ret, -EINVAL); 1218 1219 /* Punch a hole after the first page address. */ 1220 ret = munmap(buffer->ptr + self->page_size, self->page_size); 1221 ASSERT_EQ(ret, 0); 1222 1223 /* We expect an error if the vma doesn't cover the range. */ 1224 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3); 1225 ASSERT_EQ(ret, -EINVAL); 1226 1227 /* Page 2 will be a read-only zero page. */ 1228 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, 1229 PROT_READ); 1230 ASSERT_EQ(ret, 0); 1231 ptr = (int *)(buffer->ptr + 2 * self->page_size); 1232 val = *ptr + 3; 1233 ASSERT_EQ(val, 3); 1234 1235 /* Page 3 will be read-only. */ 1236 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1237 PROT_READ | PROT_WRITE); 1238 ASSERT_EQ(ret, 0); 1239 ptr = (int *)(buffer->ptr + 3 * self->page_size); 1240 *ptr = val; 1241 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1242 PROT_READ); 1243 ASSERT_EQ(ret, 0); 1244 1245 /* Page 4-5 will be read-write. */ 1246 ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size, 1247 PROT_READ | PROT_WRITE); 1248 ASSERT_EQ(ret, 0); 1249 ptr = (int *)(buffer->ptr + 4 * self->page_size); 1250 *ptr = val; 1251 ptr = (int *)(buffer->ptr + 5 * self->page_size); 1252 *ptr = val; 1253 1254 /* Now try to migrate pages 2-5 to device 1. */ 1255 buffer->ptr = p + 2 * self->page_size; 1256 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4); 1257 ASSERT_EQ(ret, 0); 1258 ASSERT_EQ(buffer->cpages, 4); 1259 1260 /* Page 5 won't be migrated to device 0 because it's on device 1. */ 1261 buffer->ptr = p + 5 * self->page_size; 1262 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); 1263 ASSERT_EQ(ret, -ENOENT); 1264 buffer->ptr = p; 1265 1266 buffer->ptr = p; 1267 hmm_buffer_free(buffer); 1268 } 1269 1270 /* 1271 * Migrate anonymous memory to device memory and back to system memory 1272 * multiple times. In case of private zone configuration, this is done 1273 * through fault pages accessed by CPU. In case of coherent zone configuration, 1274 * the pages from the device should be explicitly migrated back to system memory. 1275 * The reason is Coherent device zone has coherent access by CPU, therefore 1276 * it will not generate any page fault. 1277 */ 1278 TEST_F(hmm, migrate_multiple) 1279 { 1280 struct hmm_buffer *buffer; 1281 unsigned long npages; 1282 unsigned long size; 1283 unsigned long i; 1284 unsigned long c; 1285 int *ptr; 1286 int ret; 1287 1288 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1289 ASSERT_NE(npages, 0); 1290 size = npages << self->page_shift; 1291 1292 for (c = 0; c < NTIMES; c++) { 1293 buffer = malloc(sizeof(*buffer)); 1294 ASSERT_NE(buffer, NULL); 1295 1296 buffer->fd = -1; 1297 buffer->size = size; 1298 buffer->mirror = malloc(size); 1299 ASSERT_NE(buffer->mirror, NULL); 1300 1301 buffer->ptr = mmap(NULL, size, 1302 PROT_READ | PROT_WRITE, 1303 MAP_PRIVATE | MAP_ANONYMOUS, 1304 buffer->fd, 0); 1305 ASSERT_NE(buffer->ptr, MAP_FAILED); 1306 1307 /* Initialize buffer in system memory. */ 1308 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1309 ptr[i] = i; 1310 1311 /* Migrate memory to device. */ 1312 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1313 ASSERT_EQ(ret, 0); 1314 ASSERT_EQ(buffer->cpages, npages); 1315 1316 /* Check what the device read. */ 1317 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1318 ASSERT_EQ(ptr[i], i); 1319 1320 /* Migrate back to system memory and check them. */ 1321 if (hmm_is_coherent_type(variant->device_number)) { 1322 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages); 1323 ASSERT_EQ(ret, 0); 1324 ASSERT_EQ(buffer->cpages, npages); 1325 } 1326 1327 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1328 ASSERT_EQ(ptr[i], i); 1329 1330 hmm_buffer_free(buffer); 1331 } 1332 } 1333 1334 /* 1335 * Read anonymous memory multiple times. 1336 */ 1337 TEST_F(hmm, anon_read_multiple) 1338 { 1339 struct hmm_buffer *buffer; 1340 unsigned long npages; 1341 unsigned long size; 1342 unsigned long i; 1343 unsigned long c; 1344 int *ptr; 1345 int ret; 1346 1347 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1348 ASSERT_NE(npages, 0); 1349 size = npages << self->page_shift; 1350 1351 for (c = 0; c < NTIMES; c++) { 1352 buffer = malloc(sizeof(*buffer)); 1353 ASSERT_NE(buffer, NULL); 1354 1355 buffer->fd = -1; 1356 buffer->size = size; 1357 buffer->mirror = malloc(size); 1358 ASSERT_NE(buffer->mirror, NULL); 1359 1360 buffer->ptr = mmap(NULL, size, 1361 PROT_READ | PROT_WRITE, 1362 MAP_PRIVATE | MAP_ANONYMOUS, 1363 buffer->fd, 0); 1364 ASSERT_NE(buffer->ptr, MAP_FAILED); 1365 1366 /* Initialize buffer in system memory. */ 1367 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1368 ptr[i] = i + c; 1369 1370 /* Simulate a device reading system memory. */ 1371 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1372 npages); 1373 ASSERT_EQ(ret, 0); 1374 ASSERT_EQ(buffer->cpages, npages); 1375 ASSERT_EQ(buffer->faults, 1); 1376 1377 /* Check what the device read. */ 1378 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1379 ASSERT_EQ(ptr[i], i + c); 1380 1381 hmm_buffer_free(buffer); 1382 } 1383 } 1384 1385 void *unmap_buffer(void *p) 1386 { 1387 struct hmm_buffer *buffer = p; 1388 1389 /* Delay for a bit and then unmap buffer while it is being read. */ 1390 hmm_nanosleep(hmm_random() % 32000); 1391 munmap(buffer->ptr + buffer->size / 2, buffer->size / 2); 1392 buffer->ptr = NULL; 1393 1394 return NULL; 1395 } 1396 1397 /* 1398 * Try reading anonymous memory while it is being unmapped. 1399 */ 1400 TEST_F(hmm, anon_teardown) 1401 { 1402 unsigned long npages; 1403 unsigned long size; 1404 unsigned long c; 1405 void *ret; 1406 1407 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1408 ASSERT_NE(npages, 0); 1409 size = npages << self->page_shift; 1410 1411 for (c = 0; c < NTIMES; ++c) { 1412 pthread_t thread; 1413 struct hmm_buffer *buffer; 1414 unsigned long i; 1415 int *ptr; 1416 int rc; 1417 1418 buffer = malloc(sizeof(*buffer)); 1419 ASSERT_NE(buffer, NULL); 1420 1421 buffer->fd = -1; 1422 buffer->size = size; 1423 buffer->mirror = malloc(size); 1424 ASSERT_NE(buffer->mirror, NULL); 1425 1426 buffer->ptr = mmap(NULL, size, 1427 PROT_READ | PROT_WRITE, 1428 MAP_PRIVATE | MAP_ANONYMOUS, 1429 buffer->fd, 0); 1430 ASSERT_NE(buffer->ptr, MAP_FAILED); 1431 1432 /* Initialize buffer in system memory. */ 1433 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1434 ptr[i] = i + c; 1435 1436 rc = pthread_create(&thread, NULL, unmap_buffer, buffer); 1437 ASSERT_EQ(rc, 0); 1438 1439 /* Simulate a device reading system memory. */ 1440 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1441 npages); 1442 if (rc == 0) { 1443 ASSERT_EQ(buffer->cpages, npages); 1444 ASSERT_EQ(buffer->faults, 1); 1445 1446 /* Check what the device read. */ 1447 for (i = 0, ptr = buffer->mirror; 1448 i < size / sizeof(*ptr); 1449 ++i) 1450 ASSERT_EQ(ptr[i], i + c); 1451 } 1452 1453 pthread_join(thread, &ret); 1454 hmm_buffer_free(buffer); 1455 } 1456 } 1457 1458 /* 1459 * Test memory snapshot without faulting in pages accessed by the device. 1460 */ 1461 TEST_F(hmm, mixedmap) 1462 { 1463 struct hmm_buffer *buffer; 1464 unsigned long npages; 1465 unsigned long size; 1466 unsigned char *m; 1467 int ret; 1468 1469 npages = 1; 1470 size = npages << self->page_shift; 1471 1472 buffer = malloc(sizeof(*buffer)); 1473 ASSERT_NE(buffer, NULL); 1474 1475 buffer->fd = -1; 1476 buffer->size = size; 1477 buffer->mirror = malloc(npages); 1478 ASSERT_NE(buffer->mirror, NULL); 1479 1480 1481 /* Reserve a range of addresses. */ 1482 buffer->ptr = mmap(NULL, size, 1483 PROT_READ | PROT_WRITE, 1484 MAP_PRIVATE, 1485 self->fd, 0); 1486 ASSERT_NE(buffer->ptr, MAP_FAILED); 1487 1488 /* Simulate a device snapshotting CPU pagetables. */ 1489 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1490 ASSERT_EQ(ret, 0); 1491 ASSERT_EQ(buffer->cpages, npages); 1492 1493 /* Check what the device saw. */ 1494 m = buffer->mirror; 1495 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ); 1496 1497 hmm_buffer_free(buffer); 1498 } 1499 1500 /* 1501 * Test memory snapshot without faulting in pages accessed by the device. 1502 */ 1503 TEST_F(hmm2, snapshot) 1504 { 1505 struct hmm_buffer *buffer; 1506 unsigned long npages; 1507 unsigned long size; 1508 int *ptr; 1509 unsigned char *p; 1510 unsigned char *m; 1511 int ret; 1512 int val; 1513 1514 npages = 7; 1515 size = npages << self->page_shift; 1516 1517 buffer = malloc(sizeof(*buffer)); 1518 ASSERT_NE(buffer, NULL); 1519 1520 buffer->fd = -1; 1521 buffer->size = size; 1522 buffer->mirror = malloc(npages); 1523 ASSERT_NE(buffer->mirror, NULL); 1524 1525 /* Reserve a range of addresses. */ 1526 buffer->ptr = mmap(NULL, size, 1527 PROT_NONE, 1528 MAP_PRIVATE | MAP_ANONYMOUS, 1529 buffer->fd, 0); 1530 ASSERT_NE(buffer->ptr, MAP_FAILED); 1531 p = buffer->ptr; 1532 1533 /* Punch a hole after the first page address. */ 1534 ret = munmap(buffer->ptr + self->page_size, self->page_size); 1535 ASSERT_EQ(ret, 0); 1536 1537 /* Page 2 will be read-only zero page. */ 1538 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, 1539 PROT_READ); 1540 ASSERT_EQ(ret, 0); 1541 ptr = (int *)(buffer->ptr + 2 * self->page_size); 1542 val = *ptr + 3; 1543 ASSERT_EQ(val, 3); 1544 1545 /* Page 3 will be read-only. */ 1546 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1547 PROT_READ | PROT_WRITE); 1548 ASSERT_EQ(ret, 0); 1549 ptr = (int *)(buffer->ptr + 3 * self->page_size); 1550 *ptr = val; 1551 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1552 PROT_READ); 1553 ASSERT_EQ(ret, 0); 1554 1555 /* Page 4-6 will be read-write. */ 1556 ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size, 1557 PROT_READ | PROT_WRITE); 1558 ASSERT_EQ(ret, 0); 1559 ptr = (int *)(buffer->ptr + 4 * self->page_size); 1560 *ptr = val; 1561 1562 /* Page 5 will be migrated to device 0. */ 1563 buffer->ptr = p + 5 * self->page_size; 1564 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); 1565 ASSERT_EQ(ret, 0); 1566 ASSERT_EQ(buffer->cpages, 1); 1567 1568 /* Page 6 will be migrated to device 1. */ 1569 buffer->ptr = p + 6 * self->page_size; 1570 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1); 1571 ASSERT_EQ(ret, 0); 1572 ASSERT_EQ(buffer->cpages, 1); 1573 1574 /* Simulate a device snapshotting CPU pagetables. */ 1575 buffer->ptr = p; 1576 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1577 ASSERT_EQ(ret, 0); 1578 ASSERT_EQ(buffer->cpages, npages); 1579 1580 /* Check what the device saw. */ 1581 m = buffer->mirror; 1582 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR); 1583 ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR); 1584 ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ); 1585 ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ); 1586 ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE); 1587 if (!hmm_is_coherent_type(variant->device_number0)) { 1588 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL | 1589 HMM_DMIRROR_PROT_WRITE); 1590 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE); 1591 } else { 1592 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | 1593 HMM_DMIRROR_PROT_WRITE); 1594 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE | 1595 HMM_DMIRROR_PROT_WRITE); 1596 } 1597 1598 hmm_buffer_free(buffer); 1599 } 1600 1601 /* 1602 * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that 1603 * should be mapped by a large page table entry. 1604 */ 1605 TEST_F(hmm, compound) 1606 { 1607 struct hmm_buffer *buffer; 1608 unsigned long npages; 1609 unsigned long size; 1610 unsigned long default_hsize = default_huge_page_size(); 1611 int *ptr; 1612 unsigned char *m; 1613 int ret; 1614 unsigned long i; 1615 1616 /* Skip test if we can't allocate a hugetlbfs page. */ 1617 1618 if (!default_hsize) 1619 SKIP(return, "Huge page size could not be determined"); 1620 1621 size = ALIGN(TWOMEG, default_hsize); 1622 npages = size >> self->page_shift; 1623 1624 buffer = malloc(sizeof(*buffer)); 1625 ASSERT_NE(buffer, NULL); 1626 1627 buffer->ptr = mmap(NULL, size, 1628 PROT_READ | PROT_WRITE, 1629 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 1630 -1, 0); 1631 if (buffer->ptr == MAP_FAILED) { 1632 free(buffer); 1633 return; 1634 } 1635 1636 buffer->size = size; 1637 buffer->mirror = malloc(npages); 1638 ASSERT_NE(buffer->mirror, NULL); 1639 1640 /* Initialize the pages the device will snapshot in buffer->ptr. */ 1641 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1642 ptr[i] = i; 1643 1644 /* Simulate a device snapshotting CPU pagetables. */ 1645 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1646 ASSERT_EQ(ret, 0); 1647 ASSERT_EQ(buffer->cpages, npages); 1648 1649 /* Check what the device saw. */ 1650 m = buffer->mirror; 1651 for (i = 0; i < npages; ++i) 1652 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE | 1653 HMM_DMIRROR_PROT_PMD); 1654 1655 /* Make the region read-only. */ 1656 ret = mprotect(buffer->ptr, size, PROT_READ); 1657 ASSERT_EQ(ret, 0); 1658 1659 /* Simulate a device snapshotting CPU pagetables. */ 1660 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1661 ASSERT_EQ(ret, 0); 1662 ASSERT_EQ(buffer->cpages, npages); 1663 1664 /* Check what the device saw. */ 1665 m = buffer->mirror; 1666 for (i = 0; i < npages; ++i) 1667 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ | 1668 HMM_DMIRROR_PROT_PMD); 1669 1670 munmap(buffer->ptr, buffer->size); 1671 buffer->ptr = NULL; 1672 hmm_buffer_free(buffer); 1673 } 1674 1675 /* 1676 * Test two devices reading the same memory (double mapped). 1677 */ 1678 TEST_F(hmm2, double_map) 1679 { 1680 struct hmm_buffer *buffer; 1681 unsigned long npages; 1682 unsigned long size; 1683 unsigned long i; 1684 int *ptr; 1685 int ret; 1686 1687 npages = 6; 1688 size = npages << self->page_shift; 1689 1690 buffer = malloc(sizeof(*buffer)); 1691 ASSERT_NE(buffer, NULL); 1692 1693 buffer->fd = -1; 1694 buffer->size = size; 1695 buffer->mirror = malloc(size); 1696 ASSERT_NE(buffer->mirror, NULL); 1697 1698 /* Reserve a range of addresses. */ 1699 buffer->ptr = mmap(NULL, size, 1700 PROT_READ | PROT_WRITE, 1701 MAP_PRIVATE | MAP_ANONYMOUS, 1702 buffer->fd, 0); 1703 ASSERT_NE(buffer->ptr, MAP_FAILED); 1704 1705 /* Initialize buffer in system memory. */ 1706 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1707 ptr[i] = i; 1708 1709 /* Make region read-only. */ 1710 ret = mprotect(buffer->ptr, size, PROT_READ); 1711 ASSERT_EQ(ret, 0); 1712 1713 /* Simulate device 0 reading system memory. */ 1714 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); 1715 ASSERT_EQ(ret, 0); 1716 ASSERT_EQ(buffer->cpages, npages); 1717 ASSERT_EQ(buffer->faults, 1); 1718 1719 /* Check what the device read. */ 1720 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1721 ASSERT_EQ(ptr[i], i); 1722 1723 /* Simulate device 1 reading system memory. */ 1724 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages); 1725 ASSERT_EQ(ret, 0); 1726 ASSERT_EQ(buffer->cpages, npages); 1727 ASSERT_EQ(buffer->faults, 1); 1728 1729 /* Check what the device read. */ 1730 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1731 ASSERT_EQ(ptr[i], i); 1732 1733 /* Migrate pages to device 1 and try to read from device 0. */ 1734 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); 1735 ASSERT_EQ(ret, 0); 1736 ASSERT_EQ(buffer->cpages, npages); 1737 1738 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); 1739 ASSERT_EQ(ret, 0); 1740 ASSERT_EQ(buffer->cpages, npages); 1741 ASSERT_EQ(buffer->faults, 1); 1742 1743 /* Check what device 0 read. */ 1744 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1745 ASSERT_EQ(ptr[i], i); 1746 1747 hmm_buffer_free(buffer); 1748 } 1749 1750 /* 1751 * Basic check of exclusive faulting. 1752 */ 1753 TEST_F(hmm, exclusive) 1754 { 1755 struct hmm_buffer *buffer; 1756 unsigned long npages; 1757 unsigned long size; 1758 unsigned long i; 1759 int *ptr; 1760 int ret; 1761 1762 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1763 ASSERT_NE(npages, 0); 1764 size = npages << self->page_shift; 1765 1766 buffer = malloc(sizeof(*buffer)); 1767 ASSERT_NE(buffer, NULL); 1768 1769 buffer->fd = -1; 1770 buffer->size = size; 1771 buffer->mirror = malloc(size); 1772 ASSERT_NE(buffer->mirror, NULL); 1773 1774 buffer->ptr = mmap(NULL, size, 1775 PROT_READ | PROT_WRITE, 1776 MAP_PRIVATE | MAP_ANONYMOUS, 1777 buffer->fd, 0); 1778 ASSERT_NE(buffer->ptr, MAP_FAILED); 1779 1780 /* Initialize buffer in system memory. */ 1781 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1782 ptr[i] = i; 1783 1784 /* Map memory exclusively for device access. */ 1785 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1786 ASSERT_EQ(ret, 0); 1787 ASSERT_EQ(buffer->cpages, npages); 1788 1789 /* Check what the device read. */ 1790 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1791 ASSERT_EQ(ptr[i], i); 1792 1793 /* Fault pages back to system memory and check them. */ 1794 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1795 ASSERT_EQ(ptr[i]++, i); 1796 1797 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1798 ASSERT_EQ(ptr[i], i+1); 1799 1800 /* Check atomic access revoked */ 1801 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages); 1802 ASSERT_EQ(ret, 0); 1803 1804 hmm_buffer_free(buffer); 1805 } 1806 1807 TEST_F(hmm, exclusive_mprotect) 1808 { 1809 struct hmm_buffer *buffer; 1810 unsigned long npages; 1811 unsigned long size; 1812 unsigned long i; 1813 int *ptr; 1814 int ret; 1815 1816 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1817 ASSERT_NE(npages, 0); 1818 size = npages << self->page_shift; 1819 1820 buffer = malloc(sizeof(*buffer)); 1821 ASSERT_NE(buffer, NULL); 1822 1823 buffer->fd = -1; 1824 buffer->size = size; 1825 buffer->mirror = malloc(size); 1826 ASSERT_NE(buffer->mirror, NULL); 1827 1828 buffer->ptr = mmap(NULL, size, 1829 PROT_READ | PROT_WRITE, 1830 MAP_PRIVATE | MAP_ANONYMOUS, 1831 buffer->fd, 0); 1832 ASSERT_NE(buffer->ptr, MAP_FAILED); 1833 1834 /* Initialize buffer in system memory. */ 1835 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1836 ptr[i] = i; 1837 1838 /* Map memory exclusively for device access. */ 1839 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1840 ASSERT_EQ(ret, 0); 1841 ASSERT_EQ(buffer->cpages, npages); 1842 1843 /* Check what the device read. */ 1844 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1845 ASSERT_EQ(ptr[i], i); 1846 1847 ret = mprotect(buffer->ptr, size, PROT_READ); 1848 ASSERT_EQ(ret, 0); 1849 1850 /* Simulate a device writing system memory. */ 1851 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 1852 ASSERT_EQ(ret, -EPERM); 1853 1854 hmm_buffer_free(buffer); 1855 } 1856 1857 /* 1858 * Check copy-on-write works. 1859 */ 1860 TEST_F(hmm, exclusive_cow) 1861 { 1862 struct hmm_buffer *buffer; 1863 unsigned long npages; 1864 unsigned long size; 1865 unsigned long i; 1866 int *ptr; 1867 int ret; 1868 1869 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1870 ASSERT_NE(npages, 0); 1871 size = npages << self->page_shift; 1872 1873 buffer = malloc(sizeof(*buffer)); 1874 ASSERT_NE(buffer, NULL); 1875 1876 buffer->fd = -1; 1877 buffer->size = size; 1878 buffer->mirror = malloc(size); 1879 ASSERT_NE(buffer->mirror, NULL); 1880 1881 buffer->ptr = mmap(NULL, size, 1882 PROT_READ | PROT_WRITE, 1883 MAP_PRIVATE | MAP_ANONYMOUS, 1884 buffer->fd, 0); 1885 ASSERT_NE(buffer->ptr, MAP_FAILED); 1886 1887 /* Initialize buffer in system memory. */ 1888 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1889 ptr[i] = i; 1890 1891 /* Map memory exclusively for device access. */ 1892 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1893 ASSERT_EQ(ret, 0); 1894 ASSERT_EQ(buffer->cpages, npages); 1895 1896 fork(); 1897 1898 /* Fault pages back to system memory and check them. */ 1899 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1900 ASSERT_EQ(ptr[i]++, i); 1901 1902 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1903 ASSERT_EQ(ptr[i], i+1); 1904 1905 hmm_buffer_free(buffer); 1906 } 1907 1908 static int gup_test_exec(int gup_fd, unsigned long addr, int cmd, 1909 int npages, int size, int flags) 1910 { 1911 struct gup_test gup = { 1912 .nr_pages_per_call = npages, 1913 .addr = addr, 1914 .gup_flags = FOLL_WRITE | flags, 1915 .size = size, 1916 }; 1917 1918 if (ioctl(gup_fd, cmd, &gup)) { 1919 perror("ioctl on error\n"); 1920 return errno; 1921 } 1922 1923 return 0; 1924 } 1925 1926 /* 1927 * Test get user device pages through gup_test. Setting PIN_LONGTERM flag. 1928 * This should trigger a migration back to system memory for both, private 1929 * and coherent type pages. 1930 * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added 1931 * to your configuration before you run it. 1932 */ 1933 TEST_F(hmm, hmm_gup_test) 1934 { 1935 struct hmm_buffer *buffer; 1936 int gup_fd; 1937 unsigned long npages; 1938 unsigned long size; 1939 unsigned long i; 1940 int *ptr; 1941 int ret; 1942 unsigned char *m; 1943 1944 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); 1945 if (gup_fd == -1) 1946 SKIP(return, "Skipping test, could not find gup_test driver"); 1947 1948 npages = 4; 1949 size = npages << self->page_shift; 1950 1951 buffer = malloc(sizeof(*buffer)); 1952 ASSERT_NE(buffer, NULL); 1953 1954 buffer->fd = -1; 1955 buffer->size = size; 1956 buffer->mirror = malloc(size); 1957 ASSERT_NE(buffer->mirror, NULL); 1958 1959 buffer->ptr = mmap(NULL, size, 1960 PROT_READ | PROT_WRITE, 1961 MAP_PRIVATE | MAP_ANONYMOUS, 1962 buffer->fd, 0); 1963 ASSERT_NE(buffer->ptr, MAP_FAILED); 1964 1965 /* Initialize buffer in system memory. */ 1966 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1967 ptr[i] = i; 1968 1969 /* Migrate memory to device. */ 1970 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1971 ASSERT_EQ(ret, 0); 1972 ASSERT_EQ(buffer->cpages, npages); 1973 /* Check what the device read. */ 1974 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1975 ASSERT_EQ(ptr[i], i); 1976 1977 ASSERT_EQ(gup_test_exec(gup_fd, 1978 (unsigned long)buffer->ptr, 1979 GUP_BASIC_TEST, 1, self->page_size, 0), 0); 1980 ASSERT_EQ(gup_test_exec(gup_fd, 1981 (unsigned long)buffer->ptr + 1 * self->page_size, 1982 GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0); 1983 ASSERT_EQ(gup_test_exec(gup_fd, 1984 (unsigned long)buffer->ptr + 2 * self->page_size, 1985 PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0); 1986 ASSERT_EQ(gup_test_exec(gup_fd, 1987 (unsigned long)buffer->ptr + 3 * self->page_size, 1988 PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0); 1989 1990 /* Take snapshot to CPU pagetables */ 1991 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1992 ASSERT_EQ(ret, 0); 1993 ASSERT_EQ(buffer->cpages, npages); 1994 m = buffer->mirror; 1995 if (hmm_is_coherent_type(variant->device_number)) { 1996 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]); 1997 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]); 1998 } else { 1999 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]); 2000 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]); 2001 } 2002 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]); 2003 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]); 2004 /* 2005 * Check again the content on the pages. Make sure there's no 2006 * corrupted data. 2007 */ 2008 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2009 ASSERT_EQ(ptr[i], i); 2010 2011 close(gup_fd); 2012 hmm_buffer_free(buffer); 2013 } 2014 2015 /* 2016 * Test copy-on-write in device pages. 2017 * In case of writing to COW private page(s), a page fault will migrate pages 2018 * back to system memory first. Then, these pages will be duplicated. In case 2019 * of COW device coherent type, pages are duplicated directly from device 2020 * memory. 2021 */ 2022 TEST_F(hmm, hmm_cow_in_device) 2023 { 2024 struct hmm_buffer *buffer; 2025 unsigned long npages; 2026 unsigned long size; 2027 unsigned long i; 2028 int *ptr; 2029 int ret; 2030 unsigned char *m; 2031 pid_t pid; 2032 int status; 2033 2034 npages = 4; 2035 size = npages << self->page_shift; 2036 2037 buffer = malloc(sizeof(*buffer)); 2038 ASSERT_NE(buffer, NULL); 2039 2040 buffer->fd = -1; 2041 buffer->size = size; 2042 buffer->mirror = malloc(size); 2043 ASSERT_NE(buffer->mirror, NULL); 2044 2045 buffer->ptr = mmap(NULL, size, 2046 PROT_READ | PROT_WRITE, 2047 MAP_PRIVATE | MAP_ANONYMOUS, 2048 buffer->fd, 0); 2049 ASSERT_NE(buffer->ptr, MAP_FAILED); 2050 2051 /* Initialize buffer in system memory. */ 2052 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2053 ptr[i] = i; 2054 2055 /* Migrate memory to device. */ 2056 2057 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2058 ASSERT_EQ(ret, 0); 2059 ASSERT_EQ(buffer->cpages, npages); 2060 2061 pid = fork(); 2062 if (pid == -1) 2063 ASSERT_EQ(pid, 0); 2064 if (!pid) { 2065 /* Child process waits for SIGTERM from the parent. */ 2066 while (1) { 2067 } 2068 /* Should not reach this */ 2069 } 2070 /* Parent process writes to COW pages(s) and gets a 2071 * new copy in system. In case of device private pages, 2072 * this write causes a migration to system mem first. 2073 */ 2074 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2075 ptr[i] = i; 2076 2077 /* Terminate child and wait */ 2078 EXPECT_EQ(0, kill(pid, SIGTERM)); 2079 EXPECT_EQ(pid, waitpid(pid, &status, 0)); 2080 EXPECT_NE(0, WIFSIGNALED(status)); 2081 EXPECT_EQ(SIGTERM, WTERMSIG(status)); 2082 2083 /* Take snapshot to CPU pagetables */ 2084 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 2085 ASSERT_EQ(ret, 0); 2086 ASSERT_EQ(buffer->cpages, npages); 2087 m = buffer->mirror; 2088 for (i = 0; i < npages; i++) 2089 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]); 2090 2091 hmm_buffer_free(buffer); 2092 } 2093 2094 /* 2095 * Migrate private anonymous huge empty page. 2096 */ 2097 TEST_F(hmm, migrate_anon_huge_empty) 2098 { 2099 struct hmm_buffer *buffer; 2100 unsigned long npages; 2101 unsigned long size; 2102 unsigned long i; 2103 void *old_ptr; 2104 void *map; 2105 int *ptr; 2106 int ret; 2107 2108 size = read_pmd_pagesize(); 2109 2110 buffer = malloc(sizeof(*buffer)); 2111 ASSERT_NE(buffer, NULL); 2112 2113 buffer->fd = -1; 2114 buffer->size = 2 * size; 2115 buffer->mirror = malloc(size); 2116 ASSERT_NE(buffer->mirror, NULL); 2117 memset(buffer->mirror, 0xFF, size); 2118 2119 buffer->ptr = mmap(NULL, 2 * size, 2120 PROT_READ, 2121 MAP_PRIVATE | MAP_ANONYMOUS, 2122 buffer->fd, 0); 2123 ASSERT_NE(buffer->ptr, MAP_FAILED); 2124 2125 npages = size >> self->page_shift; 2126 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2127 ret = madvise(map, size, MADV_HUGEPAGE); 2128 ASSERT_EQ(ret, 0); 2129 old_ptr = buffer->ptr; 2130 buffer->ptr = map; 2131 2132 /* Migrate memory to device. */ 2133 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2134 ASSERT_EQ(ret, 0); 2135 ASSERT_EQ(buffer->cpages, npages); 2136 2137 /* Check what the device read. */ 2138 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2139 ASSERT_EQ(ptr[i], 0); 2140 2141 buffer->ptr = old_ptr; 2142 hmm_buffer_free(buffer); 2143 } 2144 2145 /* 2146 * Migrate private anonymous huge zero page. 2147 */ 2148 TEST_F(hmm, migrate_anon_huge_zero) 2149 { 2150 struct hmm_buffer *buffer; 2151 unsigned long npages; 2152 unsigned long size; 2153 unsigned long i; 2154 void *old_ptr; 2155 void *map; 2156 int *ptr; 2157 int ret; 2158 int val; 2159 2160 size = read_pmd_pagesize(); 2161 2162 buffer = malloc(sizeof(*buffer)); 2163 ASSERT_NE(buffer, NULL); 2164 2165 buffer->fd = -1; 2166 buffer->size = 2 * size; 2167 buffer->mirror = malloc(size); 2168 ASSERT_NE(buffer->mirror, NULL); 2169 memset(buffer->mirror, 0xFF, size); 2170 2171 buffer->ptr = mmap(NULL, 2 * size, 2172 PROT_READ, 2173 MAP_PRIVATE | MAP_ANONYMOUS, 2174 buffer->fd, 0); 2175 ASSERT_NE(buffer->ptr, MAP_FAILED); 2176 2177 npages = size >> self->page_shift; 2178 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2179 ret = madvise(map, size, MADV_HUGEPAGE); 2180 ASSERT_EQ(ret, 0); 2181 old_ptr = buffer->ptr; 2182 buffer->ptr = map; 2183 2184 /* Initialize a read-only zero huge page. */ 2185 val = *(int *)buffer->ptr; 2186 ASSERT_EQ(val, 0); 2187 2188 /* Migrate memory to device. */ 2189 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2190 ASSERT_EQ(ret, 0); 2191 ASSERT_EQ(buffer->cpages, npages); 2192 2193 /* Check what the device read. */ 2194 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2195 ASSERT_EQ(ptr[i], 0); 2196 2197 /* Fault pages back to system memory and check them. */ 2198 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) { 2199 ASSERT_EQ(ptr[i], 0); 2200 /* If it asserts once, it probably will 500,000 times */ 2201 if (ptr[i] != 0) 2202 break; 2203 } 2204 2205 buffer->ptr = old_ptr; 2206 hmm_buffer_free(buffer); 2207 } 2208 2209 /* 2210 * Migrate private anonymous huge page and free. 2211 */ 2212 TEST_F(hmm, migrate_anon_huge_free) 2213 { 2214 struct hmm_buffer *buffer; 2215 unsigned long npages; 2216 unsigned long size; 2217 unsigned long i; 2218 void *old_ptr; 2219 void *map; 2220 int *ptr; 2221 int ret; 2222 2223 size = read_pmd_pagesize(); 2224 2225 buffer = malloc(sizeof(*buffer)); 2226 ASSERT_NE(buffer, NULL); 2227 2228 buffer->fd = -1; 2229 buffer->size = 2 * size; 2230 buffer->mirror = malloc(size); 2231 ASSERT_NE(buffer->mirror, NULL); 2232 memset(buffer->mirror, 0xFF, size); 2233 2234 buffer->ptr = mmap(NULL, 2 * size, 2235 PROT_READ | PROT_WRITE, 2236 MAP_PRIVATE | MAP_ANONYMOUS, 2237 buffer->fd, 0); 2238 ASSERT_NE(buffer->ptr, MAP_FAILED); 2239 2240 npages = size >> self->page_shift; 2241 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2242 ret = madvise(map, size, MADV_HUGEPAGE); 2243 ASSERT_EQ(ret, 0); 2244 old_ptr = buffer->ptr; 2245 buffer->ptr = map; 2246 2247 /* Initialize buffer in system memory. */ 2248 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2249 ptr[i] = i; 2250 2251 /* Migrate memory to device. */ 2252 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2253 ASSERT_EQ(ret, 0); 2254 ASSERT_EQ(buffer->cpages, npages); 2255 2256 /* Check what the device read. */ 2257 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2258 ASSERT_EQ(ptr[i], i); 2259 2260 /* Try freeing it. */ 2261 ret = madvise(map, size, MADV_FREE); 2262 ASSERT_EQ(ret, 0); 2263 2264 buffer->ptr = old_ptr; 2265 hmm_buffer_free(buffer); 2266 } 2267 2268 /* 2269 * Migrate private anonymous huge page and fault back to sysmem. 2270 */ 2271 TEST_F(hmm, migrate_anon_huge_fault) 2272 { 2273 struct hmm_buffer *buffer; 2274 unsigned long npages; 2275 unsigned long size; 2276 unsigned long i; 2277 void *old_ptr; 2278 void *map; 2279 int *ptr; 2280 int ret; 2281 2282 size = read_pmd_pagesize(); 2283 2284 buffer = malloc(sizeof(*buffer)); 2285 ASSERT_NE(buffer, NULL); 2286 2287 buffer->fd = -1; 2288 buffer->size = 2 * size; 2289 buffer->mirror = malloc(size); 2290 ASSERT_NE(buffer->mirror, NULL); 2291 memset(buffer->mirror, 0xFF, size); 2292 2293 buffer->ptr = mmap(NULL, 2 * size, 2294 PROT_READ | PROT_WRITE, 2295 MAP_PRIVATE | MAP_ANONYMOUS, 2296 buffer->fd, 0); 2297 ASSERT_NE(buffer->ptr, MAP_FAILED); 2298 2299 npages = size >> self->page_shift; 2300 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2301 ret = madvise(map, size, MADV_HUGEPAGE); 2302 ASSERT_EQ(ret, 0); 2303 old_ptr = buffer->ptr; 2304 buffer->ptr = map; 2305 2306 /* Initialize buffer in system memory. */ 2307 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2308 ptr[i] = i; 2309 2310 /* Migrate memory to device. */ 2311 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2312 ASSERT_EQ(ret, 0); 2313 ASSERT_EQ(buffer->cpages, npages); 2314 2315 /* Check what the device read. */ 2316 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2317 ASSERT_EQ(ptr[i], i); 2318 2319 /* Fault pages back to system memory and check them. */ 2320 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2321 ASSERT_EQ(ptr[i], i); 2322 2323 buffer->ptr = old_ptr; 2324 hmm_buffer_free(buffer); 2325 } 2326 2327 /* 2328 * Migrate memory and fault back to sysmem after partially unmapping. 2329 */ 2330 TEST_F(hmm, migrate_partial_unmap_fault) 2331 { 2332 struct hmm_buffer *buffer; 2333 unsigned long npages; 2334 unsigned long size = read_pmd_pagesize(); 2335 unsigned long i; 2336 void *old_ptr; 2337 void *map; 2338 int *ptr; 2339 int ret, j, use_thp; 2340 int offsets[] = { 0, 512 * ONEKB, ONEMEG }; 2341 2342 for (use_thp = 0; use_thp < 2; ++use_thp) { 2343 for (j = 0; j < ARRAY_SIZE(offsets); ++j) { 2344 buffer = malloc(sizeof(*buffer)); 2345 ASSERT_NE(buffer, NULL); 2346 2347 buffer->fd = -1; 2348 buffer->size = 2 * size; 2349 buffer->mirror = malloc(size); 2350 ASSERT_NE(buffer->mirror, NULL); 2351 memset(buffer->mirror, 0xFF, size); 2352 2353 buffer->ptr = mmap(NULL, 2 * size, 2354 PROT_READ | PROT_WRITE, 2355 MAP_PRIVATE | MAP_ANONYMOUS, 2356 buffer->fd, 0); 2357 ASSERT_NE(buffer->ptr, MAP_FAILED); 2358 2359 npages = size >> self->page_shift; 2360 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2361 if (use_thp) 2362 ret = madvise(map, size, MADV_HUGEPAGE); 2363 else 2364 ret = madvise(map, size, MADV_NOHUGEPAGE); 2365 ASSERT_EQ(ret, 0); 2366 old_ptr = buffer->ptr; 2367 buffer->ptr = map; 2368 2369 /* Initialize buffer in system memory. */ 2370 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2371 ptr[i] = i; 2372 2373 /* Migrate memory to device. */ 2374 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2375 ASSERT_EQ(ret, 0); 2376 ASSERT_EQ(buffer->cpages, npages); 2377 2378 /* Check what the device read. */ 2379 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2380 ASSERT_EQ(ptr[i], i); 2381 2382 munmap(buffer->ptr + offsets[j], ONEMEG); 2383 2384 /* Fault pages back to system memory and check them. */ 2385 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2386 if (i * sizeof(int) < offsets[j] || 2387 i * sizeof(int) >= offsets[j] + ONEMEG) 2388 ASSERT_EQ(ptr[i], i); 2389 2390 buffer->ptr = old_ptr; 2391 hmm_buffer_free(buffer); 2392 } 2393 } 2394 } 2395 2396 TEST_F(hmm, migrate_remap_fault) 2397 { 2398 struct hmm_buffer *buffer; 2399 unsigned long npages; 2400 unsigned long size = read_pmd_pagesize(); 2401 unsigned long i; 2402 void *old_ptr, *new_ptr = NULL; 2403 void *map; 2404 int *ptr; 2405 int ret, j, use_thp, dont_unmap, before; 2406 int offsets[] = { 0, 512 * ONEKB, ONEMEG }; 2407 2408 for (before = 0; before < 2; ++before) { 2409 for (dont_unmap = 0; dont_unmap < 2; ++dont_unmap) { 2410 for (use_thp = 0; use_thp < 2; ++use_thp) { 2411 for (j = 0; j < ARRAY_SIZE(offsets); ++j) { 2412 int flags = MREMAP_MAYMOVE | MREMAP_FIXED; 2413 2414 if (dont_unmap) 2415 flags |= MREMAP_DONTUNMAP; 2416 2417 buffer = malloc(sizeof(*buffer)); 2418 ASSERT_NE(buffer, NULL); 2419 2420 buffer->fd = -1; 2421 buffer->size = 8 * size; 2422 buffer->mirror = malloc(size); 2423 ASSERT_NE(buffer->mirror, NULL); 2424 memset(buffer->mirror, 0xFF, size); 2425 2426 buffer->ptr = mmap(NULL, buffer->size, 2427 PROT_READ | PROT_WRITE, 2428 MAP_PRIVATE | MAP_ANONYMOUS, 2429 buffer->fd, 0); 2430 ASSERT_NE(buffer->ptr, MAP_FAILED); 2431 2432 npages = size >> self->page_shift; 2433 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 2434 if (use_thp) 2435 ret = madvise(map, size, MADV_HUGEPAGE); 2436 else 2437 ret = madvise(map, size, MADV_NOHUGEPAGE); 2438 ASSERT_EQ(ret, 0); 2439 old_ptr = buffer->ptr; 2440 munmap(map + size, size * 2); 2441 buffer->ptr = map; 2442 2443 /* Initialize buffer in system memory. */ 2444 for (i = 0, ptr = buffer->ptr; 2445 i < size / sizeof(*ptr); ++i) 2446 ptr[i] = i; 2447 2448 if (before) { 2449 new_ptr = mremap((void *)map, size, size, flags, 2450 map + size + offsets[j]); 2451 ASSERT_NE(new_ptr, MAP_FAILED); 2452 buffer->ptr = new_ptr; 2453 } 2454 2455 /* Migrate memory to device. */ 2456 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2457 ASSERT_EQ(ret, 0); 2458 ASSERT_EQ(buffer->cpages, npages); 2459 2460 /* Check what the device read. */ 2461 for (i = 0, ptr = buffer->mirror; 2462 i < size / sizeof(*ptr); ++i) 2463 ASSERT_EQ(ptr[i], i); 2464 2465 if (!before) { 2466 new_ptr = mremap((void *)map, size, size, flags, 2467 map + size + offsets[j]); 2468 ASSERT_NE(new_ptr, MAP_FAILED); 2469 buffer->ptr = new_ptr; 2470 } 2471 2472 /* Fault pages back to system memory and check them. */ 2473 for (i = 0, ptr = buffer->ptr; 2474 i < size / sizeof(*ptr); ++i) 2475 ASSERT_EQ(ptr[i], i); 2476 2477 munmap(new_ptr, size); 2478 buffer->ptr = old_ptr; 2479 hmm_buffer_free(buffer); 2480 } 2481 } 2482 } 2483 } 2484 } 2485 2486 /* 2487 * Migrate private anonymous huge page with allocation errors. 2488 */ 2489 TEST_F(hmm, migrate_anon_huge_err) 2490 { 2491 struct hmm_buffer *buffer; 2492 unsigned long npages; 2493 unsigned long size; 2494 unsigned long i; 2495 void *old_ptr; 2496 void *map; 2497 int *ptr; 2498 int ret; 2499 2500 size = read_pmd_pagesize(); 2501 2502 buffer = malloc(sizeof(*buffer)); 2503 ASSERT_NE(buffer, NULL); 2504 2505 buffer->fd = -1; 2506 buffer->size = 2 * size; 2507 buffer->mirror = malloc(2 * size); 2508 ASSERT_NE(buffer->mirror, NULL); 2509 memset(buffer->mirror, 0xFF, 2 * size); 2510 2511 old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, 2512 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2513 ASSERT_NE(old_ptr, MAP_FAILED); 2514 2515 npages = size >> self->page_shift; 2516 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2517 ret = madvise(map, size, MADV_HUGEPAGE); 2518 ASSERT_EQ(ret, 0); 2519 buffer->ptr = map; 2520 2521 /* Initialize buffer in system memory. */ 2522 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2523 ptr[i] = i; 2524 2525 /* Migrate memory to device but force a THP allocation error. */ 2526 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2527 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2528 ASSERT_EQ(ret, 0); 2529 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2530 ASSERT_EQ(ret, 0); 2531 ASSERT_EQ(buffer->cpages, npages); 2532 2533 /* Check what the device read. */ 2534 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) { 2535 ASSERT_EQ(ptr[i], i); 2536 if (ptr[i] != i) 2537 break; 2538 } 2539 2540 /* Try faulting back a single (PAGE_SIZE) page. */ 2541 ptr = buffer->ptr; 2542 ASSERT_EQ(ptr[2048], 2048); 2543 2544 /* unmap and remap the region to reset things. */ 2545 ret = munmap(old_ptr, 2 * size); 2546 ASSERT_EQ(ret, 0); 2547 old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, 2548 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2549 ASSERT_NE(old_ptr, MAP_FAILED); 2550 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2551 ret = madvise(map, size, MADV_HUGEPAGE); 2552 ASSERT_EQ(ret, 0); 2553 buffer->ptr = map; 2554 2555 /* Initialize buffer in system memory. */ 2556 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2557 ptr[i] = i; 2558 2559 /* Migrate THP to device. */ 2560 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2561 ASSERT_EQ(ret, 0); 2562 ASSERT_EQ(buffer->cpages, npages); 2563 2564 /* 2565 * Force an allocation error when faulting back a THP resident in the 2566 * device. 2567 */ 2568 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2569 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2570 ASSERT_EQ(ret, 0); 2571 2572 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages); 2573 ASSERT_EQ(ret, 0); 2574 ptr = buffer->ptr; 2575 ASSERT_EQ(ptr[2048], 2048); 2576 2577 buffer->ptr = old_ptr; 2578 hmm_buffer_free(buffer); 2579 } 2580 2581 /* 2582 * Migrate private anonymous huge zero page with allocation errors. 2583 */ 2584 TEST_F(hmm, migrate_anon_huge_zero_err) 2585 { 2586 struct hmm_buffer *buffer; 2587 unsigned long npages; 2588 unsigned long size; 2589 unsigned long i; 2590 void *old_ptr; 2591 void *map; 2592 int *ptr; 2593 int ret; 2594 2595 size = read_pmd_pagesize(); 2596 2597 buffer = malloc(sizeof(*buffer)); 2598 ASSERT_NE(buffer, NULL); 2599 2600 buffer->fd = -1; 2601 buffer->size = 2 * size; 2602 buffer->mirror = malloc(2 * size); 2603 ASSERT_NE(buffer->mirror, NULL); 2604 memset(buffer->mirror, 0xFF, 2 * size); 2605 2606 old_ptr = mmap(NULL, 2 * size, PROT_READ, 2607 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2608 ASSERT_NE(old_ptr, MAP_FAILED); 2609 2610 npages = size >> self->page_shift; 2611 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2612 ret = madvise(map, size, MADV_HUGEPAGE); 2613 ASSERT_EQ(ret, 0); 2614 buffer->ptr = map; 2615 2616 /* Migrate memory to device but force a THP allocation error. */ 2617 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2618 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2619 ASSERT_EQ(ret, 0); 2620 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2621 ASSERT_EQ(ret, 0); 2622 ASSERT_EQ(buffer->cpages, npages); 2623 2624 /* Check what the device read. */ 2625 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 2626 ASSERT_EQ(ptr[i], 0); 2627 2628 /* Try faulting back a single (PAGE_SIZE) page. */ 2629 ptr = buffer->ptr; 2630 ASSERT_EQ(ptr[2048], 0); 2631 2632 /* unmap and remap the region to reset things. */ 2633 ret = munmap(old_ptr, 2 * size); 2634 ASSERT_EQ(ret, 0); 2635 old_ptr = mmap(NULL, 2 * size, PROT_READ, 2636 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 2637 ASSERT_NE(old_ptr, MAP_FAILED); 2638 map = (void *)ALIGN((uintptr_t)old_ptr, size); 2639 ret = madvise(map, size, MADV_HUGEPAGE); 2640 ASSERT_EQ(ret, 0); 2641 buffer->ptr = map; 2642 2643 /* Initialize buffer in system memory (zero THP page). */ 2644 ret = ptr[0]; 2645 ASSERT_EQ(ret, 0); 2646 2647 /* Migrate memory to device but force a THP allocation error. */ 2648 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer, 2649 HMM_DMIRROR_FLAG_FAIL_ALLOC); 2650 ASSERT_EQ(ret, 0); 2651 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2652 ASSERT_EQ(ret, 0); 2653 ASSERT_EQ(buffer->cpages, npages); 2654 2655 /* Fault the device memory back and check it. */ 2656 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2657 ASSERT_EQ(ptr[i], 0); 2658 2659 buffer->ptr = old_ptr; 2660 hmm_buffer_free(buffer); 2661 } 2662 2663 struct benchmark_results { 2664 double sys_to_dev_time; 2665 double dev_to_sys_time; 2666 double throughput_s2d; 2667 double throughput_d2s; 2668 }; 2669 2670 static double get_time_ms(void) 2671 { 2672 struct timeval tv; 2673 2674 gettimeofday(&tv, NULL); 2675 return (tv.tv_sec * 1000.0) + (tv.tv_usec / 1000.0); 2676 } 2677 2678 static inline struct hmm_buffer *hmm_buffer_alloc(unsigned long size) 2679 { 2680 struct hmm_buffer *buffer; 2681 2682 buffer = malloc(sizeof(*buffer)); 2683 2684 buffer->fd = -1; 2685 buffer->size = size; 2686 buffer->mirror = malloc(size); 2687 memset(buffer->mirror, 0xFF, size); 2688 return buffer; 2689 } 2690 2691 static void print_benchmark_results(const char *test_name, size_t buffer_size, 2692 struct benchmark_results *thp, 2693 struct benchmark_results *regular) 2694 { 2695 double s2d_improvement = ((regular->sys_to_dev_time - thp->sys_to_dev_time) / 2696 regular->sys_to_dev_time) * 100.0; 2697 double d2s_improvement = ((regular->dev_to_sys_time - thp->dev_to_sys_time) / 2698 regular->dev_to_sys_time) * 100.0; 2699 double throughput_s2d_improvement = ((thp->throughput_s2d - regular->throughput_s2d) / 2700 regular->throughput_s2d) * 100.0; 2701 double throughput_d2s_improvement = ((thp->throughput_d2s - regular->throughput_d2s) / 2702 regular->throughput_d2s) * 100.0; 2703 2704 printf("\n=== %s (%.1f MB) ===\n", test_name, buffer_size / (1024.0 * 1024.0)); 2705 printf(" | With THP | Without THP | Improvement\n"); 2706 printf("---------------------------------------------------------------------\n"); 2707 printf("Sys->Dev Migration | %.3f ms | %.3f ms | %.1f%%\n", 2708 thp->sys_to_dev_time, regular->sys_to_dev_time, s2d_improvement); 2709 printf("Dev->Sys Migration | %.3f ms | %.3f ms | %.1f%%\n", 2710 thp->dev_to_sys_time, regular->dev_to_sys_time, d2s_improvement); 2711 printf("S->D Throughput | %.2f GB/s | %.2f GB/s | %.1f%%\n", 2712 thp->throughput_s2d, regular->throughput_s2d, throughput_s2d_improvement); 2713 printf("D->S Throughput | %.2f GB/s | %.2f GB/s | %.1f%%\n", 2714 thp->throughput_d2s, regular->throughput_d2s, throughput_d2s_improvement); 2715 } 2716 2717 /* 2718 * Run a single migration benchmark 2719 * fd: file descriptor for hmm device 2720 * use_thp: whether to use THP 2721 * buffer_size: size of buffer to allocate 2722 * iterations: number of iterations 2723 * results: where to store results 2724 */ 2725 static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_size, 2726 int iterations, struct benchmark_results *results) 2727 { 2728 struct hmm_buffer *buffer; 2729 unsigned long npages = buffer_size / sysconf(_SC_PAGESIZE); 2730 double start, end; 2731 double s2d_total = 0, d2s_total = 0; 2732 int ret, i; 2733 int *ptr; 2734 2735 buffer = hmm_buffer_alloc(buffer_size); 2736 2737 /* Map memory */ 2738 buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE, 2739 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 2740 2741 if (!buffer->ptr) 2742 return -1; 2743 2744 /* Apply THP hint if requested */ 2745 if (use_thp) 2746 ret = madvise(buffer->ptr, buffer_size, MADV_HUGEPAGE); 2747 else 2748 ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE); 2749 2750 if (ret) 2751 return ret; 2752 2753 /* Initialize memory to make sure pages are allocated */ 2754 ptr = (int *)buffer->ptr; 2755 for (i = 0; i < buffer_size / sizeof(int); i++) 2756 ptr[i] = i & 0xFF; 2757 2758 /* Warmup iteration */ 2759 ret = hmm_migrate_sys_to_dev(fd, buffer, npages); 2760 if (ret) 2761 return ret; 2762 2763 ret = hmm_migrate_dev_to_sys(fd, buffer, npages); 2764 if (ret) 2765 return ret; 2766 2767 /* Benchmark iterations */ 2768 for (i = 0; i < iterations; i++) { 2769 /* System to device migration */ 2770 start = get_time_ms(); 2771 2772 ret = hmm_migrate_sys_to_dev(fd, buffer, npages); 2773 if (ret) 2774 return ret; 2775 2776 end = get_time_ms(); 2777 s2d_total += (end - start); 2778 2779 /* Device to system migration */ 2780 start = get_time_ms(); 2781 2782 ret = hmm_migrate_dev_to_sys(fd, buffer, npages); 2783 if (ret) 2784 return ret; 2785 2786 end = get_time_ms(); 2787 d2s_total += (end - start); 2788 } 2789 2790 /* Calculate average times and throughput */ 2791 results->sys_to_dev_time = s2d_total / iterations; 2792 results->dev_to_sys_time = d2s_total / iterations; 2793 results->throughput_s2d = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / 2794 (results->sys_to_dev_time / 1000.0); 2795 results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / 2796 (results->dev_to_sys_time / 1000.0); 2797 2798 /* Cleanup */ 2799 hmm_buffer_free(buffer); 2800 return 0; 2801 } 2802 2803 /* 2804 * Benchmark THP migration with different buffer sizes 2805 */ 2806 TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120) 2807 { 2808 struct benchmark_results thp_results, regular_results; 2809 size_t thp_size = 2 * 1024 * 1024; /* 2MB - typical THP size */ 2810 int iterations = 5; 2811 2812 printf("\nHMM THP Migration Benchmark\n"); 2813 printf("---------------------------\n"); 2814 printf("System page size: %ld bytes\n", sysconf(_SC_PAGESIZE)); 2815 2816 /* Test different buffer sizes */ 2817 size_t test_sizes[] = { 2818 thp_size / 4, /* 512KB - smaller than THP */ 2819 thp_size / 2, /* 1MB - half THP */ 2820 thp_size, /* 2MB - single THP */ 2821 thp_size * 2, /* 4MB - two THPs */ 2822 thp_size * 4, /* 8MB - four THPs */ 2823 thp_size * 8, /* 16MB - eight THPs */ 2824 thp_size * 128, /* 256MB - one twenty eight THPs */ 2825 }; 2826 2827 static const char *const test_names[] = { 2828 "Small Buffer (512KB)", 2829 "Half THP Size (1MB)", 2830 "Single THP Size (2MB)", 2831 "Two THP Size (4MB)", 2832 "Four THP Size (8MB)", 2833 "Eight THP Size (16MB)", 2834 "One twenty eight THP Size (256MB)" 2835 }; 2836 2837 int num_tests = ARRAY_SIZE(test_sizes); 2838 2839 /* Run all tests */ 2840 for (int i = 0; i < num_tests; i++) { 2841 /* Test with THP */ 2842 ASSERT_EQ(run_migration_benchmark(self->fd, 1, test_sizes[i], 2843 iterations, &thp_results), 0); 2844 2845 /* Test without THP */ 2846 ASSERT_EQ(run_migration_benchmark(self->fd, 0, test_sizes[i], 2847 iterations, ®ular_results), 0); 2848 2849 /* Print results */ 2850 print_benchmark_results(test_names[i], test_sizes[i], 2851 &thp_results, ®ular_results); 2852 } 2853 } 2854 TEST_HARNESS_MAIN 2855