1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * HMM stands for Heterogeneous Memory Management, it is a helper layer inside
4 * the linux kernel to help device drivers mirror a process address space in
5 * the device. This allows the device to use the same address space which
6 * makes communication and data exchange a lot easier.
7 *
8 * This framework's sole purpose is to exercise various code paths inside
9 * the kernel to make sure that HMM performs as expected and to flush out any
10 * bugs.
11 */
12
13 #include "kselftest_harness.h"
14
15 #include <errno.h>
16 #include <fcntl.h>
17 #include <stdio.h>
18 #include <stdlib.h>
19 #include <stdint.h>
20 #include <unistd.h>
21 #include <strings.h>
22 #include <time.h>
23 #include <pthread.h>
24 #include <sys/types.h>
25 #include <sys/stat.h>
26 #include <sys/mman.h>
27 #include <sys/ioctl.h>
28 #include <sys/time.h>
29
30
31 /*
32 * This is a private UAPI to the kernel test module so it isn't exported
33 * in the usual include/uapi/... directory.
34 */
35 #include <lib/test_hmm_uapi.h>
36 #include <mm/gup_test.h>
37
38 struct hmm_buffer {
39 void *ptr;
40 void *mirror;
41 unsigned long size;
42 int fd;
43 uint64_t cpages;
44 uint64_t faults;
45 };
46
47 enum {
48 HMM_PRIVATE_DEVICE_ONE,
49 HMM_PRIVATE_DEVICE_TWO,
50 HMM_COHERENCE_DEVICE_ONE,
51 HMM_COHERENCE_DEVICE_TWO,
52 };
53
54 #define ONEKB (1 << 10)
55 #define ONEMEG (1 << 20)
56 #define TWOMEG (1 << 21)
57 #define HMM_BUFFER_SIZE (1024 << 12)
58 #define HMM_PATH_MAX 64
59 #define NTIMES 10
60
61 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
62 /* Just the flags we need, copied from mm.h: */
63
64 #ifndef FOLL_WRITE
65 #define FOLL_WRITE 0x01 /* check pte is writable */
66 #endif
67
68 #ifndef FOLL_LONGTERM
69 #define FOLL_LONGTERM 0x100 /* mapping lifetime is indefinite */
70 #endif
FIXTURE(hmm)71 FIXTURE(hmm)
72 {
73 int fd;
74 unsigned int page_size;
75 unsigned int page_shift;
76 };
77
FIXTURE_VARIANT(hmm)78 FIXTURE_VARIANT(hmm)
79 {
80 int device_number;
81 };
82
FIXTURE_VARIANT_ADD(hmm,hmm_device_private)83 FIXTURE_VARIANT_ADD(hmm, hmm_device_private)
84 {
85 .device_number = HMM_PRIVATE_DEVICE_ONE,
86 };
87
FIXTURE_VARIANT_ADD(hmm,hmm_device_coherent)88 FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent)
89 {
90 .device_number = HMM_COHERENCE_DEVICE_ONE,
91 };
92
FIXTURE(hmm2)93 FIXTURE(hmm2)
94 {
95 int fd0;
96 int fd1;
97 unsigned int page_size;
98 unsigned int page_shift;
99 };
100
FIXTURE_VARIANT(hmm2)101 FIXTURE_VARIANT(hmm2)
102 {
103 int device_number0;
104 int device_number1;
105 };
106
FIXTURE_VARIANT_ADD(hmm2,hmm2_device_private)107 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private)
108 {
109 .device_number0 = HMM_PRIVATE_DEVICE_ONE,
110 .device_number1 = HMM_PRIVATE_DEVICE_TWO,
111 };
112
FIXTURE_VARIANT_ADD(hmm2,hmm2_device_coherent)113 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent)
114 {
115 .device_number0 = HMM_COHERENCE_DEVICE_ONE,
116 .device_number1 = HMM_COHERENCE_DEVICE_TWO,
117 };
118
hmm_open(int unit)119 static int hmm_open(int unit)
120 {
121 char pathname[HMM_PATH_MAX];
122 int fd;
123
124 snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit);
125 fd = open(pathname, O_RDWR, 0);
126 if (fd < 0)
127 fprintf(stderr, "could not open hmm dmirror driver (%s)\n",
128 pathname);
129 return fd;
130 }
131
hmm_is_coherent_type(int dev_num)132 static bool hmm_is_coherent_type(int dev_num)
133 {
134 return (dev_num >= HMM_COHERENCE_DEVICE_ONE);
135 }
136
FIXTURE_SETUP(hmm)137 FIXTURE_SETUP(hmm)
138 {
139 self->page_size = sysconf(_SC_PAGE_SIZE);
140 self->page_shift = ffs(self->page_size) - 1;
141
142 self->fd = hmm_open(variant->device_number);
143 if (self->fd < 0 && hmm_is_coherent_type(variant->device_number))
144 SKIP(return, "DEVICE_COHERENT not available");
145 ASSERT_GE(self->fd, 0);
146 }
147
FIXTURE_SETUP(hmm2)148 FIXTURE_SETUP(hmm2)
149 {
150 self->page_size = sysconf(_SC_PAGE_SIZE);
151 self->page_shift = ffs(self->page_size) - 1;
152
153 self->fd0 = hmm_open(variant->device_number0);
154 if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0))
155 SKIP(return, "DEVICE_COHERENT not available");
156 ASSERT_GE(self->fd0, 0);
157 self->fd1 = hmm_open(variant->device_number1);
158 ASSERT_GE(self->fd1, 0);
159 }
160
FIXTURE_TEARDOWN(hmm)161 FIXTURE_TEARDOWN(hmm)
162 {
163 int ret = close(self->fd);
164
165 ASSERT_EQ(ret, 0);
166 self->fd = -1;
167 }
168
FIXTURE_TEARDOWN(hmm2)169 FIXTURE_TEARDOWN(hmm2)
170 {
171 int ret = close(self->fd0);
172
173 ASSERT_EQ(ret, 0);
174 self->fd0 = -1;
175
176 ret = close(self->fd1);
177 ASSERT_EQ(ret, 0);
178 self->fd1 = -1;
179 }
180
hmm_dmirror_cmd(int fd,unsigned long request,struct hmm_buffer * buffer,unsigned long npages)181 static int hmm_dmirror_cmd(int fd,
182 unsigned long request,
183 struct hmm_buffer *buffer,
184 unsigned long npages)
185 {
186 struct hmm_dmirror_cmd cmd;
187 int ret;
188
189 /* Simulate a device reading system memory. */
190 cmd.addr = (__u64)buffer->ptr;
191 cmd.ptr = (__u64)buffer->mirror;
192 cmd.npages = npages;
193
194 for (;;) {
195 ret = ioctl(fd, request, &cmd);
196 if (ret == 0)
197 break;
198 if (errno == EINTR)
199 continue;
200 return -errno;
201 }
202 buffer->cpages = cmd.cpages;
203 buffer->faults = cmd.faults;
204
205 return 0;
206 }
207
hmm_buffer_free(struct hmm_buffer * buffer)208 static void hmm_buffer_free(struct hmm_buffer *buffer)
209 {
210 if (buffer == NULL)
211 return;
212
213 if (buffer->ptr) {
214 munmap(buffer->ptr, buffer->size);
215 buffer->ptr = NULL;
216 }
217 free(buffer->mirror);
218 free(buffer);
219 }
220
221 /*
222 * Create a temporary file that will be deleted on close.
223 */
hmm_create_file(unsigned long size)224 static int hmm_create_file(unsigned long size)
225 {
226 char path[HMM_PATH_MAX];
227 int fd;
228
229 strcpy(path, "/tmp");
230 fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600);
231 if (fd >= 0) {
232 int r;
233
234 do {
235 r = ftruncate(fd, size);
236 } while (r == -1 && errno == EINTR);
237 if (!r)
238 return fd;
239 close(fd);
240 }
241 return -1;
242 }
243
244 /*
245 * Return a random unsigned number.
246 */
hmm_random(void)247 static unsigned int hmm_random(void)
248 {
249 static int fd = -1;
250 unsigned int r;
251
252 if (fd < 0) {
253 fd = open("/dev/urandom", O_RDONLY);
254 if (fd < 0) {
255 fprintf(stderr, "%s:%d failed to open /dev/urandom\n",
256 __FILE__, __LINE__);
257 return ~0U;
258 }
259 }
260 read(fd, &r, sizeof(r));
261 return r;
262 }
263
hmm_nanosleep(unsigned int n)264 static void hmm_nanosleep(unsigned int n)
265 {
266 struct timespec t;
267
268 t.tv_sec = 0;
269 t.tv_nsec = n;
270 nanosleep(&t, NULL);
271 }
272
hmm_migrate_sys_to_dev(int fd,struct hmm_buffer * buffer,unsigned long npages)273 static int hmm_migrate_sys_to_dev(int fd,
274 struct hmm_buffer *buffer,
275 unsigned long npages)
276 {
277 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages);
278 }
279
hmm_migrate_dev_to_sys(int fd,struct hmm_buffer * buffer,unsigned long npages)280 static int hmm_migrate_dev_to_sys(int fd,
281 struct hmm_buffer *buffer,
282 unsigned long npages)
283 {
284 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages);
285 }
286
287 /*
288 * Simple NULL test of device open/close.
289 */
TEST_F(hmm,open_close)290 TEST_F(hmm, open_close)
291 {
292 }
293
294 /*
295 * Read private anonymous memory.
296 */
TEST_F(hmm,anon_read)297 TEST_F(hmm, anon_read)
298 {
299 struct hmm_buffer *buffer;
300 unsigned long npages;
301 unsigned long size;
302 unsigned long i;
303 int *ptr;
304 int ret;
305 int val;
306
307 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
308 ASSERT_NE(npages, 0);
309 size = npages << self->page_shift;
310
311 buffer = malloc(sizeof(*buffer));
312 ASSERT_NE(buffer, NULL);
313
314 buffer->fd = -1;
315 buffer->size = size;
316 buffer->mirror = malloc(size);
317 ASSERT_NE(buffer->mirror, NULL);
318
319 buffer->ptr = mmap(NULL, size,
320 PROT_READ | PROT_WRITE,
321 MAP_PRIVATE | MAP_ANONYMOUS,
322 buffer->fd, 0);
323 ASSERT_NE(buffer->ptr, MAP_FAILED);
324
325 /*
326 * Initialize buffer in system memory but leave the first two pages
327 * zero (pte_none and pfn_zero).
328 */
329 i = 2 * self->page_size / sizeof(*ptr);
330 for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
331 ptr[i] = i;
332
333 /* Set buffer permission to read-only. */
334 ret = mprotect(buffer->ptr, size, PROT_READ);
335 ASSERT_EQ(ret, 0);
336
337 /* Populate the CPU page table with a special zero page. */
338 val = *(int *)(buffer->ptr + self->page_size);
339 ASSERT_EQ(val, 0);
340
341 /* Simulate a device reading system memory. */
342 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
343 ASSERT_EQ(ret, 0);
344 ASSERT_EQ(buffer->cpages, npages);
345 ASSERT_EQ(buffer->faults, 1);
346
347 /* Check what the device read. */
348 ptr = buffer->mirror;
349 for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i)
350 ASSERT_EQ(ptr[i], 0);
351 for (; i < size / sizeof(*ptr); ++i)
352 ASSERT_EQ(ptr[i], i);
353
354 hmm_buffer_free(buffer);
355 }
356
357 /*
358 * Read private anonymous memory which has been protected with
359 * mprotect() PROT_NONE.
360 */
TEST_F(hmm,anon_read_prot)361 TEST_F(hmm, anon_read_prot)
362 {
363 struct hmm_buffer *buffer;
364 unsigned long npages;
365 unsigned long size;
366 unsigned long i;
367 int *ptr;
368 int ret;
369
370 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
371 ASSERT_NE(npages, 0);
372 size = npages << self->page_shift;
373
374 buffer = malloc(sizeof(*buffer));
375 ASSERT_NE(buffer, NULL);
376
377 buffer->fd = -1;
378 buffer->size = size;
379 buffer->mirror = malloc(size);
380 ASSERT_NE(buffer->mirror, NULL);
381
382 buffer->ptr = mmap(NULL, size,
383 PROT_READ | PROT_WRITE,
384 MAP_PRIVATE | MAP_ANONYMOUS,
385 buffer->fd, 0);
386 ASSERT_NE(buffer->ptr, MAP_FAILED);
387
388 /* Initialize buffer in system memory. */
389 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
390 ptr[i] = i;
391
392 /* Initialize mirror buffer so we can verify it isn't written. */
393 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
394 ptr[i] = -i;
395
396 /* Protect buffer from reading. */
397 ret = mprotect(buffer->ptr, size, PROT_NONE);
398 ASSERT_EQ(ret, 0);
399
400 /* Simulate a device reading system memory. */
401 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
402 ASSERT_EQ(ret, -EFAULT);
403
404 /* Allow CPU to read the buffer so we can check it. */
405 ret = mprotect(buffer->ptr, size, PROT_READ);
406 ASSERT_EQ(ret, 0);
407 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
408 ASSERT_EQ(ptr[i], i);
409
410 /* Check what the device read. */
411 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
412 ASSERT_EQ(ptr[i], -i);
413
414 hmm_buffer_free(buffer);
415 }
416
417 /*
418 * Write private anonymous memory.
419 */
TEST_F(hmm,anon_write)420 TEST_F(hmm, anon_write)
421 {
422 struct hmm_buffer *buffer;
423 unsigned long npages;
424 unsigned long size;
425 unsigned long i;
426 int *ptr;
427 int ret;
428
429 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
430 ASSERT_NE(npages, 0);
431 size = npages << self->page_shift;
432
433 buffer = malloc(sizeof(*buffer));
434 ASSERT_NE(buffer, NULL);
435
436 buffer->fd = -1;
437 buffer->size = size;
438 buffer->mirror = malloc(size);
439 ASSERT_NE(buffer->mirror, NULL);
440
441 buffer->ptr = mmap(NULL, size,
442 PROT_READ | PROT_WRITE,
443 MAP_PRIVATE | MAP_ANONYMOUS,
444 buffer->fd, 0);
445 ASSERT_NE(buffer->ptr, MAP_FAILED);
446
447 /* Initialize data that the device will write to buffer->ptr. */
448 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
449 ptr[i] = i;
450
451 /* Simulate a device writing system memory. */
452 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
453 ASSERT_EQ(ret, 0);
454 ASSERT_EQ(buffer->cpages, npages);
455 ASSERT_EQ(buffer->faults, 1);
456
457 /* Check what the device wrote. */
458 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
459 ASSERT_EQ(ptr[i], i);
460
461 hmm_buffer_free(buffer);
462 }
463
464 /*
465 * Write private anonymous memory which has been protected with
466 * mprotect() PROT_READ.
467 */
TEST_F(hmm,anon_write_prot)468 TEST_F(hmm, anon_write_prot)
469 {
470 struct hmm_buffer *buffer;
471 unsigned long npages;
472 unsigned long size;
473 unsigned long i;
474 int *ptr;
475 int ret;
476
477 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
478 ASSERT_NE(npages, 0);
479 size = npages << self->page_shift;
480
481 buffer = malloc(sizeof(*buffer));
482 ASSERT_NE(buffer, NULL);
483
484 buffer->fd = -1;
485 buffer->size = size;
486 buffer->mirror = malloc(size);
487 ASSERT_NE(buffer->mirror, NULL);
488
489 buffer->ptr = mmap(NULL, size,
490 PROT_READ,
491 MAP_PRIVATE | MAP_ANONYMOUS,
492 buffer->fd, 0);
493 ASSERT_NE(buffer->ptr, MAP_FAILED);
494
495 /* Simulate a device reading a zero page of memory. */
496 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1);
497 ASSERT_EQ(ret, 0);
498 ASSERT_EQ(buffer->cpages, 1);
499 ASSERT_EQ(buffer->faults, 1);
500
501 /* Initialize data that the device will write to buffer->ptr. */
502 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
503 ptr[i] = i;
504
505 /* Simulate a device writing system memory. */
506 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
507 ASSERT_EQ(ret, -EPERM);
508
509 /* Check what the device wrote. */
510 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
511 ASSERT_EQ(ptr[i], 0);
512
513 /* Now allow writing and see that the zero page is replaced. */
514 ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ);
515 ASSERT_EQ(ret, 0);
516
517 /* Simulate a device writing system memory. */
518 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
519 ASSERT_EQ(ret, 0);
520 ASSERT_EQ(buffer->cpages, npages);
521 ASSERT_EQ(buffer->faults, 1);
522
523 /* Check what the device wrote. */
524 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
525 ASSERT_EQ(ptr[i], i);
526
527 hmm_buffer_free(buffer);
528 }
529
530 /*
531 * Check that a device writing an anonymous private mapping
532 * will copy-on-write if a child process inherits the mapping.
533 *
534 * Also verifies after fork() memory the device can be read by child.
535 */
TEST_F(hmm,anon_write_child)536 TEST_F(hmm, anon_write_child)
537 {
538 struct hmm_buffer *buffer;
539 unsigned long npages;
540 unsigned long size;
541 unsigned long i;
542 void *old_ptr;
543 void *map;
544 int *ptr;
545 pid_t pid;
546 int child_fd;
547 int ret, use_thp, migrate;
548
549 for (migrate = 0; migrate < 2; ++migrate) {
550 for (use_thp = 0; use_thp < 2; ++use_thp) {
551 npages = ALIGN(use_thp ? TWOMEG : HMM_BUFFER_SIZE,
552 self->page_size) >> self->page_shift;
553 ASSERT_NE(npages, 0);
554 size = npages << self->page_shift;
555
556 buffer = malloc(sizeof(*buffer));
557 ASSERT_NE(buffer, NULL);
558
559 buffer->fd = -1;
560 buffer->size = size * 2;
561 buffer->mirror = malloc(size);
562 ASSERT_NE(buffer->mirror, NULL);
563
564 buffer->ptr = mmap(NULL, size * 2,
565 PROT_READ | PROT_WRITE,
566 MAP_PRIVATE | MAP_ANONYMOUS,
567 buffer->fd, 0);
568 ASSERT_NE(buffer->ptr, MAP_FAILED);
569
570 old_ptr = buffer->ptr;
571 if (use_thp) {
572 map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
573 ret = madvise(map, size, MADV_HUGEPAGE);
574 ASSERT_EQ(ret, 0);
575 buffer->ptr = map;
576 }
577
578 /* Initialize buffer->ptr so we can tell if it is written. */
579 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
580 ptr[i] = i;
581
582 /* Initialize data that the device will write to buffer->ptr. */
583 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
584 ptr[i] = -i;
585
586 if (migrate) {
587 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
588 ASSERT_EQ(ret, 0);
589 ASSERT_EQ(buffer->cpages, npages);
590
591 }
592
593 pid = fork();
594 if (pid == -1)
595 ASSERT_EQ(pid, 0);
596 if (pid != 0) {
597 waitpid(pid, &ret, 0);
598 ASSERT_EQ(WIFEXITED(ret), 1);
599
600 /* Check that the parent's buffer did not change. */
601 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
602 ASSERT_EQ(ptr[i], i);
603
604 buffer->ptr = old_ptr;
605 hmm_buffer_free(buffer);
606 continue;
607 }
608
609 /* Check that we see the parent's values. */
610 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
611 ASSERT_EQ(ptr[i], i);
612 if (!migrate) {
613 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
614 ASSERT_EQ(ptr[i], -i);
615 }
616
617 /* The child process needs its own mirror to its own mm. */
618 child_fd = hmm_open(0);
619 ASSERT_GE(child_fd, 0);
620
621 /* Simulate a device writing system memory. */
622 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);
623 ASSERT_EQ(ret, 0);
624 ASSERT_EQ(buffer->cpages, npages);
625 ASSERT_EQ(buffer->faults, 1);
626
627 /* Check what the device wrote. */
628 if (!migrate) {
629 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
630 ASSERT_EQ(ptr[i], -i);
631 }
632
633 close(child_fd);
634 exit(0);
635 }
636 }
637 }
638
639 /*
640 * Check that a device writing an anonymous shared mapping
641 * will not copy-on-write if a child process inherits the mapping.
642 */
TEST_F(hmm,anon_write_child_shared)643 TEST_F(hmm, anon_write_child_shared)
644 {
645 struct hmm_buffer *buffer;
646 unsigned long npages;
647 unsigned long size;
648 unsigned long i;
649 int *ptr;
650 pid_t pid;
651 int child_fd;
652 int ret;
653
654 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
655 ASSERT_NE(npages, 0);
656 size = npages << self->page_shift;
657
658 buffer = malloc(sizeof(*buffer));
659 ASSERT_NE(buffer, NULL);
660
661 buffer->fd = -1;
662 buffer->size = size;
663 buffer->mirror = malloc(size);
664 ASSERT_NE(buffer->mirror, NULL);
665
666 buffer->ptr = mmap(NULL, size,
667 PROT_READ | PROT_WRITE,
668 MAP_SHARED | MAP_ANONYMOUS,
669 buffer->fd, 0);
670 ASSERT_NE(buffer->ptr, MAP_FAILED);
671
672 /* Initialize buffer->ptr so we can tell if it is written. */
673 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
674 ptr[i] = i;
675
676 /* Initialize data that the device will write to buffer->ptr. */
677 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
678 ptr[i] = -i;
679
680 pid = fork();
681 if (pid == -1)
682 ASSERT_EQ(pid, 0);
683 if (pid != 0) {
684 waitpid(pid, &ret, 0);
685 ASSERT_EQ(WIFEXITED(ret), 1);
686
687 /* Check that the parent's buffer did change. */
688 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
689 ASSERT_EQ(ptr[i], -i);
690 return;
691 }
692
693 /* Check that we see the parent's values. */
694 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
695 ASSERT_EQ(ptr[i], i);
696 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
697 ASSERT_EQ(ptr[i], -i);
698
699 /* The child process needs its own mirror to its own mm. */
700 child_fd = hmm_open(0);
701 ASSERT_GE(child_fd, 0);
702
703 /* Simulate a device writing system memory. */
704 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);
705 ASSERT_EQ(ret, 0);
706 ASSERT_EQ(buffer->cpages, npages);
707 ASSERT_EQ(buffer->faults, 1);
708
709 /* Check what the device wrote. */
710 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
711 ASSERT_EQ(ptr[i], -i);
712
713 close(child_fd);
714 exit(0);
715 }
716
717 /*
718 * Write private anonymous huge page.
719 */
TEST_F(hmm,anon_write_huge)720 TEST_F(hmm, anon_write_huge)
721 {
722 struct hmm_buffer *buffer;
723 unsigned long npages;
724 unsigned long size;
725 unsigned long i;
726 void *old_ptr;
727 void *map;
728 int *ptr;
729 int ret;
730
731 size = 2 * TWOMEG;
732
733 buffer = malloc(sizeof(*buffer));
734 ASSERT_NE(buffer, NULL);
735
736 buffer->fd = -1;
737 buffer->size = size;
738 buffer->mirror = malloc(size);
739 ASSERT_NE(buffer->mirror, NULL);
740
741 buffer->ptr = mmap(NULL, size,
742 PROT_READ | PROT_WRITE,
743 MAP_PRIVATE | MAP_ANONYMOUS,
744 buffer->fd, 0);
745 ASSERT_NE(buffer->ptr, MAP_FAILED);
746
747 size = TWOMEG;
748 npages = size >> self->page_shift;
749 map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
750 ret = madvise(map, size, MADV_HUGEPAGE);
751 ASSERT_EQ(ret, 0);
752 old_ptr = buffer->ptr;
753 buffer->ptr = map;
754
755 /* Initialize data that the device will write to buffer->ptr. */
756 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
757 ptr[i] = i;
758
759 /* Simulate a device writing system memory. */
760 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
761 ASSERT_EQ(ret, 0);
762 ASSERT_EQ(buffer->cpages, npages);
763 ASSERT_EQ(buffer->faults, 1);
764
765 /* Check what the device wrote. */
766 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
767 ASSERT_EQ(ptr[i], i);
768
769 buffer->ptr = old_ptr;
770 hmm_buffer_free(buffer);
771 }
772
773 /*
774 * Read numeric data from raw and tagged kernel status files. Used to read
775 * /proc and /sys data (without a tag) and from /proc/meminfo (with a tag).
776 */
file_read_ulong(char * file,const char * tag)777 static long file_read_ulong(char *file, const char *tag)
778 {
779 int fd;
780 char buf[2048];
781 int len;
782 char *p, *q;
783 long val;
784
785 fd = open(file, O_RDONLY);
786 if (fd < 0) {
787 /* Error opening the file */
788 return -1;
789 }
790
791 len = read(fd, buf, sizeof(buf));
792 close(fd);
793 if (len < 0) {
794 /* Error in reading the file */
795 return -1;
796 }
797 if (len == sizeof(buf)) {
798 /* Error file is too large */
799 return -1;
800 }
801 buf[len] = '\0';
802
803 /* Search for a tag if provided */
804 if (tag) {
805 p = strstr(buf, tag);
806 if (!p)
807 return -1; /* looks like the line we want isn't there */
808 p += strlen(tag);
809 } else
810 p = buf;
811
812 val = strtol(p, &q, 0);
813 if (*q != ' ') {
814 /* Error parsing the file */
815 return -1;
816 }
817
818 return val;
819 }
820
821 /*
822 * Write huge TLBFS page.
823 */
TEST_F(hmm,anon_write_hugetlbfs)824 TEST_F(hmm, anon_write_hugetlbfs)
825 {
826 struct hmm_buffer *buffer;
827 unsigned long npages;
828 unsigned long size;
829 unsigned long default_hsize;
830 unsigned long i;
831 int *ptr;
832 int ret;
833
834 default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:");
835 if (default_hsize < 0 || default_hsize*1024 < default_hsize)
836 SKIP(return, "Huge page size could not be determined");
837 default_hsize = default_hsize*1024; /* KB to B */
838
839 size = ALIGN(TWOMEG, default_hsize);
840 npages = size >> self->page_shift;
841
842 buffer = malloc(sizeof(*buffer));
843 ASSERT_NE(buffer, NULL);
844
845 buffer->ptr = mmap(NULL, size,
846 PROT_READ | PROT_WRITE,
847 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
848 -1, 0);
849 if (buffer->ptr == MAP_FAILED) {
850 free(buffer);
851 SKIP(return, "Huge page could not be allocated");
852 }
853
854 buffer->fd = -1;
855 buffer->size = size;
856 buffer->mirror = malloc(size);
857 ASSERT_NE(buffer->mirror, NULL);
858
859 /* Initialize data that the device will write to buffer->ptr. */
860 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
861 ptr[i] = i;
862
863 /* Simulate a device writing system memory. */
864 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
865 ASSERT_EQ(ret, 0);
866 ASSERT_EQ(buffer->cpages, npages);
867 ASSERT_EQ(buffer->faults, 1);
868
869 /* Check what the device wrote. */
870 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
871 ASSERT_EQ(ptr[i], i);
872
873 munmap(buffer->ptr, buffer->size);
874 buffer->ptr = NULL;
875 hmm_buffer_free(buffer);
876 }
877
878 /*
879 * Read mmap'ed file memory.
880 */
TEST_F(hmm,file_read)881 TEST_F(hmm, file_read)
882 {
883 struct hmm_buffer *buffer;
884 unsigned long npages;
885 unsigned long size;
886 unsigned long i;
887 int *ptr;
888 int ret;
889 int fd;
890 ssize_t len;
891
892 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
893 ASSERT_NE(npages, 0);
894 size = npages << self->page_shift;
895
896 fd = hmm_create_file(size);
897 ASSERT_GE(fd, 0);
898
899 buffer = malloc(sizeof(*buffer));
900 ASSERT_NE(buffer, NULL);
901
902 buffer->fd = fd;
903 buffer->size = size;
904 buffer->mirror = malloc(size);
905 ASSERT_NE(buffer->mirror, NULL);
906
907 /* Write initial contents of the file. */
908 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
909 ptr[i] = i;
910 len = pwrite(fd, buffer->mirror, size, 0);
911 ASSERT_EQ(len, size);
912 memset(buffer->mirror, 0, size);
913
914 buffer->ptr = mmap(NULL, size,
915 PROT_READ,
916 MAP_SHARED,
917 buffer->fd, 0);
918 ASSERT_NE(buffer->ptr, MAP_FAILED);
919
920 /* Simulate a device reading system memory. */
921 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
922 ASSERT_EQ(ret, 0);
923 ASSERT_EQ(buffer->cpages, npages);
924 ASSERT_EQ(buffer->faults, 1);
925
926 /* Check what the device read. */
927 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
928 ASSERT_EQ(ptr[i], i);
929
930 hmm_buffer_free(buffer);
931 }
932
933 /*
934 * Write mmap'ed file memory.
935 */
TEST_F(hmm,file_write)936 TEST_F(hmm, file_write)
937 {
938 struct hmm_buffer *buffer;
939 unsigned long npages;
940 unsigned long size;
941 unsigned long i;
942 int *ptr;
943 int ret;
944 int fd;
945 ssize_t len;
946
947 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
948 ASSERT_NE(npages, 0);
949 size = npages << self->page_shift;
950
951 fd = hmm_create_file(size);
952 ASSERT_GE(fd, 0);
953
954 buffer = malloc(sizeof(*buffer));
955 ASSERT_NE(buffer, NULL);
956
957 buffer->fd = fd;
958 buffer->size = size;
959 buffer->mirror = malloc(size);
960 ASSERT_NE(buffer->mirror, NULL);
961
962 buffer->ptr = mmap(NULL, size,
963 PROT_READ | PROT_WRITE,
964 MAP_SHARED,
965 buffer->fd, 0);
966 ASSERT_NE(buffer->ptr, MAP_FAILED);
967
968 /* Initialize data that the device will write to buffer->ptr. */
969 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
970 ptr[i] = i;
971
972 /* Simulate a device writing system memory. */
973 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
974 ASSERT_EQ(ret, 0);
975 ASSERT_EQ(buffer->cpages, npages);
976 ASSERT_EQ(buffer->faults, 1);
977
978 /* Check what the device wrote. */
979 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
980 ASSERT_EQ(ptr[i], i);
981
982 /* Check that the device also wrote the file. */
983 len = pread(fd, buffer->mirror, size, 0);
984 ASSERT_EQ(len, size);
985 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
986 ASSERT_EQ(ptr[i], i);
987
988 hmm_buffer_free(buffer);
989 }
990
991 /*
992 * Migrate anonymous memory to device private memory.
993 */
TEST_F(hmm,migrate)994 TEST_F(hmm, migrate)
995 {
996 struct hmm_buffer *buffer;
997 unsigned long npages;
998 unsigned long size;
999 unsigned long i;
1000 int *ptr;
1001 int ret;
1002
1003 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1004 ASSERT_NE(npages, 0);
1005 size = npages << self->page_shift;
1006
1007 buffer = malloc(sizeof(*buffer));
1008 ASSERT_NE(buffer, NULL);
1009
1010 buffer->fd = -1;
1011 buffer->size = size;
1012 buffer->mirror = malloc(size);
1013 ASSERT_NE(buffer->mirror, NULL);
1014
1015 buffer->ptr = mmap(NULL, size,
1016 PROT_READ | PROT_WRITE,
1017 MAP_PRIVATE | MAP_ANONYMOUS,
1018 buffer->fd, 0);
1019 ASSERT_NE(buffer->ptr, MAP_FAILED);
1020
1021 /* Initialize buffer in system memory. */
1022 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1023 ptr[i] = i;
1024
1025 /* Migrate memory to device. */
1026 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1027 ASSERT_EQ(ret, 0);
1028 ASSERT_EQ(buffer->cpages, npages);
1029
1030 /* Check what the device read. */
1031 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1032 ASSERT_EQ(ptr[i], i);
1033
1034 hmm_buffer_free(buffer);
1035 }
1036
1037 /*
1038 * Migrate anonymous memory to device private memory and fault some of it back
1039 * to system memory, then try migrating the resulting mix of system and device
1040 * private memory to the device.
1041 */
TEST_F(hmm,migrate_fault)1042 TEST_F(hmm, migrate_fault)
1043 {
1044 struct hmm_buffer *buffer;
1045 unsigned long npages;
1046 unsigned long size;
1047 unsigned long i;
1048 int *ptr;
1049 int ret;
1050
1051 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1052 ASSERT_NE(npages, 0);
1053 size = npages << self->page_shift;
1054
1055 buffer = malloc(sizeof(*buffer));
1056 ASSERT_NE(buffer, NULL);
1057
1058 buffer->fd = -1;
1059 buffer->size = size;
1060 buffer->mirror = malloc(size);
1061 ASSERT_NE(buffer->mirror, NULL);
1062
1063 buffer->ptr = mmap(NULL, size,
1064 PROT_READ | PROT_WRITE,
1065 MAP_PRIVATE | MAP_ANONYMOUS,
1066 buffer->fd, 0);
1067 ASSERT_NE(buffer->ptr, MAP_FAILED);
1068
1069 /* Initialize buffer in system memory. */
1070 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1071 ptr[i] = i;
1072
1073 /* Migrate memory to device. */
1074 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1075 ASSERT_EQ(ret, 0);
1076 ASSERT_EQ(buffer->cpages, npages);
1077
1078 /* Check what the device read. */
1079 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1080 ASSERT_EQ(ptr[i], i);
1081
1082 /* Fault half the pages back to system memory and check them. */
1083 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
1084 ASSERT_EQ(ptr[i], i);
1085
1086 /* Migrate memory to the device again. */
1087 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1088 ASSERT_EQ(ret, 0);
1089 ASSERT_EQ(buffer->cpages, npages);
1090
1091 /* Check what the device read. */
1092 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1093 ASSERT_EQ(ptr[i], i);
1094
1095 hmm_buffer_free(buffer);
1096 }
1097
TEST_F(hmm,migrate_release)1098 TEST_F(hmm, migrate_release)
1099 {
1100 struct hmm_buffer *buffer;
1101 unsigned long npages;
1102 unsigned long size;
1103 unsigned long i;
1104 int *ptr;
1105 int ret;
1106
1107 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1108 ASSERT_NE(npages, 0);
1109 size = npages << self->page_shift;
1110
1111 buffer = malloc(sizeof(*buffer));
1112 ASSERT_NE(buffer, NULL);
1113
1114 buffer->fd = -1;
1115 buffer->size = size;
1116 buffer->mirror = malloc(size);
1117 ASSERT_NE(buffer->mirror, NULL);
1118
1119 buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
1120 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
1121 ASSERT_NE(buffer->ptr, MAP_FAILED);
1122
1123 /* Initialize buffer in system memory. */
1124 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1125 ptr[i] = i;
1126
1127 /* Migrate memory to device. */
1128 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1129 ASSERT_EQ(ret, 0);
1130 ASSERT_EQ(buffer->cpages, npages);
1131
1132 /* Check what the device read. */
1133 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1134 ASSERT_EQ(ptr[i], i);
1135
1136 /* Release device memory. */
1137 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages);
1138 ASSERT_EQ(ret, 0);
1139
1140 /* Fault pages back to system memory and check them. */
1141 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
1142 ASSERT_EQ(ptr[i], i);
1143
1144 hmm_buffer_free(buffer);
1145 }
1146
1147 /*
1148 * Migrate anonymous shared memory to device private memory.
1149 */
TEST_F(hmm,migrate_shared)1150 TEST_F(hmm, migrate_shared)
1151 {
1152 struct hmm_buffer *buffer;
1153 unsigned long npages;
1154 unsigned long size;
1155 int ret;
1156
1157 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1158 ASSERT_NE(npages, 0);
1159 size = npages << self->page_shift;
1160
1161 buffer = malloc(sizeof(*buffer));
1162 ASSERT_NE(buffer, NULL);
1163
1164 buffer->fd = -1;
1165 buffer->size = size;
1166 buffer->mirror = malloc(size);
1167 ASSERT_NE(buffer->mirror, NULL);
1168
1169 buffer->ptr = mmap(NULL, size,
1170 PROT_READ | PROT_WRITE,
1171 MAP_SHARED | MAP_ANONYMOUS,
1172 buffer->fd, 0);
1173 ASSERT_NE(buffer->ptr, MAP_FAILED);
1174
1175 /* Migrate memory to device. */
1176 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1177 ASSERT_EQ(ret, -ENOENT);
1178
1179 hmm_buffer_free(buffer);
1180 }
1181
1182 /*
1183 * Try to migrate various memory types to device private memory.
1184 */
TEST_F(hmm2,migrate_mixed)1185 TEST_F(hmm2, migrate_mixed)
1186 {
1187 struct hmm_buffer *buffer;
1188 unsigned long npages;
1189 unsigned long size;
1190 int *ptr;
1191 unsigned char *p;
1192 int ret;
1193 int val;
1194
1195 npages = 6;
1196 size = npages << self->page_shift;
1197
1198 buffer = malloc(sizeof(*buffer));
1199 ASSERT_NE(buffer, NULL);
1200
1201 buffer->fd = -1;
1202 buffer->size = size;
1203 buffer->mirror = malloc(size);
1204 ASSERT_NE(buffer->mirror, NULL);
1205
1206 /* Reserve a range of addresses. */
1207 buffer->ptr = mmap(NULL, size,
1208 PROT_NONE,
1209 MAP_PRIVATE | MAP_ANONYMOUS,
1210 buffer->fd, 0);
1211 ASSERT_NE(buffer->ptr, MAP_FAILED);
1212 p = buffer->ptr;
1213
1214 /* Migrating a protected area should be an error. */
1215 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages);
1216 ASSERT_EQ(ret, -EINVAL);
1217
1218 /* Punch a hole after the first page address. */
1219 ret = munmap(buffer->ptr + self->page_size, self->page_size);
1220 ASSERT_EQ(ret, 0);
1221
1222 /* We expect an error if the vma doesn't cover the range. */
1223 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3);
1224 ASSERT_EQ(ret, -EINVAL);
1225
1226 /* Page 2 will be a read-only zero page. */
1227 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,
1228 PROT_READ);
1229 ASSERT_EQ(ret, 0);
1230 ptr = (int *)(buffer->ptr + 2 * self->page_size);
1231 val = *ptr + 3;
1232 ASSERT_EQ(val, 3);
1233
1234 /* Page 3 will be read-only. */
1235 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1236 PROT_READ | PROT_WRITE);
1237 ASSERT_EQ(ret, 0);
1238 ptr = (int *)(buffer->ptr + 3 * self->page_size);
1239 *ptr = val;
1240 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1241 PROT_READ);
1242 ASSERT_EQ(ret, 0);
1243
1244 /* Page 4-5 will be read-write. */
1245 ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size,
1246 PROT_READ | PROT_WRITE);
1247 ASSERT_EQ(ret, 0);
1248 ptr = (int *)(buffer->ptr + 4 * self->page_size);
1249 *ptr = val;
1250 ptr = (int *)(buffer->ptr + 5 * self->page_size);
1251 *ptr = val;
1252
1253 /* Now try to migrate pages 2-5 to device 1. */
1254 buffer->ptr = p + 2 * self->page_size;
1255 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4);
1256 ASSERT_EQ(ret, 0);
1257 ASSERT_EQ(buffer->cpages, 4);
1258
1259 /* Page 5 won't be migrated to device 0 because it's on device 1. */
1260 buffer->ptr = p + 5 * self->page_size;
1261 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);
1262 ASSERT_EQ(ret, -ENOENT);
1263 buffer->ptr = p;
1264
1265 buffer->ptr = p;
1266 hmm_buffer_free(buffer);
1267 }
1268
1269 /*
1270 * Migrate anonymous memory to device memory and back to system memory
1271 * multiple times. In case of private zone configuration, this is done
1272 * through fault pages accessed by CPU. In case of coherent zone configuration,
1273 * the pages from the device should be explicitly migrated back to system memory.
1274 * The reason is Coherent device zone has coherent access by CPU, therefore
1275 * it will not generate any page fault.
1276 */
TEST_F(hmm,migrate_multiple)1277 TEST_F(hmm, migrate_multiple)
1278 {
1279 struct hmm_buffer *buffer;
1280 unsigned long npages;
1281 unsigned long size;
1282 unsigned long i;
1283 unsigned long c;
1284 int *ptr;
1285 int ret;
1286
1287 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1288 ASSERT_NE(npages, 0);
1289 size = npages << self->page_shift;
1290
1291 for (c = 0; c < NTIMES; c++) {
1292 buffer = malloc(sizeof(*buffer));
1293 ASSERT_NE(buffer, NULL);
1294
1295 buffer->fd = -1;
1296 buffer->size = size;
1297 buffer->mirror = malloc(size);
1298 ASSERT_NE(buffer->mirror, NULL);
1299
1300 buffer->ptr = mmap(NULL, size,
1301 PROT_READ | PROT_WRITE,
1302 MAP_PRIVATE | MAP_ANONYMOUS,
1303 buffer->fd, 0);
1304 ASSERT_NE(buffer->ptr, MAP_FAILED);
1305
1306 /* Initialize buffer in system memory. */
1307 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1308 ptr[i] = i;
1309
1310 /* Migrate memory to device. */
1311 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1312 ASSERT_EQ(ret, 0);
1313 ASSERT_EQ(buffer->cpages, npages);
1314
1315 /* Check what the device read. */
1316 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1317 ASSERT_EQ(ptr[i], i);
1318
1319 /* Migrate back to system memory and check them. */
1320 if (hmm_is_coherent_type(variant->device_number)) {
1321 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages);
1322 ASSERT_EQ(ret, 0);
1323 ASSERT_EQ(buffer->cpages, npages);
1324 }
1325
1326 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1327 ASSERT_EQ(ptr[i], i);
1328
1329 hmm_buffer_free(buffer);
1330 }
1331 }
1332
1333 /*
1334 * Read anonymous memory multiple times.
1335 */
TEST_F(hmm,anon_read_multiple)1336 TEST_F(hmm, anon_read_multiple)
1337 {
1338 struct hmm_buffer *buffer;
1339 unsigned long npages;
1340 unsigned long size;
1341 unsigned long i;
1342 unsigned long c;
1343 int *ptr;
1344 int ret;
1345
1346 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1347 ASSERT_NE(npages, 0);
1348 size = npages << self->page_shift;
1349
1350 for (c = 0; c < NTIMES; c++) {
1351 buffer = malloc(sizeof(*buffer));
1352 ASSERT_NE(buffer, NULL);
1353
1354 buffer->fd = -1;
1355 buffer->size = size;
1356 buffer->mirror = malloc(size);
1357 ASSERT_NE(buffer->mirror, NULL);
1358
1359 buffer->ptr = mmap(NULL, size,
1360 PROT_READ | PROT_WRITE,
1361 MAP_PRIVATE | MAP_ANONYMOUS,
1362 buffer->fd, 0);
1363 ASSERT_NE(buffer->ptr, MAP_FAILED);
1364
1365 /* Initialize buffer in system memory. */
1366 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1367 ptr[i] = i + c;
1368
1369 /* Simulate a device reading system memory. */
1370 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,
1371 npages);
1372 ASSERT_EQ(ret, 0);
1373 ASSERT_EQ(buffer->cpages, npages);
1374 ASSERT_EQ(buffer->faults, 1);
1375
1376 /* Check what the device read. */
1377 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1378 ASSERT_EQ(ptr[i], i + c);
1379
1380 hmm_buffer_free(buffer);
1381 }
1382 }
1383
unmap_buffer(void * p)1384 void *unmap_buffer(void *p)
1385 {
1386 struct hmm_buffer *buffer = p;
1387
1388 /* Delay for a bit and then unmap buffer while it is being read. */
1389 hmm_nanosleep(hmm_random() % 32000);
1390 munmap(buffer->ptr + buffer->size / 2, buffer->size / 2);
1391 buffer->ptr = NULL;
1392
1393 return NULL;
1394 }
1395
1396 /*
1397 * Try reading anonymous memory while it is being unmapped.
1398 */
TEST_F(hmm,anon_teardown)1399 TEST_F(hmm, anon_teardown)
1400 {
1401 unsigned long npages;
1402 unsigned long size;
1403 unsigned long c;
1404 void *ret;
1405
1406 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1407 ASSERT_NE(npages, 0);
1408 size = npages << self->page_shift;
1409
1410 for (c = 0; c < NTIMES; ++c) {
1411 pthread_t thread;
1412 struct hmm_buffer *buffer;
1413 unsigned long i;
1414 int *ptr;
1415 int rc;
1416
1417 buffer = malloc(sizeof(*buffer));
1418 ASSERT_NE(buffer, NULL);
1419
1420 buffer->fd = -1;
1421 buffer->size = size;
1422 buffer->mirror = malloc(size);
1423 ASSERT_NE(buffer->mirror, NULL);
1424
1425 buffer->ptr = mmap(NULL, size,
1426 PROT_READ | PROT_WRITE,
1427 MAP_PRIVATE | MAP_ANONYMOUS,
1428 buffer->fd, 0);
1429 ASSERT_NE(buffer->ptr, MAP_FAILED);
1430
1431 /* Initialize buffer in system memory. */
1432 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1433 ptr[i] = i + c;
1434
1435 rc = pthread_create(&thread, NULL, unmap_buffer, buffer);
1436 ASSERT_EQ(rc, 0);
1437
1438 /* Simulate a device reading system memory. */
1439 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,
1440 npages);
1441 if (rc == 0) {
1442 ASSERT_EQ(buffer->cpages, npages);
1443 ASSERT_EQ(buffer->faults, 1);
1444
1445 /* Check what the device read. */
1446 for (i = 0, ptr = buffer->mirror;
1447 i < size / sizeof(*ptr);
1448 ++i)
1449 ASSERT_EQ(ptr[i], i + c);
1450 }
1451
1452 pthread_join(thread, &ret);
1453 hmm_buffer_free(buffer);
1454 }
1455 }
1456
1457 /*
1458 * Test memory snapshot without faulting in pages accessed by the device.
1459 */
TEST_F(hmm,mixedmap)1460 TEST_F(hmm, mixedmap)
1461 {
1462 struct hmm_buffer *buffer;
1463 unsigned long npages;
1464 unsigned long size;
1465 unsigned char *m;
1466 int ret;
1467
1468 npages = 1;
1469 size = npages << self->page_shift;
1470
1471 buffer = malloc(sizeof(*buffer));
1472 ASSERT_NE(buffer, NULL);
1473
1474 buffer->fd = -1;
1475 buffer->size = size;
1476 buffer->mirror = malloc(npages);
1477 ASSERT_NE(buffer->mirror, NULL);
1478
1479
1480 /* Reserve a range of addresses. */
1481 buffer->ptr = mmap(NULL, size,
1482 PROT_READ | PROT_WRITE,
1483 MAP_PRIVATE,
1484 self->fd, 0);
1485 ASSERT_NE(buffer->ptr, MAP_FAILED);
1486
1487 /* Simulate a device snapshotting CPU pagetables. */
1488 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1489 ASSERT_EQ(ret, 0);
1490 ASSERT_EQ(buffer->cpages, npages);
1491
1492 /* Check what the device saw. */
1493 m = buffer->mirror;
1494 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ);
1495
1496 hmm_buffer_free(buffer);
1497 }
1498
1499 /*
1500 * Test memory snapshot without faulting in pages accessed by the device.
1501 */
TEST_F(hmm2,snapshot)1502 TEST_F(hmm2, snapshot)
1503 {
1504 struct hmm_buffer *buffer;
1505 unsigned long npages;
1506 unsigned long size;
1507 int *ptr;
1508 unsigned char *p;
1509 unsigned char *m;
1510 int ret;
1511 int val;
1512
1513 npages = 7;
1514 size = npages << self->page_shift;
1515
1516 buffer = malloc(sizeof(*buffer));
1517 ASSERT_NE(buffer, NULL);
1518
1519 buffer->fd = -1;
1520 buffer->size = size;
1521 buffer->mirror = malloc(npages);
1522 ASSERT_NE(buffer->mirror, NULL);
1523
1524 /* Reserve a range of addresses. */
1525 buffer->ptr = mmap(NULL, size,
1526 PROT_NONE,
1527 MAP_PRIVATE | MAP_ANONYMOUS,
1528 buffer->fd, 0);
1529 ASSERT_NE(buffer->ptr, MAP_FAILED);
1530 p = buffer->ptr;
1531
1532 /* Punch a hole after the first page address. */
1533 ret = munmap(buffer->ptr + self->page_size, self->page_size);
1534 ASSERT_EQ(ret, 0);
1535
1536 /* Page 2 will be read-only zero page. */
1537 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,
1538 PROT_READ);
1539 ASSERT_EQ(ret, 0);
1540 ptr = (int *)(buffer->ptr + 2 * self->page_size);
1541 val = *ptr + 3;
1542 ASSERT_EQ(val, 3);
1543
1544 /* Page 3 will be read-only. */
1545 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1546 PROT_READ | PROT_WRITE);
1547 ASSERT_EQ(ret, 0);
1548 ptr = (int *)(buffer->ptr + 3 * self->page_size);
1549 *ptr = val;
1550 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1551 PROT_READ);
1552 ASSERT_EQ(ret, 0);
1553
1554 /* Page 4-6 will be read-write. */
1555 ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size,
1556 PROT_READ | PROT_WRITE);
1557 ASSERT_EQ(ret, 0);
1558 ptr = (int *)(buffer->ptr + 4 * self->page_size);
1559 *ptr = val;
1560
1561 /* Page 5 will be migrated to device 0. */
1562 buffer->ptr = p + 5 * self->page_size;
1563 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);
1564 ASSERT_EQ(ret, 0);
1565 ASSERT_EQ(buffer->cpages, 1);
1566
1567 /* Page 6 will be migrated to device 1. */
1568 buffer->ptr = p + 6 * self->page_size;
1569 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1);
1570 ASSERT_EQ(ret, 0);
1571 ASSERT_EQ(buffer->cpages, 1);
1572
1573 /* Simulate a device snapshotting CPU pagetables. */
1574 buffer->ptr = p;
1575 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1576 ASSERT_EQ(ret, 0);
1577 ASSERT_EQ(buffer->cpages, npages);
1578
1579 /* Check what the device saw. */
1580 m = buffer->mirror;
1581 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR);
1582 ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR);
1583 ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ);
1584 ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ);
1585 ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE);
1586 if (!hmm_is_coherent_type(variant->device_number0)) {
1587 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL |
1588 HMM_DMIRROR_PROT_WRITE);
1589 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE);
1590 } else {
1591 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL |
1592 HMM_DMIRROR_PROT_WRITE);
1593 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE |
1594 HMM_DMIRROR_PROT_WRITE);
1595 }
1596
1597 hmm_buffer_free(buffer);
1598 }
1599
1600 /*
1601 * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that
1602 * should be mapped by a large page table entry.
1603 */
TEST_F(hmm,compound)1604 TEST_F(hmm, compound)
1605 {
1606 struct hmm_buffer *buffer;
1607 unsigned long npages;
1608 unsigned long size;
1609 unsigned long default_hsize;
1610 int *ptr;
1611 unsigned char *m;
1612 int ret;
1613 unsigned long i;
1614
1615 /* Skip test if we can't allocate a hugetlbfs page. */
1616
1617 default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:");
1618 if (default_hsize < 0 || default_hsize*1024 < default_hsize)
1619 SKIP(return, "Huge page size could not be determined");
1620 default_hsize = default_hsize*1024; /* KB to B */
1621
1622 size = ALIGN(TWOMEG, default_hsize);
1623 npages = size >> self->page_shift;
1624
1625 buffer = malloc(sizeof(*buffer));
1626 ASSERT_NE(buffer, NULL);
1627
1628 buffer->ptr = mmap(NULL, size,
1629 PROT_READ | PROT_WRITE,
1630 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
1631 -1, 0);
1632 if (buffer->ptr == MAP_FAILED) {
1633 free(buffer);
1634 return;
1635 }
1636
1637 buffer->size = size;
1638 buffer->mirror = malloc(npages);
1639 ASSERT_NE(buffer->mirror, NULL);
1640
1641 /* Initialize the pages the device will snapshot in buffer->ptr. */
1642 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1643 ptr[i] = i;
1644
1645 /* Simulate a device snapshotting CPU pagetables. */
1646 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1647 ASSERT_EQ(ret, 0);
1648 ASSERT_EQ(buffer->cpages, npages);
1649
1650 /* Check what the device saw. */
1651 m = buffer->mirror;
1652 for (i = 0; i < npages; ++i)
1653 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE |
1654 HMM_DMIRROR_PROT_PMD);
1655
1656 /* Make the region read-only. */
1657 ret = mprotect(buffer->ptr, size, PROT_READ);
1658 ASSERT_EQ(ret, 0);
1659
1660 /* Simulate a device snapshotting CPU pagetables. */
1661 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1662 ASSERT_EQ(ret, 0);
1663 ASSERT_EQ(buffer->cpages, npages);
1664
1665 /* Check what the device saw. */
1666 m = buffer->mirror;
1667 for (i = 0; i < npages; ++i)
1668 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ |
1669 HMM_DMIRROR_PROT_PMD);
1670
1671 munmap(buffer->ptr, buffer->size);
1672 buffer->ptr = NULL;
1673 hmm_buffer_free(buffer);
1674 }
1675
1676 /*
1677 * Test two devices reading the same memory (double mapped).
1678 */
TEST_F(hmm2,double_map)1679 TEST_F(hmm2, double_map)
1680 {
1681 struct hmm_buffer *buffer;
1682 unsigned long npages;
1683 unsigned long size;
1684 unsigned long i;
1685 int *ptr;
1686 int ret;
1687
1688 npages = 6;
1689 size = npages << self->page_shift;
1690
1691 buffer = malloc(sizeof(*buffer));
1692 ASSERT_NE(buffer, NULL);
1693
1694 buffer->fd = -1;
1695 buffer->size = size;
1696 buffer->mirror = malloc(size);
1697 ASSERT_NE(buffer->mirror, NULL);
1698
1699 /* Reserve a range of addresses. */
1700 buffer->ptr = mmap(NULL, size,
1701 PROT_READ | PROT_WRITE,
1702 MAP_PRIVATE | MAP_ANONYMOUS,
1703 buffer->fd, 0);
1704 ASSERT_NE(buffer->ptr, MAP_FAILED);
1705
1706 /* Initialize buffer in system memory. */
1707 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1708 ptr[i] = i;
1709
1710 /* Make region read-only. */
1711 ret = mprotect(buffer->ptr, size, PROT_READ);
1712 ASSERT_EQ(ret, 0);
1713
1714 /* Simulate device 0 reading system memory. */
1715 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);
1716 ASSERT_EQ(ret, 0);
1717 ASSERT_EQ(buffer->cpages, npages);
1718 ASSERT_EQ(buffer->faults, 1);
1719
1720 /* Check what the device read. */
1721 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1722 ASSERT_EQ(ptr[i], i);
1723
1724 /* Simulate device 1 reading system memory. */
1725 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages);
1726 ASSERT_EQ(ret, 0);
1727 ASSERT_EQ(buffer->cpages, npages);
1728 ASSERT_EQ(buffer->faults, 1);
1729
1730 /* Check what the device read. */
1731 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1732 ASSERT_EQ(ptr[i], i);
1733
1734 /* Migrate pages to device 1 and try to read from device 0. */
1735 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages);
1736 ASSERT_EQ(ret, 0);
1737 ASSERT_EQ(buffer->cpages, npages);
1738
1739 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);
1740 ASSERT_EQ(ret, 0);
1741 ASSERT_EQ(buffer->cpages, npages);
1742 ASSERT_EQ(buffer->faults, 1);
1743
1744 /* Check what device 0 read. */
1745 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1746 ASSERT_EQ(ptr[i], i);
1747
1748 hmm_buffer_free(buffer);
1749 }
1750
1751 /*
1752 * Basic check of exclusive faulting.
1753 */
TEST_F(hmm,exclusive)1754 TEST_F(hmm, exclusive)
1755 {
1756 struct hmm_buffer *buffer;
1757 unsigned long npages;
1758 unsigned long size;
1759 unsigned long i;
1760 int *ptr;
1761 int ret;
1762
1763 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1764 ASSERT_NE(npages, 0);
1765 size = npages << self->page_shift;
1766
1767 buffer = malloc(sizeof(*buffer));
1768 ASSERT_NE(buffer, NULL);
1769
1770 buffer->fd = -1;
1771 buffer->size = size;
1772 buffer->mirror = malloc(size);
1773 ASSERT_NE(buffer->mirror, NULL);
1774
1775 buffer->ptr = mmap(NULL, size,
1776 PROT_READ | PROT_WRITE,
1777 MAP_PRIVATE | MAP_ANONYMOUS,
1778 buffer->fd, 0);
1779 ASSERT_NE(buffer->ptr, MAP_FAILED);
1780
1781 /* Initialize buffer in system memory. */
1782 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1783 ptr[i] = i;
1784
1785 /* Map memory exclusively for device access. */
1786 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1787 ASSERT_EQ(ret, 0);
1788 ASSERT_EQ(buffer->cpages, npages);
1789
1790 /* Check what the device read. */
1791 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1792 ASSERT_EQ(ptr[i], i);
1793
1794 /* Fault pages back to system memory and check them. */
1795 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1796 ASSERT_EQ(ptr[i]++, i);
1797
1798 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1799 ASSERT_EQ(ptr[i], i+1);
1800
1801 /* Check atomic access revoked */
1802 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages);
1803 ASSERT_EQ(ret, 0);
1804
1805 hmm_buffer_free(buffer);
1806 }
1807
TEST_F(hmm,exclusive_mprotect)1808 TEST_F(hmm, exclusive_mprotect)
1809 {
1810 struct hmm_buffer *buffer;
1811 unsigned long npages;
1812 unsigned long size;
1813 unsigned long i;
1814 int *ptr;
1815 int ret;
1816
1817 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1818 ASSERT_NE(npages, 0);
1819 size = npages << self->page_shift;
1820
1821 buffer = malloc(sizeof(*buffer));
1822 ASSERT_NE(buffer, NULL);
1823
1824 buffer->fd = -1;
1825 buffer->size = size;
1826 buffer->mirror = malloc(size);
1827 ASSERT_NE(buffer->mirror, NULL);
1828
1829 buffer->ptr = mmap(NULL, size,
1830 PROT_READ | PROT_WRITE,
1831 MAP_PRIVATE | MAP_ANONYMOUS,
1832 buffer->fd, 0);
1833 ASSERT_NE(buffer->ptr, MAP_FAILED);
1834
1835 /* Initialize buffer in system memory. */
1836 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1837 ptr[i] = i;
1838
1839 /* Map memory exclusively for device access. */
1840 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1841 ASSERT_EQ(ret, 0);
1842 ASSERT_EQ(buffer->cpages, npages);
1843
1844 /* Check what the device read. */
1845 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1846 ASSERT_EQ(ptr[i], i);
1847
1848 ret = mprotect(buffer->ptr, size, PROT_READ);
1849 ASSERT_EQ(ret, 0);
1850
1851 /* Simulate a device writing system memory. */
1852 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
1853 ASSERT_EQ(ret, -EPERM);
1854
1855 hmm_buffer_free(buffer);
1856 }
1857
1858 /*
1859 * Check copy-on-write works.
1860 */
TEST_F(hmm,exclusive_cow)1861 TEST_F(hmm, exclusive_cow)
1862 {
1863 struct hmm_buffer *buffer;
1864 unsigned long npages;
1865 unsigned long size;
1866 unsigned long i;
1867 int *ptr;
1868 int ret;
1869
1870 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1871 ASSERT_NE(npages, 0);
1872 size = npages << self->page_shift;
1873
1874 buffer = malloc(sizeof(*buffer));
1875 ASSERT_NE(buffer, NULL);
1876
1877 buffer->fd = -1;
1878 buffer->size = size;
1879 buffer->mirror = malloc(size);
1880 ASSERT_NE(buffer->mirror, NULL);
1881
1882 buffer->ptr = mmap(NULL, size,
1883 PROT_READ | PROT_WRITE,
1884 MAP_PRIVATE | MAP_ANONYMOUS,
1885 buffer->fd, 0);
1886 ASSERT_NE(buffer->ptr, MAP_FAILED);
1887
1888 /* Initialize buffer in system memory. */
1889 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1890 ptr[i] = i;
1891
1892 /* Map memory exclusively for device access. */
1893 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1894 ASSERT_EQ(ret, 0);
1895 ASSERT_EQ(buffer->cpages, npages);
1896
1897 fork();
1898
1899 /* Fault pages back to system memory and check them. */
1900 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1901 ASSERT_EQ(ptr[i]++, i);
1902
1903 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1904 ASSERT_EQ(ptr[i], i+1);
1905
1906 hmm_buffer_free(buffer);
1907 }
1908
gup_test_exec(int gup_fd,unsigned long addr,int cmd,int npages,int size,int flags)1909 static int gup_test_exec(int gup_fd, unsigned long addr, int cmd,
1910 int npages, int size, int flags)
1911 {
1912 struct gup_test gup = {
1913 .nr_pages_per_call = npages,
1914 .addr = addr,
1915 .gup_flags = FOLL_WRITE | flags,
1916 .size = size,
1917 };
1918
1919 if (ioctl(gup_fd, cmd, &gup)) {
1920 perror("ioctl on error\n");
1921 return errno;
1922 }
1923
1924 return 0;
1925 }
1926
1927 /*
1928 * Test get user device pages through gup_test. Setting PIN_LONGTERM flag.
1929 * This should trigger a migration back to system memory for both, private
1930 * and coherent type pages.
1931 * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added
1932 * to your configuration before you run it.
1933 */
TEST_F(hmm,hmm_gup_test)1934 TEST_F(hmm, hmm_gup_test)
1935 {
1936 struct hmm_buffer *buffer;
1937 int gup_fd;
1938 unsigned long npages;
1939 unsigned long size;
1940 unsigned long i;
1941 int *ptr;
1942 int ret;
1943 unsigned char *m;
1944
1945 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
1946 if (gup_fd == -1)
1947 SKIP(return, "Skipping test, could not find gup_test driver");
1948
1949 npages = 4;
1950 size = npages << self->page_shift;
1951
1952 buffer = malloc(sizeof(*buffer));
1953 ASSERT_NE(buffer, NULL);
1954
1955 buffer->fd = -1;
1956 buffer->size = size;
1957 buffer->mirror = malloc(size);
1958 ASSERT_NE(buffer->mirror, NULL);
1959
1960 buffer->ptr = mmap(NULL, size,
1961 PROT_READ | PROT_WRITE,
1962 MAP_PRIVATE | MAP_ANONYMOUS,
1963 buffer->fd, 0);
1964 ASSERT_NE(buffer->ptr, MAP_FAILED);
1965
1966 /* Initialize buffer in system memory. */
1967 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1968 ptr[i] = i;
1969
1970 /* Migrate memory to device. */
1971 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1972 ASSERT_EQ(ret, 0);
1973 ASSERT_EQ(buffer->cpages, npages);
1974 /* Check what the device read. */
1975 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1976 ASSERT_EQ(ptr[i], i);
1977
1978 ASSERT_EQ(gup_test_exec(gup_fd,
1979 (unsigned long)buffer->ptr,
1980 GUP_BASIC_TEST, 1, self->page_size, 0), 0);
1981 ASSERT_EQ(gup_test_exec(gup_fd,
1982 (unsigned long)buffer->ptr + 1 * self->page_size,
1983 GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0);
1984 ASSERT_EQ(gup_test_exec(gup_fd,
1985 (unsigned long)buffer->ptr + 2 * self->page_size,
1986 PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0);
1987 ASSERT_EQ(gup_test_exec(gup_fd,
1988 (unsigned long)buffer->ptr + 3 * self->page_size,
1989 PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0);
1990
1991 /* Take snapshot to CPU pagetables */
1992 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1993 ASSERT_EQ(ret, 0);
1994 ASSERT_EQ(buffer->cpages, npages);
1995 m = buffer->mirror;
1996 if (hmm_is_coherent_type(variant->device_number)) {
1997 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]);
1998 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]);
1999 } else {
2000 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]);
2001 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]);
2002 }
2003 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]);
2004 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]);
2005 /*
2006 * Check again the content on the pages. Make sure there's no
2007 * corrupted data.
2008 */
2009 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2010 ASSERT_EQ(ptr[i], i);
2011
2012 close(gup_fd);
2013 hmm_buffer_free(buffer);
2014 }
2015
2016 /*
2017 * Test copy-on-write in device pages.
2018 * In case of writing to COW private page(s), a page fault will migrate pages
2019 * back to system memory first. Then, these pages will be duplicated. In case
2020 * of COW device coherent type, pages are duplicated directly from device
2021 * memory.
2022 */
TEST_F(hmm,hmm_cow_in_device)2023 TEST_F(hmm, hmm_cow_in_device)
2024 {
2025 struct hmm_buffer *buffer;
2026 unsigned long npages;
2027 unsigned long size;
2028 unsigned long i;
2029 int *ptr;
2030 int ret;
2031 unsigned char *m;
2032 pid_t pid;
2033 int status;
2034
2035 npages = 4;
2036 size = npages << self->page_shift;
2037
2038 buffer = malloc(sizeof(*buffer));
2039 ASSERT_NE(buffer, NULL);
2040
2041 buffer->fd = -1;
2042 buffer->size = size;
2043 buffer->mirror = malloc(size);
2044 ASSERT_NE(buffer->mirror, NULL);
2045
2046 buffer->ptr = mmap(NULL, size,
2047 PROT_READ | PROT_WRITE,
2048 MAP_PRIVATE | MAP_ANONYMOUS,
2049 buffer->fd, 0);
2050 ASSERT_NE(buffer->ptr, MAP_FAILED);
2051
2052 /* Initialize buffer in system memory. */
2053 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2054 ptr[i] = i;
2055
2056 /* Migrate memory to device. */
2057
2058 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2059 ASSERT_EQ(ret, 0);
2060 ASSERT_EQ(buffer->cpages, npages);
2061
2062 pid = fork();
2063 if (pid == -1)
2064 ASSERT_EQ(pid, 0);
2065 if (!pid) {
2066 /* Child process waits for SIGTERM from the parent. */
2067 while (1) {
2068 }
2069 /* Should not reach this */
2070 }
2071 /* Parent process writes to COW pages(s) and gets a
2072 * new copy in system. In case of device private pages,
2073 * this write causes a migration to system mem first.
2074 */
2075 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2076 ptr[i] = i;
2077
2078 /* Terminate child and wait */
2079 EXPECT_EQ(0, kill(pid, SIGTERM));
2080 EXPECT_EQ(pid, waitpid(pid, &status, 0));
2081 EXPECT_NE(0, WIFSIGNALED(status));
2082 EXPECT_EQ(SIGTERM, WTERMSIG(status));
2083
2084 /* Take snapshot to CPU pagetables */
2085 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
2086 ASSERT_EQ(ret, 0);
2087 ASSERT_EQ(buffer->cpages, npages);
2088 m = buffer->mirror;
2089 for (i = 0; i < npages; i++)
2090 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]);
2091
2092 hmm_buffer_free(buffer);
2093 }
2094
2095 /*
2096 * Migrate private anonymous huge empty page.
2097 */
TEST_F(hmm,migrate_anon_huge_empty)2098 TEST_F(hmm, migrate_anon_huge_empty)
2099 {
2100 struct hmm_buffer *buffer;
2101 unsigned long npages;
2102 unsigned long size;
2103 unsigned long i;
2104 void *old_ptr;
2105 void *map;
2106 int *ptr;
2107 int ret;
2108
2109 size = TWOMEG;
2110
2111 buffer = malloc(sizeof(*buffer));
2112 ASSERT_NE(buffer, NULL);
2113
2114 buffer->fd = -1;
2115 buffer->size = 2 * size;
2116 buffer->mirror = malloc(size);
2117 ASSERT_NE(buffer->mirror, NULL);
2118 memset(buffer->mirror, 0xFF, size);
2119
2120 buffer->ptr = mmap(NULL, 2 * size,
2121 PROT_READ,
2122 MAP_PRIVATE | MAP_ANONYMOUS,
2123 buffer->fd, 0);
2124 ASSERT_NE(buffer->ptr, MAP_FAILED);
2125
2126 npages = size >> self->page_shift;
2127 map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2128 ret = madvise(map, size, MADV_HUGEPAGE);
2129 ASSERT_EQ(ret, 0);
2130 old_ptr = buffer->ptr;
2131 buffer->ptr = map;
2132
2133 /* Migrate memory to device. */
2134 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2135 ASSERT_EQ(ret, 0);
2136 ASSERT_EQ(buffer->cpages, npages);
2137
2138 /* Check what the device read. */
2139 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2140 ASSERT_EQ(ptr[i], 0);
2141
2142 buffer->ptr = old_ptr;
2143 hmm_buffer_free(buffer);
2144 }
2145
2146 /*
2147 * Migrate private anonymous huge zero page.
2148 */
TEST_F(hmm,migrate_anon_huge_zero)2149 TEST_F(hmm, migrate_anon_huge_zero)
2150 {
2151 struct hmm_buffer *buffer;
2152 unsigned long npages;
2153 unsigned long size;
2154 unsigned long i;
2155 void *old_ptr;
2156 void *map;
2157 int *ptr;
2158 int ret;
2159 int val;
2160
2161 size = TWOMEG;
2162
2163 buffer = malloc(sizeof(*buffer));
2164 ASSERT_NE(buffer, NULL);
2165
2166 buffer->fd = -1;
2167 buffer->size = 2 * size;
2168 buffer->mirror = malloc(size);
2169 ASSERT_NE(buffer->mirror, NULL);
2170 memset(buffer->mirror, 0xFF, size);
2171
2172 buffer->ptr = mmap(NULL, 2 * size,
2173 PROT_READ,
2174 MAP_PRIVATE | MAP_ANONYMOUS,
2175 buffer->fd, 0);
2176 ASSERT_NE(buffer->ptr, MAP_FAILED);
2177
2178 npages = size >> self->page_shift;
2179 map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2180 ret = madvise(map, size, MADV_HUGEPAGE);
2181 ASSERT_EQ(ret, 0);
2182 old_ptr = buffer->ptr;
2183 buffer->ptr = map;
2184
2185 /* Initialize a read-only zero huge page. */
2186 val = *(int *)buffer->ptr;
2187 ASSERT_EQ(val, 0);
2188
2189 /* Migrate memory to device. */
2190 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2191 ASSERT_EQ(ret, 0);
2192 ASSERT_EQ(buffer->cpages, npages);
2193
2194 /* Check what the device read. */
2195 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2196 ASSERT_EQ(ptr[i], 0);
2197
2198 /* Fault pages back to system memory and check them. */
2199 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) {
2200 ASSERT_EQ(ptr[i], 0);
2201 /* If it asserts once, it probably will 500,000 times */
2202 if (ptr[i] != 0)
2203 break;
2204 }
2205
2206 buffer->ptr = old_ptr;
2207 hmm_buffer_free(buffer);
2208 }
2209
2210 /*
2211 * Migrate private anonymous huge page and free.
2212 */
TEST_F(hmm,migrate_anon_huge_free)2213 TEST_F(hmm, migrate_anon_huge_free)
2214 {
2215 struct hmm_buffer *buffer;
2216 unsigned long npages;
2217 unsigned long size;
2218 unsigned long i;
2219 void *old_ptr;
2220 void *map;
2221 int *ptr;
2222 int ret;
2223
2224 size = TWOMEG;
2225
2226 buffer = malloc(sizeof(*buffer));
2227 ASSERT_NE(buffer, NULL);
2228
2229 buffer->fd = -1;
2230 buffer->size = 2 * size;
2231 buffer->mirror = malloc(size);
2232 ASSERT_NE(buffer->mirror, NULL);
2233 memset(buffer->mirror, 0xFF, size);
2234
2235 buffer->ptr = mmap(NULL, 2 * size,
2236 PROT_READ | PROT_WRITE,
2237 MAP_PRIVATE | MAP_ANONYMOUS,
2238 buffer->fd, 0);
2239 ASSERT_NE(buffer->ptr, MAP_FAILED);
2240
2241 npages = size >> self->page_shift;
2242 map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2243 ret = madvise(map, size, MADV_HUGEPAGE);
2244 ASSERT_EQ(ret, 0);
2245 old_ptr = buffer->ptr;
2246 buffer->ptr = map;
2247
2248 /* Initialize buffer in system memory. */
2249 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2250 ptr[i] = i;
2251
2252 /* Migrate memory to device. */
2253 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2254 ASSERT_EQ(ret, 0);
2255 ASSERT_EQ(buffer->cpages, npages);
2256
2257 /* Check what the device read. */
2258 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2259 ASSERT_EQ(ptr[i], i);
2260
2261 /* Try freeing it. */
2262 ret = madvise(map, size, MADV_FREE);
2263 ASSERT_EQ(ret, 0);
2264
2265 buffer->ptr = old_ptr;
2266 hmm_buffer_free(buffer);
2267 }
2268
2269 /*
2270 * Migrate private anonymous huge page and fault back to sysmem.
2271 */
TEST_F(hmm,migrate_anon_huge_fault)2272 TEST_F(hmm, migrate_anon_huge_fault)
2273 {
2274 struct hmm_buffer *buffer;
2275 unsigned long npages;
2276 unsigned long size;
2277 unsigned long i;
2278 void *old_ptr;
2279 void *map;
2280 int *ptr;
2281 int ret;
2282
2283 size = TWOMEG;
2284
2285 buffer = malloc(sizeof(*buffer));
2286 ASSERT_NE(buffer, NULL);
2287
2288 buffer->fd = -1;
2289 buffer->size = 2 * size;
2290 buffer->mirror = malloc(size);
2291 ASSERT_NE(buffer->mirror, NULL);
2292 memset(buffer->mirror, 0xFF, size);
2293
2294 buffer->ptr = mmap(NULL, 2 * size,
2295 PROT_READ | PROT_WRITE,
2296 MAP_PRIVATE | MAP_ANONYMOUS,
2297 buffer->fd, 0);
2298 ASSERT_NE(buffer->ptr, MAP_FAILED);
2299
2300 npages = size >> self->page_shift;
2301 map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2302 ret = madvise(map, size, MADV_HUGEPAGE);
2303 ASSERT_EQ(ret, 0);
2304 old_ptr = buffer->ptr;
2305 buffer->ptr = map;
2306
2307 /* Initialize buffer in system memory. */
2308 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2309 ptr[i] = i;
2310
2311 /* Migrate memory to device. */
2312 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2313 ASSERT_EQ(ret, 0);
2314 ASSERT_EQ(buffer->cpages, npages);
2315
2316 /* Check what the device read. */
2317 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2318 ASSERT_EQ(ptr[i], i);
2319
2320 /* Fault pages back to system memory and check them. */
2321 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2322 ASSERT_EQ(ptr[i], i);
2323
2324 buffer->ptr = old_ptr;
2325 hmm_buffer_free(buffer);
2326 }
2327
2328 /*
2329 * Migrate memory and fault back to sysmem after partially unmapping.
2330 */
TEST_F(hmm,migrate_partial_unmap_fault)2331 TEST_F(hmm, migrate_partial_unmap_fault)
2332 {
2333 struct hmm_buffer *buffer;
2334 unsigned long npages;
2335 unsigned long size = TWOMEG;
2336 unsigned long i;
2337 void *old_ptr;
2338 void *map;
2339 int *ptr;
2340 int ret, j, use_thp;
2341 int offsets[] = { 0, 512 * ONEKB, ONEMEG };
2342
2343 for (use_thp = 0; use_thp < 2; ++use_thp) {
2344 for (j = 0; j < ARRAY_SIZE(offsets); ++j) {
2345 buffer = malloc(sizeof(*buffer));
2346 ASSERT_NE(buffer, NULL);
2347
2348 buffer->fd = -1;
2349 buffer->size = 2 * size;
2350 buffer->mirror = malloc(size);
2351 ASSERT_NE(buffer->mirror, NULL);
2352 memset(buffer->mirror, 0xFF, size);
2353
2354 buffer->ptr = mmap(NULL, 2 * size,
2355 PROT_READ | PROT_WRITE,
2356 MAP_PRIVATE | MAP_ANONYMOUS,
2357 buffer->fd, 0);
2358 ASSERT_NE(buffer->ptr, MAP_FAILED);
2359
2360 npages = size >> self->page_shift;
2361 map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2362 if (use_thp)
2363 ret = madvise(map, size, MADV_HUGEPAGE);
2364 else
2365 ret = madvise(map, size, MADV_NOHUGEPAGE);
2366 ASSERT_EQ(ret, 0);
2367 old_ptr = buffer->ptr;
2368 buffer->ptr = map;
2369
2370 /* Initialize buffer in system memory. */
2371 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2372 ptr[i] = i;
2373
2374 /* Migrate memory to device. */
2375 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2376 ASSERT_EQ(ret, 0);
2377 ASSERT_EQ(buffer->cpages, npages);
2378
2379 /* Check what the device read. */
2380 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2381 ASSERT_EQ(ptr[i], i);
2382
2383 munmap(buffer->ptr + offsets[j], ONEMEG);
2384
2385 /* Fault pages back to system memory and check them. */
2386 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2387 if (i * sizeof(int) < offsets[j] ||
2388 i * sizeof(int) >= offsets[j] + ONEMEG)
2389 ASSERT_EQ(ptr[i], i);
2390
2391 buffer->ptr = old_ptr;
2392 hmm_buffer_free(buffer);
2393 }
2394 }
2395 }
2396
TEST_F(hmm,migrate_remap_fault)2397 TEST_F(hmm, migrate_remap_fault)
2398 {
2399 struct hmm_buffer *buffer;
2400 unsigned long npages;
2401 unsigned long size = TWOMEG;
2402 unsigned long i;
2403 void *old_ptr, *new_ptr = NULL;
2404 void *map;
2405 int *ptr;
2406 int ret, j, use_thp, dont_unmap, before;
2407 int offsets[] = { 0, 512 * ONEKB, ONEMEG };
2408
2409 for (before = 0; before < 2; ++before) {
2410 for (dont_unmap = 0; dont_unmap < 2; ++dont_unmap) {
2411 for (use_thp = 0; use_thp < 2; ++use_thp) {
2412 for (j = 0; j < ARRAY_SIZE(offsets); ++j) {
2413 int flags = MREMAP_MAYMOVE | MREMAP_FIXED;
2414
2415 if (dont_unmap)
2416 flags |= MREMAP_DONTUNMAP;
2417
2418 buffer = malloc(sizeof(*buffer));
2419 ASSERT_NE(buffer, NULL);
2420
2421 buffer->fd = -1;
2422 buffer->size = 8 * size;
2423 buffer->mirror = malloc(size);
2424 ASSERT_NE(buffer->mirror, NULL);
2425 memset(buffer->mirror, 0xFF, size);
2426
2427 buffer->ptr = mmap(NULL, buffer->size,
2428 PROT_READ | PROT_WRITE,
2429 MAP_PRIVATE | MAP_ANONYMOUS,
2430 buffer->fd, 0);
2431 ASSERT_NE(buffer->ptr, MAP_FAILED);
2432
2433 npages = size >> self->page_shift;
2434 map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2435 if (use_thp)
2436 ret = madvise(map, size, MADV_HUGEPAGE);
2437 else
2438 ret = madvise(map, size, MADV_NOHUGEPAGE);
2439 ASSERT_EQ(ret, 0);
2440 old_ptr = buffer->ptr;
2441 munmap(map + size, size * 2);
2442 buffer->ptr = map;
2443
2444 /* Initialize buffer in system memory. */
2445 for (i = 0, ptr = buffer->ptr;
2446 i < size / sizeof(*ptr); ++i)
2447 ptr[i] = i;
2448
2449 if (before) {
2450 new_ptr = mremap((void *)map, size, size, flags,
2451 map + size + offsets[j]);
2452 ASSERT_NE(new_ptr, MAP_FAILED);
2453 buffer->ptr = new_ptr;
2454 }
2455
2456 /* Migrate memory to device. */
2457 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2458 ASSERT_EQ(ret, 0);
2459 ASSERT_EQ(buffer->cpages, npages);
2460
2461 /* Check what the device read. */
2462 for (i = 0, ptr = buffer->mirror;
2463 i < size / sizeof(*ptr); ++i)
2464 ASSERT_EQ(ptr[i], i);
2465
2466 if (!before) {
2467 new_ptr = mremap((void *)map, size, size, flags,
2468 map + size + offsets[j]);
2469 ASSERT_NE(new_ptr, MAP_FAILED);
2470 buffer->ptr = new_ptr;
2471 }
2472
2473 /* Fault pages back to system memory and check them. */
2474 for (i = 0, ptr = buffer->ptr;
2475 i < size / sizeof(*ptr); ++i)
2476 ASSERT_EQ(ptr[i], i);
2477
2478 munmap(new_ptr, size);
2479 buffer->ptr = old_ptr;
2480 hmm_buffer_free(buffer);
2481 }
2482 }
2483 }
2484 }
2485 }
2486
2487 /*
2488 * Migrate private anonymous huge page with allocation errors.
2489 */
TEST_F(hmm,migrate_anon_huge_err)2490 TEST_F(hmm, migrate_anon_huge_err)
2491 {
2492 struct hmm_buffer *buffer;
2493 unsigned long npages;
2494 unsigned long size;
2495 unsigned long i;
2496 void *old_ptr;
2497 void *map;
2498 int *ptr;
2499 int ret;
2500
2501 size = TWOMEG;
2502
2503 buffer = malloc(sizeof(*buffer));
2504 ASSERT_NE(buffer, NULL);
2505
2506 buffer->fd = -1;
2507 buffer->size = 2 * size;
2508 buffer->mirror = malloc(2 * size);
2509 ASSERT_NE(buffer->mirror, NULL);
2510 memset(buffer->mirror, 0xFF, 2 * size);
2511
2512 old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE,
2513 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
2514 ASSERT_NE(old_ptr, MAP_FAILED);
2515
2516 npages = size >> self->page_shift;
2517 map = (void *)ALIGN((uintptr_t)old_ptr, size);
2518 ret = madvise(map, size, MADV_HUGEPAGE);
2519 ASSERT_EQ(ret, 0);
2520 buffer->ptr = map;
2521
2522 /* Initialize buffer in system memory. */
2523 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2524 ptr[i] = i;
2525
2526 /* Migrate memory to device but force a THP allocation error. */
2527 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer,
2528 HMM_DMIRROR_FLAG_FAIL_ALLOC);
2529 ASSERT_EQ(ret, 0);
2530 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2531 ASSERT_EQ(ret, 0);
2532 ASSERT_EQ(buffer->cpages, npages);
2533
2534 /* Check what the device read. */
2535 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) {
2536 ASSERT_EQ(ptr[i], i);
2537 if (ptr[i] != i)
2538 break;
2539 }
2540
2541 /* Try faulting back a single (PAGE_SIZE) page. */
2542 ptr = buffer->ptr;
2543 ASSERT_EQ(ptr[2048], 2048);
2544
2545 /* unmap and remap the region to reset things. */
2546 ret = munmap(old_ptr, 2 * size);
2547 ASSERT_EQ(ret, 0);
2548 old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE,
2549 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
2550 ASSERT_NE(old_ptr, MAP_FAILED);
2551 map = (void *)ALIGN((uintptr_t)old_ptr, size);
2552 ret = madvise(map, size, MADV_HUGEPAGE);
2553 ASSERT_EQ(ret, 0);
2554 buffer->ptr = map;
2555
2556 /* Initialize buffer in system memory. */
2557 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2558 ptr[i] = i;
2559
2560 /* Migrate THP to device. */
2561 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2562 ASSERT_EQ(ret, 0);
2563 ASSERT_EQ(buffer->cpages, npages);
2564
2565 /*
2566 * Force an allocation error when faulting back a THP resident in the
2567 * device.
2568 */
2569 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer,
2570 HMM_DMIRROR_FLAG_FAIL_ALLOC);
2571 ASSERT_EQ(ret, 0);
2572
2573 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages);
2574 ASSERT_EQ(ret, 0);
2575 ptr = buffer->ptr;
2576 ASSERT_EQ(ptr[2048], 2048);
2577
2578 buffer->ptr = old_ptr;
2579 hmm_buffer_free(buffer);
2580 }
2581
2582 /*
2583 * Migrate private anonymous huge zero page with allocation errors.
2584 */
TEST_F(hmm,migrate_anon_huge_zero_err)2585 TEST_F(hmm, migrate_anon_huge_zero_err)
2586 {
2587 struct hmm_buffer *buffer;
2588 unsigned long npages;
2589 unsigned long size;
2590 unsigned long i;
2591 void *old_ptr;
2592 void *map;
2593 int *ptr;
2594 int ret;
2595
2596 size = TWOMEG;
2597
2598 buffer = malloc(sizeof(*buffer));
2599 ASSERT_NE(buffer, NULL);
2600
2601 buffer->fd = -1;
2602 buffer->size = 2 * size;
2603 buffer->mirror = malloc(2 * size);
2604 ASSERT_NE(buffer->mirror, NULL);
2605 memset(buffer->mirror, 0xFF, 2 * size);
2606
2607 old_ptr = mmap(NULL, 2 * size, PROT_READ,
2608 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
2609 ASSERT_NE(old_ptr, MAP_FAILED);
2610
2611 npages = size >> self->page_shift;
2612 map = (void *)ALIGN((uintptr_t)old_ptr, size);
2613 ret = madvise(map, size, MADV_HUGEPAGE);
2614 ASSERT_EQ(ret, 0);
2615 buffer->ptr = map;
2616
2617 /* Migrate memory to device but force a THP allocation error. */
2618 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer,
2619 HMM_DMIRROR_FLAG_FAIL_ALLOC);
2620 ASSERT_EQ(ret, 0);
2621 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2622 ASSERT_EQ(ret, 0);
2623 ASSERT_EQ(buffer->cpages, npages);
2624
2625 /* Check what the device read. */
2626 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2627 ASSERT_EQ(ptr[i], 0);
2628
2629 /* Try faulting back a single (PAGE_SIZE) page. */
2630 ptr = buffer->ptr;
2631 ASSERT_EQ(ptr[2048], 0);
2632
2633 /* unmap and remap the region to reset things. */
2634 ret = munmap(old_ptr, 2 * size);
2635 ASSERT_EQ(ret, 0);
2636 old_ptr = mmap(NULL, 2 * size, PROT_READ,
2637 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
2638 ASSERT_NE(old_ptr, MAP_FAILED);
2639 map = (void *)ALIGN((uintptr_t)old_ptr, size);
2640 ret = madvise(map, size, MADV_HUGEPAGE);
2641 ASSERT_EQ(ret, 0);
2642 buffer->ptr = map;
2643
2644 /* Initialize buffer in system memory (zero THP page). */
2645 ret = ptr[0];
2646 ASSERT_EQ(ret, 0);
2647
2648 /* Migrate memory to device but force a THP allocation error. */
2649 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer,
2650 HMM_DMIRROR_FLAG_FAIL_ALLOC);
2651 ASSERT_EQ(ret, 0);
2652 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2653 ASSERT_EQ(ret, 0);
2654 ASSERT_EQ(buffer->cpages, npages);
2655
2656 /* Fault the device memory back and check it. */
2657 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2658 ASSERT_EQ(ptr[i], 0);
2659
2660 buffer->ptr = old_ptr;
2661 hmm_buffer_free(buffer);
2662 }
2663
2664 struct benchmark_results {
2665 double sys_to_dev_time;
2666 double dev_to_sys_time;
2667 double throughput_s2d;
2668 double throughput_d2s;
2669 };
2670
get_time_ms(void)2671 static double get_time_ms(void)
2672 {
2673 struct timeval tv;
2674
2675 gettimeofday(&tv, NULL);
2676 return (tv.tv_sec * 1000.0) + (tv.tv_usec / 1000.0);
2677 }
2678
hmm_buffer_alloc(unsigned long size)2679 static inline struct hmm_buffer *hmm_buffer_alloc(unsigned long size)
2680 {
2681 struct hmm_buffer *buffer;
2682
2683 buffer = malloc(sizeof(*buffer));
2684
2685 buffer->fd = -1;
2686 buffer->size = size;
2687 buffer->mirror = malloc(size);
2688 memset(buffer->mirror, 0xFF, size);
2689 return buffer;
2690 }
2691
print_benchmark_results(const char * test_name,size_t buffer_size,struct benchmark_results * thp,struct benchmark_results * regular)2692 static void print_benchmark_results(const char *test_name, size_t buffer_size,
2693 struct benchmark_results *thp,
2694 struct benchmark_results *regular)
2695 {
2696 double s2d_improvement = ((regular->sys_to_dev_time - thp->sys_to_dev_time) /
2697 regular->sys_to_dev_time) * 100.0;
2698 double d2s_improvement = ((regular->dev_to_sys_time - thp->dev_to_sys_time) /
2699 regular->dev_to_sys_time) * 100.0;
2700 double throughput_s2d_improvement = ((thp->throughput_s2d - regular->throughput_s2d) /
2701 regular->throughput_s2d) * 100.0;
2702 double throughput_d2s_improvement = ((thp->throughput_d2s - regular->throughput_d2s) /
2703 regular->throughput_d2s) * 100.0;
2704
2705 printf("\n=== %s (%.1f MB) ===\n", test_name, buffer_size / (1024.0 * 1024.0));
2706 printf(" | With THP | Without THP | Improvement\n");
2707 printf("---------------------------------------------------------------------\n");
2708 printf("Sys->Dev Migration | %.3f ms | %.3f ms | %.1f%%\n",
2709 thp->sys_to_dev_time, regular->sys_to_dev_time, s2d_improvement);
2710 printf("Dev->Sys Migration | %.3f ms | %.3f ms | %.1f%%\n",
2711 thp->dev_to_sys_time, regular->dev_to_sys_time, d2s_improvement);
2712 printf("S->D Throughput | %.2f GB/s | %.2f GB/s | %.1f%%\n",
2713 thp->throughput_s2d, regular->throughput_s2d, throughput_s2d_improvement);
2714 printf("D->S Throughput | %.2f GB/s | %.2f GB/s | %.1f%%\n",
2715 thp->throughput_d2s, regular->throughput_d2s, throughput_d2s_improvement);
2716 }
2717
2718 /*
2719 * Run a single migration benchmark
2720 * fd: file descriptor for hmm device
2721 * use_thp: whether to use THP
2722 * buffer_size: size of buffer to allocate
2723 * iterations: number of iterations
2724 * results: where to store results
2725 */
run_migration_benchmark(int fd,int use_thp,size_t buffer_size,int iterations,struct benchmark_results * results)2726 static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_size,
2727 int iterations, struct benchmark_results *results)
2728 {
2729 struct hmm_buffer *buffer;
2730 unsigned long npages = buffer_size / sysconf(_SC_PAGESIZE);
2731 double start, end;
2732 double s2d_total = 0, d2s_total = 0;
2733 int ret, i;
2734 int *ptr;
2735
2736 buffer = hmm_buffer_alloc(buffer_size);
2737
2738 /* Map memory */
2739 buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE,
2740 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
2741
2742 if (!buffer->ptr)
2743 return -1;
2744
2745 /* Apply THP hint if requested */
2746 if (use_thp)
2747 ret = madvise(buffer->ptr, buffer_size, MADV_HUGEPAGE);
2748 else
2749 ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE);
2750
2751 if (ret)
2752 return ret;
2753
2754 /* Initialize memory to make sure pages are allocated */
2755 ptr = (int *)buffer->ptr;
2756 for (i = 0; i < buffer_size / sizeof(int); i++)
2757 ptr[i] = i & 0xFF;
2758
2759 /* Warmup iteration */
2760 ret = hmm_migrate_sys_to_dev(fd, buffer, npages);
2761 if (ret)
2762 return ret;
2763
2764 ret = hmm_migrate_dev_to_sys(fd, buffer, npages);
2765 if (ret)
2766 return ret;
2767
2768 /* Benchmark iterations */
2769 for (i = 0; i < iterations; i++) {
2770 /* System to device migration */
2771 start = get_time_ms();
2772
2773 ret = hmm_migrate_sys_to_dev(fd, buffer, npages);
2774 if (ret)
2775 return ret;
2776
2777 end = get_time_ms();
2778 s2d_total += (end - start);
2779
2780 /* Device to system migration */
2781 start = get_time_ms();
2782
2783 ret = hmm_migrate_dev_to_sys(fd, buffer, npages);
2784 if (ret)
2785 return ret;
2786
2787 end = get_time_ms();
2788 d2s_total += (end - start);
2789 }
2790
2791 /* Calculate average times and throughput */
2792 results->sys_to_dev_time = s2d_total / iterations;
2793 results->dev_to_sys_time = d2s_total / iterations;
2794 results->throughput_s2d = (buffer_size / (1024.0 * 1024.0 * 1024.0)) /
2795 (results->sys_to_dev_time / 1000.0);
2796 results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) /
2797 (results->dev_to_sys_time / 1000.0);
2798
2799 /* Cleanup */
2800 hmm_buffer_free(buffer);
2801 return 0;
2802 }
2803
2804 /*
2805 * Benchmark THP migration with different buffer sizes
2806 */
2807 TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120)
2808 {
2809 struct benchmark_results thp_results, regular_results;
2810 size_t thp_size = 2 * 1024 * 1024; /* 2MB - typical THP size */
2811 int iterations = 5;
2812
2813 printf("\nHMM THP Migration Benchmark\n");
2814 printf("---------------------------\n");
2815 printf("System page size: %ld bytes\n", sysconf(_SC_PAGESIZE));
2816
2817 /* Test different buffer sizes */
2818 size_t test_sizes[] = {
2819 thp_size / 4, /* 512KB - smaller than THP */
2820 thp_size / 2, /* 1MB - half THP */
2821 thp_size, /* 2MB - single THP */
2822 thp_size * 2, /* 4MB - two THPs */
2823 thp_size * 4, /* 8MB - four THPs */
2824 thp_size * 8, /* 16MB - eight THPs */
2825 thp_size * 128, /* 256MB - one twenty eight THPs */
2826 };
2827
2828 static const char *const test_names[] = {
2829 "Small Buffer (512KB)",
2830 "Half THP Size (1MB)",
2831 "Single THP Size (2MB)",
2832 "Two THP Size (4MB)",
2833 "Four THP Size (8MB)",
2834 "Eight THP Size (16MB)",
2835 "One twenty eight THP Size (256MB)"
2836 };
2837
2838 int num_tests = ARRAY_SIZE(test_sizes);
2839
2840 /* Run all tests */
2841 for (int i = 0; i < num_tests; i++) {
2842 /* Test with THP */
2843 ASSERT_EQ(run_migration_benchmark(self->fd, 1, test_sizes[i],
2844 iterations, &thp_results), 0);
2845
2846 /* Test without THP */
2847 ASSERT_EQ(run_migration_benchmark(self->fd, 0, test_sizes[i],
2848 iterations, ®ular_results), 0);
2849
2850 /* Print results */
2851 print_benchmark_results(test_names[i], test_sizes[i],
2852 &thp_results, ®ular_results);
2853 }
2854 }
2855 TEST_HARNESS_MAIN
2856