40 	unsigned long	size;  member
211 		munmap(buffer->ptr, buffer->size);  in hmm_buffer_free()
219 static int hmm_create_file(unsigned long size)  in hmm_create_file()  argument
230 			r = ftruncate(fd, size);  in hmm_create_file()
296 	unsigned long size;  in TEST_F()  local
304 	size = npages << self->page_shift;  in TEST_F()
310 	buffer->size = size;  in TEST_F()
311 	buffer->mirror = malloc(size);  in TEST_F()
314 	buffer->ptr = mmap(NULL, size,  in TEST_F()
325 	for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
329 	ret = mprotect(buffer->ptr, size, PROT_READ);  in TEST_F()
346 	for (; i < size / sizeof(*ptr); ++i)  in TEST_F()
360 	unsigned long size;  in TEST_F()  local
367 	size = npages << self->page_shift;  in TEST_F()
373 	buffer->size = size;  in TEST_F()
374 	buffer->mirror = malloc(size);  in TEST_F()
377 	buffer->ptr = mmap(NULL, size,  in TEST_F()
384 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
388 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
392 	ret = mprotect(buffer->ptr, size, PROT_NONE);  in TEST_F()
400 	ret = mprotect(buffer->ptr, size, PROT_READ);  in TEST_F()
402 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
406 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
419 	unsigned long size;  in TEST_F()  local
426 	size = npages << self->page_shift;  in TEST_F()
432 	buffer->size = size;  in TEST_F()
433 	buffer->mirror = malloc(size);  in TEST_F()
436 	buffer->ptr = mmap(NULL, size,  in TEST_F()
443 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
453 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
467 	unsigned long size;  in TEST_F()  local
474 	size = npages << self->page_shift;  in TEST_F()
480 	buffer->size = size;  in TEST_F()
481 	buffer->mirror = malloc(size);  in TEST_F()
484 	buffer->ptr = mmap(NULL, size,  in TEST_F()
497 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
505 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
509 	ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ);  in TEST_F()
519 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
533 	unsigned long size;  in TEST_F()  local
542 	size = npages << self->page_shift;  in TEST_F()
548 	buffer->size = size;  in TEST_F()
549 	buffer->mirror = malloc(size);  in TEST_F()
552 	buffer->ptr = mmap(NULL, size,  in TEST_F()
559 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
563 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
574 		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
580 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
582 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
596 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
611 	unsigned long size;  in TEST_F()  local
620 	size = npages << self->page_shift;  in TEST_F()
626 	buffer->size = size;  in TEST_F()
627 	buffer->mirror = malloc(size);  in TEST_F()
630 	buffer->ptr = mmap(NULL, size,  in TEST_F()
637 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
641 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
652 		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
658 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
660 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
674 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
688 	unsigned long size;  in TEST_F()  local
695 	size = 2 * TWOMEG;  in TEST_F()
701 	buffer->size = size;  in TEST_F()
702 	buffer->mirror = malloc(size);  in TEST_F()
705 	buffer->ptr = mmap(NULL, size,  in TEST_F()
711 	size = TWOMEG;  in TEST_F()
712 	npages = size >> self->page_shift;  in TEST_F()
713 	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);  in TEST_F()
714 	ret = madvise(map, size, MADV_HUGEPAGE);  in TEST_F()
720 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
730 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
792 	unsigned long size;  in TEST_F()  local
803 	size = ALIGN(TWOMEG, default_hsize);  in TEST_F()
804 	npages = size >> self->page_shift;  in TEST_F()
809 	buffer->ptr = mmap(NULL, size,  in TEST_F()
819 	buffer->size = size;  in TEST_F()
820 	buffer->mirror = malloc(size);  in TEST_F()
824 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
834 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
837 	munmap(buffer->ptr, buffer->size);  in TEST_F()
849 	unsigned long size;  in TEST_F()  local
858 	size = npages << self->page_shift;  in TEST_F()
860 	fd = hmm_create_file(size);  in TEST_F()
867 	buffer->size = size;  in TEST_F()
868 	buffer->mirror = malloc(size);  in TEST_F()
872 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
874 	len = pwrite(fd, buffer->mirror, size, 0);  in TEST_F()
875 	ASSERT_EQ(len, size);  in TEST_F()
876 	memset(buffer->mirror, 0, size);  in TEST_F()
878 	buffer->ptr = mmap(NULL, size,  in TEST_F()
891 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
904 	unsigned long size;  in TEST_F()  local
913 	size = npages << self->page_shift;  in TEST_F()
915 	fd = hmm_create_file(size);  in TEST_F()
922 	buffer->size = size;  in TEST_F()
923 	buffer->mirror = malloc(size);  in TEST_F()
926 	buffer->ptr = mmap(NULL, size,  in TEST_F()
933 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
943 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
947 	len = pread(fd, buffer->mirror, size, 0);  in TEST_F()
948 	ASSERT_EQ(len, size);  in TEST_F()
949 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
962 	unsigned long size;  in TEST_F()  local
969 	size = npages << self->page_shift;  in TEST_F()
975 	buffer->size = size;  in TEST_F()
976 	buffer->mirror = malloc(size);  in TEST_F()
979 	buffer->ptr = mmap(NULL, size,  in TEST_F()
986 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
995 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1010 	unsigned long size;  in TEST_F()  local
1017 	size = npages << self->page_shift;  in TEST_F()
1023 	buffer->size = size;  in TEST_F()
1024 	buffer->mirror = malloc(size);  in TEST_F()
1027 	buffer->ptr = mmap(NULL, size,  in TEST_F()
1034 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1043 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1047 	for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)  in TEST_F()
1056 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1066 	unsigned long size;  in TEST_F()  local
1073 	size = npages << self->page_shift;  in TEST_F()
1079 	buffer->size = size;  in TEST_F()
1080 	buffer->mirror = malloc(size);  in TEST_F()
1083 	buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,  in TEST_F()
1088 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1097 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1105 	for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)  in TEST_F()
1118 	unsigned long size;  in TEST_F()  local
1123 	size = npages << self->page_shift;  in TEST_F()
1129 	buffer->size = size;  in TEST_F()
1130 	buffer->mirror = malloc(size);  in TEST_F()
1133 	buffer->ptr = mmap(NULL, size,  in TEST_F()
1153 	unsigned long size;  in TEST_F()  local
1160 	size = npages << self->page_shift;  in TEST_F()
1166 	buffer->size = size;  in TEST_F()
1167 	buffer->mirror = malloc(size);  in TEST_F()
1171 	buffer->ptr = mmap(NULL, size,  in TEST_F()
1245 	unsigned long size;  in TEST_F()  local
1253 	size = npages << self->page_shift;  in TEST_F()
1260 		buffer->size = size;  in TEST_F()
1261 		buffer->mirror = malloc(size);  in TEST_F()
1264 		buffer->ptr = mmap(NULL, size,  in TEST_F()
1271 		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1280 		for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1290 		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1304 	unsigned long size;  in TEST_F()  local
1312 	size = npages << self->page_shift;  in TEST_F()
1319 		buffer->size = size;  in TEST_F()
1320 		buffer->mirror = malloc(size);  in TEST_F()
1323 		buffer->ptr = mmap(NULL, size,  in TEST_F()
1330 		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1341 		for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1354 	munmap(buffer->ptr + buffer->size / 2, buffer->size / 2);  in unmap_buffer()
1366 	unsigned long size;  in TEST_F()  local
1372 	size = npages << self->page_shift;  in TEST_F()
1385 		buffer->size = size;  in TEST_F()
1386 		buffer->mirror = malloc(size);  in TEST_F()
1389 		buffer->ptr = mmap(NULL, size,  in TEST_F()
1396 		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1411 			     i < size / sizeof(*ptr);  in TEST_F()
1428 	unsigned long size;  in TEST_F()  local
1433 	size = npages << self->page_shift;  in TEST_F()
1439 	buffer->size = size;  in TEST_F()
1445 	buffer->ptr = mmap(NULL, size,  in TEST_F()
1470 	unsigned long size;  in TEST_F()  local
1478 	size = npages << self->page_shift;  in TEST_F()
1484 	buffer->size = size;  in TEST_F()
1489 	buffer->ptr = mmap(NULL, size,  in TEST_F()
1572 	unsigned long size;  in TEST_F()  local
1586 	size = ALIGN(TWOMEG, default_hsize);  in TEST_F()
1587 	npages = size >> self->page_shift;  in TEST_F()
1592 	buffer->ptr = mmap(NULL, size,  in TEST_F()
1601 	buffer->size = size;  in TEST_F()
1606 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1621 	ret = mprotect(buffer->ptr, size, PROT_READ);  in TEST_F()
1635 	munmap(buffer->ptr, buffer->size);  in TEST_F()
1647 	unsigned long size;  in TEST_F()  local
1653 	size = npages << self->page_shift;  in TEST_F()
1659 	buffer->size = size;  in TEST_F()
1660 	buffer->mirror = malloc(size);  in TEST_F()
1664 	buffer->ptr = mmap(NULL, size,  in TEST_F()
1671 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1675 	ret = mprotect(buffer->ptr, size, PROT_READ);  in TEST_F()
1685 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1695 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1709 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1722 	unsigned long size;  in TEST_F()  local
1729 	size = npages << self->page_shift;  in TEST_F()
1735 	buffer->size = size;  in TEST_F()
1736 	buffer->mirror = malloc(size);  in TEST_F()
1739 	buffer->ptr = mmap(NULL, size,  in TEST_F()
1746 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1755 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1759 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1762 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1776 	unsigned long size;  in TEST_F()  local
1783 	size = npages << self->page_shift;  in TEST_F()
1789 	buffer->size = size;  in TEST_F()
1790 	buffer->mirror = malloc(size);  in TEST_F()
1793 	buffer->ptr = mmap(NULL, size,  in TEST_F()
1800 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1809 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1812 	ret = mprotect(buffer->ptr, size, PROT_READ);  in TEST_F()
1829 	unsigned long size;  in TEST_F()  local
1836 	size = npages << self->page_shift;  in TEST_F()
1842 	buffer->size = size;  in TEST_F()
1843 	buffer->mirror = malloc(size);  in TEST_F()
1846 	buffer->ptr = mmap(NULL, size,  in TEST_F()
1853 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1864 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1867 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1874 			 int npages, int size, int flags)  in gup_test_exec()  argument
1880 		.size			= size,  in gup_test_exec()
1903 	unsigned long size;  in TEST_F()  local
1914 	size = npages << self->page_shift;  in TEST_F()
1920 	buffer->size = size;  in TEST_F()
1921 	buffer->mirror = malloc(size);  in TEST_F()
1924 	buffer->ptr = mmap(NULL, size,  in TEST_F()
1931 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1939 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)  in TEST_F()
1973 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
1991 	unsigned long size;  in TEST_F()  local
2000 	size = npages << self->page_shift;  in TEST_F()
2006 	buffer->size = size;  in TEST_F()
2007 	buffer->mirror = malloc(size);  in TEST_F()
2010 	buffer->ptr = mmap(NULL, size,  in TEST_F()
2017 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()
2039 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)  in TEST_F()