10 static void put_back_pages(unsigned int cmd, struct page **pages,  in put_back_pages()
39 static void verify_dma_pinned(unsigned int cmd, struct page **pages,  in verify_dma_pinned()
53 				 "pages[%lu] is NOT dma-pinned\n", i)) {  in verify_dma_pinned()
55 				dump_page(&folio->page, "gup_test failure");  in verify_dma_pinned()
61 				dump_page(&folio->page, "gup_test failure");  in verify_dma_pinned()
69 static void dump_pages_test(struct gup_test *gup, struct page **pages,  in dump_pages_test()
76 	 * Zero out any user-supplied page index that is out of range. Remember:  in dump_pages_test()
77 	 * .which_pages[] contains a 1-based set of page indices.  in dump_pages_test()
80 		if (gup->which_pages[i] > nr_pages) {  in dump_pages_test()
82 				i, gup->which_pages[i]);  in dump_pages_test()
83 			gup->which_pages[i] = 0;  in dump_pages_test()
88 		index_to_dump = gup->which_pages[i];  in dump_pages_test()
91 			index_to_dump--; // Decode from 1-based, to 0-based  in dump_pages_test()
92 			pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",  in dump_pages_test()
93 				index_to_dump, gup->addr);  in dump_pages_test()
106 	struct page **pages;  in __gup_test_ioctl()
111 	if (gup->size > ULONG_MAX)  in __gup_test_ioctl()
112 		return -EINVAL;  in __gup_test_ioctl()
114 	nr_pages = gup->size / PAGE_SIZE;  in __gup_test_ioctl()
117 		return -ENOMEM;  in __gup_test_ioctl()
119 	if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {  in __gup_test_ioctl()
120 		ret = -EINTR;  in __gup_test_ioctl()
125 	nr = gup->nr_pages_per_call;  in __gup_test_ioctl()
127 	for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {  in __gup_test_ioctl()
128 		if (nr != gup->nr_pages_per_call)  in __gup_test_ioctl()
132 		if (next > gup->addr + gup->size) {  in __gup_test_ioctl()
133 			next = gup->addr + gup->size;  in __gup_test_ioctl()
134 			nr = (next - addr) / PAGE_SIZE;  in __gup_test_ioctl()
139 			nr = get_user_pages_fast(addr, nr, gup->gup_flags,  in __gup_test_ioctl()
143 			nr = get_user_pages(addr, nr, gup->gup_flags, pages + i);  in __gup_test_ioctl()
146 			nr = pin_user_pages_fast(addr, nr, gup->gup_flags,  in __gup_test_ioctl()
150 			nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i);  in __gup_test_ioctl()
154 					    gup->gup_flags | FOLL_LONGTERM,  in __gup_test_ioctl()
158 			if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)  in __gup_test_ioctl()
159 				nr = pin_user_pages(addr, nr, gup->gup_flags,  in __gup_test_ioctl()
162 				nr = get_user_pages(addr, nr, gup->gup_flags,  in __gup_test_ioctl()
166 			ret = -EINVAL;  in __gup_test_ioctl()
179 	gup->get_delta_usec = ktime_us_delta(end_time, start_time);  in __gup_test_ioctl()
180 	gup->size = addr - gup->addr;  in __gup_test_ioctl()
183 	 * Take an un-benchmark-timed moment to verify DMA pinned  in __gup_test_ioctl()
184 	 * state: print a warning if any non-dma-pinned pages are found:  in __gup_test_ioctl()
193 	put_back_pages(cmd, pages, nr_pages, gup->test_flags);  in __gup_test_ioctl()
196 	gup->put_delta_usec = ktime_us_delta(end_time, start_time);  in __gup_test_ioctl()
200 		mmap_read_unlock(current->mm);  in __gup_test_ioctl()
207 static struct page **pin_longterm_test_pages;
227 	struct page **pages;  in pin_longterm_test_start()
232 		return -EINVAL;  in pin_longterm_test_start()
235 		return -EFAULT;  in pin_longterm_test_start()
239 		return -EINVAL;  in pin_longterm_test_start()
241 		return -EINVAL;  in pin_longterm_test_start()
243 		return -EINVAL;  in pin_longterm_test_start()
246 		return -EINVAL;  in pin_longterm_test_start()
250 		return -ENOMEM;  in pin_longterm_test_start()
256 	if (!fast && mmap_read_lock_killable(current->mm)) {  in pin_longterm_test_start()
258 		return -EINTR;  in pin_longterm_test_start()
264 	while (nr_pages - pin_longterm_test_nr_pages) {  in pin_longterm_test_start()
265 		remaining_pages = nr_pages - pin_longterm_test_nr_pages;  in pin_longterm_test_start()
284 		mmap_read_unlock(current->mm);  in pin_longterm_test_start()
294 		return -EINVAL;  in pin_longterm_test_read()
297 		return -EFAULT;  in pin_longterm_test_read()
307 			return -EFAULT;  in pin_longterm_test_read()
316 	int ret = -EINVAL;  in pin_longterm_test_ioctl()
319 		return -EINTR;  in pin_longterm_test_ioctl()
357 		return -EINVAL;  in gup_test_ioctl()
361 		return -EFAULT;  in gup_test_ioctl()
368 		return -EFAULT;  in gup_test_ioctl()