19c84f229SJohn Hubbard #include <linux/kernel.h> 29c84f229SJohn Hubbard #include <linux/mm.h> 39c84f229SJohn Hubbard #include <linux/slab.h> 49c84f229SJohn Hubbard #include <linux/uaccess.h> 59c84f229SJohn Hubbard #include <linux/ktime.h> 69c84f229SJohn Hubbard #include <linux/debugfs.h> 7b9dcfdffSJohn Hubbard #include "gup_test.h" 89c84f229SJohn Hubbard 99c84f229SJohn Hubbard static void put_back_pages(unsigned int cmd, struct page **pages, 10f4f9bda4SJohn Hubbard unsigned long nr_pages, unsigned int gup_test_flags) 119c84f229SJohn Hubbard { 129c84f229SJohn Hubbard unsigned long i; 139c84f229SJohn Hubbard 149c84f229SJohn Hubbard switch (cmd) { 159c84f229SJohn Hubbard case GUP_FAST_BENCHMARK: 16a9bed1e1SJohn Hubbard case GUP_BASIC_TEST: 179c84f229SJohn Hubbard for (i = 0; i < nr_pages; i++) 189c84f229SJohn Hubbard put_page(pages[i]); 199c84f229SJohn Hubbard break; 209c84f229SJohn Hubbard 219c84f229SJohn Hubbard case PIN_FAST_BENCHMARK: 22a9bed1e1SJohn Hubbard case PIN_BASIC_TEST: 239c84f229SJohn Hubbard case PIN_LONGTERM_BENCHMARK: 249c84f229SJohn Hubbard unpin_user_pages(pages, nr_pages); 259c84f229SJohn Hubbard break; 26f4f9bda4SJohn Hubbard case DUMP_USER_PAGES_TEST: 27f4f9bda4SJohn Hubbard if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) { 28f4f9bda4SJohn Hubbard unpin_user_pages(pages, nr_pages); 29f4f9bda4SJohn Hubbard } else { 30f4f9bda4SJohn Hubbard for (i = 0; i < nr_pages; i++) 31f4f9bda4SJohn Hubbard put_page(pages[i]); 32f4f9bda4SJohn Hubbard 33f4f9bda4SJohn Hubbard } 34f4f9bda4SJohn Hubbard break; 359c84f229SJohn Hubbard } 369c84f229SJohn Hubbard } 379c84f229SJohn Hubbard 389c84f229SJohn Hubbard static void verify_dma_pinned(unsigned int cmd, struct page **pages, 399c84f229SJohn Hubbard unsigned long nr_pages) 409c84f229SJohn Hubbard { 419c84f229SJohn Hubbard unsigned long i; 429c84f229SJohn Hubbard struct page *page; 439c84f229SJohn Hubbard 449c84f229SJohn Hubbard switch (cmd) { 459c84f229SJohn Hubbard case PIN_FAST_BENCHMARK: 46a9bed1e1SJohn Hubbard case PIN_BASIC_TEST: 479c84f229SJohn Hubbard case PIN_LONGTERM_BENCHMARK: 489c84f229SJohn Hubbard for (i = 0; i < nr_pages; i++) { 499c84f229SJohn Hubbard page = pages[i]; 509c84f229SJohn Hubbard if (WARN(!page_maybe_dma_pinned(page), 519c84f229SJohn Hubbard "pages[%lu] is NOT dma-pinned\n", i)) { 529c84f229SJohn Hubbard 539c84f229SJohn Hubbard dump_page(page, "gup_test failure"); 549c84f229SJohn Hubbard break; 559c84f229SJohn Hubbard } 569c84f229SJohn Hubbard } 579c84f229SJohn Hubbard break; 589c84f229SJohn Hubbard } 599c84f229SJohn Hubbard } 609c84f229SJohn Hubbard 61f4f9bda4SJohn Hubbard static void dump_pages_test(struct gup_test *gup, struct page **pages, 62f4f9bda4SJohn Hubbard unsigned long nr_pages) 63f4f9bda4SJohn Hubbard { 64f4f9bda4SJohn Hubbard unsigned int index_to_dump; 65f4f9bda4SJohn Hubbard unsigned int i; 66f4f9bda4SJohn Hubbard 67f4f9bda4SJohn Hubbard /* 68f4f9bda4SJohn Hubbard * Zero out any user-supplied page index that is out of range. Remember: 69f4f9bda4SJohn Hubbard * .which_pages[] contains a 1-based set of page indices. 70f4f9bda4SJohn Hubbard */ 71f4f9bda4SJohn Hubbard for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) { 72f4f9bda4SJohn Hubbard if (gup->which_pages[i] > nr_pages) { 73f4f9bda4SJohn Hubbard pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n", 74f4f9bda4SJohn Hubbard i, gup->which_pages[i]); 75f4f9bda4SJohn Hubbard gup->which_pages[i] = 0; 76f4f9bda4SJohn Hubbard } 77f4f9bda4SJohn Hubbard } 78f4f9bda4SJohn Hubbard 79f4f9bda4SJohn Hubbard for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) { 80f4f9bda4SJohn Hubbard index_to_dump = gup->which_pages[i]; 81f4f9bda4SJohn Hubbard 82f4f9bda4SJohn Hubbard if (index_to_dump) { 83f4f9bda4SJohn Hubbard index_to_dump--; // Decode from 1-based, to 0-based 84f4f9bda4SJohn Hubbard pr_info("---- page #%u, starting from user virt addr: 0x%llx\n", 85f4f9bda4SJohn Hubbard index_to_dump, gup->addr); 86f4f9bda4SJohn Hubbard dump_page(pages[index_to_dump], 87f4f9bda4SJohn Hubbard "gup_test: dump_pages() test"); 88f4f9bda4SJohn Hubbard } 89f4f9bda4SJohn Hubbard } 90f4f9bda4SJohn Hubbard } 91f4f9bda4SJohn Hubbard 929c84f229SJohn Hubbard static int __gup_test_ioctl(unsigned int cmd, 939c84f229SJohn Hubbard struct gup_test *gup) 949c84f229SJohn Hubbard { 959c84f229SJohn Hubbard ktime_t start_time, end_time; 969c84f229SJohn Hubbard unsigned long i, nr_pages, addr, next; 979c84f229SJohn Hubbard int nr; 989c84f229SJohn Hubbard struct page **pages; 999c84f229SJohn Hubbard int ret = 0; 1009c84f229SJohn Hubbard bool needs_mmap_lock = 1019c84f229SJohn Hubbard cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK; 1029c84f229SJohn Hubbard 1039c84f229SJohn Hubbard if (gup->size > ULONG_MAX) 1049c84f229SJohn Hubbard return -EINVAL; 1059c84f229SJohn Hubbard 1069c84f229SJohn Hubbard nr_pages = gup->size / PAGE_SIZE; 1079c84f229SJohn Hubbard pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL); 1089c84f229SJohn Hubbard if (!pages) 1099c84f229SJohn Hubbard return -ENOMEM; 1109c84f229SJohn Hubbard 1119c84f229SJohn Hubbard if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) { 1129c84f229SJohn Hubbard ret = -EINTR; 1139c84f229SJohn Hubbard goto free_pages; 1149c84f229SJohn Hubbard } 1159c84f229SJohn Hubbard 1169c84f229SJohn Hubbard i = 0; 1179c84f229SJohn Hubbard nr = gup->nr_pages_per_call; 1189c84f229SJohn Hubbard start_time = ktime_get(); 1199c84f229SJohn Hubbard for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) { 1209c84f229SJohn Hubbard if (nr != gup->nr_pages_per_call) 1219c84f229SJohn Hubbard break; 1229c84f229SJohn Hubbard 1239c84f229SJohn Hubbard next = addr + nr * PAGE_SIZE; 1249c84f229SJohn Hubbard if (next > gup->addr + gup->size) { 1259c84f229SJohn Hubbard next = gup->addr + gup->size; 1269c84f229SJohn Hubbard nr = (next - addr) / PAGE_SIZE; 1279c84f229SJohn Hubbard } 1289c84f229SJohn Hubbard 1299c84f229SJohn Hubbard /* Filter out most gup flags: only allow a tiny subset here: */ 1309c84f229SJohn Hubbard gup->flags &= FOLL_WRITE; 1319c84f229SJohn Hubbard 1329c84f229SJohn Hubbard switch (cmd) { 1339c84f229SJohn Hubbard case GUP_FAST_BENCHMARK: 1349c84f229SJohn Hubbard nr = get_user_pages_fast(addr, nr, gup->flags, 1359c84f229SJohn Hubbard pages + i); 1369c84f229SJohn Hubbard break; 137a9bed1e1SJohn Hubbard case GUP_BASIC_TEST: 1389c84f229SJohn Hubbard nr = get_user_pages(addr, nr, gup->flags, pages + i, 1399c84f229SJohn Hubbard NULL); 1409c84f229SJohn Hubbard break; 1419c84f229SJohn Hubbard case PIN_FAST_BENCHMARK: 1429c84f229SJohn Hubbard nr = pin_user_pages_fast(addr, nr, gup->flags, 1439c84f229SJohn Hubbard pages + i); 1449c84f229SJohn Hubbard break; 145a9bed1e1SJohn Hubbard case PIN_BASIC_TEST: 1469c84f229SJohn Hubbard nr = pin_user_pages(addr, nr, gup->flags, pages + i, 1479c84f229SJohn Hubbard NULL); 1489c84f229SJohn Hubbard break; 1499c84f229SJohn Hubbard case PIN_LONGTERM_BENCHMARK: 1509c84f229SJohn Hubbard nr = pin_user_pages(addr, nr, 1519c84f229SJohn Hubbard gup->flags | FOLL_LONGTERM, 1529c84f229SJohn Hubbard pages + i, NULL); 1539c84f229SJohn Hubbard break; 154f4f9bda4SJohn Hubbard case DUMP_USER_PAGES_TEST: 155f4f9bda4SJohn Hubbard if (gup->flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) 156f4f9bda4SJohn Hubbard nr = pin_user_pages(addr, nr, gup->flags, 157f4f9bda4SJohn Hubbard pages + i, NULL); 158f4f9bda4SJohn Hubbard else 159f4f9bda4SJohn Hubbard nr = get_user_pages(addr, nr, gup->flags, 160f4f9bda4SJohn Hubbard pages + i, NULL); 161f4f9bda4SJohn Hubbard break; 1629c84f229SJohn Hubbard default: 1639c84f229SJohn Hubbard ret = -EINVAL; 1649c84f229SJohn Hubbard goto unlock; 1659c84f229SJohn Hubbard } 1669c84f229SJohn Hubbard 1679c84f229SJohn Hubbard if (nr <= 0) 1689c84f229SJohn Hubbard break; 1699c84f229SJohn Hubbard i += nr; 1709c84f229SJohn Hubbard } 1719c84f229SJohn Hubbard end_time = ktime_get(); 1729c84f229SJohn Hubbard 1739c84f229SJohn Hubbard /* Shifting the meaning of nr_pages: now it is actual number pinned: */ 1749c84f229SJohn Hubbard nr_pages = i; 1759c84f229SJohn Hubbard 1769c84f229SJohn Hubbard gup->get_delta_usec = ktime_us_delta(end_time, start_time); 1779c84f229SJohn Hubbard gup->size = addr - gup->addr; 1789c84f229SJohn Hubbard 1799c84f229SJohn Hubbard /* 1809c84f229SJohn Hubbard * Take an un-benchmark-timed moment to verify DMA pinned 1819c84f229SJohn Hubbard * state: print a warning if any non-dma-pinned pages are found: 1829c84f229SJohn Hubbard */ 1839c84f229SJohn Hubbard verify_dma_pinned(cmd, pages, nr_pages); 1849c84f229SJohn Hubbard 185f4f9bda4SJohn Hubbard if (cmd == DUMP_USER_PAGES_TEST) 186f4f9bda4SJohn Hubbard dump_pages_test(gup, pages, nr_pages); 187f4f9bda4SJohn Hubbard 1889c84f229SJohn Hubbard start_time = ktime_get(); 1899c84f229SJohn Hubbard 190f4f9bda4SJohn Hubbard put_back_pages(cmd, pages, nr_pages, gup->flags); 1919c84f229SJohn Hubbard 1929c84f229SJohn Hubbard end_time = ktime_get(); 1939c84f229SJohn Hubbard gup->put_delta_usec = ktime_us_delta(end_time, start_time); 1949c84f229SJohn Hubbard 1959c84f229SJohn Hubbard unlock: 1969c84f229SJohn Hubbard if (needs_mmap_lock) 1979c84f229SJohn Hubbard mmap_read_unlock(current->mm); 1989c84f229SJohn Hubbard free_pages: 1999c84f229SJohn Hubbard kvfree(pages); 2009c84f229SJohn Hubbard return ret; 2019c84f229SJohn Hubbard } 2029c84f229SJohn Hubbard 2039c84f229SJohn Hubbard static long gup_test_ioctl(struct file *filep, unsigned int cmd, 2049c84f229SJohn Hubbard unsigned long arg) 2059c84f229SJohn Hubbard { 2069c84f229SJohn Hubbard struct gup_test gup; 2079c84f229SJohn Hubbard int ret; 2089c84f229SJohn Hubbard 2099c84f229SJohn Hubbard switch (cmd) { 2109c84f229SJohn Hubbard case GUP_FAST_BENCHMARK: 2119c84f229SJohn Hubbard case PIN_FAST_BENCHMARK: 2129c84f229SJohn Hubbard case PIN_LONGTERM_BENCHMARK: 213a9bed1e1SJohn Hubbard case GUP_BASIC_TEST: 214a9bed1e1SJohn Hubbard case PIN_BASIC_TEST: 215f4f9bda4SJohn Hubbard case DUMP_USER_PAGES_TEST: 2169c84f229SJohn Hubbard break; 2179c84f229SJohn Hubbard default: 2189c84f229SJohn Hubbard return -EINVAL; 2199c84f229SJohn Hubbard } 2209c84f229SJohn Hubbard 2219c84f229SJohn Hubbard if (copy_from_user(&gup, (void __user *)arg, sizeof(gup))) 2229c84f229SJohn Hubbard return -EFAULT; 2239c84f229SJohn Hubbard 2249c84f229SJohn Hubbard ret = __gup_test_ioctl(cmd, &gup); 2259c84f229SJohn Hubbard if (ret) 2269c84f229SJohn Hubbard return ret; 2279c84f229SJohn Hubbard 2289c84f229SJohn Hubbard if (copy_to_user((void __user *)arg, &gup, sizeof(gup))) 2299c84f229SJohn Hubbard return -EFAULT; 2309c84f229SJohn Hubbard 2319c84f229SJohn Hubbard return 0; 2329c84f229SJohn Hubbard } 2339c84f229SJohn Hubbard 2349c84f229SJohn Hubbard static const struct file_operations gup_test_fops = { 2359c84f229SJohn Hubbard .open = nonseekable_open, 2369c84f229SJohn Hubbard .unlocked_ioctl = gup_test_ioctl, 2379c84f229SJohn Hubbard }; 2389c84f229SJohn Hubbard 239*afaa7888SBarry Song static int __init gup_test_init(void) 2409c84f229SJohn Hubbard { 2419c84f229SJohn Hubbard debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL, 2429c84f229SJohn Hubbard &gup_test_fops); 2439c84f229SJohn Hubbard 2449c84f229SJohn Hubbard return 0; 2459c84f229SJohn Hubbard } 2469c84f229SJohn Hubbard 2479c84f229SJohn Hubbard late_initcall(gup_test_init); 248