1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/mm.h>
4 #include <linux/slab.h>
5 #include <linux/uaccess.h>
6 #include <linux/ktime.h>
7 #include <linux/debugfs.h>
8 #include <linux/highmem.h>
9 #include "gup_test.h"
10
11 struct gup_test_data {
12 struct mutex longterm_mutex;
13 struct page **longterm_pages;
14 unsigned long longterm_nr_pages;
15 };
16
put_back_pages(unsigned int cmd,struct page ** pages,unsigned long nr_pages,unsigned int gup_test_flags)17 static void put_back_pages(unsigned int cmd, struct page **pages,
18 unsigned long nr_pages, unsigned int gup_test_flags)
19 {
20 unsigned long i;
21
22 switch (cmd) {
23 case GUP_FAST_BENCHMARK:
24 case GUP_BASIC_TEST:
25 for (i = 0; i < nr_pages; i++)
26 put_page(pages[i]);
27 break;
28
29 case PIN_FAST_BENCHMARK:
30 case PIN_BASIC_TEST:
31 case PIN_LONGTERM_BENCHMARK:
32 unpin_user_pages(pages, nr_pages);
33 break;
34 case DUMP_USER_PAGES_TEST:
35 if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {
36 unpin_user_pages(pages, nr_pages);
37 } else {
38 for (i = 0; i < nr_pages; i++)
39 put_page(pages[i]);
40
41 }
42 break;
43 }
44 }
45
verify_dma_pinned(unsigned int cmd,struct page ** pages,unsigned long nr_pages)46 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
47 unsigned long nr_pages)
48 {
49 unsigned long i;
50 struct folio *folio;
51
52 switch (cmd) {
53 case PIN_FAST_BENCHMARK:
54 case PIN_BASIC_TEST:
55 case PIN_LONGTERM_BENCHMARK:
56 for (i = 0; i < nr_pages; i++) {
57 folio = page_folio(pages[i]);
58
59 if (WARN(!folio_maybe_dma_pinned(folio),
60 "pages[%lu] is NOT dma-pinned\n", i)) {
61
62 dump_page(&folio->page, "gup_test failure");
63 break;
64 } else if (cmd == PIN_LONGTERM_BENCHMARK &&
65 WARN(!folio_is_longterm_pinnable(folio),
66 "pages[%lu] is NOT pinnable but pinned\n",
67 i)) {
68 dump_page(&folio->page, "gup_test failure");
69 break;
70 }
71 }
72 break;
73 }
74 }
75
dump_pages_test(struct gup_test * gup,struct page ** pages,unsigned long nr_pages)76 static void dump_pages_test(struct gup_test *gup, struct page **pages,
77 unsigned long nr_pages)
78 {
79 unsigned int index_to_dump;
80 unsigned int i;
81
82 /*
83 * Zero out any user-supplied page index that is out of range. Remember:
84 * .which_pages[] contains a 1-based set of page indices.
85 */
86 for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
87 if (gup->which_pages[i] > nr_pages) {
88 pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
89 i, gup->which_pages[i]);
90 gup->which_pages[i] = 0;
91 }
92 }
93
94 for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
95 index_to_dump = gup->which_pages[i];
96
97 if (index_to_dump) {
98 index_to_dump--; // Decode from 1-based, to 0-based
99 pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
100 index_to_dump, gup->addr);
101 dump_page(pages[index_to_dump],
102 "gup_test: dump_pages() test");
103 }
104 }
105 }
106
__gup_test_ioctl(unsigned int cmd,struct gup_test * gup)107 static int __gup_test_ioctl(unsigned int cmd,
108 struct gup_test *gup)
109 {
110 ktime_t start_time, end_time;
111 unsigned long i, nr_pages, addr, next;
112 long nr;
113 struct page **pages;
114 unsigned long end;
115 int ret = 0;
116 bool needs_mmap_lock =
117 cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
118
119 if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX)
120 return -EINVAL;
121 if (check_add_overflow((unsigned long)gup->addr,
122 (unsigned long)gup->size, &end))
123 return -EINVAL;
124
125 nr_pages = gup->size / PAGE_SIZE;
126 pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
127 if (!pages)
128 return -ENOMEM;
129
130 if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
131 ret = -EINTR;
132 goto free_pages;
133 }
134
135 i = 0;
136 nr = gup->nr_pages_per_call;
137 start_time = ktime_get();
138 for (addr = gup->addr; addr < end; addr = next) {
139 if (nr != gup->nr_pages_per_call)
140 break;
141
142 next = addr + nr * PAGE_SIZE;
143 if (next > end) {
144 next = end;
145 nr = (next - addr) / PAGE_SIZE;
146 }
147
148 switch (cmd) {
149 case GUP_FAST_BENCHMARK:
150 nr = get_user_pages_fast(addr, nr, gup->gup_flags,
151 pages + i);
152 break;
153 case GUP_BASIC_TEST:
154 nr = get_user_pages(addr, nr, gup->gup_flags, pages + i);
155 break;
156 case PIN_FAST_BENCHMARK:
157 nr = pin_user_pages_fast(addr, nr, gup->gup_flags,
158 pages + i);
159 break;
160 case PIN_BASIC_TEST:
161 nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i);
162 break;
163 case PIN_LONGTERM_BENCHMARK:
164 nr = pin_user_pages(addr, nr,
165 gup->gup_flags | FOLL_LONGTERM,
166 pages + i);
167 break;
168 case DUMP_USER_PAGES_TEST:
169 if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)
170 nr = pin_user_pages(addr, nr, gup->gup_flags,
171 pages + i);
172 else
173 nr = get_user_pages(addr, nr, gup->gup_flags,
174 pages + i);
175 break;
176 default:
177 ret = -EINVAL;
178 goto unlock;
179 }
180
181 if (nr <= 0)
182 break;
183 i += nr;
184 }
185 end_time = ktime_get();
186
187 /* Shifting the meaning of nr_pages: now it is actual number pinned: */
188 nr_pages = i;
189
190 gup->get_delta_usec = ktime_us_delta(end_time, start_time);
191 gup->size = addr - gup->addr;
192
193 /*
194 * Take an un-benchmark-timed moment to verify DMA pinned
195 * state: print a warning if any non-dma-pinned pages are found:
196 */
197 verify_dma_pinned(cmd, pages, nr_pages);
198
199 if (cmd == DUMP_USER_PAGES_TEST)
200 dump_pages_test(gup, pages, nr_pages);
201
202 start_time = ktime_get();
203
204 put_back_pages(cmd, pages, nr_pages, gup->test_flags);
205
206 end_time = ktime_get();
207 gup->put_delta_usec = ktime_us_delta(end_time, start_time);
208
209 unlock:
210 if (needs_mmap_lock)
211 mmap_read_unlock(current->mm);
212 free_pages:
213 kvfree(pages);
214 return ret;
215 }
216
pin_longterm_test_stop(struct gup_test_data * data)217 static inline void pin_longterm_test_stop(struct gup_test_data *data)
218 {
219 if (data->longterm_pages) {
220 if (data->longterm_nr_pages)
221 unpin_user_pages(data->longterm_pages,
222 data->longterm_nr_pages);
223 kvfree(data->longterm_pages);
224 data->longterm_pages = NULL;
225 data->longterm_nr_pages = 0;
226 }
227 }
228
pin_longterm_test_start(struct gup_test_data * data,unsigned long arg)229 static inline int pin_longterm_test_start(struct gup_test_data *data,
230 unsigned long arg)
231 {
232 long nr_pages, cur_pages, addr, remaining_pages;
233 int gup_flags = FOLL_LONGTERM;
234 struct pin_longterm_test args;
235 struct page **pages;
236 int ret = 0;
237 bool fast;
238
239 if (data->longterm_pages)
240 return -EINVAL;
241
242 if (copy_from_user(&args, (void __user *)arg, sizeof(args)))
243 return -EFAULT;
244
245 if (args.flags &
246 ~(PIN_LONGTERM_TEST_FLAG_USE_WRITE|PIN_LONGTERM_TEST_FLAG_USE_FAST))
247 return -EINVAL;
248 if (!IS_ALIGNED(args.addr | args.size, PAGE_SIZE))
249 return -EINVAL;
250 if (args.size > LONG_MAX)
251 return -EINVAL;
252 nr_pages = args.size / PAGE_SIZE;
253 if (!nr_pages)
254 return -EINVAL;
255
256 pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
257 if (!pages)
258 return -ENOMEM;
259
260 if (args.flags & PIN_LONGTERM_TEST_FLAG_USE_WRITE)
261 gup_flags |= FOLL_WRITE;
262 fast = !!(args.flags & PIN_LONGTERM_TEST_FLAG_USE_FAST);
263
264 if (!fast && mmap_read_lock_killable(current->mm)) {
265 kvfree(pages);
266 return -EINTR;
267 }
268
269 data->longterm_pages = pages;
270 data->longterm_nr_pages = 0;
271
272 while (nr_pages - data->longterm_nr_pages) {
273 remaining_pages = nr_pages - data->longterm_nr_pages;
274 addr = args.addr + data->longterm_nr_pages * PAGE_SIZE;
275
276 if (fast)
277 cur_pages = pin_user_pages_fast(addr, remaining_pages,
278 gup_flags, pages);
279 else
280 cur_pages = pin_user_pages(addr, remaining_pages,
281 gup_flags, pages);
282 if (cur_pages < 0) {
283 pin_longterm_test_stop(data);
284 ret = cur_pages;
285 break;
286 }
287 data->longterm_nr_pages += cur_pages;
288 pages += cur_pages;
289 }
290
291 if (!fast)
292 mmap_read_unlock(current->mm);
293 return ret;
294 }
295
pin_longterm_test_read(struct gup_test_data * data,unsigned long arg)296 static inline int pin_longterm_test_read(struct gup_test_data *data,
297 unsigned long arg)
298 {
299 __u64 user_addr;
300 unsigned long i;
301
302 if (!data->longterm_pages)
303 return -EINVAL;
304
305 if (copy_from_user(&user_addr, (void __user *)arg, sizeof(user_addr)))
306 return -EFAULT;
307
308 for (i = 0; i < data->longterm_nr_pages; i++) {
309 void *addr = kmap_local_page(data->longterm_pages[i]);
310 unsigned long ret;
311
312 ret = copy_to_user((void __user *)(unsigned long)user_addr, addr,
313 PAGE_SIZE);
314 kunmap_local(addr);
315 if (ret)
316 return -EFAULT;
317 user_addr += PAGE_SIZE;
318 }
319 return 0;
320 }
321
pin_longterm_test_ioctl(struct file * filep,unsigned int cmd,unsigned long arg)322 static long pin_longterm_test_ioctl(struct file *filep, unsigned int cmd,
323 unsigned long arg)
324 {
325 struct gup_test_data *data = filep->private_data;
326 int ret = -EINVAL;
327
328 if (mutex_lock_killable(&data->longterm_mutex))
329 return -EINTR;
330
331 switch (cmd) {
332 case PIN_LONGTERM_TEST_START:
333 ret = pin_longterm_test_start(data, arg);
334 break;
335 case PIN_LONGTERM_TEST_STOP:
336 pin_longterm_test_stop(data);
337 ret = 0;
338 break;
339 case PIN_LONGTERM_TEST_READ:
340 ret = pin_longterm_test_read(data, arg);
341 break;
342 }
343
344 mutex_unlock(&data->longterm_mutex);
345 return ret;
346 }
347
gup_test_ioctl(struct file * filep,unsigned int cmd,unsigned long arg)348 static long gup_test_ioctl(struct file *filep, unsigned int cmd,
349 unsigned long arg)
350 {
351 struct gup_test gup;
352 int ret;
353
354 switch (cmd) {
355 case GUP_FAST_BENCHMARK:
356 case PIN_FAST_BENCHMARK:
357 case PIN_LONGTERM_BENCHMARK:
358 case GUP_BASIC_TEST:
359 case PIN_BASIC_TEST:
360 case DUMP_USER_PAGES_TEST:
361 break;
362 case PIN_LONGTERM_TEST_START:
363 case PIN_LONGTERM_TEST_STOP:
364 case PIN_LONGTERM_TEST_READ:
365 return pin_longterm_test_ioctl(filep, cmd, arg);
366 default:
367 return -EINVAL;
368 }
369
370 if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
371 return -EFAULT;
372
373 ret = __gup_test_ioctl(cmd, &gup);
374 if (ret)
375 return ret;
376
377 if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
378 return -EFAULT;
379
380 return 0;
381 }
382
gup_test_open(struct inode * inode,struct file * file)383 static int gup_test_open(struct inode *inode, struct file *file)
384 {
385 struct gup_test_data *data;
386 int ret;
387
388 data = kzalloc_obj(*data);
389 if (!data)
390 return -ENOMEM;
391
392 ret = nonseekable_open(inode, file);
393 if (ret) {
394 kfree(data);
395 return ret;
396 }
397
398 mutex_init(&data->longterm_mutex);
399 file->private_data = data;
400 return 0;
401 }
402
gup_test_release(struct inode * inode,struct file * file)403 static int gup_test_release(struct inode *inode, struct file *file)
404 {
405 struct gup_test_data *data = file->private_data;
406
407 pin_longterm_test_stop(data);
408 mutex_destroy(&data->longterm_mutex);
409 kfree(data);
410 file->private_data = NULL;
411
412 return 0;
413 }
414
415 static const struct file_operations gup_test_fops = {
416 .open = gup_test_open,
417 .unlocked_ioctl = gup_test_ioctl,
418 .compat_ioctl = compat_ptr_ioctl,
419 .release = gup_test_release,
420 };
421
gup_test_init(void)422 static int __init gup_test_init(void)
423 {
424 debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,
425 &gup_test_fops);
426
427 return 0;
428 }
429
430 late_initcall(gup_test_init);
431