1 #define __SANE_USERSPACE_TYPES__ // Use ll64
2 #include <fcntl.h>
3 #include <errno.h>
4 #include <stdio.h>
5 #include <stdlib.h>
6 #include <unistd.h>
7 #include <dirent.h>
8 #include <sys/ioctl.h>
9 #include <sys/mman.h>
10 #include <sys/stat.h>
11 #include <sys/types.h>
12 #include <pthread.h>
13 #include <assert.h>
14 #include <mm/gup_test.h>
15 #include "../kselftest.h"
16 #include "vm_util.h"
17
18 #define MB (1UL << 20)
19
20 /* Just the flags we need, copied from mm.h: */
21 #define FOLL_WRITE 0x01 /* check pte is writable */
22 #define FOLL_TOUCH 0x02 /* mark page accessed */
23
24 #define GUP_TEST_FILE "/sys/kernel/debug/gup_test"
25
26 static unsigned long cmd = GUP_FAST_BENCHMARK;
27 static int gup_fd, repeats = 1;
28 static unsigned long size = 128 * MB;
29 /* Serialize prints */
30 static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER;
31
cmd_to_str(unsigned long cmd)32 static char *cmd_to_str(unsigned long cmd)
33 {
34 switch (cmd) {
35 case GUP_FAST_BENCHMARK:
36 return "GUP_FAST_BENCHMARK";
37 case PIN_FAST_BENCHMARK:
38 return "PIN_FAST_BENCHMARK";
39 case PIN_LONGTERM_BENCHMARK:
40 return "PIN_LONGTERM_BENCHMARK";
41 case GUP_BASIC_TEST:
42 return "GUP_BASIC_TEST";
43 case PIN_BASIC_TEST:
44 return "PIN_BASIC_TEST";
45 case DUMP_USER_PAGES_TEST:
46 return "DUMP_USER_PAGES_TEST";
47 }
48 return "Unknown command";
49 }
50
gup_thread(void * data)51 void *gup_thread(void *data)
52 {
53 struct gup_test gup = *(struct gup_test *)data;
54 int i, status;
55
56 /* Only report timing information on the *_BENCHMARK commands: */
57 if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) ||
58 (cmd == PIN_LONGTERM_BENCHMARK)) {
59 for (i = 0; i < repeats; i++) {
60 gup.size = size;
61 status = ioctl(gup_fd, cmd, &gup);
62 if (status)
63 break;
64
65 pthread_mutex_lock(&print_mutex);
66 ksft_print_msg("%s: Time: get:%lld put:%lld us",
67 cmd_to_str(cmd), gup.get_delta_usec,
68 gup.put_delta_usec);
69 if (gup.size != size)
70 ksft_print_msg(", truncated (size: %lld)", gup.size);
71 ksft_print_msg("\n");
72 pthread_mutex_unlock(&print_mutex);
73 }
74 } else {
75 gup.size = size;
76 status = ioctl(gup_fd, cmd, &gup);
77 if (status)
78 goto return_;
79
80 pthread_mutex_lock(&print_mutex);
81 ksft_print_msg("%s: done\n", cmd_to_str(cmd));
82 if (gup.size != size)
83 ksft_print_msg("Truncated (size: %lld)\n", gup.size);
84 pthread_mutex_unlock(&print_mutex);
85 }
86
87 return_:
88 ksft_test_result(!status, "ioctl status %d\n", status);
89 return NULL;
90 }
91
main(int argc,char ** argv)92 int main(int argc, char **argv)
93 {
94 struct gup_test gup = { 0 };
95 int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret;
96 int flags = MAP_PRIVATE, touch = 0;
97 char *file = "/dev/zero";
98 pthread_t *tid;
99 char *p;
100
101 while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) {
102 switch (opt) {
103 case 'a':
104 cmd = PIN_FAST_BENCHMARK;
105 break;
106 case 'b':
107 cmd = PIN_BASIC_TEST;
108 break;
109 case 'L':
110 cmd = PIN_LONGTERM_BENCHMARK;
111 break;
112 case 'c':
113 cmd = DUMP_USER_PAGES_TEST;
114 /*
115 * Dump page 0 (index 1). May be overridden later, by
116 * user's non-option arguments.
117 *
118 * .which_pages is zero-based, so that zero can mean "do
119 * nothing".
120 */
121 gup.which_pages[0] = 1;
122 break;
123 case 'p':
124 /* works only with DUMP_USER_PAGES_TEST */
125 gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN;
126 break;
127 case 'F':
128 /* strtol, so you can pass flags in hex form */
129 gup.gup_flags = strtol(optarg, 0, 0);
130 break;
131 case 'j':
132 nthreads = atoi(optarg);
133 break;
134 case 'm':
135 size = atoi(optarg) * MB;
136 break;
137 case 'r':
138 repeats = atoi(optarg);
139 break;
140 case 'n':
141 nr_pages = atoi(optarg);
142 if (nr_pages < 0)
143 nr_pages = size / psize();
144 break;
145 case 't':
146 thp = 1;
147 break;
148 case 'T':
149 thp = 0;
150 break;
151 case 'U':
152 cmd = GUP_BASIC_TEST;
153 break;
154 case 'u':
155 cmd = GUP_FAST_BENCHMARK;
156 break;
157 case 'w':
158 write = 1;
159 break;
160 case 'W':
161 write = 0;
162 break;
163 case 'f':
164 file = optarg;
165 break;
166 case 'S':
167 flags &= ~MAP_PRIVATE;
168 flags |= MAP_SHARED;
169 break;
170 case 'H':
171 flags |= (MAP_HUGETLB | MAP_ANONYMOUS);
172 break;
173 case 'z':
174 /* fault pages in gup, do not fault in userland */
175 touch = 1;
176 break;
177 default:
178 ksft_exit_fail_msg("Wrong argument\n");
179 }
180 }
181
182 if (optind < argc) {
183 int extra_arg_count = 0;
184 /*
185 * For example:
186 *
187 * ./gup_test -c 0 1 0x1001
188 *
189 * ...to dump pages 0, 1, and 4097
190 */
191
192 while ((optind < argc) &&
193 (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) {
194 /*
195 * Do the 1-based indexing here, so that the user can
196 * use normal 0-based indexing on the command line.
197 */
198 long page_index = strtol(argv[optind], 0, 0) + 1;
199
200 gup.which_pages[extra_arg_count] = page_index;
201 extra_arg_count++;
202 optind++;
203 }
204 }
205
206 ksft_print_header();
207 ksft_set_plan(nthreads);
208
209 filed = open(file, O_RDWR|O_CREAT, 0664);
210 if (filed < 0)
211 ksft_exit_fail_msg("Unable to open %s: %s\n", file, strerror(errno));
212
213 gup.nr_pages_per_call = nr_pages;
214 if (write)
215 gup.gup_flags |= FOLL_WRITE;
216
217 gup_fd = open(GUP_TEST_FILE, O_RDWR);
218 if (gup_fd == -1) {
219 switch (errno) {
220 case EACCES:
221 if (getuid())
222 ksft_print_msg("Please run this test as root\n");
223 break;
224 case ENOENT:
225 if (opendir("/sys/kernel/debug") == NULL)
226 ksft_print_msg("mount debugfs at /sys/kernel/debug\n");
227 ksft_print_msg("check if CONFIG_GUP_TEST is enabled in kernel config\n");
228 break;
229 default:
230 ksft_print_msg("failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno));
231 break;
232 }
233 ksft_test_result_skip("Please run this test as root\n");
234 ksft_exit_pass();
235 }
236
237 p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0);
238 if (p == MAP_FAILED)
239 ksft_exit_fail_msg("mmap: %s\n", strerror(errno));
240 gup.addr = (unsigned long)p;
241
242 if (thp == 1)
243 madvise(p, size, MADV_HUGEPAGE);
244 else if (thp == 0)
245 madvise(p, size, MADV_NOHUGEPAGE);
246
247 /*
248 * FOLL_TOUCH, in gup_test, is used as an either/or case: either
249 * fault pages in from the kernel via FOLL_TOUCH, or fault them
250 * in here, from user space. This allows comparison of performance
251 * between those two cases.
252 */
253 if (touch) {
254 gup.gup_flags |= FOLL_TOUCH;
255 } else {
256 for (; (unsigned long)p < gup.addr + size; p += psize())
257 p[0] = 0;
258 }
259
260 tid = malloc(sizeof(pthread_t) * nthreads);
261 assert(tid);
262 for (i = 0; i < nthreads; i++) {
263 ret = pthread_create(&tid[i], NULL, gup_thread, &gup);
264 assert(ret == 0);
265 }
266 for (i = 0; i < nthreads; i++) {
267 ret = pthread_join(tid[i], NULL);
268 assert(ret == 0);
269 }
270
271 free(tid);
272
273 ksft_exit_pass();
274 }
275