1 #include <fcntl.h> 2 #include <errno.h> 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <unistd.h> 6 #include <dirent.h> 7 #include <sys/ioctl.h> 8 #include <sys/mman.h> 9 #include <sys/stat.h> 10 #include <sys/types.h> 11 #include <pthread.h> 12 #include <assert.h> 13 #include <mm/gup_test.h> 14 #include "../kselftest.h" 15 #include "vm_util.h" 16 17 #define MB (1UL << 20) 18 19 /* Just the flags we need, copied from mm.h: */ 20 #define FOLL_WRITE 0x01 /* check pte is writable */ 21 #define FOLL_TOUCH 0x02 /* mark page accessed */ 22 23 #define GUP_TEST_FILE "/sys/kernel/debug/gup_test" 24 25 static unsigned long cmd = GUP_FAST_BENCHMARK; 26 static int gup_fd, repeats = 1; 27 static unsigned long size = 128 * MB; 28 /* Serialize prints */ 29 static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER; 30 31 static char *cmd_to_str(unsigned long cmd) 32 { 33 switch (cmd) { 34 case GUP_FAST_BENCHMARK: 35 return "GUP_FAST_BENCHMARK"; 36 case PIN_FAST_BENCHMARK: 37 return "PIN_FAST_BENCHMARK"; 38 case PIN_LONGTERM_BENCHMARK: 39 return "PIN_LONGTERM_BENCHMARK"; 40 case GUP_BASIC_TEST: 41 return "GUP_BASIC_TEST"; 42 case PIN_BASIC_TEST: 43 return "PIN_BASIC_TEST"; 44 case DUMP_USER_PAGES_TEST: 45 return "DUMP_USER_PAGES_TEST"; 46 } 47 return "Unknown command"; 48 } 49 50 void *gup_thread(void *data) 51 { 52 struct gup_test gup = *(struct gup_test *)data; 53 int i, status; 54 55 /* Only report timing information on the *_BENCHMARK commands: */ 56 if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) || 57 (cmd == PIN_LONGTERM_BENCHMARK)) { 58 for (i = 0; i < repeats; i++) { 59 gup.size = size; 60 status = ioctl(gup_fd, cmd, &gup); 61 if (status) 62 break; 63 64 pthread_mutex_lock(&print_mutex); 65 ksft_print_msg("%s: Time: get:%lld put:%lld us", 66 cmd_to_str(cmd), gup.get_delta_usec, 67 gup.put_delta_usec); 68 if (gup.size != size) 69 ksft_print_msg(", truncated (size: %lld)", gup.size); 70 ksft_print_msg("\n"); 71 pthread_mutex_unlock(&print_mutex); 72 } 73 } else { 74 gup.size = size; 75 status = ioctl(gup_fd, cmd, &gup); 76 if (status) 77 goto return_; 78 79 pthread_mutex_lock(&print_mutex); 80 ksft_print_msg("%s: done\n", cmd_to_str(cmd)); 81 if (gup.size != size) 82 ksft_print_msg("Truncated (size: %lld)\n", gup.size); 83 pthread_mutex_unlock(&print_mutex); 84 } 85 86 return_: 87 ksft_test_result(!status, "ioctl status %d\n", status); 88 return NULL; 89 } 90 91 int main(int argc, char **argv) 92 { 93 struct gup_test gup = { 0 }; 94 int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret; 95 int flags = MAP_PRIVATE, touch = 0; 96 char *file = "/dev/zero"; 97 pthread_t *tid; 98 char *p; 99 100 while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) { 101 switch (opt) { 102 case 'a': 103 cmd = PIN_FAST_BENCHMARK; 104 break; 105 case 'b': 106 cmd = PIN_BASIC_TEST; 107 break; 108 case 'L': 109 cmd = PIN_LONGTERM_BENCHMARK; 110 break; 111 case 'c': 112 cmd = DUMP_USER_PAGES_TEST; 113 /* 114 * Dump page 0 (index 1). May be overridden later, by 115 * user's non-option arguments. 116 * 117 * .which_pages is zero-based, so that zero can mean "do 118 * nothing". 119 */ 120 gup.which_pages[0] = 1; 121 break; 122 case 'p': 123 /* works only with DUMP_USER_PAGES_TEST */ 124 gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN; 125 break; 126 case 'F': 127 /* strtol, so you can pass flags in hex form */ 128 gup.gup_flags = strtol(optarg, 0, 0); 129 break; 130 case 'j': 131 nthreads = atoi(optarg); 132 break; 133 case 'm': 134 size = atoi(optarg) * MB; 135 break; 136 case 'r': 137 repeats = atoi(optarg); 138 break; 139 case 'n': 140 nr_pages = atoi(optarg); 141 break; 142 case 't': 143 thp = 1; 144 break; 145 case 'T': 146 thp = 0; 147 break; 148 case 'U': 149 cmd = GUP_BASIC_TEST; 150 break; 151 case 'u': 152 cmd = GUP_FAST_BENCHMARK; 153 break; 154 case 'w': 155 write = 1; 156 break; 157 case 'W': 158 write = 0; 159 break; 160 case 'f': 161 file = optarg; 162 break; 163 case 'S': 164 flags &= ~MAP_PRIVATE; 165 flags |= MAP_SHARED; 166 break; 167 case 'H': 168 flags |= (MAP_HUGETLB | MAP_ANONYMOUS); 169 break; 170 case 'z': 171 /* fault pages in gup, do not fault in userland */ 172 touch = 1; 173 break; 174 default: 175 ksft_exit_fail_msg("Wrong argument\n"); 176 } 177 } 178 179 if (optind < argc) { 180 int extra_arg_count = 0; 181 /* 182 * For example: 183 * 184 * ./gup_test -c 0 1 0x1001 185 * 186 * ...to dump pages 0, 1, and 4097 187 */ 188 189 while ((optind < argc) && 190 (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) { 191 /* 192 * Do the 1-based indexing here, so that the user can 193 * use normal 0-based indexing on the command line. 194 */ 195 long page_index = strtol(argv[optind], 0, 0) + 1; 196 197 gup.which_pages[extra_arg_count] = page_index; 198 extra_arg_count++; 199 optind++; 200 } 201 } 202 203 ksft_print_header(); 204 ksft_set_plan(nthreads); 205 206 filed = open(file, O_RDWR|O_CREAT); 207 if (filed < 0) 208 ksft_exit_fail_msg("Unable to open %s: %s\n", file, strerror(errno)); 209 210 gup.nr_pages_per_call = nr_pages; 211 if (write) 212 gup.gup_flags |= FOLL_WRITE; 213 214 gup_fd = open(GUP_TEST_FILE, O_RDWR); 215 if (gup_fd == -1) { 216 switch (errno) { 217 case EACCES: 218 if (getuid()) 219 ksft_print_msg("Please run this test as root\n"); 220 break; 221 case ENOENT: 222 if (opendir("/sys/kernel/debug") == NULL) 223 ksft_print_msg("mount debugfs at /sys/kernel/debug\n"); 224 ksft_print_msg("check if CONFIG_GUP_TEST is enabled in kernel config\n"); 225 break; 226 default: 227 ksft_print_msg("failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno)); 228 break; 229 } 230 ksft_test_result_skip("Please run this test as root\n"); 231 return ksft_exit_pass(); 232 } 233 234 p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0); 235 if (p == MAP_FAILED) 236 ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); 237 gup.addr = (unsigned long)p; 238 239 if (thp == 1) 240 madvise(p, size, MADV_HUGEPAGE); 241 else if (thp == 0) 242 madvise(p, size, MADV_NOHUGEPAGE); 243 244 /* 245 * FOLL_TOUCH, in gup_test, is used as an either/or case: either 246 * fault pages in from the kernel via FOLL_TOUCH, or fault them 247 * in here, from user space. This allows comparison of performance 248 * between those two cases. 249 */ 250 if (touch) { 251 gup.gup_flags |= FOLL_TOUCH; 252 } else { 253 for (; (unsigned long)p < gup.addr + size; p += psize()) 254 p[0] = 0; 255 } 256 257 tid = malloc(sizeof(pthread_t) * nthreads); 258 assert(tid); 259 for (i = 0; i < nthreads; i++) { 260 ret = pthread_create(&tid[i], NULL, gup_thread, &gup); 261 assert(ret == 0); 262 } 263 for (i = 0; i < nthreads; i++) { 264 ret = pthread_join(tid[i], NULL); 265 assert(ret == 0); 266 } 267 268 free(tid); 269 270 return ksft_exit_pass(); 271 } 272