1 #define __SANE_USERSPACE_TYPES__ // Use ll64 2 #include <fcntl.h> 3 #include <errno.h> 4 #include <stdio.h> 5 #include <stdlib.h> 6 #include <unistd.h> 7 #include <dirent.h> 8 #include <sys/ioctl.h> 9 #include <sys/mman.h> 10 #include <sys/stat.h> 11 #include <sys/types.h> 12 #include <pthread.h> 13 #include <assert.h> 14 #include <mm/gup_test.h> 15 #include "kselftest.h" 16 #include "vm_util.h" 17 #include "hugepage_settings.h" 18 19 #define MB (1UL << 20) 20 21 /* Just the flags we need, copied from the kernel internals. */ 22 #define FOLL_WRITE 0x01 /* check pte is writable */ 23 24 #define GUP_TEST_FILE "/sys/kernel/debug/gup_test" 25 26 static unsigned long cmd = GUP_FAST_BENCHMARK; 27 static int gup_fd, repeats = 1; 28 static unsigned long size = 128 * MB; 29 /* Serialize prints */ 30 static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER; 31 32 static char *cmd_to_str(unsigned long cmd) 33 { 34 switch (cmd) { 35 case GUP_FAST_BENCHMARK: 36 return "GUP_FAST_BENCHMARK"; 37 case PIN_FAST_BENCHMARK: 38 return "PIN_FAST_BENCHMARK"; 39 case PIN_LONGTERM_BENCHMARK: 40 return "PIN_LONGTERM_BENCHMARK"; 41 case GUP_BASIC_TEST: 42 return "GUP_BASIC_TEST"; 43 case PIN_BASIC_TEST: 44 return "PIN_BASIC_TEST"; 45 case DUMP_USER_PAGES_TEST: 46 return "DUMP_USER_PAGES_TEST"; 47 } 48 return "Unknown command"; 49 } 50 51 void *gup_thread(void *data) 52 { 53 struct gup_test gup = *(struct gup_test *)data; 54 int i, status; 55 56 /* Only report timing information on the *_BENCHMARK commands: */ 57 if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) || 58 (cmd == PIN_LONGTERM_BENCHMARK)) { 59 for (i = 0; i < repeats; i++) { 60 gup.size = size; 61 status = ioctl(gup_fd, cmd, &gup); 62 if (status) 63 break; 64 65 pthread_mutex_lock(&print_mutex); 66 ksft_print_msg("%s: Time: get:%lld put:%lld us", 67 cmd_to_str(cmd), gup.get_delta_usec, 68 gup.put_delta_usec); 69 if (gup.size != size) 70 ksft_print_msg(", truncated (size: %lld)", gup.size); 71 ksft_print_msg("\n"); 72 pthread_mutex_unlock(&print_mutex); 73 } 74 } else { 75 gup.size = size; 76 status = ioctl(gup_fd, cmd, &gup); 77 if (status) 78 goto return_; 79 80 pthread_mutex_lock(&print_mutex); 81 ksft_print_msg("%s: done\n", cmd_to_str(cmd)); 82 if (gup.size != size) 83 ksft_print_msg("Truncated (size: %lld)\n", gup.size); 84 pthread_mutex_unlock(&print_mutex); 85 } 86 87 return_: 88 ksft_test_result(!status, "ioctl status %d\n", status); 89 return NULL; 90 } 91 92 int main(int argc, char **argv) 93 { 94 struct gup_test gup = { 0 }; 95 int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret; 96 int flags = MAP_PRIVATE; 97 char *file = "/dev/zero"; 98 bool hugetlb = false; 99 pthread_t *tid; 100 char *p; 101 102 while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) { 103 switch (opt) { 104 case 'a': 105 cmd = PIN_FAST_BENCHMARK; 106 break; 107 case 'b': 108 cmd = PIN_BASIC_TEST; 109 break; 110 case 'L': 111 cmd = PIN_LONGTERM_BENCHMARK; 112 break; 113 case 'c': 114 cmd = DUMP_USER_PAGES_TEST; 115 /* 116 * Dump page 0 (index 1). May be overridden later, by 117 * user's non-option arguments. 118 * 119 * .which_pages is zero-based, so that zero can mean "do 120 * nothing". 121 */ 122 gup.which_pages[0] = 1; 123 break; 124 case 'p': 125 /* works only with DUMP_USER_PAGES_TEST */ 126 gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN; 127 break; 128 case 'F': 129 /* strtol, so you can pass flags in hex form */ 130 gup.gup_flags = strtol(optarg, 0, 0); 131 break; 132 case 'j': 133 nthreads = atoi(optarg); 134 break; 135 case 'm': 136 size = atoi(optarg) * MB; 137 break; 138 case 'r': 139 repeats = atoi(optarg); 140 break; 141 case 'n': 142 nr_pages = atoi(optarg); 143 if (nr_pages < 0) 144 nr_pages = size / psize(); 145 break; 146 case 't': 147 thp = 1; 148 break; 149 case 'T': 150 thp = 0; 151 break; 152 case 'U': 153 cmd = GUP_BASIC_TEST; 154 break; 155 case 'u': 156 cmd = GUP_FAST_BENCHMARK; 157 break; 158 case 'w': 159 write = 1; 160 break; 161 case 'W': 162 write = 0; 163 break; 164 case 'f': 165 file = optarg; 166 break; 167 case 'S': 168 flags &= ~MAP_PRIVATE; 169 flags |= MAP_SHARED; 170 break; 171 case 'H': 172 flags |= (MAP_HUGETLB | MAP_ANONYMOUS); 173 hugetlb = true; 174 break; 175 default: 176 ksft_exit_fail_msg("Wrong argument\n"); 177 } 178 } 179 180 if (optind < argc) { 181 int extra_arg_count = 0; 182 /* 183 * For example: 184 * 185 * ./gup_test -c 0 1 0x1001 186 * 187 * ...to dump pages 0, 1, and 4097 188 */ 189 190 while ((optind < argc) && 191 (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) { 192 /* 193 * Do the 1-based indexing here, so that the user can 194 * use normal 0-based indexing on the command line. 195 */ 196 long page_index = strtol(argv[optind], 0, 0) + 1; 197 198 gup.which_pages[extra_arg_count] = page_index; 199 extra_arg_count++; 200 optind++; 201 } 202 } 203 204 ksft_print_header(); 205 206 if (hugetlb) { 207 unsigned long hp_size = default_huge_page_size(); 208 209 if (!hp_size) 210 ksft_exit_skip("HugeTLB is unavailable\n"); 211 212 size = (size + hp_size - 1) & ~(hp_size - 1); 213 if (!hugetlb_setup_default(size / hp_size)) 214 ksft_exit_skip("Not enough huge pages\n"); 215 } 216 217 ksft_set_plan(nthreads); 218 219 filed = open(file, O_RDWR|O_CREAT, 0664); 220 if (filed < 0) 221 ksft_exit_fail_msg("Unable to open %s: %s\n", file, strerror(errno)); 222 223 gup.nr_pages_per_call = nr_pages; 224 if (write) 225 gup.gup_flags |= FOLL_WRITE; 226 227 gup_fd = open(GUP_TEST_FILE, O_RDWR); 228 if (gup_fd == -1) { 229 switch (errno) { 230 case EACCES: 231 if (getuid()) 232 ksft_print_msg("Please run this test as root\n"); 233 break; 234 case ENOENT: 235 if (opendir("/sys/kernel/debug") == NULL) 236 ksft_print_msg("mount debugfs at /sys/kernel/debug\n"); 237 ksft_print_msg("check if CONFIG_GUP_TEST is enabled in kernel config\n"); 238 break; 239 default: 240 ksft_print_msg("failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno)); 241 break; 242 } 243 ksft_test_result_skip("Please run this test as root\n"); 244 ksft_exit_pass(); 245 } 246 247 p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0); 248 if (p == MAP_FAILED) 249 ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); 250 gup.addr = (unsigned long)p; 251 252 if (thp == 1) 253 madvise(p, size, MADV_HUGEPAGE); 254 else if (thp == 0) 255 madvise(p, size, MADV_NOHUGEPAGE); 256 257 /* Fault them in here, from user space. */ 258 for (; (unsigned long)p < gup.addr + size; p += psize()) 259 p[0] = 0; 260 261 tid = malloc(sizeof(pthread_t) * nthreads); 262 assert(tid); 263 for (i = 0; i < nthreads; i++) { 264 ret = pthread_create(&tid[i], NULL, gup_thread, &gup); 265 assert(ret == 0); 266 } 267 for (i = 0; i < nthreads; i++) { 268 ret = pthread_join(tid[i], NULL); 269 assert(ret == 0); 270 } 271 272 free(tid); 273 274 ksft_exit_pass(); 275 } 276