1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * GUP long-term page pinning tests. 4 * 5 * Copyright 2023, Red Hat, Inc. 6 * 7 * Author(s): David Hildenbrand <david@redhat.com> 8 */ 9 #define _GNU_SOURCE 10 #include <stdlib.h> 11 #include <string.h> 12 #include <stdbool.h> 13 #include <stdint.h> 14 #include <unistd.h> 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <assert.h> 18 #include <sys/mman.h> 19 #include <sys/ioctl.h> 20 #include <sys/vfs.h> 21 #include <linux/magic.h> 22 #include <linux/memfd.h> 23 24 #include "local_config.h" 25 #ifdef LOCAL_CONFIG_HAVE_LIBURING 26 #include <liburing.h> 27 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 28 29 #include "../../../../mm/gup_test.h" 30 #include "../kselftest.h" 31 #include "vm_util.h" 32 33 static size_t pagesize; 34 static int nr_hugetlbsizes; 35 static size_t hugetlbsizes[10]; 36 static int gup_fd; 37 38 static __fsword_t get_fs_type(int fd) 39 { 40 struct statfs fs; 41 int ret; 42 43 do { 44 ret = fstatfs(fd, &fs); 45 } while (ret && errno == EINTR); 46 47 return ret ? 0 : fs.f_type; 48 } 49 50 static bool fs_is_unknown(__fsword_t fs_type) 51 { 52 /* 53 * We only support some filesystems in our tests when dealing with 54 * R/W long-term pinning. For these filesystems, we can be fairly sure 55 * whether they support it or not. 56 */ 57 switch (fs_type) { 58 case TMPFS_MAGIC: 59 case HUGETLBFS_MAGIC: 60 case BTRFS_SUPER_MAGIC: 61 case EXT4_SUPER_MAGIC: 62 case XFS_SUPER_MAGIC: 63 return false; 64 default: 65 return true; 66 } 67 } 68 69 static bool fs_supports_writable_longterm_pinning(__fsword_t fs_type) 70 { 71 assert(!fs_is_unknown(fs_type)); 72 switch (fs_type) { 73 case TMPFS_MAGIC: 74 case HUGETLBFS_MAGIC: 75 return true; 76 default: 77 return false; 78 } 79 } 80 81 enum test_type { 82 TEST_TYPE_RO, 83 TEST_TYPE_RO_FAST, 84 TEST_TYPE_RW, 85 TEST_TYPE_RW_FAST, 86 #ifdef LOCAL_CONFIG_HAVE_LIBURING 87 TEST_TYPE_IOURING, 88 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 89 }; 90 91 static void do_test(int fd, size_t size, enum test_type type, bool shared) 92 { 93 __fsword_t fs_type = get_fs_type(fd); 94 bool should_work; 95 char *mem; 96 int ret; 97 98 if (ftruncate(fd, size)) { 99 ksft_test_result_fail("ftruncate() failed\n"); 100 return; 101 } 102 103 if (fallocate(fd, 0, 0, size)) { 104 if (size == pagesize) 105 ksft_test_result_fail("fallocate() failed\n"); 106 else 107 ksft_test_result_skip("need more free huge pages\n"); 108 return; 109 } 110 111 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, 112 shared ? MAP_SHARED : MAP_PRIVATE, fd, 0); 113 if (mem == MAP_FAILED) { 114 if (size == pagesize || shared) 115 ksft_test_result_fail("mmap() failed\n"); 116 else 117 ksft_test_result_skip("need more free huge pages\n"); 118 return; 119 } 120 121 /* Fault in the page such that GUP-fast can pin it directly. */ 122 memset(mem, 0, size); 123 124 switch (type) { 125 case TEST_TYPE_RO: 126 case TEST_TYPE_RO_FAST: 127 /* 128 * Cover more cases regarding unsharing decisions when 129 * long-term R/O pinning by mapping the page R/O. 130 */ 131 ret = mprotect(mem, size, PROT_READ); 132 if (ret) { 133 ksft_test_result_fail("mprotect() failed\n"); 134 goto munmap; 135 } 136 /* FALLTHROUGH */ 137 case TEST_TYPE_RW: 138 case TEST_TYPE_RW_FAST: { 139 struct pin_longterm_test args; 140 const bool fast = type == TEST_TYPE_RO_FAST || 141 type == TEST_TYPE_RW_FAST; 142 const bool rw = type == TEST_TYPE_RW || 143 type == TEST_TYPE_RW_FAST; 144 145 if (gup_fd < 0) { 146 ksft_test_result_skip("gup_test not available\n"); 147 break; 148 } 149 150 if (rw && shared && fs_is_unknown(fs_type)) { 151 ksft_test_result_skip("Unknown filesystem\n"); 152 return; 153 } 154 /* 155 * R/O pinning or pinning in a private mapping is always 156 * expected to work. Otherwise, we expect long-term R/W pinning 157 * to only succeed for special fielesystems. 158 */ 159 should_work = !shared || !rw || 160 fs_supports_writable_longterm_pinning(fs_type); 161 162 args.addr = (__u64)(uintptr_t)mem; 163 args.size = size; 164 args.flags = fast ? PIN_LONGTERM_TEST_FLAG_USE_FAST : 0; 165 args.flags |= rw ? PIN_LONGTERM_TEST_FLAG_USE_WRITE : 0; 166 ret = ioctl(gup_fd, PIN_LONGTERM_TEST_START, &args); 167 if (ret && errno == EINVAL) { 168 ksft_test_result_skip("PIN_LONGTERM_TEST_START failed\n"); 169 break; 170 } else if (ret && errno == EFAULT) { 171 ksft_test_result(!should_work, "Should have failed\n"); 172 break; 173 } else if (ret) { 174 ksft_test_result_fail("PIN_LONGTERM_TEST_START failed\n"); 175 break; 176 } 177 178 if (ioctl(gup_fd, PIN_LONGTERM_TEST_STOP)) 179 ksft_print_msg("[INFO] PIN_LONGTERM_TEST_STOP failed\n"); 180 181 /* 182 * TODO: if the kernel ever supports long-term R/W pinning on 183 * some previously unsupported filesystems, we might want to 184 * perform some additional tests for possible data corruptions. 185 */ 186 ksft_test_result(should_work, "Should have worked\n"); 187 break; 188 } 189 #ifdef LOCAL_CONFIG_HAVE_LIBURING 190 case TEST_TYPE_IOURING: { 191 struct io_uring ring; 192 struct iovec iov; 193 194 /* io_uring always pins pages writable. */ 195 if (shared && fs_is_unknown(fs_type)) { 196 ksft_test_result_skip("Unknown filesystem\n"); 197 return; 198 } 199 should_work = !shared || 200 fs_supports_writable_longterm_pinning(fs_type); 201 202 /* Skip on errors, as we might just lack kernel support. */ 203 ret = io_uring_queue_init(1, &ring, 0); 204 if (ret < 0) { 205 ksft_test_result_skip("io_uring_queue_init() failed\n"); 206 break; 207 } 208 /* 209 * Register the range as a fixed buffer. This will FOLL_WRITE | 210 * FOLL_PIN | FOLL_LONGTERM the range. 211 */ 212 iov.iov_base = mem; 213 iov.iov_len = size; 214 ret = io_uring_register_buffers(&ring, &iov, 1); 215 /* Only new kernels return EFAULT. */ 216 if (ret && (errno == ENOSPC || errno == EOPNOTSUPP || 217 errno == EFAULT)) { 218 ksft_test_result(!should_work, "Should have failed\n"); 219 } else if (ret) { 220 /* 221 * We might just lack support or have insufficient 222 * MEMLOCK limits. 223 */ 224 ksft_test_result_skip("io_uring_register_buffers() failed\n"); 225 } else { 226 ksft_test_result(should_work, "Should have worked\n"); 227 io_uring_unregister_buffers(&ring); 228 } 229 230 io_uring_queue_exit(&ring); 231 break; 232 } 233 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 234 default: 235 assert(false); 236 } 237 238 munmap: 239 munmap(mem, size); 240 } 241 242 typedef void (*test_fn)(int fd, size_t size); 243 244 static void run_with_memfd(test_fn fn, const char *desc) 245 { 246 int fd; 247 248 ksft_print_msg("[RUN] %s ... with memfd\n", desc); 249 250 fd = memfd_create("test", 0); 251 if (fd < 0) { 252 ksft_test_result_fail("memfd_create() failed\n"); 253 return; 254 } 255 256 fn(fd, pagesize); 257 close(fd); 258 } 259 260 static void run_with_tmpfile(test_fn fn, const char *desc) 261 { 262 FILE *file; 263 int fd; 264 265 ksft_print_msg("[RUN] %s ... with tmpfile\n", desc); 266 267 file = tmpfile(); 268 if (!file) { 269 ksft_test_result_fail("tmpfile() failed\n"); 270 return; 271 } 272 273 fd = fileno(file); 274 if (fd < 0) { 275 ksft_test_result_fail("fileno() failed\n"); 276 goto close; 277 } 278 279 fn(fd, pagesize); 280 close: 281 fclose(file); 282 } 283 284 static void run_with_local_tmpfile(test_fn fn, const char *desc) 285 { 286 char filename[] = __FILE__"_tmpfile_XXXXXX"; 287 int fd; 288 289 ksft_print_msg("[RUN] %s ... with local tmpfile\n", desc); 290 291 fd = mkstemp(filename); 292 if (fd < 0) { 293 ksft_test_result_fail("mkstemp() failed\n"); 294 return; 295 } 296 297 if (unlink(filename)) { 298 ksft_test_result_fail("unlink() failed\n"); 299 goto close; 300 } 301 302 fn(fd, pagesize); 303 close: 304 close(fd); 305 } 306 307 static void run_with_memfd_hugetlb(test_fn fn, const char *desc, 308 size_t hugetlbsize) 309 { 310 int flags = MFD_HUGETLB; 311 int fd; 312 313 ksft_print_msg("[RUN] %s ... with memfd hugetlb (%zu kB)\n", desc, 314 hugetlbsize / 1024); 315 316 flags |= __builtin_ctzll(hugetlbsize) << MFD_HUGE_SHIFT; 317 318 fd = memfd_create("test", flags); 319 if (fd < 0) { 320 ksft_test_result_skip("memfd_create() failed\n"); 321 return; 322 } 323 324 fn(fd, hugetlbsize); 325 close(fd); 326 } 327 328 struct test_case { 329 const char *desc; 330 test_fn fn; 331 }; 332 333 static void test_shared_rw_pin(int fd, size_t size) 334 { 335 do_test(fd, size, TEST_TYPE_RW, true); 336 } 337 338 static void test_shared_rw_fast_pin(int fd, size_t size) 339 { 340 do_test(fd, size, TEST_TYPE_RW_FAST, true); 341 } 342 343 static void test_shared_ro_pin(int fd, size_t size) 344 { 345 do_test(fd, size, TEST_TYPE_RO, true); 346 } 347 348 static void test_shared_ro_fast_pin(int fd, size_t size) 349 { 350 do_test(fd, size, TEST_TYPE_RO_FAST, true); 351 } 352 353 static void test_private_rw_pin(int fd, size_t size) 354 { 355 do_test(fd, size, TEST_TYPE_RW, false); 356 } 357 358 static void test_private_rw_fast_pin(int fd, size_t size) 359 { 360 do_test(fd, size, TEST_TYPE_RW_FAST, false); 361 } 362 363 static void test_private_ro_pin(int fd, size_t size) 364 { 365 do_test(fd, size, TEST_TYPE_RO, false); 366 } 367 368 static void test_private_ro_fast_pin(int fd, size_t size) 369 { 370 do_test(fd, size, TEST_TYPE_RO_FAST, false); 371 } 372 373 #ifdef LOCAL_CONFIG_HAVE_LIBURING 374 static void test_shared_iouring(int fd, size_t size) 375 { 376 do_test(fd, size, TEST_TYPE_IOURING, true); 377 } 378 379 static void test_private_iouring(int fd, size_t size) 380 { 381 do_test(fd, size, TEST_TYPE_IOURING, false); 382 } 383 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 384 385 static const struct test_case test_cases[] = { 386 { 387 "R/W longterm GUP pin in MAP_SHARED file mapping", 388 test_shared_rw_pin, 389 }, 390 { 391 "R/W longterm GUP-fast pin in MAP_SHARED file mapping", 392 test_shared_rw_fast_pin, 393 }, 394 { 395 "R/O longterm GUP pin in MAP_SHARED file mapping", 396 test_shared_ro_pin, 397 }, 398 { 399 "R/O longterm GUP-fast pin in MAP_SHARED file mapping", 400 test_shared_ro_fast_pin, 401 }, 402 { 403 "R/W longterm GUP pin in MAP_PRIVATE file mapping", 404 test_private_rw_pin, 405 }, 406 { 407 "R/W longterm GUP-fast pin in MAP_PRIVATE file mapping", 408 test_private_rw_fast_pin, 409 }, 410 { 411 "R/O longterm GUP pin in MAP_PRIVATE file mapping", 412 test_private_ro_pin, 413 }, 414 { 415 "R/O longterm GUP-fast pin in MAP_PRIVATE file mapping", 416 test_private_ro_fast_pin, 417 }, 418 #ifdef LOCAL_CONFIG_HAVE_LIBURING 419 { 420 "io_uring fixed buffer with MAP_SHARED file mapping", 421 test_shared_iouring, 422 }, 423 { 424 "io_uring fixed buffer with MAP_PRIVATE file mapping", 425 test_private_iouring, 426 }, 427 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 428 }; 429 430 static void run_test_case(struct test_case const *test_case) 431 { 432 int i; 433 434 run_with_memfd(test_case->fn, test_case->desc); 435 run_with_tmpfile(test_case->fn, test_case->desc); 436 run_with_local_tmpfile(test_case->fn, test_case->desc); 437 for (i = 0; i < nr_hugetlbsizes; i++) 438 run_with_memfd_hugetlb(test_case->fn, test_case->desc, 439 hugetlbsizes[i]); 440 } 441 442 static int tests_per_test_case(void) 443 { 444 return 3 + nr_hugetlbsizes; 445 } 446 447 int main(int argc, char **argv) 448 { 449 int i, err; 450 451 pagesize = getpagesize(); 452 nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes, 453 ARRAY_SIZE(hugetlbsizes)); 454 455 ksft_print_header(); 456 ksft_set_plan(ARRAY_SIZE(test_cases) * tests_per_test_case()); 457 458 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); 459 460 for (i = 0; i < ARRAY_SIZE(test_cases); i++) 461 run_test_case(&test_cases[i]); 462 463 err = ksft_get_fail_cnt(); 464 if (err) 465 ksft_exit_fail_msg("%d out of %d tests failed\n", 466 err, ksft_test_num()); 467 ksft_exit_pass(); 468 } 469