1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * GUP long-term page pinning tests. 4 * 5 * Copyright 2023, Red Hat, Inc. 6 * 7 * Author(s): David Hildenbrand <david@redhat.com> 8 */ 9 #define _GNU_SOURCE 10 #include <stdlib.h> 11 #include <string.h> 12 #include <stdbool.h> 13 #include <stdint.h> 14 #include <unistd.h> 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <assert.h> 18 #include <sys/mman.h> 19 #include <sys/ioctl.h> 20 #include <sys/vfs.h> 21 #include <linux/magic.h> 22 #include <linux/memfd.h> 23 24 #include "local_config.h" 25 #ifdef LOCAL_CONFIG_HAVE_LIBURING 26 #include <liburing.h> 27 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 28 29 #include "../../../../mm/gup_test.h" 30 #include "kselftest.h" 31 #include "vm_util.h" 32 #include "hugepage_settings.h" 33 34 static size_t pagesize; 35 static int nr_hugetlbsizes; 36 static unsigned long hugetlbsizes[10]; 37 static int gup_fd; 38 39 static __fsword_t get_fs_type(int fd) 40 { 41 struct statfs fs; 42 int ret; 43 44 do { 45 ret = fstatfs(fd, &fs); 46 } while (ret && errno == EINTR); 47 48 return ret ? 0 : fs.f_type; 49 } 50 51 static bool fs_is_unknown(__fsword_t fs_type) 52 { 53 /* 54 * We only support some filesystems in our tests when dealing with 55 * R/W long-term pinning. For these filesystems, we can be fairly sure 56 * whether they support it or not. 57 */ 58 switch (fs_type) { 59 case TMPFS_MAGIC: 60 case HUGETLBFS_MAGIC: 61 case BTRFS_SUPER_MAGIC: 62 case EXT4_SUPER_MAGIC: 63 case XFS_SUPER_MAGIC: 64 return false; 65 default: 66 return true; 67 } 68 } 69 70 static bool fs_supports_writable_longterm_pinning(__fsword_t fs_type) 71 { 72 assert(!fs_is_unknown(fs_type)); 73 switch (fs_type) { 74 case TMPFS_MAGIC: 75 case HUGETLBFS_MAGIC: 76 return true; 77 default: 78 return false; 79 } 80 } 81 82 enum test_type { 83 TEST_TYPE_RO, 84 TEST_TYPE_RO_FAST, 85 TEST_TYPE_RW, 86 TEST_TYPE_RW_FAST, 87 #ifdef LOCAL_CONFIG_HAVE_LIBURING 88 TEST_TYPE_IOURING, 89 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 90 }; 91 92 static void do_test(int fd, size_t size, enum test_type type, bool shared) 93 { 94 __fsword_t fs_type = get_fs_type(fd); 95 bool should_work; 96 char *mem; 97 int result = KSFT_PASS; 98 int ret; 99 100 if (fd < 0) { 101 result = KSFT_FAIL; 102 goto report; 103 } 104 105 if (ftruncate(fd, size)) { 106 if (errno == ENOENT) { 107 skip_test_dodgy_fs("ftruncate()"); 108 } else { 109 ksft_print_msg("ftruncate() failed (%s)\n", 110 strerror(errno)); 111 result = KSFT_FAIL; 112 goto report; 113 } 114 return; 115 } 116 117 if (fallocate(fd, 0, 0, size)) { 118 /* 119 * Some filesystems (eg, NFSv3) don't support 120 * fallocate(), report this as a skip rather than a 121 * test failure. 122 */ 123 if (errno == EOPNOTSUPP) { 124 ksft_print_msg("fallocate() not supported by filesystem\n"); 125 result = KSFT_SKIP; 126 } else if (size == pagesize) { 127 ksft_print_msg("fallocate() failed (%s)\n", strerror(errno)); 128 result = KSFT_FAIL; 129 } else { 130 ksft_print_msg("need more free huge pages\n"); 131 result = KSFT_SKIP; 132 } 133 goto report; 134 } 135 136 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, 137 shared ? MAP_SHARED : MAP_PRIVATE, fd, 0); 138 if (mem == MAP_FAILED) { 139 if (size == pagesize || shared) { 140 ksft_print_msg("mmap() failed (%s)\n", strerror(errno)); 141 result = KSFT_FAIL; 142 } else { 143 ksft_print_msg("need more free huge pages\n"); 144 result = KSFT_SKIP; 145 } 146 goto report; 147 } 148 149 /* Fault in the page such that GUP-fast can pin it directly. */ 150 memset(mem, 0, size); 151 152 switch (type) { 153 case TEST_TYPE_RO: 154 case TEST_TYPE_RO_FAST: 155 /* 156 * Cover more cases regarding unsharing decisions when 157 * long-term R/O pinning by mapping the page R/O. 158 */ 159 ret = mprotect(mem, size, PROT_READ); 160 if (ret) { 161 ksft_print_msg("mprotect() failed (%s)\n", strerror(errno)); 162 result = KSFT_FAIL; 163 goto munmap; 164 } 165 /* FALLTHROUGH */ 166 case TEST_TYPE_RW: 167 case TEST_TYPE_RW_FAST: { 168 struct pin_longterm_test args; 169 const bool fast = type == TEST_TYPE_RO_FAST || 170 type == TEST_TYPE_RW_FAST; 171 const bool rw = type == TEST_TYPE_RW || 172 type == TEST_TYPE_RW_FAST; 173 174 if (gup_fd < 0) { 175 ksft_print_msg("gup_test not available\n"); 176 result = KSFT_SKIP; 177 break; 178 } 179 180 if (rw && shared && fs_is_unknown(fs_type)) { 181 ksft_print_msg("Unknown filesystem\n"); 182 result = KSFT_SKIP; 183 break; 184 } 185 /* 186 * R/O pinning or pinning in a private mapping is always 187 * expected to work. Otherwise, we expect long-term R/W pinning 188 * to only succeed for special filesystems. 189 */ 190 should_work = !shared || !rw || 191 fs_supports_writable_longterm_pinning(fs_type); 192 193 args.addr = (__u64)(uintptr_t)mem; 194 args.size = size; 195 args.flags = fast ? PIN_LONGTERM_TEST_FLAG_USE_FAST : 0; 196 args.flags |= rw ? PIN_LONGTERM_TEST_FLAG_USE_WRITE : 0; 197 ret = ioctl(gup_fd, PIN_LONGTERM_TEST_START, &args); 198 if (ret && errno == EINVAL) { 199 ksft_print_msg("PIN_LONGTERM_TEST_START failed (EINVAL)n"); 200 result = KSFT_SKIP; 201 break; 202 } else if (ret && errno == EFAULT) { 203 if (should_work) 204 result = KSFT_FAIL; 205 else 206 result = KSFT_PASS; 207 break; 208 } else if (ret) { 209 ksft_print_msg("PIN_LONGTERM_TEST_START failed (%s)\n", 210 strerror(errno)); 211 result = KSFT_FAIL; 212 break; 213 } 214 215 if (ioctl(gup_fd, PIN_LONGTERM_TEST_STOP)) 216 ksft_print_msg("[INFO] PIN_LONGTERM_TEST_STOP failed (%s)\n", 217 strerror(errno)); 218 219 /* 220 * TODO: if the kernel ever supports long-term R/W pinning on 221 * some previously unsupported filesystems, we might want to 222 * perform some additional tests for possible data corruptions. 223 */ 224 if (should_work) 225 result = KSFT_PASS; 226 else 227 result = KSFT_FAIL; 228 break; 229 } 230 #ifdef LOCAL_CONFIG_HAVE_LIBURING 231 case TEST_TYPE_IOURING: { 232 struct io_uring ring; 233 struct iovec iov; 234 235 /* io_uring always pins pages writable. */ 236 if (shared && fs_is_unknown(fs_type)) { 237 ksft_print_msg("Unknown filesystem\n"); 238 result = KSFT_SKIP; 239 goto report; 240 } 241 should_work = !shared || 242 fs_supports_writable_longterm_pinning(fs_type); 243 244 /* Skip on errors, as we might just lack kernel support. */ 245 ret = io_uring_queue_init(1, &ring, 0); 246 if (ret < 0) { 247 ksft_print_msg("io_uring_queue_init() failed (%s)\n", 248 strerror(-ret)); 249 result = KSFT_SKIP; 250 break; 251 } 252 /* 253 * Register the range as a fixed buffer. This will FOLL_WRITE | 254 * FOLL_PIN | FOLL_LONGTERM the range. 255 */ 256 iov.iov_base = mem; 257 iov.iov_len = size; 258 ret = io_uring_register_buffers(&ring, &iov, 1); 259 /* Only new kernels return EFAULT. */ 260 if (ret && (errno == ENOSPC || errno == EOPNOTSUPP || 261 errno == EFAULT)) { 262 if (should_work) { 263 ksft_print_msg("Should have failed (%s)\n", 264 strerror(errno)); 265 result = KSFT_FAIL; 266 } else { 267 result = KSFT_PASS; 268 } 269 } else if (ret) { 270 /* 271 * We might just lack support or have insufficient 272 * MEMLOCK limits. 273 */ 274 ksft_print_msg("io_uring_register_buffers() failed (%s)\n", 275 strerror(-ret)); 276 result = KSFT_SKIP; 277 } else { 278 if (should_work) { 279 result = KSFT_PASS; 280 } else { 281 ksft_print_msg("Should have worked\n"); 282 result = KSFT_FAIL; 283 } 284 io_uring_unregister_buffers(&ring); 285 } 286 287 io_uring_queue_exit(&ring); 288 break; 289 } 290 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 291 default: 292 assert(false); 293 } 294 295 munmap: 296 munmap(mem, size); 297 report: 298 log_test_result(result); 299 } 300 301 typedef void (*test_fn)(int fd, size_t size); 302 303 static void run_with_memfd(test_fn fn, const char *desc) 304 { 305 int fd; 306 307 log_test_start("%s ... with memfd", desc); 308 309 fd = memfd_create("test", 0); 310 if (fd < 0) { 311 ksft_print_msg("memfd_create() failed (%s)\n", strerror(errno)); 312 log_test_result(KSFT_SKIP); 313 return; 314 } 315 316 fn(fd, pagesize); 317 close(fd); 318 } 319 320 static void run_with_tmpfile(test_fn fn, const char *desc) 321 { 322 FILE *file; 323 int fd; 324 325 log_test_start("%s ... with tmpfile", desc); 326 327 file = tmpfile(); 328 if (!file) { 329 ksft_print_msg("tmpfile() failed (%s)\n", strerror(errno)); 330 fd = -1; 331 } else { 332 fd = fileno(file); 333 if (fd < 0) { 334 ksft_print_msg("fileno() failed (%s)\n", strerror(errno)); 335 } 336 } 337 338 fn(fd, pagesize); 339 340 if (file) 341 fclose(file); 342 } 343 344 static void run_with_local_tmpfile(test_fn fn, const char *desc) 345 { 346 char filename[] = __FILE__"_tmpfile_XXXXXX"; 347 int fd; 348 349 log_test_start("%s ... with local tmpfile", desc); 350 351 fd = mkstemp(filename); 352 if (fd < 0) 353 ksft_print_msg("mkstemp() failed (%s)\n", strerror(errno)); 354 355 if (unlink(filename)) { 356 ksft_print_msg("unlink() failed (%s)\n", strerror(errno)); 357 close(fd); 358 fd = -1; 359 } 360 361 fn(fd, pagesize); 362 363 if (fd >= 0) 364 close(fd); 365 } 366 367 static void run_with_memfd_hugetlb(test_fn fn, const char *desc, 368 size_t hugetlbsize) 369 { 370 int flags = MFD_HUGETLB; 371 int fd; 372 373 log_test_start("%s ... with memfd hugetlb (%zu kB)", desc, 374 hugetlbsize / 1024); 375 376 flags |= __builtin_ctzll(hugetlbsize) << MFD_HUGE_SHIFT; 377 378 fd = memfd_create("test", flags); 379 if (fd < 0) { 380 ksft_print_msg("memfd_create() failed (%s)\n", strerror(errno)); 381 log_test_result(KSFT_SKIP); 382 return; 383 } 384 385 fn(fd, hugetlbsize); 386 close(fd); 387 } 388 389 struct test_case { 390 const char *desc; 391 test_fn fn; 392 }; 393 394 static void test_shared_rw_pin(int fd, size_t size) 395 { 396 do_test(fd, size, TEST_TYPE_RW, true); 397 } 398 399 static void test_shared_rw_fast_pin(int fd, size_t size) 400 { 401 do_test(fd, size, TEST_TYPE_RW_FAST, true); 402 } 403 404 static void test_shared_ro_pin(int fd, size_t size) 405 { 406 do_test(fd, size, TEST_TYPE_RO, true); 407 } 408 409 static void test_shared_ro_fast_pin(int fd, size_t size) 410 { 411 do_test(fd, size, TEST_TYPE_RO_FAST, true); 412 } 413 414 static void test_private_rw_pin(int fd, size_t size) 415 { 416 do_test(fd, size, TEST_TYPE_RW, false); 417 } 418 419 static void test_private_rw_fast_pin(int fd, size_t size) 420 { 421 do_test(fd, size, TEST_TYPE_RW_FAST, false); 422 } 423 424 static void test_private_ro_pin(int fd, size_t size) 425 { 426 do_test(fd, size, TEST_TYPE_RO, false); 427 } 428 429 static void test_private_ro_fast_pin(int fd, size_t size) 430 { 431 do_test(fd, size, TEST_TYPE_RO_FAST, false); 432 } 433 434 #ifdef LOCAL_CONFIG_HAVE_LIBURING 435 static void test_shared_iouring(int fd, size_t size) 436 { 437 do_test(fd, size, TEST_TYPE_IOURING, true); 438 } 439 440 static void test_private_iouring(int fd, size_t size) 441 { 442 do_test(fd, size, TEST_TYPE_IOURING, false); 443 } 444 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 445 446 static const struct test_case test_cases[] = { 447 { 448 "R/W longterm GUP pin in MAP_SHARED file mapping", 449 test_shared_rw_pin, 450 }, 451 { 452 "R/W longterm GUP-fast pin in MAP_SHARED file mapping", 453 test_shared_rw_fast_pin, 454 }, 455 { 456 "R/O longterm GUP pin in MAP_SHARED file mapping", 457 test_shared_ro_pin, 458 }, 459 { 460 "R/O longterm GUP-fast pin in MAP_SHARED file mapping", 461 test_shared_ro_fast_pin, 462 }, 463 { 464 "R/W longterm GUP pin in MAP_PRIVATE file mapping", 465 test_private_rw_pin, 466 }, 467 { 468 "R/W longterm GUP-fast pin in MAP_PRIVATE file mapping", 469 test_private_rw_fast_pin, 470 }, 471 { 472 "R/O longterm GUP pin in MAP_PRIVATE file mapping", 473 test_private_ro_pin, 474 }, 475 { 476 "R/O longterm GUP-fast pin in MAP_PRIVATE file mapping", 477 test_private_ro_fast_pin, 478 }, 479 #ifdef LOCAL_CONFIG_HAVE_LIBURING 480 { 481 "io_uring fixed buffer with MAP_SHARED file mapping", 482 test_shared_iouring, 483 }, 484 { 485 "io_uring fixed buffer with MAP_PRIVATE file mapping", 486 test_private_iouring, 487 }, 488 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 489 }; 490 491 static void run_test_case(struct test_case const *test_case) 492 { 493 int i; 494 495 run_with_memfd(test_case->fn, test_case->desc); 496 run_with_tmpfile(test_case->fn, test_case->desc); 497 run_with_local_tmpfile(test_case->fn, test_case->desc); 498 for (i = 0; i < nr_hugetlbsizes; i++) 499 run_with_memfd_hugetlb(test_case->fn, test_case->desc, 500 hugetlbsizes[i]); 501 } 502 503 static int tests_per_test_case(void) 504 { 505 return 3 + nr_hugetlbsizes; 506 } 507 508 int main(int argc, char **argv) 509 { 510 int i; 511 512 pagesize = getpagesize(); 513 nr_hugetlbsizes = hugetlb_setup(2, hugetlbsizes, 514 ARRAY_SIZE(hugetlbsizes)); 515 516 ksft_print_header(); 517 ksft_set_plan(ARRAY_SIZE(test_cases) * tests_per_test_case()); 518 519 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); 520 521 for (i = 0; i < ARRAY_SIZE(test_cases); i++) 522 run_test_case(&test_cases[i]); 523 524 ksft_finished(); 525 } 526