1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * GUP long-term page pinning tests. 4 * 5 * Copyright 2023, Red Hat, Inc. 6 * 7 * Author(s): David Hildenbrand <david@redhat.com> 8 */ 9 #define _GNU_SOURCE 10 #include <stdlib.h> 11 #include <string.h> 12 #include <stdbool.h> 13 #include <stdint.h> 14 #include <unistd.h> 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <assert.h> 18 #include <sys/mman.h> 19 #include <sys/ioctl.h> 20 #include <sys/vfs.h> 21 #include <linux/magic.h> 22 #include <linux/memfd.h> 23 24 #include "local_config.h" 25 #ifdef LOCAL_CONFIG_HAVE_LIBURING 26 #include <liburing.h> 27 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 28 29 #include "../../../../mm/gup_test.h" 30 #include "../kselftest.h" 31 #include "vm_util.h" 32 33 static size_t pagesize; 34 static int nr_hugetlbsizes; 35 static size_t hugetlbsizes[10]; 36 static int gup_fd; 37 38 static __fsword_t get_fs_type(int fd) 39 { 40 struct statfs fs; 41 int ret; 42 43 do { 44 ret = fstatfs(fd, &fs); 45 } while (ret && errno == EINTR); 46 47 return ret ? 0 : fs.f_type; 48 } 49 50 static bool fs_is_unknown(__fsword_t fs_type) 51 { 52 /* 53 * We only support some filesystems in our tests when dealing with 54 * R/W long-term pinning. For these filesystems, we can be fairly sure 55 * whether they support it or not. 56 */ 57 switch (fs_type) { 58 case TMPFS_MAGIC: 59 case HUGETLBFS_MAGIC: 60 case BTRFS_SUPER_MAGIC: 61 case EXT4_SUPER_MAGIC: 62 case XFS_SUPER_MAGIC: 63 return false; 64 default: 65 return true; 66 } 67 } 68 69 static bool fs_supports_writable_longterm_pinning(__fsword_t fs_type) 70 { 71 assert(!fs_is_unknown(fs_type)); 72 switch (fs_type) { 73 case TMPFS_MAGIC: 74 case HUGETLBFS_MAGIC: 75 return true; 76 default: 77 return false; 78 } 79 } 80 81 enum test_type { 82 TEST_TYPE_RO, 83 TEST_TYPE_RO_FAST, 84 TEST_TYPE_RW, 85 TEST_TYPE_RW_FAST, 86 #ifdef LOCAL_CONFIG_HAVE_LIBURING 87 TEST_TYPE_IOURING, 88 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 89 }; 90 91 static void do_test(int fd, size_t size, enum test_type type, bool shared) 92 { 93 __fsword_t fs_type = get_fs_type(fd); 94 bool should_work; 95 char *mem; 96 int ret; 97 98 if (ftruncate(fd, size)) { 99 if (errno == ENOENT) { 100 /* 101 * This can happen if the file has been unlinked and the 102 * filesystem doesn't support truncating unlinked files. 103 */ 104 ksft_test_result_skip("ftruncate() failed with ENOENT\n"); 105 } else { 106 ksft_test_result_fail("ftruncate() failed (%s)\n", strerror(errno)); 107 } 108 return; 109 } 110 111 if (fallocate(fd, 0, 0, size)) { 112 if (size == pagesize) 113 ksft_test_result_fail("fallocate() failed (%s)\n", strerror(errno)); 114 else 115 ksft_test_result_skip("need more free huge pages\n"); 116 return; 117 } 118 119 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, 120 shared ? MAP_SHARED : MAP_PRIVATE, fd, 0); 121 if (mem == MAP_FAILED) { 122 if (size == pagesize || shared) 123 ksft_test_result_fail("mmap() failed (%s)\n", strerror(errno)); 124 else 125 ksft_test_result_skip("need more free huge pages\n"); 126 return; 127 } 128 129 /* Fault in the page such that GUP-fast can pin it directly. */ 130 memset(mem, 0, size); 131 132 switch (type) { 133 case TEST_TYPE_RO: 134 case TEST_TYPE_RO_FAST: 135 /* 136 * Cover more cases regarding unsharing decisions when 137 * long-term R/O pinning by mapping the page R/O. 138 */ 139 ret = mprotect(mem, size, PROT_READ); 140 if (ret) { 141 ksft_test_result_fail("mprotect() failed (%s)\n", strerror(errno)); 142 goto munmap; 143 } 144 /* FALLTHROUGH */ 145 case TEST_TYPE_RW: 146 case TEST_TYPE_RW_FAST: { 147 struct pin_longterm_test args; 148 const bool fast = type == TEST_TYPE_RO_FAST || 149 type == TEST_TYPE_RW_FAST; 150 const bool rw = type == TEST_TYPE_RW || 151 type == TEST_TYPE_RW_FAST; 152 153 if (gup_fd < 0) { 154 ksft_test_result_skip("gup_test not available\n"); 155 break; 156 } 157 158 if (rw && shared && fs_is_unknown(fs_type)) { 159 ksft_test_result_skip("Unknown filesystem\n"); 160 return; 161 } 162 /* 163 * R/O pinning or pinning in a private mapping is always 164 * expected to work. Otherwise, we expect long-term R/W pinning 165 * to only succeed for special fielesystems. 166 */ 167 should_work = !shared || !rw || 168 fs_supports_writable_longterm_pinning(fs_type); 169 170 args.addr = (__u64)(uintptr_t)mem; 171 args.size = size; 172 args.flags = fast ? PIN_LONGTERM_TEST_FLAG_USE_FAST : 0; 173 args.flags |= rw ? PIN_LONGTERM_TEST_FLAG_USE_WRITE : 0; 174 ret = ioctl(gup_fd, PIN_LONGTERM_TEST_START, &args); 175 if (ret && errno == EINVAL) { 176 ksft_test_result_skip("PIN_LONGTERM_TEST_START failed (EINVAL)n"); 177 break; 178 } else if (ret && errno == EFAULT) { 179 ksft_test_result(!should_work, "Should have failed\n"); 180 break; 181 } else if (ret) { 182 ksft_test_result_fail("PIN_LONGTERM_TEST_START failed (%s)\n", 183 strerror(errno)); 184 break; 185 } 186 187 if (ioctl(gup_fd, PIN_LONGTERM_TEST_STOP)) 188 ksft_print_msg("[INFO] PIN_LONGTERM_TEST_STOP failed (%s)\n", 189 strerror(errno)); 190 191 /* 192 * TODO: if the kernel ever supports long-term R/W pinning on 193 * some previously unsupported filesystems, we might want to 194 * perform some additional tests for possible data corruptions. 195 */ 196 ksft_test_result(should_work, "Should have worked\n"); 197 break; 198 } 199 #ifdef LOCAL_CONFIG_HAVE_LIBURING 200 case TEST_TYPE_IOURING: { 201 struct io_uring ring; 202 struct iovec iov; 203 204 /* io_uring always pins pages writable. */ 205 if (shared && fs_is_unknown(fs_type)) { 206 ksft_test_result_skip("Unknown filesystem\n"); 207 return; 208 } 209 should_work = !shared || 210 fs_supports_writable_longterm_pinning(fs_type); 211 212 /* Skip on errors, as we might just lack kernel support. */ 213 ret = io_uring_queue_init(1, &ring, 0); 214 if (ret < 0) { 215 ksft_test_result_skip("io_uring_queue_init() failed (%s)\n", 216 strerror(-ret)); 217 break; 218 } 219 /* 220 * Register the range as a fixed buffer. This will FOLL_WRITE | 221 * FOLL_PIN | FOLL_LONGTERM the range. 222 */ 223 iov.iov_base = mem; 224 iov.iov_len = size; 225 ret = io_uring_register_buffers(&ring, &iov, 1); 226 /* Only new kernels return EFAULT. */ 227 if (ret && (errno == ENOSPC || errno == EOPNOTSUPP || 228 errno == EFAULT)) { 229 ksft_test_result(!should_work, "Should have failed (%s)\n", 230 strerror(errno)); 231 } else if (ret) { 232 /* 233 * We might just lack support or have insufficient 234 * MEMLOCK limits. 235 */ 236 ksft_test_result_skip("io_uring_register_buffers() failed (%s)\n", 237 strerror(-ret)); 238 } else { 239 ksft_test_result(should_work, "Should have worked\n"); 240 io_uring_unregister_buffers(&ring); 241 } 242 243 io_uring_queue_exit(&ring); 244 break; 245 } 246 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 247 default: 248 assert(false); 249 } 250 251 munmap: 252 munmap(mem, size); 253 } 254 255 typedef void (*test_fn)(int fd, size_t size); 256 257 static void run_with_memfd(test_fn fn, const char *desc) 258 { 259 int fd; 260 261 ksft_print_msg("[RUN] %s ... with memfd\n", desc); 262 263 fd = memfd_create("test", 0); 264 if (fd < 0) { 265 ksft_test_result_fail("memfd_create() failed (%s)\n", strerror(errno)); 266 return; 267 } 268 269 fn(fd, pagesize); 270 close(fd); 271 } 272 273 static void run_with_tmpfile(test_fn fn, const char *desc) 274 { 275 FILE *file; 276 int fd; 277 278 ksft_print_msg("[RUN] %s ... with tmpfile\n", desc); 279 280 file = tmpfile(); 281 if (!file) { 282 ksft_test_result_fail("tmpfile() failed (%s)\n", strerror(errno)); 283 return; 284 } 285 286 fd = fileno(file); 287 if (fd < 0) { 288 ksft_test_result_fail("fileno() failed (%s)\n", strerror(errno)); 289 goto close; 290 } 291 292 fn(fd, pagesize); 293 close: 294 fclose(file); 295 } 296 297 static void run_with_local_tmpfile(test_fn fn, const char *desc) 298 { 299 char filename[] = __FILE__"_tmpfile_XXXXXX"; 300 int fd; 301 302 ksft_print_msg("[RUN] %s ... with local tmpfile\n", desc); 303 304 fd = mkstemp(filename); 305 if (fd < 0) { 306 ksft_test_result_fail("mkstemp() failed (%s)\n", strerror(errno)); 307 return; 308 } 309 310 if (unlink(filename)) { 311 ksft_test_result_fail("unlink() failed (%s)\n", strerror(errno)); 312 goto close; 313 } 314 315 fn(fd, pagesize); 316 close: 317 close(fd); 318 } 319 320 static void run_with_memfd_hugetlb(test_fn fn, const char *desc, 321 size_t hugetlbsize) 322 { 323 int flags = MFD_HUGETLB; 324 int fd; 325 326 ksft_print_msg("[RUN] %s ... with memfd hugetlb (%zu kB)\n", desc, 327 hugetlbsize / 1024); 328 329 flags |= __builtin_ctzll(hugetlbsize) << MFD_HUGE_SHIFT; 330 331 fd = memfd_create("test", flags); 332 if (fd < 0) { 333 ksft_test_result_skip("memfd_create() failed (%s)\n", strerror(errno)); 334 return; 335 } 336 337 fn(fd, hugetlbsize); 338 close(fd); 339 } 340 341 struct test_case { 342 const char *desc; 343 test_fn fn; 344 }; 345 346 static void test_shared_rw_pin(int fd, size_t size) 347 { 348 do_test(fd, size, TEST_TYPE_RW, true); 349 } 350 351 static void test_shared_rw_fast_pin(int fd, size_t size) 352 { 353 do_test(fd, size, TEST_TYPE_RW_FAST, true); 354 } 355 356 static void test_shared_ro_pin(int fd, size_t size) 357 { 358 do_test(fd, size, TEST_TYPE_RO, true); 359 } 360 361 static void test_shared_ro_fast_pin(int fd, size_t size) 362 { 363 do_test(fd, size, TEST_TYPE_RO_FAST, true); 364 } 365 366 static void test_private_rw_pin(int fd, size_t size) 367 { 368 do_test(fd, size, TEST_TYPE_RW, false); 369 } 370 371 static void test_private_rw_fast_pin(int fd, size_t size) 372 { 373 do_test(fd, size, TEST_TYPE_RW_FAST, false); 374 } 375 376 static void test_private_ro_pin(int fd, size_t size) 377 { 378 do_test(fd, size, TEST_TYPE_RO, false); 379 } 380 381 static void test_private_ro_fast_pin(int fd, size_t size) 382 { 383 do_test(fd, size, TEST_TYPE_RO_FAST, false); 384 } 385 386 #ifdef LOCAL_CONFIG_HAVE_LIBURING 387 static void test_shared_iouring(int fd, size_t size) 388 { 389 do_test(fd, size, TEST_TYPE_IOURING, true); 390 } 391 392 static void test_private_iouring(int fd, size_t size) 393 { 394 do_test(fd, size, TEST_TYPE_IOURING, false); 395 } 396 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 397 398 static const struct test_case test_cases[] = { 399 { 400 "R/W longterm GUP pin in MAP_SHARED file mapping", 401 test_shared_rw_pin, 402 }, 403 { 404 "R/W longterm GUP-fast pin in MAP_SHARED file mapping", 405 test_shared_rw_fast_pin, 406 }, 407 { 408 "R/O longterm GUP pin in MAP_SHARED file mapping", 409 test_shared_ro_pin, 410 }, 411 { 412 "R/O longterm GUP-fast pin in MAP_SHARED file mapping", 413 test_shared_ro_fast_pin, 414 }, 415 { 416 "R/W longterm GUP pin in MAP_PRIVATE file mapping", 417 test_private_rw_pin, 418 }, 419 { 420 "R/W longterm GUP-fast pin in MAP_PRIVATE file mapping", 421 test_private_rw_fast_pin, 422 }, 423 { 424 "R/O longterm GUP pin in MAP_PRIVATE file mapping", 425 test_private_ro_pin, 426 }, 427 { 428 "R/O longterm GUP-fast pin in MAP_PRIVATE file mapping", 429 test_private_ro_fast_pin, 430 }, 431 #ifdef LOCAL_CONFIG_HAVE_LIBURING 432 { 433 "io_uring fixed buffer with MAP_SHARED file mapping", 434 test_shared_iouring, 435 }, 436 { 437 "io_uring fixed buffer with MAP_PRIVATE file mapping", 438 test_private_iouring, 439 }, 440 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 441 }; 442 443 static void run_test_case(struct test_case const *test_case) 444 { 445 int i; 446 447 run_with_memfd(test_case->fn, test_case->desc); 448 run_with_tmpfile(test_case->fn, test_case->desc); 449 run_with_local_tmpfile(test_case->fn, test_case->desc); 450 for (i = 0; i < nr_hugetlbsizes; i++) 451 run_with_memfd_hugetlb(test_case->fn, test_case->desc, 452 hugetlbsizes[i]); 453 } 454 455 static int tests_per_test_case(void) 456 { 457 return 3 + nr_hugetlbsizes; 458 } 459 460 int main(int argc, char **argv) 461 { 462 int i, err; 463 464 pagesize = getpagesize(); 465 nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes, 466 ARRAY_SIZE(hugetlbsizes)); 467 468 ksft_print_header(); 469 ksft_set_plan(ARRAY_SIZE(test_cases) * tests_per_test_case()); 470 471 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); 472 473 for (i = 0; i < ARRAY_SIZE(test_cases); i++) 474 run_test_case(&test_cases[i]); 475 476 err = ksft_get_fail_cnt(); 477 if (err) 478 ksft_exit_fail_msg("%d out of %d tests failed\n", 479 err, ksft_test_num()); 480 ksft_exit_pass(); 481 } 482