1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * GUP long-term page pinning tests. 4 * 5 * Copyright 2023, Red Hat, Inc. 6 * 7 * Author(s): David Hildenbrand <david@redhat.com> 8 */ 9 #define _GNU_SOURCE 10 #include <stdlib.h> 11 #include <string.h> 12 #include <stdbool.h> 13 #include <stdint.h> 14 #include <unistd.h> 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <assert.h> 18 #include <sys/mman.h> 19 #include <sys/ioctl.h> 20 #include <sys/vfs.h> 21 #include <linux/magic.h> 22 #include <linux/memfd.h> 23 24 #include "local_config.h" 25 #ifdef LOCAL_CONFIG_HAVE_LIBURING 26 #include <liburing.h> 27 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 28 29 #include "../../../../mm/gup_test.h" 30 #include "../kselftest.h" 31 #include "vm_util.h" 32 33 static size_t pagesize; 34 static int nr_hugetlbsizes; 35 static size_t hugetlbsizes[10]; 36 static int gup_fd; 37 38 static __fsword_t get_fs_type(int fd) 39 { 40 struct statfs fs; 41 int ret; 42 43 do { 44 ret = fstatfs(fd, &fs); 45 } while (ret && errno == EINTR); 46 47 return ret ? 0 : fs.f_type; 48 } 49 50 static bool fs_is_unknown(__fsword_t fs_type) 51 { 52 /* 53 * We only support some filesystems in our tests when dealing with 54 * R/W long-term pinning. For these filesystems, we can be fairly sure 55 * whether they support it or not. 56 */ 57 switch (fs_type) { 58 case TMPFS_MAGIC: 59 case HUGETLBFS_MAGIC: 60 case BTRFS_SUPER_MAGIC: 61 case EXT4_SUPER_MAGIC: 62 case XFS_SUPER_MAGIC: 63 return false; 64 default: 65 return true; 66 } 67 } 68 69 static bool fs_supports_writable_longterm_pinning(__fsword_t fs_type) 70 { 71 assert(!fs_is_unknown(fs_type)); 72 switch (fs_type) { 73 case TMPFS_MAGIC: 74 case HUGETLBFS_MAGIC: 75 return true; 76 default: 77 return false; 78 } 79 } 80 81 enum test_type { 82 TEST_TYPE_RO, 83 TEST_TYPE_RO_FAST, 84 TEST_TYPE_RW, 85 TEST_TYPE_RW_FAST, 86 #ifdef LOCAL_CONFIG_HAVE_LIBURING 87 TEST_TYPE_IOURING, 88 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 89 }; 90 91 static void do_test(int fd, size_t size, enum test_type type, bool shared) 92 { 93 __fsword_t fs_type = get_fs_type(fd); 94 bool should_work; 95 char *mem; 96 int ret; 97 98 if (ftruncate(fd, size)) { 99 if (errno == ENOENT) { 100 skip_test_dodgy_fs("ftruncate()"); 101 } else { 102 ksft_test_result_fail("ftruncate() failed (%s)\n", strerror(errno)); 103 } 104 return; 105 } 106 107 if (fallocate(fd, 0, 0, size)) { 108 if (size == pagesize) 109 ksft_test_result_fail("fallocate() failed (%s)\n", strerror(errno)); 110 else 111 ksft_test_result_skip("need more free huge pages\n"); 112 return; 113 } 114 115 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, 116 shared ? MAP_SHARED : MAP_PRIVATE, fd, 0); 117 if (mem == MAP_FAILED) { 118 if (size == pagesize || shared) 119 ksft_test_result_fail("mmap() failed (%s)\n", strerror(errno)); 120 else 121 ksft_test_result_skip("need more free huge pages\n"); 122 return; 123 } 124 125 /* Fault in the page such that GUP-fast can pin it directly. */ 126 memset(mem, 0, size); 127 128 switch (type) { 129 case TEST_TYPE_RO: 130 case TEST_TYPE_RO_FAST: 131 /* 132 * Cover more cases regarding unsharing decisions when 133 * long-term R/O pinning by mapping the page R/O. 134 */ 135 ret = mprotect(mem, size, PROT_READ); 136 if (ret) { 137 ksft_test_result_fail("mprotect() failed (%s)\n", strerror(errno)); 138 goto munmap; 139 } 140 /* FALLTHROUGH */ 141 case TEST_TYPE_RW: 142 case TEST_TYPE_RW_FAST: { 143 struct pin_longterm_test args; 144 const bool fast = type == TEST_TYPE_RO_FAST || 145 type == TEST_TYPE_RW_FAST; 146 const bool rw = type == TEST_TYPE_RW || 147 type == TEST_TYPE_RW_FAST; 148 149 if (gup_fd < 0) { 150 ksft_test_result_skip("gup_test not available\n"); 151 break; 152 } 153 154 if (rw && shared && fs_is_unknown(fs_type)) { 155 ksft_test_result_skip("Unknown filesystem\n"); 156 return; 157 } 158 /* 159 * R/O pinning or pinning in a private mapping is always 160 * expected to work. Otherwise, we expect long-term R/W pinning 161 * to only succeed for special fielesystems. 162 */ 163 should_work = !shared || !rw || 164 fs_supports_writable_longterm_pinning(fs_type); 165 166 args.addr = (__u64)(uintptr_t)mem; 167 args.size = size; 168 args.flags = fast ? PIN_LONGTERM_TEST_FLAG_USE_FAST : 0; 169 args.flags |= rw ? PIN_LONGTERM_TEST_FLAG_USE_WRITE : 0; 170 ret = ioctl(gup_fd, PIN_LONGTERM_TEST_START, &args); 171 if (ret && errno == EINVAL) { 172 ksft_test_result_skip("PIN_LONGTERM_TEST_START failed (EINVAL)n"); 173 break; 174 } else if (ret && errno == EFAULT) { 175 ksft_test_result(!should_work, "Should have failed\n"); 176 break; 177 } else if (ret) { 178 ksft_test_result_fail("PIN_LONGTERM_TEST_START failed (%s)\n", 179 strerror(errno)); 180 break; 181 } 182 183 if (ioctl(gup_fd, PIN_LONGTERM_TEST_STOP)) 184 ksft_print_msg("[INFO] PIN_LONGTERM_TEST_STOP failed (%s)\n", 185 strerror(errno)); 186 187 /* 188 * TODO: if the kernel ever supports long-term R/W pinning on 189 * some previously unsupported filesystems, we might want to 190 * perform some additional tests for possible data corruptions. 191 */ 192 ksft_test_result(should_work, "Should have worked\n"); 193 break; 194 } 195 #ifdef LOCAL_CONFIG_HAVE_LIBURING 196 case TEST_TYPE_IOURING: { 197 struct io_uring ring; 198 struct iovec iov; 199 200 /* io_uring always pins pages writable. */ 201 if (shared && fs_is_unknown(fs_type)) { 202 ksft_test_result_skip("Unknown filesystem\n"); 203 return; 204 } 205 should_work = !shared || 206 fs_supports_writable_longterm_pinning(fs_type); 207 208 /* Skip on errors, as we might just lack kernel support. */ 209 ret = io_uring_queue_init(1, &ring, 0); 210 if (ret < 0) { 211 ksft_test_result_skip("io_uring_queue_init() failed (%s)\n", 212 strerror(-ret)); 213 break; 214 } 215 /* 216 * Register the range as a fixed buffer. This will FOLL_WRITE | 217 * FOLL_PIN | FOLL_LONGTERM the range. 218 */ 219 iov.iov_base = mem; 220 iov.iov_len = size; 221 ret = io_uring_register_buffers(&ring, &iov, 1); 222 /* Only new kernels return EFAULT. */ 223 if (ret && (errno == ENOSPC || errno == EOPNOTSUPP || 224 errno == EFAULT)) { 225 ksft_test_result(!should_work, "Should have failed (%s)\n", 226 strerror(errno)); 227 } else if (ret) { 228 /* 229 * We might just lack support or have insufficient 230 * MEMLOCK limits. 231 */ 232 ksft_test_result_skip("io_uring_register_buffers() failed (%s)\n", 233 strerror(-ret)); 234 } else { 235 ksft_test_result(should_work, "Should have worked\n"); 236 io_uring_unregister_buffers(&ring); 237 } 238 239 io_uring_queue_exit(&ring); 240 break; 241 } 242 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 243 default: 244 assert(false); 245 } 246 247 munmap: 248 munmap(mem, size); 249 } 250 251 typedef void (*test_fn)(int fd, size_t size); 252 253 static void run_with_memfd(test_fn fn, const char *desc) 254 { 255 int fd; 256 257 ksft_print_msg("[RUN] %s ... with memfd\n", desc); 258 259 fd = memfd_create("test", 0); 260 if (fd < 0) { 261 ksft_test_result_fail("memfd_create() failed (%s)\n", strerror(errno)); 262 return; 263 } 264 265 fn(fd, pagesize); 266 close(fd); 267 } 268 269 static void run_with_tmpfile(test_fn fn, const char *desc) 270 { 271 FILE *file; 272 int fd; 273 274 ksft_print_msg("[RUN] %s ... with tmpfile\n", desc); 275 276 file = tmpfile(); 277 if (!file) { 278 ksft_test_result_fail("tmpfile() failed (%s)\n", strerror(errno)); 279 return; 280 } 281 282 fd = fileno(file); 283 if (fd < 0) { 284 ksft_test_result_fail("fileno() failed (%s)\n", strerror(errno)); 285 goto close; 286 } 287 288 fn(fd, pagesize); 289 close: 290 fclose(file); 291 } 292 293 static void run_with_local_tmpfile(test_fn fn, const char *desc) 294 { 295 char filename[] = __FILE__"_tmpfile_XXXXXX"; 296 int fd; 297 298 ksft_print_msg("[RUN] %s ... with local tmpfile\n", desc); 299 300 fd = mkstemp(filename); 301 if (fd < 0) { 302 ksft_test_result_fail("mkstemp() failed (%s)\n", strerror(errno)); 303 return; 304 } 305 306 if (unlink(filename)) { 307 ksft_test_result_fail("unlink() failed (%s)\n", strerror(errno)); 308 goto close; 309 } 310 311 fn(fd, pagesize); 312 close: 313 close(fd); 314 } 315 316 static void run_with_memfd_hugetlb(test_fn fn, const char *desc, 317 size_t hugetlbsize) 318 { 319 int flags = MFD_HUGETLB; 320 int fd; 321 322 ksft_print_msg("[RUN] %s ... with memfd hugetlb (%zu kB)\n", desc, 323 hugetlbsize / 1024); 324 325 flags |= __builtin_ctzll(hugetlbsize) << MFD_HUGE_SHIFT; 326 327 fd = memfd_create("test", flags); 328 if (fd < 0) { 329 ksft_test_result_skip("memfd_create() failed (%s)\n", strerror(errno)); 330 return; 331 } 332 333 fn(fd, hugetlbsize); 334 close(fd); 335 } 336 337 struct test_case { 338 const char *desc; 339 test_fn fn; 340 }; 341 342 static void test_shared_rw_pin(int fd, size_t size) 343 { 344 do_test(fd, size, TEST_TYPE_RW, true); 345 } 346 347 static void test_shared_rw_fast_pin(int fd, size_t size) 348 { 349 do_test(fd, size, TEST_TYPE_RW_FAST, true); 350 } 351 352 static void test_shared_ro_pin(int fd, size_t size) 353 { 354 do_test(fd, size, TEST_TYPE_RO, true); 355 } 356 357 static void test_shared_ro_fast_pin(int fd, size_t size) 358 { 359 do_test(fd, size, TEST_TYPE_RO_FAST, true); 360 } 361 362 static void test_private_rw_pin(int fd, size_t size) 363 { 364 do_test(fd, size, TEST_TYPE_RW, false); 365 } 366 367 static void test_private_rw_fast_pin(int fd, size_t size) 368 { 369 do_test(fd, size, TEST_TYPE_RW_FAST, false); 370 } 371 372 static void test_private_ro_pin(int fd, size_t size) 373 { 374 do_test(fd, size, TEST_TYPE_RO, false); 375 } 376 377 static void test_private_ro_fast_pin(int fd, size_t size) 378 { 379 do_test(fd, size, TEST_TYPE_RO_FAST, false); 380 } 381 382 #ifdef LOCAL_CONFIG_HAVE_LIBURING 383 static void test_shared_iouring(int fd, size_t size) 384 { 385 do_test(fd, size, TEST_TYPE_IOURING, true); 386 } 387 388 static void test_private_iouring(int fd, size_t size) 389 { 390 do_test(fd, size, TEST_TYPE_IOURING, false); 391 } 392 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 393 394 static const struct test_case test_cases[] = { 395 { 396 "R/W longterm GUP pin in MAP_SHARED file mapping", 397 test_shared_rw_pin, 398 }, 399 { 400 "R/W longterm GUP-fast pin in MAP_SHARED file mapping", 401 test_shared_rw_fast_pin, 402 }, 403 { 404 "R/O longterm GUP pin in MAP_SHARED file mapping", 405 test_shared_ro_pin, 406 }, 407 { 408 "R/O longterm GUP-fast pin in MAP_SHARED file mapping", 409 test_shared_ro_fast_pin, 410 }, 411 { 412 "R/W longterm GUP pin in MAP_PRIVATE file mapping", 413 test_private_rw_pin, 414 }, 415 { 416 "R/W longterm GUP-fast pin in MAP_PRIVATE file mapping", 417 test_private_rw_fast_pin, 418 }, 419 { 420 "R/O longterm GUP pin in MAP_PRIVATE file mapping", 421 test_private_ro_pin, 422 }, 423 { 424 "R/O longterm GUP-fast pin in MAP_PRIVATE file mapping", 425 test_private_ro_fast_pin, 426 }, 427 #ifdef LOCAL_CONFIG_HAVE_LIBURING 428 { 429 "io_uring fixed buffer with MAP_SHARED file mapping", 430 test_shared_iouring, 431 }, 432 { 433 "io_uring fixed buffer with MAP_PRIVATE file mapping", 434 test_private_iouring, 435 }, 436 #endif /* LOCAL_CONFIG_HAVE_LIBURING */ 437 }; 438 439 static void run_test_case(struct test_case const *test_case) 440 { 441 int i; 442 443 run_with_memfd(test_case->fn, test_case->desc); 444 run_with_tmpfile(test_case->fn, test_case->desc); 445 run_with_local_tmpfile(test_case->fn, test_case->desc); 446 for (i = 0; i < nr_hugetlbsizes; i++) 447 run_with_memfd_hugetlb(test_case->fn, test_case->desc, 448 hugetlbsizes[i]); 449 } 450 451 static int tests_per_test_case(void) 452 { 453 return 3 + nr_hugetlbsizes; 454 } 455 456 int main(int argc, char **argv) 457 { 458 int i, err; 459 460 pagesize = getpagesize(); 461 nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes, 462 ARRAY_SIZE(hugetlbsizes)); 463 464 ksft_print_header(); 465 ksft_set_plan(ARRAY_SIZE(test_cases) * tests_per_test_case()); 466 467 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); 468 469 for (i = 0; i < ARRAY_SIZE(test_cases); i++) 470 run_test_case(&test_cases[i]); 471 472 err = ksft_get_fail_cnt(); 473 if (err) 474 ksft_exit_fail_msg("%d out of %d tests failed\n", 475 err, ksft_test_num()); 476 ksft_exit_pass(); 477 } 478