1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Userfaultfd tests common header 4 * 5 * Copyright (C) 2015-2023 Red Hat, Inc. 6 */ 7 #ifndef __UFFD_COMMON_H__ 8 #define __UFFD_COMMON_H__ 9 10 #define _GNU_SOURCE 11 #define __SANE_USERSPACE_TYPES__ // Use ll64 12 #include <stdio.h> 13 #include <errno.h> 14 #include <unistd.h> 15 #include <stdlib.h> 16 #include <sys/types.h> 17 #include <sys/stat.h> 18 #include <fcntl.h> 19 #include <time.h> 20 #include <signal.h> 21 #include <poll.h> 22 #include <string.h> 23 #include <linux/mman.h> 24 #include <sys/mman.h> 25 #include <sys/syscall.h> 26 #include <sys/ioctl.h> 27 #include <sys/wait.h> 28 #include <pthread.h> 29 #include <linux/userfaultfd.h> 30 #include <setjmp.h> 31 #include <stdbool.h> 32 #include <assert.h> 33 #include <inttypes.h> 34 #include <stdint.h> 35 #include <sys/random.h> 36 #include <stdatomic.h> 37 38 #include "kselftest.h" 39 #include "vm_util.h" 40 41 #define UFFD_FLAGS (O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY) 42 43 #define _err(fmt, ...) \ 44 do { \ 45 int ret = errno; \ 46 ksft_print_msg("ERROR: " fmt " (errno=%d, @%s:%d)\n", \ 47 ##__VA_ARGS__, ret, __FILE__, __LINE__); \ 48 } while (0) 49 50 #define errexit(fmt, ...) \ 51 do { \ 52 _err(fmt, ##__VA_ARGS__); \ 53 ksft_exit_fail(); \ 54 } while (0) 55 56 #define err(fmt, ...) errexit(fmt, ##__VA_ARGS__) 57 58 struct uffd_global_test_opts { 59 unsigned long nr_parallel, nr_pages, nr_pages_per_cpu, page_size; 60 char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap; 61 int uffd, uffd_flags, finished, *pipefd, test_type; 62 bool map_shared; 63 bool test_uffdio_wp; 64 unsigned long long *count_verify; 65 volatile bool test_uffdio_copy_eexist; 66 atomic_bool ready_for_fork; 67 }; 68 typedef struct uffd_global_test_opts uffd_global_test_opts_t; 69 70 /* Userfaultfd test statistics */ 71 struct uffd_args { 72 int cpu; 73 /* Whether apply wr-protects when installing pages */ 74 bool apply_wp; 75 unsigned long missing_faults; 76 unsigned long wp_faults; 77 unsigned long minor_faults; 78 struct uffd_global_test_opts *gopts; 79 80 /* A custom fault handler; defaults to uffd_handle_page_fault. */ 81 void (*handle_fault)(struct uffd_global_test_opts *gopts, 82 struct uffd_msg *msg, 83 struct uffd_args *args); 84 }; 85 86 struct uffd_test_ops { 87 int (*allocate_area)(uffd_global_test_opts_t *gopts, void **alloc_area, bool is_src); 88 void (*release_pages)(uffd_global_test_opts_t *gopts, char *rel_area); 89 void (*alias_mapping)(uffd_global_test_opts_t *gopts, 90 __u64 *start, 91 size_t len, 92 unsigned long offset); 93 void (*check_pmd_mapping)(uffd_global_test_opts_t *gopts, void *p, int expect_nr_hpages); 94 }; 95 typedef struct uffd_test_ops uffd_test_ops_t; 96 97 struct uffd_test_case_ops { 98 int (*pre_alloc)(uffd_global_test_opts_t *gopts, const char **errmsg); 99 int (*post_alloc)(uffd_global_test_opts_t *gopts, const char **errmsg); 100 }; 101 typedef struct uffd_test_case_ops uffd_test_case_ops_t; 102 103 extern uffd_global_test_opts_t *uffd_gtest_opts; 104 extern uffd_test_ops_t anon_uffd_test_ops; 105 extern uffd_test_ops_t shmem_uffd_test_ops; 106 extern uffd_test_ops_t hugetlb_uffd_test_ops; 107 extern uffd_test_ops_t *uffd_test_ops; 108 extern uffd_test_case_ops_t *uffd_test_case_ops; 109 110 pthread_mutex_t *area_mutex(char *area, unsigned long nr, uffd_global_test_opts_t *gopts); 111 volatile unsigned long long *area_count(char *area, 112 unsigned long nr, 113 uffd_global_test_opts_t *gopts); 114 115 void uffd_stats_report(struct uffd_args *args, int n_cpus); 116 int uffd_test_ctx_init(uffd_global_test_opts_t *gopts, uint64_t features, const char **errmsg); 117 void uffd_test_ctx_clear(uffd_global_test_opts_t *gopts); 118 int userfaultfd_open(uffd_global_test_opts_t *gopts, uint64_t *features); 119 int uffd_read_msg(uffd_global_test_opts_t *gopts, struct uffd_msg *msg); 120 void wp_range(int ufd, __u64 start, __u64 len, bool wp); 121 void uffd_handle_page_fault(uffd_global_test_opts_t *gopts, 122 struct uffd_msg *msg, 123 struct uffd_args *args); 124 int __copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool retry, bool wp); 125 int copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool wp); 126 int move_page(uffd_global_test_opts_t *gopts, unsigned long offset, unsigned long len); 127 void *uffd_poll_thread(void *arg); 128 129 int uffd_open_dev(unsigned int flags); 130 int uffd_open_sys(unsigned int flags); 131 int uffd_open(unsigned int flags); 132 int uffd_get_features(uint64_t *features); 133 134 #define TEST_ANON 1 135 #define TEST_HUGETLB 2 136 #define TEST_SHMEM 3 137 138 #endif 139