1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #include <stdint.h> 3 #include <stdbool.h> 4 #include <sys/mman.h> 5 #include <err.h> 6 #include <stdarg.h> 7 #include <strings.h> /* ffsl() */ 8 #include <unistd.h> /* _SC_PAGESIZE */ 9 #include "kselftest.h" 10 #include <linux/fs.h> 11 #include <mm/file_utils.h> 12 #include <mm/hugepage_settings.h> 13 14 #define BIT_ULL(nr) (1ULL << (nr)) 15 #define ALIGN(x, a) (((x) + (a) - 1) & ~((a) - 1)) 16 17 #define PM_SOFT_DIRTY BIT_ULL(55) 18 #define PM_MMAP_EXCLUSIVE BIT_ULL(56) 19 #define PM_UFFD_WP BIT_ULL(57) 20 #define PM_GUARD_REGION BIT_ULL(58) 21 #define PM_FILE BIT_ULL(61) 22 #define PM_SWAP BIT_ULL(62) 23 #define PM_PRESENT BIT_ULL(63) 24 25 #define KPF_COMPOUND_HEAD BIT_ULL(15) 26 #define KPF_COMPOUND_TAIL BIT_ULL(16) 27 #define KPF_HWPOISON BIT_ULL(19) 28 #define KPF_THP BIT_ULL(22) 29 /* 30 * Ignore the checkpatch warning, we must read from x but don't want to do 31 * anything with it in order to trigger a read page fault. We therefore must use 32 * volatile to stop the compiler from optimising this away. 33 */ 34 #define FORCE_READ(x) (*(const volatile typeof(x) *)&(x)) 35 36 extern unsigned int __page_size; 37 extern unsigned int __page_shift; 38 39 /* 40 * Represents an open fd and PROCMAP_QUERY state for binary (via ioctl) 41 * /proc/$pid/[s]maps lookup. 42 */ 43 struct procmap_fd { 44 int fd; 45 struct procmap_query query; 46 }; 47 48 static inline unsigned int psize(void) 49 { 50 if (!__page_size) 51 __page_size = sysconf(_SC_PAGESIZE); 52 return __page_size; 53 } 54 55 static inline unsigned int pshift(void) 56 { 57 if (!__page_shift) 58 __page_shift = (ffsl(psize()) - 1); 59 return __page_shift; 60 } 61 62 static inline void force_read_pages(char *addr, unsigned int nr_pages, 63 size_t pagesize) 64 { 65 for (unsigned int i = 0; i < nr_pages; i++) 66 FORCE_READ(addr[i * pagesize]); 67 } 68 69 bool detect_huge_zeropage(void); 70 71 /* 72 * Plan 9 FS has bugs (at least on QEMU) where certain operations fail with 73 * ENOENT on unlinked files. See 74 * https://gitlab.com/qemu-project/qemu/-/issues/103 for some info about such 75 * bugs. There are rumours of NFS implementations with similar bugs. 76 * 77 * Ideally, tests should just detect filesystems known to have such issues and 78 * bail early. But 9pfs has the additional "feature" that it causes fstatfs to 79 * pass through the f_type field from the host filesystem. To avoid having to 80 * scrape /proc/mounts or some other hackery, tests can call this function when 81 * it seems such a bug might have been encountered. 82 */ 83 static inline void skip_test_dodgy_fs(const char *op_name) 84 { 85 ksft_test_result_skip("%s failed with ENOENT. Filesystem might be buggy (9pfs?)\n", op_name); 86 } 87 88 uint64_t pagemap_get_entry(int fd, char *start); 89 bool pagemap_is_softdirty(int fd, char *start); 90 bool pagemap_is_swapped(int fd, char *start); 91 bool pagemap_is_populated(int fd, char *start); 92 unsigned long pagemap_get_pfn(int fd, char *start); 93 void clear_softdirty(void); 94 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len); 95 uint64_t read_pmd_pagesize(void); 96 unsigned long rss_anon(void); 97 bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size); 98 bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size); 99 bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size); 100 int64_t allocate_transhuge(void *ptr, int pagemap_fd); 101 int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags); 102 int gather_folio_orders(char *vaddr_start, size_t len, 103 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders); 104 bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, 105 int kpageflags_fd); 106 bool is_range_backed_by_order(char *start, size_t len, int order, 107 int pagemap_fd, int kpageflags_fd); 108 109 int uffd_register(int uffd, void *addr, uint64_t len, 110 bool miss, bool wp, bool minor); 111 int uffd_unregister(int uffd, void *addr, uint64_t len); 112 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, 113 bool miss, bool wp, bool minor, uint64_t *ioctls); 114 bool check_vmflag_io(void *addr); 115 bool check_vmflag_pfnmap(void *addr); 116 bool check_vmflag_guard(void *addr); 117 int open_procmap(pid_t pid, struct procmap_fd *procmap_out); 118 int query_procmap(struct procmap_fd *procmap); 119 bool find_vma_procmap(struct procmap_fd *procmap, void *address); 120 int close_procmap(struct procmap_fd *procmap); 121 int write_sysfs(const char *file_path, unsigned long val); 122 int read_sysfs(const char *file_path, unsigned long *val); 123 bool softdirty_supported(void); 124 125 static inline int open_self_procmap(struct procmap_fd *procmap_out) 126 { 127 pid_t pid = getpid(); 128 129 return open_procmap(pid, procmap_out); 130 } 131 132 /* These helpers need to be inline to match the kselftest.h idiom. */ 133 static char test_name[1024]; 134 135 static inline void log_test_start(const char *name, ...) 136 { 137 va_list args; 138 va_start(args, name); 139 140 vsnprintf(test_name, sizeof(test_name), name, args); 141 ksft_print_msg("[RUN] %s\n", test_name); 142 143 va_end(args); 144 } 145 146 static inline void log_test_result(int result) 147 { 148 ksft_test_result_report(result, "%s\n", test_name); 149 } 150 151 static inline int sz2ord(size_t size, size_t pagesize) 152 { 153 return __builtin_ctzll(size / pagesize); 154 } 155 156 void *sys_mremap(void *old_address, unsigned long old_size, 157 unsigned long new_size, int flags, void *new_address); 158 159 long ksm_get_self_zero_pages(void); 160 long ksm_get_self_merging_pages(void); 161 long ksm_get_full_scans(void); 162 int ksm_use_zero_pages(void); 163 int ksm_start(void); 164 int ksm_stop(void); 165 int get_hardware_corrupted_size(unsigned long *val); 166 int unpoison_memory(unsigned long pfn); 167 168 /* 169 * On ppc64 this will only work with radix 2M hugepage size 170 */ 171 #define HPAGE_SHIFT 21 172 #define HPAGE_SIZE (1 << HPAGE_SHIFT) 173 174 #define PAGEMAP_PRESENT(ent) (((ent) & (1ull << 63)) != 0) 175 #define PAGEMAP_PFN(ent) ((ent) & ((1ull << 55) - 1)) 176 177 void shm_limits_prepare(unsigned long length); 178 void __shm_limits_restore(void); 179 180 #define SHM_LIMITS_RESTORE() \ 181 static void __attribute__((destructor)) shm_limits_restore(void) \ 182 { \ 183 __shm_limits_restore(); \ 184 } 185