1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <stdint.h>
3 #include <stdbool.h>
4 #include <sys/mman.h>
5 #include <err.h>
6 #include <stdarg.h>
7 #include <strings.h> /* ffsl() */
8 #include <unistd.h> /* _SC_PAGESIZE */
9 #include "kselftest.h"
10 #include <linux/fs.h>
11
12 #define BIT_ULL(nr) (1ULL << (nr))
13 #define PM_SOFT_DIRTY BIT_ULL(55)
14 #define PM_MMAP_EXCLUSIVE BIT_ULL(56)
15 #define PM_UFFD_WP BIT_ULL(57)
16 #define PM_GUARD_REGION BIT_ULL(58)
17 #define PM_FILE BIT_ULL(61)
18 #define PM_SWAP BIT_ULL(62)
19 #define PM_PRESENT BIT_ULL(63)
20
21 #define KPF_COMPOUND_HEAD BIT_ULL(15)
22 #define KPF_COMPOUND_TAIL BIT_ULL(16)
23 #define KPF_HWPOISON BIT_ULL(19)
24 #define KPF_THP BIT_ULL(22)
25 /*
26 * Ignore the checkpatch warning, we must read from x but don't want to do
27 * anything with it in order to trigger a read page fault. We therefore must use
28 * volatile to stop the compiler from optimising this away.
29 */
30 #define FORCE_READ(x) (*(const volatile typeof(x) *)&(x))
31
32 extern unsigned int __page_size;
33 extern unsigned int __page_shift;
34
35 /*
36 * Represents an open fd and PROCMAP_QUERY state for binary (via ioctl)
37 * /proc/$pid/[s]maps lookup.
38 */
39 struct procmap_fd {
40 int fd;
41 struct procmap_query query;
42 };
43
psize(void)44 static inline unsigned int psize(void)
45 {
46 if (!__page_size)
47 __page_size = sysconf(_SC_PAGESIZE);
48 return __page_size;
49 }
50
pshift(void)51 static inline unsigned int pshift(void)
52 {
53 if (!__page_shift)
54 __page_shift = (ffsl(psize()) - 1);
55 return __page_shift;
56 }
57
force_read_pages(char * addr,unsigned int nr_pages,size_t pagesize)58 static inline void force_read_pages(char *addr, unsigned int nr_pages,
59 size_t pagesize)
60 {
61 for (unsigned int i = 0; i < nr_pages; i++)
62 FORCE_READ(addr[i * pagesize]);
63 }
64
65 bool detect_huge_zeropage(void);
66
67 /*
68 * Plan 9 FS has bugs (at least on QEMU) where certain operations fail with
69 * ENOENT on unlinked files. See
70 * https://gitlab.com/qemu-project/qemu/-/issues/103 for some info about such
71 * bugs. There are rumours of NFS implementations with similar bugs.
72 *
73 * Ideally, tests should just detect filesystems known to have such issues and
74 * bail early. But 9pfs has the additional "feature" that it causes fstatfs to
75 * pass through the f_type field from the host filesystem. To avoid having to
76 * scrape /proc/mounts or some other hackery, tests can call this function when
77 * it seems such a bug might have been encountered.
78 */
skip_test_dodgy_fs(const char * op_name)79 static inline void skip_test_dodgy_fs(const char *op_name)
80 {
81 ksft_test_result_skip("%s failed with ENOENT. Filesystem might be buggy (9pfs?)\n", op_name);
82 }
83
84 uint64_t pagemap_get_entry(int fd, char *start);
85 bool pagemap_is_softdirty(int fd, char *start);
86 bool pagemap_is_swapped(int fd, char *start);
87 bool pagemap_is_populated(int fd, char *start);
88 unsigned long pagemap_get_pfn(int fd, char *start);
89 void clear_softdirty(void);
90 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len);
91 uint64_t read_pmd_pagesize(void);
92 unsigned long rss_anon(void);
93 bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size);
94 bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size);
95 bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size);
96 int64_t allocate_transhuge(void *ptr, int pagemap_fd);
97 int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags);
98 int gather_folio_orders(char *vaddr_start, size_t len,
99 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders);
100
101 int uffd_register(int uffd, void *addr, uint64_t len,
102 bool miss, bool wp, bool minor);
103 int uffd_unregister(int uffd, void *addr, uint64_t len);
104 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,
105 bool miss, bool wp, bool minor, uint64_t *ioctls);
106 bool check_vmflag_io(void *addr);
107 bool check_vmflag_pfnmap(void *addr);
108 bool check_vmflag_guard(void *addr);
109 int open_procmap(pid_t pid, struct procmap_fd *procmap_out);
110 int query_procmap(struct procmap_fd *procmap);
111 bool find_vma_procmap(struct procmap_fd *procmap, void *address);
112 int close_procmap(struct procmap_fd *procmap);
113 int write_sysfs(const char *file_path, unsigned long val);
114 int read_sysfs(const char *file_path, unsigned long *val);
115 bool softdirty_supported(void);
116
open_self_procmap(struct procmap_fd * procmap_out)117 static inline int open_self_procmap(struct procmap_fd *procmap_out)
118 {
119 pid_t pid = getpid();
120
121 return open_procmap(pid, procmap_out);
122 }
123
124 /* These helpers need to be inline to match the kselftest.h idiom. */
125 static char test_name[1024];
126
log_test_start(const char * name,...)127 static inline void log_test_start(const char *name, ...)
128 {
129 va_list args;
130 va_start(args, name);
131
132 vsnprintf(test_name, sizeof(test_name), name, args);
133 ksft_print_msg("[RUN] %s\n", test_name);
134
135 va_end(args);
136 }
137
log_test_result(int result)138 static inline void log_test_result(int result)
139 {
140 ksft_test_result_report(result, "%s\n", test_name);
141 }
142
sz2ord(size_t size,size_t pagesize)143 static inline int sz2ord(size_t size, size_t pagesize)
144 {
145 return __builtin_ctzll(size / pagesize);
146 }
147
148 void *sys_mremap(void *old_address, unsigned long old_size,
149 unsigned long new_size, int flags, void *new_address);
150
151 long ksm_get_self_zero_pages(void);
152 long ksm_get_self_merging_pages(void);
153 long ksm_get_full_scans(void);
154 int ksm_use_zero_pages(void);
155 int ksm_start(void);
156 int ksm_stop(void);
157 int get_hardware_corrupted_size(unsigned long *val);
158 int unpoison_memory(unsigned long pfn);
159
160 /*
161 * On ppc64 this will only work with radix 2M hugepage size
162 */
163 #define HPAGE_SHIFT 21
164 #define HPAGE_SIZE (1 << HPAGE_SHIFT)
165
166 #define PAGEMAP_PRESENT(ent) (((ent) & (1ull << 63)) != 0)
167 #define PAGEMAP_PFN(ent) ((ent) & ((1ull << 55) - 1))
168
169 void write_file(const char *path, const char *buf, size_t buflen);
170 int read_file(const char *path, char *buf, size_t buflen);
171 unsigned long read_num(const char *path);
172 void write_num(const char *path, unsigned long num);
173 void write_num_ignore_einval(const char *path, unsigned long num);
174
175 void shm_limits_prepare(unsigned long length);
176 void __shm_limits_restore(void);
177
178 #define SHM_LIMITS_RESTORE() \
179 static void __attribute__((destructor)) shm_limits_restore(void) \
180 { \
181 __shm_limits_restore(); \
182 }
183