1 // SPDX-License-Identifier: GPL-2.0
2 #include <string.h>
3 #include <fcntl.h>
4 #include <dirent.h>
5 #include <sys/ioctl.h>
6 #include <linux/userfaultfd.h>
7 #include <linux/fs.h>
8 #include <sys/syscall.h>
9 #include <unistd.h>
10 #include "../kselftest.h"
11 #include "vm_util.h"
12
13 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
14 #define SMAP_FILE_PATH "/proc/self/smaps"
15 #define STATUS_FILE_PATH "/proc/self/status"
16 #define MAX_LINE_LENGTH 500
17
18 unsigned int __page_size;
19 unsigned int __page_shift;
20
pagemap_get_entry(int fd,char * start)21 uint64_t pagemap_get_entry(int fd, char *start)
22 {
23 const unsigned long pfn = (unsigned long)start / getpagesize();
24 uint64_t entry;
25 int ret;
26
27 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
28 if (ret != sizeof(entry))
29 ksft_exit_fail_msg("reading pagemap failed\n");
30 return entry;
31 }
32
__pagemap_scan_get_categories(int fd,char * start,struct page_region * r)33 static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r)
34 {
35 struct pm_scan_arg arg;
36
37 arg.start = (uintptr_t)start;
38 arg.end = (uintptr_t)(start + psize());
39 arg.vec = (uintptr_t)r;
40 arg.vec_len = 1;
41 arg.flags = 0;
42 arg.size = sizeof(struct pm_scan_arg);
43 arg.max_pages = 0;
44 arg.category_inverted = 0;
45 arg.category_mask = 0;
46 arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE |
47 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |
48 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY;
49 arg.return_mask = arg.category_anyof_mask;
50
51 return ioctl(fd, PAGEMAP_SCAN, &arg);
52 }
53
pagemap_scan_get_categories(int fd,char * start)54 static uint64_t pagemap_scan_get_categories(int fd, char *start)
55 {
56 struct page_region r;
57 long ret;
58
59 ret = __pagemap_scan_get_categories(fd, start, &r);
60 if (ret < 0)
61 ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno));
62 if (ret == 0)
63 return 0;
64 return r.categories;
65 }
66
67 /* `start` is any valid address. */
pagemap_scan_supported(int fd,char * start)68 static bool pagemap_scan_supported(int fd, char *start)
69 {
70 static int supported = -1;
71 int ret;
72
73 if (supported != -1)
74 return supported;
75
76 /* Provide an invalid address in order to trigger EFAULT. */
77 ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL);
78 if (ret == 0)
79 ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n");
80
81 supported = errno == EFAULT;
82
83 return supported;
84 }
85
page_entry_is(int fd,char * start,char * desc,uint64_t pagemap_flags,uint64_t pagescan_flags)86 static bool page_entry_is(int fd, char *start, char *desc,
87 uint64_t pagemap_flags, uint64_t pagescan_flags)
88 {
89 bool m = pagemap_get_entry(fd, start) & pagemap_flags;
90
91 if (pagemap_scan_supported(fd, start)) {
92 bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags;
93
94 if (m == s)
95 return m;
96
97 ksft_exit_fail_msg(
98 "read and ioctl return unmatched results for %s: %d %d", desc, m, s);
99 }
100 return m;
101 }
102
pagemap_is_softdirty(int fd,char * start)103 bool pagemap_is_softdirty(int fd, char *start)
104 {
105 return page_entry_is(fd, start, "soft-dirty",
106 PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY);
107 }
108
pagemap_is_swapped(int fd,char * start)109 bool pagemap_is_swapped(int fd, char *start)
110 {
111 return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED);
112 }
113
pagemap_is_populated(int fd,char * start)114 bool pagemap_is_populated(int fd, char *start)
115 {
116 return page_entry_is(fd, start, "populated",
117 PM_PRESENT | PM_SWAP,
118 PAGE_IS_PRESENT | PAGE_IS_SWAPPED);
119 }
120
pagemap_get_pfn(int fd,char * start)121 unsigned long pagemap_get_pfn(int fd, char *start)
122 {
123 uint64_t entry = pagemap_get_entry(fd, start);
124
125 /* If present (63th bit), PFN is at bit 0 -- 54. */
126 if (entry & PM_PRESENT)
127 return entry & 0x007fffffffffffffull;
128 return -1ul;
129 }
130
clear_softdirty(void)131 void clear_softdirty(void)
132 {
133 int ret;
134 const char *ctrl = "4";
135 int fd = open("/proc/self/clear_refs", O_WRONLY);
136
137 if (fd < 0)
138 ksft_exit_fail_msg("opening clear_refs failed\n");
139 ret = write(fd, ctrl, strlen(ctrl));
140 close(fd);
141 if (ret != strlen(ctrl))
142 ksft_exit_fail_msg("writing clear_refs failed\n");
143 }
144
check_for_pattern(FILE * fp,const char * pattern,char * buf,size_t len)145 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
146 {
147 while (fgets(buf, len, fp)) {
148 if (!strncmp(buf, pattern, strlen(pattern)))
149 return true;
150 }
151 return false;
152 }
153
read_pmd_pagesize(void)154 uint64_t read_pmd_pagesize(void)
155 {
156 int fd;
157 char buf[20];
158 ssize_t num_read;
159
160 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);
161 if (fd == -1)
162 return 0;
163
164 num_read = read(fd, buf, 19);
165 if (num_read < 1) {
166 close(fd);
167 return 0;
168 }
169 buf[num_read] = '\0';
170 close(fd);
171
172 return strtoul(buf, NULL, 10);
173 }
174
rss_anon(void)175 unsigned long rss_anon(void)
176 {
177 unsigned long rss_anon = 0;
178 FILE *fp;
179 char buffer[MAX_LINE_LENGTH];
180
181 fp = fopen(STATUS_FILE_PATH, "r");
182 if (!fp)
183 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH);
184
185 if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer)))
186 goto err_out;
187
188 if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1)
189 ksft_exit_fail_msg("Reading status error\n");
190
191 err_out:
192 fclose(fp);
193 return rss_anon;
194 }
195
__check_huge(void * addr,char * pattern,int nr_hpages,uint64_t hpage_size)196 bool __check_huge(void *addr, char *pattern, int nr_hpages,
197 uint64_t hpage_size)
198 {
199 uint64_t thp = -1;
200 int ret;
201 FILE *fp;
202 char buffer[MAX_LINE_LENGTH];
203 char addr_pattern[MAX_LINE_LENGTH];
204
205 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
206 (unsigned long) addr);
207 if (ret >= MAX_LINE_LENGTH)
208 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
209
210 fp = fopen(SMAP_FILE_PATH, "r");
211 if (!fp)
212 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
213
214 if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))
215 goto err_out;
216
217 /*
218 * Fetch the pattern in the same block and check the number of
219 * hugepages.
220 */
221 if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer)))
222 goto err_out;
223
224 snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern);
225
226 if (sscanf(buffer, addr_pattern, &thp) != 1)
227 ksft_exit_fail_msg("Reading smap error\n");
228
229 err_out:
230 fclose(fp);
231 return thp == (nr_hpages * (hpage_size >> 10));
232 }
233
check_huge_anon(void * addr,int nr_hpages,uint64_t hpage_size)234 bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)
235 {
236 return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
237 }
238
check_huge_file(void * addr,int nr_hpages,uint64_t hpage_size)239 bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)
240 {
241 return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
242 }
243
check_huge_shmem(void * addr,int nr_hpages,uint64_t hpage_size)244 bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)
245 {
246 return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
247 }
248
allocate_transhuge(void * ptr,int pagemap_fd)249 int64_t allocate_transhuge(void *ptr, int pagemap_fd)
250 {
251 uint64_t ent[2];
252
253 /* drop pmd */
254 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
255 MAP_FIXED | MAP_ANONYMOUS |
256 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
257 ksft_exit_fail_msg("mmap transhuge\n");
258
259 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
260 ksft_exit_fail_msg("MADV_HUGEPAGE\n");
261
262 /* allocate transparent huge page */
263 *(volatile void **)ptr = ptr;
264
265 if (pread(pagemap_fd, ent, sizeof(ent),
266 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent))
267 ksft_exit_fail_msg("read pagemap\n");
268
269 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
270 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
271 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1)))
272 return PAGEMAP_PFN(ent[0]);
273
274 return -1;
275 }
276
default_huge_page_size(void)277 unsigned long default_huge_page_size(void)
278 {
279 unsigned long hps = 0;
280 char *line = NULL;
281 size_t linelen = 0;
282 FILE *f = fopen("/proc/meminfo", "r");
283
284 if (!f)
285 return 0;
286 while (getline(&line, &linelen, f) > 0) {
287 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) {
288 hps <<= 10;
289 break;
290 }
291 }
292
293 free(line);
294 fclose(f);
295 return hps;
296 }
297
detect_hugetlb_page_sizes(size_t sizes[],int max)298 int detect_hugetlb_page_sizes(size_t sizes[], int max)
299 {
300 DIR *dir = opendir("/sys/kernel/mm/hugepages/");
301 int count = 0;
302
303 if (!dir)
304 return 0;
305
306 while (count < max) {
307 struct dirent *entry = readdir(dir);
308 size_t kb;
309
310 if (!entry)
311 break;
312 if (entry->d_type != DT_DIR)
313 continue;
314 if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1)
315 continue;
316 sizes[count++] = kb * 1024;
317 ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n",
318 kb);
319 }
320 closedir(dir);
321 return count;
322 }
323
324 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */
uffd_register_with_ioctls(int uffd,void * addr,uint64_t len,bool miss,bool wp,bool minor,uint64_t * ioctls)325 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,
326 bool miss, bool wp, bool minor, uint64_t *ioctls)
327 {
328 struct uffdio_register uffdio_register = { 0 };
329 uint64_t mode = 0;
330 int ret = 0;
331
332 if (miss)
333 mode |= UFFDIO_REGISTER_MODE_MISSING;
334 if (wp)
335 mode |= UFFDIO_REGISTER_MODE_WP;
336 if (minor)
337 mode |= UFFDIO_REGISTER_MODE_MINOR;
338
339 uffdio_register.range.start = (unsigned long)addr;
340 uffdio_register.range.len = len;
341 uffdio_register.mode = mode;
342
343 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)
344 ret = -errno;
345 else if (ioctls)
346 *ioctls = uffdio_register.ioctls;
347
348 return ret;
349 }
350
uffd_register(int uffd,void * addr,uint64_t len,bool miss,bool wp,bool minor)351 int uffd_register(int uffd, void *addr, uint64_t len,
352 bool miss, bool wp, bool minor)
353 {
354 return uffd_register_with_ioctls(uffd, addr, len,
355 miss, wp, minor, NULL);
356 }
357
uffd_unregister(int uffd,void * addr,uint64_t len)358 int uffd_unregister(int uffd, void *addr, uint64_t len)
359 {
360 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len };
361 int ret = 0;
362
363 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1)
364 ret = -errno;
365
366 return ret;
367 }
368
get_free_hugepages(void)369 unsigned long get_free_hugepages(void)
370 {
371 unsigned long fhp = 0;
372 char *line = NULL;
373 size_t linelen = 0;
374 FILE *f = fopen("/proc/meminfo", "r");
375
376 if (!f)
377 return fhp;
378 while (getline(&line, &linelen, f) > 0) {
379 if (sscanf(line, "HugePages_Free: %lu", &fhp) == 1)
380 break;
381 }
382
383 free(line);
384 fclose(f);
385 return fhp;
386 }
387