xref: /linux/tools/testing/selftests/mm/vm_util.c (revision 55d0969c451159cff86949b38c39171cab962069)
1 // SPDX-License-Identifier: GPL-2.0
2 #include <string.h>
3 #include <fcntl.h>
4 #include <dirent.h>
5 #include <sys/ioctl.h>
6 #include <linux/userfaultfd.h>
7 #include <linux/fs.h>
8 #include <sys/syscall.h>
9 #include <unistd.h>
10 #include "../kselftest.h"
11 #include "vm_util.h"
12 
13 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
14 #define SMAP_FILE_PATH "/proc/self/smaps"
15 #define STATUS_FILE_PATH "/proc/self/status"
16 #define MAX_LINE_LENGTH 500
17 
18 unsigned int __page_size;
19 unsigned int __page_shift;
20 
21 uint64_t pagemap_get_entry(int fd, char *start)
22 {
23 	const unsigned long pfn = (unsigned long)start / getpagesize();
24 	uint64_t entry;
25 	int ret;
26 
27 	ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
28 	if (ret != sizeof(entry))
29 		ksft_exit_fail_msg("reading pagemap failed\n");
30 	return entry;
31 }
32 
33 static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r)
34 {
35 	struct pm_scan_arg arg;
36 
37 	arg.start = (uintptr_t)start;
38 	arg.end = (uintptr_t)(start + psize());
39 	arg.vec = (uintptr_t)r;
40 	arg.vec_len = 1;
41 	arg.flags = 0;
42 	arg.size = sizeof(struct pm_scan_arg);
43 	arg.max_pages = 0;
44 	arg.category_inverted = 0;
45 	arg.category_mask = 0;
46 	arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE |
47 				  PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |
48 				  PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY;
49 	arg.return_mask = arg.category_anyof_mask;
50 
51 	return ioctl(fd, PAGEMAP_SCAN, &arg);
52 }
53 
54 static uint64_t pagemap_scan_get_categories(int fd, char *start)
55 {
56 	struct page_region r;
57 	long ret;
58 
59 	ret = __pagemap_scan_get_categories(fd, start, &r);
60 	if (ret < 0)
61 		ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno));
62 	if (ret == 0)
63 		return 0;
64 	return r.categories;
65 }
66 
67 /* `start` is any valid address. */
68 static bool pagemap_scan_supported(int fd, char *start)
69 {
70 	static int supported = -1;
71 	int ret;
72 
73 	if (supported != -1)
74 		return supported;
75 
76 	/* Provide an invalid address in order to trigger EFAULT. */
77 	ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL);
78 	if (ret == 0)
79 		ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n");
80 
81 	supported = errno == EFAULT;
82 
83 	return supported;
84 }
85 
86 static bool page_entry_is(int fd, char *start, char *desc,
87 			  uint64_t pagemap_flags, uint64_t pagescan_flags)
88 {
89 	bool m = pagemap_get_entry(fd, start) & pagemap_flags;
90 
91 	if (pagemap_scan_supported(fd, start)) {
92 		bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags;
93 
94 		if (m == s)
95 			return m;
96 
97 		ksft_exit_fail_msg(
98 			"read and ioctl return unmatched results for %s: %d %d", desc, m, s);
99 	}
100 	return m;
101 }
102 
103 bool pagemap_is_softdirty(int fd, char *start)
104 {
105 	return page_entry_is(fd, start, "soft-dirty",
106 				PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY);
107 }
108 
109 bool pagemap_is_swapped(int fd, char *start)
110 {
111 	return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED);
112 }
113 
114 bool pagemap_is_populated(int fd, char *start)
115 {
116 	return page_entry_is(fd, start, "populated",
117 				PM_PRESENT | PM_SWAP,
118 				PAGE_IS_PRESENT | PAGE_IS_SWAPPED);
119 }
120 
121 unsigned long pagemap_get_pfn(int fd, char *start)
122 {
123 	uint64_t entry = pagemap_get_entry(fd, start);
124 
125 	/* If present (63th bit), PFN is at bit 0 -- 54. */
126 	if (entry & PM_PRESENT)
127 		return entry & 0x007fffffffffffffull;
128 	return -1ul;
129 }
130 
131 void clear_softdirty(void)
132 {
133 	int ret;
134 	const char *ctrl = "4";
135 	int fd = open("/proc/self/clear_refs", O_WRONLY);
136 
137 	if (fd < 0)
138 		ksft_exit_fail_msg("opening clear_refs failed\n");
139 	ret = write(fd, ctrl, strlen(ctrl));
140 	close(fd);
141 	if (ret != strlen(ctrl))
142 		ksft_exit_fail_msg("writing clear_refs failed\n");
143 }
144 
145 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
146 {
147 	while (fgets(buf, len, fp)) {
148 		if (!strncmp(buf, pattern, strlen(pattern)))
149 			return true;
150 	}
151 	return false;
152 }
153 
154 uint64_t read_pmd_pagesize(void)
155 {
156 	int fd;
157 	char buf[20];
158 	ssize_t num_read;
159 
160 	fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);
161 	if (fd == -1)
162 		return 0;
163 
164 	num_read = read(fd, buf, 19);
165 	if (num_read < 1) {
166 		close(fd);
167 		return 0;
168 	}
169 	buf[num_read] = '\0';
170 	close(fd);
171 
172 	return strtoul(buf, NULL, 10);
173 }
174 
175 unsigned long rss_anon(void)
176 {
177 	unsigned long rss_anon = 0;
178 	FILE *fp;
179 	char buffer[MAX_LINE_LENGTH];
180 
181 	fp = fopen(STATUS_FILE_PATH, "r");
182 	if (!fp)
183 		ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH);
184 
185 	if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer)))
186 		goto err_out;
187 
188 	if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1)
189 		ksft_exit_fail_msg("Reading status error\n");
190 
191 err_out:
192 	fclose(fp);
193 	return rss_anon;
194 }
195 
196 bool __check_huge(void *addr, char *pattern, int nr_hpages,
197 		  uint64_t hpage_size)
198 {
199 	uint64_t thp = -1;
200 	int ret;
201 	FILE *fp;
202 	char buffer[MAX_LINE_LENGTH];
203 	char addr_pattern[MAX_LINE_LENGTH];
204 
205 	ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
206 		       (unsigned long) addr);
207 	if (ret >= MAX_LINE_LENGTH)
208 		ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
209 
210 	fp = fopen(SMAP_FILE_PATH, "r");
211 	if (!fp)
212 		ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
213 
214 	if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))
215 		goto err_out;
216 
217 	/*
218 	 * Fetch the pattern in the same block and check the number of
219 	 * hugepages.
220 	 */
221 	if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer)))
222 		goto err_out;
223 
224 	snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern);
225 
226 	if (sscanf(buffer, addr_pattern, &thp) != 1)
227 		ksft_exit_fail_msg("Reading smap error\n");
228 
229 err_out:
230 	fclose(fp);
231 	return thp == (nr_hpages * (hpage_size >> 10));
232 }
233 
234 bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)
235 {
236 	return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
237 }
238 
239 bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)
240 {
241 	return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
242 }
243 
244 bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)
245 {
246 	return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
247 }
248 
249 int64_t allocate_transhuge(void *ptr, int pagemap_fd)
250 {
251 	uint64_t ent[2];
252 
253 	/* drop pmd */
254 	if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
255 		 MAP_FIXED | MAP_ANONYMOUS |
256 		 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
257 		ksft_exit_fail_msg("mmap transhuge\n");
258 
259 	if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
260 		ksft_exit_fail_msg("MADV_HUGEPAGE\n");
261 
262 	/* allocate transparent huge page */
263 	*(volatile void **)ptr = ptr;
264 
265 	if (pread(pagemap_fd, ent, sizeof(ent),
266 		  (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent))
267 		ksft_exit_fail_msg("read pagemap\n");
268 
269 	if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
270 	    PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
271 	    !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1)))
272 		return PAGEMAP_PFN(ent[0]);
273 
274 	return -1;
275 }
276 
277 unsigned long default_huge_page_size(void)
278 {
279 	unsigned long hps = 0;
280 	char *line = NULL;
281 	size_t linelen = 0;
282 	FILE *f = fopen("/proc/meminfo", "r");
283 
284 	if (!f)
285 		return 0;
286 	while (getline(&line, &linelen, f) > 0) {
287 		if (sscanf(line, "Hugepagesize:       %lu kB", &hps) == 1) {
288 			hps <<= 10;
289 			break;
290 		}
291 	}
292 
293 	free(line);
294 	fclose(f);
295 	return hps;
296 }
297 
298 int detect_hugetlb_page_sizes(size_t sizes[], int max)
299 {
300 	DIR *dir = opendir("/sys/kernel/mm/hugepages/");
301 	int count = 0;
302 
303 	if (!dir)
304 		return 0;
305 
306 	while (count < max) {
307 		struct dirent *entry = readdir(dir);
308 		size_t kb;
309 
310 		if (!entry)
311 			break;
312 		if (entry->d_type != DT_DIR)
313 			continue;
314 		if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1)
315 			continue;
316 		sizes[count++] = kb * 1024;
317 		ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n",
318 			       kb);
319 	}
320 	closedir(dir);
321 	return count;
322 }
323 
324 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */
325 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,
326 			      bool miss, bool wp, bool minor, uint64_t *ioctls)
327 {
328 	struct uffdio_register uffdio_register = { 0 };
329 	uint64_t mode = 0;
330 	int ret = 0;
331 
332 	if (miss)
333 		mode |= UFFDIO_REGISTER_MODE_MISSING;
334 	if (wp)
335 		mode |= UFFDIO_REGISTER_MODE_WP;
336 	if (minor)
337 		mode |= UFFDIO_REGISTER_MODE_MINOR;
338 
339 	uffdio_register.range.start = (unsigned long)addr;
340 	uffdio_register.range.len = len;
341 	uffdio_register.mode = mode;
342 
343 	if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)
344 		ret = -errno;
345 	else if (ioctls)
346 		*ioctls = uffdio_register.ioctls;
347 
348 	return ret;
349 }
350 
351 int uffd_register(int uffd, void *addr, uint64_t len,
352 		  bool miss, bool wp, bool minor)
353 {
354 	return uffd_register_with_ioctls(uffd, addr, len,
355 					 miss, wp, minor, NULL);
356 }
357 
358 int uffd_unregister(int uffd, void *addr, uint64_t len)
359 {
360 	struct uffdio_range range = { .start = (uintptr_t)addr, .len = len };
361 	int ret = 0;
362 
363 	if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1)
364 		ret = -errno;
365 
366 	return ret;
367 }
368 
369 unsigned long get_free_hugepages(void)
370 {
371 	unsigned long fhp = 0;
372 	char *line = NULL;
373 	size_t linelen = 0;
374 	FILE *f = fopen("/proc/meminfo", "r");
375 
376 	if (!f)
377 		return fhp;
378 	while (getline(&line, &linelen, f) > 0) {
379 		if (sscanf(line, "HugePages_Free:      %lu", &fhp) == 1)
380 			break;
381 	}
382 
383 	free(line);
384 	fclose(f);
385 	return fhp;
386 }
387