xref: /linux/tools/testing/selftests/mm/vm_util.c (revision 9c5968db9e625019a0ee5226c7eebef5519d366a)
1 // SPDX-License-Identifier: GPL-2.0
2 #include <string.h>
3 #include <fcntl.h>
4 #include <dirent.h>
5 #include <inttypes.h>
6 #include <sys/ioctl.h>
7 #include <linux/userfaultfd.h>
8 #include <linux/fs.h>
9 #include <sys/syscall.h>
10 #include <unistd.h>
11 #include "../kselftest.h"
12 #include "vm_util.h"
13 
14 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
15 #define SMAP_FILE_PATH "/proc/self/smaps"
16 #define STATUS_FILE_PATH "/proc/self/status"
17 #define MAX_LINE_LENGTH 500
18 
19 unsigned int __page_size;
20 unsigned int __page_shift;
21 
22 uint64_t pagemap_get_entry(int fd, char *start)
23 {
24 	const unsigned long pfn = (unsigned long)start / getpagesize();
25 	uint64_t entry;
26 	int ret;
27 
28 	ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
29 	if (ret != sizeof(entry))
30 		ksft_exit_fail_msg("reading pagemap failed\n");
31 	return entry;
32 }
33 
34 static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r)
35 {
36 	struct pm_scan_arg arg;
37 
38 	arg.start = (uintptr_t)start;
39 	arg.end = (uintptr_t)(start + psize());
40 	arg.vec = (uintptr_t)r;
41 	arg.vec_len = 1;
42 	arg.flags = 0;
43 	arg.size = sizeof(struct pm_scan_arg);
44 	arg.max_pages = 0;
45 	arg.category_inverted = 0;
46 	arg.category_mask = 0;
47 	arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE |
48 				  PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |
49 				  PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY;
50 	arg.return_mask = arg.category_anyof_mask;
51 
52 	return ioctl(fd, PAGEMAP_SCAN, &arg);
53 }
54 
55 static uint64_t pagemap_scan_get_categories(int fd, char *start)
56 {
57 	struct page_region r;
58 	long ret;
59 
60 	ret = __pagemap_scan_get_categories(fd, start, &r);
61 	if (ret < 0)
62 		ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno));
63 	if (ret == 0)
64 		return 0;
65 	return r.categories;
66 }
67 
68 /* `start` is any valid address. */
69 static bool pagemap_scan_supported(int fd, char *start)
70 {
71 	static int supported = -1;
72 	int ret;
73 
74 	if (supported != -1)
75 		return supported;
76 
77 	/* Provide an invalid address in order to trigger EFAULT. */
78 	ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL);
79 	if (ret == 0)
80 		ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n");
81 
82 	supported = errno == EFAULT;
83 
84 	return supported;
85 }
86 
87 static bool page_entry_is(int fd, char *start, char *desc,
88 			  uint64_t pagemap_flags, uint64_t pagescan_flags)
89 {
90 	bool m = pagemap_get_entry(fd, start) & pagemap_flags;
91 
92 	if (pagemap_scan_supported(fd, start)) {
93 		bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags;
94 
95 		if (m == s)
96 			return m;
97 
98 		ksft_exit_fail_msg(
99 			"read and ioctl return unmatched results for %s: %d %d", desc, m, s);
100 	}
101 	return m;
102 }
103 
104 bool pagemap_is_softdirty(int fd, char *start)
105 {
106 	return page_entry_is(fd, start, "soft-dirty",
107 				PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY);
108 }
109 
110 bool pagemap_is_swapped(int fd, char *start)
111 {
112 	return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED);
113 }
114 
115 bool pagemap_is_populated(int fd, char *start)
116 {
117 	return page_entry_is(fd, start, "populated",
118 				PM_PRESENT | PM_SWAP,
119 				PAGE_IS_PRESENT | PAGE_IS_SWAPPED);
120 }
121 
122 unsigned long pagemap_get_pfn(int fd, char *start)
123 {
124 	uint64_t entry = pagemap_get_entry(fd, start);
125 
126 	/* If present (63th bit), PFN is at bit 0 -- 54. */
127 	if (entry & PM_PRESENT)
128 		return entry & 0x007fffffffffffffull;
129 	return -1ul;
130 }
131 
132 void clear_softdirty(void)
133 {
134 	int ret;
135 	const char *ctrl = "4";
136 	int fd = open("/proc/self/clear_refs", O_WRONLY);
137 
138 	if (fd < 0)
139 		ksft_exit_fail_msg("opening clear_refs failed\n");
140 	ret = write(fd, ctrl, strlen(ctrl));
141 	close(fd);
142 	if (ret != (signed int)strlen(ctrl))
143 		ksft_exit_fail_msg("writing clear_refs failed\n");
144 }
145 
146 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
147 {
148 	while (fgets(buf, len, fp)) {
149 		if (!strncmp(buf, pattern, strlen(pattern)))
150 			return true;
151 	}
152 	return false;
153 }
154 
155 uint64_t read_pmd_pagesize(void)
156 {
157 	int fd;
158 	char buf[20];
159 	ssize_t num_read;
160 
161 	fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);
162 	if (fd == -1)
163 		return 0;
164 
165 	num_read = read(fd, buf, 19);
166 	if (num_read < 1) {
167 		close(fd);
168 		return 0;
169 	}
170 	buf[num_read] = '\0';
171 	close(fd);
172 
173 	return strtoul(buf, NULL, 10);
174 }
175 
176 unsigned long rss_anon(void)
177 {
178 	unsigned long rss_anon = 0;
179 	FILE *fp;
180 	char buffer[MAX_LINE_LENGTH];
181 
182 	fp = fopen(STATUS_FILE_PATH, "r");
183 	if (!fp)
184 		ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH);
185 
186 	if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer)))
187 		goto err_out;
188 
189 	if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1)
190 		ksft_exit_fail_msg("Reading status error\n");
191 
192 err_out:
193 	fclose(fp);
194 	return rss_anon;
195 }
196 
197 char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len)
198 {
199 	int ret;
200 	FILE *fp;
201 	char *entry = NULL;
202 	char addr_pattern[MAX_LINE_LENGTH];
203 
204 	ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
205 		       (unsigned long) addr);
206 	if (ret >= MAX_LINE_LENGTH)
207 		ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
208 
209 	fp = fopen(SMAP_FILE_PATH, "r");
210 	if (!fp)
211 		ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
212 
213 	if (!check_for_pattern(fp, addr_pattern, buf, len))
214 		goto err_out;
215 
216 	/* Fetch the pattern in the same block */
217 	if (!check_for_pattern(fp, pattern, buf, len))
218 		goto err_out;
219 
220 	/* Trim trailing newline */
221 	entry = strchr(buf, '\n');
222 	if (entry)
223 		*entry = '\0';
224 
225 	entry = buf + strlen(pattern);
226 
227 err_out:
228 	fclose(fp);
229 	return entry;
230 }
231 
232 bool __check_huge(void *addr, char *pattern, int nr_hpages,
233 		  uint64_t hpage_size)
234 {
235 	char buffer[MAX_LINE_LENGTH];
236 	uint64_t thp = -1;
237 	char *entry;
238 
239 	entry = __get_smap_entry(addr, pattern, buffer, sizeof(buffer));
240 	if (!entry)
241 		goto err_out;
242 
243 	if (sscanf(entry, "%9" SCNu64 " kB", &thp) != 1)
244 		ksft_exit_fail_msg("Reading smap error\n");
245 
246 err_out:
247 	return thp == (nr_hpages * (hpage_size >> 10));
248 }
249 
250 bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)
251 {
252 	return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
253 }
254 
255 bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)
256 {
257 	return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
258 }
259 
260 bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)
261 {
262 	return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
263 }
264 
265 int64_t allocate_transhuge(void *ptr, int pagemap_fd)
266 {
267 	uint64_t ent[2];
268 
269 	/* drop pmd */
270 	if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
271 		 MAP_FIXED | MAP_ANONYMOUS |
272 		 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
273 		ksft_exit_fail_msg("mmap transhuge\n");
274 
275 	if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
276 		ksft_exit_fail_msg("MADV_HUGEPAGE\n");
277 
278 	/* allocate transparent huge page */
279 	*(volatile void **)ptr = ptr;
280 
281 	if (pread(pagemap_fd, ent, sizeof(ent),
282 		  (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent))
283 		ksft_exit_fail_msg("read pagemap\n");
284 
285 	if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
286 	    PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
287 	    !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1)))
288 		return PAGEMAP_PFN(ent[0]);
289 
290 	return -1;
291 }
292 
293 unsigned long default_huge_page_size(void)
294 {
295 	unsigned long hps = 0;
296 	char *line = NULL;
297 	size_t linelen = 0;
298 	FILE *f = fopen("/proc/meminfo", "r");
299 
300 	if (!f)
301 		return 0;
302 	while (getline(&line, &linelen, f) > 0) {
303 		if (sscanf(line, "Hugepagesize:       %lu kB", &hps) == 1) {
304 			hps <<= 10;
305 			break;
306 		}
307 	}
308 
309 	free(line);
310 	fclose(f);
311 	return hps;
312 }
313 
314 int detect_hugetlb_page_sizes(size_t sizes[], int max)
315 {
316 	DIR *dir = opendir("/sys/kernel/mm/hugepages/");
317 	int count = 0;
318 
319 	if (!dir)
320 		return 0;
321 
322 	while (count < max) {
323 		struct dirent *entry = readdir(dir);
324 		size_t kb;
325 
326 		if (!entry)
327 			break;
328 		if (entry->d_type != DT_DIR)
329 			continue;
330 		if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1)
331 			continue;
332 		sizes[count++] = kb * 1024;
333 		ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n",
334 			       kb);
335 	}
336 	closedir(dir);
337 	return count;
338 }
339 
340 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */
341 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,
342 			      bool miss, bool wp, bool minor, uint64_t *ioctls)
343 {
344 	struct uffdio_register uffdio_register = { 0 };
345 	uint64_t mode = 0;
346 	int ret = 0;
347 
348 	if (miss)
349 		mode |= UFFDIO_REGISTER_MODE_MISSING;
350 	if (wp)
351 		mode |= UFFDIO_REGISTER_MODE_WP;
352 	if (minor)
353 		mode |= UFFDIO_REGISTER_MODE_MINOR;
354 
355 	uffdio_register.range.start = (unsigned long)addr;
356 	uffdio_register.range.len = len;
357 	uffdio_register.mode = mode;
358 
359 	if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)
360 		ret = -errno;
361 	else if (ioctls)
362 		*ioctls = uffdio_register.ioctls;
363 
364 	return ret;
365 }
366 
367 int uffd_register(int uffd, void *addr, uint64_t len,
368 		  bool miss, bool wp, bool minor)
369 {
370 	return uffd_register_with_ioctls(uffd, addr, len,
371 					 miss, wp, minor, NULL);
372 }
373 
374 int uffd_unregister(int uffd, void *addr, uint64_t len)
375 {
376 	struct uffdio_range range = { .start = (uintptr_t)addr, .len = len };
377 	int ret = 0;
378 
379 	if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1)
380 		ret = -errno;
381 
382 	return ret;
383 }
384 
385 unsigned long get_free_hugepages(void)
386 {
387 	unsigned long fhp = 0;
388 	char *line = NULL;
389 	size_t linelen = 0;
390 	FILE *f = fopen("/proc/meminfo", "r");
391 
392 	if (!f)
393 		return fhp;
394 	while (getline(&line, &linelen, f) > 0) {
395 		if (sscanf(line, "HugePages_Free:      %lu", &fhp) == 1)
396 			break;
397 	}
398 
399 	free(line);
400 	fclose(f);
401 	return fhp;
402 }
403 
404 bool check_vmflag_io(void *addr)
405 {
406 	char buffer[MAX_LINE_LENGTH];
407 	const char *flags;
408 	size_t flaglen;
409 
410 	flags = __get_smap_entry(addr, "VmFlags:", buffer, sizeof(buffer));
411 	if (!flags)
412 		ksft_exit_fail_msg("%s: No VmFlags for %p\n", __func__, addr);
413 
414 	while (true) {
415 		flags += strspn(flags, " ");
416 
417 		flaglen = strcspn(flags, " ");
418 		if (!flaglen)
419 			return false;
420 
421 		if (flaglen == strlen("io") && !memcmp(flags, "io", flaglen))
422 			return true;
423 
424 		flags += flaglen;
425 	}
426 }
427