1 // SPDX-License-Identifier: GPL-2.0
2 #include <string.h>
3 #include <errno.h>
4 #include <fcntl.h>
5 #include <inttypes.h>
6 #include <sys/ioctl.h>
7 #include <linux/userfaultfd.h>
8 #include <linux/fs.h>
9 #include <sys/syscall.h>
10 #include <unistd.h>
11 #include "kselftest.h"
12 #include "vm_util.h"
13
14 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
15 #define SMAP_FILE_PATH "/proc/self/smaps"
16 #define STATUS_FILE_PATH "/proc/self/status"
17 #define MAX_LINE_LENGTH 500
18 #define PAGEMAP_PATH "/proc/self/pagemap"
19 #define KPAGEFLAGS_PATH "/proc/kpageflags"
20 #define MAX_NR_ORDERS 20
21
22 unsigned int __page_size;
23 unsigned int __page_shift;
24
pagemap_get_entry(int fd,char * start)25 uint64_t pagemap_get_entry(int fd, char *start)
26 {
27 const unsigned long pfn = (unsigned long)start / getpagesize();
28 uint64_t entry;
29 int ret;
30
31 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
32 if (ret != sizeof(entry))
33 ksft_exit_fail_msg("reading pagemap failed\n");
34 return entry;
35 }
36
__pagemap_scan_get_categories(int fd,char * start,struct page_region * r)37 static int __pagemap_scan_get_categories(int fd, char *start, struct page_region *r)
38 {
39 struct pm_scan_arg arg;
40
41 arg.start = (uintptr_t)start;
42 arg.end = (uintptr_t)(start + psize());
43 arg.vec = (uintptr_t)r;
44 arg.vec_len = 1;
45 arg.flags = 0;
46 arg.size = sizeof(struct pm_scan_arg);
47 arg.max_pages = 0;
48 arg.category_inverted = 0;
49 arg.category_mask = 0;
50 arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE |
51 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |
52 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY;
53 arg.return_mask = arg.category_anyof_mask;
54
55 return ioctl(fd, PAGEMAP_SCAN, &arg);
56 }
57
pagemap_scan_get_categories(int fd,char * start)58 static uint64_t pagemap_scan_get_categories(int fd, char *start)
59 {
60 struct page_region r;
61 int ret;
62
63 ret = __pagemap_scan_get_categories(fd, start, &r);
64 if (ret < 0)
65 ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno));
66 if (ret == 0)
67 return 0;
68 return r.categories;
69 }
70
71 /* `start` is any valid address. */
pagemap_scan_supported(int fd,char * start)72 static bool pagemap_scan_supported(int fd, char *start)
73 {
74 static int supported = -1;
75 int ret;
76
77 if (supported != -1)
78 return supported;
79
80 /* Provide an invalid address in order to trigger EFAULT. */
81 ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL);
82 if (ret == 0)
83 ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n");
84
85 supported = errno == EFAULT;
86
87 return supported;
88 }
89
page_entry_is(int fd,char * start,char * desc,uint64_t pagemap_flags,uint64_t pagescan_flags)90 static bool page_entry_is(int fd, char *start, char *desc,
91 uint64_t pagemap_flags, uint64_t pagescan_flags)
92 {
93 bool m = pagemap_get_entry(fd, start) & pagemap_flags;
94
95 if (pagemap_scan_supported(fd, start)) {
96 bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags;
97
98 if (m == s)
99 return m;
100
101 ksft_exit_fail_msg(
102 "read and ioctl return unmatched results for %s: %d %d", desc, m, s);
103 }
104 return m;
105 }
106
pagemap_is_softdirty(int fd,char * start)107 bool pagemap_is_softdirty(int fd, char *start)
108 {
109 return page_entry_is(fd, start, "soft-dirty",
110 PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY);
111 }
112
pagemap_is_swapped(int fd,char * start)113 bool pagemap_is_swapped(int fd, char *start)
114 {
115 return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED);
116 }
117
pagemap_is_populated(int fd,char * start)118 bool pagemap_is_populated(int fd, char *start)
119 {
120 return page_entry_is(fd, start, "populated",
121 PM_PRESENT | PM_SWAP,
122 PAGE_IS_PRESENT | PAGE_IS_SWAPPED);
123 }
124
pagemap_get_pfn(int fd,char * start)125 unsigned long pagemap_get_pfn(int fd, char *start)
126 {
127 uint64_t entry = pagemap_get_entry(fd, start);
128
129 /* If present (63th bit), PFN is at bit 0 -- 54. */
130 if (entry & PM_PRESENT)
131 return entry & 0x007fffffffffffffull;
132 return -1ul;
133 }
134
clear_softdirty(void)135 void clear_softdirty(void)
136 {
137 int ret;
138 const char *ctrl = "4";
139 int fd = open("/proc/self/clear_refs", O_WRONLY);
140
141 if (fd < 0)
142 ksft_exit_fail_msg("opening clear_refs failed\n");
143 ret = write(fd, ctrl, strlen(ctrl));
144 close(fd);
145 if (ret != (signed int)strlen(ctrl))
146 ksft_exit_fail_msg("writing clear_refs failed\n");
147 }
148
check_for_pattern(FILE * fp,const char * pattern,char * buf,size_t len)149 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
150 {
151 while (fgets(buf, len, fp)) {
152 if (!strncmp(buf, pattern, strlen(pattern)))
153 return true;
154 }
155 return false;
156 }
157
read_pmd_pagesize(void)158 uint64_t read_pmd_pagesize(void)
159 {
160 int fd;
161 char buf[20];
162 ssize_t num_read;
163
164 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);
165 if (fd == -1)
166 return 0;
167
168 num_read = read(fd, buf, 19);
169 if (num_read < 1) {
170 close(fd);
171 return 0;
172 }
173 buf[num_read] = '\0';
174 close(fd);
175
176 return strtoul(buf, NULL, 10);
177 }
178
rss_anon(void)179 unsigned long rss_anon(void)
180 {
181 unsigned long rss_anon = 0;
182 FILE *fp;
183 char buffer[MAX_LINE_LENGTH];
184
185 fp = fopen(STATUS_FILE_PATH, "r");
186 if (!fp)
187 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH);
188
189 if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer)))
190 goto err_out;
191
192 if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1)
193 ksft_exit_fail_msg("Reading status error\n");
194
195 err_out:
196 fclose(fp);
197 return rss_anon;
198 }
199
vaddr_pageflags_get(char * vaddr,int pagemap_fd,int kpageflags_fd,uint64_t * flags)200 static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd,
201 uint64_t *flags)
202 {
203 unsigned long pfn;
204
205 pfn = pagemap_get_pfn(pagemap_fd, vaddr);
206
207 /* non-present PFN */
208 if (pfn == -1UL)
209 return 1;
210
211 if (pageflags_get(pfn, kpageflags_fd, flags))
212 return -1;
213
214 return 0;
215 }
216
217 /*
218 * gather_folio_orders - scan through [vaddr_start, len) and record
219 * folio orders
220 *
221 * @vaddr_start: start vaddr
222 * @len: range length
223 * @pagemap_fd: file descriptor to /proc/<pid>/pagemap
224 * @kpageflags_fd: file descriptor to /proc/kpageflags
225 * @orders: output folio order array
226 * @nr_orders: folio order array size
227 *
228 * gather_folio_orders() scan through [vaddr_start, len) and check
229 * all folios within the range and record their orders. All order-0 pages will
230 * be recorded. Non-present vaddr is skipped.
231 *
232 * Return: 0 - no error, -1 - unhandled cases
233 */
gather_folio_orders(char * vaddr_start,size_t len,int pagemap_fd,int kpageflags_fd,int orders[],int nr_orders)234 int gather_folio_orders(char *vaddr_start, size_t len,
235 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders)
236 {
237 uint64_t page_flags = 0;
238 int cur_order = -1;
239 char *vaddr;
240
241 if (pagemap_fd == -1 || kpageflags_fd == -1)
242 return -1;
243 if (!orders)
244 return -1;
245 if (nr_orders <= 0)
246 return -1;
247
248 for (vaddr = vaddr_start; vaddr < vaddr_start + len;) {
249 char *next_folio_vaddr;
250 int status;
251
252 status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd,
253 &page_flags);
254 if (status < 0)
255 return -1;
256
257 /* skip non present vaddr */
258 if (status == 1) {
259 vaddr += psize();
260 continue;
261 }
262
263 /* all order-0 pages with possible false postive (non folio) */
264 if (!(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) {
265 orders[0]++;
266 vaddr += psize();
267 continue;
268 }
269
270 /* skip non thp compound pages */
271 if (!(page_flags & KPF_THP)) {
272 vaddr += psize();
273 continue;
274 }
275
276 /* vpn points to part of a THP at this point */
277 if (page_flags & KPF_COMPOUND_HEAD)
278 cur_order = 1;
279 else {
280 vaddr += psize();
281 continue;
282 }
283
284 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift()));
285
286 if (next_folio_vaddr >= vaddr_start + len)
287 break;
288
289 while ((status = vaddr_pageflags_get(next_folio_vaddr,
290 pagemap_fd, kpageflags_fd,
291 &page_flags)) >= 0) {
292 /*
293 * non present vaddr, next compound head page, or
294 * order-0 page
295 */
296 if (status == 1 ||
297 (page_flags & KPF_COMPOUND_HEAD) ||
298 !(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) {
299 if (cur_order < nr_orders) {
300 orders[cur_order]++;
301 cur_order = -1;
302 vaddr = next_folio_vaddr;
303 }
304 break;
305 }
306
307 cur_order++;
308 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift()));
309 }
310
311 if (status < 0)
312 return status;
313 }
314 if (cur_order > 0 && cur_order < nr_orders)
315 orders[cur_order]++;
316 return 0;
317 }
318
__get_smap_entry(void * addr,const char * pattern,char * buf,size_t len)319 char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len)
320 {
321 int ret;
322 FILE *fp;
323 char *entry = NULL;
324 char addr_pattern[MAX_LINE_LENGTH];
325
326 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
327 (unsigned long) addr);
328 if (ret >= MAX_LINE_LENGTH)
329 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
330
331 fp = fopen(SMAP_FILE_PATH, "r");
332 if (!fp)
333 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
334
335 if (!check_for_pattern(fp, addr_pattern, buf, len))
336 goto err_out;
337
338 /* Fetch the pattern in the same block */
339 if (!check_for_pattern(fp, pattern, buf, len))
340 goto err_out;
341
342 /* Trim trailing newline */
343 entry = strchr(buf, '\n');
344 if (entry)
345 *entry = '\0';
346
347 entry = buf + strlen(pattern);
348
349 err_out:
350 fclose(fp);
351 return entry;
352 }
353
__check_pmd_huge(void * addr,char * pattern,int nr_hpages,uint64_t hpage_size)354 static bool __check_pmd_huge(void *addr, char *pattern, int nr_hpages,
355 uint64_t hpage_size)
356 {
357 char buffer[MAX_LINE_LENGTH];
358 uint64_t thp = -1;
359 char *entry;
360
361 entry = __get_smap_entry(addr, pattern, buffer, sizeof(buffer));
362 if (!entry)
363 goto err_out;
364
365 if (sscanf(entry, "%9" SCNu64 " kB", &thp) != 1)
366 ksft_exit_fail_msg("Reading smap error\n");
367
368 err_out:
369 return thp == (nr_hpages * (hpage_size >> 10));
370 }
371
check_large_folios(void * addr,size_t len,int nr_hpages,uint64_t hpage_size)372 static bool check_large_folios(void *addr, size_t len, int nr_hpages,
373 uint64_t hpage_size)
374 {
375 int order = 0, pagesize = getpagesize();
376 unsigned int nr_pages = hpage_size / pagesize;
377 int orders[MAX_NR_ORDERS], status;
378 int pagemap_fd, kpageflags_fd;
379 bool ret = false;
380
381 if (!nr_pages)
382 ksft_exit_fail_msg("invalid hugepage size\n");
383
384 order = 31 - __builtin_clz(nr_pages);
385 if (!order || order >= MAX_NR_ORDERS)
386 ksft_exit_fail_msg("invalid order\n");
387
388 memset(orders, 0, sizeof(int) * MAX_NR_ORDERS);
389 pagemap_fd = open(PAGEMAP_PATH, O_RDONLY);
390 if (pagemap_fd == -1)
391 ksft_exit_fail_msg("read pagemap fail\n");
392
393 kpageflags_fd = open(KPAGEFLAGS_PATH, O_RDONLY);
394 if (kpageflags_fd == -1) {
395 close(pagemap_fd);
396 ksft_exit_fail_msg("read kpageflags fail\n");
397 }
398
399 status = gather_folio_orders(addr, len, pagemap_fd,
400 kpageflags_fd, orders, MAX_NR_ORDERS);
401 if (status)
402 goto out;
403
404 if (orders[order] == nr_hpages)
405 ret = true;
406
407 out:
408 close(pagemap_fd);
409 close(kpageflags_fd);
410 return ret;
411 }
412
check_huge_anon(void * addr,size_t len,int nr_hpages,uint64_t hpage_size)413 bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size)
414 {
415 uint64_t pmd_pagesize = read_pmd_pagesize();
416
417 if (!pmd_pagesize)
418 ksft_exit_fail_msg("reading PMD pagesize failed\n");
419
420 if (hpage_size == pmd_pagesize)
421 return __check_pmd_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
422
423 return check_large_folios(addr, len, nr_hpages, hpage_size);
424 }
425
check_huge_file(void * addr,size_t len,int nr_hpages,uint64_t hpage_size)426 bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size)
427 {
428 uint64_t pmd_pagesize = read_pmd_pagesize();
429
430 if (!pmd_pagesize)
431 ksft_exit_fail_msg("reading PMD pagesize failed\n");
432
433 if (hpage_size == pmd_pagesize)
434 return __check_pmd_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
435
436 return check_large_folios(addr, len, nr_hpages, hpage_size);
437 }
438
check_huge_shmem(void * addr,size_t len,int nr_hpages,uint64_t hpage_size)439 bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size)
440 {
441 uint64_t pmd_pagesize = read_pmd_pagesize();
442
443 if (!pmd_pagesize)
444 ksft_exit_fail_msg("reading PMD pagesize failed\n");
445
446 if (hpage_size == pmd_pagesize)
447 return __check_pmd_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
448
449 return check_large_folios(addr, len, nr_hpages, hpage_size);
450 }
451
allocate_transhuge(void * ptr,int pagemap_fd)452 int64_t allocate_transhuge(void *ptr, int pagemap_fd)
453 {
454 uint64_t ent[2];
455
456 /* drop pmd */
457 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
458 MAP_FIXED | MAP_ANONYMOUS |
459 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
460 ksft_exit_fail_msg("mmap transhuge\n");
461
462 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
463 ksft_exit_fail_msg("MADV_HUGEPAGE\n");
464
465 /* allocate transparent huge page */
466 *(volatile void **)ptr = ptr;
467
468 if (pread(pagemap_fd, ent, sizeof(ent),
469 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent))
470 ksft_exit_fail_msg("read pagemap\n");
471
472 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
473 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
474 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1)))
475 return PAGEMAP_PFN(ent[0]);
476
477 return -1;
478 }
479
pageflags_get(unsigned long pfn,int kpageflags_fd,uint64_t * flags)480 int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags)
481 {
482 size_t count;
483
484 count = pread(kpageflags_fd, flags, sizeof(*flags),
485 pfn * sizeof(*flags));
486
487 if (count != sizeof(*flags))
488 return -1;
489
490 return 0;
491 }
492
493 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */
uffd_register_with_ioctls(int uffd,void * addr,uint64_t len,bool miss,bool wp,bool minor,uint64_t * ioctls)494 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,
495 bool miss, bool wp, bool minor, uint64_t *ioctls)
496 {
497 struct uffdio_register uffdio_register = { 0 };
498 uint64_t mode = 0;
499 int ret = 0;
500
501 if (miss)
502 mode |= UFFDIO_REGISTER_MODE_MISSING;
503 if (wp)
504 mode |= UFFDIO_REGISTER_MODE_WP;
505 if (minor)
506 mode |= UFFDIO_REGISTER_MODE_MINOR;
507
508 uffdio_register.range.start = (unsigned long)addr;
509 uffdio_register.range.len = len;
510 uffdio_register.mode = mode;
511
512 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)
513 ret = -errno;
514 else if (ioctls)
515 *ioctls = uffdio_register.ioctls;
516
517 return ret;
518 }
519
uffd_register(int uffd,void * addr,uint64_t len,bool miss,bool wp,bool minor)520 int uffd_register(int uffd, void *addr, uint64_t len,
521 bool miss, bool wp, bool minor)
522 {
523 return uffd_register_with_ioctls(uffd, addr, len,
524 miss, wp, minor, NULL);
525 }
526
uffd_unregister(int uffd,void * addr,uint64_t len)527 int uffd_unregister(int uffd, void *addr, uint64_t len)
528 {
529 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len };
530 int ret = 0;
531
532 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1)
533 ret = -errno;
534
535 return ret;
536 }
537
check_vmflag(void * addr,const char * flag)538 static bool check_vmflag(void *addr, const char *flag)
539 {
540 char buffer[MAX_LINE_LENGTH];
541 const char *flags;
542 size_t flaglen;
543
544 flags = __get_smap_entry(addr, "VmFlags:", buffer, sizeof(buffer));
545 if (!flags)
546 ksft_exit_fail_msg("%s: No VmFlags for %p\n", __func__, addr);
547
548 while (true) {
549 flags += strspn(flags, " ");
550
551 flaglen = strcspn(flags, " ");
552 if (!flaglen)
553 return false;
554
555 if (flaglen == strlen(flag) && !memcmp(flags, flag, flaglen))
556 return true;
557
558 flags += flaglen;
559 }
560 }
561
check_vmflag_io(void * addr)562 bool check_vmflag_io(void *addr)
563 {
564 return check_vmflag(addr, "io");
565 }
566
check_vmflag_pfnmap(void * addr)567 bool check_vmflag_pfnmap(void *addr)
568 {
569 return check_vmflag(addr, "pf");
570 }
571
check_vmflag_guard(void * addr)572 bool check_vmflag_guard(void *addr)
573 {
574 return check_vmflag(addr, "gu");
575 }
576
softdirty_supported(void)577 bool softdirty_supported(void)
578 {
579 char *addr;
580 bool supported = false;
581 const size_t pagesize = getpagesize();
582
583 /* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */
584 addr = mmap(0, pagesize, PROT_READ | PROT_WRITE,
585 MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
586 if (addr == MAP_FAILED)
587 ksft_exit_fail_msg("mmap failed\n");
588
589 supported = check_vmflag(addr, "sd");
590 munmap(addr, pagesize);
591 return supported;
592 }
593
594 /*
595 * Open an fd at /proc/$pid/maps and configure procmap_out ready for
596 * PROCMAP_QUERY query. Returns 0 on success, or an error code otherwise.
597 */
open_procmap(pid_t pid,struct procmap_fd * procmap_out)598 int open_procmap(pid_t pid, struct procmap_fd *procmap_out)
599 {
600 char path[256];
601 int ret = 0;
602
603 memset(procmap_out, '\0', sizeof(*procmap_out));
604 sprintf(path, "/proc/%d/maps", pid);
605 procmap_out->query.size = sizeof(procmap_out->query);
606 procmap_out->fd = open(path, O_RDONLY);
607 if (procmap_out->fd < 0)
608 ret = -errno;
609
610 return ret;
611 }
612
613 /* Perform PROCMAP_QUERY. Returns 0 on success, or an error code otherwise. */
query_procmap(struct procmap_fd * procmap)614 int query_procmap(struct procmap_fd *procmap)
615 {
616 int ret = 0;
617
618 if (ioctl(procmap->fd, PROCMAP_QUERY, &procmap->query) == -1)
619 ret = -errno;
620
621 return ret;
622 }
623
624 /*
625 * Try to find the VMA at specified address, returns true if found, false if not
626 * found, and the test is failed if any other error occurs.
627 *
628 * On success, procmap->query is populated with the results.
629 */
find_vma_procmap(struct procmap_fd * procmap,void * address)630 bool find_vma_procmap(struct procmap_fd *procmap, void *address)
631 {
632 int err;
633
634 procmap->query.query_flags = 0;
635 procmap->query.query_addr = (unsigned long)address;
636 err = query_procmap(procmap);
637 if (!err)
638 return true;
639
640 if (err != -ENOENT)
641 ksft_exit_fail_msg("%s: Error %d on ioctl(PROCMAP_QUERY)\n",
642 __func__, err);
643 return false;
644 }
645
646 /*
647 * Close fd used by PROCMAP_QUERY mechanism. Returns 0 on success, or an error
648 * code otherwise.
649 */
close_procmap(struct procmap_fd * procmap)650 int close_procmap(struct procmap_fd *procmap)
651 {
652 return close(procmap->fd);
653 }
654
write_sysfs(const char * file_path,unsigned long val)655 int write_sysfs(const char *file_path, unsigned long val)
656 {
657 FILE *f = fopen(file_path, "w");
658
659 if (!f) {
660 fprintf(stderr, "f %s\n", file_path);
661 perror("fopen");
662 return 1;
663 }
664 if (fprintf(f, "%lu", val) < 0) {
665 perror("fprintf");
666 fclose(f);
667 return 1;
668 }
669 fclose(f);
670
671 return 0;
672 }
673
read_sysfs(const char * file_path,unsigned long * val)674 int read_sysfs(const char *file_path, unsigned long *val)
675 {
676 FILE *f = fopen(file_path, "r");
677
678 if (!f) {
679 fprintf(stderr, "f %s\n", file_path);
680 perror("fopen");
681 return 1;
682 }
683 if (fscanf(f, "%lu", val) != 1) {
684 perror("fscanf");
685 fclose(f);
686 return 1;
687 }
688 fclose(f);
689
690 return 0;
691 }
692
sys_mremap(void * old_address,unsigned long old_size,unsigned long new_size,int flags,void * new_address)693 void *sys_mremap(void *old_address, unsigned long old_size,
694 unsigned long new_size, int flags, void *new_address)
695 {
696 return (void *)syscall(__NR_mremap, (unsigned long)old_address,
697 old_size, new_size, flags,
698 (unsigned long)new_address);
699 }
700
detect_huge_zeropage(void)701 bool detect_huge_zeropage(void)
702 {
703 int fd = open("/sys/kernel/mm/transparent_hugepage/use_zero_page",
704 O_RDONLY);
705 bool enabled = 0;
706 char buf[15];
707 int ret;
708
709 if (fd < 0)
710 return 0;
711
712 ret = pread(fd, buf, sizeof(buf), 0);
713 if (ret > 0 && ret < sizeof(buf)) {
714 buf[ret] = 0;
715
716 if (strtoul(buf, NULL, 10) == 1)
717 enabled = 1;
718 }
719
720 close(fd);
721 return enabled;
722 }
723
ksm_get_self_zero_pages(void)724 long ksm_get_self_zero_pages(void)
725 {
726 int proc_self_ksm_stat_fd;
727 char buf[200];
728 char *substr_ksm_zero;
729 size_t value_pos;
730 ssize_t read_size;
731
732 proc_self_ksm_stat_fd = open("/proc/self/ksm_stat", O_RDONLY);
733 if (proc_self_ksm_stat_fd < 0)
734 return -errno;
735
736 read_size = pread(proc_self_ksm_stat_fd, buf, sizeof(buf) - 1, 0);
737 close(proc_self_ksm_stat_fd);
738 if (read_size < 0)
739 return -errno;
740
741 buf[read_size] = 0;
742
743 substr_ksm_zero = strstr(buf, "ksm_zero_pages");
744 if (!substr_ksm_zero)
745 return 0;
746
747 value_pos = strcspn(substr_ksm_zero, "0123456789");
748 return strtol(substr_ksm_zero + value_pos, NULL, 10);
749 }
750
ksm_get_self_merging_pages(void)751 long ksm_get_self_merging_pages(void)
752 {
753 int proc_self_ksm_merging_pages_fd;
754 char buf[10];
755 ssize_t ret;
756
757 proc_self_ksm_merging_pages_fd = open("/proc/self/ksm_merging_pages",
758 O_RDONLY);
759 if (proc_self_ksm_merging_pages_fd < 0)
760 return -errno;
761
762 ret = pread(proc_self_ksm_merging_pages_fd, buf, sizeof(buf) - 1, 0);
763 close(proc_self_ksm_merging_pages_fd);
764 if (ret <= 0)
765 return -errno;
766 buf[ret] = 0;
767
768 return strtol(buf, NULL, 10);
769 }
770
ksm_get_full_scans(void)771 long ksm_get_full_scans(void)
772 {
773 int ksm_full_scans_fd;
774 char buf[10];
775 ssize_t ret;
776
777 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY);
778 if (ksm_full_scans_fd < 0)
779 return -errno;
780
781 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0);
782 close(ksm_full_scans_fd);
783 if (ret <= 0)
784 return -errno;
785 buf[ret] = 0;
786
787 return strtol(buf, NULL, 10);
788 }
789
ksm_use_zero_pages(void)790 int ksm_use_zero_pages(void)
791 {
792 int ksm_use_zero_pages_fd;
793 ssize_t ret;
794
795 ksm_use_zero_pages_fd = open("/sys/kernel/mm/ksm/use_zero_pages", O_RDWR);
796 if (ksm_use_zero_pages_fd < 0)
797 return -errno;
798
799 ret = write(ksm_use_zero_pages_fd, "1", 1);
800 close(ksm_use_zero_pages_fd);
801 return ret == 1 ? 0 : -errno;
802 }
803
ksm_start(void)804 int ksm_start(void)
805 {
806 int ksm_fd;
807 ssize_t ret;
808 long start_scans, end_scans;
809
810 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR);
811 if (ksm_fd < 0)
812 return -errno;
813
814 /* Wait for two full scans such that any possible merging happened. */
815 start_scans = ksm_get_full_scans();
816 if (start_scans < 0) {
817 close(ksm_fd);
818 return start_scans;
819 }
820 ret = write(ksm_fd, "1", 1);
821 close(ksm_fd);
822 if (ret != 1)
823 return -errno;
824 do {
825 end_scans = ksm_get_full_scans();
826 if (end_scans < 0)
827 return end_scans;
828 } while (end_scans < start_scans + 2);
829
830 return 0;
831 }
832
ksm_stop(void)833 int ksm_stop(void)
834 {
835 int ksm_fd;
836 ssize_t ret;
837
838 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR);
839 if (ksm_fd < 0)
840 return -errno;
841
842 ret = write(ksm_fd, "2", 1);
843 close(ksm_fd);
844 return ret == 1 ? 0 : -errno;
845 }
846
get_hardware_corrupted_size(unsigned long * val)847 int get_hardware_corrupted_size(unsigned long *val)
848 {
849 unsigned long size;
850 char *line = NULL;
851 size_t linelen = 0;
852 FILE *f = fopen("/proc/meminfo", "r");
853 int ret = -1;
854
855 if (!f)
856 return ret;
857
858 while (getline(&line, &linelen, f) > 0) {
859 if (sscanf(line, "HardwareCorrupted: %12lu kB", &size) == 1) {
860 *val = size;
861 ret = 0;
862 break;
863 }
864 }
865
866 free(line);
867 fclose(f);
868 return ret;
869 }
870
unpoison_memory(unsigned long pfn)871 int unpoison_memory(unsigned long pfn)
872 {
873 int unpoison_fd, len;
874 char buf[32];
875 ssize_t ret;
876
877 unpoison_fd = open("/sys/kernel/debug/hwpoison/unpoison-pfn", O_WRONLY);
878 if (unpoison_fd < 0)
879 return -errno;
880
881 len = sprintf(buf, "0x%lx\n", pfn);
882 ret = write(unpoison_fd, buf, len);
883 close(unpoison_fd);
884
885 return ret > 0 ? 0 : -errno;
886 }
887
read_file(const char * path,char * buf,size_t buflen)888 int read_file(const char *path, char *buf, size_t buflen)
889 {
890 int fd;
891 ssize_t numread;
892
893 fd = open(path, O_RDONLY);
894 if (fd == -1)
895 return 0;
896
897 numread = read(fd, buf, buflen - 1);
898 if (numread < 1) {
899 close(fd);
900 return 0;
901 }
902
903 buf[numread] = '\0';
904 close(fd);
905
906 return (unsigned int) numread;
907 }
908
__write_file(const char * path,const char * buf,size_t buflen,bool ignore_einval)909 static void __write_file(const char *path, const char *buf, size_t buflen, bool ignore_einval)
910 {
911 int fd, saved_errno;
912 ssize_t numwritten;
913
914 if (buflen < 2)
915 ksft_exit_fail_msg("Incorrect buffer len: %zu\n", buflen);
916
917 fd = open(path, O_WRONLY);
918 if (fd == -1)
919 ksft_exit_fail_msg("%s open failed: %s\n", path, strerror(errno));
920
921 numwritten = write(fd, buf, buflen - 1);
922 saved_errno = errno;
923 close(fd);
924 errno = saved_errno;
925 if (numwritten < 0) {
926 if (ignore_einval && errno == EINVAL)
927 return;
928 ksft_exit_fail_msg("%s write(%.*s) failed: %s\n", path, (int)(buflen - 1),
929 buf, strerror(errno));
930 }
931 if (numwritten != buflen - 1)
932 ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n",
933 path, (int)(buflen - 1), buf, buflen - 1, numwritten);
934 }
935
write_file(const char * path,const char * buf,size_t buflen)936 void write_file(const char *path, const char *buf, size_t buflen)
937 {
938 __write_file(path, buf, buflen, /* ignore_einval = */ false);
939 }
940
read_num(const char * path)941 unsigned long read_num(const char *path)
942 {
943 char buf[21];
944
945 if (!read_file(path, buf, sizeof(buf)))
946 ksft_exit_fail_perror("read_file()");
947
948 return strtoul(buf, NULL, 10);
949 }
950
__write_num(const char * path,unsigned long num,bool ignore_einval)951 static void __write_num(const char *path, unsigned long num, bool ignore_einval)
952 {
953 char buf[21];
954
955 sprintf(buf, "%lu", num);
956 __write_file(path, buf, strlen(buf) + 1, ignore_einval);
957 }
958
write_num(const char * path,unsigned long num)959 void write_num(const char *path, unsigned long num)
960 {
961 return __write_num(path, num, /* ignore_einval = */ false);
962 }
963
write_num_ignore_einval(const char * path,unsigned long num)964 void write_num_ignore_einval(const char *path, unsigned long num)
965 {
966 return __write_num(path, num, /* ignore_einval = */ true);
967 }
968
969 static unsigned long shmall, shmmax;
970
__shm_limits_restore(void)971 void __shm_limits_restore(void)
972 {
973 if (shmmax)
974 write_num("/proc/sys/kernel/shmmax", shmmax);
975 if (shmall)
976 write_num("/proc/sys/kernel/shmall", shmall);
977 }
978
shm_limits_prepare(unsigned long length)979 void shm_limits_prepare(unsigned long length)
980 {
981 unsigned long nr = length / psize();
982 unsigned long val;
983
984 val = read_num("/proc/sys/kernel/shmmax");
985 if (val < length) {
986 write_num("/proc/sys/kernel/shmmax", length);
987 shmmax = val;
988 }
989
990 val = read_num("/proc/sys/kernel/shmall");
991 if (val < nr) {
992 write_num("/proc/sys/kernel/shmall", nr);
993 shmall = val;
994 }
995 }
996