1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KSM functional tests 4 * 5 * Copyright 2022, Red Hat, Inc. 6 * 7 * Author(s): David Hildenbrand <david@redhat.com> 8 */ 9 #define _GNU_SOURCE 10 #include <stdlib.h> 11 #include <string.h> 12 #include <stdbool.h> 13 #include <stdint.h> 14 #include <unistd.h> 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <sys/mman.h> 18 #include <sys/syscall.h> 19 #include <sys/ioctl.h> 20 #include <linux/userfaultfd.h> 21 22 #include "../kselftest.h" 23 #include "vm_util.h" 24 25 #define KiB 1024u 26 #define MiB (1024 * KiB) 27 28 static int ksm_fd; 29 static int ksm_full_scans_fd; 30 static int pagemap_fd; 31 static size_t pagesize; 32 33 static bool range_maps_duplicates(char *addr, unsigned long size) 34 { 35 unsigned long offs_a, offs_b, pfn_a, pfn_b; 36 37 /* 38 * There is no easy way to check if there are KSM pages mapped into 39 * this range. We only check that the range does not map the same PFN 40 * twice by comparing each pair of mapped pages. 41 */ 42 for (offs_a = 0; offs_a < size; offs_a += pagesize) { 43 pfn_a = pagemap_get_pfn(pagemap_fd, addr + offs_a); 44 /* Page not present or PFN not exposed by the kernel. */ 45 if (pfn_a == -1ul || !pfn_a) 46 continue; 47 48 for (offs_b = offs_a + pagesize; offs_b < size; 49 offs_b += pagesize) { 50 pfn_b = pagemap_get_pfn(pagemap_fd, addr + offs_b); 51 if (pfn_b == -1ul || !pfn_b) 52 continue; 53 if (pfn_a == pfn_b) 54 return true; 55 } 56 } 57 return false; 58 } 59 60 static long ksm_get_full_scans(void) 61 { 62 char buf[10]; 63 ssize_t ret; 64 65 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0); 66 if (ret <= 0) 67 return -errno; 68 buf[ret] = 0; 69 70 return strtol(buf, NULL, 10); 71 } 72 73 static int ksm_merge(void) 74 { 75 long start_scans, end_scans; 76 77 /* Wait for two full scans such that any possible merging happened. */ 78 start_scans = ksm_get_full_scans(); 79 if (start_scans < 0) 80 return start_scans; 81 if (write(ksm_fd, "1", 1) != 1) 82 return -errno; 83 do { 84 end_scans = ksm_get_full_scans(); 85 if (end_scans < 0) 86 return end_scans; 87 } while (end_scans < start_scans + 2); 88 89 return 0; 90 } 91 92 static char *mmap_and_merge_range(char val, unsigned long size) 93 { 94 char *map; 95 96 map = mmap(NULL, size, PROT_READ|PROT_WRITE, 97 MAP_PRIVATE|MAP_ANON, -1, 0); 98 if (map == MAP_FAILED) { 99 ksft_test_result_fail("mmap() failed\n"); 100 return MAP_FAILED; 101 } 102 103 /* Don't use THP. Ignore if THP are not around on a kernel. */ 104 if (madvise(map, size, MADV_NOHUGEPAGE) && errno != EINVAL) { 105 ksft_test_result_fail("MADV_NOHUGEPAGE failed\n"); 106 goto unmap; 107 } 108 109 /* Make sure each page contains the same values to merge them. */ 110 memset(map, val, size); 111 if (madvise(map, size, MADV_MERGEABLE)) { 112 ksft_test_result_fail("MADV_MERGEABLE failed\n"); 113 goto unmap; 114 } 115 116 /* Run KSM to trigger merging and wait. */ 117 if (ksm_merge()) { 118 ksft_test_result_fail("Running KSM failed\n"); 119 goto unmap; 120 } 121 return map; 122 unmap: 123 munmap(map, size); 124 return MAP_FAILED; 125 } 126 127 static void test_unmerge(void) 128 { 129 const unsigned int size = 2 * MiB; 130 char *map; 131 132 ksft_print_msg("[RUN] %s\n", __func__); 133 134 map = mmap_and_merge_range(0xcf, size); 135 if (map == MAP_FAILED) 136 return; 137 138 if (madvise(map, size, MADV_UNMERGEABLE)) { 139 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 140 goto unmap; 141 } 142 143 ksft_test_result(!range_maps_duplicates(map, size), 144 "Pages were unmerged\n"); 145 unmap: 146 munmap(map, size); 147 } 148 149 static void test_unmerge_discarded(void) 150 { 151 const unsigned int size = 2 * MiB; 152 char *map; 153 154 ksft_print_msg("[RUN] %s\n", __func__); 155 156 map = mmap_and_merge_range(0xcf, size); 157 if (map == MAP_FAILED) 158 return; 159 160 /* Discard half of all mapped pages so we have pte_none() entries. */ 161 if (madvise(map, size / 2, MADV_DONTNEED)) { 162 ksft_test_result_fail("MADV_DONTNEED failed\n"); 163 goto unmap; 164 } 165 166 if (madvise(map, size, MADV_UNMERGEABLE)) { 167 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 168 goto unmap; 169 } 170 171 ksft_test_result(!range_maps_duplicates(map, size), 172 "Pages were unmerged\n"); 173 unmap: 174 munmap(map, size); 175 } 176 177 #ifdef __NR_userfaultfd 178 static void test_unmerge_uffd_wp(void) 179 { 180 struct uffdio_writeprotect uffd_writeprotect; 181 struct uffdio_register uffdio_register; 182 const unsigned int size = 2 * MiB; 183 struct uffdio_api uffdio_api; 184 char *map; 185 int uffd; 186 187 ksft_print_msg("[RUN] %s\n", __func__); 188 189 map = mmap_and_merge_range(0xcf, size); 190 if (map == MAP_FAILED) 191 return; 192 193 /* See if UFFD is around. */ 194 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); 195 if (uffd < 0) { 196 ksft_test_result_skip("__NR_userfaultfd failed\n"); 197 goto unmap; 198 } 199 200 /* See if UFFD-WP is around. */ 201 uffdio_api.api = UFFD_API; 202 uffdio_api.features = UFFD_FEATURE_PAGEFAULT_FLAG_WP; 203 if (ioctl(uffd, UFFDIO_API, &uffdio_api) < 0) { 204 ksft_test_result_fail("UFFDIO_API failed\n"); 205 goto close_uffd; 206 } 207 if (!(uffdio_api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP)) { 208 ksft_test_result_skip("UFFD_FEATURE_PAGEFAULT_FLAG_WP not available\n"); 209 goto close_uffd; 210 } 211 212 /* Register UFFD-WP, no need for an actual handler. */ 213 uffdio_register.range.start = (unsigned long) map; 214 uffdio_register.range.len = size; 215 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; 216 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) < 0) { 217 ksft_test_result_fail("UFFDIO_REGISTER_MODE_WP failed\n"); 218 goto close_uffd; 219 } 220 221 /* Write-protect the range using UFFD-WP. */ 222 uffd_writeprotect.range.start = (unsigned long) map; 223 uffd_writeprotect.range.len = size; 224 uffd_writeprotect.mode = UFFDIO_WRITEPROTECT_MODE_WP; 225 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &uffd_writeprotect)) { 226 ksft_test_result_fail("UFFDIO_WRITEPROTECT failed\n"); 227 goto close_uffd; 228 } 229 230 if (madvise(map, size, MADV_UNMERGEABLE)) { 231 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 232 goto close_uffd; 233 } 234 235 ksft_test_result(!range_maps_duplicates(map, size), 236 "Pages were unmerged\n"); 237 close_uffd: 238 close(uffd); 239 unmap: 240 munmap(map, size); 241 } 242 #endif 243 244 int main(int argc, char **argv) 245 { 246 unsigned int tests = 2; 247 int err; 248 249 #ifdef __NR_userfaultfd 250 tests++; 251 #endif 252 253 ksft_print_header(); 254 ksft_set_plan(tests); 255 256 pagesize = getpagesize(); 257 258 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR); 259 if (ksm_fd < 0) 260 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/run\") failed\n"); 261 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY); 262 if (ksm_full_scans_fd < 0) 263 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/full_scans\") failed\n"); 264 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 265 if (pagemap_fd < 0) 266 ksft_exit_skip("open(\"/proc/self/pagemap\") failed\n"); 267 268 test_unmerge(); 269 test_unmerge_discarded(); 270 #ifdef __NR_userfaultfd 271 test_unmerge_uffd_wp(); 272 #endif 273 274 err = ksft_get_fail_cnt(); 275 if (err) 276 ksft_exit_fail_msg("%d out of %d tests failed\n", 277 err, ksft_test_num()); 278 return ksft_exit_pass(); 279 } 280