1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KSM functional tests 4 * 5 * Copyright 2022, Red Hat, Inc. 6 * 7 * Author(s): David Hildenbrand <david@redhat.com> 8 */ 9 #define _GNU_SOURCE 10 #include <stdlib.h> 11 #include <string.h> 12 #include <stdbool.h> 13 #include <stdint.h> 14 #include <unistd.h> 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <sys/mman.h> 18 #include <sys/prctl.h> 19 #include <sys/syscall.h> 20 #include <sys/ioctl.h> 21 #include <sys/wait.h> 22 #include <linux/userfaultfd.h> 23 24 #include "../kselftest.h" 25 #include "vm_util.h" 26 27 #define KiB 1024u 28 #define MiB (1024 * KiB) 29 30 static int ksm_fd; 31 static int ksm_full_scans_fd; 32 static int pagemap_fd; 33 static size_t pagesize; 34 35 static bool range_maps_duplicates(char *addr, unsigned long size) 36 { 37 unsigned long offs_a, offs_b, pfn_a, pfn_b; 38 39 /* 40 * There is no easy way to check if there are KSM pages mapped into 41 * this range. We only check that the range does not map the same PFN 42 * twice by comparing each pair of mapped pages. 43 */ 44 for (offs_a = 0; offs_a < size; offs_a += pagesize) { 45 pfn_a = pagemap_get_pfn(pagemap_fd, addr + offs_a); 46 /* Page not present or PFN not exposed by the kernel. */ 47 if (pfn_a == -1ul || !pfn_a) 48 continue; 49 50 for (offs_b = offs_a + pagesize; offs_b < size; 51 offs_b += pagesize) { 52 pfn_b = pagemap_get_pfn(pagemap_fd, addr + offs_b); 53 if (pfn_b == -1ul || !pfn_b) 54 continue; 55 if (pfn_a == pfn_b) 56 return true; 57 } 58 } 59 return false; 60 } 61 62 static long ksm_get_full_scans(void) 63 { 64 char buf[10]; 65 ssize_t ret; 66 67 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0); 68 if (ret <= 0) 69 return -errno; 70 buf[ret] = 0; 71 72 return strtol(buf, NULL, 10); 73 } 74 75 static int ksm_merge(void) 76 { 77 long start_scans, end_scans; 78 79 /* Wait for two full scans such that any possible merging happened. */ 80 start_scans = ksm_get_full_scans(); 81 if (start_scans < 0) 82 return start_scans; 83 if (write(ksm_fd, "1", 1) != 1) 84 return -errno; 85 do { 86 end_scans = ksm_get_full_scans(); 87 if (end_scans < 0) 88 return end_scans; 89 } while (end_scans < start_scans + 2); 90 91 return 0; 92 } 93 94 static char *mmap_and_merge_range(char val, unsigned long size) 95 { 96 char *map; 97 98 map = mmap(NULL, size, PROT_READ|PROT_WRITE, 99 MAP_PRIVATE|MAP_ANON, -1, 0); 100 if (map == MAP_FAILED) { 101 ksft_test_result_fail("mmap() failed\n"); 102 return MAP_FAILED; 103 } 104 105 /* Don't use THP. Ignore if THP are not around on a kernel. */ 106 if (madvise(map, size, MADV_NOHUGEPAGE) && errno != EINVAL) { 107 ksft_test_result_fail("MADV_NOHUGEPAGE failed\n"); 108 goto unmap; 109 } 110 111 /* Make sure each page contains the same values to merge them. */ 112 memset(map, val, size); 113 if (madvise(map, size, MADV_MERGEABLE)) { 114 ksft_test_result_fail("MADV_MERGEABLE failed\n"); 115 goto unmap; 116 } 117 118 /* Run KSM to trigger merging and wait. */ 119 if (ksm_merge()) { 120 ksft_test_result_fail("Running KSM failed\n"); 121 goto unmap; 122 } 123 return map; 124 unmap: 125 munmap(map, size); 126 return MAP_FAILED; 127 } 128 129 static void test_unmerge(void) 130 { 131 const unsigned int size = 2 * MiB; 132 char *map; 133 134 ksft_print_msg("[RUN] %s\n", __func__); 135 136 map = mmap_and_merge_range(0xcf, size); 137 if (map == MAP_FAILED) 138 return; 139 140 if (madvise(map, size, MADV_UNMERGEABLE)) { 141 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 142 goto unmap; 143 } 144 145 ksft_test_result(!range_maps_duplicates(map, size), 146 "Pages were unmerged\n"); 147 unmap: 148 munmap(map, size); 149 } 150 151 static void test_unmerge_discarded(void) 152 { 153 const unsigned int size = 2 * MiB; 154 char *map; 155 156 ksft_print_msg("[RUN] %s\n", __func__); 157 158 map = mmap_and_merge_range(0xcf, size); 159 if (map == MAP_FAILED) 160 return; 161 162 /* Discard half of all mapped pages so we have pte_none() entries. */ 163 if (madvise(map, size / 2, MADV_DONTNEED)) { 164 ksft_test_result_fail("MADV_DONTNEED failed\n"); 165 goto unmap; 166 } 167 168 if (madvise(map, size, MADV_UNMERGEABLE)) { 169 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 170 goto unmap; 171 } 172 173 ksft_test_result(!range_maps_duplicates(map, size), 174 "Pages were unmerged\n"); 175 unmap: 176 munmap(map, size); 177 } 178 179 #ifdef __NR_userfaultfd 180 static void test_unmerge_uffd_wp(void) 181 { 182 struct uffdio_writeprotect uffd_writeprotect; 183 const unsigned int size = 2 * MiB; 184 struct uffdio_api uffdio_api; 185 char *map; 186 int uffd; 187 188 ksft_print_msg("[RUN] %s\n", __func__); 189 190 map = mmap_and_merge_range(0xcf, size); 191 if (map == MAP_FAILED) 192 return; 193 194 /* See if UFFD is around. */ 195 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); 196 if (uffd < 0) { 197 ksft_test_result_skip("__NR_userfaultfd failed\n"); 198 goto unmap; 199 } 200 201 /* See if UFFD-WP is around. */ 202 uffdio_api.api = UFFD_API; 203 uffdio_api.features = UFFD_FEATURE_PAGEFAULT_FLAG_WP; 204 if (ioctl(uffd, UFFDIO_API, &uffdio_api) < 0) { 205 ksft_test_result_fail("UFFDIO_API failed\n"); 206 goto close_uffd; 207 } 208 if (!(uffdio_api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP)) { 209 ksft_test_result_skip("UFFD_FEATURE_PAGEFAULT_FLAG_WP not available\n"); 210 goto close_uffd; 211 } 212 213 /* Register UFFD-WP, no need for an actual handler. */ 214 if (uffd_register(uffd, map, size, false, true, false)) { 215 ksft_test_result_fail("UFFDIO_REGISTER_MODE_WP failed\n"); 216 goto close_uffd; 217 } 218 219 /* Write-protect the range using UFFD-WP. */ 220 uffd_writeprotect.range.start = (unsigned long) map; 221 uffd_writeprotect.range.len = size; 222 uffd_writeprotect.mode = UFFDIO_WRITEPROTECT_MODE_WP; 223 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &uffd_writeprotect)) { 224 ksft_test_result_fail("UFFDIO_WRITEPROTECT failed\n"); 225 goto close_uffd; 226 } 227 228 if (madvise(map, size, MADV_UNMERGEABLE)) { 229 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 230 goto close_uffd; 231 } 232 233 ksft_test_result(!range_maps_duplicates(map, size), 234 "Pages were unmerged\n"); 235 close_uffd: 236 close(uffd); 237 unmap: 238 munmap(map, size); 239 } 240 #endif 241 242 /* Verify that KSM can be enabled / queried with prctl. */ 243 static void test_prctl(void) 244 { 245 int ret; 246 247 ksft_print_msg("[RUN] %s\n", __func__); 248 249 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 250 if (ret < 0 && errno == EINVAL) { 251 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); 252 return; 253 } else if (ret) { 254 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); 255 return; 256 } 257 258 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0); 259 if (ret < 0) { 260 ksft_test_result_fail("PR_GET_MEMORY_MERGE failed\n"); 261 return; 262 } else if (ret != 1) { 263 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 not effective\n"); 264 return; 265 } 266 267 ret = prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); 268 if (ret) { 269 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 270 return; 271 } 272 273 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0); 274 if (ret < 0) { 275 ksft_test_result_fail("PR_GET_MEMORY_MERGE failed\n"); 276 return; 277 } else if (ret != 0) { 278 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 not effective\n"); 279 return; 280 } 281 282 ksft_test_result_pass("Setting/clearing PR_SET_MEMORY_MERGE works\n"); 283 } 284 285 /* Verify that prctl ksm flag is inherited. */ 286 static void test_prctl_fork(void) 287 { 288 int ret, status; 289 pid_t child_pid; 290 291 ksft_print_msg("[RUN] %s\n", __func__); 292 293 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 294 if (ret < 0 && errno == EINVAL) { 295 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); 296 return; 297 } else if (ret) { 298 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); 299 return; 300 } 301 302 child_pid = fork(); 303 if (!child_pid) { 304 exit(prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0)); 305 } else if (child_pid < 0) { 306 ksft_test_result_fail("fork() failed\n"); 307 return; 308 } 309 310 if (waitpid(child_pid, &status, 0) < 0) { 311 ksft_test_result_fail("waitpid() failed\n"); 312 return; 313 } else if (WEXITSTATUS(status) != 1) { 314 ksft_test_result_fail("unexpected PR_GET_MEMORY_MERGE result in child\n"); 315 return; 316 } 317 318 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) { 319 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 320 return; 321 } 322 323 ksft_test_result_pass("PR_SET_MEMORY_MERGE value is inherited\n"); 324 } 325 326 int main(int argc, char **argv) 327 { 328 unsigned int tests = 4; 329 int err; 330 331 #ifdef __NR_userfaultfd 332 tests++; 333 #endif 334 335 ksft_print_header(); 336 ksft_set_plan(tests); 337 338 pagesize = getpagesize(); 339 340 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR); 341 if (ksm_fd < 0) 342 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/run\") failed\n"); 343 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY); 344 if (ksm_full_scans_fd < 0) 345 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/full_scans\") failed\n"); 346 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 347 if (pagemap_fd < 0) 348 ksft_exit_skip("open(\"/proc/self/pagemap\") failed\n"); 349 350 test_unmerge(); 351 test_unmerge_discarded(); 352 #ifdef __NR_userfaultfd 353 test_unmerge_uffd_wp(); 354 #endif 355 356 test_prctl(); 357 test_prctl_fork(); 358 359 err = ksft_get_fail_cnt(); 360 if (err) 361 ksft_exit_fail_msg("%d out of %d tests failed\n", 362 err, ksft_test_num()); 363 return ksft_exit_pass(); 364 } 365