1 // SPDX-License-Identifier: GPL-2.0-or-later
2
3 #define _GNU_SOURCE
4 #include "kselftest_harness.h"
5 #include <linux/prctl.h>
6 #include <fcntl.h>
7 #include <stdio.h>
8 #include <stdlib.h>
9 #include <unistd.h>
10 #include <sys/mman.h>
11 #include <sys/prctl.h>
12 #include <sys/syscall.h>
13 #include <sys/wait.h>
14 #include <linux/perf_event.h>
15 #include "vm_util.h"
16 #include <linux/mman.h>
17
FIXTURE(merge)18 FIXTURE(merge)
19 {
20 unsigned int page_size;
21 char *carveout;
22 struct procmap_fd procmap;
23 };
24
map_carveout(unsigned int page_size)25 static char *map_carveout(unsigned int page_size)
26 {
27 return mmap(NULL, 30 * page_size, PROT_NONE,
28 MAP_ANON | MAP_PRIVATE, -1, 0);
29 }
30
do_fork(struct procmap_fd * procmap)31 static pid_t do_fork(struct procmap_fd *procmap)
32 {
33 pid_t pid = fork();
34
35 if (pid == -1)
36 return -1;
37 if (pid != 0) {
38 wait(NULL);
39 return pid;
40 }
41
42 /* Reopen for child. */
43 if (close_procmap(procmap))
44 return -1;
45 if (open_self_procmap(procmap))
46 return -1;
47
48 return 0;
49 }
50
51 #ifdef __NR_mseal
sys_mseal(void * ptr,size_t len,unsigned long flags)52 static int sys_mseal(void *ptr, size_t len, unsigned long flags)
53 {
54 return syscall(__NR_mseal, (unsigned long)ptr, len, flags);
55 }
56 #else
sys_mseal(void * ptr,size_t len,unsigned long flags)57 static int sys_mseal(void *ptr, size_t len, unsigned long flags)
58 {
59 errno = ENOSYS;
60 return -1;
61 }
62 #endif
63
FIXTURE_SETUP(merge)64 FIXTURE_SETUP(merge)
65 {
66 self->page_size = psize();
67 /* Carve out PROT_NONE region to map over. */
68 self->carveout = map_carveout(self->page_size);
69 ASSERT_NE(self->carveout, MAP_FAILED);
70 /* Setup PROCMAP_QUERY interface. */
71 ASSERT_EQ(open_self_procmap(&self->procmap), 0);
72 }
73
FIXTURE_TEARDOWN(merge)74 FIXTURE_TEARDOWN(merge)
75 {
76 ASSERT_EQ(munmap(self->carveout, 30 * self->page_size), 0);
77 /* May fail for parent of forked process. */
78 close_procmap(&self->procmap);
79 /*
80 * Clear unconditionally, as some tests set this. It is no issue if this
81 * fails (KSM may be disabled for instance).
82 */
83 prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0);
84 }
85
FIXTURE(merge_with_fork)86 FIXTURE(merge_with_fork)
87 {
88 unsigned int page_size;
89 char *carveout;
90 struct procmap_fd procmap;
91 };
92
FIXTURE_VARIANT(merge_with_fork)93 FIXTURE_VARIANT(merge_with_fork)
94 {
95 bool forked;
96 };
97
FIXTURE_VARIANT_ADD(merge_with_fork,forked)98 FIXTURE_VARIANT_ADD(merge_with_fork, forked)
99 {
100 .forked = true,
101 };
102
FIXTURE_VARIANT_ADD(merge_with_fork,unforked)103 FIXTURE_VARIANT_ADD(merge_with_fork, unforked)
104 {
105 .forked = false,
106 };
107
FIXTURE_SETUP(merge_with_fork)108 FIXTURE_SETUP(merge_with_fork)
109 {
110 self->page_size = psize();
111 self->carveout = map_carveout(self->page_size);
112 ASSERT_NE(self->carveout, MAP_FAILED);
113 ASSERT_EQ(open_self_procmap(&self->procmap), 0);
114 }
115
FIXTURE_TEARDOWN(merge_with_fork)116 FIXTURE_TEARDOWN(merge_with_fork)
117 {
118 ASSERT_EQ(munmap(self->carveout, 30 * self->page_size), 0);
119 ASSERT_EQ(close_procmap(&self->procmap), 0);
120 /* See above. */
121 prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0);
122 }
123
TEST_F(merge,mprotect_unfaulted_left)124 TEST_F(merge, mprotect_unfaulted_left)
125 {
126 unsigned int page_size = self->page_size;
127 char *carveout = self->carveout;
128 struct procmap_fd *procmap = &self->procmap;
129 char *ptr;
130
131 /*
132 * Map 10 pages of R/W memory within. MAP_NORESERVE so we don't hit
133 * merge failure due to lack of VM_ACCOUNT flag by mistake.
134 *
135 * |-----------------------|
136 * | unfaulted |
137 * |-----------------------|
138 */
139 ptr = mmap(&carveout[page_size], 10 * page_size, PROT_READ | PROT_WRITE,
140 MAP_ANON | MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, -1, 0);
141 ASSERT_NE(ptr, MAP_FAILED);
142 /*
143 * Now make the first 5 pages read-only, splitting the VMA:
144 *
145 * RO RW
146 * |-----------|-----------|
147 * | unfaulted | unfaulted |
148 * |-----------|-----------|
149 */
150 ASSERT_EQ(mprotect(ptr, 5 * page_size, PROT_READ), 0);
151 /*
152 * Fault in the first of the last 5 pages so it gets an anon_vma and
153 * thus the whole VMA becomes 'faulted':
154 *
155 * RO RW
156 * |-----------|-----------|
157 * | unfaulted | faulted |
158 * |-----------|-----------|
159 */
160 ptr[5 * page_size] = 'x';
161 /*
162 * Now mprotect() the RW region read-only, we should merge (though for
163 * ~15 years we did not! :):
164 *
165 * RO
166 * |-----------------------|
167 * | faulted |
168 * |-----------------------|
169 */
170 ASSERT_EQ(mprotect(&ptr[5 * page_size], 5 * page_size, PROT_READ), 0);
171
172 /* Assert that the merge succeeded using PROCMAP_QUERY. */
173 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
174 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
175 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 10 * page_size);
176 }
177
TEST_F(merge,mprotect_unfaulted_right)178 TEST_F(merge, mprotect_unfaulted_right)
179 {
180 unsigned int page_size = self->page_size;
181 char *carveout = self->carveout;
182 struct procmap_fd *procmap = &self->procmap;
183 char *ptr;
184
185 /*
186 * |-----------------------|
187 * | unfaulted |
188 * |-----------------------|
189 */
190 ptr = mmap(&carveout[page_size], 10 * page_size, PROT_READ | PROT_WRITE,
191 MAP_ANON | MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, -1, 0);
192 ASSERT_NE(ptr, MAP_FAILED);
193 /*
194 * Now make the last 5 pages read-only, splitting the VMA:
195 *
196 * RW RO
197 * |-----------|-----------|
198 * | unfaulted | unfaulted |
199 * |-----------|-----------|
200 */
201 ASSERT_EQ(mprotect(&ptr[5 * page_size], 5 * page_size, PROT_READ), 0);
202 /*
203 * Fault in the first of the first 5 pages so it gets an anon_vma and
204 * thus the whole VMA becomes 'faulted':
205 *
206 * RW RO
207 * |-----------|-----------|
208 * | faulted | unfaulted |
209 * |-----------|-----------|
210 */
211 ptr[0] = 'x';
212 /*
213 * Now mprotect() the RW region read-only, we should merge:
214 *
215 * RO
216 * |-----------------------|
217 * | faulted |
218 * |-----------------------|
219 */
220 ASSERT_EQ(mprotect(ptr, 5 * page_size, PROT_READ), 0);
221
222 /* Assert that the merge succeeded using PROCMAP_QUERY. */
223 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
224 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
225 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 10 * page_size);
226 }
227
TEST_F(merge,mprotect_unfaulted_both)228 TEST_F(merge, mprotect_unfaulted_both)
229 {
230 unsigned int page_size = self->page_size;
231 char *carveout = self->carveout;
232 struct procmap_fd *procmap = &self->procmap;
233 char *ptr;
234
235 /*
236 * |-----------------------|
237 * | unfaulted |
238 * |-----------------------|
239 */
240 ptr = mmap(&carveout[2 * page_size], 9 * page_size, PROT_READ | PROT_WRITE,
241 MAP_ANON | MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, -1, 0);
242 ASSERT_NE(ptr, MAP_FAILED);
243 /*
244 * Now make the first and last 3 pages read-only, splitting the VMA:
245 *
246 * RO RW RO
247 * |-----------|-----------|-----------|
248 * | unfaulted | unfaulted | unfaulted |
249 * |-----------|-----------|-----------|
250 */
251 ASSERT_EQ(mprotect(ptr, 3 * page_size, PROT_READ), 0);
252 ASSERT_EQ(mprotect(&ptr[6 * page_size], 3 * page_size, PROT_READ), 0);
253 /*
254 * Fault in the first of the middle 3 pages so it gets an anon_vma and
255 * thus the whole VMA becomes 'faulted':
256 *
257 * RO RW RO
258 * |-----------|-----------|-----------|
259 * | unfaulted | faulted | unfaulted |
260 * |-----------|-----------|-----------|
261 */
262 ptr[3 * page_size] = 'x';
263 /*
264 * Now mprotect() the RW region read-only, we should merge:
265 *
266 * RO
267 * |-----------------------|
268 * | faulted |
269 * |-----------------------|
270 */
271 ASSERT_EQ(mprotect(&ptr[3 * page_size], 3 * page_size, PROT_READ), 0);
272
273 /* Assert that the merge succeeded using PROCMAP_QUERY. */
274 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
275 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
276 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 9 * page_size);
277 }
278
TEST_F(merge,mprotect_faulted_left_unfaulted_right)279 TEST_F(merge, mprotect_faulted_left_unfaulted_right)
280 {
281 unsigned int page_size = self->page_size;
282 char *carveout = self->carveout;
283 struct procmap_fd *procmap = &self->procmap;
284 char *ptr;
285
286 /*
287 * |-----------------------|
288 * | unfaulted |
289 * |-----------------------|
290 */
291 ptr = mmap(&carveout[2 * page_size], 9 * page_size, PROT_READ | PROT_WRITE,
292 MAP_ANON | MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, -1, 0);
293 ASSERT_NE(ptr, MAP_FAILED);
294 /*
295 * Now make the last 3 pages read-only, splitting the VMA:
296 *
297 * RW RO
298 * |-----------------------|-----------|
299 * | unfaulted | unfaulted |
300 * |-----------------------|-----------|
301 */
302 ASSERT_EQ(mprotect(&ptr[6 * page_size], 3 * page_size, PROT_READ), 0);
303 /*
304 * Fault in the first of the first 6 pages so it gets an anon_vma and
305 * thus the whole VMA becomes 'faulted':
306 *
307 * RW RO
308 * |-----------------------|-----------|
309 * | unfaulted | unfaulted |
310 * |-----------------------|-----------|
311 */
312 ptr[0] = 'x';
313 /*
314 * Now make the first 3 pages read-only, splitting the VMA:
315 *
316 * RO RW RO
317 * |-----------|-----------|-----------|
318 * | faulted | faulted | unfaulted |
319 * |-----------|-----------|-----------|
320 */
321 ASSERT_EQ(mprotect(ptr, 3 * page_size, PROT_READ), 0);
322 /*
323 * Now mprotect() the RW region read-only, we should merge:
324 *
325 * RO
326 * |-----------------------|
327 * | faulted |
328 * |-----------------------|
329 */
330 ASSERT_EQ(mprotect(&ptr[3 * page_size], 3 * page_size, PROT_READ), 0);
331
332 /* Assert that the merge succeeded using PROCMAP_QUERY. */
333 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
334 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
335 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 9 * page_size);
336 }
337
TEST_F(merge,mprotect_unfaulted_left_faulted_right)338 TEST_F(merge, mprotect_unfaulted_left_faulted_right)
339 {
340 unsigned int page_size = self->page_size;
341 char *carveout = self->carveout;
342 struct procmap_fd *procmap = &self->procmap;
343 char *ptr;
344
345 /*
346 * |-----------------------|
347 * | unfaulted |
348 * |-----------------------|
349 */
350 ptr = mmap(&carveout[2 * page_size], 9 * page_size, PROT_READ | PROT_WRITE,
351 MAP_ANON | MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, -1, 0);
352 ASSERT_NE(ptr, MAP_FAILED);
353 /*
354 * Now make the first 3 pages read-only, splitting the VMA:
355 *
356 * RO RW
357 * |-----------|-----------------------|
358 * | unfaulted | unfaulted |
359 * |-----------|-----------------------|
360 */
361 ASSERT_EQ(mprotect(ptr, 3 * page_size, PROT_READ), 0);
362 /*
363 * Fault in the first of the last 6 pages so it gets an anon_vma and
364 * thus the whole VMA becomes 'faulted':
365 *
366 * RO RW
367 * |-----------|-----------------------|
368 * | unfaulted | faulted |
369 * |-----------|-----------------------|
370 */
371 ptr[3 * page_size] = 'x';
372 /*
373 * Now make the last 3 pages read-only, splitting the VMA:
374 *
375 * RO RW RO
376 * |-----------|-----------|-----------|
377 * | unfaulted | faulted | faulted |
378 * |-----------|-----------|-----------|
379 */
380 ASSERT_EQ(mprotect(&ptr[6 * page_size], 3 * page_size, PROT_READ), 0);
381 /*
382 * Now mprotect() the RW region read-only, we should merge:
383 *
384 * RO
385 * |-----------------------|
386 * | faulted |
387 * |-----------------------|
388 */
389 ASSERT_EQ(mprotect(&ptr[3 * page_size], 3 * page_size, PROT_READ), 0);
390
391 /* Assert that the merge succeeded using PROCMAP_QUERY. */
392 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
393 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
394 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 9 * page_size);
395 }
396
TEST_F(merge,forked_target_vma)397 TEST_F(merge, forked_target_vma)
398 {
399 unsigned int page_size = self->page_size;
400 char *carveout = self->carveout;
401 struct procmap_fd *procmap = &self->procmap;
402 char *ptr, *ptr2;
403 pid_t pid;
404 int i;
405
406 /*
407 * |-----------|
408 * | unfaulted |
409 * |-----------|
410 */
411 ptr = mmap(&carveout[page_size], 5 * page_size, PROT_READ | PROT_WRITE,
412 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
413 ASSERT_NE(ptr, MAP_FAILED);
414
415 /*
416 * Fault in process.
417 *
418 * |-----------|
419 * | faulted |
420 * |-----------|
421 */
422 ptr[0] = 'x';
423
424 pid = do_fork(&self->procmap);
425 ASSERT_NE(pid, -1);
426 if (pid != 0)
427 return;
428
429 /* unCOWing everything does not cause the AVC to go away. */
430 for (i = 0; i < 5 * page_size; i += page_size)
431 ptr[i] = 'x';
432
433 /*
434 * Map in adjacent VMA in child.
435 *
436 * forked
437 * |-----------|-----------|
438 * | faulted | unfaulted |
439 * |-----------|-----------|
440 * ptr ptr2
441 */
442 ptr2 = mmap(&ptr[5 * page_size], 5 * page_size, PROT_READ | PROT_WRITE,
443 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
444 ASSERT_NE(ptr2, MAP_FAILED);
445
446 /* Make sure not merged. */
447 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
448 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
449 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 5 * page_size);
450 }
451
TEST_F(merge,forked_source_vma)452 TEST_F(merge, forked_source_vma)
453 {
454 unsigned int page_size = self->page_size;
455 char *carveout = self->carveout;
456 struct procmap_fd *procmap = &self->procmap;
457 char *ptr, *ptr2;
458 pid_t pid;
459 int i;
460
461 /*
462 * |-----------|------------|
463 * | unfaulted | <unmapped> |
464 * |-----------|------------|
465 */
466 ptr = mmap(&carveout[page_size], 5 * page_size, PROT_READ | PROT_WRITE,
467 MAP_ANON | MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, -1, 0);
468 ASSERT_NE(ptr, MAP_FAILED);
469
470 /*
471 * Fault in process.
472 *
473 * |-----------|------------|
474 * | faulted | <unmapped> |
475 * |-----------|------------|
476 */
477 ptr[0] = 'x';
478
479 pid = do_fork(&self->procmap);
480 ASSERT_NE(pid, -1);
481 if (pid != 0)
482 return;
483
484 /* unCOWing everything does not cause the AVC to go away. */
485 for (i = 0; i < 5 * page_size; i += page_size)
486 ptr[i] = 'x';
487
488 /*
489 * Map in adjacent VMA in child, ptr2 after ptr, but incompatible.
490 *
491 * forked RW RWX
492 * |-----------|-----------|
493 * | faulted | unfaulted |
494 * |-----------|-----------|
495 * ptr ptr2
496 */
497 ptr2 = mmap(&carveout[6 * page_size], 5 * page_size, PROT_READ | PROT_WRITE | PROT_EXEC,
498 MAP_ANON | MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, -1, 0);
499 ASSERT_NE(ptr2, MAP_FAILED);
500
501 /* Make sure not merged. */
502 ASSERT_TRUE(find_vma_procmap(procmap, ptr2));
503 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr2);
504 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr2 + 5 * page_size);
505
506 /*
507 * Now mprotect forked region to RWX so it becomes the source for the
508 * merge to unfaulted region:
509 *
510 * forked RWX RWX
511 * |-----------|-----------|
512 * | faulted | unfaulted |
513 * |-----------|-----------|
514 * ptr ptr2
515 *
516 * This should NOT result in a merge, as ptr was forked.
517 */
518 ASSERT_EQ(mprotect(ptr, 5 * page_size, PROT_READ | PROT_WRITE | PROT_EXEC), 0);
519 /* Again, make sure not merged. */
520 ASSERT_TRUE(find_vma_procmap(procmap, ptr2));
521 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr2);
522 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr2 + 5 * page_size);
523 }
524
TEST_F(merge,handle_uprobe_upon_merged_vma)525 TEST_F(merge, handle_uprobe_upon_merged_vma)
526 {
527 const size_t attr_sz = sizeof(struct perf_event_attr);
528 unsigned int page_size = self->page_size;
529 const char *probe_file = "./foo";
530 char *carveout = self->carveout;
531 struct perf_event_attr attr;
532 unsigned long type;
533 void *ptr1, *ptr2;
534 int fd;
535
536 fd = open(probe_file, O_RDWR|O_CREAT, 0600);
537 ASSERT_GE(fd, 0);
538
539 ASSERT_EQ(ftruncate(fd, page_size), 0);
540 if (read_sysfs("/sys/bus/event_source/devices/uprobe/type", &type) != 0) {
541 SKIP(goto out, "Failed to read uprobe sysfs file, skipping");
542 }
543
544 memset(&attr, 0, attr_sz);
545 attr.size = attr_sz;
546 attr.type = type;
547 attr.config1 = (__u64)(long)probe_file;
548 attr.config2 = 0x0;
549
550 ASSERT_GE(syscall(__NR_perf_event_open, &attr, 0, -1, -1, 0), 0);
551
552 ptr1 = mmap(&carveout[page_size], 10 * page_size, PROT_EXEC,
553 MAP_PRIVATE | MAP_FIXED, fd, 0);
554 ASSERT_NE(ptr1, MAP_FAILED);
555
556 ptr2 = mremap(ptr1, page_size, 2 * page_size,
557 MREMAP_MAYMOVE | MREMAP_FIXED, ptr1 + 5 * page_size);
558 ASSERT_NE(ptr2, MAP_FAILED);
559
560 ASSERT_NE(mremap(ptr2, page_size, page_size,
561 MREMAP_MAYMOVE | MREMAP_FIXED, ptr1), MAP_FAILED);
562
563 out:
564 close(fd);
565 remove(probe_file);
566 }
567
TEST_F(merge,ksm_merge)568 TEST_F(merge, ksm_merge)
569 {
570 unsigned int page_size = self->page_size;
571 char *carveout = self->carveout;
572 struct procmap_fd *procmap = &self->procmap;
573 char *ptr, *ptr2;
574 int err;
575
576 /*
577 * Map two R/W immediately adjacent to one another, they should
578 * trivially merge:
579 *
580 * |-----------|-----------|
581 * | R/W | R/W |
582 * |-----------|-----------|
583 * ptr ptr2
584 */
585
586 ptr = mmap(&carveout[page_size], page_size, PROT_READ | PROT_WRITE,
587 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
588 ASSERT_NE(ptr, MAP_FAILED);
589 ptr2 = mmap(&carveout[2 * page_size], page_size,
590 PROT_READ | PROT_WRITE,
591 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
592 ASSERT_NE(ptr2, MAP_FAILED);
593 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
594 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
595 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 2 * page_size);
596
597 /* Unmap the second half of this merged VMA. */
598 ASSERT_EQ(munmap(ptr2, page_size), 0);
599
600 /* OK, now enable global KSM merge. We clear this on test teardown. */
601 err = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0);
602 if (err == -1) {
603 int errnum = errno;
604
605 /* Only non-failure case... */
606 ASSERT_EQ(errnum, EINVAL);
607 /* ...but indicates we should skip. */
608 SKIP(return, "KSM memory merging not supported, skipping.");
609 }
610
611 /*
612 * Now map a VMA adjacent to the existing that was just made
613 * VM_MERGEABLE, this should merge as well.
614 */
615 ptr2 = mmap(&carveout[2 * page_size], page_size,
616 PROT_READ | PROT_WRITE,
617 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
618 ASSERT_NE(ptr2, MAP_FAILED);
619 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
620 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
621 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 2 * page_size);
622
623 /* Now this VMA altogether. */
624 ASSERT_EQ(munmap(ptr, 2 * page_size), 0);
625
626 /* Try the same operation as before, asserting this also merges fine. */
627 ptr = mmap(&carveout[page_size], page_size, PROT_READ | PROT_WRITE,
628 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
629 ASSERT_NE(ptr, MAP_FAILED);
630 ptr2 = mmap(&carveout[2 * page_size], page_size,
631 PROT_READ | PROT_WRITE,
632 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
633 ASSERT_NE(ptr2, MAP_FAILED);
634 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
635 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
636 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 2 * page_size);
637 }
638
TEST_F(merge,mremap_unfaulted_to_faulted)639 TEST_F(merge, mremap_unfaulted_to_faulted)
640 {
641 unsigned int page_size = self->page_size;
642 char *carveout = self->carveout;
643 struct procmap_fd *procmap = &self->procmap;
644 char *ptr, *ptr2;
645
646 /*
647 * Map two distinct areas:
648 *
649 * |-----------| |-----------|
650 * | unfaulted | | unfaulted |
651 * |-----------| |-----------|
652 * ptr ptr2
653 */
654 ptr = mmap(&carveout[page_size], 5 * page_size, PROT_READ | PROT_WRITE,
655 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
656 ASSERT_NE(ptr, MAP_FAILED);
657 ptr2 = mmap(&carveout[7 * page_size], 5 * page_size, PROT_READ | PROT_WRITE,
658 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
659 ASSERT_NE(ptr2, MAP_FAILED);
660
661 /* Offset ptr2 further away. */
662 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
663 MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000);
664 ASSERT_NE(ptr2, MAP_FAILED);
665
666 /*
667 * Fault in ptr:
668 * \
669 * |-----------| / |-----------|
670 * | faulted | \ | unfaulted |
671 * |-----------| / |-----------|
672 * ptr \ ptr2
673 */
674 ptr[0] = 'x';
675
676 /*
677 * Now move ptr2 adjacent to ptr:
678 *
679 * |-----------|-----------|
680 * | faulted | unfaulted |
681 * |-----------|-----------|
682 * ptr ptr2
683 *
684 * It should merge:
685 *
686 * |----------------------|
687 * | faulted |
688 * |----------------------|
689 * ptr
690 */
691 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
692 MREMAP_MAYMOVE | MREMAP_FIXED, &ptr[5 * page_size]);
693 ASSERT_NE(ptr2, MAP_FAILED);
694
695 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
696 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
697 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 10 * page_size);
698 }
699
TEST_F(merge,mremap_unfaulted_behind_faulted)700 TEST_F(merge, mremap_unfaulted_behind_faulted)
701 {
702 unsigned int page_size = self->page_size;
703 char *carveout = self->carveout;
704 struct procmap_fd *procmap = &self->procmap;
705 char *ptr, *ptr2;
706
707 /*
708 * Map two distinct areas:
709 *
710 * |-----------| |-----------|
711 * | unfaulted | | unfaulted |
712 * |-----------| |-----------|
713 * ptr ptr2
714 */
715 ptr = mmap(&carveout[6 * page_size], 5 * page_size, PROT_READ | PROT_WRITE,
716 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
717 ASSERT_NE(ptr, MAP_FAILED);
718 ptr2 = mmap(&carveout[14 * page_size], 5 * page_size, PROT_READ | PROT_WRITE,
719 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
720 ASSERT_NE(ptr2, MAP_FAILED);
721
722 /* Offset ptr2 further away. */
723 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
724 MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000);
725 ASSERT_NE(ptr2, MAP_FAILED);
726
727 /*
728 * Fault in ptr:
729 * \
730 * |-----------| / |-----------|
731 * | faulted | \ | unfaulted |
732 * |-----------| / |-----------|
733 * ptr \ ptr2
734 */
735 ptr[0] = 'x';
736
737 /*
738 * Now move ptr2 adjacent, but behind, ptr:
739 *
740 * |-----------|-----------|
741 * | unfaulted | faulted |
742 * |-----------|-----------|
743 * ptr2 ptr
744 *
745 * It should merge:
746 *
747 * |----------------------|
748 * | faulted |
749 * |----------------------|
750 * ptr2
751 */
752 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
753 MREMAP_MAYMOVE | MREMAP_FIXED, &carveout[page_size]);
754 ASSERT_NE(ptr2, MAP_FAILED);
755
756 ASSERT_TRUE(find_vma_procmap(procmap, ptr2));
757 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr2);
758 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr2 + 10 * page_size);
759 }
760
TEST_F(merge,mremap_unfaulted_between_faulted)761 TEST_F(merge, mremap_unfaulted_between_faulted)
762 {
763 unsigned int page_size = self->page_size;
764 char *carveout = self->carveout;
765 struct procmap_fd *procmap = &self->procmap;
766 char *ptr, *ptr2, *ptr3;
767
768 /*
769 * Map three distinct areas:
770 *
771 * |-----------| |-----------| |-----------|
772 * | unfaulted | | unfaulted | | unfaulted |
773 * |-----------| |-----------| |-----------|
774 * ptr ptr2 ptr3
775 */
776 ptr = mmap(&carveout[page_size], 5 * page_size, PROT_READ | PROT_WRITE,
777 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
778 ASSERT_NE(ptr, MAP_FAILED);
779 ptr2 = mmap(&carveout[7 * page_size], 5 * page_size, PROT_READ | PROT_WRITE,
780 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
781 ASSERT_NE(ptr2, MAP_FAILED);
782 ptr3 = mmap(&carveout[14 * page_size], 5 * page_size, PROT_READ | PROT_WRITE,
783 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
784 ASSERT_NE(ptr3, MAP_FAILED);
785
786 /* Offset ptr3 further away. */
787 ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size,
788 MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + page_size * 2000);
789 ASSERT_NE(ptr3, MAP_FAILED);
790
791 /* Offset ptr2 further away. */
792 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
793 MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000);
794 ASSERT_NE(ptr2, MAP_FAILED);
795
796 /*
797 * Fault in ptr, ptr3:
798 * \ \
799 * |-----------| / |-----------| / |-----------|
800 * | faulted | \ | unfaulted | \ | faulted |
801 * |-----------| / |-----------| / |-----------|
802 * ptr \ ptr2 \ ptr3
803 */
804 ptr[0] = 'x';
805 ptr3[0] = 'x';
806
807 /*
808 * Move ptr3 back into place, leaving a place for ptr2:
809 * \
810 * |-----------| |-----------| / |-----------|
811 * | faulted | | faulted | \ | unfaulted |
812 * |-----------| |-----------| / |-----------|
813 * ptr ptr3 \ ptr2
814 */
815 ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size,
816 MREMAP_MAYMOVE | MREMAP_FIXED, &ptr[10 * page_size]);
817 ASSERT_NE(ptr3, MAP_FAILED);
818
819 /*
820 * Finally, move ptr2 into place:
821 *
822 * |-----------|-----------|-----------|
823 * | faulted | unfaulted | faulted |
824 * |-----------|-----------|-----------|
825 * ptr ptr2 ptr3
826 *
827 * It should merge, but only ptr, ptr2:
828 *
829 * |-----------------------|-----------|
830 * | faulted | unfaulted |
831 * |-----------------------|-----------|
832 */
833 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
834 MREMAP_MAYMOVE | MREMAP_FIXED, &ptr[5 * page_size]);
835 ASSERT_NE(ptr2, MAP_FAILED);
836
837 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
838 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
839 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 10 * page_size);
840
841 ASSERT_TRUE(find_vma_procmap(procmap, ptr3));
842 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr3);
843 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr3 + 5 * page_size);
844 }
845
TEST_F(merge,mremap_unfaulted_between_faulted_unfaulted)846 TEST_F(merge, mremap_unfaulted_between_faulted_unfaulted)
847 {
848 unsigned int page_size = self->page_size;
849 char *carveout = self->carveout;
850 struct procmap_fd *procmap = &self->procmap;
851 char *ptr, *ptr2, *ptr3;
852
853 /*
854 * Map three distinct areas:
855 *
856 * |-----------| |-----------| |-----------|
857 * | unfaulted | | unfaulted | | unfaulted |
858 * |-----------| |-----------| |-----------|
859 * ptr ptr2 ptr3
860 */
861 ptr = mmap(&carveout[page_size], 5 * page_size, PROT_READ | PROT_WRITE,
862 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
863 ASSERT_NE(ptr, MAP_FAILED);
864 ptr2 = mmap(&carveout[7 * page_size], 5 * page_size, PROT_READ | PROT_WRITE,
865 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
866 ASSERT_NE(ptr2, MAP_FAILED);
867 ptr3 = mmap(&carveout[14 * page_size], 5 * page_size, PROT_READ | PROT_WRITE,
868 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
869 ASSERT_NE(ptr3, MAP_FAILED);
870
871 /* Offset ptr3 further away. */
872 ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size,
873 MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + page_size * 2000);
874 ASSERT_NE(ptr3, MAP_FAILED);
875
876
877 /* Offset ptr2 further away. */
878 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
879 MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000);
880 ASSERT_NE(ptr2, MAP_FAILED);
881
882 /*
883 * Fault in ptr:
884 * \ \
885 * |-----------| / |-----------| / |-----------|
886 * | faulted | \ | unfaulted | \ | unfaulted |
887 * |-----------| / |-----------| / |-----------|
888 * ptr \ ptr2 \ ptr3
889 */
890 ptr[0] = 'x';
891
892 /*
893 * Move ptr3 back into place, leaving a place for ptr2:
894 * \
895 * |-----------| |-----------| / |-----------|
896 * | faulted | | unfaulted | \ | unfaulted |
897 * |-----------| |-----------| / |-----------|
898 * ptr ptr3 \ ptr2
899 */
900 ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size,
901 MREMAP_MAYMOVE | MREMAP_FIXED, &ptr[10 * page_size]);
902 ASSERT_NE(ptr3, MAP_FAILED);
903
904 /*
905 * Finally, move ptr2 into place:
906 *
907 * |-----------|-----------|-----------|
908 * | faulted | unfaulted | unfaulted |
909 * |-----------|-----------|-----------|
910 * ptr ptr2 ptr3
911 *
912 * It should merge:
913 *
914 * |-----------------------------------|
915 * | faulted |
916 * |-----------------------------------|
917 */
918 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
919 MREMAP_MAYMOVE | MREMAP_FIXED, &ptr[5 * page_size]);
920 ASSERT_NE(ptr2, MAP_FAILED);
921
922 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
923 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
924 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 15 * page_size);
925 }
926
TEST_F(merge,mremap_unfaulted_between_correctly_placed_faulted)927 TEST_F(merge, mremap_unfaulted_between_correctly_placed_faulted)
928 {
929 unsigned int page_size = self->page_size;
930 char *carveout = self->carveout;
931 struct procmap_fd *procmap = &self->procmap;
932 char *ptr, *ptr2;
933
934 /*
935 * Map one larger area:
936 *
937 * |-----------------------------------|
938 * | unfaulted |
939 * |-----------------------------------|
940 */
941 ptr = mmap(&carveout[page_size], 15 * page_size, PROT_READ | PROT_WRITE,
942 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
943 ASSERT_NE(ptr, MAP_FAILED);
944
945 /*
946 * Fault in ptr:
947 *
948 * |-----------------------------------|
949 * | faulted |
950 * |-----------------------------------|
951 */
952 ptr[0] = 'x';
953
954 /*
955 * Unmap middle:
956 *
957 * |-----------| |-----------|
958 * | faulted | | faulted |
959 * |-----------| |-----------|
960 *
961 * Now the faulted areas are compatible with each other (anon_vma the
962 * same, vma->vm_pgoff equal to virtual page offset).
963 */
964 ASSERT_EQ(munmap(&ptr[5 * page_size], 5 * page_size), 0);
965
966 /*
967 * Map a new area, ptr2:
968 * \
969 * |-----------| |-----------| / |-----------|
970 * | faulted | | faulted | \ | unfaulted |
971 * |-----------| |-----------| / |-----------|
972 * ptr \ ptr2
973 */
974 ptr2 = mmap(&carveout[20 * page_size], 5 * page_size, PROT_READ | PROT_WRITE,
975 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
976 ASSERT_NE(ptr2, MAP_FAILED);
977
978 /*
979 * Finally, move ptr2 into place:
980 *
981 * |-----------|-----------|-----------|
982 * | faulted | unfaulted | faulted |
983 * |-----------|-----------|-----------|
984 * ptr ptr2 ptr3
985 *
986 * It should merge:
987 *
988 * |-----------------------------------|
989 * | faulted |
990 * |-----------------------------------|
991 */
992 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
993 MREMAP_MAYMOVE | MREMAP_FIXED, &ptr[5 * page_size]);
994 ASSERT_NE(ptr2, MAP_FAILED);
995
996 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
997 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
998 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 15 * page_size);
999 }
1000
TEST_F(merge,mremap_correct_placed_faulted)1001 TEST_F(merge, mremap_correct_placed_faulted)
1002 {
1003 unsigned int page_size = self->page_size;
1004 char *carveout = self->carveout;
1005 struct procmap_fd *procmap = &self->procmap;
1006 char *ptr, *ptr2, *ptr3;
1007
1008 /*
1009 * Map one larger area:
1010 *
1011 * |-----------------------------------|
1012 * | unfaulted |
1013 * |-----------------------------------|
1014 */
1015 ptr = mmap(&carveout[page_size], 15 * page_size, PROT_READ | PROT_WRITE,
1016 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
1017 ASSERT_NE(ptr, MAP_FAILED);
1018
1019 /*
1020 * Fault in ptr:
1021 *
1022 * |-----------------------------------|
1023 * | faulted |
1024 * |-----------------------------------|
1025 */
1026 ptr[0] = 'x';
1027
1028 /*
1029 * Offset the final and middle 5 pages further away:
1030 * \ \
1031 * |-----------| / |-----------| / |-----------|
1032 * | faulted | \ | faulted | \ | faulted |
1033 * |-----------| / |-----------| / |-----------|
1034 * ptr \ ptr2 \ ptr3
1035 */
1036 ptr3 = &ptr[10 * page_size];
1037 ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size,
1038 MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + page_size * 2000);
1039 ASSERT_NE(ptr3, MAP_FAILED);
1040 ptr2 = &ptr[5 * page_size];
1041 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
1042 MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000);
1043 ASSERT_NE(ptr2, MAP_FAILED);
1044
1045 /*
1046 * Move ptr2 into its correct place:
1047 * \
1048 * |-----------|-----------| / |-----------|
1049 * | faulted | faulted | \ | faulted |
1050 * |-----------|-----------| / |-----------|
1051 * ptr ptr2 \ ptr3
1052 *
1053 * It should merge:
1054 * \
1055 * |-----------------------| / |-----------|
1056 * | faulted | \ | faulted |
1057 * |-----------------------| / |-----------|
1058 * ptr \ ptr3
1059 */
1060
1061 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
1062 MREMAP_MAYMOVE | MREMAP_FIXED, &ptr[5 * page_size]);
1063 ASSERT_NE(ptr2, MAP_FAILED);
1064
1065 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
1066 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
1067 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 10 * page_size);
1068
1069 /*
1070 * Now move ptr out of place:
1071 * \ \
1072 * |-----------| / |-----------| / |-----------|
1073 * | faulted | \ | faulted | \ | faulted |
1074 * |-----------| / |-----------| / |-----------|
1075 * ptr2 \ ptr \ ptr3
1076 */
1077 ptr = sys_mremap(ptr, 5 * page_size, 5 * page_size,
1078 MREMAP_MAYMOVE | MREMAP_FIXED, ptr + page_size * 1000);
1079 ASSERT_NE(ptr, MAP_FAILED);
1080
1081 /*
1082 * Now move ptr back into place:
1083 * \
1084 * |-----------|-----------| / |-----------|
1085 * | faulted | faulted | \ | faulted |
1086 * |-----------|-----------| / |-----------|
1087 * ptr ptr2 \ ptr3
1088 *
1089 * It should merge:
1090 * \
1091 * |-----------------------| / |-----------|
1092 * | faulted | \ | faulted |
1093 * |-----------------------| / |-----------|
1094 * ptr \ ptr3
1095 */
1096 ptr = sys_mremap(ptr, 5 * page_size, 5 * page_size,
1097 MREMAP_MAYMOVE | MREMAP_FIXED, &carveout[page_size]);
1098 ASSERT_NE(ptr, MAP_FAILED);
1099
1100 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
1101 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
1102 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 10 * page_size);
1103
1104 /*
1105 * Now move ptr out of place again:
1106 * \ \
1107 * |-----------| / |-----------| / |-----------|
1108 * | faulted | \ | faulted | \ | faulted |
1109 * |-----------| / |-----------| / |-----------|
1110 * ptr2 \ ptr \ ptr3
1111 */
1112 ptr = sys_mremap(ptr, 5 * page_size, 5 * page_size,
1113 MREMAP_MAYMOVE | MREMAP_FIXED, ptr + page_size * 1000);
1114 ASSERT_NE(ptr, MAP_FAILED);
1115
1116 /*
1117 * Now move ptr3 back into place:
1118 * \
1119 * |-----------|-----------| / |-----------|
1120 * | faulted | faulted | \ | faulted |
1121 * |-----------|-----------| / |-----------|
1122 * ptr2 ptr3 \ ptr
1123 *
1124 * It should merge:
1125 * \
1126 * |-----------------------| / |-----------|
1127 * | faulted | \ | faulted |
1128 * |-----------------------| / |-----------|
1129 * ptr2 \ ptr
1130 */
1131 ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size,
1132 MREMAP_MAYMOVE | MREMAP_FIXED, &ptr2[5 * page_size]);
1133 ASSERT_NE(ptr3, MAP_FAILED);
1134
1135 ASSERT_TRUE(find_vma_procmap(procmap, ptr2));
1136 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr2);
1137 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr2 + 10 * page_size);
1138
1139 /*
1140 * Now move ptr back into place:
1141 *
1142 * |-----------|-----------------------|
1143 * | faulted | faulted |
1144 * |-----------|-----------------------|
1145 * ptr ptr2
1146 *
1147 * It should merge:
1148 *
1149 * |-----------------------------------|
1150 * | faulted |
1151 * |-----------------------------------|
1152 * ptr
1153 */
1154 ptr = sys_mremap(ptr, 5 * page_size, 5 * page_size,
1155 MREMAP_MAYMOVE | MREMAP_FIXED, &carveout[page_size]);
1156 ASSERT_NE(ptr, MAP_FAILED);
1157
1158 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
1159 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
1160 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 15 * page_size);
1161
1162 /*
1163 * Now move ptr2 out of the way:
1164 * \
1165 * |-----------| |-----------| / |-----------|
1166 * | faulted | | faulted | \ | faulted |
1167 * |-----------| |-----------| / |-----------|
1168 * ptr ptr3 \ ptr2
1169 */
1170 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
1171 MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000);
1172 ASSERT_NE(ptr2, MAP_FAILED);
1173
1174 /*
1175 * Now move it back:
1176 *
1177 * |-----------|-----------|-----------|
1178 * | faulted | faulted | faulted |
1179 * |-----------|-----------|-----------|
1180 * ptr ptr2 ptr3
1181 *
1182 * It should merge:
1183 *
1184 * |-----------------------------------|
1185 * | faulted |
1186 * |-----------------------------------|
1187 * ptr
1188 */
1189 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
1190 MREMAP_MAYMOVE | MREMAP_FIXED, &ptr[5 * page_size]);
1191 ASSERT_NE(ptr2, MAP_FAILED);
1192
1193 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
1194 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
1195 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 15 * page_size);
1196
1197 /*
1198 * Move ptr3 out of place:
1199 * \
1200 * |-----------------------| / |-----------|
1201 * | faulted | \ | faulted |
1202 * |-----------------------| / |-----------|
1203 * ptr \ ptr3
1204 */
1205 ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size,
1206 MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + page_size * 1000);
1207 ASSERT_NE(ptr3, MAP_FAILED);
1208
1209 /*
1210 * Now move it back:
1211 *
1212 * |-----------|-----------|-----------|
1213 * | faulted | faulted | faulted |
1214 * |-----------|-----------|-----------|
1215 * ptr ptr2 ptr3
1216 *
1217 * It should merge:
1218 *
1219 * |-----------------------------------|
1220 * | faulted |
1221 * |-----------------------------------|
1222 * ptr
1223 */
1224 ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size,
1225 MREMAP_MAYMOVE | MREMAP_FIXED, &ptr[10 * page_size]);
1226 ASSERT_NE(ptr3, MAP_FAILED);
1227
1228 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
1229 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
1230 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 15 * page_size);
1231 }
1232
TEST_F(merge,merge_vmas_with_mseal)1233 TEST_F(merge, merge_vmas_with_mseal)
1234 {
1235 unsigned int page_size = self->page_size;
1236 struct procmap_fd *procmap = &self->procmap;
1237 char *ptr, *ptr2, *ptr3;
1238 /* We need our own as cannot munmap() once sealed. */
1239 char *carveout;
1240
1241 /* Invalid mseal() call to see if implemented. */
1242 ASSERT_EQ(sys_mseal(NULL, 0, ~0UL), -1);
1243 if (errno == ENOSYS)
1244 SKIP(return, "mseal not supported, skipping.");
1245
1246 /* Map carveout. */
1247 carveout = mmap(NULL, 5 * page_size, PROT_NONE,
1248 MAP_PRIVATE | MAP_ANON, -1, 0);
1249 ASSERT_NE(carveout, MAP_FAILED);
1250
1251 /*
1252 * Map 3 separate VMAs:
1253 *
1254 * |-----------|-----------|-----------|
1255 * | RW | RWE | RO |
1256 * |-----------|-----------|-----------|
1257 * ptr ptr2 ptr3
1258 */
1259 ptr = mmap(&carveout[page_size], page_size, PROT_READ | PROT_WRITE,
1260 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
1261 ASSERT_NE(ptr, MAP_FAILED);
1262 ptr2 = mmap(&carveout[2 * page_size], page_size,
1263 PROT_READ | PROT_WRITE | PROT_EXEC,
1264 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
1265 ASSERT_NE(ptr2, MAP_FAILED);
1266 ptr3 = mmap(&carveout[3 * page_size], page_size, PROT_READ,
1267 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
1268 ASSERT_NE(ptr3, MAP_FAILED);
1269
1270 /*
1271 * mseal the second VMA:
1272 *
1273 * |-----------|-----------|-----------|
1274 * | RW | RWES | RO |
1275 * |-----------|-----------|-----------|
1276 * ptr ptr2 ptr3
1277 */
1278 ASSERT_EQ(sys_mseal(ptr2, page_size, 0), 0);
1279
1280 /* Make first VMA mergeable upon mseal. */
1281 ASSERT_EQ(mprotect(ptr, page_size,
1282 PROT_READ | PROT_WRITE | PROT_EXEC), 0);
1283 /*
1284 * At this point we have:
1285 *
1286 * |-----------|-----------|-----------|
1287 * | RWE | RWES | RO |
1288 * |-----------|-----------|-----------|
1289 * ptr ptr2 ptr3
1290 *
1291 * Now mseal all of the VMAs.
1292 */
1293 ASSERT_EQ(sys_mseal(ptr, 3 * page_size, 0), 0);
1294
1295 /*
1296 * We should end up with:
1297 *
1298 * |-----------------------|-----------|
1299 * | RWES | ROS |
1300 * |-----------------------|-----------|
1301 * ptr ptr3
1302 */
1303 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
1304 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
1305 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 2 * page_size);
1306 }
1307
TEST_F(merge,anon_and_page_offset_mismatch_memfd)1308 TEST_F(merge, anon_and_page_offset_mismatch_memfd)
1309 {
1310 struct procmap_fd *procmap = &self->procmap;
1311 unsigned int page_size = self->page_size;
1312 char *carveout = self->carveout;
1313 char *ptr, *ptr2;
1314 int fd;
1315
1316 /* Create a 10 page memfd descriptor. */
1317 fd = memfd_create("anon_page_offset_test", MFD_CLOEXEC);
1318 ASSERT_NE(fd, -1);
1319 ASSERT_EQ(ftruncate(fd, 10 * page_size), 0);
1320
1321 /* Map a region using the memfd at page offset 0. */
1322 ptr = mmap(carveout, 5 * page_size, PROT_READ | PROT_WRITE,
1323 MAP_FIXED | MAP_PRIVATE, fd, 0);
1324 ASSERT_NE(ptr, MAP_FAILED);
1325
1326 /*
1327 * Map another separately and trigger a CoW fault at page offset 5:
1328 *
1329 * |-----------| |---------|
1330 * | unfaulted | | faulted |
1331 * |-----------| |---------|
1332 */
1333 ptr2 = mmap(&carveout[10 * page_size], 5 * page_size,
1334 PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE,
1335 fd, 5 * page_size);
1336 ASSERT_NE(ptr2, MAP_FAILED);
1337 ptr2[0] = 'x';
1338
1339 /*
1340 * Now move it in place:
1341 *
1342 * |----------|
1343 * | |
1344 * v |
1345 * |-----------| |---------|
1346 * | unfaulted | | faulted |
1347 * |-----------| |---------|
1348 *
1349 * Because the anonymous page offset of the faulted region is now
1350 * &carveout[10 * page_size], despite the two regions being mergeable
1351 * due to file page offset, they are NOT mergeable due to anonymous
1352 * page offset.
1353 */
1354 ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
1355 MREMAP_MAYMOVE | MREMAP_FIXED,
1356 &carveout[5 * page_size]);
1357 ASSERT_NE(ptr2, MAP_FAILED);
1358
1359 /* Assert that they did not merge. */
1360 ASSERT_TRUE(find_vma_procmap(procmap, ptr));
1361 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
1362 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 5 * page_size);
1363 }
1364
TEST_F(merge_with_fork,mremap_faulted_to_unfaulted_prev)1365 TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev)
1366 {
1367 struct procmap_fd *procmap = &self->procmap;
1368 unsigned int page_size = self->page_size;
1369 unsigned long offset;
1370 char *ptr_a, *ptr_b;
1371
1372 /*
1373 * mremap() such that A and B merge:
1374 *
1375 * |------------|
1376 * | \ |
1377 * |-----------| | / |---------|
1378 * | unfaulted | v \ | faulted |
1379 * |-----------| / |---------|
1380 * B \ A
1381 */
1382
1383 /* Map VMA A into place. */
1384 ptr_a = mmap(&self->carveout[page_size + 3 * page_size],
1385 3 * page_size,
1386 PROT_READ | PROT_WRITE,
1387 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
1388 ASSERT_NE(ptr_a, MAP_FAILED);
1389 /* Fault it in. */
1390 ptr_a[0] = 'x';
1391
1392 if (variant->forked) {
1393 pid_t pid = do_fork(&self->procmap);
1394
1395 ASSERT_NE(pid, -1);
1396 if (pid != 0)
1397 return;
1398 }
1399
1400 /*
1401 * Now move it out of the way so we can place VMA B in position,
1402 * unfaulted.
1403 */
1404 ptr_a = mremap(ptr_a, 3 * page_size, 3 * page_size,
1405 MREMAP_FIXED | MREMAP_MAYMOVE, &self->carveout[20 * page_size]);
1406 ASSERT_NE(ptr_a, MAP_FAILED);
1407
1408 /* Map VMA B into place. */
1409 ptr_b = mmap(&self->carveout[page_size], 3 * page_size,
1410 PROT_READ | PROT_WRITE,
1411 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
1412 ASSERT_NE(ptr_b, MAP_FAILED);
1413
1414 /*
1415 * Now move VMA A into position with MREMAP_DONTUNMAP to catch incorrect
1416 * anon_vma propagation.
1417 */
1418 ptr_a = mremap(ptr_a, 3 * page_size, 3 * page_size,
1419 MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP,
1420 &self->carveout[page_size + 3 * page_size]);
1421 ASSERT_NE(ptr_a, MAP_FAILED);
1422
1423 /* The VMAs should have merged, if not forked. */
1424 ASSERT_TRUE(find_vma_procmap(procmap, ptr_b));
1425 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_b);
1426
1427 offset = variant->forked ? 3 * page_size : 6 * page_size;
1428 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_b + offset);
1429 }
1430
TEST_F(merge_with_fork,mremap_faulted_to_unfaulted_next)1431 TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_next)
1432 {
1433 struct procmap_fd *procmap = &self->procmap;
1434 unsigned int page_size = self->page_size;
1435 unsigned long offset;
1436 char *ptr_a, *ptr_b;
1437
1438 /*
1439 * mremap() such that A and B merge:
1440 *
1441 * |---------------------------|
1442 * | \ |
1443 * | |-----------| / |---------|
1444 * v | unfaulted | \ | faulted |
1445 * |-----------| / |---------|
1446 * B \ A
1447 *
1448 * Then unmap VMA A to trigger the bug.
1449 */
1450
1451 /* Map VMA A into place. */
1452 ptr_a = mmap(&self->carveout[page_size], 3 * page_size,
1453 PROT_READ | PROT_WRITE,
1454 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
1455 ASSERT_NE(ptr_a, MAP_FAILED);
1456 /* Fault it in. */
1457 ptr_a[0] = 'x';
1458
1459 if (variant->forked) {
1460 pid_t pid = do_fork(&self->procmap);
1461
1462 ASSERT_NE(pid, -1);
1463 if (pid != 0)
1464 return;
1465 }
1466
1467 /*
1468 * Now move it out of the way so we can place VMA B in position,
1469 * unfaulted.
1470 */
1471 ptr_a = mremap(ptr_a, 3 * page_size, 3 * page_size,
1472 MREMAP_FIXED | MREMAP_MAYMOVE, &self->carveout[20 * page_size]);
1473 ASSERT_NE(ptr_a, MAP_FAILED);
1474
1475 /* Map VMA B into place. */
1476 ptr_b = mmap(&self->carveout[page_size + 3 * page_size], 3 * page_size,
1477 PROT_READ | PROT_WRITE,
1478 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
1479 ASSERT_NE(ptr_b, MAP_FAILED);
1480
1481 /*
1482 * Now move VMA A into position with MREMAP_DONTUNMAP to catch incorrect
1483 * anon_vma propagation.
1484 */
1485 ptr_a = mremap(ptr_a, 3 * page_size, 3 * page_size,
1486 MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP,
1487 &self->carveout[page_size]);
1488 ASSERT_NE(ptr_a, MAP_FAILED);
1489
1490 /* The VMAs should have merged, if not forked. */
1491 ASSERT_TRUE(find_vma_procmap(procmap, ptr_a));
1492 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_a);
1493 offset = variant->forked ? 3 * page_size : 6 * page_size;
1494 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_a + offset);
1495 }
1496
TEST_F(merge_with_fork,mremap_faulted_to_unfaulted_prev_unfaulted_next)1497 TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev_unfaulted_next)
1498 {
1499 struct procmap_fd *procmap = &self->procmap;
1500 unsigned int page_size = self->page_size;
1501 unsigned long offset;
1502 char *ptr_a, *ptr_b, *ptr_c;
1503
1504 /*
1505 * mremap() with MREMAP_DONTUNMAP such that A, B and C merge:
1506 *
1507 * |---------------------------|
1508 * | \ |
1509 * |-----------| | |-----------| / |---------|
1510 * | unfaulted | v | unfaulted | \ | faulted |
1511 * |-----------| |-----------| / |---------|
1512 * A C \ B
1513 */
1514
1515 /* Map VMA B into place. */
1516 ptr_b = mmap(&self->carveout[page_size + 3 * page_size], 3 * page_size,
1517 PROT_READ | PROT_WRITE,
1518 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
1519 ASSERT_NE(ptr_b, MAP_FAILED);
1520 /* Fault it in. */
1521 ptr_b[0] = 'x';
1522
1523 if (variant->forked) {
1524 pid_t pid = do_fork(&self->procmap);
1525
1526 ASSERT_NE(pid, -1);
1527 if (pid != 0)
1528 return;
1529 }
1530
1531 /*
1532 * Now move it out of the way so we can place VMAs A, C in position,
1533 * unfaulted.
1534 */
1535 ptr_b = mremap(ptr_b, 3 * page_size, 3 * page_size,
1536 MREMAP_FIXED | MREMAP_MAYMOVE, &self->carveout[20 * page_size]);
1537 ASSERT_NE(ptr_b, MAP_FAILED);
1538
1539 /* Map VMA A into place. */
1540
1541 ptr_a = mmap(&self->carveout[page_size], 3 * page_size,
1542 PROT_READ | PROT_WRITE,
1543 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
1544 ASSERT_NE(ptr_a, MAP_FAILED);
1545
1546 /* Map VMA C into place. */
1547 ptr_c = mmap(&self->carveout[page_size + 3 * page_size + 3 * page_size],
1548 3 * page_size, PROT_READ | PROT_WRITE,
1549 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
1550 ASSERT_NE(ptr_c, MAP_FAILED);
1551
1552 /*
1553 * Now move VMA B into position with MREMAP_DONTUNMAP to catch incorrect
1554 * anon_vma propagation.
1555 */
1556 ptr_b = mremap(ptr_b, 3 * page_size, 3 * page_size,
1557 MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP,
1558 &self->carveout[page_size + 3 * page_size]);
1559 ASSERT_NE(ptr_b, MAP_FAILED);
1560
1561 /* The VMAs should have merged, if not forked. */
1562 ASSERT_TRUE(find_vma_procmap(procmap, ptr_a));
1563 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_a);
1564 offset = variant->forked ? 3 * page_size : 9 * page_size;
1565 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_a + offset);
1566
1567 /* If forked, B and C should also not have merged. */
1568 if (variant->forked) {
1569 ASSERT_TRUE(find_vma_procmap(procmap, ptr_b));
1570 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_b);
1571 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_b + 3 * page_size);
1572 }
1573 }
1574
TEST_F(merge_with_fork,mremap_faulted_to_unfaulted_prev_faulted_next)1575 TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev_faulted_next)
1576 {
1577 struct procmap_fd *procmap = &self->procmap;
1578 unsigned int page_size = self->page_size;
1579 char *ptr_a, *ptr_b, *ptr_bc;
1580
1581 /*
1582 * mremap() with MREMAP_DONTUNMAP such that A, B and C merge:
1583 *
1584 * |---------------------------|
1585 * | \ |
1586 * |-----------| | |-----------| / |---------|
1587 * | unfaulted | v | faulted | \ | faulted |
1588 * |-----------| |-----------| / |---------|
1589 * A C \ B
1590 */
1591
1592 /*
1593 * Map VMA B and C into place. We have to map them together so their
1594 * anon_vma is the same and the vma->vm_pgoff's are correctly aligned.
1595 */
1596 ptr_bc = mmap(&self->carveout[page_size + 3 * page_size],
1597 3 * page_size + 3 * page_size,
1598 PROT_READ | PROT_WRITE,
1599 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
1600 ASSERT_NE(ptr_bc, MAP_FAILED);
1601
1602 /* Fault it in. */
1603 ptr_bc[0] = 'x';
1604
1605 if (variant->forked) {
1606 pid_t pid = do_fork(&self->procmap);
1607
1608 ASSERT_NE(pid, -1);
1609 if (pid != 0)
1610 return;
1611 }
1612
1613 /*
1614 * Now move VMA B out the way (splitting VMA BC) so we can place VMA A
1615 * in position, unfaulted, and leave the remainder of the VMA we just
1616 * moved in place, faulted, as VMA C.
1617 */
1618 ptr_b = mremap(ptr_bc, 3 * page_size, 3 * page_size,
1619 MREMAP_FIXED | MREMAP_MAYMOVE, &self->carveout[20 * page_size]);
1620 ASSERT_NE(ptr_b, MAP_FAILED);
1621
1622 /* Map VMA A into place. */
1623 ptr_a = mmap(&self->carveout[page_size], 3 * page_size,
1624 PROT_READ | PROT_WRITE,
1625 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
1626 ASSERT_NE(ptr_a, MAP_FAILED);
1627
1628 /*
1629 * Now move VMA B into position with MREMAP_DONTUNMAP to catch incorrect
1630 * anon_vma propagation.
1631 */
1632 ptr_b = mremap(ptr_b, 3 * page_size, 3 * page_size,
1633 MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP,
1634 &self->carveout[page_size + 3 * page_size]);
1635 ASSERT_NE(ptr_b, MAP_FAILED);
1636
1637 /* The VMAs should have merged. A,B,C if unforked, B, C if forked. */
1638 if (variant->forked) {
1639 ASSERT_TRUE(find_vma_procmap(procmap, ptr_b));
1640 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_b);
1641 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_b + 6 * page_size);
1642 } else {
1643 ASSERT_TRUE(find_vma_procmap(procmap, ptr_a));
1644 ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_a);
1645 ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_a + 9 * page_size);
1646 }
1647 }
1648
1649 TEST_HARNESS_MAIN
1650