vma.c (cc8cb3697a8d8eabe1fb9acb8768b11c1ab607d8) vma.c (01c373e9a5ce2273812eaf83036c5357829fb3f7)
1// SPDX-License-Identifier: GPL-2.0-or-later
2
3/*
4 * VMA-specific functions.
5 */
6
7#include "vma_internal.h"
8#include "vma.h"
9
10static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_next)
11{
12 struct vm_area_struct *vma = merge_next ? vmg->next : vmg->prev;
1// SPDX-License-Identifier: GPL-2.0-or-later
2
3/*
4 * VMA-specific functions.
5 */
6
7#include "vma_internal.h"
8#include "vma.h"
9
10static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_next)
11{
12 struct vm_area_struct *vma = merge_next ? vmg->next : vmg->prev;
13 /*
14 * If the vma has a ->close operation then the driver probably needs to
15 * release per-vma resources, so we don't attempt to merge those if the
16 * caller indicates the current vma may be removed as part of the merge,
17 * which is the case if we are attempting to merge the next VMA into
18 * this one.
19 */
20 bool may_remove_vma = merge_next;
21
22 if (!mpol_equal(vmg->policy, vma_policy(vma)))
23 return false;
24 /*
25 * VM_SOFTDIRTY should not prevent from VMA merging, if we
26 * match the flags but dirty bit -- the caller should mark
27 * merged VMA as dirty. If dirty bit won't be excluded from
28 * comparison, we increase pressure on the memory system forcing
29 * the kernel to generate new VMAs when old one could be
30 * extended instead.
31 */
32 if ((vma->vm_flags ^ vmg->flags) & ~VM_SOFTDIRTY)
33 return false;
34 if (vma->vm_file != vmg->file)
35 return false;
13
14 if (!mpol_equal(vmg->policy, vma_policy(vma)))
15 return false;
16 /*
17 * VM_SOFTDIRTY should not prevent from VMA merging, if we
18 * match the flags but dirty bit -- the caller should mark
19 * merged VMA as dirty. If dirty bit won't be excluded from
20 * comparison, we increase pressure on the memory system forcing
21 * the kernel to generate new VMAs when old one could be
22 * extended instead.
23 */
24 if ((vma->vm_flags ^ vmg->flags) & ~VM_SOFTDIRTY)
25 return false;
26 if (vma->vm_file != vmg->file)
27 return false;
36 if (may_remove_vma && vma->vm_ops && vma->vm_ops->close)
37 return false;
38 if (!is_mergeable_vm_userfaultfd_ctx(vma, vmg->uffd_ctx))
39 return false;
40 if (!anon_vma_name_eq(anon_vma_name(vma), vmg->anon_name))
41 return false;
42 return true;
43}
44
45static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1,

--- 581 unchanged lines hidden (view full) ---

627 }
628 }
629
630 vma_complete(&vp, vmg->vmi, vmg->vma->vm_mm);
631
632 return 0;
633}
634
28 if (!is_mergeable_vm_userfaultfd_ctx(vma, vmg->uffd_ctx))
29 return false;
30 if (!anon_vma_name_eq(anon_vma_name(vma), vmg->anon_name))
31 return false;
32 return true;
33}
34
35static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1,

--- 581 unchanged lines hidden (view full) ---

617 }
618 }
619
620 vma_complete(&vp, vmg->vmi, vmg->vma->vm_mm);
621
622 return 0;
623}
624
625/* We can only remove VMAs when merging if they do not have a close hook. */
626static bool can_merge_remove_vma(struct vm_area_struct *vma)
627{
628 return !vma->vm_ops || !vma->vm_ops->close;
629}
630
635/*
636 * vma_merge_existing_range - Attempt to merge VMAs based on a VMA having its
637 * attributes modified.
638 *
639 * @vmg: Describes the modifications being made to a VMA and associated
640 * metadata.
641 *
642 * When the attributes of a range within a VMA change, then it might be possible

--- 77 unchanged lines hidden (view full) ---

720 if (merge_left) /* If merging prev, position iterator there. */
721 vma_prev(vmg->vmi);
722 else if (!merge_right) /* If we have nothing to merge, abort. */
723 return NULL;
724
725 merge_both = merge_left && merge_right;
726 /* If we span the entire VMA, a merge implies it will be deleted. */
727 merge_will_delete_vma = left_side && right_side;
631/*
632 * vma_merge_existing_range - Attempt to merge VMAs based on a VMA having its
633 * attributes modified.
634 *
635 * @vmg: Describes the modifications being made to a VMA and associated
636 * metadata.
637 *
638 * When the attributes of a range within a VMA change, then it might be possible

--- 77 unchanged lines hidden (view full) ---

716 if (merge_left) /* If merging prev, position iterator there. */
717 vma_prev(vmg->vmi);
718 else if (!merge_right) /* If we have nothing to merge, abort. */
719 return NULL;
720
721 merge_both = merge_left && merge_right;
722 /* If we span the entire VMA, a merge implies it will be deleted. */
723 merge_will_delete_vma = left_side && right_side;
724
728 /*
725 /*
726 * If we need to remove vma in its entirety but are unable to do so,
727 * we have no sensible recourse but to abort the merge.
728 */
729 if (merge_will_delete_vma && !can_merge_remove_vma(vma))
730 return NULL;
731
732 /*
729 * If we merge both VMAs, then next is also deleted. This implies
730 * merge_will_delete_vma also.
731 */
732 merge_will_delete_next = merge_both;
733
733 * If we merge both VMAs, then next is also deleted. This implies
734 * merge_will_delete_vma also.
735 */
736 merge_will_delete_next = merge_both;
737
738 /*
739 * If we cannot delete next, then we can reduce the operation to merging
740 * prev and vma (thereby deleting vma).
741 */
742 if (merge_will_delete_next && !can_merge_remove_vma(next)) {
743 merge_will_delete_next = false;
744 merge_right = false;
745 merge_both = false;
746 }
747
734 /* No matter what happens, we will be adjusting vma. */
735 vma_start_write(vma);
736
737 if (merge_left)
738 vma_start_write(prev);
739
740 if (merge_right)
741 vma_start_write(next);

--- 25 unchanged lines hidden (view full) ---

767 * prev vma
768 * extend shrink/delete
769 */
770
771 vmg->vma = prev;
772 vmg->start = prev->vm_start;
773 vmg->pgoff = prev->vm_pgoff;
774
748 /* No matter what happens, we will be adjusting vma. */
749 vma_start_write(vma);
750
751 if (merge_left)
752 vma_start_write(prev);
753
754 if (merge_right)
755 vma_start_write(next);

--- 25 unchanged lines hidden (view full) ---

781 * prev vma
782 * extend shrink/delete
783 */
784
785 vmg->vma = prev;
786 vmg->start = prev->vm_start;
787 vmg->pgoff = prev->vm_pgoff;
788
775 if (merge_will_delete_vma) {
776 /*
777 * can_vma_merge_after() assumed we would not be
778 * removing vma, so it skipped the check for
779 * vm_ops->close, but we are removing vma.
780 */
781 if (vma->vm_ops && vma->vm_ops->close)
782 err = -EINVAL;
783 } else {
789 if (!merge_will_delete_vma) {
784 adjust = vma;
785 adj_start = vmg->end - vma->vm_start;
786 }
787
790 adjust = vma;
791 adj_start = vmg->end - vma->vm_start;
792 }
793
788 if (!err)
789 err = dup_anon_vma(prev, vma, &anon_dup);
794 err = dup_anon_vma(prev, vma, &anon_dup);
790 } else { /* merge_right */
791 /*
792 * |<----->| OR
793 * |<--------->|
794 * *************-------|
795 * vma next
796 * shrink/delete extend
797 */

--- 137 unchanged lines hidden (view full) ---

935 }
936
937 /* If we can merge with the previous VMA, adjust vmg accordingly. */
938 if (can_merge_left) {
939 vmg->start = prev->vm_start;
940 vmg->vma = prev;
941 vmg->pgoff = prev->vm_pgoff;
942
795 } else { /* merge_right */
796 /*
797 * |<----->| OR
798 * |<--------->|
799 * *************-------|
800 * vma next
801 * shrink/delete extend
802 */

--- 137 unchanged lines hidden (view full) ---

940 }
941
942 /* If we can merge with the previous VMA, adjust vmg accordingly. */
943 if (can_merge_left) {
944 vmg->start = prev->vm_start;
945 vmg->vma = prev;
946 vmg->pgoff = prev->vm_pgoff;
947
948 /*
949 * If this merge would result in removal of the next VMA but we
950 * are not permitted to do so, reduce the operation to merging
951 * prev and vma.
952 */
953 if (can_merge_right && !can_merge_remove_vma(next))
954 vmg->end = end;
955
943 vma_prev(vmg->vmi); /* Equivalent to going to the previous range */
944 }
945
946 /*
947 * Now try to expand adjacent VMA(s). This takes care of removing the
948 * following VMA if we have VMAs on both sides.
949 */
950 if (vmg->vma && !vma_expand(vmg)) {

--- 38 unchanged lines hidden (view full) ---

989
990 mmap_assert_write_locked(vmg->mm);
991
992 vma_start_write(vma);
993 if (next && (vma != next) && (vmg->end == next->vm_end)) {
994 int ret;
995
996 remove_next = true;
956 vma_prev(vmg->vmi); /* Equivalent to going to the previous range */
957 }
958
959 /*
960 * Now try to expand adjacent VMA(s). This takes care of removing the
961 * following VMA if we have VMAs on both sides.
962 */
963 if (vmg->vma && !vma_expand(vmg)) {

--- 38 unchanged lines hidden (view full) ---

1002
1003 mmap_assert_write_locked(vmg->mm);
1004
1005 vma_start_write(vma);
1006 if (next && (vma != next) && (vmg->end == next->vm_end)) {
1007 int ret;
1008
1009 remove_next = true;
1010 /* This should already have been checked by this point. */
1011 VM_WARN_ON(!can_merge_remove_vma(next));
997 vma_start_write(next);
998 ret = dup_anon_vma(vma, next, &anon_dup);
999 if (ret)
1000 return ret;
1001 }
1002
1003 /* Not merging but overwriting any part of next is not handled. */
1004 VM_WARN_ON(next && !remove_next &&

--- 1043 unchanged lines hidden --- 		1012 vma_start_write(next);
1013 ret = dup_anon_vma(vma, next, &anon_dup);
1014 if (ret)
1015 return ret;
1016 }
1017
1018 /* Not merging but overwriting any part of next is not handled. */
1019 VM_WARN_ON(next && !remove_next &&

--- 1043 unchanged lines hidden ---