vma.c (01c373e9a5ce2273812eaf83036c5357829fb3f7) vma.c (659c55ef981bb63355a65ffc3b3b5cad562b806a)
1// SPDX-License-Identifier: GPL-2.0-or-later
2
3/*
4 * VMA-specific functions.
5 */
6
7#include "vma_internal.h"
8#include "vma.h"

--- 1157 unchanged lines hidden (view full) ---

1166/*
1167 * vms_gather_munmap_vmas() - Put all VMAs within a range into a maple tree
1168 * for removal at a later date. Handles splitting first and last if necessary
1169 * and marking the vmas as isolated.
1170 *
1171 * @vms: The vma munmap struct
1172 * @mas_detach: The maple state tracking the detached tree
1173 *
1// SPDX-License-Identifier: GPL-2.0-or-later
2
3/*
4 * VMA-specific functions.
5 */
6
7#include "vma_internal.h"
8#include "vma.h"

--- 1157 unchanged lines hidden (view full) ---

1166/*
1167 * vms_gather_munmap_vmas() - Put all VMAs within a range into a maple tree
1168 * for removal at a later date. Handles splitting first and last if necessary
1169 * and marking the vmas as isolated.
1170 *
1171 * @vms: The vma munmap struct
1172 * @mas_detach: The maple state tracking the detached tree
1173 *
1174 * Return: 0 on success, -EPERM on mseal vmas, -ENOMEM otherwise
1174 * Return: 0 on success, error otherwise
1175 */
1176int vms_gather_munmap_vmas(struct vma_munmap_struct *vms,
1177 struct ma_state *mas_detach)
1178{
1179 struct vm_area_struct *next = NULL;
1175 */
1176int vms_gather_munmap_vmas(struct vma_munmap_struct *vms,
1177 struct ma_state *mas_detach)
1178{
1179 struct vm_area_struct *next = NULL;
1180 int error = -ENOMEM;
1180 int error;
1181
1182 /*
1183 * If we need to split any vma, do it now to save pain later.
1184 * Does it split the first one?
1185 */
1186 if (vms->start > vms->vma->vm_start) {
1187
1188 /*
1189 * Make sure that map_count on return from munmap() will
1190 * not exceed its limit; but let map_count go just above
1191 * its limit temporarily, to help free resources as expected.
1192 */
1193 if (vms->end < vms->vma->vm_end &&
1181
1182 /*
1183 * If we need to split any vma, do it now to save pain later.
1184 * Does it split the first one?
1185 */
1186 if (vms->start > vms->vma->vm_start) {
1187
1188 /*
1189 * Make sure that map_count on return from munmap() will
1190 * not exceed its limit; but let map_count go just above
1191 * its limit temporarily, to help free resources as expected.
1192 */
1193 if (vms->end < vms->vma->vm_end &&
1194 vms->vma->vm_mm->map_count >= sysctl_max_map_count)
1194 vms->vma->vm_mm->map_count >= sysctl_max_map_count) {
1195 error = -ENOMEM;
1195 goto map_count_exceeded;
1196 goto map_count_exceeded;
1197 }
1196
1197 /* Don't bother splitting the VMA if we can't unmap it anyway */
1198 if (!can_modify_vma(vms->vma)) {
1199 error = -EPERM;
1200 goto start_split_failed;
1201 }
1202
1198
1199 /* Don't bother splitting the VMA if we can't unmap it anyway */
1200 if (!can_modify_vma(vms->vma)) {
1201 error = -EPERM;
1202 goto start_split_failed;
1203 }
1204
1203 if (__split_vma(vms->vmi, vms->vma, vms->start, 1))
1205 error = __split_vma(vms->vmi, vms->vma, vms->start, 1);
1206 if (error)
1204 goto start_split_failed;
1205 }
1206 vms->prev = vma_prev(vms->vmi);
1207 if (vms->prev)
1208 vms->unmap_start = vms->prev->vm_end;
1209
1210 /*
1211 * Detach a range of VMAs from the mm. Using next as a temp variable as
1212 * it is always overwritten.
1213 */
1214 for_each_vma_range(*(vms->vmi), next, vms->end) {
1215 long nrpages;
1216
1217 if (!can_modify_vma(next)) {
1218 error = -EPERM;
1219 goto modify_vma_failed;
1220 }
1221 /* Does it split the end? */
1222 if (next->vm_end > vms->end) {
1207 goto start_split_failed;
1208 }
1209 vms->prev = vma_prev(vms->vmi);
1210 if (vms->prev)
1211 vms->unmap_start = vms->prev->vm_end;
1212
1213 /*
1214 * Detach a range of VMAs from the mm. Using next as a temp variable as
1215 * it is always overwritten.
1216 */
1217 for_each_vma_range(*(vms->vmi), next, vms->end) {
1218 long nrpages;
1219
1220 if (!can_modify_vma(next)) {
1221 error = -EPERM;
1222 goto modify_vma_failed;
1223 }
1224 /* Does it split the end? */
1225 if (next->vm_end > vms->end) {
1223 if (__split_vma(vms->vmi, next, vms->end, 0))
1226 error = __split_vma(vms->vmi, next, vms->end, 0);
1227 if (error)
1224 goto end_split_failed;
1225 }
1226 vma_start_write(next);
1227 mas_set(mas_detach, vms->vma_count++);
1228 goto end_split_failed;
1229 }
1230 vma_start_write(next);
1231 mas_set(mas_detach, vms->vma_count++);
1228 if (mas_store_gfp(mas_detach, next, GFP_KERNEL))
1232 error = mas_store_gfp(mas_detach, next, GFP_KERNEL);
1233 if (error)
1229 goto munmap_gather_failed;
1230
1231 vma_mark_detached(next, true);
1232 nrpages = vma_pages(next);
1233
1234 vms->nr_pages += nrpages;
1235 if (next->vm_flags & VM_LOCKED)
1236 vms->locked_vm += nrpages;

--- 13 unchanged lines hidden (view full) ---

1250 * If userfaultfd_unmap_prep returns an error the vmas
1251 * will remain split, but userland will get a
1252 * highly unexpected error anyway. This is no
1253 * different than the case where the first of the two
1254 * __split_vma fails, but we don't undo the first
1255 * split, despite we could. This is unlikely enough
1256 * failure that it's not worth optimizing it for.
1257 */
1234 goto munmap_gather_failed;
1235
1236 vma_mark_detached(next, true);
1237 nrpages = vma_pages(next);
1238
1239 vms->nr_pages += nrpages;
1240 if (next->vm_flags & VM_LOCKED)
1241 vms->locked_vm += nrpages;

--- 13 unchanged lines hidden (view full) ---

1255 * If userfaultfd_unmap_prep returns an error the vmas
1256 * will remain split, but userland will get a
1257 * highly unexpected error anyway. This is no
1258 * different than the case where the first of the two
1259 * __split_vma fails, but we don't undo the first
1260 * split, despite we could. This is unlikely enough
1261 * failure that it's not worth optimizing it for.
1262 */
1258 if (userfaultfd_unmap_prep(next, vms->start, vms->end,
1259 vms->uf))
1263 error = userfaultfd_unmap_prep(next, vms->start,
1264 vms->end, vms->uf);
1265 if (error)
1260 goto userfaultfd_error;
1261 }
1262#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
1263 BUG_ON(next->vm_start < vms->start);
1264 BUG_ON(next->vm_start > vms->end);
1265#endif
1266 }
1267

--- 795 unchanged lines hidden --- 		1266 goto userfaultfd_error;
1267 }
1268#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
1269 BUG_ON(next->vm_start < vms->start);
1270 BUG_ON(next->vm_start > vms->end);
1271#endif
1272 }
1273

--- 795 unchanged lines hidden ---