mmap.c (3e01310d29a700b99067cb0d6ba21284f2389308) mmap.c (fc21959f74bc1138b28e90a02ec224ab8626111e)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * mm/mmap.c
4 *
5 * Written by obz.
6 *
7 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
8 */

--- 1357 unchanged lines hidden (view full) ---

1366 struct vm_area_struct *vma = NULL;
1367 struct vm_area_struct *next, *prev, *merge;
1368 pgoff_t pglen = PHYS_PFN(len);
1369 unsigned long charged = 0;
1370 struct vma_munmap_struct vms;
1371 struct ma_state mas_detach;
1372 struct maple_tree mt_detach;
1373 unsigned long end = addr + len;
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * mm/mmap.c
4 *
5 * Written by obz.
6 *
7 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
8 */

--- 1357 unchanged lines hidden (view full) ---

1366 struct vm_area_struct *vma = NULL;
1367 struct vm_area_struct *next, *prev, *merge;
1368 pgoff_t pglen = PHYS_PFN(len);
1369 unsigned long charged = 0;
1370 struct vma_munmap_struct vms;
1371 struct ma_state mas_detach;
1372 struct maple_tree mt_detach;
1373 unsigned long end = addr + len;
1374 unsigned long merge_start = addr, merge_end = end;
1375 bool writable_file_mapping = false;
1376 int error = -ENOMEM;
1377 VMA_ITERATOR(vmi, mm, addr);
1378 VMG_STATE(vmg, mm, &vmi, addr, end, vm_flags, pgoff);
1379
1380 vmg.file = file;
1381 /* Find the first overlapping VMA */
1382 vma = vma_find(&vmi, end);

--- 36 unchanged lines hidden (view full) ---

1419 }
1420
1421 if (vm_flags & VM_SPECIAL)
1422 goto cannot_expand;
1423
1424 /* Attempt to expand an old mapping */
1425 /* Check next */
1426 if (next && next->vm_start == end && can_vma_merge_before(&vmg)) {
1374 bool writable_file_mapping = false;
1375 int error = -ENOMEM;
1376 VMA_ITERATOR(vmi, mm, addr);
1377 VMG_STATE(vmg, mm, &vmi, addr, end, vm_flags, pgoff);
1378
1379 vmg.file = file;
1380 /* Find the first overlapping VMA */
1381 vma = vma_find(&vmi, end);

--- 36 unchanged lines hidden (view full) ---

1418 }
1419
1420 if (vm_flags & VM_SPECIAL)
1421 goto cannot_expand;
1422
1423 /* Attempt to expand an old mapping */
1424 /* Check next */
1425 if (next && next->vm_start == end && can_vma_merge_before(&vmg)) {
1427 merge_end = next->vm_end;
1428 vma = next;
1426 vmg.end = next->vm_end;
1427 vma = vmg.vma = next;
1429 vmg.pgoff = next->vm_pgoff - pglen;
1430 /*
1431 * We set this here so if we will merge with the previous VMA in
1432 * the code below, can_vma_merge_after() ensures anon_vma
1433 * compatibility between prev and next.
1434 */
1435 vmg.anon_vma = vma->anon_vma;
1436 vmg.uffd_ctx = vma->vm_userfaultfd_ctx;
1437 }
1438
1439 /* Check prev */
1440 if (prev && prev->vm_end == addr && can_vma_merge_after(&vmg)) {
1428 vmg.pgoff = next->vm_pgoff - pglen;
1429 /*
1430 * We set this here so if we will merge with the previous VMA in
1431 * the code below, can_vma_merge_after() ensures anon_vma
1432 * compatibility between prev and next.
1433 */
1434 vmg.anon_vma = vma->anon_vma;
1435 vmg.uffd_ctx = vma->vm_userfaultfd_ctx;
1436 }
1437
1438 /* Check prev */
1439 if (prev && prev->vm_end == addr && can_vma_merge_after(&vmg)) {
1441 merge_start = prev->vm_start;
1442 vma = prev;
1440 vmg.start = prev->vm_start;
1441 vma = vmg.vma = prev;
1443 vmg.pgoff = prev->vm_pgoff;
1444 vma_prev(&vmi); /* Equivalent to going to the previous range */
1445 }
1446
1447 if (vma) {
1448 /* Actually expand, if possible */
1442 vmg.pgoff = prev->vm_pgoff;
1443 vma_prev(&vmi); /* Equivalent to going to the previous range */
1444 }
1445
1446 if (vma) {
1447 /* Actually expand, if possible */
1449 if (!vma_expand(&vmi, vma, merge_start, merge_end, vmg.pgoff, next)) {
1448 if (!vma_expand(&vmg)) {
1450 khugepaged_enter_vma(vma, vm_flags);
1451 goto expanded;
1452 }
1453
1454 /* If the expand fails, then reposition the vma iterator */
1455 if (unlikely(vma == prev))
1456 vma_iter_set(&vmi, addr);
1457 }

--- 857 unchanged lines hidden (view full) ---

2315
2316 struct mm_struct *mm = vma->vm_mm;
2317 unsigned long old_start = vma->vm_start;
2318 unsigned long old_end = vma->vm_end;
2319 unsigned long length = old_end - old_start;
2320 unsigned long new_start = old_start - shift;
2321 unsigned long new_end = old_end - shift;
2322 VMA_ITERATOR(vmi, mm, new_start);
1449 khugepaged_enter_vma(vma, vm_flags);
1450 goto expanded;
1451 }
1452
1453 /* If the expand fails, then reposition the vma iterator */
1454 if (unlikely(vma == prev))
1455 vma_iter_set(&vmi, addr);
1456 }

--- 857 unchanged lines hidden (view full) ---

2314
2315 struct mm_struct *mm = vma->vm_mm;
2316 unsigned long old_start = vma->vm_start;
2317 unsigned long old_end = vma->vm_end;
2318 unsigned long length = old_end - old_start;
2319 unsigned long new_start = old_start - shift;
2320 unsigned long new_end = old_end - shift;
2321 VMA_ITERATOR(vmi, mm, new_start);
2322 VMG_STATE(vmg, mm, &vmi, new_start, old_end, 0, vma->vm_pgoff);
2323 struct vm_area_struct *next;
2324 struct mmu_gather tlb;
2325
2326 BUG_ON(new_start > new_end);
2327
2328 /*
2329 * ensure there are no vmas between where we want to go
2330 * and where we are
2331 */
2332 if (vma != vma_next(&vmi))
2333 return -EFAULT;
2334
2335 vma_iter_prev_range(&vmi);
2336 /*
2337 * cover the whole range: [new_start, old_end)
2338 */
2323 struct vm_area_struct *next;
2324 struct mmu_gather tlb;
2325
2326 BUG_ON(new_start > new_end);
2327
2328 /*
2329 * ensure there are no vmas between where we want to go
2330 * and where we are
2331 */
2332 if (vma != vma_next(&vmi))
2333 return -EFAULT;
2334
2335 vma_iter_prev_range(&vmi);
2336 /*
2337 * cover the whole range: [new_start, old_end)
2338 */
2339 if (vma_expand(&vmi, vma, new_start, old_end, vma->vm_pgoff, NULL))
2339 vmg.vma = vma;
2340 if (vma_expand(&vmg))
2340 return -ENOMEM;
2341
2342 /*
2343 * move the page tables downwards, on failure we rely on
2344 * process cleanup to remove whatever mess we made.
2345 */
2346 if (length != move_page_tables(vma, old_start,
2347 vma, new_start, length, false, true))

--- 27 unchanged lines hidden --- 		2341 return -ENOMEM;
2342
2343 /*
2344 * move the page tables downwards, on failure we rely on
2345 * process cleanup to remove whatever mess we made.
2346 */
2347 if (length != move_page_tables(vma, old_start,
2348 vma, new_start, length, false, true))

--- 27 unchanged lines hidden ---