mmap.c (94f59ea591f17d5fb77f68e820b27522596a7e9e) mmap.c (f8d112a4e657c65c888e6b8a8435ef61a66e4ab8)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * mm/mmap.c
4 *
5 * Written by obz.
6 *
7 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
8 */

--- 1359 unchanged lines hidden (view full) ---

1368 unsigned long charged = 0;
1369 struct vma_munmap_struct vms;
1370 struct ma_state mas_detach;
1371 struct maple_tree mt_detach;
1372 unsigned long end = addr + len;
1373 unsigned long merge_start = addr, merge_end = end;
1374 bool writable_file_mapping = false;
1375 pgoff_t vm_pgoff;
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * mm/mmap.c
4 *
5 * Written by obz.
6 *
7 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
8 */

--- 1359 unchanged lines hidden (view full) ---

1368 unsigned long charged = 0;
1369 struct vma_munmap_struct vms;
1370 struct ma_state mas_detach;
1371 struct maple_tree mt_detach;
1372 unsigned long end = addr + len;
1373 unsigned long merge_start = addr, merge_end = end;
1374 bool writable_file_mapping = false;
1375 pgoff_t vm_pgoff;
1376 int error;
1376 int error = -ENOMEM;
1377 VMA_ITERATOR(vmi, mm, addr);
1377 VMA_ITERATOR(vmi, mm, addr);
1378 unsigned long nr_pages, nr_accounted;
1378
1379
1379 /* Check against address space limit. */
1380 if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) {
1381 unsigned long nr_pages;
1380 nr_pages = count_vma_pages_range(mm, addr, end, &nr_accounted);
1382
1381
1383 /*
1384 * MAP_FIXED may remove pages of mappings that intersects with
1385 * requested mapping. Account for the pages it would unmap.
1386 */
1387 nr_pages = count_vma_pages_range(mm, addr, end);
1382 /*
1383 * Check against address space limit.
1384 * MAP_FIXED may remove pages of mappings that intersects with requested
1385 * mapping. Account for the pages it would unmap.
1386 */
1387 if (!may_expand_vm(mm, vm_flags, (len >> PAGE_SHIFT) - nr_pages))
1388 return -ENOMEM;
1388
1389
1389 if (!may_expand_vm(mm, vm_flags,
1390 (len >> PAGE_SHIFT) - nr_pages))
1391 return -ENOMEM;
1392 }
1393
1394 /* Find the first overlapping VMA */
1395 vma = vma_find(&vmi, end);
1396 init_vma_munmap(&vms, &vmi, vma, addr, end, uf, /* unlock = */ false);
1397 if (vma) {
1398 mt_init_flags(&mt_detach, vmi.mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
1399 mt_on_stack(mt_detach);
1400 mas_init(&mas_detach, &mt_detach, /* addr = */ 0);
1401 /* Prepare to unmap any existing mapping in the area */
1402 error = vms_gather_munmap_vmas(&vms, &mas_detach);
1403 if (error)
1404 goto gather_failed;
1405
1390 /* Find the first overlapping VMA */
1391 vma = vma_find(&vmi, end);
1392 init_vma_munmap(&vms, &vmi, vma, addr, end, uf, /* unlock = */ false);
1393 if (vma) {
1394 mt_init_flags(&mt_detach, vmi.mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
1395 mt_on_stack(mt_detach);
1396 mas_init(&mas_detach, &mt_detach, /* addr = */ 0);
1397 /* Prepare to unmap any existing mapping in the area */
1398 error = vms_gather_munmap_vmas(&vms, &mas_detach);
1399 if (error)
1400 goto gather_failed;
1401
1406 /* Remove any existing mappings from the vma tree */
1407 error = vma_iter_clear_gfp(&vmi, addr, end, GFP_KERNEL);
1408 if (error)
1409 goto clear_tree_failed;
1410
1411 /* Unmap any existing mapping in the area */
1412 vms_complete_munmap_vmas(&vms, &mas_detach);
1413 next = vms.next;
1414 prev = vms.prev;
1415 vma = NULL;
1416 } else {
1417 next = vma_next(&vmi);
1418 prev = vma_prev(&vmi);
1419 if (prev)
1420 vma_iter_next_range(&vmi);
1421 }
1422
1423 /*
1424 * Private writable mapping: check memory availability
1425 */
1426 if (accountable_mapping(file, vm_flags)) {
1427 charged = len >> PAGE_SHIFT;
1402 next = vms.next;
1403 prev = vms.prev;
1404 vma = NULL;
1405 } else {
1406 next = vma_next(&vmi);
1407 prev = vma_prev(&vmi);
1408 if (prev)
1409 vma_iter_next_range(&vmi);
1410 }
1411
1412 /*
1413 * Private writable mapping: check memory availability
1414 */
1415 if (accountable_mapping(file, vm_flags)) {
1416 charged = len >> PAGE_SHIFT;
1417 charged -= nr_accounted;
1428 if (security_vm_enough_memory_mm(mm, charged))
1418 if (security_vm_enough_memory_mm(mm, charged))
1429 return -ENOMEM;
1419 goto abort_munmap;
1420 vms.nr_accounted = 0;
1430 vm_flags |= VM_ACCOUNT;
1431 }
1432
1433 if (vm_flags & VM_SPECIAL)
1434 goto cannot_expand;
1435
1436 /* Attempt to expand an old mapping */
1437 /* Check next */

--- 32 unchanged lines hidden (view full) ---

1470cannot_expand:
1471
1472 /*
1473 * Determine the object being mapped and call the appropriate
1474 * specific mapper. the address has already been validated, but
1475 * not unmapped, but the maps are removed from the list.
1476 */
1477 vma = vm_area_alloc(mm);
1421 vm_flags |= VM_ACCOUNT;
1422 }
1423
1424 if (vm_flags & VM_SPECIAL)
1425 goto cannot_expand;
1426
1427 /* Attempt to expand an old mapping */
1428 /* Check next */

--- 32 unchanged lines hidden (view full) ---

1461cannot_expand:
1462
1463 /*
1464 * Determine the object being mapped and call the appropriate
1465 * specific mapper. the address has already been validated, but
1466 * not unmapped, but the maps are removed from the list.
1467 */
1468 vma = vm_area_alloc(mm);
1478 if (!vma) {
1479 error = -ENOMEM;
1469 if (!vma)
1480 goto unacct_error;
1470 goto unacct_error;
1481 }
1482
1483 vma_iter_config(&vmi, addr, end);
1484 vma_set_range(vma, addr, end, pgoff);
1485 vm_flags_init(vma, vm_flags);
1486 vma->vm_page_prot = vm_get_page_prot(vm_flags);
1487
1488 if (file) {
1489 vma->vm_file = get_file(file);
1471
1472 vma_iter_config(&vmi, addr, end);
1473 vma_set_range(vma, addr, end, pgoff);
1474 vm_flags_init(vma, vm_flags);
1475 vma->vm_page_prot = vm_get_page_prot(vm_flags);
1476
1477 if (file) {
1478 vma->vm_file = get_file(file);
1479 /*
1480 * call_mmap() may map PTE, so ensure there are no existing PTEs
1481 * call the vm_ops close function if one exists.
1482 */
1483 vms_clean_up_area(&vms, &mas_detach, true);
1490 error = call_mmap(file, vma);
1491 if (error)
1492 goto unmap_and_free_vma;
1493
1494 if (vma_is_shared_maywrite(vma)) {
1495 error = mapping_map_writable(file->f_mapping);
1496 if (error)
1497 goto close_and_free_vma;

--- 74 unchanged lines hidden (view full) ---

1572unmap_writable:
1573 if (writable_file_mapping)
1574 mapping_unmap_writable(file->f_mapping);
1575 file = vma->vm_file;
1576 ksm_add_vma(vma);
1577expanded:
1578 perf_event_mmap(vma);
1579
1484 error = call_mmap(file, vma);
1485 if (error)
1486 goto unmap_and_free_vma;
1487
1488 if (vma_is_shared_maywrite(vma)) {
1489 error = mapping_map_writable(file->f_mapping);
1490 if (error)
1491 goto close_and_free_vma;

--- 74 unchanged lines hidden (view full) ---

1566unmap_writable:
1567 if (writable_file_mapping)
1568 mapping_unmap_writable(file->f_mapping);
1569 file = vma->vm_file;
1570 ksm_add_vma(vma);
1571expanded:
1572 perf_event_mmap(vma);
1573
1574 /* Unmap any existing mapping in the area */
1575 vms_complete_munmap_vmas(&vms, &mas_detach);
1576
1580 vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
1581 if (vm_flags & VM_LOCKED) {
1582 if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) ||
1583 is_vm_hugetlb_page(vma) ||
1584 vma == get_gate_vma(current->mm))
1585 vm_flags_clear(vma, VM_LOCKED_MASK);
1586 else
1587 mm->locked_vm += (len >> PAGE_SHIFT);

--- 12 unchanged lines hidden (view full) ---

1600 vm_flags_set(vma, VM_SOFTDIRTY);
1601
1602 vma_set_page_prot(vma);
1603
1604 validate_mm(mm);
1605 return addr;
1606
1607close_and_free_vma:
1577 vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
1578 if (vm_flags & VM_LOCKED) {
1579 if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) ||
1580 is_vm_hugetlb_page(vma) ||
1581 vma == get_gate_vma(current->mm))
1582 vm_flags_clear(vma, VM_LOCKED_MASK);
1583 else
1584 mm->locked_vm += (len >> PAGE_SHIFT);

--- 12 unchanged lines hidden (view full) ---

1597 vm_flags_set(vma, VM_SOFTDIRTY);
1598
1599 vma_set_page_prot(vma);
1600
1601 validate_mm(mm);
1602 return addr;
1603
1604close_and_free_vma:
1608 if (file && vma->vm_ops && vma->vm_ops->close)
1605 if (file && !vms.closed_vm_ops && vma->vm_ops && vma->vm_ops->close)
1609 vma->vm_ops->close(vma);
1610
1611 if (file || vma->vm_file) {
1612unmap_and_free_vma:
1613 fput(vma->vm_file);
1614 vma->vm_file = NULL;
1615
1616 vma_iter_set(&vmi, vma->vm_end);
1617 /* Undo any partial mapping done by a device driver. */
1618 unmap_region(&vmi.mas, vma, prev, next);
1619 }
1620 if (writable_file_mapping)
1621 mapping_unmap_writable(file->f_mapping);
1622free_vma:
1623 vm_area_free(vma);
1624unacct_error:
1625 if (charged)
1626 vm_unacct_memory(charged);
1627
1606 vma->vm_ops->close(vma);
1607
1608 if (file || vma->vm_file) {
1609unmap_and_free_vma:
1610 fput(vma->vm_file);
1611 vma->vm_file = NULL;
1612
1613 vma_iter_set(&vmi, vma->vm_end);
1614 /* Undo any partial mapping done by a device driver. */
1615 unmap_region(&vmi.mas, vma, prev, next);
1616 }
1617 if (writable_file_mapping)
1618 mapping_unmap_writable(file->f_mapping);
1619free_vma:
1620 vm_area_free(vma);
1621unacct_error:
1622 if (charged)
1623 vm_unacct_memory(charged);
1624
1628clear_tree_failed:
1629 if (vms.vma_count)
1630 abort_munmap_vmas(&mas_detach);
1625abort_munmap:
1626 if (vms.nr_pages)
1627 abort_munmap_vmas(&mas_detach, vms.closed_vm_ops);
1631gather_failed:
1632 validate_mm(mm);
1633 return error;
1634}
1635
1636static int __vm_munmap(unsigned long start, size_t len, bool unlock)
1637{
1638 int ret;

--- 316 unchanged lines hidden (view full) ---

1955 * Walk the list again, actually closing and freeing it, with preemption
1956 * enabled, without holding any MM locks besides the unreachable
1957 * mmap_write_lock.
1958 */
1959 vma_iter_set(&vmi, vma->vm_end);
1960 do {
1961 if (vma->vm_flags & VM_ACCOUNT)
1962 nr_accounted += vma_pages(vma);
1628gather_failed:
1629 validate_mm(mm);
1630 return error;
1631}
1632
1633static int __vm_munmap(unsigned long start, size_t len, bool unlock)
1634{
1635 int ret;

--- 316 unchanged lines hidden (view full) ---

1952 * Walk the list again, actually closing and freeing it, with preemption
1953 * enabled, without holding any MM locks besides the unreachable
1954 * mmap_write_lock.
1955 */
1956 vma_iter_set(&vmi, vma->vm_end);
1957 do {
1958 if (vma->vm_flags & VM_ACCOUNT)
1959 nr_accounted += vma_pages(vma);
1963 remove_vma(vma, true);
1960 remove_vma(vma, /* unreachable = */ true, /* closed = */ false);
1964 count++;
1965 cond_resched();
1966 vma = vma_next(&vmi);
1967 } while (vma && likely(!xa_is_zero(vma)));
1968
1969 BUG_ON(count != mm->map_count);
1970
1971 trace_exit_mmap(mm);

--- 428 unchanged lines hidden --- 		1961 count++;
1962 cond_resched();
1963 vma = vma_next(&vmi);
1964 } while (vma && likely(!xa_is_zero(vma)));
1965
1966 BUG_ON(count != mm->map_count);
1967
1968 trace_exit_mmap(mm);

--- 428 unchanged lines hidden ---