task_mmu.c (3689c3ebdd542de1f8c9224a315b3389f28c331b) task_mmu.c (cdd9a571b7d8465353fe2e2d8d1edd8a58d82021)
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/pagewalk.h>
3#include <linux/mm_inline.h>
4#include <linux/hugetlb.h>
5#include <linux/huge_mm.h>
6#include <linux/mount.h>
7#include <linux/ksm.h>
8#include <linux/seq_file.h>

--- 474 unchanged lines hidden (view full) ---

483 * pages referenced by migration entries, even with the PTL held.
484 */
485 if (folio_ref_count(folio) == 1 || !present) {
486 smaps_page_accumulate(mss, folio, size, size << PSS_SHIFT,
487 dirty, locked, present);
488 return;
489 }
490 /*
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/pagewalk.h>
3#include <linux/mm_inline.h>
4#include <linux/hugetlb.h>
5#include <linux/huge_mm.h>
6#include <linux/mount.h>
7#include <linux/ksm.h>
8#include <linux/seq_file.h>

--- 474 unchanged lines hidden (view full) ---

483 * pages referenced by migration entries, even with the PTL held.
484 */
485 if (folio_ref_count(folio) == 1 || !present) {
486 smaps_page_accumulate(mss, folio, size, size << PSS_SHIFT,
487 dirty, locked, present);
488 return;
489 }
490 /*
491 * The page_mapcount() is called to get a snapshot of the mapcount.
492 * Without holding the folio lock this snapshot can be slightly wrong as
493 * we cannot always read the mapcount atomically.
491 * We obtain a snapshot of the mapcount. Without holding the folio lock
492 * this snapshot can be slightly wrong as we cannot always read the
493 * mapcount atomically.
494 */
495 for (i = 0; i < nr; i++, page++) {
494 */
495 for (i = 0; i < nr; i++, page++) {
496 int mapcount = page_mapcount(page);
496 int mapcount = folio_precise_page_mapcount(folio, page);
497 unsigned long pss = PAGE_SIZE << PSS_SHIFT;
498 if (mapcount >= 2)
499 pss /= mapcount;
500 smaps_page_accumulate(mss, folio, PAGE_SIZE, pss,
501 dirty, locked, mapcount < 2);
502 }
503}
504

--- 917 unchanged lines hidden (view full) ---

1422 return err;
1423}
1424
1425static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
1426 struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1427{
1428 u64 frame = 0, flags = 0;
1429 struct page *page = NULL;
497 unsigned long pss = PAGE_SIZE << PSS_SHIFT;
498 if (mapcount >= 2)
499 pss /= mapcount;
500 smaps_page_accumulate(mss, folio, PAGE_SIZE, pss,
501 dirty, locked, mapcount < 2);
502 }
503}
504

--- 917 unchanged lines hidden (view full) ---

1422 return err;
1423}
1424
1425static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
1426 struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1427{
1428 u64 frame = 0, flags = 0;
1429 struct page *page = NULL;
1430 struct folio *folio;
1430
1431 if (pte_present(pte)) {
1432 if (pm->show_pfn)
1433 frame = pte_pfn(pte);
1434 flags |= PM_PRESENT;
1435 page = vm_normal_page(vma, addr, pte);
1436 if (pte_soft_dirty(pte))
1437 flags |= PM_SOFT_DIRTY;

--- 21 unchanged lines hidden (view full) ---

1459 }
1460 flags |= PM_SWAP;
1461 if (is_pfn_swap_entry(entry))
1462 page = pfn_swap_entry_to_page(entry);
1463 if (pte_marker_entry_uffd_wp(entry))
1464 flags |= PM_UFFD_WP;
1465 }
1466
1431
1432 if (pte_present(pte)) {
1433 if (pm->show_pfn)
1434 frame = pte_pfn(pte);
1435 flags |= PM_PRESENT;
1436 page = vm_normal_page(vma, addr, pte);
1437 if (pte_soft_dirty(pte))
1438 flags |= PM_SOFT_DIRTY;

--- 21 unchanged lines hidden (view full) ---

1460 }
1461 flags |= PM_SWAP;
1462 if (is_pfn_swap_entry(entry))
1463 page = pfn_swap_entry_to_page(entry);
1464 if (pte_marker_entry_uffd_wp(entry))
1465 flags |= PM_UFFD_WP;
1466 }
1467
1467 if (page && !PageAnon(page))
1468 flags |= PM_FILE;
1469 if (page && (flags & PM_PRESENT) && page_mapcount(page) == 1)
1470 flags |= PM_MMAP_EXCLUSIVE;
1468 if (page) {
1469 folio = page_folio(page);
1470 if (!folio_test_anon(folio))
1471 flags |= PM_FILE;
1472 if ((flags & PM_PRESENT) &&
1473 folio_precise_page_mapcount(folio, page) == 1)
1474 flags |= PM_MMAP_EXCLUSIVE;
1475 }
1471 if (vma->vm_flags & VM_SOFTDIRTY)
1472 flags |= PM_SOFT_DIRTY;
1473
1474 return make_pme(frame, flags);
1475}
1476
1477static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
1478 struct mm_walk *walk)

--- 6 unchanged lines hidden (view full) ---

1485#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1486
1487 ptl = pmd_trans_huge_lock(pmdp, vma);
1488 if (ptl) {
1489 unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT;
1490 u64 flags = 0, frame = 0;
1491 pmd_t pmd = *pmdp;
1492 struct page *page = NULL;
1476 if (vma->vm_flags & VM_SOFTDIRTY)
1477 flags |= PM_SOFT_DIRTY;
1478
1479 return make_pme(frame, flags);
1480}
1481
1482static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
1483 struct mm_walk *walk)

--- 6 unchanged lines hidden (view full) ---

1490#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1491
1492 ptl = pmd_trans_huge_lock(pmdp, vma);
1493 if (ptl) {
1494 unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT;
1495 u64 flags = 0, frame = 0;
1496 pmd_t pmd = *pmdp;
1497 struct page *page = NULL;
1498 struct folio *folio = NULL;
1493
1494 if (vma->vm_flags & VM_SOFTDIRTY)
1495 flags |= PM_SOFT_DIRTY;
1496
1497 if (pmd_present(pmd)) {
1498 page = pmd_page(pmd);
1499
1500 flags |= PM_PRESENT;

--- 22 unchanged lines hidden (view full) ---

1523 flags |= PM_SOFT_DIRTY;
1524 if (pmd_swp_uffd_wp(pmd))
1525 flags |= PM_UFFD_WP;
1526 VM_BUG_ON(!is_pmd_migration_entry(pmd));
1527 page = pfn_swap_entry_to_page(entry);
1528 }
1529#endif
1530
1499
1500 if (vma->vm_flags & VM_SOFTDIRTY)
1501 flags |= PM_SOFT_DIRTY;
1502
1503 if (pmd_present(pmd)) {
1504 page = pmd_page(pmd);
1505
1506 flags |= PM_PRESENT;

--- 22 unchanged lines hidden (view full) ---

1529 flags |= PM_SOFT_DIRTY;
1530 if (pmd_swp_uffd_wp(pmd))
1531 flags |= PM_UFFD_WP;
1532 VM_BUG_ON(!is_pmd_migration_entry(pmd));
1533 page = pfn_swap_entry_to_page(entry);
1534 }
1535#endif
1536
1531 if (page && !PageAnon(page))
1532 flags |= PM_FILE;
1537 if (page) {
1538 folio = page_folio(page);
1539 if (!folio_test_anon(folio))
1540 flags |= PM_FILE;
1541 }
1533
1534 for (; addr != end; addr += PAGE_SIZE, idx++) {
1535 unsigned long cur_flags = flags;
1536 pagemap_entry_t pme;
1537
1542
1543 for (; addr != end; addr += PAGE_SIZE, idx++) {
1544 unsigned long cur_flags = flags;
1545 pagemap_entry_t pme;
1546
1538 if (page && (flags & PM_PRESENT) &&
1539 page_mapcount(page + idx) == 1)
1547 if (folio && (flags & PM_PRESENT) &&
1548 folio_precise_page_mapcount(folio, page + idx) == 1)
1540 cur_flags |= PM_MMAP_EXCLUSIVE;
1541
1542 pme = make_pme(frame, cur_flags);
1543 err = add_to_pagemap(&pme, pm);
1544 if (err)
1545 break;
1546 if (pm->show_pfn) {
1547 if (flags & PM_PRESENT)

--- 1022 unchanged lines hidden (view full) ---

2570 struct proc_maps_private proc_maps;
2571 struct numa_maps md;
2572};
2573
2574static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
2575 unsigned long nr_pages)
2576{
2577 struct folio *folio = page_folio(page);
1549 cur_flags |= PM_MMAP_EXCLUSIVE;
1550
1551 pme = make_pme(frame, cur_flags);
1552 err = add_to_pagemap(&pme, pm);
1553 if (err)
1554 break;
1555 if (pm->show_pfn) {
1556 if (flags & PM_PRESENT)

--- 1022 unchanged lines hidden (view full) ---

2579 struct proc_maps_private proc_maps;
2580 struct numa_maps md;
2581};
2582
2583static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
2584 unsigned long nr_pages)
2585{
2586 struct folio *folio = page_folio(page);
2578 int count = page_mapcount(page);
2587 int count = folio_precise_page_mapcount(folio, page);
2579
2580 md->pages += nr_pages;
2581 if (pte_dirty || folio_test_dirty(folio))
2582 md->dirty += nr_pages;
2583
2584 if (folio_test_swapcache(folio))
2585 md->swapcache += nr_pages;
2586

--- 237 unchanged lines hidden --- 		2588
2589 md->pages += nr_pages;
2590 if (pte_dirty || folio_test_dirty(folio))
2591 md->dirty += nr_pages;
2592
2593 if (folio_test_swapcache(folio))
2594 md->swapcache += nr_pages;
2595

--- 237 unchanged lines hidden ---