| /linux/mm/ |
| H A D | page_vma_mapped.c | 10 static inline bool not_found(struct page_vma_mapped_walk *pvmw) in not_found() argument
12 page_vma_mapped_walk_done(pvmw); in not_found()
16 static bool map_pte(struct page_vma_mapped_walk *pvmw, pmd_t *pmdvalp, in map_pte() argument
22 if (pvmw->flags & PVMW_SYNC) { in map_pte()
24 pvmw->pte = pte_offset_map_lock(pvmw->vma->vm_mm, pvmw->pmd, in map_pte()
25 pvmw->address, &pvmw->ptl); in map_pte()
26 *ptlp = pvmw->ptl; in map_pte()
27 return !!pvmw->pte; in map_pte()
30 is_migration = pvmw->flags & PVMW_MIGRATION; in map_pte()
39 pvmw->pte = pte_offset_map_rw_nolock(pvmw->vma->vm_mm, pvmw->pmd, in map_pte()
[all …]
|
| H A D | rmap.c | 922 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in folio_referenced_one()
926 while (page_vma_mapped_walk(&pvmw)) { in folio_referenced_one()
927 address = pvmw.address; in folio_referenced_one()
935 if (pvmw.pte && ptes != pvmw.nr_pages) in folio_referenced_one()
945 if (pvmw.flags & PVMW_PGTABLE_CROSSED) in folio_referenced_one()
950 page_vma_mapped_walk_done(&pvmw); in folio_referenced_one()
965 page_vma_mapped_walk_done(&pvmw); in folio_referenced_one()
969 if (pvmw.pte && folio_test_large(folio)) { in folio_referenced_one()
972 pte_t pteval = ptep_get(pvmw.pte); in folio_referenced_one()
974 nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); in folio_referenced_one()
[all …]
|
| H A D | migrate.c | 299 static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw, in try_to_map_unused_to_zeropage() argument
313 if (folio_test_mlocked(folio) || (pvmw->vma->vm_flags & VM_LOCKED) || in try_to_map_unused_to_zeropage()
314 mm_forbids_zeropage(pvmw->vma->vm_mm)) in try_to_map_unused_to_zeropage()
325 newpte = pte_mkspecial(pfn_pte(zero_pfn(pvmw->address), in try_to_map_unused_to_zeropage()
326 pvmw->vma->vm_page_prot)); in try_to_map_unused_to_zeropage()
334 if (pte_swp_uffd(old_pte) && userfaultfd_rwp(pvmw->vma)) in try_to_map_unused_to_zeropage()
337 set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte); in try_to_map_unused_to_zeropage()
339 dec_mm_counter(pvmw->vma->vm_mm, mm_counter(folio)); in try_to_map_unused_to_zeropage()
355 DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION); in remove_migration_pte()
357 while (page_vma_mapped_walk(&pvmw)) { in remove_migration_pte()
[all …]
|
| H A D | ksm.c | 1297 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, 0, 0); in write_protect_page()
1307 pvmw.address = page_address_in_vma(folio, folio_page(folio, 0), vma); in write_protect_page()
1308 if (pvmw.address == -EFAULT) in write_protect_page()
1311 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, pvmw.address, in write_protect_page()
1312 pvmw.address + PAGE_SIZE); in write_protect_page()
1315 if (!page_vma_mapped_walk(&pvmw)) in write_protect_page()
1317 if (WARN_ONCE(!pvmw.pte, "Unexpected PMD mapping?")) in write_protect_page()
1320 entry = ptep_get(pvmw.pte); in write_protect_page()
1332 flush_cache_page(vma, pvmw.address, folio_pfn(folio)); in write_protect_page()
1347 entry = ptep_clear_flush(vma, pvmw.address, pvmw.pte); in write_protect_page()
[all …]
|
| H A D | internal.h | 1080 static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) in vma_address_end() argument
1082 const pgoff_t pgoff_end = pvmw->pgoff + pvmw->nr_pages; in vma_address_end()
1083 const struct vm_area_struct *vma = pvmw->vma; in vma_address_end()
1088 if (pvmw->nr_pages == 1) in vma_address_end()
1089 return pvmw->address + PAGE_SIZE; in vma_address_end()
1091 if (pvmw->pgoff_is_anon) in vma_address_end()
|
| H A D | huge_memory.c | 5062 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, in set_pmd_migration_entry() argument
5066 struct vm_area_struct *vma = pvmw->vma; in set_pmd_migration_entry()
5068 unsigned long address = pvmw->address; in set_pmd_migration_entry()
5074 if (!(pvmw->pmd && !pvmw->pte)) in set_pmd_migration_entry()
5077 present = pmd_present(*pvmw->pmd); in set_pmd_migration_entry()
5081 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); in set_pmd_migration_entry()
5089 pmdval = pmdp_huge_get_and_clear(vma->vm_mm, address, pvmw->pmd); in set_pmd_migration_entry()
5100 set_pmd_at(mm, address, pvmw->pmd, pmdval); in set_pmd_migration_entry()
5126 set_pmd_at(mm, address, pvmw->pmd, pmdswp); in set_pmd_migration_entry()
5136 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct folio *folio) in remove_migration_pmd() argument
[all …]
|
| H A D | migrate_device.c | 201 struct page_vma_mapped_walk pvmw = { in migrate_vma_collect_huge_pmd() local
215 ret = set_pmd_migration_entry(&pvmw, folio_page(folio, 0)); in migrate_vma_collect_huge_pmd()
|
| H A D | vmscan.c | 4256 bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) in lru_gen_look_around() argument
4265 pte_t *pte = pvmw->pte; in lru_gen_look_around()
4266 unsigned long addr = pvmw->address; in lru_gen_look_around()
4267 struct vm_area_struct *vma = pvmw->vma; in lru_gen_look_around()
4268 struct folio *folio = pfn_folio(pvmw->pfn); in lru_gen_look_around()
4276 lockdep_assert_held(pvmw->ptl); in lru_gen_look_around()
4282 if (spin_is_contended(pvmw->ptl)) in lru_gen_look_around()
4362 update_bloom_filter(mm_state, max_seq, pvmw->pmd); in lru_gen_look_around()
|
| /linux/include/linux/ |
| H A D | rmap.h | 888 static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) in page_vma_mapped_walk_done() argument
891 if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma)) in page_vma_mapped_walk_done()
892 pte_unmap(pvmw->pte); in page_vma_mapped_walk_done()
893 if (pvmw->ptl) in page_vma_mapped_walk_done()
894 spin_unlock(pvmw->ptl); in page_vma_mapped_walk_done()
907 page_vma_mapped_walk_restart(struct page_vma_mapped_walk *pvmw) in page_vma_mapped_walk_restart() argument
909 WARN_ON_ONCE(!pvmw->pmd && !pvmw->pte); in page_vma_mapped_walk_restart()
911 if (likely(pvmw->ptl)) in page_vma_mapped_walk_restart()
912 spin_unlock(pvmw->ptl); in page_vma_mapped_walk_restart()
916 pvmw->ptl = NULL; in page_vma_mapped_walk_restart()
[all …]
|
| H A D | swapops.h | 326 extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
329 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
343 static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, in set_pmd_migration_entry() argument
349 static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, in remove_migration_pmd() argument
|
| H A D | mmzone.h | 692 bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr);
714 static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, in lru_gen_look_around() argument
|