/linux/mm/ |
H A D | page_vma_mapped.c | 10 static inline bool not_found(struct page_vma_mapped_walk *pvmw) in not_found() argument 12 page_vma_mapped_walk_done(pvmw); in not_found() 16 static bool map_pte(struct page_vma_mapped_walk *pvmw, spinlock_t **ptlp) in map_pte() argument 20 if (pvmw->flags & PVMW_SYNC) { in map_pte() 22 pvmw->pte = pte_offset_map_lock(pvmw->vma->vm_mm, pvmw->pmd, in map_pte() 23 pvmw->address, &pvmw->ptl); in map_pte() 24 *ptlp = pvmw in map_pte() 96 check_pte(struct page_vma_mapped_walk * pvmw) check_pte() argument 133 check_pmd(unsigned long pfn,struct page_vma_mapped_walk * pvmw) check_pmd() argument 142 step_forward(struct page_vma_mapped_walk * pvmw,unsigned long size) step_forward() argument 173 page_vma_mapped_walk(struct page_vma_mapped_walk * pvmw) page_vma_mapped_walk() argument 328 struct page_vma_mapped_walk pvmw = { page_mapped_in_vma() local [all...] |
H A D | rmap.c | 842 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in folio_referenced_one() 846 while (page_vma_mapped_walk(&pvmw)) { in folio_referenced_one() 847 address = pvmw.address; in folio_referenced_one() 850 if (!folio_test_large(folio) || !pvmw.pte) { in folio_referenced_one() 853 page_vma_mapped_walk_done(&pvmw); in folio_referenced_one() 859 * be handled after the pvmw loop. in folio_referenced_one() 873 if (pvmw.pte) { in folio_referenced_one() 875 pte_young(ptep_get(pvmw.pte))) { in folio_referenced_one() 876 lru_gen_look_around(&pvmw); in folio_referenced_one() 881 pvmw in folio_referenced_one() 999 page_vma_mkclean_one(struct page_vma_mapped_walk * pvmw) page_vma_mkclean_one() argument 1120 struct page_vma_mapped_walk pvmw = { pfn_mkclean_range() local [all...] |
H A D | page_idle.c | 56 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); in page_idle_clear_pte_refs_one() 59 while (page_vma_mapped_walk(&pvmw)) { in page_idle_clear_pte_refs_one() 60 addr = pvmw.address; in page_idle_clear_pte_refs_one() 61 if (pvmw.pte) { in page_idle_clear_pte_refs_one() 66 if (ptep_clear_young_notify(vma, addr, pvmw.pte)) in page_idle_clear_pte_refs_one() 69 if (pmdp_clear_young_notify(vma, addr, pvmw.pmd)) in page_idle_clear_pte_refs_one()
|
H A D | migrate.c | 186 DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC | PVMW_MIGRATION); in remove_migration_pte() 188 while (page_vma_mapped_walk(&pvmw)) { in remove_migration_pte() 198 idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff; in remove_migration_pte() 203 if (!pvmw.pte) { in remove_migration_pte() 206 remove_migration_pmd(&pvmw, new); in remove_migration_pte() 213 old_pte = ptep_get(pvmw.pte); in remove_migration_pte() 255 hugetlb_add_anon_rmap(folio, vma, pvmw.address, in remove_migration_pte() 259 set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw in remove_migration_pte() [all...] |
H A D | ksm.c | 1282 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, 0, 0); in write_protect_page() 1292 pvmw.address = page_address_in_vma(&folio->page, vma); in write_protect_page() 1293 if (pvmw.address == -EFAULT) in write_protect_page() 1296 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, pvmw.address, in write_protect_page() 1297 pvmw.address + PAGE_SIZE); in write_protect_page() 1300 if (!page_vma_mapped_walk(&pvmw)) in write_protect_page() 1302 if (WARN_ONCE(!pvmw.pte, "Unexpected PMD mapping?")) in write_protect_page() 1306 entry = ptep_get(pvmw.pte); in write_protect_page() 1310 flush_cache_page(vma, pvmw.address, folio_pfn(folio)); in write_protect_page() 1325 entry = ptep_clear_flush(vma, pvmw in write_protect_page() [all...] |
H A D | internal.h | 923 static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) in vma_address_end() argument 925 struct vm_area_struct *vma = pvmw->vma; in vma_address_end() 930 if (pvmw->nr_pages == 1) in vma_address_end() 931 return pvmw->address + PAGE_SIZE; in vma_address_end() 933 pgoff = pvmw->pgoff + pvmw->nr_pages; in vma_address_end()
|
H A D | huge_memory.c | 3629 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, in split_huge_pages_write() 3633 struct vm_area_struct *vma = pvmw->vma; in split_huge_pages_write() 3635 unsigned long address = pvmw->address; in split_huge_pages_write() 3641 if (!(pvmw->pmd && !pvmw->pte)) in split_huge_pages_write() 3645 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); in split_huge_pages_write() 3650 set_pmd_at(mm, address, pvmw->pmd, pmdval); in split_huge_pages_write() 3671 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 3679 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) in set_pmd_migration_entry() 3682 struct vm_area_struct *vma = pvmw in set_pmd_migration_entry() 3673 set_pmd_migration_entry(struct page_vma_mapped_walk * pvmw,struct page * page) set_pmd_migration_entry() argument 3723 remove_migration_pmd(struct page_vma_mapped_walk * pvmw,struct page * new) remove_migration_pmd() argument [all...] |
H A D | vmscan.c | 4004 void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) in lru_gen_look_around() 4011 pte_t *pte = pvmw->pte; in lru_gen_look_around() 4012 unsigned long addr = pvmw->address; in lru_gen_look_around() 4013 struct vm_area_struct *vma = pvmw->vma; in lru_gen_look_around() 4014 struct folio *folio = pfn_folio(pvmw->pfn); in lru_gen_look_around() 4023 lockdep_assert_held(pvmw->ptl); in lru_gen_look_around() 4026 if (spin_is_contended(pvmw->ptl)) in lru_gen_look_around() 4103 update_bloom_filter(mm_state, max_seq, pvmw->pmd); in lru_gen_rotate_memcg() 3988 lru_gen_look_around(struct page_vma_mapped_walk * pvmw) lru_gen_look_around() argument
|
/linux/mm/damon/ |
H A D | paddr.c | 22 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); in __damon_pa_mkold() 24 while (page_vma_mapped_walk(&pvmw)) { in __damon_pa_mkold() 25 addr = pvmw.address; in __damon_pa_mkold() 26 if (pvmw.pte) in __damon_pa_mkold() 27 damon_ptep_mkold(pvmw.pte, vma, addr); in __damon_pa_mkold() 29 damon_pmdp_mkold(pvmw.pmd, vma, addr); in __damon_pa_mkold() 91 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); in __damon_pa_young() 94 while (page_vma_mapped_walk(&pvmw)) { in __damon_pa_young() 95 addr = pvmw.address; in __damon_pa_young() 96 if (pvmw in __damon_pa_young() [all...] |
/linux/include/linux/ |
H A D | rmap.h | 704 static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) 707 if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma)) 708 pte_unmap(pvmw->pte); 709 if (pvmw->ptl) 710 spin_unlock(pvmw->ptl); 713 bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw); 672 page_vma_mapped_walk_done(struct page_vma_mapped_walk * pvmw) page_vma_mapped_walk_done() argument
|
H A D | swapops.h | 531 extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 534 extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, 564 static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, in set_pmd_migration_entry() argument 570 static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, in remove_migration_pmd() argument
|
H A D | mmzone.h | 560 void lru_gen_look_around(struct page_vma_mapped_walk *pvmw); 579 static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) in lru_gen_look_around() argument
|
/linux/kernel/events/ |
H A D | uprobes.c | 159 DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, vma, addr, 0); in __replace_page() 178 if (!page_vma_mapped_walk(&pvmw)) in __replace_page() 180 VM_BUG_ON_PAGE(addr != pvmw.address, old_page); in __replace_page() 195 flush_cache_page(vma, addr, pte_pfn(ptep_get(pvmw.pte))); in __replace_page() 196 ptep_clear_flush(vma, addr, pvmw.pte); in __replace_page() 198 set_pte_at(mm, addr, pvmw.pte, in __replace_page() 204 page_vma_mapped_walk_done(&pvmw); in __replace_page()
|