176 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,  in free_pte_range()  argument
181 	pte_free_tlb(tlb, token, addr);  in free_pte_range()
182 	mm_dec_nr_ptes(tlb->mm);  in free_pte_range()
185 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,  in free_pmd_range()  argument
199 		free_pte_range(tlb, pmd, addr);  in free_pmd_range()
215 	pmd_free_tlb(tlb, pmd, start);  in free_pmd_range()
216 	mm_dec_nr_pmds(tlb->mm);  in free_pmd_range()
219 static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,  in free_pud_range()  argument
233 		free_pmd_range(tlb, pud, addr, next, floor, ceiling);  in free_pud_range()
249 	pud_free_tlb(tlb, pud, start);  in free_pud_range()
250 	mm_dec_nr_puds(tlb->mm);  in free_pud_range()
253 static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,  in free_p4d_range()  argument
267 		free_pud_range(tlb, p4d, addr, next, floor, ceiling);  in free_p4d_range()
283 	p4d_free_tlb(tlb, p4d, start);  in free_p4d_range()
298 void free_pgd_range(struct mmu_gather *tlb,  in free_pgd_range()  argument
350 	tlb_change_page_size(tlb, PAGE_SIZE);  in free_pgd_range()
351 	pgd = pgd_offset(tlb->mm, addr);  in free_pgd_range()
356 		free_p4d_range(tlb, pgd, addr, next, floor, ceiling);  in free_pgd_range()
373 void free_pgtables(struct mmu_gather *tlb, struct unmap_desc *unmap)  in free_pgtables()  argument
387 	tlb_free_vmas(tlb);  in free_pgtables()
419 		free_pgd_range(tlb, addr, vma->vm_end, unmap->pg_start,  in free_pgtables()
1638 static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,  in zap_present_folio_ptes()  argument
1644 	struct mm_struct *mm = tlb->mm;  in zap_present_folio_ptes()
1648 		ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);  in zap_present_folio_ptes()
1651 			if (tlb_delay_rmap(tlb)) {  in zap_present_folio_ptes()
1661 		clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);  in zap_present_folio_ptes()
1666 	tlb_remove_tlb_entries(tlb, pte, nr, addr);  in zap_present_folio_ptes()
1677 	if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) {  in zap_present_folio_ptes()
1689 static inline int zap_present_ptes(struct mmu_gather *tlb,  in zap_present_ptes()  argument
1695 	struct mm_struct *mm = tlb->mm;  in zap_present_ptes()
1703 		ptep_get_and_clear_full(mm, addr, pte, tlb->fullmm);  in zap_present_ptes()
1705 		tlb_remove_tlb_entry(tlb, pte, addr);  in zap_present_ptes()
1725 		zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr,  in zap_present_ptes()
1730 	zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, 1, addr,  in zap_present_ptes()
1735 static inline int zap_nonpresent_ptes(struct mmu_gather *tlb,  in zap_nonpresent_ptes()  argument
1800 	clear_not_present_full_ptes(vma->vm_mm, addr, pte, nr, tlb->fullmm);  in zap_nonpresent_ptes()
1806 static inline int do_zap_pte_range(struct mmu_gather *tlb,  in do_zap_pte_range()  argument
1832 		nr += zap_present_ptes(tlb, vma, pte, ptent, max_nr, addr,  in do_zap_pte_range()
1836 		nr += zap_nonpresent_ptes(tlb, vma, pte, ptent, max_nr, addr,  in do_zap_pte_range()
1900 static unsigned long zap_pte_range(struct mmu_gather *tlb,  in zap_pte_range()  argument
1907 	struct mm_struct *mm = tlb->mm;  in zap_pte_range()
1918 	tlb_change_page_size(tlb, PAGE_SIZE);  in zap_pte_range()
1934 		nr = do_zap_pte_range(tlb, vma, pte, addr, end, details, rss,  in zap_pte_range()
1961 		tlb_flush_mmu_tlbonly(tlb);  in zap_pte_range()
1962 		tlb_flush_rmaps(tlb, vma);  in zap_pte_range()
1973 		tlb_flush_mmu(tlb);  in zap_pte_range()
1984 			pte_free_tlb(tlb, pmd_pgtable(pmdval), addr);  in zap_pte_range()
1992 static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,  in zap_pmd_range()  argument
2006 			else if (zap_huge_pmd(tlb, vma, pmd, addr)) {  in zap_pmd_range()
2014 			sync_with_folio_pmd_zap(tlb->mm, pmd);  in zap_pmd_range()
2020 		addr = zap_pte_range(tlb, vma, pmd, addr, next, details);  in zap_pmd_range()
2028 static inline unsigned long zap_pud_range(struct mmu_gather *tlb,  in zap_pud_range()  argument
2042 			else if (zap_huge_pud(tlb, vma, pud, addr))  in zap_pud_range()
2048 		next = zap_pmd_range(tlb, vma, pud, addr, next, details);  in zap_pud_range()
2056 static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,  in zap_p4d_range()  argument
2069 		next = zap_pud_range(tlb, vma, p4d, addr, next, details);  in zap_p4d_range()
2075 static void __zap_vma_range(struct mmu_gather *tlb, struct vm_area_struct *vma,  in __zap_vma_range()  argument
2098 		__unmap_hugepage_range(tlb, vma, start, end, NULL, zap_flags);  in __zap_vma_range()
2103 		tlb_start_vma(tlb, vma);  in __zap_vma_range()
2109 			next = zap_p4d_range(tlb, vma, pgd, addr, next, details);  in __zap_vma_range()
2111 		tlb_end_vma(tlb, vma);  in __zap_vma_range()
2130 	struct mmu_gather tlb;  in zap_vma_for_reaping()  local
2134 	tlb_gather_mmu(&tlb, vma->vm_mm);  in zap_vma_for_reaping()
2136 		tlb_finish_mmu(&tlb);  in zap_vma_for_reaping()
2139 	__zap_vma_range(&tlb, vma, range.start, range.end, &details);  in zap_vma_for_reaping()
2141 	tlb_finish_mmu(&tlb);  in zap_vma_for_reaping()
2161 void unmap_vmas(struct mmu_gather *tlb, struct unmap_desc *unmap)  in unmap_vmas()  argument
2178 		__zap_vma_range(tlb, vma, start, end, &details);  in unmap_vmas()
2199 void zap_vma_range_batched(struct mmu_gather *tlb,  in zap_vma_range_batched()  argument
2206 	VM_WARN_ON_ONCE(!tlb || tlb->mm != vma->vm_mm);  in zap_vma_range_batched()
2220 	__zap_vma_range(tlb, vma, address, end, details);  in zap_vma_range_batched()
2227 		tlb_finish_mmu(tlb);  in zap_vma_range_batched()
2229 		tlb_gather_mmu(tlb, vma->vm_mm);  in zap_vma_range_batched()
2244 	struct mmu_gather tlb;  in zap_vma_range()  local
2246 	tlb_gather_mmu(&tlb, vma->vm_mm);  in zap_vma_range()
2247 	zap_vma_range_batched(&tlb, vma, address, size, NULL);  in zap_vma_range()
2248 	tlb_finish_mmu(&tlb);  in zap_vma_range()
4346 	struct mmu_gather tlb;  in unmap_mapping_range_tree()  local
4355 		tlb_gather_mmu(&tlb, vma->vm_mm);  in unmap_mapping_range_tree()
4356 		zap_vma_range_batched(&tlb, vma, start, size, details);  in unmap_mapping_range_tree()
4357 		tlb_finish_mmu(&tlb);  in unmap_mapping_range_tree()