225 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
229 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page);
234 static inline void tlb_remove_table(struct mmu_gather *tlb, void *table)  in tlb_remove_table()  argument
239 	tlb_remove_page(tlb, ptdesc_page(ptdesc));  in tlb_remove_table()
290 extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size);
291 bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page,
299 #define tlb_delay_rmap(tlb) (((tlb)->delayed_rmap = 1), true)  argument
300 extern void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma);
313 #define tlb_delay_rmap(tlb) (false)  argument
314 static inline void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) { }  in tlb_flush_rmaps()  argument
394 void tlb_flush_mmu(struct mmu_gather *tlb);
396 static inline void __tlb_adjust_range(struct mmu_gather *tlb,  in __tlb_adjust_range()  argument
400 	tlb->start = min(tlb->start, address);  in __tlb_adjust_range()
401 	tlb->end = max(tlb->end, address + range_size);  in __tlb_adjust_range()
404 static inline void __tlb_reset_range(struct mmu_gather *tlb)  in __tlb_reset_range()  argument
406 	if (tlb->fullmm) {  in __tlb_reset_range()
407 		tlb->start = tlb->end = ~0;  in __tlb_reset_range()
409 		tlb->start = TASK_SIZE;  in __tlb_reset_range()
410 		tlb->end = 0;  in __tlb_reset_range()
412 	tlb->freed_tables = 0;  in __tlb_reset_range()
413 	tlb->cleared_ptes = 0;  in __tlb_reset_range()
414 	tlb->cleared_pmds = 0;  in __tlb_reset_range()
415 	tlb->cleared_puds = 0;  in __tlb_reset_range()
416 	tlb->cleared_p4ds = 0;  in __tlb_reset_range()
417 	tlb->unshared_tables = 0;  in __tlb_reset_range()
439 static inline void tlb_flush(struct mmu_gather *tlb)  in tlb_flush()  argument
441 	if (tlb->end)  in tlb_flush()
442 		flush_tlb_mm(tlb->mm);  in tlb_flush()
453 static inline void tlb_flush(struct mmu_gather *tlb)  in tlb_flush()  argument
455 	if (tlb->fullmm || tlb->need_flush_all) {  in tlb_flush()
456 		flush_tlb_mm(tlb->mm);  in tlb_flush()
457 	} else if (tlb->end) {  in tlb_flush()
459 			.vm_mm = tlb->mm,  in tlb_flush()
460 			.vm_flags = (tlb->vma_exec ? VM_EXEC    : 0) |  in tlb_flush()
461 				    (tlb->vma_huge ? VM_HUGETLB : 0),  in tlb_flush()
464 		flush_tlb_range(&vma, tlb->start, tlb->end);  in tlb_flush()
472 tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma)  in tlb_update_vma_flags()  argument
485 	tlb->vma_huge = is_vm_hugetlb_page(vma);  in tlb_update_vma_flags()
486 	tlb->vma_exec = !!(vma->vm_flags & VM_EXEC);  in tlb_update_vma_flags()
492 	tlb->vma_pfn |= !!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP));  in tlb_update_vma_flags()
495 static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)  in tlb_flush_mmu_tlbonly()  argument
501 	if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds ||  in tlb_flush_mmu_tlbonly()
502 	      tlb->cleared_puds || tlb->cleared_p4ds || tlb->unshared_tables))  in tlb_flush_mmu_tlbonly()
505 	tlb_flush(tlb);  in tlb_flush_mmu_tlbonly()
506 	__tlb_reset_range(tlb);  in tlb_flush_mmu_tlbonly()
509 static inline void tlb_remove_page_size(struct mmu_gather *tlb,  in tlb_remove_page_size()  argument
512 	if (__tlb_remove_page_size(tlb, page, page_size))  in tlb_remove_page_size()
513 		tlb_flush_mmu(tlb);  in tlb_remove_page_size()
516 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)  in tlb_remove_page()  argument
518 	return tlb_remove_page_size(tlb, page, PAGE_SIZE);  in tlb_remove_page()
521 static inline void tlb_remove_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt)  in tlb_remove_ptdesc()  argument
523 	tlb_remove_table(tlb, pt);  in tlb_remove_ptdesc()
526 static inline void tlb_change_page_size(struct mmu_gather *tlb,  in tlb_change_page_size()  argument
530 	if (tlb->page_size && tlb->page_size != page_size) {  in tlb_change_page_size()
531 		if (!tlb->fullmm && !tlb->need_flush_all)  in tlb_change_page_size()
532 			tlb_flush_mmu(tlb);  in tlb_change_page_size()
535 	tlb->page_size = page_size;  in tlb_change_page_size()
539 static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)  in tlb_get_unmap_shift()  argument
541 	if (tlb->cleared_ptes)  in tlb_get_unmap_shift()
543 	if (tlb->cleared_pmds)  in tlb_get_unmap_shift()
545 	if (tlb->cleared_puds)  in tlb_get_unmap_shift()
547 	if (tlb->cleared_p4ds)  in tlb_get_unmap_shift()
553 static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb)  in tlb_get_unmap_size()  argument
555 	return 1UL << tlb_get_unmap_shift(tlb);  in tlb_get_unmap_size()
563 static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)  in tlb_start_vma()  argument
565 	if (tlb->fullmm)  in tlb_start_vma()
568 	tlb_update_vma_flags(tlb, vma);  in tlb_start_vma()
574 static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)  in tlb_end_vma()  argument
576 	if (tlb->fullmm || IS_ENABLED(CONFIG_MMU_GATHER_MERGE_VMAS))  in tlb_end_vma()
585 	tlb_flush_mmu_tlbonly(tlb);  in tlb_end_vma()
588 static inline void tlb_free_vmas(struct mmu_gather *tlb)  in tlb_free_vmas()  argument
590 	if (tlb->fullmm)  in tlb_free_vmas()
606 	if (tlb->vma_pfn)  in tlb_free_vmas()
607 		tlb_flush_mmu_tlbonly(tlb);  in tlb_free_vmas()
614 static inline void tlb_flush_pte_range(struct mmu_gather *tlb,  in tlb_flush_pte_range()  argument
617 	__tlb_adjust_range(tlb, address, size);  in tlb_flush_pte_range()
618 	tlb->cleared_ptes = 1;  in tlb_flush_pte_range()
621 static inline void tlb_flush_pmd_range(struct mmu_gather *tlb,  in tlb_flush_pmd_range()  argument
624 	__tlb_adjust_range(tlb, address, size);  in tlb_flush_pmd_range()
625 	tlb->cleared_pmds = 1;  in tlb_flush_pmd_range()
628 static inline void tlb_flush_pud_range(struct mmu_gather *tlb,  in tlb_flush_pud_range()  argument
631 	__tlb_adjust_range(tlb, address, size);  in tlb_flush_pud_range()
632 	tlb->cleared_puds = 1;  in tlb_flush_pud_range()
635 static inline void tlb_flush_p4d_range(struct mmu_gather *tlb,  in tlb_flush_p4d_range()  argument
638 	__tlb_adjust_range(tlb, address, size);  in tlb_flush_p4d_range()
639 	tlb->cleared_p4ds = 1;  in tlb_flush_p4d_range()
643 static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addres…  in __tlb_remove_tlb_entry()  argument
655 #define tlb_remove_tlb_entry(tlb, ptep, address)		\  argument
657 		tlb_flush_pte_range(tlb, address, PAGE_SIZE);	\
658 		__tlb_remove_tlb_entry(tlb, ptep, address);	\
668 static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb,  in tlb_remove_tlb_entries()  argument
671 	tlb_flush_pte_range(tlb, address, PAGE_SIZE * nr);  in tlb_remove_tlb_entries()
673 		__tlb_remove_tlb_entry(tlb, ptep, address);  in tlb_remove_tlb_entries()
681 #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)	\  argument
685 			tlb_flush_p4d_range(tlb, address, _sz);	\
687 			tlb_flush_pud_range(tlb, address, _sz);	\
689 			tlb_flush_pmd_range(tlb, address, _sz);	\
691 			tlb_flush_pte_range(tlb, address, _sz);	\
692 		__tlb_remove_tlb_entry(tlb, ptep, address);	\
700 #define __tlb_remove_pmd_tlb_entry(tlb, pmdp, address) do {} while (0)  argument
703 #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address)			\  argument
705 		tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE);	\
706 		__tlb_remove_pmd_tlb_entry(tlb, pmdp, address);		\
714 #define __tlb_remove_pud_tlb_entry(tlb, pudp, address) do {} while (0)  argument
717 #define tlb_remove_pud_tlb_entry(tlb, pudp, address)			\  argument
719 		tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE);	\
720 		__tlb_remove_pud_tlb_entry(tlb, pudp, address);		\
742 #define pte_free_tlb(tlb, ptep, address)			\  argument
744 		tlb_flush_pmd_range(tlb, address, PAGE_SIZE);	\
745 		tlb->freed_tables = 1;				\
746 		__pte_free_tlb(tlb, ptep, address);		\
751 #define pmd_free_tlb(tlb, pmdp, address)			\  argument
753 		tlb_flush_pud_range(tlb, address, PAGE_SIZE);	\
754 		tlb->freed_tables = 1;				\
755 		__pmd_free_tlb(tlb, pmdp, address);		\
760 #define pud_free_tlb(tlb, pudp, address)			\  argument
762 		tlb_flush_p4d_range(tlb, address, PAGE_SIZE);	\
763 		tlb->freed_tables = 1;				\
764 		__pud_free_tlb(tlb, pudp, address);		\
769 #define p4d_free_tlb(tlb, pudp, address)			\  argument
771 		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\
772 		tlb->freed_tables = 1;				\
773 		__p4d_free_tlb(tlb, pudp, address);		\
792 static inline void tlb_unshare_pmd_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt,  in tlb_unshare_pmd_ptdesc()  argument
803 	tlb_flush_pmd_range(tlb, addr & PUD_MASK, PUD_SIZE);  in tlb_unshare_pmd_ptdesc()
810 		tlb->fully_unshared_tables = true;  in tlb_unshare_pmd_ptdesc()
811 	tlb->unshared_tables = true;  in tlb_unshare_pmd_ptdesc()
814 static inline void tlb_flush_unshared_tables(struct mmu_gather *tlb)  in tlb_flush_unshared_tables()  argument
826 	if (tlb->unshared_tables)  in tlb_flush_unshared_tables()
827 		tlb_flush_mmu_tlbonly(tlb);  in tlb_flush_unshared_tables()
841 	if (tlb->fully_unshared_tables) {  in tlb_flush_unshared_tables()
843 		tlb->fully_unshared_tables = false;  in tlb_flush_unshared_tables()