1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* include/asm-generic/tlb.h
32  * Generic MMU-gather implementation.
49  *  - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_finish_mmu()
56  *  - tlb_start_vma() / tlb_end_vma(); marks the start / end of a VMA
61  *  - tlb_remove_table()
63  *    tlb_remove_table() is the basic primitive to free page-table directories
70  *  - tlb_remove_page() / tlb_remove_page_size()
71  *  - __tlb_remove_folio_pages() / __tlb_remove_page_size()
72  *  - __tlb_remove_folio_pages_size()
86  *  - tlb_change_page_size()
88  *    call before __tlb_remove_page*() to set the current page-size; implies a
91  *  - tlb_flush_mmu() / tlb_flush_mmu_tlbonly()
93  *    tlb_flush_mmu_tlbonly() - does the TLB invalidate (and resets
96  *    tlb_flush_mmu() - in addition to the above TLB invalidate, also frees
99  *  - mmu_gather::fullmm
104  *    - We can ignore tlb_{start,end}_vma(); because we don't
107  *    - (RISC) architectures that use ASIDs can cycle to a new ASID
110  *  - mmu_gather::need_flush_all
114  *    x86-PAE needs this when changing top-level entries.
121  *  - mmu_gather::start / mmu_gather::end
126  *  - mmu_gather::freed_tables
130  *  - tlb_get_unmap_shift() / tlb_get_unmap_size()
138  * Additionally there are a few opt-in features:
153  *  Useful if your architecture has non-page page directories.
161  *  Like MMU_GATHER_TABLE_FREE, and adds semi-RCU semantics to the free (see
165  *  and therefore doesn't naturally serialize with software page-table walkers.
208 	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
239  * This allows an architecture that does not use the linux page-tables for
262  * to work on, then just handle a few from the on-stack structure.
274 	((PAGE_SIZE - sizeof(struct mmu_gather_batch)) / sizeof(void *))
278  * lockups for non-preemptible kernels on huge machines when a lot of memory
294 #define tlb_delay_rmap(tlb) (((tlb)->delayed_rmap = 1), true)
301  * We have a no-op version of the rmap removal that doesn't
381 	tlb->start = min(tlb->start, address);  in __tlb_adjust_range()
382 	tlb->end = max(tlb->end, address + range_size);  in __tlb_adjust_range()
387 	if (tlb->fullmm) {  in __tlb_reset_range()
388 		tlb->start = tlb->end = ~0;  in __tlb_reset_range()
390 		tlb->start = TASK_SIZE;  in __tlb_reset_range()
391 		tlb->end = 0;  in __tlb_reset_range()
393 	tlb->freed_tables = 0;  in __tlb_reset_range()
394 	tlb->cleared_ptes = 0;  in __tlb_reset_range()
395 	tlb->cleared_pmds = 0;  in __tlb_reset_range()
396 	tlb->cleared_puds = 0;  in __tlb_reset_range()
397 	tlb->cleared_p4ds = 0;  in __tlb_reset_range()
421 	if (tlb->end)  in tlb_flush()
422 		flush_tlb_mm(tlb->mm);  in tlb_flush()
435 	if (tlb->fullmm || tlb->need_flush_all) {  in tlb_flush()
436 		flush_tlb_mm(tlb->mm);  in tlb_flush()
437 	} else if (tlb->end) {  in tlb_flush()
439 			.vm_mm = tlb->mm,  in tlb_flush()
440 			.vm_flags = (tlb->vma_exec ? VM_EXEC    : 0) |  in tlb_flush()
441 				    (tlb->vma_huge ? VM_HUGETLB : 0),  in tlb_flush()
444 		flush_tlb_range(&vma, tlb->start, tlb->end);  in tlb_flush()
455 	 * flush_tlb_range() implementations that look at VM_HUGETLB (tile,  in tlb_update_vma_flags()
456 	 * mips-4k) flush only large pages.  in tlb_update_vma_flags()
458 	 * flush_tlb_range() implementations that flush I-TLB also flush D-TLB  in tlb_update_vma_flags()
459 	 * (tile, xtensa, arm), so it's ok to just add VM_EXEC to an existing  in tlb_update_vma_flags()
465 	tlb->vma_huge = is_vm_hugetlb_page(vma);  in tlb_update_vma_flags()
466 	tlb->vma_exec = !!(vma->vm_flags & VM_EXEC);  in tlb_update_vma_flags()
467 	tlb->vma_pfn  = !!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP));  in tlb_update_vma_flags()
476 	if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds ||  in tlb_flush_mmu_tlbonly()
477 	      tlb->cleared_puds || tlb->cleared_p4ds))  in tlb_flush_mmu_tlbonly()
505 	if (tlb->page_size && tlb->page_size != page_size) {  in tlb_change_page_size()
506 		if (!tlb->fullmm && !tlb->need_flush_all)  in tlb_change_page_size()
510 	tlb->page_size = page_size;  in tlb_change_page_size()
516 	if (tlb->cleared_ptes)  in tlb_get_unmap_shift()
518 	if (tlb->cleared_pmds)  in tlb_get_unmap_shift()
520 	if (tlb->cleared_puds)  in tlb_get_unmap_shift()
522 	if (tlb->cleared_p4ds)  in tlb_get_unmap_shift()
540 	if (tlb->fullmm)  in tlb_start_vma()
545 	flush_cache_range(vma, vma->vm_start, vma->vm_end);  in tlb_start_vma()
551 	if (tlb->fullmm)  in tlb_end_vma()
556 	 * page mapcount -- there might not be page-frames for these PFNs after  in tlb_end_vma()
560 	if (tlb->vma_pfn || !IS_ENABLED(CONFIG_MMU_GATHER_MERGE_VMAS)) {  in tlb_end_vma()
570  * tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end,
577 	tlb->cleared_ptes = 1;  in tlb_flush_pte_range()
584 	tlb->cleared_pmds = 1;  in tlb_flush_pmd_range()
591 	tlb->cleared_puds = 1;  in tlb_flush_pud_range()
598 	tlb->cleared_p4ds = 1;  in tlb_flush_p4d_range()
608  * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.
612  * userspace is unmapping already-unmapped pages, which happens quite a lot.
621  * tlb_remove_tlb_entries - remember unmapping of multiple consecutive ptes for
633 		if (--nr == 0)  in tlb_remove_tlb_entries()
655  * tlb_remove_pmd_tlb_entry - remember a pmd mapping for later tlb invalidation
669  * tlb_remove_pud_tlb_entry - remember a pud mapping for later tlb
687  * architected non-legacy page table cache (which I'm not aware of
695  * http://lkml.kernel.org/r/CA+55aFzBggoXtNXQeng5d_mRoDnaMBE5Y+URs+PHR67nUpMtaw@mail.gmail.com
697  * For now w.r.t page table cache, mark the range_size as PAGE_SIZE
704 		tlb->freed_tables = 1;				\
713 		tlb->freed_tables = 1;				\
722 		tlb->freed_tables = 1;				\
731 		tlb->freed_tables = 1;				\