1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/tlb.h 4 * 5 * Copyright (C) 2002 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASM_TLB_H 9 #define __ASM_TLB_H 10 11 #include <linux/pagemap.h> 12 13 14 #define tlb_flush tlb_flush 15 static void tlb_flush(struct mmu_gather *tlb); 16 17 #include <asm-generic/tlb.h> 18 19 /* 20 * get the tlbi levels in arm64. Default value is TLBI_TTL_UNKNOWN if more than 21 * one of cleared_* is set or neither is set - this elides the level hinting to 22 * the hardware. 23 */ 24 static inline int tlb_get_level(struct mmu_gather *tlb) 25 { 26 /* The TTL field is only valid for the leaf entry. */ 27 if (tlb->freed_tables) 28 return TLBI_TTL_UNKNOWN; 29 30 if (tlb->cleared_ptes && !(tlb->cleared_pmds || 31 tlb->cleared_puds || 32 tlb->cleared_p4ds)) 33 return 3; 34 35 if (tlb->cleared_pmds && !(tlb->cleared_ptes || 36 tlb->cleared_puds || 37 tlb->cleared_p4ds)) 38 return 2; 39 40 if (tlb->cleared_puds && !(tlb->cleared_ptes || 41 tlb->cleared_pmds || 42 tlb->cleared_p4ds)) 43 return 1; 44 45 if (tlb->cleared_p4ds && !(tlb->cleared_ptes || 46 tlb->cleared_pmds || 47 tlb->cleared_puds)) 48 return 0; 49 50 return TLBI_TTL_UNKNOWN; 51 } 52 53 static inline void tlb_flush(struct mmu_gather *tlb) 54 { 55 struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0); 56 tlbf_t flags = (tlb->freed_tables || tlb->unshared_tables) ? 57 TLBF_NONE : TLBF_NOWALKCACHE; 58 unsigned long stride = tlb_get_unmap_size(tlb); 59 int tlb_level = tlb_get_level(tlb); 60 61 /* 62 * If we're tearing down the address space then we only care about 63 * invalidating the walk-cache, since the ASID allocator won't 64 * reallocate our ASID without invalidating the entire TLB. 65 */ 66 if (tlb->fullmm) { 67 if (tlb->freed_tables) 68 flush_tlb_mm(tlb->mm); 69 return; 70 } 71 72 __flush_tlb_range(&vma, tlb->start, tlb->end, stride, 73 tlb_level, flags); 74 } 75 76 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, 77 unsigned long addr) 78 { 79 struct ptdesc *ptdesc = page_ptdesc(pte); 80 81 tlb_remove_ptdesc(tlb, ptdesc); 82 } 83 84 #if CONFIG_PGTABLE_LEVELS > 2 85 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, 86 unsigned long addr) 87 { 88 struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); 89 90 tlb_remove_ptdesc(tlb, ptdesc); 91 } 92 #endif 93 94 #if CONFIG_PGTABLE_LEVELS > 3 95 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, 96 unsigned long addr) 97 { 98 struct ptdesc *ptdesc = virt_to_ptdesc(pudp); 99 100 if (!pgtable_l4_enabled()) 101 return; 102 103 tlb_remove_ptdesc(tlb, ptdesc); 104 } 105 #endif 106 107 #if CONFIG_PGTABLE_LEVELS > 4 108 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4dp, 109 unsigned long addr) 110 { 111 struct ptdesc *ptdesc = virt_to_ptdesc(p4dp); 112 113 if (!pgtable_l5_enabled()) 114 return; 115 116 tlb_remove_ptdesc(tlb, ptdesc); 117 } 118 #endif 119 120 #endif 121