1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/tlb.h 4 * 5 * Copyright (C) 2002 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASM_TLB_H 9 #define __ASM_TLB_H 10 11 #include <linux/pagemap.h> 12 13 14 #define tlb_flush tlb_flush 15 static void tlb_flush(struct mmu_gather *tlb); 16 17 #include <asm-generic/tlb.h> 18 19 /* 20 * get the tlbi levels in arm64. Default value is TLBI_TTL_UNKNOWN if more than 21 * one of cleared_* is set or neither is set - this elides the level hinting to 22 * the hardware. 23 */ 24 static inline int tlb_get_level(struct mmu_gather *tlb) 25 { 26 /* The TTL field is only valid for the leaf entry. */ 27 if (tlb->freed_tables) 28 return TLBI_TTL_UNKNOWN; 29 30 if (tlb->cleared_ptes && !(tlb->cleared_pmds || 31 tlb->cleared_puds || 32 tlb->cleared_p4ds)) 33 return 3; 34 35 if (tlb->cleared_pmds && !(tlb->cleared_ptes || 36 tlb->cleared_puds || 37 tlb->cleared_p4ds)) 38 return 2; 39 40 if (tlb->cleared_puds && !(tlb->cleared_ptes || 41 tlb->cleared_pmds || 42 tlb->cleared_p4ds)) 43 return 1; 44 45 if (tlb->cleared_p4ds && !(tlb->cleared_ptes || 46 tlb->cleared_pmds || 47 tlb->cleared_puds)) 48 return 0; 49 50 return TLBI_TTL_UNKNOWN; 51 } 52 53 static inline void tlb_flush(struct mmu_gather *tlb) 54 { 55 struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0); 56 bool last_level = !tlb->freed_tables; 57 unsigned long stride = tlb_get_unmap_size(tlb); 58 int tlb_level = tlb_get_level(tlb); 59 60 /* 61 * If we're tearing down the address space then we only care about 62 * invalidating the walk-cache, since the ASID allocator won't 63 * reallocate our ASID without invalidating the entire TLB. 64 */ 65 if (tlb->fullmm) { 66 if (!last_level) 67 flush_tlb_mm(tlb->mm); 68 return; 69 } 70 71 __flush_tlb_range(&vma, tlb->start, tlb->end, stride, 72 last_level, tlb_level); 73 } 74 75 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, 76 unsigned long addr) 77 { 78 struct ptdesc *ptdesc = page_ptdesc(pte); 79 80 tlb_remove_ptdesc(tlb, ptdesc); 81 } 82 83 #if CONFIG_PGTABLE_LEVELS > 2 84 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, 85 unsigned long addr) 86 { 87 struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); 88 89 tlb_remove_ptdesc(tlb, ptdesc); 90 } 91 #endif 92 93 #if CONFIG_PGTABLE_LEVELS > 3 94 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, 95 unsigned long addr) 96 { 97 struct ptdesc *ptdesc = virt_to_ptdesc(pudp); 98 99 if (!pgtable_l4_enabled()) 100 return; 101 102 tlb_remove_ptdesc(tlb, ptdesc); 103 } 104 #endif 105 106 #if CONFIG_PGTABLE_LEVELS > 4 107 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4dp, 108 unsigned long addr) 109 { 110 struct ptdesc *ptdesc = virt_to_ptdesc(p4dp); 111 112 if (!pgtable_l5_enabled()) 113 return; 114 115 tlb_remove_ptdesc(tlb, ptdesc); 116 } 117 #endif 118 119 #endif 120