1 /* 2 * Based on arch/arm/include/asm/tlb.h 3 * 4 * Copyright (C) 2002 Russell King 5 * Copyright (C) 2012 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifndef __ASM_TLB_H 20 #define __ASM_TLB_H 21 22 #include <linux/pagemap.h> 23 #include <linux/swap.h> 24 25 #include <asm/pgalloc.h> 26 #include <asm/tlbflush.h> 27 28 #define MMU_GATHER_BUNDLE 8 29 30 /* 31 * TLB handling. This allows us to remove pages from the page 32 * tables, and efficiently handle the TLB issues. 33 */ 34 struct mmu_gather { 35 struct mm_struct *mm; 36 unsigned int fullmm; 37 struct vm_area_struct *vma; 38 unsigned long start, end; 39 unsigned long range_start; 40 unsigned long range_end; 41 unsigned int nr; 42 unsigned int max; 43 struct page **pages; 44 struct page *local[MMU_GATHER_BUNDLE]; 45 }; 46 47 /* 48 * This is unnecessarily complex. There's three ways the TLB shootdown 49 * code is used: 50 * 1. Unmapping a range of vmas. See zap_page_range(), unmap_region(). 51 * tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called. 52 * tlb->vma will be non-NULL. 53 * 2. Unmapping all vmas. See exit_mmap(). 54 * tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called. 55 * tlb->vma will be non-NULL. Additionally, page tables will be freed. 56 * 3. Unmapping argument pages. See shift_arg_pages(). 57 * tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called. 58 * tlb->vma will be NULL. 59 */ 60 static inline void tlb_flush(struct mmu_gather *tlb) 61 { 62 if (tlb->fullmm || !tlb->vma) 63 flush_tlb_mm(tlb->mm); 64 else if (tlb->range_end > 0) { 65 flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end); 66 tlb->range_start = TASK_SIZE; 67 tlb->range_end = 0; 68 } 69 } 70 71 static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr) 72 { 73 if (!tlb->fullmm) { 74 if (addr < tlb->range_start) 75 tlb->range_start = addr; 76 if (addr + PAGE_SIZE > tlb->range_end) 77 tlb->range_end = addr + PAGE_SIZE; 78 } 79 } 80 81 static inline void __tlb_alloc_page(struct mmu_gather *tlb) 82 { 83 unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); 84 85 if (addr) { 86 tlb->pages = (void *)addr; 87 tlb->max = PAGE_SIZE / sizeof(struct page *); 88 } 89 } 90 91 static inline void tlb_flush_mmu(struct mmu_gather *tlb) 92 { 93 tlb_flush(tlb); 94 free_pages_and_swap_cache(tlb->pages, tlb->nr); 95 tlb->nr = 0; 96 if (tlb->pages == tlb->local) 97 __tlb_alloc_page(tlb); 98 } 99 100 static inline void 101 tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end) 102 { 103 tlb->mm = mm; 104 tlb->fullmm = !(start | (end+1)); 105 tlb->start = start; 106 tlb->end = end; 107 tlb->vma = NULL; 108 tlb->max = ARRAY_SIZE(tlb->local); 109 tlb->pages = tlb->local; 110 tlb->nr = 0; 111 __tlb_alloc_page(tlb); 112 } 113 114 static inline void 115 tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) 116 { 117 tlb_flush_mmu(tlb); 118 119 /* keep the page table cache within bounds */ 120 check_pgt_cache(); 121 122 if (tlb->pages != tlb->local) 123 free_pages((unsigned long)tlb->pages, 0); 124 } 125 126 /* 127 * Memorize the range for the TLB flush. 128 */ 129 static inline void 130 tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr) 131 { 132 tlb_add_flush(tlb, addr); 133 } 134 135 /* 136 * In the case of tlb vma handling, we can optimise these away in the 137 * case where we're doing a full MM flush. When we're doing a munmap, 138 * the vmas are adjusted to only cover the region to be torn down. 139 */ 140 static inline void 141 tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) 142 { 143 if (!tlb->fullmm) { 144 tlb->vma = vma; 145 tlb->range_start = TASK_SIZE; 146 tlb->range_end = 0; 147 } 148 } 149 150 static inline void 151 tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) 152 { 153 if (!tlb->fullmm) 154 tlb_flush(tlb); 155 } 156 157 static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) 158 { 159 tlb->pages[tlb->nr++] = page; 160 VM_BUG_ON(tlb->nr > tlb->max); 161 return tlb->max - tlb->nr; 162 } 163 164 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) 165 { 166 if (!__tlb_remove_page(tlb, page)) 167 tlb_flush_mmu(tlb); 168 } 169 170 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, 171 unsigned long addr) 172 { 173 pgtable_page_dtor(pte); 174 tlb_add_flush(tlb, addr); 175 tlb_remove_page(tlb, pte); 176 } 177 178 #ifndef CONFIG_ARM64_64K_PAGES 179 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, 180 unsigned long addr) 181 { 182 tlb_add_flush(tlb, addr); 183 tlb_remove_page(tlb, virt_to_page(pmdp)); 184 } 185 #endif 186 187 #define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr) 188 #define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr) 189 #define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp) 190 191 #define tlb_migrate_finish(mm) do { } while (0) 192 193 static inline void 194 tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) 195 { 196 tlb_add_flush(tlb, addr); 197 } 198 199 #endif 200