1196d9d8bSPeter Zijlstra #include <linux/gfp.h> 2196d9d8bSPeter Zijlstra #include <linux/highmem.h> 3196d9d8bSPeter Zijlstra #include <linux/kernel.h> 4ac801e7eSAlexander Potapenko #include <linux/kmsan-checks.h> 5196d9d8bSPeter Zijlstra #include <linux/mmdebug.h> 6196d9d8bSPeter Zijlstra #include <linux/mm_types.h> 736090defSArnd Bergmann #include <linux/mm_inline.h> 8196d9d8bSPeter Zijlstra #include <linux/pagemap.h> 9196d9d8bSPeter Zijlstra #include <linux/rcupdate.h> 10196d9d8bSPeter Zijlstra #include <linux/smp.h> 11196d9d8bSPeter Zijlstra #include <linux/swap.h> 12*5df397deSLinus Torvalds #include <linux/rmap.h> 13196d9d8bSPeter Zijlstra 14196d9d8bSPeter Zijlstra #include <asm/pgalloc.h> 15196d9d8bSPeter Zijlstra #include <asm/tlb.h> 16196d9d8bSPeter Zijlstra 17580a586cSPeter Zijlstra #ifndef CONFIG_MMU_GATHER_NO_GATHER 18952a31c9SMartin Schwidefsky 19196d9d8bSPeter Zijlstra static bool tlb_next_batch(struct mmu_gather *tlb) 20196d9d8bSPeter Zijlstra { 21196d9d8bSPeter Zijlstra struct mmu_gather_batch *batch; 22196d9d8bSPeter Zijlstra 23*5df397deSLinus Torvalds /* No more batching if we have delayed rmaps pending */ 24*5df397deSLinus Torvalds if (tlb->delayed_rmap) 25*5df397deSLinus Torvalds return false; 26*5df397deSLinus Torvalds 27196d9d8bSPeter Zijlstra batch = tlb->active; 28196d9d8bSPeter Zijlstra if (batch->next) { 29196d9d8bSPeter Zijlstra tlb->active = batch->next; 30196d9d8bSPeter Zijlstra return true; 31196d9d8bSPeter Zijlstra } 32196d9d8bSPeter Zijlstra 33196d9d8bSPeter Zijlstra if (tlb->batch_count == MAX_GATHER_BATCH_COUNT) 34196d9d8bSPeter Zijlstra return false; 35196d9d8bSPeter Zijlstra 36196d9d8bSPeter Zijlstra batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); 37196d9d8bSPeter Zijlstra if (!batch) 38196d9d8bSPeter Zijlstra return false; 39196d9d8bSPeter Zijlstra 40196d9d8bSPeter Zijlstra tlb->batch_count++; 41196d9d8bSPeter Zijlstra batch->next = NULL; 42196d9d8bSPeter Zijlstra batch->nr = 0; 43196d9d8bSPeter Zijlstra batch->max = MAX_GATHER_BATCH; 44196d9d8bSPeter Zijlstra 45196d9d8bSPeter Zijlstra tlb->active->next = batch; 46196d9d8bSPeter Zijlstra tlb->active = batch; 47196d9d8bSPeter Zijlstra 48196d9d8bSPeter Zijlstra return true; 49196d9d8bSPeter Zijlstra } 50196d9d8bSPeter Zijlstra 51*5df397deSLinus Torvalds #ifdef CONFIG_SMP 52*5df397deSLinus Torvalds /** 53*5df397deSLinus Torvalds * tlb_flush_rmaps - do pending rmap removals after we have flushed the TLB 54*5df397deSLinus Torvalds * @tlb: the current mmu_gather 55*5df397deSLinus Torvalds * 56*5df397deSLinus Torvalds * Note that because of how tlb_next_batch() above works, we will 57*5df397deSLinus Torvalds * never start new batches with pending delayed rmaps, so we only 58*5df397deSLinus Torvalds * need to walk through the current active batch. 59*5df397deSLinus Torvalds */ 60*5df397deSLinus Torvalds void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) 61*5df397deSLinus Torvalds { 62*5df397deSLinus Torvalds struct mmu_gather_batch *batch; 63*5df397deSLinus Torvalds 64*5df397deSLinus Torvalds batch = tlb->active; 65*5df397deSLinus Torvalds for (int i = 0; i < batch->nr; i++) { 66*5df397deSLinus Torvalds struct encoded_page *enc = batch->encoded_pages[i]; 67*5df397deSLinus Torvalds 68*5df397deSLinus Torvalds if (encoded_page_flags(enc)) { 69*5df397deSLinus Torvalds struct page *page = encoded_page_ptr(enc); 70*5df397deSLinus Torvalds page_remove_rmap(page, vma, false); 71*5df397deSLinus Torvalds } 72*5df397deSLinus Torvalds } 73*5df397deSLinus Torvalds 74*5df397deSLinus Torvalds tlb->delayed_rmap = 0; 75*5df397deSLinus Torvalds } 76*5df397deSLinus Torvalds #endif 77*5df397deSLinus Torvalds 78952a31c9SMartin Schwidefsky static void tlb_batch_pages_flush(struct mmu_gather *tlb) 79196d9d8bSPeter Zijlstra { 80196d9d8bSPeter Zijlstra struct mmu_gather_batch *batch; 81196d9d8bSPeter Zijlstra 82196d9d8bSPeter Zijlstra for (batch = &tlb->local; batch && batch->nr; batch = batch->next) { 837cc8f9c7SLinus Torvalds struct encoded_page **pages = batch->encoded_pages; 84b191c9bcSJianxing Wang 85b191c9bcSJianxing Wang do { 86b191c9bcSJianxing Wang /* 87b191c9bcSJianxing Wang * limit free batch count when PAGE_SIZE > 4K 88b191c9bcSJianxing Wang */ 89b191c9bcSJianxing Wang unsigned int nr = min(512U, batch->nr); 90b191c9bcSJianxing Wang 91b191c9bcSJianxing Wang free_pages_and_swap_cache(pages, nr); 92b191c9bcSJianxing Wang pages += nr; 93b191c9bcSJianxing Wang batch->nr -= nr; 94b191c9bcSJianxing Wang 95b191c9bcSJianxing Wang cond_resched(); 96b191c9bcSJianxing Wang } while (batch->nr); 97196d9d8bSPeter Zijlstra } 98196d9d8bSPeter Zijlstra tlb->active = &tlb->local; 99196d9d8bSPeter Zijlstra } 100196d9d8bSPeter Zijlstra 101952a31c9SMartin Schwidefsky static void tlb_batch_list_free(struct mmu_gather *tlb) 102196d9d8bSPeter Zijlstra { 103196d9d8bSPeter Zijlstra struct mmu_gather_batch *batch, *next; 104196d9d8bSPeter Zijlstra 105196d9d8bSPeter Zijlstra for (batch = tlb->local.next; batch; batch = next) { 106196d9d8bSPeter Zijlstra next = batch->next; 107196d9d8bSPeter Zijlstra free_pages((unsigned long)batch, 0); 108196d9d8bSPeter Zijlstra } 109196d9d8bSPeter Zijlstra tlb->local.next = NULL; 110196d9d8bSPeter Zijlstra } 111196d9d8bSPeter Zijlstra 1127cc8f9c7SLinus Torvalds bool __tlb_remove_page_size(struct mmu_gather *tlb, struct encoded_page *page, int page_size) 113196d9d8bSPeter Zijlstra { 114196d9d8bSPeter Zijlstra struct mmu_gather_batch *batch; 115196d9d8bSPeter Zijlstra 116196d9d8bSPeter Zijlstra VM_BUG_ON(!tlb->end); 117ed6a7935SPeter Zijlstra 1183af4bd03SPeter Zijlstra #ifdef CONFIG_MMU_GATHER_PAGE_SIZE 119196d9d8bSPeter Zijlstra VM_WARN_ON(tlb->page_size != page_size); 120ed6a7935SPeter Zijlstra #endif 121196d9d8bSPeter Zijlstra 122196d9d8bSPeter Zijlstra batch = tlb->active; 123196d9d8bSPeter Zijlstra /* 124196d9d8bSPeter Zijlstra * Add the page and check if we are full. If so 125196d9d8bSPeter Zijlstra * force a flush. 126196d9d8bSPeter Zijlstra */ 1277cc8f9c7SLinus Torvalds batch->encoded_pages[batch->nr++] = page; 128196d9d8bSPeter Zijlstra if (batch->nr == batch->max) { 129196d9d8bSPeter Zijlstra if (!tlb_next_batch(tlb)) 130196d9d8bSPeter Zijlstra return true; 131196d9d8bSPeter Zijlstra batch = tlb->active; 132196d9d8bSPeter Zijlstra } 1337cc8f9c7SLinus Torvalds VM_BUG_ON_PAGE(batch->nr > batch->max, encoded_page_ptr(page)); 134196d9d8bSPeter Zijlstra 135196d9d8bSPeter Zijlstra return false; 136196d9d8bSPeter Zijlstra } 137196d9d8bSPeter Zijlstra 138580a586cSPeter Zijlstra #endif /* MMU_GATHER_NO_GATHER */ 139952a31c9SMartin Schwidefsky 1400d6e24d4SPeter Zijlstra #ifdef CONFIG_MMU_GATHER_TABLE_FREE 1410d6e24d4SPeter Zijlstra 1420d6e24d4SPeter Zijlstra static void __tlb_remove_table_free(struct mmu_table_batch *batch) 1430d6e24d4SPeter Zijlstra { 1440d6e24d4SPeter Zijlstra int i; 1450d6e24d4SPeter Zijlstra 1460d6e24d4SPeter Zijlstra for (i = 0; i < batch->nr; i++) 1470d6e24d4SPeter Zijlstra __tlb_remove_table(batch->tables[i]); 1480d6e24d4SPeter Zijlstra 1490d6e24d4SPeter Zijlstra free_page((unsigned long)batch); 1500d6e24d4SPeter Zijlstra } 1510d6e24d4SPeter Zijlstra 152ff2e6d72SPeter Zijlstra #ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE 153196d9d8bSPeter Zijlstra 154196d9d8bSPeter Zijlstra /* 1550d6e24d4SPeter Zijlstra * Semi RCU freeing of the page directories. 1560d6e24d4SPeter Zijlstra * 1570d6e24d4SPeter Zijlstra * This is needed by some architectures to implement software pagetable walkers. 1580d6e24d4SPeter Zijlstra * 1590d6e24d4SPeter Zijlstra * gup_fast() and other software pagetable walkers do a lockless page-table 1600d6e24d4SPeter Zijlstra * walk and therefore needs some synchronization with the freeing of the page 1610d6e24d4SPeter Zijlstra * directories. The chosen means to accomplish that is by disabling IRQs over 1620d6e24d4SPeter Zijlstra * the walk. 1630d6e24d4SPeter Zijlstra * 1640d6e24d4SPeter Zijlstra * Architectures that use IPIs to flush TLBs will then automagically DTRT, 1650d6e24d4SPeter Zijlstra * since we unlink the page, flush TLBs, free the page. Since the disabling of 1660d6e24d4SPeter Zijlstra * IRQs delays the completion of the TLB flush we can never observe an already 1670d6e24d4SPeter Zijlstra * freed page. 1680d6e24d4SPeter Zijlstra * 1690d6e24d4SPeter Zijlstra * Architectures that do not have this (PPC) need to delay the freeing by some 1700d6e24d4SPeter Zijlstra * other means, this is that means. 1710d6e24d4SPeter Zijlstra * 1720d6e24d4SPeter Zijlstra * What we do is batch the freed directory pages (tables) and RCU free them. 1730d6e24d4SPeter Zijlstra * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling 1740d6e24d4SPeter Zijlstra * holds off grace periods. 1750d6e24d4SPeter Zijlstra * 1760d6e24d4SPeter Zijlstra * However, in order to batch these pages we need to allocate storage, this 1770d6e24d4SPeter Zijlstra * allocation is deep inside the MM code and can thus easily fail on memory 1780d6e24d4SPeter Zijlstra * pressure. To guarantee progress we fall back to single table freeing, see 1790d6e24d4SPeter Zijlstra * the implementation of tlb_remove_table_one(). 1800d6e24d4SPeter Zijlstra * 181196d9d8bSPeter Zijlstra */ 182196d9d8bSPeter Zijlstra 1830d6e24d4SPeter Zijlstra static void tlb_remove_table_smp_sync(void *arg) 1840d6e24d4SPeter Zijlstra { 1850d6e24d4SPeter Zijlstra /* Simply deliver the interrupt */ 1860d6e24d4SPeter Zijlstra } 1870d6e24d4SPeter Zijlstra 1882ba99c5eSJann Horn void tlb_remove_table_sync_one(void) 1890d6e24d4SPeter Zijlstra { 1900d6e24d4SPeter Zijlstra /* 1910d6e24d4SPeter Zijlstra * This isn't an RCU grace period and hence the page-tables cannot be 1920d6e24d4SPeter Zijlstra * assumed to be actually RCU-freed. 1930d6e24d4SPeter Zijlstra * 1940d6e24d4SPeter Zijlstra * It is however sufficient for software page-table walkers that rely on 1950d6e24d4SPeter Zijlstra * IRQ disabling. 1960d6e24d4SPeter Zijlstra */ 1970d6e24d4SPeter Zijlstra smp_call_function(tlb_remove_table_smp_sync, NULL, 1); 1980d6e24d4SPeter Zijlstra } 1990d6e24d4SPeter Zijlstra 2000d6e24d4SPeter Zijlstra static void tlb_remove_table_rcu(struct rcu_head *head) 2010d6e24d4SPeter Zijlstra { 2020d6e24d4SPeter Zijlstra __tlb_remove_table_free(container_of(head, struct mmu_table_batch, rcu)); 2030d6e24d4SPeter Zijlstra } 2040d6e24d4SPeter Zijlstra 2050d6e24d4SPeter Zijlstra static void tlb_remove_table_free(struct mmu_table_batch *batch) 2060d6e24d4SPeter Zijlstra { 2070d6e24d4SPeter Zijlstra call_rcu(&batch->rcu, tlb_remove_table_rcu); 2080d6e24d4SPeter Zijlstra } 2090d6e24d4SPeter Zijlstra 2100d6e24d4SPeter Zijlstra #else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */ 2110d6e24d4SPeter Zijlstra 2120d6e24d4SPeter Zijlstra static void tlb_remove_table_free(struct mmu_table_batch *batch) 2130d6e24d4SPeter Zijlstra { 2140d6e24d4SPeter Zijlstra __tlb_remove_table_free(batch); 2150d6e24d4SPeter Zijlstra } 2160d6e24d4SPeter Zijlstra 2170d6e24d4SPeter Zijlstra #endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */ 2180d6e24d4SPeter Zijlstra 219196d9d8bSPeter Zijlstra /* 220196d9d8bSPeter Zijlstra * If we want tlb_remove_table() to imply TLB invalidates. 221196d9d8bSPeter Zijlstra */ 222196d9d8bSPeter Zijlstra static inline void tlb_table_invalidate(struct mmu_gather *tlb) 223196d9d8bSPeter Zijlstra { 2240ed13259SPeter Zijlstra if (tlb_needs_table_invalidate()) { 225196d9d8bSPeter Zijlstra /* 2260ed13259SPeter Zijlstra * Invalidate page-table caches used by hardware walkers. Then 2270ed13259SPeter Zijlstra * we still need to RCU-sched wait while freeing the pages 2280ed13259SPeter Zijlstra * because software walkers can still be in-flight. 229196d9d8bSPeter Zijlstra */ 230196d9d8bSPeter Zijlstra tlb_flush_mmu_tlbonly(tlb); 2310ed13259SPeter Zijlstra } 232196d9d8bSPeter Zijlstra } 233196d9d8bSPeter Zijlstra 234196d9d8bSPeter Zijlstra static void tlb_remove_table_one(void *table) 235196d9d8bSPeter Zijlstra { 2360d6e24d4SPeter Zijlstra tlb_remove_table_sync_one(); 237196d9d8bSPeter Zijlstra __tlb_remove_table(table); 238196d9d8bSPeter Zijlstra } 239196d9d8bSPeter Zijlstra 2400a8caf21SPeter Zijlstra static void tlb_table_flush(struct mmu_gather *tlb) 241196d9d8bSPeter Zijlstra { 242196d9d8bSPeter Zijlstra struct mmu_table_batch **batch = &tlb->batch; 243196d9d8bSPeter Zijlstra 244196d9d8bSPeter Zijlstra if (*batch) { 245196d9d8bSPeter Zijlstra tlb_table_invalidate(tlb); 2460d6e24d4SPeter Zijlstra tlb_remove_table_free(*batch); 247196d9d8bSPeter Zijlstra *batch = NULL; 248196d9d8bSPeter Zijlstra } 249196d9d8bSPeter Zijlstra } 250196d9d8bSPeter Zijlstra 251196d9d8bSPeter Zijlstra void tlb_remove_table(struct mmu_gather *tlb, void *table) 252196d9d8bSPeter Zijlstra { 253196d9d8bSPeter Zijlstra struct mmu_table_batch **batch = &tlb->batch; 254196d9d8bSPeter Zijlstra 255196d9d8bSPeter Zijlstra if (*batch == NULL) { 256196d9d8bSPeter Zijlstra *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); 257196d9d8bSPeter Zijlstra if (*batch == NULL) { 258196d9d8bSPeter Zijlstra tlb_table_invalidate(tlb); 259196d9d8bSPeter Zijlstra tlb_remove_table_one(table); 260196d9d8bSPeter Zijlstra return; 261196d9d8bSPeter Zijlstra } 262196d9d8bSPeter Zijlstra (*batch)->nr = 0; 263196d9d8bSPeter Zijlstra } 264196d9d8bSPeter Zijlstra 265196d9d8bSPeter Zijlstra (*batch)->tables[(*batch)->nr++] = table; 266196d9d8bSPeter Zijlstra if ((*batch)->nr == MAX_TABLE_BATCH) 267196d9d8bSPeter Zijlstra tlb_table_flush(tlb); 268196d9d8bSPeter Zijlstra } 269196d9d8bSPeter Zijlstra 2700d6e24d4SPeter Zijlstra static inline void tlb_table_init(struct mmu_gather *tlb) 2710d6e24d4SPeter Zijlstra { 2720d6e24d4SPeter Zijlstra tlb->batch = NULL; 2730d6e24d4SPeter Zijlstra } 2740d6e24d4SPeter Zijlstra 2750d6e24d4SPeter Zijlstra #else /* !CONFIG_MMU_GATHER_TABLE_FREE */ 2760d6e24d4SPeter Zijlstra 2770d6e24d4SPeter Zijlstra static inline void tlb_table_flush(struct mmu_gather *tlb) { } 2780d6e24d4SPeter Zijlstra static inline void tlb_table_init(struct mmu_gather *tlb) { } 2790d6e24d4SPeter Zijlstra 2800d6e24d4SPeter Zijlstra #endif /* CONFIG_MMU_GATHER_TABLE_FREE */ 281196d9d8bSPeter Zijlstra 2820a8caf21SPeter Zijlstra static void tlb_flush_mmu_free(struct mmu_gather *tlb) 2830a8caf21SPeter Zijlstra { 2840a8caf21SPeter Zijlstra tlb_table_flush(tlb); 285580a586cSPeter Zijlstra #ifndef CONFIG_MMU_GATHER_NO_GATHER 2860a8caf21SPeter Zijlstra tlb_batch_pages_flush(tlb); 2870a8caf21SPeter Zijlstra #endif 2880a8caf21SPeter Zijlstra } 2890a8caf21SPeter Zijlstra 2900a8caf21SPeter Zijlstra void tlb_flush_mmu(struct mmu_gather *tlb) 2910a8caf21SPeter Zijlstra { 2920a8caf21SPeter Zijlstra tlb_flush_mmu_tlbonly(tlb); 2930a8caf21SPeter Zijlstra tlb_flush_mmu_free(tlb); 2940a8caf21SPeter Zijlstra } 2950a8caf21SPeter Zijlstra 296d8b45053SWill Deacon static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, 297a72afd87SWill Deacon bool fullmm) 298196d9d8bSPeter Zijlstra { 299ac801e7eSAlexander Potapenko /* 300ac801e7eSAlexander Potapenko * struct mmu_gather contains 7 1-bit fields packed into a 32-bit 301ac801e7eSAlexander Potapenko * unsigned int value. The remaining 25 bits remain uninitialized 302ac801e7eSAlexander Potapenko * and are never used, but KMSAN updates the origin for them in 303ac801e7eSAlexander Potapenko * zap_pXX_range() in mm/memory.c, thus creating very long origin 304ac801e7eSAlexander Potapenko * chains. This is technically correct, but consumes too much memory. 305ac801e7eSAlexander Potapenko * Unpoisoning the whole structure will prevent creating such chains. 306ac801e7eSAlexander Potapenko */ 307ac801e7eSAlexander Potapenko kmsan_unpoison_memory(tlb, sizeof(*tlb)); 3081808d65bSPeter Zijlstra tlb->mm = mm; 309a72afd87SWill Deacon tlb->fullmm = fullmm; 3101808d65bSPeter Zijlstra 311580a586cSPeter Zijlstra #ifndef CONFIG_MMU_GATHER_NO_GATHER 3121808d65bSPeter Zijlstra tlb->need_flush_all = 0; 3131808d65bSPeter Zijlstra tlb->local.next = NULL; 3141808d65bSPeter Zijlstra tlb->local.nr = 0; 3151808d65bSPeter Zijlstra tlb->local.max = ARRAY_SIZE(tlb->__pages); 3161808d65bSPeter Zijlstra tlb->active = &tlb->local; 3171808d65bSPeter Zijlstra tlb->batch_count = 0; 3181808d65bSPeter Zijlstra #endif 319*5df397deSLinus Torvalds tlb->delayed_rmap = 0; 3201808d65bSPeter Zijlstra 3210d6e24d4SPeter Zijlstra tlb_table_init(tlb); 3223af4bd03SPeter Zijlstra #ifdef CONFIG_MMU_GATHER_PAGE_SIZE 3231808d65bSPeter Zijlstra tlb->page_size = 0; 3241808d65bSPeter Zijlstra #endif 3251808d65bSPeter Zijlstra 3261808d65bSPeter Zijlstra __tlb_reset_range(tlb); 327196d9d8bSPeter Zijlstra inc_tlb_flush_pending(tlb->mm); 328196d9d8bSPeter Zijlstra } 329196d9d8bSPeter Zijlstra 330845be1cdSRandy Dunlap /** 331845be1cdSRandy Dunlap * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down 332845be1cdSRandy Dunlap * @tlb: the mmu_gather structure to initialize 333845be1cdSRandy Dunlap * @mm: the mm_struct of the target address space 334845be1cdSRandy Dunlap * 335845be1cdSRandy Dunlap * Called to initialize an (on-stack) mmu_gather structure for page-table 336845be1cdSRandy Dunlap * tear-down from @mm. 337845be1cdSRandy Dunlap */ 338a72afd87SWill Deacon void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm) 339d8b45053SWill Deacon { 340a72afd87SWill Deacon __tlb_gather_mmu(tlb, mm, false); 341d8b45053SWill Deacon } 342d8b45053SWill Deacon 343845be1cdSRandy Dunlap /** 344845be1cdSRandy Dunlap * tlb_gather_mmu_fullmm - initialize an mmu_gather structure for page-table tear-down 345845be1cdSRandy Dunlap * @tlb: the mmu_gather structure to initialize 346845be1cdSRandy Dunlap * @mm: the mm_struct of the target address space 347845be1cdSRandy Dunlap * 348845be1cdSRandy Dunlap * In this case, @mm is without users and we're going to destroy the 349845be1cdSRandy Dunlap * full address space (exit/execve). 350845be1cdSRandy Dunlap * 351845be1cdSRandy Dunlap * Called to initialize an (on-stack) mmu_gather structure for page-table 352845be1cdSRandy Dunlap * tear-down from @mm. 353845be1cdSRandy Dunlap */ 354d8b45053SWill Deacon void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm) 355d8b45053SWill Deacon { 356a72afd87SWill Deacon __tlb_gather_mmu(tlb, mm, true); 357d8b45053SWill Deacon } 358d8b45053SWill Deacon 3591808d65bSPeter Zijlstra /** 3601808d65bSPeter Zijlstra * tlb_finish_mmu - finish an mmu_gather structure 3611808d65bSPeter Zijlstra * @tlb: the mmu_gather structure to finish 3621808d65bSPeter Zijlstra * 3631808d65bSPeter Zijlstra * Called at the end of the shootdown operation to free up any resources that 3641808d65bSPeter Zijlstra * were required. 3651808d65bSPeter Zijlstra */ 366ae8eba8bSWill Deacon void tlb_finish_mmu(struct mmu_gather *tlb) 367196d9d8bSPeter Zijlstra { 368196d9d8bSPeter Zijlstra /* 369196d9d8bSPeter Zijlstra * If there are parallel threads are doing PTE changes on same range 370c1e8d7c6SMichel Lespinasse * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB 3717a30df49SYang Shi * flush by batching, one thread may end up seeing inconsistent PTEs 3727a30df49SYang Shi * and result in having stale TLB entries. So flush TLB forcefully 3737a30df49SYang Shi * if we detect parallel PTE batching threads. 3747a30df49SYang Shi * 3757a30df49SYang Shi * However, some syscalls, e.g. munmap(), may free page tables, this 3767a30df49SYang Shi * needs force flush everything in the given range. Otherwise this 3777a30df49SYang Shi * may result in having stale TLB entries for some architectures, 3787a30df49SYang Shi * e.g. aarch64, that could specify flush what level TLB. 379196d9d8bSPeter Zijlstra */ 3801808d65bSPeter Zijlstra if (mm_tlb_flush_nested(tlb->mm)) { 3817a30df49SYang Shi /* 3827a30df49SYang Shi * The aarch64 yields better performance with fullmm by 3837a30df49SYang Shi * avoiding multiple CPUs spamming TLBI messages at the 3847a30df49SYang Shi * same time. 3857a30df49SYang Shi * 3867a30df49SYang Shi * On x86 non-fullmm doesn't yield significant difference 3877a30df49SYang Shi * against fullmm. 3887a30df49SYang Shi */ 3897a30df49SYang Shi tlb->fullmm = 1; 3901808d65bSPeter Zijlstra __tlb_reset_range(tlb); 3917a30df49SYang Shi tlb->freed_tables = 1; 3921808d65bSPeter Zijlstra } 393196d9d8bSPeter Zijlstra 3941808d65bSPeter Zijlstra tlb_flush_mmu(tlb); 3951808d65bSPeter Zijlstra 396580a586cSPeter Zijlstra #ifndef CONFIG_MMU_GATHER_NO_GATHER 3971808d65bSPeter Zijlstra tlb_batch_list_free(tlb); 3981808d65bSPeter Zijlstra #endif 399196d9d8bSPeter Zijlstra dec_tlb_flush_pending(tlb->mm); 400196d9d8bSPeter Zijlstra } 401