1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * Copyright (C) 1993 Linus Torvalds 41da177e4SLinus Torvalds * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 51da177e4SLinus Torvalds * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 61da177e4SLinus Torvalds * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 7930fc45aSChristoph Lameter * Numa awareness, Christoph Lameter, SGI, June 2005 8d758ffe6SUladzislau Rezki (Sony) * Improving global KVA allocator, Uladzislau Rezki, Sony, May 2019 91da177e4SLinus Torvalds */ 101da177e4SLinus Torvalds 11db64fe02SNick Piggin #include <linux/vmalloc.h> 121da177e4SLinus Torvalds #include <linux/mm.h> 131da177e4SLinus Torvalds #include <linux/module.h> 141da177e4SLinus Torvalds #include <linux/highmem.h> 15c3edc401SIngo Molnar #include <linux/sched/signal.h> 161da177e4SLinus Torvalds #include <linux/slab.h> 171da177e4SLinus Torvalds #include <linux/spinlock.h> 181da177e4SLinus Torvalds #include <linux/interrupt.h> 195f6a6a9cSAlexey Dobriyan #include <linux/proc_fs.h> 20a10aa579SChristoph Lameter #include <linux/seq_file.h> 21868b104dSRick Edgecombe #include <linux/set_memory.h> 223ac7fe5aSThomas Gleixner #include <linux/debugobjects.h> 2323016969SChristoph Lameter #include <linux/kallsyms.h> 24db64fe02SNick Piggin #include <linux/list.h> 254da56b99SChris Wilson #include <linux/notifier.h> 26db64fe02SNick Piggin #include <linux/rbtree.h> 270f14599cSMatthew Wilcox (Oracle) #include <linux/xarray.h> 285da96bddSMel Gorman #include <linux/io.h> 29db64fe02SNick Piggin #include <linux/rcupdate.h> 30f0aa6617STejun Heo #include <linux/pfn.h> 3189219d37SCatalin Marinas #include <linux/kmemleak.h> 3260063497SArun Sharma #include <linux/atomic.h> 333b32123dSGideon Israel Dsouza #include <linux/compiler.h> 3432fcfd40SAl Viro #include <linux/llist.h> 350f616be1SToshi Kani #include <linux/bitops.h> 3668ad4a33SUladzislau Rezki (Sony) #include <linux/rbtree_augmented.h> 37bdebd6a2SJann Horn #include <linux/overflow.h> 38c0eb315aSNicholas Piggin #include <linux/pgtable.h> 397c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 40f7ee1f13SChristophe Leroy #include <linux/hugetlb.h> 411da177e4SLinus Torvalds #include <asm/tlbflush.h> 422dca6999SDavid Miller #include <asm/shmparam.h> 431da177e4SLinus Torvalds 44dd56b046SMel Gorman #include "internal.h" 452a681cfaSJoerg Roedel #include "pgalloc-track.h" 46dd56b046SMel Gorman 47121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 48121e6f32SNicholas Piggin static bool __ro_after_init vmap_allow_huge = true; 49121e6f32SNicholas Piggin 50121e6f32SNicholas Piggin static int __init set_nohugevmalloc(char *str) 51121e6f32SNicholas Piggin { 52121e6f32SNicholas Piggin vmap_allow_huge = false; 53121e6f32SNicholas Piggin return 0; 54121e6f32SNicholas Piggin } 55121e6f32SNicholas Piggin early_param("nohugevmalloc", set_nohugevmalloc); 56121e6f32SNicholas Piggin #else /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ 57121e6f32SNicholas Piggin static const bool vmap_allow_huge = false; 58121e6f32SNicholas Piggin #endif /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ 59121e6f32SNicholas Piggin 60186525bdSIngo Molnar bool is_vmalloc_addr(const void *x) 61186525bdSIngo Molnar { 62186525bdSIngo Molnar unsigned long addr = (unsigned long)x; 63186525bdSIngo Molnar 64186525bdSIngo Molnar return addr >= VMALLOC_START && addr < VMALLOC_END; 65186525bdSIngo Molnar } 66186525bdSIngo Molnar EXPORT_SYMBOL(is_vmalloc_addr); 67186525bdSIngo Molnar 6832fcfd40SAl Viro struct vfree_deferred { 6932fcfd40SAl Viro struct llist_head list; 7032fcfd40SAl Viro struct work_struct wq; 7132fcfd40SAl Viro }; 7232fcfd40SAl Viro static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); 7332fcfd40SAl Viro 7432fcfd40SAl Viro static void __vunmap(const void *, int); 7532fcfd40SAl Viro 7632fcfd40SAl Viro static void free_work(struct work_struct *w) 7732fcfd40SAl Viro { 7832fcfd40SAl Viro struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); 79894e58c1SByungchul Park struct llist_node *t, *llnode; 80894e58c1SByungchul Park 81894e58c1SByungchul Park llist_for_each_safe(llnode, t, llist_del_all(&p->list)) 82894e58c1SByungchul Park __vunmap((void *)llnode, 1); 8332fcfd40SAl Viro } 8432fcfd40SAl Viro 85db64fe02SNick Piggin /*** Page table manipulation functions ***/ 865e9e3d77SNicholas Piggin static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 875e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 88f7ee1f13SChristophe Leroy unsigned int max_page_shift, pgtbl_mod_mask *mask) 895e9e3d77SNicholas Piggin { 905e9e3d77SNicholas Piggin pte_t *pte; 915e9e3d77SNicholas Piggin u64 pfn; 92f7ee1f13SChristophe Leroy unsigned long size = PAGE_SIZE; 935e9e3d77SNicholas Piggin 945e9e3d77SNicholas Piggin pfn = phys_addr >> PAGE_SHIFT; 955e9e3d77SNicholas Piggin pte = pte_alloc_kernel_track(pmd, addr, mask); 965e9e3d77SNicholas Piggin if (!pte) 975e9e3d77SNicholas Piggin return -ENOMEM; 985e9e3d77SNicholas Piggin do { 995e9e3d77SNicholas Piggin BUG_ON(!pte_none(*pte)); 100f7ee1f13SChristophe Leroy 101f7ee1f13SChristophe Leroy #ifdef CONFIG_HUGETLB_PAGE 102f7ee1f13SChristophe Leroy size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift); 103f7ee1f13SChristophe Leroy if (size != PAGE_SIZE) { 104f7ee1f13SChristophe Leroy pte_t entry = pfn_pte(pfn, prot); 105f7ee1f13SChristophe Leroy 106f7ee1f13SChristophe Leroy entry = pte_mkhuge(entry); 107f7ee1f13SChristophe Leroy entry = arch_make_huge_pte(entry, ilog2(size), 0); 108f7ee1f13SChristophe Leroy set_huge_pte_at(&init_mm, addr, pte, entry); 109f7ee1f13SChristophe Leroy pfn += PFN_DOWN(size); 110f7ee1f13SChristophe Leroy continue; 111f7ee1f13SChristophe Leroy } 112f7ee1f13SChristophe Leroy #endif 1135e9e3d77SNicholas Piggin set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); 1145e9e3d77SNicholas Piggin pfn++; 115f7ee1f13SChristophe Leroy } while (pte += PFN_DOWN(size), addr += size, addr != end); 1165e9e3d77SNicholas Piggin *mask |= PGTBL_PTE_MODIFIED; 1175e9e3d77SNicholas Piggin return 0; 1185e9e3d77SNicholas Piggin } 1195e9e3d77SNicholas Piggin 1205e9e3d77SNicholas Piggin static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, 1215e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1225e9e3d77SNicholas Piggin unsigned int max_page_shift) 1235e9e3d77SNicholas Piggin { 1245e9e3d77SNicholas Piggin if (max_page_shift < PMD_SHIFT) 1255e9e3d77SNicholas Piggin return 0; 1265e9e3d77SNicholas Piggin 1275e9e3d77SNicholas Piggin if (!arch_vmap_pmd_supported(prot)) 1285e9e3d77SNicholas Piggin return 0; 1295e9e3d77SNicholas Piggin 1305e9e3d77SNicholas Piggin if ((end - addr) != PMD_SIZE) 1315e9e3d77SNicholas Piggin return 0; 1325e9e3d77SNicholas Piggin 1335e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, PMD_SIZE)) 1345e9e3d77SNicholas Piggin return 0; 1355e9e3d77SNicholas Piggin 1365e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, PMD_SIZE)) 1375e9e3d77SNicholas Piggin return 0; 1385e9e3d77SNicholas Piggin 1395e9e3d77SNicholas Piggin if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) 1405e9e3d77SNicholas Piggin return 0; 1415e9e3d77SNicholas Piggin 1425e9e3d77SNicholas Piggin return pmd_set_huge(pmd, phys_addr, prot); 1435e9e3d77SNicholas Piggin } 1445e9e3d77SNicholas Piggin 1455e9e3d77SNicholas Piggin static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 1465e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1475e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 1485e9e3d77SNicholas Piggin { 1495e9e3d77SNicholas Piggin pmd_t *pmd; 1505e9e3d77SNicholas Piggin unsigned long next; 1515e9e3d77SNicholas Piggin 1525e9e3d77SNicholas Piggin pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 1535e9e3d77SNicholas Piggin if (!pmd) 1545e9e3d77SNicholas Piggin return -ENOMEM; 1555e9e3d77SNicholas Piggin do { 1565e9e3d77SNicholas Piggin next = pmd_addr_end(addr, end); 1575e9e3d77SNicholas Piggin 1585e9e3d77SNicholas Piggin if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot, 1595e9e3d77SNicholas Piggin max_page_shift)) { 1605e9e3d77SNicholas Piggin *mask |= PGTBL_PMD_MODIFIED; 1615e9e3d77SNicholas Piggin continue; 1625e9e3d77SNicholas Piggin } 1635e9e3d77SNicholas Piggin 164f7ee1f13SChristophe Leroy if (vmap_pte_range(pmd, addr, next, phys_addr, prot, max_page_shift, mask)) 1655e9e3d77SNicholas Piggin return -ENOMEM; 1665e9e3d77SNicholas Piggin } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); 1675e9e3d77SNicholas Piggin return 0; 1685e9e3d77SNicholas Piggin } 1695e9e3d77SNicholas Piggin 1705e9e3d77SNicholas Piggin static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end, 1715e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1725e9e3d77SNicholas Piggin unsigned int max_page_shift) 1735e9e3d77SNicholas Piggin { 1745e9e3d77SNicholas Piggin if (max_page_shift < PUD_SHIFT) 1755e9e3d77SNicholas Piggin return 0; 1765e9e3d77SNicholas Piggin 1775e9e3d77SNicholas Piggin if (!arch_vmap_pud_supported(prot)) 1785e9e3d77SNicholas Piggin return 0; 1795e9e3d77SNicholas Piggin 1805e9e3d77SNicholas Piggin if ((end - addr) != PUD_SIZE) 1815e9e3d77SNicholas Piggin return 0; 1825e9e3d77SNicholas Piggin 1835e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, PUD_SIZE)) 1845e9e3d77SNicholas Piggin return 0; 1855e9e3d77SNicholas Piggin 1865e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, PUD_SIZE)) 1875e9e3d77SNicholas Piggin return 0; 1885e9e3d77SNicholas Piggin 1895e9e3d77SNicholas Piggin if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) 1905e9e3d77SNicholas Piggin return 0; 1915e9e3d77SNicholas Piggin 1925e9e3d77SNicholas Piggin return pud_set_huge(pud, phys_addr, prot); 1935e9e3d77SNicholas Piggin } 1945e9e3d77SNicholas Piggin 1955e9e3d77SNicholas Piggin static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 1965e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1975e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 1985e9e3d77SNicholas Piggin { 1995e9e3d77SNicholas Piggin pud_t *pud; 2005e9e3d77SNicholas Piggin unsigned long next; 2015e9e3d77SNicholas Piggin 2025e9e3d77SNicholas Piggin pud = pud_alloc_track(&init_mm, p4d, addr, mask); 2035e9e3d77SNicholas Piggin if (!pud) 2045e9e3d77SNicholas Piggin return -ENOMEM; 2055e9e3d77SNicholas Piggin do { 2065e9e3d77SNicholas Piggin next = pud_addr_end(addr, end); 2075e9e3d77SNicholas Piggin 2085e9e3d77SNicholas Piggin if (vmap_try_huge_pud(pud, addr, next, phys_addr, prot, 2095e9e3d77SNicholas Piggin max_page_shift)) { 2105e9e3d77SNicholas Piggin *mask |= PGTBL_PUD_MODIFIED; 2115e9e3d77SNicholas Piggin continue; 2125e9e3d77SNicholas Piggin } 2135e9e3d77SNicholas Piggin 2145e9e3d77SNicholas Piggin if (vmap_pmd_range(pud, addr, next, phys_addr, prot, 2155e9e3d77SNicholas Piggin max_page_shift, mask)) 2165e9e3d77SNicholas Piggin return -ENOMEM; 2175e9e3d77SNicholas Piggin } while (pud++, phys_addr += (next - addr), addr = next, addr != end); 2185e9e3d77SNicholas Piggin return 0; 2195e9e3d77SNicholas Piggin } 2205e9e3d77SNicholas Piggin 2215e9e3d77SNicholas Piggin static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, 2225e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2235e9e3d77SNicholas Piggin unsigned int max_page_shift) 2245e9e3d77SNicholas Piggin { 2255e9e3d77SNicholas Piggin if (max_page_shift < P4D_SHIFT) 2265e9e3d77SNicholas Piggin return 0; 2275e9e3d77SNicholas Piggin 2285e9e3d77SNicholas Piggin if (!arch_vmap_p4d_supported(prot)) 2295e9e3d77SNicholas Piggin return 0; 2305e9e3d77SNicholas Piggin 2315e9e3d77SNicholas Piggin if ((end - addr) != P4D_SIZE) 2325e9e3d77SNicholas Piggin return 0; 2335e9e3d77SNicholas Piggin 2345e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, P4D_SIZE)) 2355e9e3d77SNicholas Piggin return 0; 2365e9e3d77SNicholas Piggin 2375e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, P4D_SIZE)) 2385e9e3d77SNicholas Piggin return 0; 2395e9e3d77SNicholas Piggin 2405e9e3d77SNicholas Piggin if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) 2415e9e3d77SNicholas Piggin return 0; 2425e9e3d77SNicholas Piggin 2435e9e3d77SNicholas Piggin return p4d_set_huge(p4d, phys_addr, prot); 2445e9e3d77SNicholas Piggin } 2455e9e3d77SNicholas Piggin 2465e9e3d77SNicholas Piggin static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 2475e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2485e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 2495e9e3d77SNicholas Piggin { 2505e9e3d77SNicholas Piggin p4d_t *p4d; 2515e9e3d77SNicholas Piggin unsigned long next; 2525e9e3d77SNicholas Piggin 2535e9e3d77SNicholas Piggin p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 2545e9e3d77SNicholas Piggin if (!p4d) 2555e9e3d77SNicholas Piggin return -ENOMEM; 2565e9e3d77SNicholas Piggin do { 2575e9e3d77SNicholas Piggin next = p4d_addr_end(addr, end); 2585e9e3d77SNicholas Piggin 2595e9e3d77SNicholas Piggin if (vmap_try_huge_p4d(p4d, addr, next, phys_addr, prot, 2605e9e3d77SNicholas Piggin max_page_shift)) { 2615e9e3d77SNicholas Piggin *mask |= PGTBL_P4D_MODIFIED; 2625e9e3d77SNicholas Piggin continue; 2635e9e3d77SNicholas Piggin } 2645e9e3d77SNicholas Piggin 2655e9e3d77SNicholas Piggin if (vmap_pud_range(p4d, addr, next, phys_addr, prot, 2665e9e3d77SNicholas Piggin max_page_shift, mask)) 2675e9e3d77SNicholas Piggin return -ENOMEM; 2685e9e3d77SNicholas Piggin } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); 2695e9e3d77SNicholas Piggin return 0; 2705e9e3d77SNicholas Piggin } 2715e9e3d77SNicholas Piggin 2725d87510dSNicholas Piggin static int vmap_range_noflush(unsigned long addr, unsigned long end, 2735e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2745e9e3d77SNicholas Piggin unsigned int max_page_shift) 2755e9e3d77SNicholas Piggin { 2765e9e3d77SNicholas Piggin pgd_t *pgd; 2775e9e3d77SNicholas Piggin unsigned long start; 2785e9e3d77SNicholas Piggin unsigned long next; 2795e9e3d77SNicholas Piggin int err; 2805e9e3d77SNicholas Piggin pgtbl_mod_mask mask = 0; 2815e9e3d77SNicholas Piggin 2825e9e3d77SNicholas Piggin might_sleep(); 2835e9e3d77SNicholas Piggin BUG_ON(addr >= end); 2845e9e3d77SNicholas Piggin 2855e9e3d77SNicholas Piggin start = addr; 2865e9e3d77SNicholas Piggin pgd = pgd_offset_k(addr); 2875e9e3d77SNicholas Piggin do { 2885e9e3d77SNicholas Piggin next = pgd_addr_end(addr, end); 2895e9e3d77SNicholas Piggin err = vmap_p4d_range(pgd, addr, next, phys_addr, prot, 2905e9e3d77SNicholas Piggin max_page_shift, &mask); 2915e9e3d77SNicholas Piggin if (err) 2925e9e3d77SNicholas Piggin break; 2935e9e3d77SNicholas Piggin } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); 2945e9e3d77SNicholas Piggin 2955e9e3d77SNicholas Piggin if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 2965e9e3d77SNicholas Piggin arch_sync_kernel_mappings(start, end); 2975e9e3d77SNicholas Piggin 2985e9e3d77SNicholas Piggin return err; 2995e9e3d77SNicholas Piggin } 300b221385bSAdrian Bunk 3015d87510dSNicholas Piggin int vmap_range(unsigned long addr, unsigned long end, 3025d87510dSNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 3035d87510dSNicholas Piggin unsigned int max_page_shift) 3045d87510dSNicholas Piggin { 3055d87510dSNicholas Piggin int err; 3065d87510dSNicholas Piggin 3075d87510dSNicholas Piggin err = vmap_range_noflush(addr, end, phys_addr, prot, max_page_shift); 3085d87510dSNicholas Piggin flush_cache_vmap(addr, end); 3095d87510dSNicholas Piggin 3105d87510dSNicholas Piggin return err; 3115d87510dSNicholas Piggin } 3125d87510dSNicholas Piggin 3132ba3e694SJoerg Roedel static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 3142ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3151da177e4SLinus Torvalds { 3161da177e4SLinus Torvalds pte_t *pte; 3171da177e4SLinus Torvalds 3181da177e4SLinus Torvalds pte = pte_offset_kernel(pmd, addr); 3191da177e4SLinus Torvalds do { 3201da177e4SLinus Torvalds pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); 3211da177e4SLinus Torvalds WARN_ON(!pte_none(ptent) && !pte_present(ptent)); 3221da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 3232ba3e694SJoerg Roedel *mask |= PGTBL_PTE_MODIFIED; 3241da177e4SLinus Torvalds } 3251da177e4SLinus Torvalds 3262ba3e694SJoerg Roedel static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 3272ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3281da177e4SLinus Torvalds { 3291da177e4SLinus Torvalds pmd_t *pmd; 3301da177e4SLinus Torvalds unsigned long next; 3312ba3e694SJoerg Roedel int cleared; 3321da177e4SLinus Torvalds 3331da177e4SLinus Torvalds pmd = pmd_offset(pud, addr); 3341da177e4SLinus Torvalds do { 3351da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 3362ba3e694SJoerg Roedel 3372ba3e694SJoerg Roedel cleared = pmd_clear_huge(pmd); 3382ba3e694SJoerg Roedel if (cleared || pmd_bad(*pmd)) 3392ba3e694SJoerg Roedel *mask |= PGTBL_PMD_MODIFIED; 3402ba3e694SJoerg Roedel 3412ba3e694SJoerg Roedel if (cleared) 342b9820d8fSToshi Kani continue; 3431da177e4SLinus Torvalds if (pmd_none_or_clear_bad(pmd)) 3441da177e4SLinus Torvalds continue; 3452ba3e694SJoerg Roedel vunmap_pte_range(pmd, addr, next, mask); 346e47110e9SAneesh Kumar K.V 347e47110e9SAneesh Kumar K.V cond_resched(); 3481da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 3491da177e4SLinus Torvalds } 3501da177e4SLinus Torvalds 3512ba3e694SJoerg Roedel static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 3522ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3531da177e4SLinus Torvalds { 3541da177e4SLinus Torvalds pud_t *pud; 3551da177e4SLinus Torvalds unsigned long next; 3562ba3e694SJoerg Roedel int cleared; 3571da177e4SLinus Torvalds 358c2febafcSKirill A. Shutemov pud = pud_offset(p4d, addr); 3591da177e4SLinus Torvalds do { 3601da177e4SLinus Torvalds next = pud_addr_end(addr, end); 3612ba3e694SJoerg Roedel 3622ba3e694SJoerg Roedel cleared = pud_clear_huge(pud); 3632ba3e694SJoerg Roedel if (cleared || pud_bad(*pud)) 3642ba3e694SJoerg Roedel *mask |= PGTBL_PUD_MODIFIED; 3652ba3e694SJoerg Roedel 3662ba3e694SJoerg Roedel if (cleared) 367b9820d8fSToshi Kani continue; 3681da177e4SLinus Torvalds if (pud_none_or_clear_bad(pud)) 3691da177e4SLinus Torvalds continue; 3702ba3e694SJoerg Roedel vunmap_pmd_range(pud, addr, next, mask); 3711da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 3721da177e4SLinus Torvalds } 3731da177e4SLinus Torvalds 3742ba3e694SJoerg Roedel static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 3752ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 376c2febafcSKirill A. Shutemov { 377c2febafcSKirill A. Shutemov p4d_t *p4d; 378c2febafcSKirill A. Shutemov unsigned long next; 3792ba3e694SJoerg Roedel int cleared; 380c2febafcSKirill A. Shutemov 381c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 382c2febafcSKirill A. Shutemov do { 383c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 3842ba3e694SJoerg Roedel 3852ba3e694SJoerg Roedel cleared = p4d_clear_huge(p4d); 3862ba3e694SJoerg Roedel if (cleared || p4d_bad(*p4d)) 3872ba3e694SJoerg Roedel *mask |= PGTBL_P4D_MODIFIED; 3882ba3e694SJoerg Roedel 3892ba3e694SJoerg Roedel if (cleared) 390c2febafcSKirill A. Shutemov continue; 391c2febafcSKirill A. Shutemov if (p4d_none_or_clear_bad(p4d)) 392c2febafcSKirill A. Shutemov continue; 3932ba3e694SJoerg Roedel vunmap_pud_range(p4d, addr, next, mask); 394c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 395c2febafcSKirill A. Shutemov } 396c2febafcSKirill A. Shutemov 3974ad0ae8cSNicholas Piggin /* 3984ad0ae8cSNicholas Piggin * vunmap_range_noflush is similar to vunmap_range, but does not 3994ad0ae8cSNicholas Piggin * flush caches or TLBs. 400b521c43fSChristoph Hellwig * 4014ad0ae8cSNicholas Piggin * The caller is responsible for calling flush_cache_vmap() before calling 4024ad0ae8cSNicholas Piggin * this function, and flush_tlb_kernel_range after it has returned 4034ad0ae8cSNicholas Piggin * successfully (and before the addresses are expected to cause a page fault 4044ad0ae8cSNicholas Piggin * or be re-mapped for something else, if TLB flushes are being delayed or 4054ad0ae8cSNicholas Piggin * coalesced). 406b521c43fSChristoph Hellwig * 4074ad0ae8cSNicholas Piggin * This is an internal function only. Do not use outside mm/. 408b521c43fSChristoph Hellwig */ 4094ad0ae8cSNicholas Piggin void vunmap_range_noflush(unsigned long start, unsigned long end) 4101da177e4SLinus Torvalds { 4111da177e4SLinus Torvalds unsigned long next; 412b521c43fSChristoph Hellwig pgd_t *pgd; 4132ba3e694SJoerg Roedel unsigned long addr = start; 4142ba3e694SJoerg Roedel pgtbl_mod_mask mask = 0; 4151da177e4SLinus Torvalds 4161da177e4SLinus Torvalds BUG_ON(addr >= end); 4171da177e4SLinus Torvalds pgd = pgd_offset_k(addr); 4181da177e4SLinus Torvalds do { 4191da177e4SLinus Torvalds next = pgd_addr_end(addr, end); 4202ba3e694SJoerg Roedel if (pgd_bad(*pgd)) 4212ba3e694SJoerg Roedel mask |= PGTBL_PGD_MODIFIED; 4221da177e4SLinus Torvalds if (pgd_none_or_clear_bad(pgd)) 4231da177e4SLinus Torvalds continue; 4242ba3e694SJoerg Roedel vunmap_p4d_range(pgd, addr, next, &mask); 4251da177e4SLinus Torvalds } while (pgd++, addr = next, addr != end); 4262ba3e694SJoerg Roedel 4272ba3e694SJoerg Roedel if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 4282ba3e694SJoerg Roedel arch_sync_kernel_mappings(start, end); 4291da177e4SLinus Torvalds } 4301da177e4SLinus Torvalds 4314ad0ae8cSNicholas Piggin /** 4324ad0ae8cSNicholas Piggin * vunmap_range - unmap kernel virtual addresses 4334ad0ae8cSNicholas Piggin * @addr: start of the VM area to unmap 4344ad0ae8cSNicholas Piggin * @end: end of the VM area to unmap (non-inclusive) 4354ad0ae8cSNicholas Piggin * 4364ad0ae8cSNicholas Piggin * Clears any present PTEs in the virtual address range, flushes TLBs and 4374ad0ae8cSNicholas Piggin * caches. Any subsequent access to the address before it has been re-mapped 4384ad0ae8cSNicholas Piggin * is a kernel bug. 4394ad0ae8cSNicholas Piggin */ 4404ad0ae8cSNicholas Piggin void vunmap_range(unsigned long addr, unsigned long end) 4414ad0ae8cSNicholas Piggin { 4424ad0ae8cSNicholas Piggin flush_cache_vunmap(addr, end); 4434ad0ae8cSNicholas Piggin vunmap_range_noflush(addr, end); 4444ad0ae8cSNicholas Piggin flush_tlb_kernel_range(addr, end); 4454ad0ae8cSNicholas Piggin } 4464ad0ae8cSNicholas Piggin 4470a264884SNicholas Piggin static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, 4482ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 4492ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 4501da177e4SLinus Torvalds { 4511da177e4SLinus Torvalds pte_t *pte; 4521da177e4SLinus Torvalds 453db64fe02SNick Piggin /* 454db64fe02SNick Piggin * nr is a running index into the array which helps higher level 455db64fe02SNick Piggin * callers keep track of where we're up to. 456db64fe02SNick Piggin */ 457db64fe02SNick Piggin 4582ba3e694SJoerg Roedel pte = pte_alloc_kernel_track(pmd, addr, mask); 4591da177e4SLinus Torvalds if (!pte) 4601da177e4SLinus Torvalds return -ENOMEM; 4611da177e4SLinus Torvalds do { 462db64fe02SNick Piggin struct page *page = pages[*nr]; 463db64fe02SNick Piggin 464db64fe02SNick Piggin if (WARN_ON(!pte_none(*pte))) 465db64fe02SNick Piggin return -EBUSY; 466db64fe02SNick Piggin if (WARN_ON(!page)) 4671da177e4SLinus Torvalds return -ENOMEM; 4681da177e4SLinus Torvalds set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); 469db64fe02SNick Piggin (*nr)++; 4701da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 4712ba3e694SJoerg Roedel *mask |= PGTBL_PTE_MODIFIED; 4721da177e4SLinus Torvalds return 0; 4731da177e4SLinus Torvalds } 4741da177e4SLinus Torvalds 4750a264884SNicholas Piggin static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr, 4762ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 4772ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 4781da177e4SLinus Torvalds { 4791da177e4SLinus Torvalds pmd_t *pmd; 4801da177e4SLinus Torvalds unsigned long next; 4811da177e4SLinus Torvalds 4822ba3e694SJoerg Roedel pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 4831da177e4SLinus Torvalds if (!pmd) 4841da177e4SLinus Torvalds return -ENOMEM; 4851da177e4SLinus Torvalds do { 4861da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 4870a264884SNicholas Piggin if (vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask)) 4881da177e4SLinus Torvalds return -ENOMEM; 4891da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 4901da177e4SLinus Torvalds return 0; 4911da177e4SLinus Torvalds } 4921da177e4SLinus Torvalds 4930a264884SNicholas Piggin static int vmap_pages_pud_range(p4d_t *p4d, unsigned long addr, 4942ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 4952ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 4961da177e4SLinus Torvalds { 4971da177e4SLinus Torvalds pud_t *pud; 4981da177e4SLinus Torvalds unsigned long next; 4991da177e4SLinus Torvalds 5002ba3e694SJoerg Roedel pud = pud_alloc_track(&init_mm, p4d, addr, mask); 5011da177e4SLinus Torvalds if (!pud) 5021da177e4SLinus Torvalds return -ENOMEM; 5031da177e4SLinus Torvalds do { 5041da177e4SLinus Torvalds next = pud_addr_end(addr, end); 5050a264884SNicholas Piggin if (vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask)) 5061da177e4SLinus Torvalds return -ENOMEM; 5071da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 5081da177e4SLinus Torvalds return 0; 5091da177e4SLinus Torvalds } 5101da177e4SLinus Torvalds 5110a264884SNicholas Piggin static int vmap_pages_p4d_range(pgd_t *pgd, unsigned long addr, 5122ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 5132ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 514c2febafcSKirill A. Shutemov { 515c2febafcSKirill A. Shutemov p4d_t *p4d; 516c2febafcSKirill A. Shutemov unsigned long next; 517c2febafcSKirill A. Shutemov 5182ba3e694SJoerg Roedel p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 519c2febafcSKirill A. Shutemov if (!p4d) 520c2febafcSKirill A. Shutemov return -ENOMEM; 521c2febafcSKirill A. Shutemov do { 522c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 5230a264884SNicholas Piggin if (vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask)) 524c2febafcSKirill A. Shutemov return -ENOMEM; 525c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 526c2febafcSKirill A. Shutemov return 0; 527c2febafcSKirill A. Shutemov } 528c2febafcSKirill A. Shutemov 529121e6f32SNicholas Piggin static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, 530121e6f32SNicholas Piggin pgprot_t prot, struct page **pages) 531121e6f32SNicholas Piggin { 532121e6f32SNicholas Piggin unsigned long start = addr; 533121e6f32SNicholas Piggin pgd_t *pgd; 534121e6f32SNicholas Piggin unsigned long next; 535121e6f32SNicholas Piggin int err = 0; 536121e6f32SNicholas Piggin int nr = 0; 537121e6f32SNicholas Piggin pgtbl_mod_mask mask = 0; 538121e6f32SNicholas Piggin 539121e6f32SNicholas Piggin BUG_ON(addr >= end); 540121e6f32SNicholas Piggin pgd = pgd_offset_k(addr); 541121e6f32SNicholas Piggin do { 542121e6f32SNicholas Piggin next = pgd_addr_end(addr, end); 543121e6f32SNicholas Piggin if (pgd_bad(*pgd)) 544121e6f32SNicholas Piggin mask |= PGTBL_PGD_MODIFIED; 545121e6f32SNicholas Piggin err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask); 546121e6f32SNicholas Piggin if (err) 547121e6f32SNicholas Piggin return err; 548121e6f32SNicholas Piggin } while (pgd++, addr = next, addr != end); 549121e6f32SNicholas Piggin 550121e6f32SNicholas Piggin if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 551121e6f32SNicholas Piggin arch_sync_kernel_mappings(start, end); 552121e6f32SNicholas Piggin 553121e6f32SNicholas Piggin return 0; 554121e6f32SNicholas Piggin } 555121e6f32SNicholas Piggin 556b67177ecSNicholas Piggin /* 557b67177ecSNicholas Piggin * vmap_pages_range_noflush is similar to vmap_pages_range, but does not 558b67177ecSNicholas Piggin * flush caches. 559b67177ecSNicholas Piggin * 560b67177ecSNicholas Piggin * The caller is responsible for calling flush_cache_vmap() after this 561b67177ecSNicholas Piggin * function returns successfully and before the addresses are accessed. 562b67177ecSNicholas Piggin * 563b67177ecSNicholas Piggin * This is an internal function only. Do not use outside mm/. 564b67177ecSNicholas Piggin */ 565b67177ecSNicholas Piggin int vmap_pages_range_noflush(unsigned long addr, unsigned long end, 566121e6f32SNicholas Piggin pgprot_t prot, struct page **pages, unsigned int page_shift) 567121e6f32SNicholas Piggin { 568121e6f32SNicholas Piggin unsigned int i, nr = (end - addr) >> PAGE_SHIFT; 569121e6f32SNicholas Piggin 570121e6f32SNicholas Piggin WARN_ON(page_shift < PAGE_SHIFT); 571121e6f32SNicholas Piggin 572121e6f32SNicholas Piggin if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC) || 573121e6f32SNicholas Piggin page_shift == PAGE_SHIFT) 574121e6f32SNicholas Piggin return vmap_small_pages_range_noflush(addr, end, prot, pages); 575121e6f32SNicholas Piggin 576121e6f32SNicholas Piggin for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) { 577121e6f32SNicholas Piggin int err; 578121e6f32SNicholas Piggin 579121e6f32SNicholas Piggin err = vmap_range_noflush(addr, addr + (1UL << page_shift), 580121e6f32SNicholas Piggin __pa(page_address(pages[i])), prot, 581121e6f32SNicholas Piggin page_shift); 582121e6f32SNicholas Piggin if (err) 583121e6f32SNicholas Piggin return err; 584121e6f32SNicholas Piggin 585121e6f32SNicholas Piggin addr += 1UL << page_shift; 586121e6f32SNicholas Piggin } 587121e6f32SNicholas Piggin 588121e6f32SNicholas Piggin return 0; 589121e6f32SNicholas Piggin } 590121e6f32SNicholas Piggin 591b67177ecSNicholas Piggin /** 592b67177ecSNicholas Piggin * vmap_pages_range - map pages to a kernel virtual address 593b67177ecSNicholas Piggin * @addr: start of the VM area to map 594b67177ecSNicholas Piggin * @end: end of the VM area to map (non-inclusive) 595b67177ecSNicholas Piggin * @prot: page protection flags to use 596b67177ecSNicholas Piggin * @pages: pages to map (always PAGE_SIZE pages) 597b67177ecSNicholas Piggin * @page_shift: maximum shift that the pages may be mapped with, @pages must 598b67177ecSNicholas Piggin * be aligned and contiguous up to at least this shift. 599b67177ecSNicholas Piggin * 600b67177ecSNicholas Piggin * RETURNS: 601b67177ecSNicholas Piggin * 0 on success, -errno on failure. 602b67177ecSNicholas Piggin */ 603121e6f32SNicholas Piggin static int vmap_pages_range(unsigned long addr, unsigned long end, 604121e6f32SNicholas Piggin pgprot_t prot, struct page **pages, unsigned int page_shift) 605121e6f32SNicholas Piggin { 606121e6f32SNicholas Piggin int err; 607121e6f32SNicholas Piggin 608121e6f32SNicholas Piggin err = vmap_pages_range_noflush(addr, end, prot, pages, page_shift); 609121e6f32SNicholas Piggin flush_cache_vmap(addr, end); 610121e6f32SNicholas Piggin return err; 611121e6f32SNicholas Piggin } 612121e6f32SNicholas Piggin 61381ac3ad9SKAMEZAWA Hiroyuki int is_vmalloc_or_module_addr(const void *x) 61473bdf0a6SLinus Torvalds { 61573bdf0a6SLinus Torvalds /* 616ab4f2ee1SRussell King * ARM, x86-64 and sparc64 put modules in a special place, 61773bdf0a6SLinus Torvalds * and fall back on vmalloc() if that fails. Others 61873bdf0a6SLinus Torvalds * just put it in the vmalloc space. 61973bdf0a6SLinus Torvalds */ 62073bdf0a6SLinus Torvalds #if defined(CONFIG_MODULES) && defined(MODULES_VADDR) 62173bdf0a6SLinus Torvalds unsigned long addr = (unsigned long)x; 62273bdf0a6SLinus Torvalds if (addr >= MODULES_VADDR && addr < MODULES_END) 62373bdf0a6SLinus Torvalds return 1; 62473bdf0a6SLinus Torvalds #endif 62573bdf0a6SLinus Torvalds return is_vmalloc_addr(x); 62673bdf0a6SLinus Torvalds } 62773bdf0a6SLinus Torvalds 62848667e7aSChristoph Lameter /* 629c0eb315aSNicholas Piggin * Walk a vmap address to the struct page it maps. Huge vmap mappings will 630c0eb315aSNicholas Piggin * return the tail page that corresponds to the base page address, which 631c0eb315aSNicholas Piggin * matches small vmap mappings. 63248667e7aSChristoph Lameter */ 633add688fbSmalc struct page *vmalloc_to_page(const void *vmalloc_addr) 63448667e7aSChristoph Lameter { 63548667e7aSChristoph Lameter unsigned long addr = (unsigned long) vmalloc_addr; 636add688fbSmalc struct page *page = NULL; 63748667e7aSChristoph Lameter pgd_t *pgd = pgd_offset_k(addr); 638c2febafcSKirill A. Shutemov p4d_t *p4d; 639c2febafcSKirill A. Shutemov pud_t *pud; 640c2febafcSKirill A. Shutemov pmd_t *pmd; 641c2febafcSKirill A. Shutemov pte_t *ptep, pte; 64248667e7aSChristoph Lameter 6437aa413deSIngo Molnar /* 6447aa413deSIngo Molnar * XXX we might need to change this if we add VIRTUAL_BUG_ON for 6457aa413deSIngo Molnar * architectures that do not vmalloc module space 6467aa413deSIngo Molnar */ 64773bdf0a6SLinus Torvalds VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); 64859ea7463SJiri Slaby 649c2febafcSKirill A. Shutemov if (pgd_none(*pgd)) 650c2febafcSKirill A. Shutemov return NULL; 651c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pgd_leaf(*pgd))) 652c0eb315aSNicholas Piggin return NULL; /* XXX: no allowance for huge pgd */ 653c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pgd_bad(*pgd))) 654c0eb315aSNicholas Piggin return NULL; 655c0eb315aSNicholas Piggin 656c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 657c2febafcSKirill A. Shutemov if (p4d_none(*p4d)) 658c2febafcSKirill A. Shutemov return NULL; 659c0eb315aSNicholas Piggin if (p4d_leaf(*p4d)) 660c0eb315aSNicholas Piggin return p4d_page(*p4d) + ((addr & ~P4D_MASK) >> PAGE_SHIFT); 661c0eb315aSNicholas Piggin if (WARN_ON_ONCE(p4d_bad(*p4d))) 662c2febafcSKirill A. Shutemov return NULL; 663c0eb315aSNicholas Piggin 664c0eb315aSNicholas Piggin pud = pud_offset(p4d, addr); 665c0eb315aSNicholas Piggin if (pud_none(*pud)) 666c0eb315aSNicholas Piggin return NULL; 667c0eb315aSNicholas Piggin if (pud_leaf(*pud)) 668c0eb315aSNicholas Piggin return pud_page(*pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 669c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pud_bad(*pud))) 670c0eb315aSNicholas Piggin return NULL; 671c0eb315aSNicholas Piggin 672c2febafcSKirill A. Shutemov pmd = pmd_offset(pud, addr); 673c0eb315aSNicholas Piggin if (pmd_none(*pmd)) 674c0eb315aSNicholas Piggin return NULL; 675c0eb315aSNicholas Piggin if (pmd_leaf(*pmd)) 676c0eb315aSNicholas Piggin return pmd_page(*pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 677c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pmd_bad(*pmd))) 678c2febafcSKirill A. Shutemov return NULL; 679db64fe02SNick Piggin 68048667e7aSChristoph Lameter ptep = pte_offset_map(pmd, addr); 68148667e7aSChristoph Lameter pte = *ptep; 68248667e7aSChristoph Lameter if (pte_present(pte)) 683add688fbSmalc page = pte_page(pte); 68448667e7aSChristoph Lameter pte_unmap(ptep); 685c0eb315aSNicholas Piggin 686add688fbSmalc return page; 687ece86e22SJianyu Zhan } 688ece86e22SJianyu Zhan EXPORT_SYMBOL(vmalloc_to_page); 689ece86e22SJianyu Zhan 690add688fbSmalc /* 691add688fbSmalc * Map a vmalloc()-space virtual address to the physical page frame number. 692add688fbSmalc */ 693add688fbSmalc unsigned long vmalloc_to_pfn(const void *vmalloc_addr) 694add688fbSmalc { 695add688fbSmalc return page_to_pfn(vmalloc_to_page(vmalloc_addr)); 696add688fbSmalc } 697add688fbSmalc EXPORT_SYMBOL(vmalloc_to_pfn); 698add688fbSmalc 699db64fe02SNick Piggin 700db64fe02SNick Piggin /*** Global kva allocator ***/ 701db64fe02SNick Piggin 702bb850f4dSUladzislau Rezki (Sony) #define DEBUG_AUGMENT_PROPAGATE_CHECK 0 703a6cf4e0fSUladzislau Rezki (Sony) #define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0 704bb850f4dSUladzislau Rezki (Sony) 705db64fe02SNick Piggin 706db64fe02SNick Piggin static DEFINE_SPINLOCK(vmap_area_lock); 707e36176beSUladzislau Rezki (Sony) static DEFINE_SPINLOCK(free_vmap_area_lock); 708f1c4069eSJoonsoo Kim /* Export for kexec only */ 709f1c4069eSJoonsoo Kim LIST_HEAD(vmap_area_list); 71089699605SNick Piggin static struct rb_root vmap_area_root = RB_ROOT; 71168ad4a33SUladzislau Rezki (Sony) static bool vmap_initialized __read_mostly; 71289699605SNick Piggin 71396e2db45SUladzislau Rezki (Sony) static struct rb_root purge_vmap_area_root = RB_ROOT; 71496e2db45SUladzislau Rezki (Sony) static LIST_HEAD(purge_vmap_area_list); 71596e2db45SUladzislau Rezki (Sony) static DEFINE_SPINLOCK(purge_vmap_area_lock); 71696e2db45SUladzislau Rezki (Sony) 71768ad4a33SUladzislau Rezki (Sony) /* 71868ad4a33SUladzislau Rezki (Sony) * This kmem_cache is used for vmap_area objects. Instead of 71968ad4a33SUladzislau Rezki (Sony) * allocating from slab we reuse an object from this cache to 72068ad4a33SUladzislau Rezki (Sony) * make things faster. Especially in "no edge" splitting of 72168ad4a33SUladzislau Rezki (Sony) * free block. 72268ad4a33SUladzislau Rezki (Sony) */ 72368ad4a33SUladzislau Rezki (Sony) static struct kmem_cache *vmap_area_cachep; 72489699605SNick Piggin 72568ad4a33SUladzislau Rezki (Sony) /* 72668ad4a33SUladzislau Rezki (Sony) * This linked list is used in pair with free_vmap_area_root. 72768ad4a33SUladzislau Rezki (Sony) * It gives O(1) access to prev/next to perform fast coalescing. 72868ad4a33SUladzislau Rezki (Sony) */ 72968ad4a33SUladzislau Rezki (Sony) static LIST_HEAD(free_vmap_area_list); 73068ad4a33SUladzislau Rezki (Sony) 73168ad4a33SUladzislau Rezki (Sony) /* 73268ad4a33SUladzislau Rezki (Sony) * This augment red-black tree represents the free vmap space. 73368ad4a33SUladzislau Rezki (Sony) * All vmap_area objects in this tree are sorted by va->va_start 73468ad4a33SUladzislau Rezki (Sony) * address. It is used for allocation and merging when a vmap 73568ad4a33SUladzislau Rezki (Sony) * object is released. 73668ad4a33SUladzislau Rezki (Sony) * 73768ad4a33SUladzislau Rezki (Sony) * Each vmap_area node contains a maximum available free block 73868ad4a33SUladzislau Rezki (Sony) * of its sub-tree, right or left. Therefore it is possible to 73968ad4a33SUladzislau Rezki (Sony) * find a lowest match of free area. 74068ad4a33SUladzislau Rezki (Sony) */ 74168ad4a33SUladzislau Rezki (Sony) static struct rb_root free_vmap_area_root = RB_ROOT; 74268ad4a33SUladzislau Rezki (Sony) 74382dd23e8SUladzislau Rezki (Sony) /* 74482dd23e8SUladzislau Rezki (Sony) * Preload a CPU with one object for "no edge" split case. The 74582dd23e8SUladzislau Rezki (Sony) * aim is to get rid of allocations from the atomic context, thus 74682dd23e8SUladzislau Rezki (Sony) * to use more permissive allocation masks. 74782dd23e8SUladzislau Rezki (Sony) */ 74882dd23e8SUladzislau Rezki (Sony) static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node); 74982dd23e8SUladzislau Rezki (Sony) 75068ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 75168ad4a33SUladzislau Rezki (Sony) va_size(struct vmap_area *va) 75268ad4a33SUladzislau Rezki (Sony) { 75368ad4a33SUladzislau Rezki (Sony) return (va->va_end - va->va_start); 75468ad4a33SUladzislau Rezki (Sony) } 75568ad4a33SUladzislau Rezki (Sony) 75668ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 75768ad4a33SUladzislau Rezki (Sony) get_subtree_max_size(struct rb_node *node) 75868ad4a33SUladzislau Rezki (Sony) { 75968ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 76068ad4a33SUladzislau Rezki (Sony) 76168ad4a33SUladzislau Rezki (Sony) va = rb_entry_safe(node, struct vmap_area, rb_node); 76268ad4a33SUladzislau Rezki (Sony) return va ? va->subtree_max_size : 0; 76368ad4a33SUladzislau Rezki (Sony) } 76468ad4a33SUladzislau Rezki (Sony) 76568ad4a33SUladzislau Rezki (Sony) /* 76668ad4a33SUladzislau Rezki (Sony) * Gets called when remove the node and rotate. 76768ad4a33SUladzislau Rezki (Sony) */ 76868ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 76968ad4a33SUladzislau Rezki (Sony) compute_subtree_max_size(struct vmap_area *va) 77068ad4a33SUladzislau Rezki (Sony) { 77168ad4a33SUladzislau Rezki (Sony) return max3(va_size(va), 77268ad4a33SUladzislau Rezki (Sony) get_subtree_max_size(va->rb_node.rb_left), 77368ad4a33SUladzislau Rezki (Sony) get_subtree_max_size(va->rb_node.rb_right)); 77468ad4a33SUladzislau Rezki (Sony) } 77568ad4a33SUladzislau Rezki (Sony) 776315cc066SMichel Lespinasse RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb, 777315cc066SMichel Lespinasse struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size) 77868ad4a33SUladzislau Rezki (Sony) 77968ad4a33SUladzislau Rezki (Sony) static void purge_vmap_area_lazy(void); 78068ad4a33SUladzislau Rezki (Sony) static BLOCKING_NOTIFIER_HEAD(vmap_notify_list); 78168ad4a33SUladzislau Rezki (Sony) static unsigned long lazy_max_pages(void); 782db64fe02SNick Piggin 78397105f0aSRoman Gushchin static atomic_long_t nr_vmalloc_pages; 78497105f0aSRoman Gushchin 78597105f0aSRoman Gushchin unsigned long vmalloc_nr_pages(void) 78697105f0aSRoman Gushchin { 78797105f0aSRoman Gushchin return atomic_long_read(&nr_vmalloc_pages); 78897105f0aSRoman Gushchin } 78997105f0aSRoman Gushchin 790db64fe02SNick Piggin static struct vmap_area *__find_vmap_area(unsigned long addr) 7911da177e4SLinus Torvalds { 792db64fe02SNick Piggin struct rb_node *n = vmap_area_root.rb_node; 793db64fe02SNick Piggin 794db64fe02SNick Piggin while (n) { 795db64fe02SNick Piggin struct vmap_area *va; 796db64fe02SNick Piggin 797db64fe02SNick Piggin va = rb_entry(n, struct vmap_area, rb_node); 798db64fe02SNick Piggin if (addr < va->va_start) 799db64fe02SNick Piggin n = n->rb_left; 800cef2ac3fSHATAYAMA Daisuke else if (addr >= va->va_end) 801db64fe02SNick Piggin n = n->rb_right; 802db64fe02SNick Piggin else 803db64fe02SNick Piggin return va; 804db64fe02SNick Piggin } 805db64fe02SNick Piggin 806db64fe02SNick Piggin return NULL; 807db64fe02SNick Piggin } 808db64fe02SNick Piggin 80968ad4a33SUladzislau Rezki (Sony) /* 81068ad4a33SUladzislau Rezki (Sony) * This function returns back addresses of parent node 81168ad4a33SUladzislau Rezki (Sony) * and its left or right link for further processing. 8129c801f61SUladzislau Rezki (Sony) * 8139c801f61SUladzislau Rezki (Sony) * Otherwise NULL is returned. In that case all further 8149c801f61SUladzislau Rezki (Sony) * steps regarding inserting of conflicting overlap range 8159c801f61SUladzislau Rezki (Sony) * have to be declined and actually considered as a bug. 81668ad4a33SUladzislau Rezki (Sony) */ 81768ad4a33SUladzislau Rezki (Sony) static __always_inline struct rb_node ** 81868ad4a33SUladzislau Rezki (Sony) find_va_links(struct vmap_area *va, 81968ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct rb_node *from, 82068ad4a33SUladzislau Rezki (Sony) struct rb_node **parent) 821db64fe02SNick Piggin { 822170168d0SNamhyung Kim struct vmap_area *tmp_va; 82368ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 824db64fe02SNick Piggin 82568ad4a33SUladzislau Rezki (Sony) if (root) { 82668ad4a33SUladzislau Rezki (Sony) link = &root->rb_node; 82768ad4a33SUladzislau Rezki (Sony) if (unlikely(!*link)) { 82868ad4a33SUladzislau Rezki (Sony) *parent = NULL; 82968ad4a33SUladzislau Rezki (Sony) return link; 83068ad4a33SUladzislau Rezki (Sony) } 83168ad4a33SUladzislau Rezki (Sony) } else { 83268ad4a33SUladzislau Rezki (Sony) link = &from; 83368ad4a33SUladzislau Rezki (Sony) } 83468ad4a33SUladzislau Rezki (Sony) 83568ad4a33SUladzislau Rezki (Sony) /* 83668ad4a33SUladzislau Rezki (Sony) * Go to the bottom of the tree. When we hit the last point 83768ad4a33SUladzislau Rezki (Sony) * we end up with parent rb_node and correct direction, i name 83868ad4a33SUladzislau Rezki (Sony) * it link, where the new va->rb_node will be attached to. 83968ad4a33SUladzislau Rezki (Sony) */ 84068ad4a33SUladzislau Rezki (Sony) do { 84168ad4a33SUladzislau Rezki (Sony) tmp_va = rb_entry(*link, struct vmap_area, rb_node); 84268ad4a33SUladzislau Rezki (Sony) 84368ad4a33SUladzislau Rezki (Sony) /* 84468ad4a33SUladzislau Rezki (Sony) * During the traversal we also do some sanity check. 84568ad4a33SUladzislau Rezki (Sony) * Trigger the BUG() if there are sides(left/right) 84668ad4a33SUladzislau Rezki (Sony) * or full overlaps. 84768ad4a33SUladzislau Rezki (Sony) */ 84868ad4a33SUladzislau Rezki (Sony) if (va->va_start < tmp_va->va_end && 84968ad4a33SUladzislau Rezki (Sony) va->va_end <= tmp_va->va_start) 85068ad4a33SUladzislau Rezki (Sony) link = &(*link)->rb_left; 85168ad4a33SUladzislau Rezki (Sony) else if (va->va_end > tmp_va->va_start && 85268ad4a33SUladzislau Rezki (Sony) va->va_start >= tmp_va->va_end) 85368ad4a33SUladzislau Rezki (Sony) link = &(*link)->rb_right; 8549c801f61SUladzislau Rezki (Sony) else { 8559c801f61SUladzislau Rezki (Sony) WARN(1, "vmalloc bug: 0x%lx-0x%lx overlaps with 0x%lx-0x%lx\n", 8569c801f61SUladzislau Rezki (Sony) va->va_start, va->va_end, tmp_va->va_start, tmp_va->va_end); 8579c801f61SUladzislau Rezki (Sony) 8589c801f61SUladzislau Rezki (Sony) return NULL; 8599c801f61SUladzislau Rezki (Sony) } 86068ad4a33SUladzislau Rezki (Sony) } while (*link); 86168ad4a33SUladzislau Rezki (Sony) 86268ad4a33SUladzislau Rezki (Sony) *parent = &tmp_va->rb_node; 86368ad4a33SUladzislau Rezki (Sony) return link; 864db64fe02SNick Piggin } 865db64fe02SNick Piggin 86668ad4a33SUladzislau Rezki (Sony) static __always_inline struct list_head * 86768ad4a33SUladzislau Rezki (Sony) get_va_next_sibling(struct rb_node *parent, struct rb_node **link) 86868ad4a33SUladzislau Rezki (Sony) { 86968ad4a33SUladzislau Rezki (Sony) struct list_head *list; 870db64fe02SNick Piggin 87168ad4a33SUladzislau Rezki (Sony) if (unlikely(!parent)) 87268ad4a33SUladzislau Rezki (Sony) /* 87368ad4a33SUladzislau Rezki (Sony) * The red-black tree where we try to find VA neighbors 87468ad4a33SUladzislau Rezki (Sony) * before merging or inserting is empty, i.e. it means 87568ad4a33SUladzislau Rezki (Sony) * there is no free vmap space. Normally it does not 87668ad4a33SUladzislau Rezki (Sony) * happen but we handle this case anyway. 87768ad4a33SUladzislau Rezki (Sony) */ 87868ad4a33SUladzislau Rezki (Sony) return NULL; 87968ad4a33SUladzislau Rezki (Sony) 88068ad4a33SUladzislau Rezki (Sony) list = &rb_entry(parent, struct vmap_area, rb_node)->list; 88168ad4a33SUladzislau Rezki (Sony) return (&parent->rb_right == link ? list->next : list); 882db64fe02SNick Piggin } 883db64fe02SNick Piggin 88468ad4a33SUladzislau Rezki (Sony) static __always_inline void 88568ad4a33SUladzislau Rezki (Sony) link_va(struct vmap_area *va, struct rb_root *root, 88668ad4a33SUladzislau Rezki (Sony) struct rb_node *parent, struct rb_node **link, struct list_head *head) 88768ad4a33SUladzislau Rezki (Sony) { 88868ad4a33SUladzislau Rezki (Sony) /* 88968ad4a33SUladzislau Rezki (Sony) * VA is still not in the list, but we can 89068ad4a33SUladzislau Rezki (Sony) * identify its future previous list_head node. 89168ad4a33SUladzislau Rezki (Sony) */ 89268ad4a33SUladzislau Rezki (Sony) if (likely(parent)) { 89368ad4a33SUladzislau Rezki (Sony) head = &rb_entry(parent, struct vmap_area, rb_node)->list; 89468ad4a33SUladzislau Rezki (Sony) if (&parent->rb_right != link) 89568ad4a33SUladzislau Rezki (Sony) head = head->prev; 89668ad4a33SUladzislau Rezki (Sony) } 897db64fe02SNick Piggin 89868ad4a33SUladzislau Rezki (Sony) /* Insert to the rb-tree */ 89968ad4a33SUladzislau Rezki (Sony) rb_link_node(&va->rb_node, parent, link); 90068ad4a33SUladzislau Rezki (Sony) if (root == &free_vmap_area_root) { 90168ad4a33SUladzislau Rezki (Sony) /* 90268ad4a33SUladzislau Rezki (Sony) * Some explanation here. Just perform simple insertion 90368ad4a33SUladzislau Rezki (Sony) * to the tree. We do not set va->subtree_max_size to 90468ad4a33SUladzislau Rezki (Sony) * its current size before calling rb_insert_augmented(). 90568ad4a33SUladzislau Rezki (Sony) * It is because of we populate the tree from the bottom 90668ad4a33SUladzislau Rezki (Sony) * to parent levels when the node _is_ in the tree. 90768ad4a33SUladzislau Rezki (Sony) * 90868ad4a33SUladzislau Rezki (Sony) * Therefore we set subtree_max_size to zero after insertion, 90968ad4a33SUladzislau Rezki (Sony) * to let __augment_tree_propagate_from() puts everything to 91068ad4a33SUladzislau Rezki (Sony) * the correct order later on. 91168ad4a33SUladzislau Rezki (Sony) */ 91268ad4a33SUladzislau Rezki (Sony) rb_insert_augmented(&va->rb_node, 91368ad4a33SUladzislau Rezki (Sony) root, &free_vmap_area_rb_augment_cb); 91468ad4a33SUladzislau Rezki (Sony) va->subtree_max_size = 0; 91568ad4a33SUladzislau Rezki (Sony) } else { 91668ad4a33SUladzislau Rezki (Sony) rb_insert_color(&va->rb_node, root); 91768ad4a33SUladzislau Rezki (Sony) } 91868ad4a33SUladzislau Rezki (Sony) 91968ad4a33SUladzislau Rezki (Sony) /* Address-sort this list */ 92068ad4a33SUladzislau Rezki (Sony) list_add(&va->list, head); 92168ad4a33SUladzislau Rezki (Sony) } 92268ad4a33SUladzislau Rezki (Sony) 92368ad4a33SUladzislau Rezki (Sony) static __always_inline void 92468ad4a33SUladzislau Rezki (Sony) unlink_va(struct vmap_area *va, struct rb_root *root) 92568ad4a33SUladzislau Rezki (Sony) { 926460e42d1SUladzislau Rezki (Sony) if (WARN_ON(RB_EMPTY_NODE(&va->rb_node))) 927460e42d1SUladzislau Rezki (Sony) return; 928460e42d1SUladzislau Rezki (Sony) 92968ad4a33SUladzislau Rezki (Sony) if (root == &free_vmap_area_root) 93068ad4a33SUladzislau Rezki (Sony) rb_erase_augmented(&va->rb_node, 93168ad4a33SUladzislau Rezki (Sony) root, &free_vmap_area_rb_augment_cb); 93268ad4a33SUladzislau Rezki (Sony) else 93368ad4a33SUladzislau Rezki (Sony) rb_erase(&va->rb_node, root); 93468ad4a33SUladzislau Rezki (Sony) 93568ad4a33SUladzislau Rezki (Sony) list_del(&va->list); 93668ad4a33SUladzislau Rezki (Sony) RB_CLEAR_NODE(&va->rb_node); 93768ad4a33SUladzislau Rezki (Sony) } 93868ad4a33SUladzislau Rezki (Sony) 939bb850f4dSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_PROPAGATE_CHECK 940bb850f4dSUladzislau Rezki (Sony) static void 941da27c9edSUladzislau Rezki (Sony) augment_tree_propagate_check(void) 942bb850f4dSUladzislau Rezki (Sony) { 943bb850f4dSUladzislau Rezki (Sony) struct vmap_area *va; 944da27c9edSUladzislau Rezki (Sony) unsigned long computed_size; 945bb850f4dSUladzislau Rezki (Sony) 946da27c9edSUladzislau Rezki (Sony) list_for_each_entry(va, &free_vmap_area_list, list) { 947da27c9edSUladzislau Rezki (Sony) computed_size = compute_subtree_max_size(va); 948da27c9edSUladzislau Rezki (Sony) if (computed_size != va->subtree_max_size) 949bb850f4dSUladzislau Rezki (Sony) pr_emerg("tree is corrupted: %lu, %lu\n", 950bb850f4dSUladzislau Rezki (Sony) va_size(va), va->subtree_max_size); 951bb850f4dSUladzislau Rezki (Sony) } 952bb850f4dSUladzislau Rezki (Sony) } 953bb850f4dSUladzislau Rezki (Sony) #endif 954bb850f4dSUladzislau Rezki (Sony) 95568ad4a33SUladzislau Rezki (Sony) /* 95668ad4a33SUladzislau Rezki (Sony) * This function populates subtree_max_size from bottom to upper 95768ad4a33SUladzislau Rezki (Sony) * levels starting from VA point. The propagation must be done 95868ad4a33SUladzislau Rezki (Sony) * when VA size is modified by changing its va_start/va_end. Or 95968ad4a33SUladzislau Rezki (Sony) * in case of newly inserting of VA to the tree. 96068ad4a33SUladzislau Rezki (Sony) * 96168ad4a33SUladzislau Rezki (Sony) * It means that __augment_tree_propagate_from() must be called: 96268ad4a33SUladzislau Rezki (Sony) * - After VA has been inserted to the tree(free path); 96368ad4a33SUladzislau Rezki (Sony) * - After VA has been shrunk(allocation path); 96468ad4a33SUladzislau Rezki (Sony) * - After VA has been increased(merging path). 96568ad4a33SUladzislau Rezki (Sony) * 96668ad4a33SUladzislau Rezki (Sony) * Please note that, it does not mean that upper parent nodes 96768ad4a33SUladzislau Rezki (Sony) * and their subtree_max_size are recalculated all the time up 96868ad4a33SUladzislau Rezki (Sony) * to the root node. 96968ad4a33SUladzislau Rezki (Sony) * 97068ad4a33SUladzislau Rezki (Sony) * 4--8 97168ad4a33SUladzislau Rezki (Sony) * /\ 97268ad4a33SUladzislau Rezki (Sony) * / \ 97368ad4a33SUladzislau Rezki (Sony) * / \ 97468ad4a33SUladzislau Rezki (Sony) * 2--2 8--8 97568ad4a33SUladzislau Rezki (Sony) * 97668ad4a33SUladzislau Rezki (Sony) * For example if we modify the node 4, shrinking it to 2, then 97768ad4a33SUladzislau Rezki (Sony) * no any modification is required. If we shrink the node 2 to 1 97868ad4a33SUladzislau Rezki (Sony) * its subtree_max_size is updated only, and set to 1. If we shrink 97968ad4a33SUladzislau Rezki (Sony) * the node 8 to 6, then its subtree_max_size is set to 6 and parent 98068ad4a33SUladzislau Rezki (Sony) * node becomes 4--6. 98168ad4a33SUladzislau Rezki (Sony) */ 98268ad4a33SUladzislau Rezki (Sony) static __always_inline void 98368ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(struct vmap_area *va) 98468ad4a33SUladzislau Rezki (Sony) { 98568ad4a33SUladzislau Rezki (Sony) /* 98615ae144fSUladzislau Rezki (Sony) * Populate the tree from bottom towards the root until 98715ae144fSUladzislau Rezki (Sony) * the calculated maximum available size of checked node 98815ae144fSUladzislau Rezki (Sony) * is equal to its current one. 98968ad4a33SUladzislau Rezki (Sony) */ 99015ae144fSUladzislau Rezki (Sony) free_vmap_area_rb_augment_cb_propagate(&va->rb_node, NULL); 991bb850f4dSUladzislau Rezki (Sony) 992bb850f4dSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_PROPAGATE_CHECK 993da27c9edSUladzislau Rezki (Sony) augment_tree_propagate_check(); 994bb850f4dSUladzislau Rezki (Sony) #endif 99568ad4a33SUladzislau Rezki (Sony) } 99668ad4a33SUladzislau Rezki (Sony) 99768ad4a33SUladzislau Rezki (Sony) static void 99868ad4a33SUladzislau Rezki (Sony) insert_vmap_area(struct vmap_area *va, 99968ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 100068ad4a33SUladzislau Rezki (Sony) { 100168ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 100268ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 100368ad4a33SUladzislau Rezki (Sony) 100468ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 10059c801f61SUladzislau Rezki (Sony) if (link) 100668ad4a33SUladzislau Rezki (Sony) link_va(va, root, parent, link, head); 100768ad4a33SUladzislau Rezki (Sony) } 100868ad4a33SUladzislau Rezki (Sony) 100968ad4a33SUladzislau Rezki (Sony) static void 101068ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(struct vmap_area *va, 101168ad4a33SUladzislau Rezki (Sony) struct rb_node *from, struct rb_root *root, 101268ad4a33SUladzislau Rezki (Sony) struct list_head *head) 101368ad4a33SUladzislau Rezki (Sony) { 101468ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 101568ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 101668ad4a33SUladzislau Rezki (Sony) 101768ad4a33SUladzislau Rezki (Sony) if (from) 101868ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, NULL, from, &parent); 101968ad4a33SUladzislau Rezki (Sony) else 102068ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 102168ad4a33SUladzislau Rezki (Sony) 10229c801f61SUladzislau Rezki (Sony) if (link) { 102368ad4a33SUladzislau Rezki (Sony) link_va(va, root, parent, link, head); 102468ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 102568ad4a33SUladzislau Rezki (Sony) } 10269c801f61SUladzislau Rezki (Sony) } 102768ad4a33SUladzislau Rezki (Sony) 102868ad4a33SUladzislau Rezki (Sony) /* 102968ad4a33SUladzislau Rezki (Sony) * Merge de-allocated chunk of VA memory with previous 103068ad4a33SUladzislau Rezki (Sony) * and next free blocks. If coalesce is not done a new 103168ad4a33SUladzislau Rezki (Sony) * free area is inserted. If VA has been merged, it is 103268ad4a33SUladzislau Rezki (Sony) * freed. 10339c801f61SUladzislau Rezki (Sony) * 10349c801f61SUladzislau Rezki (Sony) * Please note, it can return NULL in case of overlap 10359c801f61SUladzislau Rezki (Sony) * ranges, followed by WARN() report. Despite it is a 10369c801f61SUladzislau Rezki (Sony) * buggy behaviour, a system can be alive and keep 10379c801f61SUladzislau Rezki (Sony) * ongoing. 103868ad4a33SUladzislau Rezki (Sony) */ 10393c5c3cfbSDaniel Axtens static __always_inline struct vmap_area * 104068ad4a33SUladzislau Rezki (Sony) merge_or_add_vmap_area(struct vmap_area *va, 104168ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 104268ad4a33SUladzislau Rezki (Sony) { 104368ad4a33SUladzislau Rezki (Sony) struct vmap_area *sibling; 104468ad4a33SUladzislau Rezki (Sony) struct list_head *next; 104568ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 104668ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 104768ad4a33SUladzislau Rezki (Sony) bool merged = false; 104868ad4a33SUladzislau Rezki (Sony) 104968ad4a33SUladzislau Rezki (Sony) /* 105068ad4a33SUladzislau Rezki (Sony) * Find a place in the tree where VA potentially will be 105168ad4a33SUladzislau Rezki (Sony) * inserted, unless it is merged with its sibling/siblings. 105268ad4a33SUladzislau Rezki (Sony) */ 105368ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 10549c801f61SUladzislau Rezki (Sony) if (!link) 10559c801f61SUladzislau Rezki (Sony) return NULL; 105668ad4a33SUladzislau Rezki (Sony) 105768ad4a33SUladzislau Rezki (Sony) /* 105868ad4a33SUladzislau Rezki (Sony) * Get next node of VA to check if merging can be done. 105968ad4a33SUladzislau Rezki (Sony) */ 106068ad4a33SUladzislau Rezki (Sony) next = get_va_next_sibling(parent, link); 106168ad4a33SUladzislau Rezki (Sony) if (unlikely(next == NULL)) 106268ad4a33SUladzislau Rezki (Sony) goto insert; 106368ad4a33SUladzislau Rezki (Sony) 106468ad4a33SUladzislau Rezki (Sony) /* 106568ad4a33SUladzislau Rezki (Sony) * start end 106668ad4a33SUladzislau Rezki (Sony) * | | 106768ad4a33SUladzislau Rezki (Sony) * |<------VA------>|<-----Next----->| 106868ad4a33SUladzislau Rezki (Sony) * | | 106968ad4a33SUladzislau Rezki (Sony) * start end 107068ad4a33SUladzislau Rezki (Sony) */ 107168ad4a33SUladzislau Rezki (Sony) if (next != head) { 107268ad4a33SUladzislau Rezki (Sony) sibling = list_entry(next, struct vmap_area, list); 107368ad4a33SUladzislau Rezki (Sony) if (sibling->va_start == va->va_end) { 107468ad4a33SUladzislau Rezki (Sony) sibling->va_start = va->va_start; 107568ad4a33SUladzislau Rezki (Sony) 107668ad4a33SUladzislau Rezki (Sony) /* Free vmap_area object. */ 107768ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 107868ad4a33SUladzislau Rezki (Sony) 107968ad4a33SUladzislau Rezki (Sony) /* Point to the new merged area. */ 108068ad4a33SUladzislau Rezki (Sony) va = sibling; 108168ad4a33SUladzislau Rezki (Sony) merged = true; 108268ad4a33SUladzislau Rezki (Sony) } 108368ad4a33SUladzislau Rezki (Sony) } 108468ad4a33SUladzislau Rezki (Sony) 108568ad4a33SUladzislau Rezki (Sony) /* 108668ad4a33SUladzislau Rezki (Sony) * start end 108768ad4a33SUladzislau Rezki (Sony) * | | 108868ad4a33SUladzislau Rezki (Sony) * |<-----Prev----->|<------VA------>| 108968ad4a33SUladzislau Rezki (Sony) * | | 109068ad4a33SUladzislau Rezki (Sony) * start end 109168ad4a33SUladzislau Rezki (Sony) */ 109268ad4a33SUladzislau Rezki (Sony) if (next->prev != head) { 109368ad4a33SUladzislau Rezki (Sony) sibling = list_entry(next->prev, struct vmap_area, list); 109468ad4a33SUladzislau Rezki (Sony) if (sibling->va_end == va->va_start) { 10955dd78640SUladzislau Rezki (Sony) /* 10965dd78640SUladzislau Rezki (Sony) * If both neighbors are coalesced, it is important 10975dd78640SUladzislau Rezki (Sony) * to unlink the "next" node first, followed by merging 10985dd78640SUladzislau Rezki (Sony) * with "previous" one. Otherwise the tree might not be 10995dd78640SUladzislau Rezki (Sony) * fully populated if a sibling's augmented value is 11005dd78640SUladzislau Rezki (Sony) * "normalized" because of rotation operations. 11015dd78640SUladzislau Rezki (Sony) */ 110254f63d9dSUladzislau Rezki (Sony) if (merged) 110368ad4a33SUladzislau Rezki (Sony) unlink_va(va, root); 110468ad4a33SUladzislau Rezki (Sony) 11055dd78640SUladzislau Rezki (Sony) sibling->va_end = va->va_end; 11065dd78640SUladzislau Rezki (Sony) 110768ad4a33SUladzislau Rezki (Sony) /* Free vmap_area object. */ 110868ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 11093c5c3cfbSDaniel Axtens 11103c5c3cfbSDaniel Axtens /* Point to the new merged area. */ 11113c5c3cfbSDaniel Axtens va = sibling; 11123c5c3cfbSDaniel Axtens merged = true; 111368ad4a33SUladzislau Rezki (Sony) } 111468ad4a33SUladzislau Rezki (Sony) } 111568ad4a33SUladzislau Rezki (Sony) 111668ad4a33SUladzislau Rezki (Sony) insert: 11175dd78640SUladzislau Rezki (Sony) if (!merged) 111868ad4a33SUladzislau Rezki (Sony) link_va(va, root, parent, link, head); 11193c5c3cfbSDaniel Axtens 112096e2db45SUladzislau Rezki (Sony) return va; 112196e2db45SUladzislau Rezki (Sony) } 112296e2db45SUladzislau Rezki (Sony) 112396e2db45SUladzislau Rezki (Sony) static __always_inline struct vmap_area * 112496e2db45SUladzislau Rezki (Sony) merge_or_add_vmap_area_augment(struct vmap_area *va, 112596e2db45SUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 112696e2db45SUladzislau Rezki (Sony) { 112796e2db45SUladzislau Rezki (Sony) va = merge_or_add_vmap_area(va, root, head); 112896e2db45SUladzislau Rezki (Sony) if (va) 11295dd78640SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 113096e2db45SUladzislau Rezki (Sony) 11313c5c3cfbSDaniel Axtens return va; 113268ad4a33SUladzislau Rezki (Sony) } 113368ad4a33SUladzislau Rezki (Sony) 113468ad4a33SUladzislau Rezki (Sony) static __always_inline bool 113568ad4a33SUladzislau Rezki (Sony) is_within_this_va(struct vmap_area *va, unsigned long size, 113668ad4a33SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 113768ad4a33SUladzislau Rezki (Sony) { 113868ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr; 113968ad4a33SUladzislau Rezki (Sony) 114068ad4a33SUladzislau Rezki (Sony) if (va->va_start > vstart) 114168ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(va->va_start, align); 114268ad4a33SUladzislau Rezki (Sony) else 114368ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(vstart, align); 114468ad4a33SUladzislau Rezki (Sony) 114568ad4a33SUladzislau Rezki (Sony) /* Can be overflowed due to big size or alignment. */ 114668ad4a33SUladzislau Rezki (Sony) if (nva_start_addr + size < nva_start_addr || 114768ad4a33SUladzislau Rezki (Sony) nva_start_addr < vstart) 114868ad4a33SUladzislau Rezki (Sony) return false; 114968ad4a33SUladzislau Rezki (Sony) 115068ad4a33SUladzislau Rezki (Sony) return (nva_start_addr + size <= va->va_end); 115168ad4a33SUladzislau Rezki (Sony) } 115268ad4a33SUladzislau Rezki (Sony) 115368ad4a33SUladzislau Rezki (Sony) /* 115468ad4a33SUladzislau Rezki (Sony) * Find the first free block(lowest start address) in the tree, 115568ad4a33SUladzislau Rezki (Sony) * that will accomplish the request corresponding to passing 115668ad4a33SUladzislau Rezki (Sony) * parameters. 115768ad4a33SUladzislau Rezki (Sony) */ 115868ad4a33SUladzislau Rezki (Sony) static __always_inline struct vmap_area * 115968ad4a33SUladzislau Rezki (Sony) find_vmap_lowest_match(unsigned long size, 116068ad4a33SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 116168ad4a33SUladzislau Rezki (Sony) { 116268ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 116368ad4a33SUladzislau Rezki (Sony) struct rb_node *node; 116468ad4a33SUladzislau Rezki (Sony) unsigned long length; 116568ad4a33SUladzislau Rezki (Sony) 116668ad4a33SUladzislau Rezki (Sony) /* Start from the root. */ 116768ad4a33SUladzislau Rezki (Sony) node = free_vmap_area_root.rb_node; 116868ad4a33SUladzislau Rezki (Sony) 116968ad4a33SUladzislau Rezki (Sony) /* Adjust the search size for alignment overhead. */ 117068ad4a33SUladzislau Rezki (Sony) length = size + align - 1; 117168ad4a33SUladzislau Rezki (Sony) 117268ad4a33SUladzislau Rezki (Sony) while (node) { 117368ad4a33SUladzislau Rezki (Sony) va = rb_entry(node, struct vmap_area, rb_node); 117468ad4a33SUladzislau Rezki (Sony) 117568ad4a33SUladzislau Rezki (Sony) if (get_subtree_max_size(node->rb_left) >= length && 117668ad4a33SUladzislau Rezki (Sony) vstart < va->va_start) { 117768ad4a33SUladzislau Rezki (Sony) node = node->rb_left; 117868ad4a33SUladzislau Rezki (Sony) } else { 117968ad4a33SUladzislau Rezki (Sony) if (is_within_this_va(va, size, align, vstart)) 118068ad4a33SUladzislau Rezki (Sony) return va; 118168ad4a33SUladzislau Rezki (Sony) 118268ad4a33SUladzislau Rezki (Sony) /* 118368ad4a33SUladzislau Rezki (Sony) * Does not make sense to go deeper towards the right 118468ad4a33SUladzislau Rezki (Sony) * sub-tree if it does not have a free block that is 118568ad4a33SUladzislau Rezki (Sony) * equal or bigger to the requested search length. 118668ad4a33SUladzislau Rezki (Sony) */ 118768ad4a33SUladzislau Rezki (Sony) if (get_subtree_max_size(node->rb_right) >= length) { 118868ad4a33SUladzislau Rezki (Sony) node = node->rb_right; 118968ad4a33SUladzislau Rezki (Sony) continue; 119068ad4a33SUladzislau Rezki (Sony) } 119168ad4a33SUladzislau Rezki (Sony) 119268ad4a33SUladzislau Rezki (Sony) /* 11933806b041SAndrew Morton * OK. We roll back and find the first right sub-tree, 119468ad4a33SUladzislau Rezki (Sony) * that will satisfy the search criteria. It can happen 119568ad4a33SUladzislau Rezki (Sony) * only once due to "vstart" restriction. 119668ad4a33SUladzislau Rezki (Sony) */ 119768ad4a33SUladzislau Rezki (Sony) while ((node = rb_parent(node))) { 119868ad4a33SUladzislau Rezki (Sony) va = rb_entry(node, struct vmap_area, rb_node); 119968ad4a33SUladzislau Rezki (Sony) if (is_within_this_va(va, size, align, vstart)) 120068ad4a33SUladzislau Rezki (Sony) return va; 120168ad4a33SUladzislau Rezki (Sony) 120268ad4a33SUladzislau Rezki (Sony) if (get_subtree_max_size(node->rb_right) >= length && 120368ad4a33SUladzislau Rezki (Sony) vstart <= va->va_start) { 120468ad4a33SUladzislau Rezki (Sony) node = node->rb_right; 120568ad4a33SUladzislau Rezki (Sony) break; 120668ad4a33SUladzislau Rezki (Sony) } 120768ad4a33SUladzislau Rezki (Sony) } 120868ad4a33SUladzislau Rezki (Sony) } 120968ad4a33SUladzislau Rezki (Sony) } 121068ad4a33SUladzislau Rezki (Sony) 121168ad4a33SUladzislau Rezki (Sony) return NULL; 121268ad4a33SUladzislau Rezki (Sony) } 121368ad4a33SUladzislau Rezki (Sony) 1214a6cf4e0fSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 1215a6cf4e0fSUladzislau Rezki (Sony) #include <linux/random.h> 1216a6cf4e0fSUladzislau Rezki (Sony) 1217a6cf4e0fSUladzislau Rezki (Sony) static struct vmap_area * 1218a6cf4e0fSUladzislau Rezki (Sony) find_vmap_lowest_linear_match(unsigned long size, 1219a6cf4e0fSUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 1220a6cf4e0fSUladzislau Rezki (Sony) { 1221a6cf4e0fSUladzislau Rezki (Sony) struct vmap_area *va; 1222a6cf4e0fSUladzislau Rezki (Sony) 1223a6cf4e0fSUladzislau Rezki (Sony) list_for_each_entry(va, &free_vmap_area_list, list) { 1224a6cf4e0fSUladzislau Rezki (Sony) if (!is_within_this_va(va, size, align, vstart)) 1225a6cf4e0fSUladzislau Rezki (Sony) continue; 1226a6cf4e0fSUladzislau Rezki (Sony) 1227a6cf4e0fSUladzislau Rezki (Sony) return va; 1228a6cf4e0fSUladzislau Rezki (Sony) } 1229a6cf4e0fSUladzislau Rezki (Sony) 1230a6cf4e0fSUladzislau Rezki (Sony) return NULL; 1231a6cf4e0fSUladzislau Rezki (Sony) } 1232a6cf4e0fSUladzislau Rezki (Sony) 1233a6cf4e0fSUladzislau Rezki (Sony) static void 1234a6cf4e0fSUladzislau Rezki (Sony) find_vmap_lowest_match_check(unsigned long size) 1235a6cf4e0fSUladzislau Rezki (Sony) { 1236a6cf4e0fSUladzislau Rezki (Sony) struct vmap_area *va_1, *va_2; 1237a6cf4e0fSUladzislau Rezki (Sony) unsigned long vstart; 1238a6cf4e0fSUladzislau Rezki (Sony) unsigned int rnd; 1239a6cf4e0fSUladzislau Rezki (Sony) 1240a6cf4e0fSUladzislau Rezki (Sony) get_random_bytes(&rnd, sizeof(rnd)); 1241a6cf4e0fSUladzislau Rezki (Sony) vstart = VMALLOC_START + rnd; 1242a6cf4e0fSUladzislau Rezki (Sony) 1243a6cf4e0fSUladzislau Rezki (Sony) va_1 = find_vmap_lowest_match(size, 1, vstart); 1244a6cf4e0fSUladzislau Rezki (Sony) va_2 = find_vmap_lowest_linear_match(size, 1, vstart); 1245a6cf4e0fSUladzislau Rezki (Sony) 1246a6cf4e0fSUladzislau Rezki (Sony) if (va_1 != va_2) 1247a6cf4e0fSUladzislau Rezki (Sony) pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n", 1248a6cf4e0fSUladzislau Rezki (Sony) va_1, va_2, vstart); 1249a6cf4e0fSUladzislau Rezki (Sony) } 1250a6cf4e0fSUladzislau Rezki (Sony) #endif 1251a6cf4e0fSUladzislau Rezki (Sony) 125268ad4a33SUladzislau Rezki (Sony) enum fit_type { 125368ad4a33SUladzislau Rezki (Sony) NOTHING_FIT = 0, 125468ad4a33SUladzislau Rezki (Sony) FL_FIT_TYPE = 1, /* full fit */ 125568ad4a33SUladzislau Rezki (Sony) LE_FIT_TYPE = 2, /* left edge fit */ 125668ad4a33SUladzislau Rezki (Sony) RE_FIT_TYPE = 3, /* right edge fit */ 125768ad4a33SUladzislau Rezki (Sony) NE_FIT_TYPE = 4 /* no edge fit */ 125868ad4a33SUladzislau Rezki (Sony) }; 125968ad4a33SUladzislau Rezki (Sony) 126068ad4a33SUladzislau Rezki (Sony) static __always_inline enum fit_type 126168ad4a33SUladzislau Rezki (Sony) classify_va_fit_type(struct vmap_area *va, 126268ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr, unsigned long size) 126368ad4a33SUladzislau Rezki (Sony) { 126468ad4a33SUladzislau Rezki (Sony) enum fit_type type; 126568ad4a33SUladzislau Rezki (Sony) 126668ad4a33SUladzislau Rezki (Sony) /* Check if it is within VA. */ 126768ad4a33SUladzislau Rezki (Sony) if (nva_start_addr < va->va_start || 126868ad4a33SUladzislau Rezki (Sony) nva_start_addr + size > va->va_end) 126968ad4a33SUladzislau Rezki (Sony) return NOTHING_FIT; 127068ad4a33SUladzislau Rezki (Sony) 127168ad4a33SUladzislau Rezki (Sony) /* Now classify. */ 127268ad4a33SUladzislau Rezki (Sony) if (va->va_start == nva_start_addr) { 127368ad4a33SUladzislau Rezki (Sony) if (va->va_end == nva_start_addr + size) 127468ad4a33SUladzislau Rezki (Sony) type = FL_FIT_TYPE; 127568ad4a33SUladzislau Rezki (Sony) else 127668ad4a33SUladzislau Rezki (Sony) type = LE_FIT_TYPE; 127768ad4a33SUladzislau Rezki (Sony) } else if (va->va_end == nva_start_addr + size) { 127868ad4a33SUladzislau Rezki (Sony) type = RE_FIT_TYPE; 127968ad4a33SUladzislau Rezki (Sony) } else { 128068ad4a33SUladzislau Rezki (Sony) type = NE_FIT_TYPE; 128168ad4a33SUladzislau Rezki (Sony) } 128268ad4a33SUladzislau Rezki (Sony) 128368ad4a33SUladzislau Rezki (Sony) return type; 128468ad4a33SUladzislau Rezki (Sony) } 128568ad4a33SUladzislau Rezki (Sony) 128668ad4a33SUladzislau Rezki (Sony) static __always_inline int 128768ad4a33SUladzislau Rezki (Sony) adjust_va_to_fit_type(struct vmap_area *va, 128868ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr, unsigned long size, 128968ad4a33SUladzislau Rezki (Sony) enum fit_type type) 129068ad4a33SUladzislau Rezki (Sony) { 12912c929233SArnd Bergmann struct vmap_area *lva = NULL; 129268ad4a33SUladzislau Rezki (Sony) 129368ad4a33SUladzislau Rezki (Sony) if (type == FL_FIT_TYPE) { 129468ad4a33SUladzislau Rezki (Sony) /* 129568ad4a33SUladzislau Rezki (Sony) * No need to split VA, it fully fits. 129668ad4a33SUladzislau Rezki (Sony) * 129768ad4a33SUladzislau Rezki (Sony) * | | 129868ad4a33SUladzislau Rezki (Sony) * V NVA V 129968ad4a33SUladzislau Rezki (Sony) * |---------------| 130068ad4a33SUladzislau Rezki (Sony) */ 130168ad4a33SUladzislau Rezki (Sony) unlink_va(va, &free_vmap_area_root); 130268ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 130368ad4a33SUladzislau Rezki (Sony) } else if (type == LE_FIT_TYPE) { 130468ad4a33SUladzislau Rezki (Sony) /* 130568ad4a33SUladzislau Rezki (Sony) * Split left edge of fit VA. 130668ad4a33SUladzislau Rezki (Sony) * 130768ad4a33SUladzislau Rezki (Sony) * | | 130868ad4a33SUladzislau Rezki (Sony) * V NVA V R 130968ad4a33SUladzislau Rezki (Sony) * |-------|-------| 131068ad4a33SUladzislau Rezki (Sony) */ 131168ad4a33SUladzislau Rezki (Sony) va->va_start += size; 131268ad4a33SUladzislau Rezki (Sony) } else if (type == RE_FIT_TYPE) { 131368ad4a33SUladzislau Rezki (Sony) /* 131468ad4a33SUladzislau Rezki (Sony) * Split right edge of fit VA. 131568ad4a33SUladzislau Rezki (Sony) * 131668ad4a33SUladzislau Rezki (Sony) * | | 131768ad4a33SUladzislau Rezki (Sony) * L V NVA V 131868ad4a33SUladzislau Rezki (Sony) * |-------|-------| 131968ad4a33SUladzislau Rezki (Sony) */ 132068ad4a33SUladzislau Rezki (Sony) va->va_end = nva_start_addr; 132168ad4a33SUladzislau Rezki (Sony) } else if (type == NE_FIT_TYPE) { 132268ad4a33SUladzislau Rezki (Sony) /* 132368ad4a33SUladzislau Rezki (Sony) * Split no edge of fit VA. 132468ad4a33SUladzislau Rezki (Sony) * 132568ad4a33SUladzislau Rezki (Sony) * | | 132668ad4a33SUladzislau Rezki (Sony) * L V NVA V R 132768ad4a33SUladzislau Rezki (Sony) * |---|-------|---| 132868ad4a33SUladzislau Rezki (Sony) */ 132982dd23e8SUladzislau Rezki (Sony) lva = __this_cpu_xchg(ne_fit_preload_node, NULL); 133082dd23e8SUladzislau Rezki (Sony) if (unlikely(!lva)) { 133182dd23e8SUladzislau Rezki (Sony) /* 133282dd23e8SUladzislau Rezki (Sony) * For percpu allocator we do not do any pre-allocation 133382dd23e8SUladzislau Rezki (Sony) * and leave it as it is. The reason is it most likely 133482dd23e8SUladzislau Rezki (Sony) * never ends up with NE_FIT_TYPE splitting. In case of 133582dd23e8SUladzislau Rezki (Sony) * percpu allocations offsets and sizes are aligned to 133682dd23e8SUladzislau Rezki (Sony) * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE 133782dd23e8SUladzislau Rezki (Sony) * are its main fitting cases. 133882dd23e8SUladzislau Rezki (Sony) * 133982dd23e8SUladzislau Rezki (Sony) * There are a few exceptions though, as an example it is 134082dd23e8SUladzislau Rezki (Sony) * a first allocation (early boot up) when we have "one" 134182dd23e8SUladzislau Rezki (Sony) * big free space that has to be split. 1342060650a2SUladzislau Rezki (Sony) * 1343060650a2SUladzislau Rezki (Sony) * Also we can hit this path in case of regular "vmap" 1344060650a2SUladzislau Rezki (Sony) * allocations, if "this" current CPU was not preloaded. 1345060650a2SUladzislau Rezki (Sony) * See the comment in alloc_vmap_area() why. If so, then 1346060650a2SUladzislau Rezki (Sony) * GFP_NOWAIT is used instead to get an extra object for 1347060650a2SUladzislau Rezki (Sony) * split purpose. That is rare and most time does not 1348060650a2SUladzislau Rezki (Sony) * occur. 1349060650a2SUladzislau Rezki (Sony) * 1350060650a2SUladzislau Rezki (Sony) * What happens if an allocation gets failed. Basically, 1351060650a2SUladzislau Rezki (Sony) * an "overflow" path is triggered to purge lazily freed 1352060650a2SUladzislau Rezki (Sony) * areas to free some memory, then, the "retry" path is 1353060650a2SUladzislau Rezki (Sony) * triggered to repeat one more time. See more details 1354060650a2SUladzislau Rezki (Sony) * in alloc_vmap_area() function. 135582dd23e8SUladzislau Rezki (Sony) */ 135668ad4a33SUladzislau Rezki (Sony) lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT); 135782dd23e8SUladzislau Rezki (Sony) if (!lva) 135868ad4a33SUladzislau Rezki (Sony) return -1; 135982dd23e8SUladzislau Rezki (Sony) } 136068ad4a33SUladzislau Rezki (Sony) 136168ad4a33SUladzislau Rezki (Sony) /* 136268ad4a33SUladzislau Rezki (Sony) * Build the remainder. 136368ad4a33SUladzislau Rezki (Sony) */ 136468ad4a33SUladzislau Rezki (Sony) lva->va_start = va->va_start; 136568ad4a33SUladzislau Rezki (Sony) lva->va_end = nva_start_addr; 136668ad4a33SUladzislau Rezki (Sony) 136768ad4a33SUladzislau Rezki (Sony) /* 136868ad4a33SUladzislau Rezki (Sony) * Shrink this VA to remaining size. 136968ad4a33SUladzislau Rezki (Sony) */ 137068ad4a33SUladzislau Rezki (Sony) va->va_start = nva_start_addr + size; 137168ad4a33SUladzislau Rezki (Sony) } else { 137268ad4a33SUladzislau Rezki (Sony) return -1; 137368ad4a33SUladzislau Rezki (Sony) } 137468ad4a33SUladzislau Rezki (Sony) 137568ad4a33SUladzislau Rezki (Sony) if (type != FL_FIT_TYPE) { 137668ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 137768ad4a33SUladzislau Rezki (Sony) 13782c929233SArnd Bergmann if (lva) /* type == NE_FIT_TYPE */ 137968ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(lva, &va->rb_node, 138068ad4a33SUladzislau Rezki (Sony) &free_vmap_area_root, &free_vmap_area_list); 138168ad4a33SUladzislau Rezki (Sony) } 138268ad4a33SUladzislau Rezki (Sony) 138368ad4a33SUladzislau Rezki (Sony) return 0; 138468ad4a33SUladzislau Rezki (Sony) } 138568ad4a33SUladzislau Rezki (Sony) 138668ad4a33SUladzislau Rezki (Sony) /* 138768ad4a33SUladzislau Rezki (Sony) * Returns a start address of the newly allocated area, if success. 138868ad4a33SUladzislau Rezki (Sony) * Otherwise a vend is returned that indicates failure. 138968ad4a33SUladzislau Rezki (Sony) */ 139068ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 139168ad4a33SUladzislau Rezki (Sony) __alloc_vmap_area(unsigned long size, unsigned long align, 1392cacca6baSUladzislau Rezki (Sony) unsigned long vstart, unsigned long vend) 139368ad4a33SUladzislau Rezki (Sony) { 139468ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr; 139568ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 139668ad4a33SUladzislau Rezki (Sony) enum fit_type type; 139768ad4a33SUladzislau Rezki (Sony) int ret; 139868ad4a33SUladzislau Rezki (Sony) 139968ad4a33SUladzislau Rezki (Sony) va = find_vmap_lowest_match(size, align, vstart); 140068ad4a33SUladzislau Rezki (Sony) if (unlikely(!va)) 140168ad4a33SUladzislau Rezki (Sony) return vend; 140268ad4a33SUladzislau Rezki (Sony) 140368ad4a33SUladzislau Rezki (Sony) if (va->va_start > vstart) 140468ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(va->va_start, align); 140568ad4a33SUladzislau Rezki (Sony) else 140668ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(vstart, align); 140768ad4a33SUladzislau Rezki (Sony) 140868ad4a33SUladzislau Rezki (Sony) /* Check the "vend" restriction. */ 140968ad4a33SUladzislau Rezki (Sony) if (nva_start_addr + size > vend) 141068ad4a33SUladzislau Rezki (Sony) return vend; 141168ad4a33SUladzislau Rezki (Sony) 141268ad4a33SUladzislau Rezki (Sony) /* Classify what we have found. */ 141368ad4a33SUladzislau Rezki (Sony) type = classify_va_fit_type(va, nva_start_addr, size); 141468ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(type == NOTHING_FIT)) 141568ad4a33SUladzislau Rezki (Sony) return vend; 141668ad4a33SUladzislau Rezki (Sony) 141768ad4a33SUladzislau Rezki (Sony) /* Update the free vmap_area. */ 141868ad4a33SUladzislau Rezki (Sony) ret = adjust_va_to_fit_type(va, nva_start_addr, size, type); 141968ad4a33SUladzislau Rezki (Sony) if (ret) 142068ad4a33SUladzislau Rezki (Sony) return vend; 142168ad4a33SUladzislau Rezki (Sony) 1422a6cf4e0fSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 1423a6cf4e0fSUladzislau Rezki (Sony) find_vmap_lowest_match_check(size); 1424a6cf4e0fSUladzislau Rezki (Sony) #endif 1425a6cf4e0fSUladzislau Rezki (Sony) 142668ad4a33SUladzislau Rezki (Sony) return nva_start_addr; 142768ad4a33SUladzislau Rezki (Sony) } 14284da56b99SChris Wilson 1429db64fe02SNick Piggin /* 1430d98c9e83SAndrey Ryabinin * Free a region of KVA allocated by alloc_vmap_area 1431d98c9e83SAndrey Ryabinin */ 1432d98c9e83SAndrey Ryabinin static void free_vmap_area(struct vmap_area *va) 1433d98c9e83SAndrey Ryabinin { 1434d98c9e83SAndrey Ryabinin /* 1435d98c9e83SAndrey Ryabinin * Remove from the busy tree/list. 1436d98c9e83SAndrey Ryabinin */ 1437d98c9e83SAndrey Ryabinin spin_lock(&vmap_area_lock); 1438d98c9e83SAndrey Ryabinin unlink_va(va, &vmap_area_root); 1439d98c9e83SAndrey Ryabinin spin_unlock(&vmap_area_lock); 1440d98c9e83SAndrey Ryabinin 1441d98c9e83SAndrey Ryabinin /* 1442d98c9e83SAndrey Ryabinin * Insert/Merge it back to the free tree/list. 1443d98c9e83SAndrey Ryabinin */ 1444d98c9e83SAndrey Ryabinin spin_lock(&free_vmap_area_lock); 144596e2db45SUladzislau Rezki (Sony) merge_or_add_vmap_area_augment(va, &free_vmap_area_root, &free_vmap_area_list); 1446d98c9e83SAndrey Ryabinin spin_unlock(&free_vmap_area_lock); 1447d98c9e83SAndrey Ryabinin } 1448d98c9e83SAndrey Ryabinin 1449187f8cc4SUladzislau Rezki (Sony) static inline void 1450187f8cc4SUladzislau Rezki (Sony) preload_this_cpu_lock(spinlock_t *lock, gfp_t gfp_mask, int node) 1451187f8cc4SUladzislau Rezki (Sony) { 1452187f8cc4SUladzislau Rezki (Sony) struct vmap_area *va = NULL; 1453187f8cc4SUladzislau Rezki (Sony) 1454187f8cc4SUladzislau Rezki (Sony) /* 1455187f8cc4SUladzislau Rezki (Sony) * Preload this CPU with one extra vmap_area object. It is used 1456187f8cc4SUladzislau Rezki (Sony) * when fit type of free area is NE_FIT_TYPE. It guarantees that 1457187f8cc4SUladzislau Rezki (Sony) * a CPU that does an allocation is preloaded. 1458187f8cc4SUladzislau Rezki (Sony) * 1459187f8cc4SUladzislau Rezki (Sony) * We do it in non-atomic context, thus it allows us to use more 1460187f8cc4SUladzislau Rezki (Sony) * permissive allocation masks to be more stable under low memory 1461187f8cc4SUladzislau Rezki (Sony) * condition and high memory pressure. 1462187f8cc4SUladzislau Rezki (Sony) */ 1463187f8cc4SUladzislau Rezki (Sony) if (!this_cpu_read(ne_fit_preload_node)) 1464187f8cc4SUladzislau Rezki (Sony) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 1465187f8cc4SUladzislau Rezki (Sony) 1466187f8cc4SUladzislau Rezki (Sony) spin_lock(lock); 1467187f8cc4SUladzislau Rezki (Sony) 1468187f8cc4SUladzislau Rezki (Sony) if (va && __this_cpu_cmpxchg(ne_fit_preload_node, NULL, va)) 1469187f8cc4SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 1470187f8cc4SUladzislau Rezki (Sony) } 1471187f8cc4SUladzislau Rezki (Sony) 1472d98c9e83SAndrey Ryabinin /* 1473db64fe02SNick Piggin * Allocate a region of KVA of the specified size and alignment, within the 1474db64fe02SNick Piggin * vstart and vend. 1475db64fe02SNick Piggin */ 1476db64fe02SNick Piggin static struct vmap_area *alloc_vmap_area(unsigned long size, 1477db64fe02SNick Piggin unsigned long align, 1478db64fe02SNick Piggin unsigned long vstart, unsigned long vend, 1479db64fe02SNick Piggin int node, gfp_t gfp_mask) 1480db64fe02SNick Piggin { 1481187f8cc4SUladzislau Rezki (Sony) struct vmap_area *va; 14821da177e4SLinus Torvalds unsigned long addr; 1483db64fe02SNick Piggin int purged = 0; 1484d98c9e83SAndrey Ryabinin int ret; 1485db64fe02SNick Piggin 14867766970cSNick Piggin BUG_ON(!size); 1487891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 148889699605SNick Piggin BUG_ON(!is_power_of_2(align)); 1489db64fe02SNick Piggin 149068ad4a33SUladzislau Rezki (Sony) if (unlikely(!vmap_initialized)) 149168ad4a33SUladzislau Rezki (Sony) return ERR_PTR(-EBUSY); 149268ad4a33SUladzislau Rezki (Sony) 14935803ed29SChristoph Hellwig might_sleep(); 1494f07116d7SUladzislau Rezki (Sony) gfp_mask = gfp_mask & GFP_RECLAIM_MASK; 14954da56b99SChris Wilson 1496f07116d7SUladzislau Rezki (Sony) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 1497db64fe02SNick Piggin if (unlikely(!va)) 1498db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 1499db64fe02SNick Piggin 15007f88f88fSCatalin Marinas /* 15017f88f88fSCatalin Marinas * Only scan the relevant parts containing pointers to other objects 15027f88f88fSCatalin Marinas * to avoid false negatives. 15037f88f88fSCatalin Marinas */ 1504f07116d7SUladzislau Rezki (Sony) kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask); 15057f88f88fSCatalin Marinas 1506db64fe02SNick Piggin retry: 1507187f8cc4SUladzislau Rezki (Sony) preload_this_cpu_lock(&free_vmap_area_lock, gfp_mask, node); 1508187f8cc4SUladzislau Rezki (Sony) addr = __alloc_vmap_area(size, align, vstart, vend); 1509187f8cc4SUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 151068ad4a33SUladzislau Rezki (Sony) 151189699605SNick Piggin /* 151268ad4a33SUladzislau Rezki (Sony) * If an allocation fails, the "vend" address is 151368ad4a33SUladzislau Rezki (Sony) * returned. Therefore trigger the overflow path. 151489699605SNick Piggin */ 151568ad4a33SUladzislau Rezki (Sony) if (unlikely(addr == vend)) 151689699605SNick Piggin goto overflow; 151789699605SNick Piggin 151889699605SNick Piggin va->va_start = addr; 151989699605SNick Piggin va->va_end = addr + size; 1520688fcbfcSPengfei Li va->vm = NULL; 152168ad4a33SUladzislau Rezki (Sony) 1522e36176beSUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 1523e36176beSUladzislau Rezki (Sony) insert_vmap_area(va, &vmap_area_root, &vmap_area_list); 152489699605SNick Piggin spin_unlock(&vmap_area_lock); 152589699605SNick Piggin 152661e16557SWang Xiaoqiang BUG_ON(!IS_ALIGNED(va->va_start, align)); 152789699605SNick Piggin BUG_ON(va->va_start < vstart); 152889699605SNick Piggin BUG_ON(va->va_end > vend); 152989699605SNick Piggin 1530d98c9e83SAndrey Ryabinin ret = kasan_populate_vmalloc(addr, size); 1531d98c9e83SAndrey Ryabinin if (ret) { 1532d98c9e83SAndrey Ryabinin free_vmap_area(va); 1533d98c9e83SAndrey Ryabinin return ERR_PTR(ret); 1534d98c9e83SAndrey Ryabinin } 1535d98c9e83SAndrey Ryabinin 153689699605SNick Piggin return va; 153789699605SNick Piggin 15387766970cSNick Piggin overflow: 1539db64fe02SNick Piggin if (!purged) { 1540db64fe02SNick Piggin purge_vmap_area_lazy(); 1541db64fe02SNick Piggin purged = 1; 1542db64fe02SNick Piggin goto retry; 1543db64fe02SNick Piggin } 15444da56b99SChris Wilson 15454da56b99SChris Wilson if (gfpflags_allow_blocking(gfp_mask)) { 15464da56b99SChris Wilson unsigned long freed = 0; 15474da56b99SChris Wilson blocking_notifier_call_chain(&vmap_notify_list, 0, &freed); 15484da56b99SChris Wilson if (freed > 0) { 15494da56b99SChris Wilson purged = 0; 15504da56b99SChris Wilson goto retry; 15514da56b99SChris Wilson } 15524da56b99SChris Wilson } 15534da56b99SChris Wilson 155403497d76SFlorian Fainelli if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) 1555756a025fSJoe Perches pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n", 1556756a025fSJoe Perches size); 155768ad4a33SUladzislau Rezki (Sony) 155868ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 1559db64fe02SNick Piggin return ERR_PTR(-EBUSY); 1560db64fe02SNick Piggin } 1561db64fe02SNick Piggin 15624da56b99SChris Wilson int register_vmap_purge_notifier(struct notifier_block *nb) 15634da56b99SChris Wilson { 15644da56b99SChris Wilson return blocking_notifier_chain_register(&vmap_notify_list, nb); 15654da56b99SChris Wilson } 15664da56b99SChris Wilson EXPORT_SYMBOL_GPL(register_vmap_purge_notifier); 15674da56b99SChris Wilson 15684da56b99SChris Wilson int unregister_vmap_purge_notifier(struct notifier_block *nb) 15694da56b99SChris Wilson { 15704da56b99SChris Wilson return blocking_notifier_chain_unregister(&vmap_notify_list, nb); 15714da56b99SChris Wilson } 15724da56b99SChris Wilson EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier); 15734da56b99SChris Wilson 1574db64fe02SNick Piggin /* 1575db64fe02SNick Piggin * lazy_max_pages is the maximum amount of virtual address space we gather up 1576db64fe02SNick Piggin * before attempting to purge with a TLB flush. 1577db64fe02SNick Piggin * 1578db64fe02SNick Piggin * There is a tradeoff here: a larger number will cover more kernel page tables 1579db64fe02SNick Piggin * and take slightly longer to purge, but it will linearly reduce the number of 1580db64fe02SNick Piggin * global TLB flushes that must be performed. It would seem natural to scale 1581db64fe02SNick Piggin * this number up linearly with the number of CPUs (because vmapping activity 1582db64fe02SNick Piggin * could also scale linearly with the number of CPUs), however it is likely 1583db64fe02SNick Piggin * that in practice, workloads might be constrained in other ways that mean 1584db64fe02SNick Piggin * vmap activity will not scale linearly with CPUs. Also, I want to be 1585db64fe02SNick Piggin * conservative and not introduce a big latency on huge systems, so go with 1586db64fe02SNick Piggin * a less aggressive log scale. It will still be an improvement over the old 1587db64fe02SNick Piggin * code, and it will be simple to change the scale factor if we find that it 1588db64fe02SNick Piggin * becomes a problem on bigger systems. 1589db64fe02SNick Piggin */ 1590db64fe02SNick Piggin static unsigned long lazy_max_pages(void) 1591db64fe02SNick Piggin { 1592db64fe02SNick Piggin unsigned int log; 1593db64fe02SNick Piggin 1594db64fe02SNick Piggin log = fls(num_online_cpus()); 1595db64fe02SNick Piggin 1596db64fe02SNick Piggin return log * (32UL * 1024 * 1024 / PAGE_SIZE); 1597db64fe02SNick Piggin } 1598db64fe02SNick Piggin 15994d36e6f8SUladzislau Rezki (Sony) static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0); 1600db64fe02SNick Piggin 16010574ecd1SChristoph Hellwig /* 1602f0953a1bSIngo Molnar * Serialize vmap purging. There is no actual critical section protected 16030574ecd1SChristoph Hellwig * by this look, but we want to avoid concurrent calls for performance 16040574ecd1SChristoph Hellwig * reasons and to make the pcpu_get_vm_areas more deterministic. 16050574ecd1SChristoph Hellwig */ 1606f9e09977SChristoph Hellwig static DEFINE_MUTEX(vmap_purge_lock); 16070574ecd1SChristoph Hellwig 160802b709dfSNick Piggin /* for per-CPU blocks */ 160902b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void); 161002b709dfSNick Piggin 16115da96bddSMel Gorman #ifdef CONFIG_X86_64 1612db64fe02SNick Piggin /* 16133ee48b6aSCliff Wickman * called before a call to iounmap() if the caller wants vm_area_struct's 16143ee48b6aSCliff Wickman * immediately freed. 16153ee48b6aSCliff Wickman */ 16163ee48b6aSCliff Wickman void set_iounmap_nonlazy(void) 16173ee48b6aSCliff Wickman { 16184d36e6f8SUladzislau Rezki (Sony) atomic_long_set(&vmap_lazy_nr, lazy_max_pages()+1); 16193ee48b6aSCliff Wickman } 16205da96bddSMel Gorman #endif /* CONFIG_X86_64 */ 16213ee48b6aSCliff Wickman 16223ee48b6aSCliff Wickman /* 1623db64fe02SNick Piggin * Purges all lazily-freed vmap areas. 1624db64fe02SNick Piggin */ 16250574ecd1SChristoph Hellwig static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end) 1626db64fe02SNick Piggin { 16274d36e6f8SUladzislau Rezki (Sony) unsigned long resched_threshold; 162896e2db45SUladzislau Rezki (Sony) struct list_head local_pure_list; 162996e2db45SUladzislau Rezki (Sony) struct vmap_area *va, *n_va; 1630db64fe02SNick Piggin 16310574ecd1SChristoph Hellwig lockdep_assert_held(&vmap_purge_lock); 163202b709dfSNick Piggin 163396e2db45SUladzislau Rezki (Sony) spin_lock(&purge_vmap_area_lock); 163496e2db45SUladzislau Rezki (Sony) purge_vmap_area_root = RB_ROOT; 163596e2db45SUladzislau Rezki (Sony) list_replace_init(&purge_vmap_area_list, &local_pure_list); 163696e2db45SUladzislau Rezki (Sony) spin_unlock(&purge_vmap_area_lock); 163796e2db45SUladzislau Rezki (Sony) 163896e2db45SUladzislau Rezki (Sony) if (unlikely(list_empty(&local_pure_list))) 163968571be9SUladzislau Rezki (Sony) return false; 164068571be9SUladzislau Rezki (Sony) 164196e2db45SUladzislau Rezki (Sony) start = min(start, 164296e2db45SUladzislau Rezki (Sony) list_first_entry(&local_pure_list, 164396e2db45SUladzislau Rezki (Sony) struct vmap_area, list)->va_start); 164496e2db45SUladzislau Rezki (Sony) 164596e2db45SUladzislau Rezki (Sony) end = max(end, 164696e2db45SUladzislau Rezki (Sony) list_last_entry(&local_pure_list, 164796e2db45SUladzislau Rezki (Sony) struct vmap_area, list)->va_end); 1648db64fe02SNick Piggin 16490574ecd1SChristoph Hellwig flush_tlb_kernel_range(start, end); 16504d36e6f8SUladzislau Rezki (Sony) resched_threshold = lazy_max_pages() << 1; 1651db64fe02SNick Piggin 1652e36176beSUladzislau Rezki (Sony) spin_lock(&free_vmap_area_lock); 165396e2db45SUladzislau Rezki (Sony) list_for_each_entry_safe(va, n_va, &local_pure_list, list) { 16544d36e6f8SUladzislau Rezki (Sony) unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT; 16553c5c3cfbSDaniel Axtens unsigned long orig_start = va->va_start; 16563c5c3cfbSDaniel Axtens unsigned long orig_end = va->va_end; 1657763b218dSJoel Fernandes 1658dd3b8353SUladzislau Rezki (Sony) /* 1659dd3b8353SUladzislau Rezki (Sony) * Finally insert or merge lazily-freed area. It is 1660dd3b8353SUladzislau Rezki (Sony) * detached and there is no need to "unlink" it from 1661dd3b8353SUladzislau Rezki (Sony) * anything. 1662dd3b8353SUladzislau Rezki (Sony) */ 166396e2db45SUladzislau Rezki (Sony) va = merge_or_add_vmap_area_augment(va, &free_vmap_area_root, 16643c5c3cfbSDaniel Axtens &free_vmap_area_list); 16653c5c3cfbSDaniel Axtens 16669c801f61SUladzislau Rezki (Sony) if (!va) 16679c801f61SUladzislau Rezki (Sony) continue; 16689c801f61SUladzislau Rezki (Sony) 16693c5c3cfbSDaniel Axtens if (is_vmalloc_or_module_addr((void *)orig_start)) 16703c5c3cfbSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 16713c5c3cfbSDaniel Axtens va->va_start, va->va_end); 1672dd3b8353SUladzislau Rezki (Sony) 16734d36e6f8SUladzislau Rezki (Sony) atomic_long_sub(nr, &vmap_lazy_nr); 167468571be9SUladzislau Rezki (Sony) 16754d36e6f8SUladzislau Rezki (Sony) if (atomic_long_read(&vmap_lazy_nr) < resched_threshold) 1676e36176beSUladzislau Rezki (Sony) cond_resched_lock(&free_vmap_area_lock); 1677763b218dSJoel Fernandes } 1678e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 16790574ecd1SChristoph Hellwig return true; 1680db64fe02SNick Piggin } 1681db64fe02SNick Piggin 1682db64fe02SNick Piggin /* 1683496850e5SNick Piggin * Kick off a purge of the outstanding lazy areas. Don't bother if somebody 1684496850e5SNick Piggin * is already purging. 1685496850e5SNick Piggin */ 1686496850e5SNick Piggin static void try_purge_vmap_area_lazy(void) 1687496850e5SNick Piggin { 1688f9e09977SChristoph Hellwig if (mutex_trylock(&vmap_purge_lock)) { 16890574ecd1SChristoph Hellwig __purge_vmap_area_lazy(ULONG_MAX, 0); 1690f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 16910574ecd1SChristoph Hellwig } 1692496850e5SNick Piggin } 1693496850e5SNick Piggin 1694496850e5SNick Piggin /* 1695db64fe02SNick Piggin * Kick off a purge of the outstanding lazy areas. 1696db64fe02SNick Piggin */ 1697db64fe02SNick Piggin static void purge_vmap_area_lazy(void) 1698db64fe02SNick Piggin { 1699f9e09977SChristoph Hellwig mutex_lock(&vmap_purge_lock); 17000574ecd1SChristoph Hellwig purge_fragmented_blocks_allcpus(); 17010574ecd1SChristoph Hellwig __purge_vmap_area_lazy(ULONG_MAX, 0); 1702f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 1703db64fe02SNick Piggin } 1704db64fe02SNick Piggin 1705db64fe02SNick Piggin /* 170664141da5SJeremy Fitzhardinge * Free a vmap area, caller ensuring that the area has been unmapped 170764141da5SJeremy Fitzhardinge * and flush_cache_vunmap had been called for the correct range 170864141da5SJeremy Fitzhardinge * previously. 1709db64fe02SNick Piggin */ 171064141da5SJeremy Fitzhardinge static void free_vmap_area_noflush(struct vmap_area *va) 1711db64fe02SNick Piggin { 17124d36e6f8SUladzislau Rezki (Sony) unsigned long nr_lazy; 171380c4bd7aSChris Wilson 1714dd3b8353SUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 1715dd3b8353SUladzislau Rezki (Sony) unlink_va(va, &vmap_area_root); 1716dd3b8353SUladzislau Rezki (Sony) spin_unlock(&vmap_area_lock); 1717dd3b8353SUladzislau Rezki (Sony) 17184d36e6f8SUladzislau Rezki (Sony) nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >> 17194d36e6f8SUladzislau Rezki (Sony) PAGE_SHIFT, &vmap_lazy_nr); 172080c4bd7aSChris Wilson 172196e2db45SUladzislau Rezki (Sony) /* 172296e2db45SUladzislau Rezki (Sony) * Merge or place it to the purge tree/list. 172396e2db45SUladzislau Rezki (Sony) */ 172496e2db45SUladzislau Rezki (Sony) spin_lock(&purge_vmap_area_lock); 172596e2db45SUladzislau Rezki (Sony) merge_or_add_vmap_area(va, 172696e2db45SUladzislau Rezki (Sony) &purge_vmap_area_root, &purge_vmap_area_list); 172796e2db45SUladzislau Rezki (Sony) spin_unlock(&purge_vmap_area_lock); 172880c4bd7aSChris Wilson 172996e2db45SUladzislau Rezki (Sony) /* After this point, we may free va at any time */ 173080c4bd7aSChris Wilson if (unlikely(nr_lazy > lazy_max_pages())) 1731496850e5SNick Piggin try_purge_vmap_area_lazy(); 1732db64fe02SNick Piggin } 1733db64fe02SNick Piggin 1734b29acbdcSNick Piggin /* 1735b29acbdcSNick Piggin * Free and unmap a vmap area 1736b29acbdcSNick Piggin */ 1737b29acbdcSNick Piggin static void free_unmap_vmap_area(struct vmap_area *va) 1738b29acbdcSNick Piggin { 1739b29acbdcSNick Piggin flush_cache_vunmap(va->va_start, va->va_end); 17404ad0ae8cSNicholas Piggin vunmap_range_noflush(va->va_start, va->va_end); 17418e57f8acSVlastimil Babka if (debug_pagealloc_enabled_static()) 174282a2e924SChintan Pandya flush_tlb_kernel_range(va->va_start, va->va_end); 174382a2e924SChintan Pandya 1744c8eef01eSChristoph Hellwig free_vmap_area_noflush(va); 1745b29acbdcSNick Piggin } 1746b29acbdcSNick Piggin 1747db64fe02SNick Piggin static struct vmap_area *find_vmap_area(unsigned long addr) 1748db64fe02SNick Piggin { 1749db64fe02SNick Piggin struct vmap_area *va; 1750db64fe02SNick Piggin 1751db64fe02SNick Piggin spin_lock(&vmap_area_lock); 1752db64fe02SNick Piggin va = __find_vmap_area(addr); 1753db64fe02SNick Piggin spin_unlock(&vmap_area_lock); 1754db64fe02SNick Piggin 1755db64fe02SNick Piggin return va; 1756db64fe02SNick Piggin } 1757db64fe02SNick Piggin 1758db64fe02SNick Piggin /*** Per cpu kva allocator ***/ 1759db64fe02SNick Piggin 1760db64fe02SNick Piggin /* 1761db64fe02SNick Piggin * vmap space is limited especially on 32 bit architectures. Ensure there is 1762db64fe02SNick Piggin * room for at least 16 percpu vmap blocks per CPU. 1763db64fe02SNick Piggin */ 1764db64fe02SNick Piggin /* 1765db64fe02SNick Piggin * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able 1766db64fe02SNick Piggin * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess 1767db64fe02SNick Piggin * instead (we just need a rough idea) 1768db64fe02SNick Piggin */ 1769db64fe02SNick Piggin #if BITS_PER_LONG == 32 1770db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024) 1771db64fe02SNick Piggin #else 1772db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024*1024) 1773db64fe02SNick Piggin #endif 1774db64fe02SNick Piggin 1775db64fe02SNick Piggin #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE) 1776db64fe02SNick Piggin #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */ 1777db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */ 1778db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2) 1779db64fe02SNick Piggin #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */ 1780db64fe02SNick Piggin #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */ 1781f982f915SClemens Ladisch #define VMAP_BBMAP_BITS \ 1782f982f915SClemens Ladisch VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ 1783db64fe02SNick Piggin VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ 1784f982f915SClemens Ladisch VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16)) 1785db64fe02SNick Piggin 1786db64fe02SNick Piggin #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE) 1787db64fe02SNick Piggin 1788db64fe02SNick Piggin struct vmap_block_queue { 1789db64fe02SNick Piggin spinlock_t lock; 1790db64fe02SNick Piggin struct list_head free; 1791db64fe02SNick Piggin }; 1792db64fe02SNick Piggin 1793db64fe02SNick Piggin struct vmap_block { 1794db64fe02SNick Piggin spinlock_t lock; 1795db64fe02SNick Piggin struct vmap_area *va; 1796db64fe02SNick Piggin unsigned long free, dirty; 17977d61bfe8SRoman Pen unsigned long dirty_min, dirty_max; /*< dirty range */ 1798db64fe02SNick Piggin struct list_head free_list; 1799db64fe02SNick Piggin struct rcu_head rcu_head; 180002b709dfSNick Piggin struct list_head purge; 1801db64fe02SNick Piggin }; 1802db64fe02SNick Piggin 1803db64fe02SNick Piggin /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */ 1804db64fe02SNick Piggin static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue); 1805db64fe02SNick Piggin 1806db64fe02SNick Piggin /* 18070f14599cSMatthew Wilcox (Oracle) * XArray of vmap blocks, indexed by address, to quickly find a vmap block 1808db64fe02SNick Piggin * in the free path. Could get rid of this if we change the API to return a 1809db64fe02SNick Piggin * "cookie" from alloc, to be passed to free. But no big deal yet. 1810db64fe02SNick Piggin */ 18110f14599cSMatthew Wilcox (Oracle) static DEFINE_XARRAY(vmap_blocks); 1812db64fe02SNick Piggin 1813db64fe02SNick Piggin /* 1814db64fe02SNick Piggin * We should probably have a fallback mechanism to allocate virtual memory 1815db64fe02SNick Piggin * out of partially filled vmap blocks. However vmap block sizing should be 1816db64fe02SNick Piggin * fairly reasonable according to the vmalloc size, so it shouldn't be a 1817db64fe02SNick Piggin * big problem. 1818db64fe02SNick Piggin */ 1819db64fe02SNick Piggin 1820db64fe02SNick Piggin static unsigned long addr_to_vb_idx(unsigned long addr) 1821db64fe02SNick Piggin { 1822db64fe02SNick Piggin addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); 1823db64fe02SNick Piggin addr /= VMAP_BLOCK_SIZE; 1824db64fe02SNick Piggin return addr; 1825db64fe02SNick Piggin } 1826db64fe02SNick Piggin 1827cf725ce2SRoman Pen static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off) 1828cf725ce2SRoman Pen { 1829cf725ce2SRoman Pen unsigned long addr; 1830cf725ce2SRoman Pen 1831cf725ce2SRoman Pen addr = va_start + (pages_off << PAGE_SHIFT); 1832cf725ce2SRoman Pen BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start)); 1833cf725ce2SRoman Pen return (void *)addr; 1834cf725ce2SRoman Pen } 1835cf725ce2SRoman Pen 1836cf725ce2SRoman Pen /** 1837cf725ce2SRoman Pen * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this 1838cf725ce2SRoman Pen * block. Of course pages number can't exceed VMAP_BBMAP_BITS 1839cf725ce2SRoman Pen * @order: how many 2^order pages should be occupied in newly allocated block 1840cf725ce2SRoman Pen * @gfp_mask: flags for the page level allocator 1841cf725ce2SRoman Pen * 1842a862f68aSMike Rapoport * Return: virtual address in a newly allocated block or ERR_PTR(-errno) 1843cf725ce2SRoman Pen */ 1844cf725ce2SRoman Pen static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) 1845db64fe02SNick Piggin { 1846db64fe02SNick Piggin struct vmap_block_queue *vbq; 1847db64fe02SNick Piggin struct vmap_block *vb; 1848db64fe02SNick Piggin struct vmap_area *va; 1849db64fe02SNick Piggin unsigned long vb_idx; 1850db64fe02SNick Piggin int node, err; 1851cf725ce2SRoman Pen void *vaddr; 1852db64fe02SNick Piggin 1853db64fe02SNick Piggin node = numa_node_id(); 1854db64fe02SNick Piggin 1855db64fe02SNick Piggin vb = kmalloc_node(sizeof(struct vmap_block), 1856db64fe02SNick Piggin gfp_mask & GFP_RECLAIM_MASK, node); 1857db64fe02SNick Piggin if (unlikely(!vb)) 1858db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 1859db64fe02SNick Piggin 1860db64fe02SNick Piggin va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, 1861db64fe02SNick Piggin VMALLOC_START, VMALLOC_END, 1862db64fe02SNick Piggin node, gfp_mask); 1863ddf9c6d4STobias Klauser if (IS_ERR(va)) { 1864db64fe02SNick Piggin kfree(vb); 1865e7d86340SJulia Lawall return ERR_CAST(va); 1866db64fe02SNick Piggin } 1867db64fe02SNick Piggin 1868cf725ce2SRoman Pen vaddr = vmap_block_vaddr(va->va_start, 0); 1869db64fe02SNick Piggin spin_lock_init(&vb->lock); 1870db64fe02SNick Piggin vb->va = va; 1871cf725ce2SRoman Pen /* At least something should be left free */ 1872cf725ce2SRoman Pen BUG_ON(VMAP_BBMAP_BITS <= (1UL << order)); 1873cf725ce2SRoman Pen vb->free = VMAP_BBMAP_BITS - (1UL << order); 1874db64fe02SNick Piggin vb->dirty = 0; 18757d61bfe8SRoman Pen vb->dirty_min = VMAP_BBMAP_BITS; 18767d61bfe8SRoman Pen vb->dirty_max = 0; 1877db64fe02SNick Piggin INIT_LIST_HEAD(&vb->free_list); 1878db64fe02SNick Piggin 1879db64fe02SNick Piggin vb_idx = addr_to_vb_idx(va->va_start); 18800f14599cSMatthew Wilcox (Oracle) err = xa_insert(&vmap_blocks, vb_idx, vb, gfp_mask); 18810f14599cSMatthew Wilcox (Oracle) if (err) { 18820f14599cSMatthew Wilcox (Oracle) kfree(vb); 18830f14599cSMatthew Wilcox (Oracle) free_vmap_area(va); 18840f14599cSMatthew Wilcox (Oracle) return ERR_PTR(err); 18850f14599cSMatthew Wilcox (Oracle) } 1886db64fe02SNick Piggin 1887db64fe02SNick Piggin vbq = &get_cpu_var(vmap_block_queue); 1888db64fe02SNick Piggin spin_lock(&vbq->lock); 188968ac546fSRoman Pen list_add_tail_rcu(&vb->free_list, &vbq->free); 1890db64fe02SNick Piggin spin_unlock(&vbq->lock); 18913f04ba85STejun Heo put_cpu_var(vmap_block_queue); 1892db64fe02SNick Piggin 1893cf725ce2SRoman Pen return vaddr; 1894db64fe02SNick Piggin } 1895db64fe02SNick Piggin 1896db64fe02SNick Piggin static void free_vmap_block(struct vmap_block *vb) 1897db64fe02SNick Piggin { 1898db64fe02SNick Piggin struct vmap_block *tmp; 1899db64fe02SNick Piggin 19000f14599cSMatthew Wilcox (Oracle) tmp = xa_erase(&vmap_blocks, addr_to_vb_idx(vb->va->va_start)); 1901db64fe02SNick Piggin BUG_ON(tmp != vb); 1902db64fe02SNick Piggin 190364141da5SJeremy Fitzhardinge free_vmap_area_noflush(vb->va); 190422a3c7d1SLai Jiangshan kfree_rcu(vb, rcu_head); 1905db64fe02SNick Piggin } 1906db64fe02SNick Piggin 190702b709dfSNick Piggin static void purge_fragmented_blocks(int cpu) 190802b709dfSNick Piggin { 190902b709dfSNick Piggin LIST_HEAD(purge); 191002b709dfSNick Piggin struct vmap_block *vb; 191102b709dfSNick Piggin struct vmap_block *n_vb; 191202b709dfSNick Piggin struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 191302b709dfSNick Piggin 191402b709dfSNick Piggin rcu_read_lock(); 191502b709dfSNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 191602b709dfSNick Piggin 191702b709dfSNick Piggin if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS)) 191802b709dfSNick Piggin continue; 191902b709dfSNick Piggin 192002b709dfSNick Piggin spin_lock(&vb->lock); 192102b709dfSNick Piggin if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) { 192202b709dfSNick Piggin vb->free = 0; /* prevent further allocs after releasing lock */ 192302b709dfSNick Piggin vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */ 19247d61bfe8SRoman Pen vb->dirty_min = 0; 19257d61bfe8SRoman Pen vb->dirty_max = VMAP_BBMAP_BITS; 192602b709dfSNick Piggin spin_lock(&vbq->lock); 192702b709dfSNick Piggin list_del_rcu(&vb->free_list); 192802b709dfSNick Piggin spin_unlock(&vbq->lock); 192902b709dfSNick Piggin spin_unlock(&vb->lock); 193002b709dfSNick Piggin list_add_tail(&vb->purge, &purge); 193102b709dfSNick Piggin } else 193202b709dfSNick Piggin spin_unlock(&vb->lock); 193302b709dfSNick Piggin } 193402b709dfSNick Piggin rcu_read_unlock(); 193502b709dfSNick Piggin 193602b709dfSNick Piggin list_for_each_entry_safe(vb, n_vb, &purge, purge) { 193702b709dfSNick Piggin list_del(&vb->purge); 193802b709dfSNick Piggin free_vmap_block(vb); 193902b709dfSNick Piggin } 194002b709dfSNick Piggin } 194102b709dfSNick Piggin 194202b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void) 194302b709dfSNick Piggin { 194402b709dfSNick Piggin int cpu; 194502b709dfSNick Piggin 194602b709dfSNick Piggin for_each_possible_cpu(cpu) 194702b709dfSNick Piggin purge_fragmented_blocks(cpu); 194802b709dfSNick Piggin } 194902b709dfSNick Piggin 1950db64fe02SNick Piggin static void *vb_alloc(unsigned long size, gfp_t gfp_mask) 1951db64fe02SNick Piggin { 1952db64fe02SNick Piggin struct vmap_block_queue *vbq; 1953db64fe02SNick Piggin struct vmap_block *vb; 1954cf725ce2SRoman Pen void *vaddr = NULL; 1955db64fe02SNick Piggin unsigned int order; 1956db64fe02SNick Piggin 1957891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 1958db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 1959aa91c4d8SJan Kara if (WARN_ON(size == 0)) { 1960aa91c4d8SJan Kara /* 1961aa91c4d8SJan Kara * Allocating 0 bytes isn't what caller wants since 1962aa91c4d8SJan Kara * get_order(0) returns funny result. Just warn and terminate 1963aa91c4d8SJan Kara * early. 1964aa91c4d8SJan Kara */ 1965aa91c4d8SJan Kara return NULL; 1966aa91c4d8SJan Kara } 1967db64fe02SNick Piggin order = get_order(size); 1968db64fe02SNick Piggin 1969db64fe02SNick Piggin rcu_read_lock(); 1970db64fe02SNick Piggin vbq = &get_cpu_var(vmap_block_queue); 1971db64fe02SNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 1972cf725ce2SRoman Pen unsigned long pages_off; 1973db64fe02SNick Piggin 1974db64fe02SNick Piggin spin_lock(&vb->lock); 1975cf725ce2SRoman Pen if (vb->free < (1UL << order)) { 1976cf725ce2SRoman Pen spin_unlock(&vb->lock); 1977cf725ce2SRoman Pen continue; 1978cf725ce2SRoman Pen } 197902b709dfSNick Piggin 1980cf725ce2SRoman Pen pages_off = VMAP_BBMAP_BITS - vb->free; 1981cf725ce2SRoman Pen vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); 1982db64fe02SNick Piggin vb->free -= 1UL << order; 1983db64fe02SNick Piggin if (vb->free == 0) { 1984db64fe02SNick Piggin spin_lock(&vbq->lock); 1985de560423SNick Piggin list_del_rcu(&vb->free_list); 1986db64fe02SNick Piggin spin_unlock(&vbq->lock); 1987db64fe02SNick Piggin } 1988cf725ce2SRoman Pen 1989db64fe02SNick Piggin spin_unlock(&vb->lock); 1990db64fe02SNick Piggin break; 1991db64fe02SNick Piggin } 199202b709dfSNick Piggin 19933f04ba85STejun Heo put_cpu_var(vmap_block_queue); 1994db64fe02SNick Piggin rcu_read_unlock(); 1995db64fe02SNick Piggin 1996cf725ce2SRoman Pen /* Allocate new block if nothing was found */ 1997cf725ce2SRoman Pen if (!vaddr) 1998cf725ce2SRoman Pen vaddr = new_vmap_block(order, gfp_mask); 1999db64fe02SNick Piggin 2000cf725ce2SRoman Pen return vaddr; 2001db64fe02SNick Piggin } 2002db64fe02SNick Piggin 200378a0e8c4SChristoph Hellwig static void vb_free(unsigned long addr, unsigned long size) 2004db64fe02SNick Piggin { 2005db64fe02SNick Piggin unsigned long offset; 2006db64fe02SNick Piggin unsigned int order; 2007db64fe02SNick Piggin struct vmap_block *vb; 2008db64fe02SNick Piggin 2009891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 2010db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 2011b29acbdcSNick Piggin 201278a0e8c4SChristoph Hellwig flush_cache_vunmap(addr, addr + size); 2013b29acbdcSNick Piggin 2014db64fe02SNick Piggin order = get_order(size); 201578a0e8c4SChristoph Hellwig offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT; 20160f14599cSMatthew Wilcox (Oracle) vb = xa_load(&vmap_blocks, addr_to_vb_idx(addr)); 2017db64fe02SNick Piggin 20184ad0ae8cSNicholas Piggin vunmap_range_noflush(addr, addr + size); 201964141da5SJeremy Fitzhardinge 20208e57f8acSVlastimil Babka if (debug_pagealloc_enabled_static()) 202178a0e8c4SChristoph Hellwig flush_tlb_kernel_range(addr, addr + size); 202282a2e924SChintan Pandya 2023db64fe02SNick Piggin spin_lock(&vb->lock); 20247d61bfe8SRoman Pen 20257d61bfe8SRoman Pen /* Expand dirty range */ 20267d61bfe8SRoman Pen vb->dirty_min = min(vb->dirty_min, offset); 20277d61bfe8SRoman Pen vb->dirty_max = max(vb->dirty_max, offset + (1UL << order)); 2028d086817dSMinChan Kim 2029db64fe02SNick Piggin vb->dirty += 1UL << order; 2030db64fe02SNick Piggin if (vb->dirty == VMAP_BBMAP_BITS) { 2031de560423SNick Piggin BUG_ON(vb->free); 2032db64fe02SNick Piggin spin_unlock(&vb->lock); 2033db64fe02SNick Piggin free_vmap_block(vb); 2034db64fe02SNick Piggin } else 2035db64fe02SNick Piggin spin_unlock(&vb->lock); 2036db64fe02SNick Piggin } 2037db64fe02SNick Piggin 2038868b104dSRick Edgecombe static void _vm_unmap_aliases(unsigned long start, unsigned long end, int flush) 2039db64fe02SNick Piggin { 2040db64fe02SNick Piggin int cpu; 2041db64fe02SNick Piggin 20429b463334SJeremy Fitzhardinge if (unlikely(!vmap_initialized)) 20439b463334SJeremy Fitzhardinge return; 20449b463334SJeremy Fitzhardinge 20455803ed29SChristoph Hellwig might_sleep(); 20465803ed29SChristoph Hellwig 2047db64fe02SNick Piggin for_each_possible_cpu(cpu) { 2048db64fe02SNick Piggin struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 2049db64fe02SNick Piggin struct vmap_block *vb; 2050db64fe02SNick Piggin 2051db64fe02SNick Piggin rcu_read_lock(); 2052db64fe02SNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 2053db64fe02SNick Piggin spin_lock(&vb->lock); 2054ad216c03SVijayanand Jitta if (vb->dirty && vb->dirty != VMAP_BBMAP_BITS) { 20557d61bfe8SRoman Pen unsigned long va_start = vb->va->va_start; 2056db64fe02SNick Piggin unsigned long s, e; 2057b136be5eSJoonsoo Kim 20587d61bfe8SRoman Pen s = va_start + (vb->dirty_min << PAGE_SHIFT); 20597d61bfe8SRoman Pen e = va_start + (vb->dirty_max << PAGE_SHIFT); 2060db64fe02SNick Piggin 20617d61bfe8SRoman Pen start = min(s, start); 20627d61bfe8SRoman Pen end = max(e, end); 20637d61bfe8SRoman Pen 2064db64fe02SNick Piggin flush = 1; 2065db64fe02SNick Piggin } 2066db64fe02SNick Piggin spin_unlock(&vb->lock); 2067db64fe02SNick Piggin } 2068db64fe02SNick Piggin rcu_read_unlock(); 2069db64fe02SNick Piggin } 2070db64fe02SNick Piggin 2071f9e09977SChristoph Hellwig mutex_lock(&vmap_purge_lock); 20720574ecd1SChristoph Hellwig purge_fragmented_blocks_allcpus(); 20730574ecd1SChristoph Hellwig if (!__purge_vmap_area_lazy(start, end) && flush) 20740574ecd1SChristoph Hellwig flush_tlb_kernel_range(start, end); 2075f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 2076db64fe02SNick Piggin } 2077868b104dSRick Edgecombe 2078868b104dSRick Edgecombe /** 2079868b104dSRick Edgecombe * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer 2080868b104dSRick Edgecombe * 2081868b104dSRick Edgecombe * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily 2082868b104dSRick Edgecombe * to amortize TLB flushing overheads. What this means is that any page you 2083868b104dSRick Edgecombe * have now, may, in a former life, have been mapped into kernel virtual 2084868b104dSRick Edgecombe * address by the vmap layer and so there might be some CPUs with TLB entries 2085868b104dSRick Edgecombe * still referencing that page (additional to the regular 1:1 kernel mapping). 2086868b104dSRick Edgecombe * 2087868b104dSRick Edgecombe * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can 2088868b104dSRick Edgecombe * be sure that none of the pages we have control over will have any aliases 2089868b104dSRick Edgecombe * from the vmap layer. 2090868b104dSRick Edgecombe */ 2091868b104dSRick Edgecombe void vm_unmap_aliases(void) 2092868b104dSRick Edgecombe { 2093868b104dSRick Edgecombe unsigned long start = ULONG_MAX, end = 0; 2094868b104dSRick Edgecombe int flush = 0; 2095868b104dSRick Edgecombe 2096868b104dSRick Edgecombe _vm_unmap_aliases(start, end, flush); 2097868b104dSRick Edgecombe } 2098db64fe02SNick Piggin EXPORT_SYMBOL_GPL(vm_unmap_aliases); 2099db64fe02SNick Piggin 2100db64fe02SNick Piggin /** 2101db64fe02SNick Piggin * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram 2102db64fe02SNick Piggin * @mem: the pointer returned by vm_map_ram 2103db64fe02SNick Piggin * @count: the count passed to that vm_map_ram call (cannot unmap partial) 2104db64fe02SNick Piggin */ 2105db64fe02SNick Piggin void vm_unmap_ram(const void *mem, unsigned int count) 2106db64fe02SNick Piggin { 210765ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 2108db64fe02SNick Piggin unsigned long addr = (unsigned long)mem; 21099c3acf60SChristoph Hellwig struct vmap_area *va; 2110db64fe02SNick Piggin 21115803ed29SChristoph Hellwig might_sleep(); 2112db64fe02SNick Piggin BUG_ON(!addr); 2113db64fe02SNick Piggin BUG_ON(addr < VMALLOC_START); 2114db64fe02SNick Piggin BUG_ON(addr > VMALLOC_END); 2115a1c0b1a0SShawn Lin BUG_ON(!PAGE_ALIGNED(addr)); 2116db64fe02SNick Piggin 2117d98c9e83SAndrey Ryabinin kasan_poison_vmalloc(mem, size); 2118d98c9e83SAndrey Ryabinin 21199c3acf60SChristoph Hellwig if (likely(count <= VMAP_MAX_ALLOC)) { 212005e3ff95SChintan Pandya debug_check_no_locks_freed(mem, size); 212178a0e8c4SChristoph Hellwig vb_free(addr, size); 21229c3acf60SChristoph Hellwig return; 21239c3acf60SChristoph Hellwig } 21249c3acf60SChristoph Hellwig 21259c3acf60SChristoph Hellwig va = find_vmap_area(addr); 21269c3acf60SChristoph Hellwig BUG_ON(!va); 212705e3ff95SChintan Pandya debug_check_no_locks_freed((void *)va->va_start, 212805e3ff95SChintan Pandya (va->va_end - va->va_start)); 21299c3acf60SChristoph Hellwig free_unmap_vmap_area(va); 2130db64fe02SNick Piggin } 2131db64fe02SNick Piggin EXPORT_SYMBOL(vm_unmap_ram); 2132db64fe02SNick Piggin 2133db64fe02SNick Piggin /** 2134db64fe02SNick Piggin * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) 2135db64fe02SNick Piggin * @pages: an array of pointers to the pages to be mapped 2136db64fe02SNick Piggin * @count: number of pages 2137db64fe02SNick Piggin * @node: prefer to allocate data structures on this node 2138e99c97adSRandy Dunlap * 213936437638SGioh Kim * If you use this function for less than VMAP_MAX_ALLOC pages, it could be 214036437638SGioh Kim * faster than vmap so it's good. But if you mix long-life and short-life 214136437638SGioh Kim * objects with vm_map_ram(), it could consume lots of address space through 214236437638SGioh Kim * fragmentation (especially on a 32bit machine). You could see failures in 214336437638SGioh Kim * the end. Please use this function for short-lived objects. 214436437638SGioh Kim * 2145e99c97adSRandy Dunlap * Returns: a pointer to the address that has been mapped, or %NULL on failure 2146db64fe02SNick Piggin */ 2147d4efd79aSChristoph Hellwig void *vm_map_ram(struct page **pages, unsigned int count, int node) 2148db64fe02SNick Piggin { 214965ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 2150db64fe02SNick Piggin unsigned long addr; 2151db64fe02SNick Piggin void *mem; 2152db64fe02SNick Piggin 2153db64fe02SNick Piggin if (likely(count <= VMAP_MAX_ALLOC)) { 2154db64fe02SNick Piggin mem = vb_alloc(size, GFP_KERNEL); 2155db64fe02SNick Piggin if (IS_ERR(mem)) 2156db64fe02SNick Piggin return NULL; 2157db64fe02SNick Piggin addr = (unsigned long)mem; 2158db64fe02SNick Piggin } else { 2159db64fe02SNick Piggin struct vmap_area *va; 2160db64fe02SNick Piggin va = alloc_vmap_area(size, PAGE_SIZE, 2161db64fe02SNick Piggin VMALLOC_START, VMALLOC_END, node, GFP_KERNEL); 2162db64fe02SNick Piggin if (IS_ERR(va)) 2163db64fe02SNick Piggin return NULL; 2164db64fe02SNick Piggin 2165db64fe02SNick Piggin addr = va->va_start; 2166db64fe02SNick Piggin mem = (void *)addr; 2167db64fe02SNick Piggin } 2168d98c9e83SAndrey Ryabinin 2169d98c9e83SAndrey Ryabinin kasan_unpoison_vmalloc(mem, size); 2170d98c9e83SAndrey Ryabinin 2171b67177ecSNicholas Piggin if (vmap_pages_range(addr, addr + size, PAGE_KERNEL, 2172b67177ecSNicholas Piggin pages, PAGE_SHIFT) < 0) { 2173db64fe02SNick Piggin vm_unmap_ram(mem, count); 2174db64fe02SNick Piggin return NULL; 2175db64fe02SNick Piggin } 2176b67177ecSNicholas Piggin 2177db64fe02SNick Piggin return mem; 2178db64fe02SNick Piggin } 2179db64fe02SNick Piggin EXPORT_SYMBOL(vm_map_ram); 2180db64fe02SNick Piggin 21814341fa45SJoonsoo Kim static struct vm_struct *vmlist __initdata; 218292eac168SMike Rapoport 2183121e6f32SNicholas Piggin static inline unsigned int vm_area_page_order(struct vm_struct *vm) 2184121e6f32SNicholas Piggin { 2185121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 2186121e6f32SNicholas Piggin return vm->page_order; 2187121e6f32SNicholas Piggin #else 2188121e6f32SNicholas Piggin return 0; 2189121e6f32SNicholas Piggin #endif 2190121e6f32SNicholas Piggin } 2191121e6f32SNicholas Piggin 2192121e6f32SNicholas Piggin static inline void set_vm_area_page_order(struct vm_struct *vm, unsigned int order) 2193121e6f32SNicholas Piggin { 2194121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 2195121e6f32SNicholas Piggin vm->page_order = order; 2196121e6f32SNicholas Piggin #else 2197121e6f32SNicholas Piggin BUG_ON(order != 0); 2198121e6f32SNicholas Piggin #endif 2199121e6f32SNicholas Piggin } 2200121e6f32SNicholas Piggin 2201f0aa6617STejun Heo /** 2202be9b7335SNicolas Pitre * vm_area_add_early - add vmap area early during boot 2203be9b7335SNicolas Pitre * @vm: vm_struct to add 2204be9b7335SNicolas Pitre * 2205be9b7335SNicolas Pitre * This function is used to add fixed kernel vm area to vmlist before 2206be9b7335SNicolas Pitre * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags 2207be9b7335SNicolas Pitre * should contain proper values and the other fields should be zero. 2208be9b7335SNicolas Pitre * 2209be9b7335SNicolas Pitre * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 2210be9b7335SNicolas Pitre */ 2211be9b7335SNicolas Pitre void __init vm_area_add_early(struct vm_struct *vm) 2212be9b7335SNicolas Pitre { 2213be9b7335SNicolas Pitre struct vm_struct *tmp, **p; 2214be9b7335SNicolas Pitre 2215be9b7335SNicolas Pitre BUG_ON(vmap_initialized); 2216be9b7335SNicolas Pitre for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { 2217be9b7335SNicolas Pitre if (tmp->addr >= vm->addr) { 2218be9b7335SNicolas Pitre BUG_ON(tmp->addr < vm->addr + vm->size); 2219be9b7335SNicolas Pitre break; 2220be9b7335SNicolas Pitre } else 2221be9b7335SNicolas Pitre BUG_ON(tmp->addr + tmp->size > vm->addr); 2222be9b7335SNicolas Pitre } 2223be9b7335SNicolas Pitre vm->next = *p; 2224be9b7335SNicolas Pitre *p = vm; 2225be9b7335SNicolas Pitre } 2226be9b7335SNicolas Pitre 2227be9b7335SNicolas Pitre /** 2228f0aa6617STejun Heo * vm_area_register_early - register vmap area early during boot 2229f0aa6617STejun Heo * @vm: vm_struct to register 2230c0c0a293STejun Heo * @align: requested alignment 2231f0aa6617STejun Heo * 2232f0aa6617STejun Heo * This function is used to register kernel vm area before 2233f0aa6617STejun Heo * vmalloc_init() is called. @vm->size and @vm->flags should contain 2234f0aa6617STejun Heo * proper values on entry and other fields should be zero. On return, 2235f0aa6617STejun Heo * vm->addr contains the allocated address. 2236f0aa6617STejun Heo * 2237f0aa6617STejun Heo * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 2238f0aa6617STejun Heo */ 2239c0c0a293STejun Heo void __init vm_area_register_early(struct vm_struct *vm, size_t align) 2240f0aa6617STejun Heo { 2241f0aa6617STejun Heo static size_t vm_init_off __initdata; 2242c0c0a293STejun Heo unsigned long addr; 2243f0aa6617STejun Heo 2244c0c0a293STejun Heo addr = ALIGN(VMALLOC_START + vm_init_off, align); 2245c0c0a293STejun Heo vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START; 2246c0c0a293STejun Heo 2247c0c0a293STejun Heo vm->addr = (void *)addr; 2248f0aa6617STejun Heo 2249be9b7335SNicolas Pitre vm_area_add_early(vm); 2250f0aa6617STejun Heo } 2251f0aa6617STejun Heo 225268ad4a33SUladzislau Rezki (Sony) static void vmap_init_free_space(void) 225368ad4a33SUladzislau Rezki (Sony) { 225468ad4a33SUladzislau Rezki (Sony) unsigned long vmap_start = 1; 225568ad4a33SUladzislau Rezki (Sony) const unsigned long vmap_end = ULONG_MAX; 225668ad4a33SUladzislau Rezki (Sony) struct vmap_area *busy, *free; 225768ad4a33SUladzislau Rezki (Sony) 225868ad4a33SUladzislau Rezki (Sony) /* 225968ad4a33SUladzislau Rezki (Sony) * B F B B B F 226068ad4a33SUladzislau Rezki (Sony) * -|-----|.....|-----|-----|-----|.....|- 226168ad4a33SUladzislau Rezki (Sony) * | The KVA space | 226268ad4a33SUladzislau Rezki (Sony) * |<--------------------------------->| 226368ad4a33SUladzislau Rezki (Sony) */ 226468ad4a33SUladzislau Rezki (Sony) list_for_each_entry(busy, &vmap_area_list, list) { 226568ad4a33SUladzislau Rezki (Sony) if (busy->va_start - vmap_start > 0) { 226668ad4a33SUladzislau Rezki (Sony) free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 226768ad4a33SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(!free)) { 226868ad4a33SUladzislau Rezki (Sony) free->va_start = vmap_start; 226968ad4a33SUladzislau Rezki (Sony) free->va_end = busy->va_start; 227068ad4a33SUladzislau Rezki (Sony) 227168ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(free, NULL, 227268ad4a33SUladzislau Rezki (Sony) &free_vmap_area_root, 227368ad4a33SUladzislau Rezki (Sony) &free_vmap_area_list); 227468ad4a33SUladzislau Rezki (Sony) } 227568ad4a33SUladzislau Rezki (Sony) } 227668ad4a33SUladzislau Rezki (Sony) 227768ad4a33SUladzislau Rezki (Sony) vmap_start = busy->va_end; 227868ad4a33SUladzislau Rezki (Sony) } 227968ad4a33SUladzislau Rezki (Sony) 228068ad4a33SUladzislau Rezki (Sony) if (vmap_end - vmap_start > 0) { 228168ad4a33SUladzislau Rezki (Sony) free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 228268ad4a33SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(!free)) { 228368ad4a33SUladzislau Rezki (Sony) free->va_start = vmap_start; 228468ad4a33SUladzislau Rezki (Sony) free->va_end = vmap_end; 228568ad4a33SUladzislau Rezki (Sony) 228668ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(free, NULL, 228768ad4a33SUladzislau Rezki (Sony) &free_vmap_area_root, 228868ad4a33SUladzislau Rezki (Sony) &free_vmap_area_list); 228968ad4a33SUladzislau Rezki (Sony) } 229068ad4a33SUladzislau Rezki (Sony) } 229168ad4a33SUladzislau Rezki (Sony) } 229268ad4a33SUladzislau Rezki (Sony) 2293db64fe02SNick Piggin void __init vmalloc_init(void) 2294db64fe02SNick Piggin { 2295822c18f2SIvan Kokshaysky struct vmap_area *va; 2296822c18f2SIvan Kokshaysky struct vm_struct *tmp; 2297db64fe02SNick Piggin int i; 2298db64fe02SNick Piggin 229968ad4a33SUladzislau Rezki (Sony) /* 230068ad4a33SUladzislau Rezki (Sony) * Create the cache for vmap_area objects. 230168ad4a33SUladzislau Rezki (Sony) */ 230268ad4a33SUladzislau Rezki (Sony) vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC); 230368ad4a33SUladzislau Rezki (Sony) 2304db64fe02SNick Piggin for_each_possible_cpu(i) { 2305db64fe02SNick Piggin struct vmap_block_queue *vbq; 230632fcfd40SAl Viro struct vfree_deferred *p; 2307db64fe02SNick Piggin 2308db64fe02SNick Piggin vbq = &per_cpu(vmap_block_queue, i); 2309db64fe02SNick Piggin spin_lock_init(&vbq->lock); 2310db64fe02SNick Piggin INIT_LIST_HEAD(&vbq->free); 231132fcfd40SAl Viro p = &per_cpu(vfree_deferred, i); 231232fcfd40SAl Viro init_llist_head(&p->list); 231332fcfd40SAl Viro INIT_WORK(&p->wq, free_work); 2314db64fe02SNick Piggin } 23159b463334SJeremy Fitzhardinge 2316822c18f2SIvan Kokshaysky /* Import existing vmlist entries. */ 2317822c18f2SIvan Kokshaysky for (tmp = vmlist; tmp; tmp = tmp->next) { 231868ad4a33SUladzislau Rezki (Sony) va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 231968ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(!va)) 232068ad4a33SUladzislau Rezki (Sony) continue; 232168ad4a33SUladzislau Rezki (Sony) 2322822c18f2SIvan Kokshaysky va->va_start = (unsigned long)tmp->addr; 2323822c18f2SIvan Kokshaysky va->va_end = va->va_start + tmp->size; 2324dbda591dSKyongHo va->vm = tmp; 232568ad4a33SUladzislau Rezki (Sony) insert_vmap_area(va, &vmap_area_root, &vmap_area_list); 2326822c18f2SIvan Kokshaysky } 2327ca23e405STejun Heo 232868ad4a33SUladzislau Rezki (Sony) /* 232968ad4a33SUladzislau Rezki (Sony) * Now we can initialize a free vmap space. 233068ad4a33SUladzislau Rezki (Sony) */ 233168ad4a33SUladzislau Rezki (Sony) vmap_init_free_space(); 23329b463334SJeremy Fitzhardinge vmap_initialized = true; 2333db64fe02SNick Piggin } 2334db64fe02SNick Piggin 2335e36176beSUladzislau Rezki (Sony) static inline void setup_vmalloc_vm_locked(struct vm_struct *vm, 2336e36176beSUladzislau Rezki (Sony) struct vmap_area *va, unsigned long flags, const void *caller) 2337cf88c790STejun Heo { 2338cf88c790STejun Heo vm->flags = flags; 2339cf88c790STejun Heo vm->addr = (void *)va->va_start; 2340cf88c790STejun Heo vm->size = va->va_end - va->va_start; 2341cf88c790STejun Heo vm->caller = caller; 2342db1aecafSMinchan Kim va->vm = vm; 2343e36176beSUladzislau Rezki (Sony) } 2344e36176beSUladzislau Rezki (Sony) 2345e36176beSUladzislau Rezki (Sony) static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, 2346e36176beSUladzislau Rezki (Sony) unsigned long flags, const void *caller) 2347e36176beSUladzislau Rezki (Sony) { 2348e36176beSUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 2349e36176beSUladzislau Rezki (Sony) setup_vmalloc_vm_locked(vm, va, flags, caller); 2350c69480adSJoonsoo Kim spin_unlock(&vmap_area_lock); 2351f5252e00SMitsuo Hayasaka } 2352cf88c790STejun Heo 235320fc02b4SZhang Yanfei static void clear_vm_uninitialized_flag(struct vm_struct *vm) 2354f5252e00SMitsuo Hayasaka { 2355d4033afdSJoonsoo Kim /* 235620fc02b4SZhang Yanfei * Before removing VM_UNINITIALIZED, 2357d4033afdSJoonsoo Kim * we should make sure that vm has proper values. 2358d4033afdSJoonsoo Kim * Pair with smp_rmb() in show_numa_info(). 2359d4033afdSJoonsoo Kim */ 2360d4033afdSJoonsoo Kim smp_wmb(); 236120fc02b4SZhang Yanfei vm->flags &= ~VM_UNINITIALIZED; 2362cf88c790STejun Heo } 2363cf88c790STejun Heo 2364db64fe02SNick Piggin static struct vm_struct *__get_vm_area_node(unsigned long size, 23657ca3027bSDaniel Axtens unsigned long align, unsigned long shift, unsigned long flags, 23667ca3027bSDaniel Axtens unsigned long start, unsigned long end, int node, 23677ca3027bSDaniel Axtens gfp_t gfp_mask, const void *caller) 2368db64fe02SNick Piggin { 23690006526dSKautuk Consul struct vmap_area *va; 2370db64fe02SNick Piggin struct vm_struct *area; 2371d98c9e83SAndrey Ryabinin unsigned long requested_size = size; 23721da177e4SLinus Torvalds 237352fd24caSGiridhar Pemmasani BUG_ON(in_interrupt()); 23747ca3027bSDaniel Axtens size = ALIGN(size, 1ul << shift); 237531be8309SOGAWA Hirofumi if (unlikely(!size)) 237631be8309SOGAWA Hirofumi return NULL; 23771da177e4SLinus Torvalds 2378252e5c6eSzijun_hu if (flags & VM_IOREMAP) 2379252e5c6eSzijun_hu align = 1ul << clamp_t(int, get_count_order_long(size), 2380252e5c6eSzijun_hu PAGE_SHIFT, IOREMAP_MAX_ORDER); 2381252e5c6eSzijun_hu 2382cf88c790STejun Heo area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); 23831da177e4SLinus Torvalds if (unlikely(!area)) 23841da177e4SLinus Torvalds return NULL; 23851da177e4SLinus Torvalds 238671394fe5SAndrey Ryabinin if (!(flags & VM_NO_GUARD)) 23871da177e4SLinus Torvalds size += PAGE_SIZE; 23881da177e4SLinus Torvalds 2389db64fe02SNick Piggin va = alloc_vmap_area(size, align, start, end, node, gfp_mask); 2390db64fe02SNick Piggin if (IS_ERR(va)) { 2391db64fe02SNick Piggin kfree(area); 2392db64fe02SNick Piggin return NULL; 23931da177e4SLinus Torvalds } 23941da177e4SLinus Torvalds 2395d98c9e83SAndrey Ryabinin kasan_unpoison_vmalloc((void *)va->va_start, requested_size); 2396f5252e00SMitsuo Hayasaka 2397d98c9e83SAndrey Ryabinin setup_vmalloc_vm(area, va, flags, caller); 23983c5c3cfbSDaniel Axtens 23991da177e4SLinus Torvalds return area; 24001da177e4SLinus Torvalds } 24011da177e4SLinus Torvalds 2402c2968612SBenjamin Herrenschmidt struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, 2403c2968612SBenjamin Herrenschmidt unsigned long start, unsigned long end, 24045e6cafc8SMarek Szyprowski const void *caller) 2405c2968612SBenjamin Herrenschmidt { 24067ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, start, end, 24077ca3027bSDaniel Axtens NUMA_NO_NODE, GFP_KERNEL, caller); 2408c2968612SBenjamin Herrenschmidt } 2409c2968612SBenjamin Herrenschmidt 24101da177e4SLinus Torvalds /** 2411183ff22bSSimon Arlott * get_vm_area - reserve a contiguous kernel virtual area 24121da177e4SLinus Torvalds * @size: size of the area 24131da177e4SLinus Torvalds * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC 24141da177e4SLinus Torvalds * 24151da177e4SLinus Torvalds * Search an area of @size in the kernel virtual mapping area, 24161da177e4SLinus Torvalds * and reserved it for out purposes. Returns the area descriptor 24171da177e4SLinus Torvalds * on success or %NULL on failure. 2418a862f68aSMike Rapoport * 2419a862f68aSMike Rapoport * Return: the area descriptor on success or %NULL on failure. 24201da177e4SLinus Torvalds */ 24211da177e4SLinus Torvalds struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) 24221da177e4SLinus Torvalds { 24237ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, 24247ca3027bSDaniel Axtens VMALLOC_START, VMALLOC_END, 242500ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, 242600ef2d2fSDavid Rientjes __builtin_return_address(0)); 242723016969SChristoph Lameter } 242823016969SChristoph Lameter 242923016969SChristoph Lameter struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, 24305e6cafc8SMarek Szyprowski const void *caller) 243123016969SChristoph Lameter { 24327ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, 24337ca3027bSDaniel Axtens VMALLOC_START, VMALLOC_END, 243400ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, caller); 24351da177e4SLinus Torvalds } 24361da177e4SLinus Torvalds 2437e9da6e99SMarek Szyprowski /** 2438e9da6e99SMarek Szyprowski * find_vm_area - find a continuous kernel virtual area 2439e9da6e99SMarek Szyprowski * @addr: base address 2440e9da6e99SMarek Szyprowski * 2441e9da6e99SMarek Szyprowski * Search for the kernel VM area starting at @addr, and return it. 2442e9da6e99SMarek Szyprowski * It is up to the caller to do all required locking to keep the returned 2443e9da6e99SMarek Szyprowski * pointer valid. 2444a862f68aSMike Rapoport * 244574640617SHui Su * Return: the area descriptor on success or %NULL on failure. 2446e9da6e99SMarek Szyprowski */ 2447e9da6e99SMarek Szyprowski struct vm_struct *find_vm_area(const void *addr) 244883342314SNick Piggin { 2449db64fe02SNick Piggin struct vmap_area *va; 245083342314SNick Piggin 2451db64fe02SNick Piggin va = find_vmap_area((unsigned long)addr); 2452688fcbfcSPengfei Li if (!va) 24537856dfebSAndi Kleen return NULL; 2454688fcbfcSPengfei Li 2455688fcbfcSPengfei Li return va->vm; 24567856dfebSAndi Kleen } 24577856dfebSAndi Kleen 24581da177e4SLinus Torvalds /** 2459183ff22bSSimon Arlott * remove_vm_area - find and remove a continuous kernel virtual area 24601da177e4SLinus Torvalds * @addr: base address 24611da177e4SLinus Torvalds * 24621da177e4SLinus Torvalds * Search for the kernel VM area starting at @addr, and remove it. 24631da177e4SLinus Torvalds * This function returns the found VM area, but using it is NOT safe 24647856dfebSAndi Kleen * on SMP machines, except for its size or flags. 2465a862f68aSMike Rapoport * 246674640617SHui Su * Return: the area descriptor on success or %NULL on failure. 24671da177e4SLinus Torvalds */ 2468b3bdda02SChristoph Lameter struct vm_struct *remove_vm_area(const void *addr) 24691da177e4SLinus Torvalds { 2470db64fe02SNick Piggin struct vmap_area *va; 2471db64fe02SNick Piggin 24725803ed29SChristoph Hellwig might_sleep(); 24735803ed29SChristoph Hellwig 2474dd3b8353SUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 2475dd3b8353SUladzislau Rezki (Sony) va = __find_vmap_area((unsigned long)addr); 2476688fcbfcSPengfei Li if (va && va->vm) { 2477db1aecafSMinchan Kim struct vm_struct *vm = va->vm; 2478f5252e00SMitsuo Hayasaka 2479c69480adSJoonsoo Kim va->vm = NULL; 2480c69480adSJoonsoo Kim spin_unlock(&vmap_area_lock); 2481c69480adSJoonsoo Kim 2482a5af5aa8SAndrey Ryabinin kasan_free_shadow(vm); 2483dd32c279SKAMEZAWA Hiroyuki free_unmap_vmap_area(va); 2484dd32c279SKAMEZAWA Hiroyuki 2485db64fe02SNick Piggin return vm; 2486db64fe02SNick Piggin } 2487dd3b8353SUladzislau Rezki (Sony) 2488dd3b8353SUladzislau Rezki (Sony) spin_unlock(&vmap_area_lock); 2489db64fe02SNick Piggin return NULL; 24901da177e4SLinus Torvalds } 24911da177e4SLinus Torvalds 2492868b104dSRick Edgecombe static inline void set_area_direct_map(const struct vm_struct *area, 2493868b104dSRick Edgecombe int (*set_direct_map)(struct page *page)) 2494868b104dSRick Edgecombe { 2495868b104dSRick Edgecombe int i; 2496868b104dSRick Edgecombe 2497121e6f32SNicholas Piggin /* HUGE_VMALLOC passes small pages to set_direct_map */ 2498868b104dSRick Edgecombe for (i = 0; i < area->nr_pages; i++) 2499868b104dSRick Edgecombe if (page_address(area->pages[i])) 2500868b104dSRick Edgecombe set_direct_map(area->pages[i]); 2501868b104dSRick Edgecombe } 2502868b104dSRick Edgecombe 2503868b104dSRick Edgecombe /* Handle removing and resetting vm mappings related to the vm_struct. */ 2504868b104dSRick Edgecombe static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages) 2505868b104dSRick Edgecombe { 2506868b104dSRick Edgecombe unsigned long start = ULONG_MAX, end = 0; 2507121e6f32SNicholas Piggin unsigned int page_order = vm_area_page_order(area); 2508868b104dSRick Edgecombe int flush_reset = area->flags & VM_FLUSH_RESET_PERMS; 250931e67340SRick Edgecombe int flush_dmap = 0; 2510868b104dSRick Edgecombe int i; 2511868b104dSRick Edgecombe 2512868b104dSRick Edgecombe remove_vm_area(area->addr); 2513868b104dSRick Edgecombe 2514868b104dSRick Edgecombe /* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */ 2515868b104dSRick Edgecombe if (!flush_reset) 2516868b104dSRick Edgecombe return; 2517868b104dSRick Edgecombe 2518868b104dSRick Edgecombe /* 2519868b104dSRick Edgecombe * If not deallocating pages, just do the flush of the VM area and 2520868b104dSRick Edgecombe * return. 2521868b104dSRick Edgecombe */ 2522868b104dSRick Edgecombe if (!deallocate_pages) { 2523868b104dSRick Edgecombe vm_unmap_aliases(); 2524868b104dSRick Edgecombe return; 2525868b104dSRick Edgecombe } 2526868b104dSRick Edgecombe 2527868b104dSRick Edgecombe /* 2528868b104dSRick Edgecombe * If execution gets here, flush the vm mapping and reset the direct 2529868b104dSRick Edgecombe * map. Find the start and end range of the direct mappings to make sure 2530868b104dSRick Edgecombe * the vm_unmap_aliases() flush includes the direct map. 2531868b104dSRick Edgecombe */ 2532121e6f32SNicholas Piggin for (i = 0; i < area->nr_pages; i += 1U << page_order) { 25338e41f872SRick Edgecombe unsigned long addr = (unsigned long)page_address(area->pages[i]); 25348e41f872SRick Edgecombe if (addr) { 2535121e6f32SNicholas Piggin unsigned long page_size; 2536121e6f32SNicholas Piggin 2537121e6f32SNicholas Piggin page_size = PAGE_SIZE << page_order; 2538868b104dSRick Edgecombe start = min(addr, start); 2539121e6f32SNicholas Piggin end = max(addr + page_size, end); 254031e67340SRick Edgecombe flush_dmap = 1; 2541868b104dSRick Edgecombe } 2542868b104dSRick Edgecombe } 2543868b104dSRick Edgecombe 2544868b104dSRick Edgecombe /* 2545868b104dSRick Edgecombe * Set direct map to something invalid so that it won't be cached if 2546868b104dSRick Edgecombe * there are any accesses after the TLB flush, then flush the TLB and 2547868b104dSRick Edgecombe * reset the direct map permissions to the default. 2548868b104dSRick Edgecombe */ 2549868b104dSRick Edgecombe set_area_direct_map(area, set_direct_map_invalid_noflush); 255031e67340SRick Edgecombe _vm_unmap_aliases(start, end, flush_dmap); 2551868b104dSRick Edgecombe set_area_direct_map(area, set_direct_map_default_noflush); 2552868b104dSRick Edgecombe } 2553868b104dSRick Edgecombe 2554b3bdda02SChristoph Lameter static void __vunmap(const void *addr, int deallocate_pages) 25551da177e4SLinus Torvalds { 25561da177e4SLinus Torvalds struct vm_struct *area; 25571da177e4SLinus Torvalds 25581da177e4SLinus Torvalds if (!addr) 25591da177e4SLinus Torvalds return; 25601da177e4SLinus Torvalds 2561e69e9d4aSHATAYAMA Daisuke if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n", 2562ab15d9b4SDan Carpenter addr)) 25631da177e4SLinus Torvalds return; 25641da177e4SLinus Torvalds 25656ade2032SLiviu Dudau area = find_vm_area(addr); 25661da177e4SLinus Torvalds if (unlikely(!area)) { 25674c8573e2SArjan van de Ven WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", 25681da177e4SLinus Torvalds addr); 25691da177e4SLinus Torvalds return; 25701da177e4SLinus Torvalds } 25711da177e4SLinus Torvalds 257205e3ff95SChintan Pandya debug_check_no_locks_freed(area->addr, get_vm_area_size(area)); 257305e3ff95SChintan Pandya debug_check_no_obj_freed(area->addr, get_vm_area_size(area)); 25749a11b49aSIngo Molnar 2575c041098cSVincenzo Frascino kasan_poison_vmalloc(area->addr, get_vm_area_size(area)); 25763c5c3cfbSDaniel Axtens 2577868b104dSRick Edgecombe vm_remove_mappings(area, deallocate_pages); 2578868b104dSRick Edgecombe 25791da177e4SLinus Torvalds if (deallocate_pages) { 2580121e6f32SNicholas Piggin unsigned int page_order = vm_area_page_order(area); 25811da177e4SLinus Torvalds int i; 25821da177e4SLinus Torvalds 2583121e6f32SNicholas Piggin for (i = 0; i < area->nr_pages; i += 1U << page_order) { 2584bf53d6f8SChristoph Lameter struct page *page = area->pages[i]; 2585bf53d6f8SChristoph Lameter 2586bf53d6f8SChristoph Lameter BUG_ON(!page); 2587121e6f32SNicholas Piggin __free_pages(page, page_order); 2588a850e932SRafael Aquini cond_resched(); 25891da177e4SLinus Torvalds } 259097105f0aSRoman Gushchin atomic_long_sub(area->nr_pages, &nr_vmalloc_pages); 25911da177e4SLinus Torvalds 2592244d63eeSDavid Rientjes kvfree(area->pages); 25931da177e4SLinus Torvalds } 25941da177e4SLinus Torvalds 25951da177e4SLinus Torvalds kfree(area); 25961da177e4SLinus Torvalds } 25971da177e4SLinus Torvalds 2598bf22e37aSAndrey Ryabinin static inline void __vfree_deferred(const void *addr) 2599bf22e37aSAndrey Ryabinin { 2600bf22e37aSAndrey Ryabinin /* 2601bf22e37aSAndrey Ryabinin * Use raw_cpu_ptr() because this can be called from preemptible 2602bf22e37aSAndrey Ryabinin * context. Preemption is absolutely fine here, because the llist_add() 2603bf22e37aSAndrey Ryabinin * implementation is lockless, so it works even if we are adding to 260473221d88SJeongtae Park * another cpu's list. schedule_work() should be fine with this too. 2605bf22e37aSAndrey Ryabinin */ 2606bf22e37aSAndrey Ryabinin struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred); 2607bf22e37aSAndrey Ryabinin 2608bf22e37aSAndrey Ryabinin if (llist_add((struct llist_node *)addr, &p->list)) 2609bf22e37aSAndrey Ryabinin schedule_work(&p->wq); 2610bf22e37aSAndrey Ryabinin } 2611bf22e37aSAndrey Ryabinin 2612bf22e37aSAndrey Ryabinin /** 2613bf22e37aSAndrey Ryabinin * vfree_atomic - release memory allocated by vmalloc() 2614bf22e37aSAndrey Ryabinin * @addr: memory base address 2615bf22e37aSAndrey Ryabinin * 2616bf22e37aSAndrey Ryabinin * This one is just like vfree() but can be called in any atomic context 2617bf22e37aSAndrey Ryabinin * except NMIs. 2618bf22e37aSAndrey Ryabinin */ 2619bf22e37aSAndrey Ryabinin void vfree_atomic(const void *addr) 2620bf22e37aSAndrey Ryabinin { 2621bf22e37aSAndrey Ryabinin BUG_ON(in_nmi()); 2622bf22e37aSAndrey Ryabinin 2623bf22e37aSAndrey Ryabinin kmemleak_free(addr); 2624bf22e37aSAndrey Ryabinin 2625bf22e37aSAndrey Ryabinin if (!addr) 2626bf22e37aSAndrey Ryabinin return; 2627bf22e37aSAndrey Ryabinin __vfree_deferred(addr); 2628bf22e37aSAndrey Ryabinin } 2629bf22e37aSAndrey Ryabinin 2630c67dc624SRoman Penyaev static void __vfree(const void *addr) 2631c67dc624SRoman Penyaev { 2632c67dc624SRoman Penyaev if (unlikely(in_interrupt())) 2633c67dc624SRoman Penyaev __vfree_deferred(addr); 2634c67dc624SRoman Penyaev else 2635c67dc624SRoman Penyaev __vunmap(addr, 1); 2636c67dc624SRoman Penyaev } 2637c67dc624SRoman Penyaev 26381da177e4SLinus Torvalds /** 2639fa307474SMatthew Wilcox (Oracle) * vfree - Release memory allocated by vmalloc() 2640fa307474SMatthew Wilcox (Oracle) * @addr: Memory base address 26411da177e4SLinus Torvalds * 2642fa307474SMatthew Wilcox (Oracle) * Free the virtually continuous memory area starting at @addr, as obtained 2643fa307474SMatthew Wilcox (Oracle) * from one of the vmalloc() family of APIs. This will usually also free the 2644fa307474SMatthew Wilcox (Oracle) * physical memory underlying the virtual allocation, but that memory is 2645fa307474SMatthew Wilcox (Oracle) * reference counted, so it will not be freed until the last user goes away. 26461da177e4SLinus Torvalds * 2647fa307474SMatthew Wilcox (Oracle) * If @addr is NULL, no operation is performed. 264832fcfd40SAl Viro * 2649fa307474SMatthew Wilcox (Oracle) * Context: 26503ca4ea3aSAndrey Ryabinin * May sleep if called *not* from interrupt context. 2651fa307474SMatthew Wilcox (Oracle) * Must not be called in NMI context (strictly speaking, it could be 2652fa307474SMatthew Wilcox (Oracle) * if we have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling 2653f0953a1bSIngo Molnar * conventions for vfree() arch-dependent would be a really bad idea). 26541da177e4SLinus Torvalds */ 2655b3bdda02SChristoph Lameter void vfree(const void *addr) 26561da177e4SLinus Torvalds { 265732fcfd40SAl Viro BUG_ON(in_nmi()); 265889219d37SCatalin Marinas 265989219d37SCatalin Marinas kmemleak_free(addr); 266089219d37SCatalin Marinas 2661a8dda165SAndrey Ryabinin might_sleep_if(!in_interrupt()); 2662a8dda165SAndrey Ryabinin 266332fcfd40SAl Viro if (!addr) 266432fcfd40SAl Viro return; 2665c67dc624SRoman Penyaev 2666c67dc624SRoman Penyaev __vfree(addr); 26671da177e4SLinus Torvalds } 26681da177e4SLinus Torvalds EXPORT_SYMBOL(vfree); 26691da177e4SLinus Torvalds 26701da177e4SLinus Torvalds /** 26711da177e4SLinus Torvalds * vunmap - release virtual mapping obtained by vmap() 26721da177e4SLinus Torvalds * @addr: memory base address 26731da177e4SLinus Torvalds * 26741da177e4SLinus Torvalds * Free the virtually contiguous memory area starting at @addr, 26751da177e4SLinus Torvalds * which was created from the page array passed to vmap(). 26761da177e4SLinus Torvalds * 267780e93effSPekka Enberg * Must not be called in interrupt context. 26781da177e4SLinus Torvalds */ 2679b3bdda02SChristoph Lameter void vunmap(const void *addr) 26801da177e4SLinus Torvalds { 26811da177e4SLinus Torvalds BUG_ON(in_interrupt()); 268234754b69SPeter Zijlstra might_sleep(); 268332fcfd40SAl Viro if (addr) 26841da177e4SLinus Torvalds __vunmap(addr, 0); 26851da177e4SLinus Torvalds } 26861da177e4SLinus Torvalds EXPORT_SYMBOL(vunmap); 26871da177e4SLinus Torvalds 26881da177e4SLinus Torvalds /** 26891da177e4SLinus Torvalds * vmap - map an array of pages into virtually contiguous space 26901da177e4SLinus Torvalds * @pages: array of page pointers 26911da177e4SLinus Torvalds * @count: number of pages to map 26921da177e4SLinus Torvalds * @flags: vm_area->flags 26931da177e4SLinus Torvalds * @prot: page protection for the mapping 26941da177e4SLinus Torvalds * 2695b944afc9SChristoph Hellwig * Maps @count pages from @pages into contiguous kernel virtual space. 2696b944afc9SChristoph Hellwig * If @flags contains %VM_MAP_PUT_PAGES the ownership of the pages array itself 2697b944afc9SChristoph Hellwig * (which must be kmalloc or vmalloc memory) and one reference per pages in it 2698b944afc9SChristoph Hellwig * are transferred from the caller to vmap(), and will be freed / dropped when 2699b944afc9SChristoph Hellwig * vfree() is called on the return value. 2700a862f68aSMike Rapoport * 2701a862f68aSMike Rapoport * Return: the address of the area or %NULL on failure 27021da177e4SLinus Torvalds */ 27031da177e4SLinus Torvalds void *vmap(struct page **pages, unsigned int count, 27041da177e4SLinus Torvalds unsigned long flags, pgprot_t prot) 27051da177e4SLinus Torvalds { 27061da177e4SLinus Torvalds struct vm_struct *area; 2707b67177ecSNicholas Piggin unsigned long addr; 270865ee03c4SGuillermo Julián Moreno unsigned long size; /* In bytes */ 27091da177e4SLinus Torvalds 271034754b69SPeter Zijlstra might_sleep(); 271134754b69SPeter Zijlstra 2712ca79b0c2SArun KS if (count > totalram_pages()) 27131da177e4SLinus Torvalds return NULL; 27141da177e4SLinus Torvalds 271565ee03c4SGuillermo Julián Moreno size = (unsigned long)count << PAGE_SHIFT; 271665ee03c4SGuillermo Julián Moreno area = get_vm_area_caller(size, flags, __builtin_return_address(0)); 27171da177e4SLinus Torvalds if (!area) 27181da177e4SLinus Torvalds return NULL; 271923016969SChristoph Lameter 2720b67177ecSNicholas Piggin addr = (unsigned long)area->addr; 2721b67177ecSNicholas Piggin if (vmap_pages_range(addr, addr + size, pgprot_nx(prot), 2722b67177ecSNicholas Piggin pages, PAGE_SHIFT) < 0) { 27231da177e4SLinus Torvalds vunmap(area->addr); 27241da177e4SLinus Torvalds return NULL; 27251da177e4SLinus Torvalds } 27261da177e4SLinus Torvalds 2727c22ee528SMiaohe Lin if (flags & VM_MAP_PUT_PAGES) { 2728b944afc9SChristoph Hellwig area->pages = pages; 2729c22ee528SMiaohe Lin area->nr_pages = count; 2730c22ee528SMiaohe Lin } 27311da177e4SLinus Torvalds return area->addr; 27321da177e4SLinus Torvalds } 27331da177e4SLinus Torvalds EXPORT_SYMBOL(vmap); 27341da177e4SLinus Torvalds 27353e9a9e25SChristoph Hellwig #ifdef CONFIG_VMAP_PFN 27363e9a9e25SChristoph Hellwig struct vmap_pfn_data { 27373e9a9e25SChristoph Hellwig unsigned long *pfns; 27383e9a9e25SChristoph Hellwig pgprot_t prot; 27393e9a9e25SChristoph Hellwig unsigned int idx; 27403e9a9e25SChristoph Hellwig }; 27413e9a9e25SChristoph Hellwig 27423e9a9e25SChristoph Hellwig static int vmap_pfn_apply(pte_t *pte, unsigned long addr, void *private) 27433e9a9e25SChristoph Hellwig { 27443e9a9e25SChristoph Hellwig struct vmap_pfn_data *data = private; 27453e9a9e25SChristoph Hellwig 27463e9a9e25SChristoph Hellwig if (WARN_ON_ONCE(pfn_valid(data->pfns[data->idx]))) 27473e9a9e25SChristoph Hellwig return -EINVAL; 27483e9a9e25SChristoph Hellwig *pte = pte_mkspecial(pfn_pte(data->pfns[data->idx++], data->prot)); 27493e9a9e25SChristoph Hellwig return 0; 27503e9a9e25SChristoph Hellwig } 27513e9a9e25SChristoph Hellwig 27523e9a9e25SChristoph Hellwig /** 27533e9a9e25SChristoph Hellwig * vmap_pfn - map an array of PFNs into virtually contiguous space 27543e9a9e25SChristoph Hellwig * @pfns: array of PFNs 27553e9a9e25SChristoph Hellwig * @count: number of pages to map 27563e9a9e25SChristoph Hellwig * @prot: page protection for the mapping 27573e9a9e25SChristoph Hellwig * 27583e9a9e25SChristoph Hellwig * Maps @count PFNs from @pfns into contiguous kernel virtual space and returns 27593e9a9e25SChristoph Hellwig * the start address of the mapping. 27603e9a9e25SChristoph Hellwig */ 27613e9a9e25SChristoph Hellwig void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot) 27623e9a9e25SChristoph Hellwig { 27633e9a9e25SChristoph Hellwig struct vmap_pfn_data data = { .pfns = pfns, .prot = pgprot_nx(prot) }; 27643e9a9e25SChristoph Hellwig struct vm_struct *area; 27653e9a9e25SChristoph Hellwig 27663e9a9e25SChristoph Hellwig area = get_vm_area_caller(count * PAGE_SIZE, VM_IOREMAP, 27673e9a9e25SChristoph Hellwig __builtin_return_address(0)); 27683e9a9e25SChristoph Hellwig if (!area) 27693e9a9e25SChristoph Hellwig return NULL; 27703e9a9e25SChristoph Hellwig if (apply_to_page_range(&init_mm, (unsigned long)area->addr, 27713e9a9e25SChristoph Hellwig count * PAGE_SIZE, vmap_pfn_apply, &data)) { 27723e9a9e25SChristoph Hellwig free_vm_area(area); 27733e9a9e25SChristoph Hellwig return NULL; 27743e9a9e25SChristoph Hellwig } 27753e9a9e25SChristoph Hellwig return area->addr; 27763e9a9e25SChristoph Hellwig } 27773e9a9e25SChristoph Hellwig EXPORT_SYMBOL_GPL(vmap_pfn); 27783e9a9e25SChristoph Hellwig #endif /* CONFIG_VMAP_PFN */ 27793e9a9e25SChristoph Hellwig 278012b9f873SUladzislau Rezki static inline unsigned int 278112b9f873SUladzislau Rezki vm_area_alloc_pages(gfp_t gfp, int nid, 2782*343ab817SUladzislau Rezki (Sony) unsigned int order, unsigned int nr_pages, struct page **pages) 278312b9f873SUladzislau Rezki { 278412b9f873SUladzislau Rezki unsigned int nr_allocated = 0; 278512b9f873SUladzislau Rezki 278612b9f873SUladzislau Rezki /* 278712b9f873SUladzislau Rezki * For order-0 pages we make use of bulk allocator, if 278812b9f873SUladzislau Rezki * the page array is partly or not at all populated due 278912b9f873SUladzislau Rezki * to fails, fallback to a single page allocator that is 279012b9f873SUladzislau Rezki * more permissive. 279112b9f873SUladzislau Rezki */ 2792*343ab817SUladzislau Rezki (Sony) if (!order) { 2793*343ab817SUladzislau Rezki (Sony) while (nr_allocated < nr_pages) { 2794*343ab817SUladzislau Rezki (Sony) unsigned int nr, nr_pages_request; 2795*343ab817SUladzislau Rezki (Sony) 2796*343ab817SUladzislau Rezki (Sony) /* 2797*343ab817SUladzislau Rezki (Sony) * A maximum allowed request is hard-coded and is 100 2798*343ab817SUladzislau Rezki (Sony) * pages per call. That is done in order to prevent a 2799*343ab817SUladzislau Rezki (Sony) * long preemption off scenario in the bulk-allocator 2800*343ab817SUladzislau Rezki (Sony) * so the range is [1:100]. 2801*343ab817SUladzislau Rezki (Sony) */ 2802*343ab817SUladzislau Rezki (Sony) nr_pages_request = min(100U, nr_pages - nr_allocated); 2803*343ab817SUladzislau Rezki (Sony) 2804*343ab817SUladzislau Rezki (Sony) nr = alloc_pages_bulk_array_node(gfp, nid, 2805*343ab817SUladzislau Rezki (Sony) nr_pages_request, pages + nr_allocated); 2806*343ab817SUladzislau Rezki (Sony) 2807*343ab817SUladzislau Rezki (Sony) nr_allocated += nr; 2808*343ab817SUladzislau Rezki (Sony) cond_resched(); 2809*343ab817SUladzislau Rezki (Sony) 2810*343ab817SUladzislau Rezki (Sony) /* 2811*343ab817SUladzislau Rezki (Sony) * If zero or pages were obtained partly, 2812*343ab817SUladzislau Rezki (Sony) * fallback to a single page allocator. 2813*343ab817SUladzislau Rezki (Sony) */ 2814*343ab817SUladzislau Rezki (Sony) if (nr != nr_pages_request) 2815*343ab817SUladzislau Rezki (Sony) break; 2816*343ab817SUladzislau Rezki (Sony) } 2817*343ab817SUladzislau Rezki (Sony) } else 281812b9f873SUladzislau Rezki /* 281912b9f873SUladzislau Rezki * Compound pages required for remap_vmalloc_page if 282012b9f873SUladzislau Rezki * high-order pages. 282112b9f873SUladzislau Rezki */ 282212b9f873SUladzislau Rezki gfp |= __GFP_COMP; 282312b9f873SUladzislau Rezki 282412b9f873SUladzislau Rezki /* High-order pages or fallback path if "bulk" fails. */ 282512b9f873SUladzislau Rezki while (nr_allocated < nr_pages) { 282612b9f873SUladzislau Rezki struct page *page; 282712b9f873SUladzislau Rezki int i; 282812b9f873SUladzislau Rezki 282912b9f873SUladzislau Rezki page = alloc_pages_node(nid, gfp, order); 283012b9f873SUladzislau Rezki if (unlikely(!page)) 283112b9f873SUladzislau Rezki break; 283212b9f873SUladzislau Rezki 283312b9f873SUladzislau Rezki /* 283412b9f873SUladzislau Rezki * Careful, we allocate and map page-order pages, but 283512b9f873SUladzislau Rezki * tracking is done per PAGE_SIZE page so as to keep the 283612b9f873SUladzislau Rezki * vm_struct APIs independent of the physical/mapped size. 283712b9f873SUladzislau Rezki */ 283812b9f873SUladzislau Rezki for (i = 0; i < (1U << order); i++) 283912b9f873SUladzislau Rezki pages[nr_allocated + i] = page + i; 284012b9f873SUladzislau Rezki 284112b9f873SUladzislau Rezki if (gfpflags_allow_blocking(gfp)) 284212b9f873SUladzislau Rezki cond_resched(); 284312b9f873SUladzislau Rezki 284412b9f873SUladzislau Rezki nr_allocated += 1U << order; 284512b9f873SUladzislau Rezki } 284612b9f873SUladzislau Rezki 284712b9f873SUladzislau Rezki return nr_allocated; 284812b9f873SUladzislau Rezki } 284912b9f873SUladzislau Rezki 2850e31d9eb5SAdrian Bunk static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, 2851121e6f32SNicholas Piggin pgprot_t prot, unsigned int page_shift, 2852121e6f32SNicholas Piggin int node) 28531da177e4SLinus Torvalds { 2854930f036bSDavid Rientjes const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; 2855121e6f32SNicholas Piggin unsigned long addr = (unsigned long)area->addr; 2856121e6f32SNicholas Piggin unsigned long size = get_vm_area_size(area); 285734fe6537SAndrew Morton unsigned long array_size; 2858121e6f32SNicholas Piggin unsigned int nr_small_pages = size >> PAGE_SHIFT; 2859121e6f32SNicholas Piggin unsigned int page_order; 28601da177e4SLinus Torvalds 2861121e6f32SNicholas Piggin array_size = (unsigned long)nr_small_pages * sizeof(struct page *); 2862f255935bSChristoph Hellwig gfp_mask |= __GFP_NOWARN; 2863f255935bSChristoph Hellwig if (!(gfp_mask & (GFP_DMA | GFP_DMA32))) 2864f255935bSChristoph Hellwig gfp_mask |= __GFP_HIGHMEM; 28651da177e4SLinus Torvalds 28661da177e4SLinus Torvalds /* Please note that the recursion is strictly bounded. */ 28678757d5faSJan Kiszka if (array_size > PAGE_SIZE) { 28685c1f4e69SUladzislau Rezki (Sony) area->pages = __vmalloc_node(array_size, 1, nested_gfp, node, 2869f255935bSChristoph Hellwig area->caller); 2870286e1ea3SAndrew Morton } else { 28715c1f4e69SUladzislau Rezki (Sony) area->pages = kmalloc_node(array_size, nested_gfp, node); 2872286e1ea3SAndrew Morton } 28737ea36242SAustin Kim 28745c1f4e69SUladzislau Rezki (Sony) if (!area->pages) { 2875d70bec8cSNicholas Piggin warn_alloc(gfp_mask, NULL, 2876f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, failed to allocated page array size %lu", 2877d70bec8cSNicholas Piggin nr_small_pages * PAGE_SIZE, array_size); 2878cd61413bSUladzislau Rezki (Sony) free_vm_area(area); 28791da177e4SLinus Torvalds return NULL; 28801da177e4SLinus Torvalds } 28811da177e4SLinus Torvalds 2882121e6f32SNicholas Piggin set_vm_area_page_order(area, page_shift - PAGE_SHIFT); 2883121e6f32SNicholas Piggin page_order = vm_area_page_order(area); 2884121e6f32SNicholas Piggin 288512b9f873SUladzislau Rezki area->nr_pages = vm_area_alloc_pages(gfp_mask, node, 288612b9f873SUladzislau Rezki page_order, nr_small_pages, area->pages); 28875c1f4e69SUladzislau Rezki (Sony) 288897105f0aSRoman Gushchin atomic_long_add(area->nr_pages, &nr_vmalloc_pages); 28895c1f4e69SUladzislau Rezki (Sony) 28905c1f4e69SUladzislau Rezki (Sony) /* 28915c1f4e69SUladzislau Rezki (Sony) * If not enough pages were obtained to accomplish an 28925c1f4e69SUladzislau Rezki (Sony) * allocation request, free them via __vfree() if any. 28935c1f4e69SUladzislau Rezki (Sony) */ 28945c1f4e69SUladzislau Rezki (Sony) if (area->nr_pages != nr_small_pages) { 2895d70bec8cSNicholas Piggin warn_alloc(gfp_mask, NULL, 2896f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, page order %u, failed to allocate pages", 2897d70bec8cSNicholas Piggin area->nr_pages * PAGE_SIZE, page_order); 28981da177e4SLinus Torvalds goto fail; 28991da177e4SLinus Torvalds } 2900121e6f32SNicholas Piggin 290112b9f873SUladzislau Rezki if (vmap_pages_range(addr, addr + size, prot, area->pages, 290212b9f873SUladzislau Rezki page_shift) < 0) { 2903d70bec8cSNicholas Piggin warn_alloc(gfp_mask, NULL, 2904f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, failed to map pages", 2905d70bec8cSNicholas Piggin area->nr_pages * PAGE_SIZE); 29061da177e4SLinus Torvalds goto fail; 2907d70bec8cSNicholas Piggin } 2908ed1f324cSChristoph Hellwig 29091da177e4SLinus Torvalds return area->addr; 29101da177e4SLinus Torvalds 29111da177e4SLinus Torvalds fail: 2912c67dc624SRoman Penyaev __vfree(area->addr); 29131da177e4SLinus Torvalds return NULL; 29141da177e4SLinus Torvalds } 29151da177e4SLinus Torvalds 2916d0a21265SDavid Rientjes /** 2917d0a21265SDavid Rientjes * __vmalloc_node_range - allocate virtually contiguous memory 2918d0a21265SDavid Rientjes * @size: allocation size 2919d0a21265SDavid Rientjes * @align: desired alignment 2920d0a21265SDavid Rientjes * @start: vm area range start 2921d0a21265SDavid Rientjes * @end: vm area range end 2922d0a21265SDavid Rientjes * @gfp_mask: flags for the page level allocator 2923d0a21265SDavid Rientjes * @prot: protection mask for the allocated pages 2924cb9e3c29SAndrey Ryabinin * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) 292500ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 2926d0a21265SDavid Rientjes * @caller: caller's return address 2927d0a21265SDavid Rientjes * 2928d0a21265SDavid Rientjes * Allocate enough pages to cover @size from the page level 2929d0a21265SDavid Rientjes * allocator with @gfp_mask flags. Map them into contiguous 2930d0a21265SDavid Rientjes * kernel virtual space, using a pagetable protection of @prot. 2931a862f68aSMike Rapoport * 2932a862f68aSMike Rapoport * Return: the address of the area or %NULL on failure 2933d0a21265SDavid Rientjes */ 2934d0a21265SDavid Rientjes void *__vmalloc_node_range(unsigned long size, unsigned long align, 2935d0a21265SDavid Rientjes unsigned long start, unsigned long end, gfp_t gfp_mask, 2936cb9e3c29SAndrey Ryabinin pgprot_t prot, unsigned long vm_flags, int node, 2937cb9e3c29SAndrey Ryabinin const void *caller) 2938930fc45aSChristoph Lameter { 2939d0a21265SDavid Rientjes struct vm_struct *area; 2940d0a21265SDavid Rientjes void *addr; 2941d0a21265SDavid Rientjes unsigned long real_size = size; 2942121e6f32SNicholas Piggin unsigned long real_align = align; 2943121e6f32SNicholas Piggin unsigned int shift = PAGE_SHIFT; 2944d0a21265SDavid Rientjes 2945d70bec8cSNicholas Piggin if (WARN_ON_ONCE(!size)) 2946d70bec8cSNicholas Piggin return NULL; 2947d70bec8cSNicholas Piggin 2948d70bec8cSNicholas Piggin if ((size >> PAGE_SHIFT) > totalram_pages()) { 2949d70bec8cSNicholas Piggin warn_alloc(gfp_mask, NULL, 2950f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, exceeds total pages", 2951f4bdfeafSUladzislau Rezki (Sony) real_size); 2952d70bec8cSNicholas Piggin return NULL; 2953121e6f32SNicholas Piggin } 2954d0a21265SDavid Rientjes 29553382bbeeSChristophe Leroy if (vmap_allow_huge && !(vm_flags & VM_NO_HUGE_VMAP)) { 2956121e6f32SNicholas Piggin unsigned long size_per_node; 2957121e6f32SNicholas Piggin 2958121e6f32SNicholas Piggin /* 2959121e6f32SNicholas Piggin * Try huge pages. Only try for PAGE_KERNEL allocations, 2960121e6f32SNicholas Piggin * others like modules don't yet expect huge pages in 2961121e6f32SNicholas Piggin * their allocations due to apply_to_page_range not 2962121e6f32SNicholas Piggin * supporting them. 2963121e6f32SNicholas Piggin */ 2964121e6f32SNicholas Piggin 2965121e6f32SNicholas Piggin size_per_node = size; 2966121e6f32SNicholas Piggin if (node == NUMA_NO_NODE) 2967121e6f32SNicholas Piggin size_per_node /= num_online_nodes(); 29683382bbeeSChristophe Leroy if (arch_vmap_pmd_supported(prot) && size_per_node >= PMD_SIZE) 2969121e6f32SNicholas Piggin shift = PMD_SHIFT; 29703382bbeeSChristophe Leroy else 29713382bbeeSChristophe Leroy shift = arch_vmap_pte_supported_shift(size_per_node); 29723382bbeeSChristophe Leroy 2973121e6f32SNicholas Piggin align = max(real_align, 1UL << shift); 2974121e6f32SNicholas Piggin size = ALIGN(real_size, 1UL << shift); 2975121e6f32SNicholas Piggin } 2976121e6f32SNicholas Piggin 2977121e6f32SNicholas Piggin again: 29787ca3027bSDaniel Axtens area = __get_vm_area_node(real_size, align, shift, VM_ALLOC | 29797ca3027bSDaniel Axtens VM_UNINITIALIZED | vm_flags, start, end, node, 29807ca3027bSDaniel Axtens gfp_mask, caller); 2981d70bec8cSNicholas Piggin if (!area) { 2982d70bec8cSNicholas Piggin warn_alloc(gfp_mask, NULL, 2983f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, vm_struct allocation failed", 2984f4bdfeafSUladzislau Rezki (Sony) real_size); 2985de7d2b56SJoe Perches goto fail; 2986d70bec8cSNicholas Piggin } 2987d0a21265SDavid Rientjes 2988121e6f32SNicholas Piggin addr = __vmalloc_area_node(area, gfp_mask, prot, shift, node); 29891368edf0SMel Gorman if (!addr) 2990121e6f32SNicholas Piggin goto fail; 299189219d37SCatalin Marinas 299289219d37SCatalin Marinas /* 299320fc02b4SZhang Yanfei * In this function, newly allocated vm_struct has VM_UNINITIALIZED 299420fc02b4SZhang Yanfei * flag. It means that vm_struct is not fully initialized. 29954341fa45SJoonsoo Kim * Now, it is fully initialized, so remove this flag here. 2996f5252e00SMitsuo Hayasaka */ 299720fc02b4SZhang Yanfei clear_vm_uninitialized_flag(area); 2998f5252e00SMitsuo Hayasaka 29997ca3027bSDaniel Axtens size = PAGE_ALIGN(size); 300094f4a161SCatalin Marinas kmemleak_vmalloc(area, size, gfp_mask); 300189219d37SCatalin Marinas 300289219d37SCatalin Marinas return addr; 3003de7d2b56SJoe Perches 3004de7d2b56SJoe Perches fail: 3005121e6f32SNicholas Piggin if (shift > PAGE_SHIFT) { 3006121e6f32SNicholas Piggin shift = PAGE_SHIFT; 3007121e6f32SNicholas Piggin align = real_align; 3008121e6f32SNicholas Piggin size = real_size; 3009121e6f32SNicholas Piggin goto again; 3010121e6f32SNicholas Piggin } 3011121e6f32SNicholas Piggin 3012de7d2b56SJoe Perches return NULL; 3013930fc45aSChristoph Lameter } 3014930fc45aSChristoph Lameter 30151da177e4SLinus Torvalds /** 3016930fc45aSChristoph Lameter * __vmalloc_node - allocate virtually contiguous memory 30171da177e4SLinus Torvalds * @size: allocation size 30182dca6999SDavid Miller * @align: desired alignment 30191da177e4SLinus Torvalds * @gfp_mask: flags for the page level allocator 302000ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 3021c85d194bSRandy Dunlap * @caller: caller's return address 30221da177e4SLinus Torvalds * 3023f38fcb9cSChristoph Hellwig * Allocate enough pages to cover @size from the page level allocator with 3024f38fcb9cSChristoph Hellwig * @gfp_mask flags. Map them into contiguous kernel virtual space. 3025a7c3e901SMichal Hocko * 3026dcda9b04SMichal Hocko * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL 3027a7c3e901SMichal Hocko * and __GFP_NOFAIL are not supported 3028a7c3e901SMichal Hocko * 3029a7c3e901SMichal Hocko * Any use of gfp flags outside of GFP_KERNEL should be consulted 3030a7c3e901SMichal Hocko * with mm people. 3031a862f68aSMike Rapoport * 3032a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 30331da177e4SLinus Torvalds */ 30342b905948SChristoph Hellwig void *__vmalloc_node(unsigned long size, unsigned long align, 3035f38fcb9cSChristoph Hellwig gfp_t gfp_mask, int node, const void *caller) 30361da177e4SLinus Torvalds { 3037d0a21265SDavid Rientjes return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 3038f38fcb9cSChristoph Hellwig gfp_mask, PAGE_KERNEL, 0, node, caller); 30391da177e4SLinus Torvalds } 3040c3f896dcSChristoph Hellwig /* 3041c3f896dcSChristoph Hellwig * This is only for performance analysis of vmalloc and stress purpose. 3042c3f896dcSChristoph Hellwig * It is required by vmalloc test module, therefore do not use it other 3043c3f896dcSChristoph Hellwig * than that. 3044c3f896dcSChristoph Hellwig */ 3045c3f896dcSChristoph Hellwig #ifdef CONFIG_TEST_VMALLOC_MODULE 3046c3f896dcSChristoph Hellwig EXPORT_SYMBOL_GPL(__vmalloc_node); 3047c3f896dcSChristoph Hellwig #endif 30481da177e4SLinus Torvalds 304988dca4caSChristoph Hellwig void *__vmalloc(unsigned long size, gfp_t gfp_mask) 3050930fc45aSChristoph Lameter { 3051f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, gfp_mask, NUMA_NO_NODE, 305223016969SChristoph Lameter __builtin_return_address(0)); 3053930fc45aSChristoph Lameter } 30541da177e4SLinus Torvalds EXPORT_SYMBOL(__vmalloc); 30551da177e4SLinus Torvalds 30561da177e4SLinus Torvalds /** 30571da177e4SLinus Torvalds * vmalloc - allocate virtually contiguous memory 30581da177e4SLinus Torvalds * @size: allocation size 305992eac168SMike Rapoport * 30601da177e4SLinus Torvalds * Allocate enough pages to cover @size from the page level 30611da177e4SLinus Torvalds * allocator and map them into contiguous kernel virtual space. 30621da177e4SLinus Torvalds * 3063c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 30641da177e4SLinus Torvalds * use __vmalloc() instead. 3065a862f68aSMike Rapoport * 3066a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 30671da177e4SLinus Torvalds */ 30681da177e4SLinus Torvalds void *vmalloc(unsigned long size) 30691da177e4SLinus Torvalds { 30704d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL, NUMA_NO_NODE, 30714d39d728SChristoph Hellwig __builtin_return_address(0)); 30721da177e4SLinus Torvalds } 30731da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc); 30741da177e4SLinus Torvalds 3075930fc45aSChristoph Lameter /** 307615a64f5aSClaudio Imbrenda * vmalloc_no_huge - allocate virtually contiguous memory using small pages 307715a64f5aSClaudio Imbrenda * @size: allocation size 307815a64f5aSClaudio Imbrenda * 307915a64f5aSClaudio Imbrenda * Allocate enough non-huge pages to cover @size from the page level 308015a64f5aSClaudio Imbrenda * allocator and map them into contiguous kernel virtual space. 308115a64f5aSClaudio Imbrenda * 308215a64f5aSClaudio Imbrenda * Return: pointer to the allocated memory or %NULL on error 308315a64f5aSClaudio Imbrenda */ 308415a64f5aSClaudio Imbrenda void *vmalloc_no_huge(unsigned long size) 308515a64f5aSClaudio Imbrenda { 308615a64f5aSClaudio Imbrenda return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, 308715a64f5aSClaudio Imbrenda GFP_KERNEL, PAGE_KERNEL, VM_NO_HUGE_VMAP, 308815a64f5aSClaudio Imbrenda NUMA_NO_NODE, __builtin_return_address(0)); 308915a64f5aSClaudio Imbrenda } 309015a64f5aSClaudio Imbrenda EXPORT_SYMBOL(vmalloc_no_huge); 309115a64f5aSClaudio Imbrenda 309215a64f5aSClaudio Imbrenda /** 3093e1ca7788SDave Young * vzalloc - allocate virtually contiguous memory with zero fill 3094e1ca7788SDave Young * @size: allocation size 309592eac168SMike Rapoport * 3096e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 3097e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 3098e1ca7788SDave Young * The memory allocated is set to zero. 3099e1ca7788SDave Young * 3100e1ca7788SDave Young * For tight control over page level allocator and protection flags 3101e1ca7788SDave Young * use __vmalloc() instead. 3102a862f68aSMike Rapoport * 3103a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 3104e1ca7788SDave Young */ 3105e1ca7788SDave Young void *vzalloc(unsigned long size) 3106e1ca7788SDave Young { 31074d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, 31084d39d728SChristoph Hellwig __builtin_return_address(0)); 3109e1ca7788SDave Young } 3110e1ca7788SDave Young EXPORT_SYMBOL(vzalloc); 3111e1ca7788SDave Young 3112e1ca7788SDave Young /** 3113ead04089SRolf Eike Beer * vmalloc_user - allocate zeroed virtually contiguous memory for userspace 311483342314SNick Piggin * @size: allocation size 3115ead04089SRolf Eike Beer * 3116ead04089SRolf Eike Beer * The resulting memory area is zeroed so it can be mapped to userspace 3117ead04089SRolf Eike Beer * without leaking data. 3118a862f68aSMike Rapoport * 3119a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 312083342314SNick Piggin */ 312183342314SNick Piggin void *vmalloc_user(unsigned long size) 312283342314SNick Piggin { 3123bc84c535SRoman Penyaev return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, 3124bc84c535SRoman Penyaev GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, 3125bc84c535SRoman Penyaev VM_USERMAP, NUMA_NO_NODE, 312600ef2d2fSDavid Rientjes __builtin_return_address(0)); 312783342314SNick Piggin } 312883342314SNick Piggin EXPORT_SYMBOL(vmalloc_user); 312983342314SNick Piggin 313083342314SNick Piggin /** 3131930fc45aSChristoph Lameter * vmalloc_node - allocate memory on a specific node 3132930fc45aSChristoph Lameter * @size: allocation size 3133d44e0780SRandy Dunlap * @node: numa node 3134930fc45aSChristoph Lameter * 3135930fc45aSChristoph Lameter * Allocate enough pages to cover @size from the page level 3136930fc45aSChristoph Lameter * allocator and map them into contiguous kernel virtual space. 3137930fc45aSChristoph Lameter * 3138c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 3139930fc45aSChristoph Lameter * use __vmalloc() instead. 3140a862f68aSMike Rapoport * 3141a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 3142930fc45aSChristoph Lameter */ 3143930fc45aSChristoph Lameter void *vmalloc_node(unsigned long size, int node) 3144930fc45aSChristoph Lameter { 3145f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL, node, 3146f38fcb9cSChristoph Hellwig __builtin_return_address(0)); 3147930fc45aSChristoph Lameter } 3148930fc45aSChristoph Lameter EXPORT_SYMBOL(vmalloc_node); 3149930fc45aSChristoph Lameter 3150e1ca7788SDave Young /** 3151e1ca7788SDave Young * vzalloc_node - allocate memory on a specific node with zero fill 3152e1ca7788SDave Young * @size: allocation size 3153e1ca7788SDave Young * @node: numa node 3154e1ca7788SDave Young * 3155e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 3156e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 3157e1ca7788SDave Young * The memory allocated is set to zero. 3158e1ca7788SDave Young * 3159a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 3160e1ca7788SDave Young */ 3161e1ca7788SDave Young void *vzalloc_node(unsigned long size, int node) 3162e1ca7788SDave Young { 31634d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, node, 31644d39d728SChristoph Hellwig __builtin_return_address(0)); 3165e1ca7788SDave Young } 3166e1ca7788SDave Young EXPORT_SYMBOL(vzalloc_node); 3167e1ca7788SDave Young 31680d08e0d3SAndi Kleen #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) 3169698d0831SMichal Hocko #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) 31700d08e0d3SAndi Kleen #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) 3171698d0831SMichal Hocko #define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL) 31720d08e0d3SAndi Kleen #else 3173698d0831SMichal Hocko /* 3174698d0831SMichal Hocko * 64b systems should always have either DMA or DMA32 zones. For others 3175698d0831SMichal Hocko * GFP_DMA32 should do the right thing and use the normal zone. 3176698d0831SMichal Hocko */ 317768d68ff6SZhiyuan Dai #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) 31780d08e0d3SAndi Kleen #endif 31790d08e0d3SAndi Kleen 31801da177e4SLinus Torvalds /** 31811da177e4SLinus Torvalds * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 31821da177e4SLinus Torvalds * @size: allocation size 31831da177e4SLinus Torvalds * 31841da177e4SLinus Torvalds * Allocate enough 32bit PA addressable pages to cover @size from the 31851da177e4SLinus Torvalds * page level allocator and map them into contiguous kernel virtual space. 3186a862f68aSMike Rapoport * 3187a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 31881da177e4SLinus Torvalds */ 31891da177e4SLinus Torvalds void *vmalloc_32(unsigned long size) 31901da177e4SLinus Torvalds { 3191f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, GFP_VMALLOC32, NUMA_NO_NODE, 3192f38fcb9cSChristoph Hellwig __builtin_return_address(0)); 31931da177e4SLinus Torvalds } 31941da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc_32); 31951da177e4SLinus Torvalds 319683342314SNick Piggin /** 3197ead04089SRolf Eike Beer * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 319883342314SNick Piggin * @size: allocation size 3199ead04089SRolf Eike Beer * 3200ead04089SRolf Eike Beer * The resulting memory area is 32bit addressable and zeroed so it can be 3201ead04089SRolf Eike Beer * mapped to userspace without leaking data. 3202a862f68aSMike Rapoport * 3203a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 320483342314SNick Piggin */ 320583342314SNick Piggin void *vmalloc_32_user(unsigned long size) 320683342314SNick Piggin { 3207bc84c535SRoman Penyaev return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, 3208bc84c535SRoman Penyaev GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, 3209bc84c535SRoman Penyaev VM_USERMAP, NUMA_NO_NODE, 32105a82ac71SRoman Penyaev __builtin_return_address(0)); 321183342314SNick Piggin } 321283342314SNick Piggin EXPORT_SYMBOL(vmalloc_32_user); 321383342314SNick Piggin 3214d0107eb0SKAMEZAWA Hiroyuki /* 3215d0107eb0SKAMEZAWA Hiroyuki * small helper routine , copy contents to buf from addr. 3216d0107eb0SKAMEZAWA Hiroyuki * If the page is not present, fill zero. 3217d0107eb0SKAMEZAWA Hiroyuki */ 3218d0107eb0SKAMEZAWA Hiroyuki 3219d0107eb0SKAMEZAWA Hiroyuki static int aligned_vread(char *buf, char *addr, unsigned long count) 3220d0107eb0SKAMEZAWA Hiroyuki { 3221d0107eb0SKAMEZAWA Hiroyuki struct page *p; 3222d0107eb0SKAMEZAWA Hiroyuki int copied = 0; 3223d0107eb0SKAMEZAWA Hiroyuki 3224d0107eb0SKAMEZAWA Hiroyuki while (count) { 3225d0107eb0SKAMEZAWA Hiroyuki unsigned long offset, length; 3226d0107eb0SKAMEZAWA Hiroyuki 3227891c49abSAlexander Kuleshov offset = offset_in_page(addr); 3228d0107eb0SKAMEZAWA Hiroyuki length = PAGE_SIZE - offset; 3229d0107eb0SKAMEZAWA Hiroyuki if (length > count) 3230d0107eb0SKAMEZAWA Hiroyuki length = count; 3231d0107eb0SKAMEZAWA Hiroyuki p = vmalloc_to_page(addr); 3232d0107eb0SKAMEZAWA Hiroyuki /* 3233d0107eb0SKAMEZAWA Hiroyuki * To do safe access to this _mapped_ area, we need 3234d0107eb0SKAMEZAWA Hiroyuki * lock. But adding lock here means that we need to add 3235f0953a1bSIngo Molnar * overhead of vmalloc()/vfree() calls for this _debug_ 3236d0107eb0SKAMEZAWA Hiroyuki * interface, rarely used. Instead of that, we'll use 3237d0107eb0SKAMEZAWA Hiroyuki * kmap() and get small overhead in this access function. 3238d0107eb0SKAMEZAWA Hiroyuki */ 3239d0107eb0SKAMEZAWA Hiroyuki if (p) { 3240f7c8ce44SDavid Hildenbrand /* We can expect USER0 is not used -- see vread() */ 32419b04c5feSCong Wang void *map = kmap_atomic(p); 3242d0107eb0SKAMEZAWA Hiroyuki memcpy(buf, map + offset, length); 32439b04c5feSCong Wang kunmap_atomic(map); 3244d0107eb0SKAMEZAWA Hiroyuki } else 3245d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, length); 3246d0107eb0SKAMEZAWA Hiroyuki 3247d0107eb0SKAMEZAWA Hiroyuki addr += length; 3248d0107eb0SKAMEZAWA Hiroyuki buf += length; 3249d0107eb0SKAMEZAWA Hiroyuki copied += length; 3250d0107eb0SKAMEZAWA Hiroyuki count -= length; 3251d0107eb0SKAMEZAWA Hiroyuki } 3252d0107eb0SKAMEZAWA Hiroyuki return copied; 3253d0107eb0SKAMEZAWA Hiroyuki } 3254d0107eb0SKAMEZAWA Hiroyuki 3255d0107eb0SKAMEZAWA Hiroyuki /** 3256d0107eb0SKAMEZAWA Hiroyuki * vread() - read vmalloc area in a safe way. 3257d0107eb0SKAMEZAWA Hiroyuki * @buf: buffer for reading data 3258d0107eb0SKAMEZAWA Hiroyuki * @addr: vm address. 3259d0107eb0SKAMEZAWA Hiroyuki * @count: number of bytes to be read. 3260d0107eb0SKAMEZAWA Hiroyuki * 3261d0107eb0SKAMEZAWA Hiroyuki * This function checks that addr is a valid vmalloc'ed area, and 3262d0107eb0SKAMEZAWA Hiroyuki * copy data from that area to a given buffer. If the given memory range 3263d0107eb0SKAMEZAWA Hiroyuki * of [addr...addr+count) includes some valid address, data is copied to 3264d0107eb0SKAMEZAWA Hiroyuki * proper area of @buf. If there are memory holes, they'll be zero-filled. 3265d0107eb0SKAMEZAWA Hiroyuki * IOREMAP area is treated as memory hole and no copy is done. 3266d0107eb0SKAMEZAWA Hiroyuki * 3267d0107eb0SKAMEZAWA Hiroyuki * If [addr...addr+count) doesn't includes any intersects with alive 3268a8e5202dSCong Wang * vm_struct area, returns 0. @buf should be kernel's buffer. 3269d0107eb0SKAMEZAWA Hiroyuki * 3270d0107eb0SKAMEZAWA Hiroyuki * Note: In usual ops, vread() is never necessary because the caller 3271d0107eb0SKAMEZAWA Hiroyuki * should know vmalloc() area is valid and can use memcpy(). 3272d0107eb0SKAMEZAWA Hiroyuki * This is for routines which have to access vmalloc area without 3273bbcd53c9SDavid Hildenbrand * any information, as /proc/kcore. 3274a862f68aSMike Rapoport * 3275a862f68aSMike Rapoport * Return: number of bytes for which addr and buf should be increased 3276a862f68aSMike Rapoport * (same number as @count) or %0 if [addr...addr+count) doesn't 3277a862f68aSMike Rapoport * include any intersection with valid vmalloc area 3278d0107eb0SKAMEZAWA Hiroyuki */ 32791da177e4SLinus Torvalds long vread(char *buf, char *addr, unsigned long count) 32801da177e4SLinus Torvalds { 3281e81ce85fSJoonsoo Kim struct vmap_area *va; 3282e81ce85fSJoonsoo Kim struct vm_struct *vm; 32831da177e4SLinus Torvalds char *vaddr, *buf_start = buf; 3284d0107eb0SKAMEZAWA Hiroyuki unsigned long buflen = count; 32851da177e4SLinus Torvalds unsigned long n; 32861da177e4SLinus Torvalds 32871da177e4SLinus Torvalds /* Don't allow overflow */ 32881da177e4SLinus Torvalds if ((unsigned long) addr + count < count) 32891da177e4SLinus Torvalds count = -(unsigned long) addr; 32901da177e4SLinus Torvalds 3291e81ce85fSJoonsoo Kim spin_lock(&vmap_area_lock); 3292f608788cSSerapheim Dimitropoulos va = __find_vmap_area((unsigned long)addr); 3293f608788cSSerapheim Dimitropoulos if (!va) 3294f608788cSSerapheim Dimitropoulos goto finished; 3295f608788cSSerapheim Dimitropoulos list_for_each_entry_from(va, &vmap_area_list, list) { 3296e81ce85fSJoonsoo Kim if (!count) 3297e81ce85fSJoonsoo Kim break; 3298e81ce85fSJoonsoo Kim 3299688fcbfcSPengfei Li if (!va->vm) 3300e81ce85fSJoonsoo Kim continue; 3301e81ce85fSJoonsoo Kim 3302e81ce85fSJoonsoo Kim vm = va->vm; 3303e81ce85fSJoonsoo Kim vaddr = (char *) vm->addr; 3304762216abSWanpeng Li if (addr >= vaddr + get_vm_area_size(vm)) 33051da177e4SLinus Torvalds continue; 33061da177e4SLinus Torvalds while (addr < vaddr) { 33071da177e4SLinus Torvalds if (count == 0) 33081da177e4SLinus Torvalds goto finished; 33091da177e4SLinus Torvalds *buf = '\0'; 33101da177e4SLinus Torvalds buf++; 33111da177e4SLinus Torvalds addr++; 33121da177e4SLinus Torvalds count--; 33131da177e4SLinus Torvalds } 3314762216abSWanpeng Li n = vaddr + get_vm_area_size(vm) - addr; 3315d0107eb0SKAMEZAWA Hiroyuki if (n > count) 3316d0107eb0SKAMEZAWA Hiroyuki n = count; 3317e81ce85fSJoonsoo Kim if (!(vm->flags & VM_IOREMAP)) 3318d0107eb0SKAMEZAWA Hiroyuki aligned_vread(buf, addr, n); 3319d0107eb0SKAMEZAWA Hiroyuki else /* IOREMAP area is treated as memory hole */ 3320d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, n); 3321d0107eb0SKAMEZAWA Hiroyuki buf += n; 3322d0107eb0SKAMEZAWA Hiroyuki addr += n; 3323d0107eb0SKAMEZAWA Hiroyuki count -= n; 33241da177e4SLinus Torvalds } 33251da177e4SLinus Torvalds finished: 3326e81ce85fSJoonsoo Kim spin_unlock(&vmap_area_lock); 3327d0107eb0SKAMEZAWA Hiroyuki 3328d0107eb0SKAMEZAWA Hiroyuki if (buf == buf_start) 3329d0107eb0SKAMEZAWA Hiroyuki return 0; 3330d0107eb0SKAMEZAWA Hiroyuki /* zero-fill memory holes */ 3331d0107eb0SKAMEZAWA Hiroyuki if (buf != buf_start + buflen) 3332d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, buflen - (buf - buf_start)); 3333d0107eb0SKAMEZAWA Hiroyuki 3334d0107eb0SKAMEZAWA Hiroyuki return buflen; 33351da177e4SLinus Torvalds } 33361da177e4SLinus Torvalds 3337d0107eb0SKAMEZAWA Hiroyuki /** 3338e69e9d4aSHATAYAMA Daisuke * remap_vmalloc_range_partial - map vmalloc pages to userspace 3339e69e9d4aSHATAYAMA Daisuke * @vma: vma to cover 3340e69e9d4aSHATAYAMA Daisuke * @uaddr: target user address to start at 3341e69e9d4aSHATAYAMA Daisuke * @kaddr: virtual address of vmalloc kernel memory 3342bdebd6a2SJann Horn * @pgoff: offset from @kaddr to start at 3343e69e9d4aSHATAYAMA Daisuke * @size: size of map area 3344e69e9d4aSHATAYAMA Daisuke * 3345e69e9d4aSHATAYAMA Daisuke * Returns: 0 for success, -Exxx on failure 3346e69e9d4aSHATAYAMA Daisuke * 3347e69e9d4aSHATAYAMA Daisuke * This function checks that @kaddr is a valid vmalloc'ed area, 3348e69e9d4aSHATAYAMA Daisuke * and that it is big enough to cover the range starting at 3349e69e9d4aSHATAYAMA Daisuke * @uaddr in @vma. Will return failure if that criteria isn't 3350e69e9d4aSHATAYAMA Daisuke * met. 3351e69e9d4aSHATAYAMA Daisuke * 3352e69e9d4aSHATAYAMA Daisuke * Similar to remap_pfn_range() (see mm/memory.c) 3353e69e9d4aSHATAYAMA Daisuke */ 3354e69e9d4aSHATAYAMA Daisuke int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr, 3355bdebd6a2SJann Horn void *kaddr, unsigned long pgoff, 3356bdebd6a2SJann Horn unsigned long size) 3357e69e9d4aSHATAYAMA Daisuke { 3358e69e9d4aSHATAYAMA Daisuke struct vm_struct *area; 3359bdebd6a2SJann Horn unsigned long off; 3360bdebd6a2SJann Horn unsigned long end_index; 3361bdebd6a2SJann Horn 3362bdebd6a2SJann Horn if (check_shl_overflow(pgoff, PAGE_SHIFT, &off)) 3363bdebd6a2SJann Horn return -EINVAL; 3364e69e9d4aSHATAYAMA Daisuke 3365e69e9d4aSHATAYAMA Daisuke size = PAGE_ALIGN(size); 3366e69e9d4aSHATAYAMA Daisuke 3367e69e9d4aSHATAYAMA Daisuke if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr)) 3368e69e9d4aSHATAYAMA Daisuke return -EINVAL; 3369e69e9d4aSHATAYAMA Daisuke 3370e69e9d4aSHATAYAMA Daisuke area = find_vm_area(kaddr); 3371e69e9d4aSHATAYAMA Daisuke if (!area) 3372e69e9d4aSHATAYAMA Daisuke return -EINVAL; 3373e69e9d4aSHATAYAMA Daisuke 3374fe9041c2SChristoph Hellwig if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT))) 3375e69e9d4aSHATAYAMA Daisuke return -EINVAL; 3376e69e9d4aSHATAYAMA Daisuke 3377bdebd6a2SJann Horn if (check_add_overflow(size, off, &end_index) || 3378bdebd6a2SJann Horn end_index > get_vm_area_size(area)) 3379e69e9d4aSHATAYAMA Daisuke return -EINVAL; 3380bdebd6a2SJann Horn kaddr += off; 3381e69e9d4aSHATAYAMA Daisuke 3382e69e9d4aSHATAYAMA Daisuke do { 3383e69e9d4aSHATAYAMA Daisuke struct page *page = vmalloc_to_page(kaddr); 3384e69e9d4aSHATAYAMA Daisuke int ret; 3385e69e9d4aSHATAYAMA Daisuke 3386e69e9d4aSHATAYAMA Daisuke ret = vm_insert_page(vma, uaddr, page); 3387e69e9d4aSHATAYAMA Daisuke if (ret) 3388e69e9d4aSHATAYAMA Daisuke return ret; 3389e69e9d4aSHATAYAMA Daisuke 3390e69e9d4aSHATAYAMA Daisuke uaddr += PAGE_SIZE; 3391e69e9d4aSHATAYAMA Daisuke kaddr += PAGE_SIZE; 3392e69e9d4aSHATAYAMA Daisuke size -= PAGE_SIZE; 3393e69e9d4aSHATAYAMA Daisuke } while (size > 0); 3394e69e9d4aSHATAYAMA Daisuke 3395e69e9d4aSHATAYAMA Daisuke vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; 3396e69e9d4aSHATAYAMA Daisuke 3397e69e9d4aSHATAYAMA Daisuke return 0; 3398e69e9d4aSHATAYAMA Daisuke } 3399e69e9d4aSHATAYAMA Daisuke 3400e69e9d4aSHATAYAMA Daisuke /** 340183342314SNick Piggin * remap_vmalloc_range - map vmalloc pages to userspace 340283342314SNick Piggin * @vma: vma to cover (map full range of vma) 340383342314SNick Piggin * @addr: vmalloc memory 340483342314SNick Piggin * @pgoff: number of pages into addr before first page to map 34057682486bSRandy Dunlap * 34067682486bSRandy Dunlap * Returns: 0 for success, -Exxx on failure 340783342314SNick Piggin * 340883342314SNick Piggin * This function checks that addr is a valid vmalloc'ed area, and 340983342314SNick Piggin * that it is big enough to cover the vma. Will return failure if 341083342314SNick Piggin * that criteria isn't met. 341183342314SNick Piggin * 341272fd4a35SRobert P. J. Day * Similar to remap_pfn_range() (see mm/memory.c) 341383342314SNick Piggin */ 341483342314SNick Piggin int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 341583342314SNick Piggin unsigned long pgoff) 341683342314SNick Piggin { 3417e69e9d4aSHATAYAMA Daisuke return remap_vmalloc_range_partial(vma, vma->vm_start, 3418bdebd6a2SJann Horn addr, pgoff, 3419e69e9d4aSHATAYAMA Daisuke vma->vm_end - vma->vm_start); 342083342314SNick Piggin } 342183342314SNick Piggin EXPORT_SYMBOL(remap_vmalloc_range); 342283342314SNick Piggin 34235f4352fbSJeremy Fitzhardinge void free_vm_area(struct vm_struct *area) 34245f4352fbSJeremy Fitzhardinge { 34255f4352fbSJeremy Fitzhardinge struct vm_struct *ret; 34265f4352fbSJeremy Fitzhardinge ret = remove_vm_area(area->addr); 34275f4352fbSJeremy Fitzhardinge BUG_ON(ret != area); 34285f4352fbSJeremy Fitzhardinge kfree(area); 34295f4352fbSJeremy Fitzhardinge } 34305f4352fbSJeremy Fitzhardinge EXPORT_SYMBOL_GPL(free_vm_area); 3431a10aa579SChristoph Lameter 34324f8b02b4STejun Heo #ifdef CONFIG_SMP 3433ca23e405STejun Heo static struct vmap_area *node_to_va(struct rb_node *n) 3434ca23e405STejun Heo { 34354583e773SGeliang Tang return rb_entry_safe(n, struct vmap_area, rb_node); 3436ca23e405STejun Heo } 3437ca23e405STejun Heo 3438ca23e405STejun Heo /** 343968ad4a33SUladzislau Rezki (Sony) * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to 344068ad4a33SUladzislau Rezki (Sony) * @addr: target address 3441ca23e405STejun Heo * 344268ad4a33SUladzislau Rezki (Sony) * Returns: vmap_area if it is found. If there is no such area 344368ad4a33SUladzislau Rezki (Sony) * the first highest(reverse order) vmap_area is returned 344468ad4a33SUladzislau Rezki (Sony) * i.e. va->va_start < addr && va->va_end < addr or NULL 344568ad4a33SUladzislau Rezki (Sony) * if there are no any areas before @addr. 3446ca23e405STejun Heo */ 344768ad4a33SUladzislau Rezki (Sony) static struct vmap_area * 344868ad4a33SUladzislau Rezki (Sony) pvm_find_va_enclose_addr(unsigned long addr) 3449ca23e405STejun Heo { 345068ad4a33SUladzislau Rezki (Sony) struct vmap_area *va, *tmp; 345168ad4a33SUladzislau Rezki (Sony) struct rb_node *n; 345268ad4a33SUladzislau Rezki (Sony) 345368ad4a33SUladzislau Rezki (Sony) n = free_vmap_area_root.rb_node; 345468ad4a33SUladzislau Rezki (Sony) va = NULL; 3455ca23e405STejun Heo 3456ca23e405STejun Heo while (n) { 345768ad4a33SUladzislau Rezki (Sony) tmp = rb_entry(n, struct vmap_area, rb_node); 345868ad4a33SUladzislau Rezki (Sony) if (tmp->va_start <= addr) { 345968ad4a33SUladzislau Rezki (Sony) va = tmp; 346068ad4a33SUladzislau Rezki (Sony) if (tmp->va_end >= addr) 3461ca23e405STejun Heo break; 3462ca23e405STejun Heo 346368ad4a33SUladzislau Rezki (Sony) n = n->rb_right; 3464ca23e405STejun Heo } else { 346568ad4a33SUladzislau Rezki (Sony) n = n->rb_left; 3466ca23e405STejun Heo } 346768ad4a33SUladzislau Rezki (Sony) } 346868ad4a33SUladzislau Rezki (Sony) 346968ad4a33SUladzislau Rezki (Sony) return va; 3470ca23e405STejun Heo } 3471ca23e405STejun Heo 3472ca23e405STejun Heo /** 347368ad4a33SUladzislau Rezki (Sony) * pvm_determine_end_from_reverse - find the highest aligned address 347468ad4a33SUladzislau Rezki (Sony) * of free block below VMALLOC_END 347568ad4a33SUladzislau Rezki (Sony) * @va: 347668ad4a33SUladzislau Rezki (Sony) * in - the VA we start the search(reverse order); 347768ad4a33SUladzislau Rezki (Sony) * out - the VA with the highest aligned end address. 3478799fa85dSAlex Shi * @align: alignment for required highest address 3479ca23e405STejun Heo * 348068ad4a33SUladzislau Rezki (Sony) * Returns: determined end address within vmap_area 3481ca23e405STejun Heo */ 348268ad4a33SUladzislau Rezki (Sony) static unsigned long 348368ad4a33SUladzislau Rezki (Sony) pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) 3484ca23e405STejun Heo { 348568ad4a33SUladzislau Rezki (Sony) unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 3486ca23e405STejun Heo unsigned long addr; 3487ca23e405STejun Heo 348868ad4a33SUladzislau Rezki (Sony) if (likely(*va)) { 348968ad4a33SUladzislau Rezki (Sony) list_for_each_entry_from_reverse((*va), 349068ad4a33SUladzislau Rezki (Sony) &free_vmap_area_list, list) { 349168ad4a33SUladzislau Rezki (Sony) addr = min((*va)->va_end & ~(align - 1), vmalloc_end); 349268ad4a33SUladzislau Rezki (Sony) if ((*va)->va_start < addr) 349368ad4a33SUladzislau Rezki (Sony) return addr; 349468ad4a33SUladzislau Rezki (Sony) } 3495ca23e405STejun Heo } 3496ca23e405STejun Heo 349768ad4a33SUladzislau Rezki (Sony) return 0; 3498ca23e405STejun Heo } 3499ca23e405STejun Heo 3500ca23e405STejun Heo /** 3501ca23e405STejun Heo * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator 3502ca23e405STejun Heo * @offsets: array containing offset of each area 3503ca23e405STejun Heo * @sizes: array containing size of each area 3504ca23e405STejun Heo * @nr_vms: the number of areas to allocate 3505ca23e405STejun Heo * @align: alignment, all entries in @offsets and @sizes must be aligned to this 3506ca23e405STejun Heo * 3507ca23e405STejun Heo * Returns: kmalloc'd vm_struct pointer array pointing to allocated 3508ca23e405STejun Heo * vm_structs on success, %NULL on failure 3509ca23e405STejun Heo * 3510ca23e405STejun Heo * Percpu allocator wants to use congruent vm areas so that it can 3511ca23e405STejun Heo * maintain the offsets among percpu areas. This function allocates 3512ec3f64fcSDavid Rientjes * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to 3513ec3f64fcSDavid Rientjes * be scattered pretty far, distance between two areas easily going up 3514ec3f64fcSDavid Rientjes * to gigabytes. To avoid interacting with regular vmallocs, these 3515ec3f64fcSDavid Rientjes * areas are allocated from top. 3516ca23e405STejun Heo * 3517ca23e405STejun Heo * Despite its complicated look, this allocator is rather simple. It 351868ad4a33SUladzislau Rezki (Sony) * does everything top-down and scans free blocks from the end looking 351968ad4a33SUladzislau Rezki (Sony) * for matching base. While scanning, if any of the areas do not fit the 352068ad4a33SUladzislau Rezki (Sony) * base address is pulled down to fit the area. Scanning is repeated till 352168ad4a33SUladzislau Rezki (Sony) * all the areas fit and then all necessary data structures are inserted 352268ad4a33SUladzislau Rezki (Sony) * and the result is returned. 3523ca23e405STejun Heo */ 3524ca23e405STejun Heo struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, 3525ca23e405STejun Heo const size_t *sizes, int nr_vms, 3526ec3f64fcSDavid Rientjes size_t align) 3527ca23e405STejun Heo { 3528ca23e405STejun Heo const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); 3529ca23e405STejun Heo const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 353068ad4a33SUladzislau Rezki (Sony) struct vmap_area **vas, *va; 3531ca23e405STejun Heo struct vm_struct **vms; 3532ca23e405STejun Heo int area, area2, last_area, term_area; 3533253a496dSDaniel Axtens unsigned long base, start, size, end, last_end, orig_start, orig_end; 3534ca23e405STejun Heo bool purged = false; 353568ad4a33SUladzislau Rezki (Sony) enum fit_type type; 3536ca23e405STejun Heo 3537ca23e405STejun Heo /* verify parameters and allocate data structures */ 3538891c49abSAlexander Kuleshov BUG_ON(offset_in_page(align) || !is_power_of_2(align)); 3539ca23e405STejun Heo for (last_area = 0, area = 0; area < nr_vms; area++) { 3540ca23e405STejun Heo start = offsets[area]; 3541ca23e405STejun Heo end = start + sizes[area]; 3542ca23e405STejun Heo 3543ca23e405STejun Heo /* is everything aligned properly? */ 3544ca23e405STejun Heo BUG_ON(!IS_ALIGNED(offsets[area], align)); 3545ca23e405STejun Heo BUG_ON(!IS_ALIGNED(sizes[area], align)); 3546ca23e405STejun Heo 3547ca23e405STejun Heo /* detect the area with the highest address */ 3548ca23e405STejun Heo if (start > offsets[last_area]) 3549ca23e405STejun Heo last_area = area; 3550ca23e405STejun Heo 3551c568da28SWei Yang for (area2 = area + 1; area2 < nr_vms; area2++) { 3552ca23e405STejun Heo unsigned long start2 = offsets[area2]; 3553ca23e405STejun Heo unsigned long end2 = start2 + sizes[area2]; 3554ca23e405STejun Heo 3555c568da28SWei Yang BUG_ON(start2 < end && start < end2); 3556ca23e405STejun Heo } 3557ca23e405STejun Heo } 3558ca23e405STejun Heo last_end = offsets[last_area] + sizes[last_area]; 3559ca23e405STejun Heo 3560ca23e405STejun Heo if (vmalloc_end - vmalloc_start < last_end) { 3561ca23e405STejun Heo WARN_ON(true); 3562ca23e405STejun Heo return NULL; 3563ca23e405STejun Heo } 3564ca23e405STejun Heo 35654d67d860SThomas Meyer vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL); 35664d67d860SThomas Meyer vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL); 3567ca23e405STejun Heo if (!vas || !vms) 3568f1db7afdSKautuk Consul goto err_free2; 3569ca23e405STejun Heo 3570ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 357168ad4a33SUladzislau Rezki (Sony) vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL); 3572ec3f64fcSDavid Rientjes vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL); 3573ca23e405STejun Heo if (!vas[area] || !vms[area]) 3574ca23e405STejun Heo goto err_free; 3575ca23e405STejun Heo } 3576ca23e405STejun Heo retry: 3577e36176beSUladzislau Rezki (Sony) spin_lock(&free_vmap_area_lock); 3578ca23e405STejun Heo 3579ca23e405STejun Heo /* start scanning - we scan from the top, begin with the last area */ 3580ca23e405STejun Heo area = term_area = last_area; 3581ca23e405STejun Heo start = offsets[area]; 3582ca23e405STejun Heo end = start + sizes[area]; 3583ca23e405STejun Heo 358468ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(vmalloc_end); 358568ad4a33SUladzislau Rezki (Sony) base = pvm_determine_end_from_reverse(&va, align) - end; 3586ca23e405STejun Heo 3587ca23e405STejun Heo while (true) { 3588ca23e405STejun Heo /* 3589ca23e405STejun Heo * base might have underflowed, add last_end before 3590ca23e405STejun Heo * comparing. 3591ca23e405STejun Heo */ 359268ad4a33SUladzislau Rezki (Sony) if (base + last_end < vmalloc_start + last_end) 359368ad4a33SUladzislau Rezki (Sony) goto overflow; 3594ca23e405STejun Heo 3595ca23e405STejun Heo /* 359668ad4a33SUladzislau Rezki (Sony) * Fitting base has not been found. 3597ca23e405STejun Heo */ 359868ad4a33SUladzislau Rezki (Sony) if (va == NULL) 359968ad4a33SUladzislau Rezki (Sony) goto overflow; 3600ca23e405STejun Heo 3601ca23e405STejun Heo /* 3602d8cc323dSQiujun Huang * If required width exceeds current VA block, move 36035336e52cSKuppuswamy Sathyanarayanan * base downwards and then recheck. 36045336e52cSKuppuswamy Sathyanarayanan */ 36055336e52cSKuppuswamy Sathyanarayanan if (base + end > va->va_end) { 36065336e52cSKuppuswamy Sathyanarayanan base = pvm_determine_end_from_reverse(&va, align) - end; 36075336e52cSKuppuswamy Sathyanarayanan term_area = area; 36085336e52cSKuppuswamy Sathyanarayanan continue; 36095336e52cSKuppuswamy Sathyanarayanan } 36105336e52cSKuppuswamy Sathyanarayanan 36115336e52cSKuppuswamy Sathyanarayanan /* 361268ad4a33SUladzislau Rezki (Sony) * If this VA does not fit, move base downwards and recheck. 3613ca23e405STejun Heo */ 36145336e52cSKuppuswamy Sathyanarayanan if (base + start < va->va_start) { 361568ad4a33SUladzislau Rezki (Sony) va = node_to_va(rb_prev(&va->rb_node)); 361668ad4a33SUladzislau Rezki (Sony) base = pvm_determine_end_from_reverse(&va, align) - end; 3617ca23e405STejun Heo term_area = area; 3618ca23e405STejun Heo continue; 3619ca23e405STejun Heo } 3620ca23e405STejun Heo 3621ca23e405STejun Heo /* 3622ca23e405STejun Heo * This area fits, move on to the previous one. If 3623ca23e405STejun Heo * the previous one is the terminal one, we're done. 3624ca23e405STejun Heo */ 3625ca23e405STejun Heo area = (area + nr_vms - 1) % nr_vms; 3626ca23e405STejun Heo if (area == term_area) 3627ca23e405STejun Heo break; 362868ad4a33SUladzislau Rezki (Sony) 3629ca23e405STejun Heo start = offsets[area]; 3630ca23e405STejun Heo end = start + sizes[area]; 363168ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(base + end); 3632ca23e405STejun Heo } 363368ad4a33SUladzislau Rezki (Sony) 3634ca23e405STejun Heo /* we've found a fitting base, insert all va's */ 3635ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 363668ad4a33SUladzislau Rezki (Sony) int ret; 3637ca23e405STejun Heo 363868ad4a33SUladzislau Rezki (Sony) start = base + offsets[area]; 363968ad4a33SUladzislau Rezki (Sony) size = sizes[area]; 364068ad4a33SUladzislau Rezki (Sony) 364168ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(start); 364268ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(va == NULL)) 364368ad4a33SUladzislau Rezki (Sony) /* It is a BUG(), but trigger recovery instead. */ 364468ad4a33SUladzislau Rezki (Sony) goto recovery; 364568ad4a33SUladzislau Rezki (Sony) 364668ad4a33SUladzislau Rezki (Sony) type = classify_va_fit_type(va, start, size); 364768ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(type == NOTHING_FIT)) 364868ad4a33SUladzislau Rezki (Sony) /* It is a BUG(), but trigger recovery instead. */ 364968ad4a33SUladzislau Rezki (Sony) goto recovery; 365068ad4a33SUladzislau Rezki (Sony) 365168ad4a33SUladzislau Rezki (Sony) ret = adjust_va_to_fit_type(va, start, size, type); 365268ad4a33SUladzislau Rezki (Sony) if (unlikely(ret)) 365368ad4a33SUladzislau Rezki (Sony) goto recovery; 365468ad4a33SUladzislau Rezki (Sony) 365568ad4a33SUladzislau Rezki (Sony) /* Allocated area. */ 365668ad4a33SUladzislau Rezki (Sony) va = vas[area]; 365768ad4a33SUladzislau Rezki (Sony) va->va_start = start; 365868ad4a33SUladzislau Rezki (Sony) va->va_end = start + size; 3659ca23e405STejun Heo } 3660ca23e405STejun Heo 3661e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 3662ca23e405STejun Heo 3663253a496dSDaniel Axtens /* populate the kasan shadow space */ 3664253a496dSDaniel Axtens for (area = 0; area < nr_vms; area++) { 3665253a496dSDaniel Axtens if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area])) 3666253a496dSDaniel Axtens goto err_free_shadow; 3667253a496dSDaniel Axtens 3668253a496dSDaniel Axtens kasan_unpoison_vmalloc((void *)vas[area]->va_start, 3669253a496dSDaniel Axtens sizes[area]); 3670253a496dSDaniel Axtens } 3671253a496dSDaniel Axtens 3672ca23e405STejun Heo /* insert all vm's */ 3673e36176beSUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 3674e36176beSUladzislau Rezki (Sony) for (area = 0; area < nr_vms; area++) { 3675e36176beSUladzislau Rezki (Sony) insert_vmap_area(vas[area], &vmap_area_root, &vmap_area_list); 3676e36176beSUladzislau Rezki (Sony) 3677e36176beSUladzislau Rezki (Sony) setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC, 3678ca23e405STejun Heo pcpu_get_vm_areas); 3679e36176beSUladzislau Rezki (Sony) } 3680e36176beSUladzislau Rezki (Sony) spin_unlock(&vmap_area_lock); 3681ca23e405STejun Heo 3682ca23e405STejun Heo kfree(vas); 3683ca23e405STejun Heo return vms; 3684ca23e405STejun Heo 368568ad4a33SUladzislau Rezki (Sony) recovery: 3686e36176beSUladzislau Rezki (Sony) /* 3687e36176beSUladzislau Rezki (Sony) * Remove previously allocated areas. There is no 3688e36176beSUladzislau Rezki (Sony) * need in removing these areas from the busy tree, 3689e36176beSUladzislau Rezki (Sony) * because they are inserted only on the final step 3690e36176beSUladzislau Rezki (Sony) * and when pcpu_get_vm_areas() is success. 3691e36176beSUladzislau Rezki (Sony) */ 369268ad4a33SUladzislau Rezki (Sony) while (area--) { 3693253a496dSDaniel Axtens orig_start = vas[area]->va_start; 3694253a496dSDaniel Axtens orig_end = vas[area]->va_end; 369596e2db45SUladzislau Rezki (Sony) va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, 36963c5c3cfbSDaniel Axtens &free_vmap_area_list); 36979c801f61SUladzislau Rezki (Sony) if (va) 3698253a496dSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 3699253a496dSDaniel Axtens va->va_start, va->va_end); 370068ad4a33SUladzislau Rezki (Sony) vas[area] = NULL; 370168ad4a33SUladzislau Rezki (Sony) } 370268ad4a33SUladzislau Rezki (Sony) 370368ad4a33SUladzislau Rezki (Sony) overflow: 3704e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 370568ad4a33SUladzislau Rezki (Sony) if (!purged) { 370668ad4a33SUladzislau Rezki (Sony) purge_vmap_area_lazy(); 370768ad4a33SUladzislau Rezki (Sony) purged = true; 370868ad4a33SUladzislau Rezki (Sony) 370968ad4a33SUladzislau Rezki (Sony) /* Before "retry", check if we recover. */ 371068ad4a33SUladzislau Rezki (Sony) for (area = 0; area < nr_vms; area++) { 371168ad4a33SUladzislau Rezki (Sony) if (vas[area]) 371268ad4a33SUladzislau Rezki (Sony) continue; 371368ad4a33SUladzislau Rezki (Sony) 371468ad4a33SUladzislau Rezki (Sony) vas[area] = kmem_cache_zalloc( 371568ad4a33SUladzislau Rezki (Sony) vmap_area_cachep, GFP_KERNEL); 371668ad4a33SUladzislau Rezki (Sony) if (!vas[area]) 371768ad4a33SUladzislau Rezki (Sony) goto err_free; 371868ad4a33SUladzislau Rezki (Sony) } 371968ad4a33SUladzislau Rezki (Sony) 372068ad4a33SUladzislau Rezki (Sony) goto retry; 372168ad4a33SUladzislau Rezki (Sony) } 372268ad4a33SUladzislau Rezki (Sony) 3723ca23e405STejun Heo err_free: 3724ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 372568ad4a33SUladzislau Rezki (Sony) if (vas[area]) 372668ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, vas[area]); 372768ad4a33SUladzislau Rezki (Sony) 3728ca23e405STejun Heo kfree(vms[area]); 3729ca23e405STejun Heo } 3730f1db7afdSKautuk Consul err_free2: 3731ca23e405STejun Heo kfree(vas); 3732ca23e405STejun Heo kfree(vms); 3733ca23e405STejun Heo return NULL; 3734253a496dSDaniel Axtens 3735253a496dSDaniel Axtens err_free_shadow: 3736253a496dSDaniel Axtens spin_lock(&free_vmap_area_lock); 3737253a496dSDaniel Axtens /* 3738253a496dSDaniel Axtens * We release all the vmalloc shadows, even the ones for regions that 3739253a496dSDaniel Axtens * hadn't been successfully added. This relies on kasan_release_vmalloc 3740253a496dSDaniel Axtens * being able to tolerate this case. 3741253a496dSDaniel Axtens */ 3742253a496dSDaniel Axtens for (area = 0; area < nr_vms; area++) { 3743253a496dSDaniel Axtens orig_start = vas[area]->va_start; 3744253a496dSDaniel Axtens orig_end = vas[area]->va_end; 374596e2db45SUladzislau Rezki (Sony) va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, 3746253a496dSDaniel Axtens &free_vmap_area_list); 37479c801f61SUladzislau Rezki (Sony) if (va) 3748253a496dSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 3749253a496dSDaniel Axtens va->va_start, va->va_end); 3750253a496dSDaniel Axtens vas[area] = NULL; 3751253a496dSDaniel Axtens kfree(vms[area]); 3752253a496dSDaniel Axtens } 3753253a496dSDaniel Axtens spin_unlock(&free_vmap_area_lock); 3754253a496dSDaniel Axtens kfree(vas); 3755253a496dSDaniel Axtens kfree(vms); 3756253a496dSDaniel Axtens return NULL; 3757ca23e405STejun Heo } 3758ca23e405STejun Heo 3759ca23e405STejun Heo /** 3760ca23e405STejun Heo * pcpu_free_vm_areas - free vmalloc areas for percpu allocator 3761ca23e405STejun Heo * @vms: vm_struct pointer array returned by pcpu_get_vm_areas() 3762ca23e405STejun Heo * @nr_vms: the number of allocated areas 3763ca23e405STejun Heo * 3764ca23e405STejun Heo * Free vm_structs and the array allocated by pcpu_get_vm_areas(). 3765ca23e405STejun Heo */ 3766ca23e405STejun Heo void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) 3767ca23e405STejun Heo { 3768ca23e405STejun Heo int i; 3769ca23e405STejun Heo 3770ca23e405STejun Heo for (i = 0; i < nr_vms; i++) 3771ca23e405STejun Heo free_vm_area(vms[i]); 3772ca23e405STejun Heo kfree(vms); 3773ca23e405STejun Heo } 37744f8b02b4STejun Heo #endif /* CONFIG_SMP */ 3775a10aa579SChristoph Lameter 37765bb1bb35SPaul E. McKenney #ifdef CONFIG_PRINTK 377798f18083SPaul E. McKenney bool vmalloc_dump_obj(void *object) 377898f18083SPaul E. McKenney { 377998f18083SPaul E. McKenney struct vm_struct *vm; 378098f18083SPaul E. McKenney void *objp = (void *)PAGE_ALIGN((unsigned long)object); 378198f18083SPaul E. McKenney 378298f18083SPaul E. McKenney vm = find_vm_area(objp); 378398f18083SPaul E. McKenney if (!vm) 378498f18083SPaul E. McKenney return false; 3785bd34dcd4SPaul E. McKenney pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n", 3786bd34dcd4SPaul E. McKenney vm->nr_pages, (unsigned long)vm->addr, vm->caller); 378798f18083SPaul E. McKenney return true; 378898f18083SPaul E. McKenney } 37895bb1bb35SPaul E. McKenney #endif 379098f18083SPaul E. McKenney 3791a10aa579SChristoph Lameter #ifdef CONFIG_PROC_FS 3792a10aa579SChristoph Lameter static void *s_start(struct seq_file *m, loff_t *pos) 3793e36176beSUladzislau Rezki (Sony) __acquires(&vmap_purge_lock) 3794d4033afdSJoonsoo Kim __acquires(&vmap_area_lock) 3795a10aa579SChristoph Lameter { 3796e36176beSUladzislau Rezki (Sony) mutex_lock(&vmap_purge_lock); 3797d4033afdSJoonsoo Kim spin_lock(&vmap_area_lock); 3798e36176beSUladzislau Rezki (Sony) 37993f500069Szijun_hu return seq_list_start(&vmap_area_list, *pos); 3800a10aa579SChristoph Lameter } 3801a10aa579SChristoph Lameter 3802a10aa579SChristoph Lameter static void *s_next(struct seq_file *m, void *p, loff_t *pos) 3803a10aa579SChristoph Lameter { 38043f500069Szijun_hu return seq_list_next(p, &vmap_area_list, pos); 3805a10aa579SChristoph Lameter } 3806a10aa579SChristoph Lameter 3807a10aa579SChristoph Lameter static void s_stop(struct seq_file *m, void *p) 3808d4033afdSJoonsoo Kim __releases(&vmap_area_lock) 38090a7dd4e9SWaiman Long __releases(&vmap_purge_lock) 3810a10aa579SChristoph Lameter { 3811d4033afdSJoonsoo Kim spin_unlock(&vmap_area_lock); 38120a7dd4e9SWaiman Long mutex_unlock(&vmap_purge_lock); 3813a10aa579SChristoph Lameter } 3814a10aa579SChristoph Lameter 3815a47a126aSEric Dumazet static void show_numa_info(struct seq_file *m, struct vm_struct *v) 3816a47a126aSEric Dumazet { 3817e5adfffcSKirill A. Shutemov if (IS_ENABLED(CONFIG_NUMA)) { 3818a47a126aSEric Dumazet unsigned int nr, *counters = m->private; 3819a47a126aSEric Dumazet 3820a47a126aSEric Dumazet if (!counters) 3821a47a126aSEric Dumazet return; 3822a47a126aSEric Dumazet 3823af12346cSWanpeng Li if (v->flags & VM_UNINITIALIZED) 3824af12346cSWanpeng Li return; 38257e5b528bSDmitry Vyukov /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ 38267e5b528bSDmitry Vyukov smp_rmb(); 3827af12346cSWanpeng Li 3828a47a126aSEric Dumazet memset(counters, 0, nr_node_ids * sizeof(unsigned int)); 3829a47a126aSEric Dumazet 3830a47a126aSEric Dumazet for (nr = 0; nr < v->nr_pages; nr++) 3831a47a126aSEric Dumazet counters[page_to_nid(v->pages[nr])]++; 3832a47a126aSEric Dumazet 3833a47a126aSEric Dumazet for_each_node_state(nr, N_HIGH_MEMORY) 3834a47a126aSEric Dumazet if (counters[nr]) 3835a47a126aSEric Dumazet seq_printf(m, " N%u=%u", nr, counters[nr]); 3836a47a126aSEric Dumazet } 3837a47a126aSEric Dumazet } 3838a47a126aSEric Dumazet 3839dd3b8353SUladzislau Rezki (Sony) static void show_purge_info(struct seq_file *m) 3840dd3b8353SUladzislau Rezki (Sony) { 3841dd3b8353SUladzislau Rezki (Sony) struct vmap_area *va; 3842dd3b8353SUladzislau Rezki (Sony) 384396e2db45SUladzislau Rezki (Sony) spin_lock(&purge_vmap_area_lock); 384496e2db45SUladzislau Rezki (Sony) list_for_each_entry(va, &purge_vmap_area_list, list) { 3845dd3b8353SUladzislau Rezki (Sony) seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n", 3846dd3b8353SUladzislau Rezki (Sony) (void *)va->va_start, (void *)va->va_end, 3847dd3b8353SUladzislau Rezki (Sony) va->va_end - va->va_start); 3848dd3b8353SUladzislau Rezki (Sony) } 384996e2db45SUladzislau Rezki (Sony) spin_unlock(&purge_vmap_area_lock); 3850dd3b8353SUladzislau Rezki (Sony) } 3851dd3b8353SUladzislau Rezki (Sony) 3852a10aa579SChristoph Lameter static int s_show(struct seq_file *m, void *p) 3853a10aa579SChristoph Lameter { 38543f500069Szijun_hu struct vmap_area *va; 3855d4033afdSJoonsoo Kim struct vm_struct *v; 3856d4033afdSJoonsoo Kim 38573f500069Szijun_hu va = list_entry(p, struct vmap_area, list); 38583f500069Szijun_hu 3859c2ce8c14SWanpeng Li /* 3860688fcbfcSPengfei Li * s_show can encounter race with remove_vm_area, !vm on behalf 3861688fcbfcSPengfei Li * of vmap area is being tear down or vm_map_ram allocation. 3862c2ce8c14SWanpeng Li */ 3863688fcbfcSPengfei Li if (!va->vm) { 3864dd3b8353SUladzislau Rezki (Sony) seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n", 386578c72746SYisheng Xie (void *)va->va_start, (void *)va->va_end, 3866dd3b8353SUladzislau Rezki (Sony) va->va_end - va->va_start); 386778c72746SYisheng Xie 3868d4033afdSJoonsoo Kim return 0; 386978c72746SYisheng Xie } 3870d4033afdSJoonsoo Kim 3871d4033afdSJoonsoo Kim v = va->vm; 3872a10aa579SChristoph Lameter 387345ec1690SKees Cook seq_printf(m, "0x%pK-0x%pK %7ld", 3874a10aa579SChristoph Lameter v->addr, v->addr + v->size, v->size); 3875a10aa579SChristoph Lameter 387662c70bceSJoe Perches if (v->caller) 387762c70bceSJoe Perches seq_printf(m, " %pS", v->caller); 387823016969SChristoph Lameter 3879a10aa579SChristoph Lameter if (v->nr_pages) 3880a10aa579SChristoph Lameter seq_printf(m, " pages=%d", v->nr_pages); 3881a10aa579SChristoph Lameter 3882a10aa579SChristoph Lameter if (v->phys_addr) 3883199eaa05SMiles Chen seq_printf(m, " phys=%pa", &v->phys_addr); 3884a10aa579SChristoph Lameter 3885a10aa579SChristoph Lameter if (v->flags & VM_IOREMAP) 3886f4527c90SFabian Frederick seq_puts(m, " ioremap"); 3887a10aa579SChristoph Lameter 3888a10aa579SChristoph Lameter if (v->flags & VM_ALLOC) 3889f4527c90SFabian Frederick seq_puts(m, " vmalloc"); 3890a10aa579SChristoph Lameter 3891a10aa579SChristoph Lameter if (v->flags & VM_MAP) 3892f4527c90SFabian Frederick seq_puts(m, " vmap"); 3893a10aa579SChristoph Lameter 3894a10aa579SChristoph Lameter if (v->flags & VM_USERMAP) 3895f4527c90SFabian Frederick seq_puts(m, " user"); 3896a10aa579SChristoph Lameter 3897fe9041c2SChristoph Hellwig if (v->flags & VM_DMA_COHERENT) 3898fe9041c2SChristoph Hellwig seq_puts(m, " dma-coherent"); 3899fe9041c2SChristoph Hellwig 3900244d63eeSDavid Rientjes if (is_vmalloc_addr(v->pages)) 3901f4527c90SFabian Frederick seq_puts(m, " vpages"); 3902a10aa579SChristoph Lameter 3903a47a126aSEric Dumazet show_numa_info(m, v); 3904a10aa579SChristoph Lameter seq_putc(m, '\n'); 3905dd3b8353SUladzislau Rezki (Sony) 3906dd3b8353SUladzislau Rezki (Sony) /* 390796e2db45SUladzislau Rezki (Sony) * As a final step, dump "unpurged" areas. 3908dd3b8353SUladzislau Rezki (Sony) */ 3909dd3b8353SUladzislau Rezki (Sony) if (list_is_last(&va->list, &vmap_area_list)) 3910dd3b8353SUladzislau Rezki (Sony) show_purge_info(m); 3911dd3b8353SUladzislau Rezki (Sony) 3912a10aa579SChristoph Lameter return 0; 3913a10aa579SChristoph Lameter } 3914a10aa579SChristoph Lameter 39155f6a6a9cSAlexey Dobriyan static const struct seq_operations vmalloc_op = { 3916a10aa579SChristoph Lameter .start = s_start, 3917a10aa579SChristoph Lameter .next = s_next, 3918a10aa579SChristoph Lameter .stop = s_stop, 3919a10aa579SChristoph Lameter .show = s_show, 3920a10aa579SChristoph Lameter }; 39215f6a6a9cSAlexey Dobriyan 39225f6a6a9cSAlexey Dobriyan static int __init proc_vmalloc_init(void) 39235f6a6a9cSAlexey Dobriyan { 3924fddda2b7SChristoph Hellwig if (IS_ENABLED(CONFIG_NUMA)) 39250825a6f9SJoe Perches proc_create_seq_private("vmallocinfo", 0400, NULL, 392644414d82SChristoph Hellwig &vmalloc_op, 392744414d82SChristoph Hellwig nr_node_ids * sizeof(unsigned int), NULL); 3928fddda2b7SChristoph Hellwig else 39290825a6f9SJoe Perches proc_create_seq("vmallocinfo", 0400, NULL, &vmalloc_op); 39305f6a6a9cSAlexey Dobriyan return 0; 39315f6a6a9cSAlexey Dobriyan } 39325f6a6a9cSAlexey Dobriyan module_init(proc_vmalloc_init); 3933db3808c1SJoonsoo Kim 3934a10aa579SChristoph Lameter #endif 3935