1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * Copyright (C) 1993 Linus Torvalds 41da177e4SLinus Torvalds * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 51da177e4SLinus Torvalds * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 61da177e4SLinus Torvalds * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 7930fc45aSChristoph Lameter * Numa awareness, Christoph Lameter, SGI, June 2005 8d758ffe6SUladzislau Rezki (Sony) * Improving global KVA allocator, Uladzislau Rezki, Sony, May 2019 91da177e4SLinus Torvalds */ 101da177e4SLinus Torvalds 11db64fe02SNick Piggin #include <linux/vmalloc.h> 121da177e4SLinus Torvalds #include <linux/mm.h> 131da177e4SLinus Torvalds #include <linux/module.h> 141da177e4SLinus Torvalds #include <linux/highmem.h> 15c3edc401SIngo Molnar #include <linux/sched/signal.h> 161da177e4SLinus Torvalds #include <linux/slab.h> 171da177e4SLinus Torvalds #include <linux/spinlock.h> 181da177e4SLinus Torvalds #include <linux/interrupt.h> 195f6a6a9cSAlexey Dobriyan #include <linux/proc_fs.h> 20a10aa579SChristoph Lameter #include <linux/seq_file.h> 21868b104dSRick Edgecombe #include <linux/set_memory.h> 223ac7fe5aSThomas Gleixner #include <linux/debugobjects.h> 2323016969SChristoph Lameter #include <linux/kallsyms.h> 24db64fe02SNick Piggin #include <linux/list.h> 254da56b99SChris Wilson #include <linux/notifier.h> 26db64fe02SNick Piggin #include <linux/rbtree.h> 270f14599cSMatthew Wilcox (Oracle) #include <linux/xarray.h> 285da96bddSMel Gorman #include <linux/io.h> 29db64fe02SNick Piggin #include <linux/rcupdate.h> 30f0aa6617STejun Heo #include <linux/pfn.h> 3189219d37SCatalin Marinas #include <linux/kmemleak.h> 3260063497SArun Sharma #include <linux/atomic.h> 333b32123dSGideon Israel Dsouza #include <linux/compiler.h> 344e5aa1f4SShakeel Butt #include <linux/memcontrol.h> 3532fcfd40SAl Viro #include <linux/llist.h> 360f616be1SToshi Kani #include <linux/bitops.h> 3768ad4a33SUladzislau Rezki (Sony) #include <linux/rbtree_augmented.h> 38bdebd6a2SJann Horn #include <linux/overflow.h> 39c0eb315aSNicholas Piggin #include <linux/pgtable.h> 407c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 41f7ee1f13SChristophe Leroy #include <linux/hugetlb.h> 42451769ebSMichal Hocko #include <linux/sched/mm.h> 431da177e4SLinus Torvalds #include <asm/tlbflush.h> 442dca6999SDavid Miller #include <asm/shmparam.h> 451da177e4SLinus Torvalds 46dd56b046SMel Gorman #include "internal.h" 472a681cfaSJoerg Roedel #include "pgalloc-track.h" 48dd56b046SMel Gorman 4982a70ce0SChristoph Hellwig #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 5082a70ce0SChristoph Hellwig static unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1; 5182a70ce0SChristoph Hellwig 5282a70ce0SChristoph Hellwig static int __init set_nohugeiomap(char *str) 5382a70ce0SChristoph Hellwig { 5482a70ce0SChristoph Hellwig ioremap_max_page_shift = PAGE_SHIFT; 5582a70ce0SChristoph Hellwig return 0; 5682a70ce0SChristoph Hellwig } 5782a70ce0SChristoph Hellwig early_param("nohugeiomap", set_nohugeiomap); 5882a70ce0SChristoph Hellwig #else /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 5982a70ce0SChristoph Hellwig static const unsigned int ioremap_max_page_shift = PAGE_SHIFT; 6082a70ce0SChristoph Hellwig #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 6182a70ce0SChristoph Hellwig 62121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 63121e6f32SNicholas Piggin static bool __ro_after_init vmap_allow_huge = true; 64121e6f32SNicholas Piggin 65121e6f32SNicholas Piggin static int __init set_nohugevmalloc(char *str) 66121e6f32SNicholas Piggin { 67121e6f32SNicholas Piggin vmap_allow_huge = false; 68121e6f32SNicholas Piggin return 0; 69121e6f32SNicholas Piggin } 70121e6f32SNicholas Piggin early_param("nohugevmalloc", set_nohugevmalloc); 71121e6f32SNicholas Piggin #else /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ 72121e6f32SNicholas Piggin static const bool vmap_allow_huge = false; 73121e6f32SNicholas Piggin #endif /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ 74121e6f32SNicholas Piggin 75186525bdSIngo Molnar bool is_vmalloc_addr(const void *x) 76186525bdSIngo Molnar { 774aff1dc4SAndrey Konovalov unsigned long addr = (unsigned long)kasan_reset_tag(x); 78186525bdSIngo Molnar 79186525bdSIngo Molnar return addr >= VMALLOC_START && addr < VMALLOC_END; 80186525bdSIngo Molnar } 81186525bdSIngo Molnar EXPORT_SYMBOL(is_vmalloc_addr); 82186525bdSIngo Molnar 8332fcfd40SAl Viro struct vfree_deferred { 8432fcfd40SAl Viro struct llist_head list; 8532fcfd40SAl Viro struct work_struct wq; 8632fcfd40SAl Viro }; 8732fcfd40SAl Viro static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); 8832fcfd40SAl Viro 8932fcfd40SAl Viro static void __vunmap(const void *, int); 9032fcfd40SAl Viro 9132fcfd40SAl Viro static void free_work(struct work_struct *w) 9232fcfd40SAl Viro { 9332fcfd40SAl Viro struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); 94894e58c1SByungchul Park struct llist_node *t, *llnode; 95894e58c1SByungchul Park 96894e58c1SByungchul Park llist_for_each_safe(llnode, t, llist_del_all(&p->list)) 97894e58c1SByungchul Park __vunmap((void *)llnode, 1); 9832fcfd40SAl Viro } 9932fcfd40SAl Viro 100db64fe02SNick Piggin /*** Page table manipulation functions ***/ 1015e9e3d77SNicholas Piggin static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 1025e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 103f7ee1f13SChristophe Leroy unsigned int max_page_shift, pgtbl_mod_mask *mask) 1045e9e3d77SNicholas Piggin { 1055e9e3d77SNicholas Piggin pte_t *pte; 1065e9e3d77SNicholas Piggin u64 pfn; 107f7ee1f13SChristophe Leroy unsigned long size = PAGE_SIZE; 1085e9e3d77SNicholas Piggin 1095e9e3d77SNicholas Piggin pfn = phys_addr >> PAGE_SHIFT; 1105e9e3d77SNicholas Piggin pte = pte_alloc_kernel_track(pmd, addr, mask); 1115e9e3d77SNicholas Piggin if (!pte) 1125e9e3d77SNicholas Piggin return -ENOMEM; 1135e9e3d77SNicholas Piggin do { 1145e9e3d77SNicholas Piggin BUG_ON(!pte_none(*pte)); 115f7ee1f13SChristophe Leroy 116f7ee1f13SChristophe Leroy #ifdef CONFIG_HUGETLB_PAGE 117f7ee1f13SChristophe Leroy size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift); 118f7ee1f13SChristophe Leroy if (size != PAGE_SIZE) { 119f7ee1f13SChristophe Leroy pte_t entry = pfn_pte(pfn, prot); 120f7ee1f13SChristophe Leroy 121f7ee1f13SChristophe Leroy entry = arch_make_huge_pte(entry, ilog2(size), 0); 122f7ee1f13SChristophe Leroy set_huge_pte_at(&init_mm, addr, pte, entry); 123f7ee1f13SChristophe Leroy pfn += PFN_DOWN(size); 124f7ee1f13SChristophe Leroy continue; 125f7ee1f13SChristophe Leroy } 126f7ee1f13SChristophe Leroy #endif 1275e9e3d77SNicholas Piggin set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); 1285e9e3d77SNicholas Piggin pfn++; 129f7ee1f13SChristophe Leroy } while (pte += PFN_DOWN(size), addr += size, addr != end); 1305e9e3d77SNicholas Piggin *mask |= PGTBL_PTE_MODIFIED; 1315e9e3d77SNicholas Piggin return 0; 1325e9e3d77SNicholas Piggin } 1335e9e3d77SNicholas Piggin 1345e9e3d77SNicholas Piggin static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, 1355e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1365e9e3d77SNicholas Piggin unsigned int max_page_shift) 1375e9e3d77SNicholas Piggin { 1385e9e3d77SNicholas Piggin if (max_page_shift < PMD_SHIFT) 1395e9e3d77SNicholas Piggin return 0; 1405e9e3d77SNicholas Piggin 1415e9e3d77SNicholas Piggin if (!arch_vmap_pmd_supported(prot)) 1425e9e3d77SNicholas Piggin return 0; 1435e9e3d77SNicholas Piggin 1445e9e3d77SNicholas Piggin if ((end - addr) != PMD_SIZE) 1455e9e3d77SNicholas Piggin return 0; 1465e9e3d77SNicholas Piggin 1475e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, PMD_SIZE)) 1485e9e3d77SNicholas Piggin return 0; 1495e9e3d77SNicholas Piggin 1505e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, PMD_SIZE)) 1515e9e3d77SNicholas Piggin return 0; 1525e9e3d77SNicholas Piggin 1535e9e3d77SNicholas Piggin if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) 1545e9e3d77SNicholas Piggin return 0; 1555e9e3d77SNicholas Piggin 1565e9e3d77SNicholas Piggin return pmd_set_huge(pmd, phys_addr, prot); 1575e9e3d77SNicholas Piggin } 1585e9e3d77SNicholas Piggin 1595e9e3d77SNicholas Piggin static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 1605e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1615e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 1625e9e3d77SNicholas Piggin { 1635e9e3d77SNicholas Piggin pmd_t *pmd; 1645e9e3d77SNicholas Piggin unsigned long next; 1655e9e3d77SNicholas Piggin 1665e9e3d77SNicholas Piggin pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 1675e9e3d77SNicholas Piggin if (!pmd) 1685e9e3d77SNicholas Piggin return -ENOMEM; 1695e9e3d77SNicholas Piggin do { 1705e9e3d77SNicholas Piggin next = pmd_addr_end(addr, end); 1715e9e3d77SNicholas Piggin 1725e9e3d77SNicholas Piggin if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot, 1735e9e3d77SNicholas Piggin max_page_shift)) { 1745e9e3d77SNicholas Piggin *mask |= PGTBL_PMD_MODIFIED; 1755e9e3d77SNicholas Piggin continue; 1765e9e3d77SNicholas Piggin } 1775e9e3d77SNicholas Piggin 178f7ee1f13SChristophe Leroy if (vmap_pte_range(pmd, addr, next, phys_addr, prot, max_page_shift, mask)) 1795e9e3d77SNicholas Piggin return -ENOMEM; 1805e9e3d77SNicholas Piggin } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); 1815e9e3d77SNicholas Piggin return 0; 1825e9e3d77SNicholas Piggin } 1835e9e3d77SNicholas Piggin 1845e9e3d77SNicholas Piggin static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end, 1855e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1865e9e3d77SNicholas Piggin unsigned int max_page_shift) 1875e9e3d77SNicholas Piggin { 1885e9e3d77SNicholas Piggin if (max_page_shift < PUD_SHIFT) 1895e9e3d77SNicholas Piggin return 0; 1905e9e3d77SNicholas Piggin 1915e9e3d77SNicholas Piggin if (!arch_vmap_pud_supported(prot)) 1925e9e3d77SNicholas Piggin return 0; 1935e9e3d77SNicholas Piggin 1945e9e3d77SNicholas Piggin if ((end - addr) != PUD_SIZE) 1955e9e3d77SNicholas Piggin return 0; 1965e9e3d77SNicholas Piggin 1975e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, PUD_SIZE)) 1985e9e3d77SNicholas Piggin return 0; 1995e9e3d77SNicholas Piggin 2005e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, PUD_SIZE)) 2015e9e3d77SNicholas Piggin return 0; 2025e9e3d77SNicholas Piggin 2035e9e3d77SNicholas Piggin if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) 2045e9e3d77SNicholas Piggin return 0; 2055e9e3d77SNicholas Piggin 2065e9e3d77SNicholas Piggin return pud_set_huge(pud, phys_addr, prot); 2075e9e3d77SNicholas Piggin } 2085e9e3d77SNicholas Piggin 2095e9e3d77SNicholas Piggin static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 2105e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2115e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 2125e9e3d77SNicholas Piggin { 2135e9e3d77SNicholas Piggin pud_t *pud; 2145e9e3d77SNicholas Piggin unsigned long next; 2155e9e3d77SNicholas Piggin 2165e9e3d77SNicholas Piggin pud = pud_alloc_track(&init_mm, p4d, addr, mask); 2175e9e3d77SNicholas Piggin if (!pud) 2185e9e3d77SNicholas Piggin return -ENOMEM; 2195e9e3d77SNicholas Piggin do { 2205e9e3d77SNicholas Piggin next = pud_addr_end(addr, end); 2215e9e3d77SNicholas Piggin 2225e9e3d77SNicholas Piggin if (vmap_try_huge_pud(pud, addr, next, phys_addr, prot, 2235e9e3d77SNicholas Piggin max_page_shift)) { 2245e9e3d77SNicholas Piggin *mask |= PGTBL_PUD_MODIFIED; 2255e9e3d77SNicholas Piggin continue; 2265e9e3d77SNicholas Piggin } 2275e9e3d77SNicholas Piggin 2285e9e3d77SNicholas Piggin if (vmap_pmd_range(pud, addr, next, phys_addr, prot, 2295e9e3d77SNicholas Piggin max_page_shift, mask)) 2305e9e3d77SNicholas Piggin return -ENOMEM; 2315e9e3d77SNicholas Piggin } while (pud++, phys_addr += (next - addr), addr = next, addr != end); 2325e9e3d77SNicholas Piggin return 0; 2335e9e3d77SNicholas Piggin } 2345e9e3d77SNicholas Piggin 2355e9e3d77SNicholas Piggin static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, 2365e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2375e9e3d77SNicholas Piggin unsigned int max_page_shift) 2385e9e3d77SNicholas Piggin { 2395e9e3d77SNicholas Piggin if (max_page_shift < P4D_SHIFT) 2405e9e3d77SNicholas Piggin return 0; 2415e9e3d77SNicholas Piggin 2425e9e3d77SNicholas Piggin if (!arch_vmap_p4d_supported(prot)) 2435e9e3d77SNicholas Piggin return 0; 2445e9e3d77SNicholas Piggin 2455e9e3d77SNicholas Piggin if ((end - addr) != P4D_SIZE) 2465e9e3d77SNicholas Piggin return 0; 2475e9e3d77SNicholas Piggin 2485e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, P4D_SIZE)) 2495e9e3d77SNicholas Piggin return 0; 2505e9e3d77SNicholas Piggin 2515e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, P4D_SIZE)) 2525e9e3d77SNicholas Piggin return 0; 2535e9e3d77SNicholas Piggin 2545e9e3d77SNicholas Piggin if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) 2555e9e3d77SNicholas Piggin return 0; 2565e9e3d77SNicholas Piggin 2575e9e3d77SNicholas Piggin return p4d_set_huge(p4d, phys_addr, prot); 2585e9e3d77SNicholas Piggin } 2595e9e3d77SNicholas Piggin 2605e9e3d77SNicholas Piggin static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 2615e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2625e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 2635e9e3d77SNicholas Piggin { 2645e9e3d77SNicholas Piggin p4d_t *p4d; 2655e9e3d77SNicholas Piggin unsigned long next; 2665e9e3d77SNicholas Piggin 2675e9e3d77SNicholas Piggin p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 2685e9e3d77SNicholas Piggin if (!p4d) 2695e9e3d77SNicholas Piggin return -ENOMEM; 2705e9e3d77SNicholas Piggin do { 2715e9e3d77SNicholas Piggin next = p4d_addr_end(addr, end); 2725e9e3d77SNicholas Piggin 2735e9e3d77SNicholas Piggin if (vmap_try_huge_p4d(p4d, addr, next, phys_addr, prot, 2745e9e3d77SNicholas Piggin max_page_shift)) { 2755e9e3d77SNicholas Piggin *mask |= PGTBL_P4D_MODIFIED; 2765e9e3d77SNicholas Piggin continue; 2775e9e3d77SNicholas Piggin } 2785e9e3d77SNicholas Piggin 2795e9e3d77SNicholas Piggin if (vmap_pud_range(p4d, addr, next, phys_addr, prot, 2805e9e3d77SNicholas Piggin max_page_shift, mask)) 2815e9e3d77SNicholas Piggin return -ENOMEM; 2825e9e3d77SNicholas Piggin } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); 2835e9e3d77SNicholas Piggin return 0; 2845e9e3d77SNicholas Piggin } 2855e9e3d77SNicholas Piggin 2865d87510dSNicholas Piggin static int vmap_range_noflush(unsigned long addr, unsigned long end, 2875e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2885e9e3d77SNicholas Piggin unsigned int max_page_shift) 2895e9e3d77SNicholas Piggin { 2905e9e3d77SNicholas Piggin pgd_t *pgd; 2915e9e3d77SNicholas Piggin unsigned long start; 2925e9e3d77SNicholas Piggin unsigned long next; 2935e9e3d77SNicholas Piggin int err; 2945e9e3d77SNicholas Piggin pgtbl_mod_mask mask = 0; 2955e9e3d77SNicholas Piggin 2965e9e3d77SNicholas Piggin might_sleep(); 2975e9e3d77SNicholas Piggin BUG_ON(addr >= end); 2985e9e3d77SNicholas Piggin 2995e9e3d77SNicholas Piggin start = addr; 3005e9e3d77SNicholas Piggin pgd = pgd_offset_k(addr); 3015e9e3d77SNicholas Piggin do { 3025e9e3d77SNicholas Piggin next = pgd_addr_end(addr, end); 3035e9e3d77SNicholas Piggin err = vmap_p4d_range(pgd, addr, next, phys_addr, prot, 3045e9e3d77SNicholas Piggin max_page_shift, &mask); 3055e9e3d77SNicholas Piggin if (err) 3065e9e3d77SNicholas Piggin break; 3075e9e3d77SNicholas Piggin } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); 3085e9e3d77SNicholas Piggin 3095e9e3d77SNicholas Piggin if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 3105e9e3d77SNicholas Piggin arch_sync_kernel_mappings(start, end); 3115e9e3d77SNicholas Piggin 3125e9e3d77SNicholas Piggin return err; 3135e9e3d77SNicholas Piggin } 314b221385bSAdrian Bunk 31582a70ce0SChristoph Hellwig int ioremap_page_range(unsigned long addr, unsigned long end, 31682a70ce0SChristoph Hellwig phys_addr_t phys_addr, pgprot_t prot) 3175d87510dSNicholas Piggin { 3185d87510dSNicholas Piggin int err; 3195d87510dSNicholas Piggin 3208491502fSChristoph Hellwig err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot), 32182a70ce0SChristoph Hellwig ioremap_max_page_shift); 3225d87510dSNicholas Piggin flush_cache_vmap(addr, end); 323*b073d7f8SAlexander Potapenko if (!err) 324*b073d7f8SAlexander Potapenko kmsan_ioremap_page_range(addr, end, phys_addr, prot, 325*b073d7f8SAlexander Potapenko ioremap_max_page_shift); 3265d87510dSNicholas Piggin return err; 3275d87510dSNicholas Piggin } 3285d87510dSNicholas Piggin 3292ba3e694SJoerg Roedel static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 3302ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3311da177e4SLinus Torvalds { 3321da177e4SLinus Torvalds pte_t *pte; 3331da177e4SLinus Torvalds 3341da177e4SLinus Torvalds pte = pte_offset_kernel(pmd, addr); 3351da177e4SLinus Torvalds do { 3361da177e4SLinus Torvalds pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); 3371da177e4SLinus Torvalds WARN_ON(!pte_none(ptent) && !pte_present(ptent)); 3381da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 3392ba3e694SJoerg Roedel *mask |= PGTBL_PTE_MODIFIED; 3401da177e4SLinus Torvalds } 3411da177e4SLinus Torvalds 3422ba3e694SJoerg Roedel static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 3432ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3441da177e4SLinus Torvalds { 3451da177e4SLinus Torvalds pmd_t *pmd; 3461da177e4SLinus Torvalds unsigned long next; 3472ba3e694SJoerg Roedel int cleared; 3481da177e4SLinus Torvalds 3491da177e4SLinus Torvalds pmd = pmd_offset(pud, addr); 3501da177e4SLinus Torvalds do { 3511da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 3522ba3e694SJoerg Roedel 3532ba3e694SJoerg Roedel cleared = pmd_clear_huge(pmd); 3542ba3e694SJoerg Roedel if (cleared || pmd_bad(*pmd)) 3552ba3e694SJoerg Roedel *mask |= PGTBL_PMD_MODIFIED; 3562ba3e694SJoerg Roedel 3572ba3e694SJoerg Roedel if (cleared) 358b9820d8fSToshi Kani continue; 3591da177e4SLinus Torvalds if (pmd_none_or_clear_bad(pmd)) 3601da177e4SLinus Torvalds continue; 3612ba3e694SJoerg Roedel vunmap_pte_range(pmd, addr, next, mask); 362e47110e9SAneesh Kumar K.V 363e47110e9SAneesh Kumar K.V cond_resched(); 3641da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 3651da177e4SLinus Torvalds } 3661da177e4SLinus Torvalds 3672ba3e694SJoerg Roedel static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 3682ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3691da177e4SLinus Torvalds { 3701da177e4SLinus Torvalds pud_t *pud; 3711da177e4SLinus Torvalds unsigned long next; 3722ba3e694SJoerg Roedel int cleared; 3731da177e4SLinus Torvalds 374c2febafcSKirill A. Shutemov pud = pud_offset(p4d, addr); 3751da177e4SLinus Torvalds do { 3761da177e4SLinus Torvalds next = pud_addr_end(addr, end); 3772ba3e694SJoerg Roedel 3782ba3e694SJoerg Roedel cleared = pud_clear_huge(pud); 3792ba3e694SJoerg Roedel if (cleared || pud_bad(*pud)) 3802ba3e694SJoerg Roedel *mask |= PGTBL_PUD_MODIFIED; 3812ba3e694SJoerg Roedel 3822ba3e694SJoerg Roedel if (cleared) 383b9820d8fSToshi Kani continue; 3841da177e4SLinus Torvalds if (pud_none_or_clear_bad(pud)) 3851da177e4SLinus Torvalds continue; 3862ba3e694SJoerg Roedel vunmap_pmd_range(pud, addr, next, mask); 3871da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 3881da177e4SLinus Torvalds } 3891da177e4SLinus Torvalds 3902ba3e694SJoerg Roedel static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 3912ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 392c2febafcSKirill A. Shutemov { 393c2febafcSKirill A. Shutemov p4d_t *p4d; 394c2febafcSKirill A. Shutemov unsigned long next; 395c2febafcSKirill A. Shutemov 396c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 397c2febafcSKirill A. Shutemov do { 398c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 3992ba3e694SJoerg Roedel 400c8db8c26SLi kunyu p4d_clear_huge(p4d); 401c8db8c26SLi kunyu if (p4d_bad(*p4d)) 4022ba3e694SJoerg Roedel *mask |= PGTBL_P4D_MODIFIED; 4032ba3e694SJoerg Roedel 404c2febafcSKirill A. Shutemov if (p4d_none_or_clear_bad(p4d)) 405c2febafcSKirill A. Shutemov continue; 4062ba3e694SJoerg Roedel vunmap_pud_range(p4d, addr, next, mask); 407c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 408c2febafcSKirill A. Shutemov } 409c2febafcSKirill A. Shutemov 4104ad0ae8cSNicholas Piggin /* 4114ad0ae8cSNicholas Piggin * vunmap_range_noflush is similar to vunmap_range, but does not 4124ad0ae8cSNicholas Piggin * flush caches or TLBs. 413b521c43fSChristoph Hellwig * 4144ad0ae8cSNicholas Piggin * The caller is responsible for calling flush_cache_vmap() before calling 4154ad0ae8cSNicholas Piggin * this function, and flush_tlb_kernel_range after it has returned 4164ad0ae8cSNicholas Piggin * successfully (and before the addresses are expected to cause a page fault 4174ad0ae8cSNicholas Piggin * or be re-mapped for something else, if TLB flushes are being delayed or 4184ad0ae8cSNicholas Piggin * coalesced). 419b521c43fSChristoph Hellwig * 4204ad0ae8cSNicholas Piggin * This is an internal function only. Do not use outside mm/. 421b521c43fSChristoph Hellwig */ 422*b073d7f8SAlexander Potapenko void __vunmap_range_noflush(unsigned long start, unsigned long end) 4231da177e4SLinus Torvalds { 4241da177e4SLinus Torvalds unsigned long next; 425b521c43fSChristoph Hellwig pgd_t *pgd; 4262ba3e694SJoerg Roedel unsigned long addr = start; 4272ba3e694SJoerg Roedel pgtbl_mod_mask mask = 0; 4281da177e4SLinus Torvalds 4291da177e4SLinus Torvalds BUG_ON(addr >= end); 4301da177e4SLinus Torvalds pgd = pgd_offset_k(addr); 4311da177e4SLinus Torvalds do { 4321da177e4SLinus Torvalds next = pgd_addr_end(addr, end); 4332ba3e694SJoerg Roedel if (pgd_bad(*pgd)) 4342ba3e694SJoerg Roedel mask |= PGTBL_PGD_MODIFIED; 4351da177e4SLinus Torvalds if (pgd_none_or_clear_bad(pgd)) 4361da177e4SLinus Torvalds continue; 4372ba3e694SJoerg Roedel vunmap_p4d_range(pgd, addr, next, &mask); 4381da177e4SLinus Torvalds } while (pgd++, addr = next, addr != end); 4392ba3e694SJoerg Roedel 4402ba3e694SJoerg Roedel if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 4412ba3e694SJoerg Roedel arch_sync_kernel_mappings(start, end); 4421da177e4SLinus Torvalds } 4431da177e4SLinus Torvalds 444*b073d7f8SAlexander Potapenko void vunmap_range_noflush(unsigned long start, unsigned long end) 445*b073d7f8SAlexander Potapenko { 446*b073d7f8SAlexander Potapenko kmsan_vunmap_range_noflush(start, end); 447*b073d7f8SAlexander Potapenko __vunmap_range_noflush(start, end); 448*b073d7f8SAlexander Potapenko } 449*b073d7f8SAlexander Potapenko 4504ad0ae8cSNicholas Piggin /** 4514ad0ae8cSNicholas Piggin * vunmap_range - unmap kernel virtual addresses 4524ad0ae8cSNicholas Piggin * @addr: start of the VM area to unmap 4534ad0ae8cSNicholas Piggin * @end: end of the VM area to unmap (non-inclusive) 4544ad0ae8cSNicholas Piggin * 4554ad0ae8cSNicholas Piggin * Clears any present PTEs in the virtual address range, flushes TLBs and 4564ad0ae8cSNicholas Piggin * caches. Any subsequent access to the address before it has been re-mapped 4574ad0ae8cSNicholas Piggin * is a kernel bug. 4584ad0ae8cSNicholas Piggin */ 4594ad0ae8cSNicholas Piggin void vunmap_range(unsigned long addr, unsigned long end) 4604ad0ae8cSNicholas Piggin { 4614ad0ae8cSNicholas Piggin flush_cache_vunmap(addr, end); 4624ad0ae8cSNicholas Piggin vunmap_range_noflush(addr, end); 4634ad0ae8cSNicholas Piggin flush_tlb_kernel_range(addr, end); 4644ad0ae8cSNicholas Piggin } 4654ad0ae8cSNicholas Piggin 4660a264884SNicholas Piggin static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, 4672ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 4682ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 4691da177e4SLinus Torvalds { 4701da177e4SLinus Torvalds pte_t *pte; 4711da177e4SLinus Torvalds 472db64fe02SNick Piggin /* 473db64fe02SNick Piggin * nr is a running index into the array which helps higher level 474db64fe02SNick Piggin * callers keep track of where we're up to. 475db64fe02SNick Piggin */ 476db64fe02SNick Piggin 4772ba3e694SJoerg Roedel pte = pte_alloc_kernel_track(pmd, addr, mask); 4781da177e4SLinus Torvalds if (!pte) 4791da177e4SLinus Torvalds return -ENOMEM; 4801da177e4SLinus Torvalds do { 481db64fe02SNick Piggin struct page *page = pages[*nr]; 482db64fe02SNick Piggin 483db64fe02SNick Piggin if (WARN_ON(!pte_none(*pte))) 484db64fe02SNick Piggin return -EBUSY; 485db64fe02SNick Piggin if (WARN_ON(!page)) 4861da177e4SLinus Torvalds return -ENOMEM; 4874fcdcc12SYury Norov if (WARN_ON(!pfn_valid(page_to_pfn(page)))) 4884fcdcc12SYury Norov return -EINVAL; 4894fcdcc12SYury Norov 4901da177e4SLinus Torvalds set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); 491db64fe02SNick Piggin (*nr)++; 4921da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 4932ba3e694SJoerg Roedel *mask |= PGTBL_PTE_MODIFIED; 4941da177e4SLinus Torvalds return 0; 4951da177e4SLinus Torvalds } 4961da177e4SLinus Torvalds 4970a264884SNicholas Piggin static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr, 4982ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 4992ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 5001da177e4SLinus Torvalds { 5011da177e4SLinus Torvalds pmd_t *pmd; 5021da177e4SLinus Torvalds unsigned long next; 5031da177e4SLinus Torvalds 5042ba3e694SJoerg Roedel pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 5051da177e4SLinus Torvalds if (!pmd) 5061da177e4SLinus Torvalds return -ENOMEM; 5071da177e4SLinus Torvalds do { 5081da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 5090a264884SNicholas Piggin if (vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask)) 5101da177e4SLinus Torvalds return -ENOMEM; 5111da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 5121da177e4SLinus Torvalds return 0; 5131da177e4SLinus Torvalds } 5141da177e4SLinus Torvalds 5150a264884SNicholas Piggin static int vmap_pages_pud_range(p4d_t *p4d, unsigned long addr, 5162ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 5172ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 5181da177e4SLinus Torvalds { 5191da177e4SLinus Torvalds pud_t *pud; 5201da177e4SLinus Torvalds unsigned long next; 5211da177e4SLinus Torvalds 5222ba3e694SJoerg Roedel pud = pud_alloc_track(&init_mm, p4d, addr, mask); 5231da177e4SLinus Torvalds if (!pud) 5241da177e4SLinus Torvalds return -ENOMEM; 5251da177e4SLinus Torvalds do { 5261da177e4SLinus Torvalds next = pud_addr_end(addr, end); 5270a264884SNicholas Piggin if (vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask)) 5281da177e4SLinus Torvalds return -ENOMEM; 5291da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 5301da177e4SLinus Torvalds return 0; 5311da177e4SLinus Torvalds } 5321da177e4SLinus Torvalds 5330a264884SNicholas Piggin static int vmap_pages_p4d_range(pgd_t *pgd, unsigned long addr, 5342ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 5352ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 536c2febafcSKirill A. Shutemov { 537c2febafcSKirill A. Shutemov p4d_t *p4d; 538c2febafcSKirill A. Shutemov unsigned long next; 539c2febafcSKirill A. Shutemov 5402ba3e694SJoerg Roedel p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 541c2febafcSKirill A. Shutemov if (!p4d) 542c2febafcSKirill A. Shutemov return -ENOMEM; 543c2febafcSKirill A. Shutemov do { 544c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 5450a264884SNicholas Piggin if (vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask)) 546c2febafcSKirill A. Shutemov return -ENOMEM; 547c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 548c2febafcSKirill A. Shutemov return 0; 549c2febafcSKirill A. Shutemov } 550c2febafcSKirill A. Shutemov 551121e6f32SNicholas Piggin static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, 552121e6f32SNicholas Piggin pgprot_t prot, struct page **pages) 553121e6f32SNicholas Piggin { 554121e6f32SNicholas Piggin unsigned long start = addr; 555121e6f32SNicholas Piggin pgd_t *pgd; 556121e6f32SNicholas Piggin unsigned long next; 557121e6f32SNicholas Piggin int err = 0; 558121e6f32SNicholas Piggin int nr = 0; 559121e6f32SNicholas Piggin pgtbl_mod_mask mask = 0; 560121e6f32SNicholas Piggin 561121e6f32SNicholas Piggin BUG_ON(addr >= end); 562121e6f32SNicholas Piggin pgd = pgd_offset_k(addr); 563121e6f32SNicholas Piggin do { 564121e6f32SNicholas Piggin next = pgd_addr_end(addr, end); 565121e6f32SNicholas Piggin if (pgd_bad(*pgd)) 566121e6f32SNicholas Piggin mask |= PGTBL_PGD_MODIFIED; 567121e6f32SNicholas Piggin err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask); 568121e6f32SNicholas Piggin if (err) 569121e6f32SNicholas Piggin return err; 570121e6f32SNicholas Piggin } while (pgd++, addr = next, addr != end); 571121e6f32SNicholas Piggin 572121e6f32SNicholas Piggin if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 573121e6f32SNicholas Piggin arch_sync_kernel_mappings(start, end); 574121e6f32SNicholas Piggin 575121e6f32SNicholas Piggin return 0; 576121e6f32SNicholas Piggin } 577121e6f32SNicholas Piggin 578b67177ecSNicholas Piggin /* 579b67177ecSNicholas Piggin * vmap_pages_range_noflush is similar to vmap_pages_range, but does not 580b67177ecSNicholas Piggin * flush caches. 581b67177ecSNicholas Piggin * 582b67177ecSNicholas Piggin * The caller is responsible for calling flush_cache_vmap() after this 583b67177ecSNicholas Piggin * function returns successfully and before the addresses are accessed. 584b67177ecSNicholas Piggin * 585b67177ecSNicholas Piggin * This is an internal function only. Do not use outside mm/. 586b67177ecSNicholas Piggin */ 587*b073d7f8SAlexander Potapenko int __vmap_pages_range_noflush(unsigned long addr, unsigned long end, 588121e6f32SNicholas Piggin pgprot_t prot, struct page **pages, unsigned int page_shift) 589121e6f32SNicholas Piggin { 590121e6f32SNicholas Piggin unsigned int i, nr = (end - addr) >> PAGE_SHIFT; 591121e6f32SNicholas Piggin 592121e6f32SNicholas Piggin WARN_ON(page_shift < PAGE_SHIFT); 593121e6f32SNicholas Piggin 594121e6f32SNicholas Piggin if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC) || 595121e6f32SNicholas Piggin page_shift == PAGE_SHIFT) 596121e6f32SNicholas Piggin return vmap_small_pages_range_noflush(addr, end, prot, pages); 597121e6f32SNicholas Piggin 598121e6f32SNicholas Piggin for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) { 599121e6f32SNicholas Piggin int err; 600121e6f32SNicholas Piggin 601121e6f32SNicholas Piggin err = vmap_range_noflush(addr, addr + (1UL << page_shift), 60208262ac5SMatthew Wilcox page_to_phys(pages[i]), prot, 603121e6f32SNicholas Piggin page_shift); 604121e6f32SNicholas Piggin if (err) 605121e6f32SNicholas Piggin return err; 606121e6f32SNicholas Piggin 607121e6f32SNicholas Piggin addr += 1UL << page_shift; 608121e6f32SNicholas Piggin } 609121e6f32SNicholas Piggin 610121e6f32SNicholas Piggin return 0; 611121e6f32SNicholas Piggin } 612121e6f32SNicholas Piggin 613*b073d7f8SAlexander Potapenko int vmap_pages_range_noflush(unsigned long addr, unsigned long end, 614*b073d7f8SAlexander Potapenko pgprot_t prot, struct page **pages, unsigned int page_shift) 615*b073d7f8SAlexander Potapenko { 616*b073d7f8SAlexander Potapenko kmsan_vmap_pages_range_noflush(addr, end, prot, pages, page_shift); 617*b073d7f8SAlexander Potapenko return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift); 618*b073d7f8SAlexander Potapenko } 619*b073d7f8SAlexander Potapenko 620b67177ecSNicholas Piggin /** 621b67177ecSNicholas Piggin * vmap_pages_range - map pages to a kernel virtual address 622b67177ecSNicholas Piggin * @addr: start of the VM area to map 623b67177ecSNicholas Piggin * @end: end of the VM area to map (non-inclusive) 624b67177ecSNicholas Piggin * @prot: page protection flags to use 625b67177ecSNicholas Piggin * @pages: pages to map (always PAGE_SIZE pages) 626b67177ecSNicholas Piggin * @page_shift: maximum shift that the pages may be mapped with, @pages must 627b67177ecSNicholas Piggin * be aligned and contiguous up to at least this shift. 628b67177ecSNicholas Piggin * 629b67177ecSNicholas Piggin * RETURNS: 630b67177ecSNicholas Piggin * 0 on success, -errno on failure. 631b67177ecSNicholas Piggin */ 632121e6f32SNicholas Piggin static int vmap_pages_range(unsigned long addr, unsigned long end, 633121e6f32SNicholas Piggin pgprot_t prot, struct page **pages, unsigned int page_shift) 634121e6f32SNicholas Piggin { 635121e6f32SNicholas Piggin int err; 636121e6f32SNicholas Piggin 637121e6f32SNicholas Piggin err = vmap_pages_range_noflush(addr, end, prot, pages, page_shift); 638121e6f32SNicholas Piggin flush_cache_vmap(addr, end); 639121e6f32SNicholas Piggin return err; 640121e6f32SNicholas Piggin } 641121e6f32SNicholas Piggin 64281ac3ad9SKAMEZAWA Hiroyuki int is_vmalloc_or_module_addr(const void *x) 64373bdf0a6SLinus Torvalds { 64473bdf0a6SLinus Torvalds /* 645ab4f2ee1SRussell King * ARM, x86-64 and sparc64 put modules in a special place, 64673bdf0a6SLinus Torvalds * and fall back on vmalloc() if that fails. Others 64773bdf0a6SLinus Torvalds * just put it in the vmalloc space. 64873bdf0a6SLinus Torvalds */ 64973bdf0a6SLinus Torvalds #if defined(CONFIG_MODULES) && defined(MODULES_VADDR) 6504aff1dc4SAndrey Konovalov unsigned long addr = (unsigned long)kasan_reset_tag(x); 65173bdf0a6SLinus Torvalds if (addr >= MODULES_VADDR && addr < MODULES_END) 65273bdf0a6SLinus Torvalds return 1; 65373bdf0a6SLinus Torvalds #endif 65473bdf0a6SLinus Torvalds return is_vmalloc_addr(x); 65573bdf0a6SLinus Torvalds } 65673bdf0a6SLinus Torvalds 65748667e7aSChristoph Lameter /* 658c0eb315aSNicholas Piggin * Walk a vmap address to the struct page it maps. Huge vmap mappings will 659c0eb315aSNicholas Piggin * return the tail page that corresponds to the base page address, which 660c0eb315aSNicholas Piggin * matches small vmap mappings. 66148667e7aSChristoph Lameter */ 662add688fbSmalc struct page *vmalloc_to_page(const void *vmalloc_addr) 66348667e7aSChristoph Lameter { 66448667e7aSChristoph Lameter unsigned long addr = (unsigned long) vmalloc_addr; 665add688fbSmalc struct page *page = NULL; 66648667e7aSChristoph Lameter pgd_t *pgd = pgd_offset_k(addr); 667c2febafcSKirill A. Shutemov p4d_t *p4d; 668c2febafcSKirill A. Shutemov pud_t *pud; 669c2febafcSKirill A. Shutemov pmd_t *pmd; 670c2febafcSKirill A. Shutemov pte_t *ptep, pte; 67148667e7aSChristoph Lameter 6727aa413deSIngo Molnar /* 6737aa413deSIngo Molnar * XXX we might need to change this if we add VIRTUAL_BUG_ON for 6747aa413deSIngo Molnar * architectures that do not vmalloc module space 6757aa413deSIngo Molnar */ 67673bdf0a6SLinus Torvalds VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); 67759ea7463SJiri Slaby 678c2febafcSKirill A. Shutemov if (pgd_none(*pgd)) 679c2febafcSKirill A. Shutemov return NULL; 680c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pgd_leaf(*pgd))) 681c0eb315aSNicholas Piggin return NULL; /* XXX: no allowance for huge pgd */ 682c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pgd_bad(*pgd))) 683c0eb315aSNicholas Piggin return NULL; 684c0eb315aSNicholas Piggin 685c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 686c2febafcSKirill A. Shutemov if (p4d_none(*p4d)) 687c2febafcSKirill A. Shutemov return NULL; 688c0eb315aSNicholas Piggin if (p4d_leaf(*p4d)) 689c0eb315aSNicholas Piggin return p4d_page(*p4d) + ((addr & ~P4D_MASK) >> PAGE_SHIFT); 690c0eb315aSNicholas Piggin if (WARN_ON_ONCE(p4d_bad(*p4d))) 691c2febafcSKirill A. Shutemov return NULL; 692c0eb315aSNicholas Piggin 693c0eb315aSNicholas Piggin pud = pud_offset(p4d, addr); 694c0eb315aSNicholas Piggin if (pud_none(*pud)) 695c0eb315aSNicholas Piggin return NULL; 696c0eb315aSNicholas Piggin if (pud_leaf(*pud)) 697c0eb315aSNicholas Piggin return pud_page(*pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 698c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pud_bad(*pud))) 699c0eb315aSNicholas Piggin return NULL; 700c0eb315aSNicholas Piggin 701c2febafcSKirill A. Shutemov pmd = pmd_offset(pud, addr); 702c0eb315aSNicholas Piggin if (pmd_none(*pmd)) 703c0eb315aSNicholas Piggin return NULL; 704c0eb315aSNicholas Piggin if (pmd_leaf(*pmd)) 705c0eb315aSNicholas Piggin return pmd_page(*pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 706c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pmd_bad(*pmd))) 707c2febafcSKirill A. Shutemov return NULL; 708db64fe02SNick Piggin 70948667e7aSChristoph Lameter ptep = pte_offset_map(pmd, addr); 71048667e7aSChristoph Lameter pte = *ptep; 71148667e7aSChristoph Lameter if (pte_present(pte)) 712add688fbSmalc page = pte_page(pte); 71348667e7aSChristoph Lameter pte_unmap(ptep); 714c0eb315aSNicholas Piggin 715add688fbSmalc return page; 716ece86e22SJianyu Zhan } 717ece86e22SJianyu Zhan EXPORT_SYMBOL(vmalloc_to_page); 718ece86e22SJianyu Zhan 719add688fbSmalc /* 720add688fbSmalc * Map a vmalloc()-space virtual address to the physical page frame number. 721add688fbSmalc */ 722add688fbSmalc unsigned long vmalloc_to_pfn(const void *vmalloc_addr) 723add688fbSmalc { 724add688fbSmalc return page_to_pfn(vmalloc_to_page(vmalloc_addr)); 725add688fbSmalc } 726add688fbSmalc EXPORT_SYMBOL(vmalloc_to_pfn); 727add688fbSmalc 728db64fe02SNick Piggin 729db64fe02SNick Piggin /*** Global kva allocator ***/ 730db64fe02SNick Piggin 731bb850f4dSUladzislau Rezki (Sony) #define DEBUG_AUGMENT_PROPAGATE_CHECK 0 732a6cf4e0fSUladzislau Rezki (Sony) #define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0 733bb850f4dSUladzislau Rezki (Sony) 734db64fe02SNick Piggin 735db64fe02SNick Piggin static DEFINE_SPINLOCK(vmap_area_lock); 736e36176beSUladzislau Rezki (Sony) static DEFINE_SPINLOCK(free_vmap_area_lock); 737f1c4069eSJoonsoo Kim /* Export for kexec only */ 738f1c4069eSJoonsoo Kim LIST_HEAD(vmap_area_list); 73989699605SNick Piggin static struct rb_root vmap_area_root = RB_ROOT; 74068ad4a33SUladzislau Rezki (Sony) static bool vmap_initialized __read_mostly; 74189699605SNick Piggin 74296e2db45SUladzislau Rezki (Sony) static struct rb_root purge_vmap_area_root = RB_ROOT; 74396e2db45SUladzislau Rezki (Sony) static LIST_HEAD(purge_vmap_area_list); 74496e2db45SUladzislau Rezki (Sony) static DEFINE_SPINLOCK(purge_vmap_area_lock); 74596e2db45SUladzislau Rezki (Sony) 74668ad4a33SUladzislau Rezki (Sony) /* 74768ad4a33SUladzislau Rezki (Sony) * This kmem_cache is used for vmap_area objects. Instead of 74868ad4a33SUladzislau Rezki (Sony) * allocating from slab we reuse an object from this cache to 74968ad4a33SUladzislau Rezki (Sony) * make things faster. Especially in "no edge" splitting of 75068ad4a33SUladzislau Rezki (Sony) * free block. 75168ad4a33SUladzislau Rezki (Sony) */ 75268ad4a33SUladzislau Rezki (Sony) static struct kmem_cache *vmap_area_cachep; 75389699605SNick Piggin 75468ad4a33SUladzislau Rezki (Sony) /* 75568ad4a33SUladzislau Rezki (Sony) * This linked list is used in pair with free_vmap_area_root. 75668ad4a33SUladzislau Rezki (Sony) * It gives O(1) access to prev/next to perform fast coalescing. 75768ad4a33SUladzislau Rezki (Sony) */ 75868ad4a33SUladzislau Rezki (Sony) static LIST_HEAD(free_vmap_area_list); 75968ad4a33SUladzislau Rezki (Sony) 76068ad4a33SUladzislau Rezki (Sony) /* 76168ad4a33SUladzislau Rezki (Sony) * This augment red-black tree represents the free vmap space. 76268ad4a33SUladzislau Rezki (Sony) * All vmap_area objects in this tree are sorted by va->va_start 76368ad4a33SUladzislau Rezki (Sony) * address. It is used for allocation and merging when a vmap 76468ad4a33SUladzislau Rezki (Sony) * object is released. 76568ad4a33SUladzislau Rezki (Sony) * 76668ad4a33SUladzislau Rezki (Sony) * Each vmap_area node contains a maximum available free block 76768ad4a33SUladzislau Rezki (Sony) * of its sub-tree, right or left. Therefore it is possible to 76868ad4a33SUladzislau Rezki (Sony) * find a lowest match of free area. 76968ad4a33SUladzislau Rezki (Sony) */ 77068ad4a33SUladzislau Rezki (Sony) static struct rb_root free_vmap_area_root = RB_ROOT; 77168ad4a33SUladzislau Rezki (Sony) 77282dd23e8SUladzislau Rezki (Sony) /* 77382dd23e8SUladzislau Rezki (Sony) * Preload a CPU with one object for "no edge" split case. The 77482dd23e8SUladzislau Rezki (Sony) * aim is to get rid of allocations from the atomic context, thus 77582dd23e8SUladzislau Rezki (Sony) * to use more permissive allocation masks. 77682dd23e8SUladzislau Rezki (Sony) */ 77782dd23e8SUladzislau Rezki (Sony) static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node); 77882dd23e8SUladzislau Rezki (Sony) 77968ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 78068ad4a33SUladzislau Rezki (Sony) va_size(struct vmap_area *va) 78168ad4a33SUladzislau Rezki (Sony) { 78268ad4a33SUladzislau Rezki (Sony) return (va->va_end - va->va_start); 78368ad4a33SUladzislau Rezki (Sony) } 78468ad4a33SUladzislau Rezki (Sony) 78568ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 78668ad4a33SUladzislau Rezki (Sony) get_subtree_max_size(struct rb_node *node) 78768ad4a33SUladzislau Rezki (Sony) { 78868ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 78968ad4a33SUladzislau Rezki (Sony) 79068ad4a33SUladzislau Rezki (Sony) va = rb_entry_safe(node, struct vmap_area, rb_node); 79168ad4a33SUladzislau Rezki (Sony) return va ? va->subtree_max_size : 0; 79268ad4a33SUladzislau Rezki (Sony) } 79368ad4a33SUladzislau Rezki (Sony) 794315cc066SMichel Lespinasse RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb, 795315cc066SMichel Lespinasse struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size) 79668ad4a33SUladzislau Rezki (Sony) 79768ad4a33SUladzislau Rezki (Sony) static void purge_vmap_area_lazy(void); 79868ad4a33SUladzislau Rezki (Sony) static BLOCKING_NOTIFIER_HEAD(vmap_notify_list); 799690467c8SUladzislau Rezki (Sony) static void drain_vmap_area_work(struct work_struct *work); 800690467c8SUladzislau Rezki (Sony) static DECLARE_WORK(drain_vmap_work, drain_vmap_area_work); 801db64fe02SNick Piggin 80297105f0aSRoman Gushchin static atomic_long_t nr_vmalloc_pages; 80397105f0aSRoman Gushchin 80497105f0aSRoman Gushchin unsigned long vmalloc_nr_pages(void) 80597105f0aSRoman Gushchin { 80697105f0aSRoman Gushchin return atomic_long_read(&nr_vmalloc_pages); 80797105f0aSRoman Gushchin } 80897105f0aSRoman Gushchin 809153090f2SBaoquan He /* Look up the first VA which satisfies addr < va_end, NULL if none. */ 810f181234aSChen Wandun static struct vmap_area *find_vmap_area_exceed_addr(unsigned long addr) 811f181234aSChen Wandun { 812f181234aSChen Wandun struct vmap_area *va = NULL; 813f181234aSChen Wandun struct rb_node *n = vmap_area_root.rb_node; 814f181234aSChen Wandun 8154aff1dc4SAndrey Konovalov addr = (unsigned long)kasan_reset_tag((void *)addr); 8164aff1dc4SAndrey Konovalov 817f181234aSChen Wandun while (n) { 818f181234aSChen Wandun struct vmap_area *tmp; 819f181234aSChen Wandun 820f181234aSChen Wandun tmp = rb_entry(n, struct vmap_area, rb_node); 821f181234aSChen Wandun if (tmp->va_end > addr) { 822f181234aSChen Wandun va = tmp; 823f181234aSChen Wandun if (tmp->va_start <= addr) 824f181234aSChen Wandun break; 825f181234aSChen Wandun 826f181234aSChen Wandun n = n->rb_left; 827f181234aSChen Wandun } else 828f181234aSChen Wandun n = n->rb_right; 829f181234aSChen Wandun } 830f181234aSChen Wandun 831f181234aSChen Wandun return va; 832f181234aSChen Wandun } 833f181234aSChen Wandun 834899c6efeSUladzislau Rezki (Sony) static struct vmap_area *__find_vmap_area(unsigned long addr, struct rb_root *root) 8351da177e4SLinus Torvalds { 836899c6efeSUladzislau Rezki (Sony) struct rb_node *n = root->rb_node; 837db64fe02SNick Piggin 8384aff1dc4SAndrey Konovalov addr = (unsigned long)kasan_reset_tag((void *)addr); 8394aff1dc4SAndrey Konovalov 840db64fe02SNick Piggin while (n) { 841db64fe02SNick Piggin struct vmap_area *va; 842db64fe02SNick Piggin 843db64fe02SNick Piggin va = rb_entry(n, struct vmap_area, rb_node); 844db64fe02SNick Piggin if (addr < va->va_start) 845db64fe02SNick Piggin n = n->rb_left; 846cef2ac3fSHATAYAMA Daisuke else if (addr >= va->va_end) 847db64fe02SNick Piggin n = n->rb_right; 848db64fe02SNick Piggin else 849db64fe02SNick Piggin return va; 850db64fe02SNick Piggin } 851db64fe02SNick Piggin 852db64fe02SNick Piggin return NULL; 853db64fe02SNick Piggin } 854db64fe02SNick Piggin 85568ad4a33SUladzislau Rezki (Sony) /* 85668ad4a33SUladzislau Rezki (Sony) * This function returns back addresses of parent node 85768ad4a33SUladzislau Rezki (Sony) * and its left or right link for further processing. 8589c801f61SUladzislau Rezki (Sony) * 8599c801f61SUladzislau Rezki (Sony) * Otherwise NULL is returned. In that case all further 8609c801f61SUladzislau Rezki (Sony) * steps regarding inserting of conflicting overlap range 8619c801f61SUladzislau Rezki (Sony) * have to be declined and actually considered as a bug. 86268ad4a33SUladzislau Rezki (Sony) */ 86368ad4a33SUladzislau Rezki (Sony) static __always_inline struct rb_node ** 86468ad4a33SUladzislau Rezki (Sony) find_va_links(struct vmap_area *va, 86568ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct rb_node *from, 86668ad4a33SUladzislau Rezki (Sony) struct rb_node **parent) 867db64fe02SNick Piggin { 868170168d0SNamhyung Kim struct vmap_area *tmp_va; 86968ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 870db64fe02SNick Piggin 87168ad4a33SUladzislau Rezki (Sony) if (root) { 87268ad4a33SUladzislau Rezki (Sony) link = &root->rb_node; 87368ad4a33SUladzislau Rezki (Sony) if (unlikely(!*link)) { 87468ad4a33SUladzislau Rezki (Sony) *parent = NULL; 87568ad4a33SUladzislau Rezki (Sony) return link; 87668ad4a33SUladzislau Rezki (Sony) } 87768ad4a33SUladzislau Rezki (Sony) } else { 87868ad4a33SUladzislau Rezki (Sony) link = &from; 87968ad4a33SUladzislau Rezki (Sony) } 88068ad4a33SUladzislau Rezki (Sony) 88168ad4a33SUladzislau Rezki (Sony) /* 88268ad4a33SUladzislau Rezki (Sony) * Go to the bottom of the tree. When we hit the last point 88368ad4a33SUladzislau Rezki (Sony) * we end up with parent rb_node and correct direction, i name 88468ad4a33SUladzislau Rezki (Sony) * it link, where the new va->rb_node will be attached to. 88568ad4a33SUladzislau Rezki (Sony) */ 88668ad4a33SUladzislau Rezki (Sony) do { 88768ad4a33SUladzislau Rezki (Sony) tmp_va = rb_entry(*link, struct vmap_area, rb_node); 88868ad4a33SUladzislau Rezki (Sony) 88968ad4a33SUladzislau Rezki (Sony) /* 89068ad4a33SUladzislau Rezki (Sony) * During the traversal we also do some sanity check. 89168ad4a33SUladzislau Rezki (Sony) * Trigger the BUG() if there are sides(left/right) 89268ad4a33SUladzislau Rezki (Sony) * or full overlaps. 89368ad4a33SUladzislau Rezki (Sony) */ 894753df96bSBaoquan He if (va->va_end <= tmp_va->va_start) 89568ad4a33SUladzislau Rezki (Sony) link = &(*link)->rb_left; 896753df96bSBaoquan He else if (va->va_start >= tmp_va->va_end) 89768ad4a33SUladzislau Rezki (Sony) link = &(*link)->rb_right; 8989c801f61SUladzislau Rezki (Sony) else { 8999c801f61SUladzislau Rezki (Sony) WARN(1, "vmalloc bug: 0x%lx-0x%lx overlaps with 0x%lx-0x%lx\n", 9009c801f61SUladzislau Rezki (Sony) va->va_start, va->va_end, tmp_va->va_start, tmp_va->va_end); 9019c801f61SUladzislau Rezki (Sony) 9029c801f61SUladzislau Rezki (Sony) return NULL; 9039c801f61SUladzislau Rezki (Sony) } 90468ad4a33SUladzislau Rezki (Sony) } while (*link); 90568ad4a33SUladzislau Rezki (Sony) 90668ad4a33SUladzislau Rezki (Sony) *parent = &tmp_va->rb_node; 90768ad4a33SUladzislau Rezki (Sony) return link; 908db64fe02SNick Piggin } 909db64fe02SNick Piggin 91068ad4a33SUladzislau Rezki (Sony) static __always_inline struct list_head * 91168ad4a33SUladzislau Rezki (Sony) get_va_next_sibling(struct rb_node *parent, struct rb_node **link) 91268ad4a33SUladzislau Rezki (Sony) { 91368ad4a33SUladzislau Rezki (Sony) struct list_head *list; 914db64fe02SNick Piggin 91568ad4a33SUladzislau Rezki (Sony) if (unlikely(!parent)) 91668ad4a33SUladzislau Rezki (Sony) /* 91768ad4a33SUladzislau Rezki (Sony) * The red-black tree where we try to find VA neighbors 91868ad4a33SUladzislau Rezki (Sony) * before merging or inserting is empty, i.e. it means 91968ad4a33SUladzislau Rezki (Sony) * there is no free vmap space. Normally it does not 92068ad4a33SUladzislau Rezki (Sony) * happen but we handle this case anyway. 92168ad4a33SUladzislau Rezki (Sony) */ 92268ad4a33SUladzislau Rezki (Sony) return NULL; 92368ad4a33SUladzislau Rezki (Sony) 92468ad4a33SUladzislau Rezki (Sony) list = &rb_entry(parent, struct vmap_area, rb_node)->list; 92568ad4a33SUladzislau Rezki (Sony) return (&parent->rb_right == link ? list->next : list); 926db64fe02SNick Piggin } 927db64fe02SNick Piggin 92868ad4a33SUladzislau Rezki (Sony) static __always_inline void 9298eb510dbSUladzislau Rezki (Sony) __link_va(struct vmap_area *va, struct rb_root *root, 9308eb510dbSUladzislau Rezki (Sony) struct rb_node *parent, struct rb_node **link, 9318eb510dbSUladzislau Rezki (Sony) struct list_head *head, bool augment) 93268ad4a33SUladzislau Rezki (Sony) { 93368ad4a33SUladzislau Rezki (Sony) /* 93468ad4a33SUladzislau Rezki (Sony) * VA is still not in the list, but we can 93568ad4a33SUladzislau Rezki (Sony) * identify its future previous list_head node. 93668ad4a33SUladzislau Rezki (Sony) */ 93768ad4a33SUladzislau Rezki (Sony) if (likely(parent)) { 93868ad4a33SUladzislau Rezki (Sony) head = &rb_entry(parent, struct vmap_area, rb_node)->list; 93968ad4a33SUladzislau Rezki (Sony) if (&parent->rb_right != link) 94068ad4a33SUladzislau Rezki (Sony) head = head->prev; 94168ad4a33SUladzislau Rezki (Sony) } 942db64fe02SNick Piggin 94368ad4a33SUladzislau Rezki (Sony) /* Insert to the rb-tree */ 94468ad4a33SUladzislau Rezki (Sony) rb_link_node(&va->rb_node, parent, link); 9458eb510dbSUladzislau Rezki (Sony) if (augment) { 94668ad4a33SUladzislau Rezki (Sony) /* 94768ad4a33SUladzislau Rezki (Sony) * Some explanation here. Just perform simple insertion 94868ad4a33SUladzislau Rezki (Sony) * to the tree. We do not set va->subtree_max_size to 94968ad4a33SUladzislau Rezki (Sony) * its current size before calling rb_insert_augmented(). 950153090f2SBaoquan He * It is because we populate the tree from the bottom 95168ad4a33SUladzislau Rezki (Sony) * to parent levels when the node _is_ in the tree. 95268ad4a33SUladzislau Rezki (Sony) * 95368ad4a33SUladzislau Rezki (Sony) * Therefore we set subtree_max_size to zero after insertion, 95468ad4a33SUladzislau Rezki (Sony) * to let __augment_tree_propagate_from() puts everything to 95568ad4a33SUladzislau Rezki (Sony) * the correct order later on. 95668ad4a33SUladzislau Rezki (Sony) */ 95768ad4a33SUladzislau Rezki (Sony) rb_insert_augmented(&va->rb_node, 95868ad4a33SUladzislau Rezki (Sony) root, &free_vmap_area_rb_augment_cb); 95968ad4a33SUladzislau Rezki (Sony) va->subtree_max_size = 0; 96068ad4a33SUladzislau Rezki (Sony) } else { 96168ad4a33SUladzislau Rezki (Sony) rb_insert_color(&va->rb_node, root); 96268ad4a33SUladzislau Rezki (Sony) } 96368ad4a33SUladzislau Rezki (Sony) 96468ad4a33SUladzislau Rezki (Sony) /* Address-sort this list */ 96568ad4a33SUladzislau Rezki (Sony) list_add(&va->list, head); 96668ad4a33SUladzislau Rezki (Sony) } 96768ad4a33SUladzislau Rezki (Sony) 96868ad4a33SUladzislau Rezki (Sony) static __always_inline void 9698eb510dbSUladzislau Rezki (Sony) link_va(struct vmap_area *va, struct rb_root *root, 9708eb510dbSUladzislau Rezki (Sony) struct rb_node *parent, struct rb_node **link, 9718eb510dbSUladzislau Rezki (Sony) struct list_head *head) 9728eb510dbSUladzislau Rezki (Sony) { 9738eb510dbSUladzislau Rezki (Sony) __link_va(va, root, parent, link, head, false); 9748eb510dbSUladzislau Rezki (Sony) } 9758eb510dbSUladzislau Rezki (Sony) 9768eb510dbSUladzislau Rezki (Sony) static __always_inline void 9778eb510dbSUladzislau Rezki (Sony) link_va_augment(struct vmap_area *va, struct rb_root *root, 9788eb510dbSUladzislau Rezki (Sony) struct rb_node *parent, struct rb_node **link, 9798eb510dbSUladzislau Rezki (Sony) struct list_head *head) 9808eb510dbSUladzislau Rezki (Sony) { 9818eb510dbSUladzislau Rezki (Sony) __link_va(va, root, parent, link, head, true); 9828eb510dbSUladzislau Rezki (Sony) } 9838eb510dbSUladzislau Rezki (Sony) 9848eb510dbSUladzislau Rezki (Sony) static __always_inline void 9858eb510dbSUladzislau Rezki (Sony) __unlink_va(struct vmap_area *va, struct rb_root *root, bool augment) 98668ad4a33SUladzislau Rezki (Sony) { 987460e42d1SUladzislau Rezki (Sony) if (WARN_ON(RB_EMPTY_NODE(&va->rb_node))) 988460e42d1SUladzislau Rezki (Sony) return; 989460e42d1SUladzislau Rezki (Sony) 9908eb510dbSUladzislau Rezki (Sony) if (augment) 99168ad4a33SUladzislau Rezki (Sony) rb_erase_augmented(&va->rb_node, 99268ad4a33SUladzislau Rezki (Sony) root, &free_vmap_area_rb_augment_cb); 99368ad4a33SUladzislau Rezki (Sony) else 99468ad4a33SUladzislau Rezki (Sony) rb_erase(&va->rb_node, root); 99568ad4a33SUladzislau Rezki (Sony) 9965d7a7c54SUladzislau Rezki (Sony) list_del_init(&va->list); 99768ad4a33SUladzislau Rezki (Sony) RB_CLEAR_NODE(&va->rb_node); 99868ad4a33SUladzislau Rezki (Sony) } 99968ad4a33SUladzislau Rezki (Sony) 10008eb510dbSUladzislau Rezki (Sony) static __always_inline void 10018eb510dbSUladzislau Rezki (Sony) unlink_va(struct vmap_area *va, struct rb_root *root) 10028eb510dbSUladzislau Rezki (Sony) { 10038eb510dbSUladzislau Rezki (Sony) __unlink_va(va, root, false); 10048eb510dbSUladzislau Rezki (Sony) } 10058eb510dbSUladzislau Rezki (Sony) 10068eb510dbSUladzislau Rezki (Sony) static __always_inline void 10078eb510dbSUladzislau Rezki (Sony) unlink_va_augment(struct vmap_area *va, struct rb_root *root) 10088eb510dbSUladzislau Rezki (Sony) { 10098eb510dbSUladzislau Rezki (Sony) __unlink_va(va, root, true); 10108eb510dbSUladzislau Rezki (Sony) } 10118eb510dbSUladzislau Rezki (Sony) 1012bb850f4dSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_PROPAGATE_CHECK 1013c3385e84SJiapeng Chong /* 1014c3385e84SJiapeng Chong * Gets called when remove the node and rotate. 1015c3385e84SJiapeng Chong */ 1016c3385e84SJiapeng Chong static __always_inline unsigned long 1017c3385e84SJiapeng Chong compute_subtree_max_size(struct vmap_area *va) 1018c3385e84SJiapeng Chong { 1019c3385e84SJiapeng Chong return max3(va_size(va), 1020c3385e84SJiapeng Chong get_subtree_max_size(va->rb_node.rb_left), 1021c3385e84SJiapeng Chong get_subtree_max_size(va->rb_node.rb_right)); 1022c3385e84SJiapeng Chong } 1023c3385e84SJiapeng Chong 1024bb850f4dSUladzislau Rezki (Sony) static void 1025da27c9edSUladzislau Rezki (Sony) augment_tree_propagate_check(void) 1026bb850f4dSUladzislau Rezki (Sony) { 1027bb850f4dSUladzislau Rezki (Sony) struct vmap_area *va; 1028da27c9edSUladzislau Rezki (Sony) unsigned long computed_size; 1029bb850f4dSUladzislau Rezki (Sony) 1030da27c9edSUladzislau Rezki (Sony) list_for_each_entry(va, &free_vmap_area_list, list) { 1031da27c9edSUladzislau Rezki (Sony) computed_size = compute_subtree_max_size(va); 1032da27c9edSUladzislau Rezki (Sony) if (computed_size != va->subtree_max_size) 1033bb850f4dSUladzislau Rezki (Sony) pr_emerg("tree is corrupted: %lu, %lu\n", 1034bb850f4dSUladzislau Rezki (Sony) va_size(va), va->subtree_max_size); 1035bb850f4dSUladzislau Rezki (Sony) } 1036bb850f4dSUladzislau Rezki (Sony) } 1037bb850f4dSUladzislau Rezki (Sony) #endif 1038bb850f4dSUladzislau Rezki (Sony) 103968ad4a33SUladzislau Rezki (Sony) /* 104068ad4a33SUladzislau Rezki (Sony) * This function populates subtree_max_size from bottom to upper 104168ad4a33SUladzislau Rezki (Sony) * levels starting from VA point. The propagation must be done 104268ad4a33SUladzislau Rezki (Sony) * when VA size is modified by changing its va_start/va_end. Or 104368ad4a33SUladzislau Rezki (Sony) * in case of newly inserting of VA to the tree. 104468ad4a33SUladzislau Rezki (Sony) * 104568ad4a33SUladzislau Rezki (Sony) * It means that __augment_tree_propagate_from() must be called: 104668ad4a33SUladzislau Rezki (Sony) * - After VA has been inserted to the tree(free path); 104768ad4a33SUladzislau Rezki (Sony) * - After VA has been shrunk(allocation path); 104868ad4a33SUladzislau Rezki (Sony) * - After VA has been increased(merging path). 104968ad4a33SUladzislau Rezki (Sony) * 105068ad4a33SUladzislau Rezki (Sony) * Please note that, it does not mean that upper parent nodes 105168ad4a33SUladzislau Rezki (Sony) * and their subtree_max_size are recalculated all the time up 105268ad4a33SUladzislau Rezki (Sony) * to the root node. 105368ad4a33SUladzislau Rezki (Sony) * 105468ad4a33SUladzislau Rezki (Sony) * 4--8 105568ad4a33SUladzislau Rezki (Sony) * /\ 105668ad4a33SUladzislau Rezki (Sony) * / \ 105768ad4a33SUladzislau Rezki (Sony) * / \ 105868ad4a33SUladzislau Rezki (Sony) * 2--2 8--8 105968ad4a33SUladzislau Rezki (Sony) * 106068ad4a33SUladzislau Rezki (Sony) * For example if we modify the node 4, shrinking it to 2, then 106168ad4a33SUladzislau Rezki (Sony) * no any modification is required. If we shrink the node 2 to 1 106268ad4a33SUladzislau Rezki (Sony) * its subtree_max_size is updated only, and set to 1. If we shrink 106368ad4a33SUladzislau Rezki (Sony) * the node 8 to 6, then its subtree_max_size is set to 6 and parent 106468ad4a33SUladzislau Rezki (Sony) * node becomes 4--6. 106568ad4a33SUladzislau Rezki (Sony) */ 106668ad4a33SUladzislau Rezki (Sony) static __always_inline void 106768ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(struct vmap_area *va) 106868ad4a33SUladzislau Rezki (Sony) { 106968ad4a33SUladzislau Rezki (Sony) /* 107015ae144fSUladzislau Rezki (Sony) * Populate the tree from bottom towards the root until 107115ae144fSUladzislau Rezki (Sony) * the calculated maximum available size of checked node 107215ae144fSUladzislau Rezki (Sony) * is equal to its current one. 107368ad4a33SUladzislau Rezki (Sony) */ 107415ae144fSUladzislau Rezki (Sony) free_vmap_area_rb_augment_cb_propagate(&va->rb_node, NULL); 1075bb850f4dSUladzislau Rezki (Sony) 1076bb850f4dSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_PROPAGATE_CHECK 1077da27c9edSUladzislau Rezki (Sony) augment_tree_propagate_check(); 1078bb850f4dSUladzislau Rezki (Sony) #endif 107968ad4a33SUladzislau Rezki (Sony) } 108068ad4a33SUladzislau Rezki (Sony) 108168ad4a33SUladzislau Rezki (Sony) static void 108268ad4a33SUladzislau Rezki (Sony) insert_vmap_area(struct vmap_area *va, 108368ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 108468ad4a33SUladzislau Rezki (Sony) { 108568ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 108668ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 108768ad4a33SUladzislau Rezki (Sony) 108868ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 10899c801f61SUladzislau Rezki (Sony) if (link) 109068ad4a33SUladzislau Rezki (Sony) link_va(va, root, parent, link, head); 109168ad4a33SUladzislau Rezki (Sony) } 109268ad4a33SUladzislau Rezki (Sony) 109368ad4a33SUladzislau Rezki (Sony) static void 109468ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(struct vmap_area *va, 109568ad4a33SUladzislau Rezki (Sony) struct rb_node *from, struct rb_root *root, 109668ad4a33SUladzislau Rezki (Sony) struct list_head *head) 109768ad4a33SUladzislau Rezki (Sony) { 109868ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 109968ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 110068ad4a33SUladzislau Rezki (Sony) 110168ad4a33SUladzislau Rezki (Sony) if (from) 110268ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, NULL, from, &parent); 110368ad4a33SUladzislau Rezki (Sony) else 110468ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 110568ad4a33SUladzislau Rezki (Sony) 11069c801f61SUladzislau Rezki (Sony) if (link) { 11078eb510dbSUladzislau Rezki (Sony) link_va_augment(va, root, parent, link, head); 110868ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 110968ad4a33SUladzislau Rezki (Sony) } 11109c801f61SUladzislau Rezki (Sony) } 111168ad4a33SUladzislau Rezki (Sony) 111268ad4a33SUladzislau Rezki (Sony) /* 111368ad4a33SUladzislau Rezki (Sony) * Merge de-allocated chunk of VA memory with previous 111468ad4a33SUladzislau Rezki (Sony) * and next free blocks. If coalesce is not done a new 111568ad4a33SUladzislau Rezki (Sony) * free area is inserted. If VA has been merged, it is 111668ad4a33SUladzislau Rezki (Sony) * freed. 11179c801f61SUladzislau Rezki (Sony) * 11189c801f61SUladzislau Rezki (Sony) * Please note, it can return NULL in case of overlap 11199c801f61SUladzislau Rezki (Sony) * ranges, followed by WARN() report. Despite it is a 11209c801f61SUladzislau Rezki (Sony) * buggy behaviour, a system can be alive and keep 11219c801f61SUladzislau Rezki (Sony) * ongoing. 112268ad4a33SUladzislau Rezki (Sony) */ 11233c5c3cfbSDaniel Axtens static __always_inline struct vmap_area * 11248eb510dbSUladzislau Rezki (Sony) __merge_or_add_vmap_area(struct vmap_area *va, 11258eb510dbSUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head, bool augment) 112668ad4a33SUladzislau Rezki (Sony) { 112768ad4a33SUladzislau Rezki (Sony) struct vmap_area *sibling; 112868ad4a33SUladzislau Rezki (Sony) struct list_head *next; 112968ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 113068ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 113168ad4a33SUladzislau Rezki (Sony) bool merged = false; 113268ad4a33SUladzislau Rezki (Sony) 113368ad4a33SUladzislau Rezki (Sony) /* 113468ad4a33SUladzislau Rezki (Sony) * Find a place in the tree where VA potentially will be 113568ad4a33SUladzislau Rezki (Sony) * inserted, unless it is merged with its sibling/siblings. 113668ad4a33SUladzislau Rezki (Sony) */ 113768ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 11389c801f61SUladzislau Rezki (Sony) if (!link) 11399c801f61SUladzislau Rezki (Sony) return NULL; 114068ad4a33SUladzislau Rezki (Sony) 114168ad4a33SUladzislau Rezki (Sony) /* 114268ad4a33SUladzislau Rezki (Sony) * Get next node of VA to check if merging can be done. 114368ad4a33SUladzislau Rezki (Sony) */ 114468ad4a33SUladzislau Rezki (Sony) next = get_va_next_sibling(parent, link); 114568ad4a33SUladzislau Rezki (Sony) if (unlikely(next == NULL)) 114668ad4a33SUladzislau Rezki (Sony) goto insert; 114768ad4a33SUladzislau Rezki (Sony) 114868ad4a33SUladzislau Rezki (Sony) /* 114968ad4a33SUladzislau Rezki (Sony) * start end 115068ad4a33SUladzislau Rezki (Sony) * | | 115168ad4a33SUladzislau Rezki (Sony) * |<------VA------>|<-----Next----->| 115268ad4a33SUladzislau Rezki (Sony) * | | 115368ad4a33SUladzislau Rezki (Sony) * start end 115468ad4a33SUladzislau Rezki (Sony) */ 115568ad4a33SUladzislau Rezki (Sony) if (next != head) { 115668ad4a33SUladzislau Rezki (Sony) sibling = list_entry(next, struct vmap_area, list); 115768ad4a33SUladzislau Rezki (Sony) if (sibling->va_start == va->va_end) { 115868ad4a33SUladzislau Rezki (Sony) sibling->va_start = va->va_start; 115968ad4a33SUladzislau Rezki (Sony) 116068ad4a33SUladzislau Rezki (Sony) /* Free vmap_area object. */ 116168ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 116268ad4a33SUladzislau Rezki (Sony) 116368ad4a33SUladzislau Rezki (Sony) /* Point to the new merged area. */ 116468ad4a33SUladzislau Rezki (Sony) va = sibling; 116568ad4a33SUladzislau Rezki (Sony) merged = true; 116668ad4a33SUladzislau Rezki (Sony) } 116768ad4a33SUladzislau Rezki (Sony) } 116868ad4a33SUladzislau Rezki (Sony) 116968ad4a33SUladzislau Rezki (Sony) /* 117068ad4a33SUladzislau Rezki (Sony) * start end 117168ad4a33SUladzislau Rezki (Sony) * | | 117268ad4a33SUladzislau Rezki (Sony) * |<-----Prev----->|<------VA------>| 117368ad4a33SUladzislau Rezki (Sony) * | | 117468ad4a33SUladzislau Rezki (Sony) * start end 117568ad4a33SUladzislau Rezki (Sony) */ 117668ad4a33SUladzislau Rezki (Sony) if (next->prev != head) { 117768ad4a33SUladzislau Rezki (Sony) sibling = list_entry(next->prev, struct vmap_area, list); 117868ad4a33SUladzislau Rezki (Sony) if (sibling->va_end == va->va_start) { 11795dd78640SUladzislau Rezki (Sony) /* 11805dd78640SUladzislau Rezki (Sony) * If both neighbors are coalesced, it is important 11815dd78640SUladzislau Rezki (Sony) * to unlink the "next" node first, followed by merging 11825dd78640SUladzislau Rezki (Sony) * with "previous" one. Otherwise the tree might not be 11835dd78640SUladzislau Rezki (Sony) * fully populated if a sibling's augmented value is 11845dd78640SUladzislau Rezki (Sony) * "normalized" because of rotation operations. 11855dd78640SUladzislau Rezki (Sony) */ 118654f63d9dSUladzislau Rezki (Sony) if (merged) 11878eb510dbSUladzislau Rezki (Sony) __unlink_va(va, root, augment); 118868ad4a33SUladzislau Rezki (Sony) 11895dd78640SUladzislau Rezki (Sony) sibling->va_end = va->va_end; 11905dd78640SUladzislau Rezki (Sony) 119168ad4a33SUladzislau Rezki (Sony) /* Free vmap_area object. */ 119268ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 11933c5c3cfbSDaniel Axtens 11943c5c3cfbSDaniel Axtens /* Point to the new merged area. */ 11953c5c3cfbSDaniel Axtens va = sibling; 11963c5c3cfbSDaniel Axtens merged = true; 119768ad4a33SUladzislau Rezki (Sony) } 119868ad4a33SUladzislau Rezki (Sony) } 119968ad4a33SUladzislau Rezki (Sony) 120068ad4a33SUladzislau Rezki (Sony) insert: 12015dd78640SUladzislau Rezki (Sony) if (!merged) 12028eb510dbSUladzislau Rezki (Sony) __link_va(va, root, parent, link, head, augment); 12033c5c3cfbSDaniel Axtens 120496e2db45SUladzislau Rezki (Sony) return va; 120596e2db45SUladzislau Rezki (Sony) } 120696e2db45SUladzislau Rezki (Sony) 120796e2db45SUladzislau Rezki (Sony) static __always_inline struct vmap_area * 12088eb510dbSUladzislau Rezki (Sony) merge_or_add_vmap_area(struct vmap_area *va, 12098eb510dbSUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 12108eb510dbSUladzislau Rezki (Sony) { 12118eb510dbSUladzislau Rezki (Sony) return __merge_or_add_vmap_area(va, root, head, false); 12128eb510dbSUladzislau Rezki (Sony) } 12138eb510dbSUladzislau Rezki (Sony) 12148eb510dbSUladzislau Rezki (Sony) static __always_inline struct vmap_area * 121596e2db45SUladzislau Rezki (Sony) merge_or_add_vmap_area_augment(struct vmap_area *va, 121696e2db45SUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 121796e2db45SUladzislau Rezki (Sony) { 12188eb510dbSUladzislau Rezki (Sony) va = __merge_or_add_vmap_area(va, root, head, true); 121996e2db45SUladzislau Rezki (Sony) if (va) 12205dd78640SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 122196e2db45SUladzislau Rezki (Sony) 12223c5c3cfbSDaniel Axtens return va; 122368ad4a33SUladzislau Rezki (Sony) } 122468ad4a33SUladzislau Rezki (Sony) 122568ad4a33SUladzislau Rezki (Sony) static __always_inline bool 122668ad4a33SUladzislau Rezki (Sony) is_within_this_va(struct vmap_area *va, unsigned long size, 122768ad4a33SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 122868ad4a33SUladzislau Rezki (Sony) { 122968ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr; 123068ad4a33SUladzislau Rezki (Sony) 123168ad4a33SUladzislau Rezki (Sony) if (va->va_start > vstart) 123268ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(va->va_start, align); 123368ad4a33SUladzislau Rezki (Sony) else 123468ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(vstart, align); 123568ad4a33SUladzislau Rezki (Sony) 123668ad4a33SUladzislau Rezki (Sony) /* Can be overflowed due to big size or alignment. */ 123768ad4a33SUladzislau Rezki (Sony) if (nva_start_addr + size < nva_start_addr || 123868ad4a33SUladzislau Rezki (Sony) nva_start_addr < vstart) 123968ad4a33SUladzislau Rezki (Sony) return false; 124068ad4a33SUladzislau Rezki (Sony) 124168ad4a33SUladzislau Rezki (Sony) return (nva_start_addr + size <= va->va_end); 124268ad4a33SUladzislau Rezki (Sony) } 124368ad4a33SUladzislau Rezki (Sony) 124468ad4a33SUladzislau Rezki (Sony) /* 124568ad4a33SUladzislau Rezki (Sony) * Find the first free block(lowest start address) in the tree, 124668ad4a33SUladzislau Rezki (Sony) * that will accomplish the request corresponding to passing 12479333fe98SUladzislau Rezki * parameters. Please note, with an alignment bigger than PAGE_SIZE, 12489333fe98SUladzislau Rezki * a search length is adjusted to account for worst case alignment 12499333fe98SUladzislau Rezki * overhead. 125068ad4a33SUladzislau Rezki (Sony) */ 125168ad4a33SUladzislau Rezki (Sony) static __always_inline struct vmap_area * 1252f9863be4SUladzislau Rezki (Sony) find_vmap_lowest_match(struct rb_root *root, unsigned long size, 1253f9863be4SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart, bool adjust_search_size) 125468ad4a33SUladzislau Rezki (Sony) { 125568ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 125668ad4a33SUladzislau Rezki (Sony) struct rb_node *node; 12579333fe98SUladzislau Rezki unsigned long length; 125868ad4a33SUladzislau Rezki (Sony) 125968ad4a33SUladzislau Rezki (Sony) /* Start from the root. */ 1260f9863be4SUladzislau Rezki (Sony) node = root->rb_node; 126168ad4a33SUladzislau Rezki (Sony) 12629333fe98SUladzislau Rezki /* Adjust the search size for alignment overhead. */ 12639333fe98SUladzislau Rezki length = adjust_search_size ? size + align - 1 : size; 12649333fe98SUladzislau Rezki 126568ad4a33SUladzislau Rezki (Sony) while (node) { 126668ad4a33SUladzislau Rezki (Sony) va = rb_entry(node, struct vmap_area, rb_node); 126768ad4a33SUladzislau Rezki (Sony) 12689333fe98SUladzislau Rezki if (get_subtree_max_size(node->rb_left) >= length && 126968ad4a33SUladzislau Rezki (Sony) vstart < va->va_start) { 127068ad4a33SUladzislau Rezki (Sony) node = node->rb_left; 127168ad4a33SUladzislau Rezki (Sony) } else { 127268ad4a33SUladzislau Rezki (Sony) if (is_within_this_va(va, size, align, vstart)) 127368ad4a33SUladzislau Rezki (Sony) return va; 127468ad4a33SUladzislau Rezki (Sony) 127568ad4a33SUladzislau Rezki (Sony) /* 127668ad4a33SUladzislau Rezki (Sony) * Does not make sense to go deeper towards the right 127768ad4a33SUladzislau Rezki (Sony) * sub-tree if it does not have a free block that is 12789333fe98SUladzislau Rezki * equal or bigger to the requested search length. 127968ad4a33SUladzislau Rezki (Sony) */ 12809333fe98SUladzislau Rezki if (get_subtree_max_size(node->rb_right) >= length) { 128168ad4a33SUladzislau Rezki (Sony) node = node->rb_right; 128268ad4a33SUladzislau Rezki (Sony) continue; 128368ad4a33SUladzislau Rezki (Sony) } 128468ad4a33SUladzislau Rezki (Sony) 128568ad4a33SUladzislau Rezki (Sony) /* 12863806b041SAndrew Morton * OK. We roll back and find the first right sub-tree, 128768ad4a33SUladzislau Rezki (Sony) * that will satisfy the search criteria. It can happen 12889f531973SUladzislau Rezki (Sony) * due to "vstart" restriction or an alignment overhead 12899f531973SUladzislau Rezki (Sony) * that is bigger then PAGE_SIZE. 129068ad4a33SUladzislau Rezki (Sony) */ 129168ad4a33SUladzislau Rezki (Sony) while ((node = rb_parent(node))) { 129268ad4a33SUladzislau Rezki (Sony) va = rb_entry(node, struct vmap_area, rb_node); 129368ad4a33SUladzislau Rezki (Sony) if (is_within_this_va(va, size, align, vstart)) 129468ad4a33SUladzislau Rezki (Sony) return va; 129568ad4a33SUladzislau Rezki (Sony) 12969333fe98SUladzislau Rezki if (get_subtree_max_size(node->rb_right) >= length && 129768ad4a33SUladzislau Rezki (Sony) vstart <= va->va_start) { 12989f531973SUladzislau Rezki (Sony) /* 12999f531973SUladzislau Rezki (Sony) * Shift the vstart forward. Please note, we update it with 13009f531973SUladzislau Rezki (Sony) * parent's start address adding "1" because we do not want 13019f531973SUladzislau Rezki (Sony) * to enter same sub-tree after it has already been checked 13029f531973SUladzislau Rezki (Sony) * and no suitable free block found there. 13039f531973SUladzislau Rezki (Sony) */ 13049f531973SUladzislau Rezki (Sony) vstart = va->va_start + 1; 130568ad4a33SUladzislau Rezki (Sony) node = node->rb_right; 130668ad4a33SUladzislau Rezki (Sony) break; 130768ad4a33SUladzislau Rezki (Sony) } 130868ad4a33SUladzislau Rezki (Sony) } 130968ad4a33SUladzislau Rezki (Sony) } 131068ad4a33SUladzislau Rezki (Sony) } 131168ad4a33SUladzislau Rezki (Sony) 131268ad4a33SUladzislau Rezki (Sony) return NULL; 131368ad4a33SUladzislau Rezki (Sony) } 131468ad4a33SUladzislau Rezki (Sony) 1315a6cf4e0fSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 1316a6cf4e0fSUladzislau Rezki (Sony) #include <linux/random.h> 1317a6cf4e0fSUladzislau Rezki (Sony) 1318a6cf4e0fSUladzislau Rezki (Sony) static struct vmap_area * 1319bd1264c3SSong Liu find_vmap_lowest_linear_match(struct list_head *head, unsigned long size, 1320a6cf4e0fSUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 1321a6cf4e0fSUladzislau Rezki (Sony) { 1322a6cf4e0fSUladzislau Rezki (Sony) struct vmap_area *va; 1323a6cf4e0fSUladzislau Rezki (Sony) 1324bd1264c3SSong Liu list_for_each_entry(va, head, list) { 1325a6cf4e0fSUladzislau Rezki (Sony) if (!is_within_this_va(va, size, align, vstart)) 1326a6cf4e0fSUladzislau Rezki (Sony) continue; 1327a6cf4e0fSUladzislau Rezki (Sony) 1328a6cf4e0fSUladzislau Rezki (Sony) return va; 1329a6cf4e0fSUladzislau Rezki (Sony) } 1330a6cf4e0fSUladzislau Rezki (Sony) 1331a6cf4e0fSUladzislau Rezki (Sony) return NULL; 1332a6cf4e0fSUladzislau Rezki (Sony) } 1333a6cf4e0fSUladzislau Rezki (Sony) 1334a6cf4e0fSUladzislau Rezki (Sony) static void 1335bd1264c3SSong Liu find_vmap_lowest_match_check(struct rb_root *root, struct list_head *head, 1336bd1264c3SSong Liu unsigned long size, unsigned long align) 1337a6cf4e0fSUladzislau Rezki (Sony) { 1338a6cf4e0fSUladzislau Rezki (Sony) struct vmap_area *va_1, *va_2; 1339a6cf4e0fSUladzislau Rezki (Sony) unsigned long vstart; 1340a6cf4e0fSUladzislau Rezki (Sony) unsigned int rnd; 1341a6cf4e0fSUladzislau Rezki (Sony) 1342a6cf4e0fSUladzislau Rezki (Sony) get_random_bytes(&rnd, sizeof(rnd)); 1343a6cf4e0fSUladzislau Rezki (Sony) vstart = VMALLOC_START + rnd; 1344a6cf4e0fSUladzislau Rezki (Sony) 1345bd1264c3SSong Liu va_1 = find_vmap_lowest_match(root, size, align, vstart, false); 1346bd1264c3SSong Liu va_2 = find_vmap_lowest_linear_match(head, size, align, vstart); 1347a6cf4e0fSUladzislau Rezki (Sony) 1348a6cf4e0fSUladzislau Rezki (Sony) if (va_1 != va_2) 1349a6cf4e0fSUladzislau Rezki (Sony) pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n", 1350a6cf4e0fSUladzislau Rezki (Sony) va_1, va_2, vstart); 1351a6cf4e0fSUladzislau Rezki (Sony) } 1352a6cf4e0fSUladzislau Rezki (Sony) #endif 1353a6cf4e0fSUladzislau Rezki (Sony) 135468ad4a33SUladzislau Rezki (Sony) enum fit_type { 135568ad4a33SUladzislau Rezki (Sony) NOTHING_FIT = 0, 135668ad4a33SUladzislau Rezki (Sony) FL_FIT_TYPE = 1, /* full fit */ 135768ad4a33SUladzislau Rezki (Sony) LE_FIT_TYPE = 2, /* left edge fit */ 135868ad4a33SUladzislau Rezki (Sony) RE_FIT_TYPE = 3, /* right edge fit */ 135968ad4a33SUladzislau Rezki (Sony) NE_FIT_TYPE = 4 /* no edge fit */ 136068ad4a33SUladzislau Rezki (Sony) }; 136168ad4a33SUladzislau Rezki (Sony) 136268ad4a33SUladzislau Rezki (Sony) static __always_inline enum fit_type 136368ad4a33SUladzislau Rezki (Sony) classify_va_fit_type(struct vmap_area *va, 136468ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr, unsigned long size) 136568ad4a33SUladzislau Rezki (Sony) { 136668ad4a33SUladzislau Rezki (Sony) enum fit_type type; 136768ad4a33SUladzislau Rezki (Sony) 136868ad4a33SUladzislau Rezki (Sony) /* Check if it is within VA. */ 136968ad4a33SUladzislau Rezki (Sony) if (nva_start_addr < va->va_start || 137068ad4a33SUladzislau Rezki (Sony) nva_start_addr + size > va->va_end) 137168ad4a33SUladzislau Rezki (Sony) return NOTHING_FIT; 137268ad4a33SUladzislau Rezki (Sony) 137368ad4a33SUladzislau Rezki (Sony) /* Now classify. */ 137468ad4a33SUladzislau Rezki (Sony) if (va->va_start == nva_start_addr) { 137568ad4a33SUladzislau Rezki (Sony) if (va->va_end == nva_start_addr + size) 137668ad4a33SUladzislau Rezki (Sony) type = FL_FIT_TYPE; 137768ad4a33SUladzislau Rezki (Sony) else 137868ad4a33SUladzislau Rezki (Sony) type = LE_FIT_TYPE; 137968ad4a33SUladzislau Rezki (Sony) } else if (va->va_end == nva_start_addr + size) { 138068ad4a33SUladzislau Rezki (Sony) type = RE_FIT_TYPE; 138168ad4a33SUladzislau Rezki (Sony) } else { 138268ad4a33SUladzislau Rezki (Sony) type = NE_FIT_TYPE; 138368ad4a33SUladzislau Rezki (Sony) } 138468ad4a33SUladzislau Rezki (Sony) 138568ad4a33SUladzislau Rezki (Sony) return type; 138668ad4a33SUladzislau Rezki (Sony) } 138768ad4a33SUladzislau Rezki (Sony) 138868ad4a33SUladzislau Rezki (Sony) static __always_inline int 1389f9863be4SUladzislau Rezki (Sony) adjust_va_to_fit_type(struct rb_root *root, struct list_head *head, 1390f9863be4SUladzislau Rezki (Sony) struct vmap_area *va, unsigned long nva_start_addr, 1391f9863be4SUladzislau Rezki (Sony) unsigned long size) 139268ad4a33SUladzislau Rezki (Sony) { 13932c929233SArnd Bergmann struct vmap_area *lva = NULL; 13941b23ff80SBaoquan He enum fit_type type = classify_va_fit_type(va, nva_start_addr, size); 139568ad4a33SUladzislau Rezki (Sony) 139668ad4a33SUladzislau Rezki (Sony) if (type == FL_FIT_TYPE) { 139768ad4a33SUladzislau Rezki (Sony) /* 139868ad4a33SUladzislau Rezki (Sony) * No need to split VA, it fully fits. 139968ad4a33SUladzislau Rezki (Sony) * 140068ad4a33SUladzislau Rezki (Sony) * | | 140168ad4a33SUladzislau Rezki (Sony) * V NVA V 140268ad4a33SUladzislau Rezki (Sony) * |---------------| 140368ad4a33SUladzislau Rezki (Sony) */ 1404f9863be4SUladzislau Rezki (Sony) unlink_va_augment(va, root); 140568ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 140668ad4a33SUladzislau Rezki (Sony) } else if (type == LE_FIT_TYPE) { 140768ad4a33SUladzislau Rezki (Sony) /* 140868ad4a33SUladzislau Rezki (Sony) * Split left edge of fit VA. 140968ad4a33SUladzislau Rezki (Sony) * 141068ad4a33SUladzislau Rezki (Sony) * | | 141168ad4a33SUladzislau Rezki (Sony) * V NVA V R 141268ad4a33SUladzislau Rezki (Sony) * |-------|-------| 141368ad4a33SUladzislau Rezki (Sony) */ 141468ad4a33SUladzislau Rezki (Sony) va->va_start += size; 141568ad4a33SUladzislau Rezki (Sony) } else if (type == RE_FIT_TYPE) { 141668ad4a33SUladzislau Rezki (Sony) /* 141768ad4a33SUladzislau Rezki (Sony) * Split right edge of fit VA. 141868ad4a33SUladzislau Rezki (Sony) * 141968ad4a33SUladzislau Rezki (Sony) * | | 142068ad4a33SUladzislau Rezki (Sony) * L V NVA V 142168ad4a33SUladzislau Rezki (Sony) * |-------|-------| 142268ad4a33SUladzislau Rezki (Sony) */ 142368ad4a33SUladzislau Rezki (Sony) va->va_end = nva_start_addr; 142468ad4a33SUladzislau Rezki (Sony) } else if (type == NE_FIT_TYPE) { 142568ad4a33SUladzislau Rezki (Sony) /* 142668ad4a33SUladzislau Rezki (Sony) * Split no edge of fit VA. 142768ad4a33SUladzislau Rezki (Sony) * 142868ad4a33SUladzislau Rezki (Sony) * | | 142968ad4a33SUladzislau Rezki (Sony) * L V NVA V R 143068ad4a33SUladzislau Rezki (Sony) * |---|-------|---| 143168ad4a33SUladzislau Rezki (Sony) */ 143282dd23e8SUladzislau Rezki (Sony) lva = __this_cpu_xchg(ne_fit_preload_node, NULL); 143382dd23e8SUladzislau Rezki (Sony) if (unlikely(!lva)) { 143482dd23e8SUladzislau Rezki (Sony) /* 143582dd23e8SUladzislau Rezki (Sony) * For percpu allocator we do not do any pre-allocation 143682dd23e8SUladzislau Rezki (Sony) * and leave it as it is. The reason is it most likely 143782dd23e8SUladzislau Rezki (Sony) * never ends up with NE_FIT_TYPE splitting. In case of 143882dd23e8SUladzislau Rezki (Sony) * percpu allocations offsets and sizes are aligned to 143982dd23e8SUladzislau Rezki (Sony) * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE 144082dd23e8SUladzislau Rezki (Sony) * are its main fitting cases. 144182dd23e8SUladzislau Rezki (Sony) * 144282dd23e8SUladzislau Rezki (Sony) * There are a few exceptions though, as an example it is 144382dd23e8SUladzislau Rezki (Sony) * a first allocation (early boot up) when we have "one" 144482dd23e8SUladzislau Rezki (Sony) * big free space that has to be split. 1445060650a2SUladzislau Rezki (Sony) * 1446060650a2SUladzislau Rezki (Sony) * Also we can hit this path in case of regular "vmap" 1447060650a2SUladzislau Rezki (Sony) * allocations, if "this" current CPU was not preloaded. 1448060650a2SUladzislau Rezki (Sony) * See the comment in alloc_vmap_area() why. If so, then 1449060650a2SUladzislau Rezki (Sony) * GFP_NOWAIT is used instead to get an extra object for 1450060650a2SUladzislau Rezki (Sony) * split purpose. That is rare and most time does not 1451060650a2SUladzislau Rezki (Sony) * occur. 1452060650a2SUladzislau Rezki (Sony) * 1453060650a2SUladzislau Rezki (Sony) * What happens if an allocation gets failed. Basically, 1454060650a2SUladzislau Rezki (Sony) * an "overflow" path is triggered to purge lazily freed 1455060650a2SUladzislau Rezki (Sony) * areas to free some memory, then, the "retry" path is 1456060650a2SUladzislau Rezki (Sony) * triggered to repeat one more time. See more details 1457060650a2SUladzislau Rezki (Sony) * in alloc_vmap_area() function. 145882dd23e8SUladzislau Rezki (Sony) */ 145968ad4a33SUladzislau Rezki (Sony) lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT); 146082dd23e8SUladzislau Rezki (Sony) if (!lva) 146168ad4a33SUladzislau Rezki (Sony) return -1; 146282dd23e8SUladzislau Rezki (Sony) } 146368ad4a33SUladzislau Rezki (Sony) 146468ad4a33SUladzislau Rezki (Sony) /* 146568ad4a33SUladzislau Rezki (Sony) * Build the remainder. 146668ad4a33SUladzislau Rezki (Sony) */ 146768ad4a33SUladzislau Rezki (Sony) lva->va_start = va->va_start; 146868ad4a33SUladzislau Rezki (Sony) lva->va_end = nva_start_addr; 146968ad4a33SUladzislau Rezki (Sony) 147068ad4a33SUladzislau Rezki (Sony) /* 147168ad4a33SUladzislau Rezki (Sony) * Shrink this VA to remaining size. 147268ad4a33SUladzislau Rezki (Sony) */ 147368ad4a33SUladzislau Rezki (Sony) va->va_start = nva_start_addr + size; 147468ad4a33SUladzislau Rezki (Sony) } else { 147568ad4a33SUladzislau Rezki (Sony) return -1; 147668ad4a33SUladzislau Rezki (Sony) } 147768ad4a33SUladzislau Rezki (Sony) 147868ad4a33SUladzislau Rezki (Sony) if (type != FL_FIT_TYPE) { 147968ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 148068ad4a33SUladzislau Rezki (Sony) 14812c929233SArnd Bergmann if (lva) /* type == NE_FIT_TYPE */ 1482f9863be4SUladzislau Rezki (Sony) insert_vmap_area_augment(lva, &va->rb_node, root, head); 148368ad4a33SUladzislau Rezki (Sony) } 148468ad4a33SUladzislau Rezki (Sony) 148568ad4a33SUladzislau Rezki (Sony) return 0; 148668ad4a33SUladzislau Rezki (Sony) } 148768ad4a33SUladzislau Rezki (Sony) 148868ad4a33SUladzislau Rezki (Sony) /* 148968ad4a33SUladzislau Rezki (Sony) * Returns a start address of the newly allocated area, if success. 149068ad4a33SUladzislau Rezki (Sony) * Otherwise a vend is returned that indicates failure. 149168ad4a33SUladzislau Rezki (Sony) */ 149268ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 1493f9863be4SUladzislau Rezki (Sony) __alloc_vmap_area(struct rb_root *root, struct list_head *head, 1494f9863be4SUladzislau Rezki (Sony) unsigned long size, unsigned long align, 1495cacca6baSUladzislau Rezki (Sony) unsigned long vstart, unsigned long vend) 149668ad4a33SUladzislau Rezki (Sony) { 14979333fe98SUladzislau Rezki bool adjust_search_size = true; 149868ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr; 149968ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 150068ad4a33SUladzislau Rezki (Sony) int ret; 150168ad4a33SUladzislau Rezki (Sony) 15029333fe98SUladzislau Rezki /* 15039333fe98SUladzislau Rezki * Do not adjust when: 15049333fe98SUladzislau Rezki * a) align <= PAGE_SIZE, because it does not make any sense. 15059333fe98SUladzislau Rezki * All blocks(their start addresses) are at least PAGE_SIZE 15069333fe98SUladzislau Rezki * aligned anyway; 15079333fe98SUladzislau Rezki * b) a short range where a requested size corresponds to exactly 15089333fe98SUladzislau Rezki * specified [vstart:vend] interval and an alignment > PAGE_SIZE. 15099333fe98SUladzislau Rezki * With adjusted search length an allocation would not succeed. 15109333fe98SUladzislau Rezki */ 15119333fe98SUladzislau Rezki if (align <= PAGE_SIZE || (align > PAGE_SIZE && (vend - vstart) == size)) 15129333fe98SUladzislau Rezki adjust_search_size = false; 15139333fe98SUladzislau Rezki 1514f9863be4SUladzislau Rezki (Sony) va = find_vmap_lowest_match(root, size, align, vstart, adjust_search_size); 151568ad4a33SUladzislau Rezki (Sony) if (unlikely(!va)) 151668ad4a33SUladzislau Rezki (Sony) return vend; 151768ad4a33SUladzislau Rezki (Sony) 151868ad4a33SUladzislau Rezki (Sony) if (va->va_start > vstart) 151968ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(va->va_start, align); 152068ad4a33SUladzislau Rezki (Sony) else 152168ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(vstart, align); 152268ad4a33SUladzislau Rezki (Sony) 152368ad4a33SUladzislau Rezki (Sony) /* Check the "vend" restriction. */ 152468ad4a33SUladzislau Rezki (Sony) if (nva_start_addr + size > vend) 152568ad4a33SUladzislau Rezki (Sony) return vend; 152668ad4a33SUladzislau Rezki (Sony) 152768ad4a33SUladzislau Rezki (Sony) /* Update the free vmap_area. */ 1528f9863be4SUladzislau Rezki (Sony) ret = adjust_va_to_fit_type(root, head, va, nva_start_addr, size); 15291b23ff80SBaoquan He if (WARN_ON_ONCE(ret)) 153068ad4a33SUladzislau Rezki (Sony) return vend; 153168ad4a33SUladzislau Rezki (Sony) 1532a6cf4e0fSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 1533bd1264c3SSong Liu find_vmap_lowest_match_check(root, head, size, align); 1534a6cf4e0fSUladzislau Rezki (Sony) #endif 1535a6cf4e0fSUladzislau Rezki (Sony) 153668ad4a33SUladzislau Rezki (Sony) return nva_start_addr; 153768ad4a33SUladzislau Rezki (Sony) } 15384da56b99SChris Wilson 1539db64fe02SNick Piggin /* 1540d98c9e83SAndrey Ryabinin * Free a region of KVA allocated by alloc_vmap_area 1541d98c9e83SAndrey Ryabinin */ 1542d98c9e83SAndrey Ryabinin static void free_vmap_area(struct vmap_area *va) 1543d98c9e83SAndrey Ryabinin { 1544d98c9e83SAndrey Ryabinin /* 1545d98c9e83SAndrey Ryabinin * Remove from the busy tree/list. 1546d98c9e83SAndrey Ryabinin */ 1547d98c9e83SAndrey Ryabinin spin_lock(&vmap_area_lock); 1548d98c9e83SAndrey Ryabinin unlink_va(va, &vmap_area_root); 1549d98c9e83SAndrey Ryabinin spin_unlock(&vmap_area_lock); 1550d98c9e83SAndrey Ryabinin 1551d98c9e83SAndrey Ryabinin /* 1552d98c9e83SAndrey Ryabinin * Insert/Merge it back to the free tree/list. 1553d98c9e83SAndrey Ryabinin */ 1554d98c9e83SAndrey Ryabinin spin_lock(&free_vmap_area_lock); 155596e2db45SUladzislau Rezki (Sony) merge_or_add_vmap_area_augment(va, &free_vmap_area_root, &free_vmap_area_list); 1556d98c9e83SAndrey Ryabinin spin_unlock(&free_vmap_area_lock); 1557d98c9e83SAndrey Ryabinin } 1558d98c9e83SAndrey Ryabinin 1559187f8cc4SUladzislau Rezki (Sony) static inline void 1560187f8cc4SUladzislau Rezki (Sony) preload_this_cpu_lock(spinlock_t *lock, gfp_t gfp_mask, int node) 1561187f8cc4SUladzislau Rezki (Sony) { 1562187f8cc4SUladzislau Rezki (Sony) struct vmap_area *va = NULL; 1563187f8cc4SUladzislau Rezki (Sony) 1564187f8cc4SUladzislau Rezki (Sony) /* 1565187f8cc4SUladzislau Rezki (Sony) * Preload this CPU with one extra vmap_area object. It is used 1566187f8cc4SUladzislau Rezki (Sony) * when fit type of free area is NE_FIT_TYPE. It guarantees that 1567187f8cc4SUladzislau Rezki (Sony) * a CPU that does an allocation is preloaded. 1568187f8cc4SUladzislau Rezki (Sony) * 1569187f8cc4SUladzislau Rezki (Sony) * We do it in non-atomic context, thus it allows us to use more 1570187f8cc4SUladzislau Rezki (Sony) * permissive allocation masks to be more stable under low memory 1571187f8cc4SUladzislau Rezki (Sony) * condition and high memory pressure. 1572187f8cc4SUladzislau Rezki (Sony) */ 1573187f8cc4SUladzislau Rezki (Sony) if (!this_cpu_read(ne_fit_preload_node)) 1574187f8cc4SUladzislau Rezki (Sony) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 1575187f8cc4SUladzislau Rezki (Sony) 1576187f8cc4SUladzislau Rezki (Sony) spin_lock(lock); 1577187f8cc4SUladzislau Rezki (Sony) 1578187f8cc4SUladzislau Rezki (Sony) if (va && __this_cpu_cmpxchg(ne_fit_preload_node, NULL, va)) 1579187f8cc4SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 1580187f8cc4SUladzislau Rezki (Sony) } 1581187f8cc4SUladzislau Rezki (Sony) 1582d98c9e83SAndrey Ryabinin /* 1583db64fe02SNick Piggin * Allocate a region of KVA of the specified size and alignment, within the 1584db64fe02SNick Piggin * vstart and vend. 1585db64fe02SNick Piggin */ 1586db64fe02SNick Piggin static struct vmap_area *alloc_vmap_area(unsigned long size, 1587db64fe02SNick Piggin unsigned long align, 1588db64fe02SNick Piggin unsigned long vstart, unsigned long vend, 1589db64fe02SNick Piggin int node, gfp_t gfp_mask) 1590db64fe02SNick Piggin { 1591187f8cc4SUladzislau Rezki (Sony) struct vmap_area *va; 159212e376a6SUladzislau Rezki (Sony) unsigned long freed; 15931da177e4SLinus Torvalds unsigned long addr; 1594db64fe02SNick Piggin int purged = 0; 1595d98c9e83SAndrey Ryabinin int ret; 1596db64fe02SNick Piggin 15977766970cSNick Piggin BUG_ON(!size); 1598891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 159989699605SNick Piggin BUG_ON(!is_power_of_2(align)); 1600db64fe02SNick Piggin 160168ad4a33SUladzislau Rezki (Sony) if (unlikely(!vmap_initialized)) 160268ad4a33SUladzislau Rezki (Sony) return ERR_PTR(-EBUSY); 160368ad4a33SUladzislau Rezki (Sony) 16045803ed29SChristoph Hellwig might_sleep(); 1605f07116d7SUladzislau Rezki (Sony) gfp_mask = gfp_mask & GFP_RECLAIM_MASK; 16064da56b99SChris Wilson 1607f07116d7SUladzislau Rezki (Sony) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 1608db64fe02SNick Piggin if (unlikely(!va)) 1609db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 1610db64fe02SNick Piggin 16117f88f88fSCatalin Marinas /* 16127f88f88fSCatalin Marinas * Only scan the relevant parts containing pointers to other objects 16137f88f88fSCatalin Marinas * to avoid false negatives. 16147f88f88fSCatalin Marinas */ 1615f07116d7SUladzislau Rezki (Sony) kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask); 16167f88f88fSCatalin Marinas 1617db64fe02SNick Piggin retry: 1618187f8cc4SUladzislau Rezki (Sony) preload_this_cpu_lock(&free_vmap_area_lock, gfp_mask, node); 1619f9863be4SUladzislau Rezki (Sony) addr = __alloc_vmap_area(&free_vmap_area_root, &free_vmap_area_list, 1620f9863be4SUladzislau Rezki (Sony) size, align, vstart, vend); 1621187f8cc4SUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 162268ad4a33SUladzislau Rezki (Sony) 162389699605SNick Piggin /* 162468ad4a33SUladzislau Rezki (Sony) * If an allocation fails, the "vend" address is 162568ad4a33SUladzislau Rezki (Sony) * returned. Therefore trigger the overflow path. 162689699605SNick Piggin */ 162768ad4a33SUladzislau Rezki (Sony) if (unlikely(addr == vend)) 162889699605SNick Piggin goto overflow; 162989699605SNick Piggin 163089699605SNick Piggin va->va_start = addr; 163189699605SNick Piggin va->va_end = addr + size; 1632688fcbfcSPengfei Li va->vm = NULL; 163368ad4a33SUladzislau Rezki (Sony) 1634e36176beSUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 1635e36176beSUladzislau Rezki (Sony) insert_vmap_area(va, &vmap_area_root, &vmap_area_list); 163689699605SNick Piggin spin_unlock(&vmap_area_lock); 163789699605SNick Piggin 163861e16557SWang Xiaoqiang BUG_ON(!IS_ALIGNED(va->va_start, align)); 163989699605SNick Piggin BUG_ON(va->va_start < vstart); 164089699605SNick Piggin BUG_ON(va->va_end > vend); 164189699605SNick Piggin 1642d98c9e83SAndrey Ryabinin ret = kasan_populate_vmalloc(addr, size); 1643d98c9e83SAndrey Ryabinin if (ret) { 1644d98c9e83SAndrey Ryabinin free_vmap_area(va); 1645d98c9e83SAndrey Ryabinin return ERR_PTR(ret); 1646d98c9e83SAndrey Ryabinin } 1647d98c9e83SAndrey Ryabinin 164889699605SNick Piggin return va; 164989699605SNick Piggin 16507766970cSNick Piggin overflow: 1651db64fe02SNick Piggin if (!purged) { 1652db64fe02SNick Piggin purge_vmap_area_lazy(); 1653db64fe02SNick Piggin purged = 1; 1654db64fe02SNick Piggin goto retry; 1655db64fe02SNick Piggin } 16564da56b99SChris Wilson 165712e376a6SUladzislau Rezki (Sony) freed = 0; 16584da56b99SChris Wilson blocking_notifier_call_chain(&vmap_notify_list, 0, &freed); 165912e376a6SUladzislau Rezki (Sony) 16604da56b99SChris Wilson if (freed > 0) { 16614da56b99SChris Wilson purged = 0; 16624da56b99SChris Wilson goto retry; 16634da56b99SChris Wilson } 16644da56b99SChris Wilson 166503497d76SFlorian Fainelli if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) 1666756a025fSJoe Perches pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n", 1667756a025fSJoe Perches size); 166868ad4a33SUladzislau Rezki (Sony) 166968ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 1670db64fe02SNick Piggin return ERR_PTR(-EBUSY); 1671db64fe02SNick Piggin } 1672db64fe02SNick Piggin 16734da56b99SChris Wilson int register_vmap_purge_notifier(struct notifier_block *nb) 16744da56b99SChris Wilson { 16754da56b99SChris Wilson return blocking_notifier_chain_register(&vmap_notify_list, nb); 16764da56b99SChris Wilson } 16774da56b99SChris Wilson EXPORT_SYMBOL_GPL(register_vmap_purge_notifier); 16784da56b99SChris Wilson 16794da56b99SChris Wilson int unregister_vmap_purge_notifier(struct notifier_block *nb) 16804da56b99SChris Wilson { 16814da56b99SChris Wilson return blocking_notifier_chain_unregister(&vmap_notify_list, nb); 16824da56b99SChris Wilson } 16834da56b99SChris Wilson EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier); 16844da56b99SChris Wilson 1685db64fe02SNick Piggin /* 1686db64fe02SNick Piggin * lazy_max_pages is the maximum amount of virtual address space we gather up 1687db64fe02SNick Piggin * before attempting to purge with a TLB flush. 1688db64fe02SNick Piggin * 1689db64fe02SNick Piggin * There is a tradeoff here: a larger number will cover more kernel page tables 1690db64fe02SNick Piggin * and take slightly longer to purge, but it will linearly reduce the number of 1691db64fe02SNick Piggin * global TLB flushes that must be performed. It would seem natural to scale 1692db64fe02SNick Piggin * this number up linearly with the number of CPUs (because vmapping activity 1693db64fe02SNick Piggin * could also scale linearly with the number of CPUs), however it is likely 1694db64fe02SNick Piggin * that in practice, workloads might be constrained in other ways that mean 1695db64fe02SNick Piggin * vmap activity will not scale linearly with CPUs. Also, I want to be 1696db64fe02SNick Piggin * conservative and not introduce a big latency on huge systems, so go with 1697db64fe02SNick Piggin * a less aggressive log scale. It will still be an improvement over the old 1698db64fe02SNick Piggin * code, and it will be simple to change the scale factor if we find that it 1699db64fe02SNick Piggin * becomes a problem on bigger systems. 1700db64fe02SNick Piggin */ 1701db64fe02SNick Piggin static unsigned long lazy_max_pages(void) 1702db64fe02SNick Piggin { 1703db64fe02SNick Piggin unsigned int log; 1704db64fe02SNick Piggin 1705db64fe02SNick Piggin log = fls(num_online_cpus()); 1706db64fe02SNick Piggin 1707db64fe02SNick Piggin return log * (32UL * 1024 * 1024 / PAGE_SIZE); 1708db64fe02SNick Piggin } 1709db64fe02SNick Piggin 17104d36e6f8SUladzislau Rezki (Sony) static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0); 1711db64fe02SNick Piggin 17120574ecd1SChristoph Hellwig /* 1713f0953a1bSIngo Molnar * Serialize vmap purging. There is no actual critical section protected 1714153090f2SBaoquan He * by this lock, but we want to avoid concurrent calls for performance 17150574ecd1SChristoph Hellwig * reasons and to make the pcpu_get_vm_areas more deterministic. 17160574ecd1SChristoph Hellwig */ 1717f9e09977SChristoph Hellwig static DEFINE_MUTEX(vmap_purge_lock); 17180574ecd1SChristoph Hellwig 171902b709dfSNick Piggin /* for per-CPU blocks */ 172002b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void); 172102b709dfSNick Piggin 17223ee48b6aSCliff Wickman /* 1723db64fe02SNick Piggin * Purges all lazily-freed vmap areas. 1724db64fe02SNick Piggin */ 17250574ecd1SChristoph Hellwig static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end) 1726db64fe02SNick Piggin { 17274d36e6f8SUladzislau Rezki (Sony) unsigned long resched_threshold; 1728baa468a6SBaoquan He struct list_head local_purge_list; 172996e2db45SUladzislau Rezki (Sony) struct vmap_area *va, *n_va; 1730db64fe02SNick Piggin 17310574ecd1SChristoph Hellwig lockdep_assert_held(&vmap_purge_lock); 173202b709dfSNick Piggin 173396e2db45SUladzislau Rezki (Sony) spin_lock(&purge_vmap_area_lock); 173496e2db45SUladzislau Rezki (Sony) purge_vmap_area_root = RB_ROOT; 1735baa468a6SBaoquan He list_replace_init(&purge_vmap_area_list, &local_purge_list); 173696e2db45SUladzislau Rezki (Sony) spin_unlock(&purge_vmap_area_lock); 173796e2db45SUladzislau Rezki (Sony) 1738baa468a6SBaoquan He if (unlikely(list_empty(&local_purge_list))) 173968571be9SUladzislau Rezki (Sony) return false; 174068571be9SUladzislau Rezki (Sony) 174196e2db45SUladzislau Rezki (Sony) start = min(start, 1742baa468a6SBaoquan He list_first_entry(&local_purge_list, 174396e2db45SUladzislau Rezki (Sony) struct vmap_area, list)->va_start); 174496e2db45SUladzislau Rezki (Sony) 174596e2db45SUladzislau Rezki (Sony) end = max(end, 1746baa468a6SBaoquan He list_last_entry(&local_purge_list, 174796e2db45SUladzislau Rezki (Sony) struct vmap_area, list)->va_end); 1748db64fe02SNick Piggin 17490574ecd1SChristoph Hellwig flush_tlb_kernel_range(start, end); 17504d36e6f8SUladzislau Rezki (Sony) resched_threshold = lazy_max_pages() << 1; 1751db64fe02SNick Piggin 1752e36176beSUladzislau Rezki (Sony) spin_lock(&free_vmap_area_lock); 1753baa468a6SBaoquan He list_for_each_entry_safe(va, n_va, &local_purge_list, list) { 17544d36e6f8SUladzislau Rezki (Sony) unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT; 17553c5c3cfbSDaniel Axtens unsigned long orig_start = va->va_start; 17563c5c3cfbSDaniel Axtens unsigned long orig_end = va->va_end; 1757763b218dSJoel Fernandes 1758dd3b8353SUladzislau Rezki (Sony) /* 1759dd3b8353SUladzislau Rezki (Sony) * Finally insert or merge lazily-freed area. It is 1760dd3b8353SUladzislau Rezki (Sony) * detached and there is no need to "unlink" it from 1761dd3b8353SUladzislau Rezki (Sony) * anything. 1762dd3b8353SUladzislau Rezki (Sony) */ 176396e2db45SUladzislau Rezki (Sony) va = merge_or_add_vmap_area_augment(va, &free_vmap_area_root, 17643c5c3cfbSDaniel Axtens &free_vmap_area_list); 17653c5c3cfbSDaniel Axtens 17669c801f61SUladzislau Rezki (Sony) if (!va) 17679c801f61SUladzislau Rezki (Sony) continue; 17689c801f61SUladzislau Rezki (Sony) 17693c5c3cfbSDaniel Axtens if (is_vmalloc_or_module_addr((void *)orig_start)) 17703c5c3cfbSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 17713c5c3cfbSDaniel Axtens va->va_start, va->va_end); 1772dd3b8353SUladzislau Rezki (Sony) 17734d36e6f8SUladzislau Rezki (Sony) atomic_long_sub(nr, &vmap_lazy_nr); 177468571be9SUladzislau Rezki (Sony) 17754d36e6f8SUladzislau Rezki (Sony) if (atomic_long_read(&vmap_lazy_nr) < resched_threshold) 1776e36176beSUladzislau Rezki (Sony) cond_resched_lock(&free_vmap_area_lock); 1777763b218dSJoel Fernandes } 1778e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 17790574ecd1SChristoph Hellwig return true; 1780db64fe02SNick Piggin } 1781db64fe02SNick Piggin 1782db64fe02SNick Piggin /* 1783db64fe02SNick Piggin * Kick off a purge of the outstanding lazy areas. 1784db64fe02SNick Piggin */ 1785db64fe02SNick Piggin static void purge_vmap_area_lazy(void) 1786db64fe02SNick Piggin { 1787f9e09977SChristoph Hellwig mutex_lock(&vmap_purge_lock); 17880574ecd1SChristoph Hellwig purge_fragmented_blocks_allcpus(); 17890574ecd1SChristoph Hellwig __purge_vmap_area_lazy(ULONG_MAX, 0); 1790f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 1791db64fe02SNick Piggin } 1792db64fe02SNick Piggin 1793690467c8SUladzislau Rezki (Sony) static void drain_vmap_area_work(struct work_struct *work) 1794690467c8SUladzislau Rezki (Sony) { 1795690467c8SUladzislau Rezki (Sony) unsigned long nr_lazy; 1796690467c8SUladzislau Rezki (Sony) 1797690467c8SUladzislau Rezki (Sony) do { 1798690467c8SUladzislau Rezki (Sony) mutex_lock(&vmap_purge_lock); 1799690467c8SUladzislau Rezki (Sony) __purge_vmap_area_lazy(ULONG_MAX, 0); 1800690467c8SUladzislau Rezki (Sony) mutex_unlock(&vmap_purge_lock); 1801690467c8SUladzislau Rezki (Sony) 1802690467c8SUladzislau Rezki (Sony) /* Recheck if further work is required. */ 1803690467c8SUladzislau Rezki (Sony) nr_lazy = atomic_long_read(&vmap_lazy_nr); 1804690467c8SUladzislau Rezki (Sony) } while (nr_lazy > lazy_max_pages()); 1805690467c8SUladzislau Rezki (Sony) } 1806690467c8SUladzislau Rezki (Sony) 1807db64fe02SNick Piggin /* 180864141da5SJeremy Fitzhardinge * Free a vmap area, caller ensuring that the area has been unmapped 180964141da5SJeremy Fitzhardinge * and flush_cache_vunmap had been called for the correct range 181064141da5SJeremy Fitzhardinge * previously. 1811db64fe02SNick Piggin */ 181264141da5SJeremy Fitzhardinge static void free_vmap_area_noflush(struct vmap_area *va) 1813db64fe02SNick Piggin { 18144d36e6f8SUladzislau Rezki (Sony) unsigned long nr_lazy; 181580c4bd7aSChris Wilson 1816dd3b8353SUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 1817dd3b8353SUladzislau Rezki (Sony) unlink_va(va, &vmap_area_root); 1818dd3b8353SUladzislau Rezki (Sony) spin_unlock(&vmap_area_lock); 1819dd3b8353SUladzislau Rezki (Sony) 18204d36e6f8SUladzislau Rezki (Sony) nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >> 18214d36e6f8SUladzislau Rezki (Sony) PAGE_SHIFT, &vmap_lazy_nr); 182280c4bd7aSChris Wilson 182396e2db45SUladzislau Rezki (Sony) /* 182496e2db45SUladzislau Rezki (Sony) * Merge or place it to the purge tree/list. 182596e2db45SUladzislau Rezki (Sony) */ 182696e2db45SUladzislau Rezki (Sony) spin_lock(&purge_vmap_area_lock); 182796e2db45SUladzislau Rezki (Sony) merge_or_add_vmap_area(va, 182896e2db45SUladzislau Rezki (Sony) &purge_vmap_area_root, &purge_vmap_area_list); 182996e2db45SUladzislau Rezki (Sony) spin_unlock(&purge_vmap_area_lock); 183080c4bd7aSChris Wilson 183196e2db45SUladzislau Rezki (Sony) /* After this point, we may free va at any time */ 183280c4bd7aSChris Wilson if (unlikely(nr_lazy > lazy_max_pages())) 1833690467c8SUladzislau Rezki (Sony) schedule_work(&drain_vmap_work); 1834db64fe02SNick Piggin } 1835db64fe02SNick Piggin 1836b29acbdcSNick Piggin /* 1837b29acbdcSNick Piggin * Free and unmap a vmap area 1838b29acbdcSNick Piggin */ 1839b29acbdcSNick Piggin static void free_unmap_vmap_area(struct vmap_area *va) 1840b29acbdcSNick Piggin { 1841b29acbdcSNick Piggin flush_cache_vunmap(va->va_start, va->va_end); 18424ad0ae8cSNicholas Piggin vunmap_range_noflush(va->va_start, va->va_end); 18438e57f8acSVlastimil Babka if (debug_pagealloc_enabled_static()) 184482a2e924SChintan Pandya flush_tlb_kernel_range(va->va_start, va->va_end); 184582a2e924SChintan Pandya 1846c8eef01eSChristoph Hellwig free_vmap_area_noflush(va); 1847b29acbdcSNick Piggin } 1848b29acbdcSNick Piggin 1849993d0b28SMatthew Wilcox (Oracle) struct vmap_area *find_vmap_area(unsigned long addr) 1850db64fe02SNick Piggin { 1851db64fe02SNick Piggin struct vmap_area *va; 1852db64fe02SNick Piggin 1853db64fe02SNick Piggin spin_lock(&vmap_area_lock); 1854899c6efeSUladzislau Rezki (Sony) va = __find_vmap_area(addr, &vmap_area_root); 1855db64fe02SNick Piggin spin_unlock(&vmap_area_lock); 1856db64fe02SNick Piggin 1857db64fe02SNick Piggin return va; 1858db64fe02SNick Piggin } 1859db64fe02SNick Piggin 1860db64fe02SNick Piggin /*** Per cpu kva allocator ***/ 1861db64fe02SNick Piggin 1862db64fe02SNick Piggin /* 1863db64fe02SNick Piggin * vmap space is limited especially on 32 bit architectures. Ensure there is 1864db64fe02SNick Piggin * room for at least 16 percpu vmap blocks per CPU. 1865db64fe02SNick Piggin */ 1866db64fe02SNick Piggin /* 1867db64fe02SNick Piggin * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able 1868db64fe02SNick Piggin * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess 1869db64fe02SNick Piggin * instead (we just need a rough idea) 1870db64fe02SNick Piggin */ 1871db64fe02SNick Piggin #if BITS_PER_LONG == 32 1872db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024) 1873db64fe02SNick Piggin #else 1874db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024*1024) 1875db64fe02SNick Piggin #endif 1876db64fe02SNick Piggin 1877db64fe02SNick Piggin #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE) 1878db64fe02SNick Piggin #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */ 1879db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */ 1880db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2) 1881db64fe02SNick Piggin #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */ 1882db64fe02SNick Piggin #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */ 1883f982f915SClemens Ladisch #define VMAP_BBMAP_BITS \ 1884f982f915SClemens Ladisch VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ 1885db64fe02SNick Piggin VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ 1886f982f915SClemens Ladisch VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16)) 1887db64fe02SNick Piggin 1888db64fe02SNick Piggin #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE) 1889db64fe02SNick Piggin 1890db64fe02SNick Piggin struct vmap_block_queue { 1891db64fe02SNick Piggin spinlock_t lock; 1892db64fe02SNick Piggin struct list_head free; 1893db64fe02SNick Piggin }; 1894db64fe02SNick Piggin 1895db64fe02SNick Piggin struct vmap_block { 1896db64fe02SNick Piggin spinlock_t lock; 1897db64fe02SNick Piggin struct vmap_area *va; 1898db64fe02SNick Piggin unsigned long free, dirty; 18997d61bfe8SRoman Pen unsigned long dirty_min, dirty_max; /*< dirty range */ 1900db64fe02SNick Piggin struct list_head free_list; 1901db64fe02SNick Piggin struct rcu_head rcu_head; 190202b709dfSNick Piggin struct list_head purge; 1903db64fe02SNick Piggin }; 1904db64fe02SNick Piggin 1905db64fe02SNick Piggin /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */ 1906db64fe02SNick Piggin static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue); 1907db64fe02SNick Piggin 1908db64fe02SNick Piggin /* 19090f14599cSMatthew Wilcox (Oracle) * XArray of vmap blocks, indexed by address, to quickly find a vmap block 1910db64fe02SNick Piggin * in the free path. Could get rid of this if we change the API to return a 1911db64fe02SNick Piggin * "cookie" from alloc, to be passed to free. But no big deal yet. 1912db64fe02SNick Piggin */ 19130f14599cSMatthew Wilcox (Oracle) static DEFINE_XARRAY(vmap_blocks); 1914db64fe02SNick Piggin 1915db64fe02SNick Piggin /* 1916db64fe02SNick Piggin * We should probably have a fallback mechanism to allocate virtual memory 1917db64fe02SNick Piggin * out of partially filled vmap blocks. However vmap block sizing should be 1918db64fe02SNick Piggin * fairly reasonable according to the vmalloc size, so it shouldn't be a 1919db64fe02SNick Piggin * big problem. 1920db64fe02SNick Piggin */ 1921db64fe02SNick Piggin 1922db64fe02SNick Piggin static unsigned long addr_to_vb_idx(unsigned long addr) 1923db64fe02SNick Piggin { 1924db64fe02SNick Piggin addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); 1925db64fe02SNick Piggin addr /= VMAP_BLOCK_SIZE; 1926db64fe02SNick Piggin return addr; 1927db64fe02SNick Piggin } 1928db64fe02SNick Piggin 1929cf725ce2SRoman Pen static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off) 1930cf725ce2SRoman Pen { 1931cf725ce2SRoman Pen unsigned long addr; 1932cf725ce2SRoman Pen 1933cf725ce2SRoman Pen addr = va_start + (pages_off << PAGE_SHIFT); 1934cf725ce2SRoman Pen BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start)); 1935cf725ce2SRoman Pen return (void *)addr; 1936cf725ce2SRoman Pen } 1937cf725ce2SRoman Pen 1938cf725ce2SRoman Pen /** 1939cf725ce2SRoman Pen * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this 1940cf725ce2SRoman Pen * block. Of course pages number can't exceed VMAP_BBMAP_BITS 1941cf725ce2SRoman Pen * @order: how many 2^order pages should be occupied in newly allocated block 1942cf725ce2SRoman Pen * @gfp_mask: flags for the page level allocator 1943cf725ce2SRoman Pen * 1944a862f68aSMike Rapoport * Return: virtual address in a newly allocated block or ERR_PTR(-errno) 1945cf725ce2SRoman Pen */ 1946cf725ce2SRoman Pen static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) 1947db64fe02SNick Piggin { 1948db64fe02SNick Piggin struct vmap_block_queue *vbq; 1949db64fe02SNick Piggin struct vmap_block *vb; 1950db64fe02SNick Piggin struct vmap_area *va; 1951db64fe02SNick Piggin unsigned long vb_idx; 1952db64fe02SNick Piggin int node, err; 1953cf725ce2SRoman Pen void *vaddr; 1954db64fe02SNick Piggin 1955db64fe02SNick Piggin node = numa_node_id(); 1956db64fe02SNick Piggin 1957db64fe02SNick Piggin vb = kmalloc_node(sizeof(struct vmap_block), 1958db64fe02SNick Piggin gfp_mask & GFP_RECLAIM_MASK, node); 1959db64fe02SNick Piggin if (unlikely(!vb)) 1960db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 1961db64fe02SNick Piggin 1962db64fe02SNick Piggin va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, 1963db64fe02SNick Piggin VMALLOC_START, VMALLOC_END, 1964db64fe02SNick Piggin node, gfp_mask); 1965ddf9c6d4STobias Klauser if (IS_ERR(va)) { 1966db64fe02SNick Piggin kfree(vb); 1967e7d86340SJulia Lawall return ERR_CAST(va); 1968db64fe02SNick Piggin } 1969db64fe02SNick Piggin 1970cf725ce2SRoman Pen vaddr = vmap_block_vaddr(va->va_start, 0); 1971db64fe02SNick Piggin spin_lock_init(&vb->lock); 1972db64fe02SNick Piggin vb->va = va; 1973cf725ce2SRoman Pen /* At least something should be left free */ 1974cf725ce2SRoman Pen BUG_ON(VMAP_BBMAP_BITS <= (1UL << order)); 1975cf725ce2SRoman Pen vb->free = VMAP_BBMAP_BITS - (1UL << order); 1976db64fe02SNick Piggin vb->dirty = 0; 19777d61bfe8SRoman Pen vb->dirty_min = VMAP_BBMAP_BITS; 19787d61bfe8SRoman Pen vb->dirty_max = 0; 1979db64fe02SNick Piggin INIT_LIST_HEAD(&vb->free_list); 1980db64fe02SNick Piggin 1981db64fe02SNick Piggin vb_idx = addr_to_vb_idx(va->va_start); 19820f14599cSMatthew Wilcox (Oracle) err = xa_insert(&vmap_blocks, vb_idx, vb, gfp_mask); 19830f14599cSMatthew Wilcox (Oracle) if (err) { 19840f14599cSMatthew Wilcox (Oracle) kfree(vb); 19850f14599cSMatthew Wilcox (Oracle) free_vmap_area(va); 19860f14599cSMatthew Wilcox (Oracle) return ERR_PTR(err); 19870f14599cSMatthew Wilcox (Oracle) } 1988db64fe02SNick Piggin 19893f804920SSebastian Andrzej Siewior vbq = raw_cpu_ptr(&vmap_block_queue); 1990db64fe02SNick Piggin spin_lock(&vbq->lock); 199168ac546fSRoman Pen list_add_tail_rcu(&vb->free_list, &vbq->free); 1992db64fe02SNick Piggin spin_unlock(&vbq->lock); 1993db64fe02SNick Piggin 1994cf725ce2SRoman Pen return vaddr; 1995db64fe02SNick Piggin } 1996db64fe02SNick Piggin 1997db64fe02SNick Piggin static void free_vmap_block(struct vmap_block *vb) 1998db64fe02SNick Piggin { 1999db64fe02SNick Piggin struct vmap_block *tmp; 2000db64fe02SNick Piggin 20010f14599cSMatthew Wilcox (Oracle) tmp = xa_erase(&vmap_blocks, addr_to_vb_idx(vb->va->va_start)); 2002db64fe02SNick Piggin BUG_ON(tmp != vb); 2003db64fe02SNick Piggin 200464141da5SJeremy Fitzhardinge free_vmap_area_noflush(vb->va); 200522a3c7d1SLai Jiangshan kfree_rcu(vb, rcu_head); 2006db64fe02SNick Piggin } 2007db64fe02SNick Piggin 200802b709dfSNick Piggin static void purge_fragmented_blocks(int cpu) 200902b709dfSNick Piggin { 201002b709dfSNick Piggin LIST_HEAD(purge); 201102b709dfSNick Piggin struct vmap_block *vb; 201202b709dfSNick Piggin struct vmap_block *n_vb; 201302b709dfSNick Piggin struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 201402b709dfSNick Piggin 201502b709dfSNick Piggin rcu_read_lock(); 201602b709dfSNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 201702b709dfSNick Piggin 201802b709dfSNick Piggin if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS)) 201902b709dfSNick Piggin continue; 202002b709dfSNick Piggin 202102b709dfSNick Piggin spin_lock(&vb->lock); 202202b709dfSNick Piggin if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) { 202302b709dfSNick Piggin vb->free = 0; /* prevent further allocs after releasing lock */ 202402b709dfSNick Piggin vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */ 20257d61bfe8SRoman Pen vb->dirty_min = 0; 20267d61bfe8SRoman Pen vb->dirty_max = VMAP_BBMAP_BITS; 202702b709dfSNick Piggin spin_lock(&vbq->lock); 202802b709dfSNick Piggin list_del_rcu(&vb->free_list); 202902b709dfSNick Piggin spin_unlock(&vbq->lock); 203002b709dfSNick Piggin spin_unlock(&vb->lock); 203102b709dfSNick Piggin list_add_tail(&vb->purge, &purge); 203202b709dfSNick Piggin } else 203302b709dfSNick Piggin spin_unlock(&vb->lock); 203402b709dfSNick Piggin } 203502b709dfSNick Piggin rcu_read_unlock(); 203602b709dfSNick Piggin 203702b709dfSNick Piggin list_for_each_entry_safe(vb, n_vb, &purge, purge) { 203802b709dfSNick Piggin list_del(&vb->purge); 203902b709dfSNick Piggin free_vmap_block(vb); 204002b709dfSNick Piggin } 204102b709dfSNick Piggin } 204202b709dfSNick Piggin 204302b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void) 204402b709dfSNick Piggin { 204502b709dfSNick Piggin int cpu; 204602b709dfSNick Piggin 204702b709dfSNick Piggin for_each_possible_cpu(cpu) 204802b709dfSNick Piggin purge_fragmented_blocks(cpu); 204902b709dfSNick Piggin } 205002b709dfSNick Piggin 2051db64fe02SNick Piggin static void *vb_alloc(unsigned long size, gfp_t gfp_mask) 2052db64fe02SNick Piggin { 2053db64fe02SNick Piggin struct vmap_block_queue *vbq; 2054db64fe02SNick Piggin struct vmap_block *vb; 2055cf725ce2SRoman Pen void *vaddr = NULL; 2056db64fe02SNick Piggin unsigned int order; 2057db64fe02SNick Piggin 2058891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 2059db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 2060aa91c4d8SJan Kara if (WARN_ON(size == 0)) { 2061aa91c4d8SJan Kara /* 2062aa91c4d8SJan Kara * Allocating 0 bytes isn't what caller wants since 2063aa91c4d8SJan Kara * get_order(0) returns funny result. Just warn and terminate 2064aa91c4d8SJan Kara * early. 2065aa91c4d8SJan Kara */ 2066aa91c4d8SJan Kara return NULL; 2067aa91c4d8SJan Kara } 2068db64fe02SNick Piggin order = get_order(size); 2069db64fe02SNick Piggin 2070db64fe02SNick Piggin rcu_read_lock(); 20713f804920SSebastian Andrzej Siewior vbq = raw_cpu_ptr(&vmap_block_queue); 2072db64fe02SNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 2073cf725ce2SRoman Pen unsigned long pages_off; 2074db64fe02SNick Piggin 2075db64fe02SNick Piggin spin_lock(&vb->lock); 2076cf725ce2SRoman Pen if (vb->free < (1UL << order)) { 2077cf725ce2SRoman Pen spin_unlock(&vb->lock); 2078cf725ce2SRoman Pen continue; 2079cf725ce2SRoman Pen } 208002b709dfSNick Piggin 2081cf725ce2SRoman Pen pages_off = VMAP_BBMAP_BITS - vb->free; 2082cf725ce2SRoman Pen vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); 2083db64fe02SNick Piggin vb->free -= 1UL << order; 2084db64fe02SNick Piggin if (vb->free == 0) { 2085db64fe02SNick Piggin spin_lock(&vbq->lock); 2086de560423SNick Piggin list_del_rcu(&vb->free_list); 2087db64fe02SNick Piggin spin_unlock(&vbq->lock); 2088db64fe02SNick Piggin } 2089cf725ce2SRoman Pen 2090db64fe02SNick Piggin spin_unlock(&vb->lock); 2091db64fe02SNick Piggin break; 2092db64fe02SNick Piggin } 209302b709dfSNick Piggin 2094db64fe02SNick Piggin rcu_read_unlock(); 2095db64fe02SNick Piggin 2096cf725ce2SRoman Pen /* Allocate new block if nothing was found */ 2097cf725ce2SRoman Pen if (!vaddr) 2098cf725ce2SRoman Pen vaddr = new_vmap_block(order, gfp_mask); 2099db64fe02SNick Piggin 2100cf725ce2SRoman Pen return vaddr; 2101db64fe02SNick Piggin } 2102db64fe02SNick Piggin 210378a0e8c4SChristoph Hellwig static void vb_free(unsigned long addr, unsigned long size) 2104db64fe02SNick Piggin { 2105db64fe02SNick Piggin unsigned long offset; 2106db64fe02SNick Piggin unsigned int order; 2107db64fe02SNick Piggin struct vmap_block *vb; 2108db64fe02SNick Piggin 2109891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 2110db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 2111b29acbdcSNick Piggin 211278a0e8c4SChristoph Hellwig flush_cache_vunmap(addr, addr + size); 2113b29acbdcSNick Piggin 2114db64fe02SNick Piggin order = get_order(size); 211578a0e8c4SChristoph Hellwig offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT; 21160f14599cSMatthew Wilcox (Oracle) vb = xa_load(&vmap_blocks, addr_to_vb_idx(addr)); 2117db64fe02SNick Piggin 21184ad0ae8cSNicholas Piggin vunmap_range_noflush(addr, addr + size); 211964141da5SJeremy Fitzhardinge 21208e57f8acSVlastimil Babka if (debug_pagealloc_enabled_static()) 212178a0e8c4SChristoph Hellwig flush_tlb_kernel_range(addr, addr + size); 212282a2e924SChintan Pandya 2123db64fe02SNick Piggin spin_lock(&vb->lock); 21247d61bfe8SRoman Pen 21257d61bfe8SRoman Pen /* Expand dirty range */ 21267d61bfe8SRoman Pen vb->dirty_min = min(vb->dirty_min, offset); 21277d61bfe8SRoman Pen vb->dirty_max = max(vb->dirty_max, offset + (1UL << order)); 2128d086817dSMinChan Kim 2129db64fe02SNick Piggin vb->dirty += 1UL << order; 2130db64fe02SNick Piggin if (vb->dirty == VMAP_BBMAP_BITS) { 2131de560423SNick Piggin BUG_ON(vb->free); 2132db64fe02SNick Piggin spin_unlock(&vb->lock); 2133db64fe02SNick Piggin free_vmap_block(vb); 2134db64fe02SNick Piggin } else 2135db64fe02SNick Piggin spin_unlock(&vb->lock); 2136db64fe02SNick Piggin } 2137db64fe02SNick Piggin 2138868b104dSRick Edgecombe static void _vm_unmap_aliases(unsigned long start, unsigned long end, int flush) 2139db64fe02SNick Piggin { 2140db64fe02SNick Piggin int cpu; 2141db64fe02SNick Piggin 21429b463334SJeremy Fitzhardinge if (unlikely(!vmap_initialized)) 21439b463334SJeremy Fitzhardinge return; 21449b463334SJeremy Fitzhardinge 21455803ed29SChristoph Hellwig might_sleep(); 21465803ed29SChristoph Hellwig 2147db64fe02SNick Piggin for_each_possible_cpu(cpu) { 2148db64fe02SNick Piggin struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 2149db64fe02SNick Piggin struct vmap_block *vb; 2150db64fe02SNick Piggin 2151db64fe02SNick Piggin rcu_read_lock(); 2152db64fe02SNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 2153db64fe02SNick Piggin spin_lock(&vb->lock); 2154ad216c03SVijayanand Jitta if (vb->dirty && vb->dirty != VMAP_BBMAP_BITS) { 21557d61bfe8SRoman Pen unsigned long va_start = vb->va->va_start; 2156db64fe02SNick Piggin unsigned long s, e; 2157b136be5eSJoonsoo Kim 21587d61bfe8SRoman Pen s = va_start + (vb->dirty_min << PAGE_SHIFT); 21597d61bfe8SRoman Pen e = va_start + (vb->dirty_max << PAGE_SHIFT); 2160db64fe02SNick Piggin 21617d61bfe8SRoman Pen start = min(s, start); 21627d61bfe8SRoman Pen end = max(e, end); 21637d61bfe8SRoman Pen 2164db64fe02SNick Piggin flush = 1; 2165db64fe02SNick Piggin } 2166db64fe02SNick Piggin spin_unlock(&vb->lock); 2167db64fe02SNick Piggin } 2168db64fe02SNick Piggin rcu_read_unlock(); 2169db64fe02SNick Piggin } 2170db64fe02SNick Piggin 2171f9e09977SChristoph Hellwig mutex_lock(&vmap_purge_lock); 21720574ecd1SChristoph Hellwig purge_fragmented_blocks_allcpus(); 21730574ecd1SChristoph Hellwig if (!__purge_vmap_area_lazy(start, end) && flush) 21740574ecd1SChristoph Hellwig flush_tlb_kernel_range(start, end); 2175f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 2176db64fe02SNick Piggin } 2177868b104dSRick Edgecombe 2178868b104dSRick Edgecombe /** 2179868b104dSRick Edgecombe * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer 2180868b104dSRick Edgecombe * 2181868b104dSRick Edgecombe * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily 2182868b104dSRick Edgecombe * to amortize TLB flushing overheads. What this means is that any page you 2183868b104dSRick Edgecombe * have now, may, in a former life, have been mapped into kernel virtual 2184868b104dSRick Edgecombe * address by the vmap layer and so there might be some CPUs with TLB entries 2185868b104dSRick Edgecombe * still referencing that page (additional to the regular 1:1 kernel mapping). 2186868b104dSRick Edgecombe * 2187868b104dSRick Edgecombe * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can 2188868b104dSRick Edgecombe * be sure that none of the pages we have control over will have any aliases 2189868b104dSRick Edgecombe * from the vmap layer. 2190868b104dSRick Edgecombe */ 2191868b104dSRick Edgecombe void vm_unmap_aliases(void) 2192868b104dSRick Edgecombe { 2193868b104dSRick Edgecombe unsigned long start = ULONG_MAX, end = 0; 2194868b104dSRick Edgecombe int flush = 0; 2195868b104dSRick Edgecombe 2196868b104dSRick Edgecombe _vm_unmap_aliases(start, end, flush); 2197868b104dSRick Edgecombe } 2198db64fe02SNick Piggin EXPORT_SYMBOL_GPL(vm_unmap_aliases); 2199db64fe02SNick Piggin 2200db64fe02SNick Piggin /** 2201db64fe02SNick Piggin * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram 2202db64fe02SNick Piggin * @mem: the pointer returned by vm_map_ram 2203db64fe02SNick Piggin * @count: the count passed to that vm_map_ram call (cannot unmap partial) 2204db64fe02SNick Piggin */ 2205db64fe02SNick Piggin void vm_unmap_ram(const void *mem, unsigned int count) 2206db64fe02SNick Piggin { 220765ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 22084aff1dc4SAndrey Konovalov unsigned long addr = (unsigned long)kasan_reset_tag(mem); 22099c3acf60SChristoph Hellwig struct vmap_area *va; 2210db64fe02SNick Piggin 22115803ed29SChristoph Hellwig might_sleep(); 2212db64fe02SNick Piggin BUG_ON(!addr); 2213db64fe02SNick Piggin BUG_ON(addr < VMALLOC_START); 2214db64fe02SNick Piggin BUG_ON(addr > VMALLOC_END); 2215a1c0b1a0SShawn Lin BUG_ON(!PAGE_ALIGNED(addr)); 2216db64fe02SNick Piggin 2217d98c9e83SAndrey Ryabinin kasan_poison_vmalloc(mem, size); 2218d98c9e83SAndrey Ryabinin 22199c3acf60SChristoph Hellwig if (likely(count <= VMAP_MAX_ALLOC)) { 222005e3ff95SChintan Pandya debug_check_no_locks_freed(mem, size); 222178a0e8c4SChristoph Hellwig vb_free(addr, size); 22229c3acf60SChristoph Hellwig return; 22239c3acf60SChristoph Hellwig } 22249c3acf60SChristoph Hellwig 22259c3acf60SChristoph Hellwig va = find_vmap_area(addr); 22269c3acf60SChristoph Hellwig BUG_ON(!va); 222705e3ff95SChintan Pandya debug_check_no_locks_freed((void *)va->va_start, 222805e3ff95SChintan Pandya (va->va_end - va->va_start)); 22299c3acf60SChristoph Hellwig free_unmap_vmap_area(va); 2230db64fe02SNick Piggin } 2231db64fe02SNick Piggin EXPORT_SYMBOL(vm_unmap_ram); 2232db64fe02SNick Piggin 2233db64fe02SNick Piggin /** 2234db64fe02SNick Piggin * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) 2235db64fe02SNick Piggin * @pages: an array of pointers to the pages to be mapped 2236db64fe02SNick Piggin * @count: number of pages 2237db64fe02SNick Piggin * @node: prefer to allocate data structures on this node 2238e99c97adSRandy Dunlap * 223936437638SGioh Kim * If you use this function for less than VMAP_MAX_ALLOC pages, it could be 224036437638SGioh Kim * faster than vmap so it's good. But if you mix long-life and short-life 224136437638SGioh Kim * objects with vm_map_ram(), it could consume lots of address space through 224236437638SGioh Kim * fragmentation (especially on a 32bit machine). You could see failures in 224336437638SGioh Kim * the end. Please use this function for short-lived objects. 224436437638SGioh Kim * 2245e99c97adSRandy Dunlap * Returns: a pointer to the address that has been mapped, or %NULL on failure 2246db64fe02SNick Piggin */ 2247d4efd79aSChristoph Hellwig void *vm_map_ram(struct page **pages, unsigned int count, int node) 2248db64fe02SNick Piggin { 224965ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 2250db64fe02SNick Piggin unsigned long addr; 2251db64fe02SNick Piggin void *mem; 2252db64fe02SNick Piggin 2253db64fe02SNick Piggin if (likely(count <= VMAP_MAX_ALLOC)) { 2254db64fe02SNick Piggin mem = vb_alloc(size, GFP_KERNEL); 2255db64fe02SNick Piggin if (IS_ERR(mem)) 2256db64fe02SNick Piggin return NULL; 2257db64fe02SNick Piggin addr = (unsigned long)mem; 2258db64fe02SNick Piggin } else { 2259db64fe02SNick Piggin struct vmap_area *va; 2260db64fe02SNick Piggin va = alloc_vmap_area(size, PAGE_SIZE, 2261db64fe02SNick Piggin VMALLOC_START, VMALLOC_END, node, GFP_KERNEL); 2262db64fe02SNick Piggin if (IS_ERR(va)) 2263db64fe02SNick Piggin return NULL; 2264db64fe02SNick Piggin 2265db64fe02SNick Piggin addr = va->va_start; 2266db64fe02SNick Piggin mem = (void *)addr; 2267db64fe02SNick Piggin } 2268d98c9e83SAndrey Ryabinin 2269b67177ecSNicholas Piggin if (vmap_pages_range(addr, addr + size, PAGE_KERNEL, 2270b67177ecSNicholas Piggin pages, PAGE_SHIFT) < 0) { 2271db64fe02SNick Piggin vm_unmap_ram(mem, count); 2272db64fe02SNick Piggin return NULL; 2273db64fe02SNick Piggin } 2274b67177ecSNicholas Piggin 227523689e91SAndrey Konovalov /* 227623689e91SAndrey Konovalov * Mark the pages as accessible, now that they are mapped. 227723689e91SAndrey Konovalov * With hardware tag-based KASAN, marking is skipped for 227823689e91SAndrey Konovalov * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 227923689e91SAndrey Konovalov */ 2280f6e39794SAndrey Konovalov mem = kasan_unpoison_vmalloc(mem, size, KASAN_VMALLOC_PROT_NORMAL); 228119f1c3acSAndrey Konovalov 2282db64fe02SNick Piggin return mem; 2283db64fe02SNick Piggin } 2284db64fe02SNick Piggin EXPORT_SYMBOL(vm_map_ram); 2285db64fe02SNick Piggin 22864341fa45SJoonsoo Kim static struct vm_struct *vmlist __initdata; 228792eac168SMike Rapoport 2288121e6f32SNicholas Piggin static inline unsigned int vm_area_page_order(struct vm_struct *vm) 2289121e6f32SNicholas Piggin { 2290121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 2291121e6f32SNicholas Piggin return vm->page_order; 2292121e6f32SNicholas Piggin #else 2293121e6f32SNicholas Piggin return 0; 2294121e6f32SNicholas Piggin #endif 2295121e6f32SNicholas Piggin } 2296121e6f32SNicholas Piggin 2297121e6f32SNicholas Piggin static inline void set_vm_area_page_order(struct vm_struct *vm, unsigned int order) 2298121e6f32SNicholas Piggin { 2299121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 2300121e6f32SNicholas Piggin vm->page_order = order; 2301121e6f32SNicholas Piggin #else 2302121e6f32SNicholas Piggin BUG_ON(order != 0); 2303121e6f32SNicholas Piggin #endif 2304121e6f32SNicholas Piggin } 2305121e6f32SNicholas Piggin 2306f0aa6617STejun Heo /** 2307be9b7335SNicolas Pitre * vm_area_add_early - add vmap area early during boot 2308be9b7335SNicolas Pitre * @vm: vm_struct to add 2309be9b7335SNicolas Pitre * 2310be9b7335SNicolas Pitre * This function is used to add fixed kernel vm area to vmlist before 2311be9b7335SNicolas Pitre * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags 2312be9b7335SNicolas Pitre * should contain proper values and the other fields should be zero. 2313be9b7335SNicolas Pitre * 2314be9b7335SNicolas Pitre * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 2315be9b7335SNicolas Pitre */ 2316be9b7335SNicolas Pitre void __init vm_area_add_early(struct vm_struct *vm) 2317be9b7335SNicolas Pitre { 2318be9b7335SNicolas Pitre struct vm_struct *tmp, **p; 2319be9b7335SNicolas Pitre 2320be9b7335SNicolas Pitre BUG_ON(vmap_initialized); 2321be9b7335SNicolas Pitre for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { 2322be9b7335SNicolas Pitre if (tmp->addr >= vm->addr) { 2323be9b7335SNicolas Pitre BUG_ON(tmp->addr < vm->addr + vm->size); 2324be9b7335SNicolas Pitre break; 2325be9b7335SNicolas Pitre } else 2326be9b7335SNicolas Pitre BUG_ON(tmp->addr + tmp->size > vm->addr); 2327be9b7335SNicolas Pitre } 2328be9b7335SNicolas Pitre vm->next = *p; 2329be9b7335SNicolas Pitre *p = vm; 2330be9b7335SNicolas Pitre } 2331be9b7335SNicolas Pitre 2332be9b7335SNicolas Pitre /** 2333f0aa6617STejun Heo * vm_area_register_early - register vmap area early during boot 2334f0aa6617STejun Heo * @vm: vm_struct to register 2335c0c0a293STejun Heo * @align: requested alignment 2336f0aa6617STejun Heo * 2337f0aa6617STejun Heo * This function is used to register kernel vm area before 2338f0aa6617STejun Heo * vmalloc_init() is called. @vm->size and @vm->flags should contain 2339f0aa6617STejun Heo * proper values on entry and other fields should be zero. On return, 2340f0aa6617STejun Heo * vm->addr contains the allocated address. 2341f0aa6617STejun Heo * 2342f0aa6617STejun Heo * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 2343f0aa6617STejun Heo */ 2344c0c0a293STejun Heo void __init vm_area_register_early(struct vm_struct *vm, size_t align) 2345f0aa6617STejun Heo { 23460eb68437SKefeng Wang unsigned long addr = ALIGN(VMALLOC_START, align); 23470eb68437SKefeng Wang struct vm_struct *cur, **p; 2348f0aa6617STejun Heo 23490eb68437SKefeng Wang BUG_ON(vmap_initialized); 2350c0c0a293STejun Heo 23510eb68437SKefeng Wang for (p = &vmlist; (cur = *p) != NULL; p = &cur->next) { 23520eb68437SKefeng Wang if ((unsigned long)cur->addr - addr >= vm->size) 23530eb68437SKefeng Wang break; 23540eb68437SKefeng Wang addr = ALIGN((unsigned long)cur->addr + cur->size, align); 23550eb68437SKefeng Wang } 23560eb68437SKefeng Wang 23570eb68437SKefeng Wang BUG_ON(addr > VMALLOC_END - vm->size); 2358c0c0a293STejun Heo vm->addr = (void *)addr; 23590eb68437SKefeng Wang vm->next = *p; 23600eb68437SKefeng Wang *p = vm; 23613252b1d8SKefeng Wang kasan_populate_early_vm_area_shadow(vm->addr, vm->size); 2362f0aa6617STejun Heo } 2363f0aa6617STejun Heo 236468ad4a33SUladzislau Rezki (Sony) static void vmap_init_free_space(void) 236568ad4a33SUladzislau Rezki (Sony) { 236668ad4a33SUladzislau Rezki (Sony) unsigned long vmap_start = 1; 236768ad4a33SUladzislau Rezki (Sony) const unsigned long vmap_end = ULONG_MAX; 236868ad4a33SUladzislau Rezki (Sony) struct vmap_area *busy, *free; 236968ad4a33SUladzislau Rezki (Sony) 237068ad4a33SUladzislau Rezki (Sony) /* 237168ad4a33SUladzislau Rezki (Sony) * B F B B B F 237268ad4a33SUladzislau Rezki (Sony) * -|-----|.....|-----|-----|-----|.....|- 237368ad4a33SUladzislau Rezki (Sony) * | The KVA space | 237468ad4a33SUladzislau Rezki (Sony) * |<--------------------------------->| 237568ad4a33SUladzislau Rezki (Sony) */ 237668ad4a33SUladzislau Rezki (Sony) list_for_each_entry(busy, &vmap_area_list, list) { 237768ad4a33SUladzislau Rezki (Sony) if (busy->va_start - vmap_start > 0) { 237868ad4a33SUladzislau Rezki (Sony) free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 237968ad4a33SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(!free)) { 238068ad4a33SUladzislau Rezki (Sony) free->va_start = vmap_start; 238168ad4a33SUladzislau Rezki (Sony) free->va_end = busy->va_start; 238268ad4a33SUladzislau Rezki (Sony) 238368ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(free, NULL, 238468ad4a33SUladzislau Rezki (Sony) &free_vmap_area_root, 238568ad4a33SUladzislau Rezki (Sony) &free_vmap_area_list); 238668ad4a33SUladzislau Rezki (Sony) } 238768ad4a33SUladzislau Rezki (Sony) } 238868ad4a33SUladzislau Rezki (Sony) 238968ad4a33SUladzislau Rezki (Sony) vmap_start = busy->va_end; 239068ad4a33SUladzislau Rezki (Sony) } 239168ad4a33SUladzislau Rezki (Sony) 239268ad4a33SUladzislau Rezki (Sony) if (vmap_end - vmap_start > 0) { 239368ad4a33SUladzislau Rezki (Sony) free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 239468ad4a33SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(!free)) { 239568ad4a33SUladzislau Rezki (Sony) free->va_start = vmap_start; 239668ad4a33SUladzislau Rezki (Sony) free->va_end = vmap_end; 239768ad4a33SUladzislau Rezki (Sony) 239868ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(free, NULL, 239968ad4a33SUladzislau Rezki (Sony) &free_vmap_area_root, 240068ad4a33SUladzislau Rezki (Sony) &free_vmap_area_list); 240168ad4a33SUladzislau Rezki (Sony) } 240268ad4a33SUladzislau Rezki (Sony) } 240368ad4a33SUladzislau Rezki (Sony) } 240468ad4a33SUladzislau Rezki (Sony) 2405db64fe02SNick Piggin void __init vmalloc_init(void) 2406db64fe02SNick Piggin { 2407822c18f2SIvan Kokshaysky struct vmap_area *va; 2408822c18f2SIvan Kokshaysky struct vm_struct *tmp; 2409db64fe02SNick Piggin int i; 2410db64fe02SNick Piggin 241168ad4a33SUladzislau Rezki (Sony) /* 241268ad4a33SUladzislau Rezki (Sony) * Create the cache for vmap_area objects. 241368ad4a33SUladzislau Rezki (Sony) */ 241468ad4a33SUladzislau Rezki (Sony) vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC); 241568ad4a33SUladzislau Rezki (Sony) 2416db64fe02SNick Piggin for_each_possible_cpu(i) { 2417db64fe02SNick Piggin struct vmap_block_queue *vbq; 241832fcfd40SAl Viro struct vfree_deferred *p; 2419db64fe02SNick Piggin 2420db64fe02SNick Piggin vbq = &per_cpu(vmap_block_queue, i); 2421db64fe02SNick Piggin spin_lock_init(&vbq->lock); 2422db64fe02SNick Piggin INIT_LIST_HEAD(&vbq->free); 242332fcfd40SAl Viro p = &per_cpu(vfree_deferred, i); 242432fcfd40SAl Viro init_llist_head(&p->list); 242532fcfd40SAl Viro INIT_WORK(&p->wq, free_work); 2426db64fe02SNick Piggin } 24279b463334SJeremy Fitzhardinge 2428822c18f2SIvan Kokshaysky /* Import existing vmlist entries. */ 2429822c18f2SIvan Kokshaysky for (tmp = vmlist; tmp; tmp = tmp->next) { 243068ad4a33SUladzislau Rezki (Sony) va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 243168ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(!va)) 243268ad4a33SUladzislau Rezki (Sony) continue; 243368ad4a33SUladzislau Rezki (Sony) 2434822c18f2SIvan Kokshaysky va->va_start = (unsigned long)tmp->addr; 2435822c18f2SIvan Kokshaysky va->va_end = va->va_start + tmp->size; 2436dbda591dSKyongHo va->vm = tmp; 243768ad4a33SUladzislau Rezki (Sony) insert_vmap_area(va, &vmap_area_root, &vmap_area_list); 2438822c18f2SIvan Kokshaysky } 2439ca23e405STejun Heo 244068ad4a33SUladzislau Rezki (Sony) /* 244168ad4a33SUladzislau Rezki (Sony) * Now we can initialize a free vmap space. 244268ad4a33SUladzislau Rezki (Sony) */ 244368ad4a33SUladzislau Rezki (Sony) vmap_init_free_space(); 24449b463334SJeremy Fitzhardinge vmap_initialized = true; 2445db64fe02SNick Piggin } 2446db64fe02SNick Piggin 2447e36176beSUladzislau Rezki (Sony) static inline void setup_vmalloc_vm_locked(struct vm_struct *vm, 2448e36176beSUladzislau Rezki (Sony) struct vmap_area *va, unsigned long flags, const void *caller) 2449cf88c790STejun Heo { 2450cf88c790STejun Heo vm->flags = flags; 2451cf88c790STejun Heo vm->addr = (void *)va->va_start; 2452cf88c790STejun Heo vm->size = va->va_end - va->va_start; 2453cf88c790STejun Heo vm->caller = caller; 2454db1aecafSMinchan Kim va->vm = vm; 2455e36176beSUladzislau Rezki (Sony) } 2456e36176beSUladzislau Rezki (Sony) 2457e36176beSUladzislau Rezki (Sony) static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, 2458e36176beSUladzislau Rezki (Sony) unsigned long flags, const void *caller) 2459e36176beSUladzislau Rezki (Sony) { 2460e36176beSUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 2461e36176beSUladzislau Rezki (Sony) setup_vmalloc_vm_locked(vm, va, flags, caller); 2462c69480adSJoonsoo Kim spin_unlock(&vmap_area_lock); 2463f5252e00SMitsuo Hayasaka } 2464cf88c790STejun Heo 246520fc02b4SZhang Yanfei static void clear_vm_uninitialized_flag(struct vm_struct *vm) 2466f5252e00SMitsuo Hayasaka { 2467d4033afdSJoonsoo Kim /* 246820fc02b4SZhang Yanfei * Before removing VM_UNINITIALIZED, 2469d4033afdSJoonsoo Kim * we should make sure that vm has proper values. 2470d4033afdSJoonsoo Kim * Pair with smp_rmb() in show_numa_info(). 2471d4033afdSJoonsoo Kim */ 2472d4033afdSJoonsoo Kim smp_wmb(); 247320fc02b4SZhang Yanfei vm->flags &= ~VM_UNINITIALIZED; 2474cf88c790STejun Heo } 2475cf88c790STejun Heo 2476db64fe02SNick Piggin static struct vm_struct *__get_vm_area_node(unsigned long size, 24777ca3027bSDaniel Axtens unsigned long align, unsigned long shift, unsigned long flags, 24787ca3027bSDaniel Axtens unsigned long start, unsigned long end, int node, 24797ca3027bSDaniel Axtens gfp_t gfp_mask, const void *caller) 2480db64fe02SNick Piggin { 24810006526dSKautuk Consul struct vmap_area *va; 2482db64fe02SNick Piggin struct vm_struct *area; 2483d98c9e83SAndrey Ryabinin unsigned long requested_size = size; 24841da177e4SLinus Torvalds 248552fd24caSGiridhar Pemmasani BUG_ON(in_interrupt()); 24867ca3027bSDaniel Axtens size = ALIGN(size, 1ul << shift); 248731be8309SOGAWA Hirofumi if (unlikely(!size)) 248831be8309SOGAWA Hirofumi return NULL; 24891da177e4SLinus Torvalds 2490252e5c6eSzijun_hu if (flags & VM_IOREMAP) 2491252e5c6eSzijun_hu align = 1ul << clamp_t(int, get_count_order_long(size), 2492252e5c6eSzijun_hu PAGE_SHIFT, IOREMAP_MAX_ORDER); 2493252e5c6eSzijun_hu 2494cf88c790STejun Heo area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); 24951da177e4SLinus Torvalds if (unlikely(!area)) 24961da177e4SLinus Torvalds return NULL; 24971da177e4SLinus Torvalds 249871394fe5SAndrey Ryabinin if (!(flags & VM_NO_GUARD)) 24991da177e4SLinus Torvalds size += PAGE_SIZE; 25001da177e4SLinus Torvalds 2501db64fe02SNick Piggin va = alloc_vmap_area(size, align, start, end, node, gfp_mask); 2502db64fe02SNick Piggin if (IS_ERR(va)) { 2503db64fe02SNick Piggin kfree(area); 2504db64fe02SNick Piggin return NULL; 25051da177e4SLinus Torvalds } 25061da177e4SLinus Torvalds 2507d98c9e83SAndrey Ryabinin setup_vmalloc_vm(area, va, flags, caller); 25083c5c3cfbSDaniel Axtens 250919f1c3acSAndrey Konovalov /* 251019f1c3acSAndrey Konovalov * Mark pages for non-VM_ALLOC mappings as accessible. Do it now as a 251119f1c3acSAndrey Konovalov * best-effort approach, as they can be mapped outside of vmalloc code. 251219f1c3acSAndrey Konovalov * For VM_ALLOC mappings, the pages are marked as accessible after 251319f1c3acSAndrey Konovalov * getting mapped in __vmalloc_node_range(). 251423689e91SAndrey Konovalov * With hardware tag-based KASAN, marking is skipped for 251523689e91SAndrey Konovalov * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 251619f1c3acSAndrey Konovalov */ 251719f1c3acSAndrey Konovalov if (!(flags & VM_ALLOC)) 251823689e91SAndrey Konovalov area->addr = kasan_unpoison_vmalloc(area->addr, requested_size, 2519f6e39794SAndrey Konovalov KASAN_VMALLOC_PROT_NORMAL); 25201d96320fSAndrey Konovalov 25211da177e4SLinus Torvalds return area; 25221da177e4SLinus Torvalds } 25231da177e4SLinus Torvalds 2524c2968612SBenjamin Herrenschmidt struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, 2525c2968612SBenjamin Herrenschmidt unsigned long start, unsigned long end, 25265e6cafc8SMarek Szyprowski const void *caller) 2527c2968612SBenjamin Herrenschmidt { 25287ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, start, end, 25297ca3027bSDaniel Axtens NUMA_NO_NODE, GFP_KERNEL, caller); 2530c2968612SBenjamin Herrenschmidt } 2531c2968612SBenjamin Herrenschmidt 25321da177e4SLinus Torvalds /** 2533183ff22bSSimon Arlott * get_vm_area - reserve a contiguous kernel virtual area 25341da177e4SLinus Torvalds * @size: size of the area 25351da177e4SLinus Torvalds * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC 25361da177e4SLinus Torvalds * 25371da177e4SLinus Torvalds * Search an area of @size in the kernel virtual mapping area, 25381da177e4SLinus Torvalds * and reserved it for out purposes. Returns the area descriptor 25391da177e4SLinus Torvalds * on success or %NULL on failure. 2540a862f68aSMike Rapoport * 2541a862f68aSMike Rapoport * Return: the area descriptor on success or %NULL on failure. 25421da177e4SLinus Torvalds */ 25431da177e4SLinus Torvalds struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) 25441da177e4SLinus Torvalds { 25457ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, 25467ca3027bSDaniel Axtens VMALLOC_START, VMALLOC_END, 254700ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, 254800ef2d2fSDavid Rientjes __builtin_return_address(0)); 254923016969SChristoph Lameter } 255023016969SChristoph Lameter 255123016969SChristoph Lameter struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, 25525e6cafc8SMarek Szyprowski const void *caller) 255323016969SChristoph Lameter { 25547ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, 25557ca3027bSDaniel Axtens VMALLOC_START, VMALLOC_END, 255600ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, caller); 25571da177e4SLinus Torvalds } 25581da177e4SLinus Torvalds 2559e9da6e99SMarek Szyprowski /** 2560e9da6e99SMarek Szyprowski * find_vm_area - find a continuous kernel virtual area 2561e9da6e99SMarek Szyprowski * @addr: base address 2562e9da6e99SMarek Szyprowski * 2563e9da6e99SMarek Szyprowski * Search for the kernel VM area starting at @addr, and return it. 2564e9da6e99SMarek Szyprowski * It is up to the caller to do all required locking to keep the returned 2565e9da6e99SMarek Szyprowski * pointer valid. 2566a862f68aSMike Rapoport * 256774640617SHui Su * Return: the area descriptor on success or %NULL on failure. 2568e9da6e99SMarek Szyprowski */ 2569e9da6e99SMarek Szyprowski struct vm_struct *find_vm_area(const void *addr) 257083342314SNick Piggin { 2571db64fe02SNick Piggin struct vmap_area *va; 257283342314SNick Piggin 2573db64fe02SNick Piggin va = find_vmap_area((unsigned long)addr); 2574688fcbfcSPengfei Li if (!va) 25757856dfebSAndi Kleen return NULL; 2576688fcbfcSPengfei Li 2577688fcbfcSPengfei Li return va->vm; 25787856dfebSAndi Kleen } 25797856dfebSAndi Kleen 25801da177e4SLinus Torvalds /** 2581183ff22bSSimon Arlott * remove_vm_area - find and remove a continuous kernel virtual area 25821da177e4SLinus Torvalds * @addr: base address 25831da177e4SLinus Torvalds * 25841da177e4SLinus Torvalds * Search for the kernel VM area starting at @addr, and remove it. 25851da177e4SLinus Torvalds * This function returns the found VM area, but using it is NOT safe 25867856dfebSAndi Kleen * on SMP machines, except for its size or flags. 2587a862f68aSMike Rapoport * 258874640617SHui Su * Return: the area descriptor on success or %NULL on failure. 25891da177e4SLinus Torvalds */ 2590b3bdda02SChristoph Lameter struct vm_struct *remove_vm_area(const void *addr) 25911da177e4SLinus Torvalds { 2592db64fe02SNick Piggin struct vmap_area *va; 2593db64fe02SNick Piggin 25945803ed29SChristoph Hellwig might_sleep(); 25955803ed29SChristoph Hellwig 2596dd3b8353SUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 2597899c6efeSUladzislau Rezki (Sony) va = __find_vmap_area((unsigned long)addr, &vmap_area_root); 2598688fcbfcSPengfei Li if (va && va->vm) { 2599db1aecafSMinchan Kim struct vm_struct *vm = va->vm; 2600f5252e00SMitsuo Hayasaka 2601c69480adSJoonsoo Kim va->vm = NULL; 2602c69480adSJoonsoo Kim spin_unlock(&vmap_area_lock); 2603c69480adSJoonsoo Kim 260463840de2SAndrey Konovalov kasan_free_module_shadow(vm); 2605dd32c279SKAMEZAWA Hiroyuki free_unmap_vmap_area(va); 2606dd32c279SKAMEZAWA Hiroyuki 2607db64fe02SNick Piggin return vm; 2608db64fe02SNick Piggin } 2609dd3b8353SUladzislau Rezki (Sony) 2610dd3b8353SUladzislau Rezki (Sony) spin_unlock(&vmap_area_lock); 2611db64fe02SNick Piggin return NULL; 26121da177e4SLinus Torvalds } 26131da177e4SLinus Torvalds 2614868b104dSRick Edgecombe static inline void set_area_direct_map(const struct vm_struct *area, 2615868b104dSRick Edgecombe int (*set_direct_map)(struct page *page)) 2616868b104dSRick Edgecombe { 2617868b104dSRick Edgecombe int i; 2618868b104dSRick Edgecombe 2619121e6f32SNicholas Piggin /* HUGE_VMALLOC passes small pages to set_direct_map */ 2620868b104dSRick Edgecombe for (i = 0; i < area->nr_pages; i++) 2621868b104dSRick Edgecombe if (page_address(area->pages[i])) 2622868b104dSRick Edgecombe set_direct_map(area->pages[i]); 2623868b104dSRick Edgecombe } 2624868b104dSRick Edgecombe 2625868b104dSRick Edgecombe /* Handle removing and resetting vm mappings related to the vm_struct. */ 2626868b104dSRick Edgecombe static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages) 2627868b104dSRick Edgecombe { 2628868b104dSRick Edgecombe unsigned long start = ULONG_MAX, end = 0; 2629121e6f32SNicholas Piggin unsigned int page_order = vm_area_page_order(area); 2630868b104dSRick Edgecombe int flush_reset = area->flags & VM_FLUSH_RESET_PERMS; 263131e67340SRick Edgecombe int flush_dmap = 0; 2632868b104dSRick Edgecombe int i; 2633868b104dSRick Edgecombe 2634868b104dSRick Edgecombe remove_vm_area(area->addr); 2635868b104dSRick Edgecombe 2636868b104dSRick Edgecombe /* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */ 2637868b104dSRick Edgecombe if (!flush_reset) 2638868b104dSRick Edgecombe return; 2639868b104dSRick Edgecombe 2640868b104dSRick Edgecombe /* 2641868b104dSRick Edgecombe * If not deallocating pages, just do the flush of the VM area and 2642868b104dSRick Edgecombe * return. 2643868b104dSRick Edgecombe */ 2644868b104dSRick Edgecombe if (!deallocate_pages) { 2645868b104dSRick Edgecombe vm_unmap_aliases(); 2646868b104dSRick Edgecombe return; 2647868b104dSRick Edgecombe } 2648868b104dSRick Edgecombe 2649868b104dSRick Edgecombe /* 2650868b104dSRick Edgecombe * If execution gets here, flush the vm mapping and reset the direct 2651868b104dSRick Edgecombe * map. Find the start and end range of the direct mappings to make sure 2652868b104dSRick Edgecombe * the vm_unmap_aliases() flush includes the direct map. 2653868b104dSRick Edgecombe */ 2654121e6f32SNicholas Piggin for (i = 0; i < area->nr_pages; i += 1U << page_order) { 26558e41f872SRick Edgecombe unsigned long addr = (unsigned long)page_address(area->pages[i]); 26568e41f872SRick Edgecombe if (addr) { 2657121e6f32SNicholas Piggin unsigned long page_size; 2658121e6f32SNicholas Piggin 2659121e6f32SNicholas Piggin page_size = PAGE_SIZE << page_order; 2660868b104dSRick Edgecombe start = min(addr, start); 2661121e6f32SNicholas Piggin end = max(addr + page_size, end); 266231e67340SRick Edgecombe flush_dmap = 1; 2663868b104dSRick Edgecombe } 2664868b104dSRick Edgecombe } 2665868b104dSRick Edgecombe 2666868b104dSRick Edgecombe /* 2667868b104dSRick Edgecombe * Set direct map to something invalid so that it won't be cached if 2668868b104dSRick Edgecombe * there are any accesses after the TLB flush, then flush the TLB and 2669868b104dSRick Edgecombe * reset the direct map permissions to the default. 2670868b104dSRick Edgecombe */ 2671868b104dSRick Edgecombe set_area_direct_map(area, set_direct_map_invalid_noflush); 267231e67340SRick Edgecombe _vm_unmap_aliases(start, end, flush_dmap); 2673868b104dSRick Edgecombe set_area_direct_map(area, set_direct_map_default_noflush); 2674868b104dSRick Edgecombe } 2675868b104dSRick Edgecombe 2676b3bdda02SChristoph Lameter static void __vunmap(const void *addr, int deallocate_pages) 26771da177e4SLinus Torvalds { 26781da177e4SLinus Torvalds struct vm_struct *area; 26791da177e4SLinus Torvalds 26801da177e4SLinus Torvalds if (!addr) 26811da177e4SLinus Torvalds return; 26821da177e4SLinus Torvalds 2683e69e9d4aSHATAYAMA Daisuke if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n", 2684ab15d9b4SDan Carpenter addr)) 26851da177e4SLinus Torvalds return; 26861da177e4SLinus Torvalds 26876ade2032SLiviu Dudau area = find_vm_area(addr); 26881da177e4SLinus Torvalds if (unlikely(!area)) { 26894c8573e2SArjan van de Ven WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", 26901da177e4SLinus Torvalds addr); 26911da177e4SLinus Torvalds return; 26921da177e4SLinus Torvalds } 26931da177e4SLinus Torvalds 269405e3ff95SChintan Pandya debug_check_no_locks_freed(area->addr, get_vm_area_size(area)); 269505e3ff95SChintan Pandya debug_check_no_obj_freed(area->addr, get_vm_area_size(area)); 26969a11b49aSIngo Molnar 2697c041098cSVincenzo Frascino kasan_poison_vmalloc(area->addr, get_vm_area_size(area)); 26983c5c3cfbSDaniel Axtens 2699868b104dSRick Edgecombe vm_remove_mappings(area, deallocate_pages); 2700868b104dSRick Edgecombe 27011da177e4SLinus Torvalds if (deallocate_pages) { 27023b8000aeSNicholas Piggin int i; 27031da177e4SLinus Torvalds 27043b8000aeSNicholas Piggin for (i = 0; i < area->nr_pages; i++) { 2705bf53d6f8SChristoph Lameter struct page *page = area->pages[i]; 2706bf53d6f8SChristoph Lameter 2707bf53d6f8SChristoph Lameter BUG_ON(!page); 27083b8000aeSNicholas Piggin mod_memcg_page_state(page, MEMCG_VMALLOC, -1); 27093b8000aeSNicholas Piggin /* 27103b8000aeSNicholas Piggin * High-order allocs for huge vmallocs are split, so 27113b8000aeSNicholas Piggin * can be freed as an array of order-0 allocations 27123b8000aeSNicholas Piggin */ 27133b8000aeSNicholas Piggin __free_pages(page, 0); 2714a850e932SRafael Aquini cond_resched(); 27151da177e4SLinus Torvalds } 271697105f0aSRoman Gushchin atomic_long_sub(area->nr_pages, &nr_vmalloc_pages); 27171da177e4SLinus Torvalds 2718244d63eeSDavid Rientjes kvfree(area->pages); 27191da177e4SLinus Torvalds } 27201da177e4SLinus Torvalds 27211da177e4SLinus Torvalds kfree(area); 27221da177e4SLinus Torvalds } 27231da177e4SLinus Torvalds 2724bf22e37aSAndrey Ryabinin static inline void __vfree_deferred(const void *addr) 2725bf22e37aSAndrey Ryabinin { 2726bf22e37aSAndrey Ryabinin /* 2727bf22e37aSAndrey Ryabinin * Use raw_cpu_ptr() because this can be called from preemptible 2728bf22e37aSAndrey Ryabinin * context. Preemption is absolutely fine here, because the llist_add() 2729bf22e37aSAndrey Ryabinin * implementation is lockless, so it works even if we are adding to 273073221d88SJeongtae Park * another cpu's list. schedule_work() should be fine with this too. 2731bf22e37aSAndrey Ryabinin */ 2732bf22e37aSAndrey Ryabinin struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred); 2733bf22e37aSAndrey Ryabinin 2734bf22e37aSAndrey Ryabinin if (llist_add((struct llist_node *)addr, &p->list)) 2735bf22e37aSAndrey Ryabinin schedule_work(&p->wq); 2736bf22e37aSAndrey Ryabinin } 2737bf22e37aSAndrey Ryabinin 2738bf22e37aSAndrey Ryabinin /** 2739bf22e37aSAndrey Ryabinin * vfree_atomic - release memory allocated by vmalloc() 2740bf22e37aSAndrey Ryabinin * @addr: memory base address 2741bf22e37aSAndrey Ryabinin * 2742bf22e37aSAndrey Ryabinin * This one is just like vfree() but can be called in any atomic context 2743bf22e37aSAndrey Ryabinin * except NMIs. 2744bf22e37aSAndrey Ryabinin */ 2745bf22e37aSAndrey Ryabinin void vfree_atomic(const void *addr) 2746bf22e37aSAndrey Ryabinin { 2747bf22e37aSAndrey Ryabinin BUG_ON(in_nmi()); 2748bf22e37aSAndrey Ryabinin 2749bf22e37aSAndrey Ryabinin kmemleak_free(addr); 2750bf22e37aSAndrey Ryabinin 2751bf22e37aSAndrey Ryabinin if (!addr) 2752bf22e37aSAndrey Ryabinin return; 2753bf22e37aSAndrey Ryabinin __vfree_deferred(addr); 2754bf22e37aSAndrey Ryabinin } 2755bf22e37aSAndrey Ryabinin 2756c67dc624SRoman Penyaev static void __vfree(const void *addr) 2757c67dc624SRoman Penyaev { 2758c67dc624SRoman Penyaev if (unlikely(in_interrupt())) 2759c67dc624SRoman Penyaev __vfree_deferred(addr); 2760c67dc624SRoman Penyaev else 2761c67dc624SRoman Penyaev __vunmap(addr, 1); 2762c67dc624SRoman Penyaev } 2763c67dc624SRoman Penyaev 27641da177e4SLinus Torvalds /** 2765fa307474SMatthew Wilcox (Oracle) * vfree - Release memory allocated by vmalloc() 2766fa307474SMatthew Wilcox (Oracle) * @addr: Memory base address 27671da177e4SLinus Torvalds * 2768fa307474SMatthew Wilcox (Oracle) * Free the virtually continuous memory area starting at @addr, as obtained 2769fa307474SMatthew Wilcox (Oracle) * from one of the vmalloc() family of APIs. This will usually also free the 2770fa307474SMatthew Wilcox (Oracle) * physical memory underlying the virtual allocation, but that memory is 2771fa307474SMatthew Wilcox (Oracle) * reference counted, so it will not be freed until the last user goes away. 27721da177e4SLinus Torvalds * 2773fa307474SMatthew Wilcox (Oracle) * If @addr is NULL, no operation is performed. 277432fcfd40SAl Viro * 2775fa307474SMatthew Wilcox (Oracle) * Context: 27763ca4ea3aSAndrey Ryabinin * May sleep if called *not* from interrupt context. 2777fa307474SMatthew Wilcox (Oracle) * Must not be called in NMI context (strictly speaking, it could be 2778fa307474SMatthew Wilcox (Oracle) * if we have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling 2779f0953a1bSIngo Molnar * conventions for vfree() arch-dependent would be a really bad idea). 27801da177e4SLinus Torvalds */ 2781b3bdda02SChristoph Lameter void vfree(const void *addr) 27821da177e4SLinus Torvalds { 278332fcfd40SAl Viro BUG_ON(in_nmi()); 278489219d37SCatalin Marinas 278589219d37SCatalin Marinas kmemleak_free(addr); 278689219d37SCatalin Marinas 2787a8dda165SAndrey Ryabinin might_sleep_if(!in_interrupt()); 2788a8dda165SAndrey Ryabinin 278932fcfd40SAl Viro if (!addr) 279032fcfd40SAl Viro return; 2791c67dc624SRoman Penyaev 2792c67dc624SRoman Penyaev __vfree(addr); 27931da177e4SLinus Torvalds } 27941da177e4SLinus Torvalds EXPORT_SYMBOL(vfree); 27951da177e4SLinus Torvalds 27961da177e4SLinus Torvalds /** 27971da177e4SLinus Torvalds * vunmap - release virtual mapping obtained by vmap() 27981da177e4SLinus Torvalds * @addr: memory base address 27991da177e4SLinus Torvalds * 28001da177e4SLinus Torvalds * Free the virtually contiguous memory area starting at @addr, 28011da177e4SLinus Torvalds * which was created from the page array passed to vmap(). 28021da177e4SLinus Torvalds * 280380e93effSPekka Enberg * Must not be called in interrupt context. 28041da177e4SLinus Torvalds */ 2805b3bdda02SChristoph Lameter void vunmap(const void *addr) 28061da177e4SLinus Torvalds { 28071da177e4SLinus Torvalds BUG_ON(in_interrupt()); 280834754b69SPeter Zijlstra might_sleep(); 280932fcfd40SAl Viro if (addr) 28101da177e4SLinus Torvalds __vunmap(addr, 0); 28111da177e4SLinus Torvalds } 28121da177e4SLinus Torvalds EXPORT_SYMBOL(vunmap); 28131da177e4SLinus Torvalds 28141da177e4SLinus Torvalds /** 28151da177e4SLinus Torvalds * vmap - map an array of pages into virtually contiguous space 28161da177e4SLinus Torvalds * @pages: array of page pointers 28171da177e4SLinus Torvalds * @count: number of pages to map 28181da177e4SLinus Torvalds * @flags: vm_area->flags 28191da177e4SLinus Torvalds * @prot: page protection for the mapping 28201da177e4SLinus Torvalds * 2821b944afc9SChristoph Hellwig * Maps @count pages from @pages into contiguous kernel virtual space. 2822b944afc9SChristoph Hellwig * If @flags contains %VM_MAP_PUT_PAGES the ownership of the pages array itself 2823b944afc9SChristoph Hellwig * (which must be kmalloc or vmalloc memory) and one reference per pages in it 2824b944afc9SChristoph Hellwig * are transferred from the caller to vmap(), and will be freed / dropped when 2825b944afc9SChristoph Hellwig * vfree() is called on the return value. 2826a862f68aSMike Rapoport * 2827a862f68aSMike Rapoport * Return: the address of the area or %NULL on failure 28281da177e4SLinus Torvalds */ 28291da177e4SLinus Torvalds void *vmap(struct page **pages, unsigned int count, 28301da177e4SLinus Torvalds unsigned long flags, pgprot_t prot) 28311da177e4SLinus Torvalds { 28321da177e4SLinus Torvalds struct vm_struct *area; 2833b67177ecSNicholas Piggin unsigned long addr; 283465ee03c4SGuillermo Julián Moreno unsigned long size; /* In bytes */ 28351da177e4SLinus Torvalds 283634754b69SPeter Zijlstra might_sleep(); 283734754b69SPeter Zijlstra 2838bd1a8fb2SPeter Zijlstra /* 2839bd1a8fb2SPeter Zijlstra * Your top guard is someone else's bottom guard. Not having a top 2840bd1a8fb2SPeter Zijlstra * guard compromises someone else's mappings too. 2841bd1a8fb2SPeter Zijlstra */ 2842bd1a8fb2SPeter Zijlstra if (WARN_ON_ONCE(flags & VM_NO_GUARD)) 2843bd1a8fb2SPeter Zijlstra flags &= ~VM_NO_GUARD; 2844bd1a8fb2SPeter Zijlstra 2845ca79b0c2SArun KS if (count > totalram_pages()) 28461da177e4SLinus Torvalds return NULL; 28471da177e4SLinus Torvalds 284865ee03c4SGuillermo Julián Moreno size = (unsigned long)count << PAGE_SHIFT; 284965ee03c4SGuillermo Julián Moreno area = get_vm_area_caller(size, flags, __builtin_return_address(0)); 28501da177e4SLinus Torvalds if (!area) 28511da177e4SLinus Torvalds return NULL; 285223016969SChristoph Lameter 2853b67177ecSNicholas Piggin addr = (unsigned long)area->addr; 2854b67177ecSNicholas Piggin if (vmap_pages_range(addr, addr + size, pgprot_nx(prot), 2855b67177ecSNicholas Piggin pages, PAGE_SHIFT) < 0) { 28561da177e4SLinus Torvalds vunmap(area->addr); 28571da177e4SLinus Torvalds return NULL; 28581da177e4SLinus Torvalds } 28591da177e4SLinus Torvalds 2860c22ee528SMiaohe Lin if (flags & VM_MAP_PUT_PAGES) { 2861b944afc9SChristoph Hellwig area->pages = pages; 2862c22ee528SMiaohe Lin area->nr_pages = count; 2863c22ee528SMiaohe Lin } 28641da177e4SLinus Torvalds return area->addr; 28651da177e4SLinus Torvalds } 28661da177e4SLinus Torvalds EXPORT_SYMBOL(vmap); 28671da177e4SLinus Torvalds 28683e9a9e25SChristoph Hellwig #ifdef CONFIG_VMAP_PFN 28693e9a9e25SChristoph Hellwig struct vmap_pfn_data { 28703e9a9e25SChristoph Hellwig unsigned long *pfns; 28713e9a9e25SChristoph Hellwig pgprot_t prot; 28723e9a9e25SChristoph Hellwig unsigned int idx; 28733e9a9e25SChristoph Hellwig }; 28743e9a9e25SChristoph Hellwig 28753e9a9e25SChristoph Hellwig static int vmap_pfn_apply(pte_t *pte, unsigned long addr, void *private) 28763e9a9e25SChristoph Hellwig { 28773e9a9e25SChristoph Hellwig struct vmap_pfn_data *data = private; 28783e9a9e25SChristoph Hellwig 28793e9a9e25SChristoph Hellwig if (WARN_ON_ONCE(pfn_valid(data->pfns[data->idx]))) 28803e9a9e25SChristoph Hellwig return -EINVAL; 28813e9a9e25SChristoph Hellwig *pte = pte_mkspecial(pfn_pte(data->pfns[data->idx++], data->prot)); 28823e9a9e25SChristoph Hellwig return 0; 28833e9a9e25SChristoph Hellwig } 28843e9a9e25SChristoph Hellwig 28853e9a9e25SChristoph Hellwig /** 28863e9a9e25SChristoph Hellwig * vmap_pfn - map an array of PFNs into virtually contiguous space 28873e9a9e25SChristoph Hellwig * @pfns: array of PFNs 28883e9a9e25SChristoph Hellwig * @count: number of pages to map 28893e9a9e25SChristoph Hellwig * @prot: page protection for the mapping 28903e9a9e25SChristoph Hellwig * 28913e9a9e25SChristoph Hellwig * Maps @count PFNs from @pfns into contiguous kernel virtual space and returns 28923e9a9e25SChristoph Hellwig * the start address of the mapping. 28933e9a9e25SChristoph Hellwig */ 28943e9a9e25SChristoph Hellwig void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot) 28953e9a9e25SChristoph Hellwig { 28963e9a9e25SChristoph Hellwig struct vmap_pfn_data data = { .pfns = pfns, .prot = pgprot_nx(prot) }; 28973e9a9e25SChristoph Hellwig struct vm_struct *area; 28983e9a9e25SChristoph Hellwig 28993e9a9e25SChristoph Hellwig area = get_vm_area_caller(count * PAGE_SIZE, VM_IOREMAP, 29003e9a9e25SChristoph Hellwig __builtin_return_address(0)); 29013e9a9e25SChristoph Hellwig if (!area) 29023e9a9e25SChristoph Hellwig return NULL; 29033e9a9e25SChristoph Hellwig if (apply_to_page_range(&init_mm, (unsigned long)area->addr, 29043e9a9e25SChristoph Hellwig count * PAGE_SIZE, vmap_pfn_apply, &data)) { 29053e9a9e25SChristoph Hellwig free_vm_area(area); 29063e9a9e25SChristoph Hellwig return NULL; 29073e9a9e25SChristoph Hellwig } 29083e9a9e25SChristoph Hellwig return area->addr; 29093e9a9e25SChristoph Hellwig } 29103e9a9e25SChristoph Hellwig EXPORT_SYMBOL_GPL(vmap_pfn); 29113e9a9e25SChristoph Hellwig #endif /* CONFIG_VMAP_PFN */ 29123e9a9e25SChristoph Hellwig 291312b9f873SUladzislau Rezki static inline unsigned int 291412b9f873SUladzislau Rezki vm_area_alloc_pages(gfp_t gfp, int nid, 2915343ab817SUladzislau Rezki (Sony) unsigned int order, unsigned int nr_pages, struct page **pages) 291612b9f873SUladzislau Rezki { 291712b9f873SUladzislau Rezki unsigned int nr_allocated = 0; 2918ffb29b1cSChen Wandun struct page *page; 2919ffb29b1cSChen Wandun int i; 292012b9f873SUladzislau Rezki 292112b9f873SUladzislau Rezki /* 292212b9f873SUladzislau Rezki * For order-0 pages we make use of bulk allocator, if 292312b9f873SUladzislau Rezki * the page array is partly or not at all populated due 292412b9f873SUladzislau Rezki * to fails, fallback to a single page allocator that is 292512b9f873SUladzislau Rezki * more permissive. 292612b9f873SUladzislau Rezki */ 2927c00b6b96SChen Wandun if (!order) { 29289376130cSMichal Hocko gfp_t bulk_gfp = gfp & ~__GFP_NOFAIL; 29299376130cSMichal Hocko 2930343ab817SUladzislau Rezki (Sony) while (nr_allocated < nr_pages) { 2931343ab817SUladzislau Rezki (Sony) unsigned int nr, nr_pages_request; 2932343ab817SUladzislau Rezki (Sony) 2933343ab817SUladzislau Rezki (Sony) /* 2934343ab817SUladzislau Rezki (Sony) * A maximum allowed request is hard-coded and is 100 2935343ab817SUladzislau Rezki (Sony) * pages per call. That is done in order to prevent a 2936343ab817SUladzislau Rezki (Sony) * long preemption off scenario in the bulk-allocator 2937343ab817SUladzislau Rezki (Sony) * so the range is [1:100]. 2938343ab817SUladzislau Rezki (Sony) */ 2939343ab817SUladzislau Rezki (Sony) nr_pages_request = min(100U, nr_pages - nr_allocated); 2940343ab817SUladzislau Rezki (Sony) 2941c00b6b96SChen Wandun /* memory allocation should consider mempolicy, we can't 2942c00b6b96SChen Wandun * wrongly use nearest node when nid == NUMA_NO_NODE, 2943c00b6b96SChen Wandun * otherwise memory may be allocated in only one node, 294498af39d5SYixuan Cao * but mempolicy wants to alloc memory by interleaving. 2945c00b6b96SChen Wandun */ 2946c00b6b96SChen Wandun if (IS_ENABLED(CONFIG_NUMA) && nid == NUMA_NO_NODE) 29479376130cSMichal Hocko nr = alloc_pages_bulk_array_mempolicy(bulk_gfp, 2948c00b6b96SChen Wandun nr_pages_request, 2949c00b6b96SChen Wandun pages + nr_allocated); 2950c00b6b96SChen Wandun 2951c00b6b96SChen Wandun else 29529376130cSMichal Hocko nr = alloc_pages_bulk_array_node(bulk_gfp, nid, 2953c00b6b96SChen Wandun nr_pages_request, 2954c00b6b96SChen Wandun pages + nr_allocated); 2955343ab817SUladzislau Rezki (Sony) 2956343ab817SUladzislau Rezki (Sony) nr_allocated += nr; 2957343ab817SUladzislau Rezki (Sony) cond_resched(); 2958343ab817SUladzislau Rezki (Sony) 2959343ab817SUladzislau Rezki (Sony) /* 2960343ab817SUladzislau Rezki (Sony) * If zero or pages were obtained partly, 2961343ab817SUladzislau Rezki (Sony) * fallback to a single page allocator. 2962343ab817SUladzislau Rezki (Sony) */ 2963343ab817SUladzislau Rezki (Sony) if (nr != nr_pages_request) 2964343ab817SUladzislau Rezki (Sony) break; 2965343ab817SUladzislau Rezki (Sony) } 29663b8000aeSNicholas Piggin } 296712b9f873SUladzislau Rezki 296812b9f873SUladzislau Rezki /* High-order pages or fallback path if "bulk" fails. */ 296912b9f873SUladzislau Rezki 2970ffb29b1cSChen Wandun while (nr_allocated < nr_pages) { 2971dd544141SVasily Averin if (fatal_signal_pending(current)) 2972dd544141SVasily Averin break; 2973dd544141SVasily Averin 2974ffb29b1cSChen Wandun if (nid == NUMA_NO_NODE) 2975ffb29b1cSChen Wandun page = alloc_pages(gfp, order); 2976ffb29b1cSChen Wandun else 297712b9f873SUladzislau Rezki page = alloc_pages_node(nid, gfp, order); 297812b9f873SUladzislau Rezki if (unlikely(!page)) 297912b9f873SUladzislau Rezki break; 29803b8000aeSNicholas Piggin /* 29813b8000aeSNicholas Piggin * Higher order allocations must be able to be treated as 29823b8000aeSNicholas Piggin * indepdenent small pages by callers (as they can with 29833b8000aeSNicholas Piggin * small-page vmallocs). Some drivers do their own refcounting 29843b8000aeSNicholas Piggin * on vmalloc_to_page() pages, some use page->mapping, 29853b8000aeSNicholas Piggin * page->lru, etc. 29863b8000aeSNicholas Piggin */ 29873b8000aeSNicholas Piggin if (order) 29883b8000aeSNicholas Piggin split_page(page, order); 298912b9f873SUladzislau Rezki 299012b9f873SUladzislau Rezki /* 299112b9f873SUladzislau Rezki * Careful, we allocate and map page-order pages, but 299212b9f873SUladzislau Rezki * tracking is done per PAGE_SIZE page so as to keep the 299312b9f873SUladzislau Rezki * vm_struct APIs independent of the physical/mapped size. 299412b9f873SUladzislau Rezki */ 299512b9f873SUladzislau Rezki for (i = 0; i < (1U << order); i++) 299612b9f873SUladzislau Rezki pages[nr_allocated + i] = page + i; 299712b9f873SUladzislau Rezki 299812b9f873SUladzislau Rezki cond_resched(); 299912b9f873SUladzislau Rezki nr_allocated += 1U << order; 300012b9f873SUladzislau Rezki } 300112b9f873SUladzislau Rezki 300212b9f873SUladzislau Rezki return nr_allocated; 300312b9f873SUladzislau Rezki } 300412b9f873SUladzislau Rezki 3005e31d9eb5SAdrian Bunk static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, 3006121e6f32SNicholas Piggin pgprot_t prot, unsigned int page_shift, 3007121e6f32SNicholas Piggin int node) 30081da177e4SLinus Torvalds { 3009930f036bSDavid Rientjes const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; 30109376130cSMichal Hocko bool nofail = gfp_mask & __GFP_NOFAIL; 3011121e6f32SNicholas Piggin unsigned long addr = (unsigned long)area->addr; 3012121e6f32SNicholas Piggin unsigned long size = get_vm_area_size(area); 301334fe6537SAndrew Morton unsigned long array_size; 3014121e6f32SNicholas Piggin unsigned int nr_small_pages = size >> PAGE_SHIFT; 3015121e6f32SNicholas Piggin unsigned int page_order; 3016451769ebSMichal Hocko unsigned int flags; 3017451769ebSMichal Hocko int ret; 30181da177e4SLinus Torvalds 3019121e6f32SNicholas Piggin array_size = (unsigned long)nr_small_pages * sizeof(struct page *); 3020f255935bSChristoph Hellwig gfp_mask |= __GFP_NOWARN; 3021f255935bSChristoph Hellwig if (!(gfp_mask & (GFP_DMA | GFP_DMA32))) 3022f255935bSChristoph Hellwig gfp_mask |= __GFP_HIGHMEM; 30231da177e4SLinus Torvalds 30241da177e4SLinus Torvalds /* Please note that the recursion is strictly bounded. */ 30258757d5faSJan Kiszka if (array_size > PAGE_SIZE) { 30265c1f4e69SUladzislau Rezki (Sony) area->pages = __vmalloc_node(array_size, 1, nested_gfp, node, 3027f255935bSChristoph Hellwig area->caller); 3028286e1ea3SAndrew Morton } else { 30295c1f4e69SUladzislau Rezki (Sony) area->pages = kmalloc_node(array_size, nested_gfp, node); 3030286e1ea3SAndrew Morton } 30317ea36242SAustin Kim 30325c1f4e69SUladzislau Rezki (Sony) if (!area->pages) { 3033c3d77172SUladzislau Rezki (Sony) warn_alloc(gfp_mask, NULL, 3034f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, failed to allocated page array size %lu", 3035d70bec8cSNicholas Piggin nr_small_pages * PAGE_SIZE, array_size); 3036cd61413bSUladzislau Rezki (Sony) free_vm_area(area); 30371da177e4SLinus Torvalds return NULL; 30381da177e4SLinus Torvalds } 30391da177e4SLinus Torvalds 3040121e6f32SNicholas Piggin set_vm_area_page_order(area, page_shift - PAGE_SHIFT); 3041121e6f32SNicholas Piggin page_order = vm_area_page_order(area); 3042121e6f32SNicholas Piggin 3043c3d77172SUladzislau Rezki (Sony) area->nr_pages = vm_area_alloc_pages(gfp_mask | __GFP_NOWARN, 3044c3d77172SUladzislau Rezki (Sony) node, page_order, nr_small_pages, area->pages); 30455c1f4e69SUladzislau Rezki (Sony) 304697105f0aSRoman Gushchin atomic_long_add(area->nr_pages, &nr_vmalloc_pages); 30474e5aa1f4SShakeel Butt if (gfp_mask & __GFP_ACCOUNT) { 30483b8000aeSNicholas Piggin int i; 30494e5aa1f4SShakeel Butt 30503b8000aeSNicholas Piggin for (i = 0; i < area->nr_pages; i++) 30513b8000aeSNicholas Piggin mod_memcg_page_state(area->pages[i], MEMCG_VMALLOC, 1); 30524e5aa1f4SShakeel Butt } 30535c1f4e69SUladzislau Rezki (Sony) 30545c1f4e69SUladzislau Rezki (Sony) /* 30555c1f4e69SUladzislau Rezki (Sony) * If not enough pages were obtained to accomplish an 30565c1f4e69SUladzislau Rezki (Sony) * allocation request, free them via __vfree() if any. 30575c1f4e69SUladzislau Rezki (Sony) */ 30585c1f4e69SUladzislau Rezki (Sony) if (area->nr_pages != nr_small_pages) { 3059c3d77172SUladzislau Rezki (Sony) warn_alloc(gfp_mask, NULL, 3060f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, page order %u, failed to allocate pages", 3061d70bec8cSNicholas Piggin area->nr_pages * PAGE_SIZE, page_order); 30621da177e4SLinus Torvalds goto fail; 30631da177e4SLinus Torvalds } 3064121e6f32SNicholas Piggin 3065451769ebSMichal Hocko /* 3066451769ebSMichal Hocko * page tables allocations ignore external gfp mask, enforce it 3067451769ebSMichal Hocko * by the scope API 3068451769ebSMichal Hocko */ 3069451769ebSMichal Hocko if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO) 3070451769ebSMichal Hocko flags = memalloc_nofs_save(); 3071451769ebSMichal Hocko else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0) 3072451769ebSMichal Hocko flags = memalloc_noio_save(); 3073451769ebSMichal Hocko 30749376130cSMichal Hocko do { 3075451769ebSMichal Hocko ret = vmap_pages_range(addr, addr + size, prot, area->pages, 3076451769ebSMichal Hocko page_shift); 30779376130cSMichal Hocko if (nofail && (ret < 0)) 30789376130cSMichal Hocko schedule_timeout_uninterruptible(1); 30799376130cSMichal Hocko } while (nofail && (ret < 0)); 3080451769ebSMichal Hocko 3081451769ebSMichal Hocko if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO) 3082451769ebSMichal Hocko memalloc_nofs_restore(flags); 3083451769ebSMichal Hocko else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0) 3084451769ebSMichal Hocko memalloc_noio_restore(flags); 3085451769ebSMichal Hocko 3086451769ebSMichal Hocko if (ret < 0) { 3087c3d77172SUladzislau Rezki (Sony) warn_alloc(gfp_mask, NULL, 3088f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, failed to map pages", 3089d70bec8cSNicholas Piggin area->nr_pages * PAGE_SIZE); 30901da177e4SLinus Torvalds goto fail; 3091d70bec8cSNicholas Piggin } 3092ed1f324cSChristoph Hellwig 30931da177e4SLinus Torvalds return area->addr; 30941da177e4SLinus Torvalds 30951da177e4SLinus Torvalds fail: 3096c67dc624SRoman Penyaev __vfree(area->addr); 30971da177e4SLinus Torvalds return NULL; 30981da177e4SLinus Torvalds } 30991da177e4SLinus Torvalds 3100d0a21265SDavid Rientjes /** 3101d0a21265SDavid Rientjes * __vmalloc_node_range - allocate virtually contiguous memory 3102d0a21265SDavid Rientjes * @size: allocation size 3103d0a21265SDavid Rientjes * @align: desired alignment 3104d0a21265SDavid Rientjes * @start: vm area range start 3105d0a21265SDavid Rientjes * @end: vm area range end 3106d0a21265SDavid Rientjes * @gfp_mask: flags for the page level allocator 3107d0a21265SDavid Rientjes * @prot: protection mask for the allocated pages 3108cb9e3c29SAndrey Ryabinin * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) 310900ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 3110d0a21265SDavid Rientjes * @caller: caller's return address 3111d0a21265SDavid Rientjes * 3112d0a21265SDavid Rientjes * Allocate enough pages to cover @size from the page level 3113b7d90e7aSMichal Hocko * allocator with @gfp_mask flags. Please note that the full set of gfp 311430d3f011SMichal Hocko * flags are not supported. GFP_KERNEL, GFP_NOFS and GFP_NOIO are all 311530d3f011SMichal Hocko * supported. 311630d3f011SMichal Hocko * Zone modifiers are not supported. From the reclaim modifiers 311730d3f011SMichal Hocko * __GFP_DIRECT_RECLAIM is required (aka GFP_NOWAIT is not supported) 311830d3f011SMichal Hocko * and only __GFP_NOFAIL is supported (i.e. __GFP_NORETRY and 311930d3f011SMichal Hocko * __GFP_RETRY_MAYFAIL are not supported). 312030d3f011SMichal Hocko * 312130d3f011SMichal Hocko * __GFP_NOWARN can be used to suppress failures messages. 3122b7d90e7aSMichal Hocko * 3123b7d90e7aSMichal Hocko * Map them into contiguous kernel virtual space, using a pagetable 3124b7d90e7aSMichal Hocko * protection of @prot. 3125a862f68aSMike Rapoport * 3126a862f68aSMike Rapoport * Return: the address of the area or %NULL on failure 3127d0a21265SDavid Rientjes */ 3128d0a21265SDavid Rientjes void *__vmalloc_node_range(unsigned long size, unsigned long align, 3129d0a21265SDavid Rientjes unsigned long start, unsigned long end, gfp_t gfp_mask, 3130cb9e3c29SAndrey Ryabinin pgprot_t prot, unsigned long vm_flags, int node, 3131cb9e3c29SAndrey Ryabinin const void *caller) 3132930fc45aSChristoph Lameter { 3133d0a21265SDavid Rientjes struct vm_struct *area; 313419f1c3acSAndrey Konovalov void *ret; 3135f6e39794SAndrey Konovalov kasan_vmalloc_flags_t kasan_flags = KASAN_VMALLOC_NONE; 3136d0a21265SDavid Rientjes unsigned long real_size = size; 3137121e6f32SNicholas Piggin unsigned long real_align = align; 3138121e6f32SNicholas Piggin unsigned int shift = PAGE_SHIFT; 3139d0a21265SDavid Rientjes 3140d70bec8cSNicholas Piggin if (WARN_ON_ONCE(!size)) 3141d70bec8cSNicholas Piggin return NULL; 3142d70bec8cSNicholas Piggin 3143d70bec8cSNicholas Piggin if ((size >> PAGE_SHIFT) > totalram_pages()) { 3144d70bec8cSNicholas Piggin warn_alloc(gfp_mask, NULL, 3145f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, exceeds total pages", 3146f4bdfeafSUladzislau Rezki (Sony) real_size); 3147d70bec8cSNicholas Piggin return NULL; 3148121e6f32SNicholas Piggin } 3149d0a21265SDavid Rientjes 3150559089e0SSong Liu if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) { 3151121e6f32SNicholas Piggin unsigned long size_per_node; 3152121e6f32SNicholas Piggin 3153121e6f32SNicholas Piggin /* 3154121e6f32SNicholas Piggin * Try huge pages. Only try for PAGE_KERNEL allocations, 3155121e6f32SNicholas Piggin * others like modules don't yet expect huge pages in 3156121e6f32SNicholas Piggin * their allocations due to apply_to_page_range not 3157121e6f32SNicholas Piggin * supporting them. 3158121e6f32SNicholas Piggin */ 3159121e6f32SNicholas Piggin 3160121e6f32SNicholas Piggin size_per_node = size; 3161121e6f32SNicholas Piggin if (node == NUMA_NO_NODE) 3162121e6f32SNicholas Piggin size_per_node /= num_online_nodes(); 31633382bbeeSChristophe Leroy if (arch_vmap_pmd_supported(prot) && size_per_node >= PMD_SIZE) 3164121e6f32SNicholas Piggin shift = PMD_SHIFT; 31653382bbeeSChristophe Leroy else 31663382bbeeSChristophe Leroy shift = arch_vmap_pte_supported_shift(size_per_node); 31673382bbeeSChristophe Leroy 3168121e6f32SNicholas Piggin align = max(real_align, 1UL << shift); 3169121e6f32SNicholas Piggin size = ALIGN(real_size, 1UL << shift); 3170121e6f32SNicholas Piggin } 3171121e6f32SNicholas Piggin 3172121e6f32SNicholas Piggin again: 31737ca3027bSDaniel Axtens area = __get_vm_area_node(real_size, align, shift, VM_ALLOC | 31747ca3027bSDaniel Axtens VM_UNINITIALIZED | vm_flags, start, end, node, 31757ca3027bSDaniel Axtens gfp_mask, caller); 3176d70bec8cSNicholas Piggin if (!area) { 31779376130cSMichal Hocko bool nofail = gfp_mask & __GFP_NOFAIL; 3178d70bec8cSNicholas Piggin warn_alloc(gfp_mask, NULL, 31799376130cSMichal Hocko "vmalloc error: size %lu, vm_struct allocation failed%s", 31809376130cSMichal Hocko real_size, (nofail) ? ". Retrying." : ""); 31819376130cSMichal Hocko if (nofail) { 31829376130cSMichal Hocko schedule_timeout_uninterruptible(1); 31839376130cSMichal Hocko goto again; 31849376130cSMichal Hocko } 3185de7d2b56SJoe Perches goto fail; 3186d70bec8cSNicholas Piggin } 3187d0a21265SDavid Rientjes 3188f6e39794SAndrey Konovalov /* 3189f6e39794SAndrey Konovalov * Prepare arguments for __vmalloc_area_node() and 3190f6e39794SAndrey Konovalov * kasan_unpoison_vmalloc(). 3191f6e39794SAndrey Konovalov */ 3192f6e39794SAndrey Konovalov if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL)) { 3193f6e39794SAndrey Konovalov if (kasan_hw_tags_enabled()) { 319401d92c7fSAndrey Konovalov /* 319501d92c7fSAndrey Konovalov * Modify protection bits to allow tagging. 3196f6e39794SAndrey Konovalov * This must be done before mapping. 319701d92c7fSAndrey Konovalov */ 319801d92c7fSAndrey Konovalov prot = arch_vmap_pgprot_tagged(prot); 319901d92c7fSAndrey Konovalov 320023689e91SAndrey Konovalov /* 3201f6e39794SAndrey Konovalov * Skip page_alloc poisoning and zeroing for physical 3202f6e39794SAndrey Konovalov * pages backing VM_ALLOC mapping. Memory is instead 3203f6e39794SAndrey Konovalov * poisoned and zeroed by kasan_unpoison_vmalloc(). 320423689e91SAndrey Konovalov */ 320523689e91SAndrey Konovalov gfp_mask |= __GFP_SKIP_KASAN_UNPOISON | __GFP_SKIP_ZERO; 320623689e91SAndrey Konovalov } 320723689e91SAndrey Konovalov 3208f6e39794SAndrey Konovalov /* Take note that the mapping is PAGE_KERNEL. */ 3209f6e39794SAndrey Konovalov kasan_flags |= KASAN_VMALLOC_PROT_NORMAL; 3210f6e39794SAndrey Konovalov } 3211f6e39794SAndrey Konovalov 321201d92c7fSAndrey Konovalov /* Allocate physical pages and map them into vmalloc space. */ 321319f1c3acSAndrey Konovalov ret = __vmalloc_area_node(area, gfp_mask, prot, shift, node); 321419f1c3acSAndrey Konovalov if (!ret) 3215121e6f32SNicholas Piggin goto fail; 321689219d37SCatalin Marinas 321723689e91SAndrey Konovalov /* 321823689e91SAndrey Konovalov * Mark the pages as accessible, now that they are mapped. 32196c2f761dSAndrey Konovalov * The condition for setting KASAN_VMALLOC_INIT should complement the 32206c2f761dSAndrey Konovalov * one in post_alloc_hook() with regards to the __GFP_SKIP_ZERO check 32216c2f761dSAndrey Konovalov * to make sure that memory is initialized under the same conditions. 3222f6e39794SAndrey Konovalov * Tag-based KASAN modes only assign tags to normal non-executable 3223f6e39794SAndrey Konovalov * allocations, see __kasan_unpoison_vmalloc(). 322423689e91SAndrey Konovalov */ 3225f6e39794SAndrey Konovalov kasan_flags |= KASAN_VMALLOC_VM_ALLOC; 32266c2f761dSAndrey Konovalov if (!want_init_on_free() && want_init_on_alloc(gfp_mask) && 32276c2f761dSAndrey Konovalov (gfp_mask & __GFP_SKIP_ZERO)) 322823689e91SAndrey Konovalov kasan_flags |= KASAN_VMALLOC_INIT; 3229f6e39794SAndrey Konovalov /* KASAN_VMALLOC_PROT_NORMAL already set if required. */ 323023689e91SAndrey Konovalov area->addr = kasan_unpoison_vmalloc(area->addr, real_size, kasan_flags); 323119f1c3acSAndrey Konovalov 323289219d37SCatalin Marinas /* 323320fc02b4SZhang Yanfei * In this function, newly allocated vm_struct has VM_UNINITIALIZED 323420fc02b4SZhang Yanfei * flag. It means that vm_struct is not fully initialized. 32354341fa45SJoonsoo Kim * Now, it is fully initialized, so remove this flag here. 3236f5252e00SMitsuo Hayasaka */ 323720fc02b4SZhang Yanfei clear_vm_uninitialized_flag(area); 3238f5252e00SMitsuo Hayasaka 32397ca3027bSDaniel Axtens size = PAGE_ALIGN(size); 324060115fa5SKefeng Wang if (!(vm_flags & VM_DEFER_KMEMLEAK)) 324194f4a161SCatalin Marinas kmemleak_vmalloc(area, size, gfp_mask); 324289219d37SCatalin Marinas 324319f1c3acSAndrey Konovalov return area->addr; 3244de7d2b56SJoe Perches 3245de7d2b56SJoe Perches fail: 3246121e6f32SNicholas Piggin if (shift > PAGE_SHIFT) { 3247121e6f32SNicholas Piggin shift = PAGE_SHIFT; 3248121e6f32SNicholas Piggin align = real_align; 3249121e6f32SNicholas Piggin size = real_size; 3250121e6f32SNicholas Piggin goto again; 3251121e6f32SNicholas Piggin } 3252121e6f32SNicholas Piggin 3253de7d2b56SJoe Perches return NULL; 3254930fc45aSChristoph Lameter } 3255930fc45aSChristoph Lameter 32561da177e4SLinus Torvalds /** 3257930fc45aSChristoph Lameter * __vmalloc_node - allocate virtually contiguous memory 32581da177e4SLinus Torvalds * @size: allocation size 32592dca6999SDavid Miller * @align: desired alignment 32601da177e4SLinus Torvalds * @gfp_mask: flags for the page level allocator 326100ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 3262c85d194bSRandy Dunlap * @caller: caller's return address 32631da177e4SLinus Torvalds * 3264f38fcb9cSChristoph Hellwig * Allocate enough pages to cover @size from the page level allocator with 3265f38fcb9cSChristoph Hellwig * @gfp_mask flags. Map them into contiguous kernel virtual space. 3266a7c3e901SMichal Hocko * 3267dcda9b04SMichal Hocko * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL 3268a7c3e901SMichal Hocko * and __GFP_NOFAIL are not supported 3269a7c3e901SMichal Hocko * 3270a7c3e901SMichal Hocko * Any use of gfp flags outside of GFP_KERNEL should be consulted 3271a7c3e901SMichal Hocko * with mm people. 3272a862f68aSMike Rapoport * 3273a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 32741da177e4SLinus Torvalds */ 32752b905948SChristoph Hellwig void *__vmalloc_node(unsigned long size, unsigned long align, 3276f38fcb9cSChristoph Hellwig gfp_t gfp_mask, int node, const void *caller) 32771da177e4SLinus Torvalds { 3278d0a21265SDavid Rientjes return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 3279f38fcb9cSChristoph Hellwig gfp_mask, PAGE_KERNEL, 0, node, caller); 32801da177e4SLinus Torvalds } 3281c3f896dcSChristoph Hellwig /* 3282c3f896dcSChristoph Hellwig * This is only for performance analysis of vmalloc and stress purpose. 3283c3f896dcSChristoph Hellwig * It is required by vmalloc test module, therefore do not use it other 3284c3f896dcSChristoph Hellwig * than that. 3285c3f896dcSChristoph Hellwig */ 3286c3f896dcSChristoph Hellwig #ifdef CONFIG_TEST_VMALLOC_MODULE 3287c3f896dcSChristoph Hellwig EXPORT_SYMBOL_GPL(__vmalloc_node); 3288c3f896dcSChristoph Hellwig #endif 32891da177e4SLinus Torvalds 329088dca4caSChristoph Hellwig void *__vmalloc(unsigned long size, gfp_t gfp_mask) 3291930fc45aSChristoph Lameter { 3292f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, gfp_mask, NUMA_NO_NODE, 329323016969SChristoph Lameter __builtin_return_address(0)); 3294930fc45aSChristoph Lameter } 32951da177e4SLinus Torvalds EXPORT_SYMBOL(__vmalloc); 32961da177e4SLinus Torvalds 32971da177e4SLinus Torvalds /** 32981da177e4SLinus Torvalds * vmalloc - allocate virtually contiguous memory 32991da177e4SLinus Torvalds * @size: allocation size 330092eac168SMike Rapoport * 33011da177e4SLinus Torvalds * Allocate enough pages to cover @size from the page level 33021da177e4SLinus Torvalds * allocator and map them into contiguous kernel virtual space. 33031da177e4SLinus Torvalds * 3304c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 33051da177e4SLinus Torvalds * use __vmalloc() instead. 3306a862f68aSMike Rapoport * 3307a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 33081da177e4SLinus Torvalds */ 33091da177e4SLinus Torvalds void *vmalloc(unsigned long size) 33101da177e4SLinus Torvalds { 33114d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL, NUMA_NO_NODE, 33124d39d728SChristoph Hellwig __builtin_return_address(0)); 33131da177e4SLinus Torvalds } 33141da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc); 33151da177e4SLinus Torvalds 3316930fc45aSChristoph Lameter /** 3317559089e0SSong Liu * vmalloc_huge - allocate virtually contiguous memory, allow huge pages 331815a64f5aSClaudio Imbrenda * @size: allocation size 3319559089e0SSong Liu * @gfp_mask: flags for the page level allocator 332015a64f5aSClaudio Imbrenda * 3321559089e0SSong Liu * Allocate enough pages to cover @size from the page level 332215a64f5aSClaudio Imbrenda * allocator and map them into contiguous kernel virtual space. 3323559089e0SSong Liu * If @size is greater than or equal to PMD_SIZE, allow using 3324559089e0SSong Liu * huge pages for the memory 332515a64f5aSClaudio Imbrenda * 332615a64f5aSClaudio Imbrenda * Return: pointer to the allocated memory or %NULL on error 332715a64f5aSClaudio Imbrenda */ 3328559089e0SSong Liu void *vmalloc_huge(unsigned long size, gfp_t gfp_mask) 332915a64f5aSClaudio Imbrenda { 333015a64f5aSClaudio Imbrenda return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, 3331559089e0SSong Liu gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, 333215a64f5aSClaudio Imbrenda NUMA_NO_NODE, __builtin_return_address(0)); 333315a64f5aSClaudio Imbrenda } 3334559089e0SSong Liu EXPORT_SYMBOL_GPL(vmalloc_huge); 333515a64f5aSClaudio Imbrenda 333615a64f5aSClaudio Imbrenda /** 3337e1ca7788SDave Young * vzalloc - allocate virtually contiguous memory with zero fill 3338e1ca7788SDave Young * @size: allocation size 333992eac168SMike Rapoport * 3340e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 3341e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 3342e1ca7788SDave Young * The memory allocated is set to zero. 3343e1ca7788SDave Young * 3344e1ca7788SDave Young * For tight control over page level allocator and protection flags 3345e1ca7788SDave Young * use __vmalloc() instead. 3346a862f68aSMike Rapoport * 3347a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 3348e1ca7788SDave Young */ 3349e1ca7788SDave Young void *vzalloc(unsigned long size) 3350e1ca7788SDave Young { 33514d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, 33524d39d728SChristoph Hellwig __builtin_return_address(0)); 3353e1ca7788SDave Young } 3354e1ca7788SDave Young EXPORT_SYMBOL(vzalloc); 3355e1ca7788SDave Young 3356e1ca7788SDave Young /** 3357ead04089SRolf Eike Beer * vmalloc_user - allocate zeroed virtually contiguous memory for userspace 335883342314SNick Piggin * @size: allocation size 3359ead04089SRolf Eike Beer * 3360ead04089SRolf Eike Beer * The resulting memory area is zeroed so it can be mapped to userspace 3361ead04089SRolf Eike Beer * without leaking data. 3362a862f68aSMike Rapoport * 3363a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 336483342314SNick Piggin */ 336583342314SNick Piggin void *vmalloc_user(unsigned long size) 336683342314SNick Piggin { 3367bc84c535SRoman Penyaev return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, 3368bc84c535SRoman Penyaev GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, 3369bc84c535SRoman Penyaev VM_USERMAP, NUMA_NO_NODE, 337000ef2d2fSDavid Rientjes __builtin_return_address(0)); 337183342314SNick Piggin } 337283342314SNick Piggin EXPORT_SYMBOL(vmalloc_user); 337383342314SNick Piggin 337483342314SNick Piggin /** 3375930fc45aSChristoph Lameter * vmalloc_node - allocate memory on a specific node 3376930fc45aSChristoph Lameter * @size: allocation size 3377d44e0780SRandy Dunlap * @node: numa node 3378930fc45aSChristoph Lameter * 3379930fc45aSChristoph Lameter * Allocate enough pages to cover @size from the page level 3380930fc45aSChristoph Lameter * allocator and map them into contiguous kernel virtual space. 3381930fc45aSChristoph Lameter * 3382c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 3383930fc45aSChristoph Lameter * use __vmalloc() instead. 3384a862f68aSMike Rapoport * 3385a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 3386930fc45aSChristoph Lameter */ 3387930fc45aSChristoph Lameter void *vmalloc_node(unsigned long size, int node) 3388930fc45aSChristoph Lameter { 3389f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL, node, 3390f38fcb9cSChristoph Hellwig __builtin_return_address(0)); 3391930fc45aSChristoph Lameter } 3392930fc45aSChristoph Lameter EXPORT_SYMBOL(vmalloc_node); 3393930fc45aSChristoph Lameter 3394e1ca7788SDave Young /** 3395e1ca7788SDave Young * vzalloc_node - allocate memory on a specific node with zero fill 3396e1ca7788SDave Young * @size: allocation size 3397e1ca7788SDave Young * @node: numa node 3398e1ca7788SDave Young * 3399e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 3400e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 3401e1ca7788SDave Young * The memory allocated is set to zero. 3402e1ca7788SDave Young * 3403a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 3404e1ca7788SDave Young */ 3405e1ca7788SDave Young void *vzalloc_node(unsigned long size, int node) 3406e1ca7788SDave Young { 34074d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, node, 34084d39d728SChristoph Hellwig __builtin_return_address(0)); 3409e1ca7788SDave Young } 3410e1ca7788SDave Young EXPORT_SYMBOL(vzalloc_node); 3411e1ca7788SDave Young 34120d08e0d3SAndi Kleen #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) 3413698d0831SMichal Hocko #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) 34140d08e0d3SAndi Kleen #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) 3415698d0831SMichal Hocko #define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL) 34160d08e0d3SAndi Kleen #else 3417698d0831SMichal Hocko /* 3418698d0831SMichal Hocko * 64b systems should always have either DMA or DMA32 zones. For others 3419698d0831SMichal Hocko * GFP_DMA32 should do the right thing and use the normal zone. 3420698d0831SMichal Hocko */ 342168d68ff6SZhiyuan Dai #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) 34220d08e0d3SAndi Kleen #endif 34230d08e0d3SAndi Kleen 34241da177e4SLinus Torvalds /** 34251da177e4SLinus Torvalds * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 34261da177e4SLinus Torvalds * @size: allocation size 34271da177e4SLinus Torvalds * 34281da177e4SLinus Torvalds * Allocate enough 32bit PA addressable pages to cover @size from the 34291da177e4SLinus Torvalds * page level allocator and map them into contiguous kernel virtual space. 3430a862f68aSMike Rapoport * 3431a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 34321da177e4SLinus Torvalds */ 34331da177e4SLinus Torvalds void *vmalloc_32(unsigned long size) 34341da177e4SLinus Torvalds { 3435f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, GFP_VMALLOC32, NUMA_NO_NODE, 3436f38fcb9cSChristoph Hellwig __builtin_return_address(0)); 34371da177e4SLinus Torvalds } 34381da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc_32); 34391da177e4SLinus Torvalds 344083342314SNick Piggin /** 3441ead04089SRolf Eike Beer * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 344283342314SNick Piggin * @size: allocation size 3443ead04089SRolf Eike Beer * 3444ead04089SRolf Eike Beer * The resulting memory area is 32bit addressable and zeroed so it can be 3445ead04089SRolf Eike Beer * mapped to userspace without leaking data. 3446a862f68aSMike Rapoport * 3447a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 344883342314SNick Piggin */ 344983342314SNick Piggin void *vmalloc_32_user(unsigned long size) 345083342314SNick Piggin { 3451bc84c535SRoman Penyaev return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, 3452bc84c535SRoman Penyaev GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, 3453bc84c535SRoman Penyaev VM_USERMAP, NUMA_NO_NODE, 34545a82ac71SRoman Penyaev __builtin_return_address(0)); 345583342314SNick Piggin } 345683342314SNick Piggin EXPORT_SYMBOL(vmalloc_32_user); 345783342314SNick Piggin 3458d0107eb0SKAMEZAWA Hiroyuki /* 3459d0107eb0SKAMEZAWA Hiroyuki * small helper routine , copy contents to buf from addr. 3460d0107eb0SKAMEZAWA Hiroyuki * If the page is not present, fill zero. 3461d0107eb0SKAMEZAWA Hiroyuki */ 3462d0107eb0SKAMEZAWA Hiroyuki 3463d0107eb0SKAMEZAWA Hiroyuki static int aligned_vread(char *buf, char *addr, unsigned long count) 3464d0107eb0SKAMEZAWA Hiroyuki { 3465d0107eb0SKAMEZAWA Hiroyuki struct page *p; 3466d0107eb0SKAMEZAWA Hiroyuki int copied = 0; 3467d0107eb0SKAMEZAWA Hiroyuki 3468d0107eb0SKAMEZAWA Hiroyuki while (count) { 3469d0107eb0SKAMEZAWA Hiroyuki unsigned long offset, length; 3470d0107eb0SKAMEZAWA Hiroyuki 3471891c49abSAlexander Kuleshov offset = offset_in_page(addr); 3472d0107eb0SKAMEZAWA Hiroyuki length = PAGE_SIZE - offset; 3473d0107eb0SKAMEZAWA Hiroyuki if (length > count) 3474d0107eb0SKAMEZAWA Hiroyuki length = count; 3475d0107eb0SKAMEZAWA Hiroyuki p = vmalloc_to_page(addr); 3476d0107eb0SKAMEZAWA Hiroyuki /* 3477d0107eb0SKAMEZAWA Hiroyuki * To do safe access to this _mapped_ area, we need 3478d0107eb0SKAMEZAWA Hiroyuki * lock. But adding lock here means that we need to add 3479f0953a1bSIngo Molnar * overhead of vmalloc()/vfree() calls for this _debug_ 3480d0107eb0SKAMEZAWA Hiroyuki * interface, rarely used. Instead of that, we'll use 3481d0107eb0SKAMEZAWA Hiroyuki * kmap() and get small overhead in this access function. 3482d0107eb0SKAMEZAWA Hiroyuki */ 3483d0107eb0SKAMEZAWA Hiroyuki if (p) { 3484f7c8ce44SDavid Hildenbrand /* We can expect USER0 is not used -- see vread() */ 34859b04c5feSCong Wang void *map = kmap_atomic(p); 3486d0107eb0SKAMEZAWA Hiroyuki memcpy(buf, map + offset, length); 34879b04c5feSCong Wang kunmap_atomic(map); 3488d0107eb0SKAMEZAWA Hiroyuki } else 3489d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, length); 3490d0107eb0SKAMEZAWA Hiroyuki 3491d0107eb0SKAMEZAWA Hiroyuki addr += length; 3492d0107eb0SKAMEZAWA Hiroyuki buf += length; 3493d0107eb0SKAMEZAWA Hiroyuki copied += length; 3494d0107eb0SKAMEZAWA Hiroyuki count -= length; 3495d0107eb0SKAMEZAWA Hiroyuki } 3496d0107eb0SKAMEZAWA Hiroyuki return copied; 3497d0107eb0SKAMEZAWA Hiroyuki } 3498d0107eb0SKAMEZAWA Hiroyuki 3499d0107eb0SKAMEZAWA Hiroyuki /** 3500d0107eb0SKAMEZAWA Hiroyuki * vread() - read vmalloc area in a safe way. 3501d0107eb0SKAMEZAWA Hiroyuki * @buf: buffer for reading data 3502d0107eb0SKAMEZAWA Hiroyuki * @addr: vm address. 3503d0107eb0SKAMEZAWA Hiroyuki * @count: number of bytes to be read. 3504d0107eb0SKAMEZAWA Hiroyuki * 3505d0107eb0SKAMEZAWA Hiroyuki * This function checks that addr is a valid vmalloc'ed area, and 3506d0107eb0SKAMEZAWA Hiroyuki * copy data from that area to a given buffer. If the given memory range 3507d0107eb0SKAMEZAWA Hiroyuki * of [addr...addr+count) includes some valid address, data is copied to 3508d0107eb0SKAMEZAWA Hiroyuki * proper area of @buf. If there are memory holes, they'll be zero-filled. 3509d0107eb0SKAMEZAWA Hiroyuki * IOREMAP area is treated as memory hole and no copy is done. 3510d0107eb0SKAMEZAWA Hiroyuki * 3511d0107eb0SKAMEZAWA Hiroyuki * If [addr...addr+count) doesn't includes any intersects with alive 3512a8e5202dSCong Wang * vm_struct area, returns 0. @buf should be kernel's buffer. 3513d0107eb0SKAMEZAWA Hiroyuki * 3514d0107eb0SKAMEZAWA Hiroyuki * Note: In usual ops, vread() is never necessary because the caller 3515d0107eb0SKAMEZAWA Hiroyuki * should know vmalloc() area is valid and can use memcpy(). 3516d0107eb0SKAMEZAWA Hiroyuki * This is for routines which have to access vmalloc area without 3517bbcd53c9SDavid Hildenbrand * any information, as /proc/kcore. 3518a862f68aSMike Rapoport * 3519a862f68aSMike Rapoport * Return: number of bytes for which addr and buf should be increased 3520a862f68aSMike Rapoport * (same number as @count) or %0 if [addr...addr+count) doesn't 3521a862f68aSMike Rapoport * include any intersection with valid vmalloc area 3522d0107eb0SKAMEZAWA Hiroyuki */ 35231da177e4SLinus Torvalds long vread(char *buf, char *addr, unsigned long count) 35241da177e4SLinus Torvalds { 3525e81ce85fSJoonsoo Kim struct vmap_area *va; 3526e81ce85fSJoonsoo Kim struct vm_struct *vm; 35271da177e4SLinus Torvalds char *vaddr, *buf_start = buf; 3528d0107eb0SKAMEZAWA Hiroyuki unsigned long buflen = count; 35291da177e4SLinus Torvalds unsigned long n; 35301da177e4SLinus Torvalds 35314aff1dc4SAndrey Konovalov addr = kasan_reset_tag(addr); 35324aff1dc4SAndrey Konovalov 35331da177e4SLinus Torvalds /* Don't allow overflow */ 35341da177e4SLinus Torvalds if ((unsigned long) addr + count < count) 35351da177e4SLinus Torvalds count = -(unsigned long) addr; 35361da177e4SLinus Torvalds 3537e81ce85fSJoonsoo Kim spin_lock(&vmap_area_lock); 3538f181234aSChen Wandun va = find_vmap_area_exceed_addr((unsigned long)addr); 3539f608788cSSerapheim Dimitropoulos if (!va) 3540f608788cSSerapheim Dimitropoulos goto finished; 3541f181234aSChen Wandun 3542f181234aSChen Wandun /* no intersects with alive vmap_area */ 3543f181234aSChen Wandun if ((unsigned long)addr + count <= va->va_start) 3544f181234aSChen Wandun goto finished; 3545f181234aSChen Wandun 3546f608788cSSerapheim Dimitropoulos list_for_each_entry_from(va, &vmap_area_list, list) { 3547e81ce85fSJoonsoo Kim if (!count) 3548e81ce85fSJoonsoo Kim break; 3549e81ce85fSJoonsoo Kim 3550688fcbfcSPengfei Li if (!va->vm) 3551e81ce85fSJoonsoo Kim continue; 3552e81ce85fSJoonsoo Kim 3553e81ce85fSJoonsoo Kim vm = va->vm; 3554e81ce85fSJoonsoo Kim vaddr = (char *) vm->addr; 3555762216abSWanpeng Li if (addr >= vaddr + get_vm_area_size(vm)) 35561da177e4SLinus Torvalds continue; 35571da177e4SLinus Torvalds while (addr < vaddr) { 35581da177e4SLinus Torvalds if (count == 0) 35591da177e4SLinus Torvalds goto finished; 35601da177e4SLinus Torvalds *buf = '\0'; 35611da177e4SLinus Torvalds buf++; 35621da177e4SLinus Torvalds addr++; 35631da177e4SLinus Torvalds count--; 35641da177e4SLinus Torvalds } 3565762216abSWanpeng Li n = vaddr + get_vm_area_size(vm) - addr; 3566d0107eb0SKAMEZAWA Hiroyuki if (n > count) 3567d0107eb0SKAMEZAWA Hiroyuki n = count; 3568e81ce85fSJoonsoo Kim if (!(vm->flags & VM_IOREMAP)) 3569d0107eb0SKAMEZAWA Hiroyuki aligned_vread(buf, addr, n); 3570d0107eb0SKAMEZAWA Hiroyuki else /* IOREMAP area is treated as memory hole */ 3571d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, n); 3572d0107eb0SKAMEZAWA Hiroyuki buf += n; 3573d0107eb0SKAMEZAWA Hiroyuki addr += n; 3574d0107eb0SKAMEZAWA Hiroyuki count -= n; 35751da177e4SLinus Torvalds } 35761da177e4SLinus Torvalds finished: 3577e81ce85fSJoonsoo Kim spin_unlock(&vmap_area_lock); 3578d0107eb0SKAMEZAWA Hiroyuki 3579d0107eb0SKAMEZAWA Hiroyuki if (buf == buf_start) 3580d0107eb0SKAMEZAWA Hiroyuki return 0; 3581d0107eb0SKAMEZAWA Hiroyuki /* zero-fill memory holes */ 3582d0107eb0SKAMEZAWA Hiroyuki if (buf != buf_start + buflen) 3583d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, buflen - (buf - buf_start)); 3584d0107eb0SKAMEZAWA Hiroyuki 3585d0107eb0SKAMEZAWA Hiroyuki return buflen; 35861da177e4SLinus Torvalds } 35871da177e4SLinus Torvalds 3588d0107eb0SKAMEZAWA Hiroyuki /** 3589e69e9d4aSHATAYAMA Daisuke * remap_vmalloc_range_partial - map vmalloc pages to userspace 3590e69e9d4aSHATAYAMA Daisuke * @vma: vma to cover 3591e69e9d4aSHATAYAMA Daisuke * @uaddr: target user address to start at 3592e69e9d4aSHATAYAMA Daisuke * @kaddr: virtual address of vmalloc kernel memory 3593bdebd6a2SJann Horn * @pgoff: offset from @kaddr to start at 3594e69e9d4aSHATAYAMA Daisuke * @size: size of map area 3595e69e9d4aSHATAYAMA Daisuke * 3596e69e9d4aSHATAYAMA Daisuke * Returns: 0 for success, -Exxx on failure 3597e69e9d4aSHATAYAMA Daisuke * 3598e69e9d4aSHATAYAMA Daisuke * This function checks that @kaddr is a valid vmalloc'ed area, 3599e69e9d4aSHATAYAMA Daisuke * and that it is big enough to cover the range starting at 3600e69e9d4aSHATAYAMA Daisuke * @uaddr in @vma. Will return failure if that criteria isn't 3601e69e9d4aSHATAYAMA Daisuke * met. 3602e69e9d4aSHATAYAMA Daisuke * 3603e69e9d4aSHATAYAMA Daisuke * Similar to remap_pfn_range() (see mm/memory.c) 3604e69e9d4aSHATAYAMA Daisuke */ 3605e69e9d4aSHATAYAMA Daisuke int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr, 3606bdebd6a2SJann Horn void *kaddr, unsigned long pgoff, 3607bdebd6a2SJann Horn unsigned long size) 3608e69e9d4aSHATAYAMA Daisuke { 3609e69e9d4aSHATAYAMA Daisuke struct vm_struct *area; 3610bdebd6a2SJann Horn unsigned long off; 3611bdebd6a2SJann Horn unsigned long end_index; 3612bdebd6a2SJann Horn 3613bdebd6a2SJann Horn if (check_shl_overflow(pgoff, PAGE_SHIFT, &off)) 3614bdebd6a2SJann Horn return -EINVAL; 3615e69e9d4aSHATAYAMA Daisuke 3616e69e9d4aSHATAYAMA Daisuke size = PAGE_ALIGN(size); 3617e69e9d4aSHATAYAMA Daisuke 3618e69e9d4aSHATAYAMA Daisuke if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr)) 3619e69e9d4aSHATAYAMA Daisuke return -EINVAL; 3620e69e9d4aSHATAYAMA Daisuke 3621e69e9d4aSHATAYAMA Daisuke area = find_vm_area(kaddr); 3622e69e9d4aSHATAYAMA Daisuke if (!area) 3623e69e9d4aSHATAYAMA Daisuke return -EINVAL; 3624e69e9d4aSHATAYAMA Daisuke 3625fe9041c2SChristoph Hellwig if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT))) 3626e69e9d4aSHATAYAMA Daisuke return -EINVAL; 3627e69e9d4aSHATAYAMA Daisuke 3628bdebd6a2SJann Horn if (check_add_overflow(size, off, &end_index) || 3629bdebd6a2SJann Horn end_index > get_vm_area_size(area)) 3630e69e9d4aSHATAYAMA Daisuke return -EINVAL; 3631bdebd6a2SJann Horn kaddr += off; 3632e69e9d4aSHATAYAMA Daisuke 3633e69e9d4aSHATAYAMA Daisuke do { 3634e69e9d4aSHATAYAMA Daisuke struct page *page = vmalloc_to_page(kaddr); 3635e69e9d4aSHATAYAMA Daisuke int ret; 3636e69e9d4aSHATAYAMA Daisuke 3637e69e9d4aSHATAYAMA Daisuke ret = vm_insert_page(vma, uaddr, page); 3638e69e9d4aSHATAYAMA Daisuke if (ret) 3639e69e9d4aSHATAYAMA Daisuke return ret; 3640e69e9d4aSHATAYAMA Daisuke 3641e69e9d4aSHATAYAMA Daisuke uaddr += PAGE_SIZE; 3642e69e9d4aSHATAYAMA Daisuke kaddr += PAGE_SIZE; 3643e69e9d4aSHATAYAMA Daisuke size -= PAGE_SIZE; 3644e69e9d4aSHATAYAMA Daisuke } while (size > 0); 3645e69e9d4aSHATAYAMA Daisuke 3646e69e9d4aSHATAYAMA Daisuke vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; 3647e69e9d4aSHATAYAMA Daisuke 3648e69e9d4aSHATAYAMA Daisuke return 0; 3649e69e9d4aSHATAYAMA Daisuke } 3650e69e9d4aSHATAYAMA Daisuke 3651e69e9d4aSHATAYAMA Daisuke /** 365283342314SNick Piggin * remap_vmalloc_range - map vmalloc pages to userspace 365383342314SNick Piggin * @vma: vma to cover (map full range of vma) 365483342314SNick Piggin * @addr: vmalloc memory 365583342314SNick Piggin * @pgoff: number of pages into addr before first page to map 36567682486bSRandy Dunlap * 36577682486bSRandy Dunlap * Returns: 0 for success, -Exxx on failure 365883342314SNick Piggin * 365983342314SNick Piggin * This function checks that addr is a valid vmalloc'ed area, and 366083342314SNick Piggin * that it is big enough to cover the vma. Will return failure if 366183342314SNick Piggin * that criteria isn't met. 366283342314SNick Piggin * 366372fd4a35SRobert P. J. Day * Similar to remap_pfn_range() (see mm/memory.c) 366483342314SNick Piggin */ 366583342314SNick Piggin int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 366683342314SNick Piggin unsigned long pgoff) 366783342314SNick Piggin { 3668e69e9d4aSHATAYAMA Daisuke return remap_vmalloc_range_partial(vma, vma->vm_start, 3669bdebd6a2SJann Horn addr, pgoff, 3670e69e9d4aSHATAYAMA Daisuke vma->vm_end - vma->vm_start); 367183342314SNick Piggin } 367283342314SNick Piggin EXPORT_SYMBOL(remap_vmalloc_range); 367383342314SNick Piggin 36745f4352fbSJeremy Fitzhardinge void free_vm_area(struct vm_struct *area) 36755f4352fbSJeremy Fitzhardinge { 36765f4352fbSJeremy Fitzhardinge struct vm_struct *ret; 36775f4352fbSJeremy Fitzhardinge ret = remove_vm_area(area->addr); 36785f4352fbSJeremy Fitzhardinge BUG_ON(ret != area); 36795f4352fbSJeremy Fitzhardinge kfree(area); 36805f4352fbSJeremy Fitzhardinge } 36815f4352fbSJeremy Fitzhardinge EXPORT_SYMBOL_GPL(free_vm_area); 3682a10aa579SChristoph Lameter 36834f8b02b4STejun Heo #ifdef CONFIG_SMP 3684ca23e405STejun Heo static struct vmap_area *node_to_va(struct rb_node *n) 3685ca23e405STejun Heo { 36864583e773SGeliang Tang return rb_entry_safe(n, struct vmap_area, rb_node); 3687ca23e405STejun Heo } 3688ca23e405STejun Heo 3689ca23e405STejun Heo /** 369068ad4a33SUladzislau Rezki (Sony) * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to 369168ad4a33SUladzislau Rezki (Sony) * @addr: target address 3692ca23e405STejun Heo * 369368ad4a33SUladzislau Rezki (Sony) * Returns: vmap_area if it is found. If there is no such area 369468ad4a33SUladzislau Rezki (Sony) * the first highest(reverse order) vmap_area is returned 369568ad4a33SUladzislau Rezki (Sony) * i.e. va->va_start < addr && va->va_end < addr or NULL 369668ad4a33SUladzislau Rezki (Sony) * if there are no any areas before @addr. 3697ca23e405STejun Heo */ 369868ad4a33SUladzislau Rezki (Sony) static struct vmap_area * 369968ad4a33SUladzislau Rezki (Sony) pvm_find_va_enclose_addr(unsigned long addr) 3700ca23e405STejun Heo { 370168ad4a33SUladzislau Rezki (Sony) struct vmap_area *va, *tmp; 370268ad4a33SUladzislau Rezki (Sony) struct rb_node *n; 370368ad4a33SUladzislau Rezki (Sony) 370468ad4a33SUladzislau Rezki (Sony) n = free_vmap_area_root.rb_node; 370568ad4a33SUladzislau Rezki (Sony) va = NULL; 3706ca23e405STejun Heo 3707ca23e405STejun Heo while (n) { 370868ad4a33SUladzislau Rezki (Sony) tmp = rb_entry(n, struct vmap_area, rb_node); 370968ad4a33SUladzislau Rezki (Sony) if (tmp->va_start <= addr) { 371068ad4a33SUladzislau Rezki (Sony) va = tmp; 371168ad4a33SUladzislau Rezki (Sony) if (tmp->va_end >= addr) 3712ca23e405STejun Heo break; 3713ca23e405STejun Heo 371468ad4a33SUladzislau Rezki (Sony) n = n->rb_right; 3715ca23e405STejun Heo } else { 371668ad4a33SUladzislau Rezki (Sony) n = n->rb_left; 3717ca23e405STejun Heo } 371868ad4a33SUladzislau Rezki (Sony) } 371968ad4a33SUladzislau Rezki (Sony) 372068ad4a33SUladzislau Rezki (Sony) return va; 3721ca23e405STejun Heo } 3722ca23e405STejun Heo 3723ca23e405STejun Heo /** 372468ad4a33SUladzislau Rezki (Sony) * pvm_determine_end_from_reverse - find the highest aligned address 372568ad4a33SUladzislau Rezki (Sony) * of free block below VMALLOC_END 372668ad4a33SUladzislau Rezki (Sony) * @va: 372768ad4a33SUladzislau Rezki (Sony) * in - the VA we start the search(reverse order); 372868ad4a33SUladzislau Rezki (Sony) * out - the VA with the highest aligned end address. 3729799fa85dSAlex Shi * @align: alignment for required highest address 3730ca23e405STejun Heo * 373168ad4a33SUladzislau Rezki (Sony) * Returns: determined end address within vmap_area 3732ca23e405STejun Heo */ 373368ad4a33SUladzislau Rezki (Sony) static unsigned long 373468ad4a33SUladzislau Rezki (Sony) pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) 3735ca23e405STejun Heo { 373668ad4a33SUladzislau Rezki (Sony) unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 3737ca23e405STejun Heo unsigned long addr; 3738ca23e405STejun Heo 373968ad4a33SUladzislau Rezki (Sony) if (likely(*va)) { 374068ad4a33SUladzislau Rezki (Sony) list_for_each_entry_from_reverse((*va), 374168ad4a33SUladzislau Rezki (Sony) &free_vmap_area_list, list) { 374268ad4a33SUladzislau Rezki (Sony) addr = min((*va)->va_end & ~(align - 1), vmalloc_end); 374368ad4a33SUladzislau Rezki (Sony) if ((*va)->va_start < addr) 374468ad4a33SUladzislau Rezki (Sony) return addr; 374568ad4a33SUladzislau Rezki (Sony) } 3746ca23e405STejun Heo } 3747ca23e405STejun Heo 374868ad4a33SUladzislau Rezki (Sony) return 0; 3749ca23e405STejun Heo } 3750ca23e405STejun Heo 3751ca23e405STejun Heo /** 3752ca23e405STejun Heo * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator 3753ca23e405STejun Heo * @offsets: array containing offset of each area 3754ca23e405STejun Heo * @sizes: array containing size of each area 3755ca23e405STejun Heo * @nr_vms: the number of areas to allocate 3756ca23e405STejun Heo * @align: alignment, all entries in @offsets and @sizes must be aligned to this 3757ca23e405STejun Heo * 3758ca23e405STejun Heo * Returns: kmalloc'd vm_struct pointer array pointing to allocated 3759ca23e405STejun Heo * vm_structs on success, %NULL on failure 3760ca23e405STejun Heo * 3761ca23e405STejun Heo * Percpu allocator wants to use congruent vm areas so that it can 3762ca23e405STejun Heo * maintain the offsets among percpu areas. This function allocates 3763ec3f64fcSDavid Rientjes * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to 3764ec3f64fcSDavid Rientjes * be scattered pretty far, distance between two areas easily going up 3765ec3f64fcSDavid Rientjes * to gigabytes. To avoid interacting with regular vmallocs, these 3766ec3f64fcSDavid Rientjes * areas are allocated from top. 3767ca23e405STejun Heo * 3768ca23e405STejun Heo * Despite its complicated look, this allocator is rather simple. It 376968ad4a33SUladzislau Rezki (Sony) * does everything top-down and scans free blocks from the end looking 377068ad4a33SUladzislau Rezki (Sony) * for matching base. While scanning, if any of the areas do not fit the 377168ad4a33SUladzislau Rezki (Sony) * base address is pulled down to fit the area. Scanning is repeated till 377268ad4a33SUladzislau Rezki (Sony) * all the areas fit and then all necessary data structures are inserted 377368ad4a33SUladzislau Rezki (Sony) * and the result is returned. 3774ca23e405STejun Heo */ 3775ca23e405STejun Heo struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, 3776ca23e405STejun Heo const size_t *sizes, int nr_vms, 3777ec3f64fcSDavid Rientjes size_t align) 3778ca23e405STejun Heo { 3779ca23e405STejun Heo const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); 3780ca23e405STejun Heo const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 378168ad4a33SUladzislau Rezki (Sony) struct vmap_area **vas, *va; 3782ca23e405STejun Heo struct vm_struct **vms; 3783ca23e405STejun Heo int area, area2, last_area, term_area; 3784253a496dSDaniel Axtens unsigned long base, start, size, end, last_end, orig_start, orig_end; 3785ca23e405STejun Heo bool purged = false; 3786ca23e405STejun Heo 3787ca23e405STejun Heo /* verify parameters and allocate data structures */ 3788891c49abSAlexander Kuleshov BUG_ON(offset_in_page(align) || !is_power_of_2(align)); 3789ca23e405STejun Heo for (last_area = 0, area = 0; area < nr_vms; area++) { 3790ca23e405STejun Heo start = offsets[area]; 3791ca23e405STejun Heo end = start + sizes[area]; 3792ca23e405STejun Heo 3793ca23e405STejun Heo /* is everything aligned properly? */ 3794ca23e405STejun Heo BUG_ON(!IS_ALIGNED(offsets[area], align)); 3795ca23e405STejun Heo BUG_ON(!IS_ALIGNED(sizes[area], align)); 3796ca23e405STejun Heo 3797ca23e405STejun Heo /* detect the area with the highest address */ 3798ca23e405STejun Heo if (start > offsets[last_area]) 3799ca23e405STejun Heo last_area = area; 3800ca23e405STejun Heo 3801c568da28SWei Yang for (area2 = area + 1; area2 < nr_vms; area2++) { 3802ca23e405STejun Heo unsigned long start2 = offsets[area2]; 3803ca23e405STejun Heo unsigned long end2 = start2 + sizes[area2]; 3804ca23e405STejun Heo 3805c568da28SWei Yang BUG_ON(start2 < end && start < end2); 3806ca23e405STejun Heo } 3807ca23e405STejun Heo } 3808ca23e405STejun Heo last_end = offsets[last_area] + sizes[last_area]; 3809ca23e405STejun Heo 3810ca23e405STejun Heo if (vmalloc_end - vmalloc_start < last_end) { 3811ca23e405STejun Heo WARN_ON(true); 3812ca23e405STejun Heo return NULL; 3813ca23e405STejun Heo } 3814ca23e405STejun Heo 38154d67d860SThomas Meyer vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL); 38164d67d860SThomas Meyer vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL); 3817ca23e405STejun Heo if (!vas || !vms) 3818f1db7afdSKautuk Consul goto err_free2; 3819ca23e405STejun Heo 3820ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 382168ad4a33SUladzislau Rezki (Sony) vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL); 3822ec3f64fcSDavid Rientjes vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL); 3823ca23e405STejun Heo if (!vas[area] || !vms[area]) 3824ca23e405STejun Heo goto err_free; 3825ca23e405STejun Heo } 3826ca23e405STejun Heo retry: 3827e36176beSUladzislau Rezki (Sony) spin_lock(&free_vmap_area_lock); 3828ca23e405STejun Heo 3829ca23e405STejun Heo /* start scanning - we scan from the top, begin with the last area */ 3830ca23e405STejun Heo area = term_area = last_area; 3831ca23e405STejun Heo start = offsets[area]; 3832ca23e405STejun Heo end = start + sizes[area]; 3833ca23e405STejun Heo 383468ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(vmalloc_end); 383568ad4a33SUladzislau Rezki (Sony) base = pvm_determine_end_from_reverse(&va, align) - end; 3836ca23e405STejun Heo 3837ca23e405STejun Heo while (true) { 3838ca23e405STejun Heo /* 3839ca23e405STejun Heo * base might have underflowed, add last_end before 3840ca23e405STejun Heo * comparing. 3841ca23e405STejun Heo */ 384268ad4a33SUladzislau Rezki (Sony) if (base + last_end < vmalloc_start + last_end) 384368ad4a33SUladzislau Rezki (Sony) goto overflow; 3844ca23e405STejun Heo 3845ca23e405STejun Heo /* 384668ad4a33SUladzislau Rezki (Sony) * Fitting base has not been found. 3847ca23e405STejun Heo */ 384868ad4a33SUladzislau Rezki (Sony) if (va == NULL) 384968ad4a33SUladzislau Rezki (Sony) goto overflow; 3850ca23e405STejun Heo 3851ca23e405STejun Heo /* 3852d8cc323dSQiujun Huang * If required width exceeds current VA block, move 38535336e52cSKuppuswamy Sathyanarayanan * base downwards and then recheck. 38545336e52cSKuppuswamy Sathyanarayanan */ 38555336e52cSKuppuswamy Sathyanarayanan if (base + end > va->va_end) { 38565336e52cSKuppuswamy Sathyanarayanan base = pvm_determine_end_from_reverse(&va, align) - end; 38575336e52cSKuppuswamy Sathyanarayanan term_area = area; 38585336e52cSKuppuswamy Sathyanarayanan continue; 38595336e52cSKuppuswamy Sathyanarayanan } 38605336e52cSKuppuswamy Sathyanarayanan 38615336e52cSKuppuswamy Sathyanarayanan /* 386268ad4a33SUladzislau Rezki (Sony) * If this VA does not fit, move base downwards and recheck. 3863ca23e405STejun Heo */ 38645336e52cSKuppuswamy Sathyanarayanan if (base + start < va->va_start) { 386568ad4a33SUladzislau Rezki (Sony) va = node_to_va(rb_prev(&va->rb_node)); 386668ad4a33SUladzislau Rezki (Sony) base = pvm_determine_end_from_reverse(&va, align) - end; 3867ca23e405STejun Heo term_area = area; 3868ca23e405STejun Heo continue; 3869ca23e405STejun Heo } 3870ca23e405STejun Heo 3871ca23e405STejun Heo /* 3872ca23e405STejun Heo * This area fits, move on to the previous one. If 3873ca23e405STejun Heo * the previous one is the terminal one, we're done. 3874ca23e405STejun Heo */ 3875ca23e405STejun Heo area = (area + nr_vms - 1) % nr_vms; 3876ca23e405STejun Heo if (area == term_area) 3877ca23e405STejun Heo break; 387868ad4a33SUladzislau Rezki (Sony) 3879ca23e405STejun Heo start = offsets[area]; 3880ca23e405STejun Heo end = start + sizes[area]; 388168ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(base + end); 3882ca23e405STejun Heo } 388368ad4a33SUladzislau Rezki (Sony) 3884ca23e405STejun Heo /* we've found a fitting base, insert all va's */ 3885ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 388668ad4a33SUladzislau Rezki (Sony) int ret; 3887ca23e405STejun Heo 388868ad4a33SUladzislau Rezki (Sony) start = base + offsets[area]; 388968ad4a33SUladzislau Rezki (Sony) size = sizes[area]; 389068ad4a33SUladzislau Rezki (Sony) 389168ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(start); 389268ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(va == NULL)) 389368ad4a33SUladzislau Rezki (Sony) /* It is a BUG(), but trigger recovery instead. */ 389468ad4a33SUladzislau Rezki (Sony) goto recovery; 389568ad4a33SUladzislau Rezki (Sony) 3896f9863be4SUladzislau Rezki (Sony) ret = adjust_va_to_fit_type(&free_vmap_area_root, 3897f9863be4SUladzislau Rezki (Sony) &free_vmap_area_list, 3898f9863be4SUladzislau Rezki (Sony) va, start, size); 38991b23ff80SBaoquan He if (WARN_ON_ONCE(unlikely(ret))) 390068ad4a33SUladzislau Rezki (Sony) /* It is a BUG(), but trigger recovery instead. */ 390168ad4a33SUladzislau Rezki (Sony) goto recovery; 390268ad4a33SUladzislau Rezki (Sony) 390368ad4a33SUladzislau Rezki (Sony) /* Allocated area. */ 390468ad4a33SUladzislau Rezki (Sony) va = vas[area]; 390568ad4a33SUladzislau Rezki (Sony) va->va_start = start; 390668ad4a33SUladzislau Rezki (Sony) va->va_end = start + size; 3907ca23e405STejun Heo } 3908ca23e405STejun Heo 3909e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 3910ca23e405STejun Heo 3911253a496dSDaniel Axtens /* populate the kasan shadow space */ 3912253a496dSDaniel Axtens for (area = 0; area < nr_vms; area++) { 3913253a496dSDaniel Axtens if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area])) 3914253a496dSDaniel Axtens goto err_free_shadow; 3915253a496dSDaniel Axtens } 3916253a496dSDaniel Axtens 3917ca23e405STejun Heo /* insert all vm's */ 3918e36176beSUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 3919e36176beSUladzislau Rezki (Sony) for (area = 0; area < nr_vms; area++) { 3920e36176beSUladzislau Rezki (Sony) insert_vmap_area(vas[area], &vmap_area_root, &vmap_area_list); 3921e36176beSUladzislau Rezki (Sony) 3922e36176beSUladzislau Rezki (Sony) setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC, 3923ca23e405STejun Heo pcpu_get_vm_areas); 3924e36176beSUladzislau Rezki (Sony) } 3925e36176beSUladzislau Rezki (Sony) spin_unlock(&vmap_area_lock); 3926ca23e405STejun Heo 392719f1c3acSAndrey Konovalov /* 392819f1c3acSAndrey Konovalov * Mark allocated areas as accessible. Do it now as a best-effort 392919f1c3acSAndrey Konovalov * approach, as they can be mapped outside of vmalloc code. 393023689e91SAndrey Konovalov * With hardware tag-based KASAN, marking is skipped for 393123689e91SAndrey Konovalov * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 393219f1c3acSAndrey Konovalov */ 39331d96320fSAndrey Konovalov for (area = 0; area < nr_vms; area++) 39341d96320fSAndrey Konovalov vms[area]->addr = kasan_unpoison_vmalloc(vms[area]->addr, 3935f6e39794SAndrey Konovalov vms[area]->size, KASAN_VMALLOC_PROT_NORMAL); 39361d96320fSAndrey Konovalov 3937ca23e405STejun Heo kfree(vas); 3938ca23e405STejun Heo return vms; 3939ca23e405STejun Heo 394068ad4a33SUladzislau Rezki (Sony) recovery: 3941e36176beSUladzislau Rezki (Sony) /* 3942e36176beSUladzislau Rezki (Sony) * Remove previously allocated areas. There is no 3943e36176beSUladzislau Rezki (Sony) * need in removing these areas from the busy tree, 3944e36176beSUladzislau Rezki (Sony) * because they are inserted only on the final step 3945e36176beSUladzislau Rezki (Sony) * and when pcpu_get_vm_areas() is success. 3946e36176beSUladzislau Rezki (Sony) */ 394768ad4a33SUladzislau Rezki (Sony) while (area--) { 3948253a496dSDaniel Axtens orig_start = vas[area]->va_start; 3949253a496dSDaniel Axtens orig_end = vas[area]->va_end; 395096e2db45SUladzislau Rezki (Sony) va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, 39513c5c3cfbSDaniel Axtens &free_vmap_area_list); 39529c801f61SUladzislau Rezki (Sony) if (va) 3953253a496dSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 3954253a496dSDaniel Axtens va->va_start, va->va_end); 395568ad4a33SUladzislau Rezki (Sony) vas[area] = NULL; 395668ad4a33SUladzislau Rezki (Sony) } 395768ad4a33SUladzislau Rezki (Sony) 395868ad4a33SUladzislau Rezki (Sony) overflow: 3959e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 396068ad4a33SUladzislau Rezki (Sony) if (!purged) { 396168ad4a33SUladzislau Rezki (Sony) purge_vmap_area_lazy(); 396268ad4a33SUladzislau Rezki (Sony) purged = true; 396368ad4a33SUladzislau Rezki (Sony) 396468ad4a33SUladzislau Rezki (Sony) /* Before "retry", check if we recover. */ 396568ad4a33SUladzislau Rezki (Sony) for (area = 0; area < nr_vms; area++) { 396668ad4a33SUladzislau Rezki (Sony) if (vas[area]) 396768ad4a33SUladzislau Rezki (Sony) continue; 396868ad4a33SUladzislau Rezki (Sony) 396968ad4a33SUladzislau Rezki (Sony) vas[area] = kmem_cache_zalloc( 397068ad4a33SUladzislau Rezki (Sony) vmap_area_cachep, GFP_KERNEL); 397168ad4a33SUladzislau Rezki (Sony) if (!vas[area]) 397268ad4a33SUladzislau Rezki (Sony) goto err_free; 397368ad4a33SUladzislau Rezki (Sony) } 397468ad4a33SUladzislau Rezki (Sony) 397568ad4a33SUladzislau Rezki (Sony) goto retry; 397668ad4a33SUladzislau Rezki (Sony) } 397768ad4a33SUladzislau Rezki (Sony) 3978ca23e405STejun Heo err_free: 3979ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 398068ad4a33SUladzislau Rezki (Sony) if (vas[area]) 398168ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, vas[area]); 398268ad4a33SUladzislau Rezki (Sony) 3983ca23e405STejun Heo kfree(vms[area]); 3984ca23e405STejun Heo } 3985f1db7afdSKautuk Consul err_free2: 3986ca23e405STejun Heo kfree(vas); 3987ca23e405STejun Heo kfree(vms); 3988ca23e405STejun Heo return NULL; 3989253a496dSDaniel Axtens 3990253a496dSDaniel Axtens err_free_shadow: 3991253a496dSDaniel Axtens spin_lock(&free_vmap_area_lock); 3992253a496dSDaniel Axtens /* 3993253a496dSDaniel Axtens * We release all the vmalloc shadows, even the ones for regions that 3994253a496dSDaniel Axtens * hadn't been successfully added. This relies on kasan_release_vmalloc 3995253a496dSDaniel Axtens * being able to tolerate this case. 3996253a496dSDaniel Axtens */ 3997253a496dSDaniel Axtens for (area = 0; area < nr_vms; area++) { 3998253a496dSDaniel Axtens orig_start = vas[area]->va_start; 3999253a496dSDaniel Axtens orig_end = vas[area]->va_end; 400096e2db45SUladzislau Rezki (Sony) va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, 4001253a496dSDaniel Axtens &free_vmap_area_list); 40029c801f61SUladzislau Rezki (Sony) if (va) 4003253a496dSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 4004253a496dSDaniel Axtens va->va_start, va->va_end); 4005253a496dSDaniel Axtens vas[area] = NULL; 4006253a496dSDaniel Axtens kfree(vms[area]); 4007253a496dSDaniel Axtens } 4008253a496dSDaniel Axtens spin_unlock(&free_vmap_area_lock); 4009253a496dSDaniel Axtens kfree(vas); 4010253a496dSDaniel Axtens kfree(vms); 4011253a496dSDaniel Axtens return NULL; 4012ca23e405STejun Heo } 4013ca23e405STejun Heo 4014ca23e405STejun Heo /** 4015ca23e405STejun Heo * pcpu_free_vm_areas - free vmalloc areas for percpu allocator 4016ca23e405STejun Heo * @vms: vm_struct pointer array returned by pcpu_get_vm_areas() 4017ca23e405STejun Heo * @nr_vms: the number of allocated areas 4018ca23e405STejun Heo * 4019ca23e405STejun Heo * Free vm_structs and the array allocated by pcpu_get_vm_areas(). 4020ca23e405STejun Heo */ 4021ca23e405STejun Heo void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) 4022ca23e405STejun Heo { 4023ca23e405STejun Heo int i; 4024ca23e405STejun Heo 4025ca23e405STejun Heo for (i = 0; i < nr_vms; i++) 4026ca23e405STejun Heo free_vm_area(vms[i]); 4027ca23e405STejun Heo kfree(vms); 4028ca23e405STejun Heo } 40294f8b02b4STejun Heo #endif /* CONFIG_SMP */ 4030a10aa579SChristoph Lameter 40315bb1bb35SPaul E. McKenney #ifdef CONFIG_PRINTK 403298f18083SPaul E. McKenney bool vmalloc_dump_obj(void *object) 403398f18083SPaul E. McKenney { 403498f18083SPaul E. McKenney struct vm_struct *vm; 403598f18083SPaul E. McKenney void *objp = (void *)PAGE_ALIGN((unsigned long)object); 403698f18083SPaul E. McKenney 403798f18083SPaul E. McKenney vm = find_vm_area(objp); 403898f18083SPaul E. McKenney if (!vm) 403998f18083SPaul E. McKenney return false; 4040bd34dcd4SPaul E. McKenney pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n", 4041bd34dcd4SPaul E. McKenney vm->nr_pages, (unsigned long)vm->addr, vm->caller); 404298f18083SPaul E. McKenney return true; 404398f18083SPaul E. McKenney } 40445bb1bb35SPaul E. McKenney #endif 404598f18083SPaul E. McKenney 4046a10aa579SChristoph Lameter #ifdef CONFIG_PROC_FS 4047a10aa579SChristoph Lameter static void *s_start(struct seq_file *m, loff_t *pos) 4048e36176beSUladzislau Rezki (Sony) __acquires(&vmap_purge_lock) 4049d4033afdSJoonsoo Kim __acquires(&vmap_area_lock) 4050a10aa579SChristoph Lameter { 4051e36176beSUladzislau Rezki (Sony) mutex_lock(&vmap_purge_lock); 4052d4033afdSJoonsoo Kim spin_lock(&vmap_area_lock); 4053e36176beSUladzislau Rezki (Sony) 40543f500069Szijun_hu return seq_list_start(&vmap_area_list, *pos); 4055a10aa579SChristoph Lameter } 4056a10aa579SChristoph Lameter 4057a10aa579SChristoph Lameter static void *s_next(struct seq_file *m, void *p, loff_t *pos) 4058a10aa579SChristoph Lameter { 40593f500069Szijun_hu return seq_list_next(p, &vmap_area_list, pos); 4060a10aa579SChristoph Lameter } 4061a10aa579SChristoph Lameter 4062a10aa579SChristoph Lameter static void s_stop(struct seq_file *m, void *p) 4063d4033afdSJoonsoo Kim __releases(&vmap_area_lock) 40640a7dd4e9SWaiman Long __releases(&vmap_purge_lock) 4065a10aa579SChristoph Lameter { 4066d4033afdSJoonsoo Kim spin_unlock(&vmap_area_lock); 40670a7dd4e9SWaiman Long mutex_unlock(&vmap_purge_lock); 4068a10aa579SChristoph Lameter } 4069a10aa579SChristoph Lameter 4070a47a126aSEric Dumazet static void show_numa_info(struct seq_file *m, struct vm_struct *v) 4071a47a126aSEric Dumazet { 4072e5adfffcSKirill A. Shutemov if (IS_ENABLED(CONFIG_NUMA)) { 4073a47a126aSEric Dumazet unsigned int nr, *counters = m->private; 407451e50b3aSEric Dumazet unsigned int step = 1U << vm_area_page_order(v); 4075a47a126aSEric Dumazet 4076a47a126aSEric Dumazet if (!counters) 4077a47a126aSEric Dumazet return; 4078a47a126aSEric Dumazet 4079af12346cSWanpeng Li if (v->flags & VM_UNINITIALIZED) 4080af12346cSWanpeng Li return; 40817e5b528bSDmitry Vyukov /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ 40827e5b528bSDmitry Vyukov smp_rmb(); 4083af12346cSWanpeng Li 4084a47a126aSEric Dumazet memset(counters, 0, nr_node_ids * sizeof(unsigned int)); 4085a47a126aSEric Dumazet 408651e50b3aSEric Dumazet for (nr = 0; nr < v->nr_pages; nr += step) 408751e50b3aSEric Dumazet counters[page_to_nid(v->pages[nr])] += step; 4088a47a126aSEric Dumazet for_each_node_state(nr, N_HIGH_MEMORY) 4089a47a126aSEric Dumazet if (counters[nr]) 4090a47a126aSEric Dumazet seq_printf(m, " N%u=%u", nr, counters[nr]); 4091a47a126aSEric Dumazet } 4092a47a126aSEric Dumazet } 4093a47a126aSEric Dumazet 4094dd3b8353SUladzislau Rezki (Sony) static void show_purge_info(struct seq_file *m) 4095dd3b8353SUladzislau Rezki (Sony) { 4096dd3b8353SUladzislau Rezki (Sony) struct vmap_area *va; 4097dd3b8353SUladzislau Rezki (Sony) 409896e2db45SUladzislau Rezki (Sony) spin_lock(&purge_vmap_area_lock); 409996e2db45SUladzislau Rezki (Sony) list_for_each_entry(va, &purge_vmap_area_list, list) { 4100dd3b8353SUladzislau Rezki (Sony) seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n", 4101dd3b8353SUladzislau Rezki (Sony) (void *)va->va_start, (void *)va->va_end, 4102dd3b8353SUladzislau Rezki (Sony) va->va_end - va->va_start); 4103dd3b8353SUladzislau Rezki (Sony) } 410496e2db45SUladzislau Rezki (Sony) spin_unlock(&purge_vmap_area_lock); 4105dd3b8353SUladzislau Rezki (Sony) } 4106dd3b8353SUladzislau Rezki (Sony) 4107a10aa579SChristoph Lameter static int s_show(struct seq_file *m, void *p) 4108a10aa579SChristoph Lameter { 41093f500069Szijun_hu struct vmap_area *va; 4110d4033afdSJoonsoo Kim struct vm_struct *v; 4111d4033afdSJoonsoo Kim 41123f500069Szijun_hu va = list_entry(p, struct vmap_area, list); 41133f500069Szijun_hu 4114c2ce8c14SWanpeng Li /* 4115688fcbfcSPengfei Li * s_show can encounter race with remove_vm_area, !vm on behalf 4116688fcbfcSPengfei Li * of vmap area is being tear down or vm_map_ram allocation. 4117c2ce8c14SWanpeng Li */ 4118688fcbfcSPengfei Li if (!va->vm) { 4119dd3b8353SUladzislau Rezki (Sony) seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n", 412078c72746SYisheng Xie (void *)va->va_start, (void *)va->va_end, 4121dd3b8353SUladzislau Rezki (Sony) va->va_end - va->va_start); 412278c72746SYisheng Xie 41237cc7913eSEric Dumazet goto final; 412478c72746SYisheng Xie } 4125d4033afdSJoonsoo Kim 4126d4033afdSJoonsoo Kim v = va->vm; 4127a10aa579SChristoph Lameter 412845ec1690SKees Cook seq_printf(m, "0x%pK-0x%pK %7ld", 4129a10aa579SChristoph Lameter v->addr, v->addr + v->size, v->size); 4130a10aa579SChristoph Lameter 413162c70bceSJoe Perches if (v->caller) 413262c70bceSJoe Perches seq_printf(m, " %pS", v->caller); 413323016969SChristoph Lameter 4134a10aa579SChristoph Lameter if (v->nr_pages) 4135a10aa579SChristoph Lameter seq_printf(m, " pages=%d", v->nr_pages); 4136a10aa579SChristoph Lameter 4137a10aa579SChristoph Lameter if (v->phys_addr) 4138199eaa05SMiles Chen seq_printf(m, " phys=%pa", &v->phys_addr); 4139a10aa579SChristoph Lameter 4140a10aa579SChristoph Lameter if (v->flags & VM_IOREMAP) 4141f4527c90SFabian Frederick seq_puts(m, " ioremap"); 4142a10aa579SChristoph Lameter 4143a10aa579SChristoph Lameter if (v->flags & VM_ALLOC) 4144f4527c90SFabian Frederick seq_puts(m, " vmalloc"); 4145a10aa579SChristoph Lameter 4146a10aa579SChristoph Lameter if (v->flags & VM_MAP) 4147f4527c90SFabian Frederick seq_puts(m, " vmap"); 4148a10aa579SChristoph Lameter 4149a10aa579SChristoph Lameter if (v->flags & VM_USERMAP) 4150f4527c90SFabian Frederick seq_puts(m, " user"); 4151a10aa579SChristoph Lameter 4152fe9041c2SChristoph Hellwig if (v->flags & VM_DMA_COHERENT) 4153fe9041c2SChristoph Hellwig seq_puts(m, " dma-coherent"); 4154fe9041c2SChristoph Hellwig 4155244d63eeSDavid Rientjes if (is_vmalloc_addr(v->pages)) 4156f4527c90SFabian Frederick seq_puts(m, " vpages"); 4157a10aa579SChristoph Lameter 4158a47a126aSEric Dumazet show_numa_info(m, v); 4159a10aa579SChristoph Lameter seq_putc(m, '\n'); 4160dd3b8353SUladzislau Rezki (Sony) 4161dd3b8353SUladzislau Rezki (Sony) /* 416296e2db45SUladzislau Rezki (Sony) * As a final step, dump "unpurged" areas. 4163dd3b8353SUladzislau Rezki (Sony) */ 41647cc7913eSEric Dumazet final: 4165dd3b8353SUladzislau Rezki (Sony) if (list_is_last(&va->list, &vmap_area_list)) 4166dd3b8353SUladzislau Rezki (Sony) show_purge_info(m); 4167dd3b8353SUladzislau Rezki (Sony) 4168a10aa579SChristoph Lameter return 0; 4169a10aa579SChristoph Lameter } 4170a10aa579SChristoph Lameter 41715f6a6a9cSAlexey Dobriyan static const struct seq_operations vmalloc_op = { 4172a10aa579SChristoph Lameter .start = s_start, 4173a10aa579SChristoph Lameter .next = s_next, 4174a10aa579SChristoph Lameter .stop = s_stop, 4175a10aa579SChristoph Lameter .show = s_show, 4176a10aa579SChristoph Lameter }; 41775f6a6a9cSAlexey Dobriyan 41785f6a6a9cSAlexey Dobriyan static int __init proc_vmalloc_init(void) 41795f6a6a9cSAlexey Dobriyan { 4180fddda2b7SChristoph Hellwig if (IS_ENABLED(CONFIG_NUMA)) 41810825a6f9SJoe Perches proc_create_seq_private("vmallocinfo", 0400, NULL, 418244414d82SChristoph Hellwig &vmalloc_op, 418344414d82SChristoph Hellwig nr_node_ids * sizeof(unsigned int), NULL); 4184fddda2b7SChristoph Hellwig else 41850825a6f9SJoe Perches proc_create_seq("vmallocinfo", 0400, NULL, &vmalloc_op); 41865f6a6a9cSAlexey Dobriyan return 0; 41875f6a6a9cSAlexey Dobriyan } 41885f6a6a9cSAlexey Dobriyan module_init(proc_vmalloc_init); 4189db3808c1SJoonsoo Kim 4190a10aa579SChristoph Lameter #endif 4191