1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * Copyright (C) 1993 Linus Torvalds 41da177e4SLinus Torvalds * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 51da177e4SLinus Torvalds * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 61da177e4SLinus Torvalds * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 7930fc45aSChristoph Lameter * Numa awareness, Christoph Lameter, SGI, June 2005 8d758ffe6SUladzislau Rezki (Sony) * Improving global KVA allocator, Uladzislau Rezki, Sony, May 2019 91da177e4SLinus Torvalds */ 101da177e4SLinus Torvalds 11db64fe02SNick Piggin #include <linux/vmalloc.h> 121da177e4SLinus Torvalds #include <linux/mm.h> 131da177e4SLinus Torvalds #include <linux/module.h> 141da177e4SLinus Torvalds #include <linux/highmem.h> 15c3edc401SIngo Molnar #include <linux/sched/signal.h> 161da177e4SLinus Torvalds #include <linux/slab.h> 171da177e4SLinus Torvalds #include <linux/spinlock.h> 181da177e4SLinus Torvalds #include <linux/interrupt.h> 195f6a6a9cSAlexey Dobriyan #include <linux/proc_fs.h> 20a10aa579SChristoph Lameter #include <linux/seq_file.h> 21868b104dSRick Edgecombe #include <linux/set_memory.h> 223ac7fe5aSThomas Gleixner #include <linux/debugobjects.h> 2323016969SChristoph Lameter #include <linux/kallsyms.h> 24db64fe02SNick Piggin #include <linux/list.h> 254da56b99SChris Wilson #include <linux/notifier.h> 26db64fe02SNick Piggin #include <linux/rbtree.h> 270f14599cSMatthew Wilcox (Oracle) #include <linux/xarray.h> 285da96bddSMel Gorman #include <linux/io.h> 29db64fe02SNick Piggin #include <linux/rcupdate.h> 30f0aa6617STejun Heo #include <linux/pfn.h> 3189219d37SCatalin Marinas #include <linux/kmemleak.h> 3260063497SArun Sharma #include <linux/atomic.h> 333b32123dSGideon Israel Dsouza #include <linux/compiler.h> 344e5aa1f4SShakeel Butt #include <linux/memcontrol.h> 3532fcfd40SAl Viro #include <linux/llist.h> 364c91c07cSLorenzo Stoakes #include <linux/uio.h> 370f616be1SToshi Kani #include <linux/bitops.h> 3868ad4a33SUladzislau Rezki (Sony) #include <linux/rbtree_augmented.h> 39bdebd6a2SJann Horn #include <linux/overflow.h> 40c0eb315aSNicholas Piggin #include <linux/pgtable.h> 41f7ee1f13SChristophe Leroy #include <linux/hugetlb.h> 42451769ebSMichal Hocko #include <linux/sched/mm.h> 431da177e4SLinus Torvalds #include <asm/tlbflush.h> 442dca6999SDavid Miller #include <asm/shmparam.h> 451da177e4SLinus Torvalds 46cf243da6SUladzislau Rezki (Sony) #define CREATE_TRACE_POINTS 47cf243da6SUladzislau Rezki (Sony) #include <trace/events/vmalloc.h> 48cf243da6SUladzislau Rezki (Sony) 49dd56b046SMel Gorman #include "internal.h" 502a681cfaSJoerg Roedel #include "pgalloc-track.h" 51dd56b046SMel Gorman 5282a70ce0SChristoph Hellwig #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 5382a70ce0SChristoph Hellwig static unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1; 5482a70ce0SChristoph Hellwig 5582a70ce0SChristoph Hellwig static int __init set_nohugeiomap(char *str) 5682a70ce0SChristoph Hellwig { 5782a70ce0SChristoph Hellwig ioremap_max_page_shift = PAGE_SHIFT; 5882a70ce0SChristoph Hellwig return 0; 5982a70ce0SChristoph Hellwig } 6082a70ce0SChristoph Hellwig early_param("nohugeiomap", set_nohugeiomap); 6182a70ce0SChristoph Hellwig #else /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 6282a70ce0SChristoph Hellwig static const unsigned int ioremap_max_page_shift = PAGE_SHIFT; 6382a70ce0SChristoph Hellwig #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 6482a70ce0SChristoph Hellwig 65121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 66121e6f32SNicholas Piggin static bool __ro_after_init vmap_allow_huge = true; 67121e6f32SNicholas Piggin 68121e6f32SNicholas Piggin static int __init set_nohugevmalloc(char *str) 69121e6f32SNicholas Piggin { 70121e6f32SNicholas Piggin vmap_allow_huge = false; 71121e6f32SNicholas Piggin return 0; 72121e6f32SNicholas Piggin } 73121e6f32SNicholas Piggin early_param("nohugevmalloc", set_nohugevmalloc); 74121e6f32SNicholas Piggin #else /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ 75121e6f32SNicholas Piggin static const bool vmap_allow_huge = false; 76121e6f32SNicholas Piggin #endif /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ 77121e6f32SNicholas Piggin 78186525bdSIngo Molnar bool is_vmalloc_addr(const void *x) 79186525bdSIngo Molnar { 804aff1dc4SAndrey Konovalov unsigned long addr = (unsigned long)kasan_reset_tag(x); 81186525bdSIngo Molnar 82186525bdSIngo Molnar return addr >= VMALLOC_START && addr < VMALLOC_END; 83186525bdSIngo Molnar } 84186525bdSIngo Molnar EXPORT_SYMBOL(is_vmalloc_addr); 85186525bdSIngo Molnar 8632fcfd40SAl Viro struct vfree_deferred { 8732fcfd40SAl Viro struct llist_head list; 8832fcfd40SAl Viro struct work_struct wq; 8932fcfd40SAl Viro }; 9032fcfd40SAl Viro static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); 9132fcfd40SAl Viro 92db64fe02SNick Piggin /*** Page table manipulation functions ***/ 935e9e3d77SNicholas Piggin static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 945e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 95f7ee1f13SChristophe Leroy unsigned int max_page_shift, pgtbl_mod_mask *mask) 965e9e3d77SNicholas Piggin { 975e9e3d77SNicholas Piggin pte_t *pte; 985e9e3d77SNicholas Piggin u64 pfn; 99f7ee1f13SChristophe Leroy unsigned long size = PAGE_SIZE; 1005e9e3d77SNicholas Piggin 1015e9e3d77SNicholas Piggin pfn = phys_addr >> PAGE_SHIFT; 1025e9e3d77SNicholas Piggin pte = pte_alloc_kernel_track(pmd, addr, mask); 1035e9e3d77SNicholas Piggin if (!pte) 1045e9e3d77SNicholas Piggin return -ENOMEM; 1055e9e3d77SNicholas Piggin do { 106c33c7948SRyan Roberts BUG_ON(!pte_none(ptep_get(pte))); 107f7ee1f13SChristophe Leroy 108f7ee1f13SChristophe Leroy #ifdef CONFIG_HUGETLB_PAGE 109f7ee1f13SChristophe Leroy size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift); 110f7ee1f13SChristophe Leroy if (size != PAGE_SIZE) { 111f7ee1f13SChristophe Leroy pte_t entry = pfn_pte(pfn, prot); 112f7ee1f13SChristophe Leroy 113f7ee1f13SChristophe Leroy entry = arch_make_huge_pte(entry, ilog2(size), 0); 114935d4f0cSRyan Roberts set_huge_pte_at(&init_mm, addr, pte, entry, size); 115f7ee1f13SChristophe Leroy pfn += PFN_DOWN(size); 116f7ee1f13SChristophe Leroy continue; 117f7ee1f13SChristophe Leroy } 118f7ee1f13SChristophe Leroy #endif 1195e9e3d77SNicholas Piggin set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); 1205e9e3d77SNicholas Piggin pfn++; 121f7ee1f13SChristophe Leroy } while (pte += PFN_DOWN(size), addr += size, addr != end); 1225e9e3d77SNicholas Piggin *mask |= PGTBL_PTE_MODIFIED; 1235e9e3d77SNicholas Piggin return 0; 1245e9e3d77SNicholas Piggin } 1255e9e3d77SNicholas Piggin 1265e9e3d77SNicholas Piggin static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, 1275e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1285e9e3d77SNicholas Piggin unsigned int max_page_shift) 1295e9e3d77SNicholas Piggin { 1305e9e3d77SNicholas Piggin if (max_page_shift < PMD_SHIFT) 1315e9e3d77SNicholas Piggin return 0; 1325e9e3d77SNicholas Piggin 1335e9e3d77SNicholas Piggin if (!arch_vmap_pmd_supported(prot)) 1345e9e3d77SNicholas Piggin return 0; 1355e9e3d77SNicholas Piggin 1365e9e3d77SNicholas Piggin if ((end - addr) != PMD_SIZE) 1375e9e3d77SNicholas Piggin return 0; 1385e9e3d77SNicholas Piggin 1395e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, PMD_SIZE)) 1405e9e3d77SNicholas Piggin return 0; 1415e9e3d77SNicholas Piggin 1425e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, PMD_SIZE)) 1435e9e3d77SNicholas Piggin return 0; 1445e9e3d77SNicholas Piggin 1455e9e3d77SNicholas Piggin if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) 1465e9e3d77SNicholas Piggin return 0; 1475e9e3d77SNicholas Piggin 1485e9e3d77SNicholas Piggin return pmd_set_huge(pmd, phys_addr, prot); 1495e9e3d77SNicholas Piggin } 1505e9e3d77SNicholas Piggin 1515e9e3d77SNicholas Piggin static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 1525e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1535e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 1545e9e3d77SNicholas Piggin { 1555e9e3d77SNicholas Piggin pmd_t *pmd; 1565e9e3d77SNicholas Piggin unsigned long next; 1575e9e3d77SNicholas Piggin 1585e9e3d77SNicholas Piggin pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 1595e9e3d77SNicholas Piggin if (!pmd) 1605e9e3d77SNicholas Piggin return -ENOMEM; 1615e9e3d77SNicholas Piggin do { 1625e9e3d77SNicholas Piggin next = pmd_addr_end(addr, end); 1635e9e3d77SNicholas Piggin 1645e9e3d77SNicholas Piggin if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot, 1655e9e3d77SNicholas Piggin max_page_shift)) { 1665e9e3d77SNicholas Piggin *mask |= PGTBL_PMD_MODIFIED; 1675e9e3d77SNicholas Piggin continue; 1685e9e3d77SNicholas Piggin } 1695e9e3d77SNicholas Piggin 170f7ee1f13SChristophe Leroy if (vmap_pte_range(pmd, addr, next, phys_addr, prot, max_page_shift, mask)) 1715e9e3d77SNicholas Piggin return -ENOMEM; 1725e9e3d77SNicholas Piggin } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); 1735e9e3d77SNicholas Piggin return 0; 1745e9e3d77SNicholas Piggin } 1755e9e3d77SNicholas Piggin 1765e9e3d77SNicholas Piggin static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end, 1775e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1785e9e3d77SNicholas Piggin unsigned int max_page_shift) 1795e9e3d77SNicholas Piggin { 1805e9e3d77SNicholas Piggin if (max_page_shift < PUD_SHIFT) 1815e9e3d77SNicholas Piggin return 0; 1825e9e3d77SNicholas Piggin 1835e9e3d77SNicholas Piggin if (!arch_vmap_pud_supported(prot)) 1845e9e3d77SNicholas Piggin return 0; 1855e9e3d77SNicholas Piggin 1865e9e3d77SNicholas Piggin if ((end - addr) != PUD_SIZE) 1875e9e3d77SNicholas Piggin return 0; 1885e9e3d77SNicholas Piggin 1895e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, PUD_SIZE)) 1905e9e3d77SNicholas Piggin return 0; 1915e9e3d77SNicholas Piggin 1925e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, PUD_SIZE)) 1935e9e3d77SNicholas Piggin return 0; 1945e9e3d77SNicholas Piggin 1955e9e3d77SNicholas Piggin if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) 1965e9e3d77SNicholas Piggin return 0; 1975e9e3d77SNicholas Piggin 1985e9e3d77SNicholas Piggin return pud_set_huge(pud, phys_addr, prot); 1995e9e3d77SNicholas Piggin } 2005e9e3d77SNicholas Piggin 2015e9e3d77SNicholas Piggin static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 2025e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2035e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 2045e9e3d77SNicholas Piggin { 2055e9e3d77SNicholas Piggin pud_t *pud; 2065e9e3d77SNicholas Piggin unsigned long next; 2075e9e3d77SNicholas Piggin 2085e9e3d77SNicholas Piggin pud = pud_alloc_track(&init_mm, p4d, addr, mask); 2095e9e3d77SNicholas Piggin if (!pud) 2105e9e3d77SNicholas Piggin return -ENOMEM; 2115e9e3d77SNicholas Piggin do { 2125e9e3d77SNicholas Piggin next = pud_addr_end(addr, end); 2135e9e3d77SNicholas Piggin 2145e9e3d77SNicholas Piggin if (vmap_try_huge_pud(pud, addr, next, phys_addr, prot, 2155e9e3d77SNicholas Piggin max_page_shift)) { 2165e9e3d77SNicholas Piggin *mask |= PGTBL_PUD_MODIFIED; 2175e9e3d77SNicholas Piggin continue; 2185e9e3d77SNicholas Piggin } 2195e9e3d77SNicholas Piggin 2205e9e3d77SNicholas Piggin if (vmap_pmd_range(pud, addr, next, phys_addr, prot, 2215e9e3d77SNicholas Piggin max_page_shift, mask)) 2225e9e3d77SNicholas Piggin return -ENOMEM; 2235e9e3d77SNicholas Piggin } while (pud++, phys_addr += (next - addr), addr = next, addr != end); 2245e9e3d77SNicholas Piggin return 0; 2255e9e3d77SNicholas Piggin } 2265e9e3d77SNicholas Piggin 2275e9e3d77SNicholas Piggin static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, 2285e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2295e9e3d77SNicholas Piggin unsigned int max_page_shift) 2305e9e3d77SNicholas Piggin { 2315e9e3d77SNicholas Piggin if (max_page_shift < P4D_SHIFT) 2325e9e3d77SNicholas Piggin return 0; 2335e9e3d77SNicholas Piggin 2345e9e3d77SNicholas Piggin if (!arch_vmap_p4d_supported(prot)) 2355e9e3d77SNicholas Piggin return 0; 2365e9e3d77SNicholas Piggin 2375e9e3d77SNicholas Piggin if ((end - addr) != P4D_SIZE) 2385e9e3d77SNicholas Piggin return 0; 2395e9e3d77SNicholas Piggin 2405e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, P4D_SIZE)) 2415e9e3d77SNicholas Piggin return 0; 2425e9e3d77SNicholas Piggin 2435e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, P4D_SIZE)) 2445e9e3d77SNicholas Piggin return 0; 2455e9e3d77SNicholas Piggin 2465e9e3d77SNicholas Piggin if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) 2475e9e3d77SNicholas Piggin return 0; 2485e9e3d77SNicholas Piggin 2495e9e3d77SNicholas Piggin return p4d_set_huge(p4d, phys_addr, prot); 2505e9e3d77SNicholas Piggin } 2515e9e3d77SNicholas Piggin 2525e9e3d77SNicholas Piggin static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 2535e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2545e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 2555e9e3d77SNicholas Piggin { 2565e9e3d77SNicholas Piggin p4d_t *p4d; 2575e9e3d77SNicholas Piggin unsigned long next; 2585e9e3d77SNicholas Piggin 2595e9e3d77SNicholas Piggin p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 2605e9e3d77SNicholas Piggin if (!p4d) 2615e9e3d77SNicholas Piggin return -ENOMEM; 2625e9e3d77SNicholas Piggin do { 2635e9e3d77SNicholas Piggin next = p4d_addr_end(addr, end); 2645e9e3d77SNicholas Piggin 2655e9e3d77SNicholas Piggin if (vmap_try_huge_p4d(p4d, addr, next, phys_addr, prot, 2665e9e3d77SNicholas Piggin max_page_shift)) { 2675e9e3d77SNicholas Piggin *mask |= PGTBL_P4D_MODIFIED; 2685e9e3d77SNicholas Piggin continue; 2695e9e3d77SNicholas Piggin } 2705e9e3d77SNicholas Piggin 2715e9e3d77SNicholas Piggin if (vmap_pud_range(p4d, addr, next, phys_addr, prot, 2725e9e3d77SNicholas Piggin max_page_shift, mask)) 2735e9e3d77SNicholas Piggin return -ENOMEM; 2745e9e3d77SNicholas Piggin } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); 2755e9e3d77SNicholas Piggin return 0; 2765e9e3d77SNicholas Piggin } 2775e9e3d77SNicholas Piggin 2785d87510dSNicholas Piggin static int vmap_range_noflush(unsigned long addr, unsigned long end, 2795e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2805e9e3d77SNicholas Piggin unsigned int max_page_shift) 2815e9e3d77SNicholas Piggin { 2825e9e3d77SNicholas Piggin pgd_t *pgd; 2835e9e3d77SNicholas Piggin unsigned long start; 2845e9e3d77SNicholas Piggin unsigned long next; 2855e9e3d77SNicholas Piggin int err; 2865e9e3d77SNicholas Piggin pgtbl_mod_mask mask = 0; 2875e9e3d77SNicholas Piggin 2885e9e3d77SNicholas Piggin might_sleep(); 2895e9e3d77SNicholas Piggin BUG_ON(addr >= end); 2905e9e3d77SNicholas Piggin 2915e9e3d77SNicholas Piggin start = addr; 2925e9e3d77SNicholas Piggin pgd = pgd_offset_k(addr); 2935e9e3d77SNicholas Piggin do { 2945e9e3d77SNicholas Piggin next = pgd_addr_end(addr, end); 2955e9e3d77SNicholas Piggin err = vmap_p4d_range(pgd, addr, next, phys_addr, prot, 2965e9e3d77SNicholas Piggin max_page_shift, &mask); 2975e9e3d77SNicholas Piggin if (err) 2985e9e3d77SNicholas Piggin break; 2995e9e3d77SNicholas Piggin } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); 3005e9e3d77SNicholas Piggin 3015e9e3d77SNicholas Piggin if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 3025e9e3d77SNicholas Piggin arch_sync_kernel_mappings(start, end); 3035e9e3d77SNicholas Piggin 3045e9e3d77SNicholas Piggin return err; 3055e9e3d77SNicholas Piggin } 306b221385bSAdrian Bunk 30782a70ce0SChristoph Hellwig int ioremap_page_range(unsigned long addr, unsigned long end, 30882a70ce0SChristoph Hellwig phys_addr_t phys_addr, pgprot_t prot) 3095d87510dSNicholas Piggin { 3105d87510dSNicholas Piggin int err; 3115d87510dSNicholas Piggin 3128491502fSChristoph Hellwig err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot), 31382a70ce0SChristoph Hellwig ioremap_max_page_shift); 3145d87510dSNicholas Piggin flush_cache_vmap(addr, end); 315b073d7f8SAlexander Potapenko if (!err) 316fdea03e1SAlexander Potapenko err = kmsan_ioremap_page_range(addr, end, phys_addr, prot, 317b073d7f8SAlexander Potapenko ioremap_max_page_shift); 3185d87510dSNicholas Piggin return err; 3195d87510dSNicholas Piggin } 3205d87510dSNicholas Piggin 3212ba3e694SJoerg Roedel static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 3222ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3231da177e4SLinus Torvalds { 3241da177e4SLinus Torvalds pte_t *pte; 3251da177e4SLinus Torvalds 3261da177e4SLinus Torvalds pte = pte_offset_kernel(pmd, addr); 3271da177e4SLinus Torvalds do { 3281da177e4SLinus Torvalds pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); 3291da177e4SLinus Torvalds WARN_ON(!pte_none(ptent) && !pte_present(ptent)); 3301da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 3312ba3e694SJoerg Roedel *mask |= PGTBL_PTE_MODIFIED; 3321da177e4SLinus Torvalds } 3331da177e4SLinus Torvalds 3342ba3e694SJoerg Roedel static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 3352ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3361da177e4SLinus Torvalds { 3371da177e4SLinus Torvalds pmd_t *pmd; 3381da177e4SLinus Torvalds unsigned long next; 3392ba3e694SJoerg Roedel int cleared; 3401da177e4SLinus Torvalds 3411da177e4SLinus Torvalds pmd = pmd_offset(pud, addr); 3421da177e4SLinus Torvalds do { 3431da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 3442ba3e694SJoerg Roedel 3452ba3e694SJoerg Roedel cleared = pmd_clear_huge(pmd); 3462ba3e694SJoerg Roedel if (cleared || pmd_bad(*pmd)) 3472ba3e694SJoerg Roedel *mask |= PGTBL_PMD_MODIFIED; 3482ba3e694SJoerg Roedel 3492ba3e694SJoerg Roedel if (cleared) 350b9820d8fSToshi Kani continue; 3511da177e4SLinus Torvalds if (pmd_none_or_clear_bad(pmd)) 3521da177e4SLinus Torvalds continue; 3532ba3e694SJoerg Roedel vunmap_pte_range(pmd, addr, next, mask); 354e47110e9SAneesh Kumar K.V 355e47110e9SAneesh Kumar K.V cond_resched(); 3561da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 3571da177e4SLinus Torvalds } 3581da177e4SLinus Torvalds 3592ba3e694SJoerg Roedel static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 3602ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3611da177e4SLinus Torvalds { 3621da177e4SLinus Torvalds pud_t *pud; 3631da177e4SLinus Torvalds unsigned long next; 3642ba3e694SJoerg Roedel int cleared; 3651da177e4SLinus Torvalds 366c2febafcSKirill A. Shutemov pud = pud_offset(p4d, addr); 3671da177e4SLinus Torvalds do { 3681da177e4SLinus Torvalds next = pud_addr_end(addr, end); 3692ba3e694SJoerg Roedel 3702ba3e694SJoerg Roedel cleared = pud_clear_huge(pud); 3712ba3e694SJoerg Roedel if (cleared || pud_bad(*pud)) 3722ba3e694SJoerg Roedel *mask |= PGTBL_PUD_MODIFIED; 3732ba3e694SJoerg Roedel 3742ba3e694SJoerg Roedel if (cleared) 375b9820d8fSToshi Kani continue; 3761da177e4SLinus Torvalds if (pud_none_or_clear_bad(pud)) 3771da177e4SLinus Torvalds continue; 3782ba3e694SJoerg Roedel vunmap_pmd_range(pud, addr, next, mask); 3791da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 3801da177e4SLinus Torvalds } 3811da177e4SLinus Torvalds 3822ba3e694SJoerg Roedel static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 3832ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 384c2febafcSKirill A. Shutemov { 385c2febafcSKirill A. Shutemov p4d_t *p4d; 386c2febafcSKirill A. Shutemov unsigned long next; 387c2febafcSKirill A. Shutemov 388c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 389c2febafcSKirill A. Shutemov do { 390c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 3912ba3e694SJoerg Roedel 392c8db8c26SLi kunyu p4d_clear_huge(p4d); 393c8db8c26SLi kunyu if (p4d_bad(*p4d)) 3942ba3e694SJoerg Roedel *mask |= PGTBL_P4D_MODIFIED; 3952ba3e694SJoerg Roedel 396c2febafcSKirill A. Shutemov if (p4d_none_or_clear_bad(p4d)) 397c2febafcSKirill A. Shutemov continue; 3982ba3e694SJoerg Roedel vunmap_pud_range(p4d, addr, next, mask); 399c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 400c2febafcSKirill A. Shutemov } 401c2febafcSKirill A. Shutemov 4024ad0ae8cSNicholas Piggin /* 4034ad0ae8cSNicholas Piggin * vunmap_range_noflush is similar to vunmap_range, but does not 4044ad0ae8cSNicholas Piggin * flush caches or TLBs. 405b521c43fSChristoph Hellwig * 4064ad0ae8cSNicholas Piggin * The caller is responsible for calling flush_cache_vmap() before calling 4074ad0ae8cSNicholas Piggin * this function, and flush_tlb_kernel_range after it has returned 4084ad0ae8cSNicholas Piggin * successfully (and before the addresses are expected to cause a page fault 4094ad0ae8cSNicholas Piggin * or be re-mapped for something else, if TLB flushes are being delayed or 4104ad0ae8cSNicholas Piggin * coalesced). 411b521c43fSChristoph Hellwig * 4124ad0ae8cSNicholas Piggin * This is an internal function only. Do not use outside mm/. 413b521c43fSChristoph Hellwig */ 414b073d7f8SAlexander Potapenko void __vunmap_range_noflush(unsigned long start, unsigned long end) 4151da177e4SLinus Torvalds { 4161da177e4SLinus Torvalds unsigned long next; 417b521c43fSChristoph Hellwig pgd_t *pgd; 4182ba3e694SJoerg Roedel unsigned long addr = start; 4192ba3e694SJoerg Roedel pgtbl_mod_mask mask = 0; 4201da177e4SLinus Torvalds 4211da177e4SLinus Torvalds BUG_ON(addr >= end); 4221da177e4SLinus Torvalds pgd = pgd_offset_k(addr); 4231da177e4SLinus Torvalds do { 4241da177e4SLinus Torvalds next = pgd_addr_end(addr, end); 4252ba3e694SJoerg Roedel if (pgd_bad(*pgd)) 4262ba3e694SJoerg Roedel mask |= PGTBL_PGD_MODIFIED; 4271da177e4SLinus Torvalds if (pgd_none_or_clear_bad(pgd)) 4281da177e4SLinus Torvalds continue; 4292ba3e694SJoerg Roedel vunmap_p4d_range(pgd, addr, next, &mask); 4301da177e4SLinus Torvalds } while (pgd++, addr = next, addr != end); 4312ba3e694SJoerg Roedel 4322ba3e694SJoerg Roedel if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 4332ba3e694SJoerg Roedel arch_sync_kernel_mappings(start, end); 4341da177e4SLinus Torvalds } 4351da177e4SLinus Torvalds 436b073d7f8SAlexander Potapenko void vunmap_range_noflush(unsigned long start, unsigned long end) 437b073d7f8SAlexander Potapenko { 438b073d7f8SAlexander Potapenko kmsan_vunmap_range_noflush(start, end); 439b073d7f8SAlexander Potapenko __vunmap_range_noflush(start, end); 440b073d7f8SAlexander Potapenko } 441b073d7f8SAlexander Potapenko 4424ad0ae8cSNicholas Piggin /** 4434ad0ae8cSNicholas Piggin * vunmap_range - unmap kernel virtual addresses 4444ad0ae8cSNicholas Piggin * @addr: start of the VM area to unmap 4454ad0ae8cSNicholas Piggin * @end: end of the VM area to unmap (non-inclusive) 4464ad0ae8cSNicholas Piggin * 4474ad0ae8cSNicholas Piggin * Clears any present PTEs in the virtual address range, flushes TLBs and 4484ad0ae8cSNicholas Piggin * caches. Any subsequent access to the address before it has been re-mapped 4494ad0ae8cSNicholas Piggin * is a kernel bug. 4504ad0ae8cSNicholas Piggin */ 4514ad0ae8cSNicholas Piggin void vunmap_range(unsigned long addr, unsigned long end) 4524ad0ae8cSNicholas Piggin { 4534ad0ae8cSNicholas Piggin flush_cache_vunmap(addr, end); 4544ad0ae8cSNicholas Piggin vunmap_range_noflush(addr, end); 4554ad0ae8cSNicholas Piggin flush_tlb_kernel_range(addr, end); 4564ad0ae8cSNicholas Piggin } 4574ad0ae8cSNicholas Piggin 4580a264884SNicholas Piggin static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, 4592ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 4602ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 4611da177e4SLinus Torvalds { 4621da177e4SLinus Torvalds pte_t *pte; 4631da177e4SLinus Torvalds 464db64fe02SNick Piggin /* 465db64fe02SNick Piggin * nr is a running index into the array which helps higher level 466db64fe02SNick Piggin * callers keep track of where we're up to. 467db64fe02SNick Piggin */ 468db64fe02SNick Piggin 4692ba3e694SJoerg Roedel pte = pte_alloc_kernel_track(pmd, addr, mask); 4701da177e4SLinus Torvalds if (!pte) 4711da177e4SLinus Torvalds return -ENOMEM; 4721da177e4SLinus Torvalds do { 473db64fe02SNick Piggin struct page *page = pages[*nr]; 474db64fe02SNick Piggin 475c33c7948SRyan Roberts if (WARN_ON(!pte_none(ptep_get(pte)))) 476db64fe02SNick Piggin return -EBUSY; 477db64fe02SNick Piggin if (WARN_ON(!page)) 4781da177e4SLinus Torvalds return -ENOMEM; 4794fcdcc12SYury Norov if (WARN_ON(!pfn_valid(page_to_pfn(page)))) 4804fcdcc12SYury Norov return -EINVAL; 4814fcdcc12SYury Norov 4821da177e4SLinus Torvalds set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); 483db64fe02SNick Piggin (*nr)++; 4841da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 4852ba3e694SJoerg Roedel *mask |= PGTBL_PTE_MODIFIED; 4861da177e4SLinus Torvalds return 0; 4871da177e4SLinus Torvalds } 4881da177e4SLinus Torvalds 4890a264884SNicholas Piggin static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr, 4902ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 4912ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 4921da177e4SLinus Torvalds { 4931da177e4SLinus Torvalds pmd_t *pmd; 4941da177e4SLinus Torvalds unsigned long next; 4951da177e4SLinus Torvalds 4962ba3e694SJoerg Roedel pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 4971da177e4SLinus Torvalds if (!pmd) 4981da177e4SLinus Torvalds return -ENOMEM; 4991da177e4SLinus Torvalds do { 5001da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 5010a264884SNicholas Piggin if (vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask)) 5021da177e4SLinus Torvalds return -ENOMEM; 5031da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 5041da177e4SLinus Torvalds return 0; 5051da177e4SLinus Torvalds } 5061da177e4SLinus Torvalds 5070a264884SNicholas Piggin static int vmap_pages_pud_range(p4d_t *p4d, unsigned long addr, 5082ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 5092ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 5101da177e4SLinus Torvalds { 5111da177e4SLinus Torvalds pud_t *pud; 5121da177e4SLinus Torvalds unsigned long next; 5131da177e4SLinus Torvalds 5142ba3e694SJoerg Roedel pud = pud_alloc_track(&init_mm, p4d, addr, mask); 5151da177e4SLinus Torvalds if (!pud) 5161da177e4SLinus Torvalds return -ENOMEM; 5171da177e4SLinus Torvalds do { 5181da177e4SLinus Torvalds next = pud_addr_end(addr, end); 5190a264884SNicholas Piggin if (vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask)) 5201da177e4SLinus Torvalds return -ENOMEM; 5211da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 5221da177e4SLinus Torvalds return 0; 5231da177e4SLinus Torvalds } 5241da177e4SLinus Torvalds 5250a264884SNicholas Piggin static int vmap_pages_p4d_range(pgd_t *pgd, unsigned long addr, 5262ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 5272ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 528c2febafcSKirill A. Shutemov { 529c2febafcSKirill A. Shutemov p4d_t *p4d; 530c2febafcSKirill A. Shutemov unsigned long next; 531c2febafcSKirill A. Shutemov 5322ba3e694SJoerg Roedel p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 533c2febafcSKirill A. Shutemov if (!p4d) 534c2febafcSKirill A. Shutemov return -ENOMEM; 535c2febafcSKirill A. Shutemov do { 536c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 5370a264884SNicholas Piggin if (vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask)) 538c2febafcSKirill A. Shutemov return -ENOMEM; 539c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 540c2febafcSKirill A. Shutemov return 0; 541c2febafcSKirill A. Shutemov } 542c2febafcSKirill A. Shutemov 543121e6f32SNicholas Piggin static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, 544121e6f32SNicholas Piggin pgprot_t prot, struct page **pages) 545121e6f32SNicholas Piggin { 546121e6f32SNicholas Piggin unsigned long start = addr; 547121e6f32SNicholas Piggin pgd_t *pgd; 548121e6f32SNicholas Piggin unsigned long next; 549121e6f32SNicholas Piggin int err = 0; 550121e6f32SNicholas Piggin int nr = 0; 551121e6f32SNicholas Piggin pgtbl_mod_mask mask = 0; 552121e6f32SNicholas Piggin 553121e6f32SNicholas Piggin BUG_ON(addr >= end); 554121e6f32SNicholas Piggin pgd = pgd_offset_k(addr); 555121e6f32SNicholas Piggin do { 556121e6f32SNicholas Piggin next = pgd_addr_end(addr, end); 557121e6f32SNicholas Piggin if (pgd_bad(*pgd)) 558121e6f32SNicholas Piggin mask |= PGTBL_PGD_MODIFIED; 559121e6f32SNicholas Piggin err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask); 560121e6f32SNicholas Piggin if (err) 561121e6f32SNicholas Piggin return err; 562121e6f32SNicholas Piggin } while (pgd++, addr = next, addr != end); 563121e6f32SNicholas Piggin 564121e6f32SNicholas Piggin if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 565121e6f32SNicholas Piggin arch_sync_kernel_mappings(start, end); 566121e6f32SNicholas Piggin 567121e6f32SNicholas Piggin return 0; 568121e6f32SNicholas Piggin } 569121e6f32SNicholas Piggin 570b67177ecSNicholas Piggin /* 571b67177ecSNicholas Piggin * vmap_pages_range_noflush is similar to vmap_pages_range, but does not 572b67177ecSNicholas Piggin * flush caches. 573b67177ecSNicholas Piggin * 574b67177ecSNicholas Piggin * The caller is responsible for calling flush_cache_vmap() after this 575b67177ecSNicholas Piggin * function returns successfully and before the addresses are accessed. 576b67177ecSNicholas Piggin * 577b67177ecSNicholas Piggin * This is an internal function only. Do not use outside mm/. 578b67177ecSNicholas Piggin */ 579b073d7f8SAlexander Potapenko int __vmap_pages_range_noflush(unsigned long addr, unsigned long end, 580121e6f32SNicholas Piggin pgprot_t prot, struct page **pages, unsigned int page_shift) 581121e6f32SNicholas Piggin { 582121e6f32SNicholas Piggin unsigned int i, nr = (end - addr) >> PAGE_SHIFT; 583121e6f32SNicholas Piggin 584121e6f32SNicholas Piggin WARN_ON(page_shift < PAGE_SHIFT); 585121e6f32SNicholas Piggin 586121e6f32SNicholas Piggin if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC) || 587121e6f32SNicholas Piggin page_shift == PAGE_SHIFT) 588121e6f32SNicholas Piggin return vmap_small_pages_range_noflush(addr, end, prot, pages); 589121e6f32SNicholas Piggin 590121e6f32SNicholas Piggin for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) { 591121e6f32SNicholas Piggin int err; 592121e6f32SNicholas Piggin 593121e6f32SNicholas Piggin err = vmap_range_noflush(addr, addr + (1UL << page_shift), 59408262ac5SMatthew Wilcox page_to_phys(pages[i]), prot, 595121e6f32SNicholas Piggin page_shift); 596121e6f32SNicholas Piggin if (err) 597121e6f32SNicholas Piggin return err; 598121e6f32SNicholas Piggin 599121e6f32SNicholas Piggin addr += 1UL << page_shift; 600121e6f32SNicholas Piggin } 601121e6f32SNicholas Piggin 602121e6f32SNicholas Piggin return 0; 603121e6f32SNicholas Piggin } 604121e6f32SNicholas Piggin 605b073d7f8SAlexander Potapenko int vmap_pages_range_noflush(unsigned long addr, unsigned long end, 606b073d7f8SAlexander Potapenko pgprot_t prot, struct page **pages, unsigned int page_shift) 607b073d7f8SAlexander Potapenko { 60847ebd031SAlexander Potapenko int ret = kmsan_vmap_pages_range_noflush(addr, end, prot, pages, 60947ebd031SAlexander Potapenko page_shift); 61047ebd031SAlexander Potapenko 61147ebd031SAlexander Potapenko if (ret) 61247ebd031SAlexander Potapenko return ret; 613b073d7f8SAlexander Potapenko return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift); 614b073d7f8SAlexander Potapenko } 615b073d7f8SAlexander Potapenko 616b67177ecSNicholas Piggin /** 617b67177ecSNicholas Piggin * vmap_pages_range - map pages to a kernel virtual address 618b67177ecSNicholas Piggin * @addr: start of the VM area to map 619b67177ecSNicholas Piggin * @end: end of the VM area to map (non-inclusive) 620b67177ecSNicholas Piggin * @prot: page protection flags to use 621b67177ecSNicholas Piggin * @pages: pages to map (always PAGE_SIZE pages) 622b67177ecSNicholas Piggin * @page_shift: maximum shift that the pages may be mapped with, @pages must 623b67177ecSNicholas Piggin * be aligned and contiguous up to at least this shift. 624b67177ecSNicholas Piggin * 625b67177ecSNicholas Piggin * RETURNS: 626b67177ecSNicholas Piggin * 0 on success, -errno on failure. 627b67177ecSNicholas Piggin */ 628121e6f32SNicholas Piggin static int vmap_pages_range(unsigned long addr, unsigned long end, 629121e6f32SNicholas Piggin pgprot_t prot, struct page **pages, unsigned int page_shift) 630121e6f32SNicholas Piggin { 631121e6f32SNicholas Piggin int err; 632121e6f32SNicholas Piggin 633121e6f32SNicholas Piggin err = vmap_pages_range_noflush(addr, end, prot, pages, page_shift); 634121e6f32SNicholas Piggin flush_cache_vmap(addr, end); 635121e6f32SNicholas Piggin return err; 636121e6f32SNicholas Piggin } 637121e6f32SNicholas Piggin 63881ac3ad9SKAMEZAWA Hiroyuki int is_vmalloc_or_module_addr(const void *x) 63973bdf0a6SLinus Torvalds { 64073bdf0a6SLinus Torvalds /* 641ab4f2ee1SRussell King * ARM, x86-64 and sparc64 put modules in a special place, 64273bdf0a6SLinus Torvalds * and fall back on vmalloc() if that fails. Others 64373bdf0a6SLinus Torvalds * just put it in the vmalloc space. 64473bdf0a6SLinus Torvalds */ 64573bdf0a6SLinus Torvalds #if defined(CONFIG_MODULES) && defined(MODULES_VADDR) 6464aff1dc4SAndrey Konovalov unsigned long addr = (unsigned long)kasan_reset_tag(x); 64773bdf0a6SLinus Torvalds if (addr >= MODULES_VADDR && addr < MODULES_END) 64873bdf0a6SLinus Torvalds return 1; 64973bdf0a6SLinus Torvalds #endif 65073bdf0a6SLinus Torvalds return is_vmalloc_addr(x); 65173bdf0a6SLinus Torvalds } 65201858469SDavid Howells EXPORT_SYMBOL_GPL(is_vmalloc_or_module_addr); 65373bdf0a6SLinus Torvalds 65448667e7aSChristoph Lameter /* 655c0eb315aSNicholas Piggin * Walk a vmap address to the struct page it maps. Huge vmap mappings will 656c0eb315aSNicholas Piggin * return the tail page that corresponds to the base page address, which 657c0eb315aSNicholas Piggin * matches small vmap mappings. 65848667e7aSChristoph Lameter */ 659add688fbSmalc struct page *vmalloc_to_page(const void *vmalloc_addr) 66048667e7aSChristoph Lameter { 66148667e7aSChristoph Lameter unsigned long addr = (unsigned long) vmalloc_addr; 662add688fbSmalc struct page *page = NULL; 66348667e7aSChristoph Lameter pgd_t *pgd = pgd_offset_k(addr); 664c2febafcSKirill A. Shutemov p4d_t *p4d; 665c2febafcSKirill A. Shutemov pud_t *pud; 666c2febafcSKirill A. Shutemov pmd_t *pmd; 667c2febafcSKirill A. Shutemov pte_t *ptep, pte; 66848667e7aSChristoph Lameter 6697aa413deSIngo Molnar /* 6707aa413deSIngo Molnar * XXX we might need to change this if we add VIRTUAL_BUG_ON for 6717aa413deSIngo Molnar * architectures that do not vmalloc module space 6727aa413deSIngo Molnar */ 67373bdf0a6SLinus Torvalds VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); 67459ea7463SJiri Slaby 675c2febafcSKirill A. Shutemov if (pgd_none(*pgd)) 676c2febafcSKirill A. Shutemov return NULL; 677c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pgd_leaf(*pgd))) 678c0eb315aSNicholas Piggin return NULL; /* XXX: no allowance for huge pgd */ 679c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pgd_bad(*pgd))) 680c0eb315aSNicholas Piggin return NULL; 681c0eb315aSNicholas Piggin 682c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 683c2febafcSKirill A. Shutemov if (p4d_none(*p4d)) 684c2febafcSKirill A. Shutemov return NULL; 685c0eb315aSNicholas Piggin if (p4d_leaf(*p4d)) 686c0eb315aSNicholas Piggin return p4d_page(*p4d) + ((addr & ~P4D_MASK) >> PAGE_SHIFT); 687c0eb315aSNicholas Piggin if (WARN_ON_ONCE(p4d_bad(*p4d))) 688c2febafcSKirill A. Shutemov return NULL; 689c0eb315aSNicholas Piggin 690c0eb315aSNicholas Piggin pud = pud_offset(p4d, addr); 691c0eb315aSNicholas Piggin if (pud_none(*pud)) 692c0eb315aSNicholas Piggin return NULL; 693c0eb315aSNicholas Piggin if (pud_leaf(*pud)) 694c0eb315aSNicholas Piggin return pud_page(*pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 695c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pud_bad(*pud))) 696c0eb315aSNicholas Piggin return NULL; 697c0eb315aSNicholas Piggin 698c2febafcSKirill A. Shutemov pmd = pmd_offset(pud, addr); 699c0eb315aSNicholas Piggin if (pmd_none(*pmd)) 700c0eb315aSNicholas Piggin return NULL; 701c0eb315aSNicholas Piggin if (pmd_leaf(*pmd)) 702c0eb315aSNicholas Piggin return pmd_page(*pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 703c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pmd_bad(*pmd))) 704c2febafcSKirill A. Shutemov return NULL; 705db64fe02SNick Piggin 7060d1c81edSHugh Dickins ptep = pte_offset_kernel(pmd, addr); 707c33c7948SRyan Roberts pte = ptep_get(ptep); 70848667e7aSChristoph Lameter if (pte_present(pte)) 709add688fbSmalc page = pte_page(pte); 710c0eb315aSNicholas Piggin 711add688fbSmalc return page; 712ece86e22SJianyu Zhan } 713ece86e22SJianyu Zhan EXPORT_SYMBOL(vmalloc_to_page); 714ece86e22SJianyu Zhan 715add688fbSmalc /* 716add688fbSmalc * Map a vmalloc()-space virtual address to the physical page frame number. 717add688fbSmalc */ 718add688fbSmalc unsigned long vmalloc_to_pfn(const void *vmalloc_addr) 719add688fbSmalc { 720add688fbSmalc return page_to_pfn(vmalloc_to_page(vmalloc_addr)); 721add688fbSmalc } 722add688fbSmalc EXPORT_SYMBOL(vmalloc_to_pfn); 723add688fbSmalc 724db64fe02SNick Piggin 725db64fe02SNick Piggin /*** Global kva allocator ***/ 726db64fe02SNick Piggin 727bb850f4dSUladzislau Rezki (Sony) #define DEBUG_AUGMENT_PROPAGATE_CHECK 0 728a6cf4e0fSUladzislau Rezki (Sony) #define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0 729bb850f4dSUladzislau Rezki (Sony) 730db64fe02SNick Piggin 731e36176beSUladzislau Rezki (Sony) static DEFINE_SPINLOCK(free_vmap_area_lock); 73268ad4a33SUladzislau Rezki (Sony) static bool vmap_initialized __read_mostly; 73389699605SNick Piggin 73468ad4a33SUladzislau Rezki (Sony) /* 73568ad4a33SUladzislau Rezki (Sony) * This kmem_cache is used for vmap_area objects. Instead of 73668ad4a33SUladzislau Rezki (Sony) * allocating from slab we reuse an object from this cache to 73768ad4a33SUladzislau Rezki (Sony) * make things faster. Especially in "no edge" splitting of 73868ad4a33SUladzislau Rezki (Sony) * free block. 73968ad4a33SUladzislau Rezki (Sony) */ 74068ad4a33SUladzislau Rezki (Sony) static struct kmem_cache *vmap_area_cachep; 74189699605SNick Piggin 74268ad4a33SUladzislau Rezki (Sony) /* 74368ad4a33SUladzislau Rezki (Sony) * This linked list is used in pair with free_vmap_area_root. 74468ad4a33SUladzislau Rezki (Sony) * It gives O(1) access to prev/next to perform fast coalescing. 74568ad4a33SUladzislau Rezki (Sony) */ 74668ad4a33SUladzislau Rezki (Sony) static LIST_HEAD(free_vmap_area_list); 74768ad4a33SUladzislau Rezki (Sony) 74868ad4a33SUladzislau Rezki (Sony) /* 74968ad4a33SUladzislau Rezki (Sony) * This augment red-black tree represents the free vmap space. 75068ad4a33SUladzislau Rezki (Sony) * All vmap_area objects in this tree are sorted by va->va_start 75168ad4a33SUladzislau Rezki (Sony) * address. It is used for allocation and merging when a vmap 75268ad4a33SUladzislau Rezki (Sony) * object is released. 75368ad4a33SUladzislau Rezki (Sony) * 75468ad4a33SUladzislau Rezki (Sony) * Each vmap_area node contains a maximum available free block 75568ad4a33SUladzislau Rezki (Sony) * of its sub-tree, right or left. Therefore it is possible to 75668ad4a33SUladzislau Rezki (Sony) * find a lowest match of free area. 75768ad4a33SUladzislau Rezki (Sony) */ 75868ad4a33SUladzislau Rezki (Sony) static struct rb_root free_vmap_area_root = RB_ROOT; 75968ad4a33SUladzislau Rezki (Sony) 76082dd23e8SUladzislau Rezki (Sony) /* 76182dd23e8SUladzislau Rezki (Sony) * Preload a CPU with one object for "no edge" split case. The 76282dd23e8SUladzislau Rezki (Sony) * aim is to get rid of allocations from the atomic context, thus 76382dd23e8SUladzislau Rezki (Sony) * to use more permissive allocation masks. 76482dd23e8SUladzislau Rezki (Sony) */ 76582dd23e8SUladzislau Rezki (Sony) static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node); 76682dd23e8SUladzislau Rezki (Sony) 767d0936029SUladzislau Rezki (Sony) /* 768d0936029SUladzislau Rezki (Sony) * An effective vmap-node logic. Users make use of nodes instead 769d0936029SUladzislau Rezki (Sony) * of a global heap. It allows to balance an access and mitigate 770d0936029SUladzislau Rezki (Sony) * contention. 771d0936029SUladzislau Rezki (Sony) */ 772d0936029SUladzislau Rezki (Sony) struct rb_list { 773d0936029SUladzislau Rezki (Sony) struct rb_root root; 774d0936029SUladzislau Rezki (Sony) struct list_head head; 775d0936029SUladzislau Rezki (Sony) spinlock_t lock; 776d0936029SUladzislau Rezki (Sony) }; 777d0936029SUladzislau Rezki (Sony) 77872210662SUladzislau Rezki (Sony) struct vmap_pool { 77972210662SUladzislau Rezki (Sony) struct list_head head; 78072210662SUladzislau Rezki (Sony) unsigned long len; 78172210662SUladzislau Rezki (Sony) }; 78272210662SUladzislau Rezki (Sony) 78372210662SUladzislau Rezki (Sony) /* 78472210662SUladzislau Rezki (Sony) * A fast size storage contains VAs up to 1M size. 78572210662SUladzislau Rezki (Sony) */ 78672210662SUladzislau Rezki (Sony) #define MAX_VA_SIZE_PAGES 256 78772210662SUladzislau Rezki (Sony) 788d0936029SUladzislau Rezki (Sony) static struct vmap_node { 78972210662SUladzislau Rezki (Sony) /* Simple size segregated storage. */ 79072210662SUladzislau Rezki (Sony) struct vmap_pool pool[MAX_VA_SIZE_PAGES]; 79172210662SUladzislau Rezki (Sony) spinlock_t pool_lock; 79272210662SUladzislau Rezki (Sony) bool skip_populate; 79372210662SUladzislau Rezki (Sony) 794d0936029SUladzislau Rezki (Sony) /* Bookkeeping data of this node. */ 795d0936029SUladzislau Rezki (Sony) struct rb_list busy; 796282631cbSUladzislau Rezki (Sony) struct rb_list lazy; 797282631cbSUladzislau Rezki (Sony) 798282631cbSUladzislau Rezki (Sony) /* 799282631cbSUladzislau Rezki (Sony) * Ready-to-free areas. 800282631cbSUladzislau Rezki (Sony) */ 801282631cbSUladzislau Rezki (Sony) struct list_head purge_list; 80272210662SUladzislau Rezki (Sony) struct work_struct purge_work; 80372210662SUladzislau Rezki (Sony) unsigned long nr_purged; 804d0936029SUladzislau Rezki (Sony) } single; 805d0936029SUladzislau Rezki (Sony) 806d0936029SUladzislau Rezki (Sony) static struct vmap_node *vmap_nodes = &single; 807d0936029SUladzislau Rezki (Sony) static __read_mostly unsigned int nr_vmap_nodes = 1; 808d0936029SUladzislau Rezki (Sony) static __read_mostly unsigned int vmap_zone_size = 1; 809d0936029SUladzislau Rezki (Sony) 810d0936029SUladzislau Rezki (Sony) static inline unsigned int 811d0936029SUladzislau Rezki (Sony) addr_to_node_id(unsigned long addr) 812d0936029SUladzislau Rezki (Sony) { 813d0936029SUladzislau Rezki (Sony) return (addr / vmap_zone_size) % nr_vmap_nodes; 814d0936029SUladzislau Rezki (Sony) } 815d0936029SUladzislau Rezki (Sony) 816d0936029SUladzislau Rezki (Sony) static inline struct vmap_node * 817d0936029SUladzislau Rezki (Sony) addr_to_node(unsigned long addr) 818d0936029SUladzislau Rezki (Sony) { 819d0936029SUladzislau Rezki (Sony) return &vmap_nodes[addr_to_node_id(addr)]; 820d0936029SUladzislau Rezki (Sony) } 821d0936029SUladzislau Rezki (Sony) 82272210662SUladzislau Rezki (Sony) static inline struct vmap_node * 82372210662SUladzislau Rezki (Sony) id_to_node(unsigned int id) 82472210662SUladzislau Rezki (Sony) { 82572210662SUladzislau Rezki (Sony) return &vmap_nodes[id % nr_vmap_nodes]; 82672210662SUladzislau Rezki (Sony) } 82772210662SUladzislau Rezki (Sony) 82872210662SUladzislau Rezki (Sony) /* 82972210662SUladzislau Rezki (Sony) * We use the value 0 to represent "no node", that is why 83072210662SUladzislau Rezki (Sony) * an encoded value will be the node-id incremented by 1. 83172210662SUladzislau Rezki (Sony) * It is always greater then 0. A valid node_id which can 83272210662SUladzislau Rezki (Sony) * be encoded is [0:nr_vmap_nodes - 1]. If a passed node_id 83372210662SUladzislau Rezki (Sony) * is not valid 0 is returned. 83472210662SUladzislau Rezki (Sony) */ 83572210662SUladzislau Rezki (Sony) static unsigned int 83672210662SUladzislau Rezki (Sony) encode_vn_id(unsigned int node_id) 83772210662SUladzislau Rezki (Sony) { 83872210662SUladzislau Rezki (Sony) /* Can store U8_MAX [0:254] nodes. */ 83972210662SUladzislau Rezki (Sony) if (node_id < nr_vmap_nodes) 84072210662SUladzislau Rezki (Sony) return (node_id + 1) << BITS_PER_BYTE; 84172210662SUladzislau Rezki (Sony) 84272210662SUladzislau Rezki (Sony) /* Warn and no node encoded. */ 84372210662SUladzislau Rezki (Sony) WARN_ONCE(1, "Encode wrong node id (%u)\n", node_id); 84472210662SUladzislau Rezki (Sony) return 0; 84572210662SUladzislau Rezki (Sony) } 84672210662SUladzislau Rezki (Sony) 84772210662SUladzislau Rezki (Sony) /* 84872210662SUladzislau Rezki (Sony) * Returns an encoded node-id, the valid range is within 84972210662SUladzislau Rezki (Sony) * [0:nr_vmap_nodes-1] values. Otherwise nr_vmap_nodes is 85072210662SUladzislau Rezki (Sony) * returned if extracted data is wrong. 85172210662SUladzislau Rezki (Sony) */ 85272210662SUladzislau Rezki (Sony) static unsigned int 85372210662SUladzislau Rezki (Sony) decode_vn_id(unsigned int val) 85472210662SUladzislau Rezki (Sony) { 85572210662SUladzislau Rezki (Sony) unsigned int node_id = (val >> BITS_PER_BYTE) - 1; 85672210662SUladzislau Rezki (Sony) 85772210662SUladzislau Rezki (Sony) /* Can store U8_MAX [0:254] nodes. */ 85872210662SUladzislau Rezki (Sony) if (node_id < nr_vmap_nodes) 85972210662SUladzislau Rezki (Sony) return node_id; 86072210662SUladzislau Rezki (Sony) 86172210662SUladzislau Rezki (Sony) /* If it was _not_ zero, warn. */ 86272210662SUladzislau Rezki (Sony) WARN_ONCE(node_id != UINT_MAX, 86372210662SUladzislau Rezki (Sony) "Decode wrong node id (%d)\n", node_id); 86472210662SUladzislau Rezki (Sony) 86572210662SUladzislau Rezki (Sony) return nr_vmap_nodes; 86672210662SUladzislau Rezki (Sony) } 86772210662SUladzislau Rezki (Sony) 86872210662SUladzislau Rezki (Sony) static bool 86972210662SUladzislau Rezki (Sony) is_vn_id_valid(unsigned int node_id) 87072210662SUladzislau Rezki (Sony) { 87172210662SUladzislau Rezki (Sony) if (node_id < nr_vmap_nodes) 87272210662SUladzislau Rezki (Sony) return true; 87372210662SUladzislau Rezki (Sony) 87472210662SUladzislau Rezki (Sony) return false; 87572210662SUladzislau Rezki (Sony) } 87672210662SUladzislau Rezki (Sony) 87768ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 87868ad4a33SUladzislau Rezki (Sony) va_size(struct vmap_area *va) 87968ad4a33SUladzislau Rezki (Sony) { 88068ad4a33SUladzislau Rezki (Sony) return (va->va_end - va->va_start); 88168ad4a33SUladzislau Rezki (Sony) } 88268ad4a33SUladzislau Rezki (Sony) 88368ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 88468ad4a33SUladzislau Rezki (Sony) get_subtree_max_size(struct rb_node *node) 88568ad4a33SUladzislau Rezki (Sony) { 88668ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 88768ad4a33SUladzislau Rezki (Sony) 88868ad4a33SUladzislau Rezki (Sony) va = rb_entry_safe(node, struct vmap_area, rb_node); 88968ad4a33SUladzislau Rezki (Sony) return va ? va->subtree_max_size : 0; 89068ad4a33SUladzislau Rezki (Sony) } 89168ad4a33SUladzislau Rezki (Sony) 892315cc066SMichel Lespinasse RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb, 893315cc066SMichel Lespinasse struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size) 89468ad4a33SUladzislau Rezki (Sony) 89577e50af0SThomas Gleixner static void reclaim_and_purge_vmap_areas(void); 89668ad4a33SUladzislau Rezki (Sony) static BLOCKING_NOTIFIER_HEAD(vmap_notify_list); 897690467c8SUladzislau Rezki (Sony) static void drain_vmap_area_work(struct work_struct *work); 898690467c8SUladzislau Rezki (Sony) static DECLARE_WORK(drain_vmap_work, drain_vmap_area_work); 899db64fe02SNick Piggin 90097105f0aSRoman Gushchin static atomic_long_t nr_vmalloc_pages; 90197105f0aSRoman Gushchin 90297105f0aSRoman Gushchin unsigned long vmalloc_nr_pages(void) 90397105f0aSRoman Gushchin { 90497105f0aSRoman Gushchin return atomic_long_read(&nr_vmalloc_pages); 90597105f0aSRoman Gushchin } 90697105f0aSRoman Gushchin 907153090f2SBaoquan He /* Look up the first VA which satisfies addr < va_end, NULL if none. */ 908d0936029SUladzislau Rezki (Sony) static struct vmap_area * 90953becf32SUladzislau Rezki (Sony) __find_vmap_area_exceed_addr(unsigned long addr, struct rb_root *root) 910f181234aSChen Wandun { 911f181234aSChen Wandun struct vmap_area *va = NULL; 912d0936029SUladzislau Rezki (Sony) struct rb_node *n = root->rb_node; 913f181234aSChen Wandun 9144aff1dc4SAndrey Konovalov addr = (unsigned long)kasan_reset_tag((void *)addr); 9154aff1dc4SAndrey Konovalov 916f181234aSChen Wandun while (n) { 917f181234aSChen Wandun struct vmap_area *tmp; 918f181234aSChen Wandun 919f181234aSChen Wandun tmp = rb_entry(n, struct vmap_area, rb_node); 920f181234aSChen Wandun if (tmp->va_end > addr) { 921f181234aSChen Wandun va = tmp; 922f181234aSChen Wandun if (tmp->va_start <= addr) 923f181234aSChen Wandun break; 924f181234aSChen Wandun 925f181234aSChen Wandun n = n->rb_left; 926f181234aSChen Wandun } else 927f181234aSChen Wandun n = n->rb_right; 928f181234aSChen Wandun } 929f181234aSChen Wandun 930f181234aSChen Wandun return va; 931f181234aSChen Wandun } 932f181234aSChen Wandun 93353becf32SUladzislau Rezki (Sony) /* 93453becf32SUladzislau Rezki (Sony) * Returns a node where a first VA, that satisfies addr < va_end, resides. 93553becf32SUladzislau Rezki (Sony) * If success, a node is locked. A user is responsible to unlock it when a 93653becf32SUladzislau Rezki (Sony) * VA is no longer needed to be accessed. 93753becf32SUladzislau Rezki (Sony) * 93853becf32SUladzislau Rezki (Sony) * Returns NULL if nothing found. 93953becf32SUladzislau Rezki (Sony) */ 94053becf32SUladzislau Rezki (Sony) static struct vmap_node * 94153becf32SUladzislau Rezki (Sony) find_vmap_area_exceed_addr_lock(unsigned long addr, struct vmap_area **va) 94253becf32SUladzislau Rezki (Sony) { 94353becf32SUladzislau Rezki (Sony) struct vmap_node *vn, *va_node = NULL; 94453becf32SUladzislau Rezki (Sony) struct vmap_area *va_lowest; 94553becf32SUladzislau Rezki (Sony) int i; 94653becf32SUladzislau Rezki (Sony) 94753becf32SUladzislau Rezki (Sony) for (i = 0; i < nr_vmap_nodes; i++) { 94853becf32SUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 94953becf32SUladzislau Rezki (Sony) 95053becf32SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 95153becf32SUladzislau Rezki (Sony) va_lowest = __find_vmap_area_exceed_addr(addr, &vn->busy.root); 95253becf32SUladzislau Rezki (Sony) if (va_lowest) { 95353becf32SUladzislau Rezki (Sony) if (!va_node || va_lowest->va_start < (*va)->va_start) { 95453becf32SUladzislau Rezki (Sony) if (va_node) 95553becf32SUladzislau Rezki (Sony) spin_unlock(&va_node->busy.lock); 95653becf32SUladzislau Rezki (Sony) 95753becf32SUladzislau Rezki (Sony) *va = va_lowest; 95853becf32SUladzislau Rezki (Sony) va_node = vn; 95953becf32SUladzislau Rezki (Sony) continue; 96053becf32SUladzislau Rezki (Sony) } 96153becf32SUladzislau Rezki (Sony) } 96253becf32SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 96353becf32SUladzislau Rezki (Sony) } 96453becf32SUladzislau Rezki (Sony) 96553becf32SUladzislau Rezki (Sony) return va_node; 96653becf32SUladzislau Rezki (Sony) } 96753becf32SUladzislau Rezki (Sony) 968899c6efeSUladzislau Rezki (Sony) static struct vmap_area *__find_vmap_area(unsigned long addr, struct rb_root *root) 9691da177e4SLinus Torvalds { 970899c6efeSUladzislau Rezki (Sony) struct rb_node *n = root->rb_node; 971db64fe02SNick Piggin 9724aff1dc4SAndrey Konovalov addr = (unsigned long)kasan_reset_tag((void *)addr); 9734aff1dc4SAndrey Konovalov 974db64fe02SNick Piggin while (n) { 975db64fe02SNick Piggin struct vmap_area *va; 976db64fe02SNick Piggin 977db64fe02SNick Piggin va = rb_entry(n, struct vmap_area, rb_node); 978db64fe02SNick Piggin if (addr < va->va_start) 979db64fe02SNick Piggin n = n->rb_left; 980cef2ac3fSHATAYAMA Daisuke else if (addr >= va->va_end) 981db64fe02SNick Piggin n = n->rb_right; 982db64fe02SNick Piggin else 983db64fe02SNick Piggin return va; 984db64fe02SNick Piggin } 985db64fe02SNick Piggin 986db64fe02SNick Piggin return NULL; 987db64fe02SNick Piggin } 988db64fe02SNick Piggin 98968ad4a33SUladzislau Rezki (Sony) /* 99068ad4a33SUladzislau Rezki (Sony) * This function returns back addresses of parent node 99168ad4a33SUladzislau Rezki (Sony) * and its left or right link for further processing. 9929c801f61SUladzislau Rezki (Sony) * 9939c801f61SUladzislau Rezki (Sony) * Otherwise NULL is returned. In that case all further 9949c801f61SUladzislau Rezki (Sony) * steps regarding inserting of conflicting overlap range 9959c801f61SUladzislau Rezki (Sony) * have to be declined and actually considered as a bug. 99668ad4a33SUladzislau Rezki (Sony) */ 99768ad4a33SUladzislau Rezki (Sony) static __always_inline struct rb_node ** 99868ad4a33SUladzislau Rezki (Sony) find_va_links(struct vmap_area *va, 99968ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct rb_node *from, 100068ad4a33SUladzislau Rezki (Sony) struct rb_node **parent) 1001db64fe02SNick Piggin { 1002170168d0SNamhyung Kim struct vmap_area *tmp_va; 100368ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 1004db64fe02SNick Piggin 100568ad4a33SUladzislau Rezki (Sony) if (root) { 100668ad4a33SUladzislau Rezki (Sony) link = &root->rb_node; 100768ad4a33SUladzislau Rezki (Sony) if (unlikely(!*link)) { 100868ad4a33SUladzislau Rezki (Sony) *parent = NULL; 100968ad4a33SUladzislau Rezki (Sony) return link; 101068ad4a33SUladzislau Rezki (Sony) } 101168ad4a33SUladzislau Rezki (Sony) } else { 101268ad4a33SUladzislau Rezki (Sony) link = &from; 101368ad4a33SUladzislau Rezki (Sony) } 101468ad4a33SUladzislau Rezki (Sony) 101568ad4a33SUladzislau Rezki (Sony) /* 101668ad4a33SUladzislau Rezki (Sony) * Go to the bottom of the tree. When we hit the last point 101768ad4a33SUladzislau Rezki (Sony) * we end up with parent rb_node and correct direction, i name 101868ad4a33SUladzislau Rezki (Sony) * it link, where the new va->rb_node will be attached to. 101968ad4a33SUladzislau Rezki (Sony) */ 102068ad4a33SUladzislau Rezki (Sony) do { 102168ad4a33SUladzislau Rezki (Sony) tmp_va = rb_entry(*link, struct vmap_area, rb_node); 102268ad4a33SUladzislau Rezki (Sony) 102368ad4a33SUladzislau Rezki (Sony) /* 102468ad4a33SUladzislau Rezki (Sony) * During the traversal we also do some sanity check. 102568ad4a33SUladzislau Rezki (Sony) * Trigger the BUG() if there are sides(left/right) 102668ad4a33SUladzislau Rezki (Sony) * or full overlaps. 102768ad4a33SUladzislau Rezki (Sony) */ 1028753df96bSBaoquan He if (va->va_end <= tmp_va->va_start) 102968ad4a33SUladzislau Rezki (Sony) link = &(*link)->rb_left; 1030753df96bSBaoquan He else if (va->va_start >= tmp_va->va_end) 103168ad4a33SUladzislau Rezki (Sony) link = &(*link)->rb_right; 10329c801f61SUladzislau Rezki (Sony) else { 10339c801f61SUladzislau Rezki (Sony) WARN(1, "vmalloc bug: 0x%lx-0x%lx overlaps with 0x%lx-0x%lx\n", 10349c801f61SUladzislau Rezki (Sony) va->va_start, va->va_end, tmp_va->va_start, tmp_va->va_end); 10359c801f61SUladzislau Rezki (Sony) 10369c801f61SUladzislau Rezki (Sony) return NULL; 10379c801f61SUladzislau Rezki (Sony) } 103868ad4a33SUladzislau Rezki (Sony) } while (*link); 103968ad4a33SUladzislau Rezki (Sony) 104068ad4a33SUladzislau Rezki (Sony) *parent = &tmp_va->rb_node; 104168ad4a33SUladzislau Rezki (Sony) return link; 1042db64fe02SNick Piggin } 1043db64fe02SNick Piggin 104468ad4a33SUladzislau Rezki (Sony) static __always_inline struct list_head * 104568ad4a33SUladzislau Rezki (Sony) get_va_next_sibling(struct rb_node *parent, struct rb_node **link) 104668ad4a33SUladzislau Rezki (Sony) { 104768ad4a33SUladzislau Rezki (Sony) struct list_head *list; 1048db64fe02SNick Piggin 104968ad4a33SUladzislau Rezki (Sony) if (unlikely(!parent)) 105068ad4a33SUladzislau Rezki (Sony) /* 105168ad4a33SUladzislau Rezki (Sony) * The red-black tree where we try to find VA neighbors 105268ad4a33SUladzislau Rezki (Sony) * before merging or inserting is empty, i.e. it means 105368ad4a33SUladzislau Rezki (Sony) * there is no free vmap space. Normally it does not 105468ad4a33SUladzislau Rezki (Sony) * happen but we handle this case anyway. 105568ad4a33SUladzislau Rezki (Sony) */ 105668ad4a33SUladzislau Rezki (Sony) return NULL; 105768ad4a33SUladzislau Rezki (Sony) 105868ad4a33SUladzislau Rezki (Sony) list = &rb_entry(parent, struct vmap_area, rb_node)->list; 105968ad4a33SUladzislau Rezki (Sony) return (&parent->rb_right == link ? list->next : list); 1060db64fe02SNick Piggin } 1061db64fe02SNick Piggin 106268ad4a33SUladzislau Rezki (Sony) static __always_inline void 10638eb510dbSUladzislau Rezki (Sony) __link_va(struct vmap_area *va, struct rb_root *root, 10648eb510dbSUladzislau Rezki (Sony) struct rb_node *parent, struct rb_node **link, 10658eb510dbSUladzislau Rezki (Sony) struct list_head *head, bool augment) 106668ad4a33SUladzislau Rezki (Sony) { 106768ad4a33SUladzislau Rezki (Sony) /* 106868ad4a33SUladzislau Rezki (Sony) * VA is still not in the list, but we can 106968ad4a33SUladzislau Rezki (Sony) * identify its future previous list_head node. 107068ad4a33SUladzislau Rezki (Sony) */ 107168ad4a33SUladzislau Rezki (Sony) if (likely(parent)) { 107268ad4a33SUladzislau Rezki (Sony) head = &rb_entry(parent, struct vmap_area, rb_node)->list; 107368ad4a33SUladzislau Rezki (Sony) if (&parent->rb_right != link) 107468ad4a33SUladzislau Rezki (Sony) head = head->prev; 107568ad4a33SUladzislau Rezki (Sony) } 1076db64fe02SNick Piggin 107768ad4a33SUladzislau Rezki (Sony) /* Insert to the rb-tree */ 107868ad4a33SUladzislau Rezki (Sony) rb_link_node(&va->rb_node, parent, link); 10798eb510dbSUladzislau Rezki (Sony) if (augment) { 108068ad4a33SUladzislau Rezki (Sony) /* 108168ad4a33SUladzislau Rezki (Sony) * Some explanation here. Just perform simple insertion 108268ad4a33SUladzislau Rezki (Sony) * to the tree. We do not set va->subtree_max_size to 108368ad4a33SUladzislau Rezki (Sony) * its current size before calling rb_insert_augmented(). 1084153090f2SBaoquan He * It is because we populate the tree from the bottom 108568ad4a33SUladzislau Rezki (Sony) * to parent levels when the node _is_ in the tree. 108668ad4a33SUladzislau Rezki (Sony) * 108768ad4a33SUladzislau Rezki (Sony) * Therefore we set subtree_max_size to zero after insertion, 108868ad4a33SUladzislau Rezki (Sony) * to let __augment_tree_propagate_from() puts everything to 108968ad4a33SUladzislau Rezki (Sony) * the correct order later on. 109068ad4a33SUladzislau Rezki (Sony) */ 109168ad4a33SUladzislau Rezki (Sony) rb_insert_augmented(&va->rb_node, 109268ad4a33SUladzislau Rezki (Sony) root, &free_vmap_area_rb_augment_cb); 109368ad4a33SUladzislau Rezki (Sony) va->subtree_max_size = 0; 109468ad4a33SUladzislau Rezki (Sony) } else { 109568ad4a33SUladzislau Rezki (Sony) rb_insert_color(&va->rb_node, root); 109668ad4a33SUladzislau Rezki (Sony) } 109768ad4a33SUladzislau Rezki (Sony) 109868ad4a33SUladzislau Rezki (Sony) /* Address-sort this list */ 109968ad4a33SUladzislau Rezki (Sony) list_add(&va->list, head); 110068ad4a33SUladzislau Rezki (Sony) } 110168ad4a33SUladzislau Rezki (Sony) 110268ad4a33SUladzislau Rezki (Sony) static __always_inline void 11038eb510dbSUladzislau Rezki (Sony) link_va(struct vmap_area *va, struct rb_root *root, 11048eb510dbSUladzislau Rezki (Sony) struct rb_node *parent, struct rb_node **link, 11058eb510dbSUladzislau Rezki (Sony) struct list_head *head) 11068eb510dbSUladzislau Rezki (Sony) { 11078eb510dbSUladzislau Rezki (Sony) __link_va(va, root, parent, link, head, false); 11088eb510dbSUladzislau Rezki (Sony) } 11098eb510dbSUladzislau Rezki (Sony) 11108eb510dbSUladzislau Rezki (Sony) static __always_inline void 11118eb510dbSUladzislau Rezki (Sony) link_va_augment(struct vmap_area *va, struct rb_root *root, 11128eb510dbSUladzislau Rezki (Sony) struct rb_node *parent, struct rb_node **link, 11138eb510dbSUladzislau Rezki (Sony) struct list_head *head) 11148eb510dbSUladzislau Rezki (Sony) { 11158eb510dbSUladzislau Rezki (Sony) __link_va(va, root, parent, link, head, true); 11168eb510dbSUladzislau Rezki (Sony) } 11178eb510dbSUladzislau Rezki (Sony) 11188eb510dbSUladzislau Rezki (Sony) static __always_inline void 11198eb510dbSUladzislau Rezki (Sony) __unlink_va(struct vmap_area *va, struct rb_root *root, bool augment) 112068ad4a33SUladzislau Rezki (Sony) { 1121460e42d1SUladzislau Rezki (Sony) if (WARN_ON(RB_EMPTY_NODE(&va->rb_node))) 1122460e42d1SUladzislau Rezki (Sony) return; 1123460e42d1SUladzislau Rezki (Sony) 11248eb510dbSUladzislau Rezki (Sony) if (augment) 112568ad4a33SUladzislau Rezki (Sony) rb_erase_augmented(&va->rb_node, 112668ad4a33SUladzislau Rezki (Sony) root, &free_vmap_area_rb_augment_cb); 112768ad4a33SUladzislau Rezki (Sony) else 112868ad4a33SUladzislau Rezki (Sony) rb_erase(&va->rb_node, root); 112968ad4a33SUladzislau Rezki (Sony) 11305d7a7c54SUladzislau Rezki (Sony) list_del_init(&va->list); 113168ad4a33SUladzislau Rezki (Sony) RB_CLEAR_NODE(&va->rb_node); 113268ad4a33SUladzislau Rezki (Sony) } 113368ad4a33SUladzislau Rezki (Sony) 11348eb510dbSUladzislau Rezki (Sony) static __always_inline void 11358eb510dbSUladzislau Rezki (Sony) unlink_va(struct vmap_area *va, struct rb_root *root) 11368eb510dbSUladzislau Rezki (Sony) { 11378eb510dbSUladzislau Rezki (Sony) __unlink_va(va, root, false); 11388eb510dbSUladzislau Rezki (Sony) } 11398eb510dbSUladzislau Rezki (Sony) 11408eb510dbSUladzislau Rezki (Sony) static __always_inline void 11418eb510dbSUladzislau Rezki (Sony) unlink_va_augment(struct vmap_area *va, struct rb_root *root) 11428eb510dbSUladzislau Rezki (Sony) { 11438eb510dbSUladzislau Rezki (Sony) __unlink_va(va, root, true); 11448eb510dbSUladzislau Rezki (Sony) } 11458eb510dbSUladzislau Rezki (Sony) 1146bb850f4dSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_PROPAGATE_CHECK 1147c3385e84SJiapeng Chong /* 1148c3385e84SJiapeng Chong * Gets called when remove the node and rotate. 1149c3385e84SJiapeng Chong */ 1150c3385e84SJiapeng Chong static __always_inline unsigned long 1151c3385e84SJiapeng Chong compute_subtree_max_size(struct vmap_area *va) 1152c3385e84SJiapeng Chong { 1153c3385e84SJiapeng Chong return max3(va_size(va), 1154c3385e84SJiapeng Chong get_subtree_max_size(va->rb_node.rb_left), 1155c3385e84SJiapeng Chong get_subtree_max_size(va->rb_node.rb_right)); 1156c3385e84SJiapeng Chong } 1157c3385e84SJiapeng Chong 1158bb850f4dSUladzislau Rezki (Sony) static void 1159da27c9edSUladzislau Rezki (Sony) augment_tree_propagate_check(void) 1160bb850f4dSUladzislau Rezki (Sony) { 1161bb850f4dSUladzislau Rezki (Sony) struct vmap_area *va; 1162da27c9edSUladzislau Rezki (Sony) unsigned long computed_size; 1163bb850f4dSUladzislau Rezki (Sony) 1164da27c9edSUladzislau Rezki (Sony) list_for_each_entry(va, &free_vmap_area_list, list) { 1165da27c9edSUladzislau Rezki (Sony) computed_size = compute_subtree_max_size(va); 1166da27c9edSUladzislau Rezki (Sony) if (computed_size != va->subtree_max_size) 1167bb850f4dSUladzislau Rezki (Sony) pr_emerg("tree is corrupted: %lu, %lu\n", 1168bb850f4dSUladzislau Rezki (Sony) va_size(va), va->subtree_max_size); 1169bb850f4dSUladzislau Rezki (Sony) } 1170bb850f4dSUladzislau Rezki (Sony) } 1171bb850f4dSUladzislau Rezki (Sony) #endif 1172bb850f4dSUladzislau Rezki (Sony) 117368ad4a33SUladzislau Rezki (Sony) /* 117468ad4a33SUladzislau Rezki (Sony) * This function populates subtree_max_size from bottom to upper 117568ad4a33SUladzislau Rezki (Sony) * levels starting from VA point. The propagation must be done 117668ad4a33SUladzislau Rezki (Sony) * when VA size is modified by changing its va_start/va_end. Or 117768ad4a33SUladzislau Rezki (Sony) * in case of newly inserting of VA to the tree. 117868ad4a33SUladzislau Rezki (Sony) * 117968ad4a33SUladzislau Rezki (Sony) * It means that __augment_tree_propagate_from() must be called: 118068ad4a33SUladzislau Rezki (Sony) * - After VA has been inserted to the tree(free path); 118168ad4a33SUladzislau Rezki (Sony) * - After VA has been shrunk(allocation path); 118268ad4a33SUladzislau Rezki (Sony) * - After VA has been increased(merging path). 118368ad4a33SUladzislau Rezki (Sony) * 118468ad4a33SUladzislau Rezki (Sony) * Please note that, it does not mean that upper parent nodes 118568ad4a33SUladzislau Rezki (Sony) * and their subtree_max_size are recalculated all the time up 118668ad4a33SUladzislau Rezki (Sony) * to the root node. 118768ad4a33SUladzislau Rezki (Sony) * 118868ad4a33SUladzislau Rezki (Sony) * 4--8 118968ad4a33SUladzislau Rezki (Sony) * /\ 119068ad4a33SUladzislau Rezki (Sony) * / \ 119168ad4a33SUladzislau Rezki (Sony) * / \ 119268ad4a33SUladzislau Rezki (Sony) * 2--2 8--8 119368ad4a33SUladzislau Rezki (Sony) * 119468ad4a33SUladzislau Rezki (Sony) * For example if we modify the node 4, shrinking it to 2, then 119568ad4a33SUladzislau Rezki (Sony) * no any modification is required. If we shrink the node 2 to 1 119668ad4a33SUladzislau Rezki (Sony) * its subtree_max_size is updated only, and set to 1. If we shrink 119768ad4a33SUladzislau Rezki (Sony) * the node 8 to 6, then its subtree_max_size is set to 6 and parent 119868ad4a33SUladzislau Rezki (Sony) * node becomes 4--6. 119968ad4a33SUladzislau Rezki (Sony) */ 120068ad4a33SUladzislau Rezki (Sony) static __always_inline void 120168ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(struct vmap_area *va) 120268ad4a33SUladzislau Rezki (Sony) { 120368ad4a33SUladzislau Rezki (Sony) /* 120415ae144fSUladzislau Rezki (Sony) * Populate the tree from bottom towards the root until 120515ae144fSUladzislau Rezki (Sony) * the calculated maximum available size of checked node 120615ae144fSUladzislau Rezki (Sony) * is equal to its current one. 120768ad4a33SUladzislau Rezki (Sony) */ 120815ae144fSUladzislau Rezki (Sony) free_vmap_area_rb_augment_cb_propagate(&va->rb_node, NULL); 1209bb850f4dSUladzislau Rezki (Sony) 1210bb850f4dSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_PROPAGATE_CHECK 1211da27c9edSUladzislau Rezki (Sony) augment_tree_propagate_check(); 1212bb850f4dSUladzislau Rezki (Sony) #endif 121368ad4a33SUladzislau Rezki (Sony) } 121468ad4a33SUladzislau Rezki (Sony) 121568ad4a33SUladzislau Rezki (Sony) static void 121668ad4a33SUladzislau Rezki (Sony) insert_vmap_area(struct vmap_area *va, 121768ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 121868ad4a33SUladzislau Rezki (Sony) { 121968ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 122068ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 122168ad4a33SUladzislau Rezki (Sony) 122268ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 12239c801f61SUladzislau Rezki (Sony) if (link) 122468ad4a33SUladzislau Rezki (Sony) link_va(va, root, parent, link, head); 122568ad4a33SUladzislau Rezki (Sony) } 122668ad4a33SUladzislau Rezki (Sony) 122768ad4a33SUladzislau Rezki (Sony) static void 122868ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(struct vmap_area *va, 122968ad4a33SUladzislau Rezki (Sony) struct rb_node *from, struct rb_root *root, 123068ad4a33SUladzislau Rezki (Sony) struct list_head *head) 123168ad4a33SUladzislau Rezki (Sony) { 123268ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 123368ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 123468ad4a33SUladzislau Rezki (Sony) 123568ad4a33SUladzislau Rezki (Sony) if (from) 123668ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, NULL, from, &parent); 123768ad4a33SUladzislau Rezki (Sony) else 123868ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 123968ad4a33SUladzislau Rezki (Sony) 12409c801f61SUladzislau Rezki (Sony) if (link) { 12418eb510dbSUladzislau Rezki (Sony) link_va_augment(va, root, parent, link, head); 124268ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 124368ad4a33SUladzislau Rezki (Sony) } 12449c801f61SUladzislau Rezki (Sony) } 124568ad4a33SUladzislau Rezki (Sony) 124668ad4a33SUladzislau Rezki (Sony) /* 124768ad4a33SUladzislau Rezki (Sony) * Merge de-allocated chunk of VA memory with previous 124868ad4a33SUladzislau Rezki (Sony) * and next free blocks. If coalesce is not done a new 124968ad4a33SUladzislau Rezki (Sony) * free area is inserted. If VA has been merged, it is 125068ad4a33SUladzislau Rezki (Sony) * freed. 12519c801f61SUladzislau Rezki (Sony) * 12529c801f61SUladzislau Rezki (Sony) * Please note, it can return NULL in case of overlap 12539c801f61SUladzislau Rezki (Sony) * ranges, followed by WARN() report. Despite it is a 12549c801f61SUladzislau Rezki (Sony) * buggy behaviour, a system can be alive and keep 12559c801f61SUladzislau Rezki (Sony) * ongoing. 125668ad4a33SUladzislau Rezki (Sony) */ 12573c5c3cfbSDaniel Axtens static __always_inline struct vmap_area * 12588eb510dbSUladzislau Rezki (Sony) __merge_or_add_vmap_area(struct vmap_area *va, 12598eb510dbSUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head, bool augment) 126068ad4a33SUladzislau Rezki (Sony) { 126168ad4a33SUladzislau Rezki (Sony) struct vmap_area *sibling; 126268ad4a33SUladzislau Rezki (Sony) struct list_head *next; 126368ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 126468ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 126568ad4a33SUladzislau Rezki (Sony) bool merged = false; 126668ad4a33SUladzislau Rezki (Sony) 126768ad4a33SUladzislau Rezki (Sony) /* 126868ad4a33SUladzislau Rezki (Sony) * Find a place in the tree where VA potentially will be 126968ad4a33SUladzislau Rezki (Sony) * inserted, unless it is merged with its sibling/siblings. 127068ad4a33SUladzislau Rezki (Sony) */ 127168ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 12729c801f61SUladzislau Rezki (Sony) if (!link) 12739c801f61SUladzislau Rezki (Sony) return NULL; 127468ad4a33SUladzislau Rezki (Sony) 127568ad4a33SUladzislau Rezki (Sony) /* 127668ad4a33SUladzislau Rezki (Sony) * Get next node of VA to check if merging can be done. 127768ad4a33SUladzislau Rezki (Sony) */ 127868ad4a33SUladzislau Rezki (Sony) next = get_va_next_sibling(parent, link); 127968ad4a33SUladzislau Rezki (Sony) if (unlikely(next == NULL)) 128068ad4a33SUladzislau Rezki (Sony) goto insert; 128168ad4a33SUladzislau Rezki (Sony) 128268ad4a33SUladzislau Rezki (Sony) /* 128368ad4a33SUladzislau Rezki (Sony) * start end 128468ad4a33SUladzislau Rezki (Sony) * | | 128568ad4a33SUladzislau Rezki (Sony) * |<------VA------>|<-----Next----->| 128668ad4a33SUladzislau Rezki (Sony) * | | 128768ad4a33SUladzislau Rezki (Sony) * start end 128868ad4a33SUladzislau Rezki (Sony) */ 128968ad4a33SUladzislau Rezki (Sony) if (next != head) { 129068ad4a33SUladzislau Rezki (Sony) sibling = list_entry(next, struct vmap_area, list); 129168ad4a33SUladzislau Rezki (Sony) if (sibling->va_start == va->va_end) { 129268ad4a33SUladzislau Rezki (Sony) sibling->va_start = va->va_start; 129368ad4a33SUladzislau Rezki (Sony) 129468ad4a33SUladzislau Rezki (Sony) /* Free vmap_area object. */ 129568ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 129668ad4a33SUladzislau Rezki (Sony) 129768ad4a33SUladzislau Rezki (Sony) /* Point to the new merged area. */ 129868ad4a33SUladzislau Rezki (Sony) va = sibling; 129968ad4a33SUladzislau Rezki (Sony) merged = true; 130068ad4a33SUladzislau Rezki (Sony) } 130168ad4a33SUladzislau Rezki (Sony) } 130268ad4a33SUladzislau Rezki (Sony) 130368ad4a33SUladzislau Rezki (Sony) /* 130468ad4a33SUladzislau Rezki (Sony) * start end 130568ad4a33SUladzislau Rezki (Sony) * | | 130668ad4a33SUladzislau Rezki (Sony) * |<-----Prev----->|<------VA------>| 130768ad4a33SUladzislau Rezki (Sony) * | | 130868ad4a33SUladzislau Rezki (Sony) * start end 130968ad4a33SUladzislau Rezki (Sony) */ 131068ad4a33SUladzislau Rezki (Sony) if (next->prev != head) { 131168ad4a33SUladzislau Rezki (Sony) sibling = list_entry(next->prev, struct vmap_area, list); 131268ad4a33SUladzislau Rezki (Sony) if (sibling->va_end == va->va_start) { 13135dd78640SUladzislau Rezki (Sony) /* 13145dd78640SUladzislau Rezki (Sony) * If both neighbors are coalesced, it is important 13155dd78640SUladzislau Rezki (Sony) * to unlink the "next" node first, followed by merging 13165dd78640SUladzislau Rezki (Sony) * with "previous" one. Otherwise the tree might not be 13175dd78640SUladzislau Rezki (Sony) * fully populated if a sibling's augmented value is 13185dd78640SUladzislau Rezki (Sony) * "normalized" because of rotation operations. 13195dd78640SUladzislau Rezki (Sony) */ 132054f63d9dSUladzislau Rezki (Sony) if (merged) 13218eb510dbSUladzislau Rezki (Sony) __unlink_va(va, root, augment); 132268ad4a33SUladzislau Rezki (Sony) 13235dd78640SUladzislau Rezki (Sony) sibling->va_end = va->va_end; 13245dd78640SUladzislau Rezki (Sony) 132568ad4a33SUladzislau Rezki (Sony) /* Free vmap_area object. */ 132668ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 13273c5c3cfbSDaniel Axtens 13283c5c3cfbSDaniel Axtens /* Point to the new merged area. */ 13293c5c3cfbSDaniel Axtens va = sibling; 13303c5c3cfbSDaniel Axtens merged = true; 133168ad4a33SUladzislau Rezki (Sony) } 133268ad4a33SUladzislau Rezki (Sony) } 133368ad4a33SUladzislau Rezki (Sony) 133468ad4a33SUladzislau Rezki (Sony) insert: 13355dd78640SUladzislau Rezki (Sony) if (!merged) 13368eb510dbSUladzislau Rezki (Sony) __link_va(va, root, parent, link, head, augment); 13373c5c3cfbSDaniel Axtens 133896e2db45SUladzislau Rezki (Sony) return va; 133996e2db45SUladzislau Rezki (Sony) } 134096e2db45SUladzislau Rezki (Sony) 134196e2db45SUladzislau Rezki (Sony) static __always_inline struct vmap_area * 13428eb510dbSUladzislau Rezki (Sony) merge_or_add_vmap_area(struct vmap_area *va, 13438eb510dbSUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 13448eb510dbSUladzislau Rezki (Sony) { 13458eb510dbSUladzislau Rezki (Sony) return __merge_or_add_vmap_area(va, root, head, false); 13468eb510dbSUladzislau Rezki (Sony) } 13478eb510dbSUladzislau Rezki (Sony) 13488eb510dbSUladzislau Rezki (Sony) static __always_inline struct vmap_area * 134996e2db45SUladzislau Rezki (Sony) merge_or_add_vmap_area_augment(struct vmap_area *va, 135096e2db45SUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 135196e2db45SUladzislau Rezki (Sony) { 13528eb510dbSUladzislau Rezki (Sony) va = __merge_or_add_vmap_area(va, root, head, true); 135396e2db45SUladzislau Rezki (Sony) if (va) 13545dd78640SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 135596e2db45SUladzislau Rezki (Sony) 13563c5c3cfbSDaniel Axtens return va; 135768ad4a33SUladzislau Rezki (Sony) } 135868ad4a33SUladzislau Rezki (Sony) 135968ad4a33SUladzislau Rezki (Sony) static __always_inline bool 136068ad4a33SUladzislau Rezki (Sony) is_within_this_va(struct vmap_area *va, unsigned long size, 136168ad4a33SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 136268ad4a33SUladzislau Rezki (Sony) { 136368ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr; 136468ad4a33SUladzislau Rezki (Sony) 136568ad4a33SUladzislau Rezki (Sony) if (va->va_start > vstart) 136668ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(va->va_start, align); 136768ad4a33SUladzislau Rezki (Sony) else 136868ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(vstart, align); 136968ad4a33SUladzislau Rezki (Sony) 137068ad4a33SUladzislau Rezki (Sony) /* Can be overflowed due to big size or alignment. */ 137168ad4a33SUladzislau Rezki (Sony) if (nva_start_addr + size < nva_start_addr || 137268ad4a33SUladzislau Rezki (Sony) nva_start_addr < vstart) 137368ad4a33SUladzislau Rezki (Sony) return false; 137468ad4a33SUladzislau Rezki (Sony) 137568ad4a33SUladzislau Rezki (Sony) return (nva_start_addr + size <= va->va_end); 137668ad4a33SUladzislau Rezki (Sony) } 137768ad4a33SUladzislau Rezki (Sony) 137868ad4a33SUladzislau Rezki (Sony) /* 137968ad4a33SUladzislau Rezki (Sony) * Find the first free block(lowest start address) in the tree, 138068ad4a33SUladzislau Rezki (Sony) * that will accomplish the request corresponding to passing 13819333fe98SUladzislau Rezki * parameters. Please note, with an alignment bigger than PAGE_SIZE, 13829333fe98SUladzislau Rezki * a search length is adjusted to account for worst case alignment 13839333fe98SUladzislau Rezki * overhead. 138468ad4a33SUladzislau Rezki (Sony) */ 138568ad4a33SUladzislau Rezki (Sony) static __always_inline struct vmap_area * 1386f9863be4SUladzislau Rezki (Sony) find_vmap_lowest_match(struct rb_root *root, unsigned long size, 1387f9863be4SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart, bool adjust_search_size) 138868ad4a33SUladzislau Rezki (Sony) { 138968ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 139068ad4a33SUladzislau Rezki (Sony) struct rb_node *node; 13919333fe98SUladzislau Rezki unsigned long length; 139268ad4a33SUladzislau Rezki (Sony) 139368ad4a33SUladzislau Rezki (Sony) /* Start from the root. */ 1394f9863be4SUladzislau Rezki (Sony) node = root->rb_node; 139568ad4a33SUladzislau Rezki (Sony) 13969333fe98SUladzislau Rezki /* Adjust the search size for alignment overhead. */ 13979333fe98SUladzislau Rezki length = adjust_search_size ? size + align - 1 : size; 13989333fe98SUladzislau Rezki 139968ad4a33SUladzislau Rezki (Sony) while (node) { 140068ad4a33SUladzislau Rezki (Sony) va = rb_entry(node, struct vmap_area, rb_node); 140168ad4a33SUladzislau Rezki (Sony) 14029333fe98SUladzislau Rezki if (get_subtree_max_size(node->rb_left) >= length && 140368ad4a33SUladzislau Rezki (Sony) vstart < va->va_start) { 140468ad4a33SUladzislau Rezki (Sony) node = node->rb_left; 140568ad4a33SUladzislau Rezki (Sony) } else { 140668ad4a33SUladzislau Rezki (Sony) if (is_within_this_va(va, size, align, vstart)) 140768ad4a33SUladzislau Rezki (Sony) return va; 140868ad4a33SUladzislau Rezki (Sony) 140968ad4a33SUladzislau Rezki (Sony) /* 141068ad4a33SUladzislau Rezki (Sony) * Does not make sense to go deeper towards the right 141168ad4a33SUladzislau Rezki (Sony) * sub-tree if it does not have a free block that is 14129333fe98SUladzislau Rezki * equal or bigger to the requested search length. 141368ad4a33SUladzislau Rezki (Sony) */ 14149333fe98SUladzislau Rezki if (get_subtree_max_size(node->rb_right) >= length) { 141568ad4a33SUladzislau Rezki (Sony) node = node->rb_right; 141668ad4a33SUladzislau Rezki (Sony) continue; 141768ad4a33SUladzislau Rezki (Sony) } 141868ad4a33SUladzislau Rezki (Sony) 141968ad4a33SUladzislau Rezki (Sony) /* 14203806b041SAndrew Morton * OK. We roll back and find the first right sub-tree, 142168ad4a33SUladzislau Rezki (Sony) * that will satisfy the search criteria. It can happen 14229f531973SUladzislau Rezki (Sony) * due to "vstart" restriction or an alignment overhead 14239f531973SUladzislau Rezki (Sony) * that is bigger then PAGE_SIZE. 142468ad4a33SUladzislau Rezki (Sony) */ 142568ad4a33SUladzislau Rezki (Sony) while ((node = rb_parent(node))) { 142668ad4a33SUladzislau Rezki (Sony) va = rb_entry(node, struct vmap_area, rb_node); 142768ad4a33SUladzislau Rezki (Sony) if (is_within_this_va(va, size, align, vstart)) 142868ad4a33SUladzislau Rezki (Sony) return va; 142968ad4a33SUladzislau Rezki (Sony) 14309333fe98SUladzislau Rezki if (get_subtree_max_size(node->rb_right) >= length && 143168ad4a33SUladzislau Rezki (Sony) vstart <= va->va_start) { 14329f531973SUladzislau Rezki (Sony) /* 14339f531973SUladzislau Rezki (Sony) * Shift the vstart forward. Please note, we update it with 14349f531973SUladzislau Rezki (Sony) * parent's start address adding "1" because we do not want 14359f531973SUladzislau Rezki (Sony) * to enter same sub-tree after it has already been checked 14369f531973SUladzislau Rezki (Sony) * and no suitable free block found there. 14379f531973SUladzislau Rezki (Sony) */ 14389f531973SUladzislau Rezki (Sony) vstart = va->va_start + 1; 143968ad4a33SUladzislau Rezki (Sony) node = node->rb_right; 144068ad4a33SUladzislau Rezki (Sony) break; 144168ad4a33SUladzislau Rezki (Sony) } 144268ad4a33SUladzislau Rezki (Sony) } 144368ad4a33SUladzislau Rezki (Sony) } 144468ad4a33SUladzislau Rezki (Sony) } 144568ad4a33SUladzislau Rezki (Sony) 144668ad4a33SUladzislau Rezki (Sony) return NULL; 144768ad4a33SUladzislau Rezki (Sony) } 144868ad4a33SUladzislau Rezki (Sony) 1449a6cf4e0fSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 1450a6cf4e0fSUladzislau Rezki (Sony) #include <linux/random.h> 1451a6cf4e0fSUladzislau Rezki (Sony) 1452a6cf4e0fSUladzislau Rezki (Sony) static struct vmap_area * 1453bd1264c3SSong Liu find_vmap_lowest_linear_match(struct list_head *head, unsigned long size, 1454a6cf4e0fSUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 1455a6cf4e0fSUladzislau Rezki (Sony) { 1456a6cf4e0fSUladzislau Rezki (Sony) struct vmap_area *va; 1457a6cf4e0fSUladzislau Rezki (Sony) 1458bd1264c3SSong Liu list_for_each_entry(va, head, list) { 1459a6cf4e0fSUladzislau Rezki (Sony) if (!is_within_this_va(va, size, align, vstart)) 1460a6cf4e0fSUladzislau Rezki (Sony) continue; 1461a6cf4e0fSUladzislau Rezki (Sony) 1462a6cf4e0fSUladzislau Rezki (Sony) return va; 1463a6cf4e0fSUladzislau Rezki (Sony) } 1464a6cf4e0fSUladzislau Rezki (Sony) 1465a6cf4e0fSUladzislau Rezki (Sony) return NULL; 1466a6cf4e0fSUladzislau Rezki (Sony) } 1467a6cf4e0fSUladzislau Rezki (Sony) 1468a6cf4e0fSUladzislau Rezki (Sony) static void 1469bd1264c3SSong Liu find_vmap_lowest_match_check(struct rb_root *root, struct list_head *head, 1470bd1264c3SSong Liu unsigned long size, unsigned long align) 1471a6cf4e0fSUladzislau Rezki (Sony) { 1472a6cf4e0fSUladzislau Rezki (Sony) struct vmap_area *va_1, *va_2; 1473a6cf4e0fSUladzislau Rezki (Sony) unsigned long vstart; 1474a6cf4e0fSUladzislau Rezki (Sony) unsigned int rnd; 1475a6cf4e0fSUladzislau Rezki (Sony) 1476a6cf4e0fSUladzislau Rezki (Sony) get_random_bytes(&rnd, sizeof(rnd)); 1477a6cf4e0fSUladzislau Rezki (Sony) vstart = VMALLOC_START + rnd; 1478a6cf4e0fSUladzislau Rezki (Sony) 1479bd1264c3SSong Liu va_1 = find_vmap_lowest_match(root, size, align, vstart, false); 1480bd1264c3SSong Liu va_2 = find_vmap_lowest_linear_match(head, size, align, vstart); 1481a6cf4e0fSUladzislau Rezki (Sony) 1482a6cf4e0fSUladzislau Rezki (Sony) if (va_1 != va_2) 1483a6cf4e0fSUladzislau Rezki (Sony) pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n", 1484a6cf4e0fSUladzislau Rezki (Sony) va_1, va_2, vstart); 1485a6cf4e0fSUladzislau Rezki (Sony) } 1486a6cf4e0fSUladzislau Rezki (Sony) #endif 1487a6cf4e0fSUladzislau Rezki (Sony) 148868ad4a33SUladzislau Rezki (Sony) enum fit_type { 148968ad4a33SUladzislau Rezki (Sony) NOTHING_FIT = 0, 149068ad4a33SUladzislau Rezki (Sony) FL_FIT_TYPE = 1, /* full fit */ 149168ad4a33SUladzislau Rezki (Sony) LE_FIT_TYPE = 2, /* left edge fit */ 149268ad4a33SUladzislau Rezki (Sony) RE_FIT_TYPE = 3, /* right edge fit */ 149368ad4a33SUladzislau Rezki (Sony) NE_FIT_TYPE = 4 /* no edge fit */ 149468ad4a33SUladzislau Rezki (Sony) }; 149568ad4a33SUladzislau Rezki (Sony) 149668ad4a33SUladzislau Rezki (Sony) static __always_inline enum fit_type 149768ad4a33SUladzislau Rezki (Sony) classify_va_fit_type(struct vmap_area *va, 149868ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr, unsigned long size) 149968ad4a33SUladzislau Rezki (Sony) { 150068ad4a33SUladzislau Rezki (Sony) enum fit_type type; 150168ad4a33SUladzislau Rezki (Sony) 150268ad4a33SUladzislau Rezki (Sony) /* Check if it is within VA. */ 150368ad4a33SUladzislau Rezki (Sony) if (nva_start_addr < va->va_start || 150468ad4a33SUladzislau Rezki (Sony) nva_start_addr + size > va->va_end) 150568ad4a33SUladzislau Rezki (Sony) return NOTHING_FIT; 150668ad4a33SUladzislau Rezki (Sony) 150768ad4a33SUladzislau Rezki (Sony) /* Now classify. */ 150868ad4a33SUladzislau Rezki (Sony) if (va->va_start == nva_start_addr) { 150968ad4a33SUladzislau Rezki (Sony) if (va->va_end == nva_start_addr + size) 151068ad4a33SUladzislau Rezki (Sony) type = FL_FIT_TYPE; 151168ad4a33SUladzislau Rezki (Sony) else 151268ad4a33SUladzislau Rezki (Sony) type = LE_FIT_TYPE; 151368ad4a33SUladzislau Rezki (Sony) } else if (va->va_end == nva_start_addr + size) { 151468ad4a33SUladzislau Rezki (Sony) type = RE_FIT_TYPE; 151568ad4a33SUladzislau Rezki (Sony) } else { 151668ad4a33SUladzislau Rezki (Sony) type = NE_FIT_TYPE; 151768ad4a33SUladzislau Rezki (Sony) } 151868ad4a33SUladzislau Rezki (Sony) 151968ad4a33SUladzislau Rezki (Sony) return type; 152068ad4a33SUladzislau Rezki (Sony) } 152168ad4a33SUladzislau Rezki (Sony) 152268ad4a33SUladzislau Rezki (Sony) static __always_inline int 15235b75b8e1SUladzislau Rezki (Sony) va_clip(struct rb_root *root, struct list_head *head, 1524f9863be4SUladzislau Rezki (Sony) struct vmap_area *va, unsigned long nva_start_addr, 1525f9863be4SUladzislau Rezki (Sony) unsigned long size) 152668ad4a33SUladzislau Rezki (Sony) { 15272c929233SArnd Bergmann struct vmap_area *lva = NULL; 15281b23ff80SBaoquan He enum fit_type type = classify_va_fit_type(va, nva_start_addr, size); 152968ad4a33SUladzislau Rezki (Sony) 153068ad4a33SUladzislau Rezki (Sony) if (type == FL_FIT_TYPE) { 153168ad4a33SUladzislau Rezki (Sony) /* 153268ad4a33SUladzislau Rezki (Sony) * No need to split VA, it fully fits. 153368ad4a33SUladzislau Rezki (Sony) * 153468ad4a33SUladzislau Rezki (Sony) * | | 153568ad4a33SUladzislau Rezki (Sony) * V NVA V 153668ad4a33SUladzislau Rezki (Sony) * |---------------| 153768ad4a33SUladzislau Rezki (Sony) */ 1538f9863be4SUladzislau Rezki (Sony) unlink_va_augment(va, root); 153968ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 154068ad4a33SUladzislau Rezki (Sony) } else if (type == LE_FIT_TYPE) { 154168ad4a33SUladzislau Rezki (Sony) /* 154268ad4a33SUladzislau Rezki (Sony) * Split left edge of fit VA. 154368ad4a33SUladzislau Rezki (Sony) * 154468ad4a33SUladzislau Rezki (Sony) * | | 154568ad4a33SUladzislau Rezki (Sony) * V NVA V R 154668ad4a33SUladzislau Rezki (Sony) * |-------|-------| 154768ad4a33SUladzislau Rezki (Sony) */ 154868ad4a33SUladzislau Rezki (Sony) va->va_start += size; 154968ad4a33SUladzislau Rezki (Sony) } else if (type == RE_FIT_TYPE) { 155068ad4a33SUladzislau Rezki (Sony) /* 155168ad4a33SUladzislau Rezki (Sony) * Split right edge of fit VA. 155268ad4a33SUladzislau Rezki (Sony) * 155368ad4a33SUladzislau Rezki (Sony) * | | 155468ad4a33SUladzislau Rezki (Sony) * L V NVA V 155568ad4a33SUladzislau Rezki (Sony) * |-------|-------| 155668ad4a33SUladzislau Rezki (Sony) */ 155768ad4a33SUladzislau Rezki (Sony) va->va_end = nva_start_addr; 155868ad4a33SUladzislau Rezki (Sony) } else if (type == NE_FIT_TYPE) { 155968ad4a33SUladzislau Rezki (Sony) /* 156068ad4a33SUladzislau Rezki (Sony) * Split no edge of fit VA. 156168ad4a33SUladzislau Rezki (Sony) * 156268ad4a33SUladzislau Rezki (Sony) * | | 156368ad4a33SUladzislau Rezki (Sony) * L V NVA V R 156468ad4a33SUladzislau Rezki (Sony) * |---|-------|---| 156568ad4a33SUladzislau Rezki (Sony) */ 156682dd23e8SUladzislau Rezki (Sony) lva = __this_cpu_xchg(ne_fit_preload_node, NULL); 156782dd23e8SUladzislau Rezki (Sony) if (unlikely(!lva)) { 156882dd23e8SUladzislau Rezki (Sony) /* 156982dd23e8SUladzislau Rezki (Sony) * For percpu allocator we do not do any pre-allocation 157082dd23e8SUladzislau Rezki (Sony) * and leave it as it is. The reason is it most likely 157182dd23e8SUladzislau Rezki (Sony) * never ends up with NE_FIT_TYPE splitting. In case of 157282dd23e8SUladzislau Rezki (Sony) * percpu allocations offsets and sizes are aligned to 157382dd23e8SUladzislau Rezki (Sony) * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE 157482dd23e8SUladzislau Rezki (Sony) * are its main fitting cases. 157582dd23e8SUladzislau Rezki (Sony) * 157682dd23e8SUladzislau Rezki (Sony) * There are a few exceptions though, as an example it is 157782dd23e8SUladzislau Rezki (Sony) * a first allocation (early boot up) when we have "one" 157882dd23e8SUladzislau Rezki (Sony) * big free space that has to be split. 1579060650a2SUladzislau Rezki (Sony) * 1580060650a2SUladzislau Rezki (Sony) * Also we can hit this path in case of regular "vmap" 1581060650a2SUladzislau Rezki (Sony) * allocations, if "this" current CPU was not preloaded. 1582060650a2SUladzislau Rezki (Sony) * See the comment in alloc_vmap_area() why. If so, then 1583060650a2SUladzislau Rezki (Sony) * GFP_NOWAIT is used instead to get an extra object for 1584060650a2SUladzislau Rezki (Sony) * split purpose. That is rare and most time does not 1585060650a2SUladzislau Rezki (Sony) * occur. 1586060650a2SUladzislau Rezki (Sony) * 1587060650a2SUladzislau Rezki (Sony) * What happens if an allocation gets failed. Basically, 1588060650a2SUladzislau Rezki (Sony) * an "overflow" path is triggered to purge lazily freed 1589060650a2SUladzislau Rezki (Sony) * areas to free some memory, then, the "retry" path is 1590060650a2SUladzislau Rezki (Sony) * triggered to repeat one more time. See more details 1591060650a2SUladzislau Rezki (Sony) * in alloc_vmap_area() function. 159282dd23e8SUladzislau Rezki (Sony) */ 159368ad4a33SUladzislau Rezki (Sony) lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT); 159482dd23e8SUladzislau Rezki (Sony) if (!lva) 159568ad4a33SUladzislau Rezki (Sony) return -1; 159682dd23e8SUladzislau Rezki (Sony) } 159768ad4a33SUladzislau Rezki (Sony) 159868ad4a33SUladzislau Rezki (Sony) /* 159968ad4a33SUladzislau Rezki (Sony) * Build the remainder. 160068ad4a33SUladzislau Rezki (Sony) */ 160168ad4a33SUladzislau Rezki (Sony) lva->va_start = va->va_start; 160268ad4a33SUladzislau Rezki (Sony) lva->va_end = nva_start_addr; 160368ad4a33SUladzislau Rezki (Sony) 160468ad4a33SUladzislau Rezki (Sony) /* 160568ad4a33SUladzislau Rezki (Sony) * Shrink this VA to remaining size. 160668ad4a33SUladzislau Rezki (Sony) */ 160768ad4a33SUladzislau Rezki (Sony) va->va_start = nva_start_addr + size; 160868ad4a33SUladzislau Rezki (Sony) } else { 160968ad4a33SUladzislau Rezki (Sony) return -1; 161068ad4a33SUladzislau Rezki (Sony) } 161168ad4a33SUladzislau Rezki (Sony) 161268ad4a33SUladzislau Rezki (Sony) if (type != FL_FIT_TYPE) { 161368ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 161468ad4a33SUladzislau Rezki (Sony) 16152c929233SArnd Bergmann if (lva) /* type == NE_FIT_TYPE */ 1616f9863be4SUladzislau Rezki (Sony) insert_vmap_area_augment(lva, &va->rb_node, root, head); 161768ad4a33SUladzislau Rezki (Sony) } 161868ad4a33SUladzislau Rezki (Sony) 161968ad4a33SUladzislau Rezki (Sony) return 0; 162068ad4a33SUladzislau Rezki (Sony) } 162168ad4a33SUladzislau Rezki (Sony) 162238f6b9afSUladzislau Rezki (Sony) static unsigned long 162338f6b9afSUladzislau Rezki (Sony) va_alloc(struct vmap_area *va, 162438f6b9afSUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head, 162538f6b9afSUladzislau Rezki (Sony) unsigned long size, unsigned long align, 162638f6b9afSUladzislau Rezki (Sony) unsigned long vstart, unsigned long vend) 162738f6b9afSUladzislau Rezki (Sony) { 162838f6b9afSUladzislau Rezki (Sony) unsigned long nva_start_addr; 162938f6b9afSUladzislau Rezki (Sony) int ret; 163038f6b9afSUladzislau Rezki (Sony) 163138f6b9afSUladzislau Rezki (Sony) if (va->va_start > vstart) 163238f6b9afSUladzislau Rezki (Sony) nva_start_addr = ALIGN(va->va_start, align); 163338f6b9afSUladzislau Rezki (Sony) else 163438f6b9afSUladzislau Rezki (Sony) nva_start_addr = ALIGN(vstart, align); 163538f6b9afSUladzislau Rezki (Sony) 163638f6b9afSUladzislau Rezki (Sony) /* Check the "vend" restriction. */ 163738f6b9afSUladzislau Rezki (Sony) if (nva_start_addr + size > vend) 163838f6b9afSUladzislau Rezki (Sony) return vend; 163938f6b9afSUladzislau Rezki (Sony) 164038f6b9afSUladzislau Rezki (Sony) /* Update the free vmap_area. */ 16415b75b8e1SUladzislau Rezki (Sony) ret = va_clip(root, head, va, nva_start_addr, size); 164238f6b9afSUladzislau Rezki (Sony) if (WARN_ON_ONCE(ret)) 164338f6b9afSUladzislau Rezki (Sony) return vend; 164438f6b9afSUladzislau Rezki (Sony) 164538f6b9afSUladzislau Rezki (Sony) return nva_start_addr; 164638f6b9afSUladzislau Rezki (Sony) } 164738f6b9afSUladzislau Rezki (Sony) 164868ad4a33SUladzislau Rezki (Sony) /* 164968ad4a33SUladzislau Rezki (Sony) * Returns a start address of the newly allocated area, if success. 165068ad4a33SUladzislau Rezki (Sony) * Otherwise a vend is returned that indicates failure. 165168ad4a33SUladzislau Rezki (Sony) */ 165268ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 1653f9863be4SUladzislau Rezki (Sony) __alloc_vmap_area(struct rb_root *root, struct list_head *head, 1654f9863be4SUladzislau Rezki (Sony) unsigned long size, unsigned long align, 1655cacca6baSUladzislau Rezki (Sony) unsigned long vstart, unsigned long vend) 165668ad4a33SUladzislau Rezki (Sony) { 16579333fe98SUladzislau Rezki bool adjust_search_size = true; 165868ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr; 165968ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 166068ad4a33SUladzislau Rezki (Sony) 16619333fe98SUladzislau Rezki /* 16629333fe98SUladzislau Rezki * Do not adjust when: 16639333fe98SUladzislau Rezki * a) align <= PAGE_SIZE, because it does not make any sense. 16649333fe98SUladzislau Rezki * All blocks(their start addresses) are at least PAGE_SIZE 16659333fe98SUladzislau Rezki * aligned anyway; 16669333fe98SUladzislau Rezki * b) a short range where a requested size corresponds to exactly 16679333fe98SUladzislau Rezki * specified [vstart:vend] interval and an alignment > PAGE_SIZE. 16689333fe98SUladzislau Rezki * With adjusted search length an allocation would not succeed. 16699333fe98SUladzislau Rezki */ 16709333fe98SUladzislau Rezki if (align <= PAGE_SIZE || (align > PAGE_SIZE && (vend - vstart) == size)) 16719333fe98SUladzislau Rezki adjust_search_size = false; 16729333fe98SUladzislau Rezki 1673f9863be4SUladzislau Rezki (Sony) va = find_vmap_lowest_match(root, size, align, vstart, adjust_search_size); 167468ad4a33SUladzislau Rezki (Sony) if (unlikely(!va)) 167568ad4a33SUladzislau Rezki (Sony) return vend; 167668ad4a33SUladzislau Rezki (Sony) 167738f6b9afSUladzislau Rezki (Sony) nva_start_addr = va_alloc(va, root, head, size, align, vstart, vend); 167838f6b9afSUladzislau Rezki (Sony) if (nva_start_addr == vend) 167968ad4a33SUladzislau Rezki (Sony) return vend; 168068ad4a33SUladzislau Rezki (Sony) 1681a6cf4e0fSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 1682bd1264c3SSong Liu find_vmap_lowest_match_check(root, head, size, align); 1683a6cf4e0fSUladzislau Rezki (Sony) #endif 1684a6cf4e0fSUladzislau Rezki (Sony) 168568ad4a33SUladzislau Rezki (Sony) return nva_start_addr; 168668ad4a33SUladzislau Rezki (Sony) } 16874da56b99SChris Wilson 1688db64fe02SNick Piggin /* 1689d98c9e83SAndrey Ryabinin * Free a region of KVA allocated by alloc_vmap_area 1690d98c9e83SAndrey Ryabinin */ 1691d98c9e83SAndrey Ryabinin static void free_vmap_area(struct vmap_area *va) 1692d98c9e83SAndrey Ryabinin { 1693d0936029SUladzislau Rezki (Sony) struct vmap_node *vn = addr_to_node(va->va_start); 1694d0936029SUladzislau Rezki (Sony) 1695d98c9e83SAndrey Ryabinin /* 1696d98c9e83SAndrey Ryabinin * Remove from the busy tree/list. 1697d98c9e83SAndrey Ryabinin */ 1698d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 1699d0936029SUladzislau Rezki (Sony) unlink_va(va, &vn->busy.root); 1700d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 1701d98c9e83SAndrey Ryabinin 1702d98c9e83SAndrey Ryabinin /* 1703d98c9e83SAndrey Ryabinin * Insert/Merge it back to the free tree/list. 1704d98c9e83SAndrey Ryabinin */ 1705d98c9e83SAndrey Ryabinin spin_lock(&free_vmap_area_lock); 170696e2db45SUladzislau Rezki (Sony) merge_or_add_vmap_area_augment(va, &free_vmap_area_root, &free_vmap_area_list); 1707d98c9e83SAndrey Ryabinin spin_unlock(&free_vmap_area_lock); 1708d98c9e83SAndrey Ryabinin } 1709d98c9e83SAndrey Ryabinin 1710187f8cc4SUladzislau Rezki (Sony) static inline void 1711187f8cc4SUladzislau Rezki (Sony) preload_this_cpu_lock(spinlock_t *lock, gfp_t gfp_mask, int node) 1712187f8cc4SUladzislau Rezki (Sony) { 1713187f8cc4SUladzislau Rezki (Sony) struct vmap_area *va = NULL; 1714187f8cc4SUladzislau Rezki (Sony) 1715187f8cc4SUladzislau Rezki (Sony) /* 1716187f8cc4SUladzislau Rezki (Sony) * Preload this CPU with one extra vmap_area object. It is used 1717187f8cc4SUladzislau Rezki (Sony) * when fit type of free area is NE_FIT_TYPE. It guarantees that 1718187f8cc4SUladzislau Rezki (Sony) * a CPU that does an allocation is preloaded. 1719187f8cc4SUladzislau Rezki (Sony) * 1720187f8cc4SUladzislau Rezki (Sony) * We do it in non-atomic context, thus it allows us to use more 1721187f8cc4SUladzislau Rezki (Sony) * permissive allocation masks to be more stable under low memory 1722187f8cc4SUladzislau Rezki (Sony) * condition and high memory pressure. 1723187f8cc4SUladzislau Rezki (Sony) */ 1724187f8cc4SUladzislau Rezki (Sony) if (!this_cpu_read(ne_fit_preload_node)) 1725187f8cc4SUladzislau Rezki (Sony) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 1726187f8cc4SUladzislau Rezki (Sony) 1727187f8cc4SUladzislau Rezki (Sony) spin_lock(lock); 1728187f8cc4SUladzislau Rezki (Sony) 1729187f8cc4SUladzislau Rezki (Sony) if (va && __this_cpu_cmpxchg(ne_fit_preload_node, NULL, va)) 1730187f8cc4SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 1731187f8cc4SUladzislau Rezki (Sony) } 1732187f8cc4SUladzislau Rezki (Sony) 173372210662SUladzislau Rezki (Sony) static struct vmap_pool * 173472210662SUladzislau Rezki (Sony) size_to_va_pool(struct vmap_node *vn, unsigned long size) 173572210662SUladzislau Rezki (Sony) { 173672210662SUladzislau Rezki (Sony) unsigned int idx = (size - 1) / PAGE_SIZE; 173772210662SUladzislau Rezki (Sony) 173872210662SUladzislau Rezki (Sony) if (idx < MAX_VA_SIZE_PAGES) 173972210662SUladzislau Rezki (Sony) return &vn->pool[idx]; 174072210662SUladzislau Rezki (Sony) 174172210662SUladzislau Rezki (Sony) return NULL; 174272210662SUladzislau Rezki (Sony) } 174372210662SUladzislau Rezki (Sony) 174472210662SUladzislau Rezki (Sony) static bool 174572210662SUladzislau Rezki (Sony) node_pool_add_va(struct vmap_node *n, struct vmap_area *va) 174672210662SUladzislau Rezki (Sony) { 174772210662SUladzislau Rezki (Sony) struct vmap_pool *vp; 174872210662SUladzislau Rezki (Sony) 174972210662SUladzislau Rezki (Sony) vp = size_to_va_pool(n, va_size(va)); 175072210662SUladzislau Rezki (Sony) if (!vp) 175172210662SUladzislau Rezki (Sony) return false; 175272210662SUladzislau Rezki (Sony) 175372210662SUladzislau Rezki (Sony) spin_lock(&n->pool_lock); 175472210662SUladzislau Rezki (Sony) list_add(&va->list, &vp->head); 175572210662SUladzislau Rezki (Sony) WRITE_ONCE(vp->len, vp->len + 1); 175672210662SUladzislau Rezki (Sony) spin_unlock(&n->pool_lock); 175772210662SUladzislau Rezki (Sony) 175872210662SUladzislau Rezki (Sony) return true; 175972210662SUladzislau Rezki (Sony) } 176072210662SUladzislau Rezki (Sony) 176172210662SUladzislau Rezki (Sony) static struct vmap_area * 176272210662SUladzislau Rezki (Sony) node_pool_del_va(struct vmap_node *vn, unsigned long size, 176372210662SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart, 176472210662SUladzislau Rezki (Sony) unsigned long vend) 176572210662SUladzislau Rezki (Sony) { 176672210662SUladzislau Rezki (Sony) struct vmap_area *va = NULL; 176772210662SUladzislau Rezki (Sony) struct vmap_pool *vp; 176872210662SUladzislau Rezki (Sony) int err = 0; 176972210662SUladzislau Rezki (Sony) 177072210662SUladzislau Rezki (Sony) vp = size_to_va_pool(vn, size); 177172210662SUladzislau Rezki (Sony) if (!vp || list_empty(&vp->head)) 177272210662SUladzislau Rezki (Sony) return NULL; 177372210662SUladzislau Rezki (Sony) 177472210662SUladzislau Rezki (Sony) spin_lock(&vn->pool_lock); 177572210662SUladzislau Rezki (Sony) if (!list_empty(&vp->head)) { 177672210662SUladzislau Rezki (Sony) va = list_first_entry(&vp->head, struct vmap_area, list); 177772210662SUladzislau Rezki (Sony) 177872210662SUladzislau Rezki (Sony) if (IS_ALIGNED(va->va_start, align)) { 177972210662SUladzislau Rezki (Sony) /* 178072210662SUladzislau Rezki (Sony) * Do some sanity check and emit a warning 178172210662SUladzislau Rezki (Sony) * if one of below checks detects an error. 178272210662SUladzislau Rezki (Sony) */ 178372210662SUladzislau Rezki (Sony) err |= (va_size(va) != size); 178472210662SUladzislau Rezki (Sony) err |= (va->va_start < vstart); 178572210662SUladzislau Rezki (Sony) err |= (va->va_end > vend); 178672210662SUladzislau Rezki (Sony) 178772210662SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(err)) { 178872210662SUladzislau Rezki (Sony) list_del_init(&va->list); 178972210662SUladzislau Rezki (Sony) WRITE_ONCE(vp->len, vp->len - 1); 179072210662SUladzislau Rezki (Sony) } else { 179172210662SUladzislau Rezki (Sony) va = NULL; 179272210662SUladzislau Rezki (Sony) } 179372210662SUladzislau Rezki (Sony) } else { 179472210662SUladzislau Rezki (Sony) list_move_tail(&va->list, &vp->head); 179572210662SUladzislau Rezki (Sony) va = NULL; 179672210662SUladzislau Rezki (Sony) } 179772210662SUladzislau Rezki (Sony) } 179872210662SUladzislau Rezki (Sony) spin_unlock(&vn->pool_lock); 179972210662SUladzislau Rezki (Sony) 180072210662SUladzislau Rezki (Sony) return va; 180172210662SUladzislau Rezki (Sony) } 180272210662SUladzislau Rezki (Sony) 180372210662SUladzislau Rezki (Sony) static struct vmap_area * 180472210662SUladzislau Rezki (Sony) node_alloc(unsigned long size, unsigned long align, 180572210662SUladzislau Rezki (Sony) unsigned long vstart, unsigned long vend, 180672210662SUladzislau Rezki (Sony) unsigned long *addr, unsigned int *vn_id) 180772210662SUladzislau Rezki (Sony) { 180872210662SUladzislau Rezki (Sony) struct vmap_area *va; 180972210662SUladzislau Rezki (Sony) 181072210662SUladzislau Rezki (Sony) *vn_id = 0; 181172210662SUladzislau Rezki (Sony) *addr = vend; 181272210662SUladzislau Rezki (Sony) 181372210662SUladzislau Rezki (Sony) /* 181472210662SUladzislau Rezki (Sony) * Fallback to a global heap if not vmalloc or there 181572210662SUladzislau Rezki (Sony) * is only one node. 181672210662SUladzislau Rezki (Sony) */ 181772210662SUladzislau Rezki (Sony) if (vstart != VMALLOC_START || vend != VMALLOC_END || 181872210662SUladzislau Rezki (Sony) nr_vmap_nodes == 1) 181972210662SUladzislau Rezki (Sony) return NULL; 182072210662SUladzislau Rezki (Sony) 182172210662SUladzislau Rezki (Sony) *vn_id = raw_smp_processor_id() % nr_vmap_nodes; 182272210662SUladzislau Rezki (Sony) va = node_pool_del_va(id_to_node(*vn_id), size, align, vstart, vend); 182372210662SUladzislau Rezki (Sony) *vn_id = encode_vn_id(*vn_id); 182472210662SUladzislau Rezki (Sony) 182572210662SUladzislau Rezki (Sony) if (va) 182672210662SUladzislau Rezki (Sony) *addr = va->va_start; 182772210662SUladzislau Rezki (Sony) 182872210662SUladzislau Rezki (Sony) return va; 182972210662SUladzislau Rezki (Sony) } 183072210662SUladzislau Rezki (Sony) 1831d98c9e83SAndrey Ryabinin /* 1832db64fe02SNick Piggin * Allocate a region of KVA of the specified size and alignment, within the 1833db64fe02SNick Piggin * vstart and vend. 1834db64fe02SNick Piggin */ 1835db64fe02SNick Piggin static struct vmap_area *alloc_vmap_area(unsigned long size, 1836db64fe02SNick Piggin unsigned long align, 1837db64fe02SNick Piggin unsigned long vstart, unsigned long vend, 1838869176a0SBaoquan He int node, gfp_t gfp_mask, 1839869176a0SBaoquan He unsigned long va_flags) 1840db64fe02SNick Piggin { 1841d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 1842187f8cc4SUladzislau Rezki (Sony) struct vmap_area *va; 184312e376a6SUladzislau Rezki (Sony) unsigned long freed; 18441da177e4SLinus Torvalds unsigned long addr; 184572210662SUladzislau Rezki (Sony) unsigned int vn_id; 1846db64fe02SNick Piggin int purged = 0; 1847d98c9e83SAndrey Ryabinin int ret; 1848db64fe02SNick Piggin 18497e4a32c0SHyunmin Lee if (unlikely(!size || offset_in_page(size) || !is_power_of_2(align))) 18507e4a32c0SHyunmin Lee return ERR_PTR(-EINVAL); 1851db64fe02SNick Piggin 185268ad4a33SUladzislau Rezki (Sony) if (unlikely(!vmap_initialized)) 185368ad4a33SUladzislau Rezki (Sony) return ERR_PTR(-EBUSY); 185468ad4a33SUladzislau Rezki (Sony) 18555803ed29SChristoph Hellwig might_sleep(); 185672210662SUladzislau Rezki (Sony) 185772210662SUladzislau Rezki (Sony) /* 185872210662SUladzislau Rezki (Sony) * If a VA is obtained from a global heap(if it fails here) 185972210662SUladzislau Rezki (Sony) * it is anyway marked with this "vn_id" so it is returned 186072210662SUladzislau Rezki (Sony) * to this pool's node later. Such way gives a possibility 186172210662SUladzislau Rezki (Sony) * to populate pools based on users demand. 186272210662SUladzislau Rezki (Sony) * 186372210662SUladzislau Rezki (Sony) * On success a ready to go VA is returned. 186472210662SUladzislau Rezki (Sony) */ 186572210662SUladzislau Rezki (Sony) va = node_alloc(size, align, vstart, vend, &addr, &vn_id); 186672210662SUladzislau Rezki (Sony) if (!va) { 1867f07116d7SUladzislau Rezki (Sony) gfp_mask = gfp_mask & GFP_RECLAIM_MASK; 18684da56b99SChris Wilson 1869f07116d7SUladzislau Rezki (Sony) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 1870db64fe02SNick Piggin if (unlikely(!va)) 1871db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 1872db64fe02SNick Piggin 18737f88f88fSCatalin Marinas /* 18747f88f88fSCatalin Marinas * Only scan the relevant parts containing pointers to other objects 18757f88f88fSCatalin Marinas * to avoid false negatives. 18767f88f88fSCatalin Marinas */ 1877f07116d7SUladzislau Rezki (Sony) kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask); 187896aa8437SUladzislau Rezki (Sony) } 18797f88f88fSCatalin Marinas 1880db64fe02SNick Piggin retry: 188172210662SUladzislau Rezki (Sony) if (addr == vend) { 1882187f8cc4SUladzislau Rezki (Sony) preload_this_cpu_lock(&free_vmap_area_lock, gfp_mask, node); 1883f9863be4SUladzislau Rezki (Sony) addr = __alloc_vmap_area(&free_vmap_area_root, &free_vmap_area_list, 1884f9863be4SUladzislau Rezki (Sony) size, align, vstart, vend); 1885187f8cc4SUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 188672210662SUladzislau Rezki (Sony) } 188768ad4a33SUladzislau Rezki (Sony) 1888cf243da6SUladzislau Rezki (Sony) trace_alloc_vmap_area(addr, size, align, vstart, vend, addr == vend); 1889cf243da6SUladzislau Rezki (Sony) 189089699605SNick Piggin /* 189168ad4a33SUladzislau Rezki (Sony) * If an allocation fails, the "vend" address is 189268ad4a33SUladzislau Rezki (Sony) * returned. Therefore trigger the overflow path. 189389699605SNick Piggin */ 189468ad4a33SUladzislau Rezki (Sony) if (unlikely(addr == vend)) 189589699605SNick Piggin goto overflow; 189689699605SNick Piggin 189789699605SNick Piggin va->va_start = addr; 189889699605SNick Piggin va->va_end = addr + size; 1899688fcbfcSPengfei Li va->vm = NULL; 190072210662SUladzislau Rezki (Sony) va->flags = (va_flags | vn_id); 190168ad4a33SUladzislau Rezki (Sony) 1902d0936029SUladzislau Rezki (Sony) vn = addr_to_node(va->va_start); 1903d0936029SUladzislau Rezki (Sony) 1904d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 1905d0936029SUladzislau Rezki (Sony) insert_vmap_area(va, &vn->busy.root, &vn->busy.head); 1906d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 190789699605SNick Piggin 190861e16557SWang Xiaoqiang BUG_ON(!IS_ALIGNED(va->va_start, align)); 190989699605SNick Piggin BUG_ON(va->va_start < vstart); 191089699605SNick Piggin BUG_ON(va->va_end > vend); 191189699605SNick Piggin 1912d98c9e83SAndrey Ryabinin ret = kasan_populate_vmalloc(addr, size); 1913d98c9e83SAndrey Ryabinin if (ret) { 1914d98c9e83SAndrey Ryabinin free_vmap_area(va); 1915d98c9e83SAndrey Ryabinin return ERR_PTR(ret); 1916d98c9e83SAndrey Ryabinin } 1917d98c9e83SAndrey Ryabinin 191889699605SNick Piggin return va; 191989699605SNick Piggin 19207766970cSNick Piggin overflow: 1921db64fe02SNick Piggin if (!purged) { 192277e50af0SThomas Gleixner reclaim_and_purge_vmap_areas(); 1923db64fe02SNick Piggin purged = 1; 1924db64fe02SNick Piggin goto retry; 1925db64fe02SNick Piggin } 19264da56b99SChris Wilson 192712e376a6SUladzislau Rezki (Sony) freed = 0; 19284da56b99SChris Wilson blocking_notifier_call_chain(&vmap_notify_list, 0, &freed); 192912e376a6SUladzislau Rezki (Sony) 19304da56b99SChris Wilson if (freed > 0) { 19314da56b99SChris Wilson purged = 0; 19324da56b99SChris Wilson goto retry; 19334da56b99SChris Wilson } 19344da56b99SChris Wilson 193503497d76SFlorian Fainelli if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) 1936756a025fSJoe Perches pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n", 1937756a025fSJoe Perches size); 193868ad4a33SUladzislau Rezki (Sony) 193968ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 1940db64fe02SNick Piggin return ERR_PTR(-EBUSY); 1941db64fe02SNick Piggin } 1942db64fe02SNick Piggin 19434da56b99SChris Wilson int register_vmap_purge_notifier(struct notifier_block *nb) 19444da56b99SChris Wilson { 19454da56b99SChris Wilson return blocking_notifier_chain_register(&vmap_notify_list, nb); 19464da56b99SChris Wilson } 19474da56b99SChris Wilson EXPORT_SYMBOL_GPL(register_vmap_purge_notifier); 19484da56b99SChris Wilson 19494da56b99SChris Wilson int unregister_vmap_purge_notifier(struct notifier_block *nb) 19504da56b99SChris Wilson { 19514da56b99SChris Wilson return blocking_notifier_chain_unregister(&vmap_notify_list, nb); 19524da56b99SChris Wilson } 19534da56b99SChris Wilson EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier); 19544da56b99SChris Wilson 1955db64fe02SNick Piggin /* 1956db64fe02SNick Piggin * lazy_max_pages is the maximum amount of virtual address space we gather up 1957db64fe02SNick Piggin * before attempting to purge with a TLB flush. 1958db64fe02SNick Piggin * 1959db64fe02SNick Piggin * There is a tradeoff here: a larger number will cover more kernel page tables 1960db64fe02SNick Piggin * and take slightly longer to purge, but it will linearly reduce the number of 1961db64fe02SNick Piggin * global TLB flushes that must be performed. It would seem natural to scale 1962db64fe02SNick Piggin * this number up linearly with the number of CPUs (because vmapping activity 1963db64fe02SNick Piggin * could also scale linearly with the number of CPUs), however it is likely 1964db64fe02SNick Piggin * that in practice, workloads might be constrained in other ways that mean 1965db64fe02SNick Piggin * vmap activity will not scale linearly with CPUs. Also, I want to be 1966db64fe02SNick Piggin * conservative and not introduce a big latency on huge systems, so go with 1967db64fe02SNick Piggin * a less aggressive log scale. It will still be an improvement over the old 1968db64fe02SNick Piggin * code, and it will be simple to change the scale factor if we find that it 1969db64fe02SNick Piggin * becomes a problem on bigger systems. 1970db64fe02SNick Piggin */ 1971db64fe02SNick Piggin static unsigned long lazy_max_pages(void) 1972db64fe02SNick Piggin { 1973db64fe02SNick Piggin unsigned int log; 1974db64fe02SNick Piggin 1975db64fe02SNick Piggin log = fls(num_online_cpus()); 1976db64fe02SNick Piggin 1977db64fe02SNick Piggin return log * (32UL * 1024 * 1024 / PAGE_SIZE); 1978db64fe02SNick Piggin } 1979db64fe02SNick Piggin 19804d36e6f8SUladzislau Rezki (Sony) static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0); 1981db64fe02SNick Piggin 19820574ecd1SChristoph Hellwig /* 1983f0953a1bSIngo Molnar * Serialize vmap purging. There is no actual critical section protected 1984153090f2SBaoquan He * by this lock, but we want to avoid concurrent calls for performance 19850574ecd1SChristoph Hellwig * reasons and to make the pcpu_get_vm_areas more deterministic. 19860574ecd1SChristoph Hellwig */ 1987f9e09977SChristoph Hellwig static DEFINE_MUTEX(vmap_purge_lock); 19880574ecd1SChristoph Hellwig 198902b709dfSNick Piggin /* for per-CPU blocks */ 199002b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void); 1991282631cbSUladzislau Rezki (Sony) static cpumask_t purge_nodes; 199202b709dfSNick Piggin 199372210662SUladzislau Rezki (Sony) static void 199472210662SUladzislau Rezki (Sony) reclaim_list_global(struct list_head *head) 1995db64fe02SNick Piggin { 199672210662SUladzislau Rezki (Sony) struct vmap_area *va, *n; 1997db64fe02SNick Piggin 199872210662SUladzislau Rezki (Sony) if (list_empty(head)) 199972210662SUladzislau Rezki (Sony) return; 2000db64fe02SNick Piggin 2001e36176beSUladzislau Rezki (Sony) spin_lock(&free_vmap_area_lock); 200272210662SUladzislau Rezki (Sony) list_for_each_entry_safe(va, n, head, list) 200372210662SUladzislau Rezki (Sony) merge_or_add_vmap_area_augment(va, 200472210662SUladzislau Rezki (Sony) &free_vmap_area_root, &free_vmap_area_list); 200572210662SUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 200672210662SUladzislau Rezki (Sony) } 200772210662SUladzislau Rezki (Sony) 200872210662SUladzislau Rezki (Sony) static void 200972210662SUladzislau Rezki (Sony) decay_va_pool_node(struct vmap_node *vn, bool full_decay) 201072210662SUladzislau Rezki (Sony) { 201172210662SUladzislau Rezki (Sony) struct vmap_area *va, *nva; 201272210662SUladzislau Rezki (Sony) struct list_head decay_list; 201372210662SUladzislau Rezki (Sony) struct rb_root decay_root; 201472210662SUladzislau Rezki (Sony) unsigned long n_decay; 201572210662SUladzislau Rezki (Sony) int i; 201672210662SUladzislau Rezki (Sony) 201772210662SUladzislau Rezki (Sony) decay_root = RB_ROOT; 201872210662SUladzislau Rezki (Sony) INIT_LIST_HEAD(&decay_list); 201972210662SUladzislau Rezki (Sony) 202072210662SUladzislau Rezki (Sony) for (i = 0; i < MAX_VA_SIZE_PAGES; i++) { 202172210662SUladzislau Rezki (Sony) struct list_head tmp_list; 202272210662SUladzislau Rezki (Sony) 202372210662SUladzislau Rezki (Sony) if (list_empty(&vn->pool[i].head)) 202472210662SUladzislau Rezki (Sony) continue; 202572210662SUladzislau Rezki (Sony) 202672210662SUladzislau Rezki (Sony) INIT_LIST_HEAD(&tmp_list); 202772210662SUladzislau Rezki (Sony) 202872210662SUladzislau Rezki (Sony) /* Detach the pool, so no-one can access it. */ 202972210662SUladzislau Rezki (Sony) spin_lock(&vn->pool_lock); 203072210662SUladzislau Rezki (Sony) list_replace_init(&vn->pool[i].head, &tmp_list); 203172210662SUladzislau Rezki (Sony) spin_unlock(&vn->pool_lock); 203272210662SUladzislau Rezki (Sony) 203372210662SUladzislau Rezki (Sony) if (full_decay) 203472210662SUladzislau Rezki (Sony) WRITE_ONCE(vn->pool[i].len, 0); 203572210662SUladzislau Rezki (Sony) 203672210662SUladzislau Rezki (Sony) /* Decay a pool by ~25% out of left objects. */ 203772210662SUladzislau Rezki (Sony) n_decay = vn->pool[i].len >> 2; 203872210662SUladzislau Rezki (Sony) 203972210662SUladzislau Rezki (Sony) list_for_each_entry_safe(va, nva, &tmp_list, list) { 204072210662SUladzislau Rezki (Sony) list_del_init(&va->list); 204172210662SUladzislau Rezki (Sony) merge_or_add_vmap_area(va, &decay_root, &decay_list); 204272210662SUladzislau Rezki (Sony) 204372210662SUladzislau Rezki (Sony) if (!full_decay) { 204472210662SUladzislau Rezki (Sony) WRITE_ONCE(vn->pool[i].len, vn->pool[i].len - 1); 204572210662SUladzislau Rezki (Sony) 204672210662SUladzislau Rezki (Sony) if (!--n_decay) 204772210662SUladzislau Rezki (Sony) break; 204872210662SUladzislau Rezki (Sony) } 204972210662SUladzislau Rezki (Sony) } 205072210662SUladzislau Rezki (Sony) 205172210662SUladzislau Rezki (Sony) /* Attach the pool back if it has been partly decayed. */ 205272210662SUladzislau Rezki (Sony) if (!full_decay && !list_empty(&tmp_list)) { 205372210662SUladzislau Rezki (Sony) spin_lock(&vn->pool_lock); 205472210662SUladzislau Rezki (Sony) list_replace_init(&tmp_list, &vn->pool[i].head); 205572210662SUladzislau Rezki (Sony) spin_unlock(&vn->pool_lock); 205672210662SUladzislau Rezki (Sony) } 205772210662SUladzislau Rezki (Sony) } 205872210662SUladzislau Rezki (Sony) 205972210662SUladzislau Rezki (Sony) reclaim_list_global(&decay_list); 206072210662SUladzislau Rezki (Sony) } 206172210662SUladzislau Rezki (Sony) 206272210662SUladzislau Rezki (Sony) static void purge_vmap_node(struct work_struct *work) 206372210662SUladzislau Rezki (Sony) { 206472210662SUladzislau Rezki (Sony) struct vmap_node *vn = container_of(work, 206572210662SUladzislau Rezki (Sony) struct vmap_node, purge_work); 206672210662SUladzislau Rezki (Sony) struct vmap_area *va, *n_va; 206772210662SUladzislau Rezki (Sony) LIST_HEAD(local_list); 206872210662SUladzislau Rezki (Sony) 206972210662SUladzislau Rezki (Sony) vn->nr_purged = 0; 207072210662SUladzislau Rezki (Sony) 2071282631cbSUladzislau Rezki (Sony) list_for_each_entry_safe(va, n_va, &vn->purge_list, list) { 20724d36e6f8SUladzislau Rezki (Sony) unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT; 20733c5c3cfbSDaniel Axtens unsigned long orig_start = va->va_start; 20743c5c3cfbSDaniel Axtens unsigned long orig_end = va->va_end; 207572210662SUladzislau Rezki (Sony) unsigned int vn_id = decode_vn_id(va->flags); 2076763b218dSJoel Fernandes 207772210662SUladzislau Rezki (Sony) list_del_init(&va->list); 20789c801f61SUladzislau Rezki (Sony) 20793c5c3cfbSDaniel Axtens if (is_vmalloc_or_module_addr((void *)orig_start)) 20803c5c3cfbSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 20813c5c3cfbSDaniel Axtens va->va_start, va->va_end); 2082dd3b8353SUladzislau Rezki (Sony) 20834d36e6f8SUladzislau Rezki (Sony) atomic_long_sub(nr, &vmap_lazy_nr); 208472210662SUladzislau Rezki (Sony) vn->nr_purged++; 20856030fd5fSUladzislau Rezki (Sony) 208672210662SUladzislau Rezki (Sony) if (is_vn_id_valid(vn_id) && !vn->skip_populate) 208772210662SUladzislau Rezki (Sony) if (node_pool_add_va(vn, va)) 208872210662SUladzislau Rezki (Sony) continue; 208972210662SUladzislau Rezki (Sony) 209072210662SUladzislau Rezki (Sony) /* Go back to global. */ 209172210662SUladzislau Rezki (Sony) list_add(&va->list, &local_list); 209272210662SUladzislau Rezki (Sony) } 209372210662SUladzislau Rezki (Sony) 209472210662SUladzislau Rezki (Sony) reclaim_list_global(&local_list); 2095282631cbSUladzislau Rezki (Sony) } 2096282631cbSUladzislau Rezki (Sony) 2097282631cbSUladzislau Rezki (Sony) /* 2098282631cbSUladzislau Rezki (Sony) * Purges all lazily-freed vmap areas. 2099282631cbSUladzislau Rezki (Sony) */ 210072210662SUladzislau Rezki (Sony) static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end, 210172210662SUladzislau Rezki (Sony) bool full_pool_decay) 2102282631cbSUladzislau Rezki (Sony) { 210372210662SUladzislau Rezki (Sony) unsigned long nr_purged_areas = 0; 210472210662SUladzislau Rezki (Sony) unsigned int nr_purge_helpers; 210572210662SUladzislau Rezki (Sony) unsigned int nr_purge_nodes; 2106282631cbSUladzislau Rezki (Sony) struct vmap_node *vn; 2107282631cbSUladzislau Rezki (Sony) int i; 2108282631cbSUladzislau Rezki (Sony) 2109282631cbSUladzislau Rezki (Sony) lockdep_assert_held(&vmap_purge_lock); 211072210662SUladzislau Rezki (Sony) 211172210662SUladzislau Rezki (Sony) /* 211272210662SUladzislau Rezki (Sony) * Use cpumask to mark which node has to be processed. 211372210662SUladzislau Rezki (Sony) */ 2114282631cbSUladzislau Rezki (Sony) purge_nodes = CPU_MASK_NONE; 2115282631cbSUladzislau Rezki (Sony) 2116282631cbSUladzislau Rezki (Sony) for (i = 0; i < nr_vmap_nodes; i++) { 2117282631cbSUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 2118282631cbSUladzislau Rezki (Sony) 2119282631cbSUladzislau Rezki (Sony) INIT_LIST_HEAD(&vn->purge_list); 212072210662SUladzislau Rezki (Sony) vn->skip_populate = full_pool_decay; 212172210662SUladzislau Rezki (Sony) decay_va_pool_node(vn, full_pool_decay); 2122282631cbSUladzislau Rezki (Sony) 2123282631cbSUladzislau Rezki (Sony) if (RB_EMPTY_ROOT(&vn->lazy.root)) 2124282631cbSUladzislau Rezki (Sony) continue; 2125282631cbSUladzislau Rezki (Sony) 2126282631cbSUladzislau Rezki (Sony) spin_lock(&vn->lazy.lock); 2127282631cbSUladzislau Rezki (Sony) WRITE_ONCE(vn->lazy.root.rb_node, NULL); 2128282631cbSUladzislau Rezki (Sony) list_replace_init(&vn->lazy.head, &vn->purge_list); 2129282631cbSUladzislau Rezki (Sony) spin_unlock(&vn->lazy.lock); 2130282631cbSUladzislau Rezki (Sony) 2131282631cbSUladzislau Rezki (Sony) start = min(start, list_first_entry(&vn->purge_list, 2132282631cbSUladzislau Rezki (Sony) struct vmap_area, list)->va_start); 2133282631cbSUladzislau Rezki (Sony) 2134282631cbSUladzislau Rezki (Sony) end = max(end, list_last_entry(&vn->purge_list, 2135282631cbSUladzislau Rezki (Sony) struct vmap_area, list)->va_end); 2136282631cbSUladzislau Rezki (Sony) 2137282631cbSUladzislau Rezki (Sony) cpumask_set_cpu(i, &purge_nodes); 2138282631cbSUladzislau Rezki (Sony) } 2139282631cbSUladzislau Rezki (Sony) 214072210662SUladzislau Rezki (Sony) nr_purge_nodes = cpumask_weight(&purge_nodes); 214172210662SUladzislau Rezki (Sony) if (nr_purge_nodes > 0) { 2142282631cbSUladzislau Rezki (Sony) flush_tlb_kernel_range(start, end); 2143282631cbSUladzislau Rezki (Sony) 214472210662SUladzislau Rezki (Sony) /* One extra worker is per a lazy_max_pages() full set minus one. */ 214572210662SUladzislau Rezki (Sony) nr_purge_helpers = atomic_long_read(&vmap_lazy_nr) / lazy_max_pages(); 214672210662SUladzislau Rezki (Sony) nr_purge_helpers = clamp(nr_purge_helpers, 1U, nr_purge_nodes) - 1; 214772210662SUladzislau Rezki (Sony) 2148282631cbSUladzislau Rezki (Sony) for_each_cpu(i, &purge_nodes) { 214972210662SUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 215072210662SUladzislau Rezki (Sony) 215172210662SUladzislau Rezki (Sony) if (nr_purge_helpers > 0) { 215272210662SUladzislau Rezki (Sony) INIT_WORK(&vn->purge_work, purge_vmap_node); 215372210662SUladzislau Rezki (Sony) 215472210662SUladzislau Rezki (Sony) if (cpumask_test_cpu(i, cpu_online_mask)) 215572210662SUladzislau Rezki (Sony) schedule_work_on(i, &vn->purge_work); 215672210662SUladzislau Rezki (Sony) else 215772210662SUladzislau Rezki (Sony) schedule_work(&vn->purge_work); 215872210662SUladzislau Rezki (Sony) 215972210662SUladzislau Rezki (Sony) nr_purge_helpers--; 216072210662SUladzislau Rezki (Sony) } else { 216172210662SUladzislau Rezki (Sony) vn->purge_work.func = NULL; 216272210662SUladzislau Rezki (Sony) purge_vmap_node(&vn->purge_work); 216372210662SUladzislau Rezki (Sony) nr_purged_areas += vn->nr_purged; 2164282631cbSUladzislau Rezki (Sony) } 2165282631cbSUladzislau Rezki (Sony) } 2166282631cbSUladzislau Rezki (Sony) 216772210662SUladzislau Rezki (Sony) for_each_cpu(i, &purge_nodes) { 216872210662SUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 216972210662SUladzislau Rezki (Sony) 217072210662SUladzislau Rezki (Sony) if (vn->purge_work.func) { 217172210662SUladzislau Rezki (Sony) flush_work(&vn->purge_work); 217272210662SUladzislau Rezki (Sony) nr_purged_areas += vn->nr_purged; 217372210662SUladzislau Rezki (Sony) } 217472210662SUladzislau Rezki (Sony) } 217572210662SUladzislau Rezki (Sony) } 217672210662SUladzislau Rezki (Sony) 217772210662SUladzislau Rezki (Sony) trace_purge_vmap_area_lazy(start, end, nr_purged_areas); 217872210662SUladzislau Rezki (Sony) return nr_purged_areas > 0; 2179db64fe02SNick Piggin } 2180db64fe02SNick Piggin 2181db64fe02SNick Piggin /* 218277e50af0SThomas Gleixner * Reclaim vmap areas by purging fragmented blocks and purge_vmap_area_list. 2183db64fe02SNick Piggin */ 218477e50af0SThomas Gleixner static void reclaim_and_purge_vmap_areas(void) 218577e50af0SThomas Gleixner 2186db64fe02SNick Piggin { 2187f9e09977SChristoph Hellwig mutex_lock(&vmap_purge_lock); 21880574ecd1SChristoph Hellwig purge_fragmented_blocks_allcpus(); 218972210662SUladzislau Rezki (Sony) __purge_vmap_area_lazy(ULONG_MAX, 0, true); 2190f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 2191db64fe02SNick Piggin } 2192db64fe02SNick Piggin 2193690467c8SUladzislau Rezki (Sony) static void drain_vmap_area_work(struct work_struct *work) 2194690467c8SUladzislau Rezki (Sony) { 2195690467c8SUladzislau Rezki (Sony) mutex_lock(&vmap_purge_lock); 219672210662SUladzislau Rezki (Sony) __purge_vmap_area_lazy(ULONG_MAX, 0, false); 2197690467c8SUladzislau Rezki (Sony) mutex_unlock(&vmap_purge_lock); 2198690467c8SUladzislau Rezki (Sony) } 2199690467c8SUladzislau Rezki (Sony) 2200db64fe02SNick Piggin /* 2201edd89818SUladzislau Rezki (Sony) * Free a vmap area, caller ensuring that the area has been unmapped, 2202edd89818SUladzislau Rezki (Sony) * unlinked and flush_cache_vunmap had been called for the correct 2203edd89818SUladzislau Rezki (Sony) * range previously. 2204db64fe02SNick Piggin */ 220564141da5SJeremy Fitzhardinge static void free_vmap_area_noflush(struct vmap_area *va) 2206db64fe02SNick Piggin { 22078c4196feSUladzislau Rezki (Sony) unsigned long nr_lazy_max = lazy_max_pages(); 22088c4196feSUladzislau Rezki (Sony) unsigned long va_start = va->va_start; 220972210662SUladzislau Rezki (Sony) unsigned int vn_id = decode_vn_id(va->flags); 221072210662SUladzislau Rezki (Sony) struct vmap_node *vn; 22114d36e6f8SUladzislau Rezki (Sony) unsigned long nr_lazy; 221280c4bd7aSChris Wilson 2213edd89818SUladzislau Rezki (Sony) if (WARN_ON_ONCE(!list_empty(&va->list))) 2214edd89818SUladzislau Rezki (Sony) return; 2215dd3b8353SUladzislau Rezki (Sony) 22164d36e6f8SUladzislau Rezki (Sony) nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >> 22174d36e6f8SUladzislau Rezki (Sony) PAGE_SHIFT, &vmap_lazy_nr); 221880c4bd7aSChris Wilson 221996e2db45SUladzislau Rezki (Sony) /* 222072210662SUladzislau Rezki (Sony) * If it was request by a certain node we would like to 222172210662SUladzislau Rezki (Sony) * return it to that node, i.e. its pool for later reuse. 222296e2db45SUladzislau Rezki (Sony) */ 222372210662SUladzislau Rezki (Sony) vn = is_vn_id_valid(vn_id) ? 222472210662SUladzislau Rezki (Sony) id_to_node(vn_id):addr_to_node(va->va_start); 222572210662SUladzislau Rezki (Sony) 2226282631cbSUladzislau Rezki (Sony) spin_lock(&vn->lazy.lock); 222772210662SUladzislau Rezki (Sony) insert_vmap_area(va, &vn->lazy.root, &vn->lazy.head); 2228282631cbSUladzislau Rezki (Sony) spin_unlock(&vn->lazy.lock); 222980c4bd7aSChris Wilson 22308c4196feSUladzislau Rezki (Sony) trace_free_vmap_area_noflush(va_start, nr_lazy, nr_lazy_max); 22318c4196feSUladzislau Rezki (Sony) 223296e2db45SUladzislau Rezki (Sony) /* After this point, we may free va at any time */ 22338c4196feSUladzislau Rezki (Sony) if (unlikely(nr_lazy > nr_lazy_max)) 2234690467c8SUladzislau Rezki (Sony) schedule_work(&drain_vmap_work); 2235db64fe02SNick Piggin } 2236db64fe02SNick Piggin 2237b29acbdcSNick Piggin /* 2238b29acbdcSNick Piggin * Free and unmap a vmap area 2239b29acbdcSNick Piggin */ 2240b29acbdcSNick Piggin static void free_unmap_vmap_area(struct vmap_area *va) 2241b29acbdcSNick Piggin { 2242b29acbdcSNick Piggin flush_cache_vunmap(va->va_start, va->va_end); 22434ad0ae8cSNicholas Piggin vunmap_range_noflush(va->va_start, va->va_end); 22448e57f8acSVlastimil Babka if (debug_pagealloc_enabled_static()) 224582a2e924SChintan Pandya flush_tlb_kernel_range(va->va_start, va->va_end); 224682a2e924SChintan Pandya 2247c8eef01eSChristoph Hellwig free_vmap_area_noflush(va); 2248b29acbdcSNick Piggin } 2249b29acbdcSNick Piggin 2250993d0b28SMatthew Wilcox (Oracle) struct vmap_area *find_vmap_area(unsigned long addr) 2251db64fe02SNick Piggin { 2252d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 2253db64fe02SNick Piggin struct vmap_area *va; 2254d0936029SUladzislau Rezki (Sony) int i, j; 2255db64fe02SNick Piggin 2256d0936029SUladzislau Rezki (Sony) /* 2257d0936029SUladzislau Rezki (Sony) * An addr_to_node_id(addr) converts an address to a node index 2258d0936029SUladzislau Rezki (Sony) * where a VA is located. If VA spans several zones and passed 2259d0936029SUladzislau Rezki (Sony) * addr is not the same as va->va_start, what is not common, we 2260d0936029SUladzislau Rezki (Sony) * may need to scan an extra nodes. See an example: 2261d0936029SUladzislau Rezki (Sony) * 2262d0936029SUladzislau Rezki (Sony) * <--va--> 2263d0936029SUladzislau Rezki (Sony) * -|-----|-----|-----|-----|- 2264d0936029SUladzislau Rezki (Sony) * 1 2 0 1 2265d0936029SUladzislau Rezki (Sony) * 2266d0936029SUladzislau Rezki (Sony) * VA resides in node 1 whereas it spans 1 and 2. If passed 2267d0936029SUladzislau Rezki (Sony) * addr is within a second node we should do extra work. We 2268d0936029SUladzislau Rezki (Sony) * should mention that it is rare and is a corner case from 2269d0936029SUladzislau Rezki (Sony) * the other hand it has to be covered. 2270d0936029SUladzislau Rezki (Sony) */ 2271d0936029SUladzislau Rezki (Sony) i = j = addr_to_node_id(addr); 2272d0936029SUladzislau Rezki (Sony) do { 2273d0936029SUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 2274db64fe02SNick Piggin 2275d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 2276d0936029SUladzislau Rezki (Sony) va = __find_vmap_area(addr, &vn->busy.root); 2277d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 2278d0936029SUladzislau Rezki (Sony) 2279d0936029SUladzislau Rezki (Sony) if (va) 2280db64fe02SNick Piggin return va; 2281d0936029SUladzislau Rezki (Sony) } while ((i = (i + 1) % nr_vmap_nodes) != j); 2282d0936029SUladzislau Rezki (Sony) 2283d0936029SUladzislau Rezki (Sony) return NULL; 2284db64fe02SNick Piggin } 2285db64fe02SNick Piggin 2286edd89818SUladzislau Rezki (Sony) static struct vmap_area *find_unlink_vmap_area(unsigned long addr) 2287edd89818SUladzislau Rezki (Sony) { 2288d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 2289edd89818SUladzislau Rezki (Sony) struct vmap_area *va; 2290d0936029SUladzislau Rezki (Sony) int i, j; 2291edd89818SUladzislau Rezki (Sony) 2292d0936029SUladzislau Rezki (Sony) i = j = addr_to_node_id(addr); 2293d0936029SUladzislau Rezki (Sony) do { 2294d0936029SUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 2295d0936029SUladzislau Rezki (Sony) 2296d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 2297d0936029SUladzislau Rezki (Sony) va = __find_vmap_area(addr, &vn->busy.root); 2298edd89818SUladzislau Rezki (Sony) if (va) 2299d0936029SUladzislau Rezki (Sony) unlink_va(va, &vn->busy.root); 2300d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 2301edd89818SUladzislau Rezki (Sony) 2302d0936029SUladzislau Rezki (Sony) if (va) 2303edd89818SUladzislau Rezki (Sony) return va; 2304d0936029SUladzislau Rezki (Sony) } while ((i = (i + 1) % nr_vmap_nodes) != j); 2305d0936029SUladzislau Rezki (Sony) 2306d0936029SUladzislau Rezki (Sony) return NULL; 2307edd89818SUladzislau Rezki (Sony) } 2308edd89818SUladzislau Rezki (Sony) 2309db64fe02SNick Piggin /*** Per cpu kva allocator ***/ 2310db64fe02SNick Piggin 2311db64fe02SNick Piggin /* 2312db64fe02SNick Piggin * vmap space is limited especially on 32 bit architectures. Ensure there is 2313db64fe02SNick Piggin * room for at least 16 percpu vmap blocks per CPU. 2314db64fe02SNick Piggin */ 2315db64fe02SNick Piggin /* 2316db64fe02SNick Piggin * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able 2317db64fe02SNick Piggin * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess 2318db64fe02SNick Piggin * instead (we just need a rough idea) 2319db64fe02SNick Piggin */ 2320db64fe02SNick Piggin #if BITS_PER_LONG == 32 2321db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024) 2322db64fe02SNick Piggin #else 2323db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024*1024) 2324db64fe02SNick Piggin #endif 2325db64fe02SNick Piggin 2326db64fe02SNick Piggin #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE) 2327db64fe02SNick Piggin #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */ 2328db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */ 2329db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2) 2330db64fe02SNick Piggin #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */ 2331db64fe02SNick Piggin #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */ 2332f982f915SClemens Ladisch #define VMAP_BBMAP_BITS \ 2333f982f915SClemens Ladisch VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ 2334db64fe02SNick Piggin VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ 2335f982f915SClemens Ladisch VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16)) 2336db64fe02SNick Piggin 2337db64fe02SNick Piggin #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE) 2338db64fe02SNick Piggin 233977e50af0SThomas Gleixner /* 234077e50af0SThomas Gleixner * Purge threshold to prevent overeager purging of fragmented blocks for 234177e50af0SThomas Gleixner * regular operations: Purge if vb->free is less than 1/4 of the capacity. 234277e50af0SThomas Gleixner */ 234377e50af0SThomas Gleixner #define VMAP_PURGE_THRESHOLD (VMAP_BBMAP_BITS / 4) 234477e50af0SThomas Gleixner 2345869176a0SBaoquan He #define VMAP_RAM 0x1 /* indicates vm_map_ram area*/ 2346869176a0SBaoquan He #define VMAP_BLOCK 0x2 /* mark out the vmap_block sub-type*/ 2347869176a0SBaoquan He #define VMAP_FLAGS_MASK 0x3 2348869176a0SBaoquan He 2349db64fe02SNick Piggin struct vmap_block_queue { 2350db64fe02SNick Piggin spinlock_t lock; 2351db64fe02SNick Piggin struct list_head free; 2352062eacf5SUladzislau Rezki (Sony) 2353062eacf5SUladzislau Rezki (Sony) /* 2354062eacf5SUladzislau Rezki (Sony) * An xarray requires an extra memory dynamically to 2355062eacf5SUladzislau Rezki (Sony) * be allocated. If it is an issue, we can use rb-tree 2356062eacf5SUladzislau Rezki (Sony) * instead. 2357062eacf5SUladzislau Rezki (Sony) */ 2358062eacf5SUladzislau Rezki (Sony) struct xarray vmap_blocks; 2359db64fe02SNick Piggin }; 2360db64fe02SNick Piggin 2361db64fe02SNick Piggin struct vmap_block { 2362db64fe02SNick Piggin spinlock_t lock; 2363db64fe02SNick Piggin struct vmap_area *va; 2364db64fe02SNick Piggin unsigned long free, dirty; 2365d76f9954SBaoquan He DECLARE_BITMAP(used_map, VMAP_BBMAP_BITS); 23667d61bfe8SRoman Pen unsigned long dirty_min, dirty_max; /*< dirty range */ 2367db64fe02SNick Piggin struct list_head free_list; 2368db64fe02SNick Piggin struct rcu_head rcu_head; 236902b709dfSNick Piggin struct list_head purge; 2370db64fe02SNick Piggin }; 2371db64fe02SNick Piggin 2372db64fe02SNick Piggin /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */ 2373db64fe02SNick Piggin static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue); 2374db64fe02SNick Piggin 2375db64fe02SNick Piggin /* 2376062eacf5SUladzislau Rezki (Sony) * In order to fast access to any "vmap_block" associated with a 2377062eacf5SUladzislau Rezki (Sony) * specific address, we use a hash. 2378062eacf5SUladzislau Rezki (Sony) * 2379062eacf5SUladzislau Rezki (Sony) * A per-cpu vmap_block_queue is used in both ways, to serialize 2380062eacf5SUladzislau Rezki (Sony) * an access to free block chains among CPUs(alloc path) and it 2381062eacf5SUladzislau Rezki (Sony) * also acts as a vmap_block hash(alloc/free paths). It means we 2382062eacf5SUladzislau Rezki (Sony) * overload it, since we already have the per-cpu array which is 2383062eacf5SUladzislau Rezki (Sony) * used as a hash table. When used as a hash a 'cpu' passed to 2384062eacf5SUladzislau Rezki (Sony) * per_cpu() is not actually a CPU but rather a hash index. 2385062eacf5SUladzislau Rezki (Sony) * 2386fa1c77c1SUladzislau Rezki (Sony) * A hash function is addr_to_vb_xa() which hashes any address 2387062eacf5SUladzislau Rezki (Sony) * to a specific index(in a hash) it belongs to. This then uses a 2388062eacf5SUladzislau Rezki (Sony) * per_cpu() macro to access an array with generated index. 2389062eacf5SUladzislau Rezki (Sony) * 2390062eacf5SUladzislau Rezki (Sony) * An example: 2391062eacf5SUladzislau Rezki (Sony) * 2392062eacf5SUladzislau Rezki (Sony) * CPU_1 CPU_2 CPU_0 2393062eacf5SUladzislau Rezki (Sony) * | | | 2394062eacf5SUladzislau Rezki (Sony) * V V V 2395062eacf5SUladzislau Rezki (Sony) * 0 10 20 30 40 50 60 2396062eacf5SUladzislau Rezki (Sony) * |------|------|------|------|------|------|...<vmap address space> 2397062eacf5SUladzislau Rezki (Sony) * CPU0 CPU1 CPU2 CPU0 CPU1 CPU2 2398062eacf5SUladzislau Rezki (Sony) * 2399062eacf5SUladzislau Rezki (Sony) * - CPU_1 invokes vm_unmap_ram(6), 6 belongs to CPU0 zone, thus 2400062eacf5SUladzislau Rezki (Sony) * it access: CPU0/INDEX0 -> vmap_blocks -> xa_lock; 2401062eacf5SUladzislau Rezki (Sony) * 2402062eacf5SUladzislau Rezki (Sony) * - CPU_2 invokes vm_unmap_ram(11), 11 belongs to CPU1 zone, thus 2403062eacf5SUladzislau Rezki (Sony) * it access: CPU1/INDEX1 -> vmap_blocks -> xa_lock; 2404062eacf5SUladzislau Rezki (Sony) * 2405062eacf5SUladzislau Rezki (Sony) * - CPU_0 invokes vm_unmap_ram(20), 20 belongs to CPU2 zone, thus 2406062eacf5SUladzislau Rezki (Sony) * it access: CPU2/INDEX2 -> vmap_blocks -> xa_lock. 2407062eacf5SUladzislau Rezki (Sony) * 2408062eacf5SUladzislau Rezki (Sony) * This technique almost always avoids lock contention on insert/remove, 2409062eacf5SUladzislau Rezki (Sony) * however xarray spinlocks protect against any contention that remains. 2410db64fe02SNick Piggin */ 2411062eacf5SUladzislau Rezki (Sony) static struct xarray * 2412fa1c77c1SUladzislau Rezki (Sony) addr_to_vb_xa(unsigned long addr) 2413062eacf5SUladzislau Rezki (Sony) { 2414062eacf5SUladzislau Rezki (Sony) int index = (addr / VMAP_BLOCK_SIZE) % num_possible_cpus(); 2415062eacf5SUladzislau Rezki (Sony) 2416062eacf5SUladzislau Rezki (Sony) return &per_cpu(vmap_block_queue, index).vmap_blocks; 2417062eacf5SUladzislau Rezki (Sony) } 2418db64fe02SNick Piggin 2419db64fe02SNick Piggin /* 2420db64fe02SNick Piggin * We should probably have a fallback mechanism to allocate virtual memory 2421db64fe02SNick Piggin * out of partially filled vmap blocks. However vmap block sizing should be 2422db64fe02SNick Piggin * fairly reasonable according to the vmalloc size, so it shouldn't be a 2423db64fe02SNick Piggin * big problem. 2424db64fe02SNick Piggin */ 2425db64fe02SNick Piggin 2426db64fe02SNick Piggin static unsigned long addr_to_vb_idx(unsigned long addr) 2427db64fe02SNick Piggin { 2428db64fe02SNick Piggin addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); 2429db64fe02SNick Piggin addr /= VMAP_BLOCK_SIZE; 2430db64fe02SNick Piggin return addr; 2431db64fe02SNick Piggin } 2432db64fe02SNick Piggin 2433cf725ce2SRoman Pen static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off) 2434cf725ce2SRoman Pen { 2435cf725ce2SRoman Pen unsigned long addr; 2436cf725ce2SRoman Pen 2437cf725ce2SRoman Pen addr = va_start + (pages_off << PAGE_SHIFT); 2438cf725ce2SRoman Pen BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start)); 2439cf725ce2SRoman Pen return (void *)addr; 2440cf725ce2SRoman Pen } 2441cf725ce2SRoman Pen 2442cf725ce2SRoman Pen /** 2443cf725ce2SRoman Pen * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this 2444cf725ce2SRoman Pen * block. Of course pages number can't exceed VMAP_BBMAP_BITS 2445cf725ce2SRoman Pen * @order: how many 2^order pages should be occupied in newly allocated block 2446cf725ce2SRoman Pen * @gfp_mask: flags for the page level allocator 2447cf725ce2SRoman Pen * 2448a862f68aSMike Rapoport * Return: virtual address in a newly allocated block or ERR_PTR(-errno) 2449cf725ce2SRoman Pen */ 2450cf725ce2SRoman Pen static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) 2451db64fe02SNick Piggin { 2452db64fe02SNick Piggin struct vmap_block_queue *vbq; 2453db64fe02SNick Piggin struct vmap_block *vb; 2454db64fe02SNick Piggin struct vmap_area *va; 2455062eacf5SUladzislau Rezki (Sony) struct xarray *xa; 2456db64fe02SNick Piggin unsigned long vb_idx; 2457db64fe02SNick Piggin int node, err; 2458cf725ce2SRoman Pen void *vaddr; 2459db64fe02SNick Piggin 2460db64fe02SNick Piggin node = numa_node_id(); 2461db64fe02SNick Piggin 2462db64fe02SNick Piggin vb = kmalloc_node(sizeof(struct vmap_block), 2463db64fe02SNick Piggin gfp_mask & GFP_RECLAIM_MASK, node); 2464db64fe02SNick Piggin if (unlikely(!vb)) 2465db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 2466db64fe02SNick Piggin 2467db64fe02SNick Piggin va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, 2468db64fe02SNick Piggin VMALLOC_START, VMALLOC_END, 2469869176a0SBaoquan He node, gfp_mask, 2470869176a0SBaoquan He VMAP_RAM|VMAP_BLOCK); 2471ddf9c6d4STobias Klauser if (IS_ERR(va)) { 2472db64fe02SNick Piggin kfree(vb); 2473e7d86340SJulia Lawall return ERR_CAST(va); 2474db64fe02SNick Piggin } 2475db64fe02SNick Piggin 2476cf725ce2SRoman Pen vaddr = vmap_block_vaddr(va->va_start, 0); 2477db64fe02SNick Piggin spin_lock_init(&vb->lock); 2478db64fe02SNick Piggin vb->va = va; 2479cf725ce2SRoman Pen /* At least something should be left free */ 2480cf725ce2SRoman Pen BUG_ON(VMAP_BBMAP_BITS <= (1UL << order)); 2481d76f9954SBaoquan He bitmap_zero(vb->used_map, VMAP_BBMAP_BITS); 2482cf725ce2SRoman Pen vb->free = VMAP_BBMAP_BITS - (1UL << order); 2483db64fe02SNick Piggin vb->dirty = 0; 24847d61bfe8SRoman Pen vb->dirty_min = VMAP_BBMAP_BITS; 24857d61bfe8SRoman Pen vb->dirty_max = 0; 2486d76f9954SBaoquan He bitmap_set(vb->used_map, 0, (1UL << order)); 2487db64fe02SNick Piggin INIT_LIST_HEAD(&vb->free_list); 2488db64fe02SNick Piggin 2489fa1c77c1SUladzislau Rezki (Sony) xa = addr_to_vb_xa(va->va_start); 2490db64fe02SNick Piggin vb_idx = addr_to_vb_idx(va->va_start); 2491062eacf5SUladzislau Rezki (Sony) err = xa_insert(xa, vb_idx, vb, gfp_mask); 24920f14599cSMatthew Wilcox (Oracle) if (err) { 24930f14599cSMatthew Wilcox (Oracle) kfree(vb); 24940f14599cSMatthew Wilcox (Oracle) free_vmap_area(va); 24950f14599cSMatthew Wilcox (Oracle) return ERR_PTR(err); 24960f14599cSMatthew Wilcox (Oracle) } 2497db64fe02SNick Piggin 24983f804920SSebastian Andrzej Siewior vbq = raw_cpu_ptr(&vmap_block_queue); 2499db64fe02SNick Piggin spin_lock(&vbq->lock); 250068ac546fSRoman Pen list_add_tail_rcu(&vb->free_list, &vbq->free); 2501db64fe02SNick Piggin spin_unlock(&vbq->lock); 2502db64fe02SNick Piggin 2503cf725ce2SRoman Pen return vaddr; 2504db64fe02SNick Piggin } 2505db64fe02SNick Piggin 2506db64fe02SNick Piggin static void free_vmap_block(struct vmap_block *vb) 2507db64fe02SNick Piggin { 2508d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 2509db64fe02SNick Piggin struct vmap_block *tmp; 2510062eacf5SUladzislau Rezki (Sony) struct xarray *xa; 2511db64fe02SNick Piggin 2512fa1c77c1SUladzislau Rezki (Sony) xa = addr_to_vb_xa(vb->va->va_start); 2513062eacf5SUladzislau Rezki (Sony) tmp = xa_erase(xa, addr_to_vb_idx(vb->va->va_start)); 2514db64fe02SNick Piggin BUG_ON(tmp != vb); 2515db64fe02SNick Piggin 2516d0936029SUladzislau Rezki (Sony) vn = addr_to_node(vb->va->va_start); 2517d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 2518d0936029SUladzislau Rezki (Sony) unlink_va(vb->va, &vn->busy.root); 2519d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 2520edd89818SUladzislau Rezki (Sony) 252164141da5SJeremy Fitzhardinge free_vmap_area_noflush(vb->va); 252222a3c7d1SLai Jiangshan kfree_rcu(vb, rcu_head); 2523db64fe02SNick Piggin } 2524db64fe02SNick Piggin 2525ca5e46c3SThomas Gleixner static bool purge_fragmented_block(struct vmap_block *vb, 252677e50af0SThomas Gleixner struct vmap_block_queue *vbq, struct list_head *purge_list, 252777e50af0SThomas Gleixner bool force_purge) 252802b709dfSNick Piggin { 2529ca5e46c3SThomas Gleixner if (vb->free + vb->dirty != VMAP_BBMAP_BITS || 2530ca5e46c3SThomas Gleixner vb->dirty == VMAP_BBMAP_BITS) 2531ca5e46c3SThomas Gleixner return false; 253202b709dfSNick Piggin 253377e50af0SThomas Gleixner /* Don't overeagerly purge usable blocks unless requested */ 253477e50af0SThomas Gleixner if (!(force_purge || vb->free < VMAP_PURGE_THRESHOLD)) 253577e50af0SThomas Gleixner return false; 253677e50af0SThomas Gleixner 2537ca5e46c3SThomas Gleixner /* prevent further allocs after releasing lock */ 25387f48121eSThomas Gleixner WRITE_ONCE(vb->free, 0); 2539ca5e46c3SThomas Gleixner /* prevent purging it again */ 25407f48121eSThomas Gleixner WRITE_ONCE(vb->dirty, VMAP_BBMAP_BITS); 25417d61bfe8SRoman Pen vb->dirty_min = 0; 25427d61bfe8SRoman Pen vb->dirty_max = VMAP_BBMAP_BITS; 254302b709dfSNick Piggin spin_lock(&vbq->lock); 254402b709dfSNick Piggin list_del_rcu(&vb->free_list); 254502b709dfSNick Piggin spin_unlock(&vbq->lock); 2546ca5e46c3SThomas Gleixner list_add_tail(&vb->purge, purge_list); 2547ca5e46c3SThomas Gleixner return true; 254802b709dfSNick Piggin } 254902b709dfSNick Piggin 2550ca5e46c3SThomas Gleixner static void free_purged_blocks(struct list_head *purge_list) 2551ca5e46c3SThomas Gleixner { 2552ca5e46c3SThomas Gleixner struct vmap_block *vb, *n_vb; 2553ca5e46c3SThomas Gleixner 2554ca5e46c3SThomas Gleixner list_for_each_entry_safe(vb, n_vb, purge_list, purge) { 255502b709dfSNick Piggin list_del(&vb->purge); 255602b709dfSNick Piggin free_vmap_block(vb); 255702b709dfSNick Piggin } 255802b709dfSNick Piggin } 255902b709dfSNick Piggin 2560ca5e46c3SThomas Gleixner static void purge_fragmented_blocks(int cpu) 2561ca5e46c3SThomas Gleixner { 2562ca5e46c3SThomas Gleixner LIST_HEAD(purge); 2563ca5e46c3SThomas Gleixner struct vmap_block *vb; 2564ca5e46c3SThomas Gleixner struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 2565ca5e46c3SThomas Gleixner 2566ca5e46c3SThomas Gleixner rcu_read_lock(); 2567ca5e46c3SThomas Gleixner list_for_each_entry_rcu(vb, &vbq->free, free_list) { 25687f48121eSThomas Gleixner unsigned long free = READ_ONCE(vb->free); 25697f48121eSThomas Gleixner unsigned long dirty = READ_ONCE(vb->dirty); 25707f48121eSThomas Gleixner 25717f48121eSThomas Gleixner if (free + dirty != VMAP_BBMAP_BITS || 25727f48121eSThomas Gleixner dirty == VMAP_BBMAP_BITS) 2573ca5e46c3SThomas Gleixner continue; 2574ca5e46c3SThomas Gleixner 2575ca5e46c3SThomas Gleixner spin_lock(&vb->lock); 257677e50af0SThomas Gleixner purge_fragmented_block(vb, vbq, &purge, true); 2577ca5e46c3SThomas Gleixner spin_unlock(&vb->lock); 2578ca5e46c3SThomas Gleixner } 2579ca5e46c3SThomas Gleixner rcu_read_unlock(); 2580ca5e46c3SThomas Gleixner free_purged_blocks(&purge); 2581ca5e46c3SThomas Gleixner } 2582ca5e46c3SThomas Gleixner 258302b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void) 258402b709dfSNick Piggin { 258502b709dfSNick Piggin int cpu; 258602b709dfSNick Piggin 258702b709dfSNick Piggin for_each_possible_cpu(cpu) 258802b709dfSNick Piggin purge_fragmented_blocks(cpu); 258902b709dfSNick Piggin } 259002b709dfSNick Piggin 2591db64fe02SNick Piggin static void *vb_alloc(unsigned long size, gfp_t gfp_mask) 2592db64fe02SNick Piggin { 2593db64fe02SNick Piggin struct vmap_block_queue *vbq; 2594db64fe02SNick Piggin struct vmap_block *vb; 2595cf725ce2SRoman Pen void *vaddr = NULL; 2596db64fe02SNick Piggin unsigned int order; 2597db64fe02SNick Piggin 2598891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 2599db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 2600aa91c4d8SJan Kara if (WARN_ON(size == 0)) { 2601aa91c4d8SJan Kara /* 2602aa91c4d8SJan Kara * Allocating 0 bytes isn't what caller wants since 2603aa91c4d8SJan Kara * get_order(0) returns funny result. Just warn and terminate 2604aa91c4d8SJan Kara * early. 2605aa91c4d8SJan Kara */ 2606aa91c4d8SJan Kara return NULL; 2607aa91c4d8SJan Kara } 2608db64fe02SNick Piggin order = get_order(size); 2609db64fe02SNick Piggin 2610db64fe02SNick Piggin rcu_read_lock(); 26113f804920SSebastian Andrzej Siewior vbq = raw_cpu_ptr(&vmap_block_queue); 2612db64fe02SNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 2613cf725ce2SRoman Pen unsigned long pages_off; 2614db64fe02SNick Piggin 261543d76502SThomas Gleixner if (READ_ONCE(vb->free) < (1UL << order)) 261643d76502SThomas Gleixner continue; 261743d76502SThomas Gleixner 2618db64fe02SNick Piggin spin_lock(&vb->lock); 2619cf725ce2SRoman Pen if (vb->free < (1UL << order)) { 2620cf725ce2SRoman Pen spin_unlock(&vb->lock); 2621cf725ce2SRoman Pen continue; 2622cf725ce2SRoman Pen } 262302b709dfSNick Piggin 2624cf725ce2SRoman Pen pages_off = VMAP_BBMAP_BITS - vb->free; 2625cf725ce2SRoman Pen vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); 262643d76502SThomas Gleixner WRITE_ONCE(vb->free, vb->free - (1UL << order)); 2627d76f9954SBaoquan He bitmap_set(vb->used_map, pages_off, (1UL << order)); 2628db64fe02SNick Piggin if (vb->free == 0) { 2629db64fe02SNick Piggin spin_lock(&vbq->lock); 2630de560423SNick Piggin list_del_rcu(&vb->free_list); 2631db64fe02SNick Piggin spin_unlock(&vbq->lock); 2632db64fe02SNick Piggin } 2633cf725ce2SRoman Pen 2634db64fe02SNick Piggin spin_unlock(&vb->lock); 2635db64fe02SNick Piggin break; 2636db64fe02SNick Piggin } 263702b709dfSNick Piggin 2638db64fe02SNick Piggin rcu_read_unlock(); 2639db64fe02SNick Piggin 2640cf725ce2SRoman Pen /* Allocate new block if nothing was found */ 2641cf725ce2SRoman Pen if (!vaddr) 2642cf725ce2SRoman Pen vaddr = new_vmap_block(order, gfp_mask); 2643db64fe02SNick Piggin 2644cf725ce2SRoman Pen return vaddr; 2645db64fe02SNick Piggin } 2646db64fe02SNick Piggin 264778a0e8c4SChristoph Hellwig static void vb_free(unsigned long addr, unsigned long size) 2648db64fe02SNick Piggin { 2649db64fe02SNick Piggin unsigned long offset; 2650db64fe02SNick Piggin unsigned int order; 2651db64fe02SNick Piggin struct vmap_block *vb; 2652062eacf5SUladzislau Rezki (Sony) struct xarray *xa; 2653db64fe02SNick Piggin 2654891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 2655db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 2656b29acbdcSNick Piggin 265778a0e8c4SChristoph Hellwig flush_cache_vunmap(addr, addr + size); 2658b29acbdcSNick Piggin 2659db64fe02SNick Piggin order = get_order(size); 266078a0e8c4SChristoph Hellwig offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT; 2661062eacf5SUladzislau Rezki (Sony) 2662fa1c77c1SUladzislau Rezki (Sony) xa = addr_to_vb_xa(addr); 2663062eacf5SUladzislau Rezki (Sony) vb = xa_load(xa, addr_to_vb_idx(addr)); 2664062eacf5SUladzislau Rezki (Sony) 2665d76f9954SBaoquan He spin_lock(&vb->lock); 2666d76f9954SBaoquan He bitmap_clear(vb->used_map, offset, (1UL << order)); 2667d76f9954SBaoquan He spin_unlock(&vb->lock); 2668db64fe02SNick Piggin 26694ad0ae8cSNicholas Piggin vunmap_range_noflush(addr, addr + size); 267064141da5SJeremy Fitzhardinge 26718e57f8acSVlastimil Babka if (debug_pagealloc_enabled_static()) 267278a0e8c4SChristoph Hellwig flush_tlb_kernel_range(addr, addr + size); 267382a2e924SChintan Pandya 2674db64fe02SNick Piggin spin_lock(&vb->lock); 26757d61bfe8SRoman Pen 2676a09fad96SThomas Gleixner /* Expand the not yet TLB flushed dirty range */ 26777d61bfe8SRoman Pen vb->dirty_min = min(vb->dirty_min, offset); 26787d61bfe8SRoman Pen vb->dirty_max = max(vb->dirty_max, offset + (1UL << order)); 2679d086817dSMinChan Kim 26807f48121eSThomas Gleixner WRITE_ONCE(vb->dirty, vb->dirty + (1UL << order)); 2681db64fe02SNick Piggin if (vb->dirty == VMAP_BBMAP_BITS) { 2682de560423SNick Piggin BUG_ON(vb->free); 2683db64fe02SNick Piggin spin_unlock(&vb->lock); 2684db64fe02SNick Piggin free_vmap_block(vb); 2685db64fe02SNick Piggin } else 2686db64fe02SNick Piggin spin_unlock(&vb->lock); 2687db64fe02SNick Piggin } 2688db64fe02SNick Piggin 2689868b104dSRick Edgecombe static void _vm_unmap_aliases(unsigned long start, unsigned long end, int flush) 2690db64fe02SNick Piggin { 2691ca5e46c3SThomas Gleixner LIST_HEAD(purge_list); 2692db64fe02SNick Piggin int cpu; 2693db64fe02SNick Piggin 26949b463334SJeremy Fitzhardinge if (unlikely(!vmap_initialized)) 26959b463334SJeremy Fitzhardinge return; 26969b463334SJeremy Fitzhardinge 2697ca5e46c3SThomas Gleixner mutex_lock(&vmap_purge_lock); 26985803ed29SChristoph Hellwig 2699db64fe02SNick Piggin for_each_possible_cpu(cpu) { 2700db64fe02SNick Piggin struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 2701db64fe02SNick Piggin struct vmap_block *vb; 2702fc1e0d98SThomas Gleixner unsigned long idx; 2703db64fe02SNick Piggin 2704db64fe02SNick Piggin rcu_read_lock(); 2705fc1e0d98SThomas Gleixner xa_for_each(&vbq->vmap_blocks, idx, vb) { 2706db64fe02SNick Piggin spin_lock(&vb->lock); 2707ca5e46c3SThomas Gleixner 2708ca5e46c3SThomas Gleixner /* 2709ca5e46c3SThomas Gleixner * Try to purge a fragmented block first. If it's 2710ca5e46c3SThomas Gleixner * not purgeable, check whether there is dirty 2711ca5e46c3SThomas Gleixner * space to be flushed. 2712ca5e46c3SThomas Gleixner */ 271377e50af0SThomas Gleixner if (!purge_fragmented_block(vb, vbq, &purge_list, false) && 2714a09fad96SThomas Gleixner vb->dirty_max && vb->dirty != VMAP_BBMAP_BITS) { 27157d61bfe8SRoman Pen unsigned long va_start = vb->va->va_start; 2716db64fe02SNick Piggin unsigned long s, e; 2717b136be5eSJoonsoo Kim 27187d61bfe8SRoman Pen s = va_start + (vb->dirty_min << PAGE_SHIFT); 27197d61bfe8SRoman Pen e = va_start + (vb->dirty_max << PAGE_SHIFT); 2720db64fe02SNick Piggin 27217d61bfe8SRoman Pen start = min(s, start); 27227d61bfe8SRoman Pen end = max(e, end); 27237d61bfe8SRoman Pen 2724a09fad96SThomas Gleixner /* Prevent that this is flushed again */ 2725a09fad96SThomas Gleixner vb->dirty_min = VMAP_BBMAP_BITS; 2726a09fad96SThomas Gleixner vb->dirty_max = 0; 2727a09fad96SThomas Gleixner 2728db64fe02SNick Piggin flush = 1; 2729db64fe02SNick Piggin } 2730db64fe02SNick Piggin spin_unlock(&vb->lock); 2731db64fe02SNick Piggin } 2732db64fe02SNick Piggin rcu_read_unlock(); 2733db64fe02SNick Piggin } 2734ca5e46c3SThomas Gleixner free_purged_blocks(&purge_list); 2735db64fe02SNick Piggin 273672210662SUladzislau Rezki (Sony) if (!__purge_vmap_area_lazy(start, end, false) && flush) 27370574ecd1SChristoph Hellwig flush_tlb_kernel_range(start, end); 2738f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 2739db64fe02SNick Piggin } 2740868b104dSRick Edgecombe 2741868b104dSRick Edgecombe /** 2742868b104dSRick Edgecombe * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer 2743868b104dSRick Edgecombe * 2744868b104dSRick Edgecombe * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily 2745868b104dSRick Edgecombe * to amortize TLB flushing overheads. What this means is that any page you 2746868b104dSRick Edgecombe * have now, may, in a former life, have been mapped into kernel virtual 2747868b104dSRick Edgecombe * address by the vmap layer and so there might be some CPUs with TLB entries 2748868b104dSRick Edgecombe * still referencing that page (additional to the regular 1:1 kernel mapping). 2749868b104dSRick Edgecombe * 2750868b104dSRick Edgecombe * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can 2751868b104dSRick Edgecombe * be sure that none of the pages we have control over will have any aliases 2752868b104dSRick Edgecombe * from the vmap layer. 2753868b104dSRick Edgecombe */ 2754868b104dSRick Edgecombe void vm_unmap_aliases(void) 2755868b104dSRick Edgecombe { 2756868b104dSRick Edgecombe unsigned long start = ULONG_MAX, end = 0; 2757868b104dSRick Edgecombe int flush = 0; 2758868b104dSRick Edgecombe 2759868b104dSRick Edgecombe _vm_unmap_aliases(start, end, flush); 2760868b104dSRick Edgecombe } 2761db64fe02SNick Piggin EXPORT_SYMBOL_GPL(vm_unmap_aliases); 2762db64fe02SNick Piggin 2763db64fe02SNick Piggin /** 2764db64fe02SNick Piggin * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram 2765db64fe02SNick Piggin * @mem: the pointer returned by vm_map_ram 2766db64fe02SNick Piggin * @count: the count passed to that vm_map_ram call (cannot unmap partial) 2767db64fe02SNick Piggin */ 2768db64fe02SNick Piggin void vm_unmap_ram(const void *mem, unsigned int count) 2769db64fe02SNick Piggin { 277065ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 27714aff1dc4SAndrey Konovalov unsigned long addr = (unsigned long)kasan_reset_tag(mem); 27729c3acf60SChristoph Hellwig struct vmap_area *va; 2773db64fe02SNick Piggin 27745803ed29SChristoph Hellwig might_sleep(); 2775db64fe02SNick Piggin BUG_ON(!addr); 2776db64fe02SNick Piggin BUG_ON(addr < VMALLOC_START); 2777db64fe02SNick Piggin BUG_ON(addr > VMALLOC_END); 2778a1c0b1a0SShawn Lin BUG_ON(!PAGE_ALIGNED(addr)); 2779db64fe02SNick Piggin 2780d98c9e83SAndrey Ryabinin kasan_poison_vmalloc(mem, size); 2781d98c9e83SAndrey Ryabinin 27829c3acf60SChristoph Hellwig if (likely(count <= VMAP_MAX_ALLOC)) { 278305e3ff95SChintan Pandya debug_check_no_locks_freed(mem, size); 278478a0e8c4SChristoph Hellwig vb_free(addr, size); 27859c3acf60SChristoph Hellwig return; 27869c3acf60SChristoph Hellwig } 27879c3acf60SChristoph Hellwig 2788edd89818SUladzislau Rezki (Sony) va = find_unlink_vmap_area(addr); 278914687619SUladzislau Rezki (Sony) if (WARN_ON_ONCE(!va)) 279014687619SUladzislau Rezki (Sony) return; 279114687619SUladzislau Rezki (Sony) 279205e3ff95SChintan Pandya debug_check_no_locks_freed((void *)va->va_start, 279305e3ff95SChintan Pandya (va->va_end - va->va_start)); 27949c3acf60SChristoph Hellwig free_unmap_vmap_area(va); 2795db64fe02SNick Piggin } 2796db64fe02SNick Piggin EXPORT_SYMBOL(vm_unmap_ram); 2797db64fe02SNick Piggin 2798db64fe02SNick Piggin /** 2799db64fe02SNick Piggin * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) 2800db64fe02SNick Piggin * @pages: an array of pointers to the pages to be mapped 2801db64fe02SNick Piggin * @count: number of pages 2802db64fe02SNick Piggin * @node: prefer to allocate data structures on this node 2803e99c97adSRandy Dunlap * 280436437638SGioh Kim * If you use this function for less than VMAP_MAX_ALLOC pages, it could be 280536437638SGioh Kim * faster than vmap so it's good. But if you mix long-life and short-life 280636437638SGioh Kim * objects with vm_map_ram(), it could consume lots of address space through 280736437638SGioh Kim * fragmentation (especially on a 32bit machine). You could see failures in 280836437638SGioh Kim * the end. Please use this function for short-lived objects. 280936437638SGioh Kim * 2810e99c97adSRandy Dunlap * Returns: a pointer to the address that has been mapped, or %NULL on failure 2811db64fe02SNick Piggin */ 2812d4efd79aSChristoph Hellwig void *vm_map_ram(struct page **pages, unsigned int count, int node) 2813db64fe02SNick Piggin { 281465ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 2815db64fe02SNick Piggin unsigned long addr; 2816db64fe02SNick Piggin void *mem; 2817db64fe02SNick Piggin 2818db64fe02SNick Piggin if (likely(count <= VMAP_MAX_ALLOC)) { 2819db64fe02SNick Piggin mem = vb_alloc(size, GFP_KERNEL); 2820db64fe02SNick Piggin if (IS_ERR(mem)) 2821db64fe02SNick Piggin return NULL; 2822db64fe02SNick Piggin addr = (unsigned long)mem; 2823db64fe02SNick Piggin } else { 2824db64fe02SNick Piggin struct vmap_area *va; 2825db64fe02SNick Piggin va = alloc_vmap_area(size, PAGE_SIZE, 2826869176a0SBaoquan He VMALLOC_START, VMALLOC_END, 2827869176a0SBaoquan He node, GFP_KERNEL, VMAP_RAM); 2828db64fe02SNick Piggin if (IS_ERR(va)) 2829db64fe02SNick Piggin return NULL; 2830db64fe02SNick Piggin 2831db64fe02SNick Piggin addr = va->va_start; 2832db64fe02SNick Piggin mem = (void *)addr; 2833db64fe02SNick Piggin } 2834d98c9e83SAndrey Ryabinin 2835b67177ecSNicholas Piggin if (vmap_pages_range(addr, addr + size, PAGE_KERNEL, 2836b67177ecSNicholas Piggin pages, PAGE_SHIFT) < 0) { 2837db64fe02SNick Piggin vm_unmap_ram(mem, count); 2838db64fe02SNick Piggin return NULL; 2839db64fe02SNick Piggin } 2840b67177ecSNicholas Piggin 284123689e91SAndrey Konovalov /* 284223689e91SAndrey Konovalov * Mark the pages as accessible, now that they are mapped. 284323689e91SAndrey Konovalov * With hardware tag-based KASAN, marking is skipped for 284423689e91SAndrey Konovalov * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 284523689e91SAndrey Konovalov */ 2846f6e39794SAndrey Konovalov mem = kasan_unpoison_vmalloc(mem, size, KASAN_VMALLOC_PROT_NORMAL); 284719f1c3acSAndrey Konovalov 2848db64fe02SNick Piggin return mem; 2849db64fe02SNick Piggin } 2850db64fe02SNick Piggin EXPORT_SYMBOL(vm_map_ram); 2851db64fe02SNick Piggin 28524341fa45SJoonsoo Kim static struct vm_struct *vmlist __initdata; 285392eac168SMike Rapoport 2854121e6f32SNicholas Piggin static inline unsigned int vm_area_page_order(struct vm_struct *vm) 2855121e6f32SNicholas Piggin { 2856121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 2857121e6f32SNicholas Piggin return vm->page_order; 2858121e6f32SNicholas Piggin #else 2859121e6f32SNicholas Piggin return 0; 2860121e6f32SNicholas Piggin #endif 2861121e6f32SNicholas Piggin } 2862121e6f32SNicholas Piggin 2863121e6f32SNicholas Piggin static inline void set_vm_area_page_order(struct vm_struct *vm, unsigned int order) 2864121e6f32SNicholas Piggin { 2865121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 2866121e6f32SNicholas Piggin vm->page_order = order; 2867121e6f32SNicholas Piggin #else 2868121e6f32SNicholas Piggin BUG_ON(order != 0); 2869121e6f32SNicholas Piggin #endif 2870121e6f32SNicholas Piggin } 2871121e6f32SNicholas Piggin 2872f0aa6617STejun Heo /** 2873be9b7335SNicolas Pitre * vm_area_add_early - add vmap area early during boot 2874be9b7335SNicolas Pitre * @vm: vm_struct to add 2875be9b7335SNicolas Pitre * 2876be9b7335SNicolas Pitre * This function is used to add fixed kernel vm area to vmlist before 2877be9b7335SNicolas Pitre * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags 2878be9b7335SNicolas Pitre * should contain proper values and the other fields should be zero. 2879be9b7335SNicolas Pitre * 2880be9b7335SNicolas Pitre * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 2881be9b7335SNicolas Pitre */ 2882be9b7335SNicolas Pitre void __init vm_area_add_early(struct vm_struct *vm) 2883be9b7335SNicolas Pitre { 2884be9b7335SNicolas Pitre struct vm_struct *tmp, **p; 2885be9b7335SNicolas Pitre 2886be9b7335SNicolas Pitre BUG_ON(vmap_initialized); 2887be9b7335SNicolas Pitre for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { 2888be9b7335SNicolas Pitre if (tmp->addr >= vm->addr) { 2889be9b7335SNicolas Pitre BUG_ON(tmp->addr < vm->addr + vm->size); 2890be9b7335SNicolas Pitre break; 2891be9b7335SNicolas Pitre } else 2892be9b7335SNicolas Pitre BUG_ON(tmp->addr + tmp->size > vm->addr); 2893be9b7335SNicolas Pitre } 2894be9b7335SNicolas Pitre vm->next = *p; 2895be9b7335SNicolas Pitre *p = vm; 2896be9b7335SNicolas Pitre } 2897be9b7335SNicolas Pitre 2898be9b7335SNicolas Pitre /** 2899f0aa6617STejun Heo * vm_area_register_early - register vmap area early during boot 2900f0aa6617STejun Heo * @vm: vm_struct to register 2901c0c0a293STejun Heo * @align: requested alignment 2902f0aa6617STejun Heo * 2903f0aa6617STejun Heo * This function is used to register kernel vm area before 2904f0aa6617STejun Heo * vmalloc_init() is called. @vm->size and @vm->flags should contain 2905f0aa6617STejun Heo * proper values on entry and other fields should be zero. On return, 2906f0aa6617STejun Heo * vm->addr contains the allocated address. 2907f0aa6617STejun Heo * 2908f0aa6617STejun Heo * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 2909f0aa6617STejun Heo */ 2910c0c0a293STejun Heo void __init vm_area_register_early(struct vm_struct *vm, size_t align) 2911f0aa6617STejun Heo { 29120eb68437SKefeng Wang unsigned long addr = ALIGN(VMALLOC_START, align); 29130eb68437SKefeng Wang struct vm_struct *cur, **p; 2914f0aa6617STejun Heo 29150eb68437SKefeng Wang BUG_ON(vmap_initialized); 2916c0c0a293STejun Heo 29170eb68437SKefeng Wang for (p = &vmlist; (cur = *p) != NULL; p = &cur->next) { 29180eb68437SKefeng Wang if ((unsigned long)cur->addr - addr >= vm->size) 29190eb68437SKefeng Wang break; 29200eb68437SKefeng Wang addr = ALIGN((unsigned long)cur->addr + cur->size, align); 29210eb68437SKefeng Wang } 29220eb68437SKefeng Wang 29230eb68437SKefeng Wang BUG_ON(addr > VMALLOC_END - vm->size); 2924c0c0a293STejun Heo vm->addr = (void *)addr; 29250eb68437SKefeng Wang vm->next = *p; 29260eb68437SKefeng Wang *p = vm; 29273252b1d8SKefeng Wang kasan_populate_early_vm_area_shadow(vm->addr, vm->size); 2928f0aa6617STejun Heo } 2929f0aa6617STejun Heo 2930e36176beSUladzislau Rezki (Sony) static inline void setup_vmalloc_vm_locked(struct vm_struct *vm, 2931e36176beSUladzislau Rezki (Sony) struct vmap_area *va, unsigned long flags, const void *caller) 2932cf88c790STejun Heo { 2933cf88c790STejun Heo vm->flags = flags; 2934cf88c790STejun Heo vm->addr = (void *)va->va_start; 2935cf88c790STejun Heo vm->size = va->va_end - va->va_start; 2936cf88c790STejun Heo vm->caller = caller; 2937db1aecafSMinchan Kim va->vm = vm; 2938e36176beSUladzislau Rezki (Sony) } 2939e36176beSUladzislau Rezki (Sony) 2940e36176beSUladzislau Rezki (Sony) static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, 2941e36176beSUladzislau Rezki (Sony) unsigned long flags, const void *caller) 2942e36176beSUladzislau Rezki (Sony) { 2943d0936029SUladzislau Rezki (Sony) struct vmap_node *vn = addr_to_node(va->va_start); 2944d0936029SUladzislau Rezki (Sony) 2945d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 2946e36176beSUladzislau Rezki (Sony) setup_vmalloc_vm_locked(vm, va, flags, caller); 2947d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 2948f5252e00SMitsuo Hayasaka } 2949cf88c790STejun Heo 295020fc02b4SZhang Yanfei static void clear_vm_uninitialized_flag(struct vm_struct *vm) 2951f5252e00SMitsuo Hayasaka { 2952d4033afdSJoonsoo Kim /* 295320fc02b4SZhang Yanfei * Before removing VM_UNINITIALIZED, 2954d4033afdSJoonsoo Kim * we should make sure that vm has proper values. 2955d4033afdSJoonsoo Kim * Pair with smp_rmb() in show_numa_info(). 2956d4033afdSJoonsoo Kim */ 2957d4033afdSJoonsoo Kim smp_wmb(); 295820fc02b4SZhang Yanfei vm->flags &= ~VM_UNINITIALIZED; 2959cf88c790STejun Heo } 2960cf88c790STejun Heo 2961db64fe02SNick Piggin static struct vm_struct *__get_vm_area_node(unsigned long size, 29627ca3027bSDaniel Axtens unsigned long align, unsigned long shift, unsigned long flags, 29637ca3027bSDaniel Axtens unsigned long start, unsigned long end, int node, 29647ca3027bSDaniel Axtens gfp_t gfp_mask, const void *caller) 2965db64fe02SNick Piggin { 29660006526dSKautuk Consul struct vmap_area *va; 2967db64fe02SNick Piggin struct vm_struct *area; 2968d98c9e83SAndrey Ryabinin unsigned long requested_size = size; 29691da177e4SLinus Torvalds 297052fd24caSGiridhar Pemmasani BUG_ON(in_interrupt()); 29717ca3027bSDaniel Axtens size = ALIGN(size, 1ul << shift); 297231be8309SOGAWA Hirofumi if (unlikely(!size)) 297331be8309SOGAWA Hirofumi return NULL; 29741da177e4SLinus Torvalds 2975252e5c6eSzijun_hu if (flags & VM_IOREMAP) 2976252e5c6eSzijun_hu align = 1ul << clamp_t(int, get_count_order_long(size), 2977252e5c6eSzijun_hu PAGE_SHIFT, IOREMAP_MAX_ORDER); 2978252e5c6eSzijun_hu 2979cf88c790STejun Heo area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); 29801da177e4SLinus Torvalds if (unlikely(!area)) 29811da177e4SLinus Torvalds return NULL; 29821da177e4SLinus Torvalds 298371394fe5SAndrey Ryabinin if (!(flags & VM_NO_GUARD)) 29841da177e4SLinus Torvalds size += PAGE_SIZE; 29851da177e4SLinus Torvalds 2986869176a0SBaoquan He va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0); 2987db64fe02SNick Piggin if (IS_ERR(va)) { 2988db64fe02SNick Piggin kfree(area); 2989db64fe02SNick Piggin return NULL; 29901da177e4SLinus Torvalds } 29911da177e4SLinus Torvalds 2992d98c9e83SAndrey Ryabinin setup_vmalloc_vm(area, va, flags, caller); 29933c5c3cfbSDaniel Axtens 299419f1c3acSAndrey Konovalov /* 299519f1c3acSAndrey Konovalov * Mark pages for non-VM_ALLOC mappings as accessible. Do it now as a 299619f1c3acSAndrey Konovalov * best-effort approach, as they can be mapped outside of vmalloc code. 299719f1c3acSAndrey Konovalov * For VM_ALLOC mappings, the pages are marked as accessible after 299819f1c3acSAndrey Konovalov * getting mapped in __vmalloc_node_range(). 299923689e91SAndrey Konovalov * With hardware tag-based KASAN, marking is skipped for 300023689e91SAndrey Konovalov * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 300119f1c3acSAndrey Konovalov */ 300219f1c3acSAndrey Konovalov if (!(flags & VM_ALLOC)) 300323689e91SAndrey Konovalov area->addr = kasan_unpoison_vmalloc(area->addr, requested_size, 3004f6e39794SAndrey Konovalov KASAN_VMALLOC_PROT_NORMAL); 30051d96320fSAndrey Konovalov 30061da177e4SLinus Torvalds return area; 30071da177e4SLinus Torvalds } 30081da177e4SLinus Torvalds 3009c2968612SBenjamin Herrenschmidt struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, 3010c2968612SBenjamin Herrenschmidt unsigned long start, unsigned long end, 30115e6cafc8SMarek Szyprowski const void *caller) 3012c2968612SBenjamin Herrenschmidt { 30137ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, start, end, 30147ca3027bSDaniel Axtens NUMA_NO_NODE, GFP_KERNEL, caller); 3015c2968612SBenjamin Herrenschmidt } 3016c2968612SBenjamin Herrenschmidt 30171da177e4SLinus Torvalds /** 3018183ff22bSSimon Arlott * get_vm_area - reserve a contiguous kernel virtual area 30191da177e4SLinus Torvalds * @size: size of the area 30201da177e4SLinus Torvalds * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC 30211da177e4SLinus Torvalds * 30221da177e4SLinus Torvalds * Search an area of @size in the kernel virtual mapping area, 30231da177e4SLinus Torvalds * and reserved it for out purposes. Returns the area descriptor 30241da177e4SLinus Torvalds * on success or %NULL on failure. 3025a862f68aSMike Rapoport * 3026a862f68aSMike Rapoport * Return: the area descriptor on success or %NULL on failure. 30271da177e4SLinus Torvalds */ 30281da177e4SLinus Torvalds struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) 30291da177e4SLinus Torvalds { 30307ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, 30317ca3027bSDaniel Axtens VMALLOC_START, VMALLOC_END, 303200ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, 303300ef2d2fSDavid Rientjes __builtin_return_address(0)); 303423016969SChristoph Lameter } 303523016969SChristoph Lameter 303623016969SChristoph Lameter struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, 30375e6cafc8SMarek Szyprowski const void *caller) 303823016969SChristoph Lameter { 30397ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, 30407ca3027bSDaniel Axtens VMALLOC_START, VMALLOC_END, 304100ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, caller); 30421da177e4SLinus Torvalds } 30431da177e4SLinus Torvalds 3044e9da6e99SMarek Szyprowski /** 3045e9da6e99SMarek Szyprowski * find_vm_area - find a continuous kernel virtual area 3046e9da6e99SMarek Szyprowski * @addr: base address 3047e9da6e99SMarek Szyprowski * 3048e9da6e99SMarek Szyprowski * Search for the kernel VM area starting at @addr, and return it. 3049e9da6e99SMarek Szyprowski * It is up to the caller to do all required locking to keep the returned 3050e9da6e99SMarek Szyprowski * pointer valid. 3051a862f68aSMike Rapoport * 305274640617SHui Su * Return: the area descriptor on success or %NULL on failure. 3053e9da6e99SMarek Szyprowski */ 3054e9da6e99SMarek Szyprowski struct vm_struct *find_vm_area(const void *addr) 305583342314SNick Piggin { 3056db64fe02SNick Piggin struct vmap_area *va; 305783342314SNick Piggin 3058db64fe02SNick Piggin va = find_vmap_area((unsigned long)addr); 3059688fcbfcSPengfei Li if (!va) 30607856dfebSAndi Kleen return NULL; 3061688fcbfcSPengfei Li 3062688fcbfcSPengfei Li return va->vm; 30637856dfebSAndi Kleen } 30647856dfebSAndi Kleen 30651da177e4SLinus Torvalds /** 3066183ff22bSSimon Arlott * remove_vm_area - find and remove a continuous kernel virtual area 30671da177e4SLinus Torvalds * @addr: base address 30681da177e4SLinus Torvalds * 30691da177e4SLinus Torvalds * Search for the kernel VM area starting at @addr, and remove it. 30701da177e4SLinus Torvalds * This function returns the found VM area, but using it is NOT safe 30717856dfebSAndi Kleen * on SMP machines, except for its size or flags. 3072a862f68aSMike Rapoport * 307374640617SHui Su * Return: the area descriptor on success or %NULL on failure. 30741da177e4SLinus Torvalds */ 3075b3bdda02SChristoph Lameter struct vm_struct *remove_vm_area(const void *addr) 30761da177e4SLinus Torvalds { 3077db64fe02SNick Piggin struct vmap_area *va; 307875c59ce7SChristoph Hellwig struct vm_struct *vm; 3079db64fe02SNick Piggin 30805803ed29SChristoph Hellwig might_sleep(); 30815803ed29SChristoph Hellwig 308217d3ef43SChristoph Hellwig if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n", 308317d3ef43SChristoph Hellwig addr)) 3084db64fe02SNick Piggin return NULL; 308517d3ef43SChristoph Hellwig 308675c59ce7SChristoph Hellwig va = find_unlink_vmap_area((unsigned long)addr); 308775c59ce7SChristoph Hellwig if (!va || !va->vm) 308875c59ce7SChristoph Hellwig return NULL; 308975c59ce7SChristoph Hellwig vm = va->vm; 309017d3ef43SChristoph Hellwig 309117d3ef43SChristoph Hellwig debug_check_no_locks_freed(vm->addr, get_vm_area_size(vm)); 309217d3ef43SChristoph Hellwig debug_check_no_obj_freed(vm->addr, get_vm_area_size(vm)); 309375c59ce7SChristoph Hellwig kasan_free_module_shadow(vm); 309417d3ef43SChristoph Hellwig kasan_poison_vmalloc(vm->addr, get_vm_area_size(vm)); 309517d3ef43SChristoph Hellwig 309675c59ce7SChristoph Hellwig free_unmap_vmap_area(va); 309775c59ce7SChristoph Hellwig return vm; 30981da177e4SLinus Torvalds } 30991da177e4SLinus Torvalds 3100868b104dSRick Edgecombe static inline void set_area_direct_map(const struct vm_struct *area, 3101868b104dSRick Edgecombe int (*set_direct_map)(struct page *page)) 3102868b104dSRick Edgecombe { 3103868b104dSRick Edgecombe int i; 3104868b104dSRick Edgecombe 3105121e6f32SNicholas Piggin /* HUGE_VMALLOC passes small pages to set_direct_map */ 3106868b104dSRick Edgecombe for (i = 0; i < area->nr_pages; i++) 3107868b104dSRick Edgecombe if (page_address(area->pages[i])) 3108868b104dSRick Edgecombe set_direct_map(area->pages[i]); 3109868b104dSRick Edgecombe } 3110868b104dSRick Edgecombe 31119e5fa0aeSChristoph Hellwig /* 31129e5fa0aeSChristoph Hellwig * Flush the vm mapping and reset the direct map. 31139e5fa0aeSChristoph Hellwig */ 31149e5fa0aeSChristoph Hellwig static void vm_reset_perms(struct vm_struct *area) 3115868b104dSRick Edgecombe { 3116868b104dSRick Edgecombe unsigned long start = ULONG_MAX, end = 0; 3117121e6f32SNicholas Piggin unsigned int page_order = vm_area_page_order(area); 311831e67340SRick Edgecombe int flush_dmap = 0; 3119868b104dSRick Edgecombe int i; 3120868b104dSRick Edgecombe 3121868b104dSRick Edgecombe /* 31229e5fa0aeSChristoph Hellwig * Find the start and end range of the direct mappings to make sure that 3123868b104dSRick Edgecombe * the vm_unmap_aliases() flush includes the direct map. 3124868b104dSRick Edgecombe */ 3125121e6f32SNicholas Piggin for (i = 0; i < area->nr_pages; i += 1U << page_order) { 31268e41f872SRick Edgecombe unsigned long addr = (unsigned long)page_address(area->pages[i]); 31279e5fa0aeSChristoph Hellwig 31288e41f872SRick Edgecombe if (addr) { 3129121e6f32SNicholas Piggin unsigned long page_size; 3130121e6f32SNicholas Piggin 3131121e6f32SNicholas Piggin page_size = PAGE_SIZE << page_order; 3132868b104dSRick Edgecombe start = min(addr, start); 3133121e6f32SNicholas Piggin end = max(addr + page_size, end); 313431e67340SRick Edgecombe flush_dmap = 1; 3135868b104dSRick Edgecombe } 3136868b104dSRick Edgecombe } 3137868b104dSRick Edgecombe 3138868b104dSRick Edgecombe /* 3139868b104dSRick Edgecombe * Set direct map to something invalid so that it won't be cached if 3140868b104dSRick Edgecombe * there are any accesses after the TLB flush, then flush the TLB and 3141868b104dSRick Edgecombe * reset the direct map permissions to the default. 3142868b104dSRick Edgecombe */ 3143868b104dSRick Edgecombe set_area_direct_map(area, set_direct_map_invalid_noflush); 314431e67340SRick Edgecombe _vm_unmap_aliases(start, end, flush_dmap); 3145868b104dSRick Edgecombe set_area_direct_map(area, set_direct_map_default_noflush); 3146868b104dSRick Edgecombe } 3147868b104dSRick Edgecombe 3148208162f4SChristoph Hellwig static void delayed_vfree_work(struct work_struct *w) 31491da177e4SLinus Torvalds { 3150208162f4SChristoph Hellwig struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); 3151208162f4SChristoph Hellwig struct llist_node *t, *llnode; 31521da177e4SLinus Torvalds 3153208162f4SChristoph Hellwig llist_for_each_safe(llnode, t, llist_del_all(&p->list)) 31545d3d31d6SChristoph Hellwig vfree(llnode); 3155bf22e37aSAndrey Ryabinin } 3156bf22e37aSAndrey Ryabinin 3157bf22e37aSAndrey Ryabinin /** 3158bf22e37aSAndrey Ryabinin * vfree_atomic - release memory allocated by vmalloc() 3159bf22e37aSAndrey Ryabinin * @addr: memory base address 3160bf22e37aSAndrey Ryabinin * 3161bf22e37aSAndrey Ryabinin * This one is just like vfree() but can be called in any atomic context 3162bf22e37aSAndrey Ryabinin * except NMIs. 3163bf22e37aSAndrey Ryabinin */ 3164bf22e37aSAndrey Ryabinin void vfree_atomic(const void *addr) 3165bf22e37aSAndrey Ryabinin { 316601e2e839SChristoph Hellwig struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred); 3167bf22e37aSAndrey Ryabinin 316801e2e839SChristoph Hellwig BUG_ON(in_nmi()); 3169bf22e37aSAndrey Ryabinin kmemleak_free(addr); 3170bf22e37aSAndrey Ryabinin 317101e2e839SChristoph Hellwig /* 317201e2e839SChristoph Hellwig * Use raw_cpu_ptr() because this can be called from preemptible 317301e2e839SChristoph Hellwig * context. Preemption is absolutely fine here, because the llist_add() 317401e2e839SChristoph Hellwig * implementation is lockless, so it works even if we are adding to 317501e2e839SChristoph Hellwig * another cpu's list. schedule_work() should be fine with this too. 317601e2e839SChristoph Hellwig */ 317701e2e839SChristoph Hellwig if (addr && llist_add((struct llist_node *)addr, &p->list)) 317801e2e839SChristoph Hellwig schedule_work(&p->wq); 3179c67dc624SRoman Penyaev } 3180c67dc624SRoman Penyaev 31811da177e4SLinus Torvalds /** 3182fa307474SMatthew Wilcox (Oracle) * vfree - Release memory allocated by vmalloc() 3183fa307474SMatthew Wilcox (Oracle) * @addr: Memory base address 31841da177e4SLinus Torvalds * 3185fa307474SMatthew Wilcox (Oracle) * Free the virtually continuous memory area starting at @addr, as obtained 3186fa307474SMatthew Wilcox (Oracle) * from one of the vmalloc() family of APIs. This will usually also free the 3187fa307474SMatthew Wilcox (Oracle) * physical memory underlying the virtual allocation, but that memory is 3188fa307474SMatthew Wilcox (Oracle) * reference counted, so it will not be freed until the last user goes away. 31891da177e4SLinus Torvalds * 3190fa307474SMatthew Wilcox (Oracle) * If @addr is NULL, no operation is performed. 319132fcfd40SAl Viro * 3192fa307474SMatthew Wilcox (Oracle) * Context: 31933ca4ea3aSAndrey Ryabinin * May sleep if called *not* from interrupt context. 3194fa307474SMatthew Wilcox (Oracle) * Must not be called in NMI context (strictly speaking, it could be 3195fa307474SMatthew Wilcox (Oracle) * if we have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling 3196f0953a1bSIngo Molnar * conventions for vfree() arch-dependent would be a really bad idea). 31971da177e4SLinus Torvalds */ 3198b3bdda02SChristoph Lameter void vfree(const void *addr) 31991da177e4SLinus Torvalds { 320079311c1fSChristoph Hellwig struct vm_struct *vm; 320179311c1fSChristoph Hellwig int i; 320279311c1fSChristoph Hellwig 320301e2e839SChristoph Hellwig if (unlikely(in_interrupt())) { 320401e2e839SChristoph Hellwig vfree_atomic(addr); 320532fcfd40SAl Viro return; 320601e2e839SChristoph Hellwig } 320701e2e839SChristoph Hellwig 32081da177e4SLinus Torvalds BUG_ON(in_nmi()); 320989219d37SCatalin Marinas kmemleak_free(addr); 321001e2e839SChristoph Hellwig might_sleep(); 321132fcfd40SAl Viro 3212bf22e37aSAndrey Ryabinin if (!addr) 3213bf22e37aSAndrey Ryabinin return; 3214c67dc624SRoman Penyaev 321579311c1fSChristoph Hellwig vm = remove_vm_area(addr); 321679311c1fSChristoph Hellwig if (unlikely(!vm)) { 321779311c1fSChristoph Hellwig WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", 321879311c1fSChristoph Hellwig addr); 321979311c1fSChristoph Hellwig return; 322079311c1fSChristoph Hellwig } 322179311c1fSChristoph Hellwig 32229e5fa0aeSChristoph Hellwig if (unlikely(vm->flags & VM_FLUSH_RESET_PERMS)) 32239e5fa0aeSChristoph Hellwig vm_reset_perms(vm); 322479311c1fSChristoph Hellwig for (i = 0; i < vm->nr_pages; i++) { 322579311c1fSChristoph Hellwig struct page *page = vm->pages[i]; 322679311c1fSChristoph Hellwig 322779311c1fSChristoph Hellwig BUG_ON(!page); 322879311c1fSChristoph Hellwig mod_memcg_page_state(page, MEMCG_VMALLOC, -1); 322979311c1fSChristoph Hellwig /* 323079311c1fSChristoph Hellwig * High-order allocs for huge vmallocs are split, so 323179311c1fSChristoph Hellwig * can be freed as an array of order-0 allocations 323279311c1fSChristoph Hellwig */ 3233dcc1be11SLorenzo Stoakes __free_page(page); 323479311c1fSChristoph Hellwig cond_resched(); 323579311c1fSChristoph Hellwig } 323679311c1fSChristoph Hellwig atomic_long_sub(vm->nr_pages, &nr_vmalloc_pages); 323779311c1fSChristoph Hellwig kvfree(vm->pages); 323879311c1fSChristoph Hellwig kfree(vm); 32391da177e4SLinus Torvalds } 32401da177e4SLinus Torvalds EXPORT_SYMBOL(vfree); 32411da177e4SLinus Torvalds 32421da177e4SLinus Torvalds /** 32431da177e4SLinus Torvalds * vunmap - release virtual mapping obtained by vmap() 32441da177e4SLinus Torvalds * @addr: memory base address 32451da177e4SLinus Torvalds * 32461da177e4SLinus Torvalds * Free the virtually contiguous memory area starting at @addr, 32471da177e4SLinus Torvalds * which was created from the page array passed to vmap(). 32481da177e4SLinus Torvalds * 324980e93effSPekka Enberg * Must not be called in interrupt context. 32501da177e4SLinus Torvalds */ 3251b3bdda02SChristoph Lameter void vunmap(const void *addr) 32521da177e4SLinus Torvalds { 325379311c1fSChristoph Hellwig struct vm_struct *vm; 325479311c1fSChristoph Hellwig 32551da177e4SLinus Torvalds BUG_ON(in_interrupt()); 325634754b69SPeter Zijlstra might_sleep(); 325779311c1fSChristoph Hellwig 325879311c1fSChristoph Hellwig if (!addr) 325979311c1fSChristoph Hellwig return; 326079311c1fSChristoph Hellwig vm = remove_vm_area(addr); 326179311c1fSChristoph Hellwig if (unlikely(!vm)) { 326279311c1fSChristoph Hellwig WARN(1, KERN_ERR "Trying to vunmap() nonexistent vm area (%p)\n", 326379311c1fSChristoph Hellwig addr); 326479311c1fSChristoph Hellwig return; 326579311c1fSChristoph Hellwig } 326679311c1fSChristoph Hellwig kfree(vm); 32671da177e4SLinus Torvalds } 32681da177e4SLinus Torvalds EXPORT_SYMBOL(vunmap); 32691da177e4SLinus Torvalds 32701da177e4SLinus Torvalds /** 32711da177e4SLinus Torvalds * vmap - map an array of pages into virtually contiguous space 32721da177e4SLinus Torvalds * @pages: array of page pointers 32731da177e4SLinus Torvalds * @count: number of pages to map 32741da177e4SLinus Torvalds * @flags: vm_area->flags 32751da177e4SLinus Torvalds * @prot: page protection for the mapping 32761da177e4SLinus Torvalds * 3277b944afc9SChristoph Hellwig * Maps @count pages from @pages into contiguous kernel virtual space. 3278b944afc9SChristoph Hellwig * If @flags contains %VM_MAP_PUT_PAGES the ownership of the pages array itself 3279b944afc9SChristoph Hellwig * (which must be kmalloc or vmalloc memory) and one reference per pages in it 3280b944afc9SChristoph Hellwig * are transferred from the caller to vmap(), and will be freed / dropped when 3281b944afc9SChristoph Hellwig * vfree() is called on the return value. 3282a862f68aSMike Rapoport * 3283a862f68aSMike Rapoport * Return: the address of the area or %NULL on failure 32841da177e4SLinus Torvalds */ 32851da177e4SLinus Torvalds void *vmap(struct page **pages, unsigned int count, 32861da177e4SLinus Torvalds unsigned long flags, pgprot_t prot) 32871da177e4SLinus Torvalds { 32881da177e4SLinus Torvalds struct vm_struct *area; 3289b67177ecSNicholas Piggin unsigned long addr; 329065ee03c4SGuillermo Julián Moreno unsigned long size; /* In bytes */ 32911da177e4SLinus Torvalds 329234754b69SPeter Zijlstra might_sleep(); 329334754b69SPeter Zijlstra 329437f3605eSChristoph Hellwig if (WARN_ON_ONCE(flags & VM_FLUSH_RESET_PERMS)) 329537f3605eSChristoph Hellwig return NULL; 329637f3605eSChristoph Hellwig 3297bd1a8fb2SPeter Zijlstra /* 3298bd1a8fb2SPeter Zijlstra * Your top guard is someone else's bottom guard. Not having a top 3299bd1a8fb2SPeter Zijlstra * guard compromises someone else's mappings too. 3300bd1a8fb2SPeter Zijlstra */ 3301bd1a8fb2SPeter Zijlstra if (WARN_ON_ONCE(flags & VM_NO_GUARD)) 3302bd1a8fb2SPeter Zijlstra flags &= ~VM_NO_GUARD; 3303bd1a8fb2SPeter Zijlstra 3304ca79b0c2SArun KS if (count > totalram_pages()) 33051da177e4SLinus Torvalds return NULL; 33061da177e4SLinus Torvalds 330765ee03c4SGuillermo Julián Moreno size = (unsigned long)count << PAGE_SHIFT; 330865ee03c4SGuillermo Julián Moreno area = get_vm_area_caller(size, flags, __builtin_return_address(0)); 33091da177e4SLinus Torvalds if (!area) 33101da177e4SLinus Torvalds return NULL; 331123016969SChristoph Lameter 3312b67177ecSNicholas Piggin addr = (unsigned long)area->addr; 3313b67177ecSNicholas Piggin if (vmap_pages_range(addr, addr + size, pgprot_nx(prot), 3314b67177ecSNicholas Piggin pages, PAGE_SHIFT) < 0) { 33151da177e4SLinus Torvalds vunmap(area->addr); 33161da177e4SLinus Torvalds return NULL; 33171da177e4SLinus Torvalds } 33181da177e4SLinus Torvalds 3319c22ee528SMiaohe Lin if (flags & VM_MAP_PUT_PAGES) { 3320b944afc9SChristoph Hellwig area->pages = pages; 3321c22ee528SMiaohe Lin area->nr_pages = count; 3322c22ee528SMiaohe Lin } 33231da177e4SLinus Torvalds return area->addr; 33241da177e4SLinus Torvalds } 33251da177e4SLinus Torvalds EXPORT_SYMBOL(vmap); 33261da177e4SLinus Torvalds 33273e9a9e25SChristoph Hellwig #ifdef CONFIG_VMAP_PFN 33283e9a9e25SChristoph Hellwig struct vmap_pfn_data { 33293e9a9e25SChristoph Hellwig unsigned long *pfns; 33303e9a9e25SChristoph Hellwig pgprot_t prot; 33313e9a9e25SChristoph Hellwig unsigned int idx; 33323e9a9e25SChristoph Hellwig }; 33333e9a9e25SChristoph Hellwig 33343e9a9e25SChristoph Hellwig static int vmap_pfn_apply(pte_t *pte, unsigned long addr, void *private) 33353e9a9e25SChristoph Hellwig { 33363e9a9e25SChristoph Hellwig struct vmap_pfn_data *data = private; 3337b3f78e74SRyan Roberts unsigned long pfn = data->pfns[data->idx]; 3338b3f78e74SRyan Roberts pte_t ptent; 33393e9a9e25SChristoph Hellwig 3340b3f78e74SRyan Roberts if (WARN_ON_ONCE(pfn_valid(pfn))) 33413e9a9e25SChristoph Hellwig return -EINVAL; 3342b3f78e74SRyan Roberts 3343b3f78e74SRyan Roberts ptent = pte_mkspecial(pfn_pte(pfn, data->prot)); 3344b3f78e74SRyan Roberts set_pte_at(&init_mm, addr, pte, ptent); 3345b3f78e74SRyan Roberts 3346b3f78e74SRyan Roberts data->idx++; 33473e9a9e25SChristoph Hellwig return 0; 33483e9a9e25SChristoph Hellwig } 33493e9a9e25SChristoph Hellwig 33503e9a9e25SChristoph Hellwig /** 33513e9a9e25SChristoph Hellwig * vmap_pfn - map an array of PFNs into virtually contiguous space 33523e9a9e25SChristoph Hellwig * @pfns: array of PFNs 33533e9a9e25SChristoph Hellwig * @count: number of pages to map 33543e9a9e25SChristoph Hellwig * @prot: page protection for the mapping 33553e9a9e25SChristoph Hellwig * 33563e9a9e25SChristoph Hellwig * Maps @count PFNs from @pfns into contiguous kernel virtual space and returns 33573e9a9e25SChristoph Hellwig * the start address of the mapping. 33583e9a9e25SChristoph Hellwig */ 33593e9a9e25SChristoph Hellwig void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot) 33603e9a9e25SChristoph Hellwig { 33613e9a9e25SChristoph Hellwig struct vmap_pfn_data data = { .pfns = pfns, .prot = pgprot_nx(prot) }; 33623e9a9e25SChristoph Hellwig struct vm_struct *area; 33633e9a9e25SChristoph Hellwig 33643e9a9e25SChristoph Hellwig area = get_vm_area_caller(count * PAGE_SIZE, VM_IOREMAP, 33653e9a9e25SChristoph Hellwig __builtin_return_address(0)); 33663e9a9e25SChristoph Hellwig if (!area) 33673e9a9e25SChristoph Hellwig return NULL; 33683e9a9e25SChristoph Hellwig if (apply_to_page_range(&init_mm, (unsigned long)area->addr, 33693e9a9e25SChristoph Hellwig count * PAGE_SIZE, vmap_pfn_apply, &data)) { 33703e9a9e25SChristoph Hellwig free_vm_area(area); 33713e9a9e25SChristoph Hellwig return NULL; 33723e9a9e25SChristoph Hellwig } 3373a50420c7SAlexandre Ghiti 3374a50420c7SAlexandre Ghiti flush_cache_vmap((unsigned long)area->addr, 3375a50420c7SAlexandre Ghiti (unsigned long)area->addr + count * PAGE_SIZE); 3376a50420c7SAlexandre Ghiti 33773e9a9e25SChristoph Hellwig return area->addr; 33783e9a9e25SChristoph Hellwig } 33793e9a9e25SChristoph Hellwig EXPORT_SYMBOL_GPL(vmap_pfn); 33803e9a9e25SChristoph Hellwig #endif /* CONFIG_VMAP_PFN */ 33813e9a9e25SChristoph Hellwig 338212b9f873SUladzislau Rezki static inline unsigned int 338312b9f873SUladzislau Rezki vm_area_alloc_pages(gfp_t gfp, int nid, 3384343ab817SUladzislau Rezki (Sony) unsigned int order, unsigned int nr_pages, struct page **pages) 338512b9f873SUladzislau Rezki { 338612b9f873SUladzislau Rezki unsigned int nr_allocated = 0; 3387e9c3cda4SMichal Hocko gfp_t alloc_gfp = gfp; 3388e9c3cda4SMichal Hocko bool nofail = false; 3389ffb29b1cSChen Wandun struct page *page; 3390ffb29b1cSChen Wandun int i; 339112b9f873SUladzislau Rezki 339212b9f873SUladzislau Rezki /* 339312b9f873SUladzislau Rezki * For order-0 pages we make use of bulk allocator, if 339412b9f873SUladzislau Rezki * the page array is partly or not at all populated due 339512b9f873SUladzislau Rezki * to fails, fallback to a single page allocator that is 339612b9f873SUladzislau Rezki * more permissive. 339712b9f873SUladzislau Rezki */ 3398c00b6b96SChen Wandun if (!order) { 3399e9c3cda4SMichal Hocko /* bulk allocator doesn't support nofail req. officially */ 34009376130cSMichal Hocko gfp_t bulk_gfp = gfp & ~__GFP_NOFAIL; 34019376130cSMichal Hocko 3402343ab817SUladzislau Rezki (Sony) while (nr_allocated < nr_pages) { 3403343ab817SUladzislau Rezki (Sony) unsigned int nr, nr_pages_request; 3404343ab817SUladzislau Rezki (Sony) 3405343ab817SUladzislau Rezki (Sony) /* 3406343ab817SUladzislau Rezki (Sony) * A maximum allowed request is hard-coded and is 100 3407343ab817SUladzislau Rezki (Sony) * pages per call. That is done in order to prevent a 3408343ab817SUladzislau Rezki (Sony) * long preemption off scenario in the bulk-allocator 3409343ab817SUladzislau Rezki (Sony) * so the range is [1:100]. 3410343ab817SUladzislau Rezki (Sony) */ 3411343ab817SUladzislau Rezki (Sony) nr_pages_request = min(100U, nr_pages - nr_allocated); 3412343ab817SUladzislau Rezki (Sony) 3413c00b6b96SChen Wandun /* memory allocation should consider mempolicy, we can't 3414c00b6b96SChen Wandun * wrongly use nearest node when nid == NUMA_NO_NODE, 3415c00b6b96SChen Wandun * otherwise memory may be allocated in only one node, 341698af39d5SYixuan Cao * but mempolicy wants to alloc memory by interleaving. 3417c00b6b96SChen Wandun */ 3418c00b6b96SChen Wandun if (IS_ENABLED(CONFIG_NUMA) && nid == NUMA_NO_NODE) 34199376130cSMichal Hocko nr = alloc_pages_bulk_array_mempolicy(bulk_gfp, 3420c00b6b96SChen Wandun nr_pages_request, 3421c00b6b96SChen Wandun pages + nr_allocated); 3422c00b6b96SChen Wandun 3423c00b6b96SChen Wandun else 34249376130cSMichal Hocko nr = alloc_pages_bulk_array_node(bulk_gfp, nid, 3425c00b6b96SChen Wandun nr_pages_request, 3426c00b6b96SChen Wandun pages + nr_allocated); 3427343ab817SUladzislau Rezki (Sony) 3428343ab817SUladzislau Rezki (Sony) nr_allocated += nr; 3429343ab817SUladzislau Rezki (Sony) cond_resched(); 3430343ab817SUladzislau Rezki (Sony) 3431343ab817SUladzislau Rezki (Sony) /* 3432343ab817SUladzislau Rezki (Sony) * If zero or pages were obtained partly, 3433343ab817SUladzislau Rezki (Sony) * fallback to a single page allocator. 3434343ab817SUladzislau Rezki (Sony) */ 3435343ab817SUladzislau Rezki (Sony) if (nr != nr_pages_request) 3436343ab817SUladzislau Rezki (Sony) break; 3437343ab817SUladzislau Rezki (Sony) } 3438e9c3cda4SMichal Hocko } else if (gfp & __GFP_NOFAIL) { 3439e9c3cda4SMichal Hocko /* 3440e9c3cda4SMichal Hocko * Higher order nofail allocations are really expensive and 3441e9c3cda4SMichal Hocko * potentially dangerous (pre-mature OOM, disruptive reclaim 3442e9c3cda4SMichal Hocko * and compaction etc. 3443e9c3cda4SMichal Hocko */ 3444e9c3cda4SMichal Hocko alloc_gfp &= ~__GFP_NOFAIL; 3445e9c3cda4SMichal Hocko nofail = true; 34463b8000aeSNicholas Piggin } 344712b9f873SUladzislau Rezki 344812b9f873SUladzislau Rezki /* High-order pages or fallback path if "bulk" fails. */ 3449ffb29b1cSChen Wandun while (nr_allocated < nr_pages) { 3450dd544141SVasily Averin if (fatal_signal_pending(current)) 3451dd544141SVasily Averin break; 3452dd544141SVasily Averin 3453ffb29b1cSChen Wandun if (nid == NUMA_NO_NODE) 3454e9c3cda4SMichal Hocko page = alloc_pages(alloc_gfp, order); 3455ffb29b1cSChen Wandun else 3456e9c3cda4SMichal Hocko page = alloc_pages_node(nid, alloc_gfp, order); 3457e9c3cda4SMichal Hocko if (unlikely(!page)) { 3458e9c3cda4SMichal Hocko if (!nofail) 345912b9f873SUladzislau Rezki break; 3460e9c3cda4SMichal Hocko 3461e9c3cda4SMichal Hocko /* fall back to the zero order allocations */ 3462e9c3cda4SMichal Hocko alloc_gfp |= __GFP_NOFAIL; 3463e9c3cda4SMichal Hocko order = 0; 3464e9c3cda4SMichal Hocko continue; 3465e9c3cda4SMichal Hocko } 3466e9c3cda4SMichal Hocko 34673b8000aeSNicholas Piggin /* 34683b8000aeSNicholas Piggin * Higher order allocations must be able to be treated as 34693b8000aeSNicholas Piggin * indepdenent small pages by callers (as they can with 34703b8000aeSNicholas Piggin * small-page vmallocs). Some drivers do their own refcounting 34713b8000aeSNicholas Piggin * on vmalloc_to_page() pages, some use page->mapping, 34723b8000aeSNicholas Piggin * page->lru, etc. 34733b8000aeSNicholas Piggin */ 34743b8000aeSNicholas Piggin if (order) 34753b8000aeSNicholas Piggin split_page(page, order); 347612b9f873SUladzislau Rezki 347712b9f873SUladzislau Rezki /* 347812b9f873SUladzislau Rezki * Careful, we allocate and map page-order pages, but 347912b9f873SUladzislau Rezki * tracking is done per PAGE_SIZE page so as to keep the 348012b9f873SUladzislau Rezki * vm_struct APIs independent of the physical/mapped size. 348112b9f873SUladzislau Rezki */ 348212b9f873SUladzislau Rezki for (i = 0; i < (1U << order); i++) 348312b9f873SUladzislau Rezki pages[nr_allocated + i] = page + i; 348412b9f873SUladzislau Rezki 348512b9f873SUladzislau Rezki cond_resched(); 348612b9f873SUladzislau Rezki nr_allocated += 1U << order; 348712b9f873SUladzislau Rezki } 348812b9f873SUladzislau Rezki 348912b9f873SUladzislau Rezki return nr_allocated; 349012b9f873SUladzislau Rezki } 349112b9f873SUladzislau Rezki 3492e31d9eb5SAdrian Bunk static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, 3493121e6f32SNicholas Piggin pgprot_t prot, unsigned int page_shift, 3494121e6f32SNicholas Piggin int node) 34951da177e4SLinus Torvalds { 3496930f036bSDavid Rientjes const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; 34979376130cSMichal Hocko bool nofail = gfp_mask & __GFP_NOFAIL; 3498121e6f32SNicholas Piggin unsigned long addr = (unsigned long)area->addr; 3499121e6f32SNicholas Piggin unsigned long size = get_vm_area_size(area); 350034fe6537SAndrew Morton unsigned long array_size; 3501121e6f32SNicholas Piggin unsigned int nr_small_pages = size >> PAGE_SHIFT; 3502121e6f32SNicholas Piggin unsigned int page_order; 3503451769ebSMichal Hocko unsigned int flags; 3504451769ebSMichal Hocko int ret; 35051da177e4SLinus Torvalds 3506121e6f32SNicholas Piggin array_size = (unsigned long)nr_small_pages * sizeof(struct page *); 350780b1d8fdSLorenzo Stoakes 3508f255935bSChristoph Hellwig if (!(gfp_mask & (GFP_DMA | GFP_DMA32))) 3509f255935bSChristoph Hellwig gfp_mask |= __GFP_HIGHMEM; 35101da177e4SLinus Torvalds 35111da177e4SLinus Torvalds /* Please note that the recursion is strictly bounded. */ 35128757d5faSJan Kiszka if (array_size > PAGE_SIZE) { 35135c1f4e69SUladzislau Rezki (Sony) area->pages = __vmalloc_node(array_size, 1, nested_gfp, node, 3514f255935bSChristoph Hellwig area->caller); 3515286e1ea3SAndrew Morton } else { 35165c1f4e69SUladzislau Rezki (Sony) area->pages = kmalloc_node(array_size, nested_gfp, node); 3517286e1ea3SAndrew Morton } 35187ea36242SAustin Kim 35195c1f4e69SUladzislau Rezki (Sony) if (!area->pages) { 3520c3d77172SUladzislau Rezki (Sony) warn_alloc(gfp_mask, NULL, 3521f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, failed to allocated page array size %lu", 3522d70bec8cSNicholas Piggin nr_small_pages * PAGE_SIZE, array_size); 3523cd61413bSUladzislau Rezki (Sony) free_vm_area(area); 35241da177e4SLinus Torvalds return NULL; 35251da177e4SLinus Torvalds } 35261da177e4SLinus Torvalds 3527121e6f32SNicholas Piggin set_vm_area_page_order(area, page_shift - PAGE_SHIFT); 3528121e6f32SNicholas Piggin page_order = vm_area_page_order(area); 3529121e6f32SNicholas Piggin 3530c3d77172SUladzislau Rezki (Sony) area->nr_pages = vm_area_alloc_pages(gfp_mask | __GFP_NOWARN, 3531c3d77172SUladzislau Rezki (Sony) node, page_order, nr_small_pages, area->pages); 35325c1f4e69SUladzislau Rezki (Sony) 353397105f0aSRoman Gushchin atomic_long_add(area->nr_pages, &nr_vmalloc_pages); 35344e5aa1f4SShakeel Butt if (gfp_mask & __GFP_ACCOUNT) { 35353b8000aeSNicholas Piggin int i; 35364e5aa1f4SShakeel Butt 35373b8000aeSNicholas Piggin for (i = 0; i < area->nr_pages; i++) 35383b8000aeSNicholas Piggin mod_memcg_page_state(area->pages[i], MEMCG_VMALLOC, 1); 35394e5aa1f4SShakeel Butt } 35405c1f4e69SUladzislau Rezki (Sony) 35415c1f4e69SUladzislau Rezki (Sony) /* 35425c1f4e69SUladzislau Rezki (Sony) * If not enough pages were obtained to accomplish an 3543f41f036bSChristoph Hellwig * allocation request, free them via vfree() if any. 35445c1f4e69SUladzislau Rezki (Sony) */ 35455c1f4e69SUladzislau Rezki (Sony) if (area->nr_pages != nr_small_pages) { 354695a301eeSLorenzo Stoakes /* 354795a301eeSLorenzo Stoakes * vm_area_alloc_pages() can fail due to insufficient memory but 354895a301eeSLorenzo Stoakes * also:- 354995a301eeSLorenzo Stoakes * 355095a301eeSLorenzo Stoakes * - a pending fatal signal 355195a301eeSLorenzo Stoakes * - insufficient huge page-order pages 355295a301eeSLorenzo Stoakes * 355395a301eeSLorenzo Stoakes * Since we always retry allocations at order-0 in the huge page 355495a301eeSLorenzo Stoakes * case a warning for either is spurious. 355595a301eeSLorenzo Stoakes */ 355695a301eeSLorenzo Stoakes if (!fatal_signal_pending(current) && page_order == 0) 3557c3d77172SUladzislau Rezki (Sony) warn_alloc(gfp_mask, NULL, 355895a301eeSLorenzo Stoakes "vmalloc error: size %lu, failed to allocate pages", 355995a301eeSLorenzo Stoakes area->nr_pages * PAGE_SIZE); 35601da177e4SLinus Torvalds goto fail; 35611da177e4SLinus Torvalds } 3562121e6f32SNicholas Piggin 3563451769ebSMichal Hocko /* 3564451769ebSMichal Hocko * page tables allocations ignore external gfp mask, enforce it 3565451769ebSMichal Hocko * by the scope API 3566451769ebSMichal Hocko */ 3567451769ebSMichal Hocko if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO) 3568451769ebSMichal Hocko flags = memalloc_nofs_save(); 3569451769ebSMichal Hocko else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0) 3570451769ebSMichal Hocko flags = memalloc_noio_save(); 3571451769ebSMichal Hocko 35729376130cSMichal Hocko do { 3573451769ebSMichal Hocko ret = vmap_pages_range(addr, addr + size, prot, area->pages, 3574451769ebSMichal Hocko page_shift); 35759376130cSMichal Hocko if (nofail && (ret < 0)) 35769376130cSMichal Hocko schedule_timeout_uninterruptible(1); 35779376130cSMichal Hocko } while (nofail && (ret < 0)); 3578451769ebSMichal Hocko 3579451769ebSMichal Hocko if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO) 3580451769ebSMichal Hocko memalloc_nofs_restore(flags); 3581451769ebSMichal Hocko else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0) 3582451769ebSMichal Hocko memalloc_noio_restore(flags); 3583451769ebSMichal Hocko 3584451769ebSMichal Hocko if (ret < 0) { 3585c3d77172SUladzislau Rezki (Sony) warn_alloc(gfp_mask, NULL, 3586f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, failed to map pages", 3587d70bec8cSNicholas Piggin area->nr_pages * PAGE_SIZE); 35881da177e4SLinus Torvalds goto fail; 3589d70bec8cSNicholas Piggin } 3590ed1f324cSChristoph Hellwig 35911da177e4SLinus Torvalds return area->addr; 35921da177e4SLinus Torvalds 35931da177e4SLinus Torvalds fail: 3594f41f036bSChristoph Hellwig vfree(area->addr); 35951da177e4SLinus Torvalds return NULL; 35961da177e4SLinus Torvalds } 35971da177e4SLinus Torvalds 3598d0a21265SDavid Rientjes /** 3599d0a21265SDavid Rientjes * __vmalloc_node_range - allocate virtually contiguous memory 3600d0a21265SDavid Rientjes * @size: allocation size 3601d0a21265SDavid Rientjes * @align: desired alignment 3602d0a21265SDavid Rientjes * @start: vm area range start 3603d0a21265SDavid Rientjes * @end: vm area range end 3604d0a21265SDavid Rientjes * @gfp_mask: flags for the page level allocator 3605d0a21265SDavid Rientjes * @prot: protection mask for the allocated pages 3606cb9e3c29SAndrey Ryabinin * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) 360700ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 3608d0a21265SDavid Rientjes * @caller: caller's return address 3609d0a21265SDavid Rientjes * 3610d0a21265SDavid Rientjes * Allocate enough pages to cover @size from the page level 3611b7d90e7aSMichal Hocko * allocator with @gfp_mask flags. Please note that the full set of gfp 361230d3f011SMichal Hocko * flags are not supported. GFP_KERNEL, GFP_NOFS and GFP_NOIO are all 361330d3f011SMichal Hocko * supported. 361430d3f011SMichal Hocko * Zone modifiers are not supported. From the reclaim modifiers 361530d3f011SMichal Hocko * __GFP_DIRECT_RECLAIM is required (aka GFP_NOWAIT is not supported) 361630d3f011SMichal Hocko * and only __GFP_NOFAIL is supported (i.e. __GFP_NORETRY and 361730d3f011SMichal Hocko * __GFP_RETRY_MAYFAIL are not supported). 361830d3f011SMichal Hocko * 361930d3f011SMichal Hocko * __GFP_NOWARN can be used to suppress failures messages. 3620b7d90e7aSMichal Hocko * 3621b7d90e7aSMichal Hocko * Map them into contiguous kernel virtual space, using a pagetable 3622b7d90e7aSMichal Hocko * protection of @prot. 3623a862f68aSMike Rapoport * 3624a862f68aSMike Rapoport * Return: the address of the area or %NULL on failure 3625d0a21265SDavid Rientjes */ 3626d0a21265SDavid Rientjes void *__vmalloc_node_range(unsigned long size, unsigned long align, 3627d0a21265SDavid Rientjes unsigned long start, unsigned long end, gfp_t gfp_mask, 3628cb9e3c29SAndrey Ryabinin pgprot_t prot, unsigned long vm_flags, int node, 3629cb9e3c29SAndrey Ryabinin const void *caller) 3630930fc45aSChristoph Lameter { 3631d0a21265SDavid Rientjes struct vm_struct *area; 363219f1c3acSAndrey Konovalov void *ret; 3633f6e39794SAndrey Konovalov kasan_vmalloc_flags_t kasan_flags = KASAN_VMALLOC_NONE; 3634d0a21265SDavid Rientjes unsigned long real_size = size; 3635121e6f32SNicholas Piggin unsigned long real_align = align; 3636121e6f32SNicholas Piggin unsigned int shift = PAGE_SHIFT; 3637d0a21265SDavid Rientjes 3638d70bec8cSNicholas Piggin if (WARN_ON_ONCE(!size)) 3639d70bec8cSNicholas Piggin return NULL; 3640d70bec8cSNicholas Piggin 3641d70bec8cSNicholas Piggin if ((size >> PAGE_SHIFT) > totalram_pages()) { 3642d70bec8cSNicholas Piggin warn_alloc(gfp_mask, NULL, 3643f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, exceeds total pages", 3644f4bdfeafSUladzislau Rezki (Sony) real_size); 3645d70bec8cSNicholas Piggin return NULL; 3646121e6f32SNicholas Piggin } 3647d0a21265SDavid Rientjes 3648559089e0SSong Liu if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) { 3649121e6f32SNicholas Piggin unsigned long size_per_node; 3650121e6f32SNicholas Piggin 3651121e6f32SNicholas Piggin /* 3652121e6f32SNicholas Piggin * Try huge pages. Only try for PAGE_KERNEL allocations, 3653121e6f32SNicholas Piggin * others like modules don't yet expect huge pages in 3654121e6f32SNicholas Piggin * their allocations due to apply_to_page_range not 3655121e6f32SNicholas Piggin * supporting them. 3656121e6f32SNicholas Piggin */ 3657121e6f32SNicholas Piggin 3658121e6f32SNicholas Piggin size_per_node = size; 3659121e6f32SNicholas Piggin if (node == NUMA_NO_NODE) 3660121e6f32SNicholas Piggin size_per_node /= num_online_nodes(); 36613382bbeeSChristophe Leroy if (arch_vmap_pmd_supported(prot) && size_per_node >= PMD_SIZE) 3662121e6f32SNicholas Piggin shift = PMD_SHIFT; 36633382bbeeSChristophe Leroy else 36643382bbeeSChristophe Leroy shift = arch_vmap_pte_supported_shift(size_per_node); 36653382bbeeSChristophe Leroy 3666121e6f32SNicholas Piggin align = max(real_align, 1UL << shift); 3667121e6f32SNicholas Piggin size = ALIGN(real_size, 1UL << shift); 3668121e6f32SNicholas Piggin } 3669121e6f32SNicholas Piggin 3670121e6f32SNicholas Piggin again: 36717ca3027bSDaniel Axtens area = __get_vm_area_node(real_size, align, shift, VM_ALLOC | 36727ca3027bSDaniel Axtens VM_UNINITIALIZED | vm_flags, start, end, node, 36737ca3027bSDaniel Axtens gfp_mask, caller); 3674d70bec8cSNicholas Piggin if (!area) { 36759376130cSMichal Hocko bool nofail = gfp_mask & __GFP_NOFAIL; 3676d70bec8cSNicholas Piggin warn_alloc(gfp_mask, NULL, 36779376130cSMichal Hocko "vmalloc error: size %lu, vm_struct allocation failed%s", 36789376130cSMichal Hocko real_size, (nofail) ? ". Retrying." : ""); 36799376130cSMichal Hocko if (nofail) { 36809376130cSMichal Hocko schedule_timeout_uninterruptible(1); 36819376130cSMichal Hocko goto again; 36829376130cSMichal Hocko } 3683de7d2b56SJoe Perches goto fail; 3684d70bec8cSNicholas Piggin } 3685d0a21265SDavid Rientjes 3686f6e39794SAndrey Konovalov /* 3687f6e39794SAndrey Konovalov * Prepare arguments for __vmalloc_area_node() and 3688f6e39794SAndrey Konovalov * kasan_unpoison_vmalloc(). 3689f6e39794SAndrey Konovalov */ 3690f6e39794SAndrey Konovalov if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL)) { 3691f6e39794SAndrey Konovalov if (kasan_hw_tags_enabled()) { 369201d92c7fSAndrey Konovalov /* 369301d92c7fSAndrey Konovalov * Modify protection bits to allow tagging. 3694f6e39794SAndrey Konovalov * This must be done before mapping. 369501d92c7fSAndrey Konovalov */ 369601d92c7fSAndrey Konovalov prot = arch_vmap_pgprot_tagged(prot); 369701d92c7fSAndrey Konovalov 369823689e91SAndrey Konovalov /* 3699f6e39794SAndrey Konovalov * Skip page_alloc poisoning and zeroing for physical 3700f6e39794SAndrey Konovalov * pages backing VM_ALLOC mapping. Memory is instead 3701f6e39794SAndrey Konovalov * poisoned and zeroed by kasan_unpoison_vmalloc(). 370223689e91SAndrey Konovalov */ 37030a54864fSPeter Collingbourne gfp_mask |= __GFP_SKIP_KASAN | __GFP_SKIP_ZERO; 370423689e91SAndrey Konovalov } 370523689e91SAndrey Konovalov 3706f6e39794SAndrey Konovalov /* Take note that the mapping is PAGE_KERNEL. */ 3707f6e39794SAndrey Konovalov kasan_flags |= KASAN_VMALLOC_PROT_NORMAL; 3708f6e39794SAndrey Konovalov } 3709f6e39794SAndrey Konovalov 371001d92c7fSAndrey Konovalov /* Allocate physical pages and map them into vmalloc space. */ 371119f1c3acSAndrey Konovalov ret = __vmalloc_area_node(area, gfp_mask, prot, shift, node); 371219f1c3acSAndrey Konovalov if (!ret) 3713121e6f32SNicholas Piggin goto fail; 371489219d37SCatalin Marinas 371523689e91SAndrey Konovalov /* 371623689e91SAndrey Konovalov * Mark the pages as accessible, now that they are mapped. 37176c2f761dSAndrey Konovalov * The condition for setting KASAN_VMALLOC_INIT should complement the 37186c2f761dSAndrey Konovalov * one in post_alloc_hook() with regards to the __GFP_SKIP_ZERO check 37196c2f761dSAndrey Konovalov * to make sure that memory is initialized under the same conditions. 3720f6e39794SAndrey Konovalov * Tag-based KASAN modes only assign tags to normal non-executable 3721f6e39794SAndrey Konovalov * allocations, see __kasan_unpoison_vmalloc(). 372223689e91SAndrey Konovalov */ 3723f6e39794SAndrey Konovalov kasan_flags |= KASAN_VMALLOC_VM_ALLOC; 37246c2f761dSAndrey Konovalov if (!want_init_on_free() && want_init_on_alloc(gfp_mask) && 37256c2f761dSAndrey Konovalov (gfp_mask & __GFP_SKIP_ZERO)) 372623689e91SAndrey Konovalov kasan_flags |= KASAN_VMALLOC_INIT; 3727f6e39794SAndrey Konovalov /* KASAN_VMALLOC_PROT_NORMAL already set if required. */ 372823689e91SAndrey Konovalov area->addr = kasan_unpoison_vmalloc(area->addr, real_size, kasan_flags); 372919f1c3acSAndrey Konovalov 373089219d37SCatalin Marinas /* 373120fc02b4SZhang Yanfei * In this function, newly allocated vm_struct has VM_UNINITIALIZED 373220fc02b4SZhang Yanfei * flag. It means that vm_struct is not fully initialized. 37334341fa45SJoonsoo Kim * Now, it is fully initialized, so remove this flag here. 3734f5252e00SMitsuo Hayasaka */ 373520fc02b4SZhang Yanfei clear_vm_uninitialized_flag(area); 3736f5252e00SMitsuo Hayasaka 37377ca3027bSDaniel Axtens size = PAGE_ALIGN(size); 373860115fa5SKefeng Wang if (!(vm_flags & VM_DEFER_KMEMLEAK)) 373994f4a161SCatalin Marinas kmemleak_vmalloc(area, size, gfp_mask); 374089219d37SCatalin Marinas 374119f1c3acSAndrey Konovalov return area->addr; 3742de7d2b56SJoe Perches 3743de7d2b56SJoe Perches fail: 3744121e6f32SNicholas Piggin if (shift > PAGE_SHIFT) { 3745121e6f32SNicholas Piggin shift = PAGE_SHIFT; 3746121e6f32SNicholas Piggin align = real_align; 3747121e6f32SNicholas Piggin size = real_size; 3748121e6f32SNicholas Piggin goto again; 3749121e6f32SNicholas Piggin } 3750121e6f32SNicholas Piggin 3751de7d2b56SJoe Perches return NULL; 3752930fc45aSChristoph Lameter } 3753930fc45aSChristoph Lameter 37541da177e4SLinus Torvalds /** 3755930fc45aSChristoph Lameter * __vmalloc_node - allocate virtually contiguous memory 37561da177e4SLinus Torvalds * @size: allocation size 37572dca6999SDavid Miller * @align: desired alignment 37581da177e4SLinus Torvalds * @gfp_mask: flags for the page level allocator 375900ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 3760c85d194bSRandy Dunlap * @caller: caller's return address 37611da177e4SLinus Torvalds * 3762f38fcb9cSChristoph Hellwig * Allocate enough pages to cover @size from the page level allocator with 3763f38fcb9cSChristoph Hellwig * @gfp_mask flags. Map them into contiguous kernel virtual space. 3764a7c3e901SMichal Hocko * 3765dcda9b04SMichal Hocko * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL 3766a7c3e901SMichal Hocko * and __GFP_NOFAIL are not supported 3767a7c3e901SMichal Hocko * 3768a7c3e901SMichal Hocko * Any use of gfp flags outside of GFP_KERNEL should be consulted 3769a7c3e901SMichal Hocko * with mm people. 3770a862f68aSMike Rapoport * 3771a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 37721da177e4SLinus Torvalds */ 37732b905948SChristoph Hellwig void *__vmalloc_node(unsigned long size, unsigned long align, 3774f38fcb9cSChristoph Hellwig gfp_t gfp_mask, int node, const void *caller) 37751da177e4SLinus Torvalds { 3776d0a21265SDavid Rientjes return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 3777f38fcb9cSChristoph Hellwig gfp_mask, PAGE_KERNEL, 0, node, caller); 37781da177e4SLinus Torvalds } 3779c3f896dcSChristoph Hellwig /* 3780c3f896dcSChristoph Hellwig * This is only for performance analysis of vmalloc and stress purpose. 3781c3f896dcSChristoph Hellwig * It is required by vmalloc test module, therefore do not use it other 3782c3f896dcSChristoph Hellwig * than that. 3783c3f896dcSChristoph Hellwig */ 3784c3f896dcSChristoph Hellwig #ifdef CONFIG_TEST_VMALLOC_MODULE 3785c3f896dcSChristoph Hellwig EXPORT_SYMBOL_GPL(__vmalloc_node); 3786c3f896dcSChristoph Hellwig #endif 37871da177e4SLinus Torvalds 378888dca4caSChristoph Hellwig void *__vmalloc(unsigned long size, gfp_t gfp_mask) 3789930fc45aSChristoph Lameter { 3790f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, gfp_mask, NUMA_NO_NODE, 379123016969SChristoph Lameter __builtin_return_address(0)); 3792930fc45aSChristoph Lameter } 37931da177e4SLinus Torvalds EXPORT_SYMBOL(__vmalloc); 37941da177e4SLinus Torvalds 37951da177e4SLinus Torvalds /** 37961da177e4SLinus Torvalds * vmalloc - allocate virtually contiguous memory 37971da177e4SLinus Torvalds * @size: allocation size 379892eac168SMike Rapoport * 37991da177e4SLinus Torvalds * Allocate enough pages to cover @size from the page level 38001da177e4SLinus Torvalds * allocator and map them into contiguous kernel virtual space. 38011da177e4SLinus Torvalds * 3802c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 38031da177e4SLinus Torvalds * use __vmalloc() instead. 3804a862f68aSMike Rapoport * 3805a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 38061da177e4SLinus Torvalds */ 38071da177e4SLinus Torvalds void *vmalloc(unsigned long size) 38081da177e4SLinus Torvalds { 38094d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL, NUMA_NO_NODE, 38104d39d728SChristoph Hellwig __builtin_return_address(0)); 38111da177e4SLinus Torvalds } 38121da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc); 38131da177e4SLinus Torvalds 3814930fc45aSChristoph Lameter /** 3815559089e0SSong Liu * vmalloc_huge - allocate virtually contiguous memory, allow huge pages 381615a64f5aSClaudio Imbrenda * @size: allocation size 3817559089e0SSong Liu * @gfp_mask: flags for the page level allocator 381815a64f5aSClaudio Imbrenda * 3819559089e0SSong Liu * Allocate enough pages to cover @size from the page level 382015a64f5aSClaudio Imbrenda * allocator and map them into contiguous kernel virtual space. 3821559089e0SSong Liu * If @size is greater than or equal to PMD_SIZE, allow using 3822559089e0SSong Liu * huge pages for the memory 382315a64f5aSClaudio Imbrenda * 382415a64f5aSClaudio Imbrenda * Return: pointer to the allocated memory or %NULL on error 382515a64f5aSClaudio Imbrenda */ 3826559089e0SSong Liu void *vmalloc_huge(unsigned long size, gfp_t gfp_mask) 382715a64f5aSClaudio Imbrenda { 382815a64f5aSClaudio Imbrenda return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, 3829559089e0SSong Liu gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, 383015a64f5aSClaudio Imbrenda NUMA_NO_NODE, __builtin_return_address(0)); 383115a64f5aSClaudio Imbrenda } 3832559089e0SSong Liu EXPORT_SYMBOL_GPL(vmalloc_huge); 383315a64f5aSClaudio Imbrenda 383415a64f5aSClaudio Imbrenda /** 3835e1ca7788SDave Young * vzalloc - allocate virtually contiguous memory with zero fill 3836e1ca7788SDave Young * @size: allocation size 383792eac168SMike Rapoport * 3838e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 3839e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 3840e1ca7788SDave Young * The memory allocated is set to zero. 3841e1ca7788SDave Young * 3842e1ca7788SDave Young * For tight control over page level allocator and protection flags 3843e1ca7788SDave Young * use __vmalloc() instead. 3844a862f68aSMike Rapoport * 3845a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 3846e1ca7788SDave Young */ 3847e1ca7788SDave Young void *vzalloc(unsigned long size) 3848e1ca7788SDave Young { 38494d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, 38504d39d728SChristoph Hellwig __builtin_return_address(0)); 3851e1ca7788SDave Young } 3852e1ca7788SDave Young EXPORT_SYMBOL(vzalloc); 3853e1ca7788SDave Young 3854e1ca7788SDave Young /** 3855ead04089SRolf Eike Beer * vmalloc_user - allocate zeroed virtually contiguous memory for userspace 385683342314SNick Piggin * @size: allocation size 3857ead04089SRolf Eike Beer * 3858ead04089SRolf Eike Beer * The resulting memory area is zeroed so it can be mapped to userspace 3859ead04089SRolf Eike Beer * without leaking data. 3860a862f68aSMike Rapoport * 3861a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 386283342314SNick Piggin */ 386383342314SNick Piggin void *vmalloc_user(unsigned long size) 386483342314SNick Piggin { 3865bc84c535SRoman Penyaev return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, 3866bc84c535SRoman Penyaev GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, 3867bc84c535SRoman Penyaev VM_USERMAP, NUMA_NO_NODE, 386800ef2d2fSDavid Rientjes __builtin_return_address(0)); 386983342314SNick Piggin } 387083342314SNick Piggin EXPORT_SYMBOL(vmalloc_user); 387183342314SNick Piggin 387283342314SNick Piggin /** 3873930fc45aSChristoph Lameter * vmalloc_node - allocate memory on a specific node 3874930fc45aSChristoph Lameter * @size: allocation size 3875d44e0780SRandy Dunlap * @node: numa node 3876930fc45aSChristoph Lameter * 3877930fc45aSChristoph Lameter * Allocate enough pages to cover @size from the page level 3878930fc45aSChristoph Lameter * allocator and map them into contiguous kernel virtual space. 3879930fc45aSChristoph Lameter * 3880c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 3881930fc45aSChristoph Lameter * use __vmalloc() instead. 3882a862f68aSMike Rapoport * 3883a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 3884930fc45aSChristoph Lameter */ 3885930fc45aSChristoph Lameter void *vmalloc_node(unsigned long size, int node) 3886930fc45aSChristoph Lameter { 3887f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL, node, 3888f38fcb9cSChristoph Hellwig __builtin_return_address(0)); 3889930fc45aSChristoph Lameter } 3890930fc45aSChristoph Lameter EXPORT_SYMBOL(vmalloc_node); 3891930fc45aSChristoph Lameter 3892e1ca7788SDave Young /** 3893e1ca7788SDave Young * vzalloc_node - allocate memory on a specific node with zero fill 3894e1ca7788SDave Young * @size: allocation size 3895e1ca7788SDave Young * @node: numa node 3896e1ca7788SDave Young * 3897e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 3898e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 3899e1ca7788SDave Young * The memory allocated is set to zero. 3900e1ca7788SDave Young * 3901a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 3902e1ca7788SDave Young */ 3903e1ca7788SDave Young void *vzalloc_node(unsigned long size, int node) 3904e1ca7788SDave Young { 39054d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, node, 39064d39d728SChristoph Hellwig __builtin_return_address(0)); 3907e1ca7788SDave Young } 3908e1ca7788SDave Young EXPORT_SYMBOL(vzalloc_node); 3909e1ca7788SDave Young 39100d08e0d3SAndi Kleen #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) 3911698d0831SMichal Hocko #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) 39120d08e0d3SAndi Kleen #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) 3913698d0831SMichal Hocko #define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL) 39140d08e0d3SAndi Kleen #else 3915698d0831SMichal Hocko /* 3916698d0831SMichal Hocko * 64b systems should always have either DMA or DMA32 zones. For others 3917698d0831SMichal Hocko * GFP_DMA32 should do the right thing and use the normal zone. 3918698d0831SMichal Hocko */ 391968d68ff6SZhiyuan Dai #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) 39200d08e0d3SAndi Kleen #endif 39210d08e0d3SAndi Kleen 39221da177e4SLinus Torvalds /** 39231da177e4SLinus Torvalds * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 39241da177e4SLinus Torvalds * @size: allocation size 39251da177e4SLinus Torvalds * 39261da177e4SLinus Torvalds * Allocate enough 32bit PA addressable pages to cover @size from the 39271da177e4SLinus Torvalds * page level allocator and map them into contiguous kernel virtual space. 3928a862f68aSMike Rapoport * 3929a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 39301da177e4SLinus Torvalds */ 39311da177e4SLinus Torvalds void *vmalloc_32(unsigned long size) 39321da177e4SLinus Torvalds { 3933f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, GFP_VMALLOC32, NUMA_NO_NODE, 3934f38fcb9cSChristoph Hellwig __builtin_return_address(0)); 39351da177e4SLinus Torvalds } 39361da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc_32); 39371da177e4SLinus Torvalds 393883342314SNick Piggin /** 3939ead04089SRolf Eike Beer * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 394083342314SNick Piggin * @size: allocation size 3941ead04089SRolf Eike Beer * 3942ead04089SRolf Eike Beer * The resulting memory area is 32bit addressable and zeroed so it can be 3943ead04089SRolf Eike Beer * mapped to userspace without leaking data. 3944a862f68aSMike Rapoport * 3945a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 394683342314SNick Piggin */ 394783342314SNick Piggin void *vmalloc_32_user(unsigned long size) 394883342314SNick Piggin { 3949bc84c535SRoman Penyaev return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, 3950bc84c535SRoman Penyaev GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, 3951bc84c535SRoman Penyaev VM_USERMAP, NUMA_NO_NODE, 39525a82ac71SRoman Penyaev __builtin_return_address(0)); 395383342314SNick Piggin } 395483342314SNick Piggin EXPORT_SYMBOL(vmalloc_32_user); 395583342314SNick Piggin 3956d0107eb0SKAMEZAWA Hiroyuki /* 39574c91c07cSLorenzo Stoakes * Atomically zero bytes in the iterator. 39584c91c07cSLorenzo Stoakes * 39594c91c07cSLorenzo Stoakes * Returns the number of zeroed bytes. 3960d0107eb0SKAMEZAWA Hiroyuki */ 39614c91c07cSLorenzo Stoakes static size_t zero_iter(struct iov_iter *iter, size_t count) 3962d0107eb0SKAMEZAWA Hiroyuki { 39634c91c07cSLorenzo Stoakes size_t remains = count; 3964d0107eb0SKAMEZAWA Hiroyuki 39654c91c07cSLorenzo Stoakes while (remains > 0) { 39664c91c07cSLorenzo Stoakes size_t num, copied; 39674c91c07cSLorenzo Stoakes 39680e4bc271SLu Hongfei num = min_t(size_t, remains, PAGE_SIZE); 39694c91c07cSLorenzo Stoakes copied = copy_page_to_iter_nofault(ZERO_PAGE(0), 0, num, iter); 39704c91c07cSLorenzo Stoakes remains -= copied; 39714c91c07cSLorenzo Stoakes 39724c91c07cSLorenzo Stoakes if (copied < num) 39734c91c07cSLorenzo Stoakes break; 39744c91c07cSLorenzo Stoakes } 39754c91c07cSLorenzo Stoakes 39764c91c07cSLorenzo Stoakes return count - remains; 39774c91c07cSLorenzo Stoakes } 39784c91c07cSLorenzo Stoakes 39794c91c07cSLorenzo Stoakes /* 39804c91c07cSLorenzo Stoakes * small helper routine, copy contents to iter from addr. 39814c91c07cSLorenzo Stoakes * If the page is not present, fill zero. 39824c91c07cSLorenzo Stoakes * 39834c91c07cSLorenzo Stoakes * Returns the number of copied bytes. 39844c91c07cSLorenzo Stoakes */ 39854c91c07cSLorenzo Stoakes static size_t aligned_vread_iter(struct iov_iter *iter, 39864c91c07cSLorenzo Stoakes const char *addr, size_t count) 39874c91c07cSLorenzo Stoakes { 39884c91c07cSLorenzo Stoakes size_t remains = count; 39894c91c07cSLorenzo Stoakes struct page *page; 39904c91c07cSLorenzo Stoakes 39914c91c07cSLorenzo Stoakes while (remains > 0) { 3992d0107eb0SKAMEZAWA Hiroyuki unsigned long offset, length; 39934c91c07cSLorenzo Stoakes size_t copied = 0; 3994d0107eb0SKAMEZAWA Hiroyuki 3995891c49abSAlexander Kuleshov offset = offset_in_page(addr); 3996d0107eb0SKAMEZAWA Hiroyuki length = PAGE_SIZE - offset; 39974c91c07cSLorenzo Stoakes if (length > remains) 39984c91c07cSLorenzo Stoakes length = remains; 39994c91c07cSLorenzo Stoakes page = vmalloc_to_page(addr); 4000d0107eb0SKAMEZAWA Hiroyuki /* 40014c91c07cSLorenzo Stoakes * To do safe access to this _mapped_ area, we need lock. But 40024c91c07cSLorenzo Stoakes * adding lock here means that we need to add overhead of 40034c91c07cSLorenzo Stoakes * vmalloc()/vfree() calls for this _debug_ interface, rarely 40044c91c07cSLorenzo Stoakes * used. Instead of that, we'll use an local mapping via 40054c91c07cSLorenzo Stoakes * copy_page_to_iter_nofault() and accept a small overhead in 40064c91c07cSLorenzo Stoakes * this access function. 4007d0107eb0SKAMEZAWA Hiroyuki */ 40084c91c07cSLorenzo Stoakes if (page) 40094c91c07cSLorenzo Stoakes copied = copy_page_to_iter_nofault(page, offset, 40104c91c07cSLorenzo Stoakes length, iter); 40114c91c07cSLorenzo Stoakes else 40124c91c07cSLorenzo Stoakes copied = zero_iter(iter, length); 4013d0107eb0SKAMEZAWA Hiroyuki 40144c91c07cSLorenzo Stoakes addr += copied; 40154c91c07cSLorenzo Stoakes remains -= copied; 40164c91c07cSLorenzo Stoakes 40174c91c07cSLorenzo Stoakes if (copied != length) 40184c91c07cSLorenzo Stoakes break; 4019d0107eb0SKAMEZAWA Hiroyuki } 4020d0107eb0SKAMEZAWA Hiroyuki 40214c91c07cSLorenzo Stoakes return count - remains; 40224c91c07cSLorenzo Stoakes } 40234c91c07cSLorenzo Stoakes 40244c91c07cSLorenzo Stoakes /* 40254c91c07cSLorenzo Stoakes * Read from a vm_map_ram region of memory. 40264c91c07cSLorenzo Stoakes * 40274c91c07cSLorenzo Stoakes * Returns the number of copied bytes. 40284c91c07cSLorenzo Stoakes */ 40294c91c07cSLorenzo Stoakes static size_t vmap_ram_vread_iter(struct iov_iter *iter, const char *addr, 40304c91c07cSLorenzo Stoakes size_t count, unsigned long flags) 403106c89946SBaoquan He { 403206c89946SBaoquan He char *start; 403306c89946SBaoquan He struct vmap_block *vb; 4034062eacf5SUladzislau Rezki (Sony) struct xarray *xa; 403506c89946SBaoquan He unsigned long offset; 40364c91c07cSLorenzo Stoakes unsigned int rs, re; 40374c91c07cSLorenzo Stoakes size_t remains, n; 403806c89946SBaoquan He 403906c89946SBaoquan He /* 404006c89946SBaoquan He * If it's area created by vm_map_ram() interface directly, but 404106c89946SBaoquan He * not further subdividing and delegating management to vmap_block, 404206c89946SBaoquan He * handle it here. 404306c89946SBaoquan He */ 40444c91c07cSLorenzo Stoakes if (!(flags & VMAP_BLOCK)) 40454c91c07cSLorenzo Stoakes return aligned_vread_iter(iter, addr, count); 40464c91c07cSLorenzo Stoakes 40474c91c07cSLorenzo Stoakes remains = count; 404806c89946SBaoquan He 404906c89946SBaoquan He /* 405006c89946SBaoquan He * Area is split into regions and tracked with vmap_block, read out 405106c89946SBaoquan He * each region and zero fill the hole between regions. 405206c89946SBaoquan He */ 4053fa1c77c1SUladzislau Rezki (Sony) xa = addr_to_vb_xa((unsigned long) addr); 4054062eacf5SUladzislau Rezki (Sony) vb = xa_load(xa, addr_to_vb_idx((unsigned long)addr)); 405506c89946SBaoquan He if (!vb) 40564c91c07cSLorenzo Stoakes goto finished_zero; 405706c89946SBaoquan He 405806c89946SBaoquan He spin_lock(&vb->lock); 405906c89946SBaoquan He if (bitmap_empty(vb->used_map, VMAP_BBMAP_BITS)) { 406006c89946SBaoquan He spin_unlock(&vb->lock); 40614c91c07cSLorenzo Stoakes goto finished_zero; 40624c91c07cSLorenzo Stoakes } 40634c91c07cSLorenzo Stoakes 40644c91c07cSLorenzo Stoakes for_each_set_bitrange(rs, re, vb->used_map, VMAP_BBMAP_BITS) { 40654c91c07cSLorenzo Stoakes size_t copied; 40664c91c07cSLorenzo Stoakes 40674c91c07cSLorenzo Stoakes if (remains == 0) 40684c91c07cSLorenzo Stoakes goto finished; 40694c91c07cSLorenzo Stoakes 40704c91c07cSLorenzo Stoakes start = vmap_block_vaddr(vb->va->va_start, rs); 40714c91c07cSLorenzo Stoakes 40724c91c07cSLorenzo Stoakes if (addr < start) { 40734c91c07cSLorenzo Stoakes size_t to_zero = min_t(size_t, start - addr, remains); 40744c91c07cSLorenzo Stoakes size_t zeroed = zero_iter(iter, to_zero); 40754c91c07cSLorenzo Stoakes 40764c91c07cSLorenzo Stoakes addr += zeroed; 40774c91c07cSLorenzo Stoakes remains -= zeroed; 40784c91c07cSLorenzo Stoakes 40794c91c07cSLorenzo Stoakes if (remains == 0 || zeroed != to_zero) 408006c89946SBaoquan He goto finished; 408106c89946SBaoquan He } 40824c91c07cSLorenzo Stoakes 408306c89946SBaoquan He /*it could start reading from the middle of used region*/ 408406c89946SBaoquan He offset = offset_in_page(addr); 408506c89946SBaoquan He n = ((re - rs + 1) << PAGE_SHIFT) - offset; 40864c91c07cSLorenzo Stoakes if (n > remains) 40874c91c07cSLorenzo Stoakes n = remains; 408806c89946SBaoquan He 40894c91c07cSLorenzo Stoakes copied = aligned_vread_iter(iter, start + offset, n); 40904c91c07cSLorenzo Stoakes 40914c91c07cSLorenzo Stoakes addr += copied; 40924c91c07cSLorenzo Stoakes remains -= copied; 40934c91c07cSLorenzo Stoakes 40944c91c07cSLorenzo Stoakes if (copied != n) 40954c91c07cSLorenzo Stoakes goto finished; 409606c89946SBaoquan He } 40974c91c07cSLorenzo Stoakes 409806c89946SBaoquan He spin_unlock(&vb->lock); 409906c89946SBaoquan He 41004c91c07cSLorenzo Stoakes finished_zero: 410106c89946SBaoquan He /* zero-fill the left dirty or free regions */ 41024c91c07cSLorenzo Stoakes return count - remains + zero_iter(iter, remains); 41034c91c07cSLorenzo Stoakes finished: 41044c91c07cSLorenzo Stoakes /* We couldn't copy/zero everything */ 41054c91c07cSLorenzo Stoakes spin_unlock(&vb->lock); 41064c91c07cSLorenzo Stoakes return count - remains; 410706c89946SBaoquan He } 410806c89946SBaoquan He 4109d0107eb0SKAMEZAWA Hiroyuki /** 41104c91c07cSLorenzo Stoakes * vread_iter() - read vmalloc area in a safe way to an iterator. 41114c91c07cSLorenzo Stoakes * @iter: the iterator to which data should be written. 4112d0107eb0SKAMEZAWA Hiroyuki * @addr: vm address. 4113d0107eb0SKAMEZAWA Hiroyuki * @count: number of bytes to be read. 4114d0107eb0SKAMEZAWA Hiroyuki * 4115d0107eb0SKAMEZAWA Hiroyuki * This function checks that addr is a valid vmalloc'ed area, and 4116d0107eb0SKAMEZAWA Hiroyuki * copy data from that area to a given buffer. If the given memory range 4117d0107eb0SKAMEZAWA Hiroyuki * of [addr...addr+count) includes some valid address, data is copied to 4118d0107eb0SKAMEZAWA Hiroyuki * proper area of @buf. If there are memory holes, they'll be zero-filled. 4119d0107eb0SKAMEZAWA Hiroyuki * IOREMAP area is treated as memory hole and no copy is done. 4120d0107eb0SKAMEZAWA Hiroyuki * 4121d0107eb0SKAMEZAWA Hiroyuki * If [addr...addr+count) doesn't includes any intersects with alive 4122a8e5202dSCong Wang * vm_struct area, returns 0. @buf should be kernel's buffer. 4123d0107eb0SKAMEZAWA Hiroyuki * 4124d0107eb0SKAMEZAWA Hiroyuki * Note: In usual ops, vread() is never necessary because the caller 4125d0107eb0SKAMEZAWA Hiroyuki * should know vmalloc() area is valid and can use memcpy(). 4126d0107eb0SKAMEZAWA Hiroyuki * This is for routines which have to access vmalloc area without 4127bbcd53c9SDavid Hildenbrand * any information, as /proc/kcore. 4128a862f68aSMike Rapoport * 4129a862f68aSMike Rapoport * Return: number of bytes for which addr and buf should be increased 4130a862f68aSMike Rapoport * (same number as @count) or %0 if [addr...addr+count) doesn't 4131a862f68aSMike Rapoport * include any intersection with valid vmalloc area 4132d0107eb0SKAMEZAWA Hiroyuki */ 41334c91c07cSLorenzo Stoakes long vread_iter(struct iov_iter *iter, const char *addr, size_t count) 41341da177e4SLinus Torvalds { 4135d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 4136e81ce85fSJoonsoo Kim struct vmap_area *va; 4137e81ce85fSJoonsoo Kim struct vm_struct *vm; 41384c91c07cSLorenzo Stoakes char *vaddr; 41394c91c07cSLorenzo Stoakes size_t n, size, flags, remains; 414053becf32SUladzislau Rezki (Sony) unsigned long next; 41411da177e4SLinus Torvalds 41424aff1dc4SAndrey Konovalov addr = kasan_reset_tag(addr); 41434aff1dc4SAndrey Konovalov 41441da177e4SLinus Torvalds /* Don't allow overflow */ 41451da177e4SLinus Torvalds if ((unsigned long) addr + count < count) 41461da177e4SLinus Torvalds count = -(unsigned long) addr; 41471da177e4SLinus Torvalds 41484c91c07cSLorenzo Stoakes remains = count; 41494c91c07cSLorenzo Stoakes 415053becf32SUladzislau Rezki (Sony) vn = find_vmap_area_exceed_addr_lock((unsigned long) addr, &va); 415153becf32SUladzislau Rezki (Sony) if (!vn) 41524c91c07cSLorenzo Stoakes goto finished_zero; 4153f181234aSChen Wandun 4154f181234aSChen Wandun /* no intersects with alive vmap_area */ 41554c91c07cSLorenzo Stoakes if ((unsigned long)addr + remains <= va->va_start) 41564c91c07cSLorenzo Stoakes goto finished_zero; 4157f181234aSChen Wandun 415853becf32SUladzislau Rezki (Sony) do { 41594c91c07cSLorenzo Stoakes size_t copied; 41604c91c07cSLorenzo Stoakes 41614c91c07cSLorenzo Stoakes if (remains == 0) 41624c91c07cSLorenzo Stoakes goto finished; 4163e81ce85fSJoonsoo Kim 416406c89946SBaoquan He vm = va->vm; 416506c89946SBaoquan He flags = va->flags & VMAP_FLAGS_MASK; 416606c89946SBaoquan He /* 416706c89946SBaoquan He * VMAP_BLOCK indicates a sub-type of vm_map_ram area, need 416806c89946SBaoquan He * be set together with VMAP_RAM. 416906c89946SBaoquan He */ 417006c89946SBaoquan He WARN_ON(flags == VMAP_BLOCK); 417106c89946SBaoquan He 417206c89946SBaoquan He if (!vm && !flags) 417353becf32SUladzislau Rezki (Sony) goto next_va; 4174e81ce85fSJoonsoo Kim 417530a7a9b1SBaoquan He if (vm && (vm->flags & VM_UNINITIALIZED)) 417653becf32SUladzislau Rezki (Sony) goto next_va; 41774c91c07cSLorenzo Stoakes 417830a7a9b1SBaoquan He /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ 417930a7a9b1SBaoquan He smp_rmb(); 418030a7a9b1SBaoquan He 418106c89946SBaoquan He vaddr = (char *) va->va_start; 418206c89946SBaoquan He size = vm ? get_vm_area_size(vm) : va_size(va); 418306c89946SBaoquan He 418406c89946SBaoquan He if (addr >= vaddr + size) 418553becf32SUladzislau Rezki (Sony) goto next_va; 41864c91c07cSLorenzo Stoakes 41874c91c07cSLorenzo Stoakes if (addr < vaddr) { 41884c91c07cSLorenzo Stoakes size_t to_zero = min_t(size_t, vaddr - addr, remains); 41894c91c07cSLorenzo Stoakes size_t zeroed = zero_iter(iter, to_zero); 41904c91c07cSLorenzo Stoakes 41914c91c07cSLorenzo Stoakes addr += zeroed; 41924c91c07cSLorenzo Stoakes remains -= zeroed; 41934c91c07cSLorenzo Stoakes 41944c91c07cSLorenzo Stoakes if (remains == 0 || zeroed != to_zero) 41951da177e4SLinus Torvalds goto finished; 41961da177e4SLinus Torvalds } 41974c91c07cSLorenzo Stoakes 419806c89946SBaoquan He n = vaddr + size - addr; 41994c91c07cSLorenzo Stoakes if (n > remains) 42004c91c07cSLorenzo Stoakes n = remains; 420106c89946SBaoquan He 420206c89946SBaoquan He if (flags & VMAP_RAM) 42034c91c07cSLorenzo Stoakes copied = vmap_ram_vread_iter(iter, addr, n, flags); 4204ca6c2ce1SBaoquan He else if (!(vm && (vm->flags & VM_IOREMAP))) 42054c91c07cSLorenzo Stoakes copied = aligned_vread_iter(iter, addr, n); 4206d0107eb0SKAMEZAWA Hiroyuki else /* IOREMAP area is treated as memory hole */ 42074c91c07cSLorenzo Stoakes copied = zero_iter(iter, n); 42084c91c07cSLorenzo Stoakes 42094c91c07cSLorenzo Stoakes addr += copied; 42104c91c07cSLorenzo Stoakes remains -= copied; 42114c91c07cSLorenzo Stoakes 42124c91c07cSLorenzo Stoakes if (copied != n) 42134c91c07cSLorenzo Stoakes goto finished; 421453becf32SUladzislau Rezki (Sony) 421553becf32SUladzislau Rezki (Sony) next_va: 421653becf32SUladzislau Rezki (Sony) next = va->va_end; 421753becf32SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 421853becf32SUladzislau Rezki (Sony) } while ((vn = find_vmap_area_exceed_addr_lock(next, &va))); 42194c91c07cSLorenzo Stoakes 42204c91c07cSLorenzo Stoakes finished_zero: 422153becf32SUladzislau Rezki (Sony) if (vn) 4222d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 422353becf32SUladzislau Rezki (Sony) 42244c91c07cSLorenzo Stoakes /* zero-fill memory holes */ 42254c91c07cSLorenzo Stoakes return count - remains + zero_iter(iter, remains); 42261da177e4SLinus Torvalds finished: 42274c91c07cSLorenzo Stoakes /* Nothing remains, or We couldn't copy/zero everything. */ 422853becf32SUladzislau Rezki (Sony) if (vn) 4229d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 4230d0107eb0SKAMEZAWA Hiroyuki 42314c91c07cSLorenzo Stoakes return count - remains; 42321da177e4SLinus Torvalds } 42331da177e4SLinus Torvalds 4234d0107eb0SKAMEZAWA Hiroyuki /** 4235e69e9d4aSHATAYAMA Daisuke * remap_vmalloc_range_partial - map vmalloc pages to userspace 4236e69e9d4aSHATAYAMA Daisuke * @vma: vma to cover 4237e69e9d4aSHATAYAMA Daisuke * @uaddr: target user address to start at 4238e69e9d4aSHATAYAMA Daisuke * @kaddr: virtual address of vmalloc kernel memory 4239bdebd6a2SJann Horn * @pgoff: offset from @kaddr to start at 4240e69e9d4aSHATAYAMA Daisuke * @size: size of map area 4241e69e9d4aSHATAYAMA Daisuke * 4242e69e9d4aSHATAYAMA Daisuke * Returns: 0 for success, -Exxx on failure 4243e69e9d4aSHATAYAMA Daisuke * 4244e69e9d4aSHATAYAMA Daisuke * This function checks that @kaddr is a valid vmalloc'ed area, 4245e69e9d4aSHATAYAMA Daisuke * and that it is big enough to cover the range starting at 4246e69e9d4aSHATAYAMA Daisuke * @uaddr in @vma. Will return failure if that criteria isn't 4247e69e9d4aSHATAYAMA Daisuke * met. 4248e69e9d4aSHATAYAMA Daisuke * 4249e69e9d4aSHATAYAMA Daisuke * Similar to remap_pfn_range() (see mm/memory.c) 4250e69e9d4aSHATAYAMA Daisuke */ 4251e69e9d4aSHATAYAMA Daisuke int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr, 4252bdebd6a2SJann Horn void *kaddr, unsigned long pgoff, 4253bdebd6a2SJann Horn unsigned long size) 4254e69e9d4aSHATAYAMA Daisuke { 4255e69e9d4aSHATAYAMA Daisuke struct vm_struct *area; 4256bdebd6a2SJann Horn unsigned long off; 4257bdebd6a2SJann Horn unsigned long end_index; 4258bdebd6a2SJann Horn 4259bdebd6a2SJann Horn if (check_shl_overflow(pgoff, PAGE_SHIFT, &off)) 4260bdebd6a2SJann Horn return -EINVAL; 4261e69e9d4aSHATAYAMA Daisuke 4262e69e9d4aSHATAYAMA Daisuke size = PAGE_ALIGN(size); 4263e69e9d4aSHATAYAMA Daisuke 4264e69e9d4aSHATAYAMA Daisuke if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr)) 4265e69e9d4aSHATAYAMA Daisuke return -EINVAL; 4266e69e9d4aSHATAYAMA Daisuke 4267e69e9d4aSHATAYAMA Daisuke area = find_vm_area(kaddr); 4268e69e9d4aSHATAYAMA Daisuke if (!area) 4269e69e9d4aSHATAYAMA Daisuke return -EINVAL; 4270e69e9d4aSHATAYAMA Daisuke 4271fe9041c2SChristoph Hellwig if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT))) 4272e69e9d4aSHATAYAMA Daisuke return -EINVAL; 4273e69e9d4aSHATAYAMA Daisuke 4274bdebd6a2SJann Horn if (check_add_overflow(size, off, &end_index) || 4275bdebd6a2SJann Horn end_index > get_vm_area_size(area)) 4276e69e9d4aSHATAYAMA Daisuke return -EINVAL; 4277bdebd6a2SJann Horn kaddr += off; 4278e69e9d4aSHATAYAMA Daisuke 4279e69e9d4aSHATAYAMA Daisuke do { 4280e69e9d4aSHATAYAMA Daisuke struct page *page = vmalloc_to_page(kaddr); 4281e69e9d4aSHATAYAMA Daisuke int ret; 4282e69e9d4aSHATAYAMA Daisuke 4283e69e9d4aSHATAYAMA Daisuke ret = vm_insert_page(vma, uaddr, page); 4284e69e9d4aSHATAYAMA Daisuke if (ret) 4285e69e9d4aSHATAYAMA Daisuke return ret; 4286e69e9d4aSHATAYAMA Daisuke 4287e69e9d4aSHATAYAMA Daisuke uaddr += PAGE_SIZE; 4288e69e9d4aSHATAYAMA Daisuke kaddr += PAGE_SIZE; 4289e69e9d4aSHATAYAMA Daisuke size -= PAGE_SIZE; 4290e69e9d4aSHATAYAMA Daisuke } while (size > 0); 4291e69e9d4aSHATAYAMA Daisuke 42921c71222eSSuren Baghdasaryan vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); 4293e69e9d4aSHATAYAMA Daisuke 4294e69e9d4aSHATAYAMA Daisuke return 0; 4295e69e9d4aSHATAYAMA Daisuke } 4296e69e9d4aSHATAYAMA Daisuke 4297e69e9d4aSHATAYAMA Daisuke /** 429883342314SNick Piggin * remap_vmalloc_range - map vmalloc pages to userspace 429983342314SNick Piggin * @vma: vma to cover (map full range of vma) 430083342314SNick Piggin * @addr: vmalloc memory 430183342314SNick Piggin * @pgoff: number of pages into addr before first page to map 43027682486bSRandy Dunlap * 43037682486bSRandy Dunlap * Returns: 0 for success, -Exxx on failure 430483342314SNick Piggin * 430583342314SNick Piggin * This function checks that addr is a valid vmalloc'ed area, and 430683342314SNick Piggin * that it is big enough to cover the vma. Will return failure if 430783342314SNick Piggin * that criteria isn't met. 430883342314SNick Piggin * 430972fd4a35SRobert P. J. Day * Similar to remap_pfn_range() (see mm/memory.c) 431083342314SNick Piggin */ 431183342314SNick Piggin int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 431283342314SNick Piggin unsigned long pgoff) 431383342314SNick Piggin { 4314e69e9d4aSHATAYAMA Daisuke return remap_vmalloc_range_partial(vma, vma->vm_start, 4315bdebd6a2SJann Horn addr, pgoff, 4316e69e9d4aSHATAYAMA Daisuke vma->vm_end - vma->vm_start); 431783342314SNick Piggin } 431883342314SNick Piggin EXPORT_SYMBOL(remap_vmalloc_range); 431983342314SNick Piggin 43205f4352fbSJeremy Fitzhardinge void free_vm_area(struct vm_struct *area) 43215f4352fbSJeremy Fitzhardinge { 43225f4352fbSJeremy Fitzhardinge struct vm_struct *ret; 43235f4352fbSJeremy Fitzhardinge ret = remove_vm_area(area->addr); 43245f4352fbSJeremy Fitzhardinge BUG_ON(ret != area); 43255f4352fbSJeremy Fitzhardinge kfree(area); 43265f4352fbSJeremy Fitzhardinge } 43275f4352fbSJeremy Fitzhardinge EXPORT_SYMBOL_GPL(free_vm_area); 4328a10aa579SChristoph Lameter 43294f8b02b4STejun Heo #ifdef CONFIG_SMP 4330ca23e405STejun Heo static struct vmap_area *node_to_va(struct rb_node *n) 4331ca23e405STejun Heo { 43324583e773SGeliang Tang return rb_entry_safe(n, struct vmap_area, rb_node); 4333ca23e405STejun Heo } 4334ca23e405STejun Heo 4335ca23e405STejun Heo /** 433668ad4a33SUladzislau Rezki (Sony) * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to 433768ad4a33SUladzislau Rezki (Sony) * @addr: target address 4338ca23e405STejun Heo * 433968ad4a33SUladzislau Rezki (Sony) * Returns: vmap_area if it is found. If there is no such area 434068ad4a33SUladzislau Rezki (Sony) * the first highest(reverse order) vmap_area is returned 434168ad4a33SUladzislau Rezki (Sony) * i.e. va->va_start < addr && va->va_end < addr or NULL 434268ad4a33SUladzislau Rezki (Sony) * if there are no any areas before @addr. 4343ca23e405STejun Heo */ 434468ad4a33SUladzislau Rezki (Sony) static struct vmap_area * 434568ad4a33SUladzislau Rezki (Sony) pvm_find_va_enclose_addr(unsigned long addr) 4346ca23e405STejun Heo { 434768ad4a33SUladzislau Rezki (Sony) struct vmap_area *va, *tmp; 434868ad4a33SUladzislau Rezki (Sony) struct rb_node *n; 434968ad4a33SUladzislau Rezki (Sony) 435068ad4a33SUladzislau Rezki (Sony) n = free_vmap_area_root.rb_node; 435168ad4a33SUladzislau Rezki (Sony) va = NULL; 4352ca23e405STejun Heo 4353ca23e405STejun Heo while (n) { 435468ad4a33SUladzislau Rezki (Sony) tmp = rb_entry(n, struct vmap_area, rb_node); 435568ad4a33SUladzislau Rezki (Sony) if (tmp->va_start <= addr) { 435668ad4a33SUladzislau Rezki (Sony) va = tmp; 435768ad4a33SUladzislau Rezki (Sony) if (tmp->va_end >= addr) 4358ca23e405STejun Heo break; 4359ca23e405STejun Heo 436068ad4a33SUladzislau Rezki (Sony) n = n->rb_right; 4361ca23e405STejun Heo } else { 436268ad4a33SUladzislau Rezki (Sony) n = n->rb_left; 4363ca23e405STejun Heo } 436468ad4a33SUladzislau Rezki (Sony) } 436568ad4a33SUladzislau Rezki (Sony) 436668ad4a33SUladzislau Rezki (Sony) return va; 4367ca23e405STejun Heo } 4368ca23e405STejun Heo 4369ca23e405STejun Heo /** 437068ad4a33SUladzislau Rezki (Sony) * pvm_determine_end_from_reverse - find the highest aligned address 437168ad4a33SUladzislau Rezki (Sony) * of free block below VMALLOC_END 437268ad4a33SUladzislau Rezki (Sony) * @va: 437368ad4a33SUladzislau Rezki (Sony) * in - the VA we start the search(reverse order); 437468ad4a33SUladzislau Rezki (Sony) * out - the VA with the highest aligned end address. 4375799fa85dSAlex Shi * @align: alignment for required highest address 4376ca23e405STejun Heo * 437768ad4a33SUladzislau Rezki (Sony) * Returns: determined end address within vmap_area 4378ca23e405STejun Heo */ 437968ad4a33SUladzislau Rezki (Sony) static unsigned long 438068ad4a33SUladzislau Rezki (Sony) pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) 4381ca23e405STejun Heo { 438268ad4a33SUladzislau Rezki (Sony) unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 4383ca23e405STejun Heo unsigned long addr; 4384ca23e405STejun Heo 438568ad4a33SUladzislau Rezki (Sony) if (likely(*va)) { 438668ad4a33SUladzislau Rezki (Sony) list_for_each_entry_from_reverse((*va), 438768ad4a33SUladzislau Rezki (Sony) &free_vmap_area_list, list) { 438868ad4a33SUladzislau Rezki (Sony) addr = min((*va)->va_end & ~(align - 1), vmalloc_end); 438968ad4a33SUladzislau Rezki (Sony) if ((*va)->va_start < addr) 439068ad4a33SUladzislau Rezki (Sony) return addr; 439168ad4a33SUladzislau Rezki (Sony) } 4392ca23e405STejun Heo } 4393ca23e405STejun Heo 439468ad4a33SUladzislau Rezki (Sony) return 0; 4395ca23e405STejun Heo } 4396ca23e405STejun Heo 4397ca23e405STejun Heo /** 4398ca23e405STejun Heo * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator 4399ca23e405STejun Heo * @offsets: array containing offset of each area 4400ca23e405STejun Heo * @sizes: array containing size of each area 4401ca23e405STejun Heo * @nr_vms: the number of areas to allocate 4402ca23e405STejun Heo * @align: alignment, all entries in @offsets and @sizes must be aligned to this 4403ca23e405STejun Heo * 4404ca23e405STejun Heo * Returns: kmalloc'd vm_struct pointer array pointing to allocated 4405ca23e405STejun Heo * vm_structs on success, %NULL on failure 4406ca23e405STejun Heo * 4407ca23e405STejun Heo * Percpu allocator wants to use congruent vm areas so that it can 4408ca23e405STejun Heo * maintain the offsets among percpu areas. This function allocates 4409ec3f64fcSDavid Rientjes * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to 4410ec3f64fcSDavid Rientjes * be scattered pretty far, distance between two areas easily going up 4411ec3f64fcSDavid Rientjes * to gigabytes. To avoid interacting with regular vmallocs, these 4412ec3f64fcSDavid Rientjes * areas are allocated from top. 4413ca23e405STejun Heo * 4414ca23e405STejun Heo * Despite its complicated look, this allocator is rather simple. It 441568ad4a33SUladzislau Rezki (Sony) * does everything top-down and scans free blocks from the end looking 441668ad4a33SUladzislau Rezki (Sony) * for matching base. While scanning, if any of the areas do not fit the 441768ad4a33SUladzislau Rezki (Sony) * base address is pulled down to fit the area. Scanning is repeated till 441868ad4a33SUladzislau Rezki (Sony) * all the areas fit and then all necessary data structures are inserted 441968ad4a33SUladzislau Rezki (Sony) * and the result is returned. 4420ca23e405STejun Heo */ 4421ca23e405STejun Heo struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, 4422ca23e405STejun Heo const size_t *sizes, int nr_vms, 4423ec3f64fcSDavid Rientjes size_t align) 4424ca23e405STejun Heo { 4425ca23e405STejun Heo const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); 4426ca23e405STejun Heo const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 442768ad4a33SUladzislau Rezki (Sony) struct vmap_area **vas, *va; 4428ca23e405STejun Heo struct vm_struct **vms; 4429ca23e405STejun Heo int area, area2, last_area, term_area; 4430253a496dSDaniel Axtens unsigned long base, start, size, end, last_end, orig_start, orig_end; 4431ca23e405STejun Heo bool purged = false; 4432ca23e405STejun Heo 4433ca23e405STejun Heo /* verify parameters and allocate data structures */ 4434891c49abSAlexander Kuleshov BUG_ON(offset_in_page(align) || !is_power_of_2(align)); 4435ca23e405STejun Heo for (last_area = 0, area = 0; area < nr_vms; area++) { 4436ca23e405STejun Heo start = offsets[area]; 4437ca23e405STejun Heo end = start + sizes[area]; 4438ca23e405STejun Heo 4439ca23e405STejun Heo /* is everything aligned properly? */ 4440ca23e405STejun Heo BUG_ON(!IS_ALIGNED(offsets[area], align)); 4441ca23e405STejun Heo BUG_ON(!IS_ALIGNED(sizes[area], align)); 4442ca23e405STejun Heo 4443ca23e405STejun Heo /* detect the area with the highest address */ 4444ca23e405STejun Heo if (start > offsets[last_area]) 4445ca23e405STejun Heo last_area = area; 4446ca23e405STejun Heo 4447c568da28SWei Yang for (area2 = area + 1; area2 < nr_vms; area2++) { 4448ca23e405STejun Heo unsigned long start2 = offsets[area2]; 4449ca23e405STejun Heo unsigned long end2 = start2 + sizes[area2]; 4450ca23e405STejun Heo 4451c568da28SWei Yang BUG_ON(start2 < end && start < end2); 4452ca23e405STejun Heo } 4453ca23e405STejun Heo } 4454ca23e405STejun Heo last_end = offsets[last_area] + sizes[last_area]; 4455ca23e405STejun Heo 4456ca23e405STejun Heo if (vmalloc_end - vmalloc_start < last_end) { 4457ca23e405STejun Heo WARN_ON(true); 4458ca23e405STejun Heo return NULL; 4459ca23e405STejun Heo } 4460ca23e405STejun Heo 44614d67d860SThomas Meyer vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL); 44624d67d860SThomas Meyer vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL); 4463ca23e405STejun Heo if (!vas || !vms) 4464f1db7afdSKautuk Consul goto err_free2; 4465ca23e405STejun Heo 4466ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 446768ad4a33SUladzislau Rezki (Sony) vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL); 4468ec3f64fcSDavid Rientjes vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL); 4469ca23e405STejun Heo if (!vas[area] || !vms[area]) 4470ca23e405STejun Heo goto err_free; 4471ca23e405STejun Heo } 4472ca23e405STejun Heo retry: 4473e36176beSUladzislau Rezki (Sony) spin_lock(&free_vmap_area_lock); 4474ca23e405STejun Heo 4475ca23e405STejun Heo /* start scanning - we scan from the top, begin with the last area */ 4476ca23e405STejun Heo area = term_area = last_area; 4477ca23e405STejun Heo start = offsets[area]; 4478ca23e405STejun Heo end = start + sizes[area]; 4479ca23e405STejun Heo 448068ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(vmalloc_end); 448168ad4a33SUladzislau Rezki (Sony) base = pvm_determine_end_from_reverse(&va, align) - end; 4482ca23e405STejun Heo 4483ca23e405STejun Heo while (true) { 4484ca23e405STejun Heo /* 4485ca23e405STejun Heo * base might have underflowed, add last_end before 4486ca23e405STejun Heo * comparing. 4487ca23e405STejun Heo */ 448868ad4a33SUladzislau Rezki (Sony) if (base + last_end < vmalloc_start + last_end) 448968ad4a33SUladzislau Rezki (Sony) goto overflow; 4490ca23e405STejun Heo 4491ca23e405STejun Heo /* 449268ad4a33SUladzislau Rezki (Sony) * Fitting base has not been found. 4493ca23e405STejun Heo */ 449468ad4a33SUladzislau Rezki (Sony) if (va == NULL) 449568ad4a33SUladzislau Rezki (Sony) goto overflow; 4496ca23e405STejun Heo 4497ca23e405STejun Heo /* 4498d8cc323dSQiujun Huang * If required width exceeds current VA block, move 44995336e52cSKuppuswamy Sathyanarayanan * base downwards and then recheck. 45005336e52cSKuppuswamy Sathyanarayanan */ 45015336e52cSKuppuswamy Sathyanarayanan if (base + end > va->va_end) { 45025336e52cSKuppuswamy Sathyanarayanan base = pvm_determine_end_from_reverse(&va, align) - end; 45035336e52cSKuppuswamy Sathyanarayanan term_area = area; 45045336e52cSKuppuswamy Sathyanarayanan continue; 45055336e52cSKuppuswamy Sathyanarayanan } 45065336e52cSKuppuswamy Sathyanarayanan 45075336e52cSKuppuswamy Sathyanarayanan /* 450868ad4a33SUladzislau Rezki (Sony) * If this VA does not fit, move base downwards and recheck. 4509ca23e405STejun Heo */ 45105336e52cSKuppuswamy Sathyanarayanan if (base + start < va->va_start) { 451168ad4a33SUladzislau Rezki (Sony) va = node_to_va(rb_prev(&va->rb_node)); 451268ad4a33SUladzislau Rezki (Sony) base = pvm_determine_end_from_reverse(&va, align) - end; 4513ca23e405STejun Heo term_area = area; 4514ca23e405STejun Heo continue; 4515ca23e405STejun Heo } 4516ca23e405STejun Heo 4517ca23e405STejun Heo /* 4518ca23e405STejun Heo * This area fits, move on to the previous one. If 4519ca23e405STejun Heo * the previous one is the terminal one, we're done. 4520ca23e405STejun Heo */ 4521ca23e405STejun Heo area = (area + nr_vms - 1) % nr_vms; 4522ca23e405STejun Heo if (area == term_area) 4523ca23e405STejun Heo break; 452468ad4a33SUladzislau Rezki (Sony) 4525ca23e405STejun Heo start = offsets[area]; 4526ca23e405STejun Heo end = start + sizes[area]; 452768ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(base + end); 4528ca23e405STejun Heo } 452968ad4a33SUladzislau Rezki (Sony) 4530ca23e405STejun Heo /* we've found a fitting base, insert all va's */ 4531ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 453268ad4a33SUladzislau Rezki (Sony) int ret; 4533ca23e405STejun Heo 453468ad4a33SUladzislau Rezki (Sony) start = base + offsets[area]; 453568ad4a33SUladzislau Rezki (Sony) size = sizes[area]; 453668ad4a33SUladzislau Rezki (Sony) 453768ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(start); 453868ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(va == NULL)) 453968ad4a33SUladzislau Rezki (Sony) /* It is a BUG(), but trigger recovery instead. */ 454068ad4a33SUladzislau Rezki (Sony) goto recovery; 454168ad4a33SUladzislau Rezki (Sony) 45425b75b8e1SUladzislau Rezki (Sony) ret = va_clip(&free_vmap_area_root, 45435b75b8e1SUladzislau Rezki (Sony) &free_vmap_area_list, va, start, size); 45441b23ff80SBaoquan He if (WARN_ON_ONCE(unlikely(ret))) 454568ad4a33SUladzislau Rezki (Sony) /* It is a BUG(), but trigger recovery instead. */ 454668ad4a33SUladzislau Rezki (Sony) goto recovery; 454768ad4a33SUladzislau Rezki (Sony) 454868ad4a33SUladzislau Rezki (Sony) /* Allocated area. */ 454968ad4a33SUladzislau Rezki (Sony) va = vas[area]; 455068ad4a33SUladzislau Rezki (Sony) va->va_start = start; 455168ad4a33SUladzislau Rezki (Sony) va->va_end = start + size; 4552ca23e405STejun Heo } 4553ca23e405STejun Heo 4554e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 4555ca23e405STejun Heo 4556253a496dSDaniel Axtens /* populate the kasan shadow space */ 4557253a496dSDaniel Axtens for (area = 0; area < nr_vms; area++) { 4558253a496dSDaniel Axtens if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area])) 4559253a496dSDaniel Axtens goto err_free_shadow; 4560253a496dSDaniel Axtens } 4561253a496dSDaniel Axtens 4562ca23e405STejun Heo /* insert all vm's */ 4563e36176beSUladzislau Rezki (Sony) for (area = 0; area < nr_vms; area++) { 4564d0936029SUladzislau Rezki (Sony) struct vmap_node *vn = addr_to_node(vas[area]->va_start); 4565e36176beSUladzislau Rezki (Sony) 4566d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 4567d0936029SUladzislau Rezki (Sony) insert_vmap_area(vas[area], &vn->busy.root, &vn->busy.head); 4568e36176beSUladzislau Rezki (Sony) setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC, 4569ca23e405STejun Heo pcpu_get_vm_areas); 4570d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 4571e36176beSUladzislau Rezki (Sony) } 4572ca23e405STejun Heo 457319f1c3acSAndrey Konovalov /* 457419f1c3acSAndrey Konovalov * Mark allocated areas as accessible. Do it now as a best-effort 457519f1c3acSAndrey Konovalov * approach, as they can be mapped outside of vmalloc code. 457623689e91SAndrey Konovalov * With hardware tag-based KASAN, marking is skipped for 457723689e91SAndrey Konovalov * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 457819f1c3acSAndrey Konovalov */ 45791d96320fSAndrey Konovalov for (area = 0; area < nr_vms; area++) 45801d96320fSAndrey Konovalov vms[area]->addr = kasan_unpoison_vmalloc(vms[area]->addr, 4581f6e39794SAndrey Konovalov vms[area]->size, KASAN_VMALLOC_PROT_NORMAL); 45821d96320fSAndrey Konovalov 4583ca23e405STejun Heo kfree(vas); 4584ca23e405STejun Heo return vms; 4585ca23e405STejun Heo 458668ad4a33SUladzislau Rezki (Sony) recovery: 4587e36176beSUladzislau Rezki (Sony) /* 4588e36176beSUladzislau Rezki (Sony) * Remove previously allocated areas. There is no 4589e36176beSUladzislau Rezki (Sony) * need in removing these areas from the busy tree, 4590e36176beSUladzislau Rezki (Sony) * because they are inserted only on the final step 4591e36176beSUladzislau Rezki (Sony) * and when pcpu_get_vm_areas() is success. 4592e36176beSUladzislau Rezki (Sony) */ 459368ad4a33SUladzislau Rezki (Sony) while (area--) { 4594253a496dSDaniel Axtens orig_start = vas[area]->va_start; 4595253a496dSDaniel Axtens orig_end = vas[area]->va_end; 459696e2db45SUladzislau Rezki (Sony) va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, 45973c5c3cfbSDaniel Axtens &free_vmap_area_list); 45989c801f61SUladzislau Rezki (Sony) if (va) 4599253a496dSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 4600253a496dSDaniel Axtens va->va_start, va->va_end); 460168ad4a33SUladzislau Rezki (Sony) vas[area] = NULL; 460268ad4a33SUladzislau Rezki (Sony) } 460368ad4a33SUladzislau Rezki (Sony) 460468ad4a33SUladzislau Rezki (Sony) overflow: 4605e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 460668ad4a33SUladzislau Rezki (Sony) if (!purged) { 460777e50af0SThomas Gleixner reclaim_and_purge_vmap_areas(); 460868ad4a33SUladzislau Rezki (Sony) purged = true; 460968ad4a33SUladzislau Rezki (Sony) 461068ad4a33SUladzislau Rezki (Sony) /* Before "retry", check if we recover. */ 461168ad4a33SUladzislau Rezki (Sony) for (area = 0; area < nr_vms; area++) { 461268ad4a33SUladzislau Rezki (Sony) if (vas[area]) 461368ad4a33SUladzislau Rezki (Sony) continue; 461468ad4a33SUladzislau Rezki (Sony) 461568ad4a33SUladzislau Rezki (Sony) vas[area] = kmem_cache_zalloc( 461668ad4a33SUladzislau Rezki (Sony) vmap_area_cachep, GFP_KERNEL); 461768ad4a33SUladzislau Rezki (Sony) if (!vas[area]) 461868ad4a33SUladzislau Rezki (Sony) goto err_free; 461968ad4a33SUladzislau Rezki (Sony) } 462068ad4a33SUladzislau Rezki (Sony) 462168ad4a33SUladzislau Rezki (Sony) goto retry; 462268ad4a33SUladzislau Rezki (Sony) } 462368ad4a33SUladzislau Rezki (Sony) 4624ca23e405STejun Heo err_free: 4625ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 462668ad4a33SUladzislau Rezki (Sony) if (vas[area]) 462768ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, vas[area]); 462868ad4a33SUladzislau Rezki (Sony) 4629ca23e405STejun Heo kfree(vms[area]); 4630ca23e405STejun Heo } 4631f1db7afdSKautuk Consul err_free2: 4632ca23e405STejun Heo kfree(vas); 4633ca23e405STejun Heo kfree(vms); 4634ca23e405STejun Heo return NULL; 4635253a496dSDaniel Axtens 4636253a496dSDaniel Axtens err_free_shadow: 4637253a496dSDaniel Axtens spin_lock(&free_vmap_area_lock); 4638253a496dSDaniel Axtens /* 4639253a496dSDaniel Axtens * We release all the vmalloc shadows, even the ones for regions that 4640253a496dSDaniel Axtens * hadn't been successfully added. This relies on kasan_release_vmalloc 4641253a496dSDaniel Axtens * being able to tolerate this case. 4642253a496dSDaniel Axtens */ 4643253a496dSDaniel Axtens for (area = 0; area < nr_vms; area++) { 4644253a496dSDaniel Axtens orig_start = vas[area]->va_start; 4645253a496dSDaniel Axtens orig_end = vas[area]->va_end; 464696e2db45SUladzislau Rezki (Sony) va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, 4647253a496dSDaniel Axtens &free_vmap_area_list); 46489c801f61SUladzislau Rezki (Sony) if (va) 4649253a496dSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 4650253a496dSDaniel Axtens va->va_start, va->va_end); 4651253a496dSDaniel Axtens vas[area] = NULL; 4652253a496dSDaniel Axtens kfree(vms[area]); 4653253a496dSDaniel Axtens } 4654253a496dSDaniel Axtens spin_unlock(&free_vmap_area_lock); 4655253a496dSDaniel Axtens kfree(vas); 4656253a496dSDaniel Axtens kfree(vms); 4657253a496dSDaniel Axtens return NULL; 4658ca23e405STejun Heo } 4659ca23e405STejun Heo 4660ca23e405STejun Heo /** 4661ca23e405STejun Heo * pcpu_free_vm_areas - free vmalloc areas for percpu allocator 4662ca23e405STejun Heo * @vms: vm_struct pointer array returned by pcpu_get_vm_areas() 4663ca23e405STejun Heo * @nr_vms: the number of allocated areas 4664ca23e405STejun Heo * 4665ca23e405STejun Heo * Free vm_structs and the array allocated by pcpu_get_vm_areas(). 4666ca23e405STejun Heo */ 4667ca23e405STejun Heo void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) 4668ca23e405STejun Heo { 4669ca23e405STejun Heo int i; 4670ca23e405STejun Heo 4671ca23e405STejun Heo for (i = 0; i < nr_vms; i++) 4672ca23e405STejun Heo free_vm_area(vms[i]); 4673ca23e405STejun Heo kfree(vms); 4674ca23e405STejun Heo } 46754f8b02b4STejun Heo #endif /* CONFIG_SMP */ 4676a10aa579SChristoph Lameter 46775bb1bb35SPaul E. McKenney #ifdef CONFIG_PRINTK 467898f18083SPaul E. McKenney bool vmalloc_dump_obj(void *object) 467998f18083SPaul E. McKenney { 468098f18083SPaul E. McKenney void *objp = (void *)PAGE_ALIGN((unsigned long)object); 46810818e739SJoel Fernandes (Google) const void *caller; 46820818e739SJoel Fernandes (Google) struct vmap_area *va; 4683d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 46840818e739SJoel Fernandes (Google) unsigned long addr; 46850818e739SJoel Fernandes (Google) unsigned int nr_pages; 4686d0936029SUladzislau Rezki (Sony) bool success = false; 468798f18083SPaul E. McKenney 4688d0936029SUladzislau Rezki (Sony) vn = addr_to_node((unsigned long)objp); 4689d0936029SUladzislau Rezki (Sony) 4690d0936029SUladzislau Rezki (Sony) if (spin_trylock(&vn->busy.lock)) { 4691d0936029SUladzislau Rezki (Sony) va = __find_vmap_area((unsigned long)objp, &vn->busy.root); 4692d0936029SUladzislau Rezki (Sony) 4693d0936029SUladzislau Rezki (Sony) if (va && va->vm) { 4694d0936029SUladzislau Rezki (Sony) addr = (unsigned long)va->vm->addr; 4695d0936029SUladzislau Rezki (Sony) caller = va->vm->caller; 4696d0936029SUladzislau Rezki (Sony) nr_pages = va->vm->nr_pages; 4697d0936029SUladzislau Rezki (Sony) success = true; 46980818e739SJoel Fernandes (Google) } 46990818e739SJoel Fernandes (Google) 4700d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 47010818e739SJoel Fernandes (Google) } 4702d0936029SUladzislau Rezki (Sony) 4703d0936029SUladzislau Rezki (Sony) if (success) 4704bd34dcd4SPaul E. McKenney pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n", 47050818e739SJoel Fernandes (Google) nr_pages, addr, caller); 4706d0936029SUladzislau Rezki (Sony) 4707d0936029SUladzislau Rezki (Sony) return success; 470898f18083SPaul E. McKenney } 47095bb1bb35SPaul E. McKenney #endif 471098f18083SPaul E. McKenney 4711a10aa579SChristoph Lameter #ifdef CONFIG_PROC_FS 4712a47a126aSEric Dumazet static void show_numa_info(struct seq_file *m, struct vm_struct *v) 4713a47a126aSEric Dumazet { 4714e5adfffcSKirill A. Shutemov if (IS_ENABLED(CONFIG_NUMA)) { 4715a47a126aSEric Dumazet unsigned int nr, *counters = m->private; 471651e50b3aSEric Dumazet unsigned int step = 1U << vm_area_page_order(v); 4717a47a126aSEric Dumazet 4718a47a126aSEric Dumazet if (!counters) 4719a47a126aSEric Dumazet return; 4720a47a126aSEric Dumazet 4721af12346cSWanpeng Li if (v->flags & VM_UNINITIALIZED) 4722af12346cSWanpeng Li return; 47237e5b528bSDmitry Vyukov /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ 47247e5b528bSDmitry Vyukov smp_rmb(); 4725af12346cSWanpeng Li 4726a47a126aSEric Dumazet memset(counters, 0, nr_node_ids * sizeof(unsigned int)); 4727a47a126aSEric Dumazet 472851e50b3aSEric Dumazet for (nr = 0; nr < v->nr_pages; nr += step) 472951e50b3aSEric Dumazet counters[page_to_nid(v->pages[nr])] += step; 4730a47a126aSEric Dumazet for_each_node_state(nr, N_HIGH_MEMORY) 4731a47a126aSEric Dumazet if (counters[nr]) 4732a47a126aSEric Dumazet seq_printf(m, " N%u=%u", nr, counters[nr]); 4733a47a126aSEric Dumazet } 4734a47a126aSEric Dumazet } 4735a47a126aSEric Dumazet 4736dd3b8353SUladzislau Rezki (Sony) static void show_purge_info(struct seq_file *m) 4737dd3b8353SUladzislau Rezki (Sony) { 4738282631cbSUladzislau Rezki (Sony) struct vmap_node *vn; 4739dd3b8353SUladzislau Rezki (Sony) struct vmap_area *va; 4740282631cbSUladzislau Rezki (Sony) int i; 4741dd3b8353SUladzislau Rezki (Sony) 4742282631cbSUladzislau Rezki (Sony) for (i = 0; i < nr_vmap_nodes; i++) { 4743282631cbSUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 4744282631cbSUladzislau Rezki (Sony) 4745282631cbSUladzislau Rezki (Sony) spin_lock(&vn->lazy.lock); 4746282631cbSUladzislau Rezki (Sony) list_for_each_entry(va, &vn->lazy.head, list) { 4747dd3b8353SUladzislau Rezki (Sony) seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n", 4748dd3b8353SUladzislau Rezki (Sony) (void *)va->va_start, (void *)va->va_end, 4749dd3b8353SUladzislau Rezki (Sony) va->va_end - va->va_start); 4750dd3b8353SUladzislau Rezki (Sony) } 4751282631cbSUladzislau Rezki (Sony) spin_unlock(&vn->lazy.lock); 4752282631cbSUladzislau Rezki (Sony) } 4753dd3b8353SUladzislau Rezki (Sony) } 4754dd3b8353SUladzislau Rezki (Sony) 47558e1d743fSUladzislau Rezki (Sony) static int vmalloc_info_show(struct seq_file *m, void *p) 4756a10aa579SChristoph Lameter { 4757d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 47583f500069Szijun_hu struct vmap_area *va; 4759d4033afdSJoonsoo Kim struct vm_struct *v; 47608e1d743fSUladzislau Rezki (Sony) int i; 4761d4033afdSJoonsoo Kim 47628e1d743fSUladzislau Rezki (Sony) for (i = 0; i < nr_vmap_nodes; i++) { 47638e1d743fSUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 47643f500069Szijun_hu 47658e1d743fSUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 47668e1d743fSUladzislau Rezki (Sony) list_for_each_entry(va, &vn->busy.head, list) { 4767688fcbfcSPengfei Li if (!va->vm) { 4768bba9697bSBaoquan He if (va->flags & VMAP_RAM) 4769dd3b8353SUladzislau Rezki (Sony) seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n", 477078c72746SYisheng Xie (void *)va->va_start, (void *)va->va_end, 4771dd3b8353SUladzislau Rezki (Sony) va->va_end - va->va_start); 477278c72746SYisheng Xie 47738e1d743fSUladzislau Rezki (Sony) continue; 477478c72746SYisheng Xie } 4775d4033afdSJoonsoo Kim 4776d4033afdSJoonsoo Kim v = va->vm; 4777a10aa579SChristoph Lameter 477845ec1690SKees Cook seq_printf(m, "0x%pK-0x%pK %7ld", 4779a10aa579SChristoph Lameter v->addr, v->addr + v->size, v->size); 4780a10aa579SChristoph Lameter 478162c70bceSJoe Perches if (v->caller) 478262c70bceSJoe Perches seq_printf(m, " %pS", v->caller); 478323016969SChristoph Lameter 4784a10aa579SChristoph Lameter if (v->nr_pages) 4785a10aa579SChristoph Lameter seq_printf(m, " pages=%d", v->nr_pages); 4786a10aa579SChristoph Lameter 4787a10aa579SChristoph Lameter if (v->phys_addr) 4788199eaa05SMiles Chen seq_printf(m, " phys=%pa", &v->phys_addr); 4789a10aa579SChristoph Lameter 4790a10aa579SChristoph Lameter if (v->flags & VM_IOREMAP) 4791f4527c90SFabian Frederick seq_puts(m, " ioremap"); 4792a10aa579SChristoph Lameter 4793a10aa579SChristoph Lameter if (v->flags & VM_ALLOC) 4794f4527c90SFabian Frederick seq_puts(m, " vmalloc"); 4795a10aa579SChristoph Lameter 4796a10aa579SChristoph Lameter if (v->flags & VM_MAP) 4797f4527c90SFabian Frederick seq_puts(m, " vmap"); 4798a10aa579SChristoph Lameter 4799a10aa579SChristoph Lameter if (v->flags & VM_USERMAP) 4800f4527c90SFabian Frederick seq_puts(m, " user"); 4801a10aa579SChristoph Lameter 4802fe9041c2SChristoph Hellwig if (v->flags & VM_DMA_COHERENT) 4803fe9041c2SChristoph Hellwig seq_puts(m, " dma-coherent"); 4804fe9041c2SChristoph Hellwig 4805244d63eeSDavid Rientjes if (is_vmalloc_addr(v->pages)) 4806f4527c90SFabian Frederick seq_puts(m, " vpages"); 4807a10aa579SChristoph Lameter 4808a47a126aSEric Dumazet show_numa_info(m, v); 4809a10aa579SChristoph Lameter seq_putc(m, '\n'); 48108e1d743fSUladzislau Rezki (Sony) } 48118e1d743fSUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 48128e1d743fSUladzislau Rezki (Sony) } 4813dd3b8353SUladzislau Rezki (Sony) 4814dd3b8353SUladzislau Rezki (Sony) /* 481596e2db45SUladzislau Rezki (Sony) * As a final step, dump "unpurged" areas. 4816dd3b8353SUladzislau Rezki (Sony) */ 4817dd3b8353SUladzislau Rezki (Sony) show_purge_info(m); 4818a10aa579SChristoph Lameter return 0; 4819a10aa579SChristoph Lameter } 4820a10aa579SChristoph Lameter 48215f6a6a9cSAlexey Dobriyan static int __init proc_vmalloc_init(void) 48225f6a6a9cSAlexey Dobriyan { 48238e1d743fSUladzislau Rezki (Sony) void *priv_data = NULL; 48248e1d743fSUladzislau Rezki (Sony) 4825fddda2b7SChristoph Hellwig if (IS_ENABLED(CONFIG_NUMA)) 48268e1d743fSUladzislau Rezki (Sony) priv_data = kmalloc(nr_node_ids * sizeof(unsigned int), GFP_KERNEL); 48278e1d743fSUladzislau Rezki (Sony) 48288e1d743fSUladzislau Rezki (Sony) proc_create_single_data("vmallocinfo", 48298e1d743fSUladzislau Rezki (Sony) 0400, NULL, vmalloc_info_show, priv_data); 48308e1d743fSUladzislau Rezki (Sony) 48315f6a6a9cSAlexey Dobriyan return 0; 48325f6a6a9cSAlexey Dobriyan } 48335f6a6a9cSAlexey Dobriyan module_init(proc_vmalloc_init); 4834db3808c1SJoonsoo Kim 4835a10aa579SChristoph Lameter #endif 4836208162f4SChristoph Hellwig 4837d0936029SUladzislau Rezki (Sony) static void __init vmap_init_free_space(void) 48387fa8cee0SUladzislau Rezki (Sony) { 48397fa8cee0SUladzislau Rezki (Sony) unsigned long vmap_start = 1; 48407fa8cee0SUladzislau Rezki (Sony) const unsigned long vmap_end = ULONG_MAX; 4841d0936029SUladzislau Rezki (Sony) struct vmap_area *free; 4842d0936029SUladzislau Rezki (Sony) struct vm_struct *busy; 48437fa8cee0SUladzislau Rezki (Sony) 48447fa8cee0SUladzislau Rezki (Sony) /* 48457fa8cee0SUladzislau Rezki (Sony) * B F B B B F 48467fa8cee0SUladzislau Rezki (Sony) * -|-----|.....|-----|-----|-----|.....|- 48477fa8cee0SUladzislau Rezki (Sony) * | The KVA space | 48487fa8cee0SUladzislau Rezki (Sony) * |<--------------------------------->| 48497fa8cee0SUladzislau Rezki (Sony) */ 4850d0936029SUladzislau Rezki (Sony) for (busy = vmlist; busy; busy = busy->next) { 4851d0936029SUladzislau Rezki (Sony) if ((unsigned long) busy->addr - vmap_start > 0) { 48527fa8cee0SUladzislau Rezki (Sony) free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 48537fa8cee0SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(!free)) { 48547fa8cee0SUladzislau Rezki (Sony) free->va_start = vmap_start; 4855d0936029SUladzislau Rezki (Sony) free->va_end = (unsigned long) busy->addr; 48567fa8cee0SUladzislau Rezki (Sony) 48577fa8cee0SUladzislau Rezki (Sony) insert_vmap_area_augment(free, NULL, 48587fa8cee0SUladzislau Rezki (Sony) &free_vmap_area_root, 48597fa8cee0SUladzislau Rezki (Sony) &free_vmap_area_list); 48607fa8cee0SUladzislau Rezki (Sony) } 48617fa8cee0SUladzislau Rezki (Sony) } 48627fa8cee0SUladzislau Rezki (Sony) 4863d0936029SUladzislau Rezki (Sony) vmap_start = (unsigned long) busy->addr + busy->size; 48647fa8cee0SUladzislau Rezki (Sony) } 48657fa8cee0SUladzislau Rezki (Sony) 48667fa8cee0SUladzislau Rezki (Sony) if (vmap_end - vmap_start > 0) { 48677fa8cee0SUladzislau Rezki (Sony) free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 48687fa8cee0SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(!free)) { 48697fa8cee0SUladzislau Rezki (Sony) free->va_start = vmap_start; 48707fa8cee0SUladzislau Rezki (Sony) free->va_end = vmap_end; 48717fa8cee0SUladzislau Rezki (Sony) 48727fa8cee0SUladzislau Rezki (Sony) insert_vmap_area_augment(free, NULL, 48737fa8cee0SUladzislau Rezki (Sony) &free_vmap_area_root, 48747fa8cee0SUladzislau Rezki (Sony) &free_vmap_area_list); 48757fa8cee0SUladzislau Rezki (Sony) } 48767fa8cee0SUladzislau Rezki (Sony) } 48777fa8cee0SUladzislau Rezki (Sony) } 48787fa8cee0SUladzislau Rezki (Sony) 4879d0936029SUladzislau Rezki (Sony) static void vmap_init_nodes(void) 4880d0936029SUladzislau Rezki (Sony) { 4881d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 48828f33a2ffSUladzislau Rezki (Sony) int i, n; 4883d0936029SUladzislau Rezki (Sony) 48848f33a2ffSUladzislau Rezki (Sony) #if BITS_PER_LONG == 64 48858f33a2ffSUladzislau Rezki (Sony) /* A high threshold of max nodes is fixed and bound to 128. */ 48868f33a2ffSUladzislau Rezki (Sony) n = clamp_t(unsigned int, num_possible_cpus(), 1, 128); 48878f33a2ffSUladzislau Rezki (Sony) 48888f33a2ffSUladzislau Rezki (Sony) if (n > 1) { 48898f33a2ffSUladzislau Rezki (Sony) vn = kmalloc_array(n, sizeof(*vn), GFP_NOWAIT | __GFP_NOWARN); 48908f33a2ffSUladzislau Rezki (Sony) if (vn) { 48918f33a2ffSUladzislau Rezki (Sony) /* Node partition is 16 pages. */ 48928f33a2ffSUladzislau Rezki (Sony) vmap_zone_size = (1 << 4) * PAGE_SIZE; 48938f33a2ffSUladzislau Rezki (Sony) nr_vmap_nodes = n; 48948f33a2ffSUladzislau Rezki (Sony) vmap_nodes = vn; 48958f33a2ffSUladzislau Rezki (Sony) } else { 48968f33a2ffSUladzislau Rezki (Sony) pr_err("Failed to allocate an array. Disable a node layer\n"); 48978f33a2ffSUladzislau Rezki (Sony) } 48988f33a2ffSUladzislau Rezki (Sony) } 48998f33a2ffSUladzislau Rezki (Sony) #endif 49008f33a2ffSUladzislau Rezki (Sony) 49018f33a2ffSUladzislau Rezki (Sony) for (n = 0; n < nr_vmap_nodes; n++) { 49028f33a2ffSUladzislau Rezki (Sony) vn = &vmap_nodes[n]; 4903d0936029SUladzislau Rezki (Sony) vn->busy.root = RB_ROOT; 4904d0936029SUladzislau Rezki (Sony) INIT_LIST_HEAD(&vn->busy.head); 4905d0936029SUladzislau Rezki (Sony) spin_lock_init(&vn->busy.lock); 4906282631cbSUladzislau Rezki (Sony) 4907282631cbSUladzislau Rezki (Sony) vn->lazy.root = RB_ROOT; 4908282631cbSUladzislau Rezki (Sony) INIT_LIST_HEAD(&vn->lazy.head); 4909282631cbSUladzislau Rezki (Sony) spin_lock_init(&vn->lazy.lock); 491072210662SUladzislau Rezki (Sony) 49118f33a2ffSUladzislau Rezki (Sony) for (i = 0; i < MAX_VA_SIZE_PAGES; i++) { 49128f33a2ffSUladzislau Rezki (Sony) INIT_LIST_HEAD(&vn->pool[i].head); 49138f33a2ffSUladzislau Rezki (Sony) WRITE_ONCE(vn->pool[i].len, 0); 491472210662SUladzislau Rezki (Sony) } 491572210662SUladzislau Rezki (Sony) 491672210662SUladzislau Rezki (Sony) spin_lock_init(&vn->pool_lock); 4917d0936029SUladzislau Rezki (Sony) } 4918d0936029SUladzislau Rezki (Sony) } 4919d0936029SUladzislau Rezki (Sony) 4920*7679ba6bSUladzislau Rezki (Sony) static unsigned long 4921*7679ba6bSUladzislau Rezki (Sony) vmap_node_shrink_count(struct shrinker *shrink, struct shrink_control *sc) 4922*7679ba6bSUladzislau Rezki (Sony) { 4923*7679ba6bSUladzislau Rezki (Sony) unsigned long count; 4924*7679ba6bSUladzislau Rezki (Sony) struct vmap_node *vn; 4925*7679ba6bSUladzislau Rezki (Sony) int i, j; 4926*7679ba6bSUladzislau Rezki (Sony) 4927*7679ba6bSUladzislau Rezki (Sony) for (count = 0, i = 0; i < nr_vmap_nodes; i++) { 4928*7679ba6bSUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 4929*7679ba6bSUladzislau Rezki (Sony) 4930*7679ba6bSUladzislau Rezki (Sony) for (j = 0; j < MAX_VA_SIZE_PAGES; j++) 4931*7679ba6bSUladzislau Rezki (Sony) count += READ_ONCE(vn->pool[j].len); 4932*7679ba6bSUladzislau Rezki (Sony) } 4933*7679ba6bSUladzislau Rezki (Sony) 4934*7679ba6bSUladzislau Rezki (Sony) return count ? count : SHRINK_EMPTY; 4935*7679ba6bSUladzislau Rezki (Sony) } 4936*7679ba6bSUladzislau Rezki (Sony) 4937*7679ba6bSUladzislau Rezki (Sony) static unsigned long 4938*7679ba6bSUladzislau Rezki (Sony) vmap_node_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) 4939*7679ba6bSUladzislau Rezki (Sony) { 4940*7679ba6bSUladzislau Rezki (Sony) int i; 4941*7679ba6bSUladzislau Rezki (Sony) 4942*7679ba6bSUladzislau Rezki (Sony) for (i = 0; i < nr_vmap_nodes; i++) 4943*7679ba6bSUladzislau Rezki (Sony) decay_va_pool_node(&vmap_nodes[i], true); 4944*7679ba6bSUladzislau Rezki (Sony) 4945*7679ba6bSUladzislau Rezki (Sony) return SHRINK_STOP; 4946*7679ba6bSUladzislau Rezki (Sony) } 4947*7679ba6bSUladzislau Rezki (Sony) 4948208162f4SChristoph Hellwig void __init vmalloc_init(void) 4949208162f4SChristoph Hellwig { 4950*7679ba6bSUladzislau Rezki (Sony) struct shrinker *vmap_node_shrinker; 4951208162f4SChristoph Hellwig struct vmap_area *va; 4952d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 4953208162f4SChristoph Hellwig struct vm_struct *tmp; 4954208162f4SChristoph Hellwig int i; 4955208162f4SChristoph Hellwig 4956208162f4SChristoph Hellwig /* 4957208162f4SChristoph Hellwig * Create the cache for vmap_area objects. 4958208162f4SChristoph Hellwig */ 4959208162f4SChristoph Hellwig vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC); 4960208162f4SChristoph Hellwig 4961208162f4SChristoph Hellwig for_each_possible_cpu(i) { 4962208162f4SChristoph Hellwig struct vmap_block_queue *vbq; 4963208162f4SChristoph Hellwig struct vfree_deferred *p; 4964208162f4SChristoph Hellwig 4965208162f4SChristoph Hellwig vbq = &per_cpu(vmap_block_queue, i); 4966208162f4SChristoph Hellwig spin_lock_init(&vbq->lock); 4967208162f4SChristoph Hellwig INIT_LIST_HEAD(&vbq->free); 4968208162f4SChristoph Hellwig p = &per_cpu(vfree_deferred, i); 4969208162f4SChristoph Hellwig init_llist_head(&p->list); 4970208162f4SChristoph Hellwig INIT_WORK(&p->wq, delayed_vfree_work); 4971062eacf5SUladzislau Rezki (Sony) xa_init(&vbq->vmap_blocks); 4972208162f4SChristoph Hellwig } 4973208162f4SChristoph Hellwig 4974d0936029SUladzislau Rezki (Sony) /* 4975d0936029SUladzislau Rezki (Sony) * Setup nodes before importing vmlist. 4976d0936029SUladzislau Rezki (Sony) */ 4977d0936029SUladzislau Rezki (Sony) vmap_init_nodes(); 4978d0936029SUladzislau Rezki (Sony) 4979208162f4SChristoph Hellwig /* Import existing vmlist entries. */ 4980208162f4SChristoph Hellwig for (tmp = vmlist; tmp; tmp = tmp->next) { 4981208162f4SChristoph Hellwig va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 4982208162f4SChristoph Hellwig if (WARN_ON_ONCE(!va)) 4983208162f4SChristoph Hellwig continue; 4984208162f4SChristoph Hellwig 4985208162f4SChristoph Hellwig va->va_start = (unsigned long)tmp->addr; 4986208162f4SChristoph Hellwig va->va_end = va->va_start + tmp->size; 4987208162f4SChristoph Hellwig va->vm = tmp; 4988d0936029SUladzislau Rezki (Sony) 4989d0936029SUladzislau Rezki (Sony) vn = addr_to_node(va->va_start); 4990d0936029SUladzislau Rezki (Sony) insert_vmap_area(va, &vn->busy.root, &vn->busy.head); 4991208162f4SChristoph Hellwig } 4992208162f4SChristoph Hellwig 4993208162f4SChristoph Hellwig /* 4994208162f4SChristoph Hellwig * Now we can initialize a free vmap space. 4995208162f4SChristoph Hellwig */ 4996208162f4SChristoph Hellwig vmap_init_free_space(); 4997208162f4SChristoph Hellwig vmap_initialized = true; 4998*7679ba6bSUladzislau Rezki (Sony) 4999*7679ba6bSUladzislau Rezki (Sony) vmap_node_shrinker = shrinker_alloc(0, "vmap-node"); 5000*7679ba6bSUladzislau Rezki (Sony) if (!vmap_node_shrinker) { 5001*7679ba6bSUladzislau Rezki (Sony) pr_err("Failed to allocate vmap-node shrinker!\n"); 5002*7679ba6bSUladzislau Rezki (Sony) return; 5003*7679ba6bSUladzislau Rezki (Sony) } 5004*7679ba6bSUladzislau Rezki (Sony) 5005*7679ba6bSUladzislau Rezki (Sony) vmap_node_shrinker->count_objects = vmap_node_shrink_count; 5006*7679ba6bSUladzislau Rezki (Sony) vmap_node_shrinker->scan_objects = vmap_node_shrink_scan; 5007*7679ba6bSUladzislau Rezki (Sony) shrinker_register(vmap_node_shrinker); 5008208162f4SChristoph Hellwig } 5009