1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * Copyright (C) 1993 Linus Torvalds 41da177e4SLinus Torvalds * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 51da177e4SLinus Torvalds * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 61da177e4SLinus Torvalds * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 7930fc45aSChristoph Lameter * Numa awareness, Christoph Lameter, SGI, June 2005 8d758ffe6SUladzislau Rezki (Sony) * Improving global KVA allocator, Uladzislau Rezki, Sony, May 2019 91da177e4SLinus Torvalds */ 101da177e4SLinus Torvalds 11db64fe02SNick Piggin #include <linux/vmalloc.h> 121da177e4SLinus Torvalds #include <linux/mm.h> 131da177e4SLinus Torvalds #include <linux/module.h> 141da177e4SLinus Torvalds #include <linux/highmem.h> 15c3edc401SIngo Molnar #include <linux/sched/signal.h> 161da177e4SLinus Torvalds #include <linux/slab.h> 171da177e4SLinus Torvalds #include <linux/spinlock.h> 181da177e4SLinus Torvalds #include <linux/interrupt.h> 195f6a6a9cSAlexey Dobriyan #include <linux/proc_fs.h> 20a10aa579SChristoph Lameter #include <linux/seq_file.h> 21868b104dSRick Edgecombe #include <linux/set_memory.h> 223ac7fe5aSThomas Gleixner #include <linux/debugobjects.h> 2323016969SChristoph Lameter #include <linux/kallsyms.h> 24db64fe02SNick Piggin #include <linux/list.h> 254da56b99SChris Wilson #include <linux/notifier.h> 26db64fe02SNick Piggin #include <linux/rbtree.h> 270f14599cSMatthew Wilcox (Oracle) #include <linux/xarray.h> 285da96bddSMel Gorman #include <linux/io.h> 29db64fe02SNick Piggin #include <linux/rcupdate.h> 30f0aa6617STejun Heo #include <linux/pfn.h> 3189219d37SCatalin Marinas #include <linux/kmemleak.h> 3260063497SArun Sharma #include <linux/atomic.h> 333b32123dSGideon Israel Dsouza #include <linux/compiler.h> 344e5aa1f4SShakeel Butt #include <linux/memcontrol.h> 3532fcfd40SAl Viro #include <linux/llist.h> 364c91c07cSLorenzo Stoakes #include <linux/uio.h> 370f616be1SToshi Kani #include <linux/bitops.h> 3868ad4a33SUladzislau Rezki (Sony) #include <linux/rbtree_augmented.h> 39bdebd6a2SJann Horn #include <linux/overflow.h> 40c0eb315aSNicholas Piggin #include <linux/pgtable.h> 41f7ee1f13SChristophe Leroy #include <linux/hugetlb.h> 42451769ebSMichal Hocko #include <linux/sched/mm.h> 431da177e4SLinus Torvalds #include <asm/tlbflush.h> 442dca6999SDavid Miller #include <asm/shmparam.h> 451da177e4SLinus Torvalds 46cf243da6SUladzislau Rezki (Sony) #define CREATE_TRACE_POINTS 47cf243da6SUladzislau Rezki (Sony) #include <trace/events/vmalloc.h> 48cf243da6SUladzislau Rezki (Sony) 49dd56b046SMel Gorman #include "internal.h" 502a681cfaSJoerg Roedel #include "pgalloc-track.h" 51dd56b046SMel Gorman 5282a70ce0SChristoph Hellwig #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 5382a70ce0SChristoph Hellwig static unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1; 5482a70ce0SChristoph Hellwig 5582a70ce0SChristoph Hellwig static int __init set_nohugeiomap(char *str) 5682a70ce0SChristoph Hellwig { 5782a70ce0SChristoph Hellwig ioremap_max_page_shift = PAGE_SHIFT; 5882a70ce0SChristoph Hellwig return 0; 5982a70ce0SChristoph Hellwig } 6082a70ce0SChristoph Hellwig early_param("nohugeiomap", set_nohugeiomap); 6182a70ce0SChristoph Hellwig #else /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 6282a70ce0SChristoph Hellwig static const unsigned int ioremap_max_page_shift = PAGE_SHIFT; 6382a70ce0SChristoph Hellwig #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 6482a70ce0SChristoph Hellwig 65121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 66121e6f32SNicholas Piggin static bool __ro_after_init vmap_allow_huge = true; 67121e6f32SNicholas Piggin 68121e6f32SNicholas Piggin static int __init set_nohugevmalloc(char *str) 69121e6f32SNicholas Piggin { 70121e6f32SNicholas Piggin vmap_allow_huge = false; 71121e6f32SNicholas Piggin return 0; 72121e6f32SNicholas Piggin } 73121e6f32SNicholas Piggin early_param("nohugevmalloc", set_nohugevmalloc); 74121e6f32SNicholas Piggin #else /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ 75121e6f32SNicholas Piggin static const bool vmap_allow_huge = false; 76121e6f32SNicholas Piggin #endif /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ 77121e6f32SNicholas Piggin 78186525bdSIngo Molnar bool is_vmalloc_addr(const void *x) 79186525bdSIngo Molnar { 804aff1dc4SAndrey Konovalov unsigned long addr = (unsigned long)kasan_reset_tag(x); 81186525bdSIngo Molnar 82186525bdSIngo Molnar return addr >= VMALLOC_START && addr < VMALLOC_END; 83186525bdSIngo Molnar } 84186525bdSIngo Molnar EXPORT_SYMBOL(is_vmalloc_addr); 85186525bdSIngo Molnar 8632fcfd40SAl Viro struct vfree_deferred { 8732fcfd40SAl Viro struct llist_head list; 8832fcfd40SAl Viro struct work_struct wq; 8932fcfd40SAl Viro }; 9032fcfd40SAl Viro static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); 9132fcfd40SAl Viro 92db64fe02SNick Piggin /*** Page table manipulation functions ***/ 935e9e3d77SNicholas Piggin static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 945e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 95f7ee1f13SChristophe Leroy unsigned int max_page_shift, pgtbl_mod_mask *mask) 965e9e3d77SNicholas Piggin { 975e9e3d77SNicholas Piggin pte_t *pte; 985e9e3d77SNicholas Piggin u64 pfn; 99f7ee1f13SChristophe Leroy unsigned long size = PAGE_SIZE; 1005e9e3d77SNicholas Piggin 1015e9e3d77SNicholas Piggin pfn = phys_addr >> PAGE_SHIFT; 1025e9e3d77SNicholas Piggin pte = pte_alloc_kernel_track(pmd, addr, mask); 1035e9e3d77SNicholas Piggin if (!pte) 1045e9e3d77SNicholas Piggin return -ENOMEM; 1055e9e3d77SNicholas Piggin do { 106c33c7948SRyan Roberts BUG_ON(!pte_none(ptep_get(pte))); 107f7ee1f13SChristophe Leroy 108f7ee1f13SChristophe Leroy #ifdef CONFIG_HUGETLB_PAGE 109f7ee1f13SChristophe Leroy size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift); 110f7ee1f13SChristophe Leroy if (size != PAGE_SIZE) { 111f7ee1f13SChristophe Leroy pte_t entry = pfn_pte(pfn, prot); 112f7ee1f13SChristophe Leroy 113f7ee1f13SChristophe Leroy entry = arch_make_huge_pte(entry, ilog2(size), 0); 114935d4f0cSRyan Roberts set_huge_pte_at(&init_mm, addr, pte, entry, size); 115f7ee1f13SChristophe Leroy pfn += PFN_DOWN(size); 116f7ee1f13SChristophe Leroy continue; 117f7ee1f13SChristophe Leroy } 118f7ee1f13SChristophe Leroy #endif 1195e9e3d77SNicholas Piggin set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); 1205e9e3d77SNicholas Piggin pfn++; 121f7ee1f13SChristophe Leroy } while (pte += PFN_DOWN(size), addr += size, addr != end); 1225e9e3d77SNicholas Piggin *mask |= PGTBL_PTE_MODIFIED; 1235e9e3d77SNicholas Piggin return 0; 1245e9e3d77SNicholas Piggin } 1255e9e3d77SNicholas Piggin 1265e9e3d77SNicholas Piggin static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, 1275e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1285e9e3d77SNicholas Piggin unsigned int max_page_shift) 1295e9e3d77SNicholas Piggin { 1305e9e3d77SNicholas Piggin if (max_page_shift < PMD_SHIFT) 1315e9e3d77SNicholas Piggin return 0; 1325e9e3d77SNicholas Piggin 1335e9e3d77SNicholas Piggin if (!arch_vmap_pmd_supported(prot)) 1345e9e3d77SNicholas Piggin return 0; 1355e9e3d77SNicholas Piggin 1365e9e3d77SNicholas Piggin if ((end - addr) != PMD_SIZE) 1375e9e3d77SNicholas Piggin return 0; 1385e9e3d77SNicholas Piggin 1395e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, PMD_SIZE)) 1405e9e3d77SNicholas Piggin return 0; 1415e9e3d77SNicholas Piggin 1425e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, PMD_SIZE)) 1435e9e3d77SNicholas Piggin return 0; 1445e9e3d77SNicholas Piggin 1455e9e3d77SNicholas Piggin if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) 1465e9e3d77SNicholas Piggin return 0; 1475e9e3d77SNicholas Piggin 1485e9e3d77SNicholas Piggin return pmd_set_huge(pmd, phys_addr, prot); 1495e9e3d77SNicholas Piggin } 1505e9e3d77SNicholas Piggin 1515e9e3d77SNicholas Piggin static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 1525e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1535e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 1545e9e3d77SNicholas Piggin { 1555e9e3d77SNicholas Piggin pmd_t *pmd; 1565e9e3d77SNicholas Piggin unsigned long next; 1575e9e3d77SNicholas Piggin 1585e9e3d77SNicholas Piggin pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 1595e9e3d77SNicholas Piggin if (!pmd) 1605e9e3d77SNicholas Piggin return -ENOMEM; 1615e9e3d77SNicholas Piggin do { 1625e9e3d77SNicholas Piggin next = pmd_addr_end(addr, end); 1635e9e3d77SNicholas Piggin 1645e9e3d77SNicholas Piggin if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot, 1655e9e3d77SNicholas Piggin max_page_shift)) { 1665e9e3d77SNicholas Piggin *mask |= PGTBL_PMD_MODIFIED; 1675e9e3d77SNicholas Piggin continue; 1685e9e3d77SNicholas Piggin } 1695e9e3d77SNicholas Piggin 170f7ee1f13SChristophe Leroy if (vmap_pte_range(pmd, addr, next, phys_addr, prot, max_page_shift, mask)) 1715e9e3d77SNicholas Piggin return -ENOMEM; 1725e9e3d77SNicholas Piggin } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); 1735e9e3d77SNicholas Piggin return 0; 1745e9e3d77SNicholas Piggin } 1755e9e3d77SNicholas Piggin 1765e9e3d77SNicholas Piggin static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end, 1775e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 1785e9e3d77SNicholas Piggin unsigned int max_page_shift) 1795e9e3d77SNicholas Piggin { 1805e9e3d77SNicholas Piggin if (max_page_shift < PUD_SHIFT) 1815e9e3d77SNicholas Piggin return 0; 1825e9e3d77SNicholas Piggin 1835e9e3d77SNicholas Piggin if (!arch_vmap_pud_supported(prot)) 1845e9e3d77SNicholas Piggin return 0; 1855e9e3d77SNicholas Piggin 1865e9e3d77SNicholas Piggin if ((end - addr) != PUD_SIZE) 1875e9e3d77SNicholas Piggin return 0; 1885e9e3d77SNicholas Piggin 1895e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, PUD_SIZE)) 1905e9e3d77SNicholas Piggin return 0; 1915e9e3d77SNicholas Piggin 1925e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, PUD_SIZE)) 1935e9e3d77SNicholas Piggin return 0; 1945e9e3d77SNicholas Piggin 1955e9e3d77SNicholas Piggin if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) 1965e9e3d77SNicholas Piggin return 0; 1975e9e3d77SNicholas Piggin 1985e9e3d77SNicholas Piggin return pud_set_huge(pud, phys_addr, prot); 1995e9e3d77SNicholas Piggin } 2005e9e3d77SNicholas Piggin 2015e9e3d77SNicholas Piggin static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 2025e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2035e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 2045e9e3d77SNicholas Piggin { 2055e9e3d77SNicholas Piggin pud_t *pud; 2065e9e3d77SNicholas Piggin unsigned long next; 2075e9e3d77SNicholas Piggin 2085e9e3d77SNicholas Piggin pud = pud_alloc_track(&init_mm, p4d, addr, mask); 2095e9e3d77SNicholas Piggin if (!pud) 2105e9e3d77SNicholas Piggin return -ENOMEM; 2115e9e3d77SNicholas Piggin do { 2125e9e3d77SNicholas Piggin next = pud_addr_end(addr, end); 2135e9e3d77SNicholas Piggin 2145e9e3d77SNicholas Piggin if (vmap_try_huge_pud(pud, addr, next, phys_addr, prot, 2155e9e3d77SNicholas Piggin max_page_shift)) { 2165e9e3d77SNicholas Piggin *mask |= PGTBL_PUD_MODIFIED; 2175e9e3d77SNicholas Piggin continue; 2185e9e3d77SNicholas Piggin } 2195e9e3d77SNicholas Piggin 2205e9e3d77SNicholas Piggin if (vmap_pmd_range(pud, addr, next, phys_addr, prot, 2215e9e3d77SNicholas Piggin max_page_shift, mask)) 2225e9e3d77SNicholas Piggin return -ENOMEM; 2235e9e3d77SNicholas Piggin } while (pud++, phys_addr += (next - addr), addr = next, addr != end); 2245e9e3d77SNicholas Piggin return 0; 2255e9e3d77SNicholas Piggin } 2265e9e3d77SNicholas Piggin 2275e9e3d77SNicholas Piggin static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, 2285e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2295e9e3d77SNicholas Piggin unsigned int max_page_shift) 2305e9e3d77SNicholas Piggin { 2315e9e3d77SNicholas Piggin if (max_page_shift < P4D_SHIFT) 2325e9e3d77SNicholas Piggin return 0; 2335e9e3d77SNicholas Piggin 2345e9e3d77SNicholas Piggin if (!arch_vmap_p4d_supported(prot)) 2355e9e3d77SNicholas Piggin return 0; 2365e9e3d77SNicholas Piggin 2375e9e3d77SNicholas Piggin if ((end - addr) != P4D_SIZE) 2385e9e3d77SNicholas Piggin return 0; 2395e9e3d77SNicholas Piggin 2405e9e3d77SNicholas Piggin if (!IS_ALIGNED(addr, P4D_SIZE)) 2415e9e3d77SNicholas Piggin return 0; 2425e9e3d77SNicholas Piggin 2435e9e3d77SNicholas Piggin if (!IS_ALIGNED(phys_addr, P4D_SIZE)) 2445e9e3d77SNicholas Piggin return 0; 2455e9e3d77SNicholas Piggin 2465e9e3d77SNicholas Piggin if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) 2475e9e3d77SNicholas Piggin return 0; 2485e9e3d77SNicholas Piggin 2495e9e3d77SNicholas Piggin return p4d_set_huge(p4d, phys_addr, prot); 2505e9e3d77SNicholas Piggin } 2515e9e3d77SNicholas Piggin 2525e9e3d77SNicholas Piggin static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 2535e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2545e9e3d77SNicholas Piggin unsigned int max_page_shift, pgtbl_mod_mask *mask) 2555e9e3d77SNicholas Piggin { 2565e9e3d77SNicholas Piggin p4d_t *p4d; 2575e9e3d77SNicholas Piggin unsigned long next; 2585e9e3d77SNicholas Piggin 2595e9e3d77SNicholas Piggin p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 2605e9e3d77SNicholas Piggin if (!p4d) 2615e9e3d77SNicholas Piggin return -ENOMEM; 2625e9e3d77SNicholas Piggin do { 2635e9e3d77SNicholas Piggin next = p4d_addr_end(addr, end); 2645e9e3d77SNicholas Piggin 2655e9e3d77SNicholas Piggin if (vmap_try_huge_p4d(p4d, addr, next, phys_addr, prot, 2665e9e3d77SNicholas Piggin max_page_shift)) { 2675e9e3d77SNicholas Piggin *mask |= PGTBL_P4D_MODIFIED; 2685e9e3d77SNicholas Piggin continue; 2695e9e3d77SNicholas Piggin } 2705e9e3d77SNicholas Piggin 2715e9e3d77SNicholas Piggin if (vmap_pud_range(p4d, addr, next, phys_addr, prot, 2725e9e3d77SNicholas Piggin max_page_shift, mask)) 2735e9e3d77SNicholas Piggin return -ENOMEM; 2745e9e3d77SNicholas Piggin } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); 2755e9e3d77SNicholas Piggin return 0; 2765e9e3d77SNicholas Piggin } 2775e9e3d77SNicholas Piggin 2785d87510dSNicholas Piggin static int vmap_range_noflush(unsigned long addr, unsigned long end, 2795e9e3d77SNicholas Piggin phys_addr_t phys_addr, pgprot_t prot, 2805e9e3d77SNicholas Piggin unsigned int max_page_shift) 2815e9e3d77SNicholas Piggin { 2825e9e3d77SNicholas Piggin pgd_t *pgd; 2835e9e3d77SNicholas Piggin unsigned long start; 2845e9e3d77SNicholas Piggin unsigned long next; 2855e9e3d77SNicholas Piggin int err; 2865e9e3d77SNicholas Piggin pgtbl_mod_mask mask = 0; 2875e9e3d77SNicholas Piggin 2885e9e3d77SNicholas Piggin might_sleep(); 2895e9e3d77SNicholas Piggin BUG_ON(addr >= end); 2905e9e3d77SNicholas Piggin 2915e9e3d77SNicholas Piggin start = addr; 2925e9e3d77SNicholas Piggin pgd = pgd_offset_k(addr); 2935e9e3d77SNicholas Piggin do { 2945e9e3d77SNicholas Piggin next = pgd_addr_end(addr, end); 2955e9e3d77SNicholas Piggin err = vmap_p4d_range(pgd, addr, next, phys_addr, prot, 2965e9e3d77SNicholas Piggin max_page_shift, &mask); 2975e9e3d77SNicholas Piggin if (err) 2985e9e3d77SNicholas Piggin break; 2995e9e3d77SNicholas Piggin } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); 3005e9e3d77SNicholas Piggin 3015e9e3d77SNicholas Piggin if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 3025e9e3d77SNicholas Piggin arch_sync_kernel_mappings(start, end); 3035e9e3d77SNicholas Piggin 3045e9e3d77SNicholas Piggin return err; 3055e9e3d77SNicholas Piggin } 306b221385bSAdrian Bunk 307d7bca919SAlexei Starovoitov int vmap_page_range(unsigned long addr, unsigned long end, 308d7bca919SAlexei Starovoitov phys_addr_t phys_addr, pgprot_t prot) 309d7bca919SAlexei Starovoitov { 310d7bca919SAlexei Starovoitov int err; 311d7bca919SAlexei Starovoitov 312d7bca919SAlexei Starovoitov err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot), 313d7bca919SAlexei Starovoitov ioremap_max_page_shift); 314d7bca919SAlexei Starovoitov flush_cache_vmap(addr, end); 315d7bca919SAlexei Starovoitov if (!err) 316d7bca919SAlexei Starovoitov err = kmsan_ioremap_page_range(addr, end, phys_addr, prot, 317d7bca919SAlexei Starovoitov ioremap_max_page_shift); 318d7bca919SAlexei Starovoitov return err; 319d7bca919SAlexei Starovoitov } 320d7bca919SAlexei Starovoitov 32182a70ce0SChristoph Hellwig int ioremap_page_range(unsigned long addr, unsigned long end, 32282a70ce0SChristoph Hellwig phys_addr_t phys_addr, pgprot_t prot) 3235d87510dSNicholas Piggin { 3243e49a866SAlexei Starovoitov struct vm_struct *area; 3255d87510dSNicholas Piggin 3263e49a866SAlexei Starovoitov area = find_vm_area((void *)addr); 3273e49a866SAlexei Starovoitov if (!area || !(area->flags & VM_IOREMAP)) { 3283e49a866SAlexei Starovoitov WARN_ONCE(1, "vm_area at addr %lx is not marked as VM_IOREMAP\n", addr); 3293e49a866SAlexei Starovoitov return -EINVAL; 3303e49a866SAlexei Starovoitov } 3313e49a866SAlexei Starovoitov if (addr != (unsigned long)area->addr || 3323e49a866SAlexei Starovoitov (void *)end != area->addr + get_vm_area_size(area)) { 3333e49a866SAlexei Starovoitov WARN_ONCE(1, "ioremap request [%lx,%lx) doesn't match vm_area [%lx, %lx)\n", 3343e49a866SAlexei Starovoitov addr, end, (long)area->addr, 3353e49a866SAlexei Starovoitov (long)area->addr + get_vm_area_size(area)); 3363e49a866SAlexei Starovoitov return -ERANGE; 3373e49a866SAlexei Starovoitov } 338d7bca919SAlexei Starovoitov return vmap_page_range(addr, end, phys_addr, prot); 3395d87510dSNicholas Piggin } 3405d87510dSNicholas Piggin 3412ba3e694SJoerg Roedel static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 3422ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3431da177e4SLinus Torvalds { 3441da177e4SLinus Torvalds pte_t *pte; 3451da177e4SLinus Torvalds 3461da177e4SLinus Torvalds pte = pte_offset_kernel(pmd, addr); 3471da177e4SLinus Torvalds do { 3481da177e4SLinus Torvalds pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); 3491da177e4SLinus Torvalds WARN_ON(!pte_none(ptent) && !pte_present(ptent)); 3501da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 3512ba3e694SJoerg Roedel *mask |= PGTBL_PTE_MODIFIED; 3521da177e4SLinus Torvalds } 3531da177e4SLinus Torvalds 3542ba3e694SJoerg Roedel static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 3552ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3561da177e4SLinus Torvalds { 3571da177e4SLinus Torvalds pmd_t *pmd; 3581da177e4SLinus Torvalds unsigned long next; 3592ba3e694SJoerg Roedel int cleared; 3601da177e4SLinus Torvalds 3611da177e4SLinus Torvalds pmd = pmd_offset(pud, addr); 3621da177e4SLinus Torvalds do { 3631da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 3642ba3e694SJoerg Roedel 3652ba3e694SJoerg Roedel cleared = pmd_clear_huge(pmd); 3662ba3e694SJoerg Roedel if (cleared || pmd_bad(*pmd)) 3672ba3e694SJoerg Roedel *mask |= PGTBL_PMD_MODIFIED; 3682ba3e694SJoerg Roedel 3692ba3e694SJoerg Roedel if (cleared) 370b9820d8fSToshi Kani continue; 3711da177e4SLinus Torvalds if (pmd_none_or_clear_bad(pmd)) 3721da177e4SLinus Torvalds continue; 3732ba3e694SJoerg Roedel vunmap_pte_range(pmd, addr, next, mask); 374e47110e9SAneesh Kumar K.V 375e47110e9SAneesh Kumar K.V cond_resched(); 3761da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 3771da177e4SLinus Torvalds } 3781da177e4SLinus Torvalds 3792ba3e694SJoerg Roedel static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 3802ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 3811da177e4SLinus Torvalds { 3821da177e4SLinus Torvalds pud_t *pud; 3831da177e4SLinus Torvalds unsigned long next; 3842ba3e694SJoerg Roedel int cleared; 3851da177e4SLinus Torvalds 386c2febafcSKirill A. Shutemov pud = pud_offset(p4d, addr); 3871da177e4SLinus Torvalds do { 3881da177e4SLinus Torvalds next = pud_addr_end(addr, end); 3892ba3e694SJoerg Roedel 3902ba3e694SJoerg Roedel cleared = pud_clear_huge(pud); 3912ba3e694SJoerg Roedel if (cleared || pud_bad(*pud)) 3922ba3e694SJoerg Roedel *mask |= PGTBL_PUD_MODIFIED; 3932ba3e694SJoerg Roedel 3942ba3e694SJoerg Roedel if (cleared) 395b9820d8fSToshi Kani continue; 3961da177e4SLinus Torvalds if (pud_none_or_clear_bad(pud)) 3971da177e4SLinus Torvalds continue; 3982ba3e694SJoerg Roedel vunmap_pmd_range(pud, addr, next, mask); 3991da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 4001da177e4SLinus Torvalds } 4011da177e4SLinus Torvalds 4022ba3e694SJoerg Roedel static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 4032ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 404c2febafcSKirill A. Shutemov { 405c2febafcSKirill A. Shutemov p4d_t *p4d; 406c2febafcSKirill A. Shutemov unsigned long next; 407c2febafcSKirill A. Shutemov 408c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 409c2febafcSKirill A. Shutemov do { 410c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 4112ba3e694SJoerg Roedel 412c8db8c26SLi kunyu p4d_clear_huge(p4d); 413c8db8c26SLi kunyu if (p4d_bad(*p4d)) 4142ba3e694SJoerg Roedel *mask |= PGTBL_P4D_MODIFIED; 4152ba3e694SJoerg Roedel 416c2febafcSKirill A. Shutemov if (p4d_none_or_clear_bad(p4d)) 417c2febafcSKirill A. Shutemov continue; 4182ba3e694SJoerg Roedel vunmap_pud_range(p4d, addr, next, mask); 419c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 420c2febafcSKirill A. Shutemov } 421c2febafcSKirill A. Shutemov 4224ad0ae8cSNicholas Piggin /* 4234ad0ae8cSNicholas Piggin * vunmap_range_noflush is similar to vunmap_range, but does not 4244ad0ae8cSNicholas Piggin * flush caches or TLBs. 425b521c43fSChristoph Hellwig * 4264ad0ae8cSNicholas Piggin * The caller is responsible for calling flush_cache_vmap() before calling 4274ad0ae8cSNicholas Piggin * this function, and flush_tlb_kernel_range after it has returned 4284ad0ae8cSNicholas Piggin * successfully (and before the addresses are expected to cause a page fault 4294ad0ae8cSNicholas Piggin * or be re-mapped for something else, if TLB flushes are being delayed or 4304ad0ae8cSNicholas Piggin * coalesced). 431b521c43fSChristoph Hellwig * 4324ad0ae8cSNicholas Piggin * This is an internal function only. Do not use outside mm/. 433b521c43fSChristoph Hellwig */ 434b073d7f8SAlexander Potapenko void __vunmap_range_noflush(unsigned long start, unsigned long end) 4351da177e4SLinus Torvalds { 4361da177e4SLinus Torvalds unsigned long next; 437b521c43fSChristoph Hellwig pgd_t *pgd; 4382ba3e694SJoerg Roedel unsigned long addr = start; 4392ba3e694SJoerg Roedel pgtbl_mod_mask mask = 0; 4401da177e4SLinus Torvalds 4411da177e4SLinus Torvalds BUG_ON(addr >= end); 4421da177e4SLinus Torvalds pgd = pgd_offset_k(addr); 4431da177e4SLinus Torvalds do { 4441da177e4SLinus Torvalds next = pgd_addr_end(addr, end); 4452ba3e694SJoerg Roedel if (pgd_bad(*pgd)) 4462ba3e694SJoerg Roedel mask |= PGTBL_PGD_MODIFIED; 4471da177e4SLinus Torvalds if (pgd_none_or_clear_bad(pgd)) 4481da177e4SLinus Torvalds continue; 4492ba3e694SJoerg Roedel vunmap_p4d_range(pgd, addr, next, &mask); 4501da177e4SLinus Torvalds } while (pgd++, addr = next, addr != end); 4512ba3e694SJoerg Roedel 4522ba3e694SJoerg Roedel if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 4532ba3e694SJoerg Roedel arch_sync_kernel_mappings(start, end); 4541da177e4SLinus Torvalds } 4551da177e4SLinus Torvalds 456b073d7f8SAlexander Potapenko void vunmap_range_noflush(unsigned long start, unsigned long end) 457b073d7f8SAlexander Potapenko { 458b073d7f8SAlexander Potapenko kmsan_vunmap_range_noflush(start, end); 459b073d7f8SAlexander Potapenko __vunmap_range_noflush(start, end); 460b073d7f8SAlexander Potapenko } 461b073d7f8SAlexander Potapenko 4624ad0ae8cSNicholas Piggin /** 4634ad0ae8cSNicholas Piggin * vunmap_range - unmap kernel virtual addresses 4644ad0ae8cSNicholas Piggin * @addr: start of the VM area to unmap 4654ad0ae8cSNicholas Piggin * @end: end of the VM area to unmap (non-inclusive) 4664ad0ae8cSNicholas Piggin * 4674ad0ae8cSNicholas Piggin * Clears any present PTEs in the virtual address range, flushes TLBs and 4684ad0ae8cSNicholas Piggin * caches. Any subsequent access to the address before it has been re-mapped 4694ad0ae8cSNicholas Piggin * is a kernel bug. 4704ad0ae8cSNicholas Piggin */ 4714ad0ae8cSNicholas Piggin void vunmap_range(unsigned long addr, unsigned long end) 4724ad0ae8cSNicholas Piggin { 4734ad0ae8cSNicholas Piggin flush_cache_vunmap(addr, end); 4744ad0ae8cSNicholas Piggin vunmap_range_noflush(addr, end); 4754ad0ae8cSNicholas Piggin flush_tlb_kernel_range(addr, end); 4764ad0ae8cSNicholas Piggin } 4774ad0ae8cSNicholas Piggin 4780a264884SNicholas Piggin static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, 4792ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 4802ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 4811da177e4SLinus Torvalds { 4821da177e4SLinus Torvalds pte_t *pte; 4831da177e4SLinus Torvalds 484db64fe02SNick Piggin /* 485db64fe02SNick Piggin * nr is a running index into the array which helps higher level 486db64fe02SNick Piggin * callers keep track of where we're up to. 487db64fe02SNick Piggin */ 488db64fe02SNick Piggin 4892ba3e694SJoerg Roedel pte = pte_alloc_kernel_track(pmd, addr, mask); 4901da177e4SLinus Torvalds if (!pte) 4911da177e4SLinus Torvalds return -ENOMEM; 4921da177e4SLinus Torvalds do { 493db64fe02SNick Piggin struct page *page = pages[*nr]; 494db64fe02SNick Piggin 495c33c7948SRyan Roberts if (WARN_ON(!pte_none(ptep_get(pte)))) 496db64fe02SNick Piggin return -EBUSY; 497db64fe02SNick Piggin if (WARN_ON(!page)) 4981da177e4SLinus Torvalds return -ENOMEM; 4994fcdcc12SYury Norov if (WARN_ON(!pfn_valid(page_to_pfn(page)))) 5004fcdcc12SYury Norov return -EINVAL; 5014fcdcc12SYury Norov 5021da177e4SLinus Torvalds set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); 503db64fe02SNick Piggin (*nr)++; 5041da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 5052ba3e694SJoerg Roedel *mask |= PGTBL_PTE_MODIFIED; 5061da177e4SLinus Torvalds return 0; 5071da177e4SLinus Torvalds } 5081da177e4SLinus Torvalds 5090a264884SNicholas Piggin static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr, 5102ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 5112ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 5121da177e4SLinus Torvalds { 5131da177e4SLinus Torvalds pmd_t *pmd; 5141da177e4SLinus Torvalds unsigned long next; 5151da177e4SLinus Torvalds 5162ba3e694SJoerg Roedel pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 5171da177e4SLinus Torvalds if (!pmd) 5181da177e4SLinus Torvalds return -ENOMEM; 5191da177e4SLinus Torvalds do { 5201da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 5210a264884SNicholas Piggin if (vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask)) 5221da177e4SLinus Torvalds return -ENOMEM; 5231da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 5241da177e4SLinus Torvalds return 0; 5251da177e4SLinus Torvalds } 5261da177e4SLinus Torvalds 5270a264884SNicholas Piggin static int vmap_pages_pud_range(p4d_t *p4d, unsigned long addr, 5282ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 5292ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 5301da177e4SLinus Torvalds { 5311da177e4SLinus Torvalds pud_t *pud; 5321da177e4SLinus Torvalds unsigned long next; 5331da177e4SLinus Torvalds 5342ba3e694SJoerg Roedel pud = pud_alloc_track(&init_mm, p4d, addr, mask); 5351da177e4SLinus Torvalds if (!pud) 5361da177e4SLinus Torvalds return -ENOMEM; 5371da177e4SLinus Torvalds do { 5381da177e4SLinus Torvalds next = pud_addr_end(addr, end); 5390a264884SNicholas Piggin if (vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask)) 5401da177e4SLinus Torvalds return -ENOMEM; 5411da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 5421da177e4SLinus Torvalds return 0; 5431da177e4SLinus Torvalds } 5441da177e4SLinus Torvalds 5450a264884SNicholas Piggin static int vmap_pages_p4d_range(pgd_t *pgd, unsigned long addr, 5462ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 5472ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 548c2febafcSKirill A. Shutemov { 549c2febafcSKirill A. Shutemov p4d_t *p4d; 550c2febafcSKirill A. Shutemov unsigned long next; 551c2febafcSKirill A. Shutemov 5522ba3e694SJoerg Roedel p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 553c2febafcSKirill A. Shutemov if (!p4d) 554c2febafcSKirill A. Shutemov return -ENOMEM; 555c2febafcSKirill A. Shutemov do { 556c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 5570a264884SNicholas Piggin if (vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask)) 558c2febafcSKirill A. Shutemov return -ENOMEM; 559c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 560c2febafcSKirill A. Shutemov return 0; 561c2febafcSKirill A. Shutemov } 562c2febafcSKirill A. Shutemov 563121e6f32SNicholas Piggin static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, 564121e6f32SNicholas Piggin pgprot_t prot, struct page **pages) 565121e6f32SNicholas Piggin { 566121e6f32SNicholas Piggin unsigned long start = addr; 567121e6f32SNicholas Piggin pgd_t *pgd; 568121e6f32SNicholas Piggin unsigned long next; 569121e6f32SNicholas Piggin int err = 0; 570121e6f32SNicholas Piggin int nr = 0; 571121e6f32SNicholas Piggin pgtbl_mod_mask mask = 0; 572121e6f32SNicholas Piggin 573121e6f32SNicholas Piggin BUG_ON(addr >= end); 574121e6f32SNicholas Piggin pgd = pgd_offset_k(addr); 575121e6f32SNicholas Piggin do { 576121e6f32SNicholas Piggin next = pgd_addr_end(addr, end); 577121e6f32SNicholas Piggin if (pgd_bad(*pgd)) 578121e6f32SNicholas Piggin mask |= PGTBL_PGD_MODIFIED; 579121e6f32SNicholas Piggin err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask); 580121e6f32SNicholas Piggin if (err) 581121e6f32SNicholas Piggin return err; 582121e6f32SNicholas Piggin } while (pgd++, addr = next, addr != end); 583121e6f32SNicholas Piggin 584121e6f32SNicholas Piggin if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 585121e6f32SNicholas Piggin arch_sync_kernel_mappings(start, end); 586121e6f32SNicholas Piggin 587121e6f32SNicholas Piggin return 0; 588121e6f32SNicholas Piggin } 589121e6f32SNicholas Piggin 590b67177ecSNicholas Piggin /* 591b67177ecSNicholas Piggin * vmap_pages_range_noflush is similar to vmap_pages_range, but does not 592b67177ecSNicholas Piggin * flush caches. 593b67177ecSNicholas Piggin * 594b67177ecSNicholas Piggin * The caller is responsible for calling flush_cache_vmap() after this 595b67177ecSNicholas Piggin * function returns successfully and before the addresses are accessed. 596b67177ecSNicholas Piggin * 597b67177ecSNicholas Piggin * This is an internal function only. Do not use outside mm/. 598b67177ecSNicholas Piggin */ 599b073d7f8SAlexander Potapenko int __vmap_pages_range_noflush(unsigned long addr, unsigned long end, 600121e6f32SNicholas Piggin pgprot_t prot, struct page **pages, unsigned int page_shift) 601121e6f32SNicholas Piggin { 602121e6f32SNicholas Piggin unsigned int i, nr = (end - addr) >> PAGE_SHIFT; 603121e6f32SNicholas Piggin 604121e6f32SNicholas Piggin WARN_ON(page_shift < PAGE_SHIFT); 605121e6f32SNicholas Piggin 606121e6f32SNicholas Piggin if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC) || 607121e6f32SNicholas Piggin page_shift == PAGE_SHIFT) 608121e6f32SNicholas Piggin return vmap_small_pages_range_noflush(addr, end, prot, pages); 609121e6f32SNicholas Piggin 610121e6f32SNicholas Piggin for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) { 611121e6f32SNicholas Piggin int err; 612121e6f32SNicholas Piggin 613121e6f32SNicholas Piggin err = vmap_range_noflush(addr, addr + (1UL << page_shift), 61408262ac5SMatthew Wilcox page_to_phys(pages[i]), prot, 615121e6f32SNicholas Piggin page_shift); 616121e6f32SNicholas Piggin if (err) 617121e6f32SNicholas Piggin return err; 618121e6f32SNicholas Piggin 619121e6f32SNicholas Piggin addr += 1UL << page_shift; 620121e6f32SNicholas Piggin } 621121e6f32SNicholas Piggin 622121e6f32SNicholas Piggin return 0; 623121e6f32SNicholas Piggin } 624121e6f32SNicholas Piggin 625b073d7f8SAlexander Potapenko int vmap_pages_range_noflush(unsigned long addr, unsigned long end, 626b073d7f8SAlexander Potapenko pgprot_t prot, struct page **pages, unsigned int page_shift) 627b073d7f8SAlexander Potapenko { 62847ebd031SAlexander Potapenko int ret = kmsan_vmap_pages_range_noflush(addr, end, prot, pages, 62947ebd031SAlexander Potapenko page_shift); 63047ebd031SAlexander Potapenko 63147ebd031SAlexander Potapenko if (ret) 63247ebd031SAlexander Potapenko return ret; 633b073d7f8SAlexander Potapenko return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift); 634b073d7f8SAlexander Potapenko } 635b073d7f8SAlexander Potapenko 636b67177ecSNicholas Piggin /** 637b67177ecSNicholas Piggin * vmap_pages_range - map pages to a kernel virtual address 638b67177ecSNicholas Piggin * @addr: start of the VM area to map 639b67177ecSNicholas Piggin * @end: end of the VM area to map (non-inclusive) 640b67177ecSNicholas Piggin * @prot: page protection flags to use 641b67177ecSNicholas Piggin * @pages: pages to map (always PAGE_SIZE pages) 642b67177ecSNicholas Piggin * @page_shift: maximum shift that the pages may be mapped with, @pages must 643b67177ecSNicholas Piggin * be aligned and contiguous up to at least this shift. 644b67177ecSNicholas Piggin * 645b67177ecSNicholas Piggin * RETURNS: 646b67177ecSNicholas Piggin * 0 on success, -errno on failure. 647b67177ecSNicholas Piggin */ 648121e6f32SNicholas Piggin static int vmap_pages_range(unsigned long addr, unsigned long end, 649121e6f32SNicholas Piggin pgprot_t prot, struct page **pages, unsigned int page_shift) 650121e6f32SNicholas Piggin { 651121e6f32SNicholas Piggin int err; 652121e6f32SNicholas Piggin 653121e6f32SNicholas Piggin err = vmap_pages_range_noflush(addr, end, prot, pages, page_shift); 654121e6f32SNicholas Piggin flush_cache_vmap(addr, end); 655121e6f32SNicholas Piggin return err; 656121e6f32SNicholas Piggin } 657121e6f32SNicholas Piggin 658e6f79822SAlexei Starovoitov static int check_sparse_vm_area(struct vm_struct *area, unsigned long start, 659e6f79822SAlexei Starovoitov unsigned long end) 660e6f79822SAlexei Starovoitov { 661e6f79822SAlexei Starovoitov might_sleep(); 662e6f79822SAlexei Starovoitov if (WARN_ON_ONCE(area->flags & VM_FLUSH_RESET_PERMS)) 663e6f79822SAlexei Starovoitov return -EINVAL; 664e6f79822SAlexei Starovoitov if (WARN_ON_ONCE(area->flags & VM_NO_GUARD)) 665e6f79822SAlexei Starovoitov return -EINVAL; 666e6f79822SAlexei Starovoitov if (WARN_ON_ONCE(!(area->flags & VM_SPARSE))) 667e6f79822SAlexei Starovoitov return -EINVAL; 668e6f79822SAlexei Starovoitov if ((end - start) >> PAGE_SHIFT > totalram_pages()) 669e6f79822SAlexei Starovoitov return -E2BIG; 670e6f79822SAlexei Starovoitov if (start < (unsigned long)area->addr || 671e6f79822SAlexei Starovoitov (void *)end > area->addr + get_vm_area_size(area)) 672e6f79822SAlexei Starovoitov return -ERANGE; 673e6f79822SAlexei Starovoitov return 0; 674e6f79822SAlexei Starovoitov } 675e6f79822SAlexei Starovoitov 676e6f79822SAlexei Starovoitov /** 677e6f79822SAlexei Starovoitov * vm_area_map_pages - map pages inside given sparse vm_area 678e6f79822SAlexei Starovoitov * @area: vm_area 679e6f79822SAlexei Starovoitov * @start: start address inside vm_area 680e6f79822SAlexei Starovoitov * @end: end address inside vm_area 681e6f79822SAlexei Starovoitov * @pages: pages to map (always PAGE_SIZE pages) 682e6f79822SAlexei Starovoitov */ 683e6f79822SAlexei Starovoitov int vm_area_map_pages(struct vm_struct *area, unsigned long start, 684e6f79822SAlexei Starovoitov unsigned long end, struct page **pages) 685e6f79822SAlexei Starovoitov { 686e6f79822SAlexei Starovoitov int err; 687e6f79822SAlexei Starovoitov 688e6f79822SAlexei Starovoitov err = check_sparse_vm_area(area, start, end); 689e6f79822SAlexei Starovoitov if (err) 690e6f79822SAlexei Starovoitov return err; 691e6f79822SAlexei Starovoitov 692e6f79822SAlexei Starovoitov return vmap_pages_range(start, end, PAGE_KERNEL, pages, PAGE_SHIFT); 693e6f79822SAlexei Starovoitov } 694e6f79822SAlexei Starovoitov 695e6f79822SAlexei Starovoitov /** 696e6f79822SAlexei Starovoitov * vm_area_unmap_pages - unmap pages inside given sparse vm_area 697e6f79822SAlexei Starovoitov * @area: vm_area 698e6f79822SAlexei Starovoitov * @start: start address inside vm_area 699e6f79822SAlexei Starovoitov * @end: end address inside vm_area 700e6f79822SAlexei Starovoitov */ 701e6f79822SAlexei Starovoitov void vm_area_unmap_pages(struct vm_struct *area, unsigned long start, 702e6f79822SAlexei Starovoitov unsigned long end) 703e6f79822SAlexei Starovoitov { 704e6f79822SAlexei Starovoitov if (check_sparse_vm_area(area, start, end)) 705e6f79822SAlexei Starovoitov return; 706e6f79822SAlexei Starovoitov 707e6f79822SAlexei Starovoitov vunmap_range(start, end); 708e6f79822SAlexei Starovoitov } 709e6f79822SAlexei Starovoitov 71081ac3ad9SKAMEZAWA Hiroyuki int is_vmalloc_or_module_addr(const void *x) 71173bdf0a6SLinus Torvalds { 71273bdf0a6SLinus Torvalds /* 713ab4f2ee1SRussell King * ARM, x86-64 and sparc64 put modules in a special place, 71473bdf0a6SLinus Torvalds * and fall back on vmalloc() if that fails. Others 71573bdf0a6SLinus Torvalds * just put it in the vmalloc space. 71673bdf0a6SLinus Torvalds */ 71773bdf0a6SLinus Torvalds #if defined(CONFIG_MODULES) && defined(MODULES_VADDR) 7184aff1dc4SAndrey Konovalov unsigned long addr = (unsigned long)kasan_reset_tag(x); 71973bdf0a6SLinus Torvalds if (addr >= MODULES_VADDR && addr < MODULES_END) 72073bdf0a6SLinus Torvalds return 1; 72173bdf0a6SLinus Torvalds #endif 72273bdf0a6SLinus Torvalds return is_vmalloc_addr(x); 72373bdf0a6SLinus Torvalds } 72401858469SDavid Howells EXPORT_SYMBOL_GPL(is_vmalloc_or_module_addr); 72573bdf0a6SLinus Torvalds 72648667e7aSChristoph Lameter /* 727c0eb315aSNicholas Piggin * Walk a vmap address to the struct page it maps. Huge vmap mappings will 728c0eb315aSNicholas Piggin * return the tail page that corresponds to the base page address, which 729c0eb315aSNicholas Piggin * matches small vmap mappings. 73048667e7aSChristoph Lameter */ 731add688fbSmalc struct page *vmalloc_to_page(const void *vmalloc_addr) 73248667e7aSChristoph Lameter { 73348667e7aSChristoph Lameter unsigned long addr = (unsigned long) vmalloc_addr; 734add688fbSmalc struct page *page = NULL; 73548667e7aSChristoph Lameter pgd_t *pgd = pgd_offset_k(addr); 736c2febafcSKirill A. Shutemov p4d_t *p4d; 737c2febafcSKirill A. Shutemov pud_t *pud; 738c2febafcSKirill A. Shutemov pmd_t *pmd; 739c2febafcSKirill A. Shutemov pte_t *ptep, pte; 74048667e7aSChristoph Lameter 7417aa413deSIngo Molnar /* 7427aa413deSIngo Molnar * XXX we might need to change this if we add VIRTUAL_BUG_ON for 7437aa413deSIngo Molnar * architectures that do not vmalloc module space 7447aa413deSIngo Molnar */ 74573bdf0a6SLinus Torvalds VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); 74659ea7463SJiri Slaby 747c2febafcSKirill A. Shutemov if (pgd_none(*pgd)) 748c2febafcSKirill A. Shutemov return NULL; 749c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pgd_leaf(*pgd))) 750c0eb315aSNicholas Piggin return NULL; /* XXX: no allowance for huge pgd */ 751c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pgd_bad(*pgd))) 752c0eb315aSNicholas Piggin return NULL; 753c0eb315aSNicholas Piggin 754c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 755c2febafcSKirill A. Shutemov if (p4d_none(*p4d)) 756c2febafcSKirill A. Shutemov return NULL; 757c0eb315aSNicholas Piggin if (p4d_leaf(*p4d)) 758c0eb315aSNicholas Piggin return p4d_page(*p4d) + ((addr & ~P4D_MASK) >> PAGE_SHIFT); 759c0eb315aSNicholas Piggin if (WARN_ON_ONCE(p4d_bad(*p4d))) 760c2febafcSKirill A. Shutemov return NULL; 761c0eb315aSNicholas Piggin 762c0eb315aSNicholas Piggin pud = pud_offset(p4d, addr); 763c0eb315aSNicholas Piggin if (pud_none(*pud)) 764c0eb315aSNicholas Piggin return NULL; 765c0eb315aSNicholas Piggin if (pud_leaf(*pud)) 766c0eb315aSNicholas Piggin return pud_page(*pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 767c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pud_bad(*pud))) 768c0eb315aSNicholas Piggin return NULL; 769c0eb315aSNicholas Piggin 770c2febafcSKirill A. Shutemov pmd = pmd_offset(pud, addr); 771c0eb315aSNicholas Piggin if (pmd_none(*pmd)) 772c0eb315aSNicholas Piggin return NULL; 773c0eb315aSNicholas Piggin if (pmd_leaf(*pmd)) 774c0eb315aSNicholas Piggin return pmd_page(*pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 775c0eb315aSNicholas Piggin if (WARN_ON_ONCE(pmd_bad(*pmd))) 776c2febafcSKirill A. Shutemov return NULL; 777db64fe02SNick Piggin 7780d1c81edSHugh Dickins ptep = pte_offset_kernel(pmd, addr); 779c33c7948SRyan Roberts pte = ptep_get(ptep); 78048667e7aSChristoph Lameter if (pte_present(pte)) 781add688fbSmalc page = pte_page(pte); 782c0eb315aSNicholas Piggin 783add688fbSmalc return page; 784ece86e22SJianyu Zhan } 785ece86e22SJianyu Zhan EXPORT_SYMBOL(vmalloc_to_page); 786ece86e22SJianyu Zhan 787add688fbSmalc /* 788add688fbSmalc * Map a vmalloc()-space virtual address to the physical page frame number. 789add688fbSmalc */ 790add688fbSmalc unsigned long vmalloc_to_pfn(const void *vmalloc_addr) 791add688fbSmalc { 792add688fbSmalc return page_to_pfn(vmalloc_to_page(vmalloc_addr)); 793add688fbSmalc } 794add688fbSmalc EXPORT_SYMBOL(vmalloc_to_pfn); 795add688fbSmalc 796db64fe02SNick Piggin 797db64fe02SNick Piggin /*** Global kva allocator ***/ 798db64fe02SNick Piggin 799bb850f4dSUladzislau Rezki (Sony) #define DEBUG_AUGMENT_PROPAGATE_CHECK 0 800a6cf4e0fSUladzislau Rezki (Sony) #define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0 801bb850f4dSUladzislau Rezki (Sony) 802db64fe02SNick Piggin 803e36176beSUladzislau Rezki (Sony) static DEFINE_SPINLOCK(free_vmap_area_lock); 80468ad4a33SUladzislau Rezki (Sony) static bool vmap_initialized __read_mostly; 80589699605SNick Piggin 80668ad4a33SUladzislau Rezki (Sony) /* 80768ad4a33SUladzislau Rezki (Sony) * This kmem_cache is used for vmap_area objects. Instead of 80868ad4a33SUladzislau Rezki (Sony) * allocating from slab we reuse an object from this cache to 80968ad4a33SUladzislau Rezki (Sony) * make things faster. Especially in "no edge" splitting of 81068ad4a33SUladzislau Rezki (Sony) * free block. 81168ad4a33SUladzislau Rezki (Sony) */ 81268ad4a33SUladzislau Rezki (Sony) static struct kmem_cache *vmap_area_cachep; 81389699605SNick Piggin 81468ad4a33SUladzislau Rezki (Sony) /* 81568ad4a33SUladzislau Rezki (Sony) * This linked list is used in pair with free_vmap_area_root. 81668ad4a33SUladzislau Rezki (Sony) * It gives O(1) access to prev/next to perform fast coalescing. 81768ad4a33SUladzislau Rezki (Sony) */ 81868ad4a33SUladzislau Rezki (Sony) static LIST_HEAD(free_vmap_area_list); 81968ad4a33SUladzislau Rezki (Sony) 82068ad4a33SUladzislau Rezki (Sony) /* 82168ad4a33SUladzislau Rezki (Sony) * This augment red-black tree represents the free vmap space. 82268ad4a33SUladzislau Rezki (Sony) * All vmap_area objects in this tree are sorted by va->va_start 82368ad4a33SUladzislau Rezki (Sony) * address. It is used for allocation and merging when a vmap 82468ad4a33SUladzislau Rezki (Sony) * object is released. 82568ad4a33SUladzislau Rezki (Sony) * 82668ad4a33SUladzislau Rezki (Sony) * Each vmap_area node contains a maximum available free block 82768ad4a33SUladzislau Rezki (Sony) * of its sub-tree, right or left. Therefore it is possible to 82868ad4a33SUladzislau Rezki (Sony) * find a lowest match of free area. 82968ad4a33SUladzislau Rezki (Sony) */ 83068ad4a33SUladzislau Rezki (Sony) static struct rb_root free_vmap_area_root = RB_ROOT; 83168ad4a33SUladzislau Rezki (Sony) 83282dd23e8SUladzislau Rezki (Sony) /* 83382dd23e8SUladzislau Rezki (Sony) * Preload a CPU with one object for "no edge" split case. The 83482dd23e8SUladzislau Rezki (Sony) * aim is to get rid of allocations from the atomic context, thus 83582dd23e8SUladzislau Rezki (Sony) * to use more permissive allocation masks. 83682dd23e8SUladzislau Rezki (Sony) */ 83782dd23e8SUladzislau Rezki (Sony) static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node); 83882dd23e8SUladzislau Rezki (Sony) 839d0936029SUladzislau Rezki (Sony) /* 84015e02a39SUladzislau Rezki (Sony) * This structure defines a single, solid model where a list and 84115e02a39SUladzislau Rezki (Sony) * rb-tree are part of one entity protected by the lock. Nodes are 84215e02a39SUladzislau Rezki (Sony) * sorted in ascending order, thus for O(1) access to left/right 84315e02a39SUladzislau Rezki (Sony) * neighbors a list is used as well as for sequential traversal. 844d0936029SUladzislau Rezki (Sony) */ 845d0936029SUladzislau Rezki (Sony) struct rb_list { 846d0936029SUladzislau Rezki (Sony) struct rb_root root; 847d0936029SUladzislau Rezki (Sony) struct list_head head; 848d0936029SUladzislau Rezki (Sony) spinlock_t lock; 849d0936029SUladzislau Rezki (Sony) }; 850d0936029SUladzislau Rezki (Sony) 85115e02a39SUladzislau Rezki (Sony) /* 85215e02a39SUladzislau Rezki (Sony) * A fast size storage contains VAs up to 1M size. A pool consists 85315e02a39SUladzislau Rezki (Sony) * of linked between each other ready to go VAs of certain sizes. 85415e02a39SUladzislau Rezki (Sony) * An index in the pool-array corresponds to number of pages + 1. 85515e02a39SUladzislau Rezki (Sony) */ 85615e02a39SUladzislau Rezki (Sony) #define MAX_VA_SIZE_PAGES 256 85715e02a39SUladzislau Rezki (Sony) 85872210662SUladzislau Rezki (Sony) struct vmap_pool { 85972210662SUladzislau Rezki (Sony) struct list_head head; 86072210662SUladzislau Rezki (Sony) unsigned long len; 86172210662SUladzislau Rezki (Sony) }; 86272210662SUladzislau Rezki (Sony) 86372210662SUladzislau Rezki (Sony) /* 86415e02a39SUladzislau Rezki (Sony) * An effective vmap-node logic. Users make use of nodes instead 86515e02a39SUladzislau Rezki (Sony) * of a global heap. It allows to balance an access and mitigate 86615e02a39SUladzislau Rezki (Sony) * contention. 86772210662SUladzislau Rezki (Sony) */ 868d0936029SUladzislau Rezki (Sony) static struct vmap_node { 86972210662SUladzislau Rezki (Sony) /* Simple size segregated storage. */ 87072210662SUladzislau Rezki (Sony) struct vmap_pool pool[MAX_VA_SIZE_PAGES]; 87172210662SUladzislau Rezki (Sony) spinlock_t pool_lock; 87272210662SUladzislau Rezki (Sony) bool skip_populate; 87372210662SUladzislau Rezki (Sony) 874d0936029SUladzislau Rezki (Sony) /* Bookkeeping data of this node. */ 875d0936029SUladzislau Rezki (Sony) struct rb_list busy; 876282631cbSUladzislau Rezki (Sony) struct rb_list lazy; 877282631cbSUladzislau Rezki (Sony) 878282631cbSUladzislau Rezki (Sony) /* 879282631cbSUladzislau Rezki (Sony) * Ready-to-free areas. 880282631cbSUladzislau Rezki (Sony) */ 881282631cbSUladzislau Rezki (Sony) struct list_head purge_list; 88272210662SUladzislau Rezki (Sony) struct work_struct purge_work; 88372210662SUladzislau Rezki (Sony) unsigned long nr_purged; 884d0936029SUladzislau Rezki (Sony) } single; 885d0936029SUladzislau Rezki (Sony) 88615e02a39SUladzislau Rezki (Sony) /* 88715e02a39SUladzislau Rezki (Sony) * Initial setup consists of one single node, i.e. a balancing 88815e02a39SUladzislau Rezki (Sony) * is fully disabled. Later on, after vmap is initialized these 88915e02a39SUladzislau Rezki (Sony) * parameters are updated based on a system capacity. 89015e02a39SUladzislau Rezki (Sony) */ 891d0936029SUladzislau Rezki (Sony) static struct vmap_node *vmap_nodes = &single; 892d0936029SUladzislau Rezki (Sony) static __read_mostly unsigned int nr_vmap_nodes = 1; 893d0936029SUladzislau Rezki (Sony) static __read_mostly unsigned int vmap_zone_size = 1; 894d0936029SUladzislau Rezki (Sony) 895d0936029SUladzislau Rezki (Sony) static inline unsigned int 896d0936029SUladzislau Rezki (Sony) addr_to_node_id(unsigned long addr) 897d0936029SUladzislau Rezki (Sony) { 898d0936029SUladzislau Rezki (Sony) return (addr / vmap_zone_size) % nr_vmap_nodes; 899d0936029SUladzislau Rezki (Sony) } 900d0936029SUladzislau Rezki (Sony) 901d0936029SUladzislau Rezki (Sony) static inline struct vmap_node * 902d0936029SUladzislau Rezki (Sony) addr_to_node(unsigned long addr) 903d0936029SUladzislau Rezki (Sony) { 904d0936029SUladzislau Rezki (Sony) return &vmap_nodes[addr_to_node_id(addr)]; 905d0936029SUladzislau Rezki (Sony) } 906d0936029SUladzislau Rezki (Sony) 90772210662SUladzislau Rezki (Sony) static inline struct vmap_node * 90872210662SUladzislau Rezki (Sony) id_to_node(unsigned int id) 90972210662SUladzislau Rezki (Sony) { 91072210662SUladzislau Rezki (Sony) return &vmap_nodes[id % nr_vmap_nodes]; 91172210662SUladzislau Rezki (Sony) } 91272210662SUladzislau Rezki (Sony) 91372210662SUladzislau Rezki (Sony) /* 91472210662SUladzislau Rezki (Sony) * We use the value 0 to represent "no node", that is why 91572210662SUladzislau Rezki (Sony) * an encoded value will be the node-id incremented by 1. 91672210662SUladzislau Rezki (Sony) * It is always greater then 0. A valid node_id which can 91772210662SUladzislau Rezki (Sony) * be encoded is [0:nr_vmap_nodes - 1]. If a passed node_id 91872210662SUladzislau Rezki (Sony) * is not valid 0 is returned. 91972210662SUladzislau Rezki (Sony) */ 92072210662SUladzislau Rezki (Sony) static unsigned int 92172210662SUladzislau Rezki (Sony) encode_vn_id(unsigned int node_id) 92272210662SUladzislau Rezki (Sony) { 92372210662SUladzislau Rezki (Sony) /* Can store U8_MAX [0:254] nodes. */ 92472210662SUladzislau Rezki (Sony) if (node_id < nr_vmap_nodes) 92572210662SUladzislau Rezki (Sony) return (node_id + 1) << BITS_PER_BYTE; 92672210662SUladzislau Rezki (Sony) 92772210662SUladzislau Rezki (Sony) /* Warn and no node encoded. */ 92872210662SUladzislau Rezki (Sony) WARN_ONCE(1, "Encode wrong node id (%u)\n", node_id); 92972210662SUladzislau Rezki (Sony) return 0; 93072210662SUladzislau Rezki (Sony) } 93172210662SUladzislau Rezki (Sony) 93272210662SUladzislau Rezki (Sony) /* 93372210662SUladzislau Rezki (Sony) * Returns an encoded node-id, the valid range is within 93472210662SUladzislau Rezki (Sony) * [0:nr_vmap_nodes-1] values. Otherwise nr_vmap_nodes is 93572210662SUladzislau Rezki (Sony) * returned if extracted data is wrong. 93672210662SUladzislau Rezki (Sony) */ 93772210662SUladzislau Rezki (Sony) static unsigned int 93872210662SUladzislau Rezki (Sony) decode_vn_id(unsigned int val) 93972210662SUladzislau Rezki (Sony) { 94072210662SUladzislau Rezki (Sony) unsigned int node_id = (val >> BITS_PER_BYTE) - 1; 94172210662SUladzislau Rezki (Sony) 94272210662SUladzislau Rezki (Sony) /* Can store U8_MAX [0:254] nodes. */ 94372210662SUladzislau Rezki (Sony) if (node_id < nr_vmap_nodes) 94472210662SUladzislau Rezki (Sony) return node_id; 94572210662SUladzislau Rezki (Sony) 94672210662SUladzislau Rezki (Sony) /* If it was _not_ zero, warn. */ 94772210662SUladzislau Rezki (Sony) WARN_ONCE(node_id != UINT_MAX, 94872210662SUladzislau Rezki (Sony) "Decode wrong node id (%d)\n", node_id); 94972210662SUladzislau Rezki (Sony) 95072210662SUladzislau Rezki (Sony) return nr_vmap_nodes; 95172210662SUladzislau Rezki (Sony) } 95272210662SUladzislau Rezki (Sony) 95372210662SUladzislau Rezki (Sony) static bool 95472210662SUladzislau Rezki (Sony) is_vn_id_valid(unsigned int node_id) 95572210662SUladzislau Rezki (Sony) { 95672210662SUladzislau Rezki (Sony) if (node_id < nr_vmap_nodes) 95772210662SUladzislau Rezki (Sony) return true; 95872210662SUladzislau Rezki (Sony) 95972210662SUladzislau Rezki (Sony) return false; 96072210662SUladzislau Rezki (Sony) } 96172210662SUladzislau Rezki (Sony) 96268ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 96368ad4a33SUladzislau Rezki (Sony) va_size(struct vmap_area *va) 96468ad4a33SUladzislau Rezki (Sony) { 96568ad4a33SUladzislau Rezki (Sony) return (va->va_end - va->va_start); 96668ad4a33SUladzislau Rezki (Sony) } 96768ad4a33SUladzislau Rezki (Sony) 96868ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 96968ad4a33SUladzislau Rezki (Sony) get_subtree_max_size(struct rb_node *node) 97068ad4a33SUladzislau Rezki (Sony) { 97168ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 97268ad4a33SUladzislau Rezki (Sony) 97368ad4a33SUladzislau Rezki (Sony) va = rb_entry_safe(node, struct vmap_area, rb_node); 97468ad4a33SUladzislau Rezki (Sony) return va ? va->subtree_max_size : 0; 97568ad4a33SUladzislau Rezki (Sony) } 97668ad4a33SUladzislau Rezki (Sony) 977315cc066SMichel Lespinasse RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb, 978315cc066SMichel Lespinasse struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size) 97968ad4a33SUladzislau Rezki (Sony) 98077e50af0SThomas Gleixner static void reclaim_and_purge_vmap_areas(void); 98168ad4a33SUladzislau Rezki (Sony) static BLOCKING_NOTIFIER_HEAD(vmap_notify_list); 982690467c8SUladzislau Rezki (Sony) static void drain_vmap_area_work(struct work_struct *work); 983690467c8SUladzislau Rezki (Sony) static DECLARE_WORK(drain_vmap_work, drain_vmap_area_work); 984db64fe02SNick Piggin 98597105f0aSRoman Gushchin static atomic_long_t nr_vmalloc_pages; 98697105f0aSRoman Gushchin 98797105f0aSRoman Gushchin unsigned long vmalloc_nr_pages(void) 98897105f0aSRoman Gushchin { 98997105f0aSRoman Gushchin return atomic_long_read(&nr_vmalloc_pages); 99097105f0aSRoman Gushchin } 99197105f0aSRoman Gushchin 992*fc2c2269SUladzislau Rezki (Sony) static struct vmap_area *__find_vmap_area(unsigned long addr, struct rb_root *root) 993*fc2c2269SUladzislau Rezki (Sony) { 994*fc2c2269SUladzislau Rezki (Sony) struct rb_node *n = root->rb_node; 995*fc2c2269SUladzislau Rezki (Sony) 996*fc2c2269SUladzislau Rezki (Sony) addr = (unsigned long)kasan_reset_tag((void *)addr); 997*fc2c2269SUladzislau Rezki (Sony) 998*fc2c2269SUladzislau Rezki (Sony) while (n) { 999*fc2c2269SUladzislau Rezki (Sony) struct vmap_area *va; 1000*fc2c2269SUladzislau Rezki (Sony) 1001*fc2c2269SUladzislau Rezki (Sony) va = rb_entry(n, struct vmap_area, rb_node); 1002*fc2c2269SUladzislau Rezki (Sony) if (addr < va->va_start) 1003*fc2c2269SUladzislau Rezki (Sony) n = n->rb_left; 1004*fc2c2269SUladzislau Rezki (Sony) else if (addr >= va->va_end) 1005*fc2c2269SUladzislau Rezki (Sony) n = n->rb_right; 1006*fc2c2269SUladzislau Rezki (Sony) else 1007*fc2c2269SUladzislau Rezki (Sony) return va; 1008*fc2c2269SUladzislau Rezki (Sony) } 1009*fc2c2269SUladzislau Rezki (Sony) 1010*fc2c2269SUladzislau Rezki (Sony) return NULL; 1011*fc2c2269SUladzislau Rezki (Sony) } 1012*fc2c2269SUladzislau Rezki (Sony) 1013153090f2SBaoquan He /* Look up the first VA which satisfies addr < va_end, NULL if none. */ 1014d0936029SUladzislau Rezki (Sony) static struct vmap_area * 101553becf32SUladzislau Rezki (Sony) __find_vmap_area_exceed_addr(unsigned long addr, struct rb_root *root) 1016f181234aSChen Wandun { 1017f181234aSChen Wandun struct vmap_area *va = NULL; 1018d0936029SUladzislau Rezki (Sony) struct rb_node *n = root->rb_node; 1019f181234aSChen Wandun 10204aff1dc4SAndrey Konovalov addr = (unsigned long)kasan_reset_tag((void *)addr); 10214aff1dc4SAndrey Konovalov 1022f181234aSChen Wandun while (n) { 1023f181234aSChen Wandun struct vmap_area *tmp; 1024f181234aSChen Wandun 1025f181234aSChen Wandun tmp = rb_entry(n, struct vmap_area, rb_node); 1026f181234aSChen Wandun if (tmp->va_end > addr) { 1027f181234aSChen Wandun va = tmp; 1028f181234aSChen Wandun if (tmp->va_start <= addr) 1029f181234aSChen Wandun break; 1030f181234aSChen Wandun 1031f181234aSChen Wandun n = n->rb_left; 1032f181234aSChen Wandun } else 1033f181234aSChen Wandun n = n->rb_right; 1034f181234aSChen Wandun } 1035f181234aSChen Wandun 1036f181234aSChen Wandun return va; 1037f181234aSChen Wandun } 1038f181234aSChen Wandun 103953becf32SUladzislau Rezki (Sony) /* 104053becf32SUladzislau Rezki (Sony) * Returns a node where a first VA, that satisfies addr < va_end, resides. 104153becf32SUladzislau Rezki (Sony) * If success, a node is locked. A user is responsible to unlock it when a 104253becf32SUladzislau Rezki (Sony) * VA is no longer needed to be accessed. 104353becf32SUladzislau Rezki (Sony) * 104453becf32SUladzislau Rezki (Sony) * Returns NULL if nothing found. 104553becf32SUladzislau Rezki (Sony) */ 104653becf32SUladzislau Rezki (Sony) static struct vmap_node * 104753becf32SUladzislau Rezki (Sony) find_vmap_area_exceed_addr_lock(unsigned long addr, struct vmap_area **va) 104853becf32SUladzislau Rezki (Sony) { 1049*fc2c2269SUladzislau Rezki (Sony) unsigned long va_start_lowest; 1050*fc2c2269SUladzislau Rezki (Sony) struct vmap_node *vn; 105153becf32SUladzislau Rezki (Sony) int i; 105253becf32SUladzislau Rezki (Sony) 1053*fc2c2269SUladzislau Rezki (Sony) repeat: 1054*fc2c2269SUladzislau Rezki (Sony) for (i = 0, va_start_lowest = 0; i < nr_vmap_nodes; i++) { 105553becf32SUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 105653becf32SUladzislau Rezki (Sony) 105753becf32SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 1058*fc2c2269SUladzislau Rezki (Sony) *va = __find_vmap_area_exceed_addr(addr, &vn->busy.root); 105953becf32SUladzislau Rezki (Sony) 1060*fc2c2269SUladzislau Rezki (Sony) if (*va) 1061*fc2c2269SUladzislau Rezki (Sony) if (!va_start_lowest || (*va)->va_start < va_start_lowest) 1062*fc2c2269SUladzislau Rezki (Sony) va_start_lowest = (*va)->va_start; 106353becf32SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 106453becf32SUladzislau Rezki (Sony) } 106553becf32SUladzislau Rezki (Sony) 1066*fc2c2269SUladzislau Rezki (Sony) /* 1067*fc2c2269SUladzislau Rezki (Sony) * Check if found VA exists, it might have gone away. In this case we 1068*fc2c2269SUladzislau Rezki (Sony) * repeat the search because a VA has been removed concurrently and we 1069*fc2c2269SUladzislau Rezki (Sony) * need to proceed to the next one, which is a rare case. 1070*fc2c2269SUladzislau Rezki (Sony) */ 1071*fc2c2269SUladzislau Rezki (Sony) if (va_start_lowest) { 1072*fc2c2269SUladzislau Rezki (Sony) vn = addr_to_node(va_start_lowest); 107353becf32SUladzislau Rezki (Sony) 1074*fc2c2269SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 1075*fc2c2269SUladzislau Rezki (Sony) *va = __find_vmap_area(va_start_lowest, &vn->busy.root); 1076db64fe02SNick Piggin 1077*fc2c2269SUladzislau Rezki (Sony) if (*va) 1078*fc2c2269SUladzislau Rezki (Sony) return vn; 10794aff1dc4SAndrey Konovalov 1080*fc2c2269SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 1081*fc2c2269SUladzislau Rezki (Sony) goto repeat; 1082db64fe02SNick Piggin } 1083db64fe02SNick Piggin 1084db64fe02SNick Piggin return NULL; 1085db64fe02SNick Piggin } 1086db64fe02SNick Piggin 108768ad4a33SUladzislau Rezki (Sony) /* 108868ad4a33SUladzislau Rezki (Sony) * This function returns back addresses of parent node 108968ad4a33SUladzislau Rezki (Sony) * and its left or right link for further processing. 10909c801f61SUladzislau Rezki (Sony) * 10919c801f61SUladzislau Rezki (Sony) * Otherwise NULL is returned. In that case all further 10929c801f61SUladzislau Rezki (Sony) * steps regarding inserting of conflicting overlap range 10939c801f61SUladzislau Rezki (Sony) * have to be declined and actually considered as a bug. 109468ad4a33SUladzislau Rezki (Sony) */ 109568ad4a33SUladzislau Rezki (Sony) static __always_inline struct rb_node ** 109668ad4a33SUladzislau Rezki (Sony) find_va_links(struct vmap_area *va, 109768ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct rb_node *from, 109868ad4a33SUladzislau Rezki (Sony) struct rb_node **parent) 1099db64fe02SNick Piggin { 1100170168d0SNamhyung Kim struct vmap_area *tmp_va; 110168ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 1102db64fe02SNick Piggin 110368ad4a33SUladzislau Rezki (Sony) if (root) { 110468ad4a33SUladzislau Rezki (Sony) link = &root->rb_node; 110568ad4a33SUladzislau Rezki (Sony) if (unlikely(!*link)) { 110668ad4a33SUladzislau Rezki (Sony) *parent = NULL; 110768ad4a33SUladzislau Rezki (Sony) return link; 110868ad4a33SUladzislau Rezki (Sony) } 110968ad4a33SUladzislau Rezki (Sony) } else { 111068ad4a33SUladzislau Rezki (Sony) link = &from; 111168ad4a33SUladzislau Rezki (Sony) } 111268ad4a33SUladzislau Rezki (Sony) 111368ad4a33SUladzislau Rezki (Sony) /* 111468ad4a33SUladzislau Rezki (Sony) * Go to the bottom of the tree. When we hit the last point 111568ad4a33SUladzislau Rezki (Sony) * we end up with parent rb_node and correct direction, i name 111668ad4a33SUladzislau Rezki (Sony) * it link, where the new va->rb_node will be attached to. 111768ad4a33SUladzislau Rezki (Sony) */ 111868ad4a33SUladzislau Rezki (Sony) do { 111968ad4a33SUladzislau Rezki (Sony) tmp_va = rb_entry(*link, struct vmap_area, rb_node); 112068ad4a33SUladzislau Rezki (Sony) 112168ad4a33SUladzislau Rezki (Sony) /* 112268ad4a33SUladzislau Rezki (Sony) * During the traversal we also do some sanity check. 112368ad4a33SUladzislau Rezki (Sony) * Trigger the BUG() if there are sides(left/right) 112468ad4a33SUladzislau Rezki (Sony) * or full overlaps. 112568ad4a33SUladzislau Rezki (Sony) */ 1126753df96bSBaoquan He if (va->va_end <= tmp_va->va_start) 112768ad4a33SUladzislau Rezki (Sony) link = &(*link)->rb_left; 1128753df96bSBaoquan He else if (va->va_start >= tmp_va->va_end) 112968ad4a33SUladzislau Rezki (Sony) link = &(*link)->rb_right; 11309c801f61SUladzislau Rezki (Sony) else { 11319c801f61SUladzislau Rezki (Sony) WARN(1, "vmalloc bug: 0x%lx-0x%lx overlaps with 0x%lx-0x%lx\n", 11329c801f61SUladzislau Rezki (Sony) va->va_start, va->va_end, tmp_va->va_start, tmp_va->va_end); 11339c801f61SUladzislau Rezki (Sony) 11349c801f61SUladzislau Rezki (Sony) return NULL; 11359c801f61SUladzislau Rezki (Sony) } 113668ad4a33SUladzislau Rezki (Sony) } while (*link); 113768ad4a33SUladzislau Rezki (Sony) 113868ad4a33SUladzislau Rezki (Sony) *parent = &tmp_va->rb_node; 113968ad4a33SUladzislau Rezki (Sony) return link; 1140db64fe02SNick Piggin } 1141db64fe02SNick Piggin 114268ad4a33SUladzislau Rezki (Sony) static __always_inline struct list_head * 114368ad4a33SUladzislau Rezki (Sony) get_va_next_sibling(struct rb_node *parent, struct rb_node **link) 114468ad4a33SUladzislau Rezki (Sony) { 114568ad4a33SUladzislau Rezki (Sony) struct list_head *list; 1146db64fe02SNick Piggin 114768ad4a33SUladzislau Rezki (Sony) if (unlikely(!parent)) 114868ad4a33SUladzislau Rezki (Sony) /* 114968ad4a33SUladzislau Rezki (Sony) * The red-black tree where we try to find VA neighbors 115068ad4a33SUladzislau Rezki (Sony) * before merging or inserting is empty, i.e. it means 115168ad4a33SUladzislau Rezki (Sony) * there is no free vmap space. Normally it does not 115268ad4a33SUladzislau Rezki (Sony) * happen but we handle this case anyway. 115368ad4a33SUladzislau Rezki (Sony) */ 115468ad4a33SUladzislau Rezki (Sony) return NULL; 115568ad4a33SUladzislau Rezki (Sony) 115668ad4a33SUladzislau Rezki (Sony) list = &rb_entry(parent, struct vmap_area, rb_node)->list; 115768ad4a33SUladzislau Rezki (Sony) return (&parent->rb_right == link ? list->next : list); 1158db64fe02SNick Piggin } 1159db64fe02SNick Piggin 116068ad4a33SUladzislau Rezki (Sony) static __always_inline void 11618eb510dbSUladzislau Rezki (Sony) __link_va(struct vmap_area *va, struct rb_root *root, 11628eb510dbSUladzislau Rezki (Sony) struct rb_node *parent, struct rb_node **link, 11638eb510dbSUladzislau Rezki (Sony) struct list_head *head, bool augment) 116468ad4a33SUladzislau Rezki (Sony) { 116568ad4a33SUladzislau Rezki (Sony) /* 116668ad4a33SUladzislau Rezki (Sony) * VA is still not in the list, but we can 116768ad4a33SUladzislau Rezki (Sony) * identify its future previous list_head node. 116868ad4a33SUladzislau Rezki (Sony) */ 116968ad4a33SUladzislau Rezki (Sony) if (likely(parent)) { 117068ad4a33SUladzislau Rezki (Sony) head = &rb_entry(parent, struct vmap_area, rb_node)->list; 117168ad4a33SUladzislau Rezki (Sony) if (&parent->rb_right != link) 117268ad4a33SUladzislau Rezki (Sony) head = head->prev; 117368ad4a33SUladzislau Rezki (Sony) } 1174db64fe02SNick Piggin 117568ad4a33SUladzislau Rezki (Sony) /* Insert to the rb-tree */ 117668ad4a33SUladzislau Rezki (Sony) rb_link_node(&va->rb_node, parent, link); 11778eb510dbSUladzislau Rezki (Sony) if (augment) { 117868ad4a33SUladzislau Rezki (Sony) /* 117968ad4a33SUladzislau Rezki (Sony) * Some explanation here. Just perform simple insertion 118068ad4a33SUladzislau Rezki (Sony) * to the tree. We do not set va->subtree_max_size to 118168ad4a33SUladzislau Rezki (Sony) * its current size before calling rb_insert_augmented(). 1182153090f2SBaoquan He * It is because we populate the tree from the bottom 118368ad4a33SUladzislau Rezki (Sony) * to parent levels when the node _is_ in the tree. 118468ad4a33SUladzislau Rezki (Sony) * 118568ad4a33SUladzislau Rezki (Sony) * Therefore we set subtree_max_size to zero after insertion, 118668ad4a33SUladzislau Rezki (Sony) * to let __augment_tree_propagate_from() puts everything to 118768ad4a33SUladzislau Rezki (Sony) * the correct order later on. 118868ad4a33SUladzislau Rezki (Sony) */ 118968ad4a33SUladzislau Rezki (Sony) rb_insert_augmented(&va->rb_node, 119068ad4a33SUladzislau Rezki (Sony) root, &free_vmap_area_rb_augment_cb); 119168ad4a33SUladzislau Rezki (Sony) va->subtree_max_size = 0; 119268ad4a33SUladzislau Rezki (Sony) } else { 119368ad4a33SUladzislau Rezki (Sony) rb_insert_color(&va->rb_node, root); 119468ad4a33SUladzislau Rezki (Sony) } 119568ad4a33SUladzislau Rezki (Sony) 119668ad4a33SUladzislau Rezki (Sony) /* Address-sort this list */ 119768ad4a33SUladzislau Rezki (Sony) list_add(&va->list, head); 119868ad4a33SUladzislau Rezki (Sony) } 119968ad4a33SUladzislau Rezki (Sony) 120068ad4a33SUladzislau Rezki (Sony) static __always_inline void 12018eb510dbSUladzislau Rezki (Sony) link_va(struct vmap_area *va, struct rb_root *root, 12028eb510dbSUladzislau Rezki (Sony) struct rb_node *parent, struct rb_node **link, 12038eb510dbSUladzislau Rezki (Sony) struct list_head *head) 12048eb510dbSUladzislau Rezki (Sony) { 12058eb510dbSUladzislau Rezki (Sony) __link_va(va, root, parent, link, head, false); 12068eb510dbSUladzislau Rezki (Sony) } 12078eb510dbSUladzislau Rezki (Sony) 12088eb510dbSUladzislau Rezki (Sony) static __always_inline void 12098eb510dbSUladzislau Rezki (Sony) link_va_augment(struct vmap_area *va, struct rb_root *root, 12108eb510dbSUladzislau Rezki (Sony) struct rb_node *parent, struct rb_node **link, 12118eb510dbSUladzislau Rezki (Sony) struct list_head *head) 12128eb510dbSUladzislau Rezki (Sony) { 12138eb510dbSUladzislau Rezki (Sony) __link_va(va, root, parent, link, head, true); 12148eb510dbSUladzislau Rezki (Sony) } 12158eb510dbSUladzislau Rezki (Sony) 12168eb510dbSUladzislau Rezki (Sony) static __always_inline void 12178eb510dbSUladzislau Rezki (Sony) __unlink_va(struct vmap_area *va, struct rb_root *root, bool augment) 121868ad4a33SUladzislau Rezki (Sony) { 1219460e42d1SUladzislau Rezki (Sony) if (WARN_ON(RB_EMPTY_NODE(&va->rb_node))) 1220460e42d1SUladzislau Rezki (Sony) return; 1221460e42d1SUladzislau Rezki (Sony) 12228eb510dbSUladzislau Rezki (Sony) if (augment) 122368ad4a33SUladzislau Rezki (Sony) rb_erase_augmented(&va->rb_node, 122468ad4a33SUladzislau Rezki (Sony) root, &free_vmap_area_rb_augment_cb); 122568ad4a33SUladzislau Rezki (Sony) else 122668ad4a33SUladzislau Rezki (Sony) rb_erase(&va->rb_node, root); 122768ad4a33SUladzislau Rezki (Sony) 12285d7a7c54SUladzislau Rezki (Sony) list_del_init(&va->list); 122968ad4a33SUladzislau Rezki (Sony) RB_CLEAR_NODE(&va->rb_node); 123068ad4a33SUladzislau Rezki (Sony) } 123168ad4a33SUladzislau Rezki (Sony) 12328eb510dbSUladzislau Rezki (Sony) static __always_inline void 12338eb510dbSUladzislau Rezki (Sony) unlink_va(struct vmap_area *va, struct rb_root *root) 12348eb510dbSUladzislau Rezki (Sony) { 12358eb510dbSUladzislau Rezki (Sony) __unlink_va(va, root, false); 12368eb510dbSUladzislau Rezki (Sony) } 12378eb510dbSUladzislau Rezki (Sony) 12388eb510dbSUladzislau Rezki (Sony) static __always_inline void 12398eb510dbSUladzislau Rezki (Sony) unlink_va_augment(struct vmap_area *va, struct rb_root *root) 12408eb510dbSUladzislau Rezki (Sony) { 12418eb510dbSUladzislau Rezki (Sony) __unlink_va(va, root, true); 12428eb510dbSUladzislau Rezki (Sony) } 12438eb510dbSUladzislau Rezki (Sony) 1244bb850f4dSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_PROPAGATE_CHECK 1245c3385e84SJiapeng Chong /* 1246c3385e84SJiapeng Chong * Gets called when remove the node and rotate. 1247c3385e84SJiapeng Chong */ 1248c3385e84SJiapeng Chong static __always_inline unsigned long 1249c3385e84SJiapeng Chong compute_subtree_max_size(struct vmap_area *va) 1250c3385e84SJiapeng Chong { 1251c3385e84SJiapeng Chong return max3(va_size(va), 1252c3385e84SJiapeng Chong get_subtree_max_size(va->rb_node.rb_left), 1253c3385e84SJiapeng Chong get_subtree_max_size(va->rb_node.rb_right)); 1254c3385e84SJiapeng Chong } 1255c3385e84SJiapeng Chong 1256bb850f4dSUladzislau Rezki (Sony) static void 1257da27c9edSUladzislau Rezki (Sony) augment_tree_propagate_check(void) 1258bb850f4dSUladzislau Rezki (Sony) { 1259bb850f4dSUladzislau Rezki (Sony) struct vmap_area *va; 1260da27c9edSUladzislau Rezki (Sony) unsigned long computed_size; 1261bb850f4dSUladzislau Rezki (Sony) 1262da27c9edSUladzislau Rezki (Sony) list_for_each_entry(va, &free_vmap_area_list, list) { 1263da27c9edSUladzislau Rezki (Sony) computed_size = compute_subtree_max_size(va); 1264da27c9edSUladzislau Rezki (Sony) if (computed_size != va->subtree_max_size) 1265bb850f4dSUladzislau Rezki (Sony) pr_emerg("tree is corrupted: %lu, %lu\n", 1266bb850f4dSUladzislau Rezki (Sony) va_size(va), va->subtree_max_size); 1267bb850f4dSUladzislau Rezki (Sony) } 1268bb850f4dSUladzislau Rezki (Sony) } 1269bb850f4dSUladzislau Rezki (Sony) #endif 1270bb850f4dSUladzislau Rezki (Sony) 127168ad4a33SUladzislau Rezki (Sony) /* 127268ad4a33SUladzislau Rezki (Sony) * This function populates subtree_max_size from bottom to upper 127368ad4a33SUladzislau Rezki (Sony) * levels starting from VA point. The propagation must be done 127468ad4a33SUladzislau Rezki (Sony) * when VA size is modified by changing its va_start/va_end. Or 127568ad4a33SUladzislau Rezki (Sony) * in case of newly inserting of VA to the tree. 127668ad4a33SUladzislau Rezki (Sony) * 127768ad4a33SUladzislau Rezki (Sony) * It means that __augment_tree_propagate_from() must be called: 127868ad4a33SUladzislau Rezki (Sony) * - After VA has been inserted to the tree(free path); 127968ad4a33SUladzislau Rezki (Sony) * - After VA has been shrunk(allocation path); 128068ad4a33SUladzislau Rezki (Sony) * - After VA has been increased(merging path). 128168ad4a33SUladzislau Rezki (Sony) * 128268ad4a33SUladzislau Rezki (Sony) * Please note that, it does not mean that upper parent nodes 128368ad4a33SUladzislau Rezki (Sony) * and their subtree_max_size are recalculated all the time up 128468ad4a33SUladzislau Rezki (Sony) * to the root node. 128568ad4a33SUladzislau Rezki (Sony) * 128668ad4a33SUladzislau Rezki (Sony) * 4--8 128768ad4a33SUladzislau Rezki (Sony) * /\ 128868ad4a33SUladzislau Rezki (Sony) * / \ 128968ad4a33SUladzislau Rezki (Sony) * / \ 129068ad4a33SUladzislau Rezki (Sony) * 2--2 8--8 129168ad4a33SUladzislau Rezki (Sony) * 129268ad4a33SUladzislau Rezki (Sony) * For example if we modify the node 4, shrinking it to 2, then 129368ad4a33SUladzislau Rezki (Sony) * no any modification is required. If we shrink the node 2 to 1 129468ad4a33SUladzislau Rezki (Sony) * its subtree_max_size is updated only, and set to 1. If we shrink 129568ad4a33SUladzislau Rezki (Sony) * the node 8 to 6, then its subtree_max_size is set to 6 and parent 129668ad4a33SUladzislau Rezki (Sony) * node becomes 4--6. 129768ad4a33SUladzislau Rezki (Sony) */ 129868ad4a33SUladzislau Rezki (Sony) static __always_inline void 129968ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(struct vmap_area *va) 130068ad4a33SUladzislau Rezki (Sony) { 130168ad4a33SUladzislau Rezki (Sony) /* 130215ae144fSUladzislau Rezki (Sony) * Populate the tree from bottom towards the root until 130315ae144fSUladzislau Rezki (Sony) * the calculated maximum available size of checked node 130415ae144fSUladzislau Rezki (Sony) * is equal to its current one. 130568ad4a33SUladzislau Rezki (Sony) */ 130615ae144fSUladzislau Rezki (Sony) free_vmap_area_rb_augment_cb_propagate(&va->rb_node, NULL); 1307bb850f4dSUladzislau Rezki (Sony) 1308bb850f4dSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_PROPAGATE_CHECK 1309da27c9edSUladzislau Rezki (Sony) augment_tree_propagate_check(); 1310bb850f4dSUladzislau Rezki (Sony) #endif 131168ad4a33SUladzislau Rezki (Sony) } 131268ad4a33SUladzislau Rezki (Sony) 131368ad4a33SUladzislau Rezki (Sony) static void 131468ad4a33SUladzislau Rezki (Sony) insert_vmap_area(struct vmap_area *va, 131568ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 131668ad4a33SUladzislau Rezki (Sony) { 131768ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 131868ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 131968ad4a33SUladzislau Rezki (Sony) 132068ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 13219c801f61SUladzislau Rezki (Sony) if (link) 132268ad4a33SUladzislau Rezki (Sony) link_va(va, root, parent, link, head); 132368ad4a33SUladzislau Rezki (Sony) } 132468ad4a33SUladzislau Rezki (Sony) 132568ad4a33SUladzislau Rezki (Sony) static void 132668ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(struct vmap_area *va, 132768ad4a33SUladzislau Rezki (Sony) struct rb_node *from, struct rb_root *root, 132868ad4a33SUladzislau Rezki (Sony) struct list_head *head) 132968ad4a33SUladzislau Rezki (Sony) { 133068ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 133168ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 133268ad4a33SUladzislau Rezki (Sony) 133368ad4a33SUladzislau Rezki (Sony) if (from) 133468ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, NULL, from, &parent); 133568ad4a33SUladzislau Rezki (Sony) else 133668ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 133768ad4a33SUladzislau Rezki (Sony) 13389c801f61SUladzislau Rezki (Sony) if (link) { 13398eb510dbSUladzislau Rezki (Sony) link_va_augment(va, root, parent, link, head); 134068ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 134168ad4a33SUladzislau Rezki (Sony) } 13429c801f61SUladzislau Rezki (Sony) } 134368ad4a33SUladzislau Rezki (Sony) 134468ad4a33SUladzislau Rezki (Sony) /* 134568ad4a33SUladzislau Rezki (Sony) * Merge de-allocated chunk of VA memory with previous 134668ad4a33SUladzislau Rezki (Sony) * and next free blocks. If coalesce is not done a new 134768ad4a33SUladzislau Rezki (Sony) * free area is inserted. If VA has been merged, it is 134868ad4a33SUladzislau Rezki (Sony) * freed. 13499c801f61SUladzislau Rezki (Sony) * 13509c801f61SUladzislau Rezki (Sony) * Please note, it can return NULL in case of overlap 13519c801f61SUladzislau Rezki (Sony) * ranges, followed by WARN() report. Despite it is a 13529c801f61SUladzislau Rezki (Sony) * buggy behaviour, a system can be alive and keep 13539c801f61SUladzislau Rezki (Sony) * ongoing. 135468ad4a33SUladzislau Rezki (Sony) */ 13553c5c3cfbSDaniel Axtens static __always_inline struct vmap_area * 13568eb510dbSUladzislau Rezki (Sony) __merge_or_add_vmap_area(struct vmap_area *va, 13578eb510dbSUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head, bool augment) 135868ad4a33SUladzislau Rezki (Sony) { 135968ad4a33SUladzislau Rezki (Sony) struct vmap_area *sibling; 136068ad4a33SUladzislau Rezki (Sony) struct list_head *next; 136168ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 136268ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 136368ad4a33SUladzislau Rezki (Sony) bool merged = false; 136468ad4a33SUladzislau Rezki (Sony) 136568ad4a33SUladzislau Rezki (Sony) /* 136668ad4a33SUladzislau Rezki (Sony) * Find a place in the tree where VA potentially will be 136768ad4a33SUladzislau Rezki (Sony) * inserted, unless it is merged with its sibling/siblings. 136868ad4a33SUladzislau Rezki (Sony) */ 136968ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 13709c801f61SUladzislau Rezki (Sony) if (!link) 13719c801f61SUladzislau Rezki (Sony) return NULL; 137268ad4a33SUladzislau Rezki (Sony) 137368ad4a33SUladzislau Rezki (Sony) /* 137468ad4a33SUladzislau Rezki (Sony) * Get next node of VA to check if merging can be done. 137568ad4a33SUladzislau Rezki (Sony) */ 137668ad4a33SUladzislau Rezki (Sony) next = get_va_next_sibling(parent, link); 137768ad4a33SUladzislau Rezki (Sony) if (unlikely(next == NULL)) 137868ad4a33SUladzislau Rezki (Sony) goto insert; 137968ad4a33SUladzislau Rezki (Sony) 138068ad4a33SUladzislau Rezki (Sony) /* 138168ad4a33SUladzislau Rezki (Sony) * start end 138268ad4a33SUladzislau Rezki (Sony) * | | 138368ad4a33SUladzislau Rezki (Sony) * |<------VA------>|<-----Next----->| 138468ad4a33SUladzislau Rezki (Sony) * | | 138568ad4a33SUladzislau Rezki (Sony) * start end 138668ad4a33SUladzislau Rezki (Sony) */ 138768ad4a33SUladzislau Rezki (Sony) if (next != head) { 138868ad4a33SUladzislau Rezki (Sony) sibling = list_entry(next, struct vmap_area, list); 138968ad4a33SUladzislau Rezki (Sony) if (sibling->va_start == va->va_end) { 139068ad4a33SUladzislau Rezki (Sony) sibling->va_start = va->va_start; 139168ad4a33SUladzislau Rezki (Sony) 139268ad4a33SUladzislau Rezki (Sony) /* Free vmap_area object. */ 139368ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 139468ad4a33SUladzislau Rezki (Sony) 139568ad4a33SUladzislau Rezki (Sony) /* Point to the new merged area. */ 139668ad4a33SUladzislau Rezki (Sony) va = sibling; 139768ad4a33SUladzislau Rezki (Sony) merged = true; 139868ad4a33SUladzislau Rezki (Sony) } 139968ad4a33SUladzislau Rezki (Sony) } 140068ad4a33SUladzislau Rezki (Sony) 140168ad4a33SUladzislau Rezki (Sony) /* 140268ad4a33SUladzislau Rezki (Sony) * start end 140368ad4a33SUladzislau Rezki (Sony) * | | 140468ad4a33SUladzislau Rezki (Sony) * |<-----Prev----->|<------VA------>| 140568ad4a33SUladzislau Rezki (Sony) * | | 140668ad4a33SUladzislau Rezki (Sony) * start end 140768ad4a33SUladzislau Rezki (Sony) */ 140868ad4a33SUladzislau Rezki (Sony) if (next->prev != head) { 140968ad4a33SUladzislau Rezki (Sony) sibling = list_entry(next->prev, struct vmap_area, list); 141068ad4a33SUladzislau Rezki (Sony) if (sibling->va_end == va->va_start) { 14115dd78640SUladzislau Rezki (Sony) /* 14125dd78640SUladzislau Rezki (Sony) * If both neighbors are coalesced, it is important 14135dd78640SUladzislau Rezki (Sony) * to unlink the "next" node first, followed by merging 14145dd78640SUladzislau Rezki (Sony) * with "previous" one. Otherwise the tree might not be 14155dd78640SUladzislau Rezki (Sony) * fully populated if a sibling's augmented value is 14165dd78640SUladzislau Rezki (Sony) * "normalized" because of rotation operations. 14175dd78640SUladzislau Rezki (Sony) */ 141854f63d9dSUladzislau Rezki (Sony) if (merged) 14198eb510dbSUladzislau Rezki (Sony) __unlink_va(va, root, augment); 142068ad4a33SUladzislau Rezki (Sony) 14215dd78640SUladzislau Rezki (Sony) sibling->va_end = va->va_end; 14225dd78640SUladzislau Rezki (Sony) 142368ad4a33SUladzislau Rezki (Sony) /* Free vmap_area object. */ 142468ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 14253c5c3cfbSDaniel Axtens 14263c5c3cfbSDaniel Axtens /* Point to the new merged area. */ 14273c5c3cfbSDaniel Axtens va = sibling; 14283c5c3cfbSDaniel Axtens merged = true; 142968ad4a33SUladzislau Rezki (Sony) } 143068ad4a33SUladzislau Rezki (Sony) } 143168ad4a33SUladzislau Rezki (Sony) 143268ad4a33SUladzislau Rezki (Sony) insert: 14335dd78640SUladzislau Rezki (Sony) if (!merged) 14348eb510dbSUladzislau Rezki (Sony) __link_va(va, root, parent, link, head, augment); 14353c5c3cfbSDaniel Axtens 143696e2db45SUladzislau Rezki (Sony) return va; 143796e2db45SUladzislau Rezki (Sony) } 143896e2db45SUladzislau Rezki (Sony) 143996e2db45SUladzislau Rezki (Sony) static __always_inline struct vmap_area * 14408eb510dbSUladzislau Rezki (Sony) merge_or_add_vmap_area(struct vmap_area *va, 14418eb510dbSUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 14428eb510dbSUladzislau Rezki (Sony) { 14438eb510dbSUladzislau Rezki (Sony) return __merge_or_add_vmap_area(va, root, head, false); 14448eb510dbSUladzislau Rezki (Sony) } 14458eb510dbSUladzislau Rezki (Sony) 14468eb510dbSUladzislau Rezki (Sony) static __always_inline struct vmap_area * 144796e2db45SUladzislau Rezki (Sony) merge_or_add_vmap_area_augment(struct vmap_area *va, 144896e2db45SUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 144996e2db45SUladzislau Rezki (Sony) { 14508eb510dbSUladzislau Rezki (Sony) va = __merge_or_add_vmap_area(va, root, head, true); 145196e2db45SUladzislau Rezki (Sony) if (va) 14525dd78640SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 145396e2db45SUladzislau Rezki (Sony) 14543c5c3cfbSDaniel Axtens return va; 145568ad4a33SUladzislau Rezki (Sony) } 145668ad4a33SUladzislau Rezki (Sony) 145768ad4a33SUladzislau Rezki (Sony) static __always_inline bool 145868ad4a33SUladzislau Rezki (Sony) is_within_this_va(struct vmap_area *va, unsigned long size, 145968ad4a33SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 146068ad4a33SUladzislau Rezki (Sony) { 146168ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr; 146268ad4a33SUladzislau Rezki (Sony) 146368ad4a33SUladzislau Rezki (Sony) if (va->va_start > vstart) 146468ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(va->va_start, align); 146568ad4a33SUladzislau Rezki (Sony) else 146668ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(vstart, align); 146768ad4a33SUladzislau Rezki (Sony) 146868ad4a33SUladzislau Rezki (Sony) /* Can be overflowed due to big size or alignment. */ 146968ad4a33SUladzislau Rezki (Sony) if (nva_start_addr + size < nva_start_addr || 147068ad4a33SUladzislau Rezki (Sony) nva_start_addr < vstart) 147168ad4a33SUladzislau Rezki (Sony) return false; 147268ad4a33SUladzislau Rezki (Sony) 147368ad4a33SUladzislau Rezki (Sony) return (nva_start_addr + size <= va->va_end); 147468ad4a33SUladzislau Rezki (Sony) } 147568ad4a33SUladzislau Rezki (Sony) 147668ad4a33SUladzislau Rezki (Sony) /* 147768ad4a33SUladzislau Rezki (Sony) * Find the first free block(lowest start address) in the tree, 147868ad4a33SUladzislau Rezki (Sony) * that will accomplish the request corresponding to passing 14799333fe98SUladzislau Rezki * parameters. Please note, with an alignment bigger than PAGE_SIZE, 14809333fe98SUladzislau Rezki * a search length is adjusted to account for worst case alignment 14819333fe98SUladzislau Rezki * overhead. 148268ad4a33SUladzislau Rezki (Sony) */ 148368ad4a33SUladzislau Rezki (Sony) static __always_inline struct vmap_area * 1484f9863be4SUladzislau Rezki (Sony) find_vmap_lowest_match(struct rb_root *root, unsigned long size, 1485f9863be4SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart, bool adjust_search_size) 148668ad4a33SUladzislau Rezki (Sony) { 148768ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 148868ad4a33SUladzislau Rezki (Sony) struct rb_node *node; 14899333fe98SUladzislau Rezki unsigned long length; 149068ad4a33SUladzislau Rezki (Sony) 149168ad4a33SUladzislau Rezki (Sony) /* Start from the root. */ 1492f9863be4SUladzislau Rezki (Sony) node = root->rb_node; 149368ad4a33SUladzislau Rezki (Sony) 14949333fe98SUladzislau Rezki /* Adjust the search size for alignment overhead. */ 14959333fe98SUladzislau Rezki length = adjust_search_size ? size + align - 1 : size; 14969333fe98SUladzislau Rezki 149768ad4a33SUladzislau Rezki (Sony) while (node) { 149868ad4a33SUladzislau Rezki (Sony) va = rb_entry(node, struct vmap_area, rb_node); 149968ad4a33SUladzislau Rezki (Sony) 15009333fe98SUladzislau Rezki if (get_subtree_max_size(node->rb_left) >= length && 150168ad4a33SUladzislau Rezki (Sony) vstart < va->va_start) { 150268ad4a33SUladzislau Rezki (Sony) node = node->rb_left; 150368ad4a33SUladzislau Rezki (Sony) } else { 150468ad4a33SUladzislau Rezki (Sony) if (is_within_this_va(va, size, align, vstart)) 150568ad4a33SUladzislau Rezki (Sony) return va; 150668ad4a33SUladzislau Rezki (Sony) 150768ad4a33SUladzislau Rezki (Sony) /* 150868ad4a33SUladzislau Rezki (Sony) * Does not make sense to go deeper towards the right 150968ad4a33SUladzislau Rezki (Sony) * sub-tree if it does not have a free block that is 15109333fe98SUladzislau Rezki * equal or bigger to the requested search length. 151168ad4a33SUladzislau Rezki (Sony) */ 15129333fe98SUladzislau Rezki if (get_subtree_max_size(node->rb_right) >= length) { 151368ad4a33SUladzislau Rezki (Sony) node = node->rb_right; 151468ad4a33SUladzislau Rezki (Sony) continue; 151568ad4a33SUladzislau Rezki (Sony) } 151668ad4a33SUladzislau Rezki (Sony) 151768ad4a33SUladzislau Rezki (Sony) /* 15183806b041SAndrew Morton * OK. We roll back and find the first right sub-tree, 151968ad4a33SUladzislau Rezki (Sony) * that will satisfy the search criteria. It can happen 15209f531973SUladzislau Rezki (Sony) * due to "vstart" restriction or an alignment overhead 15219f531973SUladzislau Rezki (Sony) * that is bigger then PAGE_SIZE. 152268ad4a33SUladzislau Rezki (Sony) */ 152368ad4a33SUladzislau Rezki (Sony) while ((node = rb_parent(node))) { 152468ad4a33SUladzislau Rezki (Sony) va = rb_entry(node, struct vmap_area, rb_node); 152568ad4a33SUladzislau Rezki (Sony) if (is_within_this_va(va, size, align, vstart)) 152668ad4a33SUladzislau Rezki (Sony) return va; 152768ad4a33SUladzislau Rezki (Sony) 15289333fe98SUladzislau Rezki if (get_subtree_max_size(node->rb_right) >= length && 152968ad4a33SUladzislau Rezki (Sony) vstart <= va->va_start) { 15309f531973SUladzislau Rezki (Sony) /* 15319f531973SUladzislau Rezki (Sony) * Shift the vstart forward. Please note, we update it with 15329f531973SUladzislau Rezki (Sony) * parent's start address adding "1" because we do not want 15339f531973SUladzislau Rezki (Sony) * to enter same sub-tree after it has already been checked 15349f531973SUladzislau Rezki (Sony) * and no suitable free block found there. 15359f531973SUladzislau Rezki (Sony) */ 15369f531973SUladzislau Rezki (Sony) vstart = va->va_start + 1; 153768ad4a33SUladzislau Rezki (Sony) node = node->rb_right; 153868ad4a33SUladzislau Rezki (Sony) break; 153968ad4a33SUladzislau Rezki (Sony) } 154068ad4a33SUladzislau Rezki (Sony) } 154168ad4a33SUladzislau Rezki (Sony) } 154268ad4a33SUladzislau Rezki (Sony) } 154368ad4a33SUladzislau Rezki (Sony) 154468ad4a33SUladzislau Rezki (Sony) return NULL; 154568ad4a33SUladzislau Rezki (Sony) } 154668ad4a33SUladzislau Rezki (Sony) 1547a6cf4e0fSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 1548a6cf4e0fSUladzislau Rezki (Sony) #include <linux/random.h> 1549a6cf4e0fSUladzislau Rezki (Sony) 1550a6cf4e0fSUladzislau Rezki (Sony) static struct vmap_area * 1551bd1264c3SSong Liu find_vmap_lowest_linear_match(struct list_head *head, unsigned long size, 1552a6cf4e0fSUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 1553a6cf4e0fSUladzislau Rezki (Sony) { 1554a6cf4e0fSUladzislau Rezki (Sony) struct vmap_area *va; 1555a6cf4e0fSUladzislau Rezki (Sony) 1556bd1264c3SSong Liu list_for_each_entry(va, head, list) { 1557a6cf4e0fSUladzislau Rezki (Sony) if (!is_within_this_va(va, size, align, vstart)) 1558a6cf4e0fSUladzislau Rezki (Sony) continue; 1559a6cf4e0fSUladzislau Rezki (Sony) 1560a6cf4e0fSUladzislau Rezki (Sony) return va; 1561a6cf4e0fSUladzislau Rezki (Sony) } 1562a6cf4e0fSUladzislau Rezki (Sony) 1563a6cf4e0fSUladzislau Rezki (Sony) return NULL; 1564a6cf4e0fSUladzislau Rezki (Sony) } 1565a6cf4e0fSUladzislau Rezki (Sony) 1566a6cf4e0fSUladzislau Rezki (Sony) static void 1567bd1264c3SSong Liu find_vmap_lowest_match_check(struct rb_root *root, struct list_head *head, 1568bd1264c3SSong Liu unsigned long size, unsigned long align) 1569a6cf4e0fSUladzislau Rezki (Sony) { 1570a6cf4e0fSUladzislau Rezki (Sony) struct vmap_area *va_1, *va_2; 1571a6cf4e0fSUladzislau Rezki (Sony) unsigned long vstart; 1572a6cf4e0fSUladzislau Rezki (Sony) unsigned int rnd; 1573a6cf4e0fSUladzislau Rezki (Sony) 1574a6cf4e0fSUladzislau Rezki (Sony) get_random_bytes(&rnd, sizeof(rnd)); 1575a6cf4e0fSUladzislau Rezki (Sony) vstart = VMALLOC_START + rnd; 1576a6cf4e0fSUladzislau Rezki (Sony) 1577bd1264c3SSong Liu va_1 = find_vmap_lowest_match(root, size, align, vstart, false); 1578bd1264c3SSong Liu va_2 = find_vmap_lowest_linear_match(head, size, align, vstart); 1579a6cf4e0fSUladzislau Rezki (Sony) 1580a6cf4e0fSUladzislau Rezki (Sony) if (va_1 != va_2) 1581a6cf4e0fSUladzislau Rezki (Sony) pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n", 1582a6cf4e0fSUladzislau Rezki (Sony) va_1, va_2, vstart); 1583a6cf4e0fSUladzislau Rezki (Sony) } 1584a6cf4e0fSUladzislau Rezki (Sony) #endif 1585a6cf4e0fSUladzislau Rezki (Sony) 158668ad4a33SUladzislau Rezki (Sony) enum fit_type { 158768ad4a33SUladzislau Rezki (Sony) NOTHING_FIT = 0, 158868ad4a33SUladzislau Rezki (Sony) FL_FIT_TYPE = 1, /* full fit */ 158968ad4a33SUladzislau Rezki (Sony) LE_FIT_TYPE = 2, /* left edge fit */ 159068ad4a33SUladzislau Rezki (Sony) RE_FIT_TYPE = 3, /* right edge fit */ 159168ad4a33SUladzislau Rezki (Sony) NE_FIT_TYPE = 4 /* no edge fit */ 159268ad4a33SUladzislau Rezki (Sony) }; 159368ad4a33SUladzislau Rezki (Sony) 159468ad4a33SUladzislau Rezki (Sony) static __always_inline enum fit_type 159568ad4a33SUladzislau Rezki (Sony) classify_va_fit_type(struct vmap_area *va, 159668ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr, unsigned long size) 159768ad4a33SUladzislau Rezki (Sony) { 159868ad4a33SUladzislau Rezki (Sony) enum fit_type type; 159968ad4a33SUladzislau Rezki (Sony) 160068ad4a33SUladzislau Rezki (Sony) /* Check if it is within VA. */ 160168ad4a33SUladzislau Rezki (Sony) if (nva_start_addr < va->va_start || 160268ad4a33SUladzislau Rezki (Sony) nva_start_addr + size > va->va_end) 160368ad4a33SUladzislau Rezki (Sony) return NOTHING_FIT; 160468ad4a33SUladzislau Rezki (Sony) 160568ad4a33SUladzislau Rezki (Sony) /* Now classify. */ 160668ad4a33SUladzislau Rezki (Sony) if (va->va_start == nva_start_addr) { 160768ad4a33SUladzislau Rezki (Sony) if (va->va_end == nva_start_addr + size) 160868ad4a33SUladzislau Rezki (Sony) type = FL_FIT_TYPE; 160968ad4a33SUladzislau Rezki (Sony) else 161068ad4a33SUladzislau Rezki (Sony) type = LE_FIT_TYPE; 161168ad4a33SUladzislau Rezki (Sony) } else if (va->va_end == nva_start_addr + size) { 161268ad4a33SUladzislau Rezki (Sony) type = RE_FIT_TYPE; 161368ad4a33SUladzislau Rezki (Sony) } else { 161468ad4a33SUladzislau Rezki (Sony) type = NE_FIT_TYPE; 161568ad4a33SUladzislau Rezki (Sony) } 161668ad4a33SUladzislau Rezki (Sony) 161768ad4a33SUladzislau Rezki (Sony) return type; 161868ad4a33SUladzislau Rezki (Sony) } 161968ad4a33SUladzislau Rezki (Sony) 162068ad4a33SUladzislau Rezki (Sony) static __always_inline int 16215b75b8e1SUladzislau Rezki (Sony) va_clip(struct rb_root *root, struct list_head *head, 1622f9863be4SUladzislau Rezki (Sony) struct vmap_area *va, unsigned long nva_start_addr, 1623f9863be4SUladzislau Rezki (Sony) unsigned long size) 162468ad4a33SUladzislau Rezki (Sony) { 16252c929233SArnd Bergmann struct vmap_area *lva = NULL; 16261b23ff80SBaoquan He enum fit_type type = classify_va_fit_type(va, nva_start_addr, size); 162768ad4a33SUladzislau Rezki (Sony) 162868ad4a33SUladzislau Rezki (Sony) if (type == FL_FIT_TYPE) { 162968ad4a33SUladzislau Rezki (Sony) /* 163068ad4a33SUladzislau Rezki (Sony) * No need to split VA, it fully fits. 163168ad4a33SUladzislau Rezki (Sony) * 163268ad4a33SUladzislau Rezki (Sony) * | | 163368ad4a33SUladzislau Rezki (Sony) * V NVA V 163468ad4a33SUladzislau Rezki (Sony) * |---------------| 163568ad4a33SUladzislau Rezki (Sony) */ 1636f9863be4SUladzislau Rezki (Sony) unlink_va_augment(va, root); 163768ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 163868ad4a33SUladzislau Rezki (Sony) } else if (type == LE_FIT_TYPE) { 163968ad4a33SUladzislau Rezki (Sony) /* 164068ad4a33SUladzislau Rezki (Sony) * Split left edge of fit VA. 164168ad4a33SUladzislau Rezki (Sony) * 164268ad4a33SUladzislau Rezki (Sony) * | | 164368ad4a33SUladzislau Rezki (Sony) * V NVA V R 164468ad4a33SUladzislau Rezki (Sony) * |-------|-------| 164568ad4a33SUladzislau Rezki (Sony) */ 164668ad4a33SUladzislau Rezki (Sony) va->va_start += size; 164768ad4a33SUladzislau Rezki (Sony) } else if (type == RE_FIT_TYPE) { 164868ad4a33SUladzislau Rezki (Sony) /* 164968ad4a33SUladzislau Rezki (Sony) * Split right edge of fit VA. 165068ad4a33SUladzislau Rezki (Sony) * 165168ad4a33SUladzislau Rezki (Sony) * | | 165268ad4a33SUladzislau Rezki (Sony) * L V NVA V 165368ad4a33SUladzislau Rezki (Sony) * |-------|-------| 165468ad4a33SUladzislau Rezki (Sony) */ 165568ad4a33SUladzislau Rezki (Sony) va->va_end = nva_start_addr; 165668ad4a33SUladzislau Rezki (Sony) } else if (type == NE_FIT_TYPE) { 165768ad4a33SUladzislau Rezki (Sony) /* 165868ad4a33SUladzislau Rezki (Sony) * Split no edge of fit VA. 165968ad4a33SUladzislau Rezki (Sony) * 166068ad4a33SUladzislau Rezki (Sony) * | | 166168ad4a33SUladzislau Rezki (Sony) * L V NVA V R 166268ad4a33SUladzislau Rezki (Sony) * |---|-------|---| 166368ad4a33SUladzislau Rezki (Sony) */ 166482dd23e8SUladzislau Rezki (Sony) lva = __this_cpu_xchg(ne_fit_preload_node, NULL); 166582dd23e8SUladzislau Rezki (Sony) if (unlikely(!lva)) { 166682dd23e8SUladzislau Rezki (Sony) /* 166782dd23e8SUladzislau Rezki (Sony) * For percpu allocator we do not do any pre-allocation 166882dd23e8SUladzislau Rezki (Sony) * and leave it as it is. The reason is it most likely 166982dd23e8SUladzislau Rezki (Sony) * never ends up with NE_FIT_TYPE splitting. In case of 167082dd23e8SUladzislau Rezki (Sony) * percpu allocations offsets and sizes are aligned to 167182dd23e8SUladzislau Rezki (Sony) * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE 167282dd23e8SUladzislau Rezki (Sony) * are its main fitting cases. 167382dd23e8SUladzislau Rezki (Sony) * 167482dd23e8SUladzislau Rezki (Sony) * There are a few exceptions though, as an example it is 167582dd23e8SUladzislau Rezki (Sony) * a first allocation (early boot up) when we have "one" 167682dd23e8SUladzislau Rezki (Sony) * big free space that has to be split. 1677060650a2SUladzislau Rezki (Sony) * 1678060650a2SUladzislau Rezki (Sony) * Also we can hit this path in case of regular "vmap" 1679060650a2SUladzislau Rezki (Sony) * allocations, if "this" current CPU was not preloaded. 1680060650a2SUladzislau Rezki (Sony) * See the comment in alloc_vmap_area() why. If so, then 1681060650a2SUladzislau Rezki (Sony) * GFP_NOWAIT is used instead to get an extra object for 1682060650a2SUladzislau Rezki (Sony) * split purpose. That is rare and most time does not 1683060650a2SUladzislau Rezki (Sony) * occur. 1684060650a2SUladzislau Rezki (Sony) * 1685060650a2SUladzislau Rezki (Sony) * What happens if an allocation gets failed. Basically, 1686060650a2SUladzislau Rezki (Sony) * an "overflow" path is triggered to purge lazily freed 1687060650a2SUladzislau Rezki (Sony) * areas to free some memory, then, the "retry" path is 1688060650a2SUladzislau Rezki (Sony) * triggered to repeat one more time. See more details 1689060650a2SUladzislau Rezki (Sony) * in alloc_vmap_area() function. 169082dd23e8SUladzislau Rezki (Sony) */ 169168ad4a33SUladzislau Rezki (Sony) lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT); 169282dd23e8SUladzislau Rezki (Sony) if (!lva) 169368ad4a33SUladzislau Rezki (Sony) return -1; 169482dd23e8SUladzislau Rezki (Sony) } 169568ad4a33SUladzislau Rezki (Sony) 169668ad4a33SUladzislau Rezki (Sony) /* 169768ad4a33SUladzislau Rezki (Sony) * Build the remainder. 169868ad4a33SUladzislau Rezki (Sony) */ 169968ad4a33SUladzislau Rezki (Sony) lva->va_start = va->va_start; 170068ad4a33SUladzislau Rezki (Sony) lva->va_end = nva_start_addr; 170168ad4a33SUladzislau Rezki (Sony) 170268ad4a33SUladzislau Rezki (Sony) /* 170368ad4a33SUladzislau Rezki (Sony) * Shrink this VA to remaining size. 170468ad4a33SUladzislau Rezki (Sony) */ 170568ad4a33SUladzislau Rezki (Sony) va->va_start = nva_start_addr + size; 170668ad4a33SUladzislau Rezki (Sony) } else { 170768ad4a33SUladzislau Rezki (Sony) return -1; 170868ad4a33SUladzislau Rezki (Sony) } 170968ad4a33SUladzislau Rezki (Sony) 171068ad4a33SUladzislau Rezki (Sony) if (type != FL_FIT_TYPE) { 171168ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 171268ad4a33SUladzislau Rezki (Sony) 17132c929233SArnd Bergmann if (lva) /* type == NE_FIT_TYPE */ 1714f9863be4SUladzislau Rezki (Sony) insert_vmap_area_augment(lva, &va->rb_node, root, head); 171568ad4a33SUladzislau Rezki (Sony) } 171668ad4a33SUladzislau Rezki (Sony) 171768ad4a33SUladzislau Rezki (Sony) return 0; 171868ad4a33SUladzislau Rezki (Sony) } 171968ad4a33SUladzislau Rezki (Sony) 172038f6b9afSUladzislau Rezki (Sony) static unsigned long 172138f6b9afSUladzislau Rezki (Sony) va_alloc(struct vmap_area *va, 172238f6b9afSUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head, 172338f6b9afSUladzislau Rezki (Sony) unsigned long size, unsigned long align, 172438f6b9afSUladzislau Rezki (Sony) unsigned long vstart, unsigned long vend) 172538f6b9afSUladzislau Rezki (Sony) { 172638f6b9afSUladzislau Rezki (Sony) unsigned long nva_start_addr; 172738f6b9afSUladzislau Rezki (Sony) int ret; 172838f6b9afSUladzislau Rezki (Sony) 172938f6b9afSUladzislau Rezki (Sony) if (va->va_start > vstart) 173038f6b9afSUladzislau Rezki (Sony) nva_start_addr = ALIGN(va->va_start, align); 173138f6b9afSUladzislau Rezki (Sony) else 173238f6b9afSUladzislau Rezki (Sony) nva_start_addr = ALIGN(vstart, align); 173338f6b9afSUladzislau Rezki (Sony) 173438f6b9afSUladzislau Rezki (Sony) /* Check the "vend" restriction. */ 173538f6b9afSUladzislau Rezki (Sony) if (nva_start_addr + size > vend) 173638f6b9afSUladzislau Rezki (Sony) return vend; 173738f6b9afSUladzislau Rezki (Sony) 173838f6b9afSUladzislau Rezki (Sony) /* Update the free vmap_area. */ 17395b75b8e1SUladzislau Rezki (Sony) ret = va_clip(root, head, va, nva_start_addr, size); 174038f6b9afSUladzislau Rezki (Sony) if (WARN_ON_ONCE(ret)) 174138f6b9afSUladzislau Rezki (Sony) return vend; 174238f6b9afSUladzislau Rezki (Sony) 174338f6b9afSUladzislau Rezki (Sony) return nva_start_addr; 174438f6b9afSUladzislau Rezki (Sony) } 174538f6b9afSUladzislau Rezki (Sony) 174668ad4a33SUladzislau Rezki (Sony) /* 174768ad4a33SUladzislau Rezki (Sony) * Returns a start address of the newly allocated area, if success. 174868ad4a33SUladzislau Rezki (Sony) * Otherwise a vend is returned that indicates failure. 174968ad4a33SUladzislau Rezki (Sony) */ 175068ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 1751f9863be4SUladzislau Rezki (Sony) __alloc_vmap_area(struct rb_root *root, struct list_head *head, 1752f9863be4SUladzislau Rezki (Sony) unsigned long size, unsigned long align, 1753cacca6baSUladzislau Rezki (Sony) unsigned long vstart, unsigned long vend) 175468ad4a33SUladzislau Rezki (Sony) { 17559333fe98SUladzislau Rezki bool adjust_search_size = true; 175668ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr; 175768ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 175868ad4a33SUladzislau Rezki (Sony) 17599333fe98SUladzislau Rezki /* 17609333fe98SUladzislau Rezki * Do not adjust when: 17619333fe98SUladzislau Rezki * a) align <= PAGE_SIZE, because it does not make any sense. 17629333fe98SUladzislau Rezki * All blocks(their start addresses) are at least PAGE_SIZE 17639333fe98SUladzislau Rezki * aligned anyway; 17649333fe98SUladzislau Rezki * b) a short range where a requested size corresponds to exactly 17659333fe98SUladzislau Rezki * specified [vstart:vend] interval and an alignment > PAGE_SIZE. 17669333fe98SUladzislau Rezki * With adjusted search length an allocation would not succeed. 17679333fe98SUladzislau Rezki */ 17689333fe98SUladzislau Rezki if (align <= PAGE_SIZE || (align > PAGE_SIZE && (vend - vstart) == size)) 17699333fe98SUladzislau Rezki adjust_search_size = false; 17709333fe98SUladzislau Rezki 1771f9863be4SUladzislau Rezki (Sony) va = find_vmap_lowest_match(root, size, align, vstart, adjust_search_size); 177268ad4a33SUladzislau Rezki (Sony) if (unlikely(!va)) 177368ad4a33SUladzislau Rezki (Sony) return vend; 177468ad4a33SUladzislau Rezki (Sony) 177538f6b9afSUladzislau Rezki (Sony) nva_start_addr = va_alloc(va, root, head, size, align, vstart, vend); 177638f6b9afSUladzislau Rezki (Sony) if (nva_start_addr == vend) 177768ad4a33SUladzislau Rezki (Sony) return vend; 177868ad4a33SUladzislau Rezki (Sony) 1779a6cf4e0fSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 1780bd1264c3SSong Liu find_vmap_lowest_match_check(root, head, size, align); 1781a6cf4e0fSUladzislau Rezki (Sony) #endif 1782a6cf4e0fSUladzislau Rezki (Sony) 178368ad4a33SUladzislau Rezki (Sony) return nva_start_addr; 178468ad4a33SUladzislau Rezki (Sony) } 17854da56b99SChris Wilson 1786db64fe02SNick Piggin /* 1787d98c9e83SAndrey Ryabinin * Free a region of KVA allocated by alloc_vmap_area 1788d98c9e83SAndrey Ryabinin */ 1789d98c9e83SAndrey Ryabinin static void free_vmap_area(struct vmap_area *va) 1790d98c9e83SAndrey Ryabinin { 1791d0936029SUladzislau Rezki (Sony) struct vmap_node *vn = addr_to_node(va->va_start); 1792d0936029SUladzislau Rezki (Sony) 1793d98c9e83SAndrey Ryabinin /* 1794d98c9e83SAndrey Ryabinin * Remove from the busy tree/list. 1795d98c9e83SAndrey Ryabinin */ 1796d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 1797d0936029SUladzislau Rezki (Sony) unlink_va(va, &vn->busy.root); 1798d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 1799d98c9e83SAndrey Ryabinin 1800d98c9e83SAndrey Ryabinin /* 1801d98c9e83SAndrey Ryabinin * Insert/Merge it back to the free tree/list. 1802d98c9e83SAndrey Ryabinin */ 1803d98c9e83SAndrey Ryabinin spin_lock(&free_vmap_area_lock); 180496e2db45SUladzislau Rezki (Sony) merge_or_add_vmap_area_augment(va, &free_vmap_area_root, &free_vmap_area_list); 1805d98c9e83SAndrey Ryabinin spin_unlock(&free_vmap_area_lock); 1806d98c9e83SAndrey Ryabinin } 1807d98c9e83SAndrey Ryabinin 1808187f8cc4SUladzislau Rezki (Sony) static inline void 1809187f8cc4SUladzislau Rezki (Sony) preload_this_cpu_lock(spinlock_t *lock, gfp_t gfp_mask, int node) 1810187f8cc4SUladzislau Rezki (Sony) { 1811187f8cc4SUladzislau Rezki (Sony) struct vmap_area *va = NULL; 1812187f8cc4SUladzislau Rezki (Sony) 1813187f8cc4SUladzislau Rezki (Sony) /* 1814187f8cc4SUladzislau Rezki (Sony) * Preload this CPU with one extra vmap_area object. It is used 1815187f8cc4SUladzislau Rezki (Sony) * when fit type of free area is NE_FIT_TYPE. It guarantees that 1816187f8cc4SUladzislau Rezki (Sony) * a CPU that does an allocation is preloaded. 1817187f8cc4SUladzislau Rezki (Sony) * 1818187f8cc4SUladzislau Rezki (Sony) * We do it in non-atomic context, thus it allows us to use more 1819187f8cc4SUladzislau Rezki (Sony) * permissive allocation masks to be more stable under low memory 1820187f8cc4SUladzislau Rezki (Sony) * condition and high memory pressure. 1821187f8cc4SUladzislau Rezki (Sony) */ 1822187f8cc4SUladzislau Rezki (Sony) if (!this_cpu_read(ne_fit_preload_node)) 1823187f8cc4SUladzislau Rezki (Sony) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 1824187f8cc4SUladzislau Rezki (Sony) 1825187f8cc4SUladzislau Rezki (Sony) spin_lock(lock); 1826187f8cc4SUladzislau Rezki (Sony) 1827187f8cc4SUladzislau Rezki (Sony) if (va && __this_cpu_cmpxchg(ne_fit_preload_node, NULL, va)) 1828187f8cc4SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 1829187f8cc4SUladzislau Rezki (Sony) } 1830187f8cc4SUladzislau Rezki (Sony) 183172210662SUladzislau Rezki (Sony) static struct vmap_pool * 183272210662SUladzislau Rezki (Sony) size_to_va_pool(struct vmap_node *vn, unsigned long size) 183372210662SUladzislau Rezki (Sony) { 183472210662SUladzislau Rezki (Sony) unsigned int idx = (size - 1) / PAGE_SIZE; 183572210662SUladzislau Rezki (Sony) 183672210662SUladzislau Rezki (Sony) if (idx < MAX_VA_SIZE_PAGES) 183772210662SUladzislau Rezki (Sony) return &vn->pool[idx]; 183872210662SUladzislau Rezki (Sony) 183972210662SUladzislau Rezki (Sony) return NULL; 184072210662SUladzislau Rezki (Sony) } 184172210662SUladzislau Rezki (Sony) 184272210662SUladzislau Rezki (Sony) static bool 184372210662SUladzislau Rezki (Sony) node_pool_add_va(struct vmap_node *n, struct vmap_area *va) 184472210662SUladzislau Rezki (Sony) { 184572210662SUladzislau Rezki (Sony) struct vmap_pool *vp; 184672210662SUladzislau Rezki (Sony) 184772210662SUladzislau Rezki (Sony) vp = size_to_va_pool(n, va_size(va)); 184872210662SUladzislau Rezki (Sony) if (!vp) 184972210662SUladzislau Rezki (Sony) return false; 185072210662SUladzislau Rezki (Sony) 185172210662SUladzislau Rezki (Sony) spin_lock(&n->pool_lock); 185272210662SUladzislau Rezki (Sony) list_add(&va->list, &vp->head); 185372210662SUladzislau Rezki (Sony) WRITE_ONCE(vp->len, vp->len + 1); 185472210662SUladzislau Rezki (Sony) spin_unlock(&n->pool_lock); 185572210662SUladzislau Rezki (Sony) 185672210662SUladzislau Rezki (Sony) return true; 185772210662SUladzislau Rezki (Sony) } 185872210662SUladzislau Rezki (Sony) 185972210662SUladzislau Rezki (Sony) static struct vmap_area * 186072210662SUladzislau Rezki (Sony) node_pool_del_va(struct vmap_node *vn, unsigned long size, 186172210662SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart, 186272210662SUladzislau Rezki (Sony) unsigned long vend) 186372210662SUladzislau Rezki (Sony) { 186472210662SUladzislau Rezki (Sony) struct vmap_area *va = NULL; 186572210662SUladzislau Rezki (Sony) struct vmap_pool *vp; 186672210662SUladzislau Rezki (Sony) int err = 0; 186772210662SUladzislau Rezki (Sony) 186872210662SUladzislau Rezki (Sony) vp = size_to_va_pool(vn, size); 186972210662SUladzislau Rezki (Sony) if (!vp || list_empty(&vp->head)) 187072210662SUladzislau Rezki (Sony) return NULL; 187172210662SUladzislau Rezki (Sony) 187272210662SUladzislau Rezki (Sony) spin_lock(&vn->pool_lock); 187372210662SUladzislau Rezki (Sony) if (!list_empty(&vp->head)) { 187472210662SUladzislau Rezki (Sony) va = list_first_entry(&vp->head, struct vmap_area, list); 187572210662SUladzislau Rezki (Sony) 187672210662SUladzislau Rezki (Sony) if (IS_ALIGNED(va->va_start, align)) { 187772210662SUladzislau Rezki (Sony) /* 187872210662SUladzislau Rezki (Sony) * Do some sanity check and emit a warning 187972210662SUladzislau Rezki (Sony) * if one of below checks detects an error. 188072210662SUladzislau Rezki (Sony) */ 188172210662SUladzislau Rezki (Sony) err |= (va_size(va) != size); 188272210662SUladzislau Rezki (Sony) err |= (va->va_start < vstart); 188372210662SUladzislau Rezki (Sony) err |= (va->va_end > vend); 188472210662SUladzislau Rezki (Sony) 188572210662SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(err)) { 188672210662SUladzislau Rezki (Sony) list_del_init(&va->list); 188772210662SUladzislau Rezki (Sony) WRITE_ONCE(vp->len, vp->len - 1); 188872210662SUladzislau Rezki (Sony) } else { 188972210662SUladzislau Rezki (Sony) va = NULL; 189072210662SUladzislau Rezki (Sony) } 189172210662SUladzislau Rezki (Sony) } else { 189272210662SUladzislau Rezki (Sony) list_move_tail(&va->list, &vp->head); 189372210662SUladzislau Rezki (Sony) va = NULL; 189472210662SUladzislau Rezki (Sony) } 189572210662SUladzislau Rezki (Sony) } 189672210662SUladzislau Rezki (Sony) spin_unlock(&vn->pool_lock); 189772210662SUladzislau Rezki (Sony) 189872210662SUladzislau Rezki (Sony) return va; 189972210662SUladzislau Rezki (Sony) } 190072210662SUladzislau Rezki (Sony) 190172210662SUladzislau Rezki (Sony) static struct vmap_area * 190272210662SUladzislau Rezki (Sony) node_alloc(unsigned long size, unsigned long align, 190372210662SUladzislau Rezki (Sony) unsigned long vstart, unsigned long vend, 190472210662SUladzislau Rezki (Sony) unsigned long *addr, unsigned int *vn_id) 190572210662SUladzislau Rezki (Sony) { 190672210662SUladzislau Rezki (Sony) struct vmap_area *va; 190772210662SUladzislau Rezki (Sony) 190872210662SUladzislau Rezki (Sony) *vn_id = 0; 190972210662SUladzislau Rezki (Sony) *addr = vend; 191072210662SUladzislau Rezki (Sony) 191172210662SUladzislau Rezki (Sony) /* 191272210662SUladzislau Rezki (Sony) * Fallback to a global heap if not vmalloc or there 191372210662SUladzislau Rezki (Sony) * is only one node. 191472210662SUladzislau Rezki (Sony) */ 191572210662SUladzislau Rezki (Sony) if (vstart != VMALLOC_START || vend != VMALLOC_END || 191672210662SUladzislau Rezki (Sony) nr_vmap_nodes == 1) 191772210662SUladzislau Rezki (Sony) return NULL; 191872210662SUladzislau Rezki (Sony) 191972210662SUladzislau Rezki (Sony) *vn_id = raw_smp_processor_id() % nr_vmap_nodes; 192072210662SUladzislau Rezki (Sony) va = node_pool_del_va(id_to_node(*vn_id), size, align, vstart, vend); 192172210662SUladzislau Rezki (Sony) *vn_id = encode_vn_id(*vn_id); 192272210662SUladzislau Rezki (Sony) 192372210662SUladzislau Rezki (Sony) if (va) 192472210662SUladzislau Rezki (Sony) *addr = va->va_start; 192572210662SUladzislau Rezki (Sony) 192672210662SUladzislau Rezki (Sony) return va; 192772210662SUladzislau Rezki (Sony) } 192872210662SUladzislau Rezki (Sony) 1929d98c9e83SAndrey Ryabinin /* 1930db64fe02SNick Piggin * Allocate a region of KVA of the specified size and alignment, within the 1931db64fe02SNick Piggin * vstart and vend. 1932db64fe02SNick Piggin */ 1933db64fe02SNick Piggin static struct vmap_area *alloc_vmap_area(unsigned long size, 1934db64fe02SNick Piggin unsigned long align, 1935db64fe02SNick Piggin unsigned long vstart, unsigned long vend, 1936869176a0SBaoquan He int node, gfp_t gfp_mask, 1937869176a0SBaoquan He unsigned long va_flags) 1938db64fe02SNick Piggin { 1939d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 1940187f8cc4SUladzislau Rezki (Sony) struct vmap_area *va; 194112e376a6SUladzislau Rezki (Sony) unsigned long freed; 19421da177e4SLinus Torvalds unsigned long addr; 194372210662SUladzislau Rezki (Sony) unsigned int vn_id; 1944db64fe02SNick Piggin int purged = 0; 1945d98c9e83SAndrey Ryabinin int ret; 1946db64fe02SNick Piggin 19477e4a32c0SHyunmin Lee if (unlikely(!size || offset_in_page(size) || !is_power_of_2(align))) 19487e4a32c0SHyunmin Lee return ERR_PTR(-EINVAL); 1949db64fe02SNick Piggin 195068ad4a33SUladzislau Rezki (Sony) if (unlikely(!vmap_initialized)) 195168ad4a33SUladzislau Rezki (Sony) return ERR_PTR(-EBUSY); 195268ad4a33SUladzislau Rezki (Sony) 19535803ed29SChristoph Hellwig might_sleep(); 195472210662SUladzislau Rezki (Sony) 195572210662SUladzislau Rezki (Sony) /* 195672210662SUladzislau Rezki (Sony) * If a VA is obtained from a global heap(if it fails here) 195772210662SUladzislau Rezki (Sony) * it is anyway marked with this "vn_id" so it is returned 195872210662SUladzislau Rezki (Sony) * to this pool's node later. Such way gives a possibility 195972210662SUladzislau Rezki (Sony) * to populate pools based on users demand. 196072210662SUladzislau Rezki (Sony) * 196172210662SUladzislau Rezki (Sony) * On success a ready to go VA is returned. 196272210662SUladzislau Rezki (Sony) */ 196372210662SUladzislau Rezki (Sony) va = node_alloc(size, align, vstart, vend, &addr, &vn_id); 196472210662SUladzislau Rezki (Sony) if (!va) { 1965f07116d7SUladzislau Rezki (Sony) gfp_mask = gfp_mask & GFP_RECLAIM_MASK; 19664da56b99SChris Wilson 1967f07116d7SUladzislau Rezki (Sony) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 1968db64fe02SNick Piggin if (unlikely(!va)) 1969db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 1970db64fe02SNick Piggin 19717f88f88fSCatalin Marinas /* 19727f88f88fSCatalin Marinas * Only scan the relevant parts containing pointers to other objects 19737f88f88fSCatalin Marinas * to avoid false negatives. 19747f88f88fSCatalin Marinas */ 1975f07116d7SUladzislau Rezki (Sony) kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask); 197696aa8437SUladzislau Rezki (Sony) } 19777f88f88fSCatalin Marinas 1978db64fe02SNick Piggin retry: 197972210662SUladzislau Rezki (Sony) if (addr == vend) { 1980187f8cc4SUladzislau Rezki (Sony) preload_this_cpu_lock(&free_vmap_area_lock, gfp_mask, node); 1981f9863be4SUladzislau Rezki (Sony) addr = __alloc_vmap_area(&free_vmap_area_root, &free_vmap_area_list, 1982f9863be4SUladzislau Rezki (Sony) size, align, vstart, vend); 1983187f8cc4SUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 198472210662SUladzislau Rezki (Sony) } 198568ad4a33SUladzislau Rezki (Sony) 1986cf243da6SUladzislau Rezki (Sony) trace_alloc_vmap_area(addr, size, align, vstart, vend, addr == vend); 1987cf243da6SUladzislau Rezki (Sony) 198889699605SNick Piggin /* 198968ad4a33SUladzislau Rezki (Sony) * If an allocation fails, the "vend" address is 199068ad4a33SUladzislau Rezki (Sony) * returned. Therefore trigger the overflow path. 199189699605SNick Piggin */ 199268ad4a33SUladzislau Rezki (Sony) if (unlikely(addr == vend)) 199389699605SNick Piggin goto overflow; 199489699605SNick Piggin 199589699605SNick Piggin va->va_start = addr; 199689699605SNick Piggin va->va_end = addr + size; 1997688fcbfcSPengfei Li va->vm = NULL; 199872210662SUladzislau Rezki (Sony) va->flags = (va_flags | vn_id); 199968ad4a33SUladzislau Rezki (Sony) 2000d0936029SUladzislau Rezki (Sony) vn = addr_to_node(va->va_start); 2001d0936029SUladzislau Rezki (Sony) 2002d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 2003d0936029SUladzislau Rezki (Sony) insert_vmap_area(va, &vn->busy.root, &vn->busy.head); 2004d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 200589699605SNick Piggin 200661e16557SWang Xiaoqiang BUG_ON(!IS_ALIGNED(va->va_start, align)); 200789699605SNick Piggin BUG_ON(va->va_start < vstart); 200889699605SNick Piggin BUG_ON(va->va_end > vend); 200989699605SNick Piggin 2010d98c9e83SAndrey Ryabinin ret = kasan_populate_vmalloc(addr, size); 2011d98c9e83SAndrey Ryabinin if (ret) { 2012d98c9e83SAndrey Ryabinin free_vmap_area(va); 2013d98c9e83SAndrey Ryabinin return ERR_PTR(ret); 2014d98c9e83SAndrey Ryabinin } 2015d98c9e83SAndrey Ryabinin 201689699605SNick Piggin return va; 201789699605SNick Piggin 20187766970cSNick Piggin overflow: 2019db64fe02SNick Piggin if (!purged) { 202077e50af0SThomas Gleixner reclaim_and_purge_vmap_areas(); 2021db64fe02SNick Piggin purged = 1; 2022db64fe02SNick Piggin goto retry; 2023db64fe02SNick Piggin } 20244da56b99SChris Wilson 202512e376a6SUladzislau Rezki (Sony) freed = 0; 20264da56b99SChris Wilson blocking_notifier_call_chain(&vmap_notify_list, 0, &freed); 202712e376a6SUladzislau Rezki (Sony) 20284da56b99SChris Wilson if (freed > 0) { 20294da56b99SChris Wilson purged = 0; 20304da56b99SChris Wilson goto retry; 20314da56b99SChris Wilson } 20324da56b99SChris Wilson 203303497d76SFlorian Fainelli if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) 2034756a025fSJoe Perches pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n", 2035756a025fSJoe Perches size); 203668ad4a33SUladzislau Rezki (Sony) 203768ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 2038db64fe02SNick Piggin return ERR_PTR(-EBUSY); 2039db64fe02SNick Piggin } 2040db64fe02SNick Piggin 20414da56b99SChris Wilson int register_vmap_purge_notifier(struct notifier_block *nb) 20424da56b99SChris Wilson { 20434da56b99SChris Wilson return blocking_notifier_chain_register(&vmap_notify_list, nb); 20444da56b99SChris Wilson } 20454da56b99SChris Wilson EXPORT_SYMBOL_GPL(register_vmap_purge_notifier); 20464da56b99SChris Wilson 20474da56b99SChris Wilson int unregister_vmap_purge_notifier(struct notifier_block *nb) 20484da56b99SChris Wilson { 20494da56b99SChris Wilson return blocking_notifier_chain_unregister(&vmap_notify_list, nb); 20504da56b99SChris Wilson } 20514da56b99SChris Wilson EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier); 20524da56b99SChris Wilson 2053db64fe02SNick Piggin /* 2054db64fe02SNick Piggin * lazy_max_pages is the maximum amount of virtual address space we gather up 2055db64fe02SNick Piggin * before attempting to purge with a TLB flush. 2056db64fe02SNick Piggin * 2057db64fe02SNick Piggin * There is a tradeoff here: a larger number will cover more kernel page tables 2058db64fe02SNick Piggin * and take slightly longer to purge, but it will linearly reduce the number of 2059db64fe02SNick Piggin * global TLB flushes that must be performed. It would seem natural to scale 2060db64fe02SNick Piggin * this number up linearly with the number of CPUs (because vmapping activity 2061db64fe02SNick Piggin * could also scale linearly with the number of CPUs), however it is likely 2062db64fe02SNick Piggin * that in practice, workloads might be constrained in other ways that mean 2063db64fe02SNick Piggin * vmap activity will not scale linearly with CPUs. Also, I want to be 2064db64fe02SNick Piggin * conservative and not introduce a big latency on huge systems, so go with 2065db64fe02SNick Piggin * a less aggressive log scale. It will still be an improvement over the old 2066db64fe02SNick Piggin * code, and it will be simple to change the scale factor if we find that it 2067db64fe02SNick Piggin * becomes a problem on bigger systems. 2068db64fe02SNick Piggin */ 2069db64fe02SNick Piggin static unsigned long lazy_max_pages(void) 2070db64fe02SNick Piggin { 2071db64fe02SNick Piggin unsigned int log; 2072db64fe02SNick Piggin 2073db64fe02SNick Piggin log = fls(num_online_cpus()); 2074db64fe02SNick Piggin 2075db64fe02SNick Piggin return log * (32UL * 1024 * 1024 / PAGE_SIZE); 2076db64fe02SNick Piggin } 2077db64fe02SNick Piggin 20784d36e6f8SUladzislau Rezki (Sony) static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0); 2079db64fe02SNick Piggin 20800574ecd1SChristoph Hellwig /* 2081f0953a1bSIngo Molnar * Serialize vmap purging. There is no actual critical section protected 2082153090f2SBaoquan He * by this lock, but we want to avoid concurrent calls for performance 20830574ecd1SChristoph Hellwig * reasons and to make the pcpu_get_vm_areas more deterministic. 20840574ecd1SChristoph Hellwig */ 2085f9e09977SChristoph Hellwig static DEFINE_MUTEX(vmap_purge_lock); 20860574ecd1SChristoph Hellwig 208702b709dfSNick Piggin /* for per-CPU blocks */ 208802b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void); 2089282631cbSUladzislau Rezki (Sony) static cpumask_t purge_nodes; 209002b709dfSNick Piggin 209172210662SUladzislau Rezki (Sony) static void 209272210662SUladzislau Rezki (Sony) reclaim_list_global(struct list_head *head) 2093db64fe02SNick Piggin { 209472210662SUladzislau Rezki (Sony) struct vmap_area *va, *n; 2095db64fe02SNick Piggin 209672210662SUladzislau Rezki (Sony) if (list_empty(head)) 209772210662SUladzislau Rezki (Sony) return; 2098db64fe02SNick Piggin 2099e36176beSUladzislau Rezki (Sony) spin_lock(&free_vmap_area_lock); 210072210662SUladzislau Rezki (Sony) list_for_each_entry_safe(va, n, head, list) 210172210662SUladzislau Rezki (Sony) merge_or_add_vmap_area_augment(va, 210272210662SUladzislau Rezki (Sony) &free_vmap_area_root, &free_vmap_area_list); 210372210662SUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 210472210662SUladzislau Rezki (Sony) } 210572210662SUladzislau Rezki (Sony) 210672210662SUladzislau Rezki (Sony) static void 210772210662SUladzislau Rezki (Sony) decay_va_pool_node(struct vmap_node *vn, bool full_decay) 210872210662SUladzislau Rezki (Sony) { 210972210662SUladzislau Rezki (Sony) struct vmap_area *va, *nva; 211072210662SUladzislau Rezki (Sony) struct list_head decay_list; 211172210662SUladzislau Rezki (Sony) struct rb_root decay_root; 211272210662SUladzislau Rezki (Sony) unsigned long n_decay; 211372210662SUladzislau Rezki (Sony) int i; 211472210662SUladzislau Rezki (Sony) 211572210662SUladzislau Rezki (Sony) decay_root = RB_ROOT; 211672210662SUladzislau Rezki (Sony) INIT_LIST_HEAD(&decay_list); 211772210662SUladzislau Rezki (Sony) 211872210662SUladzislau Rezki (Sony) for (i = 0; i < MAX_VA_SIZE_PAGES; i++) { 211972210662SUladzislau Rezki (Sony) struct list_head tmp_list; 212072210662SUladzislau Rezki (Sony) 212172210662SUladzislau Rezki (Sony) if (list_empty(&vn->pool[i].head)) 212272210662SUladzislau Rezki (Sony) continue; 212372210662SUladzislau Rezki (Sony) 212472210662SUladzislau Rezki (Sony) INIT_LIST_HEAD(&tmp_list); 212572210662SUladzislau Rezki (Sony) 212672210662SUladzislau Rezki (Sony) /* Detach the pool, so no-one can access it. */ 212772210662SUladzislau Rezki (Sony) spin_lock(&vn->pool_lock); 212872210662SUladzislau Rezki (Sony) list_replace_init(&vn->pool[i].head, &tmp_list); 212972210662SUladzislau Rezki (Sony) spin_unlock(&vn->pool_lock); 213072210662SUladzislau Rezki (Sony) 213172210662SUladzislau Rezki (Sony) if (full_decay) 213272210662SUladzislau Rezki (Sony) WRITE_ONCE(vn->pool[i].len, 0); 213372210662SUladzislau Rezki (Sony) 213472210662SUladzislau Rezki (Sony) /* Decay a pool by ~25% out of left objects. */ 213572210662SUladzislau Rezki (Sony) n_decay = vn->pool[i].len >> 2; 213672210662SUladzislau Rezki (Sony) 213772210662SUladzislau Rezki (Sony) list_for_each_entry_safe(va, nva, &tmp_list, list) { 213872210662SUladzislau Rezki (Sony) list_del_init(&va->list); 213972210662SUladzislau Rezki (Sony) merge_or_add_vmap_area(va, &decay_root, &decay_list); 214072210662SUladzislau Rezki (Sony) 214172210662SUladzislau Rezki (Sony) if (!full_decay) { 214272210662SUladzislau Rezki (Sony) WRITE_ONCE(vn->pool[i].len, vn->pool[i].len - 1); 214372210662SUladzislau Rezki (Sony) 214472210662SUladzislau Rezki (Sony) if (!--n_decay) 214572210662SUladzislau Rezki (Sony) break; 214672210662SUladzislau Rezki (Sony) } 214772210662SUladzislau Rezki (Sony) } 214872210662SUladzislau Rezki (Sony) 214915e02a39SUladzislau Rezki (Sony) /* 215015e02a39SUladzislau Rezki (Sony) * Attach the pool back if it has been partly decayed. 215115e02a39SUladzislau Rezki (Sony) * Please note, it is supposed that nobody(other contexts) 215215e02a39SUladzislau Rezki (Sony) * can populate the pool therefore a simple list replace 215315e02a39SUladzislau Rezki (Sony) * operation takes place here. 215415e02a39SUladzislau Rezki (Sony) */ 215572210662SUladzislau Rezki (Sony) if (!full_decay && !list_empty(&tmp_list)) { 215672210662SUladzislau Rezki (Sony) spin_lock(&vn->pool_lock); 215772210662SUladzislau Rezki (Sony) list_replace_init(&tmp_list, &vn->pool[i].head); 215872210662SUladzislau Rezki (Sony) spin_unlock(&vn->pool_lock); 215972210662SUladzislau Rezki (Sony) } 216072210662SUladzislau Rezki (Sony) } 216172210662SUladzislau Rezki (Sony) 216272210662SUladzislau Rezki (Sony) reclaim_list_global(&decay_list); 216372210662SUladzislau Rezki (Sony) } 216472210662SUladzislau Rezki (Sony) 216572210662SUladzislau Rezki (Sony) static void purge_vmap_node(struct work_struct *work) 216672210662SUladzislau Rezki (Sony) { 216772210662SUladzislau Rezki (Sony) struct vmap_node *vn = container_of(work, 216872210662SUladzislau Rezki (Sony) struct vmap_node, purge_work); 216972210662SUladzislau Rezki (Sony) struct vmap_area *va, *n_va; 217072210662SUladzislau Rezki (Sony) LIST_HEAD(local_list); 217172210662SUladzislau Rezki (Sony) 217272210662SUladzislau Rezki (Sony) vn->nr_purged = 0; 217372210662SUladzislau Rezki (Sony) 2174282631cbSUladzislau Rezki (Sony) list_for_each_entry_safe(va, n_va, &vn->purge_list, list) { 21754d36e6f8SUladzislau Rezki (Sony) unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT; 21763c5c3cfbSDaniel Axtens unsigned long orig_start = va->va_start; 21773c5c3cfbSDaniel Axtens unsigned long orig_end = va->va_end; 217872210662SUladzislau Rezki (Sony) unsigned int vn_id = decode_vn_id(va->flags); 2179763b218dSJoel Fernandes 218072210662SUladzislau Rezki (Sony) list_del_init(&va->list); 21819c801f61SUladzislau Rezki (Sony) 21823c5c3cfbSDaniel Axtens if (is_vmalloc_or_module_addr((void *)orig_start)) 21833c5c3cfbSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 21843c5c3cfbSDaniel Axtens va->va_start, va->va_end); 2185dd3b8353SUladzislau Rezki (Sony) 21864d36e6f8SUladzislau Rezki (Sony) atomic_long_sub(nr, &vmap_lazy_nr); 218772210662SUladzislau Rezki (Sony) vn->nr_purged++; 218868571be9SUladzislau Rezki (Sony) 218972210662SUladzislau Rezki (Sony) if (is_vn_id_valid(vn_id) && !vn->skip_populate) 219072210662SUladzislau Rezki (Sony) if (node_pool_add_va(vn, va)) 219172210662SUladzislau Rezki (Sony) continue; 219272210662SUladzislau Rezki (Sony) 219372210662SUladzislau Rezki (Sony) /* Go back to global. */ 219472210662SUladzislau Rezki (Sony) list_add(&va->list, &local_list); 2195763b218dSJoel Fernandes } 21966030fd5fSUladzislau Rezki (Sony) 219772210662SUladzislau Rezki (Sony) reclaim_list_global(&local_list); 2198282631cbSUladzislau Rezki (Sony) } 2199282631cbSUladzislau Rezki (Sony) 2200282631cbSUladzislau Rezki (Sony) /* 2201282631cbSUladzislau Rezki (Sony) * Purges all lazily-freed vmap areas. 2202282631cbSUladzislau Rezki (Sony) */ 220372210662SUladzislau Rezki (Sony) static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end, 220472210662SUladzislau Rezki (Sony) bool full_pool_decay) 2205282631cbSUladzislau Rezki (Sony) { 220672210662SUladzislau Rezki (Sony) unsigned long nr_purged_areas = 0; 220772210662SUladzislau Rezki (Sony) unsigned int nr_purge_helpers; 220872210662SUladzislau Rezki (Sony) unsigned int nr_purge_nodes; 2209282631cbSUladzislau Rezki (Sony) struct vmap_node *vn; 2210282631cbSUladzislau Rezki (Sony) int i; 2211282631cbSUladzislau Rezki (Sony) 2212282631cbSUladzislau Rezki (Sony) lockdep_assert_held(&vmap_purge_lock); 221372210662SUladzislau Rezki (Sony) 221472210662SUladzislau Rezki (Sony) /* 221572210662SUladzislau Rezki (Sony) * Use cpumask to mark which node has to be processed. 221672210662SUladzislau Rezki (Sony) */ 2217282631cbSUladzislau Rezki (Sony) purge_nodes = CPU_MASK_NONE; 2218282631cbSUladzislau Rezki (Sony) 2219282631cbSUladzislau Rezki (Sony) for (i = 0; i < nr_vmap_nodes; i++) { 2220282631cbSUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 2221282631cbSUladzislau Rezki (Sony) 2222282631cbSUladzislau Rezki (Sony) INIT_LIST_HEAD(&vn->purge_list); 222372210662SUladzislau Rezki (Sony) vn->skip_populate = full_pool_decay; 222472210662SUladzislau Rezki (Sony) decay_va_pool_node(vn, full_pool_decay); 2225282631cbSUladzislau Rezki (Sony) 2226282631cbSUladzislau Rezki (Sony) if (RB_EMPTY_ROOT(&vn->lazy.root)) 2227282631cbSUladzislau Rezki (Sony) continue; 2228282631cbSUladzislau Rezki (Sony) 2229282631cbSUladzislau Rezki (Sony) spin_lock(&vn->lazy.lock); 2230282631cbSUladzislau Rezki (Sony) WRITE_ONCE(vn->lazy.root.rb_node, NULL); 2231282631cbSUladzislau Rezki (Sony) list_replace_init(&vn->lazy.head, &vn->purge_list); 2232282631cbSUladzislau Rezki (Sony) spin_unlock(&vn->lazy.lock); 2233282631cbSUladzislau Rezki (Sony) 2234282631cbSUladzislau Rezki (Sony) start = min(start, list_first_entry(&vn->purge_list, 2235282631cbSUladzislau Rezki (Sony) struct vmap_area, list)->va_start); 2236282631cbSUladzislau Rezki (Sony) 2237282631cbSUladzislau Rezki (Sony) end = max(end, list_last_entry(&vn->purge_list, 2238282631cbSUladzislau Rezki (Sony) struct vmap_area, list)->va_end); 2239282631cbSUladzislau Rezki (Sony) 2240282631cbSUladzislau Rezki (Sony) cpumask_set_cpu(i, &purge_nodes); 2241282631cbSUladzislau Rezki (Sony) } 2242282631cbSUladzislau Rezki (Sony) 224372210662SUladzislau Rezki (Sony) nr_purge_nodes = cpumask_weight(&purge_nodes); 224472210662SUladzislau Rezki (Sony) if (nr_purge_nodes > 0) { 2245282631cbSUladzislau Rezki (Sony) flush_tlb_kernel_range(start, end); 2246282631cbSUladzislau Rezki (Sony) 224772210662SUladzislau Rezki (Sony) /* One extra worker is per a lazy_max_pages() full set minus one. */ 224872210662SUladzislau Rezki (Sony) nr_purge_helpers = atomic_long_read(&vmap_lazy_nr) / lazy_max_pages(); 224972210662SUladzislau Rezki (Sony) nr_purge_helpers = clamp(nr_purge_helpers, 1U, nr_purge_nodes) - 1; 225072210662SUladzislau Rezki (Sony) 2251282631cbSUladzislau Rezki (Sony) for_each_cpu(i, &purge_nodes) { 225272210662SUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 225372210662SUladzislau Rezki (Sony) 225472210662SUladzislau Rezki (Sony) if (nr_purge_helpers > 0) { 225572210662SUladzislau Rezki (Sony) INIT_WORK(&vn->purge_work, purge_vmap_node); 225672210662SUladzislau Rezki (Sony) 225772210662SUladzislau Rezki (Sony) if (cpumask_test_cpu(i, cpu_online_mask)) 225872210662SUladzislau Rezki (Sony) schedule_work_on(i, &vn->purge_work); 225972210662SUladzislau Rezki (Sony) else 226072210662SUladzislau Rezki (Sony) schedule_work(&vn->purge_work); 226172210662SUladzislau Rezki (Sony) 226272210662SUladzislau Rezki (Sony) nr_purge_helpers--; 226372210662SUladzislau Rezki (Sony) } else { 226472210662SUladzislau Rezki (Sony) vn->purge_work.func = NULL; 226572210662SUladzislau Rezki (Sony) purge_vmap_node(&vn->purge_work); 226672210662SUladzislau Rezki (Sony) nr_purged_areas += vn->nr_purged; 2267282631cbSUladzislau Rezki (Sony) } 2268282631cbSUladzislau Rezki (Sony) } 2269282631cbSUladzislau Rezki (Sony) 227072210662SUladzislau Rezki (Sony) for_each_cpu(i, &purge_nodes) { 227172210662SUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 227272210662SUladzislau Rezki (Sony) 227372210662SUladzislau Rezki (Sony) if (vn->purge_work.func) { 227472210662SUladzislau Rezki (Sony) flush_work(&vn->purge_work); 227572210662SUladzislau Rezki (Sony) nr_purged_areas += vn->nr_purged; 227672210662SUladzislau Rezki (Sony) } 227772210662SUladzislau Rezki (Sony) } 227872210662SUladzislau Rezki (Sony) } 227972210662SUladzislau Rezki (Sony) 228072210662SUladzislau Rezki (Sony) trace_purge_vmap_area_lazy(start, end, nr_purged_areas); 228172210662SUladzislau Rezki (Sony) return nr_purged_areas > 0; 2282db64fe02SNick Piggin } 2283db64fe02SNick Piggin 2284db64fe02SNick Piggin /* 228577e50af0SThomas Gleixner * Reclaim vmap areas by purging fragmented blocks and purge_vmap_area_list. 2286db64fe02SNick Piggin */ 228777e50af0SThomas Gleixner static void reclaim_and_purge_vmap_areas(void) 228877e50af0SThomas Gleixner 2289db64fe02SNick Piggin { 2290f9e09977SChristoph Hellwig mutex_lock(&vmap_purge_lock); 22910574ecd1SChristoph Hellwig purge_fragmented_blocks_allcpus(); 229272210662SUladzislau Rezki (Sony) __purge_vmap_area_lazy(ULONG_MAX, 0, true); 2293f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 2294db64fe02SNick Piggin } 2295db64fe02SNick Piggin 2296690467c8SUladzislau Rezki (Sony) static void drain_vmap_area_work(struct work_struct *work) 2297690467c8SUladzislau Rezki (Sony) { 2298690467c8SUladzislau Rezki (Sony) mutex_lock(&vmap_purge_lock); 229972210662SUladzislau Rezki (Sony) __purge_vmap_area_lazy(ULONG_MAX, 0, false); 2300690467c8SUladzislau Rezki (Sony) mutex_unlock(&vmap_purge_lock); 2301690467c8SUladzislau Rezki (Sony) } 2302690467c8SUladzislau Rezki (Sony) 2303db64fe02SNick Piggin /* 2304edd89818SUladzislau Rezki (Sony) * Free a vmap area, caller ensuring that the area has been unmapped, 2305edd89818SUladzislau Rezki (Sony) * unlinked and flush_cache_vunmap had been called for the correct 2306edd89818SUladzislau Rezki (Sony) * range previously. 2307db64fe02SNick Piggin */ 230864141da5SJeremy Fitzhardinge static void free_vmap_area_noflush(struct vmap_area *va) 2309db64fe02SNick Piggin { 23108c4196feSUladzislau Rezki (Sony) unsigned long nr_lazy_max = lazy_max_pages(); 23118c4196feSUladzislau Rezki (Sony) unsigned long va_start = va->va_start; 231272210662SUladzislau Rezki (Sony) unsigned int vn_id = decode_vn_id(va->flags); 231372210662SUladzislau Rezki (Sony) struct vmap_node *vn; 23144d36e6f8SUladzislau Rezki (Sony) unsigned long nr_lazy; 231580c4bd7aSChris Wilson 2316edd89818SUladzislau Rezki (Sony) if (WARN_ON_ONCE(!list_empty(&va->list))) 2317edd89818SUladzislau Rezki (Sony) return; 2318dd3b8353SUladzislau Rezki (Sony) 23194d36e6f8SUladzislau Rezki (Sony) nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >> 23204d36e6f8SUladzislau Rezki (Sony) PAGE_SHIFT, &vmap_lazy_nr); 232180c4bd7aSChris Wilson 232296e2db45SUladzislau Rezki (Sony) /* 232372210662SUladzislau Rezki (Sony) * If it was request by a certain node we would like to 232472210662SUladzislau Rezki (Sony) * return it to that node, i.e. its pool for later reuse. 232596e2db45SUladzislau Rezki (Sony) */ 232672210662SUladzislau Rezki (Sony) vn = is_vn_id_valid(vn_id) ? 232772210662SUladzislau Rezki (Sony) id_to_node(vn_id):addr_to_node(va->va_start); 232872210662SUladzislau Rezki (Sony) 2329282631cbSUladzislau Rezki (Sony) spin_lock(&vn->lazy.lock); 233072210662SUladzislau Rezki (Sony) insert_vmap_area(va, &vn->lazy.root, &vn->lazy.head); 2331282631cbSUladzislau Rezki (Sony) spin_unlock(&vn->lazy.lock); 233280c4bd7aSChris Wilson 23338c4196feSUladzislau Rezki (Sony) trace_free_vmap_area_noflush(va_start, nr_lazy, nr_lazy_max); 23348c4196feSUladzislau Rezki (Sony) 233596e2db45SUladzislau Rezki (Sony) /* After this point, we may free va at any time */ 23368c4196feSUladzislau Rezki (Sony) if (unlikely(nr_lazy > nr_lazy_max)) 2337690467c8SUladzislau Rezki (Sony) schedule_work(&drain_vmap_work); 2338db64fe02SNick Piggin } 2339db64fe02SNick Piggin 2340b29acbdcSNick Piggin /* 2341b29acbdcSNick Piggin * Free and unmap a vmap area 2342b29acbdcSNick Piggin */ 2343b29acbdcSNick Piggin static void free_unmap_vmap_area(struct vmap_area *va) 2344b29acbdcSNick Piggin { 2345b29acbdcSNick Piggin flush_cache_vunmap(va->va_start, va->va_end); 23464ad0ae8cSNicholas Piggin vunmap_range_noflush(va->va_start, va->va_end); 23478e57f8acSVlastimil Babka if (debug_pagealloc_enabled_static()) 234882a2e924SChintan Pandya flush_tlb_kernel_range(va->va_start, va->va_end); 234982a2e924SChintan Pandya 2350c8eef01eSChristoph Hellwig free_vmap_area_noflush(va); 2351b29acbdcSNick Piggin } 2352b29acbdcSNick Piggin 2353993d0b28SMatthew Wilcox (Oracle) struct vmap_area *find_vmap_area(unsigned long addr) 2354db64fe02SNick Piggin { 2355d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 2356db64fe02SNick Piggin struct vmap_area *va; 2357d0936029SUladzislau Rezki (Sony) int i, j; 2358db64fe02SNick Piggin 23594ed91fa9SUladzislau Rezki (Sony) if (unlikely(!vmap_initialized)) 23604ed91fa9SUladzislau Rezki (Sony) return NULL; 23614ed91fa9SUladzislau Rezki (Sony) 2362d0936029SUladzislau Rezki (Sony) /* 2363d0936029SUladzislau Rezki (Sony) * An addr_to_node_id(addr) converts an address to a node index 2364d0936029SUladzislau Rezki (Sony) * where a VA is located. If VA spans several zones and passed 2365d0936029SUladzislau Rezki (Sony) * addr is not the same as va->va_start, what is not common, we 236615e02a39SUladzislau Rezki (Sony) * may need to scan extra nodes. See an example: 2367d0936029SUladzislau Rezki (Sony) * 236815e02a39SUladzislau Rezki (Sony) * <----va----> 2369d0936029SUladzislau Rezki (Sony) * -|-----|-----|-----|-----|- 2370d0936029SUladzislau Rezki (Sony) * 1 2 0 1 2371d0936029SUladzislau Rezki (Sony) * 237215e02a39SUladzislau Rezki (Sony) * VA resides in node 1 whereas it spans 1, 2 an 0. If passed 237315e02a39SUladzislau Rezki (Sony) * addr is within 2 or 0 nodes we should do extra work. 2374d0936029SUladzislau Rezki (Sony) */ 2375d0936029SUladzislau Rezki (Sony) i = j = addr_to_node_id(addr); 2376d0936029SUladzislau Rezki (Sony) do { 2377d0936029SUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 2378db64fe02SNick Piggin 2379d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 2380d0936029SUladzislau Rezki (Sony) va = __find_vmap_area(addr, &vn->busy.root); 2381d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 2382d0936029SUladzislau Rezki (Sony) 2383d0936029SUladzislau Rezki (Sony) if (va) 2384db64fe02SNick Piggin return va; 2385d0936029SUladzislau Rezki (Sony) } while ((i = (i + 1) % nr_vmap_nodes) != j); 2386d0936029SUladzislau Rezki (Sony) 2387d0936029SUladzislau Rezki (Sony) return NULL; 2388db64fe02SNick Piggin } 2389db64fe02SNick Piggin 2390edd89818SUladzislau Rezki (Sony) static struct vmap_area *find_unlink_vmap_area(unsigned long addr) 2391edd89818SUladzislau Rezki (Sony) { 2392d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 2393edd89818SUladzislau Rezki (Sony) struct vmap_area *va; 2394d0936029SUladzislau Rezki (Sony) int i, j; 2395edd89818SUladzislau Rezki (Sony) 239615e02a39SUladzislau Rezki (Sony) /* 239715e02a39SUladzislau Rezki (Sony) * Check the comment in the find_vmap_area() about the loop. 239815e02a39SUladzislau Rezki (Sony) */ 2399d0936029SUladzislau Rezki (Sony) i = j = addr_to_node_id(addr); 2400d0936029SUladzislau Rezki (Sony) do { 2401d0936029SUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 2402d0936029SUladzislau Rezki (Sony) 2403d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 2404d0936029SUladzislau Rezki (Sony) va = __find_vmap_area(addr, &vn->busy.root); 2405edd89818SUladzislau Rezki (Sony) if (va) 2406d0936029SUladzislau Rezki (Sony) unlink_va(va, &vn->busy.root); 2407d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 2408edd89818SUladzislau Rezki (Sony) 2409d0936029SUladzislau Rezki (Sony) if (va) 2410edd89818SUladzislau Rezki (Sony) return va; 2411d0936029SUladzislau Rezki (Sony) } while ((i = (i + 1) % nr_vmap_nodes) != j); 2412d0936029SUladzislau Rezki (Sony) 2413d0936029SUladzislau Rezki (Sony) return NULL; 2414edd89818SUladzislau Rezki (Sony) } 2415edd89818SUladzislau Rezki (Sony) 2416db64fe02SNick Piggin /*** Per cpu kva allocator ***/ 2417db64fe02SNick Piggin 2418db64fe02SNick Piggin /* 2419db64fe02SNick Piggin * vmap space is limited especially on 32 bit architectures. Ensure there is 2420db64fe02SNick Piggin * room for at least 16 percpu vmap blocks per CPU. 2421db64fe02SNick Piggin */ 2422db64fe02SNick Piggin /* 2423db64fe02SNick Piggin * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able 2424db64fe02SNick Piggin * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess 2425db64fe02SNick Piggin * instead (we just need a rough idea) 2426db64fe02SNick Piggin */ 2427db64fe02SNick Piggin #if BITS_PER_LONG == 32 2428db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024) 2429db64fe02SNick Piggin #else 2430db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024*1024) 2431db64fe02SNick Piggin #endif 2432db64fe02SNick Piggin 2433db64fe02SNick Piggin #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE) 2434db64fe02SNick Piggin #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */ 2435db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */ 2436db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2) 2437db64fe02SNick Piggin #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */ 2438db64fe02SNick Piggin #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */ 2439f982f915SClemens Ladisch #define VMAP_BBMAP_BITS \ 2440f982f915SClemens Ladisch VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ 2441db64fe02SNick Piggin VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ 2442f982f915SClemens Ladisch VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16)) 2443db64fe02SNick Piggin 2444db64fe02SNick Piggin #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE) 2445db64fe02SNick Piggin 244677e50af0SThomas Gleixner /* 244777e50af0SThomas Gleixner * Purge threshold to prevent overeager purging of fragmented blocks for 244877e50af0SThomas Gleixner * regular operations: Purge if vb->free is less than 1/4 of the capacity. 244977e50af0SThomas Gleixner */ 245077e50af0SThomas Gleixner #define VMAP_PURGE_THRESHOLD (VMAP_BBMAP_BITS / 4) 245177e50af0SThomas Gleixner 2452869176a0SBaoquan He #define VMAP_RAM 0x1 /* indicates vm_map_ram area*/ 2453869176a0SBaoquan He #define VMAP_BLOCK 0x2 /* mark out the vmap_block sub-type*/ 2454869176a0SBaoquan He #define VMAP_FLAGS_MASK 0x3 2455869176a0SBaoquan He 2456db64fe02SNick Piggin struct vmap_block_queue { 2457db64fe02SNick Piggin spinlock_t lock; 2458db64fe02SNick Piggin struct list_head free; 2459062eacf5SUladzislau Rezki (Sony) 2460062eacf5SUladzislau Rezki (Sony) /* 2461062eacf5SUladzislau Rezki (Sony) * An xarray requires an extra memory dynamically to 2462062eacf5SUladzislau Rezki (Sony) * be allocated. If it is an issue, we can use rb-tree 2463062eacf5SUladzislau Rezki (Sony) * instead. 2464062eacf5SUladzislau Rezki (Sony) */ 2465062eacf5SUladzislau Rezki (Sony) struct xarray vmap_blocks; 2466db64fe02SNick Piggin }; 2467db64fe02SNick Piggin 2468db64fe02SNick Piggin struct vmap_block { 2469db64fe02SNick Piggin spinlock_t lock; 2470db64fe02SNick Piggin struct vmap_area *va; 2471db64fe02SNick Piggin unsigned long free, dirty; 2472d76f9954SBaoquan He DECLARE_BITMAP(used_map, VMAP_BBMAP_BITS); 24737d61bfe8SRoman Pen unsigned long dirty_min, dirty_max; /*< dirty range */ 2474db64fe02SNick Piggin struct list_head free_list; 2475db64fe02SNick Piggin struct rcu_head rcu_head; 247602b709dfSNick Piggin struct list_head purge; 2477db64fe02SNick Piggin }; 2478db64fe02SNick Piggin 2479db64fe02SNick Piggin /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */ 2480db64fe02SNick Piggin static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue); 2481db64fe02SNick Piggin 2482db64fe02SNick Piggin /* 2483062eacf5SUladzislau Rezki (Sony) * In order to fast access to any "vmap_block" associated with a 2484062eacf5SUladzislau Rezki (Sony) * specific address, we use a hash. 2485062eacf5SUladzislau Rezki (Sony) * 2486062eacf5SUladzislau Rezki (Sony) * A per-cpu vmap_block_queue is used in both ways, to serialize 2487062eacf5SUladzislau Rezki (Sony) * an access to free block chains among CPUs(alloc path) and it 2488062eacf5SUladzislau Rezki (Sony) * also acts as a vmap_block hash(alloc/free paths). It means we 2489062eacf5SUladzislau Rezki (Sony) * overload it, since we already have the per-cpu array which is 2490062eacf5SUladzislau Rezki (Sony) * used as a hash table. When used as a hash a 'cpu' passed to 2491062eacf5SUladzislau Rezki (Sony) * per_cpu() is not actually a CPU but rather a hash index. 2492062eacf5SUladzislau Rezki (Sony) * 2493fa1c77c1SUladzislau Rezki (Sony) * A hash function is addr_to_vb_xa() which hashes any address 2494062eacf5SUladzislau Rezki (Sony) * to a specific index(in a hash) it belongs to. This then uses a 2495062eacf5SUladzislau Rezki (Sony) * per_cpu() macro to access an array with generated index. 2496062eacf5SUladzislau Rezki (Sony) * 2497062eacf5SUladzislau Rezki (Sony) * An example: 2498062eacf5SUladzislau Rezki (Sony) * 2499062eacf5SUladzislau Rezki (Sony) * CPU_1 CPU_2 CPU_0 2500062eacf5SUladzislau Rezki (Sony) * | | | 2501062eacf5SUladzislau Rezki (Sony) * V V V 2502062eacf5SUladzislau Rezki (Sony) * 0 10 20 30 40 50 60 2503062eacf5SUladzislau Rezki (Sony) * |------|------|------|------|------|------|...<vmap address space> 2504062eacf5SUladzislau Rezki (Sony) * CPU0 CPU1 CPU2 CPU0 CPU1 CPU2 2505062eacf5SUladzislau Rezki (Sony) * 2506062eacf5SUladzislau Rezki (Sony) * - CPU_1 invokes vm_unmap_ram(6), 6 belongs to CPU0 zone, thus 2507062eacf5SUladzislau Rezki (Sony) * it access: CPU0/INDEX0 -> vmap_blocks -> xa_lock; 2508062eacf5SUladzislau Rezki (Sony) * 2509062eacf5SUladzislau Rezki (Sony) * - CPU_2 invokes vm_unmap_ram(11), 11 belongs to CPU1 zone, thus 2510062eacf5SUladzislau Rezki (Sony) * it access: CPU1/INDEX1 -> vmap_blocks -> xa_lock; 2511062eacf5SUladzislau Rezki (Sony) * 2512062eacf5SUladzislau Rezki (Sony) * - CPU_0 invokes vm_unmap_ram(20), 20 belongs to CPU2 zone, thus 2513062eacf5SUladzislau Rezki (Sony) * it access: CPU2/INDEX2 -> vmap_blocks -> xa_lock. 2514062eacf5SUladzislau Rezki (Sony) * 2515062eacf5SUladzislau Rezki (Sony) * This technique almost always avoids lock contention on insert/remove, 2516062eacf5SUladzislau Rezki (Sony) * however xarray spinlocks protect against any contention that remains. 2517db64fe02SNick Piggin */ 2518062eacf5SUladzislau Rezki (Sony) static struct xarray * 2519fa1c77c1SUladzislau Rezki (Sony) addr_to_vb_xa(unsigned long addr) 2520062eacf5SUladzislau Rezki (Sony) { 2521062eacf5SUladzislau Rezki (Sony) int index = (addr / VMAP_BLOCK_SIZE) % num_possible_cpus(); 2522062eacf5SUladzislau Rezki (Sony) 2523062eacf5SUladzislau Rezki (Sony) return &per_cpu(vmap_block_queue, index).vmap_blocks; 2524062eacf5SUladzislau Rezki (Sony) } 2525db64fe02SNick Piggin 2526db64fe02SNick Piggin /* 2527db64fe02SNick Piggin * We should probably have a fallback mechanism to allocate virtual memory 2528db64fe02SNick Piggin * out of partially filled vmap blocks. However vmap block sizing should be 2529db64fe02SNick Piggin * fairly reasonable according to the vmalloc size, so it shouldn't be a 2530db64fe02SNick Piggin * big problem. 2531db64fe02SNick Piggin */ 2532db64fe02SNick Piggin 2533db64fe02SNick Piggin static unsigned long addr_to_vb_idx(unsigned long addr) 2534db64fe02SNick Piggin { 2535db64fe02SNick Piggin addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); 2536db64fe02SNick Piggin addr /= VMAP_BLOCK_SIZE; 2537db64fe02SNick Piggin return addr; 2538db64fe02SNick Piggin } 2539db64fe02SNick Piggin 2540cf725ce2SRoman Pen static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off) 2541cf725ce2SRoman Pen { 2542cf725ce2SRoman Pen unsigned long addr; 2543cf725ce2SRoman Pen 2544cf725ce2SRoman Pen addr = va_start + (pages_off << PAGE_SHIFT); 2545cf725ce2SRoman Pen BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start)); 2546cf725ce2SRoman Pen return (void *)addr; 2547cf725ce2SRoman Pen } 2548cf725ce2SRoman Pen 2549cf725ce2SRoman Pen /** 2550cf725ce2SRoman Pen * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this 2551cf725ce2SRoman Pen * block. Of course pages number can't exceed VMAP_BBMAP_BITS 2552cf725ce2SRoman Pen * @order: how many 2^order pages should be occupied in newly allocated block 2553cf725ce2SRoman Pen * @gfp_mask: flags for the page level allocator 2554cf725ce2SRoman Pen * 2555a862f68aSMike Rapoport * Return: virtual address in a newly allocated block or ERR_PTR(-errno) 2556cf725ce2SRoman Pen */ 2557cf725ce2SRoman Pen static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) 2558db64fe02SNick Piggin { 2559db64fe02SNick Piggin struct vmap_block_queue *vbq; 2560db64fe02SNick Piggin struct vmap_block *vb; 2561db64fe02SNick Piggin struct vmap_area *va; 2562062eacf5SUladzislau Rezki (Sony) struct xarray *xa; 2563db64fe02SNick Piggin unsigned long vb_idx; 2564db64fe02SNick Piggin int node, err; 2565cf725ce2SRoman Pen void *vaddr; 2566db64fe02SNick Piggin 2567db64fe02SNick Piggin node = numa_node_id(); 2568db64fe02SNick Piggin 2569db64fe02SNick Piggin vb = kmalloc_node(sizeof(struct vmap_block), 2570db64fe02SNick Piggin gfp_mask & GFP_RECLAIM_MASK, node); 2571db64fe02SNick Piggin if (unlikely(!vb)) 2572db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 2573db64fe02SNick Piggin 2574db64fe02SNick Piggin va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, 2575db64fe02SNick Piggin VMALLOC_START, VMALLOC_END, 2576869176a0SBaoquan He node, gfp_mask, 2577869176a0SBaoquan He VMAP_RAM|VMAP_BLOCK); 2578ddf9c6d4STobias Klauser if (IS_ERR(va)) { 2579db64fe02SNick Piggin kfree(vb); 2580e7d86340SJulia Lawall return ERR_CAST(va); 2581db64fe02SNick Piggin } 2582db64fe02SNick Piggin 2583cf725ce2SRoman Pen vaddr = vmap_block_vaddr(va->va_start, 0); 2584db64fe02SNick Piggin spin_lock_init(&vb->lock); 2585db64fe02SNick Piggin vb->va = va; 2586cf725ce2SRoman Pen /* At least something should be left free */ 2587cf725ce2SRoman Pen BUG_ON(VMAP_BBMAP_BITS <= (1UL << order)); 2588d76f9954SBaoquan He bitmap_zero(vb->used_map, VMAP_BBMAP_BITS); 2589cf725ce2SRoman Pen vb->free = VMAP_BBMAP_BITS - (1UL << order); 2590db64fe02SNick Piggin vb->dirty = 0; 25917d61bfe8SRoman Pen vb->dirty_min = VMAP_BBMAP_BITS; 25927d61bfe8SRoman Pen vb->dirty_max = 0; 2593d76f9954SBaoquan He bitmap_set(vb->used_map, 0, (1UL << order)); 2594db64fe02SNick Piggin INIT_LIST_HEAD(&vb->free_list); 2595db64fe02SNick Piggin 2596fa1c77c1SUladzislau Rezki (Sony) xa = addr_to_vb_xa(va->va_start); 2597db64fe02SNick Piggin vb_idx = addr_to_vb_idx(va->va_start); 2598062eacf5SUladzislau Rezki (Sony) err = xa_insert(xa, vb_idx, vb, gfp_mask); 25990f14599cSMatthew Wilcox (Oracle) if (err) { 26000f14599cSMatthew Wilcox (Oracle) kfree(vb); 26010f14599cSMatthew Wilcox (Oracle) free_vmap_area(va); 26020f14599cSMatthew Wilcox (Oracle) return ERR_PTR(err); 26030f14599cSMatthew Wilcox (Oracle) } 2604db64fe02SNick Piggin 26053f804920SSebastian Andrzej Siewior vbq = raw_cpu_ptr(&vmap_block_queue); 2606db64fe02SNick Piggin spin_lock(&vbq->lock); 260768ac546fSRoman Pen list_add_tail_rcu(&vb->free_list, &vbq->free); 2608db64fe02SNick Piggin spin_unlock(&vbq->lock); 2609db64fe02SNick Piggin 2610cf725ce2SRoman Pen return vaddr; 2611db64fe02SNick Piggin } 2612db64fe02SNick Piggin 2613db64fe02SNick Piggin static void free_vmap_block(struct vmap_block *vb) 2614db64fe02SNick Piggin { 2615d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 2616db64fe02SNick Piggin struct vmap_block *tmp; 2617062eacf5SUladzislau Rezki (Sony) struct xarray *xa; 2618db64fe02SNick Piggin 2619fa1c77c1SUladzislau Rezki (Sony) xa = addr_to_vb_xa(vb->va->va_start); 2620062eacf5SUladzislau Rezki (Sony) tmp = xa_erase(xa, addr_to_vb_idx(vb->va->va_start)); 2621db64fe02SNick Piggin BUG_ON(tmp != vb); 2622db64fe02SNick Piggin 2623d0936029SUladzislau Rezki (Sony) vn = addr_to_node(vb->va->va_start); 2624d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 2625d0936029SUladzislau Rezki (Sony) unlink_va(vb->va, &vn->busy.root); 2626d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 2627edd89818SUladzislau Rezki (Sony) 262864141da5SJeremy Fitzhardinge free_vmap_area_noflush(vb->va); 262922a3c7d1SLai Jiangshan kfree_rcu(vb, rcu_head); 2630db64fe02SNick Piggin } 2631db64fe02SNick Piggin 2632ca5e46c3SThomas Gleixner static bool purge_fragmented_block(struct vmap_block *vb, 263377e50af0SThomas Gleixner struct vmap_block_queue *vbq, struct list_head *purge_list, 263477e50af0SThomas Gleixner bool force_purge) 263502b709dfSNick Piggin { 2636ca5e46c3SThomas Gleixner if (vb->free + vb->dirty != VMAP_BBMAP_BITS || 2637ca5e46c3SThomas Gleixner vb->dirty == VMAP_BBMAP_BITS) 2638ca5e46c3SThomas Gleixner return false; 263902b709dfSNick Piggin 264077e50af0SThomas Gleixner /* Don't overeagerly purge usable blocks unless requested */ 264177e50af0SThomas Gleixner if (!(force_purge || vb->free < VMAP_PURGE_THRESHOLD)) 264277e50af0SThomas Gleixner return false; 264377e50af0SThomas Gleixner 2644ca5e46c3SThomas Gleixner /* prevent further allocs after releasing lock */ 26457f48121eSThomas Gleixner WRITE_ONCE(vb->free, 0); 2646ca5e46c3SThomas Gleixner /* prevent purging it again */ 26477f48121eSThomas Gleixner WRITE_ONCE(vb->dirty, VMAP_BBMAP_BITS); 26487d61bfe8SRoman Pen vb->dirty_min = 0; 26497d61bfe8SRoman Pen vb->dirty_max = VMAP_BBMAP_BITS; 265002b709dfSNick Piggin spin_lock(&vbq->lock); 265102b709dfSNick Piggin list_del_rcu(&vb->free_list); 265202b709dfSNick Piggin spin_unlock(&vbq->lock); 2653ca5e46c3SThomas Gleixner list_add_tail(&vb->purge, purge_list); 2654ca5e46c3SThomas Gleixner return true; 265502b709dfSNick Piggin } 265602b709dfSNick Piggin 2657ca5e46c3SThomas Gleixner static void free_purged_blocks(struct list_head *purge_list) 2658ca5e46c3SThomas Gleixner { 2659ca5e46c3SThomas Gleixner struct vmap_block *vb, *n_vb; 2660ca5e46c3SThomas Gleixner 2661ca5e46c3SThomas Gleixner list_for_each_entry_safe(vb, n_vb, purge_list, purge) { 266202b709dfSNick Piggin list_del(&vb->purge); 266302b709dfSNick Piggin free_vmap_block(vb); 266402b709dfSNick Piggin } 266502b709dfSNick Piggin } 266602b709dfSNick Piggin 2667ca5e46c3SThomas Gleixner static void purge_fragmented_blocks(int cpu) 2668ca5e46c3SThomas Gleixner { 2669ca5e46c3SThomas Gleixner LIST_HEAD(purge); 2670ca5e46c3SThomas Gleixner struct vmap_block *vb; 2671ca5e46c3SThomas Gleixner struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 2672ca5e46c3SThomas Gleixner 2673ca5e46c3SThomas Gleixner rcu_read_lock(); 2674ca5e46c3SThomas Gleixner list_for_each_entry_rcu(vb, &vbq->free, free_list) { 26757f48121eSThomas Gleixner unsigned long free = READ_ONCE(vb->free); 26767f48121eSThomas Gleixner unsigned long dirty = READ_ONCE(vb->dirty); 26777f48121eSThomas Gleixner 26787f48121eSThomas Gleixner if (free + dirty != VMAP_BBMAP_BITS || 26797f48121eSThomas Gleixner dirty == VMAP_BBMAP_BITS) 2680ca5e46c3SThomas Gleixner continue; 2681ca5e46c3SThomas Gleixner 2682ca5e46c3SThomas Gleixner spin_lock(&vb->lock); 268377e50af0SThomas Gleixner purge_fragmented_block(vb, vbq, &purge, true); 2684ca5e46c3SThomas Gleixner spin_unlock(&vb->lock); 2685ca5e46c3SThomas Gleixner } 2686ca5e46c3SThomas Gleixner rcu_read_unlock(); 2687ca5e46c3SThomas Gleixner free_purged_blocks(&purge); 2688ca5e46c3SThomas Gleixner } 2689ca5e46c3SThomas Gleixner 269002b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void) 269102b709dfSNick Piggin { 269202b709dfSNick Piggin int cpu; 269302b709dfSNick Piggin 269402b709dfSNick Piggin for_each_possible_cpu(cpu) 269502b709dfSNick Piggin purge_fragmented_blocks(cpu); 269602b709dfSNick Piggin } 269702b709dfSNick Piggin 2698db64fe02SNick Piggin static void *vb_alloc(unsigned long size, gfp_t gfp_mask) 2699db64fe02SNick Piggin { 2700db64fe02SNick Piggin struct vmap_block_queue *vbq; 2701db64fe02SNick Piggin struct vmap_block *vb; 2702cf725ce2SRoman Pen void *vaddr = NULL; 2703db64fe02SNick Piggin unsigned int order; 2704db64fe02SNick Piggin 2705891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 2706db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 2707aa91c4d8SJan Kara if (WARN_ON(size == 0)) { 2708aa91c4d8SJan Kara /* 2709aa91c4d8SJan Kara * Allocating 0 bytes isn't what caller wants since 2710aa91c4d8SJan Kara * get_order(0) returns funny result. Just warn and terminate 2711aa91c4d8SJan Kara * early. 2712aa91c4d8SJan Kara */ 2713aa91c4d8SJan Kara return NULL; 2714aa91c4d8SJan Kara } 2715db64fe02SNick Piggin order = get_order(size); 2716db64fe02SNick Piggin 2717db64fe02SNick Piggin rcu_read_lock(); 27183f804920SSebastian Andrzej Siewior vbq = raw_cpu_ptr(&vmap_block_queue); 2719db64fe02SNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 2720cf725ce2SRoman Pen unsigned long pages_off; 2721db64fe02SNick Piggin 272243d76502SThomas Gleixner if (READ_ONCE(vb->free) < (1UL << order)) 272343d76502SThomas Gleixner continue; 272443d76502SThomas Gleixner 2725db64fe02SNick Piggin spin_lock(&vb->lock); 2726cf725ce2SRoman Pen if (vb->free < (1UL << order)) { 2727cf725ce2SRoman Pen spin_unlock(&vb->lock); 2728cf725ce2SRoman Pen continue; 2729cf725ce2SRoman Pen } 273002b709dfSNick Piggin 2731cf725ce2SRoman Pen pages_off = VMAP_BBMAP_BITS - vb->free; 2732cf725ce2SRoman Pen vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); 273343d76502SThomas Gleixner WRITE_ONCE(vb->free, vb->free - (1UL << order)); 2734d76f9954SBaoquan He bitmap_set(vb->used_map, pages_off, (1UL << order)); 2735db64fe02SNick Piggin if (vb->free == 0) { 2736db64fe02SNick Piggin spin_lock(&vbq->lock); 2737de560423SNick Piggin list_del_rcu(&vb->free_list); 2738db64fe02SNick Piggin spin_unlock(&vbq->lock); 2739db64fe02SNick Piggin } 2740cf725ce2SRoman Pen 2741db64fe02SNick Piggin spin_unlock(&vb->lock); 2742db64fe02SNick Piggin break; 2743db64fe02SNick Piggin } 274402b709dfSNick Piggin 2745db64fe02SNick Piggin rcu_read_unlock(); 2746db64fe02SNick Piggin 2747cf725ce2SRoman Pen /* Allocate new block if nothing was found */ 2748cf725ce2SRoman Pen if (!vaddr) 2749cf725ce2SRoman Pen vaddr = new_vmap_block(order, gfp_mask); 2750db64fe02SNick Piggin 2751cf725ce2SRoman Pen return vaddr; 2752db64fe02SNick Piggin } 2753db64fe02SNick Piggin 275478a0e8c4SChristoph Hellwig static void vb_free(unsigned long addr, unsigned long size) 2755db64fe02SNick Piggin { 2756db64fe02SNick Piggin unsigned long offset; 2757db64fe02SNick Piggin unsigned int order; 2758db64fe02SNick Piggin struct vmap_block *vb; 2759062eacf5SUladzislau Rezki (Sony) struct xarray *xa; 2760db64fe02SNick Piggin 2761891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 2762db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 2763b29acbdcSNick Piggin 276478a0e8c4SChristoph Hellwig flush_cache_vunmap(addr, addr + size); 2765b29acbdcSNick Piggin 2766db64fe02SNick Piggin order = get_order(size); 276778a0e8c4SChristoph Hellwig offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT; 2768062eacf5SUladzislau Rezki (Sony) 2769fa1c77c1SUladzislau Rezki (Sony) xa = addr_to_vb_xa(addr); 2770062eacf5SUladzislau Rezki (Sony) vb = xa_load(xa, addr_to_vb_idx(addr)); 2771062eacf5SUladzislau Rezki (Sony) 2772d76f9954SBaoquan He spin_lock(&vb->lock); 2773d76f9954SBaoquan He bitmap_clear(vb->used_map, offset, (1UL << order)); 2774d76f9954SBaoquan He spin_unlock(&vb->lock); 2775db64fe02SNick Piggin 27764ad0ae8cSNicholas Piggin vunmap_range_noflush(addr, addr + size); 277764141da5SJeremy Fitzhardinge 27788e57f8acSVlastimil Babka if (debug_pagealloc_enabled_static()) 277978a0e8c4SChristoph Hellwig flush_tlb_kernel_range(addr, addr + size); 278082a2e924SChintan Pandya 2781db64fe02SNick Piggin spin_lock(&vb->lock); 27827d61bfe8SRoman Pen 2783a09fad96SThomas Gleixner /* Expand the not yet TLB flushed dirty range */ 27847d61bfe8SRoman Pen vb->dirty_min = min(vb->dirty_min, offset); 27857d61bfe8SRoman Pen vb->dirty_max = max(vb->dirty_max, offset + (1UL << order)); 2786d086817dSMinChan Kim 27877f48121eSThomas Gleixner WRITE_ONCE(vb->dirty, vb->dirty + (1UL << order)); 2788db64fe02SNick Piggin if (vb->dirty == VMAP_BBMAP_BITS) { 2789de560423SNick Piggin BUG_ON(vb->free); 2790db64fe02SNick Piggin spin_unlock(&vb->lock); 2791db64fe02SNick Piggin free_vmap_block(vb); 2792db64fe02SNick Piggin } else 2793db64fe02SNick Piggin spin_unlock(&vb->lock); 2794db64fe02SNick Piggin } 2795db64fe02SNick Piggin 2796868b104dSRick Edgecombe static void _vm_unmap_aliases(unsigned long start, unsigned long end, int flush) 2797db64fe02SNick Piggin { 2798ca5e46c3SThomas Gleixner LIST_HEAD(purge_list); 2799db64fe02SNick Piggin int cpu; 2800db64fe02SNick Piggin 28019b463334SJeremy Fitzhardinge if (unlikely(!vmap_initialized)) 28029b463334SJeremy Fitzhardinge return; 28039b463334SJeremy Fitzhardinge 2804ca5e46c3SThomas Gleixner mutex_lock(&vmap_purge_lock); 28055803ed29SChristoph Hellwig 2806db64fe02SNick Piggin for_each_possible_cpu(cpu) { 2807db64fe02SNick Piggin struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 2808db64fe02SNick Piggin struct vmap_block *vb; 2809fc1e0d98SThomas Gleixner unsigned long idx; 2810db64fe02SNick Piggin 2811db64fe02SNick Piggin rcu_read_lock(); 2812fc1e0d98SThomas Gleixner xa_for_each(&vbq->vmap_blocks, idx, vb) { 2813db64fe02SNick Piggin spin_lock(&vb->lock); 2814ca5e46c3SThomas Gleixner 2815ca5e46c3SThomas Gleixner /* 2816ca5e46c3SThomas Gleixner * Try to purge a fragmented block first. If it's 2817ca5e46c3SThomas Gleixner * not purgeable, check whether there is dirty 2818ca5e46c3SThomas Gleixner * space to be flushed. 2819ca5e46c3SThomas Gleixner */ 282077e50af0SThomas Gleixner if (!purge_fragmented_block(vb, vbq, &purge_list, false) && 2821a09fad96SThomas Gleixner vb->dirty_max && vb->dirty != VMAP_BBMAP_BITS) { 28227d61bfe8SRoman Pen unsigned long va_start = vb->va->va_start; 2823db64fe02SNick Piggin unsigned long s, e; 2824b136be5eSJoonsoo Kim 28257d61bfe8SRoman Pen s = va_start + (vb->dirty_min << PAGE_SHIFT); 28267d61bfe8SRoman Pen e = va_start + (vb->dirty_max << PAGE_SHIFT); 2827db64fe02SNick Piggin 28287d61bfe8SRoman Pen start = min(s, start); 28297d61bfe8SRoman Pen end = max(e, end); 28307d61bfe8SRoman Pen 2831a09fad96SThomas Gleixner /* Prevent that this is flushed again */ 2832a09fad96SThomas Gleixner vb->dirty_min = VMAP_BBMAP_BITS; 2833a09fad96SThomas Gleixner vb->dirty_max = 0; 2834a09fad96SThomas Gleixner 2835db64fe02SNick Piggin flush = 1; 2836db64fe02SNick Piggin } 2837db64fe02SNick Piggin spin_unlock(&vb->lock); 2838db64fe02SNick Piggin } 2839db64fe02SNick Piggin rcu_read_unlock(); 2840db64fe02SNick Piggin } 2841ca5e46c3SThomas Gleixner free_purged_blocks(&purge_list); 2842db64fe02SNick Piggin 284372210662SUladzislau Rezki (Sony) if (!__purge_vmap_area_lazy(start, end, false) && flush) 28440574ecd1SChristoph Hellwig flush_tlb_kernel_range(start, end); 2845f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 2846db64fe02SNick Piggin } 2847868b104dSRick Edgecombe 2848868b104dSRick Edgecombe /** 2849868b104dSRick Edgecombe * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer 2850868b104dSRick Edgecombe * 2851868b104dSRick Edgecombe * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily 2852868b104dSRick Edgecombe * to amortize TLB flushing overheads. What this means is that any page you 2853868b104dSRick Edgecombe * have now, may, in a former life, have been mapped into kernel virtual 2854868b104dSRick Edgecombe * address by the vmap layer and so there might be some CPUs with TLB entries 2855868b104dSRick Edgecombe * still referencing that page (additional to the regular 1:1 kernel mapping). 2856868b104dSRick Edgecombe * 2857868b104dSRick Edgecombe * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can 2858868b104dSRick Edgecombe * be sure that none of the pages we have control over will have any aliases 2859868b104dSRick Edgecombe * from the vmap layer. 2860868b104dSRick Edgecombe */ 2861868b104dSRick Edgecombe void vm_unmap_aliases(void) 2862868b104dSRick Edgecombe { 2863868b104dSRick Edgecombe unsigned long start = ULONG_MAX, end = 0; 2864868b104dSRick Edgecombe int flush = 0; 2865868b104dSRick Edgecombe 2866868b104dSRick Edgecombe _vm_unmap_aliases(start, end, flush); 2867868b104dSRick Edgecombe } 2868db64fe02SNick Piggin EXPORT_SYMBOL_GPL(vm_unmap_aliases); 2869db64fe02SNick Piggin 2870db64fe02SNick Piggin /** 2871db64fe02SNick Piggin * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram 2872db64fe02SNick Piggin * @mem: the pointer returned by vm_map_ram 2873db64fe02SNick Piggin * @count: the count passed to that vm_map_ram call (cannot unmap partial) 2874db64fe02SNick Piggin */ 2875db64fe02SNick Piggin void vm_unmap_ram(const void *mem, unsigned int count) 2876db64fe02SNick Piggin { 287765ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 28784aff1dc4SAndrey Konovalov unsigned long addr = (unsigned long)kasan_reset_tag(mem); 28799c3acf60SChristoph Hellwig struct vmap_area *va; 2880db64fe02SNick Piggin 28815803ed29SChristoph Hellwig might_sleep(); 2882db64fe02SNick Piggin BUG_ON(!addr); 2883db64fe02SNick Piggin BUG_ON(addr < VMALLOC_START); 2884db64fe02SNick Piggin BUG_ON(addr > VMALLOC_END); 2885a1c0b1a0SShawn Lin BUG_ON(!PAGE_ALIGNED(addr)); 2886db64fe02SNick Piggin 2887d98c9e83SAndrey Ryabinin kasan_poison_vmalloc(mem, size); 2888d98c9e83SAndrey Ryabinin 28899c3acf60SChristoph Hellwig if (likely(count <= VMAP_MAX_ALLOC)) { 289005e3ff95SChintan Pandya debug_check_no_locks_freed(mem, size); 289178a0e8c4SChristoph Hellwig vb_free(addr, size); 28929c3acf60SChristoph Hellwig return; 28939c3acf60SChristoph Hellwig } 28949c3acf60SChristoph Hellwig 2895edd89818SUladzislau Rezki (Sony) va = find_unlink_vmap_area(addr); 289614687619SUladzislau Rezki (Sony) if (WARN_ON_ONCE(!va)) 289714687619SUladzislau Rezki (Sony) return; 289814687619SUladzislau Rezki (Sony) 289905e3ff95SChintan Pandya debug_check_no_locks_freed((void *)va->va_start, 290005e3ff95SChintan Pandya (va->va_end - va->va_start)); 29019c3acf60SChristoph Hellwig free_unmap_vmap_area(va); 2902db64fe02SNick Piggin } 2903db64fe02SNick Piggin EXPORT_SYMBOL(vm_unmap_ram); 2904db64fe02SNick Piggin 2905db64fe02SNick Piggin /** 2906db64fe02SNick Piggin * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) 2907db64fe02SNick Piggin * @pages: an array of pointers to the pages to be mapped 2908db64fe02SNick Piggin * @count: number of pages 2909db64fe02SNick Piggin * @node: prefer to allocate data structures on this node 2910e99c97adSRandy Dunlap * 291136437638SGioh Kim * If you use this function for less than VMAP_MAX_ALLOC pages, it could be 291236437638SGioh Kim * faster than vmap so it's good. But if you mix long-life and short-life 291336437638SGioh Kim * objects with vm_map_ram(), it could consume lots of address space through 291436437638SGioh Kim * fragmentation (especially on a 32bit machine). You could see failures in 291536437638SGioh Kim * the end. Please use this function for short-lived objects. 291636437638SGioh Kim * 2917e99c97adSRandy Dunlap * Returns: a pointer to the address that has been mapped, or %NULL on failure 2918db64fe02SNick Piggin */ 2919d4efd79aSChristoph Hellwig void *vm_map_ram(struct page **pages, unsigned int count, int node) 2920db64fe02SNick Piggin { 292165ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 2922db64fe02SNick Piggin unsigned long addr; 2923db64fe02SNick Piggin void *mem; 2924db64fe02SNick Piggin 2925db64fe02SNick Piggin if (likely(count <= VMAP_MAX_ALLOC)) { 2926db64fe02SNick Piggin mem = vb_alloc(size, GFP_KERNEL); 2927db64fe02SNick Piggin if (IS_ERR(mem)) 2928db64fe02SNick Piggin return NULL; 2929db64fe02SNick Piggin addr = (unsigned long)mem; 2930db64fe02SNick Piggin } else { 2931db64fe02SNick Piggin struct vmap_area *va; 2932db64fe02SNick Piggin va = alloc_vmap_area(size, PAGE_SIZE, 2933869176a0SBaoquan He VMALLOC_START, VMALLOC_END, 2934869176a0SBaoquan He node, GFP_KERNEL, VMAP_RAM); 2935db64fe02SNick Piggin if (IS_ERR(va)) 2936db64fe02SNick Piggin return NULL; 2937db64fe02SNick Piggin 2938db64fe02SNick Piggin addr = va->va_start; 2939db64fe02SNick Piggin mem = (void *)addr; 2940db64fe02SNick Piggin } 2941d98c9e83SAndrey Ryabinin 2942b67177ecSNicholas Piggin if (vmap_pages_range(addr, addr + size, PAGE_KERNEL, 2943b67177ecSNicholas Piggin pages, PAGE_SHIFT) < 0) { 2944db64fe02SNick Piggin vm_unmap_ram(mem, count); 2945db64fe02SNick Piggin return NULL; 2946db64fe02SNick Piggin } 2947b67177ecSNicholas Piggin 294823689e91SAndrey Konovalov /* 294923689e91SAndrey Konovalov * Mark the pages as accessible, now that they are mapped. 295023689e91SAndrey Konovalov * With hardware tag-based KASAN, marking is skipped for 295123689e91SAndrey Konovalov * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 295223689e91SAndrey Konovalov */ 2953f6e39794SAndrey Konovalov mem = kasan_unpoison_vmalloc(mem, size, KASAN_VMALLOC_PROT_NORMAL); 295419f1c3acSAndrey Konovalov 2955db64fe02SNick Piggin return mem; 2956db64fe02SNick Piggin } 2957db64fe02SNick Piggin EXPORT_SYMBOL(vm_map_ram); 2958db64fe02SNick Piggin 29594341fa45SJoonsoo Kim static struct vm_struct *vmlist __initdata; 296092eac168SMike Rapoport 2961121e6f32SNicholas Piggin static inline unsigned int vm_area_page_order(struct vm_struct *vm) 2962121e6f32SNicholas Piggin { 2963121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 2964121e6f32SNicholas Piggin return vm->page_order; 2965121e6f32SNicholas Piggin #else 2966121e6f32SNicholas Piggin return 0; 2967121e6f32SNicholas Piggin #endif 2968121e6f32SNicholas Piggin } 2969121e6f32SNicholas Piggin 2970121e6f32SNicholas Piggin static inline void set_vm_area_page_order(struct vm_struct *vm, unsigned int order) 2971121e6f32SNicholas Piggin { 2972121e6f32SNicholas Piggin #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 2973121e6f32SNicholas Piggin vm->page_order = order; 2974121e6f32SNicholas Piggin #else 2975121e6f32SNicholas Piggin BUG_ON(order != 0); 2976121e6f32SNicholas Piggin #endif 2977121e6f32SNicholas Piggin } 2978121e6f32SNicholas Piggin 2979f0aa6617STejun Heo /** 2980be9b7335SNicolas Pitre * vm_area_add_early - add vmap area early during boot 2981be9b7335SNicolas Pitre * @vm: vm_struct to add 2982be9b7335SNicolas Pitre * 2983be9b7335SNicolas Pitre * This function is used to add fixed kernel vm area to vmlist before 2984be9b7335SNicolas Pitre * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags 2985be9b7335SNicolas Pitre * should contain proper values and the other fields should be zero. 2986be9b7335SNicolas Pitre * 2987be9b7335SNicolas Pitre * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 2988be9b7335SNicolas Pitre */ 2989be9b7335SNicolas Pitre void __init vm_area_add_early(struct vm_struct *vm) 2990be9b7335SNicolas Pitre { 2991be9b7335SNicolas Pitre struct vm_struct *tmp, **p; 2992be9b7335SNicolas Pitre 2993be9b7335SNicolas Pitre BUG_ON(vmap_initialized); 2994be9b7335SNicolas Pitre for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { 2995be9b7335SNicolas Pitre if (tmp->addr >= vm->addr) { 2996be9b7335SNicolas Pitre BUG_ON(tmp->addr < vm->addr + vm->size); 2997be9b7335SNicolas Pitre break; 2998be9b7335SNicolas Pitre } else 2999be9b7335SNicolas Pitre BUG_ON(tmp->addr + tmp->size > vm->addr); 3000be9b7335SNicolas Pitre } 3001be9b7335SNicolas Pitre vm->next = *p; 3002be9b7335SNicolas Pitre *p = vm; 3003be9b7335SNicolas Pitre } 3004be9b7335SNicolas Pitre 3005be9b7335SNicolas Pitre /** 3006f0aa6617STejun Heo * vm_area_register_early - register vmap area early during boot 3007f0aa6617STejun Heo * @vm: vm_struct to register 3008c0c0a293STejun Heo * @align: requested alignment 3009f0aa6617STejun Heo * 3010f0aa6617STejun Heo * This function is used to register kernel vm area before 3011f0aa6617STejun Heo * vmalloc_init() is called. @vm->size and @vm->flags should contain 3012f0aa6617STejun Heo * proper values on entry and other fields should be zero. On return, 3013f0aa6617STejun Heo * vm->addr contains the allocated address. 3014f0aa6617STejun Heo * 3015f0aa6617STejun Heo * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 3016f0aa6617STejun Heo */ 3017c0c0a293STejun Heo void __init vm_area_register_early(struct vm_struct *vm, size_t align) 3018f0aa6617STejun Heo { 30190eb68437SKefeng Wang unsigned long addr = ALIGN(VMALLOC_START, align); 30200eb68437SKefeng Wang struct vm_struct *cur, **p; 3021f0aa6617STejun Heo 30220eb68437SKefeng Wang BUG_ON(vmap_initialized); 3023c0c0a293STejun Heo 30240eb68437SKefeng Wang for (p = &vmlist; (cur = *p) != NULL; p = &cur->next) { 30250eb68437SKefeng Wang if ((unsigned long)cur->addr - addr >= vm->size) 30260eb68437SKefeng Wang break; 30270eb68437SKefeng Wang addr = ALIGN((unsigned long)cur->addr + cur->size, align); 30280eb68437SKefeng Wang } 30290eb68437SKefeng Wang 30300eb68437SKefeng Wang BUG_ON(addr > VMALLOC_END - vm->size); 3031c0c0a293STejun Heo vm->addr = (void *)addr; 30320eb68437SKefeng Wang vm->next = *p; 30330eb68437SKefeng Wang *p = vm; 30343252b1d8SKefeng Wang kasan_populate_early_vm_area_shadow(vm->addr, vm->size); 3035f0aa6617STejun Heo } 3036f0aa6617STejun Heo 3037e36176beSUladzislau Rezki (Sony) static inline void setup_vmalloc_vm_locked(struct vm_struct *vm, 3038e36176beSUladzislau Rezki (Sony) struct vmap_area *va, unsigned long flags, const void *caller) 3039cf88c790STejun Heo { 3040cf88c790STejun Heo vm->flags = flags; 3041cf88c790STejun Heo vm->addr = (void *)va->va_start; 3042cf88c790STejun Heo vm->size = va->va_end - va->va_start; 3043cf88c790STejun Heo vm->caller = caller; 3044db1aecafSMinchan Kim va->vm = vm; 3045e36176beSUladzislau Rezki (Sony) } 3046e36176beSUladzislau Rezki (Sony) 3047e36176beSUladzislau Rezki (Sony) static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, 3048e36176beSUladzislau Rezki (Sony) unsigned long flags, const void *caller) 3049e36176beSUladzislau Rezki (Sony) { 3050d0936029SUladzislau Rezki (Sony) struct vmap_node *vn = addr_to_node(va->va_start); 3051d0936029SUladzislau Rezki (Sony) 3052d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 3053e36176beSUladzislau Rezki (Sony) setup_vmalloc_vm_locked(vm, va, flags, caller); 3054d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 3055f5252e00SMitsuo Hayasaka } 3056cf88c790STejun Heo 305720fc02b4SZhang Yanfei static void clear_vm_uninitialized_flag(struct vm_struct *vm) 3058f5252e00SMitsuo Hayasaka { 3059d4033afdSJoonsoo Kim /* 306020fc02b4SZhang Yanfei * Before removing VM_UNINITIALIZED, 3061d4033afdSJoonsoo Kim * we should make sure that vm has proper values. 3062d4033afdSJoonsoo Kim * Pair with smp_rmb() in show_numa_info(). 3063d4033afdSJoonsoo Kim */ 3064d4033afdSJoonsoo Kim smp_wmb(); 306520fc02b4SZhang Yanfei vm->flags &= ~VM_UNINITIALIZED; 3066cf88c790STejun Heo } 3067cf88c790STejun Heo 3068db64fe02SNick Piggin static struct vm_struct *__get_vm_area_node(unsigned long size, 30697ca3027bSDaniel Axtens unsigned long align, unsigned long shift, unsigned long flags, 30707ca3027bSDaniel Axtens unsigned long start, unsigned long end, int node, 30717ca3027bSDaniel Axtens gfp_t gfp_mask, const void *caller) 3072db64fe02SNick Piggin { 30730006526dSKautuk Consul struct vmap_area *va; 3074db64fe02SNick Piggin struct vm_struct *area; 3075d98c9e83SAndrey Ryabinin unsigned long requested_size = size; 30761da177e4SLinus Torvalds 307752fd24caSGiridhar Pemmasani BUG_ON(in_interrupt()); 30787ca3027bSDaniel Axtens size = ALIGN(size, 1ul << shift); 307931be8309SOGAWA Hirofumi if (unlikely(!size)) 308031be8309SOGAWA Hirofumi return NULL; 30811da177e4SLinus Torvalds 3082252e5c6eSzijun_hu if (flags & VM_IOREMAP) 3083252e5c6eSzijun_hu align = 1ul << clamp_t(int, get_count_order_long(size), 3084252e5c6eSzijun_hu PAGE_SHIFT, IOREMAP_MAX_ORDER); 3085252e5c6eSzijun_hu 3086cf88c790STejun Heo area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); 30871da177e4SLinus Torvalds if (unlikely(!area)) 30881da177e4SLinus Torvalds return NULL; 30891da177e4SLinus Torvalds 309071394fe5SAndrey Ryabinin if (!(flags & VM_NO_GUARD)) 30911da177e4SLinus Torvalds size += PAGE_SIZE; 30921da177e4SLinus Torvalds 3093869176a0SBaoquan He va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0); 3094db64fe02SNick Piggin if (IS_ERR(va)) { 3095db64fe02SNick Piggin kfree(area); 3096db64fe02SNick Piggin return NULL; 30971da177e4SLinus Torvalds } 30981da177e4SLinus Torvalds 3099d98c9e83SAndrey Ryabinin setup_vmalloc_vm(area, va, flags, caller); 31003c5c3cfbSDaniel Axtens 310119f1c3acSAndrey Konovalov /* 310219f1c3acSAndrey Konovalov * Mark pages for non-VM_ALLOC mappings as accessible. Do it now as a 310319f1c3acSAndrey Konovalov * best-effort approach, as they can be mapped outside of vmalloc code. 310419f1c3acSAndrey Konovalov * For VM_ALLOC mappings, the pages are marked as accessible after 310519f1c3acSAndrey Konovalov * getting mapped in __vmalloc_node_range(). 310623689e91SAndrey Konovalov * With hardware tag-based KASAN, marking is skipped for 310723689e91SAndrey Konovalov * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 310819f1c3acSAndrey Konovalov */ 310919f1c3acSAndrey Konovalov if (!(flags & VM_ALLOC)) 311023689e91SAndrey Konovalov area->addr = kasan_unpoison_vmalloc(area->addr, requested_size, 3111f6e39794SAndrey Konovalov KASAN_VMALLOC_PROT_NORMAL); 31121d96320fSAndrey Konovalov 31131da177e4SLinus Torvalds return area; 31141da177e4SLinus Torvalds } 31151da177e4SLinus Torvalds 3116c2968612SBenjamin Herrenschmidt struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, 3117c2968612SBenjamin Herrenschmidt unsigned long start, unsigned long end, 31185e6cafc8SMarek Szyprowski const void *caller) 3119c2968612SBenjamin Herrenschmidt { 31207ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, start, end, 31217ca3027bSDaniel Axtens NUMA_NO_NODE, GFP_KERNEL, caller); 3122c2968612SBenjamin Herrenschmidt } 3123c2968612SBenjamin Herrenschmidt 31241da177e4SLinus Torvalds /** 3125183ff22bSSimon Arlott * get_vm_area - reserve a contiguous kernel virtual area 31261da177e4SLinus Torvalds * @size: size of the area 31271da177e4SLinus Torvalds * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC 31281da177e4SLinus Torvalds * 31291da177e4SLinus Torvalds * Search an area of @size in the kernel virtual mapping area, 31301da177e4SLinus Torvalds * and reserved it for out purposes. Returns the area descriptor 31311da177e4SLinus Torvalds * on success or %NULL on failure. 3132a862f68aSMike Rapoport * 3133a862f68aSMike Rapoport * Return: the area descriptor on success or %NULL on failure. 31341da177e4SLinus Torvalds */ 31351da177e4SLinus Torvalds struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) 31361da177e4SLinus Torvalds { 31377ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, 31387ca3027bSDaniel Axtens VMALLOC_START, VMALLOC_END, 313900ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, 314000ef2d2fSDavid Rientjes __builtin_return_address(0)); 314123016969SChristoph Lameter } 314223016969SChristoph Lameter 314323016969SChristoph Lameter struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, 31445e6cafc8SMarek Szyprowski const void *caller) 314523016969SChristoph Lameter { 31467ca3027bSDaniel Axtens return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, 31477ca3027bSDaniel Axtens VMALLOC_START, VMALLOC_END, 314800ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, caller); 31491da177e4SLinus Torvalds } 31501da177e4SLinus Torvalds 3151e9da6e99SMarek Szyprowski /** 3152e9da6e99SMarek Szyprowski * find_vm_area - find a continuous kernel virtual area 3153e9da6e99SMarek Szyprowski * @addr: base address 3154e9da6e99SMarek Szyprowski * 3155e9da6e99SMarek Szyprowski * Search for the kernel VM area starting at @addr, and return it. 3156e9da6e99SMarek Szyprowski * It is up to the caller to do all required locking to keep the returned 3157e9da6e99SMarek Szyprowski * pointer valid. 3158a862f68aSMike Rapoport * 315974640617SHui Su * Return: the area descriptor on success or %NULL on failure. 3160e9da6e99SMarek Szyprowski */ 3161e9da6e99SMarek Szyprowski struct vm_struct *find_vm_area(const void *addr) 316283342314SNick Piggin { 3163db64fe02SNick Piggin struct vmap_area *va; 316483342314SNick Piggin 3165db64fe02SNick Piggin va = find_vmap_area((unsigned long)addr); 3166688fcbfcSPengfei Li if (!va) 31677856dfebSAndi Kleen return NULL; 3168688fcbfcSPengfei Li 3169688fcbfcSPengfei Li return va->vm; 31707856dfebSAndi Kleen } 31717856dfebSAndi Kleen 31721da177e4SLinus Torvalds /** 3173183ff22bSSimon Arlott * remove_vm_area - find and remove a continuous kernel virtual area 31741da177e4SLinus Torvalds * @addr: base address 31751da177e4SLinus Torvalds * 31761da177e4SLinus Torvalds * Search for the kernel VM area starting at @addr, and remove it. 31771da177e4SLinus Torvalds * This function returns the found VM area, but using it is NOT safe 31787856dfebSAndi Kleen * on SMP machines, except for its size or flags. 3179a862f68aSMike Rapoport * 318074640617SHui Su * Return: the area descriptor on success or %NULL on failure. 31811da177e4SLinus Torvalds */ 3182b3bdda02SChristoph Lameter struct vm_struct *remove_vm_area(const void *addr) 31831da177e4SLinus Torvalds { 3184db64fe02SNick Piggin struct vmap_area *va; 318575c59ce7SChristoph Hellwig struct vm_struct *vm; 3186db64fe02SNick Piggin 31875803ed29SChristoph Hellwig might_sleep(); 31885803ed29SChristoph Hellwig 318917d3ef43SChristoph Hellwig if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n", 319017d3ef43SChristoph Hellwig addr)) 3191db64fe02SNick Piggin return NULL; 319217d3ef43SChristoph Hellwig 319375c59ce7SChristoph Hellwig va = find_unlink_vmap_area((unsigned long)addr); 319475c59ce7SChristoph Hellwig if (!va || !va->vm) 319575c59ce7SChristoph Hellwig return NULL; 319675c59ce7SChristoph Hellwig vm = va->vm; 319717d3ef43SChristoph Hellwig 319817d3ef43SChristoph Hellwig debug_check_no_locks_freed(vm->addr, get_vm_area_size(vm)); 319917d3ef43SChristoph Hellwig debug_check_no_obj_freed(vm->addr, get_vm_area_size(vm)); 320075c59ce7SChristoph Hellwig kasan_free_module_shadow(vm); 320117d3ef43SChristoph Hellwig kasan_poison_vmalloc(vm->addr, get_vm_area_size(vm)); 320217d3ef43SChristoph Hellwig 320375c59ce7SChristoph Hellwig free_unmap_vmap_area(va); 320475c59ce7SChristoph Hellwig return vm; 32051da177e4SLinus Torvalds } 32061da177e4SLinus Torvalds 3207868b104dSRick Edgecombe static inline void set_area_direct_map(const struct vm_struct *area, 3208868b104dSRick Edgecombe int (*set_direct_map)(struct page *page)) 3209868b104dSRick Edgecombe { 3210868b104dSRick Edgecombe int i; 3211868b104dSRick Edgecombe 3212121e6f32SNicholas Piggin /* HUGE_VMALLOC passes small pages to set_direct_map */ 3213868b104dSRick Edgecombe for (i = 0; i < area->nr_pages; i++) 3214868b104dSRick Edgecombe if (page_address(area->pages[i])) 3215868b104dSRick Edgecombe set_direct_map(area->pages[i]); 3216868b104dSRick Edgecombe } 3217868b104dSRick Edgecombe 32189e5fa0aeSChristoph Hellwig /* 32199e5fa0aeSChristoph Hellwig * Flush the vm mapping and reset the direct map. 32209e5fa0aeSChristoph Hellwig */ 32219e5fa0aeSChristoph Hellwig static void vm_reset_perms(struct vm_struct *area) 3222868b104dSRick Edgecombe { 3223868b104dSRick Edgecombe unsigned long start = ULONG_MAX, end = 0; 3224121e6f32SNicholas Piggin unsigned int page_order = vm_area_page_order(area); 322531e67340SRick Edgecombe int flush_dmap = 0; 3226868b104dSRick Edgecombe int i; 3227868b104dSRick Edgecombe 3228868b104dSRick Edgecombe /* 32299e5fa0aeSChristoph Hellwig * Find the start and end range of the direct mappings to make sure that 3230868b104dSRick Edgecombe * the vm_unmap_aliases() flush includes the direct map. 3231868b104dSRick Edgecombe */ 3232121e6f32SNicholas Piggin for (i = 0; i < area->nr_pages; i += 1U << page_order) { 32338e41f872SRick Edgecombe unsigned long addr = (unsigned long)page_address(area->pages[i]); 32349e5fa0aeSChristoph Hellwig 32358e41f872SRick Edgecombe if (addr) { 3236121e6f32SNicholas Piggin unsigned long page_size; 3237121e6f32SNicholas Piggin 3238121e6f32SNicholas Piggin page_size = PAGE_SIZE << page_order; 3239868b104dSRick Edgecombe start = min(addr, start); 3240121e6f32SNicholas Piggin end = max(addr + page_size, end); 324131e67340SRick Edgecombe flush_dmap = 1; 3242868b104dSRick Edgecombe } 3243868b104dSRick Edgecombe } 3244868b104dSRick Edgecombe 3245868b104dSRick Edgecombe /* 3246868b104dSRick Edgecombe * Set direct map to something invalid so that it won't be cached if 3247868b104dSRick Edgecombe * there are any accesses after the TLB flush, then flush the TLB and 3248868b104dSRick Edgecombe * reset the direct map permissions to the default. 3249868b104dSRick Edgecombe */ 3250868b104dSRick Edgecombe set_area_direct_map(area, set_direct_map_invalid_noflush); 325131e67340SRick Edgecombe _vm_unmap_aliases(start, end, flush_dmap); 3252868b104dSRick Edgecombe set_area_direct_map(area, set_direct_map_default_noflush); 3253868b104dSRick Edgecombe } 3254868b104dSRick Edgecombe 3255208162f4SChristoph Hellwig static void delayed_vfree_work(struct work_struct *w) 32561da177e4SLinus Torvalds { 3257208162f4SChristoph Hellwig struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); 3258208162f4SChristoph Hellwig struct llist_node *t, *llnode; 32591da177e4SLinus Torvalds 3260208162f4SChristoph Hellwig llist_for_each_safe(llnode, t, llist_del_all(&p->list)) 32615d3d31d6SChristoph Hellwig vfree(llnode); 3262bf22e37aSAndrey Ryabinin } 3263bf22e37aSAndrey Ryabinin 3264bf22e37aSAndrey Ryabinin /** 3265bf22e37aSAndrey Ryabinin * vfree_atomic - release memory allocated by vmalloc() 3266bf22e37aSAndrey Ryabinin * @addr: memory base address 3267bf22e37aSAndrey Ryabinin * 3268bf22e37aSAndrey Ryabinin * This one is just like vfree() but can be called in any atomic context 3269bf22e37aSAndrey Ryabinin * except NMIs. 3270bf22e37aSAndrey Ryabinin */ 3271bf22e37aSAndrey Ryabinin void vfree_atomic(const void *addr) 3272bf22e37aSAndrey Ryabinin { 327301e2e839SChristoph Hellwig struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred); 3274bf22e37aSAndrey Ryabinin 327501e2e839SChristoph Hellwig BUG_ON(in_nmi()); 3276bf22e37aSAndrey Ryabinin kmemleak_free(addr); 3277bf22e37aSAndrey Ryabinin 327801e2e839SChristoph Hellwig /* 327901e2e839SChristoph Hellwig * Use raw_cpu_ptr() because this can be called from preemptible 328001e2e839SChristoph Hellwig * context. Preemption is absolutely fine here, because the llist_add() 328101e2e839SChristoph Hellwig * implementation is lockless, so it works even if we are adding to 328201e2e839SChristoph Hellwig * another cpu's list. schedule_work() should be fine with this too. 328301e2e839SChristoph Hellwig */ 328401e2e839SChristoph Hellwig if (addr && llist_add((struct llist_node *)addr, &p->list)) 328501e2e839SChristoph Hellwig schedule_work(&p->wq); 3286c67dc624SRoman Penyaev } 3287c67dc624SRoman Penyaev 32881da177e4SLinus Torvalds /** 3289fa307474SMatthew Wilcox (Oracle) * vfree - Release memory allocated by vmalloc() 3290fa307474SMatthew Wilcox (Oracle) * @addr: Memory base address 32911da177e4SLinus Torvalds * 3292fa307474SMatthew Wilcox (Oracle) * Free the virtually continuous memory area starting at @addr, as obtained 3293fa307474SMatthew Wilcox (Oracle) * from one of the vmalloc() family of APIs. This will usually also free the 3294fa307474SMatthew Wilcox (Oracle) * physical memory underlying the virtual allocation, but that memory is 3295fa307474SMatthew Wilcox (Oracle) * reference counted, so it will not be freed until the last user goes away. 32961da177e4SLinus Torvalds * 3297fa307474SMatthew Wilcox (Oracle) * If @addr is NULL, no operation is performed. 329832fcfd40SAl Viro * 3299fa307474SMatthew Wilcox (Oracle) * Context: 33003ca4ea3aSAndrey Ryabinin * May sleep if called *not* from interrupt context. 3301fa307474SMatthew Wilcox (Oracle) * Must not be called in NMI context (strictly speaking, it could be 3302fa307474SMatthew Wilcox (Oracle) * if we have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling 3303f0953a1bSIngo Molnar * conventions for vfree() arch-dependent would be a really bad idea). 33041da177e4SLinus Torvalds */ 3305b3bdda02SChristoph Lameter void vfree(const void *addr) 33061da177e4SLinus Torvalds { 330779311c1fSChristoph Hellwig struct vm_struct *vm; 330879311c1fSChristoph Hellwig int i; 330979311c1fSChristoph Hellwig 331001e2e839SChristoph Hellwig if (unlikely(in_interrupt())) { 331101e2e839SChristoph Hellwig vfree_atomic(addr); 331232fcfd40SAl Viro return; 331301e2e839SChristoph Hellwig } 331401e2e839SChristoph Hellwig 33151da177e4SLinus Torvalds BUG_ON(in_nmi()); 331689219d37SCatalin Marinas kmemleak_free(addr); 331701e2e839SChristoph Hellwig might_sleep(); 331832fcfd40SAl Viro 3319bf22e37aSAndrey Ryabinin if (!addr) 3320bf22e37aSAndrey Ryabinin return; 3321c67dc624SRoman Penyaev 332279311c1fSChristoph Hellwig vm = remove_vm_area(addr); 332379311c1fSChristoph Hellwig if (unlikely(!vm)) { 332479311c1fSChristoph Hellwig WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", 332579311c1fSChristoph Hellwig addr); 332679311c1fSChristoph Hellwig return; 332779311c1fSChristoph Hellwig } 332879311c1fSChristoph Hellwig 33299e5fa0aeSChristoph Hellwig if (unlikely(vm->flags & VM_FLUSH_RESET_PERMS)) 33309e5fa0aeSChristoph Hellwig vm_reset_perms(vm); 333179311c1fSChristoph Hellwig for (i = 0; i < vm->nr_pages; i++) { 333279311c1fSChristoph Hellwig struct page *page = vm->pages[i]; 333379311c1fSChristoph Hellwig 333479311c1fSChristoph Hellwig BUG_ON(!page); 333579311c1fSChristoph Hellwig mod_memcg_page_state(page, MEMCG_VMALLOC, -1); 333679311c1fSChristoph Hellwig /* 333779311c1fSChristoph Hellwig * High-order allocs for huge vmallocs are split, so 333879311c1fSChristoph Hellwig * can be freed as an array of order-0 allocations 333979311c1fSChristoph Hellwig */ 3340dcc1be11SLorenzo Stoakes __free_page(page); 334179311c1fSChristoph Hellwig cond_resched(); 334279311c1fSChristoph Hellwig } 334379311c1fSChristoph Hellwig atomic_long_sub(vm->nr_pages, &nr_vmalloc_pages); 334479311c1fSChristoph Hellwig kvfree(vm->pages); 334579311c1fSChristoph Hellwig kfree(vm); 33461da177e4SLinus Torvalds } 33471da177e4SLinus Torvalds EXPORT_SYMBOL(vfree); 33481da177e4SLinus Torvalds 33491da177e4SLinus Torvalds /** 33501da177e4SLinus Torvalds * vunmap - release virtual mapping obtained by vmap() 33511da177e4SLinus Torvalds * @addr: memory base address 33521da177e4SLinus Torvalds * 33531da177e4SLinus Torvalds * Free the virtually contiguous memory area starting at @addr, 33541da177e4SLinus Torvalds * which was created from the page array passed to vmap(). 33551da177e4SLinus Torvalds * 335680e93effSPekka Enberg * Must not be called in interrupt context. 33571da177e4SLinus Torvalds */ 3358b3bdda02SChristoph Lameter void vunmap(const void *addr) 33591da177e4SLinus Torvalds { 336079311c1fSChristoph Hellwig struct vm_struct *vm; 336179311c1fSChristoph Hellwig 33621da177e4SLinus Torvalds BUG_ON(in_interrupt()); 336334754b69SPeter Zijlstra might_sleep(); 336479311c1fSChristoph Hellwig 336579311c1fSChristoph Hellwig if (!addr) 336679311c1fSChristoph Hellwig return; 336779311c1fSChristoph Hellwig vm = remove_vm_area(addr); 336879311c1fSChristoph Hellwig if (unlikely(!vm)) { 336979311c1fSChristoph Hellwig WARN(1, KERN_ERR "Trying to vunmap() nonexistent vm area (%p)\n", 337079311c1fSChristoph Hellwig addr); 337179311c1fSChristoph Hellwig return; 337279311c1fSChristoph Hellwig } 337379311c1fSChristoph Hellwig kfree(vm); 33741da177e4SLinus Torvalds } 33751da177e4SLinus Torvalds EXPORT_SYMBOL(vunmap); 33761da177e4SLinus Torvalds 33771da177e4SLinus Torvalds /** 33781da177e4SLinus Torvalds * vmap - map an array of pages into virtually contiguous space 33791da177e4SLinus Torvalds * @pages: array of page pointers 33801da177e4SLinus Torvalds * @count: number of pages to map 33811da177e4SLinus Torvalds * @flags: vm_area->flags 33821da177e4SLinus Torvalds * @prot: page protection for the mapping 33831da177e4SLinus Torvalds * 3384b944afc9SChristoph Hellwig * Maps @count pages from @pages into contiguous kernel virtual space. 3385b944afc9SChristoph Hellwig * If @flags contains %VM_MAP_PUT_PAGES the ownership of the pages array itself 3386b944afc9SChristoph Hellwig * (which must be kmalloc or vmalloc memory) and one reference per pages in it 3387b944afc9SChristoph Hellwig * are transferred from the caller to vmap(), and will be freed / dropped when 3388b944afc9SChristoph Hellwig * vfree() is called on the return value. 3389a862f68aSMike Rapoport * 3390a862f68aSMike Rapoport * Return: the address of the area or %NULL on failure 33911da177e4SLinus Torvalds */ 33921da177e4SLinus Torvalds void *vmap(struct page **pages, unsigned int count, 33931da177e4SLinus Torvalds unsigned long flags, pgprot_t prot) 33941da177e4SLinus Torvalds { 33951da177e4SLinus Torvalds struct vm_struct *area; 3396b67177ecSNicholas Piggin unsigned long addr; 339765ee03c4SGuillermo Julián Moreno unsigned long size; /* In bytes */ 33981da177e4SLinus Torvalds 339934754b69SPeter Zijlstra might_sleep(); 340034754b69SPeter Zijlstra 340137f3605eSChristoph Hellwig if (WARN_ON_ONCE(flags & VM_FLUSH_RESET_PERMS)) 340237f3605eSChristoph Hellwig return NULL; 340337f3605eSChristoph Hellwig 3404bd1a8fb2SPeter Zijlstra /* 3405bd1a8fb2SPeter Zijlstra * Your top guard is someone else's bottom guard. Not having a top 3406bd1a8fb2SPeter Zijlstra * guard compromises someone else's mappings too. 3407bd1a8fb2SPeter Zijlstra */ 3408bd1a8fb2SPeter Zijlstra if (WARN_ON_ONCE(flags & VM_NO_GUARD)) 3409bd1a8fb2SPeter Zijlstra flags &= ~VM_NO_GUARD; 3410bd1a8fb2SPeter Zijlstra 3411ca79b0c2SArun KS if (count > totalram_pages()) 34121da177e4SLinus Torvalds return NULL; 34131da177e4SLinus Torvalds 341465ee03c4SGuillermo Julián Moreno size = (unsigned long)count << PAGE_SHIFT; 341565ee03c4SGuillermo Julián Moreno area = get_vm_area_caller(size, flags, __builtin_return_address(0)); 34161da177e4SLinus Torvalds if (!area) 34171da177e4SLinus Torvalds return NULL; 341823016969SChristoph Lameter 3419b67177ecSNicholas Piggin addr = (unsigned long)area->addr; 3420b67177ecSNicholas Piggin if (vmap_pages_range(addr, addr + size, pgprot_nx(prot), 3421b67177ecSNicholas Piggin pages, PAGE_SHIFT) < 0) { 34221da177e4SLinus Torvalds vunmap(area->addr); 34231da177e4SLinus Torvalds return NULL; 34241da177e4SLinus Torvalds } 34251da177e4SLinus Torvalds 3426c22ee528SMiaohe Lin if (flags & VM_MAP_PUT_PAGES) { 3427b944afc9SChristoph Hellwig area->pages = pages; 3428c22ee528SMiaohe Lin area->nr_pages = count; 3429c22ee528SMiaohe Lin } 34301da177e4SLinus Torvalds return area->addr; 34311da177e4SLinus Torvalds } 34321da177e4SLinus Torvalds EXPORT_SYMBOL(vmap); 34331da177e4SLinus Torvalds 34343e9a9e25SChristoph Hellwig #ifdef CONFIG_VMAP_PFN 34353e9a9e25SChristoph Hellwig struct vmap_pfn_data { 34363e9a9e25SChristoph Hellwig unsigned long *pfns; 34373e9a9e25SChristoph Hellwig pgprot_t prot; 34383e9a9e25SChristoph Hellwig unsigned int idx; 34393e9a9e25SChristoph Hellwig }; 34403e9a9e25SChristoph Hellwig 34413e9a9e25SChristoph Hellwig static int vmap_pfn_apply(pte_t *pte, unsigned long addr, void *private) 34423e9a9e25SChristoph Hellwig { 34433e9a9e25SChristoph Hellwig struct vmap_pfn_data *data = private; 3444b3f78e74SRyan Roberts unsigned long pfn = data->pfns[data->idx]; 3445b3f78e74SRyan Roberts pte_t ptent; 34463e9a9e25SChristoph Hellwig 3447b3f78e74SRyan Roberts if (WARN_ON_ONCE(pfn_valid(pfn))) 34483e9a9e25SChristoph Hellwig return -EINVAL; 3449b3f78e74SRyan Roberts 3450b3f78e74SRyan Roberts ptent = pte_mkspecial(pfn_pte(pfn, data->prot)); 3451b3f78e74SRyan Roberts set_pte_at(&init_mm, addr, pte, ptent); 3452b3f78e74SRyan Roberts 3453b3f78e74SRyan Roberts data->idx++; 34543e9a9e25SChristoph Hellwig return 0; 34553e9a9e25SChristoph Hellwig } 34563e9a9e25SChristoph Hellwig 34573e9a9e25SChristoph Hellwig /** 34583e9a9e25SChristoph Hellwig * vmap_pfn - map an array of PFNs into virtually contiguous space 34593e9a9e25SChristoph Hellwig * @pfns: array of PFNs 34603e9a9e25SChristoph Hellwig * @count: number of pages to map 34613e9a9e25SChristoph Hellwig * @prot: page protection for the mapping 34623e9a9e25SChristoph Hellwig * 34633e9a9e25SChristoph Hellwig * Maps @count PFNs from @pfns into contiguous kernel virtual space and returns 34643e9a9e25SChristoph Hellwig * the start address of the mapping. 34653e9a9e25SChristoph Hellwig */ 34663e9a9e25SChristoph Hellwig void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot) 34673e9a9e25SChristoph Hellwig { 34683e9a9e25SChristoph Hellwig struct vmap_pfn_data data = { .pfns = pfns, .prot = pgprot_nx(prot) }; 34693e9a9e25SChristoph Hellwig struct vm_struct *area; 34703e9a9e25SChristoph Hellwig 34713e9a9e25SChristoph Hellwig area = get_vm_area_caller(count * PAGE_SIZE, VM_IOREMAP, 34723e9a9e25SChristoph Hellwig __builtin_return_address(0)); 34733e9a9e25SChristoph Hellwig if (!area) 34743e9a9e25SChristoph Hellwig return NULL; 34753e9a9e25SChristoph Hellwig if (apply_to_page_range(&init_mm, (unsigned long)area->addr, 34763e9a9e25SChristoph Hellwig count * PAGE_SIZE, vmap_pfn_apply, &data)) { 34773e9a9e25SChristoph Hellwig free_vm_area(area); 34783e9a9e25SChristoph Hellwig return NULL; 34793e9a9e25SChristoph Hellwig } 3480a50420c7SAlexandre Ghiti 3481a50420c7SAlexandre Ghiti flush_cache_vmap((unsigned long)area->addr, 3482a50420c7SAlexandre Ghiti (unsigned long)area->addr + count * PAGE_SIZE); 3483a50420c7SAlexandre Ghiti 34843e9a9e25SChristoph Hellwig return area->addr; 34853e9a9e25SChristoph Hellwig } 34863e9a9e25SChristoph Hellwig EXPORT_SYMBOL_GPL(vmap_pfn); 34873e9a9e25SChristoph Hellwig #endif /* CONFIG_VMAP_PFN */ 34883e9a9e25SChristoph Hellwig 348912b9f873SUladzislau Rezki static inline unsigned int 349012b9f873SUladzislau Rezki vm_area_alloc_pages(gfp_t gfp, int nid, 3491343ab817SUladzislau Rezki (Sony) unsigned int order, unsigned int nr_pages, struct page **pages) 349212b9f873SUladzislau Rezki { 349312b9f873SUladzislau Rezki unsigned int nr_allocated = 0; 3494e9c3cda4SMichal Hocko gfp_t alloc_gfp = gfp; 3495e9c3cda4SMichal Hocko bool nofail = false; 3496ffb29b1cSChen Wandun struct page *page; 3497ffb29b1cSChen Wandun int i; 349812b9f873SUladzislau Rezki 349912b9f873SUladzislau Rezki /* 350012b9f873SUladzislau Rezki * For order-0 pages we make use of bulk allocator, if 350112b9f873SUladzislau Rezki * the page array is partly or not at all populated due 350212b9f873SUladzislau Rezki * to fails, fallback to a single page allocator that is 350312b9f873SUladzislau Rezki * more permissive. 350412b9f873SUladzislau Rezki */ 3505c00b6b96SChen Wandun if (!order) { 3506e9c3cda4SMichal Hocko /* bulk allocator doesn't support nofail req. officially */ 35079376130cSMichal Hocko gfp_t bulk_gfp = gfp & ~__GFP_NOFAIL; 35089376130cSMichal Hocko 3509343ab817SUladzislau Rezki (Sony) while (nr_allocated < nr_pages) { 3510343ab817SUladzislau Rezki (Sony) unsigned int nr, nr_pages_request; 3511343ab817SUladzislau Rezki (Sony) 3512343ab817SUladzislau Rezki (Sony) /* 3513343ab817SUladzislau Rezki (Sony) * A maximum allowed request is hard-coded and is 100 3514343ab817SUladzislau Rezki (Sony) * pages per call. That is done in order to prevent a 3515343ab817SUladzislau Rezki (Sony) * long preemption off scenario in the bulk-allocator 3516343ab817SUladzislau Rezki (Sony) * so the range is [1:100]. 3517343ab817SUladzislau Rezki (Sony) */ 3518343ab817SUladzislau Rezki (Sony) nr_pages_request = min(100U, nr_pages - nr_allocated); 3519343ab817SUladzislau Rezki (Sony) 3520c00b6b96SChen Wandun /* memory allocation should consider mempolicy, we can't 3521c00b6b96SChen Wandun * wrongly use nearest node when nid == NUMA_NO_NODE, 3522c00b6b96SChen Wandun * otherwise memory may be allocated in only one node, 352398af39d5SYixuan Cao * but mempolicy wants to alloc memory by interleaving. 3524c00b6b96SChen Wandun */ 3525c00b6b96SChen Wandun if (IS_ENABLED(CONFIG_NUMA) && nid == NUMA_NO_NODE) 35269376130cSMichal Hocko nr = alloc_pages_bulk_array_mempolicy(bulk_gfp, 3527c00b6b96SChen Wandun nr_pages_request, 3528c00b6b96SChen Wandun pages + nr_allocated); 3529c00b6b96SChen Wandun 3530c00b6b96SChen Wandun else 35319376130cSMichal Hocko nr = alloc_pages_bulk_array_node(bulk_gfp, nid, 3532c00b6b96SChen Wandun nr_pages_request, 3533c00b6b96SChen Wandun pages + nr_allocated); 3534343ab817SUladzislau Rezki (Sony) 3535343ab817SUladzislau Rezki (Sony) nr_allocated += nr; 3536343ab817SUladzislau Rezki (Sony) cond_resched(); 3537343ab817SUladzislau Rezki (Sony) 3538343ab817SUladzislau Rezki (Sony) /* 3539343ab817SUladzislau Rezki (Sony) * If zero or pages were obtained partly, 3540343ab817SUladzislau Rezki (Sony) * fallback to a single page allocator. 3541343ab817SUladzislau Rezki (Sony) */ 3542343ab817SUladzislau Rezki (Sony) if (nr != nr_pages_request) 3543343ab817SUladzislau Rezki (Sony) break; 3544343ab817SUladzislau Rezki (Sony) } 3545e9c3cda4SMichal Hocko } else if (gfp & __GFP_NOFAIL) { 3546e9c3cda4SMichal Hocko /* 3547e9c3cda4SMichal Hocko * Higher order nofail allocations are really expensive and 3548e9c3cda4SMichal Hocko * potentially dangerous (pre-mature OOM, disruptive reclaim 3549e9c3cda4SMichal Hocko * and compaction etc. 3550e9c3cda4SMichal Hocko */ 3551e9c3cda4SMichal Hocko alloc_gfp &= ~__GFP_NOFAIL; 3552e9c3cda4SMichal Hocko nofail = true; 35533b8000aeSNicholas Piggin } 355412b9f873SUladzislau Rezki 355512b9f873SUladzislau Rezki /* High-order pages or fallback path if "bulk" fails. */ 3556ffb29b1cSChen Wandun while (nr_allocated < nr_pages) { 3557dd544141SVasily Averin if (fatal_signal_pending(current)) 3558dd544141SVasily Averin break; 3559dd544141SVasily Averin 3560ffb29b1cSChen Wandun if (nid == NUMA_NO_NODE) 3561e9c3cda4SMichal Hocko page = alloc_pages(alloc_gfp, order); 3562ffb29b1cSChen Wandun else 3563e9c3cda4SMichal Hocko page = alloc_pages_node(nid, alloc_gfp, order); 3564e9c3cda4SMichal Hocko if (unlikely(!page)) { 3565e9c3cda4SMichal Hocko if (!nofail) 356612b9f873SUladzislau Rezki break; 3567e9c3cda4SMichal Hocko 3568e9c3cda4SMichal Hocko /* fall back to the zero order allocations */ 3569e9c3cda4SMichal Hocko alloc_gfp |= __GFP_NOFAIL; 3570e9c3cda4SMichal Hocko order = 0; 3571e9c3cda4SMichal Hocko continue; 3572e9c3cda4SMichal Hocko } 3573e9c3cda4SMichal Hocko 35743b8000aeSNicholas Piggin /* 35753b8000aeSNicholas Piggin * Higher order allocations must be able to be treated as 35763b8000aeSNicholas Piggin * indepdenent small pages by callers (as they can with 35773b8000aeSNicholas Piggin * small-page vmallocs). Some drivers do their own refcounting 35783b8000aeSNicholas Piggin * on vmalloc_to_page() pages, some use page->mapping, 35793b8000aeSNicholas Piggin * page->lru, etc. 35803b8000aeSNicholas Piggin */ 35813b8000aeSNicholas Piggin if (order) 35823b8000aeSNicholas Piggin split_page(page, order); 358312b9f873SUladzislau Rezki 358412b9f873SUladzislau Rezki /* 358512b9f873SUladzislau Rezki * Careful, we allocate and map page-order pages, but 358612b9f873SUladzislau Rezki * tracking is done per PAGE_SIZE page so as to keep the 358712b9f873SUladzislau Rezki * vm_struct APIs independent of the physical/mapped size. 358812b9f873SUladzislau Rezki */ 358912b9f873SUladzislau Rezki for (i = 0; i < (1U << order); i++) 359012b9f873SUladzislau Rezki pages[nr_allocated + i] = page + i; 359112b9f873SUladzislau Rezki 359212b9f873SUladzislau Rezki cond_resched(); 359312b9f873SUladzislau Rezki nr_allocated += 1U << order; 359412b9f873SUladzislau Rezki } 359512b9f873SUladzislau Rezki 359612b9f873SUladzislau Rezki return nr_allocated; 359712b9f873SUladzislau Rezki } 359812b9f873SUladzislau Rezki 3599e31d9eb5SAdrian Bunk static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, 3600121e6f32SNicholas Piggin pgprot_t prot, unsigned int page_shift, 3601121e6f32SNicholas Piggin int node) 36021da177e4SLinus Torvalds { 3603930f036bSDavid Rientjes const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; 36049376130cSMichal Hocko bool nofail = gfp_mask & __GFP_NOFAIL; 3605121e6f32SNicholas Piggin unsigned long addr = (unsigned long)area->addr; 3606121e6f32SNicholas Piggin unsigned long size = get_vm_area_size(area); 360734fe6537SAndrew Morton unsigned long array_size; 3608121e6f32SNicholas Piggin unsigned int nr_small_pages = size >> PAGE_SHIFT; 3609121e6f32SNicholas Piggin unsigned int page_order; 3610451769ebSMichal Hocko unsigned int flags; 3611451769ebSMichal Hocko int ret; 36121da177e4SLinus Torvalds 3613121e6f32SNicholas Piggin array_size = (unsigned long)nr_small_pages * sizeof(struct page *); 361480b1d8fdSLorenzo Stoakes 3615f255935bSChristoph Hellwig if (!(gfp_mask & (GFP_DMA | GFP_DMA32))) 3616f255935bSChristoph Hellwig gfp_mask |= __GFP_HIGHMEM; 36171da177e4SLinus Torvalds 36181da177e4SLinus Torvalds /* Please note that the recursion is strictly bounded. */ 36198757d5faSJan Kiszka if (array_size > PAGE_SIZE) { 36205c1f4e69SUladzislau Rezki (Sony) area->pages = __vmalloc_node(array_size, 1, nested_gfp, node, 3621f255935bSChristoph Hellwig area->caller); 3622286e1ea3SAndrew Morton } else { 36235c1f4e69SUladzislau Rezki (Sony) area->pages = kmalloc_node(array_size, nested_gfp, node); 3624286e1ea3SAndrew Morton } 36257ea36242SAustin Kim 36265c1f4e69SUladzislau Rezki (Sony) if (!area->pages) { 3627c3d77172SUladzislau Rezki (Sony) warn_alloc(gfp_mask, NULL, 3628f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, failed to allocated page array size %lu", 3629d70bec8cSNicholas Piggin nr_small_pages * PAGE_SIZE, array_size); 3630cd61413bSUladzislau Rezki (Sony) free_vm_area(area); 36311da177e4SLinus Torvalds return NULL; 36321da177e4SLinus Torvalds } 36331da177e4SLinus Torvalds 3634121e6f32SNicholas Piggin set_vm_area_page_order(area, page_shift - PAGE_SHIFT); 3635121e6f32SNicholas Piggin page_order = vm_area_page_order(area); 3636121e6f32SNicholas Piggin 3637c3d77172SUladzislau Rezki (Sony) area->nr_pages = vm_area_alloc_pages(gfp_mask | __GFP_NOWARN, 3638c3d77172SUladzislau Rezki (Sony) node, page_order, nr_small_pages, area->pages); 36395c1f4e69SUladzislau Rezki (Sony) 364097105f0aSRoman Gushchin atomic_long_add(area->nr_pages, &nr_vmalloc_pages); 36414e5aa1f4SShakeel Butt if (gfp_mask & __GFP_ACCOUNT) { 36423b8000aeSNicholas Piggin int i; 36434e5aa1f4SShakeel Butt 36443b8000aeSNicholas Piggin for (i = 0; i < area->nr_pages; i++) 36453b8000aeSNicholas Piggin mod_memcg_page_state(area->pages[i], MEMCG_VMALLOC, 1); 36464e5aa1f4SShakeel Butt } 36475c1f4e69SUladzislau Rezki (Sony) 36485c1f4e69SUladzislau Rezki (Sony) /* 36495c1f4e69SUladzislau Rezki (Sony) * If not enough pages were obtained to accomplish an 3650f41f036bSChristoph Hellwig * allocation request, free them via vfree() if any. 36515c1f4e69SUladzislau Rezki (Sony) */ 36525c1f4e69SUladzislau Rezki (Sony) if (area->nr_pages != nr_small_pages) { 365395a301eeSLorenzo Stoakes /* 365495a301eeSLorenzo Stoakes * vm_area_alloc_pages() can fail due to insufficient memory but 365595a301eeSLorenzo Stoakes * also:- 365695a301eeSLorenzo Stoakes * 365795a301eeSLorenzo Stoakes * - a pending fatal signal 365895a301eeSLorenzo Stoakes * - insufficient huge page-order pages 365995a301eeSLorenzo Stoakes * 366095a301eeSLorenzo Stoakes * Since we always retry allocations at order-0 in the huge page 366195a301eeSLorenzo Stoakes * case a warning for either is spurious. 366295a301eeSLorenzo Stoakes */ 366395a301eeSLorenzo Stoakes if (!fatal_signal_pending(current) && page_order == 0) 3664c3d77172SUladzislau Rezki (Sony) warn_alloc(gfp_mask, NULL, 366595a301eeSLorenzo Stoakes "vmalloc error: size %lu, failed to allocate pages", 366695a301eeSLorenzo Stoakes area->nr_pages * PAGE_SIZE); 36671da177e4SLinus Torvalds goto fail; 36681da177e4SLinus Torvalds } 3669121e6f32SNicholas Piggin 3670451769ebSMichal Hocko /* 3671451769ebSMichal Hocko * page tables allocations ignore external gfp mask, enforce it 3672451769ebSMichal Hocko * by the scope API 3673451769ebSMichal Hocko */ 3674451769ebSMichal Hocko if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO) 3675451769ebSMichal Hocko flags = memalloc_nofs_save(); 3676451769ebSMichal Hocko else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0) 3677451769ebSMichal Hocko flags = memalloc_noio_save(); 3678451769ebSMichal Hocko 36799376130cSMichal Hocko do { 3680451769ebSMichal Hocko ret = vmap_pages_range(addr, addr + size, prot, area->pages, 3681451769ebSMichal Hocko page_shift); 36829376130cSMichal Hocko if (nofail && (ret < 0)) 36839376130cSMichal Hocko schedule_timeout_uninterruptible(1); 36849376130cSMichal Hocko } while (nofail && (ret < 0)); 3685451769ebSMichal Hocko 3686451769ebSMichal Hocko if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO) 3687451769ebSMichal Hocko memalloc_nofs_restore(flags); 3688451769ebSMichal Hocko else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0) 3689451769ebSMichal Hocko memalloc_noio_restore(flags); 3690451769ebSMichal Hocko 3691451769ebSMichal Hocko if (ret < 0) { 3692c3d77172SUladzislau Rezki (Sony) warn_alloc(gfp_mask, NULL, 3693f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, failed to map pages", 3694d70bec8cSNicholas Piggin area->nr_pages * PAGE_SIZE); 36951da177e4SLinus Torvalds goto fail; 3696d70bec8cSNicholas Piggin } 3697ed1f324cSChristoph Hellwig 36981da177e4SLinus Torvalds return area->addr; 36991da177e4SLinus Torvalds 37001da177e4SLinus Torvalds fail: 3701f41f036bSChristoph Hellwig vfree(area->addr); 37021da177e4SLinus Torvalds return NULL; 37031da177e4SLinus Torvalds } 37041da177e4SLinus Torvalds 3705d0a21265SDavid Rientjes /** 3706d0a21265SDavid Rientjes * __vmalloc_node_range - allocate virtually contiguous memory 3707d0a21265SDavid Rientjes * @size: allocation size 3708d0a21265SDavid Rientjes * @align: desired alignment 3709d0a21265SDavid Rientjes * @start: vm area range start 3710d0a21265SDavid Rientjes * @end: vm area range end 3711d0a21265SDavid Rientjes * @gfp_mask: flags for the page level allocator 3712d0a21265SDavid Rientjes * @prot: protection mask for the allocated pages 3713cb9e3c29SAndrey Ryabinin * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) 371400ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 3715d0a21265SDavid Rientjes * @caller: caller's return address 3716d0a21265SDavid Rientjes * 3717d0a21265SDavid Rientjes * Allocate enough pages to cover @size from the page level 3718b7d90e7aSMichal Hocko * allocator with @gfp_mask flags. Please note that the full set of gfp 371930d3f011SMichal Hocko * flags are not supported. GFP_KERNEL, GFP_NOFS and GFP_NOIO are all 372030d3f011SMichal Hocko * supported. 372130d3f011SMichal Hocko * Zone modifiers are not supported. From the reclaim modifiers 372230d3f011SMichal Hocko * __GFP_DIRECT_RECLAIM is required (aka GFP_NOWAIT is not supported) 372330d3f011SMichal Hocko * and only __GFP_NOFAIL is supported (i.e. __GFP_NORETRY and 372430d3f011SMichal Hocko * __GFP_RETRY_MAYFAIL are not supported). 372530d3f011SMichal Hocko * 372630d3f011SMichal Hocko * __GFP_NOWARN can be used to suppress failures messages. 3727b7d90e7aSMichal Hocko * 3728b7d90e7aSMichal Hocko * Map them into contiguous kernel virtual space, using a pagetable 3729b7d90e7aSMichal Hocko * protection of @prot. 3730a862f68aSMike Rapoport * 3731a862f68aSMike Rapoport * Return: the address of the area or %NULL on failure 3732d0a21265SDavid Rientjes */ 3733d0a21265SDavid Rientjes void *__vmalloc_node_range(unsigned long size, unsigned long align, 3734d0a21265SDavid Rientjes unsigned long start, unsigned long end, gfp_t gfp_mask, 3735cb9e3c29SAndrey Ryabinin pgprot_t prot, unsigned long vm_flags, int node, 3736cb9e3c29SAndrey Ryabinin const void *caller) 3737930fc45aSChristoph Lameter { 3738d0a21265SDavid Rientjes struct vm_struct *area; 373919f1c3acSAndrey Konovalov void *ret; 3740f6e39794SAndrey Konovalov kasan_vmalloc_flags_t kasan_flags = KASAN_VMALLOC_NONE; 3741d0a21265SDavid Rientjes unsigned long real_size = size; 3742121e6f32SNicholas Piggin unsigned long real_align = align; 3743121e6f32SNicholas Piggin unsigned int shift = PAGE_SHIFT; 3744d0a21265SDavid Rientjes 3745d70bec8cSNicholas Piggin if (WARN_ON_ONCE(!size)) 3746d70bec8cSNicholas Piggin return NULL; 3747d70bec8cSNicholas Piggin 3748d70bec8cSNicholas Piggin if ((size >> PAGE_SHIFT) > totalram_pages()) { 3749d70bec8cSNicholas Piggin warn_alloc(gfp_mask, NULL, 3750f4bdfeafSUladzislau Rezki (Sony) "vmalloc error: size %lu, exceeds total pages", 3751f4bdfeafSUladzislau Rezki (Sony) real_size); 3752d70bec8cSNicholas Piggin return NULL; 3753121e6f32SNicholas Piggin } 3754d0a21265SDavid Rientjes 3755559089e0SSong Liu if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) { 3756121e6f32SNicholas Piggin unsigned long size_per_node; 3757121e6f32SNicholas Piggin 3758121e6f32SNicholas Piggin /* 3759121e6f32SNicholas Piggin * Try huge pages. Only try for PAGE_KERNEL allocations, 3760121e6f32SNicholas Piggin * others like modules don't yet expect huge pages in 3761121e6f32SNicholas Piggin * their allocations due to apply_to_page_range not 3762121e6f32SNicholas Piggin * supporting them. 3763121e6f32SNicholas Piggin */ 3764121e6f32SNicholas Piggin 3765121e6f32SNicholas Piggin size_per_node = size; 3766121e6f32SNicholas Piggin if (node == NUMA_NO_NODE) 3767121e6f32SNicholas Piggin size_per_node /= num_online_nodes(); 37683382bbeeSChristophe Leroy if (arch_vmap_pmd_supported(prot) && size_per_node >= PMD_SIZE) 3769121e6f32SNicholas Piggin shift = PMD_SHIFT; 37703382bbeeSChristophe Leroy else 37713382bbeeSChristophe Leroy shift = arch_vmap_pte_supported_shift(size_per_node); 37723382bbeeSChristophe Leroy 3773121e6f32SNicholas Piggin align = max(real_align, 1UL << shift); 3774121e6f32SNicholas Piggin size = ALIGN(real_size, 1UL << shift); 3775121e6f32SNicholas Piggin } 3776121e6f32SNicholas Piggin 3777121e6f32SNicholas Piggin again: 37787ca3027bSDaniel Axtens area = __get_vm_area_node(real_size, align, shift, VM_ALLOC | 37797ca3027bSDaniel Axtens VM_UNINITIALIZED | vm_flags, start, end, node, 37807ca3027bSDaniel Axtens gfp_mask, caller); 3781d70bec8cSNicholas Piggin if (!area) { 37829376130cSMichal Hocko bool nofail = gfp_mask & __GFP_NOFAIL; 3783d70bec8cSNicholas Piggin warn_alloc(gfp_mask, NULL, 37849376130cSMichal Hocko "vmalloc error: size %lu, vm_struct allocation failed%s", 37859376130cSMichal Hocko real_size, (nofail) ? ". Retrying." : ""); 37869376130cSMichal Hocko if (nofail) { 37879376130cSMichal Hocko schedule_timeout_uninterruptible(1); 37889376130cSMichal Hocko goto again; 37899376130cSMichal Hocko } 3790de7d2b56SJoe Perches goto fail; 3791d70bec8cSNicholas Piggin } 3792d0a21265SDavid Rientjes 3793f6e39794SAndrey Konovalov /* 3794f6e39794SAndrey Konovalov * Prepare arguments for __vmalloc_area_node() and 3795f6e39794SAndrey Konovalov * kasan_unpoison_vmalloc(). 3796f6e39794SAndrey Konovalov */ 3797f6e39794SAndrey Konovalov if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL)) { 3798f6e39794SAndrey Konovalov if (kasan_hw_tags_enabled()) { 379901d92c7fSAndrey Konovalov /* 380001d92c7fSAndrey Konovalov * Modify protection bits to allow tagging. 3801f6e39794SAndrey Konovalov * This must be done before mapping. 380201d92c7fSAndrey Konovalov */ 380301d92c7fSAndrey Konovalov prot = arch_vmap_pgprot_tagged(prot); 380401d92c7fSAndrey Konovalov 380523689e91SAndrey Konovalov /* 3806f6e39794SAndrey Konovalov * Skip page_alloc poisoning and zeroing for physical 3807f6e39794SAndrey Konovalov * pages backing VM_ALLOC mapping. Memory is instead 3808f6e39794SAndrey Konovalov * poisoned and zeroed by kasan_unpoison_vmalloc(). 380923689e91SAndrey Konovalov */ 38100a54864fSPeter Collingbourne gfp_mask |= __GFP_SKIP_KASAN | __GFP_SKIP_ZERO; 381123689e91SAndrey Konovalov } 381223689e91SAndrey Konovalov 3813f6e39794SAndrey Konovalov /* Take note that the mapping is PAGE_KERNEL. */ 3814f6e39794SAndrey Konovalov kasan_flags |= KASAN_VMALLOC_PROT_NORMAL; 3815f6e39794SAndrey Konovalov } 3816f6e39794SAndrey Konovalov 381701d92c7fSAndrey Konovalov /* Allocate physical pages and map them into vmalloc space. */ 381819f1c3acSAndrey Konovalov ret = __vmalloc_area_node(area, gfp_mask, prot, shift, node); 381919f1c3acSAndrey Konovalov if (!ret) 3820121e6f32SNicholas Piggin goto fail; 382189219d37SCatalin Marinas 382223689e91SAndrey Konovalov /* 382323689e91SAndrey Konovalov * Mark the pages as accessible, now that they are mapped. 38246c2f761dSAndrey Konovalov * The condition for setting KASAN_VMALLOC_INIT should complement the 38256c2f761dSAndrey Konovalov * one in post_alloc_hook() with regards to the __GFP_SKIP_ZERO check 38266c2f761dSAndrey Konovalov * to make sure that memory is initialized under the same conditions. 3827f6e39794SAndrey Konovalov * Tag-based KASAN modes only assign tags to normal non-executable 3828f6e39794SAndrey Konovalov * allocations, see __kasan_unpoison_vmalloc(). 382923689e91SAndrey Konovalov */ 3830f6e39794SAndrey Konovalov kasan_flags |= KASAN_VMALLOC_VM_ALLOC; 38316c2f761dSAndrey Konovalov if (!want_init_on_free() && want_init_on_alloc(gfp_mask) && 38326c2f761dSAndrey Konovalov (gfp_mask & __GFP_SKIP_ZERO)) 383323689e91SAndrey Konovalov kasan_flags |= KASAN_VMALLOC_INIT; 3834f6e39794SAndrey Konovalov /* KASAN_VMALLOC_PROT_NORMAL already set if required. */ 383523689e91SAndrey Konovalov area->addr = kasan_unpoison_vmalloc(area->addr, real_size, kasan_flags); 383619f1c3acSAndrey Konovalov 383789219d37SCatalin Marinas /* 383820fc02b4SZhang Yanfei * In this function, newly allocated vm_struct has VM_UNINITIALIZED 383920fc02b4SZhang Yanfei * flag. It means that vm_struct is not fully initialized. 38404341fa45SJoonsoo Kim * Now, it is fully initialized, so remove this flag here. 3841f5252e00SMitsuo Hayasaka */ 384220fc02b4SZhang Yanfei clear_vm_uninitialized_flag(area); 3843f5252e00SMitsuo Hayasaka 38447ca3027bSDaniel Axtens size = PAGE_ALIGN(size); 384560115fa5SKefeng Wang if (!(vm_flags & VM_DEFER_KMEMLEAK)) 384694f4a161SCatalin Marinas kmemleak_vmalloc(area, size, gfp_mask); 384789219d37SCatalin Marinas 384819f1c3acSAndrey Konovalov return area->addr; 3849de7d2b56SJoe Perches 3850de7d2b56SJoe Perches fail: 3851121e6f32SNicholas Piggin if (shift > PAGE_SHIFT) { 3852121e6f32SNicholas Piggin shift = PAGE_SHIFT; 3853121e6f32SNicholas Piggin align = real_align; 3854121e6f32SNicholas Piggin size = real_size; 3855121e6f32SNicholas Piggin goto again; 3856121e6f32SNicholas Piggin } 3857121e6f32SNicholas Piggin 3858de7d2b56SJoe Perches return NULL; 3859930fc45aSChristoph Lameter } 3860930fc45aSChristoph Lameter 38611da177e4SLinus Torvalds /** 3862930fc45aSChristoph Lameter * __vmalloc_node - allocate virtually contiguous memory 38631da177e4SLinus Torvalds * @size: allocation size 38642dca6999SDavid Miller * @align: desired alignment 38651da177e4SLinus Torvalds * @gfp_mask: flags for the page level allocator 386600ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 3867c85d194bSRandy Dunlap * @caller: caller's return address 38681da177e4SLinus Torvalds * 3869f38fcb9cSChristoph Hellwig * Allocate enough pages to cover @size from the page level allocator with 3870f38fcb9cSChristoph Hellwig * @gfp_mask flags. Map them into contiguous kernel virtual space. 3871a7c3e901SMichal Hocko * 3872dcda9b04SMichal Hocko * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL 3873a7c3e901SMichal Hocko * and __GFP_NOFAIL are not supported 3874a7c3e901SMichal Hocko * 3875a7c3e901SMichal Hocko * Any use of gfp flags outside of GFP_KERNEL should be consulted 3876a7c3e901SMichal Hocko * with mm people. 3877a862f68aSMike Rapoport * 3878a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 38791da177e4SLinus Torvalds */ 38802b905948SChristoph Hellwig void *__vmalloc_node(unsigned long size, unsigned long align, 3881f38fcb9cSChristoph Hellwig gfp_t gfp_mask, int node, const void *caller) 38821da177e4SLinus Torvalds { 3883d0a21265SDavid Rientjes return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 3884f38fcb9cSChristoph Hellwig gfp_mask, PAGE_KERNEL, 0, node, caller); 38851da177e4SLinus Torvalds } 3886c3f896dcSChristoph Hellwig /* 3887c3f896dcSChristoph Hellwig * This is only for performance analysis of vmalloc and stress purpose. 3888c3f896dcSChristoph Hellwig * It is required by vmalloc test module, therefore do not use it other 3889c3f896dcSChristoph Hellwig * than that. 3890c3f896dcSChristoph Hellwig */ 3891c3f896dcSChristoph Hellwig #ifdef CONFIG_TEST_VMALLOC_MODULE 3892c3f896dcSChristoph Hellwig EXPORT_SYMBOL_GPL(__vmalloc_node); 3893c3f896dcSChristoph Hellwig #endif 38941da177e4SLinus Torvalds 389588dca4caSChristoph Hellwig void *__vmalloc(unsigned long size, gfp_t gfp_mask) 3896930fc45aSChristoph Lameter { 3897f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, gfp_mask, NUMA_NO_NODE, 389823016969SChristoph Lameter __builtin_return_address(0)); 3899930fc45aSChristoph Lameter } 39001da177e4SLinus Torvalds EXPORT_SYMBOL(__vmalloc); 39011da177e4SLinus Torvalds 39021da177e4SLinus Torvalds /** 39031da177e4SLinus Torvalds * vmalloc - allocate virtually contiguous memory 39041da177e4SLinus Torvalds * @size: allocation size 390592eac168SMike Rapoport * 39061da177e4SLinus Torvalds * Allocate enough pages to cover @size from the page level 39071da177e4SLinus Torvalds * allocator and map them into contiguous kernel virtual space. 39081da177e4SLinus Torvalds * 3909c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 39101da177e4SLinus Torvalds * use __vmalloc() instead. 3911a862f68aSMike Rapoport * 3912a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 39131da177e4SLinus Torvalds */ 39141da177e4SLinus Torvalds void *vmalloc(unsigned long size) 39151da177e4SLinus Torvalds { 39164d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL, NUMA_NO_NODE, 39174d39d728SChristoph Hellwig __builtin_return_address(0)); 39181da177e4SLinus Torvalds } 39191da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc); 39201da177e4SLinus Torvalds 3921930fc45aSChristoph Lameter /** 3922559089e0SSong Liu * vmalloc_huge - allocate virtually contiguous memory, allow huge pages 392315a64f5aSClaudio Imbrenda * @size: allocation size 3924559089e0SSong Liu * @gfp_mask: flags for the page level allocator 392515a64f5aSClaudio Imbrenda * 3926559089e0SSong Liu * Allocate enough pages to cover @size from the page level 392715a64f5aSClaudio Imbrenda * allocator and map them into contiguous kernel virtual space. 3928559089e0SSong Liu * If @size is greater than or equal to PMD_SIZE, allow using 3929559089e0SSong Liu * huge pages for the memory 393015a64f5aSClaudio Imbrenda * 393115a64f5aSClaudio Imbrenda * Return: pointer to the allocated memory or %NULL on error 393215a64f5aSClaudio Imbrenda */ 3933559089e0SSong Liu void *vmalloc_huge(unsigned long size, gfp_t gfp_mask) 393415a64f5aSClaudio Imbrenda { 393515a64f5aSClaudio Imbrenda return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, 3936559089e0SSong Liu gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, 393715a64f5aSClaudio Imbrenda NUMA_NO_NODE, __builtin_return_address(0)); 393815a64f5aSClaudio Imbrenda } 3939559089e0SSong Liu EXPORT_SYMBOL_GPL(vmalloc_huge); 394015a64f5aSClaudio Imbrenda 394115a64f5aSClaudio Imbrenda /** 3942e1ca7788SDave Young * vzalloc - allocate virtually contiguous memory with zero fill 3943e1ca7788SDave Young * @size: allocation size 394492eac168SMike Rapoport * 3945e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 3946e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 3947e1ca7788SDave Young * The memory allocated is set to zero. 3948e1ca7788SDave Young * 3949e1ca7788SDave Young * For tight control over page level allocator and protection flags 3950e1ca7788SDave Young * use __vmalloc() instead. 3951a862f68aSMike Rapoport * 3952a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 3953e1ca7788SDave Young */ 3954e1ca7788SDave Young void *vzalloc(unsigned long size) 3955e1ca7788SDave Young { 39564d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, 39574d39d728SChristoph Hellwig __builtin_return_address(0)); 3958e1ca7788SDave Young } 3959e1ca7788SDave Young EXPORT_SYMBOL(vzalloc); 3960e1ca7788SDave Young 3961e1ca7788SDave Young /** 3962ead04089SRolf Eike Beer * vmalloc_user - allocate zeroed virtually contiguous memory for userspace 396383342314SNick Piggin * @size: allocation size 3964ead04089SRolf Eike Beer * 3965ead04089SRolf Eike Beer * The resulting memory area is zeroed so it can be mapped to userspace 3966ead04089SRolf Eike Beer * without leaking data. 3967a862f68aSMike Rapoport * 3968a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 396983342314SNick Piggin */ 397083342314SNick Piggin void *vmalloc_user(unsigned long size) 397183342314SNick Piggin { 3972bc84c535SRoman Penyaev return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, 3973bc84c535SRoman Penyaev GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, 3974bc84c535SRoman Penyaev VM_USERMAP, NUMA_NO_NODE, 397500ef2d2fSDavid Rientjes __builtin_return_address(0)); 397683342314SNick Piggin } 397783342314SNick Piggin EXPORT_SYMBOL(vmalloc_user); 397883342314SNick Piggin 397983342314SNick Piggin /** 3980930fc45aSChristoph Lameter * vmalloc_node - allocate memory on a specific node 3981930fc45aSChristoph Lameter * @size: allocation size 3982d44e0780SRandy Dunlap * @node: numa node 3983930fc45aSChristoph Lameter * 3984930fc45aSChristoph Lameter * Allocate enough pages to cover @size from the page level 3985930fc45aSChristoph Lameter * allocator and map them into contiguous kernel virtual space. 3986930fc45aSChristoph Lameter * 3987c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 3988930fc45aSChristoph Lameter * use __vmalloc() instead. 3989a862f68aSMike Rapoport * 3990a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 3991930fc45aSChristoph Lameter */ 3992930fc45aSChristoph Lameter void *vmalloc_node(unsigned long size, int node) 3993930fc45aSChristoph Lameter { 3994f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL, node, 3995f38fcb9cSChristoph Hellwig __builtin_return_address(0)); 3996930fc45aSChristoph Lameter } 3997930fc45aSChristoph Lameter EXPORT_SYMBOL(vmalloc_node); 3998930fc45aSChristoph Lameter 3999e1ca7788SDave Young /** 4000e1ca7788SDave Young * vzalloc_node - allocate memory on a specific node with zero fill 4001e1ca7788SDave Young * @size: allocation size 4002e1ca7788SDave Young * @node: numa node 4003e1ca7788SDave Young * 4004e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 4005e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 4006e1ca7788SDave Young * The memory allocated is set to zero. 4007e1ca7788SDave Young * 4008a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 4009e1ca7788SDave Young */ 4010e1ca7788SDave Young void *vzalloc_node(unsigned long size, int node) 4011e1ca7788SDave Young { 40124d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, node, 40134d39d728SChristoph Hellwig __builtin_return_address(0)); 4014e1ca7788SDave Young } 4015e1ca7788SDave Young EXPORT_SYMBOL(vzalloc_node); 4016e1ca7788SDave Young 40170d08e0d3SAndi Kleen #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) 4018698d0831SMichal Hocko #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) 40190d08e0d3SAndi Kleen #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) 4020698d0831SMichal Hocko #define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL) 40210d08e0d3SAndi Kleen #else 4022698d0831SMichal Hocko /* 4023698d0831SMichal Hocko * 64b systems should always have either DMA or DMA32 zones. For others 4024698d0831SMichal Hocko * GFP_DMA32 should do the right thing and use the normal zone. 4025698d0831SMichal Hocko */ 402668d68ff6SZhiyuan Dai #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) 40270d08e0d3SAndi Kleen #endif 40280d08e0d3SAndi Kleen 40291da177e4SLinus Torvalds /** 40301da177e4SLinus Torvalds * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 40311da177e4SLinus Torvalds * @size: allocation size 40321da177e4SLinus Torvalds * 40331da177e4SLinus Torvalds * Allocate enough 32bit PA addressable pages to cover @size from the 40341da177e4SLinus Torvalds * page level allocator and map them into contiguous kernel virtual space. 4035a862f68aSMike Rapoport * 4036a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 40371da177e4SLinus Torvalds */ 40381da177e4SLinus Torvalds void *vmalloc_32(unsigned long size) 40391da177e4SLinus Torvalds { 4040f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, GFP_VMALLOC32, NUMA_NO_NODE, 4041f38fcb9cSChristoph Hellwig __builtin_return_address(0)); 40421da177e4SLinus Torvalds } 40431da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc_32); 40441da177e4SLinus Torvalds 404583342314SNick Piggin /** 4046ead04089SRolf Eike Beer * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 404783342314SNick Piggin * @size: allocation size 4048ead04089SRolf Eike Beer * 4049ead04089SRolf Eike Beer * The resulting memory area is 32bit addressable and zeroed so it can be 4050ead04089SRolf Eike Beer * mapped to userspace without leaking data. 4051a862f68aSMike Rapoport * 4052a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 405383342314SNick Piggin */ 405483342314SNick Piggin void *vmalloc_32_user(unsigned long size) 405583342314SNick Piggin { 4056bc84c535SRoman Penyaev return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, 4057bc84c535SRoman Penyaev GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, 4058bc84c535SRoman Penyaev VM_USERMAP, NUMA_NO_NODE, 40595a82ac71SRoman Penyaev __builtin_return_address(0)); 406083342314SNick Piggin } 406183342314SNick Piggin EXPORT_SYMBOL(vmalloc_32_user); 406283342314SNick Piggin 4063d0107eb0SKAMEZAWA Hiroyuki /* 40644c91c07cSLorenzo Stoakes * Atomically zero bytes in the iterator. 40654c91c07cSLorenzo Stoakes * 40664c91c07cSLorenzo Stoakes * Returns the number of zeroed bytes. 4067d0107eb0SKAMEZAWA Hiroyuki */ 40684c91c07cSLorenzo Stoakes static size_t zero_iter(struct iov_iter *iter, size_t count) 4069d0107eb0SKAMEZAWA Hiroyuki { 40704c91c07cSLorenzo Stoakes size_t remains = count; 4071d0107eb0SKAMEZAWA Hiroyuki 40724c91c07cSLorenzo Stoakes while (remains > 0) { 40734c91c07cSLorenzo Stoakes size_t num, copied; 40744c91c07cSLorenzo Stoakes 40750e4bc271SLu Hongfei num = min_t(size_t, remains, PAGE_SIZE); 40764c91c07cSLorenzo Stoakes copied = copy_page_to_iter_nofault(ZERO_PAGE(0), 0, num, iter); 40774c91c07cSLorenzo Stoakes remains -= copied; 40784c91c07cSLorenzo Stoakes 40794c91c07cSLorenzo Stoakes if (copied < num) 40804c91c07cSLorenzo Stoakes break; 40814c91c07cSLorenzo Stoakes } 40824c91c07cSLorenzo Stoakes 40834c91c07cSLorenzo Stoakes return count - remains; 40844c91c07cSLorenzo Stoakes } 40854c91c07cSLorenzo Stoakes 40864c91c07cSLorenzo Stoakes /* 40874c91c07cSLorenzo Stoakes * small helper routine, copy contents to iter from addr. 40884c91c07cSLorenzo Stoakes * If the page is not present, fill zero. 40894c91c07cSLorenzo Stoakes * 40904c91c07cSLorenzo Stoakes * Returns the number of copied bytes. 40914c91c07cSLorenzo Stoakes */ 40924c91c07cSLorenzo Stoakes static size_t aligned_vread_iter(struct iov_iter *iter, 40934c91c07cSLorenzo Stoakes const char *addr, size_t count) 40944c91c07cSLorenzo Stoakes { 40954c91c07cSLorenzo Stoakes size_t remains = count; 40964c91c07cSLorenzo Stoakes struct page *page; 40974c91c07cSLorenzo Stoakes 40984c91c07cSLorenzo Stoakes while (remains > 0) { 4099d0107eb0SKAMEZAWA Hiroyuki unsigned long offset, length; 41004c91c07cSLorenzo Stoakes size_t copied = 0; 4101d0107eb0SKAMEZAWA Hiroyuki 4102891c49abSAlexander Kuleshov offset = offset_in_page(addr); 4103d0107eb0SKAMEZAWA Hiroyuki length = PAGE_SIZE - offset; 41044c91c07cSLorenzo Stoakes if (length > remains) 41054c91c07cSLorenzo Stoakes length = remains; 41064c91c07cSLorenzo Stoakes page = vmalloc_to_page(addr); 4107d0107eb0SKAMEZAWA Hiroyuki /* 41084c91c07cSLorenzo Stoakes * To do safe access to this _mapped_ area, we need lock. But 41094c91c07cSLorenzo Stoakes * adding lock here means that we need to add overhead of 41104c91c07cSLorenzo Stoakes * vmalloc()/vfree() calls for this _debug_ interface, rarely 41114c91c07cSLorenzo Stoakes * used. Instead of that, we'll use an local mapping via 41124c91c07cSLorenzo Stoakes * copy_page_to_iter_nofault() and accept a small overhead in 41134c91c07cSLorenzo Stoakes * this access function. 4114d0107eb0SKAMEZAWA Hiroyuki */ 41154c91c07cSLorenzo Stoakes if (page) 41164c91c07cSLorenzo Stoakes copied = copy_page_to_iter_nofault(page, offset, 41174c91c07cSLorenzo Stoakes length, iter); 41184c91c07cSLorenzo Stoakes else 41194c91c07cSLorenzo Stoakes copied = zero_iter(iter, length); 4120d0107eb0SKAMEZAWA Hiroyuki 41214c91c07cSLorenzo Stoakes addr += copied; 41224c91c07cSLorenzo Stoakes remains -= copied; 41234c91c07cSLorenzo Stoakes 41244c91c07cSLorenzo Stoakes if (copied != length) 41254c91c07cSLorenzo Stoakes break; 4126d0107eb0SKAMEZAWA Hiroyuki } 4127d0107eb0SKAMEZAWA Hiroyuki 41284c91c07cSLorenzo Stoakes return count - remains; 41294c91c07cSLorenzo Stoakes } 41304c91c07cSLorenzo Stoakes 41314c91c07cSLorenzo Stoakes /* 41324c91c07cSLorenzo Stoakes * Read from a vm_map_ram region of memory. 41334c91c07cSLorenzo Stoakes * 41344c91c07cSLorenzo Stoakes * Returns the number of copied bytes. 41354c91c07cSLorenzo Stoakes */ 41364c91c07cSLorenzo Stoakes static size_t vmap_ram_vread_iter(struct iov_iter *iter, const char *addr, 41374c91c07cSLorenzo Stoakes size_t count, unsigned long flags) 413806c89946SBaoquan He { 413906c89946SBaoquan He char *start; 414006c89946SBaoquan He struct vmap_block *vb; 4141062eacf5SUladzislau Rezki (Sony) struct xarray *xa; 414206c89946SBaoquan He unsigned long offset; 41434c91c07cSLorenzo Stoakes unsigned int rs, re; 41444c91c07cSLorenzo Stoakes size_t remains, n; 414506c89946SBaoquan He 414606c89946SBaoquan He /* 414706c89946SBaoquan He * If it's area created by vm_map_ram() interface directly, but 414806c89946SBaoquan He * not further subdividing and delegating management to vmap_block, 414906c89946SBaoquan He * handle it here. 415006c89946SBaoquan He */ 41514c91c07cSLorenzo Stoakes if (!(flags & VMAP_BLOCK)) 41524c91c07cSLorenzo Stoakes return aligned_vread_iter(iter, addr, count); 41534c91c07cSLorenzo Stoakes 41544c91c07cSLorenzo Stoakes remains = count; 415506c89946SBaoquan He 415606c89946SBaoquan He /* 415706c89946SBaoquan He * Area is split into regions and tracked with vmap_block, read out 415806c89946SBaoquan He * each region and zero fill the hole between regions. 415906c89946SBaoquan He */ 4160fa1c77c1SUladzislau Rezki (Sony) xa = addr_to_vb_xa((unsigned long) addr); 4161062eacf5SUladzislau Rezki (Sony) vb = xa_load(xa, addr_to_vb_idx((unsigned long)addr)); 416206c89946SBaoquan He if (!vb) 41634c91c07cSLorenzo Stoakes goto finished_zero; 416406c89946SBaoquan He 416506c89946SBaoquan He spin_lock(&vb->lock); 416606c89946SBaoquan He if (bitmap_empty(vb->used_map, VMAP_BBMAP_BITS)) { 416706c89946SBaoquan He spin_unlock(&vb->lock); 41684c91c07cSLorenzo Stoakes goto finished_zero; 41694c91c07cSLorenzo Stoakes } 41704c91c07cSLorenzo Stoakes 41714c91c07cSLorenzo Stoakes for_each_set_bitrange(rs, re, vb->used_map, VMAP_BBMAP_BITS) { 41724c91c07cSLorenzo Stoakes size_t copied; 41734c91c07cSLorenzo Stoakes 41744c91c07cSLorenzo Stoakes if (remains == 0) 41754c91c07cSLorenzo Stoakes goto finished; 41764c91c07cSLorenzo Stoakes 41774c91c07cSLorenzo Stoakes start = vmap_block_vaddr(vb->va->va_start, rs); 41784c91c07cSLorenzo Stoakes 41794c91c07cSLorenzo Stoakes if (addr < start) { 41804c91c07cSLorenzo Stoakes size_t to_zero = min_t(size_t, start - addr, remains); 41814c91c07cSLorenzo Stoakes size_t zeroed = zero_iter(iter, to_zero); 41824c91c07cSLorenzo Stoakes 41834c91c07cSLorenzo Stoakes addr += zeroed; 41844c91c07cSLorenzo Stoakes remains -= zeroed; 41854c91c07cSLorenzo Stoakes 41864c91c07cSLorenzo Stoakes if (remains == 0 || zeroed != to_zero) 418706c89946SBaoquan He goto finished; 418806c89946SBaoquan He } 41894c91c07cSLorenzo Stoakes 419006c89946SBaoquan He /*it could start reading from the middle of used region*/ 419106c89946SBaoquan He offset = offset_in_page(addr); 419206c89946SBaoquan He n = ((re - rs + 1) << PAGE_SHIFT) - offset; 41934c91c07cSLorenzo Stoakes if (n > remains) 41944c91c07cSLorenzo Stoakes n = remains; 419506c89946SBaoquan He 41964c91c07cSLorenzo Stoakes copied = aligned_vread_iter(iter, start + offset, n); 41974c91c07cSLorenzo Stoakes 41984c91c07cSLorenzo Stoakes addr += copied; 41994c91c07cSLorenzo Stoakes remains -= copied; 42004c91c07cSLorenzo Stoakes 42014c91c07cSLorenzo Stoakes if (copied != n) 42024c91c07cSLorenzo Stoakes goto finished; 420306c89946SBaoquan He } 42044c91c07cSLorenzo Stoakes 420506c89946SBaoquan He spin_unlock(&vb->lock); 420606c89946SBaoquan He 42074c91c07cSLorenzo Stoakes finished_zero: 420806c89946SBaoquan He /* zero-fill the left dirty or free regions */ 42094c91c07cSLorenzo Stoakes return count - remains + zero_iter(iter, remains); 42104c91c07cSLorenzo Stoakes finished: 42114c91c07cSLorenzo Stoakes /* We couldn't copy/zero everything */ 42124c91c07cSLorenzo Stoakes spin_unlock(&vb->lock); 42134c91c07cSLorenzo Stoakes return count - remains; 421406c89946SBaoquan He } 421506c89946SBaoquan He 4216d0107eb0SKAMEZAWA Hiroyuki /** 42174c91c07cSLorenzo Stoakes * vread_iter() - read vmalloc area in a safe way to an iterator. 42184c91c07cSLorenzo Stoakes * @iter: the iterator to which data should be written. 4219d0107eb0SKAMEZAWA Hiroyuki * @addr: vm address. 4220d0107eb0SKAMEZAWA Hiroyuki * @count: number of bytes to be read. 4221d0107eb0SKAMEZAWA Hiroyuki * 4222d0107eb0SKAMEZAWA Hiroyuki * This function checks that addr is a valid vmalloc'ed area, and 4223d0107eb0SKAMEZAWA Hiroyuki * copy data from that area to a given buffer. If the given memory range 4224d0107eb0SKAMEZAWA Hiroyuki * of [addr...addr+count) includes some valid address, data is copied to 4225d0107eb0SKAMEZAWA Hiroyuki * proper area of @buf. If there are memory holes, they'll be zero-filled. 4226d0107eb0SKAMEZAWA Hiroyuki * IOREMAP area is treated as memory hole and no copy is done. 4227d0107eb0SKAMEZAWA Hiroyuki * 4228d0107eb0SKAMEZAWA Hiroyuki * If [addr...addr+count) doesn't includes any intersects with alive 4229a8e5202dSCong Wang * vm_struct area, returns 0. @buf should be kernel's buffer. 4230d0107eb0SKAMEZAWA Hiroyuki * 4231d0107eb0SKAMEZAWA Hiroyuki * Note: In usual ops, vread() is never necessary because the caller 4232d0107eb0SKAMEZAWA Hiroyuki * should know vmalloc() area is valid and can use memcpy(). 4233d0107eb0SKAMEZAWA Hiroyuki * This is for routines which have to access vmalloc area without 4234bbcd53c9SDavid Hildenbrand * any information, as /proc/kcore. 4235a862f68aSMike Rapoport * 4236a862f68aSMike Rapoport * Return: number of bytes for which addr and buf should be increased 4237a862f68aSMike Rapoport * (same number as @count) or %0 if [addr...addr+count) doesn't 4238a862f68aSMike Rapoport * include any intersection with valid vmalloc area 4239d0107eb0SKAMEZAWA Hiroyuki */ 42404c91c07cSLorenzo Stoakes long vread_iter(struct iov_iter *iter, const char *addr, size_t count) 42411da177e4SLinus Torvalds { 4242d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 4243e81ce85fSJoonsoo Kim struct vmap_area *va; 4244e81ce85fSJoonsoo Kim struct vm_struct *vm; 42454c91c07cSLorenzo Stoakes char *vaddr; 42464c91c07cSLorenzo Stoakes size_t n, size, flags, remains; 424753becf32SUladzislau Rezki (Sony) unsigned long next; 42481da177e4SLinus Torvalds 42494aff1dc4SAndrey Konovalov addr = kasan_reset_tag(addr); 42504aff1dc4SAndrey Konovalov 42511da177e4SLinus Torvalds /* Don't allow overflow */ 42521da177e4SLinus Torvalds if ((unsigned long) addr + count < count) 42531da177e4SLinus Torvalds count = -(unsigned long) addr; 42541da177e4SLinus Torvalds 42554c91c07cSLorenzo Stoakes remains = count; 42564c91c07cSLorenzo Stoakes 425753becf32SUladzislau Rezki (Sony) vn = find_vmap_area_exceed_addr_lock((unsigned long) addr, &va); 425853becf32SUladzislau Rezki (Sony) if (!vn) 42594c91c07cSLorenzo Stoakes goto finished_zero; 4260f181234aSChen Wandun 4261f181234aSChen Wandun /* no intersects with alive vmap_area */ 42624c91c07cSLorenzo Stoakes if ((unsigned long)addr + remains <= va->va_start) 42634c91c07cSLorenzo Stoakes goto finished_zero; 4264f181234aSChen Wandun 426553becf32SUladzislau Rezki (Sony) do { 42664c91c07cSLorenzo Stoakes size_t copied; 42674c91c07cSLorenzo Stoakes 42684c91c07cSLorenzo Stoakes if (remains == 0) 42694c91c07cSLorenzo Stoakes goto finished; 4270e81ce85fSJoonsoo Kim 427106c89946SBaoquan He vm = va->vm; 427206c89946SBaoquan He flags = va->flags & VMAP_FLAGS_MASK; 427306c89946SBaoquan He /* 427406c89946SBaoquan He * VMAP_BLOCK indicates a sub-type of vm_map_ram area, need 427506c89946SBaoquan He * be set together with VMAP_RAM. 427606c89946SBaoquan He */ 427706c89946SBaoquan He WARN_ON(flags == VMAP_BLOCK); 427806c89946SBaoquan He 427906c89946SBaoquan He if (!vm && !flags) 428053becf32SUladzislau Rezki (Sony) goto next_va; 4281e81ce85fSJoonsoo Kim 428230a7a9b1SBaoquan He if (vm && (vm->flags & VM_UNINITIALIZED)) 428353becf32SUladzislau Rezki (Sony) goto next_va; 42844c91c07cSLorenzo Stoakes 428530a7a9b1SBaoquan He /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ 428630a7a9b1SBaoquan He smp_rmb(); 428730a7a9b1SBaoquan He 428806c89946SBaoquan He vaddr = (char *) va->va_start; 428906c89946SBaoquan He size = vm ? get_vm_area_size(vm) : va_size(va); 429006c89946SBaoquan He 429106c89946SBaoquan He if (addr >= vaddr + size) 429253becf32SUladzislau Rezki (Sony) goto next_va; 42934c91c07cSLorenzo Stoakes 42944c91c07cSLorenzo Stoakes if (addr < vaddr) { 42954c91c07cSLorenzo Stoakes size_t to_zero = min_t(size_t, vaddr - addr, remains); 42964c91c07cSLorenzo Stoakes size_t zeroed = zero_iter(iter, to_zero); 42974c91c07cSLorenzo Stoakes 42984c91c07cSLorenzo Stoakes addr += zeroed; 42994c91c07cSLorenzo Stoakes remains -= zeroed; 43004c91c07cSLorenzo Stoakes 43014c91c07cSLorenzo Stoakes if (remains == 0 || zeroed != to_zero) 43021da177e4SLinus Torvalds goto finished; 43031da177e4SLinus Torvalds } 43044c91c07cSLorenzo Stoakes 430506c89946SBaoquan He n = vaddr + size - addr; 43064c91c07cSLorenzo Stoakes if (n > remains) 43074c91c07cSLorenzo Stoakes n = remains; 430806c89946SBaoquan He 430906c89946SBaoquan He if (flags & VMAP_RAM) 43104c91c07cSLorenzo Stoakes copied = vmap_ram_vread_iter(iter, addr, n, flags); 4311e6f79822SAlexei Starovoitov else if (!(vm && (vm->flags & (VM_IOREMAP | VM_SPARSE)))) 43124c91c07cSLorenzo Stoakes copied = aligned_vread_iter(iter, addr, n); 4313e6f79822SAlexei Starovoitov else /* IOREMAP | SPARSE area is treated as memory hole */ 43144c91c07cSLorenzo Stoakes copied = zero_iter(iter, n); 43154c91c07cSLorenzo Stoakes 43164c91c07cSLorenzo Stoakes addr += copied; 43174c91c07cSLorenzo Stoakes remains -= copied; 43184c91c07cSLorenzo Stoakes 43194c91c07cSLorenzo Stoakes if (copied != n) 43204c91c07cSLorenzo Stoakes goto finished; 432153becf32SUladzislau Rezki (Sony) 432253becf32SUladzislau Rezki (Sony) next_va: 432353becf32SUladzislau Rezki (Sony) next = va->va_end; 432453becf32SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 432553becf32SUladzislau Rezki (Sony) } while ((vn = find_vmap_area_exceed_addr_lock(next, &va))); 43264c91c07cSLorenzo Stoakes 43274c91c07cSLorenzo Stoakes finished_zero: 432853becf32SUladzislau Rezki (Sony) if (vn) 4329d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 433053becf32SUladzislau Rezki (Sony) 43314c91c07cSLorenzo Stoakes /* zero-fill memory holes */ 43324c91c07cSLorenzo Stoakes return count - remains + zero_iter(iter, remains); 43331da177e4SLinus Torvalds finished: 43344c91c07cSLorenzo Stoakes /* Nothing remains, or We couldn't copy/zero everything. */ 433553becf32SUladzislau Rezki (Sony) if (vn) 4336d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 4337d0107eb0SKAMEZAWA Hiroyuki 43384c91c07cSLorenzo Stoakes return count - remains; 43391da177e4SLinus Torvalds } 43401da177e4SLinus Torvalds 4341d0107eb0SKAMEZAWA Hiroyuki /** 4342e69e9d4aSHATAYAMA Daisuke * remap_vmalloc_range_partial - map vmalloc pages to userspace 4343e69e9d4aSHATAYAMA Daisuke * @vma: vma to cover 4344e69e9d4aSHATAYAMA Daisuke * @uaddr: target user address to start at 4345e69e9d4aSHATAYAMA Daisuke * @kaddr: virtual address of vmalloc kernel memory 4346bdebd6a2SJann Horn * @pgoff: offset from @kaddr to start at 4347e69e9d4aSHATAYAMA Daisuke * @size: size of map area 4348e69e9d4aSHATAYAMA Daisuke * 4349e69e9d4aSHATAYAMA Daisuke * Returns: 0 for success, -Exxx on failure 4350e69e9d4aSHATAYAMA Daisuke * 4351e69e9d4aSHATAYAMA Daisuke * This function checks that @kaddr is a valid vmalloc'ed area, 4352e69e9d4aSHATAYAMA Daisuke * and that it is big enough to cover the range starting at 4353e69e9d4aSHATAYAMA Daisuke * @uaddr in @vma. Will return failure if that criteria isn't 4354e69e9d4aSHATAYAMA Daisuke * met. 4355e69e9d4aSHATAYAMA Daisuke * 4356e69e9d4aSHATAYAMA Daisuke * Similar to remap_pfn_range() (see mm/memory.c) 4357e69e9d4aSHATAYAMA Daisuke */ 4358e69e9d4aSHATAYAMA Daisuke int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr, 4359bdebd6a2SJann Horn void *kaddr, unsigned long pgoff, 4360bdebd6a2SJann Horn unsigned long size) 4361e69e9d4aSHATAYAMA Daisuke { 4362e69e9d4aSHATAYAMA Daisuke struct vm_struct *area; 4363bdebd6a2SJann Horn unsigned long off; 4364bdebd6a2SJann Horn unsigned long end_index; 4365bdebd6a2SJann Horn 4366bdebd6a2SJann Horn if (check_shl_overflow(pgoff, PAGE_SHIFT, &off)) 4367bdebd6a2SJann Horn return -EINVAL; 4368e69e9d4aSHATAYAMA Daisuke 4369e69e9d4aSHATAYAMA Daisuke size = PAGE_ALIGN(size); 4370e69e9d4aSHATAYAMA Daisuke 4371e69e9d4aSHATAYAMA Daisuke if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr)) 4372e69e9d4aSHATAYAMA Daisuke return -EINVAL; 4373e69e9d4aSHATAYAMA Daisuke 4374e69e9d4aSHATAYAMA Daisuke area = find_vm_area(kaddr); 4375e69e9d4aSHATAYAMA Daisuke if (!area) 4376e69e9d4aSHATAYAMA Daisuke return -EINVAL; 4377e69e9d4aSHATAYAMA Daisuke 4378fe9041c2SChristoph Hellwig if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT))) 4379e69e9d4aSHATAYAMA Daisuke return -EINVAL; 4380e69e9d4aSHATAYAMA Daisuke 4381bdebd6a2SJann Horn if (check_add_overflow(size, off, &end_index) || 4382bdebd6a2SJann Horn end_index > get_vm_area_size(area)) 4383e69e9d4aSHATAYAMA Daisuke return -EINVAL; 4384bdebd6a2SJann Horn kaddr += off; 4385e69e9d4aSHATAYAMA Daisuke 4386e69e9d4aSHATAYAMA Daisuke do { 4387e69e9d4aSHATAYAMA Daisuke struct page *page = vmalloc_to_page(kaddr); 4388e69e9d4aSHATAYAMA Daisuke int ret; 4389e69e9d4aSHATAYAMA Daisuke 4390e69e9d4aSHATAYAMA Daisuke ret = vm_insert_page(vma, uaddr, page); 4391e69e9d4aSHATAYAMA Daisuke if (ret) 4392e69e9d4aSHATAYAMA Daisuke return ret; 4393e69e9d4aSHATAYAMA Daisuke 4394e69e9d4aSHATAYAMA Daisuke uaddr += PAGE_SIZE; 4395e69e9d4aSHATAYAMA Daisuke kaddr += PAGE_SIZE; 4396e69e9d4aSHATAYAMA Daisuke size -= PAGE_SIZE; 4397e69e9d4aSHATAYAMA Daisuke } while (size > 0); 4398e69e9d4aSHATAYAMA Daisuke 43991c71222eSSuren Baghdasaryan vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); 4400e69e9d4aSHATAYAMA Daisuke 4401e69e9d4aSHATAYAMA Daisuke return 0; 4402e69e9d4aSHATAYAMA Daisuke } 4403e69e9d4aSHATAYAMA Daisuke 4404e69e9d4aSHATAYAMA Daisuke /** 440583342314SNick Piggin * remap_vmalloc_range - map vmalloc pages to userspace 440683342314SNick Piggin * @vma: vma to cover (map full range of vma) 440783342314SNick Piggin * @addr: vmalloc memory 440883342314SNick Piggin * @pgoff: number of pages into addr before first page to map 44097682486bSRandy Dunlap * 44107682486bSRandy Dunlap * Returns: 0 for success, -Exxx on failure 441183342314SNick Piggin * 441283342314SNick Piggin * This function checks that addr is a valid vmalloc'ed area, and 441383342314SNick Piggin * that it is big enough to cover the vma. Will return failure if 441483342314SNick Piggin * that criteria isn't met. 441583342314SNick Piggin * 441672fd4a35SRobert P. J. Day * Similar to remap_pfn_range() (see mm/memory.c) 441783342314SNick Piggin */ 441883342314SNick Piggin int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 441983342314SNick Piggin unsigned long pgoff) 442083342314SNick Piggin { 4421e69e9d4aSHATAYAMA Daisuke return remap_vmalloc_range_partial(vma, vma->vm_start, 4422bdebd6a2SJann Horn addr, pgoff, 4423e69e9d4aSHATAYAMA Daisuke vma->vm_end - vma->vm_start); 442483342314SNick Piggin } 442583342314SNick Piggin EXPORT_SYMBOL(remap_vmalloc_range); 442683342314SNick Piggin 44275f4352fbSJeremy Fitzhardinge void free_vm_area(struct vm_struct *area) 44285f4352fbSJeremy Fitzhardinge { 44295f4352fbSJeremy Fitzhardinge struct vm_struct *ret; 44305f4352fbSJeremy Fitzhardinge ret = remove_vm_area(area->addr); 44315f4352fbSJeremy Fitzhardinge BUG_ON(ret != area); 44325f4352fbSJeremy Fitzhardinge kfree(area); 44335f4352fbSJeremy Fitzhardinge } 44345f4352fbSJeremy Fitzhardinge EXPORT_SYMBOL_GPL(free_vm_area); 4435a10aa579SChristoph Lameter 44364f8b02b4STejun Heo #ifdef CONFIG_SMP 4437ca23e405STejun Heo static struct vmap_area *node_to_va(struct rb_node *n) 4438ca23e405STejun Heo { 44394583e773SGeliang Tang return rb_entry_safe(n, struct vmap_area, rb_node); 4440ca23e405STejun Heo } 4441ca23e405STejun Heo 4442ca23e405STejun Heo /** 444368ad4a33SUladzislau Rezki (Sony) * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to 444468ad4a33SUladzislau Rezki (Sony) * @addr: target address 4445ca23e405STejun Heo * 444668ad4a33SUladzislau Rezki (Sony) * Returns: vmap_area if it is found. If there is no such area 444768ad4a33SUladzislau Rezki (Sony) * the first highest(reverse order) vmap_area is returned 444868ad4a33SUladzislau Rezki (Sony) * i.e. va->va_start < addr && va->va_end < addr or NULL 444968ad4a33SUladzislau Rezki (Sony) * if there are no any areas before @addr. 4450ca23e405STejun Heo */ 445168ad4a33SUladzislau Rezki (Sony) static struct vmap_area * 445268ad4a33SUladzislau Rezki (Sony) pvm_find_va_enclose_addr(unsigned long addr) 4453ca23e405STejun Heo { 445468ad4a33SUladzislau Rezki (Sony) struct vmap_area *va, *tmp; 445568ad4a33SUladzislau Rezki (Sony) struct rb_node *n; 445668ad4a33SUladzislau Rezki (Sony) 445768ad4a33SUladzislau Rezki (Sony) n = free_vmap_area_root.rb_node; 445868ad4a33SUladzislau Rezki (Sony) va = NULL; 4459ca23e405STejun Heo 4460ca23e405STejun Heo while (n) { 446168ad4a33SUladzislau Rezki (Sony) tmp = rb_entry(n, struct vmap_area, rb_node); 446268ad4a33SUladzislau Rezki (Sony) if (tmp->va_start <= addr) { 446368ad4a33SUladzislau Rezki (Sony) va = tmp; 446468ad4a33SUladzislau Rezki (Sony) if (tmp->va_end >= addr) 4465ca23e405STejun Heo break; 4466ca23e405STejun Heo 446768ad4a33SUladzislau Rezki (Sony) n = n->rb_right; 4468ca23e405STejun Heo } else { 446968ad4a33SUladzislau Rezki (Sony) n = n->rb_left; 4470ca23e405STejun Heo } 447168ad4a33SUladzislau Rezki (Sony) } 447268ad4a33SUladzislau Rezki (Sony) 447368ad4a33SUladzislau Rezki (Sony) return va; 4474ca23e405STejun Heo } 4475ca23e405STejun Heo 4476ca23e405STejun Heo /** 447768ad4a33SUladzislau Rezki (Sony) * pvm_determine_end_from_reverse - find the highest aligned address 447868ad4a33SUladzislau Rezki (Sony) * of free block below VMALLOC_END 447968ad4a33SUladzislau Rezki (Sony) * @va: 448068ad4a33SUladzislau Rezki (Sony) * in - the VA we start the search(reverse order); 448168ad4a33SUladzislau Rezki (Sony) * out - the VA with the highest aligned end address. 4482799fa85dSAlex Shi * @align: alignment for required highest address 4483ca23e405STejun Heo * 448468ad4a33SUladzislau Rezki (Sony) * Returns: determined end address within vmap_area 4485ca23e405STejun Heo */ 448668ad4a33SUladzislau Rezki (Sony) static unsigned long 448768ad4a33SUladzislau Rezki (Sony) pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) 4488ca23e405STejun Heo { 448968ad4a33SUladzislau Rezki (Sony) unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 4490ca23e405STejun Heo unsigned long addr; 4491ca23e405STejun Heo 449268ad4a33SUladzislau Rezki (Sony) if (likely(*va)) { 449368ad4a33SUladzislau Rezki (Sony) list_for_each_entry_from_reverse((*va), 449468ad4a33SUladzislau Rezki (Sony) &free_vmap_area_list, list) { 449568ad4a33SUladzislau Rezki (Sony) addr = min((*va)->va_end & ~(align - 1), vmalloc_end); 449668ad4a33SUladzislau Rezki (Sony) if ((*va)->va_start < addr) 449768ad4a33SUladzislau Rezki (Sony) return addr; 449868ad4a33SUladzislau Rezki (Sony) } 4499ca23e405STejun Heo } 4500ca23e405STejun Heo 450168ad4a33SUladzislau Rezki (Sony) return 0; 4502ca23e405STejun Heo } 4503ca23e405STejun Heo 4504ca23e405STejun Heo /** 4505ca23e405STejun Heo * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator 4506ca23e405STejun Heo * @offsets: array containing offset of each area 4507ca23e405STejun Heo * @sizes: array containing size of each area 4508ca23e405STejun Heo * @nr_vms: the number of areas to allocate 4509ca23e405STejun Heo * @align: alignment, all entries in @offsets and @sizes must be aligned to this 4510ca23e405STejun Heo * 4511ca23e405STejun Heo * Returns: kmalloc'd vm_struct pointer array pointing to allocated 4512ca23e405STejun Heo * vm_structs on success, %NULL on failure 4513ca23e405STejun Heo * 4514ca23e405STejun Heo * Percpu allocator wants to use congruent vm areas so that it can 4515ca23e405STejun Heo * maintain the offsets among percpu areas. This function allocates 4516ec3f64fcSDavid Rientjes * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to 4517ec3f64fcSDavid Rientjes * be scattered pretty far, distance between two areas easily going up 4518ec3f64fcSDavid Rientjes * to gigabytes. To avoid interacting with regular vmallocs, these 4519ec3f64fcSDavid Rientjes * areas are allocated from top. 4520ca23e405STejun Heo * 4521ca23e405STejun Heo * Despite its complicated look, this allocator is rather simple. It 452268ad4a33SUladzislau Rezki (Sony) * does everything top-down and scans free blocks from the end looking 452368ad4a33SUladzislau Rezki (Sony) * for matching base. While scanning, if any of the areas do not fit the 452468ad4a33SUladzislau Rezki (Sony) * base address is pulled down to fit the area. Scanning is repeated till 452568ad4a33SUladzislau Rezki (Sony) * all the areas fit and then all necessary data structures are inserted 452668ad4a33SUladzislau Rezki (Sony) * and the result is returned. 4527ca23e405STejun Heo */ 4528ca23e405STejun Heo struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, 4529ca23e405STejun Heo const size_t *sizes, int nr_vms, 4530ec3f64fcSDavid Rientjes size_t align) 4531ca23e405STejun Heo { 4532ca23e405STejun Heo const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); 4533ca23e405STejun Heo const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 453468ad4a33SUladzislau Rezki (Sony) struct vmap_area **vas, *va; 4535ca23e405STejun Heo struct vm_struct **vms; 4536ca23e405STejun Heo int area, area2, last_area, term_area; 4537253a496dSDaniel Axtens unsigned long base, start, size, end, last_end, orig_start, orig_end; 4538ca23e405STejun Heo bool purged = false; 4539ca23e405STejun Heo 4540ca23e405STejun Heo /* verify parameters and allocate data structures */ 4541891c49abSAlexander Kuleshov BUG_ON(offset_in_page(align) || !is_power_of_2(align)); 4542ca23e405STejun Heo for (last_area = 0, area = 0; area < nr_vms; area++) { 4543ca23e405STejun Heo start = offsets[area]; 4544ca23e405STejun Heo end = start + sizes[area]; 4545ca23e405STejun Heo 4546ca23e405STejun Heo /* is everything aligned properly? */ 4547ca23e405STejun Heo BUG_ON(!IS_ALIGNED(offsets[area], align)); 4548ca23e405STejun Heo BUG_ON(!IS_ALIGNED(sizes[area], align)); 4549ca23e405STejun Heo 4550ca23e405STejun Heo /* detect the area with the highest address */ 4551ca23e405STejun Heo if (start > offsets[last_area]) 4552ca23e405STejun Heo last_area = area; 4553ca23e405STejun Heo 4554c568da28SWei Yang for (area2 = area + 1; area2 < nr_vms; area2++) { 4555ca23e405STejun Heo unsigned long start2 = offsets[area2]; 4556ca23e405STejun Heo unsigned long end2 = start2 + sizes[area2]; 4557ca23e405STejun Heo 4558c568da28SWei Yang BUG_ON(start2 < end && start < end2); 4559ca23e405STejun Heo } 4560ca23e405STejun Heo } 4561ca23e405STejun Heo last_end = offsets[last_area] + sizes[last_area]; 4562ca23e405STejun Heo 4563ca23e405STejun Heo if (vmalloc_end - vmalloc_start < last_end) { 4564ca23e405STejun Heo WARN_ON(true); 4565ca23e405STejun Heo return NULL; 4566ca23e405STejun Heo } 4567ca23e405STejun Heo 45684d67d860SThomas Meyer vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL); 45694d67d860SThomas Meyer vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL); 4570ca23e405STejun Heo if (!vas || !vms) 4571f1db7afdSKautuk Consul goto err_free2; 4572ca23e405STejun Heo 4573ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 457468ad4a33SUladzislau Rezki (Sony) vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL); 4575ec3f64fcSDavid Rientjes vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL); 4576ca23e405STejun Heo if (!vas[area] || !vms[area]) 4577ca23e405STejun Heo goto err_free; 4578ca23e405STejun Heo } 4579ca23e405STejun Heo retry: 4580e36176beSUladzislau Rezki (Sony) spin_lock(&free_vmap_area_lock); 4581ca23e405STejun Heo 4582ca23e405STejun Heo /* start scanning - we scan from the top, begin with the last area */ 4583ca23e405STejun Heo area = term_area = last_area; 4584ca23e405STejun Heo start = offsets[area]; 4585ca23e405STejun Heo end = start + sizes[area]; 4586ca23e405STejun Heo 458768ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(vmalloc_end); 458868ad4a33SUladzislau Rezki (Sony) base = pvm_determine_end_from_reverse(&va, align) - end; 4589ca23e405STejun Heo 4590ca23e405STejun Heo while (true) { 4591ca23e405STejun Heo /* 4592ca23e405STejun Heo * base might have underflowed, add last_end before 4593ca23e405STejun Heo * comparing. 4594ca23e405STejun Heo */ 459568ad4a33SUladzislau Rezki (Sony) if (base + last_end < vmalloc_start + last_end) 459668ad4a33SUladzislau Rezki (Sony) goto overflow; 4597ca23e405STejun Heo 4598ca23e405STejun Heo /* 459968ad4a33SUladzislau Rezki (Sony) * Fitting base has not been found. 4600ca23e405STejun Heo */ 460168ad4a33SUladzislau Rezki (Sony) if (va == NULL) 460268ad4a33SUladzislau Rezki (Sony) goto overflow; 4603ca23e405STejun Heo 4604ca23e405STejun Heo /* 4605d8cc323dSQiujun Huang * If required width exceeds current VA block, move 46065336e52cSKuppuswamy Sathyanarayanan * base downwards and then recheck. 46075336e52cSKuppuswamy Sathyanarayanan */ 46085336e52cSKuppuswamy Sathyanarayanan if (base + end > va->va_end) { 46095336e52cSKuppuswamy Sathyanarayanan base = pvm_determine_end_from_reverse(&va, align) - end; 46105336e52cSKuppuswamy Sathyanarayanan term_area = area; 46115336e52cSKuppuswamy Sathyanarayanan continue; 46125336e52cSKuppuswamy Sathyanarayanan } 46135336e52cSKuppuswamy Sathyanarayanan 46145336e52cSKuppuswamy Sathyanarayanan /* 461568ad4a33SUladzislau Rezki (Sony) * If this VA does not fit, move base downwards and recheck. 4616ca23e405STejun Heo */ 46175336e52cSKuppuswamy Sathyanarayanan if (base + start < va->va_start) { 461868ad4a33SUladzislau Rezki (Sony) va = node_to_va(rb_prev(&va->rb_node)); 461968ad4a33SUladzislau Rezki (Sony) base = pvm_determine_end_from_reverse(&va, align) - end; 4620ca23e405STejun Heo term_area = area; 4621ca23e405STejun Heo continue; 4622ca23e405STejun Heo } 4623ca23e405STejun Heo 4624ca23e405STejun Heo /* 4625ca23e405STejun Heo * This area fits, move on to the previous one. If 4626ca23e405STejun Heo * the previous one is the terminal one, we're done. 4627ca23e405STejun Heo */ 4628ca23e405STejun Heo area = (area + nr_vms - 1) % nr_vms; 4629ca23e405STejun Heo if (area == term_area) 4630ca23e405STejun Heo break; 463168ad4a33SUladzislau Rezki (Sony) 4632ca23e405STejun Heo start = offsets[area]; 4633ca23e405STejun Heo end = start + sizes[area]; 463468ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(base + end); 4635ca23e405STejun Heo } 463668ad4a33SUladzislau Rezki (Sony) 4637ca23e405STejun Heo /* we've found a fitting base, insert all va's */ 4638ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 463968ad4a33SUladzislau Rezki (Sony) int ret; 4640ca23e405STejun Heo 464168ad4a33SUladzislau Rezki (Sony) start = base + offsets[area]; 464268ad4a33SUladzislau Rezki (Sony) size = sizes[area]; 464368ad4a33SUladzislau Rezki (Sony) 464468ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(start); 464568ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(va == NULL)) 464668ad4a33SUladzislau Rezki (Sony) /* It is a BUG(), but trigger recovery instead. */ 464768ad4a33SUladzislau Rezki (Sony) goto recovery; 464868ad4a33SUladzislau Rezki (Sony) 46495b75b8e1SUladzislau Rezki (Sony) ret = va_clip(&free_vmap_area_root, 46505b75b8e1SUladzislau Rezki (Sony) &free_vmap_area_list, va, start, size); 46511b23ff80SBaoquan He if (WARN_ON_ONCE(unlikely(ret))) 465268ad4a33SUladzislau Rezki (Sony) /* It is a BUG(), but trigger recovery instead. */ 465368ad4a33SUladzislau Rezki (Sony) goto recovery; 465468ad4a33SUladzislau Rezki (Sony) 465568ad4a33SUladzislau Rezki (Sony) /* Allocated area. */ 465668ad4a33SUladzislau Rezki (Sony) va = vas[area]; 465768ad4a33SUladzislau Rezki (Sony) va->va_start = start; 465868ad4a33SUladzislau Rezki (Sony) va->va_end = start + size; 4659ca23e405STejun Heo } 4660ca23e405STejun Heo 4661e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 4662ca23e405STejun Heo 4663253a496dSDaniel Axtens /* populate the kasan shadow space */ 4664253a496dSDaniel Axtens for (area = 0; area < nr_vms; area++) { 4665253a496dSDaniel Axtens if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area])) 4666253a496dSDaniel Axtens goto err_free_shadow; 4667253a496dSDaniel Axtens } 4668253a496dSDaniel Axtens 4669ca23e405STejun Heo /* insert all vm's */ 4670e36176beSUladzislau Rezki (Sony) for (area = 0; area < nr_vms; area++) { 4671d0936029SUladzislau Rezki (Sony) struct vmap_node *vn = addr_to_node(vas[area]->va_start); 4672e36176beSUladzislau Rezki (Sony) 4673d0936029SUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 4674d0936029SUladzislau Rezki (Sony) insert_vmap_area(vas[area], &vn->busy.root, &vn->busy.head); 4675e36176beSUladzislau Rezki (Sony) setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC, 4676ca23e405STejun Heo pcpu_get_vm_areas); 4677d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 4678e36176beSUladzislau Rezki (Sony) } 4679ca23e405STejun Heo 468019f1c3acSAndrey Konovalov /* 468119f1c3acSAndrey Konovalov * Mark allocated areas as accessible. Do it now as a best-effort 468219f1c3acSAndrey Konovalov * approach, as they can be mapped outside of vmalloc code. 468323689e91SAndrey Konovalov * With hardware tag-based KASAN, marking is skipped for 468423689e91SAndrey Konovalov * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 468519f1c3acSAndrey Konovalov */ 46861d96320fSAndrey Konovalov for (area = 0; area < nr_vms; area++) 46871d96320fSAndrey Konovalov vms[area]->addr = kasan_unpoison_vmalloc(vms[area]->addr, 4688f6e39794SAndrey Konovalov vms[area]->size, KASAN_VMALLOC_PROT_NORMAL); 46891d96320fSAndrey Konovalov 4690ca23e405STejun Heo kfree(vas); 4691ca23e405STejun Heo return vms; 4692ca23e405STejun Heo 469368ad4a33SUladzislau Rezki (Sony) recovery: 4694e36176beSUladzislau Rezki (Sony) /* 4695e36176beSUladzislau Rezki (Sony) * Remove previously allocated areas. There is no 4696e36176beSUladzislau Rezki (Sony) * need in removing these areas from the busy tree, 4697e36176beSUladzislau Rezki (Sony) * because they are inserted only on the final step 4698e36176beSUladzislau Rezki (Sony) * and when pcpu_get_vm_areas() is success. 4699e36176beSUladzislau Rezki (Sony) */ 470068ad4a33SUladzislau Rezki (Sony) while (area--) { 4701253a496dSDaniel Axtens orig_start = vas[area]->va_start; 4702253a496dSDaniel Axtens orig_end = vas[area]->va_end; 470396e2db45SUladzislau Rezki (Sony) va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, 47043c5c3cfbSDaniel Axtens &free_vmap_area_list); 47059c801f61SUladzislau Rezki (Sony) if (va) 4706253a496dSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 4707253a496dSDaniel Axtens va->va_start, va->va_end); 470868ad4a33SUladzislau Rezki (Sony) vas[area] = NULL; 470968ad4a33SUladzislau Rezki (Sony) } 471068ad4a33SUladzislau Rezki (Sony) 471168ad4a33SUladzislau Rezki (Sony) overflow: 4712e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 471368ad4a33SUladzislau Rezki (Sony) if (!purged) { 471477e50af0SThomas Gleixner reclaim_and_purge_vmap_areas(); 471568ad4a33SUladzislau Rezki (Sony) purged = true; 471668ad4a33SUladzislau Rezki (Sony) 471768ad4a33SUladzislau Rezki (Sony) /* Before "retry", check if we recover. */ 471868ad4a33SUladzislau Rezki (Sony) for (area = 0; area < nr_vms; area++) { 471968ad4a33SUladzislau Rezki (Sony) if (vas[area]) 472068ad4a33SUladzislau Rezki (Sony) continue; 472168ad4a33SUladzislau Rezki (Sony) 472268ad4a33SUladzislau Rezki (Sony) vas[area] = kmem_cache_zalloc( 472368ad4a33SUladzislau Rezki (Sony) vmap_area_cachep, GFP_KERNEL); 472468ad4a33SUladzislau Rezki (Sony) if (!vas[area]) 472568ad4a33SUladzislau Rezki (Sony) goto err_free; 472668ad4a33SUladzislau Rezki (Sony) } 472768ad4a33SUladzislau Rezki (Sony) 472868ad4a33SUladzislau Rezki (Sony) goto retry; 472968ad4a33SUladzislau Rezki (Sony) } 473068ad4a33SUladzislau Rezki (Sony) 4731ca23e405STejun Heo err_free: 4732ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 473368ad4a33SUladzislau Rezki (Sony) if (vas[area]) 473468ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, vas[area]); 473568ad4a33SUladzislau Rezki (Sony) 4736ca23e405STejun Heo kfree(vms[area]); 4737ca23e405STejun Heo } 4738f1db7afdSKautuk Consul err_free2: 4739ca23e405STejun Heo kfree(vas); 4740ca23e405STejun Heo kfree(vms); 4741ca23e405STejun Heo return NULL; 4742253a496dSDaniel Axtens 4743253a496dSDaniel Axtens err_free_shadow: 4744253a496dSDaniel Axtens spin_lock(&free_vmap_area_lock); 4745253a496dSDaniel Axtens /* 4746253a496dSDaniel Axtens * We release all the vmalloc shadows, even the ones for regions that 4747253a496dSDaniel Axtens * hadn't been successfully added. This relies on kasan_release_vmalloc 4748253a496dSDaniel Axtens * being able to tolerate this case. 4749253a496dSDaniel Axtens */ 4750253a496dSDaniel Axtens for (area = 0; area < nr_vms; area++) { 4751253a496dSDaniel Axtens orig_start = vas[area]->va_start; 4752253a496dSDaniel Axtens orig_end = vas[area]->va_end; 475396e2db45SUladzislau Rezki (Sony) va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, 4754253a496dSDaniel Axtens &free_vmap_area_list); 47559c801f61SUladzislau Rezki (Sony) if (va) 4756253a496dSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 4757253a496dSDaniel Axtens va->va_start, va->va_end); 4758253a496dSDaniel Axtens vas[area] = NULL; 4759253a496dSDaniel Axtens kfree(vms[area]); 4760253a496dSDaniel Axtens } 4761253a496dSDaniel Axtens spin_unlock(&free_vmap_area_lock); 4762253a496dSDaniel Axtens kfree(vas); 4763253a496dSDaniel Axtens kfree(vms); 4764253a496dSDaniel Axtens return NULL; 4765ca23e405STejun Heo } 4766ca23e405STejun Heo 4767ca23e405STejun Heo /** 4768ca23e405STejun Heo * pcpu_free_vm_areas - free vmalloc areas for percpu allocator 4769ca23e405STejun Heo * @vms: vm_struct pointer array returned by pcpu_get_vm_areas() 4770ca23e405STejun Heo * @nr_vms: the number of allocated areas 4771ca23e405STejun Heo * 4772ca23e405STejun Heo * Free vm_structs and the array allocated by pcpu_get_vm_areas(). 4773ca23e405STejun Heo */ 4774ca23e405STejun Heo void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) 4775ca23e405STejun Heo { 4776ca23e405STejun Heo int i; 4777ca23e405STejun Heo 4778ca23e405STejun Heo for (i = 0; i < nr_vms; i++) 4779ca23e405STejun Heo free_vm_area(vms[i]); 4780ca23e405STejun Heo kfree(vms); 4781ca23e405STejun Heo } 47824f8b02b4STejun Heo #endif /* CONFIG_SMP */ 4783a10aa579SChristoph Lameter 47845bb1bb35SPaul E. McKenney #ifdef CONFIG_PRINTK 478598f18083SPaul E. McKenney bool vmalloc_dump_obj(void *object) 478698f18083SPaul E. McKenney { 47870818e739SJoel Fernandes (Google) const void *caller; 47880818e739SJoel Fernandes (Google) struct vm_struct *vm; 47890818e739SJoel Fernandes (Google) struct vmap_area *va; 4790d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 47910818e739SJoel Fernandes (Google) unsigned long addr; 47920818e739SJoel Fernandes (Google) unsigned int nr_pages; 479398f18083SPaul E. McKenney 47948be4d46eSUladzislau Rezki (Sony) addr = PAGE_ALIGN((unsigned long) object); 47958be4d46eSUladzislau Rezki (Sony) vn = addr_to_node(addr); 4796d0936029SUladzislau Rezki (Sony) 47978be4d46eSUladzislau Rezki (Sony) if (!spin_trylock(&vn->busy.lock)) 479898f18083SPaul E. McKenney return false; 4799d0936029SUladzislau Rezki (Sony) 48008be4d46eSUladzislau Rezki (Sony) va = __find_vmap_area(addr, &vn->busy.root); 48018be4d46eSUladzislau Rezki (Sony) if (!va || !va->vm) { 4802d0936029SUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 48030818e739SJoel Fernandes (Google) return false; 48040818e739SJoel Fernandes (Google) } 48050818e739SJoel Fernandes (Google) 48060818e739SJoel Fernandes (Google) vm = va->vm; 48070818e739SJoel Fernandes (Google) addr = (unsigned long) vm->addr; 48080818e739SJoel Fernandes (Google) caller = vm->caller; 48090818e739SJoel Fernandes (Google) nr_pages = vm->nr_pages; 48108be4d46eSUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 48118be4d46eSUladzislau Rezki (Sony) 4812bd34dcd4SPaul E. McKenney pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n", 48130818e739SJoel Fernandes (Google) nr_pages, addr, caller); 4814d0936029SUladzislau Rezki (Sony) 481598f18083SPaul E. McKenney return true; 481698f18083SPaul E. McKenney } 48175bb1bb35SPaul E. McKenney #endif 481898f18083SPaul E. McKenney 4819a10aa579SChristoph Lameter #ifdef CONFIG_PROC_FS 4820a47a126aSEric Dumazet static void show_numa_info(struct seq_file *m, struct vm_struct *v) 4821a47a126aSEric Dumazet { 4822e5adfffcSKirill A. Shutemov if (IS_ENABLED(CONFIG_NUMA)) { 4823a47a126aSEric Dumazet unsigned int nr, *counters = m->private; 482451e50b3aSEric Dumazet unsigned int step = 1U << vm_area_page_order(v); 4825a47a126aSEric Dumazet 4826a47a126aSEric Dumazet if (!counters) 4827a47a126aSEric Dumazet return; 4828a47a126aSEric Dumazet 4829af12346cSWanpeng Li if (v->flags & VM_UNINITIALIZED) 4830af12346cSWanpeng Li return; 48317e5b528bSDmitry Vyukov /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ 48327e5b528bSDmitry Vyukov smp_rmb(); 4833af12346cSWanpeng Li 4834a47a126aSEric Dumazet memset(counters, 0, nr_node_ids * sizeof(unsigned int)); 4835a47a126aSEric Dumazet 483651e50b3aSEric Dumazet for (nr = 0; nr < v->nr_pages; nr += step) 483751e50b3aSEric Dumazet counters[page_to_nid(v->pages[nr])] += step; 4838a47a126aSEric Dumazet for_each_node_state(nr, N_HIGH_MEMORY) 4839a47a126aSEric Dumazet if (counters[nr]) 4840a47a126aSEric Dumazet seq_printf(m, " N%u=%u", nr, counters[nr]); 4841a47a126aSEric Dumazet } 4842a47a126aSEric Dumazet } 4843a47a126aSEric Dumazet 4844dd3b8353SUladzislau Rezki (Sony) static void show_purge_info(struct seq_file *m) 4845dd3b8353SUladzislau Rezki (Sony) { 4846282631cbSUladzislau Rezki (Sony) struct vmap_node *vn; 4847dd3b8353SUladzislau Rezki (Sony) struct vmap_area *va; 4848282631cbSUladzislau Rezki (Sony) int i; 4849dd3b8353SUladzislau Rezki (Sony) 4850282631cbSUladzislau Rezki (Sony) for (i = 0; i < nr_vmap_nodes; i++) { 4851282631cbSUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 4852282631cbSUladzislau Rezki (Sony) 4853282631cbSUladzislau Rezki (Sony) spin_lock(&vn->lazy.lock); 4854282631cbSUladzislau Rezki (Sony) list_for_each_entry(va, &vn->lazy.head, list) { 4855dd3b8353SUladzislau Rezki (Sony) seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n", 4856dd3b8353SUladzislau Rezki (Sony) (void *)va->va_start, (void *)va->va_end, 4857dd3b8353SUladzislau Rezki (Sony) va->va_end - va->va_start); 4858dd3b8353SUladzislau Rezki (Sony) } 4859282631cbSUladzislau Rezki (Sony) spin_unlock(&vn->lazy.lock); 4860282631cbSUladzislau Rezki (Sony) } 4861dd3b8353SUladzislau Rezki (Sony) } 4862dd3b8353SUladzislau Rezki (Sony) 48638e1d743fSUladzislau Rezki (Sony) static int vmalloc_info_show(struct seq_file *m, void *p) 4864a10aa579SChristoph Lameter { 4865d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 48663f500069Szijun_hu struct vmap_area *va; 4867d4033afdSJoonsoo Kim struct vm_struct *v; 48688e1d743fSUladzislau Rezki (Sony) int i; 4869d4033afdSJoonsoo Kim 48708e1d743fSUladzislau Rezki (Sony) for (i = 0; i < nr_vmap_nodes; i++) { 48718e1d743fSUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 48723f500069Szijun_hu 48738e1d743fSUladzislau Rezki (Sony) spin_lock(&vn->busy.lock); 48748e1d743fSUladzislau Rezki (Sony) list_for_each_entry(va, &vn->busy.head, list) { 4875688fcbfcSPengfei Li if (!va->vm) { 4876bba9697bSBaoquan He if (va->flags & VMAP_RAM) 4877dd3b8353SUladzislau Rezki (Sony) seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n", 487878c72746SYisheng Xie (void *)va->va_start, (void *)va->va_end, 4879dd3b8353SUladzislau Rezki (Sony) va->va_end - va->va_start); 488078c72746SYisheng Xie 48818e1d743fSUladzislau Rezki (Sony) continue; 488278c72746SYisheng Xie } 4883d4033afdSJoonsoo Kim 4884d4033afdSJoonsoo Kim v = va->vm; 4885a10aa579SChristoph Lameter 488645ec1690SKees Cook seq_printf(m, "0x%pK-0x%pK %7ld", 4887a10aa579SChristoph Lameter v->addr, v->addr + v->size, v->size); 4888a10aa579SChristoph Lameter 488962c70bceSJoe Perches if (v->caller) 489062c70bceSJoe Perches seq_printf(m, " %pS", v->caller); 489123016969SChristoph Lameter 4892a10aa579SChristoph Lameter if (v->nr_pages) 4893a10aa579SChristoph Lameter seq_printf(m, " pages=%d", v->nr_pages); 4894a10aa579SChristoph Lameter 4895a10aa579SChristoph Lameter if (v->phys_addr) 4896199eaa05SMiles Chen seq_printf(m, " phys=%pa", &v->phys_addr); 4897a10aa579SChristoph Lameter 4898a10aa579SChristoph Lameter if (v->flags & VM_IOREMAP) 4899f4527c90SFabian Frederick seq_puts(m, " ioremap"); 4900a10aa579SChristoph Lameter 4901e6f79822SAlexei Starovoitov if (v->flags & VM_SPARSE) 4902e6f79822SAlexei Starovoitov seq_puts(m, " sparse"); 4903e6f79822SAlexei Starovoitov 4904a10aa579SChristoph Lameter if (v->flags & VM_ALLOC) 4905f4527c90SFabian Frederick seq_puts(m, " vmalloc"); 4906a10aa579SChristoph Lameter 4907a10aa579SChristoph Lameter if (v->flags & VM_MAP) 4908f4527c90SFabian Frederick seq_puts(m, " vmap"); 4909a10aa579SChristoph Lameter 4910a10aa579SChristoph Lameter if (v->flags & VM_USERMAP) 4911f4527c90SFabian Frederick seq_puts(m, " user"); 4912a10aa579SChristoph Lameter 4913fe9041c2SChristoph Hellwig if (v->flags & VM_DMA_COHERENT) 4914fe9041c2SChristoph Hellwig seq_puts(m, " dma-coherent"); 4915fe9041c2SChristoph Hellwig 4916244d63eeSDavid Rientjes if (is_vmalloc_addr(v->pages)) 4917f4527c90SFabian Frederick seq_puts(m, " vpages"); 4918a10aa579SChristoph Lameter 4919a47a126aSEric Dumazet show_numa_info(m, v); 4920a10aa579SChristoph Lameter seq_putc(m, '\n'); 49218e1d743fSUladzislau Rezki (Sony) } 49228e1d743fSUladzislau Rezki (Sony) spin_unlock(&vn->busy.lock); 49238e1d743fSUladzislau Rezki (Sony) } 4924dd3b8353SUladzislau Rezki (Sony) 4925dd3b8353SUladzislau Rezki (Sony) /* 492696e2db45SUladzislau Rezki (Sony) * As a final step, dump "unpurged" areas. 4927dd3b8353SUladzislau Rezki (Sony) */ 4928dd3b8353SUladzislau Rezki (Sony) show_purge_info(m); 4929a10aa579SChristoph Lameter return 0; 4930a10aa579SChristoph Lameter } 4931a10aa579SChristoph Lameter 49325f6a6a9cSAlexey Dobriyan static int __init proc_vmalloc_init(void) 49335f6a6a9cSAlexey Dobriyan { 49348e1d743fSUladzislau Rezki (Sony) void *priv_data = NULL; 49358e1d743fSUladzislau Rezki (Sony) 4936fddda2b7SChristoph Hellwig if (IS_ENABLED(CONFIG_NUMA)) 49378e1d743fSUladzislau Rezki (Sony) priv_data = kmalloc(nr_node_ids * sizeof(unsigned int), GFP_KERNEL); 49388e1d743fSUladzislau Rezki (Sony) 49398e1d743fSUladzislau Rezki (Sony) proc_create_single_data("vmallocinfo", 49408e1d743fSUladzislau Rezki (Sony) 0400, NULL, vmalloc_info_show, priv_data); 49418e1d743fSUladzislau Rezki (Sony) 49425f6a6a9cSAlexey Dobriyan return 0; 49435f6a6a9cSAlexey Dobriyan } 49445f6a6a9cSAlexey Dobriyan module_init(proc_vmalloc_init); 4945db3808c1SJoonsoo Kim 4946a10aa579SChristoph Lameter #endif 4947208162f4SChristoph Hellwig 4948d0936029SUladzislau Rezki (Sony) static void __init vmap_init_free_space(void) 49497fa8cee0SUladzislau Rezki (Sony) { 49507fa8cee0SUladzislau Rezki (Sony) unsigned long vmap_start = 1; 49517fa8cee0SUladzislau Rezki (Sony) const unsigned long vmap_end = ULONG_MAX; 4952d0936029SUladzislau Rezki (Sony) struct vmap_area *free; 4953d0936029SUladzislau Rezki (Sony) struct vm_struct *busy; 49547fa8cee0SUladzislau Rezki (Sony) 49557fa8cee0SUladzislau Rezki (Sony) /* 49567fa8cee0SUladzislau Rezki (Sony) * B F B B B F 49577fa8cee0SUladzislau Rezki (Sony) * -|-----|.....|-----|-----|-----|.....|- 49587fa8cee0SUladzislau Rezki (Sony) * | The KVA space | 49597fa8cee0SUladzislau Rezki (Sony) * |<--------------------------------->| 49607fa8cee0SUladzislau Rezki (Sony) */ 4961d0936029SUladzislau Rezki (Sony) for (busy = vmlist; busy; busy = busy->next) { 4962d0936029SUladzislau Rezki (Sony) if ((unsigned long) busy->addr - vmap_start > 0) { 49637fa8cee0SUladzislau Rezki (Sony) free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 49647fa8cee0SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(!free)) { 49657fa8cee0SUladzislau Rezki (Sony) free->va_start = vmap_start; 4966d0936029SUladzislau Rezki (Sony) free->va_end = (unsigned long) busy->addr; 49677fa8cee0SUladzislau Rezki (Sony) 49687fa8cee0SUladzislau Rezki (Sony) insert_vmap_area_augment(free, NULL, 49697fa8cee0SUladzislau Rezki (Sony) &free_vmap_area_root, 49707fa8cee0SUladzislau Rezki (Sony) &free_vmap_area_list); 49717fa8cee0SUladzislau Rezki (Sony) } 49727fa8cee0SUladzislau Rezki (Sony) } 49737fa8cee0SUladzislau Rezki (Sony) 4974d0936029SUladzislau Rezki (Sony) vmap_start = (unsigned long) busy->addr + busy->size; 49757fa8cee0SUladzislau Rezki (Sony) } 49767fa8cee0SUladzislau Rezki (Sony) 49777fa8cee0SUladzislau Rezki (Sony) if (vmap_end - vmap_start > 0) { 49787fa8cee0SUladzislau Rezki (Sony) free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 49797fa8cee0SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(!free)) { 49807fa8cee0SUladzislau Rezki (Sony) free->va_start = vmap_start; 49817fa8cee0SUladzislau Rezki (Sony) free->va_end = vmap_end; 49827fa8cee0SUladzislau Rezki (Sony) 49837fa8cee0SUladzislau Rezki (Sony) insert_vmap_area_augment(free, NULL, 49847fa8cee0SUladzislau Rezki (Sony) &free_vmap_area_root, 49857fa8cee0SUladzislau Rezki (Sony) &free_vmap_area_list); 49867fa8cee0SUladzislau Rezki (Sony) } 49877fa8cee0SUladzislau Rezki (Sony) } 49887fa8cee0SUladzislau Rezki (Sony) } 49897fa8cee0SUladzislau Rezki (Sony) 4990d0936029SUladzislau Rezki (Sony) static void vmap_init_nodes(void) 4991d0936029SUladzislau Rezki (Sony) { 4992d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 49938f33a2ffSUladzislau Rezki (Sony) int i, n; 4994d0936029SUladzislau Rezki (Sony) 49958f33a2ffSUladzislau Rezki (Sony) #if BITS_PER_LONG == 64 499615e02a39SUladzislau Rezki (Sony) /* 499715e02a39SUladzislau Rezki (Sony) * A high threshold of max nodes is fixed and bound to 128, 499815e02a39SUladzislau Rezki (Sony) * thus a scale factor is 1 for systems where number of cores 499915e02a39SUladzislau Rezki (Sony) * are less or equal to specified threshold. 500015e02a39SUladzislau Rezki (Sony) * 500115e02a39SUladzislau Rezki (Sony) * As for NUMA-aware notes. For bigger systems, for example 500215e02a39SUladzislau Rezki (Sony) * NUMA with multi-sockets, where we can end-up with thousands 500315e02a39SUladzislau Rezki (Sony) * of cores in total, a "sub-numa-clustering" should be added. 500415e02a39SUladzislau Rezki (Sony) * 500515e02a39SUladzislau Rezki (Sony) * In this case a NUMA domain is considered as a single entity 500615e02a39SUladzislau Rezki (Sony) * with dedicated sub-nodes in it which describe one group or 500715e02a39SUladzislau Rezki (Sony) * set of cores. Therefore a per-domain purging is supposed to 500815e02a39SUladzislau Rezki (Sony) * be added as well as a per-domain balancing. 500915e02a39SUladzislau Rezki (Sony) */ 50108f33a2ffSUladzislau Rezki (Sony) n = clamp_t(unsigned int, num_possible_cpus(), 1, 128); 50118f33a2ffSUladzislau Rezki (Sony) 50128f33a2ffSUladzislau Rezki (Sony) if (n > 1) { 50138f33a2ffSUladzislau Rezki (Sony) vn = kmalloc_array(n, sizeof(*vn), GFP_NOWAIT | __GFP_NOWARN); 50148f33a2ffSUladzislau Rezki (Sony) if (vn) { 50158f33a2ffSUladzislau Rezki (Sony) /* Node partition is 16 pages. */ 50168f33a2ffSUladzislau Rezki (Sony) vmap_zone_size = (1 << 4) * PAGE_SIZE; 50178f33a2ffSUladzislau Rezki (Sony) nr_vmap_nodes = n; 50188f33a2ffSUladzislau Rezki (Sony) vmap_nodes = vn; 50198f33a2ffSUladzislau Rezki (Sony) } else { 50208f33a2ffSUladzislau Rezki (Sony) pr_err("Failed to allocate an array. Disable a node layer\n"); 50218f33a2ffSUladzislau Rezki (Sony) } 50228f33a2ffSUladzislau Rezki (Sony) } 50238f33a2ffSUladzislau Rezki (Sony) #endif 50248f33a2ffSUladzislau Rezki (Sony) 50258f33a2ffSUladzislau Rezki (Sony) for (n = 0; n < nr_vmap_nodes; n++) { 50268f33a2ffSUladzislau Rezki (Sony) vn = &vmap_nodes[n]; 5027d0936029SUladzislau Rezki (Sony) vn->busy.root = RB_ROOT; 5028d0936029SUladzislau Rezki (Sony) INIT_LIST_HEAD(&vn->busy.head); 5029d0936029SUladzislau Rezki (Sony) spin_lock_init(&vn->busy.lock); 5030282631cbSUladzislau Rezki (Sony) 5031282631cbSUladzislau Rezki (Sony) vn->lazy.root = RB_ROOT; 5032282631cbSUladzislau Rezki (Sony) INIT_LIST_HEAD(&vn->lazy.head); 5033282631cbSUladzislau Rezki (Sony) spin_lock_init(&vn->lazy.lock); 503472210662SUladzislau Rezki (Sony) 50358f33a2ffSUladzislau Rezki (Sony) for (i = 0; i < MAX_VA_SIZE_PAGES; i++) { 50368f33a2ffSUladzislau Rezki (Sony) INIT_LIST_HEAD(&vn->pool[i].head); 50378f33a2ffSUladzislau Rezki (Sony) WRITE_ONCE(vn->pool[i].len, 0); 503872210662SUladzislau Rezki (Sony) } 503972210662SUladzislau Rezki (Sony) 504072210662SUladzislau Rezki (Sony) spin_lock_init(&vn->pool_lock); 5041d0936029SUladzislau Rezki (Sony) } 5042d0936029SUladzislau Rezki (Sony) } 5043d0936029SUladzislau Rezki (Sony) 50447679ba6bSUladzislau Rezki (Sony) static unsigned long 50457679ba6bSUladzislau Rezki (Sony) vmap_node_shrink_count(struct shrinker *shrink, struct shrink_control *sc) 50467679ba6bSUladzislau Rezki (Sony) { 50477679ba6bSUladzislau Rezki (Sony) unsigned long count; 50487679ba6bSUladzislau Rezki (Sony) struct vmap_node *vn; 50497679ba6bSUladzislau Rezki (Sony) int i, j; 50507679ba6bSUladzislau Rezki (Sony) 50517679ba6bSUladzislau Rezki (Sony) for (count = 0, i = 0; i < nr_vmap_nodes; i++) { 50527679ba6bSUladzislau Rezki (Sony) vn = &vmap_nodes[i]; 50537679ba6bSUladzislau Rezki (Sony) 50547679ba6bSUladzislau Rezki (Sony) for (j = 0; j < MAX_VA_SIZE_PAGES; j++) 50557679ba6bSUladzislau Rezki (Sony) count += READ_ONCE(vn->pool[j].len); 50567679ba6bSUladzislau Rezki (Sony) } 50577679ba6bSUladzislau Rezki (Sony) 50587679ba6bSUladzislau Rezki (Sony) return count ? count : SHRINK_EMPTY; 50597679ba6bSUladzislau Rezki (Sony) } 50607679ba6bSUladzislau Rezki (Sony) 50617679ba6bSUladzislau Rezki (Sony) static unsigned long 50627679ba6bSUladzislau Rezki (Sony) vmap_node_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) 50637679ba6bSUladzislau Rezki (Sony) { 50647679ba6bSUladzislau Rezki (Sony) int i; 50657679ba6bSUladzislau Rezki (Sony) 50667679ba6bSUladzislau Rezki (Sony) for (i = 0; i < nr_vmap_nodes; i++) 50677679ba6bSUladzislau Rezki (Sony) decay_va_pool_node(&vmap_nodes[i], true); 50687679ba6bSUladzislau Rezki (Sony) 50697679ba6bSUladzislau Rezki (Sony) return SHRINK_STOP; 50707679ba6bSUladzislau Rezki (Sony) } 50717679ba6bSUladzislau Rezki (Sony) 5072208162f4SChristoph Hellwig void __init vmalloc_init(void) 5073208162f4SChristoph Hellwig { 50747679ba6bSUladzislau Rezki (Sony) struct shrinker *vmap_node_shrinker; 5075208162f4SChristoph Hellwig struct vmap_area *va; 5076d0936029SUladzislau Rezki (Sony) struct vmap_node *vn; 5077208162f4SChristoph Hellwig struct vm_struct *tmp; 5078208162f4SChristoph Hellwig int i; 5079208162f4SChristoph Hellwig 5080208162f4SChristoph Hellwig /* 5081208162f4SChristoph Hellwig * Create the cache for vmap_area objects. 5082208162f4SChristoph Hellwig */ 5083208162f4SChristoph Hellwig vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC); 5084208162f4SChristoph Hellwig 5085208162f4SChristoph Hellwig for_each_possible_cpu(i) { 5086208162f4SChristoph Hellwig struct vmap_block_queue *vbq; 5087208162f4SChristoph Hellwig struct vfree_deferred *p; 5088208162f4SChristoph Hellwig 5089208162f4SChristoph Hellwig vbq = &per_cpu(vmap_block_queue, i); 5090208162f4SChristoph Hellwig spin_lock_init(&vbq->lock); 5091208162f4SChristoph Hellwig INIT_LIST_HEAD(&vbq->free); 5092208162f4SChristoph Hellwig p = &per_cpu(vfree_deferred, i); 5093208162f4SChristoph Hellwig init_llist_head(&p->list); 5094208162f4SChristoph Hellwig INIT_WORK(&p->wq, delayed_vfree_work); 5095062eacf5SUladzislau Rezki (Sony) xa_init(&vbq->vmap_blocks); 5096208162f4SChristoph Hellwig } 5097208162f4SChristoph Hellwig 5098d0936029SUladzislau Rezki (Sony) /* 5099d0936029SUladzislau Rezki (Sony) * Setup nodes before importing vmlist. 5100d0936029SUladzislau Rezki (Sony) */ 5101d0936029SUladzislau Rezki (Sony) vmap_init_nodes(); 5102d0936029SUladzislau Rezki (Sony) 5103208162f4SChristoph Hellwig /* Import existing vmlist entries. */ 5104208162f4SChristoph Hellwig for (tmp = vmlist; tmp; tmp = tmp->next) { 5105208162f4SChristoph Hellwig va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 5106208162f4SChristoph Hellwig if (WARN_ON_ONCE(!va)) 5107208162f4SChristoph Hellwig continue; 5108208162f4SChristoph Hellwig 5109208162f4SChristoph Hellwig va->va_start = (unsigned long)tmp->addr; 5110208162f4SChristoph Hellwig va->va_end = va->va_start + tmp->size; 5111208162f4SChristoph Hellwig va->vm = tmp; 5112d0936029SUladzislau Rezki (Sony) 5113d0936029SUladzislau Rezki (Sony) vn = addr_to_node(va->va_start); 5114d0936029SUladzislau Rezki (Sony) insert_vmap_area(va, &vn->busy.root, &vn->busy.head); 5115208162f4SChristoph Hellwig } 5116208162f4SChristoph Hellwig 5117208162f4SChristoph Hellwig /* 5118208162f4SChristoph Hellwig * Now we can initialize a free vmap space. 5119208162f4SChristoph Hellwig */ 5120208162f4SChristoph Hellwig vmap_init_free_space(); 5121208162f4SChristoph Hellwig vmap_initialized = true; 51227679ba6bSUladzislau Rezki (Sony) 51237679ba6bSUladzislau Rezki (Sony) vmap_node_shrinker = shrinker_alloc(0, "vmap-node"); 51247679ba6bSUladzislau Rezki (Sony) if (!vmap_node_shrinker) { 51257679ba6bSUladzislau Rezki (Sony) pr_err("Failed to allocate vmap-node shrinker!\n"); 51267679ba6bSUladzislau Rezki (Sony) return; 51277679ba6bSUladzislau Rezki (Sony) } 51287679ba6bSUladzislau Rezki (Sony) 51297679ba6bSUladzislau Rezki (Sony) vmap_node_shrinker->count_objects = vmap_node_shrink_count; 51307679ba6bSUladzislau Rezki (Sony) vmap_node_shrinker->scan_objects = vmap_node_shrink_scan; 51317679ba6bSUladzislau Rezki (Sony) shrinker_register(vmap_node_shrinker); 5132208162f4SChristoph Hellwig } 5133