1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * linux/mm/vmalloc.c 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Copyright (C) 1993 Linus Torvalds 61da177e4SLinus Torvalds * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 71da177e4SLinus Torvalds * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 81da177e4SLinus Torvalds * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 9930fc45aSChristoph Lameter * Numa awareness, Christoph Lameter, SGI, June 2005 101da177e4SLinus Torvalds */ 111da177e4SLinus Torvalds 12db64fe02SNick Piggin #include <linux/vmalloc.h> 131da177e4SLinus Torvalds #include <linux/mm.h> 141da177e4SLinus Torvalds #include <linux/module.h> 151da177e4SLinus Torvalds #include <linux/highmem.h> 16c3edc401SIngo Molnar #include <linux/sched/signal.h> 171da177e4SLinus Torvalds #include <linux/slab.h> 181da177e4SLinus Torvalds #include <linux/spinlock.h> 191da177e4SLinus Torvalds #include <linux/interrupt.h> 205f6a6a9cSAlexey Dobriyan #include <linux/proc_fs.h> 21a10aa579SChristoph Lameter #include <linux/seq_file.h> 22868b104dSRick Edgecombe #include <linux/set_memory.h> 233ac7fe5aSThomas Gleixner #include <linux/debugobjects.h> 2423016969SChristoph Lameter #include <linux/kallsyms.h> 25db64fe02SNick Piggin #include <linux/list.h> 264da56b99SChris Wilson #include <linux/notifier.h> 27db64fe02SNick Piggin #include <linux/rbtree.h> 280f14599cSMatthew Wilcox (Oracle) #include <linux/xarray.h> 29db64fe02SNick Piggin #include <linux/rcupdate.h> 30f0aa6617STejun Heo #include <linux/pfn.h> 3189219d37SCatalin Marinas #include <linux/kmemleak.h> 3260063497SArun Sharma #include <linux/atomic.h> 333b32123dSGideon Israel Dsouza #include <linux/compiler.h> 3432fcfd40SAl Viro #include <linux/llist.h> 350f616be1SToshi Kani #include <linux/bitops.h> 3668ad4a33SUladzislau Rezki (Sony) #include <linux/rbtree_augmented.h> 37bdebd6a2SJann Horn #include <linux/overflow.h> 383b32123dSGideon Israel Dsouza 397c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 401da177e4SLinus Torvalds #include <asm/tlbflush.h> 412dca6999SDavid Miller #include <asm/shmparam.h> 421da177e4SLinus Torvalds 43dd56b046SMel Gorman #include "internal.h" 442a681cfaSJoerg Roedel #include "pgalloc-track.h" 45dd56b046SMel Gorman 46186525bdSIngo Molnar bool is_vmalloc_addr(const void *x) 47186525bdSIngo Molnar { 48186525bdSIngo Molnar unsigned long addr = (unsigned long)x; 49186525bdSIngo Molnar 50186525bdSIngo Molnar return addr >= VMALLOC_START && addr < VMALLOC_END; 51186525bdSIngo Molnar } 52186525bdSIngo Molnar EXPORT_SYMBOL(is_vmalloc_addr); 53186525bdSIngo Molnar 5432fcfd40SAl Viro struct vfree_deferred { 5532fcfd40SAl Viro struct llist_head list; 5632fcfd40SAl Viro struct work_struct wq; 5732fcfd40SAl Viro }; 5832fcfd40SAl Viro static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); 5932fcfd40SAl Viro 6032fcfd40SAl Viro static void __vunmap(const void *, int); 6132fcfd40SAl Viro 6232fcfd40SAl Viro static void free_work(struct work_struct *w) 6332fcfd40SAl Viro { 6432fcfd40SAl Viro struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); 65894e58c1SByungchul Park struct llist_node *t, *llnode; 66894e58c1SByungchul Park 67894e58c1SByungchul Park llist_for_each_safe(llnode, t, llist_del_all(&p->list)) 68894e58c1SByungchul Park __vunmap((void *)llnode, 1); 6932fcfd40SAl Viro } 7032fcfd40SAl Viro 71db64fe02SNick Piggin /*** Page table manipulation functions ***/ 72b221385bSAdrian Bunk 732ba3e694SJoerg Roedel static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 742ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 751da177e4SLinus Torvalds { 761da177e4SLinus Torvalds pte_t *pte; 771da177e4SLinus Torvalds 781da177e4SLinus Torvalds pte = pte_offset_kernel(pmd, addr); 791da177e4SLinus Torvalds do { 801da177e4SLinus Torvalds pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); 811da177e4SLinus Torvalds WARN_ON(!pte_none(ptent) && !pte_present(ptent)); 821da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 832ba3e694SJoerg Roedel *mask |= PGTBL_PTE_MODIFIED; 841da177e4SLinus Torvalds } 851da177e4SLinus Torvalds 862ba3e694SJoerg Roedel static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 872ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 881da177e4SLinus Torvalds { 891da177e4SLinus Torvalds pmd_t *pmd; 901da177e4SLinus Torvalds unsigned long next; 912ba3e694SJoerg Roedel int cleared; 921da177e4SLinus Torvalds 931da177e4SLinus Torvalds pmd = pmd_offset(pud, addr); 941da177e4SLinus Torvalds do { 951da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 962ba3e694SJoerg Roedel 972ba3e694SJoerg Roedel cleared = pmd_clear_huge(pmd); 982ba3e694SJoerg Roedel if (cleared || pmd_bad(*pmd)) 992ba3e694SJoerg Roedel *mask |= PGTBL_PMD_MODIFIED; 1002ba3e694SJoerg Roedel 1012ba3e694SJoerg Roedel if (cleared) 102b9820d8fSToshi Kani continue; 1031da177e4SLinus Torvalds if (pmd_none_or_clear_bad(pmd)) 1041da177e4SLinus Torvalds continue; 1052ba3e694SJoerg Roedel vunmap_pte_range(pmd, addr, next, mask); 1061da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 1071da177e4SLinus Torvalds } 1081da177e4SLinus Torvalds 1092ba3e694SJoerg Roedel static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 1102ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 1111da177e4SLinus Torvalds { 1121da177e4SLinus Torvalds pud_t *pud; 1131da177e4SLinus Torvalds unsigned long next; 1142ba3e694SJoerg Roedel int cleared; 1151da177e4SLinus Torvalds 116c2febafcSKirill A. Shutemov pud = pud_offset(p4d, addr); 1171da177e4SLinus Torvalds do { 1181da177e4SLinus Torvalds next = pud_addr_end(addr, end); 1192ba3e694SJoerg Roedel 1202ba3e694SJoerg Roedel cleared = pud_clear_huge(pud); 1212ba3e694SJoerg Roedel if (cleared || pud_bad(*pud)) 1222ba3e694SJoerg Roedel *mask |= PGTBL_PUD_MODIFIED; 1232ba3e694SJoerg Roedel 1242ba3e694SJoerg Roedel if (cleared) 125b9820d8fSToshi Kani continue; 1261da177e4SLinus Torvalds if (pud_none_or_clear_bad(pud)) 1271da177e4SLinus Torvalds continue; 1282ba3e694SJoerg Roedel vunmap_pmd_range(pud, addr, next, mask); 1291da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 1301da177e4SLinus Torvalds } 1311da177e4SLinus Torvalds 1322ba3e694SJoerg Roedel static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 1332ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 134c2febafcSKirill A. Shutemov { 135c2febafcSKirill A. Shutemov p4d_t *p4d; 136c2febafcSKirill A. Shutemov unsigned long next; 1372ba3e694SJoerg Roedel int cleared; 138c2febafcSKirill A. Shutemov 139c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 140c2febafcSKirill A. Shutemov do { 141c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 1422ba3e694SJoerg Roedel 1432ba3e694SJoerg Roedel cleared = p4d_clear_huge(p4d); 1442ba3e694SJoerg Roedel if (cleared || p4d_bad(*p4d)) 1452ba3e694SJoerg Roedel *mask |= PGTBL_P4D_MODIFIED; 1462ba3e694SJoerg Roedel 1472ba3e694SJoerg Roedel if (cleared) 148c2febafcSKirill A. Shutemov continue; 149c2febafcSKirill A. Shutemov if (p4d_none_or_clear_bad(p4d)) 150c2febafcSKirill A. Shutemov continue; 1512ba3e694SJoerg Roedel vunmap_pud_range(p4d, addr, next, mask); 152c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 153c2febafcSKirill A. Shutemov } 154c2febafcSKirill A. Shutemov 155b521c43fSChristoph Hellwig /** 156b521c43fSChristoph Hellwig * unmap_kernel_range_noflush - unmap kernel VM area 1572ba3e694SJoerg Roedel * @start: start of the VM area to unmap 158b521c43fSChristoph Hellwig * @size: size of the VM area to unmap 159b521c43fSChristoph Hellwig * 160b521c43fSChristoph Hellwig * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size specify 161b521c43fSChristoph Hellwig * should have been allocated using get_vm_area() and its friends. 162b521c43fSChristoph Hellwig * 163b521c43fSChristoph Hellwig * NOTE: 164b521c43fSChristoph Hellwig * This function does NOT do any cache flushing. The caller is responsible 165b521c43fSChristoph Hellwig * for calling flush_cache_vunmap() on to-be-mapped areas before calling this 166b521c43fSChristoph Hellwig * function and flush_tlb_kernel_range() after. 167b521c43fSChristoph Hellwig */ 1682ba3e694SJoerg Roedel void unmap_kernel_range_noflush(unsigned long start, unsigned long size) 1691da177e4SLinus Torvalds { 1702ba3e694SJoerg Roedel unsigned long end = start + size; 1711da177e4SLinus Torvalds unsigned long next; 172b521c43fSChristoph Hellwig pgd_t *pgd; 1732ba3e694SJoerg Roedel unsigned long addr = start; 1742ba3e694SJoerg Roedel pgtbl_mod_mask mask = 0; 1751da177e4SLinus Torvalds 1761da177e4SLinus Torvalds BUG_ON(addr >= end); 1772ba3e694SJoerg Roedel start = addr; 1781da177e4SLinus Torvalds pgd = pgd_offset_k(addr); 1791da177e4SLinus Torvalds do { 1801da177e4SLinus Torvalds next = pgd_addr_end(addr, end); 1812ba3e694SJoerg Roedel if (pgd_bad(*pgd)) 1822ba3e694SJoerg Roedel mask |= PGTBL_PGD_MODIFIED; 1831da177e4SLinus Torvalds if (pgd_none_or_clear_bad(pgd)) 1841da177e4SLinus Torvalds continue; 1852ba3e694SJoerg Roedel vunmap_p4d_range(pgd, addr, next, &mask); 1861da177e4SLinus Torvalds } while (pgd++, addr = next, addr != end); 1872ba3e694SJoerg Roedel 1882ba3e694SJoerg Roedel if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 1892ba3e694SJoerg Roedel arch_sync_kernel_mappings(start, end); 1901da177e4SLinus Torvalds } 1911da177e4SLinus Torvalds 1921da177e4SLinus Torvalds static int vmap_pte_range(pmd_t *pmd, unsigned long addr, 1932ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 1942ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 1951da177e4SLinus Torvalds { 1961da177e4SLinus Torvalds pte_t *pte; 1971da177e4SLinus Torvalds 198db64fe02SNick Piggin /* 199db64fe02SNick Piggin * nr is a running index into the array which helps higher level 200db64fe02SNick Piggin * callers keep track of where we're up to. 201db64fe02SNick Piggin */ 202db64fe02SNick Piggin 2032ba3e694SJoerg Roedel pte = pte_alloc_kernel_track(pmd, addr, mask); 2041da177e4SLinus Torvalds if (!pte) 2051da177e4SLinus Torvalds return -ENOMEM; 2061da177e4SLinus Torvalds do { 207db64fe02SNick Piggin struct page *page = pages[*nr]; 208db64fe02SNick Piggin 209db64fe02SNick Piggin if (WARN_ON(!pte_none(*pte))) 210db64fe02SNick Piggin return -EBUSY; 211db64fe02SNick Piggin if (WARN_ON(!page)) 2121da177e4SLinus Torvalds return -ENOMEM; 2131da177e4SLinus Torvalds set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); 214db64fe02SNick Piggin (*nr)++; 2151da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 2162ba3e694SJoerg Roedel *mask |= PGTBL_PTE_MODIFIED; 2171da177e4SLinus Torvalds return 0; 2181da177e4SLinus Torvalds } 2191da177e4SLinus Torvalds 220db64fe02SNick Piggin static int vmap_pmd_range(pud_t *pud, unsigned long addr, 2212ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 2222ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 2231da177e4SLinus Torvalds { 2241da177e4SLinus Torvalds pmd_t *pmd; 2251da177e4SLinus Torvalds unsigned long next; 2261da177e4SLinus Torvalds 2272ba3e694SJoerg Roedel pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 2281da177e4SLinus Torvalds if (!pmd) 2291da177e4SLinus Torvalds return -ENOMEM; 2301da177e4SLinus Torvalds do { 2311da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 2322ba3e694SJoerg Roedel if (vmap_pte_range(pmd, addr, next, prot, pages, nr, mask)) 2331da177e4SLinus Torvalds return -ENOMEM; 2341da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 2351da177e4SLinus Torvalds return 0; 2361da177e4SLinus Torvalds } 2371da177e4SLinus Torvalds 238c2febafcSKirill A. Shutemov static int vmap_pud_range(p4d_t *p4d, unsigned long addr, 2392ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 2402ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 2411da177e4SLinus Torvalds { 2421da177e4SLinus Torvalds pud_t *pud; 2431da177e4SLinus Torvalds unsigned long next; 2441da177e4SLinus Torvalds 2452ba3e694SJoerg Roedel pud = pud_alloc_track(&init_mm, p4d, addr, mask); 2461da177e4SLinus Torvalds if (!pud) 2471da177e4SLinus Torvalds return -ENOMEM; 2481da177e4SLinus Torvalds do { 2491da177e4SLinus Torvalds next = pud_addr_end(addr, end); 2502ba3e694SJoerg Roedel if (vmap_pmd_range(pud, addr, next, prot, pages, nr, mask)) 2511da177e4SLinus Torvalds return -ENOMEM; 2521da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 2531da177e4SLinus Torvalds return 0; 2541da177e4SLinus Torvalds } 2551da177e4SLinus Torvalds 256c2febafcSKirill A. Shutemov static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, 2572ba3e694SJoerg Roedel unsigned long end, pgprot_t prot, struct page **pages, int *nr, 2582ba3e694SJoerg Roedel pgtbl_mod_mask *mask) 259c2febafcSKirill A. Shutemov { 260c2febafcSKirill A. Shutemov p4d_t *p4d; 261c2febafcSKirill A. Shutemov unsigned long next; 262c2febafcSKirill A. Shutemov 2632ba3e694SJoerg Roedel p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 264c2febafcSKirill A. Shutemov if (!p4d) 265c2febafcSKirill A. Shutemov return -ENOMEM; 266c2febafcSKirill A. Shutemov do { 267c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 2682ba3e694SJoerg Roedel if (vmap_pud_range(p4d, addr, next, prot, pages, nr, mask)) 269c2febafcSKirill A. Shutemov return -ENOMEM; 270c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 271c2febafcSKirill A. Shutemov return 0; 272c2febafcSKirill A. Shutemov } 273c2febafcSKirill A. Shutemov 274b521c43fSChristoph Hellwig /** 275b521c43fSChristoph Hellwig * map_kernel_range_noflush - map kernel VM area with the specified pages 276b521c43fSChristoph Hellwig * @addr: start of the VM area to map 277b521c43fSChristoph Hellwig * @size: size of the VM area to map 278b521c43fSChristoph Hellwig * @prot: page protection flags to use 279b521c43fSChristoph Hellwig * @pages: pages to map 280db64fe02SNick Piggin * 281b521c43fSChristoph Hellwig * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size specify should 282b521c43fSChristoph Hellwig * have been allocated using get_vm_area() and its friends. 283b521c43fSChristoph Hellwig * 284b521c43fSChristoph Hellwig * NOTE: 285b521c43fSChristoph Hellwig * This function does NOT do any cache flushing. The caller is responsible for 286b521c43fSChristoph Hellwig * calling flush_cache_vmap() on to-be-mapped areas before calling this 287b521c43fSChristoph Hellwig * function. 288b521c43fSChristoph Hellwig * 289b521c43fSChristoph Hellwig * RETURNS: 29060bb4465SChristoph Hellwig * 0 on success, -errno on failure. 291db64fe02SNick Piggin */ 292b521c43fSChristoph Hellwig int map_kernel_range_noflush(unsigned long addr, unsigned long size, 293db64fe02SNick Piggin pgprot_t prot, struct page **pages) 2941da177e4SLinus Torvalds { 2952ba3e694SJoerg Roedel unsigned long start = addr; 296b521c43fSChristoph Hellwig unsigned long end = addr + size; 2971da177e4SLinus Torvalds unsigned long next; 298b521c43fSChristoph Hellwig pgd_t *pgd; 299db64fe02SNick Piggin int err = 0; 300db64fe02SNick Piggin int nr = 0; 3012ba3e694SJoerg Roedel pgtbl_mod_mask mask = 0; 3021da177e4SLinus Torvalds 3031da177e4SLinus Torvalds BUG_ON(addr >= end); 3041da177e4SLinus Torvalds pgd = pgd_offset_k(addr); 3051da177e4SLinus Torvalds do { 3061da177e4SLinus Torvalds next = pgd_addr_end(addr, end); 3072ba3e694SJoerg Roedel if (pgd_bad(*pgd)) 3082ba3e694SJoerg Roedel mask |= PGTBL_PGD_MODIFIED; 3092ba3e694SJoerg Roedel err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr, &mask); 3101da177e4SLinus Torvalds if (err) 311bf88c8c8SFigo.zhang return err; 3121da177e4SLinus Torvalds } while (pgd++, addr = next, addr != end); 313db64fe02SNick Piggin 3142ba3e694SJoerg Roedel if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 3152ba3e694SJoerg Roedel arch_sync_kernel_mappings(start, end); 3162ba3e694SJoerg Roedel 31760bb4465SChristoph Hellwig return 0; 3181da177e4SLinus Torvalds } 3191da177e4SLinus Torvalds 320ed1f324cSChristoph Hellwig int map_kernel_range(unsigned long start, unsigned long size, pgprot_t prot, 321ed1f324cSChristoph Hellwig struct page **pages) 3228fc48985STejun Heo { 3238fc48985STejun Heo int ret; 3248fc48985STejun Heo 325a29adb62SChristoph Hellwig ret = map_kernel_range_noflush(start, size, prot, pages); 326a29adb62SChristoph Hellwig flush_cache_vmap(start, start + size); 3278fc48985STejun Heo return ret; 3288fc48985STejun Heo } 3298fc48985STejun Heo 33081ac3ad9SKAMEZAWA Hiroyuki int is_vmalloc_or_module_addr(const void *x) 33173bdf0a6SLinus Torvalds { 33273bdf0a6SLinus Torvalds /* 333ab4f2ee1SRussell King * ARM, x86-64 and sparc64 put modules in a special place, 33473bdf0a6SLinus Torvalds * and fall back on vmalloc() if that fails. Others 33573bdf0a6SLinus Torvalds * just put it in the vmalloc space. 33673bdf0a6SLinus Torvalds */ 33773bdf0a6SLinus Torvalds #if defined(CONFIG_MODULES) && defined(MODULES_VADDR) 33873bdf0a6SLinus Torvalds unsigned long addr = (unsigned long)x; 33973bdf0a6SLinus Torvalds if (addr >= MODULES_VADDR && addr < MODULES_END) 34073bdf0a6SLinus Torvalds return 1; 34173bdf0a6SLinus Torvalds #endif 34273bdf0a6SLinus Torvalds return is_vmalloc_addr(x); 34373bdf0a6SLinus Torvalds } 34473bdf0a6SLinus Torvalds 34548667e7aSChristoph Lameter /* 346add688fbSmalc * Walk a vmap address to the struct page it maps. 34748667e7aSChristoph Lameter */ 348add688fbSmalc struct page *vmalloc_to_page(const void *vmalloc_addr) 34948667e7aSChristoph Lameter { 35048667e7aSChristoph Lameter unsigned long addr = (unsigned long) vmalloc_addr; 351add688fbSmalc struct page *page = NULL; 35248667e7aSChristoph Lameter pgd_t *pgd = pgd_offset_k(addr); 353c2febafcSKirill A. Shutemov p4d_t *p4d; 354c2febafcSKirill A. Shutemov pud_t *pud; 355c2febafcSKirill A. Shutemov pmd_t *pmd; 356c2febafcSKirill A. Shutemov pte_t *ptep, pte; 35748667e7aSChristoph Lameter 3587aa413deSIngo Molnar /* 3597aa413deSIngo Molnar * XXX we might need to change this if we add VIRTUAL_BUG_ON for 3607aa413deSIngo Molnar * architectures that do not vmalloc module space 3617aa413deSIngo Molnar */ 36273bdf0a6SLinus Torvalds VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); 36359ea7463SJiri Slaby 364c2febafcSKirill A. Shutemov if (pgd_none(*pgd)) 365c2febafcSKirill A. Shutemov return NULL; 366c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 367c2febafcSKirill A. Shutemov if (p4d_none(*p4d)) 368c2febafcSKirill A. Shutemov return NULL; 369c2febafcSKirill A. Shutemov pud = pud_offset(p4d, addr); 370029c54b0SArd Biesheuvel 371029c54b0SArd Biesheuvel /* 372029c54b0SArd Biesheuvel * Don't dereference bad PUD or PMD (below) entries. This will also 373029c54b0SArd Biesheuvel * identify huge mappings, which we may encounter on architectures 374029c54b0SArd Biesheuvel * that define CONFIG_HAVE_ARCH_HUGE_VMAP=y. Such regions will be 375029c54b0SArd Biesheuvel * identified as vmalloc addresses by is_vmalloc_addr(), but are 376029c54b0SArd Biesheuvel * not [unambiguously] associated with a struct page, so there is 377029c54b0SArd Biesheuvel * no correct value to return for them. 378029c54b0SArd Biesheuvel */ 379029c54b0SArd Biesheuvel WARN_ON_ONCE(pud_bad(*pud)); 380029c54b0SArd Biesheuvel if (pud_none(*pud) || pud_bad(*pud)) 381c2febafcSKirill A. Shutemov return NULL; 382c2febafcSKirill A. Shutemov pmd = pmd_offset(pud, addr); 383029c54b0SArd Biesheuvel WARN_ON_ONCE(pmd_bad(*pmd)); 384029c54b0SArd Biesheuvel if (pmd_none(*pmd) || pmd_bad(*pmd)) 385c2febafcSKirill A. Shutemov return NULL; 386db64fe02SNick Piggin 38748667e7aSChristoph Lameter ptep = pte_offset_map(pmd, addr); 38848667e7aSChristoph Lameter pte = *ptep; 38948667e7aSChristoph Lameter if (pte_present(pte)) 390add688fbSmalc page = pte_page(pte); 39148667e7aSChristoph Lameter pte_unmap(ptep); 392add688fbSmalc return page; 393ece86e22SJianyu Zhan } 394ece86e22SJianyu Zhan EXPORT_SYMBOL(vmalloc_to_page); 395ece86e22SJianyu Zhan 396add688fbSmalc /* 397add688fbSmalc * Map a vmalloc()-space virtual address to the physical page frame number. 398add688fbSmalc */ 399add688fbSmalc unsigned long vmalloc_to_pfn(const void *vmalloc_addr) 400add688fbSmalc { 401add688fbSmalc return page_to_pfn(vmalloc_to_page(vmalloc_addr)); 402add688fbSmalc } 403add688fbSmalc EXPORT_SYMBOL(vmalloc_to_pfn); 404add688fbSmalc 405db64fe02SNick Piggin 406db64fe02SNick Piggin /*** Global kva allocator ***/ 407db64fe02SNick Piggin 408bb850f4dSUladzislau Rezki (Sony) #define DEBUG_AUGMENT_PROPAGATE_CHECK 0 409a6cf4e0fSUladzislau Rezki (Sony) #define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0 410bb850f4dSUladzislau Rezki (Sony) 411db64fe02SNick Piggin 412db64fe02SNick Piggin static DEFINE_SPINLOCK(vmap_area_lock); 413e36176beSUladzislau Rezki (Sony) static DEFINE_SPINLOCK(free_vmap_area_lock); 414f1c4069eSJoonsoo Kim /* Export for kexec only */ 415f1c4069eSJoonsoo Kim LIST_HEAD(vmap_area_list); 41680c4bd7aSChris Wilson static LLIST_HEAD(vmap_purge_list); 41789699605SNick Piggin static struct rb_root vmap_area_root = RB_ROOT; 41868ad4a33SUladzislau Rezki (Sony) static bool vmap_initialized __read_mostly; 41989699605SNick Piggin 42068ad4a33SUladzislau Rezki (Sony) /* 42168ad4a33SUladzislau Rezki (Sony) * This kmem_cache is used for vmap_area objects. Instead of 42268ad4a33SUladzislau Rezki (Sony) * allocating from slab we reuse an object from this cache to 42368ad4a33SUladzislau Rezki (Sony) * make things faster. Especially in "no edge" splitting of 42468ad4a33SUladzislau Rezki (Sony) * free block. 42568ad4a33SUladzislau Rezki (Sony) */ 42668ad4a33SUladzislau Rezki (Sony) static struct kmem_cache *vmap_area_cachep; 42789699605SNick Piggin 42868ad4a33SUladzislau Rezki (Sony) /* 42968ad4a33SUladzislau Rezki (Sony) * This linked list is used in pair with free_vmap_area_root. 43068ad4a33SUladzislau Rezki (Sony) * It gives O(1) access to prev/next to perform fast coalescing. 43168ad4a33SUladzislau Rezki (Sony) */ 43268ad4a33SUladzislau Rezki (Sony) static LIST_HEAD(free_vmap_area_list); 43368ad4a33SUladzislau Rezki (Sony) 43468ad4a33SUladzislau Rezki (Sony) /* 43568ad4a33SUladzislau Rezki (Sony) * This augment red-black tree represents the free vmap space. 43668ad4a33SUladzislau Rezki (Sony) * All vmap_area objects in this tree are sorted by va->va_start 43768ad4a33SUladzislau Rezki (Sony) * address. It is used for allocation and merging when a vmap 43868ad4a33SUladzislau Rezki (Sony) * object is released. 43968ad4a33SUladzislau Rezki (Sony) * 44068ad4a33SUladzislau Rezki (Sony) * Each vmap_area node contains a maximum available free block 44168ad4a33SUladzislau Rezki (Sony) * of its sub-tree, right or left. Therefore it is possible to 44268ad4a33SUladzislau Rezki (Sony) * find a lowest match of free area. 44368ad4a33SUladzislau Rezki (Sony) */ 44468ad4a33SUladzislau Rezki (Sony) static struct rb_root free_vmap_area_root = RB_ROOT; 44568ad4a33SUladzislau Rezki (Sony) 44682dd23e8SUladzislau Rezki (Sony) /* 44782dd23e8SUladzislau Rezki (Sony) * Preload a CPU with one object for "no edge" split case. The 44882dd23e8SUladzislau Rezki (Sony) * aim is to get rid of allocations from the atomic context, thus 44982dd23e8SUladzislau Rezki (Sony) * to use more permissive allocation masks. 45082dd23e8SUladzislau Rezki (Sony) */ 45182dd23e8SUladzislau Rezki (Sony) static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node); 45282dd23e8SUladzislau Rezki (Sony) 45368ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 45468ad4a33SUladzislau Rezki (Sony) va_size(struct vmap_area *va) 45568ad4a33SUladzislau Rezki (Sony) { 45668ad4a33SUladzislau Rezki (Sony) return (va->va_end - va->va_start); 45768ad4a33SUladzislau Rezki (Sony) } 45868ad4a33SUladzislau Rezki (Sony) 45968ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 46068ad4a33SUladzislau Rezki (Sony) get_subtree_max_size(struct rb_node *node) 46168ad4a33SUladzislau Rezki (Sony) { 46268ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 46368ad4a33SUladzislau Rezki (Sony) 46468ad4a33SUladzislau Rezki (Sony) va = rb_entry_safe(node, struct vmap_area, rb_node); 46568ad4a33SUladzislau Rezki (Sony) return va ? va->subtree_max_size : 0; 46668ad4a33SUladzislau Rezki (Sony) } 46768ad4a33SUladzislau Rezki (Sony) 46868ad4a33SUladzislau Rezki (Sony) /* 46968ad4a33SUladzislau Rezki (Sony) * Gets called when remove the node and rotate. 47068ad4a33SUladzislau Rezki (Sony) */ 47168ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 47268ad4a33SUladzislau Rezki (Sony) compute_subtree_max_size(struct vmap_area *va) 47368ad4a33SUladzislau Rezki (Sony) { 47468ad4a33SUladzislau Rezki (Sony) return max3(va_size(va), 47568ad4a33SUladzislau Rezki (Sony) get_subtree_max_size(va->rb_node.rb_left), 47668ad4a33SUladzislau Rezki (Sony) get_subtree_max_size(va->rb_node.rb_right)); 47768ad4a33SUladzislau Rezki (Sony) } 47868ad4a33SUladzislau Rezki (Sony) 479315cc066SMichel Lespinasse RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb, 480315cc066SMichel Lespinasse struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size) 48168ad4a33SUladzislau Rezki (Sony) 48268ad4a33SUladzislau Rezki (Sony) static void purge_vmap_area_lazy(void); 48368ad4a33SUladzislau Rezki (Sony) static BLOCKING_NOTIFIER_HEAD(vmap_notify_list); 48468ad4a33SUladzislau Rezki (Sony) static unsigned long lazy_max_pages(void); 485db64fe02SNick Piggin 48697105f0aSRoman Gushchin static atomic_long_t nr_vmalloc_pages; 48797105f0aSRoman Gushchin 48897105f0aSRoman Gushchin unsigned long vmalloc_nr_pages(void) 48997105f0aSRoman Gushchin { 49097105f0aSRoman Gushchin return atomic_long_read(&nr_vmalloc_pages); 49197105f0aSRoman Gushchin } 49297105f0aSRoman Gushchin 493db64fe02SNick Piggin static struct vmap_area *__find_vmap_area(unsigned long addr) 4941da177e4SLinus Torvalds { 495db64fe02SNick Piggin struct rb_node *n = vmap_area_root.rb_node; 496db64fe02SNick Piggin 497db64fe02SNick Piggin while (n) { 498db64fe02SNick Piggin struct vmap_area *va; 499db64fe02SNick Piggin 500db64fe02SNick Piggin va = rb_entry(n, struct vmap_area, rb_node); 501db64fe02SNick Piggin if (addr < va->va_start) 502db64fe02SNick Piggin n = n->rb_left; 503cef2ac3fSHATAYAMA Daisuke else if (addr >= va->va_end) 504db64fe02SNick Piggin n = n->rb_right; 505db64fe02SNick Piggin else 506db64fe02SNick Piggin return va; 507db64fe02SNick Piggin } 508db64fe02SNick Piggin 509db64fe02SNick Piggin return NULL; 510db64fe02SNick Piggin } 511db64fe02SNick Piggin 51268ad4a33SUladzislau Rezki (Sony) /* 51368ad4a33SUladzislau Rezki (Sony) * This function returns back addresses of parent node 51468ad4a33SUladzislau Rezki (Sony) * and its left or right link for further processing. 51568ad4a33SUladzislau Rezki (Sony) */ 51668ad4a33SUladzislau Rezki (Sony) static __always_inline struct rb_node ** 51768ad4a33SUladzislau Rezki (Sony) find_va_links(struct vmap_area *va, 51868ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct rb_node *from, 51968ad4a33SUladzislau Rezki (Sony) struct rb_node **parent) 520db64fe02SNick Piggin { 521170168d0SNamhyung Kim struct vmap_area *tmp_va; 52268ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 523db64fe02SNick Piggin 52468ad4a33SUladzislau Rezki (Sony) if (root) { 52568ad4a33SUladzislau Rezki (Sony) link = &root->rb_node; 52668ad4a33SUladzislau Rezki (Sony) if (unlikely(!*link)) { 52768ad4a33SUladzislau Rezki (Sony) *parent = NULL; 52868ad4a33SUladzislau Rezki (Sony) return link; 52968ad4a33SUladzislau Rezki (Sony) } 53068ad4a33SUladzislau Rezki (Sony) } else { 53168ad4a33SUladzislau Rezki (Sony) link = &from; 53268ad4a33SUladzislau Rezki (Sony) } 53368ad4a33SUladzislau Rezki (Sony) 53468ad4a33SUladzislau Rezki (Sony) /* 53568ad4a33SUladzislau Rezki (Sony) * Go to the bottom of the tree. When we hit the last point 53668ad4a33SUladzislau Rezki (Sony) * we end up with parent rb_node and correct direction, i name 53768ad4a33SUladzislau Rezki (Sony) * it link, where the new va->rb_node will be attached to. 53868ad4a33SUladzislau Rezki (Sony) */ 53968ad4a33SUladzislau Rezki (Sony) do { 54068ad4a33SUladzislau Rezki (Sony) tmp_va = rb_entry(*link, struct vmap_area, rb_node); 54168ad4a33SUladzislau Rezki (Sony) 54268ad4a33SUladzislau Rezki (Sony) /* 54368ad4a33SUladzislau Rezki (Sony) * During the traversal we also do some sanity check. 54468ad4a33SUladzislau Rezki (Sony) * Trigger the BUG() if there are sides(left/right) 54568ad4a33SUladzislau Rezki (Sony) * or full overlaps. 54668ad4a33SUladzislau Rezki (Sony) */ 54768ad4a33SUladzislau Rezki (Sony) if (va->va_start < tmp_va->va_end && 54868ad4a33SUladzislau Rezki (Sony) va->va_end <= tmp_va->va_start) 54968ad4a33SUladzislau Rezki (Sony) link = &(*link)->rb_left; 55068ad4a33SUladzislau Rezki (Sony) else if (va->va_end > tmp_va->va_start && 55168ad4a33SUladzislau Rezki (Sony) va->va_start >= tmp_va->va_end) 55268ad4a33SUladzislau Rezki (Sony) link = &(*link)->rb_right; 553db64fe02SNick Piggin else 554db64fe02SNick Piggin BUG(); 55568ad4a33SUladzislau Rezki (Sony) } while (*link); 55668ad4a33SUladzislau Rezki (Sony) 55768ad4a33SUladzislau Rezki (Sony) *parent = &tmp_va->rb_node; 55868ad4a33SUladzislau Rezki (Sony) return link; 559db64fe02SNick Piggin } 560db64fe02SNick Piggin 56168ad4a33SUladzislau Rezki (Sony) static __always_inline struct list_head * 56268ad4a33SUladzislau Rezki (Sony) get_va_next_sibling(struct rb_node *parent, struct rb_node **link) 56368ad4a33SUladzislau Rezki (Sony) { 56468ad4a33SUladzislau Rezki (Sony) struct list_head *list; 565db64fe02SNick Piggin 56668ad4a33SUladzislau Rezki (Sony) if (unlikely(!parent)) 56768ad4a33SUladzislau Rezki (Sony) /* 56868ad4a33SUladzislau Rezki (Sony) * The red-black tree where we try to find VA neighbors 56968ad4a33SUladzislau Rezki (Sony) * before merging or inserting is empty, i.e. it means 57068ad4a33SUladzislau Rezki (Sony) * there is no free vmap space. Normally it does not 57168ad4a33SUladzislau Rezki (Sony) * happen but we handle this case anyway. 57268ad4a33SUladzislau Rezki (Sony) */ 57368ad4a33SUladzislau Rezki (Sony) return NULL; 57468ad4a33SUladzislau Rezki (Sony) 57568ad4a33SUladzislau Rezki (Sony) list = &rb_entry(parent, struct vmap_area, rb_node)->list; 57668ad4a33SUladzislau Rezki (Sony) return (&parent->rb_right == link ? list->next : list); 577db64fe02SNick Piggin } 578db64fe02SNick Piggin 57968ad4a33SUladzislau Rezki (Sony) static __always_inline void 58068ad4a33SUladzislau Rezki (Sony) link_va(struct vmap_area *va, struct rb_root *root, 58168ad4a33SUladzislau Rezki (Sony) struct rb_node *parent, struct rb_node **link, struct list_head *head) 58268ad4a33SUladzislau Rezki (Sony) { 58368ad4a33SUladzislau Rezki (Sony) /* 58468ad4a33SUladzislau Rezki (Sony) * VA is still not in the list, but we can 58568ad4a33SUladzislau Rezki (Sony) * identify its future previous list_head node. 58668ad4a33SUladzislau Rezki (Sony) */ 58768ad4a33SUladzislau Rezki (Sony) if (likely(parent)) { 58868ad4a33SUladzislau Rezki (Sony) head = &rb_entry(parent, struct vmap_area, rb_node)->list; 58968ad4a33SUladzislau Rezki (Sony) if (&parent->rb_right != link) 59068ad4a33SUladzislau Rezki (Sony) head = head->prev; 59168ad4a33SUladzislau Rezki (Sony) } 592db64fe02SNick Piggin 59368ad4a33SUladzislau Rezki (Sony) /* Insert to the rb-tree */ 59468ad4a33SUladzislau Rezki (Sony) rb_link_node(&va->rb_node, parent, link); 59568ad4a33SUladzislau Rezki (Sony) if (root == &free_vmap_area_root) { 59668ad4a33SUladzislau Rezki (Sony) /* 59768ad4a33SUladzislau Rezki (Sony) * Some explanation here. Just perform simple insertion 59868ad4a33SUladzislau Rezki (Sony) * to the tree. We do not set va->subtree_max_size to 59968ad4a33SUladzislau Rezki (Sony) * its current size before calling rb_insert_augmented(). 60068ad4a33SUladzislau Rezki (Sony) * It is because of we populate the tree from the bottom 60168ad4a33SUladzislau Rezki (Sony) * to parent levels when the node _is_ in the tree. 60268ad4a33SUladzislau Rezki (Sony) * 60368ad4a33SUladzislau Rezki (Sony) * Therefore we set subtree_max_size to zero after insertion, 60468ad4a33SUladzislau Rezki (Sony) * to let __augment_tree_propagate_from() puts everything to 60568ad4a33SUladzislau Rezki (Sony) * the correct order later on. 60668ad4a33SUladzislau Rezki (Sony) */ 60768ad4a33SUladzislau Rezki (Sony) rb_insert_augmented(&va->rb_node, 60868ad4a33SUladzislau Rezki (Sony) root, &free_vmap_area_rb_augment_cb); 60968ad4a33SUladzislau Rezki (Sony) va->subtree_max_size = 0; 61068ad4a33SUladzislau Rezki (Sony) } else { 61168ad4a33SUladzislau Rezki (Sony) rb_insert_color(&va->rb_node, root); 61268ad4a33SUladzislau Rezki (Sony) } 61368ad4a33SUladzislau Rezki (Sony) 61468ad4a33SUladzislau Rezki (Sony) /* Address-sort this list */ 61568ad4a33SUladzislau Rezki (Sony) list_add(&va->list, head); 61668ad4a33SUladzislau Rezki (Sony) } 61768ad4a33SUladzislau Rezki (Sony) 61868ad4a33SUladzislau Rezki (Sony) static __always_inline void 61968ad4a33SUladzislau Rezki (Sony) unlink_va(struct vmap_area *va, struct rb_root *root) 62068ad4a33SUladzislau Rezki (Sony) { 621460e42d1SUladzislau Rezki (Sony) if (WARN_ON(RB_EMPTY_NODE(&va->rb_node))) 622460e42d1SUladzislau Rezki (Sony) return; 623460e42d1SUladzislau Rezki (Sony) 62468ad4a33SUladzislau Rezki (Sony) if (root == &free_vmap_area_root) 62568ad4a33SUladzislau Rezki (Sony) rb_erase_augmented(&va->rb_node, 62668ad4a33SUladzislau Rezki (Sony) root, &free_vmap_area_rb_augment_cb); 62768ad4a33SUladzislau Rezki (Sony) else 62868ad4a33SUladzislau Rezki (Sony) rb_erase(&va->rb_node, root); 62968ad4a33SUladzislau Rezki (Sony) 63068ad4a33SUladzislau Rezki (Sony) list_del(&va->list); 63168ad4a33SUladzislau Rezki (Sony) RB_CLEAR_NODE(&va->rb_node); 63268ad4a33SUladzislau Rezki (Sony) } 63368ad4a33SUladzislau Rezki (Sony) 634bb850f4dSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_PROPAGATE_CHECK 635bb850f4dSUladzislau Rezki (Sony) static void 636da27c9edSUladzislau Rezki (Sony) augment_tree_propagate_check(void) 637bb850f4dSUladzislau Rezki (Sony) { 638bb850f4dSUladzislau Rezki (Sony) struct vmap_area *va; 639da27c9edSUladzislau Rezki (Sony) unsigned long computed_size; 640bb850f4dSUladzislau Rezki (Sony) 641da27c9edSUladzislau Rezki (Sony) list_for_each_entry(va, &free_vmap_area_list, list) { 642da27c9edSUladzislau Rezki (Sony) computed_size = compute_subtree_max_size(va); 643da27c9edSUladzislau Rezki (Sony) if (computed_size != va->subtree_max_size) 644bb850f4dSUladzislau Rezki (Sony) pr_emerg("tree is corrupted: %lu, %lu\n", 645bb850f4dSUladzislau Rezki (Sony) va_size(va), va->subtree_max_size); 646bb850f4dSUladzislau Rezki (Sony) } 647bb850f4dSUladzislau Rezki (Sony) } 648bb850f4dSUladzislau Rezki (Sony) #endif 649bb850f4dSUladzislau Rezki (Sony) 65068ad4a33SUladzislau Rezki (Sony) /* 65168ad4a33SUladzislau Rezki (Sony) * This function populates subtree_max_size from bottom to upper 65268ad4a33SUladzislau Rezki (Sony) * levels starting from VA point. The propagation must be done 65368ad4a33SUladzislau Rezki (Sony) * when VA size is modified by changing its va_start/va_end. Or 65468ad4a33SUladzislau Rezki (Sony) * in case of newly inserting of VA to the tree. 65568ad4a33SUladzislau Rezki (Sony) * 65668ad4a33SUladzislau Rezki (Sony) * It means that __augment_tree_propagate_from() must be called: 65768ad4a33SUladzislau Rezki (Sony) * - After VA has been inserted to the tree(free path); 65868ad4a33SUladzislau Rezki (Sony) * - After VA has been shrunk(allocation path); 65968ad4a33SUladzislau Rezki (Sony) * - After VA has been increased(merging path). 66068ad4a33SUladzislau Rezki (Sony) * 66168ad4a33SUladzislau Rezki (Sony) * Please note that, it does not mean that upper parent nodes 66268ad4a33SUladzislau Rezki (Sony) * and their subtree_max_size are recalculated all the time up 66368ad4a33SUladzislau Rezki (Sony) * to the root node. 66468ad4a33SUladzislau Rezki (Sony) * 66568ad4a33SUladzislau Rezki (Sony) * 4--8 66668ad4a33SUladzislau Rezki (Sony) * /\ 66768ad4a33SUladzislau Rezki (Sony) * / \ 66868ad4a33SUladzislau Rezki (Sony) * / \ 66968ad4a33SUladzislau Rezki (Sony) * 2--2 8--8 67068ad4a33SUladzislau Rezki (Sony) * 67168ad4a33SUladzislau Rezki (Sony) * For example if we modify the node 4, shrinking it to 2, then 67268ad4a33SUladzislau Rezki (Sony) * no any modification is required. If we shrink the node 2 to 1 67368ad4a33SUladzislau Rezki (Sony) * its subtree_max_size is updated only, and set to 1. If we shrink 67468ad4a33SUladzislau Rezki (Sony) * the node 8 to 6, then its subtree_max_size is set to 6 and parent 67568ad4a33SUladzislau Rezki (Sony) * node becomes 4--6. 67668ad4a33SUladzislau Rezki (Sony) */ 67768ad4a33SUladzislau Rezki (Sony) static __always_inline void 67868ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(struct vmap_area *va) 67968ad4a33SUladzislau Rezki (Sony) { 68068ad4a33SUladzislau Rezki (Sony) /* 681*15ae144fSUladzislau Rezki (Sony) * Populate the tree from bottom towards the root until 682*15ae144fSUladzislau Rezki (Sony) * the calculated maximum available size of checked node 683*15ae144fSUladzislau Rezki (Sony) * is equal to its current one. 68468ad4a33SUladzislau Rezki (Sony) */ 685*15ae144fSUladzislau Rezki (Sony) free_vmap_area_rb_augment_cb_propagate(&va->rb_node, NULL); 686bb850f4dSUladzislau Rezki (Sony) 687bb850f4dSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_PROPAGATE_CHECK 688da27c9edSUladzislau Rezki (Sony) augment_tree_propagate_check(); 689bb850f4dSUladzislau Rezki (Sony) #endif 69068ad4a33SUladzislau Rezki (Sony) } 69168ad4a33SUladzislau Rezki (Sony) 69268ad4a33SUladzislau Rezki (Sony) static void 69368ad4a33SUladzislau Rezki (Sony) insert_vmap_area(struct vmap_area *va, 69468ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 69568ad4a33SUladzislau Rezki (Sony) { 69668ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 69768ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 69868ad4a33SUladzislau Rezki (Sony) 69968ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 70068ad4a33SUladzislau Rezki (Sony) link_va(va, root, parent, link, head); 70168ad4a33SUladzislau Rezki (Sony) } 70268ad4a33SUladzislau Rezki (Sony) 70368ad4a33SUladzislau Rezki (Sony) static void 70468ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(struct vmap_area *va, 70568ad4a33SUladzislau Rezki (Sony) struct rb_node *from, struct rb_root *root, 70668ad4a33SUladzislau Rezki (Sony) struct list_head *head) 70768ad4a33SUladzislau Rezki (Sony) { 70868ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 70968ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 71068ad4a33SUladzislau Rezki (Sony) 71168ad4a33SUladzislau Rezki (Sony) if (from) 71268ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, NULL, from, &parent); 71368ad4a33SUladzislau Rezki (Sony) else 71468ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 71568ad4a33SUladzislau Rezki (Sony) 71668ad4a33SUladzislau Rezki (Sony) link_va(va, root, parent, link, head); 71768ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 71868ad4a33SUladzislau Rezki (Sony) } 71968ad4a33SUladzislau Rezki (Sony) 72068ad4a33SUladzislau Rezki (Sony) /* 72168ad4a33SUladzislau Rezki (Sony) * Merge de-allocated chunk of VA memory with previous 72268ad4a33SUladzislau Rezki (Sony) * and next free blocks. If coalesce is not done a new 72368ad4a33SUladzislau Rezki (Sony) * free area is inserted. If VA has been merged, it is 72468ad4a33SUladzislau Rezki (Sony) * freed. 72568ad4a33SUladzislau Rezki (Sony) */ 7263c5c3cfbSDaniel Axtens static __always_inline struct vmap_area * 72768ad4a33SUladzislau Rezki (Sony) merge_or_add_vmap_area(struct vmap_area *va, 72868ad4a33SUladzislau Rezki (Sony) struct rb_root *root, struct list_head *head) 72968ad4a33SUladzislau Rezki (Sony) { 73068ad4a33SUladzislau Rezki (Sony) struct vmap_area *sibling; 73168ad4a33SUladzislau Rezki (Sony) struct list_head *next; 73268ad4a33SUladzislau Rezki (Sony) struct rb_node **link; 73368ad4a33SUladzislau Rezki (Sony) struct rb_node *parent; 73468ad4a33SUladzislau Rezki (Sony) bool merged = false; 73568ad4a33SUladzislau Rezki (Sony) 73668ad4a33SUladzislau Rezki (Sony) /* 73768ad4a33SUladzislau Rezki (Sony) * Find a place in the tree where VA potentially will be 73868ad4a33SUladzislau Rezki (Sony) * inserted, unless it is merged with its sibling/siblings. 73968ad4a33SUladzislau Rezki (Sony) */ 74068ad4a33SUladzislau Rezki (Sony) link = find_va_links(va, root, NULL, &parent); 74168ad4a33SUladzislau Rezki (Sony) 74268ad4a33SUladzislau Rezki (Sony) /* 74368ad4a33SUladzislau Rezki (Sony) * Get next node of VA to check if merging can be done. 74468ad4a33SUladzislau Rezki (Sony) */ 74568ad4a33SUladzislau Rezki (Sony) next = get_va_next_sibling(parent, link); 74668ad4a33SUladzislau Rezki (Sony) if (unlikely(next == NULL)) 74768ad4a33SUladzislau Rezki (Sony) goto insert; 74868ad4a33SUladzislau Rezki (Sony) 74968ad4a33SUladzislau Rezki (Sony) /* 75068ad4a33SUladzislau Rezki (Sony) * start end 75168ad4a33SUladzislau Rezki (Sony) * | | 75268ad4a33SUladzislau Rezki (Sony) * |<------VA------>|<-----Next----->| 75368ad4a33SUladzislau Rezki (Sony) * | | 75468ad4a33SUladzislau Rezki (Sony) * start end 75568ad4a33SUladzislau Rezki (Sony) */ 75668ad4a33SUladzislau Rezki (Sony) if (next != head) { 75768ad4a33SUladzislau Rezki (Sony) sibling = list_entry(next, struct vmap_area, list); 75868ad4a33SUladzislau Rezki (Sony) if (sibling->va_start == va->va_end) { 75968ad4a33SUladzislau Rezki (Sony) sibling->va_start = va->va_start; 76068ad4a33SUladzislau Rezki (Sony) 76168ad4a33SUladzislau Rezki (Sony) /* Free vmap_area object. */ 76268ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 76368ad4a33SUladzislau Rezki (Sony) 76468ad4a33SUladzislau Rezki (Sony) /* Point to the new merged area. */ 76568ad4a33SUladzislau Rezki (Sony) va = sibling; 76668ad4a33SUladzislau Rezki (Sony) merged = true; 76768ad4a33SUladzislau Rezki (Sony) } 76868ad4a33SUladzislau Rezki (Sony) } 76968ad4a33SUladzislau Rezki (Sony) 77068ad4a33SUladzislau Rezki (Sony) /* 77168ad4a33SUladzislau Rezki (Sony) * start end 77268ad4a33SUladzislau Rezki (Sony) * | | 77368ad4a33SUladzislau Rezki (Sony) * |<-----Prev----->|<------VA------>| 77468ad4a33SUladzislau Rezki (Sony) * | | 77568ad4a33SUladzislau Rezki (Sony) * start end 77668ad4a33SUladzislau Rezki (Sony) */ 77768ad4a33SUladzislau Rezki (Sony) if (next->prev != head) { 77868ad4a33SUladzislau Rezki (Sony) sibling = list_entry(next->prev, struct vmap_area, list); 77968ad4a33SUladzislau Rezki (Sony) if (sibling->va_end == va->va_start) { 7805dd78640SUladzislau Rezki (Sony) /* 7815dd78640SUladzislau Rezki (Sony) * If both neighbors are coalesced, it is important 7825dd78640SUladzislau Rezki (Sony) * to unlink the "next" node first, followed by merging 7835dd78640SUladzislau Rezki (Sony) * with "previous" one. Otherwise the tree might not be 7845dd78640SUladzislau Rezki (Sony) * fully populated if a sibling's augmented value is 7855dd78640SUladzislau Rezki (Sony) * "normalized" because of rotation operations. 7865dd78640SUladzislau Rezki (Sony) */ 78754f63d9dSUladzislau Rezki (Sony) if (merged) 78868ad4a33SUladzislau Rezki (Sony) unlink_va(va, root); 78968ad4a33SUladzislau Rezki (Sony) 7905dd78640SUladzislau Rezki (Sony) sibling->va_end = va->va_end; 7915dd78640SUladzislau Rezki (Sony) 79268ad4a33SUladzislau Rezki (Sony) /* Free vmap_area object. */ 79368ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 7943c5c3cfbSDaniel Axtens 7953c5c3cfbSDaniel Axtens /* Point to the new merged area. */ 7963c5c3cfbSDaniel Axtens va = sibling; 7973c5c3cfbSDaniel Axtens merged = true; 79868ad4a33SUladzislau Rezki (Sony) } 79968ad4a33SUladzislau Rezki (Sony) } 80068ad4a33SUladzislau Rezki (Sony) 80168ad4a33SUladzislau Rezki (Sony) insert: 8025dd78640SUladzislau Rezki (Sony) if (!merged) 80368ad4a33SUladzislau Rezki (Sony) link_va(va, root, parent, link, head); 8043c5c3cfbSDaniel Axtens 8055dd78640SUladzislau Rezki (Sony) /* 8065dd78640SUladzislau Rezki (Sony) * Last step is to check and update the tree. 8075dd78640SUladzislau Rezki (Sony) */ 8085dd78640SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 8093c5c3cfbSDaniel Axtens return va; 81068ad4a33SUladzislau Rezki (Sony) } 81168ad4a33SUladzislau Rezki (Sony) 81268ad4a33SUladzislau Rezki (Sony) static __always_inline bool 81368ad4a33SUladzislau Rezki (Sony) is_within_this_va(struct vmap_area *va, unsigned long size, 81468ad4a33SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 81568ad4a33SUladzislau Rezki (Sony) { 81668ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr; 81768ad4a33SUladzislau Rezki (Sony) 81868ad4a33SUladzislau Rezki (Sony) if (va->va_start > vstart) 81968ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(va->va_start, align); 82068ad4a33SUladzislau Rezki (Sony) else 82168ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(vstart, align); 82268ad4a33SUladzislau Rezki (Sony) 82368ad4a33SUladzislau Rezki (Sony) /* Can be overflowed due to big size or alignment. */ 82468ad4a33SUladzislau Rezki (Sony) if (nva_start_addr + size < nva_start_addr || 82568ad4a33SUladzislau Rezki (Sony) nva_start_addr < vstart) 82668ad4a33SUladzislau Rezki (Sony) return false; 82768ad4a33SUladzislau Rezki (Sony) 82868ad4a33SUladzislau Rezki (Sony) return (nva_start_addr + size <= va->va_end); 82968ad4a33SUladzislau Rezki (Sony) } 83068ad4a33SUladzislau Rezki (Sony) 83168ad4a33SUladzislau Rezki (Sony) /* 83268ad4a33SUladzislau Rezki (Sony) * Find the first free block(lowest start address) in the tree, 83368ad4a33SUladzislau Rezki (Sony) * that will accomplish the request corresponding to passing 83468ad4a33SUladzislau Rezki (Sony) * parameters. 83568ad4a33SUladzislau Rezki (Sony) */ 83668ad4a33SUladzislau Rezki (Sony) static __always_inline struct vmap_area * 83768ad4a33SUladzislau Rezki (Sony) find_vmap_lowest_match(unsigned long size, 83868ad4a33SUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 83968ad4a33SUladzislau Rezki (Sony) { 84068ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 84168ad4a33SUladzislau Rezki (Sony) struct rb_node *node; 84268ad4a33SUladzislau Rezki (Sony) unsigned long length; 84368ad4a33SUladzislau Rezki (Sony) 84468ad4a33SUladzislau Rezki (Sony) /* Start from the root. */ 84568ad4a33SUladzislau Rezki (Sony) node = free_vmap_area_root.rb_node; 84668ad4a33SUladzislau Rezki (Sony) 84768ad4a33SUladzislau Rezki (Sony) /* Adjust the search size for alignment overhead. */ 84868ad4a33SUladzislau Rezki (Sony) length = size + align - 1; 84968ad4a33SUladzislau Rezki (Sony) 85068ad4a33SUladzislau Rezki (Sony) while (node) { 85168ad4a33SUladzislau Rezki (Sony) va = rb_entry(node, struct vmap_area, rb_node); 85268ad4a33SUladzislau Rezki (Sony) 85368ad4a33SUladzislau Rezki (Sony) if (get_subtree_max_size(node->rb_left) >= length && 85468ad4a33SUladzislau Rezki (Sony) vstart < va->va_start) { 85568ad4a33SUladzislau Rezki (Sony) node = node->rb_left; 85668ad4a33SUladzislau Rezki (Sony) } else { 85768ad4a33SUladzislau Rezki (Sony) if (is_within_this_va(va, size, align, vstart)) 85868ad4a33SUladzislau Rezki (Sony) return va; 85968ad4a33SUladzislau Rezki (Sony) 86068ad4a33SUladzislau Rezki (Sony) /* 86168ad4a33SUladzislau Rezki (Sony) * Does not make sense to go deeper towards the right 86268ad4a33SUladzislau Rezki (Sony) * sub-tree if it does not have a free block that is 86368ad4a33SUladzislau Rezki (Sony) * equal or bigger to the requested search length. 86468ad4a33SUladzislau Rezki (Sony) */ 86568ad4a33SUladzislau Rezki (Sony) if (get_subtree_max_size(node->rb_right) >= length) { 86668ad4a33SUladzislau Rezki (Sony) node = node->rb_right; 86768ad4a33SUladzislau Rezki (Sony) continue; 86868ad4a33SUladzislau Rezki (Sony) } 86968ad4a33SUladzislau Rezki (Sony) 87068ad4a33SUladzislau Rezki (Sony) /* 8713806b041SAndrew Morton * OK. We roll back and find the first right sub-tree, 87268ad4a33SUladzislau Rezki (Sony) * that will satisfy the search criteria. It can happen 87368ad4a33SUladzislau Rezki (Sony) * only once due to "vstart" restriction. 87468ad4a33SUladzislau Rezki (Sony) */ 87568ad4a33SUladzislau Rezki (Sony) while ((node = rb_parent(node))) { 87668ad4a33SUladzislau Rezki (Sony) va = rb_entry(node, struct vmap_area, rb_node); 87768ad4a33SUladzislau Rezki (Sony) if (is_within_this_va(va, size, align, vstart)) 87868ad4a33SUladzislau Rezki (Sony) return va; 87968ad4a33SUladzislau Rezki (Sony) 88068ad4a33SUladzislau Rezki (Sony) if (get_subtree_max_size(node->rb_right) >= length && 88168ad4a33SUladzislau Rezki (Sony) vstart <= va->va_start) { 88268ad4a33SUladzislau Rezki (Sony) node = node->rb_right; 88368ad4a33SUladzislau Rezki (Sony) break; 88468ad4a33SUladzislau Rezki (Sony) } 88568ad4a33SUladzislau Rezki (Sony) } 88668ad4a33SUladzislau Rezki (Sony) } 88768ad4a33SUladzislau Rezki (Sony) } 88868ad4a33SUladzislau Rezki (Sony) 88968ad4a33SUladzislau Rezki (Sony) return NULL; 89068ad4a33SUladzislau Rezki (Sony) } 89168ad4a33SUladzislau Rezki (Sony) 892a6cf4e0fSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 893a6cf4e0fSUladzislau Rezki (Sony) #include <linux/random.h> 894a6cf4e0fSUladzislau Rezki (Sony) 895a6cf4e0fSUladzislau Rezki (Sony) static struct vmap_area * 896a6cf4e0fSUladzislau Rezki (Sony) find_vmap_lowest_linear_match(unsigned long size, 897a6cf4e0fSUladzislau Rezki (Sony) unsigned long align, unsigned long vstart) 898a6cf4e0fSUladzislau Rezki (Sony) { 899a6cf4e0fSUladzislau Rezki (Sony) struct vmap_area *va; 900a6cf4e0fSUladzislau Rezki (Sony) 901a6cf4e0fSUladzislau Rezki (Sony) list_for_each_entry(va, &free_vmap_area_list, list) { 902a6cf4e0fSUladzislau Rezki (Sony) if (!is_within_this_va(va, size, align, vstart)) 903a6cf4e0fSUladzislau Rezki (Sony) continue; 904a6cf4e0fSUladzislau Rezki (Sony) 905a6cf4e0fSUladzislau Rezki (Sony) return va; 906a6cf4e0fSUladzislau Rezki (Sony) } 907a6cf4e0fSUladzislau Rezki (Sony) 908a6cf4e0fSUladzislau Rezki (Sony) return NULL; 909a6cf4e0fSUladzislau Rezki (Sony) } 910a6cf4e0fSUladzislau Rezki (Sony) 911a6cf4e0fSUladzislau Rezki (Sony) static void 912a6cf4e0fSUladzislau Rezki (Sony) find_vmap_lowest_match_check(unsigned long size) 913a6cf4e0fSUladzislau Rezki (Sony) { 914a6cf4e0fSUladzislau Rezki (Sony) struct vmap_area *va_1, *va_2; 915a6cf4e0fSUladzislau Rezki (Sony) unsigned long vstart; 916a6cf4e0fSUladzislau Rezki (Sony) unsigned int rnd; 917a6cf4e0fSUladzislau Rezki (Sony) 918a6cf4e0fSUladzislau Rezki (Sony) get_random_bytes(&rnd, sizeof(rnd)); 919a6cf4e0fSUladzislau Rezki (Sony) vstart = VMALLOC_START + rnd; 920a6cf4e0fSUladzislau Rezki (Sony) 921a6cf4e0fSUladzislau Rezki (Sony) va_1 = find_vmap_lowest_match(size, 1, vstart); 922a6cf4e0fSUladzislau Rezki (Sony) va_2 = find_vmap_lowest_linear_match(size, 1, vstart); 923a6cf4e0fSUladzislau Rezki (Sony) 924a6cf4e0fSUladzislau Rezki (Sony) if (va_1 != va_2) 925a6cf4e0fSUladzislau Rezki (Sony) pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n", 926a6cf4e0fSUladzislau Rezki (Sony) va_1, va_2, vstart); 927a6cf4e0fSUladzislau Rezki (Sony) } 928a6cf4e0fSUladzislau Rezki (Sony) #endif 929a6cf4e0fSUladzislau Rezki (Sony) 93068ad4a33SUladzislau Rezki (Sony) enum fit_type { 93168ad4a33SUladzislau Rezki (Sony) NOTHING_FIT = 0, 93268ad4a33SUladzislau Rezki (Sony) FL_FIT_TYPE = 1, /* full fit */ 93368ad4a33SUladzislau Rezki (Sony) LE_FIT_TYPE = 2, /* left edge fit */ 93468ad4a33SUladzislau Rezki (Sony) RE_FIT_TYPE = 3, /* right edge fit */ 93568ad4a33SUladzislau Rezki (Sony) NE_FIT_TYPE = 4 /* no edge fit */ 93668ad4a33SUladzislau Rezki (Sony) }; 93768ad4a33SUladzislau Rezki (Sony) 93868ad4a33SUladzislau Rezki (Sony) static __always_inline enum fit_type 93968ad4a33SUladzislau Rezki (Sony) classify_va_fit_type(struct vmap_area *va, 94068ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr, unsigned long size) 94168ad4a33SUladzislau Rezki (Sony) { 94268ad4a33SUladzislau Rezki (Sony) enum fit_type type; 94368ad4a33SUladzislau Rezki (Sony) 94468ad4a33SUladzislau Rezki (Sony) /* Check if it is within VA. */ 94568ad4a33SUladzislau Rezki (Sony) if (nva_start_addr < va->va_start || 94668ad4a33SUladzislau Rezki (Sony) nva_start_addr + size > va->va_end) 94768ad4a33SUladzislau Rezki (Sony) return NOTHING_FIT; 94868ad4a33SUladzislau Rezki (Sony) 94968ad4a33SUladzislau Rezki (Sony) /* Now classify. */ 95068ad4a33SUladzislau Rezki (Sony) if (va->va_start == nva_start_addr) { 95168ad4a33SUladzislau Rezki (Sony) if (va->va_end == nva_start_addr + size) 95268ad4a33SUladzislau Rezki (Sony) type = FL_FIT_TYPE; 95368ad4a33SUladzislau Rezki (Sony) else 95468ad4a33SUladzislau Rezki (Sony) type = LE_FIT_TYPE; 95568ad4a33SUladzislau Rezki (Sony) } else if (va->va_end == nva_start_addr + size) { 95668ad4a33SUladzislau Rezki (Sony) type = RE_FIT_TYPE; 95768ad4a33SUladzislau Rezki (Sony) } else { 95868ad4a33SUladzislau Rezki (Sony) type = NE_FIT_TYPE; 95968ad4a33SUladzislau Rezki (Sony) } 96068ad4a33SUladzislau Rezki (Sony) 96168ad4a33SUladzislau Rezki (Sony) return type; 96268ad4a33SUladzislau Rezki (Sony) } 96368ad4a33SUladzislau Rezki (Sony) 96468ad4a33SUladzislau Rezki (Sony) static __always_inline int 96568ad4a33SUladzislau Rezki (Sony) adjust_va_to_fit_type(struct vmap_area *va, 96668ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr, unsigned long size, 96768ad4a33SUladzislau Rezki (Sony) enum fit_type type) 96868ad4a33SUladzislau Rezki (Sony) { 9692c929233SArnd Bergmann struct vmap_area *lva = NULL; 97068ad4a33SUladzislau Rezki (Sony) 97168ad4a33SUladzislau Rezki (Sony) if (type == FL_FIT_TYPE) { 97268ad4a33SUladzislau Rezki (Sony) /* 97368ad4a33SUladzislau Rezki (Sony) * No need to split VA, it fully fits. 97468ad4a33SUladzislau Rezki (Sony) * 97568ad4a33SUladzislau Rezki (Sony) * | | 97668ad4a33SUladzislau Rezki (Sony) * V NVA V 97768ad4a33SUladzislau Rezki (Sony) * |---------------| 97868ad4a33SUladzislau Rezki (Sony) */ 97968ad4a33SUladzislau Rezki (Sony) unlink_va(va, &free_vmap_area_root); 98068ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 98168ad4a33SUladzislau Rezki (Sony) } else if (type == LE_FIT_TYPE) { 98268ad4a33SUladzislau Rezki (Sony) /* 98368ad4a33SUladzislau Rezki (Sony) * Split left edge of fit VA. 98468ad4a33SUladzislau Rezki (Sony) * 98568ad4a33SUladzislau Rezki (Sony) * | | 98668ad4a33SUladzislau Rezki (Sony) * V NVA V R 98768ad4a33SUladzislau Rezki (Sony) * |-------|-------| 98868ad4a33SUladzislau Rezki (Sony) */ 98968ad4a33SUladzislau Rezki (Sony) va->va_start += size; 99068ad4a33SUladzislau Rezki (Sony) } else if (type == RE_FIT_TYPE) { 99168ad4a33SUladzislau Rezki (Sony) /* 99268ad4a33SUladzislau Rezki (Sony) * Split right edge of fit VA. 99368ad4a33SUladzislau Rezki (Sony) * 99468ad4a33SUladzislau Rezki (Sony) * | | 99568ad4a33SUladzislau Rezki (Sony) * L V NVA V 99668ad4a33SUladzislau Rezki (Sony) * |-------|-------| 99768ad4a33SUladzislau Rezki (Sony) */ 99868ad4a33SUladzislau Rezki (Sony) va->va_end = nva_start_addr; 99968ad4a33SUladzislau Rezki (Sony) } else if (type == NE_FIT_TYPE) { 100068ad4a33SUladzislau Rezki (Sony) /* 100168ad4a33SUladzislau Rezki (Sony) * Split no edge of fit VA. 100268ad4a33SUladzislau Rezki (Sony) * 100368ad4a33SUladzislau Rezki (Sony) * | | 100468ad4a33SUladzislau Rezki (Sony) * L V NVA V R 100568ad4a33SUladzislau Rezki (Sony) * |---|-------|---| 100668ad4a33SUladzislau Rezki (Sony) */ 100782dd23e8SUladzislau Rezki (Sony) lva = __this_cpu_xchg(ne_fit_preload_node, NULL); 100882dd23e8SUladzislau Rezki (Sony) if (unlikely(!lva)) { 100982dd23e8SUladzislau Rezki (Sony) /* 101082dd23e8SUladzislau Rezki (Sony) * For percpu allocator we do not do any pre-allocation 101182dd23e8SUladzislau Rezki (Sony) * and leave it as it is. The reason is it most likely 101282dd23e8SUladzislau Rezki (Sony) * never ends up with NE_FIT_TYPE splitting. In case of 101382dd23e8SUladzislau Rezki (Sony) * percpu allocations offsets and sizes are aligned to 101482dd23e8SUladzislau Rezki (Sony) * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE 101582dd23e8SUladzislau Rezki (Sony) * are its main fitting cases. 101682dd23e8SUladzislau Rezki (Sony) * 101782dd23e8SUladzislau Rezki (Sony) * There are a few exceptions though, as an example it is 101882dd23e8SUladzislau Rezki (Sony) * a first allocation (early boot up) when we have "one" 101982dd23e8SUladzislau Rezki (Sony) * big free space that has to be split. 1020060650a2SUladzislau Rezki (Sony) * 1021060650a2SUladzislau Rezki (Sony) * Also we can hit this path in case of regular "vmap" 1022060650a2SUladzislau Rezki (Sony) * allocations, if "this" current CPU was not preloaded. 1023060650a2SUladzislau Rezki (Sony) * See the comment in alloc_vmap_area() why. If so, then 1024060650a2SUladzislau Rezki (Sony) * GFP_NOWAIT is used instead to get an extra object for 1025060650a2SUladzislau Rezki (Sony) * split purpose. That is rare and most time does not 1026060650a2SUladzislau Rezki (Sony) * occur. 1027060650a2SUladzislau Rezki (Sony) * 1028060650a2SUladzislau Rezki (Sony) * What happens if an allocation gets failed. Basically, 1029060650a2SUladzislau Rezki (Sony) * an "overflow" path is triggered to purge lazily freed 1030060650a2SUladzislau Rezki (Sony) * areas to free some memory, then, the "retry" path is 1031060650a2SUladzislau Rezki (Sony) * triggered to repeat one more time. See more details 1032060650a2SUladzislau Rezki (Sony) * in alloc_vmap_area() function. 103382dd23e8SUladzislau Rezki (Sony) */ 103468ad4a33SUladzislau Rezki (Sony) lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT); 103582dd23e8SUladzislau Rezki (Sony) if (!lva) 103668ad4a33SUladzislau Rezki (Sony) return -1; 103782dd23e8SUladzislau Rezki (Sony) } 103868ad4a33SUladzislau Rezki (Sony) 103968ad4a33SUladzislau Rezki (Sony) /* 104068ad4a33SUladzislau Rezki (Sony) * Build the remainder. 104168ad4a33SUladzislau Rezki (Sony) */ 104268ad4a33SUladzislau Rezki (Sony) lva->va_start = va->va_start; 104368ad4a33SUladzislau Rezki (Sony) lva->va_end = nva_start_addr; 104468ad4a33SUladzislau Rezki (Sony) 104568ad4a33SUladzislau Rezki (Sony) /* 104668ad4a33SUladzislau Rezki (Sony) * Shrink this VA to remaining size. 104768ad4a33SUladzislau Rezki (Sony) */ 104868ad4a33SUladzislau Rezki (Sony) va->va_start = nva_start_addr + size; 104968ad4a33SUladzislau Rezki (Sony) } else { 105068ad4a33SUladzislau Rezki (Sony) return -1; 105168ad4a33SUladzislau Rezki (Sony) } 105268ad4a33SUladzislau Rezki (Sony) 105368ad4a33SUladzislau Rezki (Sony) if (type != FL_FIT_TYPE) { 105468ad4a33SUladzislau Rezki (Sony) augment_tree_propagate_from(va); 105568ad4a33SUladzislau Rezki (Sony) 10562c929233SArnd Bergmann if (lva) /* type == NE_FIT_TYPE */ 105768ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(lva, &va->rb_node, 105868ad4a33SUladzislau Rezki (Sony) &free_vmap_area_root, &free_vmap_area_list); 105968ad4a33SUladzislau Rezki (Sony) } 106068ad4a33SUladzislau Rezki (Sony) 106168ad4a33SUladzislau Rezki (Sony) return 0; 106268ad4a33SUladzislau Rezki (Sony) } 106368ad4a33SUladzislau Rezki (Sony) 106468ad4a33SUladzislau Rezki (Sony) /* 106568ad4a33SUladzislau Rezki (Sony) * Returns a start address of the newly allocated area, if success. 106668ad4a33SUladzislau Rezki (Sony) * Otherwise a vend is returned that indicates failure. 106768ad4a33SUladzislau Rezki (Sony) */ 106868ad4a33SUladzislau Rezki (Sony) static __always_inline unsigned long 106968ad4a33SUladzislau Rezki (Sony) __alloc_vmap_area(unsigned long size, unsigned long align, 1070cacca6baSUladzislau Rezki (Sony) unsigned long vstart, unsigned long vend) 107168ad4a33SUladzislau Rezki (Sony) { 107268ad4a33SUladzislau Rezki (Sony) unsigned long nva_start_addr; 107368ad4a33SUladzislau Rezki (Sony) struct vmap_area *va; 107468ad4a33SUladzislau Rezki (Sony) enum fit_type type; 107568ad4a33SUladzislau Rezki (Sony) int ret; 107668ad4a33SUladzislau Rezki (Sony) 107768ad4a33SUladzislau Rezki (Sony) va = find_vmap_lowest_match(size, align, vstart); 107868ad4a33SUladzislau Rezki (Sony) if (unlikely(!va)) 107968ad4a33SUladzislau Rezki (Sony) return vend; 108068ad4a33SUladzislau Rezki (Sony) 108168ad4a33SUladzislau Rezki (Sony) if (va->va_start > vstart) 108268ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(va->va_start, align); 108368ad4a33SUladzislau Rezki (Sony) else 108468ad4a33SUladzislau Rezki (Sony) nva_start_addr = ALIGN(vstart, align); 108568ad4a33SUladzislau Rezki (Sony) 108668ad4a33SUladzislau Rezki (Sony) /* Check the "vend" restriction. */ 108768ad4a33SUladzislau Rezki (Sony) if (nva_start_addr + size > vend) 108868ad4a33SUladzislau Rezki (Sony) return vend; 108968ad4a33SUladzislau Rezki (Sony) 109068ad4a33SUladzislau Rezki (Sony) /* Classify what we have found. */ 109168ad4a33SUladzislau Rezki (Sony) type = classify_va_fit_type(va, nva_start_addr, size); 109268ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(type == NOTHING_FIT)) 109368ad4a33SUladzislau Rezki (Sony) return vend; 109468ad4a33SUladzislau Rezki (Sony) 109568ad4a33SUladzislau Rezki (Sony) /* Update the free vmap_area. */ 109668ad4a33SUladzislau Rezki (Sony) ret = adjust_va_to_fit_type(va, nva_start_addr, size, type); 109768ad4a33SUladzislau Rezki (Sony) if (ret) 109868ad4a33SUladzislau Rezki (Sony) return vend; 109968ad4a33SUladzislau Rezki (Sony) 1100a6cf4e0fSUladzislau Rezki (Sony) #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 1101a6cf4e0fSUladzislau Rezki (Sony) find_vmap_lowest_match_check(size); 1102a6cf4e0fSUladzislau Rezki (Sony) #endif 1103a6cf4e0fSUladzislau Rezki (Sony) 110468ad4a33SUladzislau Rezki (Sony) return nva_start_addr; 110568ad4a33SUladzislau Rezki (Sony) } 11064da56b99SChris Wilson 1107db64fe02SNick Piggin /* 1108d98c9e83SAndrey Ryabinin * Free a region of KVA allocated by alloc_vmap_area 1109d98c9e83SAndrey Ryabinin */ 1110d98c9e83SAndrey Ryabinin static void free_vmap_area(struct vmap_area *va) 1111d98c9e83SAndrey Ryabinin { 1112d98c9e83SAndrey Ryabinin /* 1113d98c9e83SAndrey Ryabinin * Remove from the busy tree/list. 1114d98c9e83SAndrey Ryabinin */ 1115d98c9e83SAndrey Ryabinin spin_lock(&vmap_area_lock); 1116d98c9e83SAndrey Ryabinin unlink_va(va, &vmap_area_root); 1117d98c9e83SAndrey Ryabinin spin_unlock(&vmap_area_lock); 1118d98c9e83SAndrey Ryabinin 1119d98c9e83SAndrey Ryabinin /* 1120d98c9e83SAndrey Ryabinin * Insert/Merge it back to the free tree/list. 1121d98c9e83SAndrey Ryabinin */ 1122d98c9e83SAndrey Ryabinin spin_lock(&free_vmap_area_lock); 1123d98c9e83SAndrey Ryabinin merge_or_add_vmap_area(va, &free_vmap_area_root, &free_vmap_area_list); 1124d98c9e83SAndrey Ryabinin spin_unlock(&free_vmap_area_lock); 1125d98c9e83SAndrey Ryabinin } 1126d98c9e83SAndrey Ryabinin 1127d98c9e83SAndrey Ryabinin /* 1128db64fe02SNick Piggin * Allocate a region of KVA of the specified size and alignment, within the 1129db64fe02SNick Piggin * vstart and vend. 1130db64fe02SNick Piggin */ 1131db64fe02SNick Piggin static struct vmap_area *alloc_vmap_area(unsigned long size, 1132db64fe02SNick Piggin unsigned long align, 1133db64fe02SNick Piggin unsigned long vstart, unsigned long vend, 1134db64fe02SNick Piggin int node, gfp_t gfp_mask) 1135db64fe02SNick Piggin { 113682dd23e8SUladzislau Rezki (Sony) struct vmap_area *va, *pva; 11371da177e4SLinus Torvalds unsigned long addr; 1138db64fe02SNick Piggin int purged = 0; 1139d98c9e83SAndrey Ryabinin int ret; 1140db64fe02SNick Piggin 11417766970cSNick Piggin BUG_ON(!size); 1142891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 114389699605SNick Piggin BUG_ON(!is_power_of_2(align)); 1144db64fe02SNick Piggin 114568ad4a33SUladzislau Rezki (Sony) if (unlikely(!vmap_initialized)) 114668ad4a33SUladzislau Rezki (Sony) return ERR_PTR(-EBUSY); 114768ad4a33SUladzislau Rezki (Sony) 11485803ed29SChristoph Hellwig might_sleep(); 1149f07116d7SUladzislau Rezki (Sony) gfp_mask = gfp_mask & GFP_RECLAIM_MASK; 11504da56b99SChris Wilson 1151f07116d7SUladzislau Rezki (Sony) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 1152db64fe02SNick Piggin if (unlikely(!va)) 1153db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 1154db64fe02SNick Piggin 11557f88f88fSCatalin Marinas /* 11567f88f88fSCatalin Marinas * Only scan the relevant parts containing pointers to other objects 11577f88f88fSCatalin Marinas * to avoid false negatives. 11587f88f88fSCatalin Marinas */ 1159f07116d7SUladzislau Rezki (Sony) kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask); 11607f88f88fSCatalin Marinas 1161db64fe02SNick Piggin retry: 116282dd23e8SUladzislau Rezki (Sony) /* 116381f1ba58SUladzislau Rezki (Sony) * Preload this CPU with one extra vmap_area object. It is used 116481f1ba58SUladzislau Rezki (Sony) * when fit type of free area is NE_FIT_TYPE. Please note, it 116581f1ba58SUladzislau Rezki (Sony) * does not guarantee that an allocation occurs on a CPU that 116681f1ba58SUladzislau Rezki (Sony) * is preloaded, instead we minimize the case when it is not. 116781f1ba58SUladzislau Rezki (Sony) * It can happen because of cpu migration, because there is a 116881f1ba58SUladzislau Rezki (Sony) * race until the below spinlock is taken. 116982dd23e8SUladzislau Rezki (Sony) * 117082dd23e8SUladzislau Rezki (Sony) * The preload is done in non-atomic context, thus it allows us 117182dd23e8SUladzislau Rezki (Sony) * to use more permissive allocation masks to be more stable under 117281f1ba58SUladzislau Rezki (Sony) * low memory condition and high memory pressure. In rare case, 117381f1ba58SUladzislau Rezki (Sony) * if not preloaded, GFP_NOWAIT is used. 117482dd23e8SUladzislau Rezki (Sony) * 117581f1ba58SUladzislau Rezki (Sony) * Set "pva" to NULL here, because of "retry" path. 117682dd23e8SUladzislau Rezki (Sony) */ 117781f1ba58SUladzislau Rezki (Sony) pva = NULL; 117882dd23e8SUladzislau Rezki (Sony) 117981f1ba58SUladzislau Rezki (Sony) if (!this_cpu_read(ne_fit_preload_node)) 118081f1ba58SUladzislau Rezki (Sony) /* 118181f1ba58SUladzislau Rezki (Sony) * Even if it fails we do not really care about that. 118281f1ba58SUladzislau Rezki (Sony) * Just proceed as it is. If needed "overflow" path 118381f1ba58SUladzislau Rezki (Sony) * will refill the cache we allocate from. 118481f1ba58SUladzislau Rezki (Sony) */ 1185f07116d7SUladzislau Rezki (Sony) pva = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 118682dd23e8SUladzislau Rezki (Sony) 1187e36176beSUladzislau Rezki (Sony) spin_lock(&free_vmap_area_lock); 118881f1ba58SUladzislau Rezki (Sony) 118981f1ba58SUladzislau Rezki (Sony) if (pva && __this_cpu_cmpxchg(ne_fit_preload_node, NULL, pva)) 119081f1ba58SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, pva); 119168ad4a33SUladzislau Rezki (Sony) 119289699605SNick Piggin /* 119368ad4a33SUladzislau Rezki (Sony) * If an allocation fails, the "vend" address is 119468ad4a33SUladzislau Rezki (Sony) * returned. Therefore trigger the overflow path. 119589699605SNick Piggin */ 1196cacca6baSUladzislau Rezki (Sony) addr = __alloc_vmap_area(size, align, vstart, vend); 1197e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 1198e36176beSUladzislau Rezki (Sony) 119968ad4a33SUladzislau Rezki (Sony) if (unlikely(addr == vend)) 120089699605SNick Piggin goto overflow; 120189699605SNick Piggin 120289699605SNick Piggin va->va_start = addr; 120389699605SNick Piggin va->va_end = addr + size; 1204688fcbfcSPengfei Li va->vm = NULL; 120568ad4a33SUladzislau Rezki (Sony) 1206d98c9e83SAndrey Ryabinin 1207e36176beSUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 1208e36176beSUladzislau Rezki (Sony) insert_vmap_area(va, &vmap_area_root, &vmap_area_list); 120989699605SNick Piggin spin_unlock(&vmap_area_lock); 121089699605SNick Piggin 121161e16557SWang Xiaoqiang BUG_ON(!IS_ALIGNED(va->va_start, align)); 121289699605SNick Piggin BUG_ON(va->va_start < vstart); 121389699605SNick Piggin BUG_ON(va->va_end > vend); 121489699605SNick Piggin 1215d98c9e83SAndrey Ryabinin ret = kasan_populate_vmalloc(addr, size); 1216d98c9e83SAndrey Ryabinin if (ret) { 1217d98c9e83SAndrey Ryabinin free_vmap_area(va); 1218d98c9e83SAndrey Ryabinin return ERR_PTR(ret); 1219d98c9e83SAndrey Ryabinin } 1220d98c9e83SAndrey Ryabinin 122189699605SNick Piggin return va; 122289699605SNick Piggin 12237766970cSNick Piggin overflow: 1224db64fe02SNick Piggin if (!purged) { 1225db64fe02SNick Piggin purge_vmap_area_lazy(); 1226db64fe02SNick Piggin purged = 1; 1227db64fe02SNick Piggin goto retry; 1228db64fe02SNick Piggin } 12294da56b99SChris Wilson 12304da56b99SChris Wilson if (gfpflags_allow_blocking(gfp_mask)) { 12314da56b99SChris Wilson unsigned long freed = 0; 12324da56b99SChris Wilson blocking_notifier_call_chain(&vmap_notify_list, 0, &freed); 12334da56b99SChris Wilson if (freed > 0) { 12344da56b99SChris Wilson purged = 0; 12354da56b99SChris Wilson goto retry; 12364da56b99SChris Wilson } 12374da56b99SChris Wilson } 12384da56b99SChris Wilson 123903497d76SFlorian Fainelli if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) 1240756a025fSJoe Perches pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n", 1241756a025fSJoe Perches size); 124268ad4a33SUladzislau Rezki (Sony) 124368ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, va); 1244db64fe02SNick Piggin return ERR_PTR(-EBUSY); 1245db64fe02SNick Piggin } 1246db64fe02SNick Piggin 12474da56b99SChris Wilson int register_vmap_purge_notifier(struct notifier_block *nb) 12484da56b99SChris Wilson { 12494da56b99SChris Wilson return blocking_notifier_chain_register(&vmap_notify_list, nb); 12504da56b99SChris Wilson } 12514da56b99SChris Wilson EXPORT_SYMBOL_GPL(register_vmap_purge_notifier); 12524da56b99SChris Wilson 12534da56b99SChris Wilson int unregister_vmap_purge_notifier(struct notifier_block *nb) 12544da56b99SChris Wilson { 12554da56b99SChris Wilson return blocking_notifier_chain_unregister(&vmap_notify_list, nb); 12564da56b99SChris Wilson } 12574da56b99SChris Wilson EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier); 12584da56b99SChris Wilson 1259db64fe02SNick Piggin /* 1260db64fe02SNick Piggin * lazy_max_pages is the maximum amount of virtual address space we gather up 1261db64fe02SNick Piggin * before attempting to purge with a TLB flush. 1262db64fe02SNick Piggin * 1263db64fe02SNick Piggin * There is a tradeoff here: a larger number will cover more kernel page tables 1264db64fe02SNick Piggin * and take slightly longer to purge, but it will linearly reduce the number of 1265db64fe02SNick Piggin * global TLB flushes that must be performed. It would seem natural to scale 1266db64fe02SNick Piggin * this number up linearly with the number of CPUs (because vmapping activity 1267db64fe02SNick Piggin * could also scale linearly with the number of CPUs), however it is likely 1268db64fe02SNick Piggin * that in practice, workloads might be constrained in other ways that mean 1269db64fe02SNick Piggin * vmap activity will not scale linearly with CPUs. Also, I want to be 1270db64fe02SNick Piggin * conservative and not introduce a big latency on huge systems, so go with 1271db64fe02SNick Piggin * a less aggressive log scale. It will still be an improvement over the old 1272db64fe02SNick Piggin * code, and it will be simple to change the scale factor if we find that it 1273db64fe02SNick Piggin * becomes a problem on bigger systems. 1274db64fe02SNick Piggin */ 1275db64fe02SNick Piggin static unsigned long lazy_max_pages(void) 1276db64fe02SNick Piggin { 1277db64fe02SNick Piggin unsigned int log; 1278db64fe02SNick Piggin 1279db64fe02SNick Piggin log = fls(num_online_cpus()); 1280db64fe02SNick Piggin 1281db64fe02SNick Piggin return log * (32UL * 1024 * 1024 / PAGE_SIZE); 1282db64fe02SNick Piggin } 1283db64fe02SNick Piggin 12844d36e6f8SUladzislau Rezki (Sony) static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0); 1285db64fe02SNick Piggin 12860574ecd1SChristoph Hellwig /* 12870574ecd1SChristoph Hellwig * Serialize vmap purging. There is no actual criticial section protected 12880574ecd1SChristoph Hellwig * by this look, but we want to avoid concurrent calls for performance 12890574ecd1SChristoph Hellwig * reasons and to make the pcpu_get_vm_areas more deterministic. 12900574ecd1SChristoph Hellwig */ 1291f9e09977SChristoph Hellwig static DEFINE_MUTEX(vmap_purge_lock); 12920574ecd1SChristoph Hellwig 129302b709dfSNick Piggin /* for per-CPU blocks */ 129402b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void); 129502b709dfSNick Piggin 1296db64fe02SNick Piggin /* 12973ee48b6aSCliff Wickman * called before a call to iounmap() if the caller wants vm_area_struct's 12983ee48b6aSCliff Wickman * immediately freed. 12993ee48b6aSCliff Wickman */ 13003ee48b6aSCliff Wickman void set_iounmap_nonlazy(void) 13013ee48b6aSCliff Wickman { 13024d36e6f8SUladzislau Rezki (Sony) atomic_long_set(&vmap_lazy_nr, lazy_max_pages()+1); 13033ee48b6aSCliff Wickman } 13043ee48b6aSCliff Wickman 13053ee48b6aSCliff Wickman /* 1306db64fe02SNick Piggin * Purges all lazily-freed vmap areas. 1307db64fe02SNick Piggin */ 13080574ecd1SChristoph Hellwig static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end) 1309db64fe02SNick Piggin { 13104d36e6f8SUladzislau Rezki (Sony) unsigned long resched_threshold; 131180c4bd7aSChris Wilson struct llist_node *valist; 1312db64fe02SNick Piggin struct vmap_area *va; 1313cbb76676SVegard Nossum struct vmap_area *n_va; 1314db64fe02SNick Piggin 13150574ecd1SChristoph Hellwig lockdep_assert_held(&vmap_purge_lock); 131602b709dfSNick Piggin 131780c4bd7aSChris Wilson valist = llist_del_all(&vmap_purge_list); 131868571be9SUladzislau Rezki (Sony) if (unlikely(valist == NULL)) 131968571be9SUladzislau Rezki (Sony) return false; 132068571be9SUladzislau Rezki (Sony) 132168571be9SUladzislau Rezki (Sony) /* 132268571be9SUladzislau Rezki (Sony) * TODO: to calculate a flush range without looping. 132368571be9SUladzislau Rezki (Sony) * The list can be up to lazy_max_pages() elements. 132468571be9SUladzislau Rezki (Sony) */ 132580c4bd7aSChris Wilson llist_for_each_entry(va, valist, purge_list) { 13260574ecd1SChristoph Hellwig if (va->va_start < start) 13270574ecd1SChristoph Hellwig start = va->va_start; 13280574ecd1SChristoph Hellwig if (va->va_end > end) 13290574ecd1SChristoph Hellwig end = va->va_end; 1330db64fe02SNick Piggin } 1331db64fe02SNick Piggin 13320574ecd1SChristoph Hellwig flush_tlb_kernel_range(start, end); 13334d36e6f8SUladzislau Rezki (Sony) resched_threshold = lazy_max_pages() << 1; 1334db64fe02SNick Piggin 1335e36176beSUladzislau Rezki (Sony) spin_lock(&free_vmap_area_lock); 1336763b218dSJoel Fernandes llist_for_each_entry_safe(va, n_va, valist, purge_list) { 13374d36e6f8SUladzislau Rezki (Sony) unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT; 13383c5c3cfbSDaniel Axtens unsigned long orig_start = va->va_start; 13393c5c3cfbSDaniel Axtens unsigned long orig_end = va->va_end; 1340763b218dSJoel Fernandes 1341dd3b8353SUladzislau Rezki (Sony) /* 1342dd3b8353SUladzislau Rezki (Sony) * Finally insert or merge lazily-freed area. It is 1343dd3b8353SUladzislau Rezki (Sony) * detached and there is no need to "unlink" it from 1344dd3b8353SUladzislau Rezki (Sony) * anything. 1345dd3b8353SUladzislau Rezki (Sony) */ 13463c5c3cfbSDaniel Axtens va = merge_or_add_vmap_area(va, &free_vmap_area_root, 13473c5c3cfbSDaniel Axtens &free_vmap_area_list); 13483c5c3cfbSDaniel Axtens 13493c5c3cfbSDaniel Axtens if (is_vmalloc_or_module_addr((void *)orig_start)) 13503c5c3cfbSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 13513c5c3cfbSDaniel Axtens va->va_start, va->va_end); 1352dd3b8353SUladzislau Rezki (Sony) 13534d36e6f8SUladzislau Rezki (Sony) atomic_long_sub(nr, &vmap_lazy_nr); 135468571be9SUladzislau Rezki (Sony) 13554d36e6f8SUladzislau Rezki (Sony) if (atomic_long_read(&vmap_lazy_nr) < resched_threshold) 1356e36176beSUladzislau Rezki (Sony) cond_resched_lock(&free_vmap_area_lock); 1357763b218dSJoel Fernandes } 1358e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 13590574ecd1SChristoph Hellwig return true; 1360db64fe02SNick Piggin } 1361db64fe02SNick Piggin 1362db64fe02SNick Piggin /* 1363496850e5SNick Piggin * Kick off a purge of the outstanding lazy areas. Don't bother if somebody 1364496850e5SNick Piggin * is already purging. 1365496850e5SNick Piggin */ 1366496850e5SNick Piggin static void try_purge_vmap_area_lazy(void) 1367496850e5SNick Piggin { 1368f9e09977SChristoph Hellwig if (mutex_trylock(&vmap_purge_lock)) { 13690574ecd1SChristoph Hellwig __purge_vmap_area_lazy(ULONG_MAX, 0); 1370f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 13710574ecd1SChristoph Hellwig } 1372496850e5SNick Piggin } 1373496850e5SNick Piggin 1374496850e5SNick Piggin /* 1375db64fe02SNick Piggin * Kick off a purge of the outstanding lazy areas. 1376db64fe02SNick Piggin */ 1377db64fe02SNick Piggin static void purge_vmap_area_lazy(void) 1378db64fe02SNick Piggin { 1379f9e09977SChristoph Hellwig mutex_lock(&vmap_purge_lock); 13800574ecd1SChristoph Hellwig purge_fragmented_blocks_allcpus(); 13810574ecd1SChristoph Hellwig __purge_vmap_area_lazy(ULONG_MAX, 0); 1382f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 1383db64fe02SNick Piggin } 1384db64fe02SNick Piggin 1385db64fe02SNick Piggin /* 138664141da5SJeremy Fitzhardinge * Free a vmap area, caller ensuring that the area has been unmapped 138764141da5SJeremy Fitzhardinge * and flush_cache_vunmap had been called for the correct range 138864141da5SJeremy Fitzhardinge * previously. 1389db64fe02SNick Piggin */ 139064141da5SJeremy Fitzhardinge static void free_vmap_area_noflush(struct vmap_area *va) 1391db64fe02SNick Piggin { 13924d36e6f8SUladzislau Rezki (Sony) unsigned long nr_lazy; 139380c4bd7aSChris Wilson 1394dd3b8353SUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 1395dd3b8353SUladzislau Rezki (Sony) unlink_va(va, &vmap_area_root); 1396dd3b8353SUladzislau Rezki (Sony) spin_unlock(&vmap_area_lock); 1397dd3b8353SUladzislau Rezki (Sony) 13984d36e6f8SUladzislau Rezki (Sony) nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >> 13994d36e6f8SUladzislau Rezki (Sony) PAGE_SHIFT, &vmap_lazy_nr); 140080c4bd7aSChris Wilson 140180c4bd7aSChris Wilson /* After this point, we may free va at any time */ 140280c4bd7aSChris Wilson llist_add(&va->purge_list, &vmap_purge_list); 140380c4bd7aSChris Wilson 140480c4bd7aSChris Wilson if (unlikely(nr_lazy > lazy_max_pages())) 1405496850e5SNick Piggin try_purge_vmap_area_lazy(); 1406db64fe02SNick Piggin } 1407db64fe02SNick Piggin 1408b29acbdcSNick Piggin /* 1409b29acbdcSNick Piggin * Free and unmap a vmap area 1410b29acbdcSNick Piggin */ 1411b29acbdcSNick Piggin static void free_unmap_vmap_area(struct vmap_area *va) 1412b29acbdcSNick Piggin { 1413b29acbdcSNick Piggin flush_cache_vunmap(va->va_start, va->va_end); 1414855e57a1SChristoph Hellwig unmap_kernel_range_noflush(va->va_start, va->va_end - va->va_start); 14158e57f8acSVlastimil Babka if (debug_pagealloc_enabled_static()) 141682a2e924SChintan Pandya flush_tlb_kernel_range(va->va_start, va->va_end); 141782a2e924SChintan Pandya 1418c8eef01eSChristoph Hellwig free_vmap_area_noflush(va); 1419b29acbdcSNick Piggin } 1420b29acbdcSNick Piggin 1421db64fe02SNick Piggin static struct vmap_area *find_vmap_area(unsigned long addr) 1422db64fe02SNick Piggin { 1423db64fe02SNick Piggin struct vmap_area *va; 1424db64fe02SNick Piggin 1425db64fe02SNick Piggin spin_lock(&vmap_area_lock); 1426db64fe02SNick Piggin va = __find_vmap_area(addr); 1427db64fe02SNick Piggin spin_unlock(&vmap_area_lock); 1428db64fe02SNick Piggin 1429db64fe02SNick Piggin return va; 1430db64fe02SNick Piggin } 1431db64fe02SNick Piggin 1432db64fe02SNick Piggin /*** Per cpu kva allocator ***/ 1433db64fe02SNick Piggin 1434db64fe02SNick Piggin /* 1435db64fe02SNick Piggin * vmap space is limited especially on 32 bit architectures. Ensure there is 1436db64fe02SNick Piggin * room for at least 16 percpu vmap blocks per CPU. 1437db64fe02SNick Piggin */ 1438db64fe02SNick Piggin /* 1439db64fe02SNick Piggin * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able 1440db64fe02SNick Piggin * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess 1441db64fe02SNick Piggin * instead (we just need a rough idea) 1442db64fe02SNick Piggin */ 1443db64fe02SNick Piggin #if BITS_PER_LONG == 32 1444db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024) 1445db64fe02SNick Piggin #else 1446db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024*1024) 1447db64fe02SNick Piggin #endif 1448db64fe02SNick Piggin 1449db64fe02SNick Piggin #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE) 1450db64fe02SNick Piggin #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */ 1451db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */ 1452db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2) 1453db64fe02SNick Piggin #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */ 1454db64fe02SNick Piggin #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */ 1455f982f915SClemens Ladisch #define VMAP_BBMAP_BITS \ 1456f982f915SClemens Ladisch VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ 1457db64fe02SNick Piggin VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ 1458f982f915SClemens Ladisch VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16)) 1459db64fe02SNick Piggin 1460db64fe02SNick Piggin #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE) 1461db64fe02SNick Piggin 1462db64fe02SNick Piggin struct vmap_block_queue { 1463db64fe02SNick Piggin spinlock_t lock; 1464db64fe02SNick Piggin struct list_head free; 1465db64fe02SNick Piggin }; 1466db64fe02SNick Piggin 1467db64fe02SNick Piggin struct vmap_block { 1468db64fe02SNick Piggin spinlock_t lock; 1469db64fe02SNick Piggin struct vmap_area *va; 1470db64fe02SNick Piggin unsigned long free, dirty; 14717d61bfe8SRoman Pen unsigned long dirty_min, dirty_max; /*< dirty range */ 1472db64fe02SNick Piggin struct list_head free_list; 1473db64fe02SNick Piggin struct rcu_head rcu_head; 147402b709dfSNick Piggin struct list_head purge; 1475db64fe02SNick Piggin }; 1476db64fe02SNick Piggin 1477db64fe02SNick Piggin /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */ 1478db64fe02SNick Piggin static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue); 1479db64fe02SNick Piggin 1480db64fe02SNick Piggin /* 14810f14599cSMatthew Wilcox (Oracle) * XArray of vmap blocks, indexed by address, to quickly find a vmap block 1482db64fe02SNick Piggin * in the free path. Could get rid of this if we change the API to return a 1483db64fe02SNick Piggin * "cookie" from alloc, to be passed to free. But no big deal yet. 1484db64fe02SNick Piggin */ 14850f14599cSMatthew Wilcox (Oracle) static DEFINE_XARRAY(vmap_blocks); 1486db64fe02SNick Piggin 1487db64fe02SNick Piggin /* 1488db64fe02SNick Piggin * We should probably have a fallback mechanism to allocate virtual memory 1489db64fe02SNick Piggin * out of partially filled vmap blocks. However vmap block sizing should be 1490db64fe02SNick Piggin * fairly reasonable according to the vmalloc size, so it shouldn't be a 1491db64fe02SNick Piggin * big problem. 1492db64fe02SNick Piggin */ 1493db64fe02SNick Piggin 1494db64fe02SNick Piggin static unsigned long addr_to_vb_idx(unsigned long addr) 1495db64fe02SNick Piggin { 1496db64fe02SNick Piggin addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); 1497db64fe02SNick Piggin addr /= VMAP_BLOCK_SIZE; 1498db64fe02SNick Piggin return addr; 1499db64fe02SNick Piggin } 1500db64fe02SNick Piggin 1501cf725ce2SRoman Pen static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off) 1502cf725ce2SRoman Pen { 1503cf725ce2SRoman Pen unsigned long addr; 1504cf725ce2SRoman Pen 1505cf725ce2SRoman Pen addr = va_start + (pages_off << PAGE_SHIFT); 1506cf725ce2SRoman Pen BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start)); 1507cf725ce2SRoman Pen return (void *)addr; 1508cf725ce2SRoman Pen } 1509cf725ce2SRoman Pen 1510cf725ce2SRoman Pen /** 1511cf725ce2SRoman Pen * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this 1512cf725ce2SRoman Pen * block. Of course pages number can't exceed VMAP_BBMAP_BITS 1513cf725ce2SRoman Pen * @order: how many 2^order pages should be occupied in newly allocated block 1514cf725ce2SRoman Pen * @gfp_mask: flags for the page level allocator 1515cf725ce2SRoman Pen * 1516a862f68aSMike Rapoport * Return: virtual address in a newly allocated block or ERR_PTR(-errno) 1517cf725ce2SRoman Pen */ 1518cf725ce2SRoman Pen static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) 1519db64fe02SNick Piggin { 1520db64fe02SNick Piggin struct vmap_block_queue *vbq; 1521db64fe02SNick Piggin struct vmap_block *vb; 1522db64fe02SNick Piggin struct vmap_area *va; 1523db64fe02SNick Piggin unsigned long vb_idx; 1524db64fe02SNick Piggin int node, err; 1525cf725ce2SRoman Pen void *vaddr; 1526db64fe02SNick Piggin 1527db64fe02SNick Piggin node = numa_node_id(); 1528db64fe02SNick Piggin 1529db64fe02SNick Piggin vb = kmalloc_node(sizeof(struct vmap_block), 1530db64fe02SNick Piggin gfp_mask & GFP_RECLAIM_MASK, node); 1531db64fe02SNick Piggin if (unlikely(!vb)) 1532db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 1533db64fe02SNick Piggin 1534db64fe02SNick Piggin va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, 1535db64fe02SNick Piggin VMALLOC_START, VMALLOC_END, 1536db64fe02SNick Piggin node, gfp_mask); 1537ddf9c6d4STobias Klauser if (IS_ERR(va)) { 1538db64fe02SNick Piggin kfree(vb); 1539e7d86340SJulia Lawall return ERR_CAST(va); 1540db64fe02SNick Piggin } 1541db64fe02SNick Piggin 1542cf725ce2SRoman Pen vaddr = vmap_block_vaddr(va->va_start, 0); 1543db64fe02SNick Piggin spin_lock_init(&vb->lock); 1544db64fe02SNick Piggin vb->va = va; 1545cf725ce2SRoman Pen /* At least something should be left free */ 1546cf725ce2SRoman Pen BUG_ON(VMAP_BBMAP_BITS <= (1UL << order)); 1547cf725ce2SRoman Pen vb->free = VMAP_BBMAP_BITS - (1UL << order); 1548db64fe02SNick Piggin vb->dirty = 0; 15497d61bfe8SRoman Pen vb->dirty_min = VMAP_BBMAP_BITS; 15507d61bfe8SRoman Pen vb->dirty_max = 0; 1551db64fe02SNick Piggin INIT_LIST_HEAD(&vb->free_list); 1552db64fe02SNick Piggin 1553db64fe02SNick Piggin vb_idx = addr_to_vb_idx(va->va_start); 15540f14599cSMatthew Wilcox (Oracle) err = xa_insert(&vmap_blocks, vb_idx, vb, gfp_mask); 15550f14599cSMatthew Wilcox (Oracle) if (err) { 15560f14599cSMatthew Wilcox (Oracle) kfree(vb); 15570f14599cSMatthew Wilcox (Oracle) free_vmap_area(va); 15580f14599cSMatthew Wilcox (Oracle) return ERR_PTR(err); 15590f14599cSMatthew Wilcox (Oracle) } 1560db64fe02SNick Piggin 1561db64fe02SNick Piggin vbq = &get_cpu_var(vmap_block_queue); 1562db64fe02SNick Piggin spin_lock(&vbq->lock); 156368ac546fSRoman Pen list_add_tail_rcu(&vb->free_list, &vbq->free); 1564db64fe02SNick Piggin spin_unlock(&vbq->lock); 15653f04ba85STejun Heo put_cpu_var(vmap_block_queue); 1566db64fe02SNick Piggin 1567cf725ce2SRoman Pen return vaddr; 1568db64fe02SNick Piggin } 1569db64fe02SNick Piggin 1570db64fe02SNick Piggin static void free_vmap_block(struct vmap_block *vb) 1571db64fe02SNick Piggin { 1572db64fe02SNick Piggin struct vmap_block *tmp; 1573db64fe02SNick Piggin 15740f14599cSMatthew Wilcox (Oracle) tmp = xa_erase(&vmap_blocks, addr_to_vb_idx(vb->va->va_start)); 1575db64fe02SNick Piggin BUG_ON(tmp != vb); 1576db64fe02SNick Piggin 157764141da5SJeremy Fitzhardinge free_vmap_area_noflush(vb->va); 157822a3c7d1SLai Jiangshan kfree_rcu(vb, rcu_head); 1579db64fe02SNick Piggin } 1580db64fe02SNick Piggin 158102b709dfSNick Piggin static void purge_fragmented_blocks(int cpu) 158202b709dfSNick Piggin { 158302b709dfSNick Piggin LIST_HEAD(purge); 158402b709dfSNick Piggin struct vmap_block *vb; 158502b709dfSNick Piggin struct vmap_block *n_vb; 158602b709dfSNick Piggin struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 158702b709dfSNick Piggin 158802b709dfSNick Piggin rcu_read_lock(); 158902b709dfSNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 159002b709dfSNick Piggin 159102b709dfSNick Piggin if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS)) 159202b709dfSNick Piggin continue; 159302b709dfSNick Piggin 159402b709dfSNick Piggin spin_lock(&vb->lock); 159502b709dfSNick Piggin if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) { 159602b709dfSNick Piggin vb->free = 0; /* prevent further allocs after releasing lock */ 159702b709dfSNick Piggin vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */ 15987d61bfe8SRoman Pen vb->dirty_min = 0; 15997d61bfe8SRoman Pen vb->dirty_max = VMAP_BBMAP_BITS; 160002b709dfSNick Piggin spin_lock(&vbq->lock); 160102b709dfSNick Piggin list_del_rcu(&vb->free_list); 160202b709dfSNick Piggin spin_unlock(&vbq->lock); 160302b709dfSNick Piggin spin_unlock(&vb->lock); 160402b709dfSNick Piggin list_add_tail(&vb->purge, &purge); 160502b709dfSNick Piggin } else 160602b709dfSNick Piggin spin_unlock(&vb->lock); 160702b709dfSNick Piggin } 160802b709dfSNick Piggin rcu_read_unlock(); 160902b709dfSNick Piggin 161002b709dfSNick Piggin list_for_each_entry_safe(vb, n_vb, &purge, purge) { 161102b709dfSNick Piggin list_del(&vb->purge); 161202b709dfSNick Piggin free_vmap_block(vb); 161302b709dfSNick Piggin } 161402b709dfSNick Piggin } 161502b709dfSNick Piggin 161602b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void) 161702b709dfSNick Piggin { 161802b709dfSNick Piggin int cpu; 161902b709dfSNick Piggin 162002b709dfSNick Piggin for_each_possible_cpu(cpu) 162102b709dfSNick Piggin purge_fragmented_blocks(cpu); 162202b709dfSNick Piggin } 162302b709dfSNick Piggin 1624db64fe02SNick Piggin static void *vb_alloc(unsigned long size, gfp_t gfp_mask) 1625db64fe02SNick Piggin { 1626db64fe02SNick Piggin struct vmap_block_queue *vbq; 1627db64fe02SNick Piggin struct vmap_block *vb; 1628cf725ce2SRoman Pen void *vaddr = NULL; 1629db64fe02SNick Piggin unsigned int order; 1630db64fe02SNick Piggin 1631891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 1632db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 1633aa91c4d8SJan Kara if (WARN_ON(size == 0)) { 1634aa91c4d8SJan Kara /* 1635aa91c4d8SJan Kara * Allocating 0 bytes isn't what caller wants since 1636aa91c4d8SJan Kara * get_order(0) returns funny result. Just warn and terminate 1637aa91c4d8SJan Kara * early. 1638aa91c4d8SJan Kara */ 1639aa91c4d8SJan Kara return NULL; 1640aa91c4d8SJan Kara } 1641db64fe02SNick Piggin order = get_order(size); 1642db64fe02SNick Piggin 1643db64fe02SNick Piggin rcu_read_lock(); 1644db64fe02SNick Piggin vbq = &get_cpu_var(vmap_block_queue); 1645db64fe02SNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 1646cf725ce2SRoman Pen unsigned long pages_off; 1647db64fe02SNick Piggin 1648db64fe02SNick Piggin spin_lock(&vb->lock); 1649cf725ce2SRoman Pen if (vb->free < (1UL << order)) { 1650cf725ce2SRoman Pen spin_unlock(&vb->lock); 1651cf725ce2SRoman Pen continue; 1652cf725ce2SRoman Pen } 165302b709dfSNick Piggin 1654cf725ce2SRoman Pen pages_off = VMAP_BBMAP_BITS - vb->free; 1655cf725ce2SRoman Pen vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); 1656db64fe02SNick Piggin vb->free -= 1UL << order; 1657db64fe02SNick Piggin if (vb->free == 0) { 1658db64fe02SNick Piggin spin_lock(&vbq->lock); 1659de560423SNick Piggin list_del_rcu(&vb->free_list); 1660db64fe02SNick Piggin spin_unlock(&vbq->lock); 1661db64fe02SNick Piggin } 1662cf725ce2SRoman Pen 1663db64fe02SNick Piggin spin_unlock(&vb->lock); 1664db64fe02SNick Piggin break; 1665db64fe02SNick Piggin } 166602b709dfSNick Piggin 16673f04ba85STejun Heo put_cpu_var(vmap_block_queue); 1668db64fe02SNick Piggin rcu_read_unlock(); 1669db64fe02SNick Piggin 1670cf725ce2SRoman Pen /* Allocate new block if nothing was found */ 1671cf725ce2SRoman Pen if (!vaddr) 1672cf725ce2SRoman Pen vaddr = new_vmap_block(order, gfp_mask); 1673db64fe02SNick Piggin 1674cf725ce2SRoman Pen return vaddr; 1675db64fe02SNick Piggin } 1676db64fe02SNick Piggin 167778a0e8c4SChristoph Hellwig static void vb_free(unsigned long addr, unsigned long size) 1678db64fe02SNick Piggin { 1679db64fe02SNick Piggin unsigned long offset; 1680db64fe02SNick Piggin unsigned int order; 1681db64fe02SNick Piggin struct vmap_block *vb; 1682db64fe02SNick Piggin 1683891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 1684db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 1685b29acbdcSNick Piggin 168678a0e8c4SChristoph Hellwig flush_cache_vunmap(addr, addr + size); 1687b29acbdcSNick Piggin 1688db64fe02SNick Piggin order = get_order(size); 168978a0e8c4SChristoph Hellwig offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT; 16900f14599cSMatthew Wilcox (Oracle) vb = xa_load(&vmap_blocks, addr_to_vb_idx(addr)); 1691db64fe02SNick Piggin 1692b521c43fSChristoph Hellwig unmap_kernel_range_noflush(addr, size); 169364141da5SJeremy Fitzhardinge 16948e57f8acSVlastimil Babka if (debug_pagealloc_enabled_static()) 169578a0e8c4SChristoph Hellwig flush_tlb_kernel_range(addr, addr + size); 169682a2e924SChintan Pandya 1697db64fe02SNick Piggin spin_lock(&vb->lock); 16987d61bfe8SRoman Pen 16997d61bfe8SRoman Pen /* Expand dirty range */ 17007d61bfe8SRoman Pen vb->dirty_min = min(vb->dirty_min, offset); 17017d61bfe8SRoman Pen vb->dirty_max = max(vb->dirty_max, offset + (1UL << order)); 1702d086817dSMinChan Kim 1703db64fe02SNick Piggin vb->dirty += 1UL << order; 1704db64fe02SNick Piggin if (vb->dirty == VMAP_BBMAP_BITS) { 1705de560423SNick Piggin BUG_ON(vb->free); 1706db64fe02SNick Piggin spin_unlock(&vb->lock); 1707db64fe02SNick Piggin free_vmap_block(vb); 1708db64fe02SNick Piggin } else 1709db64fe02SNick Piggin spin_unlock(&vb->lock); 1710db64fe02SNick Piggin } 1711db64fe02SNick Piggin 1712868b104dSRick Edgecombe static void _vm_unmap_aliases(unsigned long start, unsigned long end, int flush) 1713db64fe02SNick Piggin { 1714db64fe02SNick Piggin int cpu; 1715db64fe02SNick Piggin 17169b463334SJeremy Fitzhardinge if (unlikely(!vmap_initialized)) 17179b463334SJeremy Fitzhardinge return; 17189b463334SJeremy Fitzhardinge 17195803ed29SChristoph Hellwig might_sleep(); 17205803ed29SChristoph Hellwig 1721db64fe02SNick Piggin for_each_possible_cpu(cpu) { 1722db64fe02SNick Piggin struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 1723db64fe02SNick Piggin struct vmap_block *vb; 1724db64fe02SNick Piggin 1725db64fe02SNick Piggin rcu_read_lock(); 1726db64fe02SNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 1727db64fe02SNick Piggin spin_lock(&vb->lock); 17287d61bfe8SRoman Pen if (vb->dirty) { 17297d61bfe8SRoman Pen unsigned long va_start = vb->va->va_start; 1730db64fe02SNick Piggin unsigned long s, e; 1731b136be5eSJoonsoo Kim 17327d61bfe8SRoman Pen s = va_start + (vb->dirty_min << PAGE_SHIFT); 17337d61bfe8SRoman Pen e = va_start + (vb->dirty_max << PAGE_SHIFT); 1734db64fe02SNick Piggin 17357d61bfe8SRoman Pen start = min(s, start); 17367d61bfe8SRoman Pen end = max(e, end); 17377d61bfe8SRoman Pen 1738db64fe02SNick Piggin flush = 1; 1739db64fe02SNick Piggin } 1740db64fe02SNick Piggin spin_unlock(&vb->lock); 1741db64fe02SNick Piggin } 1742db64fe02SNick Piggin rcu_read_unlock(); 1743db64fe02SNick Piggin } 1744db64fe02SNick Piggin 1745f9e09977SChristoph Hellwig mutex_lock(&vmap_purge_lock); 17460574ecd1SChristoph Hellwig purge_fragmented_blocks_allcpus(); 17470574ecd1SChristoph Hellwig if (!__purge_vmap_area_lazy(start, end) && flush) 17480574ecd1SChristoph Hellwig flush_tlb_kernel_range(start, end); 1749f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 1750db64fe02SNick Piggin } 1751868b104dSRick Edgecombe 1752868b104dSRick Edgecombe /** 1753868b104dSRick Edgecombe * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer 1754868b104dSRick Edgecombe * 1755868b104dSRick Edgecombe * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily 1756868b104dSRick Edgecombe * to amortize TLB flushing overheads. What this means is that any page you 1757868b104dSRick Edgecombe * have now, may, in a former life, have been mapped into kernel virtual 1758868b104dSRick Edgecombe * address by the vmap layer and so there might be some CPUs with TLB entries 1759868b104dSRick Edgecombe * still referencing that page (additional to the regular 1:1 kernel mapping). 1760868b104dSRick Edgecombe * 1761868b104dSRick Edgecombe * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can 1762868b104dSRick Edgecombe * be sure that none of the pages we have control over will have any aliases 1763868b104dSRick Edgecombe * from the vmap layer. 1764868b104dSRick Edgecombe */ 1765868b104dSRick Edgecombe void vm_unmap_aliases(void) 1766868b104dSRick Edgecombe { 1767868b104dSRick Edgecombe unsigned long start = ULONG_MAX, end = 0; 1768868b104dSRick Edgecombe int flush = 0; 1769868b104dSRick Edgecombe 1770868b104dSRick Edgecombe _vm_unmap_aliases(start, end, flush); 1771868b104dSRick Edgecombe } 1772db64fe02SNick Piggin EXPORT_SYMBOL_GPL(vm_unmap_aliases); 1773db64fe02SNick Piggin 1774db64fe02SNick Piggin /** 1775db64fe02SNick Piggin * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram 1776db64fe02SNick Piggin * @mem: the pointer returned by vm_map_ram 1777db64fe02SNick Piggin * @count: the count passed to that vm_map_ram call (cannot unmap partial) 1778db64fe02SNick Piggin */ 1779db64fe02SNick Piggin void vm_unmap_ram(const void *mem, unsigned int count) 1780db64fe02SNick Piggin { 178165ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 1782db64fe02SNick Piggin unsigned long addr = (unsigned long)mem; 17839c3acf60SChristoph Hellwig struct vmap_area *va; 1784db64fe02SNick Piggin 17855803ed29SChristoph Hellwig might_sleep(); 1786db64fe02SNick Piggin BUG_ON(!addr); 1787db64fe02SNick Piggin BUG_ON(addr < VMALLOC_START); 1788db64fe02SNick Piggin BUG_ON(addr > VMALLOC_END); 1789a1c0b1a0SShawn Lin BUG_ON(!PAGE_ALIGNED(addr)); 1790db64fe02SNick Piggin 1791d98c9e83SAndrey Ryabinin kasan_poison_vmalloc(mem, size); 1792d98c9e83SAndrey Ryabinin 17939c3acf60SChristoph Hellwig if (likely(count <= VMAP_MAX_ALLOC)) { 179405e3ff95SChintan Pandya debug_check_no_locks_freed(mem, size); 179578a0e8c4SChristoph Hellwig vb_free(addr, size); 17969c3acf60SChristoph Hellwig return; 17979c3acf60SChristoph Hellwig } 17989c3acf60SChristoph Hellwig 17999c3acf60SChristoph Hellwig va = find_vmap_area(addr); 18009c3acf60SChristoph Hellwig BUG_ON(!va); 180105e3ff95SChintan Pandya debug_check_no_locks_freed((void *)va->va_start, 180205e3ff95SChintan Pandya (va->va_end - va->va_start)); 18039c3acf60SChristoph Hellwig free_unmap_vmap_area(va); 1804db64fe02SNick Piggin } 1805db64fe02SNick Piggin EXPORT_SYMBOL(vm_unmap_ram); 1806db64fe02SNick Piggin 1807db64fe02SNick Piggin /** 1808db64fe02SNick Piggin * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) 1809db64fe02SNick Piggin * @pages: an array of pointers to the pages to be mapped 1810db64fe02SNick Piggin * @count: number of pages 1811db64fe02SNick Piggin * @node: prefer to allocate data structures on this node 1812e99c97adSRandy Dunlap * 181336437638SGioh Kim * If you use this function for less than VMAP_MAX_ALLOC pages, it could be 181436437638SGioh Kim * faster than vmap so it's good. But if you mix long-life and short-life 181536437638SGioh Kim * objects with vm_map_ram(), it could consume lots of address space through 181636437638SGioh Kim * fragmentation (especially on a 32bit machine). You could see failures in 181736437638SGioh Kim * the end. Please use this function for short-lived objects. 181836437638SGioh Kim * 1819e99c97adSRandy Dunlap * Returns: a pointer to the address that has been mapped, or %NULL on failure 1820db64fe02SNick Piggin */ 1821d4efd79aSChristoph Hellwig void *vm_map_ram(struct page **pages, unsigned int count, int node) 1822db64fe02SNick Piggin { 182365ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 1824db64fe02SNick Piggin unsigned long addr; 1825db64fe02SNick Piggin void *mem; 1826db64fe02SNick Piggin 1827db64fe02SNick Piggin if (likely(count <= VMAP_MAX_ALLOC)) { 1828db64fe02SNick Piggin mem = vb_alloc(size, GFP_KERNEL); 1829db64fe02SNick Piggin if (IS_ERR(mem)) 1830db64fe02SNick Piggin return NULL; 1831db64fe02SNick Piggin addr = (unsigned long)mem; 1832db64fe02SNick Piggin } else { 1833db64fe02SNick Piggin struct vmap_area *va; 1834db64fe02SNick Piggin va = alloc_vmap_area(size, PAGE_SIZE, 1835db64fe02SNick Piggin VMALLOC_START, VMALLOC_END, node, GFP_KERNEL); 1836db64fe02SNick Piggin if (IS_ERR(va)) 1837db64fe02SNick Piggin return NULL; 1838db64fe02SNick Piggin 1839db64fe02SNick Piggin addr = va->va_start; 1840db64fe02SNick Piggin mem = (void *)addr; 1841db64fe02SNick Piggin } 1842d98c9e83SAndrey Ryabinin 1843d98c9e83SAndrey Ryabinin kasan_unpoison_vmalloc(mem, size); 1844d98c9e83SAndrey Ryabinin 1845d4efd79aSChristoph Hellwig if (map_kernel_range(addr, size, PAGE_KERNEL, pages) < 0) { 1846db64fe02SNick Piggin vm_unmap_ram(mem, count); 1847db64fe02SNick Piggin return NULL; 1848db64fe02SNick Piggin } 1849db64fe02SNick Piggin return mem; 1850db64fe02SNick Piggin } 1851db64fe02SNick Piggin EXPORT_SYMBOL(vm_map_ram); 1852db64fe02SNick Piggin 18534341fa45SJoonsoo Kim static struct vm_struct *vmlist __initdata; 185492eac168SMike Rapoport 1855f0aa6617STejun Heo /** 1856be9b7335SNicolas Pitre * vm_area_add_early - add vmap area early during boot 1857be9b7335SNicolas Pitre * @vm: vm_struct to add 1858be9b7335SNicolas Pitre * 1859be9b7335SNicolas Pitre * This function is used to add fixed kernel vm area to vmlist before 1860be9b7335SNicolas Pitre * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags 1861be9b7335SNicolas Pitre * should contain proper values and the other fields should be zero. 1862be9b7335SNicolas Pitre * 1863be9b7335SNicolas Pitre * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 1864be9b7335SNicolas Pitre */ 1865be9b7335SNicolas Pitre void __init vm_area_add_early(struct vm_struct *vm) 1866be9b7335SNicolas Pitre { 1867be9b7335SNicolas Pitre struct vm_struct *tmp, **p; 1868be9b7335SNicolas Pitre 1869be9b7335SNicolas Pitre BUG_ON(vmap_initialized); 1870be9b7335SNicolas Pitre for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { 1871be9b7335SNicolas Pitre if (tmp->addr >= vm->addr) { 1872be9b7335SNicolas Pitre BUG_ON(tmp->addr < vm->addr + vm->size); 1873be9b7335SNicolas Pitre break; 1874be9b7335SNicolas Pitre } else 1875be9b7335SNicolas Pitre BUG_ON(tmp->addr + tmp->size > vm->addr); 1876be9b7335SNicolas Pitre } 1877be9b7335SNicolas Pitre vm->next = *p; 1878be9b7335SNicolas Pitre *p = vm; 1879be9b7335SNicolas Pitre } 1880be9b7335SNicolas Pitre 1881be9b7335SNicolas Pitre /** 1882f0aa6617STejun Heo * vm_area_register_early - register vmap area early during boot 1883f0aa6617STejun Heo * @vm: vm_struct to register 1884c0c0a293STejun Heo * @align: requested alignment 1885f0aa6617STejun Heo * 1886f0aa6617STejun Heo * This function is used to register kernel vm area before 1887f0aa6617STejun Heo * vmalloc_init() is called. @vm->size and @vm->flags should contain 1888f0aa6617STejun Heo * proper values on entry and other fields should be zero. On return, 1889f0aa6617STejun Heo * vm->addr contains the allocated address. 1890f0aa6617STejun Heo * 1891f0aa6617STejun Heo * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 1892f0aa6617STejun Heo */ 1893c0c0a293STejun Heo void __init vm_area_register_early(struct vm_struct *vm, size_t align) 1894f0aa6617STejun Heo { 1895f0aa6617STejun Heo static size_t vm_init_off __initdata; 1896c0c0a293STejun Heo unsigned long addr; 1897f0aa6617STejun Heo 1898c0c0a293STejun Heo addr = ALIGN(VMALLOC_START + vm_init_off, align); 1899c0c0a293STejun Heo vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START; 1900c0c0a293STejun Heo 1901c0c0a293STejun Heo vm->addr = (void *)addr; 1902f0aa6617STejun Heo 1903be9b7335SNicolas Pitre vm_area_add_early(vm); 1904f0aa6617STejun Heo } 1905f0aa6617STejun Heo 190668ad4a33SUladzislau Rezki (Sony) static void vmap_init_free_space(void) 190768ad4a33SUladzislau Rezki (Sony) { 190868ad4a33SUladzislau Rezki (Sony) unsigned long vmap_start = 1; 190968ad4a33SUladzislau Rezki (Sony) const unsigned long vmap_end = ULONG_MAX; 191068ad4a33SUladzislau Rezki (Sony) struct vmap_area *busy, *free; 191168ad4a33SUladzislau Rezki (Sony) 191268ad4a33SUladzislau Rezki (Sony) /* 191368ad4a33SUladzislau Rezki (Sony) * B F B B B F 191468ad4a33SUladzislau Rezki (Sony) * -|-----|.....|-----|-----|-----|.....|- 191568ad4a33SUladzislau Rezki (Sony) * | The KVA space | 191668ad4a33SUladzislau Rezki (Sony) * |<--------------------------------->| 191768ad4a33SUladzislau Rezki (Sony) */ 191868ad4a33SUladzislau Rezki (Sony) list_for_each_entry(busy, &vmap_area_list, list) { 191968ad4a33SUladzislau Rezki (Sony) if (busy->va_start - vmap_start > 0) { 192068ad4a33SUladzislau Rezki (Sony) free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 192168ad4a33SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(!free)) { 192268ad4a33SUladzislau Rezki (Sony) free->va_start = vmap_start; 192368ad4a33SUladzislau Rezki (Sony) free->va_end = busy->va_start; 192468ad4a33SUladzislau Rezki (Sony) 192568ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(free, NULL, 192668ad4a33SUladzislau Rezki (Sony) &free_vmap_area_root, 192768ad4a33SUladzislau Rezki (Sony) &free_vmap_area_list); 192868ad4a33SUladzislau Rezki (Sony) } 192968ad4a33SUladzislau Rezki (Sony) } 193068ad4a33SUladzislau Rezki (Sony) 193168ad4a33SUladzislau Rezki (Sony) vmap_start = busy->va_end; 193268ad4a33SUladzislau Rezki (Sony) } 193368ad4a33SUladzislau Rezki (Sony) 193468ad4a33SUladzislau Rezki (Sony) if (vmap_end - vmap_start > 0) { 193568ad4a33SUladzislau Rezki (Sony) free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 193668ad4a33SUladzislau Rezki (Sony) if (!WARN_ON_ONCE(!free)) { 193768ad4a33SUladzislau Rezki (Sony) free->va_start = vmap_start; 193868ad4a33SUladzislau Rezki (Sony) free->va_end = vmap_end; 193968ad4a33SUladzislau Rezki (Sony) 194068ad4a33SUladzislau Rezki (Sony) insert_vmap_area_augment(free, NULL, 194168ad4a33SUladzislau Rezki (Sony) &free_vmap_area_root, 194268ad4a33SUladzislau Rezki (Sony) &free_vmap_area_list); 194368ad4a33SUladzislau Rezki (Sony) } 194468ad4a33SUladzislau Rezki (Sony) } 194568ad4a33SUladzislau Rezki (Sony) } 194668ad4a33SUladzislau Rezki (Sony) 1947db64fe02SNick Piggin void __init vmalloc_init(void) 1948db64fe02SNick Piggin { 1949822c18f2SIvan Kokshaysky struct vmap_area *va; 1950822c18f2SIvan Kokshaysky struct vm_struct *tmp; 1951db64fe02SNick Piggin int i; 1952db64fe02SNick Piggin 195368ad4a33SUladzislau Rezki (Sony) /* 195468ad4a33SUladzislau Rezki (Sony) * Create the cache for vmap_area objects. 195568ad4a33SUladzislau Rezki (Sony) */ 195668ad4a33SUladzislau Rezki (Sony) vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC); 195768ad4a33SUladzislau Rezki (Sony) 1958db64fe02SNick Piggin for_each_possible_cpu(i) { 1959db64fe02SNick Piggin struct vmap_block_queue *vbq; 196032fcfd40SAl Viro struct vfree_deferred *p; 1961db64fe02SNick Piggin 1962db64fe02SNick Piggin vbq = &per_cpu(vmap_block_queue, i); 1963db64fe02SNick Piggin spin_lock_init(&vbq->lock); 1964db64fe02SNick Piggin INIT_LIST_HEAD(&vbq->free); 196532fcfd40SAl Viro p = &per_cpu(vfree_deferred, i); 196632fcfd40SAl Viro init_llist_head(&p->list); 196732fcfd40SAl Viro INIT_WORK(&p->wq, free_work); 1968db64fe02SNick Piggin } 19699b463334SJeremy Fitzhardinge 1970822c18f2SIvan Kokshaysky /* Import existing vmlist entries. */ 1971822c18f2SIvan Kokshaysky for (tmp = vmlist; tmp; tmp = tmp->next) { 197268ad4a33SUladzislau Rezki (Sony) va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 197368ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(!va)) 197468ad4a33SUladzislau Rezki (Sony) continue; 197568ad4a33SUladzislau Rezki (Sony) 1976822c18f2SIvan Kokshaysky va->va_start = (unsigned long)tmp->addr; 1977822c18f2SIvan Kokshaysky va->va_end = va->va_start + tmp->size; 1978dbda591dSKyongHo va->vm = tmp; 197968ad4a33SUladzislau Rezki (Sony) insert_vmap_area(va, &vmap_area_root, &vmap_area_list); 1980822c18f2SIvan Kokshaysky } 1981ca23e405STejun Heo 198268ad4a33SUladzislau Rezki (Sony) /* 198368ad4a33SUladzislau Rezki (Sony) * Now we can initialize a free vmap space. 198468ad4a33SUladzislau Rezki (Sony) */ 198568ad4a33SUladzislau Rezki (Sony) vmap_init_free_space(); 19869b463334SJeremy Fitzhardinge vmap_initialized = true; 1987db64fe02SNick Piggin } 1988db64fe02SNick Piggin 19898fc48985STejun Heo /** 19908fc48985STejun Heo * unmap_kernel_range - unmap kernel VM area and flush cache and TLB 19918fc48985STejun Heo * @addr: start of the VM area to unmap 19928fc48985STejun Heo * @size: size of the VM area to unmap 19938fc48985STejun Heo * 19948fc48985STejun Heo * Similar to unmap_kernel_range_noflush() but flushes vcache before 19958fc48985STejun Heo * the unmapping and tlb after. 19968fc48985STejun Heo */ 1997db64fe02SNick Piggin void unmap_kernel_range(unsigned long addr, unsigned long size) 1998db64fe02SNick Piggin { 1999db64fe02SNick Piggin unsigned long end = addr + size; 2000f6fcba70STejun Heo 2001f6fcba70STejun Heo flush_cache_vunmap(addr, end); 2002b521c43fSChristoph Hellwig unmap_kernel_range_noflush(addr, size); 2003db64fe02SNick Piggin flush_tlb_kernel_range(addr, end); 2004db64fe02SNick Piggin } 2005db64fe02SNick Piggin 2006e36176beSUladzislau Rezki (Sony) static inline void setup_vmalloc_vm_locked(struct vm_struct *vm, 2007e36176beSUladzislau Rezki (Sony) struct vmap_area *va, unsigned long flags, const void *caller) 2008cf88c790STejun Heo { 2009cf88c790STejun Heo vm->flags = flags; 2010cf88c790STejun Heo vm->addr = (void *)va->va_start; 2011cf88c790STejun Heo vm->size = va->va_end - va->va_start; 2012cf88c790STejun Heo vm->caller = caller; 2013db1aecafSMinchan Kim va->vm = vm; 2014e36176beSUladzislau Rezki (Sony) } 2015e36176beSUladzislau Rezki (Sony) 2016e36176beSUladzislau Rezki (Sony) static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, 2017e36176beSUladzislau Rezki (Sony) unsigned long flags, const void *caller) 2018e36176beSUladzislau Rezki (Sony) { 2019e36176beSUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 2020e36176beSUladzislau Rezki (Sony) setup_vmalloc_vm_locked(vm, va, flags, caller); 2021c69480adSJoonsoo Kim spin_unlock(&vmap_area_lock); 2022f5252e00SMitsuo Hayasaka } 2023cf88c790STejun Heo 202420fc02b4SZhang Yanfei static void clear_vm_uninitialized_flag(struct vm_struct *vm) 2025f5252e00SMitsuo Hayasaka { 2026d4033afdSJoonsoo Kim /* 202720fc02b4SZhang Yanfei * Before removing VM_UNINITIALIZED, 2028d4033afdSJoonsoo Kim * we should make sure that vm has proper values. 2029d4033afdSJoonsoo Kim * Pair with smp_rmb() in show_numa_info(). 2030d4033afdSJoonsoo Kim */ 2031d4033afdSJoonsoo Kim smp_wmb(); 203220fc02b4SZhang Yanfei vm->flags &= ~VM_UNINITIALIZED; 2033cf88c790STejun Heo } 2034cf88c790STejun Heo 2035db64fe02SNick Piggin static struct vm_struct *__get_vm_area_node(unsigned long size, 20362dca6999SDavid Miller unsigned long align, unsigned long flags, unsigned long start, 20375e6cafc8SMarek Szyprowski unsigned long end, int node, gfp_t gfp_mask, const void *caller) 2038db64fe02SNick Piggin { 20390006526dSKautuk Consul struct vmap_area *va; 2040db64fe02SNick Piggin struct vm_struct *area; 2041d98c9e83SAndrey Ryabinin unsigned long requested_size = size; 20421da177e4SLinus Torvalds 204352fd24caSGiridhar Pemmasani BUG_ON(in_interrupt()); 20441da177e4SLinus Torvalds size = PAGE_ALIGN(size); 204531be8309SOGAWA Hirofumi if (unlikely(!size)) 204631be8309SOGAWA Hirofumi return NULL; 20471da177e4SLinus Torvalds 2048252e5c6eSzijun_hu if (flags & VM_IOREMAP) 2049252e5c6eSzijun_hu align = 1ul << clamp_t(int, get_count_order_long(size), 2050252e5c6eSzijun_hu PAGE_SHIFT, IOREMAP_MAX_ORDER); 2051252e5c6eSzijun_hu 2052cf88c790STejun Heo area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); 20531da177e4SLinus Torvalds if (unlikely(!area)) 20541da177e4SLinus Torvalds return NULL; 20551da177e4SLinus Torvalds 205671394fe5SAndrey Ryabinin if (!(flags & VM_NO_GUARD)) 20571da177e4SLinus Torvalds size += PAGE_SIZE; 20581da177e4SLinus Torvalds 2059db64fe02SNick Piggin va = alloc_vmap_area(size, align, start, end, node, gfp_mask); 2060db64fe02SNick Piggin if (IS_ERR(va)) { 2061db64fe02SNick Piggin kfree(area); 2062db64fe02SNick Piggin return NULL; 20631da177e4SLinus Torvalds } 20641da177e4SLinus Torvalds 2065d98c9e83SAndrey Ryabinin kasan_unpoison_vmalloc((void *)va->va_start, requested_size); 2066f5252e00SMitsuo Hayasaka 2067d98c9e83SAndrey Ryabinin setup_vmalloc_vm(area, va, flags, caller); 20683c5c3cfbSDaniel Axtens 20691da177e4SLinus Torvalds return area; 20701da177e4SLinus Torvalds } 20711da177e4SLinus Torvalds 2072c2968612SBenjamin Herrenschmidt struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, 2073c2968612SBenjamin Herrenschmidt unsigned long start, unsigned long end, 20745e6cafc8SMarek Szyprowski const void *caller) 2075c2968612SBenjamin Herrenschmidt { 207600ef2d2fSDavid Rientjes return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE, 207700ef2d2fSDavid Rientjes GFP_KERNEL, caller); 2078c2968612SBenjamin Herrenschmidt } 2079c2968612SBenjamin Herrenschmidt 20801da177e4SLinus Torvalds /** 2081183ff22bSSimon Arlott * get_vm_area - reserve a contiguous kernel virtual area 20821da177e4SLinus Torvalds * @size: size of the area 20831da177e4SLinus Torvalds * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC 20841da177e4SLinus Torvalds * 20851da177e4SLinus Torvalds * Search an area of @size in the kernel virtual mapping area, 20861da177e4SLinus Torvalds * and reserved it for out purposes. Returns the area descriptor 20871da177e4SLinus Torvalds * on success or %NULL on failure. 2088a862f68aSMike Rapoport * 2089a862f68aSMike Rapoport * Return: the area descriptor on success or %NULL on failure. 20901da177e4SLinus Torvalds */ 20911da177e4SLinus Torvalds struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) 20921da177e4SLinus Torvalds { 20932dca6999SDavid Miller return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, 209400ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, 209500ef2d2fSDavid Rientjes __builtin_return_address(0)); 209623016969SChristoph Lameter } 209723016969SChristoph Lameter 209823016969SChristoph Lameter struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, 20995e6cafc8SMarek Szyprowski const void *caller) 210023016969SChristoph Lameter { 21012dca6999SDavid Miller return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, 210200ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, caller); 21031da177e4SLinus Torvalds } 21041da177e4SLinus Torvalds 2105e9da6e99SMarek Szyprowski /** 2106e9da6e99SMarek Szyprowski * find_vm_area - find a continuous kernel virtual area 2107e9da6e99SMarek Szyprowski * @addr: base address 2108e9da6e99SMarek Szyprowski * 2109e9da6e99SMarek Szyprowski * Search for the kernel VM area starting at @addr, and return it. 2110e9da6e99SMarek Szyprowski * It is up to the caller to do all required locking to keep the returned 2111e9da6e99SMarek Szyprowski * pointer valid. 2112a862f68aSMike Rapoport * 2113a862f68aSMike Rapoport * Return: pointer to the found area or %NULL on faulure 2114e9da6e99SMarek Szyprowski */ 2115e9da6e99SMarek Szyprowski struct vm_struct *find_vm_area(const void *addr) 211683342314SNick Piggin { 2117db64fe02SNick Piggin struct vmap_area *va; 211883342314SNick Piggin 2119db64fe02SNick Piggin va = find_vmap_area((unsigned long)addr); 2120688fcbfcSPengfei Li if (!va) 21217856dfebSAndi Kleen return NULL; 2122688fcbfcSPengfei Li 2123688fcbfcSPengfei Li return va->vm; 21247856dfebSAndi Kleen } 21257856dfebSAndi Kleen 21261da177e4SLinus Torvalds /** 2127183ff22bSSimon Arlott * remove_vm_area - find and remove a continuous kernel virtual area 21281da177e4SLinus Torvalds * @addr: base address 21291da177e4SLinus Torvalds * 21301da177e4SLinus Torvalds * Search for the kernel VM area starting at @addr, and remove it. 21311da177e4SLinus Torvalds * This function returns the found VM area, but using it is NOT safe 21327856dfebSAndi Kleen * on SMP machines, except for its size or flags. 2133a862f68aSMike Rapoport * 2134a862f68aSMike Rapoport * Return: pointer to the found area or %NULL on faulure 21351da177e4SLinus Torvalds */ 2136b3bdda02SChristoph Lameter struct vm_struct *remove_vm_area(const void *addr) 21371da177e4SLinus Torvalds { 2138db64fe02SNick Piggin struct vmap_area *va; 2139db64fe02SNick Piggin 21405803ed29SChristoph Hellwig might_sleep(); 21415803ed29SChristoph Hellwig 2142dd3b8353SUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 2143dd3b8353SUladzislau Rezki (Sony) va = __find_vmap_area((unsigned long)addr); 2144688fcbfcSPengfei Li if (va && va->vm) { 2145db1aecafSMinchan Kim struct vm_struct *vm = va->vm; 2146f5252e00SMitsuo Hayasaka 2147c69480adSJoonsoo Kim va->vm = NULL; 2148c69480adSJoonsoo Kim spin_unlock(&vmap_area_lock); 2149c69480adSJoonsoo Kim 2150a5af5aa8SAndrey Ryabinin kasan_free_shadow(vm); 2151dd32c279SKAMEZAWA Hiroyuki free_unmap_vmap_area(va); 2152dd32c279SKAMEZAWA Hiroyuki 2153db64fe02SNick Piggin return vm; 2154db64fe02SNick Piggin } 2155dd3b8353SUladzislau Rezki (Sony) 2156dd3b8353SUladzislau Rezki (Sony) spin_unlock(&vmap_area_lock); 2157db64fe02SNick Piggin return NULL; 21581da177e4SLinus Torvalds } 21591da177e4SLinus Torvalds 2160868b104dSRick Edgecombe static inline void set_area_direct_map(const struct vm_struct *area, 2161868b104dSRick Edgecombe int (*set_direct_map)(struct page *page)) 2162868b104dSRick Edgecombe { 2163868b104dSRick Edgecombe int i; 2164868b104dSRick Edgecombe 2165868b104dSRick Edgecombe for (i = 0; i < area->nr_pages; i++) 2166868b104dSRick Edgecombe if (page_address(area->pages[i])) 2167868b104dSRick Edgecombe set_direct_map(area->pages[i]); 2168868b104dSRick Edgecombe } 2169868b104dSRick Edgecombe 2170868b104dSRick Edgecombe /* Handle removing and resetting vm mappings related to the vm_struct. */ 2171868b104dSRick Edgecombe static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages) 2172868b104dSRick Edgecombe { 2173868b104dSRick Edgecombe unsigned long start = ULONG_MAX, end = 0; 2174868b104dSRick Edgecombe int flush_reset = area->flags & VM_FLUSH_RESET_PERMS; 217531e67340SRick Edgecombe int flush_dmap = 0; 2176868b104dSRick Edgecombe int i; 2177868b104dSRick Edgecombe 2178868b104dSRick Edgecombe remove_vm_area(area->addr); 2179868b104dSRick Edgecombe 2180868b104dSRick Edgecombe /* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */ 2181868b104dSRick Edgecombe if (!flush_reset) 2182868b104dSRick Edgecombe return; 2183868b104dSRick Edgecombe 2184868b104dSRick Edgecombe /* 2185868b104dSRick Edgecombe * If not deallocating pages, just do the flush of the VM area and 2186868b104dSRick Edgecombe * return. 2187868b104dSRick Edgecombe */ 2188868b104dSRick Edgecombe if (!deallocate_pages) { 2189868b104dSRick Edgecombe vm_unmap_aliases(); 2190868b104dSRick Edgecombe return; 2191868b104dSRick Edgecombe } 2192868b104dSRick Edgecombe 2193868b104dSRick Edgecombe /* 2194868b104dSRick Edgecombe * If execution gets here, flush the vm mapping and reset the direct 2195868b104dSRick Edgecombe * map. Find the start and end range of the direct mappings to make sure 2196868b104dSRick Edgecombe * the vm_unmap_aliases() flush includes the direct map. 2197868b104dSRick Edgecombe */ 2198868b104dSRick Edgecombe for (i = 0; i < area->nr_pages; i++) { 21998e41f872SRick Edgecombe unsigned long addr = (unsigned long)page_address(area->pages[i]); 22008e41f872SRick Edgecombe if (addr) { 2201868b104dSRick Edgecombe start = min(addr, start); 22028e41f872SRick Edgecombe end = max(addr + PAGE_SIZE, end); 220331e67340SRick Edgecombe flush_dmap = 1; 2204868b104dSRick Edgecombe } 2205868b104dSRick Edgecombe } 2206868b104dSRick Edgecombe 2207868b104dSRick Edgecombe /* 2208868b104dSRick Edgecombe * Set direct map to something invalid so that it won't be cached if 2209868b104dSRick Edgecombe * there are any accesses after the TLB flush, then flush the TLB and 2210868b104dSRick Edgecombe * reset the direct map permissions to the default. 2211868b104dSRick Edgecombe */ 2212868b104dSRick Edgecombe set_area_direct_map(area, set_direct_map_invalid_noflush); 221331e67340SRick Edgecombe _vm_unmap_aliases(start, end, flush_dmap); 2214868b104dSRick Edgecombe set_area_direct_map(area, set_direct_map_default_noflush); 2215868b104dSRick Edgecombe } 2216868b104dSRick Edgecombe 2217b3bdda02SChristoph Lameter static void __vunmap(const void *addr, int deallocate_pages) 22181da177e4SLinus Torvalds { 22191da177e4SLinus Torvalds struct vm_struct *area; 22201da177e4SLinus Torvalds 22211da177e4SLinus Torvalds if (!addr) 22221da177e4SLinus Torvalds return; 22231da177e4SLinus Torvalds 2224e69e9d4aSHATAYAMA Daisuke if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n", 2225ab15d9b4SDan Carpenter addr)) 22261da177e4SLinus Torvalds return; 22271da177e4SLinus Torvalds 22286ade2032SLiviu Dudau area = find_vm_area(addr); 22291da177e4SLinus Torvalds if (unlikely(!area)) { 22304c8573e2SArjan van de Ven WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", 22311da177e4SLinus Torvalds addr); 22321da177e4SLinus Torvalds return; 22331da177e4SLinus Torvalds } 22341da177e4SLinus Torvalds 223505e3ff95SChintan Pandya debug_check_no_locks_freed(area->addr, get_vm_area_size(area)); 223605e3ff95SChintan Pandya debug_check_no_obj_freed(area->addr, get_vm_area_size(area)); 22379a11b49aSIngo Molnar 22383c5c3cfbSDaniel Axtens kasan_poison_vmalloc(area->addr, area->size); 22393c5c3cfbSDaniel Axtens 2240868b104dSRick Edgecombe vm_remove_mappings(area, deallocate_pages); 2241868b104dSRick Edgecombe 22421da177e4SLinus Torvalds if (deallocate_pages) { 22431da177e4SLinus Torvalds int i; 22441da177e4SLinus Torvalds 22451da177e4SLinus Torvalds for (i = 0; i < area->nr_pages; i++) { 2246bf53d6f8SChristoph Lameter struct page *page = area->pages[i]; 2247bf53d6f8SChristoph Lameter 2248bf53d6f8SChristoph Lameter BUG_ON(!page); 22494949148aSVladimir Davydov __free_pages(page, 0); 22501da177e4SLinus Torvalds } 225197105f0aSRoman Gushchin atomic_long_sub(area->nr_pages, &nr_vmalloc_pages); 22521da177e4SLinus Torvalds 2253244d63eeSDavid Rientjes kvfree(area->pages); 22541da177e4SLinus Torvalds } 22551da177e4SLinus Torvalds 22561da177e4SLinus Torvalds kfree(area); 22571da177e4SLinus Torvalds return; 22581da177e4SLinus Torvalds } 22591da177e4SLinus Torvalds 2260bf22e37aSAndrey Ryabinin static inline void __vfree_deferred(const void *addr) 2261bf22e37aSAndrey Ryabinin { 2262bf22e37aSAndrey Ryabinin /* 2263bf22e37aSAndrey Ryabinin * Use raw_cpu_ptr() because this can be called from preemptible 2264bf22e37aSAndrey Ryabinin * context. Preemption is absolutely fine here, because the llist_add() 2265bf22e37aSAndrey Ryabinin * implementation is lockless, so it works even if we are adding to 226673221d88SJeongtae Park * another cpu's list. schedule_work() should be fine with this too. 2267bf22e37aSAndrey Ryabinin */ 2268bf22e37aSAndrey Ryabinin struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred); 2269bf22e37aSAndrey Ryabinin 2270bf22e37aSAndrey Ryabinin if (llist_add((struct llist_node *)addr, &p->list)) 2271bf22e37aSAndrey Ryabinin schedule_work(&p->wq); 2272bf22e37aSAndrey Ryabinin } 2273bf22e37aSAndrey Ryabinin 2274bf22e37aSAndrey Ryabinin /** 2275bf22e37aSAndrey Ryabinin * vfree_atomic - release memory allocated by vmalloc() 2276bf22e37aSAndrey Ryabinin * @addr: memory base address 2277bf22e37aSAndrey Ryabinin * 2278bf22e37aSAndrey Ryabinin * This one is just like vfree() but can be called in any atomic context 2279bf22e37aSAndrey Ryabinin * except NMIs. 2280bf22e37aSAndrey Ryabinin */ 2281bf22e37aSAndrey Ryabinin void vfree_atomic(const void *addr) 2282bf22e37aSAndrey Ryabinin { 2283bf22e37aSAndrey Ryabinin BUG_ON(in_nmi()); 2284bf22e37aSAndrey Ryabinin 2285bf22e37aSAndrey Ryabinin kmemleak_free(addr); 2286bf22e37aSAndrey Ryabinin 2287bf22e37aSAndrey Ryabinin if (!addr) 2288bf22e37aSAndrey Ryabinin return; 2289bf22e37aSAndrey Ryabinin __vfree_deferred(addr); 2290bf22e37aSAndrey Ryabinin } 2291bf22e37aSAndrey Ryabinin 2292c67dc624SRoman Penyaev static void __vfree(const void *addr) 2293c67dc624SRoman Penyaev { 2294c67dc624SRoman Penyaev if (unlikely(in_interrupt())) 2295c67dc624SRoman Penyaev __vfree_deferred(addr); 2296c67dc624SRoman Penyaev else 2297c67dc624SRoman Penyaev __vunmap(addr, 1); 2298c67dc624SRoman Penyaev } 2299c67dc624SRoman Penyaev 23001da177e4SLinus Torvalds /** 23011da177e4SLinus Torvalds * vfree - release memory allocated by vmalloc() 23021da177e4SLinus Torvalds * @addr: memory base address 23031da177e4SLinus Torvalds * 2304183ff22bSSimon Arlott * Free the virtually continuous memory area starting at @addr, as 230580e93effSPekka Enberg * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is 230680e93effSPekka Enberg * NULL, no operation is performed. 23071da177e4SLinus Torvalds * 230832fcfd40SAl Viro * Must not be called in NMI context (strictly speaking, only if we don't 230932fcfd40SAl Viro * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling 231032fcfd40SAl Viro * conventions for vfree() arch-depenedent would be a really bad idea) 231132fcfd40SAl Viro * 23123ca4ea3aSAndrey Ryabinin * May sleep if called *not* from interrupt context. 23133ca4ea3aSAndrey Ryabinin * 23140e056eb5Smchehab@s-opensource.com * NOTE: assumes that the object at @addr has a size >= sizeof(llist_node) 23151da177e4SLinus Torvalds */ 2316b3bdda02SChristoph Lameter void vfree(const void *addr) 23171da177e4SLinus Torvalds { 231832fcfd40SAl Viro BUG_ON(in_nmi()); 231989219d37SCatalin Marinas 232089219d37SCatalin Marinas kmemleak_free(addr); 232189219d37SCatalin Marinas 2322a8dda165SAndrey Ryabinin might_sleep_if(!in_interrupt()); 2323a8dda165SAndrey Ryabinin 232432fcfd40SAl Viro if (!addr) 232532fcfd40SAl Viro return; 2326c67dc624SRoman Penyaev 2327c67dc624SRoman Penyaev __vfree(addr); 23281da177e4SLinus Torvalds } 23291da177e4SLinus Torvalds EXPORT_SYMBOL(vfree); 23301da177e4SLinus Torvalds 23311da177e4SLinus Torvalds /** 23321da177e4SLinus Torvalds * vunmap - release virtual mapping obtained by vmap() 23331da177e4SLinus Torvalds * @addr: memory base address 23341da177e4SLinus Torvalds * 23351da177e4SLinus Torvalds * Free the virtually contiguous memory area starting at @addr, 23361da177e4SLinus Torvalds * which was created from the page array passed to vmap(). 23371da177e4SLinus Torvalds * 233880e93effSPekka Enberg * Must not be called in interrupt context. 23391da177e4SLinus Torvalds */ 2340b3bdda02SChristoph Lameter void vunmap(const void *addr) 23411da177e4SLinus Torvalds { 23421da177e4SLinus Torvalds BUG_ON(in_interrupt()); 234334754b69SPeter Zijlstra might_sleep(); 234432fcfd40SAl Viro if (addr) 23451da177e4SLinus Torvalds __vunmap(addr, 0); 23461da177e4SLinus Torvalds } 23471da177e4SLinus Torvalds EXPORT_SYMBOL(vunmap); 23481da177e4SLinus Torvalds 23491da177e4SLinus Torvalds /** 23501da177e4SLinus Torvalds * vmap - map an array of pages into virtually contiguous space 23511da177e4SLinus Torvalds * @pages: array of page pointers 23521da177e4SLinus Torvalds * @count: number of pages to map 23531da177e4SLinus Torvalds * @flags: vm_area->flags 23541da177e4SLinus Torvalds * @prot: page protection for the mapping 23551da177e4SLinus Torvalds * 23561da177e4SLinus Torvalds * Maps @count pages from @pages into contiguous kernel virtual 23571da177e4SLinus Torvalds * space. 2358a862f68aSMike Rapoport * 2359a862f68aSMike Rapoport * Return: the address of the area or %NULL on failure 23601da177e4SLinus Torvalds */ 23611da177e4SLinus Torvalds void *vmap(struct page **pages, unsigned int count, 23621da177e4SLinus Torvalds unsigned long flags, pgprot_t prot) 23631da177e4SLinus Torvalds { 23641da177e4SLinus Torvalds struct vm_struct *area; 236565ee03c4SGuillermo Julián Moreno unsigned long size; /* In bytes */ 23661da177e4SLinus Torvalds 236734754b69SPeter Zijlstra might_sleep(); 236834754b69SPeter Zijlstra 2369ca79b0c2SArun KS if (count > totalram_pages()) 23701da177e4SLinus Torvalds return NULL; 23711da177e4SLinus Torvalds 237265ee03c4SGuillermo Julián Moreno size = (unsigned long)count << PAGE_SHIFT; 237365ee03c4SGuillermo Julián Moreno area = get_vm_area_caller(size, flags, __builtin_return_address(0)); 23741da177e4SLinus Torvalds if (!area) 23751da177e4SLinus Torvalds return NULL; 237623016969SChristoph Lameter 2377cca98e9fSChristoph Hellwig if (map_kernel_range((unsigned long)area->addr, size, pgprot_nx(prot), 2378ed1f324cSChristoph Hellwig pages) < 0) { 23791da177e4SLinus Torvalds vunmap(area->addr); 23801da177e4SLinus Torvalds return NULL; 23811da177e4SLinus Torvalds } 23821da177e4SLinus Torvalds 23831da177e4SLinus Torvalds return area->addr; 23841da177e4SLinus Torvalds } 23851da177e4SLinus Torvalds EXPORT_SYMBOL(vmap); 23861da177e4SLinus Torvalds 2387e31d9eb5SAdrian Bunk static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, 23883722e13cSWanpeng Li pgprot_t prot, int node) 23891da177e4SLinus Torvalds { 23901da177e4SLinus Torvalds struct page **pages; 23911da177e4SLinus Torvalds unsigned int nr_pages, array_size, i; 2392930f036bSDavid Rientjes const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; 2393704b862fSLaura Abbott const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN; 2394704b862fSLaura Abbott const gfp_t highmem_mask = (gfp_mask & (GFP_DMA | GFP_DMA32)) ? 2395704b862fSLaura Abbott 0 : 2396704b862fSLaura Abbott __GFP_HIGHMEM; 23971da177e4SLinus Torvalds 2398762216abSWanpeng Li nr_pages = get_vm_area_size(area) >> PAGE_SHIFT; 23991da177e4SLinus Torvalds array_size = (nr_pages * sizeof(struct page *)); 24001da177e4SLinus Torvalds 24011da177e4SLinus Torvalds /* Please note that the recursion is strictly bounded. */ 24028757d5faSJan Kiszka if (array_size > PAGE_SIZE) { 2403704b862fSLaura Abbott pages = __vmalloc_node(array_size, 1, nested_gfp|highmem_mask, 2404f38fcb9cSChristoph Hellwig node, area->caller); 2405286e1ea3SAndrew Morton } else { 2406976d6dfbSJan Beulich pages = kmalloc_node(array_size, nested_gfp, node); 2407286e1ea3SAndrew Morton } 24087ea36242SAustin Kim 24097ea36242SAustin Kim if (!pages) { 24101da177e4SLinus Torvalds remove_vm_area(area->addr); 24111da177e4SLinus Torvalds kfree(area); 24121da177e4SLinus Torvalds return NULL; 24131da177e4SLinus Torvalds } 24141da177e4SLinus Torvalds 24157ea36242SAustin Kim area->pages = pages; 24167ea36242SAustin Kim area->nr_pages = nr_pages; 24177ea36242SAustin Kim 24181da177e4SLinus Torvalds for (i = 0; i < area->nr_pages; i++) { 2419bf53d6f8SChristoph Lameter struct page *page; 2420bf53d6f8SChristoph Lameter 24214b90951cSJianguo Wu if (node == NUMA_NO_NODE) 2422704b862fSLaura Abbott page = alloc_page(alloc_mask|highmem_mask); 2423930fc45aSChristoph Lameter else 2424704b862fSLaura Abbott page = alloc_pages_node(node, alloc_mask|highmem_mask, 0); 2425bf53d6f8SChristoph Lameter 2426bf53d6f8SChristoph Lameter if (unlikely(!page)) { 24271da177e4SLinus Torvalds /* Successfully allocated i pages, free them in __vunmap() */ 24281da177e4SLinus Torvalds area->nr_pages = i; 242997105f0aSRoman Gushchin atomic_long_add(area->nr_pages, &nr_vmalloc_pages); 24301da177e4SLinus Torvalds goto fail; 24311da177e4SLinus Torvalds } 2432bf53d6f8SChristoph Lameter area->pages[i] = page; 2433dcf61ff0SLiu Xiang if (gfpflags_allow_blocking(gfp_mask)) 2434660654f9SEric Dumazet cond_resched(); 24351da177e4SLinus Torvalds } 243697105f0aSRoman Gushchin atomic_long_add(area->nr_pages, &nr_vmalloc_pages); 24371da177e4SLinus Torvalds 2438ed1f324cSChristoph Hellwig if (map_kernel_range((unsigned long)area->addr, get_vm_area_size(area), 2439ed1f324cSChristoph Hellwig prot, pages) < 0) 24401da177e4SLinus Torvalds goto fail; 2441ed1f324cSChristoph Hellwig 24421da177e4SLinus Torvalds return area->addr; 24431da177e4SLinus Torvalds 24441da177e4SLinus Torvalds fail: 2445a8e99259SMichal Hocko warn_alloc(gfp_mask, NULL, 24467877cdccSMichal Hocko "vmalloc: allocation failure, allocated %ld of %ld bytes", 244722943ab1SDave Hansen (area->nr_pages*PAGE_SIZE), area->size); 2448c67dc624SRoman Penyaev __vfree(area->addr); 24491da177e4SLinus Torvalds return NULL; 24501da177e4SLinus Torvalds } 24511da177e4SLinus Torvalds 2452d0a21265SDavid Rientjes /** 2453d0a21265SDavid Rientjes * __vmalloc_node_range - allocate virtually contiguous memory 2454d0a21265SDavid Rientjes * @size: allocation size 2455d0a21265SDavid Rientjes * @align: desired alignment 2456d0a21265SDavid Rientjes * @start: vm area range start 2457d0a21265SDavid Rientjes * @end: vm area range end 2458d0a21265SDavid Rientjes * @gfp_mask: flags for the page level allocator 2459d0a21265SDavid Rientjes * @prot: protection mask for the allocated pages 2460cb9e3c29SAndrey Ryabinin * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) 246100ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 2462d0a21265SDavid Rientjes * @caller: caller's return address 2463d0a21265SDavid Rientjes * 2464d0a21265SDavid Rientjes * Allocate enough pages to cover @size from the page level 2465d0a21265SDavid Rientjes * allocator with @gfp_mask flags. Map them into contiguous 2466d0a21265SDavid Rientjes * kernel virtual space, using a pagetable protection of @prot. 2467a862f68aSMike Rapoport * 2468a862f68aSMike Rapoport * Return: the address of the area or %NULL on failure 2469d0a21265SDavid Rientjes */ 2470d0a21265SDavid Rientjes void *__vmalloc_node_range(unsigned long size, unsigned long align, 2471d0a21265SDavid Rientjes unsigned long start, unsigned long end, gfp_t gfp_mask, 2472cb9e3c29SAndrey Ryabinin pgprot_t prot, unsigned long vm_flags, int node, 2473cb9e3c29SAndrey Ryabinin const void *caller) 2474930fc45aSChristoph Lameter { 2475d0a21265SDavid Rientjes struct vm_struct *area; 2476d0a21265SDavid Rientjes void *addr; 2477d0a21265SDavid Rientjes unsigned long real_size = size; 2478d0a21265SDavid Rientjes 2479d0a21265SDavid Rientjes size = PAGE_ALIGN(size); 2480ca79b0c2SArun KS if (!size || (size >> PAGE_SHIFT) > totalram_pages()) 2481de7d2b56SJoe Perches goto fail; 2482d0a21265SDavid Rientjes 2483d98c9e83SAndrey Ryabinin area = __get_vm_area_node(real_size, align, VM_ALLOC | VM_UNINITIALIZED | 2484cb9e3c29SAndrey Ryabinin vm_flags, start, end, node, gfp_mask, caller); 2485d0a21265SDavid Rientjes if (!area) 2486de7d2b56SJoe Perches goto fail; 2487d0a21265SDavid Rientjes 24883722e13cSWanpeng Li addr = __vmalloc_area_node(area, gfp_mask, prot, node); 24891368edf0SMel Gorman if (!addr) 2490b82225f3SWanpeng Li return NULL; 249189219d37SCatalin Marinas 249289219d37SCatalin Marinas /* 249320fc02b4SZhang Yanfei * In this function, newly allocated vm_struct has VM_UNINITIALIZED 249420fc02b4SZhang Yanfei * flag. It means that vm_struct is not fully initialized. 24954341fa45SJoonsoo Kim * Now, it is fully initialized, so remove this flag here. 2496f5252e00SMitsuo Hayasaka */ 249720fc02b4SZhang Yanfei clear_vm_uninitialized_flag(area); 2498f5252e00SMitsuo Hayasaka 249994f4a161SCatalin Marinas kmemleak_vmalloc(area, size, gfp_mask); 250089219d37SCatalin Marinas 250189219d37SCatalin Marinas return addr; 2502de7d2b56SJoe Perches 2503de7d2b56SJoe Perches fail: 2504a8e99259SMichal Hocko warn_alloc(gfp_mask, NULL, 25057877cdccSMichal Hocko "vmalloc: allocation failure: %lu bytes", real_size); 2506de7d2b56SJoe Perches return NULL; 2507930fc45aSChristoph Lameter } 2508930fc45aSChristoph Lameter 25091da177e4SLinus Torvalds /** 2510930fc45aSChristoph Lameter * __vmalloc_node - allocate virtually contiguous memory 25111da177e4SLinus Torvalds * @size: allocation size 25122dca6999SDavid Miller * @align: desired alignment 25131da177e4SLinus Torvalds * @gfp_mask: flags for the page level allocator 251400ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 2515c85d194bSRandy Dunlap * @caller: caller's return address 25161da177e4SLinus Torvalds * 2517f38fcb9cSChristoph Hellwig * Allocate enough pages to cover @size from the page level allocator with 2518f38fcb9cSChristoph Hellwig * @gfp_mask flags. Map them into contiguous kernel virtual space. 2519a7c3e901SMichal Hocko * 2520dcda9b04SMichal Hocko * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL 2521a7c3e901SMichal Hocko * and __GFP_NOFAIL are not supported 2522a7c3e901SMichal Hocko * 2523a7c3e901SMichal Hocko * Any use of gfp flags outside of GFP_KERNEL should be consulted 2524a7c3e901SMichal Hocko * with mm people. 2525a862f68aSMike Rapoport * 2526a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 25271da177e4SLinus Torvalds */ 25282b905948SChristoph Hellwig void *__vmalloc_node(unsigned long size, unsigned long align, 2529f38fcb9cSChristoph Hellwig gfp_t gfp_mask, int node, const void *caller) 25301da177e4SLinus Torvalds { 2531d0a21265SDavid Rientjes return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 2532f38fcb9cSChristoph Hellwig gfp_mask, PAGE_KERNEL, 0, node, caller); 25331da177e4SLinus Torvalds } 2534c3f896dcSChristoph Hellwig /* 2535c3f896dcSChristoph Hellwig * This is only for performance analysis of vmalloc and stress purpose. 2536c3f896dcSChristoph Hellwig * It is required by vmalloc test module, therefore do not use it other 2537c3f896dcSChristoph Hellwig * than that. 2538c3f896dcSChristoph Hellwig */ 2539c3f896dcSChristoph Hellwig #ifdef CONFIG_TEST_VMALLOC_MODULE 2540c3f896dcSChristoph Hellwig EXPORT_SYMBOL_GPL(__vmalloc_node); 2541c3f896dcSChristoph Hellwig #endif 25421da177e4SLinus Torvalds 254388dca4caSChristoph Hellwig void *__vmalloc(unsigned long size, gfp_t gfp_mask) 2544930fc45aSChristoph Lameter { 2545f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, gfp_mask, NUMA_NO_NODE, 254623016969SChristoph Lameter __builtin_return_address(0)); 2547930fc45aSChristoph Lameter } 25481da177e4SLinus Torvalds EXPORT_SYMBOL(__vmalloc); 25491da177e4SLinus Torvalds 25501da177e4SLinus Torvalds /** 25511da177e4SLinus Torvalds * vmalloc - allocate virtually contiguous memory 25521da177e4SLinus Torvalds * @size: allocation size 255392eac168SMike Rapoport * 25541da177e4SLinus Torvalds * Allocate enough pages to cover @size from the page level 25551da177e4SLinus Torvalds * allocator and map them into contiguous kernel virtual space. 25561da177e4SLinus Torvalds * 2557c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 25581da177e4SLinus Torvalds * use __vmalloc() instead. 2559a862f68aSMike Rapoport * 2560a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 25611da177e4SLinus Torvalds */ 25621da177e4SLinus Torvalds void *vmalloc(unsigned long size) 25631da177e4SLinus Torvalds { 25644d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL, NUMA_NO_NODE, 25654d39d728SChristoph Hellwig __builtin_return_address(0)); 25661da177e4SLinus Torvalds } 25671da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc); 25681da177e4SLinus Torvalds 2569930fc45aSChristoph Lameter /** 2570e1ca7788SDave Young * vzalloc - allocate virtually contiguous memory with zero fill 2571e1ca7788SDave Young * @size: allocation size 257292eac168SMike Rapoport * 2573e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 2574e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 2575e1ca7788SDave Young * The memory allocated is set to zero. 2576e1ca7788SDave Young * 2577e1ca7788SDave Young * For tight control over page level allocator and protection flags 2578e1ca7788SDave Young * use __vmalloc() instead. 2579a862f68aSMike Rapoport * 2580a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 2581e1ca7788SDave Young */ 2582e1ca7788SDave Young void *vzalloc(unsigned long size) 2583e1ca7788SDave Young { 25844d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, 25854d39d728SChristoph Hellwig __builtin_return_address(0)); 2586e1ca7788SDave Young } 2587e1ca7788SDave Young EXPORT_SYMBOL(vzalloc); 2588e1ca7788SDave Young 2589e1ca7788SDave Young /** 2590ead04089SRolf Eike Beer * vmalloc_user - allocate zeroed virtually contiguous memory for userspace 259183342314SNick Piggin * @size: allocation size 2592ead04089SRolf Eike Beer * 2593ead04089SRolf Eike Beer * The resulting memory area is zeroed so it can be mapped to userspace 2594ead04089SRolf Eike Beer * without leaking data. 2595a862f68aSMike Rapoport * 2596a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 259783342314SNick Piggin */ 259883342314SNick Piggin void *vmalloc_user(unsigned long size) 259983342314SNick Piggin { 2600bc84c535SRoman Penyaev return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, 2601bc84c535SRoman Penyaev GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, 2602bc84c535SRoman Penyaev VM_USERMAP, NUMA_NO_NODE, 260300ef2d2fSDavid Rientjes __builtin_return_address(0)); 260483342314SNick Piggin } 260583342314SNick Piggin EXPORT_SYMBOL(vmalloc_user); 260683342314SNick Piggin 260783342314SNick Piggin /** 2608930fc45aSChristoph Lameter * vmalloc_node - allocate memory on a specific node 2609930fc45aSChristoph Lameter * @size: allocation size 2610d44e0780SRandy Dunlap * @node: numa node 2611930fc45aSChristoph Lameter * 2612930fc45aSChristoph Lameter * Allocate enough pages to cover @size from the page level 2613930fc45aSChristoph Lameter * allocator and map them into contiguous kernel virtual space. 2614930fc45aSChristoph Lameter * 2615c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 2616930fc45aSChristoph Lameter * use __vmalloc() instead. 2617a862f68aSMike Rapoport * 2618a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 2619930fc45aSChristoph Lameter */ 2620930fc45aSChristoph Lameter void *vmalloc_node(unsigned long size, int node) 2621930fc45aSChristoph Lameter { 2622f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL, node, 2623f38fcb9cSChristoph Hellwig __builtin_return_address(0)); 2624930fc45aSChristoph Lameter } 2625930fc45aSChristoph Lameter EXPORT_SYMBOL(vmalloc_node); 2626930fc45aSChristoph Lameter 2627e1ca7788SDave Young /** 2628e1ca7788SDave Young * vzalloc_node - allocate memory on a specific node with zero fill 2629e1ca7788SDave Young * @size: allocation size 2630e1ca7788SDave Young * @node: numa node 2631e1ca7788SDave Young * 2632e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 2633e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 2634e1ca7788SDave Young * The memory allocated is set to zero. 2635e1ca7788SDave Young * 2636a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 2637e1ca7788SDave Young */ 2638e1ca7788SDave Young void *vzalloc_node(unsigned long size, int node) 2639e1ca7788SDave Young { 26404d39d728SChristoph Hellwig return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, node, 26414d39d728SChristoph Hellwig __builtin_return_address(0)); 2642e1ca7788SDave Young } 2643e1ca7788SDave Young EXPORT_SYMBOL(vzalloc_node); 2644e1ca7788SDave Young 26450d08e0d3SAndi Kleen #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) 2646698d0831SMichal Hocko #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) 26470d08e0d3SAndi Kleen #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) 2648698d0831SMichal Hocko #define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL) 26490d08e0d3SAndi Kleen #else 2650698d0831SMichal Hocko /* 2651698d0831SMichal Hocko * 64b systems should always have either DMA or DMA32 zones. For others 2652698d0831SMichal Hocko * GFP_DMA32 should do the right thing and use the normal zone. 2653698d0831SMichal Hocko */ 2654698d0831SMichal Hocko #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL 26550d08e0d3SAndi Kleen #endif 26560d08e0d3SAndi Kleen 26571da177e4SLinus Torvalds /** 26581da177e4SLinus Torvalds * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 26591da177e4SLinus Torvalds * @size: allocation size 26601da177e4SLinus Torvalds * 26611da177e4SLinus Torvalds * Allocate enough 32bit PA addressable pages to cover @size from the 26621da177e4SLinus Torvalds * page level allocator and map them into contiguous kernel virtual space. 2663a862f68aSMike Rapoport * 2664a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 26651da177e4SLinus Torvalds */ 26661da177e4SLinus Torvalds void *vmalloc_32(unsigned long size) 26671da177e4SLinus Torvalds { 2668f38fcb9cSChristoph Hellwig return __vmalloc_node(size, 1, GFP_VMALLOC32, NUMA_NO_NODE, 2669f38fcb9cSChristoph Hellwig __builtin_return_address(0)); 26701da177e4SLinus Torvalds } 26711da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc_32); 26721da177e4SLinus Torvalds 267383342314SNick Piggin /** 2674ead04089SRolf Eike Beer * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 267583342314SNick Piggin * @size: allocation size 2676ead04089SRolf Eike Beer * 2677ead04089SRolf Eike Beer * The resulting memory area is 32bit addressable and zeroed so it can be 2678ead04089SRolf Eike Beer * mapped to userspace without leaking data. 2679a862f68aSMike Rapoport * 2680a862f68aSMike Rapoport * Return: pointer to the allocated memory or %NULL on error 268183342314SNick Piggin */ 268283342314SNick Piggin void *vmalloc_32_user(unsigned long size) 268383342314SNick Piggin { 2684bc84c535SRoman Penyaev return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, 2685bc84c535SRoman Penyaev GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, 2686bc84c535SRoman Penyaev VM_USERMAP, NUMA_NO_NODE, 26875a82ac71SRoman Penyaev __builtin_return_address(0)); 268883342314SNick Piggin } 268983342314SNick Piggin EXPORT_SYMBOL(vmalloc_32_user); 269083342314SNick Piggin 2691d0107eb0SKAMEZAWA Hiroyuki /* 2692d0107eb0SKAMEZAWA Hiroyuki * small helper routine , copy contents to buf from addr. 2693d0107eb0SKAMEZAWA Hiroyuki * If the page is not present, fill zero. 2694d0107eb0SKAMEZAWA Hiroyuki */ 2695d0107eb0SKAMEZAWA Hiroyuki 2696d0107eb0SKAMEZAWA Hiroyuki static int aligned_vread(char *buf, char *addr, unsigned long count) 2697d0107eb0SKAMEZAWA Hiroyuki { 2698d0107eb0SKAMEZAWA Hiroyuki struct page *p; 2699d0107eb0SKAMEZAWA Hiroyuki int copied = 0; 2700d0107eb0SKAMEZAWA Hiroyuki 2701d0107eb0SKAMEZAWA Hiroyuki while (count) { 2702d0107eb0SKAMEZAWA Hiroyuki unsigned long offset, length; 2703d0107eb0SKAMEZAWA Hiroyuki 2704891c49abSAlexander Kuleshov offset = offset_in_page(addr); 2705d0107eb0SKAMEZAWA Hiroyuki length = PAGE_SIZE - offset; 2706d0107eb0SKAMEZAWA Hiroyuki if (length > count) 2707d0107eb0SKAMEZAWA Hiroyuki length = count; 2708d0107eb0SKAMEZAWA Hiroyuki p = vmalloc_to_page(addr); 2709d0107eb0SKAMEZAWA Hiroyuki /* 2710d0107eb0SKAMEZAWA Hiroyuki * To do safe access to this _mapped_ area, we need 2711d0107eb0SKAMEZAWA Hiroyuki * lock. But adding lock here means that we need to add 2712d0107eb0SKAMEZAWA Hiroyuki * overhead of vmalloc()/vfree() calles for this _debug_ 2713d0107eb0SKAMEZAWA Hiroyuki * interface, rarely used. Instead of that, we'll use 2714d0107eb0SKAMEZAWA Hiroyuki * kmap() and get small overhead in this access function. 2715d0107eb0SKAMEZAWA Hiroyuki */ 2716d0107eb0SKAMEZAWA Hiroyuki if (p) { 2717d0107eb0SKAMEZAWA Hiroyuki /* 2718d0107eb0SKAMEZAWA Hiroyuki * we can expect USER0 is not used (see vread/vwrite's 2719d0107eb0SKAMEZAWA Hiroyuki * function description) 2720d0107eb0SKAMEZAWA Hiroyuki */ 27219b04c5feSCong Wang void *map = kmap_atomic(p); 2722d0107eb0SKAMEZAWA Hiroyuki memcpy(buf, map + offset, length); 27239b04c5feSCong Wang kunmap_atomic(map); 2724d0107eb0SKAMEZAWA Hiroyuki } else 2725d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, length); 2726d0107eb0SKAMEZAWA Hiroyuki 2727d0107eb0SKAMEZAWA Hiroyuki addr += length; 2728d0107eb0SKAMEZAWA Hiroyuki buf += length; 2729d0107eb0SKAMEZAWA Hiroyuki copied += length; 2730d0107eb0SKAMEZAWA Hiroyuki count -= length; 2731d0107eb0SKAMEZAWA Hiroyuki } 2732d0107eb0SKAMEZAWA Hiroyuki return copied; 2733d0107eb0SKAMEZAWA Hiroyuki } 2734d0107eb0SKAMEZAWA Hiroyuki 2735d0107eb0SKAMEZAWA Hiroyuki static int aligned_vwrite(char *buf, char *addr, unsigned long count) 2736d0107eb0SKAMEZAWA Hiroyuki { 2737d0107eb0SKAMEZAWA Hiroyuki struct page *p; 2738d0107eb0SKAMEZAWA Hiroyuki int copied = 0; 2739d0107eb0SKAMEZAWA Hiroyuki 2740d0107eb0SKAMEZAWA Hiroyuki while (count) { 2741d0107eb0SKAMEZAWA Hiroyuki unsigned long offset, length; 2742d0107eb0SKAMEZAWA Hiroyuki 2743891c49abSAlexander Kuleshov offset = offset_in_page(addr); 2744d0107eb0SKAMEZAWA Hiroyuki length = PAGE_SIZE - offset; 2745d0107eb0SKAMEZAWA Hiroyuki if (length > count) 2746d0107eb0SKAMEZAWA Hiroyuki length = count; 2747d0107eb0SKAMEZAWA Hiroyuki p = vmalloc_to_page(addr); 2748d0107eb0SKAMEZAWA Hiroyuki /* 2749d0107eb0SKAMEZAWA Hiroyuki * To do safe access to this _mapped_ area, we need 2750d0107eb0SKAMEZAWA Hiroyuki * lock. But adding lock here means that we need to add 2751d0107eb0SKAMEZAWA Hiroyuki * overhead of vmalloc()/vfree() calles for this _debug_ 2752d0107eb0SKAMEZAWA Hiroyuki * interface, rarely used. Instead of that, we'll use 2753d0107eb0SKAMEZAWA Hiroyuki * kmap() and get small overhead in this access function. 2754d0107eb0SKAMEZAWA Hiroyuki */ 2755d0107eb0SKAMEZAWA Hiroyuki if (p) { 2756d0107eb0SKAMEZAWA Hiroyuki /* 2757d0107eb0SKAMEZAWA Hiroyuki * we can expect USER0 is not used (see vread/vwrite's 2758d0107eb0SKAMEZAWA Hiroyuki * function description) 2759d0107eb0SKAMEZAWA Hiroyuki */ 27609b04c5feSCong Wang void *map = kmap_atomic(p); 2761d0107eb0SKAMEZAWA Hiroyuki memcpy(map + offset, buf, length); 27629b04c5feSCong Wang kunmap_atomic(map); 2763d0107eb0SKAMEZAWA Hiroyuki } 2764d0107eb0SKAMEZAWA Hiroyuki addr += length; 2765d0107eb0SKAMEZAWA Hiroyuki buf += length; 2766d0107eb0SKAMEZAWA Hiroyuki copied += length; 2767d0107eb0SKAMEZAWA Hiroyuki count -= length; 2768d0107eb0SKAMEZAWA Hiroyuki } 2769d0107eb0SKAMEZAWA Hiroyuki return copied; 2770d0107eb0SKAMEZAWA Hiroyuki } 2771d0107eb0SKAMEZAWA Hiroyuki 2772d0107eb0SKAMEZAWA Hiroyuki /** 2773d0107eb0SKAMEZAWA Hiroyuki * vread() - read vmalloc area in a safe way. 2774d0107eb0SKAMEZAWA Hiroyuki * @buf: buffer for reading data 2775d0107eb0SKAMEZAWA Hiroyuki * @addr: vm address. 2776d0107eb0SKAMEZAWA Hiroyuki * @count: number of bytes to be read. 2777d0107eb0SKAMEZAWA Hiroyuki * 2778d0107eb0SKAMEZAWA Hiroyuki * This function checks that addr is a valid vmalloc'ed area, and 2779d0107eb0SKAMEZAWA Hiroyuki * copy data from that area to a given buffer. If the given memory range 2780d0107eb0SKAMEZAWA Hiroyuki * of [addr...addr+count) includes some valid address, data is copied to 2781d0107eb0SKAMEZAWA Hiroyuki * proper area of @buf. If there are memory holes, they'll be zero-filled. 2782d0107eb0SKAMEZAWA Hiroyuki * IOREMAP area is treated as memory hole and no copy is done. 2783d0107eb0SKAMEZAWA Hiroyuki * 2784d0107eb0SKAMEZAWA Hiroyuki * If [addr...addr+count) doesn't includes any intersects with alive 2785a8e5202dSCong Wang * vm_struct area, returns 0. @buf should be kernel's buffer. 2786d0107eb0SKAMEZAWA Hiroyuki * 2787d0107eb0SKAMEZAWA Hiroyuki * Note: In usual ops, vread() is never necessary because the caller 2788d0107eb0SKAMEZAWA Hiroyuki * should know vmalloc() area is valid and can use memcpy(). 2789d0107eb0SKAMEZAWA Hiroyuki * This is for routines which have to access vmalloc area without 2790d9009d67SGeert Uytterhoeven * any information, as /dev/kmem. 2791a862f68aSMike Rapoport * 2792a862f68aSMike Rapoport * Return: number of bytes for which addr and buf should be increased 2793a862f68aSMike Rapoport * (same number as @count) or %0 if [addr...addr+count) doesn't 2794a862f68aSMike Rapoport * include any intersection with valid vmalloc area 2795d0107eb0SKAMEZAWA Hiroyuki */ 27961da177e4SLinus Torvalds long vread(char *buf, char *addr, unsigned long count) 27971da177e4SLinus Torvalds { 2798e81ce85fSJoonsoo Kim struct vmap_area *va; 2799e81ce85fSJoonsoo Kim struct vm_struct *vm; 28001da177e4SLinus Torvalds char *vaddr, *buf_start = buf; 2801d0107eb0SKAMEZAWA Hiroyuki unsigned long buflen = count; 28021da177e4SLinus Torvalds unsigned long n; 28031da177e4SLinus Torvalds 28041da177e4SLinus Torvalds /* Don't allow overflow */ 28051da177e4SLinus Torvalds if ((unsigned long) addr + count < count) 28061da177e4SLinus Torvalds count = -(unsigned long) addr; 28071da177e4SLinus Torvalds 2808e81ce85fSJoonsoo Kim spin_lock(&vmap_area_lock); 2809e81ce85fSJoonsoo Kim list_for_each_entry(va, &vmap_area_list, list) { 2810e81ce85fSJoonsoo Kim if (!count) 2811e81ce85fSJoonsoo Kim break; 2812e81ce85fSJoonsoo Kim 2813688fcbfcSPengfei Li if (!va->vm) 2814e81ce85fSJoonsoo Kim continue; 2815e81ce85fSJoonsoo Kim 2816e81ce85fSJoonsoo Kim vm = va->vm; 2817e81ce85fSJoonsoo Kim vaddr = (char *) vm->addr; 2818762216abSWanpeng Li if (addr >= vaddr + get_vm_area_size(vm)) 28191da177e4SLinus Torvalds continue; 28201da177e4SLinus Torvalds while (addr < vaddr) { 28211da177e4SLinus Torvalds if (count == 0) 28221da177e4SLinus Torvalds goto finished; 28231da177e4SLinus Torvalds *buf = '\0'; 28241da177e4SLinus Torvalds buf++; 28251da177e4SLinus Torvalds addr++; 28261da177e4SLinus Torvalds count--; 28271da177e4SLinus Torvalds } 2828762216abSWanpeng Li n = vaddr + get_vm_area_size(vm) - addr; 2829d0107eb0SKAMEZAWA Hiroyuki if (n > count) 2830d0107eb0SKAMEZAWA Hiroyuki n = count; 2831e81ce85fSJoonsoo Kim if (!(vm->flags & VM_IOREMAP)) 2832d0107eb0SKAMEZAWA Hiroyuki aligned_vread(buf, addr, n); 2833d0107eb0SKAMEZAWA Hiroyuki else /* IOREMAP area is treated as memory hole */ 2834d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, n); 2835d0107eb0SKAMEZAWA Hiroyuki buf += n; 2836d0107eb0SKAMEZAWA Hiroyuki addr += n; 2837d0107eb0SKAMEZAWA Hiroyuki count -= n; 28381da177e4SLinus Torvalds } 28391da177e4SLinus Torvalds finished: 2840e81ce85fSJoonsoo Kim spin_unlock(&vmap_area_lock); 2841d0107eb0SKAMEZAWA Hiroyuki 2842d0107eb0SKAMEZAWA Hiroyuki if (buf == buf_start) 2843d0107eb0SKAMEZAWA Hiroyuki return 0; 2844d0107eb0SKAMEZAWA Hiroyuki /* zero-fill memory holes */ 2845d0107eb0SKAMEZAWA Hiroyuki if (buf != buf_start + buflen) 2846d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, buflen - (buf - buf_start)); 2847d0107eb0SKAMEZAWA Hiroyuki 2848d0107eb0SKAMEZAWA Hiroyuki return buflen; 28491da177e4SLinus Torvalds } 28501da177e4SLinus Torvalds 2851d0107eb0SKAMEZAWA Hiroyuki /** 2852d0107eb0SKAMEZAWA Hiroyuki * vwrite() - write vmalloc area in a safe way. 2853d0107eb0SKAMEZAWA Hiroyuki * @buf: buffer for source data 2854d0107eb0SKAMEZAWA Hiroyuki * @addr: vm address. 2855d0107eb0SKAMEZAWA Hiroyuki * @count: number of bytes to be read. 2856d0107eb0SKAMEZAWA Hiroyuki * 2857d0107eb0SKAMEZAWA Hiroyuki * This function checks that addr is a valid vmalloc'ed area, and 2858d0107eb0SKAMEZAWA Hiroyuki * copy data from a buffer to the given addr. If specified range of 2859d0107eb0SKAMEZAWA Hiroyuki * [addr...addr+count) includes some valid address, data is copied from 2860d0107eb0SKAMEZAWA Hiroyuki * proper area of @buf. If there are memory holes, no copy to hole. 2861d0107eb0SKAMEZAWA Hiroyuki * IOREMAP area is treated as memory hole and no copy is done. 2862d0107eb0SKAMEZAWA Hiroyuki * 2863d0107eb0SKAMEZAWA Hiroyuki * If [addr...addr+count) doesn't includes any intersects with alive 2864a8e5202dSCong Wang * vm_struct area, returns 0. @buf should be kernel's buffer. 2865d0107eb0SKAMEZAWA Hiroyuki * 2866d0107eb0SKAMEZAWA Hiroyuki * Note: In usual ops, vwrite() is never necessary because the caller 2867d0107eb0SKAMEZAWA Hiroyuki * should know vmalloc() area is valid and can use memcpy(). 2868d0107eb0SKAMEZAWA Hiroyuki * This is for routines which have to access vmalloc area without 2869d9009d67SGeert Uytterhoeven * any information, as /dev/kmem. 2870a862f68aSMike Rapoport * 2871a862f68aSMike Rapoport * Return: number of bytes for which addr and buf should be 2872a862f68aSMike Rapoport * increased (same number as @count) or %0 if [addr...addr+count) 2873a862f68aSMike Rapoport * doesn't include any intersection with valid vmalloc area 2874d0107eb0SKAMEZAWA Hiroyuki */ 28751da177e4SLinus Torvalds long vwrite(char *buf, char *addr, unsigned long count) 28761da177e4SLinus Torvalds { 2877e81ce85fSJoonsoo Kim struct vmap_area *va; 2878e81ce85fSJoonsoo Kim struct vm_struct *vm; 2879d0107eb0SKAMEZAWA Hiroyuki char *vaddr; 2880d0107eb0SKAMEZAWA Hiroyuki unsigned long n, buflen; 2881d0107eb0SKAMEZAWA Hiroyuki int copied = 0; 28821da177e4SLinus Torvalds 28831da177e4SLinus Torvalds /* Don't allow overflow */ 28841da177e4SLinus Torvalds if ((unsigned long) addr + count < count) 28851da177e4SLinus Torvalds count = -(unsigned long) addr; 2886d0107eb0SKAMEZAWA Hiroyuki buflen = count; 28871da177e4SLinus Torvalds 2888e81ce85fSJoonsoo Kim spin_lock(&vmap_area_lock); 2889e81ce85fSJoonsoo Kim list_for_each_entry(va, &vmap_area_list, list) { 2890e81ce85fSJoonsoo Kim if (!count) 2891e81ce85fSJoonsoo Kim break; 2892e81ce85fSJoonsoo Kim 2893688fcbfcSPengfei Li if (!va->vm) 2894e81ce85fSJoonsoo Kim continue; 2895e81ce85fSJoonsoo Kim 2896e81ce85fSJoonsoo Kim vm = va->vm; 2897e81ce85fSJoonsoo Kim vaddr = (char *) vm->addr; 2898762216abSWanpeng Li if (addr >= vaddr + get_vm_area_size(vm)) 28991da177e4SLinus Torvalds continue; 29001da177e4SLinus Torvalds while (addr < vaddr) { 29011da177e4SLinus Torvalds if (count == 0) 29021da177e4SLinus Torvalds goto finished; 29031da177e4SLinus Torvalds buf++; 29041da177e4SLinus Torvalds addr++; 29051da177e4SLinus Torvalds count--; 29061da177e4SLinus Torvalds } 2907762216abSWanpeng Li n = vaddr + get_vm_area_size(vm) - addr; 2908d0107eb0SKAMEZAWA Hiroyuki if (n > count) 2909d0107eb0SKAMEZAWA Hiroyuki n = count; 2910e81ce85fSJoonsoo Kim if (!(vm->flags & VM_IOREMAP)) { 2911d0107eb0SKAMEZAWA Hiroyuki aligned_vwrite(buf, addr, n); 2912d0107eb0SKAMEZAWA Hiroyuki copied++; 2913d0107eb0SKAMEZAWA Hiroyuki } 2914d0107eb0SKAMEZAWA Hiroyuki buf += n; 2915d0107eb0SKAMEZAWA Hiroyuki addr += n; 2916d0107eb0SKAMEZAWA Hiroyuki count -= n; 29171da177e4SLinus Torvalds } 29181da177e4SLinus Torvalds finished: 2919e81ce85fSJoonsoo Kim spin_unlock(&vmap_area_lock); 2920d0107eb0SKAMEZAWA Hiroyuki if (!copied) 2921d0107eb0SKAMEZAWA Hiroyuki return 0; 2922d0107eb0SKAMEZAWA Hiroyuki return buflen; 29231da177e4SLinus Torvalds } 292483342314SNick Piggin 292583342314SNick Piggin /** 2926e69e9d4aSHATAYAMA Daisuke * remap_vmalloc_range_partial - map vmalloc pages to userspace 2927e69e9d4aSHATAYAMA Daisuke * @vma: vma to cover 2928e69e9d4aSHATAYAMA Daisuke * @uaddr: target user address to start at 2929e69e9d4aSHATAYAMA Daisuke * @kaddr: virtual address of vmalloc kernel memory 2930bdebd6a2SJann Horn * @pgoff: offset from @kaddr to start at 2931e69e9d4aSHATAYAMA Daisuke * @size: size of map area 2932e69e9d4aSHATAYAMA Daisuke * 2933e69e9d4aSHATAYAMA Daisuke * Returns: 0 for success, -Exxx on failure 2934e69e9d4aSHATAYAMA Daisuke * 2935e69e9d4aSHATAYAMA Daisuke * This function checks that @kaddr is a valid vmalloc'ed area, 2936e69e9d4aSHATAYAMA Daisuke * and that it is big enough to cover the range starting at 2937e69e9d4aSHATAYAMA Daisuke * @uaddr in @vma. Will return failure if that criteria isn't 2938e69e9d4aSHATAYAMA Daisuke * met. 2939e69e9d4aSHATAYAMA Daisuke * 2940e69e9d4aSHATAYAMA Daisuke * Similar to remap_pfn_range() (see mm/memory.c) 2941e69e9d4aSHATAYAMA Daisuke */ 2942e69e9d4aSHATAYAMA Daisuke int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr, 2943bdebd6a2SJann Horn void *kaddr, unsigned long pgoff, 2944bdebd6a2SJann Horn unsigned long size) 2945e69e9d4aSHATAYAMA Daisuke { 2946e69e9d4aSHATAYAMA Daisuke struct vm_struct *area; 2947bdebd6a2SJann Horn unsigned long off; 2948bdebd6a2SJann Horn unsigned long end_index; 2949bdebd6a2SJann Horn 2950bdebd6a2SJann Horn if (check_shl_overflow(pgoff, PAGE_SHIFT, &off)) 2951bdebd6a2SJann Horn return -EINVAL; 2952e69e9d4aSHATAYAMA Daisuke 2953e69e9d4aSHATAYAMA Daisuke size = PAGE_ALIGN(size); 2954e69e9d4aSHATAYAMA Daisuke 2955e69e9d4aSHATAYAMA Daisuke if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr)) 2956e69e9d4aSHATAYAMA Daisuke return -EINVAL; 2957e69e9d4aSHATAYAMA Daisuke 2958e69e9d4aSHATAYAMA Daisuke area = find_vm_area(kaddr); 2959e69e9d4aSHATAYAMA Daisuke if (!area) 2960e69e9d4aSHATAYAMA Daisuke return -EINVAL; 2961e69e9d4aSHATAYAMA Daisuke 2962fe9041c2SChristoph Hellwig if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT))) 2963e69e9d4aSHATAYAMA Daisuke return -EINVAL; 2964e69e9d4aSHATAYAMA Daisuke 2965bdebd6a2SJann Horn if (check_add_overflow(size, off, &end_index) || 2966bdebd6a2SJann Horn end_index > get_vm_area_size(area)) 2967e69e9d4aSHATAYAMA Daisuke return -EINVAL; 2968bdebd6a2SJann Horn kaddr += off; 2969e69e9d4aSHATAYAMA Daisuke 2970e69e9d4aSHATAYAMA Daisuke do { 2971e69e9d4aSHATAYAMA Daisuke struct page *page = vmalloc_to_page(kaddr); 2972e69e9d4aSHATAYAMA Daisuke int ret; 2973e69e9d4aSHATAYAMA Daisuke 2974e69e9d4aSHATAYAMA Daisuke ret = vm_insert_page(vma, uaddr, page); 2975e69e9d4aSHATAYAMA Daisuke if (ret) 2976e69e9d4aSHATAYAMA Daisuke return ret; 2977e69e9d4aSHATAYAMA Daisuke 2978e69e9d4aSHATAYAMA Daisuke uaddr += PAGE_SIZE; 2979e69e9d4aSHATAYAMA Daisuke kaddr += PAGE_SIZE; 2980e69e9d4aSHATAYAMA Daisuke size -= PAGE_SIZE; 2981e69e9d4aSHATAYAMA Daisuke } while (size > 0); 2982e69e9d4aSHATAYAMA Daisuke 2983e69e9d4aSHATAYAMA Daisuke vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; 2984e69e9d4aSHATAYAMA Daisuke 2985e69e9d4aSHATAYAMA Daisuke return 0; 2986e69e9d4aSHATAYAMA Daisuke } 2987e69e9d4aSHATAYAMA Daisuke EXPORT_SYMBOL(remap_vmalloc_range_partial); 2988e69e9d4aSHATAYAMA Daisuke 2989e69e9d4aSHATAYAMA Daisuke /** 299083342314SNick Piggin * remap_vmalloc_range - map vmalloc pages to userspace 299183342314SNick Piggin * @vma: vma to cover (map full range of vma) 299283342314SNick Piggin * @addr: vmalloc memory 299383342314SNick Piggin * @pgoff: number of pages into addr before first page to map 29947682486bSRandy Dunlap * 29957682486bSRandy Dunlap * Returns: 0 for success, -Exxx on failure 299683342314SNick Piggin * 299783342314SNick Piggin * This function checks that addr is a valid vmalloc'ed area, and 299883342314SNick Piggin * that it is big enough to cover the vma. Will return failure if 299983342314SNick Piggin * that criteria isn't met. 300083342314SNick Piggin * 300172fd4a35SRobert P. J. Day * Similar to remap_pfn_range() (see mm/memory.c) 300283342314SNick Piggin */ 300383342314SNick Piggin int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 300483342314SNick Piggin unsigned long pgoff) 300583342314SNick Piggin { 3006e69e9d4aSHATAYAMA Daisuke return remap_vmalloc_range_partial(vma, vma->vm_start, 3007bdebd6a2SJann Horn addr, pgoff, 3008e69e9d4aSHATAYAMA Daisuke vma->vm_end - vma->vm_start); 300983342314SNick Piggin } 301083342314SNick Piggin EXPORT_SYMBOL(remap_vmalloc_range); 301183342314SNick Piggin 30128b1e0f81SAnshuman Khandual static int f(pte_t *pte, unsigned long addr, void *data) 30135f4352fbSJeremy Fitzhardinge { 3014cd12909cSDavid Vrabel pte_t ***p = data; 3015cd12909cSDavid Vrabel 3016cd12909cSDavid Vrabel if (p) { 3017cd12909cSDavid Vrabel *(*p) = pte; 3018cd12909cSDavid Vrabel (*p)++; 3019cd12909cSDavid Vrabel } 30205f4352fbSJeremy Fitzhardinge return 0; 30215f4352fbSJeremy Fitzhardinge } 30225f4352fbSJeremy Fitzhardinge 30235f4352fbSJeremy Fitzhardinge /** 30245f4352fbSJeremy Fitzhardinge * alloc_vm_area - allocate a range of kernel address space 30255f4352fbSJeremy Fitzhardinge * @size: size of the area 3026cd12909cSDavid Vrabel * @ptes: returns the PTEs for the address space 30277682486bSRandy Dunlap * 30287682486bSRandy Dunlap * Returns: NULL on failure, vm_struct on success 30295f4352fbSJeremy Fitzhardinge * 30305f4352fbSJeremy Fitzhardinge * This function reserves a range of kernel address space, and 30315f4352fbSJeremy Fitzhardinge * allocates pagetables to map that range. No actual mappings 3032cd12909cSDavid Vrabel * are created. 3033cd12909cSDavid Vrabel * 3034cd12909cSDavid Vrabel * If @ptes is non-NULL, pointers to the PTEs (in init_mm) 3035cd12909cSDavid Vrabel * allocated for the VM area are returned. 30365f4352fbSJeremy Fitzhardinge */ 3037cd12909cSDavid Vrabel struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes) 30385f4352fbSJeremy Fitzhardinge { 30395f4352fbSJeremy Fitzhardinge struct vm_struct *area; 30405f4352fbSJeremy Fitzhardinge 304123016969SChristoph Lameter area = get_vm_area_caller(size, VM_IOREMAP, 304223016969SChristoph Lameter __builtin_return_address(0)); 30435f4352fbSJeremy Fitzhardinge if (area == NULL) 30445f4352fbSJeremy Fitzhardinge return NULL; 30455f4352fbSJeremy Fitzhardinge 30465f4352fbSJeremy Fitzhardinge /* 30475f4352fbSJeremy Fitzhardinge * This ensures that page tables are constructed for this region 30485f4352fbSJeremy Fitzhardinge * of kernel virtual address space and mapped into init_mm. 30495f4352fbSJeremy Fitzhardinge */ 30505f4352fbSJeremy Fitzhardinge if (apply_to_page_range(&init_mm, (unsigned long)area->addr, 3051cd12909cSDavid Vrabel size, f, ptes ? &ptes : NULL)) { 30525f4352fbSJeremy Fitzhardinge free_vm_area(area); 30535f4352fbSJeremy Fitzhardinge return NULL; 30545f4352fbSJeremy Fitzhardinge } 30555f4352fbSJeremy Fitzhardinge 30565f4352fbSJeremy Fitzhardinge return area; 30575f4352fbSJeremy Fitzhardinge } 30585f4352fbSJeremy Fitzhardinge EXPORT_SYMBOL_GPL(alloc_vm_area); 30595f4352fbSJeremy Fitzhardinge 30605f4352fbSJeremy Fitzhardinge void free_vm_area(struct vm_struct *area) 30615f4352fbSJeremy Fitzhardinge { 30625f4352fbSJeremy Fitzhardinge struct vm_struct *ret; 30635f4352fbSJeremy Fitzhardinge ret = remove_vm_area(area->addr); 30645f4352fbSJeremy Fitzhardinge BUG_ON(ret != area); 30655f4352fbSJeremy Fitzhardinge kfree(area); 30665f4352fbSJeremy Fitzhardinge } 30675f4352fbSJeremy Fitzhardinge EXPORT_SYMBOL_GPL(free_vm_area); 3068a10aa579SChristoph Lameter 30694f8b02b4STejun Heo #ifdef CONFIG_SMP 3070ca23e405STejun Heo static struct vmap_area *node_to_va(struct rb_node *n) 3071ca23e405STejun Heo { 30724583e773SGeliang Tang return rb_entry_safe(n, struct vmap_area, rb_node); 3073ca23e405STejun Heo } 3074ca23e405STejun Heo 3075ca23e405STejun Heo /** 307668ad4a33SUladzislau Rezki (Sony) * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to 307768ad4a33SUladzislau Rezki (Sony) * @addr: target address 3078ca23e405STejun Heo * 307968ad4a33SUladzislau Rezki (Sony) * Returns: vmap_area if it is found. If there is no such area 308068ad4a33SUladzislau Rezki (Sony) * the first highest(reverse order) vmap_area is returned 308168ad4a33SUladzislau Rezki (Sony) * i.e. va->va_start < addr && va->va_end < addr or NULL 308268ad4a33SUladzislau Rezki (Sony) * if there are no any areas before @addr. 3083ca23e405STejun Heo */ 308468ad4a33SUladzislau Rezki (Sony) static struct vmap_area * 308568ad4a33SUladzislau Rezki (Sony) pvm_find_va_enclose_addr(unsigned long addr) 3086ca23e405STejun Heo { 308768ad4a33SUladzislau Rezki (Sony) struct vmap_area *va, *tmp; 308868ad4a33SUladzislau Rezki (Sony) struct rb_node *n; 308968ad4a33SUladzislau Rezki (Sony) 309068ad4a33SUladzislau Rezki (Sony) n = free_vmap_area_root.rb_node; 309168ad4a33SUladzislau Rezki (Sony) va = NULL; 3092ca23e405STejun Heo 3093ca23e405STejun Heo while (n) { 309468ad4a33SUladzislau Rezki (Sony) tmp = rb_entry(n, struct vmap_area, rb_node); 309568ad4a33SUladzislau Rezki (Sony) if (tmp->va_start <= addr) { 309668ad4a33SUladzislau Rezki (Sony) va = tmp; 309768ad4a33SUladzislau Rezki (Sony) if (tmp->va_end >= addr) 3098ca23e405STejun Heo break; 3099ca23e405STejun Heo 310068ad4a33SUladzislau Rezki (Sony) n = n->rb_right; 3101ca23e405STejun Heo } else { 310268ad4a33SUladzislau Rezki (Sony) n = n->rb_left; 3103ca23e405STejun Heo } 310468ad4a33SUladzislau Rezki (Sony) } 310568ad4a33SUladzislau Rezki (Sony) 310668ad4a33SUladzislau Rezki (Sony) return va; 3107ca23e405STejun Heo } 3108ca23e405STejun Heo 3109ca23e405STejun Heo /** 311068ad4a33SUladzislau Rezki (Sony) * pvm_determine_end_from_reverse - find the highest aligned address 311168ad4a33SUladzislau Rezki (Sony) * of free block below VMALLOC_END 311268ad4a33SUladzislau Rezki (Sony) * @va: 311368ad4a33SUladzislau Rezki (Sony) * in - the VA we start the search(reverse order); 311468ad4a33SUladzislau Rezki (Sony) * out - the VA with the highest aligned end address. 3115ca23e405STejun Heo * 311668ad4a33SUladzislau Rezki (Sony) * Returns: determined end address within vmap_area 3117ca23e405STejun Heo */ 311868ad4a33SUladzislau Rezki (Sony) static unsigned long 311968ad4a33SUladzislau Rezki (Sony) pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) 3120ca23e405STejun Heo { 312168ad4a33SUladzislau Rezki (Sony) unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 3122ca23e405STejun Heo unsigned long addr; 3123ca23e405STejun Heo 312468ad4a33SUladzislau Rezki (Sony) if (likely(*va)) { 312568ad4a33SUladzislau Rezki (Sony) list_for_each_entry_from_reverse((*va), 312668ad4a33SUladzislau Rezki (Sony) &free_vmap_area_list, list) { 312768ad4a33SUladzislau Rezki (Sony) addr = min((*va)->va_end & ~(align - 1), vmalloc_end); 312868ad4a33SUladzislau Rezki (Sony) if ((*va)->va_start < addr) 312968ad4a33SUladzislau Rezki (Sony) return addr; 313068ad4a33SUladzislau Rezki (Sony) } 3131ca23e405STejun Heo } 3132ca23e405STejun Heo 313368ad4a33SUladzislau Rezki (Sony) return 0; 3134ca23e405STejun Heo } 3135ca23e405STejun Heo 3136ca23e405STejun Heo /** 3137ca23e405STejun Heo * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator 3138ca23e405STejun Heo * @offsets: array containing offset of each area 3139ca23e405STejun Heo * @sizes: array containing size of each area 3140ca23e405STejun Heo * @nr_vms: the number of areas to allocate 3141ca23e405STejun Heo * @align: alignment, all entries in @offsets and @sizes must be aligned to this 3142ca23e405STejun Heo * 3143ca23e405STejun Heo * Returns: kmalloc'd vm_struct pointer array pointing to allocated 3144ca23e405STejun Heo * vm_structs on success, %NULL on failure 3145ca23e405STejun Heo * 3146ca23e405STejun Heo * Percpu allocator wants to use congruent vm areas so that it can 3147ca23e405STejun Heo * maintain the offsets among percpu areas. This function allocates 3148ec3f64fcSDavid Rientjes * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to 3149ec3f64fcSDavid Rientjes * be scattered pretty far, distance between two areas easily going up 3150ec3f64fcSDavid Rientjes * to gigabytes. To avoid interacting with regular vmallocs, these 3151ec3f64fcSDavid Rientjes * areas are allocated from top. 3152ca23e405STejun Heo * 3153ca23e405STejun Heo * Despite its complicated look, this allocator is rather simple. It 315468ad4a33SUladzislau Rezki (Sony) * does everything top-down and scans free blocks from the end looking 315568ad4a33SUladzislau Rezki (Sony) * for matching base. While scanning, if any of the areas do not fit the 315668ad4a33SUladzislau Rezki (Sony) * base address is pulled down to fit the area. Scanning is repeated till 315768ad4a33SUladzislau Rezki (Sony) * all the areas fit and then all necessary data structures are inserted 315868ad4a33SUladzislau Rezki (Sony) * and the result is returned. 3159ca23e405STejun Heo */ 3160ca23e405STejun Heo struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, 3161ca23e405STejun Heo const size_t *sizes, int nr_vms, 3162ec3f64fcSDavid Rientjes size_t align) 3163ca23e405STejun Heo { 3164ca23e405STejun Heo const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); 3165ca23e405STejun Heo const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 316668ad4a33SUladzislau Rezki (Sony) struct vmap_area **vas, *va; 3167ca23e405STejun Heo struct vm_struct **vms; 3168ca23e405STejun Heo int area, area2, last_area, term_area; 3169253a496dSDaniel Axtens unsigned long base, start, size, end, last_end, orig_start, orig_end; 3170ca23e405STejun Heo bool purged = false; 317168ad4a33SUladzislau Rezki (Sony) enum fit_type type; 3172ca23e405STejun Heo 3173ca23e405STejun Heo /* verify parameters and allocate data structures */ 3174891c49abSAlexander Kuleshov BUG_ON(offset_in_page(align) || !is_power_of_2(align)); 3175ca23e405STejun Heo for (last_area = 0, area = 0; area < nr_vms; area++) { 3176ca23e405STejun Heo start = offsets[area]; 3177ca23e405STejun Heo end = start + sizes[area]; 3178ca23e405STejun Heo 3179ca23e405STejun Heo /* is everything aligned properly? */ 3180ca23e405STejun Heo BUG_ON(!IS_ALIGNED(offsets[area], align)); 3181ca23e405STejun Heo BUG_ON(!IS_ALIGNED(sizes[area], align)); 3182ca23e405STejun Heo 3183ca23e405STejun Heo /* detect the area with the highest address */ 3184ca23e405STejun Heo if (start > offsets[last_area]) 3185ca23e405STejun Heo last_area = area; 3186ca23e405STejun Heo 3187c568da28SWei Yang for (area2 = area + 1; area2 < nr_vms; area2++) { 3188ca23e405STejun Heo unsigned long start2 = offsets[area2]; 3189ca23e405STejun Heo unsigned long end2 = start2 + sizes[area2]; 3190ca23e405STejun Heo 3191c568da28SWei Yang BUG_ON(start2 < end && start < end2); 3192ca23e405STejun Heo } 3193ca23e405STejun Heo } 3194ca23e405STejun Heo last_end = offsets[last_area] + sizes[last_area]; 3195ca23e405STejun Heo 3196ca23e405STejun Heo if (vmalloc_end - vmalloc_start < last_end) { 3197ca23e405STejun Heo WARN_ON(true); 3198ca23e405STejun Heo return NULL; 3199ca23e405STejun Heo } 3200ca23e405STejun Heo 32014d67d860SThomas Meyer vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL); 32024d67d860SThomas Meyer vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL); 3203ca23e405STejun Heo if (!vas || !vms) 3204f1db7afdSKautuk Consul goto err_free2; 3205ca23e405STejun Heo 3206ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 320768ad4a33SUladzislau Rezki (Sony) vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL); 3208ec3f64fcSDavid Rientjes vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL); 3209ca23e405STejun Heo if (!vas[area] || !vms[area]) 3210ca23e405STejun Heo goto err_free; 3211ca23e405STejun Heo } 3212ca23e405STejun Heo retry: 3213e36176beSUladzislau Rezki (Sony) spin_lock(&free_vmap_area_lock); 3214ca23e405STejun Heo 3215ca23e405STejun Heo /* start scanning - we scan from the top, begin with the last area */ 3216ca23e405STejun Heo area = term_area = last_area; 3217ca23e405STejun Heo start = offsets[area]; 3218ca23e405STejun Heo end = start + sizes[area]; 3219ca23e405STejun Heo 322068ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(vmalloc_end); 322168ad4a33SUladzislau Rezki (Sony) base = pvm_determine_end_from_reverse(&va, align) - end; 3222ca23e405STejun Heo 3223ca23e405STejun Heo while (true) { 3224ca23e405STejun Heo /* 3225ca23e405STejun Heo * base might have underflowed, add last_end before 3226ca23e405STejun Heo * comparing. 3227ca23e405STejun Heo */ 322868ad4a33SUladzislau Rezki (Sony) if (base + last_end < vmalloc_start + last_end) 322968ad4a33SUladzislau Rezki (Sony) goto overflow; 3230ca23e405STejun Heo 3231ca23e405STejun Heo /* 323268ad4a33SUladzislau Rezki (Sony) * Fitting base has not been found. 3233ca23e405STejun Heo */ 323468ad4a33SUladzislau Rezki (Sony) if (va == NULL) 323568ad4a33SUladzislau Rezki (Sony) goto overflow; 3236ca23e405STejun Heo 3237ca23e405STejun Heo /* 3238d8cc323dSQiujun Huang * If required width exceeds current VA block, move 32395336e52cSKuppuswamy Sathyanarayanan * base downwards and then recheck. 32405336e52cSKuppuswamy Sathyanarayanan */ 32415336e52cSKuppuswamy Sathyanarayanan if (base + end > va->va_end) { 32425336e52cSKuppuswamy Sathyanarayanan base = pvm_determine_end_from_reverse(&va, align) - end; 32435336e52cSKuppuswamy Sathyanarayanan term_area = area; 32445336e52cSKuppuswamy Sathyanarayanan continue; 32455336e52cSKuppuswamy Sathyanarayanan } 32465336e52cSKuppuswamy Sathyanarayanan 32475336e52cSKuppuswamy Sathyanarayanan /* 324868ad4a33SUladzislau Rezki (Sony) * If this VA does not fit, move base downwards and recheck. 3249ca23e405STejun Heo */ 32505336e52cSKuppuswamy Sathyanarayanan if (base + start < va->va_start) { 325168ad4a33SUladzislau Rezki (Sony) va = node_to_va(rb_prev(&va->rb_node)); 325268ad4a33SUladzislau Rezki (Sony) base = pvm_determine_end_from_reverse(&va, align) - end; 3253ca23e405STejun Heo term_area = area; 3254ca23e405STejun Heo continue; 3255ca23e405STejun Heo } 3256ca23e405STejun Heo 3257ca23e405STejun Heo /* 3258ca23e405STejun Heo * This area fits, move on to the previous one. If 3259ca23e405STejun Heo * the previous one is the terminal one, we're done. 3260ca23e405STejun Heo */ 3261ca23e405STejun Heo area = (area + nr_vms - 1) % nr_vms; 3262ca23e405STejun Heo if (area == term_area) 3263ca23e405STejun Heo break; 326468ad4a33SUladzislau Rezki (Sony) 3265ca23e405STejun Heo start = offsets[area]; 3266ca23e405STejun Heo end = start + sizes[area]; 326768ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(base + end); 3268ca23e405STejun Heo } 326968ad4a33SUladzislau Rezki (Sony) 3270ca23e405STejun Heo /* we've found a fitting base, insert all va's */ 3271ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 327268ad4a33SUladzislau Rezki (Sony) int ret; 3273ca23e405STejun Heo 327468ad4a33SUladzislau Rezki (Sony) start = base + offsets[area]; 327568ad4a33SUladzislau Rezki (Sony) size = sizes[area]; 327668ad4a33SUladzislau Rezki (Sony) 327768ad4a33SUladzislau Rezki (Sony) va = pvm_find_va_enclose_addr(start); 327868ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(va == NULL)) 327968ad4a33SUladzislau Rezki (Sony) /* It is a BUG(), but trigger recovery instead. */ 328068ad4a33SUladzislau Rezki (Sony) goto recovery; 328168ad4a33SUladzislau Rezki (Sony) 328268ad4a33SUladzislau Rezki (Sony) type = classify_va_fit_type(va, start, size); 328368ad4a33SUladzislau Rezki (Sony) if (WARN_ON_ONCE(type == NOTHING_FIT)) 328468ad4a33SUladzislau Rezki (Sony) /* It is a BUG(), but trigger recovery instead. */ 328568ad4a33SUladzislau Rezki (Sony) goto recovery; 328668ad4a33SUladzislau Rezki (Sony) 328768ad4a33SUladzislau Rezki (Sony) ret = adjust_va_to_fit_type(va, start, size, type); 328868ad4a33SUladzislau Rezki (Sony) if (unlikely(ret)) 328968ad4a33SUladzislau Rezki (Sony) goto recovery; 329068ad4a33SUladzislau Rezki (Sony) 329168ad4a33SUladzislau Rezki (Sony) /* Allocated area. */ 329268ad4a33SUladzislau Rezki (Sony) va = vas[area]; 329368ad4a33SUladzislau Rezki (Sony) va->va_start = start; 329468ad4a33SUladzislau Rezki (Sony) va->va_end = start + size; 3295ca23e405STejun Heo } 3296ca23e405STejun Heo 3297e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 3298ca23e405STejun Heo 3299253a496dSDaniel Axtens /* populate the kasan shadow space */ 3300253a496dSDaniel Axtens for (area = 0; area < nr_vms; area++) { 3301253a496dSDaniel Axtens if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area])) 3302253a496dSDaniel Axtens goto err_free_shadow; 3303253a496dSDaniel Axtens 3304253a496dSDaniel Axtens kasan_unpoison_vmalloc((void *)vas[area]->va_start, 3305253a496dSDaniel Axtens sizes[area]); 3306253a496dSDaniel Axtens } 3307253a496dSDaniel Axtens 3308ca23e405STejun Heo /* insert all vm's */ 3309e36176beSUladzislau Rezki (Sony) spin_lock(&vmap_area_lock); 3310e36176beSUladzislau Rezki (Sony) for (area = 0; area < nr_vms; area++) { 3311e36176beSUladzislau Rezki (Sony) insert_vmap_area(vas[area], &vmap_area_root, &vmap_area_list); 3312e36176beSUladzislau Rezki (Sony) 3313e36176beSUladzislau Rezki (Sony) setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC, 3314ca23e405STejun Heo pcpu_get_vm_areas); 3315e36176beSUladzislau Rezki (Sony) } 3316e36176beSUladzislau Rezki (Sony) spin_unlock(&vmap_area_lock); 3317ca23e405STejun Heo 3318ca23e405STejun Heo kfree(vas); 3319ca23e405STejun Heo return vms; 3320ca23e405STejun Heo 332168ad4a33SUladzislau Rezki (Sony) recovery: 3322e36176beSUladzislau Rezki (Sony) /* 3323e36176beSUladzislau Rezki (Sony) * Remove previously allocated areas. There is no 3324e36176beSUladzislau Rezki (Sony) * need in removing these areas from the busy tree, 3325e36176beSUladzislau Rezki (Sony) * because they are inserted only on the final step 3326e36176beSUladzislau Rezki (Sony) * and when pcpu_get_vm_areas() is success. 3327e36176beSUladzislau Rezki (Sony) */ 332868ad4a33SUladzislau Rezki (Sony) while (area--) { 3329253a496dSDaniel Axtens orig_start = vas[area]->va_start; 3330253a496dSDaniel Axtens orig_end = vas[area]->va_end; 3331253a496dSDaniel Axtens va = merge_or_add_vmap_area(vas[area], &free_vmap_area_root, 33323c5c3cfbSDaniel Axtens &free_vmap_area_list); 3333253a496dSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 3334253a496dSDaniel Axtens va->va_start, va->va_end); 333568ad4a33SUladzislau Rezki (Sony) vas[area] = NULL; 333668ad4a33SUladzislau Rezki (Sony) } 333768ad4a33SUladzislau Rezki (Sony) 333868ad4a33SUladzislau Rezki (Sony) overflow: 3339e36176beSUladzislau Rezki (Sony) spin_unlock(&free_vmap_area_lock); 334068ad4a33SUladzislau Rezki (Sony) if (!purged) { 334168ad4a33SUladzislau Rezki (Sony) purge_vmap_area_lazy(); 334268ad4a33SUladzislau Rezki (Sony) purged = true; 334368ad4a33SUladzislau Rezki (Sony) 334468ad4a33SUladzislau Rezki (Sony) /* Before "retry", check if we recover. */ 334568ad4a33SUladzislau Rezki (Sony) for (area = 0; area < nr_vms; area++) { 334668ad4a33SUladzislau Rezki (Sony) if (vas[area]) 334768ad4a33SUladzislau Rezki (Sony) continue; 334868ad4a33SUladzislau Rezki (Sony) 334968ad4a33SUladzislau Rezki (Sony) vas[area] = kmem_cache_zalloc( 335068ad4a33SUladzislau Rezki (Sony) vmap_area_cachep, GFP_KERNEL); 335168ad4a33SUladzislau Rezki (Sony) if (!vas[area]) 335268ad4a33SUladzislau Rezki (Sony) goto err_free; 335368ad4a33SUladzislau Rezki (Sony) } 335468ad4a33SUladzislau Rezki (Sony) 335568ad4a33SUladzislau Rezki (Sony) goto retry; 335668ad4a33SUladzislau Rezki (Sony) } 335768ad4a33SUladzislau Rezki (Sony) 3358ca23e405STejun Heo err_free: 3359ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 336068ad4a33SUladzislau Rezki (Sony) if (vas[area]) 336168ad4a33SUladzislau Rezki (Sony) kmem_cache_free(vmap_area_cachep, vas[area]); 336268ad4a33SUladzislau Rezki (Sony) 3363ca23e405STejun Heo kfree(vms[area]); 3364ca23e405STejun Heo } 3365f1db7afdSKautuk Consul err_free2: 3366ca23e405STejun Heo kfree(vas); 3367ca23e405STejun Heo kfree(vms); 3368ca23e405STejun Heo return NULL; 3369253a496dSDaniel Axtens 3370253a496dSDaniel Axtens err_free_shadow: 3371253a496dSDaniel Axtens spin_lock(&free_vmap_area_lock); 3372253a496dSDaniel Axtens /* 3373253a496dSDaniel Axtens * We release all the vmalloc shadows, even the ones for regions that 3374253a496dSDaniel Axtens * hadn't been successfully added. This relies on kasan_release_vmalloc 3375253a496dSDaniel Axtens * being able to tolerate this case. 3376253a496dSDaniel Axtens */ 3377253a496dSDaniel Axtens for (area = 0; area < nr_vms; area++) { 3378253a496dSDaniel Axtens orig_start = vas[area]->va_start; 3379253a496dSDaniel Axtens orig_end = vas[area]->va_end; 3380253a496dSDaniel Axtens va = merge_or_add_vmap_area(vas[area], &free_vmap_area_root, 3381253a496dSDaniel Axtens &free_vmap_area_list); 3382253a496dSDaniel Axtens kasan_release_vmalloc(orig_start, orig_end, 3383253a496dSDaniel Axtens va->va_start, va->va_end); 3384253a496dSDaniel Axtens vas[area] = NULL; 3385253a496dSDaniel Axtens kfree(vms[area]); 3386253a496dSDaniel Axtens } 3387253a496dSDaniel Axtens spin_unlock(&free_vmap_area_lock); 3388253a496dSDaniel Axtens kfree(vas); 3389253a496dSDaniel Axtens kfree(vms); 3390253a496dSDaniel Axtens return NULL; 3391ca23e405STejun Heo } 3392ca23e405STejun Heo 3393ca23e405STejun Heo /** 3394ca23e405STejun Heo * pcpu_free_vm_areas - free vmalloc areas for percpu allocator 3395ca23e405STejun Heo * @vms: vm_struct pointer array returned by pcpu_get_vm_areas() 3396ca23e405STejun Heo * @nr_vms: the number of allocated areas 3397ca23e405STejun Heo * 3398ca23e405STejun Heo * Free vm_structs and the array allocated by pcpu_get_vm_areas(). 3399ca23e405STejun Heo */ 3400ca23e405STejun Heo void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) 3401ca23e405STejun Heo { 3402ca23e405STejun Heo int i; 3403ca23e405STejun Heo 3404ca23e405STejun Heo for (i = 0; i < nr_vms; i++) 3405ca23e405STejun Heo free_vm_area(vms[i]); 3406ca23e405STejun Heo kfree(vms); 3407ca23e405STejun Heo } 34084f8b02b4STejun Heo #endif /* CONFIG_SMP */ 3409a10aa579SChristoph Lameter 3410a10aa579SChristoph Lameter #ifdef CONFIG_PROC_FS 3411a10aa579SChristoph Lameter static void *s_start(struct seq_file *m, loff_t *pos) 3412e36176beSUladzislau Rezki (Sony) __acquires(&vmap_purge_lock) 3413d4033afdSJoonsoo Kim __acquires(&vmap_area_lock) 3414a10aa579SChristoph Lameter { 3415e36176beSUladzislau Rezki (Sony) mutex_lock(&vmap_purge_lock); 3416d4033afdSJoonsoo Kim spin_lock(&vmap_area_lock); 3417e36176beSUladzislau Rezki (Sony) 34183f500069Szijun_hu return seq_list_start(&vmap_area_list, *pos); 3419a10aa579SChristoph Lameter } 3420a10aa579SChristoph Lameter 3421a10aa579SChristoph Lameter static void *s_next(struct seq_file *m, void *p, loff_t *pos) 3422a10aa579SChristoph Lameter { 34233f500069Szijun_hu return seq_list_next(p, &vmap_area_list, pos); 3424a10aa579SChristoph Lameter } 3425a10aa579SChristoph Lameter 3426a10aa579SChristoph Lameter static void s_stop(struct seq_file *m, void *p) 3427e36176beSUladzislau Rezki (Sony) __releases(&vmap_purge_lock) 3428d4033afdSJoonsoo Kim __releases(&vmap_area_lock) 3429a10aa579SChristoph Lameter { 3430e36176beSUladzislau Rezki (Sony) mutex_unlock(&vmap_purge_lock); 3431d4033afdSJoonsoo Kim spin_unlock(&vmap_area_lock); 3432a10aa579SChristoph Lameter } 3433a10aa579SChristoph Lameter 3434a47a126aSEric Dumazet static void show_numa_info(struct seq_file *m, struct vm_struct *v) 3435a47a126aSEric Dumazet { 3436e5adfffcSKirill A. Shutemov if (IS_ENABLED(CONFIG_NUMA)) { 3437a47a126aSEric Dumazet unsigned int nr, *counters = m->private; 3438a47a126aSEric Dumazet 3439a47a126aSEric Dumazet if (!counters) 3440a47a126aSEric Dumazet return; 3441a47a126aSEric Dumazet 3442af12346cSWanpeng Li if (v->flags & VM_UNINITIALIZED) 3443af12346cSWanpeng Li return; 34447e5b528bSDmitry Vyukov /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ 34457e5b528bSDmitry Vyukov smp_rmb(); 3446af12346cSWanpeng Li 3447a47a126aSEric Dumazet memset(counters, 0, nr_node_ids * sizeof(unsigned int)); 3448a47a126aSEric Dumazet 3449a47a126aSEric Dumazet for (nr = 0; nr < v->nr_pages; nr++) 3450a47a126aSEric Dumazet counters[page_to_nid(v->pages[nr])]++; 3451a47a126aSEric Dumazet 3452a47a126aSEric Dumazet for_each_node_state(nr, N_HIGH_MEMORY) 3453a47a126aSEric Dumazet if (counters[nr]) 3454a47a126aSEric Dumazet seq_printf(m, " N%u=%u", nr, counters[nr]); 3455a47a126aSEric Dumazet } 3456a47a126aSEric Dumazet } 3457a47a126aSEric Dumazet 3458dd3b8353SUladzislau Rezki (Sony) static void show_purge_info(struct seq_file *m) 3459dd3b8353SUladzislau Rezki (Sony) { 3460dd3b8353SUladzislau Rezki (Sony) struct llist_node *head; 3461dd3b8353SUladzislau Rezki (Sony) struct vmap_area *va; 3462dd3b8353SUladzislau Rezki (Sony) 3463dd3b8353SUladzislau Rezki (Sony) head = READ_ONCE(vmap_purge_list.first); 3464dd3b8353SUladzislau Rezki (Sony) if (head == NULL) 3465dd3b8353SUladzislau Rezki (Sony) return; 3466dd3b8353SUladzislau Rezki (Sony) 3467dd3b8353SUladzislau Rezki (Sony) llist_for_each_entry(va, head, purge_list) { 3468dd3b8353SUladzislau Rezki (Sony) seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n", 3469dd3b8353SUladzislau Rezki (Sony) (void *)va->va_start, (void *)va->va_end, 3470dd3b8353SUladzislau Rezki (Sony) va->va_end - va->va_start); 3471dd3b8353SUladzislau Rezki (Sony) } 3472dd3b8353SUladzislau Rezki (Sony) } 3473dd3b8353SUladzislau Rezki (Sony) 3474a10aa579SChristoph Lameter static int s_show(struct seq_file *m, void *p) 3475a10aa579SChristoph Lameter { 34763f500069Szijun_hu struct vmap_area *va; 3477d4033afdSJoonsoo Kim struct vm_struct *v; 3478d4033afdSJoonsoo Kim 34793f500069Szijun_hu va = list_entry(p, struct vmap_area, list); 34803f500069Szijun_hu 3481c2ce8c14SWanpeng Li /* 3482688fcbfcSPengfei Li * s_show can encounter race with remove_vm_area, !vm on behalf 3483688fcbfcSPengfei Li * of vmap area is being tear down or vm_map_ram allocation. 3484c2ce8c14SWanpeng Li */ 3485688fcbfcSPengfei Li if (!va->vm) { 3486dd3b8353SUladzislau Rezki (Sony) seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n", 348778c72746SYisheng Xie (void *)va->va_start, (void *)va->va_end, 3488dd3b8353SUladzislau Rezki (Sony) va->va_end - va->va_start); 348978c72746SYisheng Xie 3490d4033afdSJoonsoo Kim return 0; 349178c72746SYisheng Xie } 3492d4033afdSJoonsoo Kim 3493d4033afdSJoonsoo Kim v = va->vm; 3494a10aa579SChristoph Lameter 349545ec1690SKees Cook seq_printf(m, "0x%pK-0x%pK %7ld", 3496a10aa579SChristoph Lameter v->addr, v->addr + v->size, v->size); 3497a10aa579SChristoph Lameter 349862c70bceSJoe Perches if (v->caller) 349962c70bceSJoe Perches seq_printf(m, " %pS", v->caller); 350023016969SChristoph Lameter 3501a10aa579SChristoph Lameter if (v->nr_pages) 3502a10aa579SChristoph Lameter seq_printf(m, " pages=%d", v->nr_pages); 3503a10aa579SChristoph Lameter 3504a10aa579SChristoph Lameter if (v->phys_addr) 3505199eaa05SMiles Chen seq_printf(m, " phys=%pa", &v->phys_addr); 3506a10aa579SChristoph Lameter 3507a10aa579SChristoph Lameter if (v->flags & VM_IOREMAP) 3508f4527c90SFabian Frederick seq_puts(m, " ioremap"); 3509a10aa579SChristoph Lameter 3510a10aa579SChristoph Lameter if (v->flags & VM_ALLOC) 3511f4527c90SFabian Frederick seq_puts(m, " vmalloc"); 3512a10aa579SChristoph Lameter 3513a10aa579SChristoph Lameter if (v->flags & VM_MAP) 3514f4527c90SFabian Frederick seq_puts(m, " vmap"); 3515a10aa579SChristoph Lameter 3516a10aa579SChristoph Lameter if (v->flags & VM_USERMAP) 3517f4527c90SFabian Frederick seq_puts(m, " user"); 3518a10aa579SChristoph Lameter 3519fe9041c2SChristoph Hellwig if (v->flags & VM_DMA_COHERENT) 3520fe9041c2SChristoph Hellwig seq_puts(m, " dma-coherent"); 3521fe9041c2SChristoph Hellwig 3522244d63eeSDavid Rientjes if (is_vmalloc_addr(v->pages)) 3523f4527c90SFabian Frederick seq_puts(m, " vpages"); 3524a10aa579SChristoph Lameter 3525a47a126aSEric Dumazet show_numa_info(m, v); 3526a10aa579SChristoph Lameter seq_putc(m, '\n'); 3527dd3b8353SUladzislau Rezki (Sony) 3528dd3b8353SUladzislau Rezki (Sony) /* 3529dd3b8353SUladzislau Rezki (Sony) * As a final step, dump "unpurged" areas. Note, 3530dd3b8353SUladzislau Rezki (Sony) * that entire "/proc/vmallocinfo" output will not 3531dd3b8353SUladzislau Rezki (Sony) * be address sorted, because the purge list is not 3532dd3b8353SUladzislau Rezki (Sony) * sorted. 3533dd3b8353SUladzislau Rezki (Sony) */ 3534dd3b8353SUladzislau Rezki (Sony) if (list_is_last(&va->list, &vmap_area_list)) 3535dd3b8353SUladzislau Rezki (Sony) show_purge_info(m); 3536dd3b8353SUladzislau Rezki (Sony) 3537a10aa579SChristoph Lameter return 0; 3538a10aa579SChristoph Lameter } 3539a10aa579SChristoph Lameter 35405f6a6a9cSAlexey Dobriyan static const struct seq_operations vmalloc_op = { 3541a10aa579SChristoph Lameter .start = s_start, 3542a10aa579SChristoph Lameter .next = s_next, 3543a10aa579SChristoph Lameter .stop = s_stop, 3544a10aa579SChristoph Lameter .show = s_show, 3545a10aa579SChristoph Lameter }; 35465f6a6a9cSAlexey Dobriyan 35475f6a6a9cSAlexey Dobriyan static int __init proc_vmalloc_init(void) 35485f6a6a9cSAlexey Dobriyan { 3549fddda2b7SChristoph Hellwig if (IS_ENABLED(CONFIG_NUMA)) 35500825a6f9SJoe Perches proc_create_seq_private("vmallocinfo", 0400, NULL, 355144414d82SChristoph Hellwig &vmalloc_op, 355244414d82SChristoph Hellwig nr_node_ids * sizeof(unsigned int), NULL); 3553fddda2b7SChristoph Hellwig else 35540825a6f9SJoe Perches proc_create_seq("vmallocinfo", 0400, NULL, &vmalloc_op); 35555f6a6a9cSAlexey Dobriyan return 0; 35565f6a6a9cSAlexey Dobriyan } 35575f6a6a9cSAlexey Dobriyan module_init(proc_vmalloc_init); 3558db3808c1SJoonsoo Kim 3559a10aa579SChristoph Lameter #endif 3560