11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/mm/vmalloc.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1993 Linus Torvalds 51da177e4SLinus Torvalds * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 61da177e4SLinus Torvalds * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 71da177e4SLinus Torvalds * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 8930fc45aSChristoph Lameter * Numa awareness, Christoph Lameter, SGI, June 2005 91da177e4SLinus Torvalds */ 101da177e4SLinus Torvalds 11db64fe02SNick Piggin #include <linux/vmalloc.h> 121da177e4SLinus Torvalds #include <linux/mm.h> 131da177e4SLinus Torvalds #include <linux/module.h> 141da177e4SLinus Torvalds #include <linux/highmem.h> 15c3edc401SIngo Molnar #include <linux/sched/signal.h> 161da177e4SLinus Torvalds #include <linux/slab.h> 171da177e4SLinus Torvalds #include <linux/spinlock.h> 181da177e4SLinus Torvalds #include <linux/interrupt.h> 195f6a6a9cSAlexey Dobriyan #include <linux/proc_fs.h> 20a10aa579SChristoph Lameter #include <linux/seq_file.h> 213ac7fe5aSThomas Gleixner #include <linux/debugobjects.h> 2223016969SChristoph Lameter #include <linux/kallsyms.h> 23db64fe02SNick Piggin #include <linux/list.h> 244da56b99SChris Wilson #include <linux/notifier.h> 25db64fe02SNick Piggin #include <linux/rbtree.h> 26db64fe02SNick Piggin #include <linux/radix-tree.h> 27db64fe02SNick Piggin #include <linux/rcupdate.h> 28f0aa6617STejun Heo #include <linux/pfn.h> 2989219d37SCatalin Marinas #include <linux/kmemleak.h> 3060063497SArun Sharma #include <linux/atomic.h> 313b32123dSGideon Israel Dsouza #include <linux/compiler.h> 3232fcfd40SAl Viro #include <linux/llist.h> 330f616be1SToshi Kani #include <linux/bitops.h> 343b32123dSGideon Israel Dsouza 357c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 361da177e4SLinus Torvalds #include <asm/tlbflush.h> 372dca6999SDavid Miller #include <asm/shmparam.h> 381da177e4SLinus Torvalds 39dd56b046SMel Gorman #include "internal.h" 40dd56b046SMel Gorman 4132fcfd40SAl Viro struct vfree_deferred { 4232fcfd40SAl Viro struct llist_head list; 4332fcfd40SAl Viro struct work_struct wq; 4432fcfd40SAl Viro }; 4532fcfd40SAl Viro static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); 4632fcfd40SAl Viro 4732fcfd40SAl Viro static void __vunmap(const void *, int); 4832fcfd40SAl Viro 4932fcfd40SAl Viro static void free_work(struct work_struct *w) 5032fcfd40SAl Viro { 5132fcfd40SAl Viro struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); 5232fcfd40SAl Viro struct llist_node *llnode = llist_del_all(&p->list); 5332fcfd40SAl Viro while (llnode) { 5432fcfd40SAl Viro void *p = llnode; 5532fcfd40SAl Viro llnode = llist_next(llnode); 5632fcfd40SAl Viro __vunmap(p, 1); 5732fcfd40SAl Viro } 5832fcfd40SAl Viro } 5932fcfd40SAl Viro 60db64fe02SNick Piggin /*** Page table manipulation functions ***/ 61b221385bSAdrian Bunk 621da177e4SLinus Torvalds static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end) 631da177e4SLinus Torvalds { 641da177e4SLinus Torvalds pte_t *pte; 651da177e4SLinus Torvalds 661da177e4SLinus Torvalds pte = pte_offset_kernel(pmd, addr); 671da177e4SLinus Torvalds do { 681da177e4SLinus Torvalds pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); 691da177e4SLinus Torvalds WARN_ON(!pte_none(ptent) && !pte_present(ptent)); 701da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 711da177e4SLinus Torvalds } 721da177e4SLinus Torvalds 73db64fe02SNick Piggin static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end) 741da177e4SLinus Torvalds { 751da177e4SLinus Torvalds pmd_t *pmd; 761da177e4SLinus Torvalds unsigned long next; 771da177e4SLinus Torvalds 781da177e4SLinus Torvalds pmd = pmd_offset(pud, addr); 791da177e4SLinus Torvalds do { 801da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 81b9820d8fSToshi Kani if (pmd_clear_huge(pmd)) 82b9820d8fSToshi Kani continue; 831da177e4SLinus Torvalds if (pmd_none_or_clear_bad(pmd)) 841da177e4SLinus Torvalds continue; 851da177e4SLinus Torvalds vunmap_pte_range(pmd, addr, next); 861da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 871da177e4SLinus Torvalds } 881da177e4SLinus Torvalds 89c2febafcSKirill A. Shutemov static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end) 901da177e4SLinus Torvalds { 911da177e4SLinus Torvalds pud_t *pud; 921da177e4SLinus Torvalds unsigned long next; 931da177e4SLinus Torvalds 94c2febafcSKirill A. Shutemov pud = pud_offset(p4d, addr); 951da177e4SLinus Torvalds do { 961da177e4SLinus Torvalds next = pud_addr_end(addr, end); 97b9820d8fSToshi Kani if (pud_clear_huge(pud)) 98b9820d8fSToshi Kani continue; 991da177e4SLinus Torvalds if (pud_none_or_clear_bad(pud)) 1001da177e4SLinus Torvalds continue; 1011da177e4SLinus Torvalds vunmap_pmd_range(pud, addr, next); 1021da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 1031da177e4SLinus Torvalds } 1041da177e4SLinus Torvalds 105c2febafcSKirill A. Shutemov static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end) 106c2febafcSKirill A. Shutemov { 107c2febafcSKirill A. Shutemov p4d_t *p4d; 108c2febafcSKirill A. Shutemov unsigned long next; 109c2febafcSKirill A. Shutemov 110c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 111c2febafcSKirill A. Shutemov do { 112c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 113c2febafcSKirill A. Shutemov if (p4d_clear_huge(p4d)) 114c2febafcSKirill A. Shutemov continue; 115c2febafcSKirill A. Shutemov if (p4d_none_or_clear_bad(p4d)) 116c2febafcSKirill A. Shutemov continue; 117c2febafcSKirill A. Shutemov vunmap_pud_range(p4d, addr, next); 118c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 119c2febafcSKirill A. Shutemov } 120c2febafcSKirill A. Shutemov 121db64fe02SNick Piggin static void vunmap_page_range(unsigned long addr, unsigned long end) 1221da177e4SLinus Torvalds { 1231da177e4SLinus Torvalds pgd_t *pgd; 1241da177e4SLinus Torvalds unsigned long next; 1251da177e4SLinus Torvalds 1261da177e4SLinus Torvalds BUG_ON(addr >= end); 1271da177e4SLinus Torvalds pgd = pgd_offset_k(addr); 1281da177e4SLinus Torvalds do { 1291da177e4SLinus Torvalds next = pgd_addr_end(addr, end); 1301da177e4SLinus Torvalds if (pgd_none_or_clear_bad(pgd)) 1311da177e4SLinus Torvalds continue; 132c2febafcSKirill A. Shutemov vunmap_p4d_range(pgd, addr, next); 1331da177e4SLinus Torvalds } while (pgd++, addr = next, addr != end); 1341da177e4SLinus Torvalds } 1351da177e4SLinus Torvalds 1361da177e4SLinus Torvalds static int vmap_pte_range(pmd_t *pmd, unsigned long addr, 137db64fe02SNick Piggin unsigned long end, pgprot_t prot, struct page **pages, int *nr) 1381da177e4SLinus Torvalds { 1391da177e4SLinus Torvalds pte_t *pte; 1401da177e4SLinus Torvalds 141db64fe02SNick Piggin /* 142db64fe02SNick Piggin * nr is a running index into the array which helps higher level 143db64fe02SNick Piggin * callers keep track of where we're up to. 144db64fe02SNick Piggin */ 145db64fe02SNick Piggin 146872fec16SHugh Dickins pte = pte_alloc_kernel(pmd, addr); 1471da177e4SLinus Torvalds if (!pte) 1481da177e4SLinus Torvalds return -ENOMEM; 1491da177e4SLinus Torvalds do { 150db64fe02SNick Piggin struct page *page = pages[*nr]; 151db64fe02SNick Piggin 152db64fe02SNick Piggin if (WARN_ON(!pte_none(*pte))) 153db64fe02SNick Piggin return -EBUSY; 154db64fe02SNick Piggin if (WARN_ON(!page)) 1551da177e4SLinus Torvalds return -ENOMEM; 1561da177e4SLinus Torvalds set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); 157db64fe02SNick Piggin (*nr)++; 1581da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 1591da177e4SLinus Torvalds return 0; 1601da177e4SLinus Torvalds } 1611da177e4SLinus Torvalds 162db64fe02SNick Piggin static int vmap_pmd_range(pud_t *pud, unsigned long addr, 163db64fe02SNick Piggin unsigned long end, pgprot_t prot, struct page **pages, int *nr) 1641da177e4SLinus Torvalds { 1651da177e4SLinus Torvalds pmd_t *pmd; 1661da177e4SLinus Torvalds unsigned long next; 1671da177e4SLinus Torvalds 1681da177e4SLinus Torvalds pmd = pmd_alloc(&init_mm, pud, addr); 1691da177e4SLinus Torvalds if (!pmd) 1701da177e4SLinus Torvalds return -ENOMEM; 1711da177e4SLinus Torvalds do { 1721da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 173db64fe02SNick Piggin if (vmap_pte_range(pmd, addr, next, prot, pages, nr)) 1741da177e4SLinus Torvalds return -ENOMEM; 1751da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 1761da177e4SLinus Torvalds return 0; 1771da177e4SLinus Torvalds } 1781da177e4SLinus Torvalds 179c2febafcSKirill A. Shutemov static int vmap_pud_range(p4d_t *p4d, unsigned long addr, 180db64fe02SNick Piggin unsigned long end, pgprot_t prot, struct page **pages, int *nr) 1811da177e4SLinus Torvalds { 1821da177e4SLinus Torvalds pud_t *pud; 1831da177e4SLinus Torvalds unsigned long next; 1841da177e4SLinus Torvalds 185c2febafcSKirill A. Shutemov pud = pud_alloc(&init_mm, p4d, addr); 1861da177e4SLinus Torvalds if (!pud) 1871da177e4SLinus Torvalds return -ENOMEM; 1881da177e4SLinus Torvalds do { 1891da177e4SLinus Torvalds next = pud_addr_end(addr, end); 190db64fe02SNick Piggin if (vmap_pmd_range(pud, addr, next, prot, pages, nr)) 1911da177e4SLinus Torvalds return -ENOMEM; 1921da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 1931da177e4SLinus Torvalds return 0; 1941da177e4SLinus Torvalds } 1951da177e4SLinus Torvalds 196c2febafcSKirill A. Shutemov static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, 197c2febafcSKirill A. Shutemov unsigned long end, pgprot_t prot, struct page **pages, int *nr) 198c2febafcSKirill A. Shutemov { 199c2febafcSKirill A. Shutemov p4d_t *p4d; 200c2febafcSKirill A. Shutemov unsigned long next; 201c2febafcSKirill A. Shutemov 202c2febafcSKirill A. Shutemov p4d = p4d_alloc(&init_mm, pgd, addr); 203c2febafcSKirill A. Shutemov if (!p4d) 204c2febafcSKirill A. Shutemov return -ENOMEM; 205c2febafcSKirill A. Shutemov do { 206c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 207c2febafcSKirill A. Shutemov if (vmap_pud_range(p4d, addr, next, prot, pages, nr)) 208c2febafcSKirill A. Shutemov return -ENOMEM; 209c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 210c2febafcSKirill A. Shutemov return 0; 211c2febafcSKirill A. Shutemov } 212c2febafcSKirill A. Shutemov 213db64fe02SNick Piggin /* 214db64fe02SNick Piggin * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and 215db64fe02SNick Piggin * will have pfns corresponding to the "pages" array. 216db64fe02SNick Piggin * 217db64fe02SNick Piggin * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N] 218db64fe02SNick Piggin */ 2198fc48985STejun Heo static int vmap_page_range_noflush(unsigned long start, unsigned long end, 220db64fe02SNick Piggin pgprot_t prot, struct page **pages) 2211da177e4SLinus Torvalds { 2221da177e4SLinus Torvalds pgd_t *pgd; 2231da177e4SLinus Torvalds unsigned long next; 2242e4e27c7SAdam Lackorzynski unsigned long addr = start; 225db64fe02SNick Piggin int err = 0; 226db64fe02SNick Piggin int nr = 0; 2271da177e4SLinus Torvalds 2281da177e4SLinus Torvalds BUG_ON(addr >= end); 2291da177e4SLinus Torvalds pgd = pgd_offset_k(addr); 2301da177e4SLinus Torvalds do { 2311da177e4SLinus Torvalds next = pgd_addr_end(addr, end); 232c2febafcSKirill A. Shutemov err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr); 2331da177e4SLinus Torvalds if (err) 234bf88c8c8SFigo.zhang return err; 2351da177e4SLinus Torvalds } while (pgd++, addr = next, addr != end); 236db64fe02SNick Piggin 237db64fe02SNick Piggin return nr; 2381da177e4SLinus Torvalds } 2391da177e4SLinus Torvalds 2408fc48985STejun Heo static int vmap_page_range(unsigned long start, unsigned long end, 2418fc48985STejun Heo pgprot_t prot, struct page **pages) 2428fc48985STejun Heo { 2438fc48985STejun Heo int ret; 2448fc48985STejun Heo 2458fc48985STejun Heo ret = vmap_page_range_noflush(start, end, prot, pages); 2468fc48985STejun Heo flush_cache_vmap(start, end); 2478fc48985STejun Heo return ret; 2488fc48985STejun Heo } 2498fc48985STejun Heo 25081ac3ad9SKAMEZAWA Hiroyuki int is_vmalloc_or_module_addr(const void *x) 25173bdf0a6SLinus Torvalds { 25273bdf0a6SLinus Torvalds /* 253ab4f2ee1SRussell King * ARM, x86-64 and sparc64 put modules in a special place, 25473bdf0a6SLinus Torvalds * and fall back on vmalloc() if that fails. Others 25573bdf0a6SLinus Torvalds * just put it in the vmalloc space. 25673bdf0a6SLinus Torvalds */ 25773bdf0a6SLinus Torvalds #if defined(CONFIG_MODULES) && defined(MODULES_VADDR) 25873bdf0a6SLinus Torvalds unsigned long addr = (unsigned long)x; 25973bdf0a6SLinus Torvalds if (addr >= MODULES_VADDR && addr < MODULES_END) 26073bdf0a6SLinus Torvalds return 1; 26173bdf0a6SLinus Torvalds #endif 26273bdf0a6SLinus Torvalds return is_vmalloc_addr(x); 26373bdf0a6SLinus Torvalds } 26473bdf0a6SLinus Torvalds 26548667e7aSChristoph Lameter /* 266add688fbSmalc * Walk a vmap address to the struct page it maps. 26748667e7aSChristoph Lameter */ 268add688fbSmalc struct page *vmalloc_to_page(const void *vmalloc_addr) 26948667e7aSChristoph Lameter { 27048667e7aSChristoph Lameter unsigned long addr = (unsigned long) vmalloc_addr; 271add688fbSmalc struct page *page = NULL; 27248667e7aSChristoph Lameter pgd_t *pgd = pgd_offset_k(addr); 273c2febafcSKirill A. Shutemov p4d_t *p4d; 274c2febafcSKirill A. Shutemov pud_t *pud; 275c2febafcSKirill A. Shutemov pmd_t *pmd; 276c2febafcSKirill A. Shutemov pte_t *ptep, pte; 27748667e7aSChristoph Lameter 2787aa413deSIngo Molnar /* 2797aa413deSIngo Molnar * XXX we might need to change this if we add VIRTUAL_BUG_ON for 2807aa413deSIngo Molnar * architectures that do not vmalloc module space 2817aa413deSIngo Molnar */ 28273bdf0a6SLinus Torvalds VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); 28359ea7463SJiri Slaby 284c2febafcSKirill A. Shutemov if (pgd_none(*pgd)) 285c2febafcSKirill A. Shutemov return NULL; 286c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 287c2febafcSKirill A. Shutemov if (p4d_none(*p4d)) 288c2febafcSKirill A. Shutemov return NULL; 289c2febafcSKirill A. Shutemov pud = pud_offset(p4d, addr); 290029c54b0SArd Biesheuvel 291029c54b0SArd Biesheuvel /* 292029c54b0SArd Biesheuvel * Don't dereference bad PUD or PMD (below) entries. This will also 293029c54b0SArd Biesheuvel * identify huge mappings, which we may encounter on architectures 294029c54b0SArd Biesheuvel * that define CONFIG_HAVE_ARCH_HUGE_VMAP=y. Such regions will be 295029c54b0SArd Biesheuvel * identified as vmalloc addresses by is_vmalloc_addr(), but are 296029c54b0SArd Biesheuvel * not [unambiguously] associated with a struct page, so there is 297029c54b0SArd Biesheuvel * no correct value to return for them. 298029c54b0SArd Biesheuvel */ 299029c54b0SArd Biesheuvel WARN_ON_ONCE(pud_bad(*pud)); 300029c54b0SArd Biesheuvel if (pud_none(*pud) || pud_bad(*pud)) 301c2febafcSKirill A. Shutemov return NULL; 302c2febafcSKirill A. Shutemov pmd = pmd_offset(pud, addr); 303029c54b0SArd Biesheuvel WARN_ON_ONCE(pmd_bad(*pmd)); 304029c54b0SArd Biesheuvel if (pmd_none(*pmd) || pmd_bad(*pmd)) 305c2febafcSKirill A. Shutemov return NULL; 306db64fe02SNick Piggin 30748667e7aSChristoph Lameter ptep = pte_offset_map(pmd, addr); 30848667e7aSChristoph Lameter pte = *ptep; 30948667e7aSChristoph Lameter if (pte_present(pte)) 310add688fbSmalc page = pte_page(pte); 31148667e7aSChristoph Lameter pte_unmap(ptep); 312add688fbSmalc return page; 313ece86e22SJianyu Zhan } 314ece86e22SJianyu Zhan EXPORT_SYMBOL(vmalloc_to_page); 315ece86e22SJianyu Zhan 316add688fbSmalc /* 317add688fbSmalc * Map a vmalloc()-space virtual address to the physical page frame number. 318add688fbSmalc */ 319add688fbSmalc unsigned long vmalloc_to_pfn(const void *vmalloc_addr) 320add688fbSmalc { 321add688fbSmalc return page_to_pfn(vmalloc_to_page(vmalloc_addr)); 322add688fbSmalc } 323add688fbSmalc EXPORT_SYMBOL(vmalloc_to_pfn); 324add688fbSmalc 325db64fe02SNick Piggin 326db64fe02SNick Piggin /*** Global kva allocator ***/ 327db64fe02SNick Piggin 328db64fe02SNick Piggin #define VM_VM_AREA 0x04 329db64fe02SNick Piggin 330db64fe02SNick Piggin static DEFINE_SPINLOCK(vmap_area_lock); 331f1c4069eSJoonsoo Kim /* Export for kexec only */ 332f1c4069eSJoonsoo Kim LIST_HEAD(vmap_area_list); 33380c4bd7aSChris Wilson static LLIST_HEAD(vmap_purge_list); 33489699605SNick Piggin static struct rb_root vmap_area_root = RB_ROOT; 33589699605SNick Piggin 33689699605SNick Piggin /* The vmap cache globals are protected by vmap_area_lock */ 33789699605SNick Piggin static struct rb_node *free_vmap_cache; 33889699605SNick Piggin static unsigned long cached_hole_size; 33989699605SNick Piggin static unsigned long cached_vstart; 34089699605SNick Piggin static unsigned long cached_align; 34189699605SNick Piggin 342ca23e405STejun Heo static unsigned long vmap_area_pcpu_hole; 343db64fe02SNick Piggin 344db64fe02SNick Piggin static struct vmap_area *__find_vmap_area(unsigned long addr) 3451da177e4SLinus Torvalds { 346db64fe02SNick Piggin struct rb_node *n = vmap_area_root.rb_node; 347db64fe02SNick Piggin 348db64fe02SNick Piggin while (n) { 349db64fe02SNick Piggin struct vmap_area *va; 350db64fe02SNick Piggin 351db64fe02SNick Piggin va = rb_entry(n, struct vmap_area, rb_node); 352db64fe02SNick Piggin if (addr < va->va_start) 353db64fe02SNick Piggin n = n->rb_left; 354cef2ac3fSHATAYAMA Daisuke else if (addr >= va->va_end) 355db64fe02SNick Piggin n = n->rb_right; 356db64fe02SNick Piggin else 357db64fe02SNick Piggin return va; 358db64fe02SNick Piggin } 359db64fe02SNick Piggin 360db64fe02SNick Piggin return NULL; 361db64fe02SNick Piggin } 362db64fe02SNick Piggin 363db64fe02SNick Piggin static void __insert_vmap_area(struct vmap_area *va) 364db64fe02SNick Piggin { 365db64fe02SNick Piggin struct rb_node **p = &vmap_area_root.rb_node; 366db64fe02SNick Piggin struct rb_node *parent = NULL; 367db64fe02SNick Piggin struct rb_node *tmp; 368db64fe02SNick Piggin 369db64fe02SNick Piggin while (*p) { 370170168d0SNamhyung Kim struct vmap_area *tmp_va; 371db64fe02SNick Piggin 372db64fe02SNick Piggin parent = *p; 373170168d0SNamhyung Kim tmp_va = rb_entry(parent, struct vmap_area, rb_node); 374170168d0SNamhyung Kim if (va->va_start < tmp_va->va_end) 375db64fe02SNick Piggin p = &(*p)->rb_left; 376170168d0SNamhyung Kim else if (va->va_end > tmp_va->va_start) 377db64fe02SNick Piggin p = &(*p)->rb_right; 378db64fe02SNick Piggin else 379db64fe02SNick Piggin BUG(); 380db64fe02SNick Piggin } 381db64fe02SNick Piggin 382db64fe02SNick Piggin rb_link_node(&va->rb_node, parent, p); 383db64fe02SNick Piggin rb_insert_color(&va->rb_node, &vmap_area_root); 384db64fe02SNick Piggin 3854341fa45SJoonsoo Kim /* address-sort this list */ 386db64fe02SNick Piggin tmp = rb_prev(&va->rb_node); 387db64fe02SNick Piggin if (tmp) { 388db64fe02SNick Piggin struct vmap_area *prev; 389db64fe02SNick Piggin prev = rb_entry(tmp, struct vmap_area, rb_node); 390db64fe02SNick Piggin list_add_rcu(&va->list, &prev->list); 391db64fe02SNick Piggin } else 392db64fe02SNick Piggin list_add_rcu(&va->list, &vmap_area_list); 393db64fe02SNick Piggin } 394db64fe02SNick Piggin 395db64fe02SNick Piggin static void purge_vmap_area_lazy(void); 396db64fe02SNick Piggin 3974da56b99SChris Wilson static BLOCKING_NOTIFIER_HEAD(vmap_notify_list); 3984da56b99SChris Wilson 399db64fe02SNick Piggin /* 400db64fe02SNick Piggin * Allocate a region of KVA of the specified size and alignment, within the 401db64fe02SNick Piggin * vstart and vend. 402db64fe02SNick Piggin */ 403db64fe02SNick Piggin static struct vmap_area *alloc_vmap_area(unsigned long size, 404db64fe02SNick Piggin unsigned long align, 405db64fe02SNick Piggin unsigned long vstart, unsigned long vend, 406db64fe02SNick Piggin int node, gfp_t gfp_mask) 407db64fe02SNick Piggin { 408db64fe02SNick Piggin struct vmap_area *va; 409db64fe02SNick Piggin struct rb_node *n; 4101da177e4SLinus Torvalds unsigned long addr; 411db64fe02SNick Piggin int purged = 0; 41289699605SNick Piggin struct vmap_area *first; 413db64fe02SNick Piggin 4147766970cSNick Piggin BUG_ON(!size); 415891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 41689699605SNick Piggin BUG_ON(!is_power_of_2(align)); 417db64fe02SNick Piggin 4185803ed29SChristoph Hellwig might_sleep(); 4194da56b99SChris Wilson 420db64fe02SNick Piggin va = kmalloc_node(sizeof(struct vmap_area), 421db64fe02SNick Piggin gfp_mask & GFP_RECLAIM_MASK, node); 422db64fe02SNick Piggin if (unlikely(!va)) 423db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 424db64fe02SNick Piggin 4257f88f88fSCatalin Marinas /* 4267f88f88fSCatalin Marinas * Only scan the relevant parts containing pointers to other objects 4277f88f88fSCatalin Marinas * to avoid false negatives. 4287f88f88fSCatalin Marinas */ 4297f88f88fSCatalin Marinas kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK); 4307f88f88fSCatalin Marinas 431db64fe02SNick Piggin retry: 432db64fe02SNick Piggin spin_lock(&vmap_area_lock); 43389699605SNick Piggin /* 43489699605SNick Piggin * Invalidate cache if we have more permissive parameters. 43589699605SNick Piggin * cached_hole_size notes the largest hole noticed _below_ 43689699605SNick Piggin * the vmap_area cached in free_vmap_cache: if size fits 43789699605SNick Piggin * into that hole, we want to scan from vstart to reuse 43889699605SNick Piggin * the hole instead of allocating above free_vmap_cache. 43989699605SNick Piggin * Note that __free_vmap_area may update free_vmap_cache 44089699605SNick Piggin * without updating cached_hole_size or cached_align. 44189699605SNick Piggin */ 44289699605SNick Piggin if (!free_vmap_cache || 44389699605SNick Piggin size < cached_hole_size || 44489699605SNick Piggin vstart < cached_vstart || 44589699605SNick Piggin align < cached_align) { 44689699605SNick Piggin nocache: 44789699605SNick Piggin cached_hole_size = 0; 44889699605SNick Piggin free_vmap_cache = NULL; 44989699605SNick Piggin } 45089699605SNick Piggin /* record if we encounter less permissive parameters */ 45189699605SNick Piggin cached_vstart = vstart; 45289699605SNick Piggin cached_align = align; 45389699605SNick Piggin 45489699605SNick Piggin /* find starting point for our search */ 45589699605SNick Piggin if (free_vmap_cache) { 45689699605SNick Piggin first = rb_entry(free_vmap_cache, struct vmap_area, rb_node); 457248ac0e1SJohannes Weiner addr = ALIGN(first->va_end, align); 45889699605SNick Piggin if (addr < vstart) 45989699605SNick Piggin goto nocache; 460bcb615a8SZhang Yanfei if (addr + size < addr) 4617766970cSNick Piggin goto overflow; 4627766970cSNick Piggin 46389699605SNick Piggin } else { 46489699605SNick Piggin addr = ALIGN(vstart, align); 465bcb615a8SZhang Yanfei if (addr + size < addr) 46689699605SNick Piggin goto overflow; 467db64fe02SNick Piggin 46889699605SNick Piggin n = vmap_area_root.rb_node; 46989699605SNick Piggin first = NULL; 47089699605SNick Piggin 47189699605SNick Piggin while (n) { 472db64fe02SNick Piggin struct vmap_area *tmp; 473db64fe02SNick Piggin tmp = rb_entry(n, struct vmap_area, rb_node); 474db64fe02SNick Piggin if (tmp->va_end >= addr) { 475db64fe02SNick Piggin first = tmp; 47689699605SNick Piggin if (tmp->va_start <= addr) 47789699605SNick Piggin break; 478db64fe02SNick Piggin n = n->rb_left; 47989699605SNick Piggin } else 480db64fe02SNick Piggin n = n->rb_right; 481db64fe02SNick Piggin } 482db64fe02SNick Piggin 483db64fe02SNick Piggin if (!first) 484db64fe02SNick Piggin goto found; 485db64fe02SNick Piggin } 486db64fe02SNick Piggin 48789699605SNick Piggin /* from the starting point, walk areas until a suitable hole is found */ 488248ac0e1SJohannes Weiner while (addr + size > first->va_start && addr + size <= vend) { 48989699605SNick Piggin if (addr + cached_hole_size < first->va_start) 49089699605SNick Piggin cached_hole_size = first->va_start - addr; 491248ac0e1SJohannes Weiner addr = ALIGN(first->va_end, align); 492bcb615a8SZhang Yanfei if (addr + size < addr) 4937766970cSNick Piggin goto overflow; 494db64fe02SNick Piggin 49592ca922fSHong zhi guo if (list_is_last(&first->list, &vmap_area_list)) 496db64fe02SNick Piggin goto found; 49792ca922fSHong zhi guo 4986219c2a2SGeliang Tang first = list_next_entry(first, list); 499db64fe02SNick Piggin } 50089699605SNick Piggin 501db64fe02SNick Piggin found: 50289699605SNick Piggin if (addr + size > vend) 50389699605SNick Piggin goto overflow; 50489699605SNick Piggin 50589699605SNick Piggin va->va_start = addr; 50689699605SNick Piggin va->va_end = addr + size; 50789699605SNick Piggin va->flags = 0; 50889699605SNick Piggin __insert_vmap_area(va); 50989699605SNick Piggin free_vmap_cache = &va->rb_node; 51089699605SNick Piggin spin_unlock(&vmap_area_lock); 51189699605SNick Piggin 51261e16557SWang Xiaoqiang BUG_ON(!IS_ALIGNED(va->va_start, align)); 51389699605SNick Piggin BUG_ON(va->va_start < vstart); 51489699605SNick Piggin BUG_ON(va->va_end > vend); 51589699605SNick Piggin 51689699605SNick Piggin return va; 51789699605SNick Piggin 5187766970cSNick Piggin overflow: 519db64fe02SNick Piggin spin_unlock(&vmap_area_lock); 520db64fe02SNick Piggin if (!purged) { 521db64fe02SNick Piggin purge_vmap_area_lazy(); 522db64fe02SNick Piggin purged = 1; 523db64fe02SNick Piggin goto retry; 524db64fe02SNick Piggin } 5254da56b99SChris Wilson 5264da56b99SChris Wilson if (gfpflags_allow_blocking(gfp_mask)) { 5274da56b99SChris Wilson unsigned long freed = 0; 5284da56b99SChris Wilson blocking_notifier_call_chain(&vmap_notify_list, 0, &freed); 5294da56b99SChris Wilson if (freed > 0) { 5304da56b99SChris Wilson purged = 0; 5314da56b99SChris Wilson goto retry; 5324da56b99SChris Wilson } 5334da56b99SChris Wilson } 5344da56b99SChris Wilson 53503497d76SFlorian Fainelli if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) 536756a025fSJoe Perches pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n", 537756a025fSJoe Perches size); 5382498ce42SRalph Wuerthner kfree(va); 539db64fe02SNick Piggin return ERR_PTR(-EBUSY); 540db64fe02SNick Piggin } 541db64fe02SNick Piggin 5424da56b99SChris Wilson int register_vmap_purge_notifier(struct notifier_block *nb) 5434da56b99SChris Wilson { 5444da56b99SChris Wilson return blocking_notifier_chain_register(&vmap_notify_list, nb); 5454da56b99SChris Wilson } 5464da56b99SChris Wilson EXPORT_SYMBOL_GPL(register_vmap_purge_notifier); 5474da56b99SChris Wilson 5484da56b99SChris Wilson int unregister_vmap_purge_notifier(struct notifier_block *nb) 5494da56b99SChris Wilson { 5504da56b99SChris Wilson return blocking_notifier_chain_unregister(&vmap_notify_list, nb); 5514da56b99SChris Wilson } 5524da56b99SChris Wilson EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier); 5534da56b99SChris Wilson 554db64fe02SNick Piggin static void __free_vmap_area(struct vmap_area *va) 555db64fe02SNick Piggin { 556db64fe02SNick Piggin BUG_ON(RB_EMPTY_NODE(&va->rb_node)); 55789699605SNick Piggin 55889699605SNick Piggin if (free_vmap_cache) { 55989699605SNick Piggin if (va->va_end < cached_vstart) { 56089699605SNick Piggin free_vmap_cache = NULL; 56189699605SNick Piggin } else { 56289699605SNick Piggin struct vmap_area *cache; 56389699605SNick Piggin cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node); 56489699605SNick Piggin if (va->va_start <= cache->va_start) { 56589699605SNick Piggin free_vmap_cache = rb_prev(&va->rb_node); 56689699605SNick Piggin /* 56789699605SNick Piggin * We don't try to update cached_hole_size or 56889699605SNick Piggin * cached_align, but it won't go very wrong. 56989699605SNick Piggin */ 57089699605SNick Piggin } 57189699605SNick Piggin } 57289699605SNick Piggin } 573db64fe02SNick Piggin rb_erase(&va->rb_node, &vmap_area_root); 574db64fe02SNick Piggin RB_CLEAR_NODE(&va->rb_node); 575db64fe02SNick Piggin list_del_rcu(&va->list); 576db64fe02SNick Piggin 577ca23e405STejun Heo /* 578ca23e405STejun Heo * Track the highest possible candidate for pcpu area 579ca23e405STejun Heo * allocation. Areas outside of vmalloc area can be returned 580ca23e405STejun Heo * here too, consider only end addresses which fall inside 581ca23e405STejun Heo * vmalloc area proper. 582ca23e405STejun Heo */ 583ca23e405STejun Heo if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END) 584ca23e405STejun Heo vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end); 585ca23e405STejun Heo 58614769de9SLai Jiangshan kfree_rcu(va, rcu_head); 587db64fe02SNick Piggin } 588db64fe02SNick Piggin 589db64fe02SNick Piggin /* 590db64fe02SNick Piggin * Free a region of KVA allocated by alloc_vmap_area 591db64fe02SNick Piggin */ 592db64fe02SNick Piggin static void free_vmap_area(struct vmap_area *va) 593db64fe02SNick Piggin { 594db64fe02SNick Piggin spin_lock(&vmap_area_lock); 595db64fe02SNick Piggin __free_vmap_area(va); 596db64fe02SNick Piggin spin_unlock(&vmap_area_lock); 597db64fe02SNick Piggin } 598db64fe02SNick Piggin 599db64fe02SNick Piggin /* 600db64fe02SNick Piggin * Clear the pagetable entries of a given vmap_area 601db64fe02SNick Piggin */ 602db64fe02SNick Piggin static void unmap_vmap_area(struct vmap_area *va) 603db64fe02SNick Piggin { 604db64fe02SNick Piggin vunmap_page_range(va->va_start, va->va_end); 605db64fe02SNick Piggin } 606db64fe02SNick Piggin 607cd52858cSNick Piggin static void vmap_debug_free_range(unsigned long start, unsigned long end) 608cd52858cSNick Piggin { 609cd52858cSNick Piggin /* 610f48d97f3SJoonsoo Kim * Unmap page tables and force a TLB flush immediately if pagealloc 611f48d97f3SJoonsoo Kim * debugging is enabled. This catches use after free bugs similarly to 612f48d97f3SJoonsoo Kim * those in linear kernel virtual address space after a page has been 613f48d97f3SJoonsoo Kim * freed. 614cd52858cSNick Piggin * 615f48d97f3SJoonsoo Kim * All the lazy freeing logic is still retained, in order to minimise 616f48d97f3SJoonsoo Kim * intrusiveness of this debugging feature. 617cd52858cSNick Piggin * 618f48d97f3SJoonsoo Kim * This is going to be *slow* (linear kernel virtual address debugging 619f48d97f3SJoonsoo Kim * doesn't do a broadcast TLB flush so it is a lot faster). 620cd52858cSNick Piggin */ 621f48d97f3SJoonsoo Kim if (debug_pagealloc_enabled()) { 622cd52858cSNick Piggin vunmap_page_range(start, end); 623cd52858cSNick Piggin flush_tlb_kernel_range(start, end); 624f48d97f3SJoonsoo Kim } 625cd52858cSNick Piggin } 626cd52858cSNick Piggin 627db64fe02SNick Piggin /* 628db64fe02SNick Piggin * lazy_max_pages is the maximum amount of virtual address space we gather up 629db64fe02SNick Piggin * before attempting to purge with a TLB flush. 630db64fe02SNick Piggin * 631db64fe02SNick Piggin * There is a tradeoff here: a larger number will cover more kernel page tables 632db64fe02SNick Piggin * and take slightly longer to purge, but it will linearly reduce the number of 633db64fe02SNick Piggin * global TLB flushes that must be performed. It would seem natural to scale 634db64fe02SNick Piggin * this number up linearly with the number of CPUs (because vmapping activity 635db64fe02SNick Piggin * could also scale linearly with the number of CPUs), however it is likely 636db64fe02SNick Piggin * that in practice, workloads might be constrained in other ways that mean 637db64fe02SNick Piggin * vmap activity will not scale linearly with CPUs. Also, I want to be 638db64fe02SNick Piggin * conservative and not introduce a big latency on huge systems, so go with 639db64fe02SNick Piggin * a less aggressive log scale. It will still be an improvement over the old 640db64fe02SNick Piggin * code, and it will be simple to change the scale factor if we find that it 641db64fe02SNick Piggin * becomes a problem on bigger systems. 642db64fe02SNick Piggin */ 643db64fe02SNick Piggin static unsigned long lazy_max_pages(void) 644db64fe02SNick Piggin { 645db64fe02SNick Piggin unsigned int log; 646db64fe02SNick Piggin 647db64fe02SNick Piggin log = fls(num_online_cpus()); 648db64fe02SNick Piggin 649db64fe02SNick Piggin return log * (32UL * 1024 * 1024 / PAGE_SIZE); 650db64fe02SNick Piggin } 651db64fe02SNick Piggin 652db64fe02SNick Piggin static atomic_t vmap_lazy_nr = ATOMIC_INIT(0); 653db64fe02SNick Piggin 6540574ecd1SChristoph Hellwig /* 6550574ecd1SChristoph Hellwig * Serialize vmap purging. There is no actual criticial section protected 6560574ecd1SChristoph Hellwig * by this look, but we want to avoid concurrent calls for performance 6570574ecd1SChristoph Hellwig * reasons and to make the pcpu_get_vm_areas more deterministic. 6580574ecd1SChristoph Hellwig */ 659f9e09977SChristoph Hellwig static DEFINE_MUTEX(vmap_purge_lock); 6600574ecd1SChristoph Hellwig 66102b709dfSNick Piggin /* for per-CPU blocks */ 66202b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void); 66302b709dfSNick Piggin 664db64fe02SNick Piggin /* 6653ee48b6aSCliff Wickman * called before a call to iounmap() if the caller wants vm_area_struct's 6663ee48b6aSCliff Wickman * immediately freed. 6673ee48b6aSCliff Wickman */ 6683ee48b6aSCliff Wickman void set_iounmap_nonlazy(void) 6693ee48b6aSCliff Wickman { 6703ee48b6aSCliff Wickman atomic_set(&vmap_lazy_nr, lazy_max_pages()+1); 6713ee48b6aSCliff Wickman } 6723ee48b6aSCliff Wickman 6733ee48b6aSCliff Wickman /* 674db64fe02SNick Piggin * Purges all lazily-freed vmap areas. 675db64fe02SNick Piggin */ 6760574ecd1SChristoph Hellwig static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end) 677db64fe02SNick Piggin { 67880c4bd7aSChris Wilson struct llist_node *valist; 679db64fe02SNick Piggin struct vmap_area *va; 680cbb76676SVegard Nossum struct vmap_area *n_va; 681763b218dSJoel Fernandes bool do_free = false; 682db64fe02SNick Piggin 6830574ecd1SChristoph Hellwig lockdep_assert_held(&vmap_purge_lock); 68402b709dfSNick Piggin 68580c4bd7aSChris Wilson valist = llist_del_all(&vmap_purge_list); 68680c4bd7aSChris Wilson llist_for_each_entry(va, valist, purge_list) { 6870574ecd1SChristoph Hellwig if (va->va_start < start) 6880574ecd1SChristoph Hellwig start = va->va_start; 6890574ecd1SChristoph Hellwig if (va->va_end > end) 6900574ecd1SChristoph Hellwig end = va->va_end; 691763b218dSJoel Fernandes do_free = true; 692db64fe02SNick Piggin } 693db64fe02SNick Piggin 694763b218dSJoel Fernandes if (!do_free) 6950574ecd1SChristoph Hellwig return false; 6960574ecd1SChristoph Hellwig 6970574ecd1SChristoph Hellwig flush_tlb_kernel_range(start, end); 698db64fe02SNick Piggin 699db64fe02SNick Piggin spin_lock(&vmap_area_lock); 700763b218dSJoel Fernandes llist_for_each_entry_safe(va, n_va, valist, purge_list) { 701763b218dSJoel Fernandes int nr = (va->va_end - va->va_start) >> PAGE_SHIFT; 702763b218dSJoel Fernandes 703db64fe02SNick Piggin __free_vmap_area(va); 704763b218dSJoel Fernandes atomic_sub(nr, &vmap_lazy_nr); 705763b218dSJoel Fernandes cond_resched_lock(&vmap_area_lock); 706763b218dSJoel Fernandes } 707db64fe02SNick Piggin spin_unlock(&vmap_area_lock); 7080574ecd1SChristoph Hellwig return true; 709db64fe02SNick Piggin } 710db64fe02SNick Piggin 711db64fe02SNick Piggin /* 712496850e5SNick Piggin * Kick off a purge of the outstanding lazy areas. Don't bother if somebody 713496850e5SNick Piggin * is already purging. 714496850e5SNick Piggin */ 715496850e5SNick Piggin static void try_purge_vmap_area_lazy(void) 716496850e5SNick Piggin { 717f9e09977SChristoph Hellwig if (mutex_trylock(&vmap_purge_lock)) { 7180574ecd1SChristoph Hellwig __purge_vmap_area_lazy(ULONG_MAX, 0); 719f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 7200574ecd1SChristoph Hellwig } 721496850e5SNick Piggin } 722496850e5SNick Piggin 723496850e5SNick Piggin /* 724db64fe02SNick Piggin * Kick off a purge of the outstanding lazy areas. 725db64fe02SNick Piggin */ 726db64fe02SNick Piggin static void purge_vmap_area_lazy(void) 727db64fe02SNick Piggin { 728f9e09977SChristoph Hellwig mutex_lock(&vmap_purge_lock); 7290574ecd1SChristoph Hellwig purge_fragmented_blocks_allcpus(); 7300574ecd1SChristoph Hellwig __purge_vmap_area_lazy(ULONG_MAX, 0); 731f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 732db64fe02SNick Piggin } 733db64fe02SNick Piggin 734db64fe02SNick Piggin /* 73564141da5SJeremy Fitzhardinge * Free a vmap area, caller ensuring that the area has been unmapped 73664141da5SJeremy Fitzhardinge * and flush_cache_vunmap had been called for the correct range 73764141da5SJeremy Fitzhardinge * previously. 738db64fe02SNick Piggin */ 73964141da5SJeremy Fitzhardinge static void free_vmap_area_noflush(struct vmap_area *va) 740db64fe02SNick Piggin { 74180c4bd7aSChris Wilson int nr_lazy; 74280c4bd7aSChris Wilson 74380c4bd7aSChris Wilson nr_lazy = atomic_add_return((va->va_end - va->va_start) >> PAGE_SHIFT, 74480c4bd7aSChris Wilson &vmap_lazy_nr); 74580c4bd7aSChris Wilson 74680c4bd7aSChris Wilson /* After this point, we may free va at any time */ 74780c4bd7aSChris Wilson llist_add(&va->purge_list, &vmap_purge_list); 74880c4bd7aSChris Wilson 74980c4bd7aSChris Wilson if (unlikely(nr_lazy > lazy_max_pages())) 750496850e5SNick Piggin try_purge_vmap_area_lazy(); 751db64fe02SNick Piggin } 752db64fe02SNick Piggin 753b29acbdcSNick Piggin /* 754b29acbdcSNick Piggin * Free and unmap a vmap area 755b29acbdcSNick Piggin */ 756b29acbdcSNick Piggin static void free_unmap_vmap_area(struct vmap_area *va) 757b29acbdcSNick Piggin { 758b29acbdcSNick Piggin flush_cache_vunmap(va->va_start, va->va_end); 759c8eef01eSChristoph Hellwig unmap_vmap_area(va); 760c8eef01eSChristoph Hellwig free_vmap_area_noflush(va); 761b29acbdcSNick Piggin } 762b29acbdcSNick Piggin 763db64fe02SNick Piggin static struct vmap_area *find_vmap_area(unsigned long addr) 764db64fe02SNick Piggin { 765db64fe02SNick Piggin struct vmap_area *va; 766db64fe02SNick Piggin 767db64fe02SNick Piggin spin_lock(&vmap_area_lock); 768db64fe02SNick Piggin va = __find_vmap_area(addr); 769db64fe02SNick Piggin spin_unlock(&vmap_area_lock); 770db64fe02SNick Piggin 771db64fe02SNick Piggin return va; 772db64fe02SNick Piggin } 773db64fe02SNick Piggin 774db64fe02SNick Piggin /*** Per cpu kva allocator ***/ 775db64fe02SNick Piggin 776db64fe02SNick Piggin /* 777db64fe02SNick Piggin * vmap space is limited especially on 32 bit architectures. Ensure there is 778db64fe02SNick Piggin * room for at least 16 percpu vmap blocks per CPU. 779db64fe02SNick Piggin */ 780db64fe02SNick Piggin /* 781db64fe02SNick Piggin * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able 782db64fe02SNick Piggin * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess 783db64fe02SNick Piggin * instead (we just need a rough idea) 784db64fe02SNick Piggin */ 785db64fe02SNick Piggin #if BITS_PER_LONG == 32 786db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024) 787db64fe02SNick Piggin #else 788db64fe02SNick Piggin #define VMALLOC_SPACE (128UL*1024*1024*1024) 789db64fe02SNick Piggin #endif 790db64fe02SNick Piggin 791db64fe02SNick Piggin #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE) 792db64fe02SNick Piggin #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */ 793db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */ 794db64fe02SNick Piggin #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2) 795db64fe02SNick Piggin #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */ 796db64fe02SNick Piggin #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */ 797f982f915SClemens Ladisch #define VMAP_BBMAP_BITS \ 798f982f915SClemens Ladisch VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ 799db64fe02SNick Piggin VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ 800f982f915SClemens Ladisch VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16)) 801db64fe02SNick Piggin 802db64fe02SNick Piggin #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE) 803db64fe02SNick Piggin 8049b463334SJeremy Fitzhardinge static bool vmap_initialized __read_mostly = false; 8059b463334SJeremy Fitzhardinge 806db64fe02SNick Piggin struct vmap_block_queue { 807db64fe02SNick Piggin spinlock_t lock; 808db64fe02SNick Piggin struct list_head free; 809db64fe02SNick Piggin }; 810db64fe02SNick Piggin 811db64fe02SNick Piggin struct vmap_block { 812db64fe02SNick Piggin spinlock_t lock; 813db64fe02SNick Piggin struct vmap_area *va; 814db64fe02SNick Piggin unsigned long free, dirty; 8157d61bfe8SRoman Pen unsigned long dirty_min, dirty_max; /*< dirty range */ 816db64fe02SNick Piggin struct list_head free_list; 817db64fe02SNick Piggin struct rcu_head rcu_head; 81802b709dfSNick Piggin struct list_head purge; 819db64fe02SNick Piggin }; 820db64fe02SNick Piggin 821db64fe02SNick Piggin /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */ 822db64fe02SNick Piggin static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue); 823db64fe02SNick Piggin 824db64fe02SNick Piggin /* 825db64fe02SNick Piggin * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block 826db64fe02SNick Piggin * in the free path. Could get rid of this if we change the API to return a 827db64fe02SNick Piggin * "cookie" from alloc, to be passed to free. But no big deal yet. 828db64fe02SNick Piggin */ 829db64fe02SNick Piggin static DEFINE_SPINLOCK(vmap_block_tree_lock); 830db64fe02SNick Piggin static RADIX_TREE(vmap_block_tree, GFP_ATOMIC); 831db64fe02SNick Piggin 832db64fe02SNick Piggin /* 833db64fe02SNick Piggin * We should probably have a fallback mechanism to allocate virtual memory 834db64fe02SNick Piggin * out of partially filled vmap blocks. However vmap block sizing should be 835db64fe02SNick Piggin * fairly reasonable according to the vmalloc size, so it shouldn't be a 836db64fe02SNick Piggin * big problem. 837db64fe02SNick Piggin */ 838db64fe02SNick Piggin 839db64fe02SNick Piggin static unsigned long addr_to_vb_idx(unsigned long addr) 840db64fe02SNick Piggin { 841db64fe02SNick Piggin addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); 842db64fe02SNick Piggin addr /= VMAP_BLOCK_SIZE; 843db64fe02SNick Piggin return addr; 844db64fe02SNick Piggin } 845db64fe02SNick Piggin 846cf725ce2SRoman Pen static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off) 847cf725ce2SRoman Pen { 848cf725ce2SRoman Pen unsigned long addr; 849cf725ce2SRoman Pen 850cf725ce2SRoman Pen addr = va_start + (pages_off << PAGE_SHIFT); 851cf725ce2SRoman Pen BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start)); 852cf725ce2SRoman Pen return (void *)addr; 853cf725ce2SRoman Pen } 854cf725ce2SRoman Pen 855cf725ce2SRoman Pen /** 856cf725ce2SRoman Pen * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this 857cf725ce2SRoman Pen * block. Of course pages number can't exceed VMAP_BBMAP_BITS 858cf725ce2SRoman Pen * @order: how many 2^order pages should be occupied in newly allocated block 859cf725ce2SRoman Pen * @gfp_mask: flags for the page level allocator 860cf725ce2SRoman Pen * 861cf725ce2SRoman Pen * Returns: virtual address in a newly allocated block or ERR_PTR(-errno) 862cf725ce2SRoman Pen */ 863cf725ce2SRoman Pen static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) 864db64fe02SNick Piggin { 865db64fe02SNick Piggin struct vmap_block_queue *vbq; 866db64fe02SNick Piggin struct vmap_block *vb; 867db64fe02SNick Piggin struct vmap_area *va; 868db64fe02SNick Piggin unsigned long vb_idx; 869db64fe02SNick Piggin int node, err; 870cf725ce2SRoman Pen void *vaddr; 871db64fe02SNick Piggin 872db64fe02SNick Piggin node = numa_node_id(); 873db64fe02SNick Piggin 874db64fe02SNick Piggin vb = kmalloc_node(sizeof(struct vmap_block), 875db64fe02SNick Piggin gfp_mask & GFP_RECLAIM_MASK, node); 876db64fe02SNick Piggin if (unlikely(!vb)) 877db64fe02SNick Piggin return ERR_PTR(-ENOMEM); 878db64fe02SNick Piggin 879db64fe02SNick Piggin va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, 880db64fe02SNick Piggin VMALLOC_START, VMALLOC_END, 881db64fe02SNick Piggin node, gfp_mask); 882ddf9c6d4STobias Klauser if (IS_ERR(va)) { 883db64fe02SNick Piggin kfree(vb); 884e7d86340SJulia Lawall return ERR_CAST(va); 885db64fe02SNick Piggin } 886db64fe02SNick Piggin 887db64fe02SNick Piggin err = radix_tree_preload(gfp_mask); 888db64fe02SNick Piggin if (unlikely(err)) { 889db64fe02SNick Piggin kfree(vb); 890db64fe02SNick Piggin free_vmap_area(va); 891db64fe02SNick Piggin return ERR_PTR(err); 892db64fe02SNick Piggin } 893db64fe02SNick Piggin 894cf725ce2SRoman Pen vaddr = vmap_block_vaddr(va->va_start, 0); 895db64fe02SNick Piggin spin_lock_init(&vb->lock); 896db64fe02SNick Piggin vb->va = va; 897cf725ce2SRoman Pen /* At least something should be left free */ 898cf725ce2SRoman Pen BUG_ON(VMAP_BBMAP_BITS <= (1UL << order)); 899cf725ce2SRoman Pen vb->free = VMAP_BBMAP_BITS - (1UL << order); 900db64fe02SNick Piggin vb->dirty = 0; 9017d61bfe8SRoman Pen vb->dirty_min = VMAP_BBMAP_BITS; 9027d61bfe8SRoman Pen vb->dirty_max = 0; 903db64fe02SNick Piggin INIT_LIST_HEAD(&vb->free_list); 904db64fe02SNick Piggin 905db64fe02SNick Piggin vb_idx = addr_to_vb_idx(va->va_start); 906db64fe02SNick Piggin spin_lock(&vmap_block_tree_lock); 907db64fe02SNick Piggin err = radix_tree_insert(&vmap_block_tree, vb_idx, vb); 908db64fe02SNick Piggin spin_unlock(&vmap_block_tree_lock); 909db64fe02SNick Piggin BUG_ON(err); 910db64fe02SNick Piggin radix_tree_preload_end(); 911db64fe02SNick Piggin 912db64fe02SNick Piggin vbq = &get_cpu_var(vmap_block_queue); 913db64fe02SNick Piggin spin_lock(&vbq->lock); 91468ac546fSRoman Pen list_add_tail_rcu(&vb->free_list, &vbq->free); 915db64fe02SNick Piggin spin_unlock(&vbq->lock); 9163f04ba85STejun Heo put_cpu_var(vmap_block_queue); 917db64fe02SNick Piggin 918cf725ce2SRoman Pen return vaddr; 919db64fe02SNick Piggin } 920db64fe02SNick Piggin 921db64fe02SNick Piggin static void free_vmap_block(struct vmap_block *vb) 922db64fe02SNick Piggin { 923db64fe02SNick Piggin struct vmap_block *tmp; 924db64fe02SNick Piggin unsigned long vb_idx; 925db64fe02SNick Piggin 926db64fe02SNick Piggin vb_idx = addr_to_vb_idx(vb->va->va_start); 927db64fe02SNick Piggin spin_lock(&vmap_block_tree_lock); 928db64fe02SNick Piggin tmp = radix_tree_delete(&vmap_block_tree, vb_idx); 929db64fe02SNick Piggin spin_unlock(&vmap_block_tree_lock); 930db64fe02SNick Piggin BUG_ON(tmp != vb); 931db64fe02SNick Piggin 93264141da5SJeremy Fitzhardinge free_vmap_area_noflush(vb->va); 93322a3c7d1SLai Jiangshan kfree_rcu(vb, rcu_head); 934db64fe02SNick Piggin } 935db64fe02SNick Piggin 93602b709dfSNick Piggin static void purge_fragmented_blocks(int cpu) 93702b709dfSNick Piggin { 93802b709dfSNick Piggin LIST_HEAD(purge); 93902b709dfSNick Piggin struct vmap_block *vb; 94002b709dfSNick Piggin struct vmap_block *n_vb; 94102b709dfSNick Piggin struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 94202b709dfSNick Piggin 94302b709dfSNick Piggin rcu_read_lock(); 94402b709dfSNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 94502b709dfSNick Piggin 94602b709dfSNick Piggin if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS)) 94702b709dfSNick Piggin continue; 94802b709dfSNick Piggin 94902b709dfSNick Piggin spin_lock(&vb->lock); 95002b709dfSNick Piggin if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) { 95102b709dfSNick Piggin vb->free = 0; /* prevent further allocs after releasing lock */ 95202b709dfSNick Piggin vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */ 9537d61bfe8SRoman Pen vb->dirty_min = 0; 9547d61bfe8SRoman Pen vb->dirty_max = VMAP_BBMAP_BITS; 95502b709dfSNick Piggin spin_lock(&vbq->lock); 95602b709dfSNick Piggin list_del_rcu(&vb->free_list); 95702b709dfSNick Piggin spin_unlock(&vbq->lock); 95802b709dfSNick Piggin spin_unlock(&vb->lock); 95902b709dfSNick Piggin list_add_tail(&vb->purge, &purge); 96002b709dfSNick Piggin } else 96102b709dfSNick Piggin spin_unlock(&vb->lock); 96202b709dfSNick Piggin } 96302b709dfSNick Piggin rcu_read_unlock(); 96402b709dfSNick Piggin 96502b709dfSNick Piggin list_for_each_entry_safe(vb, n_vb, &purge, purge) { 96602b709dfSNick Piggin list_del(&vb->purge); 96702b709dfSNick Piggin free_vmap_block(vb); 96802b709dfSNick Piggin } 96902b709dfSNick Piggin } 97002b709dfSNick Piggin 97102b709dfSNick Piggin static void purge_fragmented_blocks_allcpus(void) 97202b709dfSNick Piggin { 97302b709dfSNick Piggin int cpu; 97402b709dfSNick Piggin 97502b709dfSNick Piggin for_each_possible_cpu(cpu) 97602b709dfSNick Piggin purge_fragmented_blocks(cpu); 97702b709dfSNick Piggin } 97802b709dfSNick Piggin 979db64fe02SNick Piggin static void *vb_alloc(unsigned long size, gfp_t gfp_mask) 980db64fe02SNick Piggin { 981db64fe02SNick Piggin struct vmap_block_queue *vbq; 982db64fe02SNick Piggin struct vmap_block *vb; 983cf725ce2SRoman Pen void *vaddr = NULL; 984db64fe02SNick Piggin unsigned int order; 985db64fe02SNick Piggin 986891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 987db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 988aa91c4d8SJan Kara if (WARN_ON(size == 0)) { 989aa91c4d8SJan Kara /* 990aa91c4d8SJan Kara * Allocating 0 bytes isn't what caller wants since 991aa91c4d8SJan Kara * get_order(0) returns funny result. Just warn and terminate 992aa91c4d8SJan Kara * early. 993aa91c4d8SJan Kara */ 994aa91c4d8SJan Kara return NULL; 995aa91c4d8SJan Kara } 996db64fe02SNick Piggin order = get_order(size); 997db64fe02SNick Piggin 998db64fe02SNick Piggin rcu_read_lock(); 999db64fe02SNick Piggin vbq = &get_cpu_var(vmap_block_queue); 1000db64fe02SNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 1001cf725ce2SRoman Pen unsigned long pages_off; 1002db64fe02SNick Piggin 1003db64fe02SNick Piggin spin_lock(&vb->lock); 1004cf725ce2SRoman Pen if (vb->free < (1UL << order)) { 1005cf725ce2SRoman Pen spin_unlock(&vb->lock); 1006cf725ce2SRoman Pen continue; 1007cf725ce2SRoman Pen } 100802b709dfSNick Piggin 1009cf725ce2SRoman Pen pages_off = VMAP_BBMAP_BITS - vb->free; 1010cf725ce2SRoman Pen vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); 1011db64fe02SNick Piggin vb->free -= 1UL << order; 1012db64fe02SNick Piggin if (vb->free == 0) { 1013db64fe02SNick Piggin spin_lock(&vbq->lock); 1014de560423SNick Piggin list_del_rcu(&vb->free_list); 1015db64fe02SNick Piggin spin_unlock(&vbq->lock); 1016db64fe02SNick Piggin } 1017cf725ce2SRoman Pen 1018db64fe02SNick Piggin spin_unlock(&vb->lock); 1019db64fe02SNick Piggin break; 1020db64fe02SNick Piggin } 102102b709dfSNick Piggin 10223f04ba85STejun Heo put_cpu_var(vmap_block_queue); 1023db64fe02SNick Piggin rcu_read_unlock(); 1024db64fe02SNick Piggin 1025cf725ce2SRoman Pen /* Allocate new block if nothing was found */ 1026cf725ce2SRoman Pen if (!vaddr) 1027cf725ce2SRoman Pen vaddr = new_vmap_block(order, gfp_mask); 1028db64fe02SNick Piggin 1029cf725ce2SRoman Pen return vaddr; 1030db64fe02SNick Piggin } 1031db64fe02SNick Piggin 1032db64fe02SNick Piggin static void vb_free(const void *addr, unsigned long size) 1033db64fe02SNick Piggin { 1034db64fe02SNick Piggin unsigned long offset; 1035db64fe02SNick Piggin unsigned long vb_idx; 1036db64fe02SNick Piggin unsigned int order; 1037db64fe02SNick Piggin struct vmap_block *vb; 1038db64fe02SNick Piggin 1039891c49abSAlexander Kuleshov BUG_ON(offset_in_page(size)); 1040db64fe02SNick Piggin BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 1041b29acbdcSNick Piggin 1042b29acbdcSNick Piggin flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size); 1043b29acbdcSNick Piggin 1044db64fe02SNick Piggin order = get_order(size); 1045db64fe02SNick Piggin 1046db64fe02SNick Piggin offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1); 10477d61bfe8SRoman Pen offset >>= PAGE_SHIFT; 1048db64fe02SNick Piggin 1049db64fe02SNick Piggin vb_idx = addr_to_vb_idx((unsigned long)addr); 1050db64fe02SNick Piggin rcu_read_lock(); 1051db64fe02SNick Piggin vb = radix_tree_lookup(&vmap_block_tree, vb_idx); 1052db64fe02SNick Piggin rcu_read_unlock(); 1053db64fe02SNick Piggin BUG_ON(!vb); 1054db64fe02SNick Piggin 105564141da5SJeremy Fitzhardinge vunmap_page_range((unsigned long)addr, (unsigned long)addr + size); 105664141da5SJeremy Fitzhardinge 1057db64fe02SNick Piggin spin_lock(&vb->lock); 10587d61bfe8SRoman Pen 10597d61bfe8SRoman Pen /* Expand dirty range */ 10607d61bfe8SRoman Pen vb->dirty_min = min(vb->dirty_min, offset); 10617d61bfe8SRoman Pen vb->dirty_max = max(vb->dirty_max, offset + (1UL << order)); 1062d086817dSMinChan Kim 1063db64fe02SNick Piggin vb->dirty += 1UL << order; 1064db64fe02SNick Piggin if (vb->dirty == VMAP_BBMAP_BITS) { 1065de560423SNick Piggin BUG_ON(vb->free); 1066db64fe02SNick Piggin spin_unlock(&vb->lock); 1067db64fe02SNick Piggin free_vmap_block(vb); 1068db64fe02SNick Piggin } else 1069db64fe02SNick Piggin spin_unlock(&vb->lock); 1070db64fe02SNick Piggin } 1071db64fe02SNick Piggin 1072db64fe02SNick Piggin /** 1073db64fe02SNick Piggin * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer 1074db64fe02SNick Piggin * 1075db64fe02SNick Piggin * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily 1076db64fe02SNick Piggin * to amortize TLB flushing overheads. What this means is that any page you 1077db64fe02SNick Piggin * have now, may, in a former life, have been mapped into kernel virtual 1078db64fe02SNick Piggin * address by the vmap layer and so there might be some CPUs with TLB entries 1079db64fe02SNick Piggin * still referencing that page (additional to the regular 1:1 kernel mapping). 1080db64fe02SNick Piggin * 1081db64fe02SNick Piggin * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can 1082db64fe02SNick Piggin * be sure that none of the pages we have control over will have any aliases 1083db64fe02SNick Piggin * from the vmap layer. 1084db64fe02SNick Piggin */ 1085db64fe02SNick Piggin void vm_unmap_aliases(void) 1086db64fe02SNick Piggin { 1087db64fe02SNick Piggin unsigned long start = ULONG_MAX, end = 0; 1088db64fe02SNick Piggin int cpu; 1089db64fe02SNick Piggin int flush = 0; 1090db64fe02SNick Piggin 10919b463334SJeremy Fitzhardinge if (unlikely(!vmap_initialized)) 10929b463334SJeremy Fitzhardinge return; 10939b463334SJeremy Fitzhardinge 10945803ed29SChristoph Hellwig might_sleep(); 10955803ed29SChristoph Hellwig 1096db64fe02SNick Piggin for_each_possible_cpu(cpu) { 1097db64fe02SNick Piggin struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 1098db64fe02SNick Piggin struct vmap_block *vb; 1099db64fe02SNick Piggin 1100db64fe02SNick Piggin rcu_read_lock(); 1101db64fe02SNick Piggin list_for_each_entry_rcu(vb, &vbq->free, free_list) { 1102db64fe02SNick Piggin spin_lock(&vb->lock); 11037d61bfe8SRoman Pen if (vb->dirty) { 11047d61bfe8SRoman Pen unsigned long va_start = vb->va->va_start; 1105db64fe02SNick Piggin unsigned long s, e; 1106b136be5eSJoonsoo Kim 11077d61bfe8SRoman Pen s = va_start + (vb->dirty_min << PAGE_SHIFT); 11087d61bfe8SRoman Pen e = va_start + (vb->dirty_max << PAGE_SHIFT); 1109db64fe02SNick Piggin 11107d61bfe8SRoman Pen start = min(s, start); 11117d61bfe8SRoman Pen end = max(e, end); 11127d61bfe8SRoman Pen 1113db64fe02SNick Piggin flush = 1; 1114db64fe02SNick Piggin } 1115db64fe02SNick Piggin spin_unlock(&vb->lock); 1116db64fe02SNick Piggin } 1117db64fe02SNick Piggin rcu_read_unlock(); 1118db64fe02SNick Piggin } 1119db64fe02SNick Piggin 1120f9e09977SChristoph Hellwig mutex_lock(&vmap_purge_lock); 11210574ecd1SChristoph Hellwig purge_fragmented_blocks_allcpus(); 11220574ecd1SChristoph Hellwig if (!__purge_vmap_area_lazy(start, end) && flush) 11230574ecd1SChristoph Hellwig flush_tlb_kernel_range(start, end); 1124f9e09977SChristoph Hellwig mutex_unlock(&vmap_purge_lock); 1125db64fe02SNick Piggin } 1126db64fe02SNick Piggin EXPORT_SYMBOL_GPL(vm_unmap_aliases); 1127db64fe02SNick Piggin 1128db64fe02SNick Piggin /** 1129db64fe02SNick Piggin * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram 1130db64fe02SNick Piggin * @mem: the pointer returned by vm_map_ram 1131db64fe02SNick Piggin * @count: the count passed to that vm_map_ram call (cannot unmap partial) 1132db64fe02SNick Piggin */ 1133db64fe02SNick Piggin void vm_unmap_ram(const void *mem, unsigned int count) 1134db64fe02SNick Piggin { 113565ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 1136db64fe02SNick Piggin unsigned long addr = (unsigned long)mem; 11379c3acf60SChristoph Hellwig struct vmap_area *va; 1138db64fe02SNick Piggin 11395803ed29SChristoph Hellwig might_sleep(); 1140db64fe02SNick Piggin BUG_ON(!addr); 1141db64fe02SNick Piggin BUG_ON(addr < VMALLOC_START); 1142db64fe02SNick Piggin BUG_ON(addr > VMALLOC_END); 1143a1c0b1a0SShawn Lin BUG_ON(!PAGE_ALIGNED(addr)); 1144db64fe02SNick Piggin 1145db64fe02SNick Piggin debug_check_no_locks_freed(mem, size); 1146cd52858cSNick Piggin vmap_debug_free_range(addr, addr+size); 1147db64fe02SNick Piggin 11489c3acf60SChristoph Hellwig if (likely(count <= VMAP_MAX_ALLOC)) { 1149db64fe02SNick Piggin vb_free(mem, size); 11509c3acf60SChristoph Hellwig return; 11519c3acf60SChristoph Hellwig } 11529c3acf60SChristoph Hellwig 11539c3acf60SChristoph Hellwig va = find_vmap_area(addr); 11549c3acf60SChristoph Hellwig BUG_ON(!va); 11559c3acf60SChristoph Hellwig free_unmap_vmap_area(va); 1156db64fe02SNick Piggin } 1157db64fe02SNick Piggin EXPORT_SYMBOL(vm_unmap_ram); 1158db64fe02SNick Piggin 1159db64fe02SNick Piggin /** 1160db64fe02SNick Piggin * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) 1161db64fe02SNick Piggin * @pages: an array of pointers to the pages to be mapped 1162db64fe02SNick Piggin * @count: number of pages 1163db64fe02SNick Piggin * @node: prefer to allocate data structures on this node 1164db64fe02SNick Piggin * @prot: memory protection to use. PAGE_KERNEL for regular RAM 1165e99c97adSRandy Dunlap * 116636437638SGioh Kim * If you use this function for less than VMAP_MAX_ALLOC pages, it could be 116736437638SGioh Kim * faster than vmap so it's good. But if you mix long-life and short-life 116836437638SGioh Kim * objects with vm_map_ram(), it could consume lots of address space through 116936437638SGioh Kim * fragmentation (especially on a 32bit machine). You could see failures in 117036437638SGioh Kim * the end. Please use this function for short-lived objects. 117136437638SGioh Kim * 1172e99c97adSRandy Dunlap * Returns: a pointer to the address that has been mapped, or %NULL on failure 1173db64fe02SNick Piggin */ 1174db64fe02SNick Piggin void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot) 1175db64fe02SNick Piggin { 117665ee03c4SGuillermo Julián Moreno unsigned long size = (unsigned long)count << PAGE_SHIFT; 1177db64fe02SNick Piggin unsigned long addr; 1178db64fe02SNick Piggin void *mem; 1179db64fe02SNick Piggin 1180db64fe02SNick Piggin if (likely(count <= VMAP_MAX_ALLOC)) { 1181db64fe02SNick Piggin mem = vb_alloc(size, GFP_KERNEL); 1182db64fe02SNick Piggin if (IS_ERR(mem)) 1183db64fe02SNick Piggin return NULL; 1184db64fe02SNick Piggin addr = (unsigned long)mem; 1185db64fe02SNick Piggin } else { 1186db64fe02SNick Piggin struct vmap_area *va; 1187db64fe02SNick Piggin va = alloc_vmap_area(size, PAGE_SIZE, 1188db64fe02SNick Piggin VMALLOC_START, VMALLOC_END, node, GFP_KERNEL); 1189db64fe02SNick Piggin if (IS_ERR(va)) 1190db64fe02SNick Piggin return NULL; 1191db64fe02SNick Piggin 1192db64fe02SNick Piggin addr = va->va_start; 1193db64fe02SNick Piggin mem = (void *)addr; 1194db64fe02SNick Piggin } 1195db64fe02SNick Piggin if (vmap_page_range(addr, addr + size, prot, pages) < 0) { 1196db64fe02SNick Piggin vm_unmap_ram(mem, count); 1197db64fe02SNick Piggin return NULL; 1198db64fe02SNick Piggin } 1199db64fe02SNick Piggin return mem; 1200db64fe02SNick Piggin } 1201db64fe02SNick Piggin EXPORT_SYMBOL(vm_map_ram); 1202db64fe02SNick Piggin 12034341fa45SJoonsoo Kim static struct vm_struct *vmlist __initdata; 1204f0aa6617STejun Heo /** 1205be9b7335SNicolas Pitre * vm_area_add_early - add vmap area early during boot 1206be9b7335SNicolas Pitre * @vm: vm_struct to add 1207be9b7335SNicolas Pitre * 1208be9b7335SNicolas Pitre * This function is used to add fixed kernel vm area to vmlist before 1209be9b7335SNicolas Pitre * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags 1210be9b7335SNicolas Pitre * should contain proper values and the other fields should be zero. 1211be9b7335SNicolas Pitre * 1212be9b7335SNicolas Pitre * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 1213be9b7335SNicolas Pitre */ 1214be9b7335SNicolas Pitre void __init vm_area_add_early(struct vm_struct *vm) 1215be9b7335SNicolas Pitre { 1216be9b7335SNicolas Pitre struct vm_struct *tmp, **p; 1217be9b7335SNicolas Pitre 1218be9b7335SNicolas Pitre BUG_ON(vmap_initialized); 1219be9b7335SNicolas Pitre for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { 1220be9b7335SNicolas Pitre if (tmp->addr >= vm->addr) { 1221be9b7335SNicolas Pitre BUG_ON(tmp->addr < vm->addr + vm->size); 1222be9b7335SNicolas Pitre break; 1223be9b7335SNicolas Pitre } else 1224be9b7335SNicolas Pitre BUG_ON(tmp->addr + tmp->size > vm->addr); 1225be9b7335SNicolas Pitre } 1226be9b7335SNicolas Pitre vm->next = *p; 1227be9b7335SNicolas Pitre *p = vm; 1228be9b7335SNicolas Pitre } 1229be9b7335SNicolas Pitre 1230be9b7335SNicolas Pitre /** 1231f0aa6617STejun Heo * vm_area_register_early - register vmap area early during boot 1232f0aa6617STejun Heo * @vm: vm_struct to register 1233c0c0a293STejun Heo * @align: requested alignment 1234f0aa6617STejun Heo * 1235f0aa6617STejun Heo * This function is used to register kernel vm area before 1236f0aa6617STejun Heo * vmalloc_init() is called. @vm->size and @vm->flags should contain 1237f0aa6617STejun Heo * proper values on entry and other fields should be zero. On return, 1238f0aa6617STejun Heo * vm->addr contains the allocated address. 1239f0aa6617STejun Heo * 1240f0aa6617STejun Heo * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 1241f0aa6617STejun Heo */ 1242c0c0a293STejun Heo void __init vm_area_register_early(struct vm_struct *vm, size_t align) 1243f0aa6617STejun Heo { 1244f0aa6617STejun Heo static size_t vm_init_off __initdata; 1245c0c0a293STejun Heo unsigned long addr; 1246f0aa6617STejun Heo 1247c0c0a293STejun Heo addr = ALIGN(VMALLOC_START + vm_init_off, align); 1248c0c0a293STejun Heo vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START; 1249c0c0a293STejun Heo 1250c0c0a293STejun Heo vm->addr = (void *)addr; 1251f0aa6617STejun Heo 1252be9b7335SNicolas Pitre vm_area_add_early(vm); 1253f0aa6617STejun Heo } 1254f0aa6617STejun Heo 1255db64fe02SNick Piggin void __init vmalloc_init(void) 1256db64fe02SNick Piggin { 1257822c18f2SIvan Kokshaysky struct vmap_area *va; 1258822c18f2SIvan Kokshaysky struct vm_struct *tmp; 1259db64fe02SNick Piggin int i; 1260db64fe02SNick Piggin 1261db64fe02SNick Piggin for_each_possible_cpu(i) { 1262db64fe02SNick Piggin struct vmap_block_queue *vbq; 126332fcfd40SAl Viro struct vfree_deferred *p; 1264db64fe02SNick Piggin 1265db64fe02SNick Piggin vbq = &per_cpu(vmap_block_queue, i); 1266db64fe02SNick Piggin spin_lock_init(&vbq->lock); 1267db64fe02SNick Piggin INIT_LIST_HEAD(&vbq->free); 126832fcfd40SAl Viro p = &per_cpu(vfree_deferred, i); 126932fcfd40SAl Viro init_llist_head(&p->list); 127032fcfd40SAl Viro INIT_WORK(&p->wq, free_work); 1271db64fe02SNick Piggin } 12729b463334SJeremy Fitzhardinge 1273822c18f2SIvan Kokshaysky /* Import existing vmlist entries. */ 1274822c18f2SIvan Kokshaysky for (tmp = vmlist; tmp; tmp = tmp->next) { 127543ebdac4SPekka Enberg va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT); 1276dbda591dSKyongHo va->flags = VM_VM_AREA; 1277822c18f2SIvan Kokshaysky va->va_start = (unsigned long)tmp->addr; 1278822c18f2SIvan Kokshaysky va->va_end = va->va_start + tmp->size; 1279dbda591dSKyongHo va->vm = tmp; 1280822c18f2SIvan Kokshaysky __insert_vmap_area(va); 1281822c18f2SIvan Kokshaysky } 1282ca23e405STejun Heo 1283ca23e405STejun Heo vmap_area_pcpu_hole = VMALLOC_END; 1284ca23e405STejun Heo 12859b463334SJeremy Fitzhardinge vmap_initialized = true; 1286db64fe02SNick Piggin } 1287db64fe02SNick Piggin 12888fc48985STejun Heo /** 12898fc48985STejun Heo * map_kernel_range_noflush - map kernel VM area with the specified pages 12908fc48985STejun Heo * @addr: start of the VM area to map 12918fc48985STejun Heo * @size: size of the VM area to map 12928fc48985STejun Heo * @prot: page protection flags to use 12938fc48985STejun Heo * @pages: pages to map 12948fc48985STejun Heo * 12958fc48985STejun Heo * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size 12968fc48985STejun Heo * specify should have been allocated using get_vm_area() and its 12978fc48985STejun Heo * friends. 12988fc48985STejun Heo * 12998fc48985STejun Heo * NOTE: 13008fc48985STejun Heo * This function does NOT do any cache flushing. The caller is 13018fc48985STejun Heo * responsible for calling flush_cache_vmap() on to-be-mapped areas 13028fc48985STejun Heo * before calling this function. 13038fc48985STejun Heo * 13048fc48985STejun Heo * RETURNS: 13058fc48985STejun Heo * The number of pages mapped on success, -errno on failure. 13068fc48985STejun Heo */ 13078fc48985STejun Heo int map_kernel_range_noflush(unsigned long addr, unsigned long size, 13088fc48985STejun Heo pgprot_t prot, struct page **pages) 13098fc48985STejun Heo { 13108fc48985STejun Heo return vmap_page_range_noflush(addr, addr + size, prot, pages); 13118fc48985STejun Heo } 13128fc48985STejun Heo 13138fc48985STejun Heo /** 13148fc48985STejun Heo * unmap_kernel_range_noflush - unmap kernel VM area 13158fc48985STejun Heo * @addr: start of the VM area to unmap 13168fc48985STejun Heo * @size: size of the VM area to unmap 13178fc48985STejun Heo * 13188fc48985STejun Heo * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size 13198fc48985STejun Heo * specify should have been allocated using get_vm_area() and its 13208fc48985STejun Heo * friends. 13218fc48985STejun Heo * 13228fc48985STejun Heo * NOTE: 13238fc48985STejun Heo * This function does NOT do any cache flushing. The caller is 13248fc48985STejun Heo * responsible for calling flush_cache_vunmap() on to-be-mapped areas 13258fc48985STejun Heo * before calling this function and flush_tlb_kernel_range() after. 13268fc48985STejun Heo */ 13278fc48985STejun Heo void unmap_kernel_range_noflush(unsigned long addr, unsigned long size) 13288fc48985STejun Heo { 13298fc48985STejun Heo vunmap_page_range(addr, addr + size); 13308fc48985STejun Heo } 133181e88fdcSHuang Ying EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush); 13328fc48985STejun Heo 13338fc48985STejun Heo /** 13348fc48985STejun Heo * unmap_kernel_range - unmap kernel VM area and flush cache and TLB 13358fc48985STejun Heo * @addr: start of the VM area to unmap 13368fc48985STejun Heo * @size: size of the VM area to unmap 13378fc48985STejun Heo * 13388fc48985STejun Heo * Similar to unmap_kernel_range_noflush() but flushes vcache before 13398fc48985STejun Heo * the unmapping and tlb after. 13408fc48985STejun Heo */ 1341db64fe02SNick Piggin void unmap_kernel_range(unsigned long addr, unsigned long size) 1342db64fe02SNick Piggin { 1343db64fe02SNick Piggin unsigned long end = addr + size; 1344f6fcba70STejun Heo 1345f6fcba70STejun Heo flush_cache_vunmap(addr, end); 1346db64fe02SNick Piggin vunmap_page_range(addr, end); 1347db64fe02SNick Piggin flush_tlb_kernel_range(addr, end); 1348db64fe02SNick Piggin } 134993ef6d6cSMinchan Kim EXPORT_SYMBOL_GPL(unmap_kernel_range); 1350db64fe02SNick Piggin 1351f6f8ed47SWANG Chao int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages) 1352db64fe02SNick Piggin { 1353db64fe02SNick Piggin unsigned long addr = (unsigned long)area->addr; 1354762216abSWanpeng Li unsigned long end = addr + get_vm_area_size(area); 1355db64fe02SNick Piggin int err; 1356db64fe02SNick Piggin 1357f6f8ed47SWANG Chao err = vmap_page_range(addr, end, prot, pages); 1358db64fe02SNick Piggin 1359f6f8ed47SWANG Chao return err > 0 ? 0 : err; 1360db64fe02SNick Piggin } 1361db64fe02SNick Piggin EXPORT_SYMBOL_GPL(map_vm_area); 1362db64fe02SNick Piggin 1363f5252e00SMitsuo Hayasaka static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, 13645e6cafc8SMarek Szyprowski unsigned long flags, const void *caller) 1365cf88c790STejun Heo { 1366c69480adSJoonsoo Kim spin_lock(&vmap_area_lock); 1367cf88c790STejun Heo vm->flags = flags; 1368cf88c790STejun Heo vm->addr = (void *)va->va_start; 1369cf88c790STejun Heo vm->size = va->va_end - va->va_start; 1370cf88c790STejun Heo vm->caller = caller; 1371db1aecafSMinchan Kim va->vm = vm; 1372cf88c790STejun Heo va->flags |= VM_VM_AREA; 1373c69480adSJoonsoo Kim spin_unlock(&vmap_area_lock); 1374f5252e00SMitsuo Hayasaka } 1375cf88c790STejun Heo 137620fc02b4SZhang Yanfei static void clear_vm_uninitialized_flag(struct vm_struct *vm) 1377f5252e00SMitsuo Hayasaka { 1378d4033afdSJoonsoo Kim /* 137920fc02b4SZhang Yanfei * Before removing VM_UNINITIALIZED, 1380d4033afdSJoonsoo Kim * we should make sure that vm has proper values. 1381d4033afdSJoonsoo Kim * Pair with smp_rmb() in show_numa_info(). 1382d4033afdSJoonsoo Kim */ 1383d4033afdSJoonsoo Kim smp_wmb(); 138420fc02b4SZhang Yanfei vm->flags &= ~VM_UNINITIALIZED; 1385cf88c790STejun Heo } 1386cf88c790STejun Heo 1387db64fe02SNick Piggin static struct vm_struct *__get_vm_area_node(unsigned long size, 13882dca6999SDavid Miller unsigned long align, unsigned long flags, unsigned long start, 13895e6cafc8SMarek Szyprowski unsigned long end, int node, gfp_t gfp_mask, const void *caller) 1390db64fe02SNick Piggin { 13910006526dSKautuk Consul struct vmap_area *va; 1392db64fe02SNick Piggin struct vm_struct *area; 13931da177e4SLinus Torvalds 139452fd24caSGiridhar Pemmasani BUG_ON(in_interrupt()); 13951da177e4SLinus Torvalds size = PAGE_ALIGN(size); 139631be8309SOGAWA Hirofumi if (unlikely(!size)) 139731be8309SOGAWA Hirofumi return NULL; 13981da177e4SLinus Torvalds 1399252e5c6eSzijun_hu if (flags & VM_IOREMAP) 1400252e5c6eSzijun_hu align = 1ul << clamp_t(int, get_count_order_long(size), 1401252e5c6eSzijun_hu PAGE_SHIFT, IOREMAP_MAX_ORDER); 1402252e5c6eSzijun_hu 1403cf88c790STejun Heo area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); 14041da177e4SLinus Torvalds if (unlikely(!area)) 14051da177e4SLinus Torvalds return NULL; 14061da177e4SLinus Torvalds 140771394fe5SAndrey Ryabinin if (!(flags & VM_NO_GUARD)) 14081da177e4SLinus Torvalds size += PAGE_SIZE; 14091da177e4SLinus Torvalds 1410db64fe02SNick Piggin va = alloc_vmap_area(size, align, start, end, node, gfp_mask); 1411db64fe02SNick Piggin if (IS_ERR(va)) { 1412db64fe02SNick Piggin kfree(area); 1413db64fe02SNick Piggin return NULL; 14141da177e4SLinus Torvalds } 14151da177e4SLinus Torvalds 1416f5252e00SMitsuo Hayasaka setup_vmalloc_vm(area, va, flags, caller); 1417f5252e00SMitsuo Hayasaka 14181da177e4SLinus Torvalds return area; 14191da177e4SLinus Torvalds } 14201da177e4SLinus Torvalds 1421930fc45aSChristoph Lameter struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags, 1422930fc45aSChristoph Lameter unsigned long start, unsigned long end) 1423930fc45aSChristoph Lameter { 142400ef2d2fSDavid Rientjes return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE, 142500ef2d2fSDavid Rientjes GFP_KERNEL, __builtin_return_address(0)); 1426930fc45aSChristoph Lameter } 14275992b6daSRusty Russell EXPORT_SYMBOL_GPL(__get_vm_area); 1428930fc45aSChristoph Lameter 1429c2968612SBenjamin Herrenschmidt struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, 1430c2968612SBenjamin Herrenschmidt unsigned long start, unsigned long end, 14315e6cafc8SMarek Szyprowski const void *caller) 1432c2968612SBenjamin Herrenschmidt { 143300ef2d2fSDavid Rientjes return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE, 143400ef2d2fSDavid Rientjes GFP_KERNEL, caller); 1435c2968612SBenjamin Herrenschmidt } 1436c2968612SBenjamin Herrenschmidt 14371da177e4SLinus Torvalds /** 1438183ff22bSSimon Arlott * get_vm_area - reserve a contiguous kernel virtual area 14391da177e4SLinus Torvalds * @size: size of the area 14401da177e4SLinus Torvalds * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC 14411da177e4SLinus Torvalds * 14421da177e4SLinus Torvalds * Search an area of @size in the kernel virtual mapping area, 14431da177e4SLinus Torvalds * and reserved it for out purposes. Returns the area descriptor 14441da177e4SLinus Torvalds * on success or %NULL on failure. 14451da177e4SLinus Torvalds */ 14461da177e4SLinus Torvalds struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) 14471da177e4SLinus Torvalds { 14482dca6999SDavid Miller return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, 144900ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, 145000ef2d2fSDavid Rientjes __builtin_return_address(0)); 145123016969SChristoph Lameter } 145223016969SChristoph Lameter 145323016969SChristoph Lameter struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, 14545e6cafc8SMarek Szyprowski const void *caller) 145523016969SChristoph Lameter { 14562dca6999SDavid Miller return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, 145700ef2d2fSDavid Rientjes NUMA_NO_NODE, GFP_KERNEL, caller); 14581da177e4SLinus Torvalds } 14591da177e4SLinus Torvalds 1460e9da6e99SMarek Szyprowski /** 1461e9da6e99SMarek Szyprowski * find_vm_area - find a continuous kernel virtual area 1462e9da6e99SMarek Szyprowski * @addr: base address 1463e9da6e99SMarek Szyprowski * 1464e9da6e99SMarek Szyprowski * Search for the kernel VM area starting at @addr, and return it. 1465e9da6e99SMarek Szyprowski * It is up to the caller to do all required locking to keep the returned 1466e9da6e99SMarek Szyprowski * pointer valid. 1467e9da6e99SMarek Szyprowski */ 1468e9da6e99SMarek Szyprowski struct vm_struct *find_vm_area(const void *addr) 146983342314SNick Piggin { 1470db64fe02SNick Piggin struct vmap_area *va; 147183342314SNick Piggin 1472db64fe02SNick Piggin va = find_vmap_area((unsigned long)addr); 1473db64fe02SNick Piggin if (va && va->flags & VM_VM_AREA) 1474db1aecafSMinchan Kim return va->vm; 147583342314SNick Piggin 14767856dfebSAndi Kleen return NULL; 14777856dfebSAndi Kleen } 14787856dfebSAndi Kleen 14791da177e4SLinus Torvalds /** 1480183ff22bSSimon Arlott * remove_vm_area - find and remove a continuous kernel virtual area 14811da177e4SLinus Torvalds * @addr: base address 14821da177e4SLinus Torvalds * 14831da177e4SLinus Torvalds * Search for the kernel VM area starting at @addr, and remove it. 14841da177e4SLinus Torvalds * This function returns the found VM area, but using it is NOT safe 14857856dfebSAndi Kleen * on SMP machines, except for its size or flags. 14861da177e4SLinus Torvalds */ 1487b3bdda02SChristoph Lameter struct vm_struct *remove_vm_area(const void *addr) 14881da177e4SLinus Torvalds { 1489db64fe02SNick Piggin struct vmap_area *va; 1490db64fe02SNick Piggin 14915803ed29SChristoph Hellwig might_sleep(); 14925803ed29SChristoph Hellwig 1493db64fe02SNick Piggin va = find_vmap_area((unsigned long)addr); 1494db64fe02SNick Piggin if (va && va->flags & VM_VM_AREA) { 1495db1aecafSMinchan Kim struct vm_struct *vm = va->vm; 1496f5252e00SMitsuo Hayasaka 1497c69480adSJoonsoo Kim spin_lock(&vmap_area_lock); 1498c69480adSJoonsoo Kim va->vm = NULL; 1499c69480adSJoonsoo Kim va->flags &= ~VM_VM_AREA; 1500c69480adSJoonsoo Kim spin_unlock(&vmap_area_lock); 1501c69480adSJoonsoo Kim 1502dd32c279SKAMEZAWA Hiroyuki vmap_debug_free_range(va->va_start, va->va_end); 1503a5af5aa8SAndrey Ryabinin kasan_free_shadow(vm); 1504dd32c279SKAMEZAWA Hiroyuki free_unmap_vmap_area(va); 1505dd32c279SKAMEZAWA Hiroyuki 1506db64fe02SNick Piggin return vm; 1507db64fe02SNick Piggin } 1508db64fe02SNick Piggin return NULL; 15091da177e4SLinus Torvalds } 15101da177e4SLinus Torvalds 1511b3bdda02SChristoph Lameter static void __vunmap(const void *addr, int deallocate_pages) 15121da177e4SLinus Torvalds { 15131da177e4SLinus Torvalds struct vm_struct *area; 15141da177e4SLinus Torvalds 15151da177e4SLinus Torvalds if (!addr) 15161da177e4SLinus Torvalds return; 15171da177e4SLinus Torvalds 1518e69e9d4aSHATAYAMA Daisuke if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n", 1519ab15d9b4SDan Carpenter addr)) 15201da177e4SLinus Torvalds return; 15211da177e4SLinus Torvalds 15221da177e4SLinus Torvalds area = remove_vm_area(addr); 15231da177e4SLinus Torvalds if (unlikely(!area)) { 15244c8573e2SArjan van de Ven WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", 15251da177e4SLinus Torvalds addr); 15261da177e4SLinus Torvalds return; 15271da177e4SLinus Torvalds } 15281da177e4SLinus Torvalds 15297511c3edSJerome Marchand debug_check_no_locks_freed(addr, get_vm_area_size(area)); 15307511c3edSJerome Marchand debug_check_no_obj_freed(addr, get_vm_area_size(area)); 15319a11b49aSIngo Molnar 15321da177e4SLinus Torvalds if (deallocate_pages) { 15331da177e4SLinus Torvalds int i; 15341da177e4SLinus Torvalds 15351da177e4SLinus Torvalds for (i = 0; i < area->nr_pages; i++) { 1536bf53d6f8SChristoph Lameter struct page *page = area->pages[i]; 1537bf53d6f8SChristoph Lameter 1538bf53d6f8SChristoph Lameter BUG_ON(!page); 15394949148aSVladimir Davydov __free_pages(page, 0); 15401da177e4SLinus Torvalds } 15411da177e4SLinus Torvalds 1542244d63eeSDavid Rientjes kvfree(area->pages); 15431da177e4SLinus Torvalds } 15441da177e4SLinus Torvalds 15451da177e4SLinus Torvalds kfree(area); 15461da177e4SLinus Torvalds return; 15471da177e4SLinus Torvalds } 15481da177e4SLinus Torvalds 1549bf22e37aSAndrey Ryabinin static inline void __vfree_deferred(const void *addr) 1550bf22e37aSAndrey Ryabinin { 1551bf22e37aSAndrey Ryabinin /* 1552bf22e37aSAndrey Ryabinin * Use raw_cpu_ptr() because this can be called from preemptible 1553bf22e37aSAndrey Ryabinin * context. Preemption is absolutely fine here, because the llist_add() 1554bf22e37aSAndrey Ryabinin * implementation is lockless, so it works even if we are adding to 1555bf22e37aSAndrey Ryabinin * nother cpu's list. schedule_work() should be fine with this too. 1556bf22e37aSAndrey Ryabinin */ 1557bf22e37aSAndrey Ryabinin struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred); 1558bf22e37aSAndrey Ryabinin 1559bf22e37aSAndrey Ryabinin if (llist_add((struct llist_node *)addr, &p->list)) 1560bf22e37aSAndrey Ryabinin schedule_work(&p->wq); 1561bf22e37aSAndrey Ryabinin } 1562bf22e37aSAndrey Ryabinin 1563bf22e37aSAndrey Ryabinin /** 1564bf22e37aSAndrey Ryabinin * vfree_atomic - release memory allocated by vmalloc() 1565bf22e37aSAndrey Ryabinin * @addr: memory base address 1566bf22e37aSAndrey Ryabinin * 1567bf22e37aSAndrey Ryabinin * This one is just like vfree() but can be called in any atomic context 1568bf22e37aSAndrey Ryabinin * except NMIs. 1569bf22e37aSAndrey Ryabinin */ 1570bf22e37aSAndrey Ryabinin void vfree_atomic(const void *addr) 1571bf22e37aSAndrey Ryabinin { 1572bf22e37aSAndrey Ryabinin BUG_ON(in_nmi()); 1573bf22e37aSAndrey Ryabinin 1574bf22e37aSAndrey Ryabinin kmemleak_free(addr); 1575bf22e37aSAndrey Ryabinin 1576bf22e37aSAndrey Ryabinin if (!addr) 1577bf22e37aSAndrey Ryabinin return; 1578bf22e37aSAndrey Ryabinin __vfree_deferred(addr); 1579bf22e37aSAndrey Ryabinin } 1580bf22e37aSAndrey Ryabinin 15811da177e4SLinus Torvalds /** 15821da177e4SLinus Torvalds * vfree - release memory allocated by vmalloc() 15831da177e4SLinus Torvalds * @addr: memory base address 15841da177e4SLinus Torvalds * 1585183ff22bSSimon Arlott * Free the virtually continuous memory area starting at @addr, as 158680e93effSPekka Enberg * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is 158780e93effSPekka Enberg * NULL, no operation is performed. 15881da177e4SLinus Torvalds * 158932fcfd40SAl Viro * Must not be called in NMI context (strictly speaking, only if we don't 159032fcfd40SAl Viro * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling 159132fcfd40SAl Viro * conventions for vfree() arch-depenedent would be a really bad idea) 159232fcfd40SAl Viro * 15930e056eb5Smchehab@s-opensource.com * NOTE: assumes that the object at @addr has a size >= sizeof(llist_node) 15941da177e4SLinus Torvalds */ 1595b3bdda02SChristoph Lameter void vfree(const void *addr) 15961da177e4SLinus Torvalds { 159732fcfd40SAl Viro BUG_ON(in_nmi()); 159889219d37SCatalin Marinas 159989219d37SCatalin Marinas kmemleak_free(addr); 160089219d37SCatalin Marinas 160132fcfd40SAl Viro if (!addr) 160232fcfd40SAl Viro return; 1603bf22e37aSAndrey Ryabinin if (unlikely(in_interrupt())) 1604bf22e37aSAndrey Ryabinin __vfree_deferred(addr); 1605bf22e37aSAndrey Ryabinin else 16061da177e4SLinus Torvalds __vunmap(addr, 1); 16071da177e4SLinus Torvalds } 16081da177e4SLinus Torvalds EXPORT_SYMBOL(vfree); 16091da177e4SLinus Torvalds 16101da177e4SLinus Torvalds /** 16111da177e4SLinus Torvalds * vunmap - release virtual mapping obtained by vmap() 16121da177e4SLinus Torvalds * @addr: memory base address 16131da177e4SLinus Torvalds * 16141da177e4SLinus Torvalds * Free the virtually contiguous memory area starting at @addr, 16151da177e4SLinus Torvalds * which was created from the page array passed to vmap(). 16161da177e4SLinus Torvalds * 161780e93effSPekka Enberg * Must not be called in interrupt context. 16181da177e4SLinus Torvalds */ 1619b3bdda02SChristoph Lameter void vunmap(const void *addr) 16201da177e4SLinus Torvalds { 16211da177e4SLinus Torvalds BUG_ON(in_interrupt()); 162234754b69SPeter Zijlstra might_sleep(); 162332fcfd40SAl Viro if (addr) 16241da177e4SLinus Torvalds __vunmap(addr, 0); 16251da177e4SLinus Torvalds } 16261da177e4SLinus Torvalds EXPORT_SYMBOL(vunmap); 16271da177e4SLinus Torvalds 16281da177e4SLinus Torvalds /** 16291da177e4SLinus Torvalds * vmap - map an array of pages into virtually contiguous space 16301da177e4SLinus Torvalds * @pages: array of page pointers 16311da177e4SLinus Torvalds * @count: number of pages to map 16321da177e4SLinus Torvalds * @flags: vm_area->flags 16331da177e4SLinus Torvalds * @prot: page protection for the mapping 16341da177e4SLinus Torvalds * 16351da177e4SLinus Torvalds * Maps @count pages from @pages into contiguous kernel virtual 16361da177e4SLinus Torvalds * space. 16371da177e4SLinus Torvalds */ 16381da177e4SLinus Torvalds void *vmap(struct page **pages, unsigned int count, 16391da177e4SLinus Torvalds unsigned long flags, pgprot_t prot) 16401da177e4SLinus Torvalds { 16411da177e4SLinus Torvalds struct vm_struct *area; 164265ee03c4SGuillermo Julián Moreno unsigned long size; /* In bytes */ 16431da177e4SLinus Torvalds 164434754b69SPeter Zijlstra might_sleep(); 164534754b69SPeter Zijlstra 16464481374cSJan Beulich if (count > totalram_pages) 16471da177e4SLinus Torvalds return NULL; 16481da177e4SLinus Torvalds 164965ee03c4SGuillermo Julián Moreno size = (unsigned long)count << PAGE_SHIFT; 165065ee03c4SGuillermo Julián Moreno area = get_vm_area_caller(size, flags, __builtin_return_address(0)); 16511da177e4SLinus Torvalds if (!area) 16521da177e4SLinus Torvalds return NULL; 165323016969SChristoph Lameter 1654f6f8ed47SWANG Chao if (map_vm_area(area, prot, pages)) { 16551da177e4SLinus Torvalds vunmap(area->addr); 16561da177e4SLinus Torvalds return NULL; 16571da177e4SLinus Torvalds } 16581da177e4SLinus Torvalds 16591da177e4SLinus Torvalds return area->addr; 16601da177e4SLinus Torvalds } 16611da177e4SLinus Torvalds EXPORT_SYMBOL(vmap); 16621da177e4SLinus Torvalds 16638594a21cSMichal Hocko static void *__vmalloc_node(unsigned long size, unsigned long align, 16648594a21cSMichal Hocko gfp_t gfp_mask, pgprot_t prot, 16658594a21cSMichal Hocko int node, const void *caller); 1666e31d9eb5SAdrian Bunk static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, 16673722e13cSWanpeng Li pgprot_t prot, int node) 16681da177e4SLinus Torvalds { 16691da177e4SLinus Torvalds struct page **pages; 16701da177e4SLinus Torvalds unsigned int nr_pages, array_size, i; 1671930f036bSDavid Rientjes const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; 167219809c2dSMichal Hocko const gfp_t alloc_mask = gfp_mask | __GFP_HIGHMEM | __GFP_NOWARN; 16731da177e4SLinus Torvalds 1674762216abSWanpeng Li nr_pages = get_vm_area_size(area) >> PAGE_SHIFT; 16751da177e4SLinus Torvalds array_size = (nr_pages * sizeof(struct page *)); 16761da177e4SLinus Torvalds 16771da177e4SLinus Torvalds area->nr_pages = nr_pages; 16781da177e4SLinus Torvalds /* Please note that the recursion is strictly bounded. */ 16798757d5faSJan Kiszka if (array_size > PAGE_SIZE) { 1680976d6dfbSJan Beulich pages = __vmalloc_node(array_size, 1, nested_gfp|__GFP_HIGHMEM, 16813722e13cSWanpeng Li PAGE_KERNEL, node, area->caller); 1682286e1ea3SAndrew Morton } else { 1683976d6dfbSJan Beulich pages = kmalloc_node(array_size, nested_gfp, node); 1684286e1ea3SAndrew Morton } 16851da177e4SLinus Torvalds area->pages = pages; 16861da177e4SLinus Torvalds if (!area->pages) { 16871da177e4SLinus Torvalds remove_vm_area(area->addr); 16881da177e4SLinus Torvalds kfree(area); 16891da177e4SLinus Torvalds return NULL; 16901da177e4SLinus Torvalds } 16911da177e4SLinus Torvalds 16921da177e4SLinus Torvalds for (i = 0; i < area->nr_pages; i++) { 1693bf53d6f8SChristoph Lameter struct page *page; 1694bf53d6f8SChristoph Lameter 16955d17a73aSMichal Hocko if (fatal_signal_pending(current)) { 16965d17a73aSMichal Hocko area->nr_pages = i; 1697171012f5SDmitry Vyukov goto fail_no_warn; 16985d17a73aSMichal Hocko } 16995d17a73aSMichal Hocko 17004b90951cSJianguo Wu if (node == NUMA_NO_NODE) 17017877cdccSMichal Hocko page = alloc_page(alloc_mask); 1702930fc45aSChristoph Lameter else 17037877cdccSMichal Hocko page = alloc_pages_node(node, alloc_mask, 0); 1704bf53d6f8SChristoph Lameter 1705bf53d6f8SChristoph Lameter if (unlikely(!page)) { 17061da177e4SLinus Torvalds /* Successfully allocated i pages, free them in __vunmap() */ 17071da177e4SLinus Torvalds area->nr_pages = i; 17081da177e4SLinus Torvalds goto fail; 17091da177e4SLinus Torvalds } 1710bf53d6f8SChristoph Lameter area->pages[i] = page; 1711d0164adcSMel Gorman if (gfpflags_allow_blocking(gfp_mask)) 1712660654f9SEric Dumazet cond_resched(); 17131da177e4SLinus Torvalds } 17141da177e4SLinus Torvalds 1715f6f8ed47SWANG Chao if (map_vm_area(area, prot, pages)) 17161da177e4SLinus Torvalds goto fail; 17171da177e4SLinus Torvalds return area->addr; 17181da177e4SLinus Torvalds 17191da177e4SLinus Torvalds fail: 1720a8e99259SMichal Hocko warn_alloc(gfp_mask, NULL, 17217877cdccSMichal Hocko "vmalloc: allocation failure, allocated %ld of %ld bytes", 172222943ab1SDave Hansen (area->nr_pages*PAGE_SIZE), area->size); 1723171012f5SDmitry Vyukov fail_no_warn: 17241da177e4SLinus Torvalds vfree(area->addr); 17251da177e4SLinus Torvalds return NULL; 17261da177e4SLinus Torvalds } 17271da177e4SLinus Torvalds 1728d0a21265SDavid Rientjes /** 1729d0a21265SDavid Rientjes * __vmalloc_node_range - allocate virtually contiguous memory 1730d0a21265SDavid Rientjes * @size: allocation size 1731d0a21265SDavid Rientjes * @align: desired alignment 1732d0a21265SDavid Rientjes * @start: vm area range start 1733d0a21265SDavid Rientjes * @end: vm area range end 1734d0a21265SDavid Rientjes * @gfp_mask: flags for the page level allocator 1735d0a21265SDavid Rientjes * @prot: protection mask for the allocated pages 1736cb9e3c29SAndrey Ryabinin * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) 173700ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 1738d0a21265SDavid Rientjes * @caller: caller's return address 1739d0a21265SDavid Rientjes * 1740d0a21265SDavid Rientjes * Allocate enough pages to cover @size from the page level 1741d0a21265SDavid Rientjes * allocator with @gfp_mask flags. Map them into contiguous 1742d0a21265SDavid Rientjes * kernel virtual space, using a pagetable protection of @prot. 1743d0a21265SDavid Rientjes */ 1744d0a21265SDavid Rientjes void *__vmalloc_node_range(unsigned long size, unsigned long align, 1745d0a21265SDavid Rientjes unsigned long start, unsigned long end, gfp_t gfp_mask, 1746cb9e3c29SAndrey Ryabinin pgprot_t prot, unsigned long vm_flags, int node, 1747cb9e3c29SAndrey Ryabinin const void *caller) 1748930fc45aSChristoph Lameter { 1749d0a21265SDavid Rientjes struct vm_struct *area; 1750d0a21265SDavid Rientjes void *addr; 1751d0a21265SDavid Rientjes unsigned long real_size = size; 1752d0a21265SDavid Rientjes 1753d0a21265SDavid Rientjes size = PAGE_ALIGN(size); 1754d0a21265SDavid Rientjes if (!size || (size >> PAGE_SHIFT) > totalram_pages) 1755de7d2b56SJoe Perches goto fail; 1756d0a21265SDavid Rientjes 1757cb9e3c29SAndrey Ryabinin area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED | 1758cb9e3c29SAndrey Ryabinin vm_flags, start, end, node, gfp_mask, caller); 1759d0a21265SDavid Rientjes if (!area) 1760de7d2b56SJoe Perches goto fail; 1761d0a21265SDavid Rientjes 17623722e13cSWanpeng Li addr = __vmalloc_area_node(area, gfp_mask, prot, node); 17631368edf0SMel Gorman if (!addr) 1764b82225f3SWanpeng Li return NULL; 176589219d37SCatalin Marinas 176689219d37SCatalin Marinas /* 176720fc02b4SZhang Yanfei * In this function, newly allocated vm_struct has VM_UNINITIALIZED 176820fc02b4SZhang Yanfei * flag. It means that vm_struct is not fully initialized. 17694341fa45SJoonsoo Kim * Now, it is fully initialized, so remove this flag here. 1770f5252e00SMitsuo Hayasaka */ 177120fc02b4SZhang Yanfei clear_vm_uninitialized_flag(area); 1772f5252e00SMitsuo Hayasaka 1773*94f4a161SCatalin Marinas kmemleak_vmalloc(area, size, gfp_mask); 177489219d37SCatalin Marinas 177589219d37SCatalin Marinas return addr; 1776de7d2b56SJoe Perches 1777de7d2b56SJoe Perches fail: 1778a8e99259SMichal Hocko warn_alloc(gfp_mask, NULL, 17797877cdccSMichal Hocko "vmalloc: allocation failure: %lu bytes", real_size); 1780de7d2b56SJoe Perches return NULL; 1781930fc45aSChristoph Lameter } 1782930fc45aSChristoph Lameter 17831da177e4SLinus Torvalds /** 1784930fc45aSChristoph Lameter * __vmalloc_node - allocate virtually contiguous memory 17851da177e4SLinus Torvalds * @size: allocation size 17862dca6999SDavid Miller * @align: desired alignment 17871da177e4SLinus Torvalds * @gfp_mask: flags for the page level allocator 17881da177e4SLinus Torvalds * @prot: protection mask for the allocated pages 178900ef2d2fSDavid Rientjes * @node: node to use for allocation or NUMA_NO_NODE 1790c85d194bSRandy Dunlap * @caller: caller's return address 17911da177e4SLinus Torvalds * 17921da177e4SLinus Torvalds * Allocate enough pages to cover @size from the page level 17931da177e4SLinus Torvalds * allocator with @gfp_mask flags. Map them into contiguous 17941da177e4SLinus Torvalds * kernel virtual space, using a pagetable protection of @prot. 1795a7c3e901SMichal Hocko * 1796a7c3e901SMichal Hocko * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_REPEAT 1797a7c3e901SMichal Hocko * and __GFP_NOFAIL are not supported 1798a7c3e901SMichal Hocko * 1799a7c3e901SMichal Hocko * Any use of gfp flags outside of GFP_KERNEL should be consulted 1800a7c3e901SMichal Hocko * with mm people. 1801a7c3e901SMichal Hocko * 18021da177e4SLinus Torvalds */ 18038594a21cSMichal Hocko static void *__vmalloc_node(unsigned long size, unsigned long align, 18042dca6999SDavid Miller gfp_t gfp_mask, pgprot_t prot, 18055e6cafc8SMarek Szyprowski int node, const void *caller) 18061da177e4SLinus Torvalds { 1807d0a21265SDavid Rientjes return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 1808cb9e3c29SAndrey Ryabinin gfp_mask, prot, 0, node, caller); 18091da177e4SLinus Torvalds } 18101da177e4SLinus Torvalds 1811930fc45aSChristoph Lameter void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) 1812930fc45aSChristoph Lameter { 181300ef2d2fSDavid Rientjes return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE, 181423016969SChristoph Lameter __builtin_return_address(0)); 1815930fc45aSChristoph Lameter } 18161da177e4SLinus Torvalds EXPORT_SYMBOL(__vmalloc); 18171da177e4SLinus Torvalds 18188594a21cSMichal Hocko static inline void *__vmalloc_node_flags(unsigned long size, 18198594a21cSMichal Hocko int node, gfp_t flags) 18208594a21cSMichal Hocko { 18218594a21cSMichal Hocko return __vmalloc_node(size, 1, flags, PAGE_KERNEL, 18228594a21cSMichal Hocko node, __builtin_return_address(0)); 18238594a21cSMichal Hocko } 18248594a21cSMichal Hocko 18258594a21cSMichal Hocko 18268594a21cSMichal Hocko void *__vmalloc_node_flags_caller(unsigned long size, int node, gfp_t flags, 18278594a21cSMichal Hocko void *caller) 18288594a21cSMichal Hocko { 18298594a21cSMichal Hocko return __vmalloc_node(size, 1, flags, PAGE_KERNEL, node, caller); 18308594a21cSMichal Hocko } 18318594a21cSMichal Hocko 18321da177e4SLinus Torvalds /** 18331da177e4SLinus Torvalds * vmalloc - allocate virtually contiguous memory 18341da177e4SLinus Torvalds * @size: allocation size 18351da177e4SLinus Torvalds * Allocate enough pages to cover @size from the page level 18361da177e4SLinus Torvalds * allocator and map them into contiguous kernel virtual space. 18371da177e4SLinus Torvalds * 1838c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 18391da177e4SLinus Torvalds * use __vmalloc() instead. 18401da177e4SLinus Torvalds */ 18411da177e4SLinus Torvalds void *vmalloc(unsigned long size) 18421da177e4SLinus Torvalds { 184300ef2d2fSDavid Rientjes return __vmalloc_node_flags(size, NUMA_NO_NODE, 184419809c2dSMichal Hocko GFP_KERNEL); 18451da177e4SLinus Torvalds } 18461da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc); 18471da177e4SLinus Torvalds 1848930fc45aSChristoph Lameter /** 1849e1ca7788SDave Young * vzalloc - allocate virtually contiguous memory with zero fill 1850e1ca7788SDave Young * @size: allocation size 1851e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 1852e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 1853e1ca7788SDave Young * The memory allocated is set to zero. 1854e1ca7788SDave Young * 1855e1ca7788SDave Young * For tight control over page level allocator and protection flags 1856e1ca7788SDave Young * use __vmalloc() instead. 1857e1ca7788SDave Young */ 1858e1ca7788SDave Young void *vzalloc(unsigned long size) 1859e1ca7788SDave Young { 186000ef2d2fSDavid Rientjes return __vmalloc_node_flags(size, NUMA_NO_NODE, 186119809c2dSMichal Hocko GFP_KERNEL | __GFP_ZERO); 1862e1ca7788SDave Young } 1863e1ca7788SDave Young EXPORT_SYMBOL(vzalloc); 1864e1ca7788SDave Young 1865e1ca7788SDave Young /** 1866ead04089SRolf Eike Beer * vmalloc_user - allocate zeroed virtually contiguous memory for userspace 186783342314SNick Piggin * @size: allocation size 1868ead04089SRolf Eike Beer * 1869ead04089SRolf Eike Beer * The resulting memory area is zeroed so it can be mapped to userspace 1870ead04089SRolf Eike Beer * without leaking data. 187183342314SNick Piggin */ 187283342314SNick Piggin void *vmalloc_user(unsigned long size) 187383342314SNick Piggin { 187483342314SNick Piggin struct vm_struct *area; 187583342314SNick Piggin void *ret; 187683342314SNick Piggin 18772dca6999SDavid Miller ret = __vmalloc_node(size, SHMLBA, 187819809c2dSMichal Hocko GFP_KERNEL | __GFP_ZERO, 187900ef2d2fSDavid Rientjes PAGE_KERNEL, NUMA_NO_NODE, 188000ef2d2fSDavid Rientjes __builtin_return_address(0)); 18812b4ac44eSEric Dumazet if (ret) { 1882db64fe02SNick Piggin area = find_vm_area(ret); 188383342314SNick Piggin area->flags |= VM_USERMAP; 18842b4ac44eSEric Dumazet } 188583342314SNick Piggin return ret; 188683342314SNick Piggin } 188783342314SNick Piggin EXPORT_SYMBOL(vmalloc_user); 188883342314SNick Piggin 188983342314SNick Piggin /** 1890930fc45aSChristoph Lameter * vmalloc_node - allocate memory on a specific node 1891930fc45aSChristoph Lameter * @size: allocation size 1892d44e0780SRandy Dunlap * @node: numa node 1893930fc45aSChristoph Lameter * 1894930fc45aSChristoph Lameter * Allocate enough pages to cover @size from the page level 1895930fc45aSChristoph Lameter * allocator and map them into contiguous kernel virtual space. 1896930fc45aSChristoph Lameter * 1897c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 1898930fc45aSChristoph Lameter * use __vmalloc() instead. 1899930fc45aSChristoph Lameter */ 1900930fc45aSChristoph Lameter void *vmalloc_node(unsigned long size, int node) 1901930fc45aSChristoph Lameter { 190219809c2dSMichal Hocko return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL, 190323016969SChristoph Lameter node, __builtin_return_address(0)); 1904930fc45aSChristoph Lameter } 1905930fc45aSChristoph Lameter EXPORT_SYMBOL(vmalloc_node); 1906930fc45aSChristoph Lameter 1907e1ca7788SDave Young /** 1908e1ca7788SDave Young * vzalloc_node - allocate memory on a specific node with zero fill 1909e1ca7788SDave Young * @size: allocation size 1910e1ca7788SDave Young * @node: numa node 1911e1ca7788SDave Young * 1912e1ca7788SDave Young * Allocate enough pages to cover @size from the page level 1913e1ca7788SDave Young * allocator and map them into contiguous kernel virtual space. 1914e1ca7788SDave Young * The memory allocated is set to zero. 1915e1ca7788SDave Young * 1916e1ca7788SDave Young * For tight control over page level allocator and protection flags 1917e1ca7788SDave Young * use __vmalloc_node() instead. 1918e1ca7788SDave Young */ 1919e1ca7788SDave Young void *vzalloc_node(unsigned long size, int node) 1920e1ca7788SDave Young { 1921e1ca7788SDave Young return __vmalloc_node_flags(size, node, 192219809c2dSMichal Hocko GFP_KERNEL | __GFP_ZERO); 1923e1ca7788SDave Young } 1924e1ca7788SDave Young EXPORT_SYMBOL(vzalloc_node); 1925e1ca7788SDave Young 19264dc3b16bSPavel Pisa #ifndef PAGE_KERNEL_EXEC 19274dc3b16bSPavel Pisa # define PAGE_KERNEL_EXEC PAGE_KERNEL 19284dc3b16bSPavel Pisa #endif 19294dc3b16bSPavel Pisa 19301da177e4SLinus Torvalds /** 19311da177e4SLinus Torvalds * vmalloc_exec - allocate virtually contiguous, executable memory 19321da177e4SLinus Torvalds * @size: allocation size 19331da177e4SLinus Torvalds * 19341da177e4SLinus Torvalds * Kernel-internal function to allocate enough pages to cover @size 19351da177e4SLinus Torvalds * the page level allocator and map them into contiguous and 19361da177e4SLinus Torvalds * executable kernel virtual space. 19371da177e4SLinus Torvalds * 1938c1c8897fSMichael Opdenacker * For tight control over page level allocator and protection flags 19391da177e4SLinus Torvalds * use __vmalloc() instead. 19401da177e4SLinus Torvalds */ 19411da177e4SLinus Torvalds 19421da177e4SLinus Torvalds void *vmalloc_exec(unsigned long size) 19431da177e4SLinus Torvalds { 194419809c2dSMichal Hocko return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL_EXEC, 194500ef2d2fSDavid Rientjes NUMA_NO_NODE, __builtin_return_address(0)); 19461da177e4SLinus Torvalds } 19471da177e4SLinus Torvalds 19480d08e0d3SAndi Kleen #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) 19497ac674f5SBenjamin Herrenschmidt #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL 19500d08e0d3SAndi Kleen #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) 19517ac674f5SBenjamin Herrenschmidt #define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL 19520d08e0d3SAndi Kleen #else 19530d08e0d3SAndi Kleen #define GFP_VMALLOC32 GFP_KERNEL 19540d08e0d3SAndi Kleen #endif 19550d08e0d3SAndi Kleen 19561da177e4SLinus Torvalds /** 19571da177e4SLinus Torvalds * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 19581da177e4SLinus Torvalds * @size: allocation size 19591da177e4SLinus Torvalds * 19601da177e4SLinus Torvalds * Allocate enough 32bit PA addressable pages to cover @size from the 19611da177e4SLinus Torvalds * page level allocator and map them into contiguous kernel virtual space. 19621da177e4SLinus Torvalds */ 19631da177e4SLinus Torvalds void *vmalloc_32(unsigned long size) 19641da177e4SLinus Torvalds { 19652dca6999SDavid Miller return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL, 196600ef2d2fSDavid Rientjes NUMA_NO_NODE, __builtin_return_address(0)); 19671da177e4SLinus Torvalds } 19681da177e4SLinus Torvalds EXPORT_SYMBOL(vmalloc_32); 19691da177e4SLinus Torvalds 197083342314SNick Piggin /** 1971ead04089SRolf Eike Beer * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 197283342314SNick Piggin * @size: allocation size 1973ead04089SRolf Eike Beer * 1974ead04089SRolf Eike Beer * The resulting memory area is 32bit addressable and zeroed so it can be 1975ead04089SRolf Eike Beer * mapped to userspace without leaking data. 197683342314SNick Piggin */ 197783342314SNick Piggin void *vmalloc_32_user(unsigned long size) 197883342314SNick Piggin { 197983342314SNick Piggin struct vm_struct *area; 198083342314SNick Piggin void *ret; 198183342314SNick Piggin 19822dca6999SDavid Miller ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, 198300ef2d2fSDavid Rientjes NUMA_NO_NODE, __builtin_return_address(0)); 19842b4ac44eSEric Dumazet if (ret) { 1985db64fe02SNick Piggin area = find_vm_area(ret); 198683342314SNick Piggin area->flags |= VM_USERMAP; 19872b4ac44eSEric Dumazet } 198883342314SNick Piggin return ret; 198983342314SNick Piggin } 199083342314SNick Piggin EXPORT_SYMBOL(vmalloc_32_user); 199183342314SNick Piggin 1992d0107eb0SKAMEZAWA Hiroyuki /* 1993d0107eb0SKAMEZAWA Hiroyuki * small helper routine , copy contents to buf from addr. 1994d0107eb0SKAMEZAWA Hiroyuki * If the page is not present, fill zero. 1995d0107eb0SKAMEZAWA Hiroyuki */ 1996d0107eb0SKAMEZAWA Hiroyuki 1997d0107eb0SKAMEZAWA Hiroyuki static int aligned_vread(char *buf, char *addr, unsigned long count) 1998d0107eb0SKAMEZAWA Hiroyuki { 1999d0107eb0SKAMEZAWA Hiroyuki struct page *p; 2000d0107eb0SKAMEZAWA Hiroyuki int copied = 0; 2001d0107eb0SKAMEZAWA Hiroyuki 2002d0107eb0SKAMEZAWA Hiroyuki while (count) { 2003d0107eb0SKAMEZAWA Hiroyuki unsigned long offset, length; 2004d0107eb0SKAMEZAWA Hiroyuki 2005891c49abSAlexander Kuleshov offset = offset_in_page(addr); 2006d0107eb0SKAMEZAWA Hiroyuki length = PAGE_SIZE - offset; 2007d0107eb0SKAMEZAWA Hiroyuki if (length > count) 2008d0107eb0SKAMEZAWA Hiroyuki length = count; 2009d0107eb0SKAMEZAWA Hiroyuki p = vmalloc_to_page(addr); 2010d0107eb0SKAMEZAWA Hiroyuki /* 2011d0107eb0SKAMEZAWA Hiroyuki * To do safe access to this _mapped_ area, we need 2012d0107eb0SKAMEZAWA Hiroyuki * lock. But adding lock here means that we need to add 2013d0107eb0SKAMEZAWA Hiroyuki * overhead of vmalloc()/vfree() calles for this _debug_ 2014d0107eb0SKAMEZAWA Hiroyuki * interface, rarely used. Instead of that, we'll use 2015d0107eb0SKAMEZAWA Hiroyuki * kmap() and get small overhead in this access function. 2016d0107eb0SKAMEZAWA Hiroyuki */ 2017d0107eb0SKAMEZAWA Hiroyuki if (p) { 2018d0107eb0SKAMEZAWA Hiroyuki /* 2019d0107eb0SKAMEZAWA Hiroyuki * we can expect USER0 is not used (see vread/vwrite's 2020d0107eb0SKAMEZAWA Hiroyuki * function description) 2021d0107eb0SKAMEZAWA Hiroyuki */ 20229b04c5feSCong Wang void *map = kmap_atomic(p); 2023d0107eb0SKAMEZAWA Hiroyuki memcpy(buf, map + offset, length); 20249b04c5feSCong Wang kunmap_atomic(map); 2025d0107eb0SKAMEZAWA Hiroyuki } else 2026d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, length); 2027d0107eb0SKAMEZAWA Hiroyuki 2028d0107eb0SKAMEZAWA Hiroyuki addr += length; 2029d0107eb0SKAMEZAWA Hiroyuki buf += length; 2030d0107eb0SKAMEZAWA Hiroyuki copied += length; 2031d0107eb0SKAMEZAWA Hiroyuki count -= length; 2032d0107eb0SKAMEZAWA Hiroyuki } 2033d0107eb0SKAMEZAWA Hiroyuki return copied; 2034d0107eb0SKAMEZAWA Hiroyuki } 2035d0107eb0SKAMEZAWA Hiroyuki 2036d0107eb0SKAMEZAWA Hiroyuki static int aligned_vwrite(char *buf, char *addr, unsigned long count) 2037d0107eb0SKAMEZAWA Hiroyuki { 2038d0107eb0SKAMEZAWA Hiroyuki struct page *p; 2039d0107eb0SKAMEZAWA Hiroyuki int copied = 0; 2040d0107eb0SKAMEZAWA Hiroyuki 2041d0107eb0SKAMEZAWA Hiroyuki while (count) { 2042d0107eb0SKAMEZAWA Hiroyuki unsigned long offset, length; 2043d0107eb0SKAMEZAWA Hiroyuki 2044891c49abSAlexander Kuleshov offset = offset_in_page(addr); 2045d0107eb0SKAMEZAWA Hiroyuki length = PAGE_SIZE - offset; 2046d0107eb0SKAMEZAWA Hiroyuki if (length > count) 2047d0107eb0SKAMEZAWA Hiroyuki length = count; 2048d0107eb0SKAMEZAWA Hiroyuki p = vmalloc_to_page(addr); 2049d0107eb0SKAMEZAWA Hiroyuki /* 2050d0107eb0SKAMEZAWA Hiroyuki * To do safe access to this _mapped_ area, we need 2051d0107eb0SKAMEZAWA Hiroyuki * lock. But adding lock here means that we need to add 2052d0107eb0SKAMEZAWA Hiroyuki * overhead of vmalloc()/vfree() calles for this _debug_ 2053d0107eb0SKAMEZAWA Hiroyuki * interface, rarely used. Instead of that, we'll use 2054d0107eb0SKAMEZAWA Hiroyuki * kmap() and get small overhead in this access function. 2055d0107eb0SKAMEZAWA Hiroyuki */ 2056d0107eb0SKAMEZAWA Hiroyuki if (p) { 2057d0107eb0SKAMEZAWA Hiroyuki /* 2058d0107eb0SKAMEZAWA Hiroyuki * we can expect USER0 is not used (see vread/vwrite's 2059d0107eb0SKAMEZAWA Hiroyuki * function description) 2060d0107eb0SKAMEZAWA Hiroyuki */ 20619b04c5feSCong Wang void *map = kmap_atomic(p); 2062d0107eb0SKAMEZAWA Hiroyuki memcpy(map + offset, buf, length); 20639b04c5feSCong Wang kunmap_atomic(map); 2064d0107eb0SKAMEZAWA Hiroyuki } 2065d0107eb0SKAMEZAWA Hiroyuki addr += length; 2066d0107eb0SKAMEZAWA Hiroyuki buf += length; 2067d0107eb0SKAMEZAWA Hiroyuki copied += length; 2068d0107eb0SKAMEZAWA Hiroyuki count -= length; 2069d0107eb0SKAMEZAWA Hiroyuki } 2070d0107eb0SKAMEZAWA Hiroyuki return copied; 2071d0107eb0SKAMEZAWA Hiroyuki } 2072d0107eb0SKAMEZAWA Hiroyuki 2073d0107eb0SKAMEZAWA Hiroyuki /** 2074d0107eb0SKAMEZAWA Hiroyuki * vread() - read vmalloc area in a safe way. 2075d0107eb0SKAMEZAWA Hiroyuki * @buf: buffer for reading data 2076d0107eb0SKAMEZAWA Hiroyuki * @addr: vm address. 2077d0107eb0SKAMEZAWA Hiroyuki * @count: number of bytes to be read. 2078d0107eb0SKAMEZAWA Hiroyuki * 2079d0107eb0SKAMEZAWA Hiroyuki * Returns # of bytes which addr and buf should be increased. 2080d0107eb0SKAMEZAWA Hiroyuki * (same number to @count). Returns 0 if [addr...addr+count) doesn't 2081d0107eb0SKAMEZAWA Hiroyuki * includes any intersect with alive vmalloc area. 2082d0107eb0SKAMEZAWA Hiroyuki * 2083d0107eb0SKAMEZAWA Hiroyuki * This function checks that addr is a valid vmalloc'ed area, and 2084d0107eb0SKAMEZAWA Hiroyuki * copy data from that area to a given buffer. If the given memory range 2085d0107eb0SKAMEZAWA Hiroyuki * of [addr...addr+count) includes some valid address, data is copied to 2086d0107eb0SKAMEZAWA Hiroyuki * proper area of @buf. If there are memory holes, they'll be zero-filled. 2087d0107eb0SKAMEZAWA Hiroyuki * IOREMAP area is treated as memory hole and no copy is done. 2088d0107eb0SKAMEZAWA Hiroyuki * 2089d0107eb0SKAMEZAWA Hiroyuki * If [addr...addr+count) doesn't includes any intersects with alive 2090a8e5202dSCong Wang * vm_struct area, returns 0. @buf should be kernel's buffer. 2091d0107eb0SKAMEZAWA Hiroyuki * 2092d0107eb0SKAMEZAWA Hiroyuki * Note: In usual ops, vread() is never necessary because the caller 2093d0107eb0SKAMEZAWA Hiroyuki * should know vmalloc() area is valid and can use memcpy(). 2094d0107eb0SKAMEZAWA Hiroyuki * This is for routines which have to access vmalloc area without 2095d0107eb0SKAMEZAWA Hiroyuki * any informaion, as /dev/kmem. 2096d0107eb0SKAMEZAWA Hiroyuki * 2097d0107eb0SKAMEZAWA Hiroyuki */ 2098d0107eb0SKAMEZAWA Hiroyuki 20991da177e4SLinus Torvalds long vread(char *buf, char *addr, unsigned long count) 21001da177e4SLinus Torvalds { 2101e81ce85fSJoonsoo Kim struct vmap_area *va; 2102e81ce85fSJoonsoo Kim struct vm_struct *vm; 21031da177e4SLinus Torvalds char *vaddr, *buf_start = buf; 2104d0107eb0SKAMEZAWA Hiroyuki unsigned long buflen = count; 21051da177e4SLinus Torvalds unsigned long n; 21061da177e4SLinus Torvalds 21071da177e4SLinus Torvalds /* Don't allow overflow */ 21081da177e4SLinus Torvalds if ((unsigned long) addr + count < count) 21091da177e4SLinus Torvalds count = -(unsigned long) addr; 21101da177e4SLinus Torvalds 2111e81ce85fSJoonsoo Kim spin_lock(&vmap_area_lock); 2112e81ce85fSJoonsoo Kim list_for_each_entry(va, &vmap_area_list, list) { 2113e81ce85fSJoonsoo Kim if (!count) 2114e81ce85fSJoonsoo Kim break; 2115e81ce85fSJoonsoo Kim 2116e81ce85fSJoonsoo Kim if (!(va->flags & VM_VM_AREA)) 2117e81ce85fSJoonsoo Kim continue; 2118e81ce85fSJoonsoo Kim 2119e81ce85fSJoonsoo Kim vm = va->vm; 2120e81ce85fSJoonsoo Kim vaddr = (char *) vm->addr; 2121762216abSWanpeng Li if (addr >= vaddr + get_vm_area_size(vm)) 21221da177e4SLinus Torvalds continue; 21231da177e4SLinus Torvalds while (addr < vaddr) { 21241da177e4SLinus Torvalds if (count == 0) 21251da177e4SLinus Torvalds goto finished; 21261da177e4SLinus Torvalds *buf = '\0'; 21271da177e4SLinus Torvalds buf++; 21281da177e4SLinus Torvalds addr++; 21291da177e4SLinus Torvalds count--; 21301da177e4SLinus Torvalds } 2131762216abSWanpeng Li n = vaddr + get_vm_area_size(vm) - addr; 2132d0107eb0SKAMEZAWA Hiroyuki if (n > count) 2133d0107eb0SKAMEZAWA Hiroyuki n = count; 2134e81ce85fSJoonsoo Kim if (!(vm->flags & VM_IOREMAP)) 2135d0107eb0SKAMEZAWA Hiroyuki aligned_vread(buf, addr, n); 2136d0107eb0SKAMEZAWA Hiroyuki else /* IOREMAP area is treated as memory hole */ 2137d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, n); 2138d0107eb0SKAMEZAWA Hiroyuki buf += n; 2139d0107eb0SKAMEZAWA Hiroyuki addr += n; 2140d0107eb0SKAMEZAWA Hiroyuki count -= n; 21411da177e4SLinus Torvalds } 21421da177e4SLinus Torvalds finished: 2143e81ce85fSJoonsoo Kim spin_unlock(&vmap_area_lock); 2144d0107eb0SKAMEZAWA Hiroyuki 2145d0107eb0SKAMEZAWA Hiroyuki if (buf == buf_start) 2146d0107eb0SKAMEZAWA Hiroyuki return 0; 2147d0107eb0SKAMEZAWA Hiroyuki /* zero-fill memory holes */ 2148d0107eb0SKAMEZAWA Hiroyuki if (buf != buf_start + buflen) 2149d0107eb0SKAMEZAWA Hiroyuki memset(buf, 0, buflen - (buf - buf_start)); 2150d0107eb0SKAMEZAWA Hiroyuki 2151d0107eb0SKAMEZAWA Hiroyuki return buflen; 21521da177e4SLinus Torvalds } 21531da177e4SLinus Torvalds 2154d0107eb0SKAMEZAWA Hiroyuki /** 2155d0107eb0SKAMEZAWA Hiroyuki * vwrite() - write vmalloc area in a safe way. 2156d0107eb0SKAMEZAWA Hiroyuki * @buf: buffer for source data 2157d0107eb0SKAMEZAWA Hiroyuki * @addr: vm address. 2158d0107eb0SKAMEZAWA Hiroyuki * @count: number of bytes to be read. 2159d0107eb0SKAMEZAWA Hiroyuki * 2160d0107eb0SKAMEZAWA Hiroyuki * Returns # of bytes which addr and buf should be incresed. 2161d0107eb0SKAMEZAWA Hiroyuki * (same number to @count). 2162d0107eb0SKAMEZAWA Hiroyuki * If [addr...addr+count) doesn't includes any intersect with valid 2163d0107eb0SKAMEZAWA Hiroyuki * vmalloc area, returns 0. 2164d0107eb0SKAMEZAWA Hiroyuki * 2165d0107eb0SKAMEZAWA Hiroyuki * This function checks that addr is a valid vmalloc'ed area, and 2166d0107eb0SKAMEZAWA Hiroyuki * copy data from a buffer to the given addr. If specified range of 2167d0107eb0SKAMEZAWA Hiroyuki * [addr...addr+count) includes some valid address, data is copied from 2168d0107eb0SKAMEZAWA Hiroyuki * proper area of @buf. If there are memory holes, no copy to hole. 2169d0107eb0SKAMEZAWA Hiroyuki * IOREMAP area is treated as memory hole and no copy is done. 2170d0107eb0SKAMEZAWA Hiroyuki * 2171d0107eb0SKAMEZAWA Hiroyuki * If [addr...addr+count) doesn't includes any intersects with alive 2172a8e5202dSCong Wang * vm_struct area, returns 0. @buf should be kernel's buffer. 2173d0107eb0SKAMEZAWA Hiroyuki * 2174d0107eb0SKAMEZAWA Hiroyuki * Note: In usual ops, vwrite() is never necessary because the caller 2175d0107eb0SKAMEZAWA Hiroyuki * should know vmalloc() area is valid and can use memcpy(). 2176d0107eb0SKAMEZAWA Hiroyuki * This is for routines which have to access vmalloc area without 2177d0107eb0SKAMEZAWA Hiroyuki * any informaion, as /dev/kmem. 2178d0107eb0SKAMEZAWA Hiroyuki */ 2179d0107eb0SKAMEZAWA Hiroyuki 21801da177e4SLinus Torvalds long vwrite(char *buf, char *addr, unsigned long count) 21811da177e4SLinus Torvalds { 2182e81ce85fSJoonsoo Kim struct vmap_area *va; 2183e81ce85fSJoonsoo Kim struct vm_struct *vm; 2184d0107eb0SKAMEZAWA Hiroyuki char *vaddr; 2185d0107eb0SKAMEZAWA Hiroyuki unsigned long n, buflen; 2186d0107eb0SKAMEZAWA Hiroyuki int copied = 0; 21871da177e4SLinus Torvalds 21881da177e4SLinus Torvalds /* Don't allow overflow */ 21891da177e4SLinus Torvalds if ((unsigned long) addr + count < count) 21901da177e4SLinus Torvalds count = -(unsigned long) addr; 2191d0107eb0SKAMEZAWA Hiroyuki buflen = count; 21921da177e4SLinus Torvalds 2193e81ce85fSJoonsoo Kim spin_lock(&vmap_area_lock); 2194e81ce85fSJoonsoo Kim list_for_each_entry(va, &vmap_area_list, list) { 2195e81ce85fSJoonsoo Kim if (!count) 2196e81ce85fSJoonsoo Kim break; 2197e81ce85fSJoonsoo Kim 2198e81ce85fSJoonsoo Kim if (!(va->flags & VM_VM_AREA)) 2199e81ce85fSJoonsoo Kim continue; 2200e81ce85fSJoonsoo Kim 2201e81ce85fSJoonsoo Kim vm = va->vm; 2202e81ce85fSJoonsoo Kim vaddr = (char *) vm->addr; 2203762216abSWanpeng Li if (addr >= vaddr + get_vm_area_size(vm)) 22041da177e4SLinus Torvalds continue; 22051da177e4SLinus Torvalds while (addr < vaddr) { 22061da177e4SLinus Torvalds if (count == 0) 22071da177e4SLinus Torvalds goto finished; 22081da177e4SLinus Torvalds buf++; 22091da177e4SLinus Torvalds addr++; 22101da177e4SLinus Torvalds count--; 22111da177e4SLinus Torvalds } 2212762216abSWanpeng Li n = vaddr + get_vm_area_size(vm) - addr; 2213d0107eb0SKAMEZAWA Hiroyuki if (n > count) 2214d0107eb0SKAMEZAWA Hiroyuki n = count; 2215e81ce85fSJoonsoo Kim if (!(vm->flags & VM_IOREMAP)) { 2216d0107eb0SKAMEZAWA Hiroyuki aligned_vwrite(buf, addr, n); 2217d0107eb0SKAMEZAWA Hiroyuki copied++; 2218d0107eb0SKAMEZAWA Hiroyuki } 2219d0107eb0SKAMEZAWA Hiroyuki buf += n; 2220d0107eb0SKAMEZAWA Hiroyuki addr += n; 2221d0107eb0SKAMEZAWA Hiroyuki count -= n; 22221da177e4SLinus Torvalds } 22231da177e4SLinus Torvalds finished: 2224e81ce85fSJoonsoo Kim spin_unlock(&vmap_area_lock); 2225d0107eb0SKAMEZAWA Hiroyuki if (!copied) 2226d0107eb0SKAMEZAWA Hiroyuki return 0; 2227d0107eb0SKAMEZAWA Hiroyuki return buflen; 22281da177e4SLinus Torvalds } 222983342314SNick Piggin 223083342314SNick Piggin /** 2231e69e9d4aSHATAYAMA Daisuke * remap_vmalloc_range_partial - map vmalloc pages to userspace 2232e69e9d4aSHATAYAMA Daisuke * @vma: vma to cover 2233e69e9d4aSHATAYAMA Daisuke * @uaddr: target user address to start at 2234e69e9d4aSHATAYAMA Daisuke * @kaddr: virtual address of vmalloc kernel memory 2235e69e9d4aSHATAYAMA Daisuke * @size: size of map area 2236e69e9d4aSHATAYAMA Daisuke * 2237e69e9d4aSHATAYAMA Daisuke * Returns: 0 for success, -Exxx on failure 2238e69e9d4aSHATAYAMA Daisuke * 2239e69e9d4aSHATAYAMA Daisuke * This function checks that @kaddr is a valid vmalloc'ed area, 2240e69e9d4aSHATAYAMA Daisuke * and that it is big enough to cover the range starting at 2241e69e9d4aSHATAYAMA Daisuke * @uaddr in @vma. Will return failure if that criteria isn't 2242e69e9d4aSHATAYAMA Daisuke * met. 2243e69e9d4aSHATAYAMA Daisuke * 2244e69e9d4aSHATAYAMA Daisuke * Similar to remap_pfn_range() (see mm/memory.c) 2245e69e9d4aSHATAYAMA Daisuke */ 2246e69e9d4aSHATAYAMA Daisuke int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr, 2247e69e9d4aSHATAYAMA Daisuke void *kaddr, unsigned long size) 2248e69e9d4aSHATAYAMA Daisuke { 2249e69e9d4aSHATAYAMA Daisuke struct vm_struct *area; 2250e69e9d4aSHATAYAMA Daisuke 2251e69e9d4aSHATAYAMA Daisuke size = PAGE_ALIGN(size); 2252e69e9d4aSHATAYAMA Daisuke 2253e69e9d4aSHATAYAMA Daisuke if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr)) 2254e69e9d4aSHATAYAMA Daisuke return -EINVAL; 2255e69e9d4aSHATAYAMA Daisuke 2256e69e9d4aSHATAYAMA Daisuke area = find_vm_area(kaddr); 2257e69e9d4aSHATAYAMA Daisuke if (!area) 2258e69e9d4aSHATAYAMA Daisuke return -EINVAL; 2259e69e9d4aSHATAYAMA Daisuke 2260e69e9d4aSHATAYAMA Daisuke if (!(area->flags & VM_USERMAP)) 2261e69e9d4aSHATAYAMA Daisuke return -EINVAL; 2262e69e9d4aSHATAYAMA Daisuke 2263e69e9d4aSHATAYAMA Daisuke if (kaddr + size > area->addr + area->size) 2264e69e9d4aSHATAYAMA Daisuke return -EINVAL; 2265e69e9d4aSHATAYAMA Daisuke 2266e69e9d4aSHATAYAMA Daisuke do { 2267e69e9d4aSHATAYAMA Daisuke struct page *page = vmalloc_to_page(kaddr); 2268e69e9d4aSHATAYAMA Daisuke int ret; 2269e69e9d4aSHATAYAMA Daisuke 2270e69e9d4aSHATAYAMA Daisuke ret = vm_insert_page(vma, uaddr, page); 2271e69e9d4aSHATAYAMA Daisuke if (ret) 2272e69e9d4aSHATAYAMA Daisuke return ret; 2273e69e9d4aSHATAYAMA Daisuke 2274e69e9d4aSHATAYAMA Daisuke uaddr += PAGE_SIZE; 2275e69e9d4aSHATAYAMA Daisuke kaddr += PAGE_SIZE; 2276e69e9d4aSHATAYAMA Daisuke size -= PAGE_SIZE; 2277e69e9d4aSHATAYAMA Daisuke } while (size > 0); 2278e69e9d4aSHATAYAMA Daisuke 2279e69e9d4aSHATAYAMA Daisuke vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; 2280e69e9d4aSHATAYAMA Daisuke 2281e69e9d4aSHATAYAMA Daisuke return 0; 2282e69e9d4aSHATAYAMA Daisuke } 2283e69e9d4aSHATAYAMA Daisuke EXPORT_SYMBOL(remap_vmalloc_range_partial); 2284e69e9d4aSHATAYAMA Daisuke 2285e69e9d4aSHATAYAMA Daisuke /** 228683342314SNick Piggin * remap_vmalloc_range - map vmalloc pages to userspace 228783342314SNick Piggin * @vma: vma to cover (map full range of vma) 228883342314SNick Piggin * @addr: vmalloc memory 228983342314SNick Piggin * @pgoff: number of pages into addr before first page to map 22907682486bSRandy Dunlap * 22917682486bSRandy Dunlap * Returns: 0 for success, -Exxx on failure 229283342314SNick Piggin * 229383342314SNick Piggin * This function checks that addr is a valid vmalloc'ed area, and 229483342314SNick Piggin * that it is big enough to cover the vma. Will return failure if 229583342314SNick Piggin * that criteria isn't met. 229683342314SNick Piggin * 229772fd4a35SRobert P. J. Day * Similar to remap_pfn_range() (see mm/memory.c) 229883342314SNick Piggin */ 229983342314SNick Piggin int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 230083342314SNick Piggin unsigned long pgoff) 230183342314SNick Piggin { 2302e69e9d4aSHATAYAMA Daisuke return remap_vmalloc_range_partial(vma, vma->vm_start, 2303e69e9d4aSHATAYAMA Daisuke addr + (pgoff << PAGE_SHIFT), 2304e69e9d4aSHATAYAMA Daisuke vma->vm_end - vma->vm_start); 230583342314SNick Piggin } 230683342314SNick Piggin EXPORT_SYMBOL(remap_vmalloc_range); 230783342314SNick Piggin 23081eeb66a1SChristoph Hellwig /* 23091eeb66a1SChristoph Hellwig * Implement a stub for vmalloc_sync_all() if the architecture chose not to 23101eeb66a1SChristoph Hellwig * have one. 23111eeb66a1SChristoph Hellwig */ 23123b32123dSGideon Israel Dsouza void __weak vmalloc_sync_all(void) 23131eeb66a1SChristoph Hellwig { 23141eeb66a1SChristoph Hellwig } 23155f4352fbSJeremy Fitzhardinge 23165f4352fbSJeremy Fitzhardinge 23172f569afdSMartin Schwidefsky static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data) 23185f4352fbSJeremy Fitzhardinge { 2319cd12909cSDavid Vrabel pte_t ***p = data; 2320cd12909cSDavid Vrabel 2321cd12909cSDavid Vrabel if (p) { 2322cd12909cSDavid Vrabel *(*p) = pte; 2323cd12909cSDavid Vrabel (*p)++; 2324cd12909cSDavid Vrabel } 23255f4352fbSJeremy Fitzhardinge return 0; 23265f4352fbSJeremy Fitzhardinge } 23275f4352fbSJeremy Fitzhardinge 23285f4352fbSJeremy Fitzhardinge /** 23295f4352fbSJeremy Fitzhardinge * alloc_vm_area - allocate a range of kernel address space 23305f4352fbSJeremy Fitzhardinge * @size: size of the area 2331cd12909cSDavid Vrabel * @ptes: returns the PTEs for the address space 23327682486bSRandy Dunlap * 23337682486bSRandy Dunlap * Returns: NULL on failure, vm_struct on success 23345f4352fbSJeremy Fitzhardinge * 23355f4352fbSJeremy Fitzhardinge * This function reserves a range of kernel address space, and 23365f4352fbSJeremy Fitzhardinge * allocates pagetables to map that range. No actual mappings 2337cd12909cSDavid Vrabel * are created. 2338cd12909cSDavid Vrabel * 2339cd12909cSDavid Vrabel * If @ptes is non-NULL, pointers to the PTEs (in init_mm) 2340cd12909cSDavid Vrabel * allocated for the VM area are returned. 23415f4352fbSJeremy Fitzhardinge */ 2342cd12909cSDavid Vrabel struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes) 23435f4352fbSJeremy Fitzhardinge { 23445f4352fbSJeremy Fitzhardinge struct vm_struct *area; 23455f4352fbSJeremy Fitzhardinge 234623016969SChristoph Lameter area = get_vm_area_caller(size, VM_IOREMAP, 234723016969SChristoph Lameter __builtin_return_address(0)); 23485f4352fbSJeremy Fitzhardinge if (area == NULL) 23495f4352fbSJeremy Fitzhardinge return NULL; 23505f4352fbSJeremy Fitzhardinge 23515f4352fbSJeremy Fitzhardinge /* 23525f4352fbSJeremy Fitzhardinge * This ensures that page tables are constructed for this region 23535f4352fbSJeremy Fitzhardinge * of kernel virtual address space and mapped into init_mm. 23545f4352fbSJeremy Fitzhardinge */ 23555f4352fbSJeremy Fitzhardinge if (apply_to_page_range(&init_mm, (unsigned long)area->addr, 2356cd12909cSDavid Vrabel size, f, ptes ? &ptes : NULL)) { 23575f4352fbSJeremy Fitzhardinge free_vm_area(area); 23585f4352fbSJeremy Fitzhardinge return NULL; 23595f4352fbSJeremy Fitzhardinge } 23605f4352fbSJeremy Fitzhardinge 23615f4352fbSJeremy Fitzhardinge return area; 23625f4352fbSJeremy Fitzhardinge } 23635f4352fbSJeremy Fitzhardinge EXPORT_SYMBOL_GPL(alloc_vm_area); 23645f4352fbSJeremy Fitzhardinge 23655f4352fbSJeremy Fitzhardinge void free_vm_area(struct vm_struct *area) 23665f4352fbSJeremy Fitzhardinge { 23675f4352fbSJeremy Fitzhardinge struct vm_struct *ret; 23685f4352fbSJeremy Fitzhardinge ret = remove_vm_area(area->addr); 23695f4352fbSJeremy Fitzhardinge BUG_ON(ret != area); 23705f4352fbSJeremy Fitzhardinge kfree(area); 23715f4352fbSJeremy Fitzhardinge } 23725f4352fbSJeremy Fitzhardinge EXPORT_SYMBOL_GPL(free_vm_area); 2373a10aa579SChristoph Lameter 23744f8b02b4STejun Heo #ifdef CONFIG_SMP 2375ca23e405STejun Heo static struct vmap_area *node_to_va(struct rb_node *n) 2376ca23e405STejun Heo { 23774583e773SGeliang Tang return rb_entry_safe(n, struct vmap_area, rb_node); 2378ca23e405STejun Heo } 2379ca23e405STejun Heo 2380ca23e405STejun Heo /** 2381ca23e405STejun Heo * pvm_find_next_prev - find the next and prev vmap_area surrounding @end 2382ca23e405STejun Heo * @end: target address 2383ca23e405STejun Heo * @pnext: out arg for the next vmap_area 2384ca23e405STejun Heo * @pprev: out arg for the previous vmap_area 2385ca23e405STejun Heo * 2386ca23e405STejun Heo * Returns: %true if either or both of next and prev are found, 2387ca23e405STejun Heo * %false if no vmap_area exists 2388ca23e405STejun Heo * 2389ca23e405STejun Heo * Find vmap_areas end addresses of which enclose @end. ie. if not 2390ca23e405STejun Heo * NULL, *pnext->va_end > @end and *pprev->va_end <= @end. 2391ca23e405STejun Heo */ 2392ca23e405STejun Heo static bool pvm_find_next_prev(unsigned long end, 2393ca23e405STejun Heo struct vmap_area **pnext, 2394ca23e405STejun Heo struct vmap_area **pprev) 2395ca23e405STejun Heo { 2396ca23e405STejun Heo struct rb_node *n = vmap_area_root.rb_node; 2397ca23e405STejun Heo struct vmap_area *va = NULL; 2398ca23e405STejun Heo 2399ca23e405STejun Heo while (n) { 2400ca23e405STejun Heo va = rb_entry(n, struct vmap_area, rb_node); 2401ca23e405STejun Heo if (end < va->va_end) 2402ca23e405STejun Heo n = n->rb_left; 2403ca23e405STejun Heo else if (end > va->va_end) 2404ca23e405STejun Heo n = n->rb_right; 2405ca23e405STejun Heo else 2406ca23e405STejun Heo break; 2407ca23e405STejun Heo } 2408ca23e405STejun Heo 2409ca23e405STejun Heo if (!va) 2410ca23e405STejun Heo return false; 2411ca23e405STejun Heo 2412ca23e405STejun Heo if (va->va_end > end) { 2413ca23e405STejun Heo *pnext = va; 2414ca23e405STejun Heo *pprev = node_to_va(rb_prev(&(*pnext)->rb_node)); 2415ca23e405STejun Heo } else { 2416ca23e405STejun Heo *pprev = va; 2417ca23e405STejun Heo *pnext = node_to_va(rb_next(&(*pprev)->rb_node)); 2418ca23e405STejun Heo } 2419ca23e405STejun Heo return true; 2420ca23e405STejun Heo } 2421ca23e405STejun Heo 2422ca23e405STejun Heo /** 2423ca23e405STejun Heo * pvm_determine_end - find the highest aligned address between two vmap_areas 2424ca23e405STejun Heo * @pnext: in/out arg for the next vmap_area 2425ca23e405STejun Heo * @pprev: in/out arg for the previous vmap_area 2426ca23e405STejun Heo * @align: alignment 2427ca23e405STejun Heo * 2428ca23e405STejun Heo * Returns: determined end address 2429ca23e405STejun Heo * 2430ca23e405STejun Heo * Find the highest aligned address between *@pnext and *@pprev below 2431ca23e405STejun Heo * VMALLOC_END. *@pnext and *@pprev are adjusted so that the aligned 2432ca23e405STejun Heo * down address is between the end addresses of the two vmap_areas. 2433ca23e405STejun Heo * 2434ca23e405STejun Heo * Please note that the address returned by this function may fall 2435ca23e405STejun Heo * inside *@pnext vmap_area. The caller is responsible for checking 2436ca23e405STejun Heo * that. 2437ca23e405STejun Heo */ 2438ca23e405STejun Heo static unsigned long pvm_determine_end(struct vmap_area **pnext, 2439ca23e405STejun Heo struct vmap_area **pprev, 2440ca23e405STejun Heo unsigned long align) 2441ca23e405STejun Heo { 2442ca23e405STejun Heo const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 2443ca23e405STejun Heo unsigned long addr; 2444ca23e405STejun Heo 2445ca23e405STejun Heo if (*pnext) 2446ca23e405STejun Heo addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end); 2447ca23e405STejun Heo else 2448ca23e405STejun Heo addr = vmalloc_end; 2449ca23e405STejun Heo 2450ca23e405STejun Heo while (*pprev && (*pprev)->va_end > addr) { 2451ca23e405STejun Heo *pnext = *pprev; 2452ca23e405STejun Heo *pprev = node_to_va(rb_prev(&(*pnext)->rb_node)); 2453ca23e405STejun Heo } 2454ca23e405STejun Heo 2455ca23e405STejun Heo return addr; 2456ca23e405STejun Heo } 2457ca23e405STejun Heo 2458ca23e405STejun Heo /** 2459ca23e405STejun Heo * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator 2460ca23e405STejun Heo * @offsets: array containing offset of each area 2461ca23e405STejun Heo * @sizes: array containing size of each area 2462ca23e405STejun Heo * @nr_vms: the number of areas to allocate 2463ca23e405STejun Heo * @align: alignment, all entries in @offsets and @sizes must be aligned to this 2464ca23e405STejun Heo * 2465ca23e405STejun Heo * Returns: kmalloc'd vm_struct pointer array pointing to allocated 2466ca23e405STejun Heo * vm_structs on success, %NULL on failure 2467ca23e405STejun Heo * 2468ca23e405STejun Heo * Percpu allocator wants to use congruent vm areas so that it can 2469ca23e405STejun Heo * maintain the offsets among percpu areas. This function allocates 2470ec3f64fcSDavid Rientjes * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to 2471ec3f64fcSDavid Rientjes * be scattered pretty far, distance between two areas easily going up 2472ec3f64fcSDavid Rientjes * to gigabytes. To avoid interacting with regular vmallocs, these 2473ec3f64fcSDavid Rientjes * areas are allocated from top. 2474ca23e405STejun Heo * 2475ca23e405STejun Heo * Despite its complicated look, this allocator is rather simple. It 2476ca23e405STejun Heo * does everything top-down and scans areas from the end looking for 2477ca23e405STejun Heo * matching slot. While scanning, if any of the areas overlaps with 2478ca23e405STejun Heo * existing vmap_area, the base address is pulled down to fit the 2479ca23e405STejun Heo * area. Scanning is repeated till all the areas fit and then all 2480ca23e405STejun Heo * necessary data structres are inserted and the result is returned. 2481ca23e405STejun Heo */ 2482ca23e405STejun Heo struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, 2483ca23e405STejun Heo const size_t *sizes, int nr_vms, 2484ec3f64fcSDavid Rientjes size_t align) 2485ca23e405STejun Heo { 2486ca23e405STejun Heo const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); 2487ca23e405STejun Heo const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 2488ca23e405STejun Heo struct vmap_area **vas, *prev, *next; 2489ca23e405STejun Heo struct vm_struct **vms; 2490ca23e405STejun Heo int area, area2, last_area, term_area; 2491ca23e405STejun Heo unsigned long base, start, end, last_end; 2492ca23e405STejun Heo bool purged = false; 2493ca23e405STejun Heo 2494ca23e405STejun Heo /* verify parameters and allocate data structures */ 2495891c49abSAlexander Kuleshov BUG_ON(offset_in_page(align) || !is_power_of_2(align)); 2496ca23e405STejun Heo for (last_area = 0, area = 0; area < nr_vms; area++) { 2497ca23e405STejun Heo start = offsets[area]; 2498ca23e405STejun Heo end = start + sizes[area]; 2499ca23e405STejun Heo 2500ca23e405STejun Heo /* is everything aligned properly? */ 2501ca23e405STejun Heo BUG_ON(!IS_ALIGNED(offsets[area], align)); 2502ca23e405STejun Heo BUG_ON(!IS_ALIGNED(sizes[area], align)); 2503ca23e405STejun Heo 2504ca23e405STejun Heo /* detect the area with the highest address */ 2505ca23e405STejun Heo if (start > offsets[last_area]) 2506ca23e405STejun Heo last_area = area; 2507ca23e405STejun Heo 2508ca23e405STejun Heo for (area2 = 0; area2 < nr_vms; area2++) { 2509ca23e405STejun Heo unsigned long start2 = offsets[area2]; 2510ca23e405STejun Heo unsigned long end2 = start2 + sizes[area2]; 2511ca23e405STejun Heo 2512ca23e405STejun Heo if (area2 == area) 2513ca23e405STejun Heo continue; 2514ca23e405STejun Heo 2515ca23e405STejun Heo BUG_ON(start2 >= start && start2 < end); 2516ca23e405STejun Heo BUG_ON(end2 <= end && end2 > start); 2517ca23e405STejun Heo } 2518ca23e405STejun Heo } 2519ca23e405STejun Heo last_end = offsets[last_area] + sizes[last_area]; 2520ca23e405STejun Heo 2521ca23e405STejun Heo if (vmalloc_end - vmalloc_start < last_end) { 2522ca23e405STejun Heo WARN_ON(true); 2523ca23e405STejun Heo return NULL; 2524ca23e405STejun Heo } 2525ca23e405STejun Heo 25264d67d860SThomas Meyer vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL); 25274d67d860SThomas Meyer vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL); 2528ca23e405STejun Heo if (!vas || !vms) 2529f1db7afdSKautuk Consul goto err_free2; 2530ca23e405STejun Heo 2531ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 2532ec3f64fcSDavid Rientjes vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL); 2533ec3f64fcSDavid Rientjes vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL); 2534ca23e405STejun Heo if (!vas[area] || !vms[area]) 2535ca23e405STejun Heo goto err_free; 2536ca23e405STejun Heo } 2537ca23e405STejun Heo retry: 2538ca23e405STejun Heo spin_lock(&vmap_area_lock); 2539ca23e405STejun Heo 2540ca23e405STejun Heo /* start scanning - we scan from the top, begin with the last area */ 2541ca23e405STejun Heo area = term_area = last_area; 2542ca23e405STejun Heo start = offsets[area]; 2543ca23e405STejun Heo end = start + sizes[area]; 2544ca23e405STejun Heo 2545ca23e405STejun Heo if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) { 2546ca23e405STejun Heo base = vmalloc_end - last_end; 2547ca23e405STejun Heo goto found; 2548ca23e405STejun Heo } 2549ca23e405STejun Heo base = pvm_determine_end(&next, &prev, align) - end; 2550ca23e405STejun Heo 2551ca23e405STejun Heo while (true) { 2552ca23e405STejun Heo BUG_ON(next && next->va_end <= base + end); 2553ca23e405STejun Heo BUG_ON(prev && prev->va_end > base + end); 2554ca23e405STejun Heo 2555ca23e405STejun Heo /* 2556ca23e405STejun Heo * base might have underflowed, add last_end before 2557ca23e405STejun Heo * comparing. 2558ca23e405STejun Heo */ 2559ca23e405STejun Heo if (base + last_end < vmalloc_start + last_end) { 2560ca23e405STejun Heo spin_unlock(&vmap_area_lock); 2561ca23e405STejun Heo if (!purged) { 2562ca23e405STejun Heo purge_vmap_area_lazy(); 2563ca23e405STejun Heo purged = true; 2564ca23e405STejun Heo goto retry; 2565ca23e405STejun Heo } 2566ca23e405STejun Heo goto err_free; 2567ca23e405STejun Heo } 2568ca23e405STejun Heo 2569ca23e405STejun Heo /* 2570ca23e405STejun Heo * If next overlaps, move base downwards so that it's 2571ca23e405STejun Heo * right below next and then recheck. 2572ca23e405STejun Heo */ 2573ca23e405STejun Heo if (next && next->va_start < base + end) { 2574ca23e405STejun Heo base = pvm_determine_end(&next, &prev, align) - end; 2575ca23e405STejun Heo term_area = area; 2576ca23e405STejun Heo continue; 2577ca23e405STejun Heo } 2578ca23e405STejun Heo 2579ca23e405STejun Heo /* 2580ca23e405STejun Heo * If prev overlaps, shift down next and prev and move 2581ca23e405STejun Heo * base so that it's right below new next and then 2582ca23e405STejun Heo * recheck. 2583ca23e405STejun Heo */ 2584ca23e405STejun Heo if (prev && prev->va_end > base + start) { 2585ca23e405STejun Heo next = prev; 2586ca23e405STejun Heo prev = node_to_va(rb_prev(&next->rb_node)); 2587ca23e405STejun Heo base = pvm_determine_end(&next, &prev, align) - end; 2588ca23e405STejun Heo term_area = area; 2589ca23e405STejun Heo continue; 2590ca23e405STejun Heo } 2591ca23e405STejun Heo 2592ca23e405STejun Heo /* 2593ca23e405STejun Heo * This area fits, move on to the previous one. If 2594ca23e405STejun Heo * the previous one is the terminal one, we're done. 2595ca23e405STejun Heo */ 2596ca23e405STejun Heo area = (area + nr_vms - 1) % nr_vms; 2597ca23e405STejun Heo if (area == term_area) 2598ca23e405STejun Heo break; 2599ca23e405STejun Heo start = offsets[area]; 2600ca23e405STejun Heo end = start + sizes[area]; 2601ca23e405STejun Heo pvm_find_next_prev(base + end, &next, &prev); 2602ca23e405STejun Heo } 2603ca23e405STejun Heo found: 2604ca23e405STejun Heo /* we've found a fitting base, insert all va's */ 2605ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 2606ca23e405STejun Heo struct vmap_area *va = vas[area]; 2607ca23e405STejun Heo 2608ca23e405STejun Heo va->va_start = base + offsets[area]; 2609ca23e405STejun Heo va->va_end = va->va_start + sizes[area]; 2610ca23e405STejun Heo __insert_vmap_area(va); 2611ca23e405STejun Heo } 2612ca23e405STejun Heo 2613ca23e405STejun Heo vmap_area_pcpu_hole = base + offsets[last_area]; 2614ca23e405STejun Heo 2615ca23e405STejun Heo spin_unlock(&vmap_area_lock); 2616ca23e405STejun Heo 2617ca23e405STejun Heo /* insert all vm's */ 2618ca23e405STejun Heo for (area = 0; area < nr_vms; area++) 26193645cb4aSZhang Yanfei setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC, 2620ca23e405STejun Heo pcpu_get_vm_areas); 2621ca23e405STejun Heo 2622ca23e405STejun Heo kfree(vas); 2623ca23e405STejun Heo return vms; 2624ca23e405STejun Heo 2625ca23e405STejun Heo err_free: 2626ca23e405STejun Heo for (area = 0; area < nr_vms; area++) { 2627ca23e405STejun Heo kfree(vas[area]); 2628ca23e405STejun Heo kfree(vms[area]); 2629ca23e405STejun Heo } 2630f1db7afdSKautuk Consul err_free2: 2631ca23e405STejun Heo kfree(vas); 2632ca23e405STejun Heo kfree(vms); 2633ca23e405STejun Heo return NULL; 2634ca23e405STejun Heo } 2635ca23e405STejun Heo 2636ca23e405STejun Heo /** 2637ca23e405STejun Heo * pcpu_free_vm_areas - free vmalloc areas for percpu allocator 2638ca23e405STejun Heo * @vms: vm_struct pointer array returned by pcpu_get_vm_areas() 2639ca23e405STejun Heo * @nr_vms: the number of allocated areas 2640ca23e405STejun Heo * 2641ca23e405STejun Heo * Free vm_structs and the array allocated by pcpu_get_vm_areas(). 2642ca23e405STejun Heo */ 2643ca23e405STejun Heo void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) 2644ca23e405STejun Heo { 2645ca23e405STejun Heo int i; 2646ca23e405STejun Heo 2647ca23e405STejun Heo for (i = 0; i < nr_vms; i++) 2648ca23e405STejun Heo free_vm_area(vms[i]); 2649ca23e405STejun Heo kfree(vms); 2650ca23e405STejun Heo } 26514f8b02b4STejun Heo #endif /* CONFIG_SMP */ 2652a10aa579SChristoph Lameter 2653a10aa579SChristoph Lameter #ifdef CONFIG_PROC_FS 2654a10aa579SChristoph Lameter static void *s_start(struct seq_file *m, loff_t *pos) 2655d4033afdSJoonsoo Kim __acquires(&vmap_area_lock) 2656a10aa579SChristoph Lameter { 2657d4033afdSJoonsoo Kim spin_lock(&vmap_area_lock); 26583f500069Szijun_hu return seq_list_start(&vmap_area_list, *pos); 2659a10aa579SChristoph Lameter } 2660a10aa579SChristoph Lameter 2661a10aa579SChristoph Lameter static void *s_next(struct seq_file *m, void *p, loff_t *pos) 2662a10aa579SChristoph Lameter { 26633f500069Szijun_hu return seq_list_next(p, &vmap_area_list, pos); 2664a10aa579SChristoph Lameter } 2665a10aa579SChristoph Lameter 2666a10aa579SChristoph Lameter static void s_stop(struct seq_file *m, void *p) 2667d4033afdSJoonsoo Kim __releases(&vmap_area_lock) 2668a10aa579SChristoph Lameter { 2669d4033afdSJoonsoo Kim spin_unlock(&vmap_area_lock); 2670a10aa579SChristoph Lameter } 2671a10aa579SChristoph Lameter 2672a47a126aSEric Dumazet static void show_numa_info(struct seq_file *m, struct vm_struct *v) 2673a47a126aSEric Dumazet { 2674e5adfffcSKirill A. Shutemov if (IS_ENABLED(CONFIG_NUMA)) { 2675a47a126aSEric Dumazet unsigned int nr, *counters = m->private; 2676a47a126aSEric Dumazet 2677a47a126aSEric Dumazet if (!counters) 2678a47a126aSEric Dumazet return; 2679a47a126aSEric Dumazet 2680af12346cSWanpeng Li if (v->flags & VM_UNINITIALIZED) 2681af12346cSWanpeng Li return; 26827e5b528bSDmitry Vyukov /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ 26837e5b528bSDmitry Vyukov smp_rmb(); 2684af12346cSWanpeng Li 2685a47a126aSEric Dumazet memset(counters, 0, nr_node_ids * sizeof(unsigned int)); 2686a47a126aSEric Dumazet 2687a47a126aSEric Dumazet for (nr = 0; nr < v->nr_pages; nr++) 2688a47a126aSEric Dumazet counters[page_to_nid(v->pages[nr])]++; 2689a47a126aSEric Dumazet 2690a47a126aSEric Dumazet for_each_node_state(nr, N_HIGH_MEMORY) 2691a47a126aSEric Dumazet if (counters[nr]) 2692a47a126aSEric Dumazet seq_printf(m, " N%u=%u", nr, counters[nr]); 2693a47a126aSEric Dumazet } 2694a47a126aSEric Dumazet } 2695a47a126aSEric Dumazet 2696a10aa579SChristoph Lameter static int s_show(struct seq_file *m, void *p) 2697a10aa579SChristoph Lameter { 26983f500069Szijun_hu struct vmap_area *va; 2699d4033afdSJoonsoo Kim struct vm_struct *v; 2700d4033afdSJoonsoo Kim 27013f500069Szijun_hu va = list_entry(p, struct vmap_area, list); 27023f500069Szijun_hu 2703c2ce8c14SWanpeng Li /* 2704c2ce8c14SWanpeng Li * s_show can encounter race with remove_vm_area, !VM_VM_AREA on 2705c2ce8c14SWanpeng Li * behalf of vmap area is being tear down or vm_map_ram allocation. 2706c2ce8c14SWanpeng Li */ 2707c2ce8c14SWanpeng Li if (!(va->flags & VM_VM_AREA)) 2708d4033afdSJoonsoo Kim return 0; 2709d4033afdSJoonsoo Kim 2710d4033afdSJoonsoo Kim v = va->vm; 2711a10aa579SChristoph Lameter 271245ec1690SKees Cook seq_printf(m, "0x%pK-0x%pK %7ld", 2713a10aa579SChristoph Lameter v->addr, v->addr + v->size, v->size); 2714a10aa579SChristoph Lameter 271562c70bceSJoe Perches if (v->caller) 271662c70bceSJoe Perches seq_printf(m, " %pS", v->caller); 271723016969SChristoph Lameter 2718a10aa579SChristoph Lameter if (v->nr_pages) 2719a10aa579SChristoph Lameter seq_printf(m, " pages=%d", v->nr_pages); 2720a10aa579SChristoph Lameter 2721a10aa579SChristoph Lameter if (v->phys_addr) 2722199eaa05SMiles Chen seq_printf(m, " phys=%pa", &v->phys_addr); 2723a10aa579SChristoph Lameter 2724a10aa579SChristoph Lameter if (v->flags & VM_IOREMAP) 2725f4527c90SFabian Frederick seq_puts(m, " ioremap"); 2726a10aa579SChristoph Lameter 2727a10aa579SChristoph Lameter if (v->flags & VM_ALLOC) 2728f4527c90SFabian Frederick seq_puts(m, " vmalloc"); 2729a10aa579SChristoph Lameter 2730a10aa579SChristoph Lameter if (v->flags & VM_MAP) 2731f4527c90SFabian Frederick seq_puts(m, " vmap"); 2732a10aa579SChristoph Lameter 2733a10aa579SChristoph Lameter if (v->flags & VM_USERMAP) 2734f4527c90SFabian Frederick seq_puts(m, " user"); 2735a10aa579SChristoph Lameter 2736244d63eeSDavid Rientjes if (is_vmalloc_addr(v->pages)) 2737f4527c90SFabian Frederick seq_puts(m, " vpages"); 2738a10aa579SChristoph Lameter 2739a47a126aSEric Dumazet show_numa_info(m, v); 2740a10aa579SChristoph Lameter seq_putc(m, '\n'); 2741a10aa579SChristoph Lameter return 0; 2742a10aa579SChristoph Lameter } 2743a10aa579SChristoph Lameter 27445f6a6a9cSAlexey Dobriyan static const struct seq_operations vmalloc_op = { 2745a10aa579SChristoph Lameter .start = s_start, 2746a10aa579SChristoph Lameter .next = s_next, 2747a10aa579SChristoph Lameter .stop = s_stop, 2748a10aa579SChristoph Lameter .show = s_show, 2749a10aa579SChristoph Lameter }; 27505f6a6a9cSAlexey Dobriyan 27515f6a6a9cSAlexey Dobriyan static int vmalloc_open(struct inode *inode, struct file *file) 27525f6a6a9cSAlexey Dobriyan { 2753703394c1SRob Jones if (IS_ENABLED(CONFIG_NUMA)) 2754703394c1SRob Jones return seq_open_private(file, &vmalloc_op, 2755703394c1SRob Jones nr_node_ids * sizeof(unsigned int)); 2756703394c1SRob Jones else 2757703394c1SRob Jones return seq_open(file, &vmalloc_op); 27585f6a6a9cSAlexey Dobriyan } 27595f6a6a9cSAlexey Dobriyan 27605f6a6a9cSAlexey Dobriyan static const struct file_operations proc_vmalloc_operations = { 27615f6a6a9cSAlexey Dobriyan .open = vmalloc_open, 27625f6a6a9cSAlexey Dobriyan .read = seq_read, 27635f6a6a9cSAlexey Dobriyan .llseek = seq_lseek, 27645f6a6a9cSAlexey Dobriyan .release = seq_release_private, 27655f6a6a9cSAlexey Dobriyan }; 27665f6a6a9cSAlexey Dobriyan 27675f6a6a9cSAlexey Dobriyan static int __init proc_vmalloc_init(void) 27685f6a6a9cSAlexey Dobriyan { 27695f6a6a9cSAlexey Dobriyan proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations); 27705f6a6a9cSAlexey Dobriyan return 0; 27715f6a6a9cSAlexey Dobriyan } 27725f6a6a9cSAlexey Dobriyan module_init(proc_vmalloc_init); 2773db3808c1SJoonsoo Kim 2774a10aa579SChristoph Lameter #endif 2775a10aa579SChristoph Lameter 2776