xref: /linux/mm/vmalloc.c (revision dd9a41bc61cc62d38306465ed62373b98df0049e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/mm/vmalloc.c
4  *
5  *  Copyright (C) 1993  Linus Torvalds
6  *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
7  *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
8  *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
9  *  Numa awareness, Christoph Lameter, SGI, June 2005
10  */
11 
12 #include <linux/vmalloc.h>
13 #include <linux/mm.h>
14 #include <linux/module.h>
15 #include <linux/highmem.h>
16 #include <linux/sched/signal.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 #include <linux/interrupt.h>
20 #include <linux/proc_fs.h>
21 #include <linux/seq_file.h>
22 #include <linux/set_memory.h>
23 #include <linux/debugobjects.h>
24 #include <linux/kallsyms.h>
25 #include <linux/list.h>
26 #include <linux/notifier.h>
27 #include <linux/rbtree.h>
28 #include <linux/radix-tree.h>
29 #include <linux/rcupdate.h>
30 #include <linux/pfn.h>
31 #include <linux/kmemleak.h>
32 #include <linux/atomic.h>
33 #include <linux/compiler.h>
34 #include <linux/llist.h>
35 #include <linux/bitops.h>
36 #include <linux/rbtree_augmented.h>
37 
38 #include <linux/uaccess.h>
39 #include <asm/tlbflush.h>
40 #include <asm/shmparam.h>
41 
42 #include "internal.h"
43 
44 bool is_vmalloc_addr(const void *x)
45 {
46 	unsigned long addr = (unsigned long)x;
47 
48 	return addr >= VMALLOC_START && addr < VMALLOC_END;
49 }
50 EXPORT_SYMBOL(is_vmalloc_addr);
51 
52 struct vfree_deferred {
53 	struct llist_head list;
54 	struct work_struct wq;
55 };
56 static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
57 
58 static void __vunmap(const void *, int);
59 
60 static void free_work(struct work_struct *w)
61 {
62 	struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
63 	struct llist_node *t, *llnode;
64 
65 	llist_for_each_safe(llnode, t, llist_del_all(&p->list))
66 		__vunmap((void *)llnode, 1);
67 }
68 
69 /*** Page table manipulation functions ***/
70 
71 static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
72 {
73 	pte_t *pte;
74 
75 	pte = pte_offset_kernel(pmd, addr);
76 	do {
77 		pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
78 		WARN_ON(!pte_none(ptent) && !pte_present(ptent));
79 	} while (pte++, addr += PAGE_SIZE, addr != end);
80 }
81 
82 static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
83 {
84 	pmd_t *pmd;
85 	unsigned long next;
86 
87 	pmd = pmd_offset(pud, addr);
88 	do {
89 		next = pmd_addr_end(addr, end);
90 		if (pmd_clear_huge(pmd))
91 			continue;
92 		if (pmd_none_or_clear_bad(pmd))
93 			continue;
94 		vunmap_pte_range(pmd, addr, next);
95 	} while (pmd++, addr = next, addr != end);
96 }
97 
98 static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end)
99 {
100 	pud_t *pud;
101 	unsigned long next;
102 
103 	pud = pud_offset(p4d, addr);
104 	do {
105 		next = pud_addr_end(addr, end);
106 		if (pud_clear_huge(pud))
107 			continue;
108 		if (pud_none_or_clear_bad(pud))
109 			continue;
110 		vunmap_pmd_range(pud, addr, next);
111 	} while (pud++, addr = next, addr != end);
112 }
113 
114 static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end)
115 {
116 	p4d_t *p4d;
117 	unsigned long next;
118 
119 	p4d = p4d_offset(pgd, addr);
120 	do {
121 		next = p4d_addr_end(addr, end);
122 		if (p4d_clear_huge(p4d))
123 			continue;
124 		if (p4d_none_or_clear_bad(p4d))
125 			continue;
126 		vunmap_pud_range(p4d, addr, next);
127 	} while (p4d++, addr = next, addr != end);
128 }
129 
130 static void vunmap_page_range(unsigned long addr, unsigned long end)
131 {
132 	pgd_t *pgd;
133 	unsigned long next;
134 
135 	BUG_ON(addr >= end);
136 	pgd = pgd_offset_k(addr);
137 	do {
138 		next = pgd_addr_end(addr, end);
139 		if (pgd_none_or_clear_bad(pgd))
140 			continue;
141 		vunmap_p4d_range(pgd, addr, next);
142 	} while (pgd++, addr = next, addr != end);
143 }
144 
145 static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
146 		unsigned long end, pgprot_t prot, struct page **pages, int *nr)
147 {
148 	pte_t *pte;
149 
150 	/*
151 	 * nr is a running index into the array which helps higher level
152 	 * callers keep track of where we're up to.
153 	 */
154 
155 	pte = pte_alloc_kernel(pmd, addr);
156 	if (!pte)
157 		return -ENOMEM;
158 	do {
159 		struct page *page = pages[*nr];
160 
161 		if (WARN_ON(!pte_none(*pte)))
162 			return -EBUSY;
163 		if (WARN_ON(!page))
164 			return -ENOMEM;
165 		set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
166 		(*nr)++;
167 	} while (pte++, addr += PAGE_SIZE, addr != end);
168 	return 0;
169 }
170 
171 static int vmap_pmd_range(pud_t *pud, unsigned long addr,
172 		unsigned long end, pgprot_t prot, struct page **pages, int *nr)
173 {
174 	pmd_t *pmd;
175 	unsigned long next;
176 
177 	pmd = pmd_alloc(&init_mm, pud, addr);
178 	if (!pmd)
179 		return -ENOMEM;
180 	do {
181 		next = pmd_addr_end(addr, end);
182 		if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
183 			return -ENOMEM;
184 	} while (pmd++, addr = next, addr != end);
185 	return 0;
186 }
187 
188 static int vmap_pud_range(p4d_t *p4d, unsigned long addr,
189 		unsigned long end, pgprot_t prot, struct page **pages, int *nr)
190 {
191 	pud_t *pud;
192 	unsigned long next;
193 
194 	pud = pud_alloc(&init_mm, p4d, addr);
195 	if (!pud)
196 		return -ENOMEM;
197 	do {
198 		next = pud_addr_end(addr, end);
199 		if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
200 			return -ENOMEM;
201 	} while (pud++, addr = next, addr != end);
202 	return 0;
203 }
204 
205 static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,
206 		unsigned long end, pgprot_t prot, struct page **pages, int *nr)
207 {
208 	p4d_t *p4d;
209 	unsigned long next;
210 
211 	p4d = p4d_alloc(&init_mm, pgd, addr);
212 	if (!p4d)
213 		return -ENOMEM;
214 	do {
215 		next = p4d_addr_end(addr, end);
216 		if (vmap_pud_range(p4d, addr, next, prot, pages, nr))
217 			return -ENOMEM;
218 	} while (p4d++, addr = next, addr != end);
219 	return 0;
220 }
221 
222 /*
223  * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
224  * will have pfns corresponding to the "pages" array.
225  *
226  * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
227  */
228 static int vmap_page_range_noflush(unsigned long start, unsigned long end,
229 				   pgprot_t prot, struct page **pages)
230 {
231 	pgd_t *pgd;
232 	unsigned long next;
233 	unsigned long addr = start;
234 	int err = 0;
235 	int nr = 0;
236 
237 	BUG_ON(addr >= end);
238 	pgd = pgd_offset_k(addr);
239 	do {
240 		next = pgd_addr_end(addr, end);
241 		err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr);
242 		if (err)
243 			return err;
244 	} while (pgd++, addr = next, addr != end);
245 
246 	return nr;
247 }
248 
249 static int vmap_page_range(unsigned long start, unsigned long end,
250 			   pgprot_t prot, struct page **pages)
251 {
252 	int ret;
253 
254 	ret = vmap_page_range_noflush(start, end, prot, pages);
255 	flush_cache_vmap(start, end);
256 	return ret;
257 }
258 
259 int is_vmalloc_or_module_addr(const void *x)
260 {
261 	/*
262 	 * ARM, x86-64 and sparc64 put modules in a special place,
263 	 * and fall back on vmalloc() if that fails. Others
264 	 * just put it in the vmalloc space.
265 	 */
266 #if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
267 	unsigned long addr = (unsigned long)x;
268 	if (addr >= MODULES_VADDR && addr < MODULES_END)
269 		return 1;
270 #endif
271 	return is_vmalloc_addr(x);
272 }
273 
274 /*
275  * Walk a vmap address to the struct page it maps.
276  */
277 struct page *vmalloc_to_page(const void *vmalloc_addr)
278 {
279 	unsigned long addr = (unsigned long) vmalloc_addr;
280 	struct page *page = NULL;
281 	pgd_t *pgd = pgd_offset_k(addr);
282 	p4d_t *p4d;
283 	pud_t *pud;
284 	pmd_t *pmd;
285 	pte_t *ptep, pte;
286 
287 	/*
288 	 * XXX we might need to change this if we add VIRTUAL_BUG_ON for
289 	 * architectures that do not vmalloc module space
290 	 */
291 	VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
292 
293 	if (pgd_none(*pgd))
294 		return NULL;
295 	p4d = p4d_offset(pgd, addr);
296 	if (p4d_none(*p4d))
297 		return NULL;
298 	pud = pud_offset(p4d, addr);
299 
300 	/*
301 	 * Don't dereference bad PUD or PMD (below) entries. This will also
302 	 * identify huge mappings, which we may encounter on architectures
303 	 * that define CONFIG_HAVE_ARCH_HUGE_VMAP=y. Such regions will be
304 	 * identified as vmalloc addresses by is_vmalloc_addr(), but are
305 	 * not [unambiguously] associated with a struct page, so there is
306 	 * no correct value to return for them.
307 	 */
308 	WARN_ON_ONCE(pud_bad(*pud));
309 	if (pud_none(*pud) || pud_bad(*pud))
310 		return NULL;
311 	pmd = pmd_offset(pud, addr);
312 	WARN_ON_ONCE(pmd_bad(*pmd));
313 	if (pmd_none(*pmd) || pmd_bad(*pmd))
314 		return NULL;
315 
316 	ptep = pte_offset_map(pmd, addr);
317 	pte = *ptep;
318 	if (pte_present(pte))
319 		page = pte_page(pte);
320 	pte_unmap(ptep);
321 	return page;
322 }
323 EXPORT_SYMBOL(vmalloc_to_page);
324 
325 /*
326  * Map a vmalloc()-space virtual address to the physical page frame number.
327  */
328 unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
329 {
330 	return page_to_pfn(vmalloc_to_page(vmalloc_addr));
331 }
332 EXPORT_SYMBOL(vmalloc_to_pfn);
333 
334 
335 /*** Global kva allocator ***/
336 
337 #define DEBUG_AUGMENT_PROPAGATE_CHECK 0
338 #define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0
339 
340 
341 static DEFINE_SPINLOCK(vmap_area_lock);
342 static DEFINE_SPINLOCK(free_vmap_area_lock);
343 /* Export for kexec only */
344 LIST_HEAD(vmap_area_list);
345 static LLIST_HEAD(vmap_purge_list);
346 static struct rb_root vmap_area_root = RB_ROOT;
347 static bool vmap_initialized __read_mostly;
348 
349 /*
350  * This kmem_cache is used for vmap_area objects. Instead of
351  * allocating from slab we reuse an object from this cache to
352  * make things faster. Especially in "no edge" splitting of
353  * free block.
354  */
355 static struct kmem_cache *vmap_area_cachep;
356 
357 /*
358  * This linked list is used in pair with free_vmap_area_root.
359  * It gives O(1) access to prev/next to perform fast coalescing.
360  */
361 static LIST_HEAD(free_vmap_area_list);
362 
363 /*
364  * This augment red-black tree represents the free vmap space.
365  * All vmap_area objects in this tree are sorted by va->va_start
366  * address. It is used for allocation and merging when a vmap
367  * object is released.
368  *
369  * Each vmap_area node contains a maximum available free block
370  * of its sub-tree, right or left. Therefore it is possible to
371  * find a lowest match of free area.
372  */
373 static struct rb_root free_vmap_area_root = RB_ROOT;
374 
375 /*
376  * Preload a CPU with one object for "no edge" split case. The
377  * aim is to get rid of allocations from the atomic context, thus
378  * to use more permissive allocation masks.
379  */
380 static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node);
381 
382 static __always_inline unsigned long
383 va_size(struct vmap_area *va)
384 {
385 	return (va->va_end - va->va_start);
386 }
387 
388 static __always_inline unsigned long
389 get_subtree_max_size(struct rb_node *node)
390 {
391 	struct vmap_area *va;
392 
393 	va = rb_entry_safe(node, struct vmap_area, rb_node);
394 	return va ? va->subtree_max_size : 0;
395 }
396 
397 /*
398  * Gets called when remove the node and rotate.
399  */
400 static __always_inline unsigned long
401 compute_subtree_max_size(struct vmap_area *va)
402 {
403 	return max3(va_size(va),
404 		get_subtree_max_size(va->rb_node.rb_left),
405 		get_subtree_max_size(va->rb_node.rb_right));
406 }
407 
408 RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb,
409 	struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size)
410 
411 static void purge_vmap_area_lazy(void);
412 static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
413 static unsigned long lazy_max_pages(void);
414 
415 static atomic_long_t nr_vmalloc_pages;
416 
417 unsigned long vmalloc_nr_pages(void)
418 {
419 	return atomic_long_read(&nr_vmalloc_pages);
420 }
421 
422 static struct vmap_area *__find_vmap_area(unsigned long addr)
423 {
424 	struct rb_node *n = vmap_area_root.rb_node;
425 
426 	while (n) {
427 		struct vmap_area *va;
428 
429 		va = rb_entry(n, struct vmap_area, rb_node);
430 		if (addr < va->va_start)
431 			n = n->rb_left;
432 		else if (addr >= va->va_end)
433 			n = n->rb_right;
434 		else
435 			return va;
436 	}
437 
438 	return NULL;
439 }
440 
441 /*
442  * This function returns back addresses of parent node
443  * and its left or right link for further processing.
444  */
445 static __always_inline struct rb_node **
446 find_va_links(struct vmap_area *va,
447 	struct rb_root *root, struct rb_node *from,
448 	struct rb_node **parent)
449 {
450 	struct vmap_area *tmp_va;
451 	struct rb_node **link;
452 
453 	if (root) {
454 		link = &root->rb_node;
455 		if (unlikely(!*link)) {
456 			*parent = NULL;
457 			return link;
458 		}
459 	} else {
460 		link = &from;
461 	}
462 
463 	/*
464 	 * Go to the bottom of the tree. When we hit the last point
465 	 * we end up with parent rb_node and correct direction, i name
466 	 * it link, where the new va->rb_node will be attached to.
467 	 */
468 	do {
469 		tmp_va = rb_entry(*link, struct vmap_area, rb_node);
470 
471 		/*
472 		 * During the traversal we also do some sanity check.
473 		 * Trigger the BUG() if there are sides(left/right)
474 		 * or full overlaps.
475 		 */
476 		if (va->va_start < tmp_va->va_end &&
477 				va->va_end <= tmp_va->va_start)
478 			link = &(*link)->rb_left;
479 		else if (va->va_end > tmp_va->va_start &&
480 				va->va_start >= tmp_va->va_end)
481 			link = &(*link)->rb_right;
482 		else
483 			BUG();
484 	} while (*link);
485 
486 	*parent = &tmp_va->rb_node;
487 	return link;
488 }
489 
490 static __always_inline struct list_head *
491 get_va_next_sibling(struct rb_node *parent, struct rb_node **link)
492 {
493 	struct list_head *list;
494 
495 	if (unlikely(!parent))
496 		/*
497 		 * The red-black tree where we try to find VA neighbors
498 		 * before merging or inserting is empty, i.e. it means
499 		 * there is no free vmap space. Normally it does not
500 		 * happen but we handle this case anyway.
501 		 */
502 		return NULL;
503 
504 	list = &rb_entry(parent, struct vmap_area, rb_node)->list;
505 	return (&parent->rb_right == link ? list->next : list);
506 }
507 
508 static __always_inline void
509 link_va(struct vmap_area *va, struct rb_root *root,
510 	struct rb_node *parent, struct rb_node **link, struct list_head *head)
511 {
512 	/*
513 	 * VA is still not in the list, but we can
514 	 * identify its future previous list_head node.
515 	 */
516 	if (likely(parent)) {
517 		head = &rb_entry(parent, struct vmap_area, rb_node)->list;
518 		if (&parent->rb_right != link)
519 			head = head->prev;
520 	}
521 
522 	/* Insert to the rb-tree */
523 	rb_link_node(&va->rb_node, parent, link);
524 	if (root == &free_vmap_area_root) {
525 		/*
526 		 * Some explanation here. Just perform simple insertion
527 		 * to the tree. We do not set va->subtree_max_size to
528 		 * its current size before calling rb_insert_augmented().
529 		 * It is because of we populate the tree from the bottom
530 		 * to parent levels when the node _is_ in the tree.
531 		 *
532 		 * Therefore we set subtree_max_size to zero after insertion,
533 		 * to let __augment_tree_propagate_from() puts everything to
534 		 * the correct order later on.
535 		 */
536 		rb_insert_augmented(&va->rb_node,
537 			root, &free_vmap_area_rb_augment_cb);
538 		va->subtree_max_size = 0;
539 	} else {
540 		rb_insert_color(&va->rb_node, root);
541 	}
542 
543 	/* Address-sort this list */
544 	list_add(&va->list, head);
545 }
546 
547 static __always_inline void
548 unlink_va(struct vmap_area *va, struct rb_root *root)
549 {
550 	if (WARN_ON(RB_EMPTY_NODE(&va->rb_node)))
551 		return;
552 
553 	if (root == &free_vmap_area_root)
554 		rb_erase_augmented(&va->rb_node,
555 			root, &free_vmap_area_rb_augment_cb);
556 	else
557 		rb_erase(&va->rb_node, root);
558 
559 	list_del(&va->list);
560 	RB_CLEAR_NODE(&va->rb_node);
561 }
562 
563 #if DEBUG_AUGMENT_PROPAGATE_CHECK
564 static void
565 augment_tree_propagate_check(struct rb_node *n)
566 {
567 	struct vmap_area *va;
568 	struct rb_node *node;
569 	unsigned long size;
570 	bool found = false;
571 
572 	if (n == NULL)
573 		return;
574 
575 	va = rb_entry(n, struct vmap_area, rb_node);
576 	size = va->subtree_max_size;
577 	node = n;
578 
579 	while (node) {
580 		va = rb_entry(node, struct vmap_area, rb_node);
581 
582 		if (get_subtree_max_size(node->rb_left) == size) {
583 			node = node->rb_left;
584 		} else {
585 			if (va_size(va) == size) {
586 				found = true;
587 				break;
588 			}
589 
590 			node = node->rb_right;
591 		}
592 	}
593 
594 	if (!found) {
595 		va = rb_entry(n, struct vmap_area, rb_node);
596 		pr_emerg("tree is corrupted: %lu, %lu\n",
597 			va_size(va), va->subtree_max_size);
598 	}
599 
600 	augment_tree_propagate_check(n->rb_left);
601 	augment_tree_propagate_check(n->rb_right);
602 }
603 #endif
604 
605 /*
606  * This function populates subtree_max_size from bottom to upper
607  * levels starting from VA point. The propagation must be done
608  * when VA size is modified by changing its va_start/va_end. Or
609  * in case of newly inserting of VA to the tree.
610  *
611  * It means that __augment_tree_propagate_from() must be called:
612  * - After VA has been inserted to the tree(free path);
613  * - After VA has been shrunk(allocation path);
614  * - After VA has been increased(merging path).
615  *
616  * Please note that, it does not mean that upper parent nodes
617  * and their subtree_max_size are recalculated all the time up
618  * to the root node.
619  *
620  *       4--8
621  *        /\
622  *       /  \
623  *      /    \
624  *    2--2  8--8
625  *
626  * For example if we modify the node 4, shrinking it to 2, then
627  * no any modification is required. If we shrink the node 2 to 1
628  * its subtree_max_size is updated only, and set to 1. If we shrink
629  * the node 8 to 6, then its subtree_max_size is set to 6 and parent
630  * node becomes 4--6.
631  */
632 static __always_inline void
633 augment_tree_propagate_from(struct vmap_area *va)
634 {
635 	struct rb_node *node = &va->rb_node;
636 	unsigned long new_va_sub_max_size;
637 
638 	while (node) {
639 		va = rb_entry(node, struct vmap_area, rb_node);
640 		new_va_sub_max_size = compute_subtree_max_size(va);
641 
642 		/*
643 		 * If the newly calculated maximum available size of the
644 		 * subtree is equal to the current one, then it means that
645 		 * the tree is propagated correctly. So we have to stop at
646 		 * this point to save cycles.
647 		 */
648 		if (va->subtree_max_size == new_va_sub_max_size)
649 			break;
650 
651 		va->subtree_max_size = new_va_sub_max_size;
652 		node = rb_parent(&va->rb_node);
653 	}
654 
655 #if DEBUG_AUGMENT_PROPAGATE_CHECK
656 	augment_tree_propagate_check(free_vmap_area_root.rb_node);
657 #endif
658 }
659 
660 static void
661 insert_vmap_area(struct vmap_area *va,
662 	struct rb_root *root, struct list_head *head)
663 {
664 	struct rb_node **link;
665 	struct rb_node *parent;
666 
667 	link = find_va_links(va, root, NULL, &parent);
668 	link_va(va, root, parent, link, head);
669 }
670 
671 static void
672 insert_vmap_area_augment(struct vmap_area *va,
673 	struct rb_node *from, struct rb_root *root,
674 	struct list_head *head)
675 {
676 	struct rb_node **link;
677 	struct rb_node *parent;
678 
679 	if (from)
680 		link = find_va_links(va, NULL, from, &parent);
681 	else
682 		link = find_va_links(va, root, NULL, &parent);
683 
684 	link_va(va, root, parent, link, head);
685 	augment_tree_propagate_from(va);
686 }
687 
688 /*
689  * Merge de-allocated chunk of VA memory with previous
690  * and next free blocks. If coalesce is not done a new
691  * free area is inserted. If VA has been merged, it is
692  * freed.
693  */
694 static __always_inline struct vmap_area *
695 merge_or_add_vmap_area(struct vmap_area *va,
696 	struct rb_root *root, struct list_head *head)
697 {
698 	struct vmap_area *sibling;
699 	struct list_head *next;
700 	struct rb_node **link;
701 	struct rb_node *parent;
702 	bool merged = false;
703 
704 	/*
705 	 * Find a place in the tree where VA potentially will be
706 	 * inserted, unless it is merged with its sibling/siblings.
707 	 */
708 	link = find_va_links(va, root, NULL, &parent);
709 
710 	/*
711 	 * Get next node of VA to check if merging can be done.
712 	 */
713 	next = get_va_next_sibling(parent, link);
714 	if (unlikely(next == NULL))
715 		goto insert;
716 
717 	/*
718 	 * start            end
719 	 * |                |
720 	 * |<------VA------>|<-----Next----->|
721 	 *                  |                |
722 	 *                  start            end
723 	 */
724 	if (next != head) {
725 		sibling = list_entry(next, struct vmap_area, list);
726 		if (sibling->va_start == va->va_end) {
727 			sibling->va_start = va->va_start;
728 
729 			/* Check and update the tree if needed. */
730 			augment_tree_propagate_from(sibling);
731 
732 			/* Free vmap_area object. */
733 			kmem_cache_free(vmap_area_cachep, va);
734 
735 			/* Point to the new merged area. */
736 			va = sibling;
737 			merged = true;
738 		}
739 	}
740 
741 	/*
742 	 * start            end
743 	 * |                |
744 	 * |<-----Prev----->|<------VA------>|
745 	 *                  |                |
746 	 *                  start            end
747 	 */
748 	if (next->prev != head) {
749 		sibling = list_entry(next->prev, struct vmap_area, list);
750 		if (sibling->va_end == va->va_start) {
751 			sibling->va_end = va->va_end;
752 
753 			/* Check and update the tree if needed. */
754 			augment_tree_propagate_from(sibling);
755 
756 			if (merged)
757 				unlink_va(va, root);
758 
759 			/* Free vmap_area object. */
760 			kmem_cache_free(vmap_area_cachep, va);
761 
762 			/* Point to the new merged area. */
763 			va = sibling;
764 			merged = true;
765 		}
766 	}
767 
768 insert:
769 	if (!merged) {
770 		link_va(va, root, parent, link, head);
771 		augment_tree_propagate_from(va);
772 	}
773 
774 	return va;
775 }
776 
777 static __always_inline bool
778 is_within_this_va(struct vmap_area *va, unsigned long size,
779 	unsigned long align, unsigned long vstart)
780 {
781 	unsigned long nva_start_addr;
782 
783 	if (va->va_start > vstart)
784 		nva_start_addr = ALIGN(va->va_start, align);
785 	else
786 		nva_start_addr = ALIGN(vstart, align);
787 
788 	/* Can be overflowed due to big size or alignment. */
789 	if (nva_start_addr + size < nva_start_addr ||
790 			nva_start_addr < vstart)
791 		return false;
792 
793 	return (nva_start_addr + size <= va->va_end);
794 }
795 
796 /*
797  * Find the first free block(lowest start address) in the tree,
798  * that will accomplish the request corresponding to passing
799  * parameters.
800  */
801 static __always_inline struct vmap_area *
802 find_vmap_lowest_match(unsigned long size,
803 	unsigned long align, unsigned long vstart)
804 {
805 	struct vmap_area *va;
806 	struct rb_node *node;
807 	unsigned long length;
808 
809 	/* Start from the root. */
810 	node = free_vmap_area_root.rb_node;
811 
812 	/* Adjust the search size for alignment overhead. */
813 	length = size + align - 1;
814 
815 	while (node) {
816 		va = rb_entry(node, struct vmap_area, rb_node);
817 
818 		if (get_subtree_max_size(node->rb_left) >= length &&
819 				vstart < va->va_start) {
820 			node = node->rb_left;
821 		} else {
822 			if (is_within_this_va(va, size, align, vstart))
823 				return va;
824 
825 			/*
826 			 * Does not make sense to go deeper towards the right
827 			 * sub-tree if it does not have a free block that is
828 			 * equal or bigger to the requested search length.
829 			 */
830 			if (get_subtree_max_size(node->rb_right) >= length) {
831 				node = node->rb_right;
832 				continue;
833 			}
834 
835 			/*
836 			 * OK. We roll back and find the first right sub-tree,
837 			 * that will satisfy the search criteria. It can happen
838 			 * only once due to "vstart" restriction.
839 			 */
840 			while ((node = rb_parent(node))) {
841 				va = rb_entry(node, struct vmap_area, rb_node);
842 				if (is_within_this_va(va, size, align, vstart))
843 					return va;
844 
845 				if (get_subtree_max_size(node->rb_right) >= length &&
846 						vstart <= va->va_start) {
847 					node = node->rb_right;
848 					break;
849 				}
850 			}
851 		}
852 	}
853 
854 	return NULL;
855 }
856 
857 #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
858 #include <linux/random.h>
859 
860 static struct vmap_area *
861 find_vmap_lowest_linear_match(unsigned long size,
862 	unsigned long align, unsigned long vstart)
863 {
864 	struct vmap_area *va;
865 
866 	list_for_each_entry(va, &free_vmap_area_list, list) {
867 		if (!is_within_this_va(va, size, align, vstart))
868 			continue;
869 
870 		return va;
871 	}
872 
873 	return NULL;
874 }
875 
876 static void
877 find_vmap_lowest_match_check(unsigned long size)
878 {
879 	struct vmap_area *va_1, *va_2;
880 	unsigned long vstart;
881 	unsigned int rnd;
882 
883 	get_random_bytes(&rnd, sizeof(rnd));
884 	vstart = VMALLOC_START + rnd;
885 
886 	va_1 = find_vmap_lowest_match(size, 1, vstart);
887 	va_2 = find_vmap_lowest_linear_match(size, 1, vstart);
888 
889 	if (va_1 != va_2)
890 		pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n",
891 			va_1, va_2, vstart);
892 }
893 #endif
894 
895 enum fit_type {
896 	NOTHING_FIT = 0,
897 	FL_FIT_TYPE = 1,	/* full fit */
898 	LE_FIT_TYPE = 2,	/* left edge fit */
899 	RE_FIT_TYPE = 3,	/* right edge fit */
900 	NE_FIT_TYPE = 4		/* no edge fit */
901 };
902 
903 static __always_inline enum fit_type
904 classify_va_fit_type(struct vmap_area *va,
905 	unsigned long nva_start_addr, unsigned long size)
906 {
907 	enum fit_type type;
908 
909 	/* Check if it is within VA. */
910 	if (nva_start_addr < va->va_start ||
911 			nva_start_addr + size > va->va_end)
912 		return NOTHING_FIT;
913 
914 	/* Now classify. */
915 	if (va->va_start == nva_start_addr) {
916 		if (va->va_end == nva_start_addr + size)
917 			type = FL_FIT_TYPE;
918 		else
919 			type = LE_FIT_TYPE;
920 	} else if (va->va_end == nva_start_addr + size) {
921 		type = RE_FIT_TYPE;
922 	} else {
923 		type = NE_FIT_TYPE;
924 	}
925 
926 	return type;
927 }
928 
929 static __always_inline int
930 adjust_va_to_fit_type(struct vmap_area *va,
931 	unsigned long nva_start_addr, unsigned long size,
932 	enum fit_type type)
933 {
934 	struct vmap_area *lva = NULL;
935 
936 	if (type == FL_FIT_TYPE) {
937 		/*
938 		 * No need to split VA, it fully fits.
939 		 *
940 		 * |               |
941 		 * V      NVA      V
942 		 * |---------------|
943 		 */
944 		unlink_va(va, &free_vmap_area_root);
945 		kmem_cache_free(vmap_area_cachep, va);
946 	} else if (type == LE_FIT_TYPE) {
947 		/*
948 		 * Split left edge of fit VA.
949 		 *
950 		 * |       |
951 		 * V  NVA  V   R
952 		 * |-------|-------|
953 		 */
954 		va->va_start += size;
955 	} else if (type == RE_FIT_TYPE) {
956 		/*
957 		 * Split right edge of fit VA.
958 		 *
959 		 *         |       |
960 		 *     L   V  NVA  V
961 		 * |-------|-------|
962 		 */
963 		va->va_end = nva_start_addr;
964 	} else if (type == NE_FIT_TYPE) {
965 		/*
966 		 * Split no edge of fit VA.
967 		 *
968 		 *     |       |
969 		 *   L V  NVA  V R
970 		 * |---|-------|---|
971 		 */
972 		lva = __this_cpu_xchg(ne_fit_preload_node, NULL);
973 		if (unlikely(!lva)) {
974 			/*
975 			 * For percpu allocator we do not do any pre-allocation
976 			 * and leave it as it is. The reason is it most likely
977 			 * never ends up with NE_FIT_TYPE splitting. In case of
978 			 * percpu allocations offsets and sizes are aligned to
979 			 * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE
980 			 * are its main fitting cases.
981 			 *
982 			 * There are a few exceptions though, as an example it is
983 			 * a first allocation (early boot up) when we have "one"
984 			 * big free space that has to be split.
985 			 *
986 			 * Also we can hit this path in case of regular "vmap"
987 			 * allocations, if "this" current CPU was not preloaded.
988 			 * See the comment in alloc_vmap_area() why. If so, then
989 			 * GFP_NOWAIT is used instead to get an extra object for
990 			 * split purpose. That is rare and most time does not
991 			 * occur.
992 			 *
993 			 * What happens if an allocation gets failed. Basically,
994 			 * an "overflow" path is triggered to purge lazily freed
995 			 * areas to free some memory, then, the "retry" path is
996 			 * triggered to repeat one more time. See more details
997 			 * in alloc_vmap_area() function.
998 			 */
999 			lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);
1000 			if (!lva)
1001 				return -1;
1002 		}
1003 
1004 		/*
1005 		 * Build the remainder.
1006 		 */
1007 		lva->va_start = va->va_start;
1008 		lva->va_end = nva_start_addr;
1009 
1010 		/*
1011 		 * Shrink this VA to remaining size.
1012 		 */
1013 		va->va_start = nva_start_addr + size;
1014 	} else {
1015 		return -1;
1016 	}
1017 
1018 	if (type != FL_FIT_TYPE) {
1019 		augment_tree_propagate_from(va);
1020 
1021 		if (lva)	/* type == NE_FIT_TYPE */
1022 			insert_vmap_area_augment(lva, &va->rb_node,
1023 				&free_vmap_area_root, &free_vmap_area_list);
1024 	}
1025 
1026 	return 0;
1027 }
1028 
1029 /*
1030  * Returns a start address of the newly allocated area, if success.
1031  * Otherwise a vend is returned that indicates failure.
1032  */
1033 static __always_inline unsigned long
1034 __alloc_vmap_area(unsigned long size, unsigned long align,
1035 	unsigned long vstart, unsigned long vend)
1036 {
1037 	unsigned long nva_start_addr;
1038 	struct vmap_area *va;
1039 	enum fit_type type;
1040 	int ret;
1041 
1042 	va = find_vmap_lowest_match(size, align, vstart);
1043 	if (unlikely(!va))
1044 		return vend;
1045 
1046 	if (va->va_start > vstart)
1047 		nva_start_addr = ALIGN(va->va_start, align);
1048 	else
1049 		nva_start_addr = ALIGN(vstart, align);
1050 
1051 	/* Check the "vend" restriction. */
1052 	if (nva_start_addr + size > vend)
1053 		return vend;
1054 
1055 	/* Classify what we have found. */
1056 	type = classify_va_fit_type(va, nva_start_addr, size);
1057 	if (WARN_ON_ONCE(type == NOTHING_FIT))
1058 		return vend;
1059 
1060 	/* Update the free vmap_area. */
1061 	ret = adjust_va_to_fit_type(va, nva_start_addr, size, type);
1062 	if (ret)
1063 		return vend;
1064 
1065 #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
1066 	find_vmap_lowest_match_check(size);
1067 #endif
1068 
1069 	return nva_start_addr;
1070 }
1071 
1072 /*
1073  * Free a region of KVA allocated by alloc_vmap_area
1074  */
1075 static void free_vmap_area(struct vmap_area *va)
1076 {
1077 	/*
1078 	 * Remove from the busy tree/list.
1079 	 */
1080 	spin_lock(&vmap_area_lock);
1081 	unlink_va(va, &vmap_area_root);
1082 	spin_unlock(&vmap_area_lock);
1083 
1084 	/*
1085 	 * Insert/Merge it back to the free tree/list.
1086 	 */
1087 	spin_lock(&free_vmap_area_lock);
1088 	merge_or_add_vmap_area(va, &free_vmap_area_root, &free_vmap_area_list);
1089 	spin_unlock(&free_vmap_area_lock);
1090 }
1091 
1092 /*
1093  * Allocate a region of KVA of the specified size and alignment, within the
1094  * vstart and vend.
1095  */
1096 static struct vmap_area *alloc_vmap_area(unsigned long size,
1097 				unsigned long align,
1098 				unsigned long vstart, unsigned long vend,
1099 				int node, gfp_t gfp_mask)
1100 {
1101 	struct vmap_area *va, *pva;
1102 	unsigned long addr;
1103 	int purged = 0;
1104 	int ret;
1105 
1106 	BUG_ON(!size);
1107 	BUG_ON(offset_in_page(size));
1108 	BUG_ON(!is_power_of_2(align));
1109 
1110 	if (unlikely(!vmap_initialized))
1111 		return ERR_PTR(-EBUSY);
1112 
1113 	might_sleep();
1114 	gfp_mask = gfp_mask & GFP_RECLAIM_MASK;
1115 
1116 	va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
1117 	if (unlikely(!va))
1118 		return ERR_PTR(-ENOMEM);
1119 
1120 	/*
1121 	 * Only scan the relevant parts containing pointers to other objects
1122 	 * to avoid false negatives.
1123 	 */
1124 	kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask);
1125 
1126 retry:
1127 	/*
1128 	 * Preload this CPU with one extra vmap_area object. It is used
1129 	 * when fit type of free area is NE_FIT_TYPE. Please note, it
1130 	 * does not guarantee that an allocation occurs on a CPU that
1131 	 * is preloaded, instead we minimize the case when it is not.
1132 	 * It can happen because of cpu migration, because there is a
1133 	 * race until the below spinlock is taken.
1134 	 *
1135 	 * The preload is done in non-atomic context, thus it allows us
1136 	 * to use more permissive allocation masks to be more stable under
1137 	 * low memory condition and high memory pressure. In rare case,
1138 	 * if not preloaded, GFP_NOWAIT is used.
1139 	 *
1140 	 * Set "pva" to NULL here, because of "retry" path.
1141 	 */
1142 	pva = NULL;
1143 
1144 	if (!this_cpu_read(ne_fit_preload_node))
1145 		/*
1146 		 * Even if it fails we do not really care about that.
1147 		 * Just proceed as it is. If needed "overflow" path
1148 		 * will refill the cache we allocate from.
1149 		 */
1150 		pva = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
1151 
1152 	spin_lock(&free_vmap_area_lock);
1153 
1154 	if (pva && __this_cpu_cmpxchg(ne_fit_preload_node, NULL, pva))
1155 		kmem_cache_free(vmap_area_cachep, pva);
1156 
1157 	/*
1158 	 * If an allocation fails, the "vend" address is
1159 	 * returned. Therefore trigger the overflow path.
1160 	 */
1161 	addr = __alloc_vmap_area(size, align, vstart, vend);
1162 	spin_unlock(&free_vmap_area_lock);
1163 
1164 	if (unlikely(addr == vend))
1165 		goto overflow;
1166 
1167 	va->va_start = addr;
1168 	va->va_end = addr + size;
1169 	va->vm = NULL;
1170 
1171 
1172 	spin_lock(&vmap_area_lock);
1173 	insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
1174 	spin_unlock(&vmap_area_lock);
1175 
1176 	BUG_ON(!IS_ALIGNED(va->va_start, align));
1177 	BUG_ON(va->va_start < vstart);
1178 	BUG_ON(va->va_end > vend);
1179 
1180 	ret = kasan_populate_vmalloc(addr, size);
1181 	if (ret) {
1182 		free_vmap_area(va);
1183 		return ERR_PTR(ret);
1184 	}
1185 
1186 	return va;
1187 
1188 overflow:
1189 	if (!purged) {
1190 		purge_vmap_area_lazy();
1191 		purged = 1;
1192 		goto retry;
1193 	}
1194 
1195 	if (gfpflags_allow_blocking(gfp_mask)) {
1196 		unsigned long freed = 0;
1197 		blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
1198 		if (freed > 0) {
1199 			purged = 0;
1200 			goto retry;
1201 		}
1202 	}
1203 
1204 	if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit())
1205 		pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
1206 			size);
1207 
1208 	kmem_cache_free(vmap_area_cachep, va);
1209 	return ERR_PTR(-EBUSY);
1210 }
1211 
1212 int register_vmap_purge_notifier(struct notifier_block *nb)
1213 {
1214 	return blocking_notifier_chain_register(&vmap_notify_list, nb);
1215 }
1216 EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
1217 
1218 int unregister_vmap_purge_notifier(struct notifier_block *nb)
1219 {
1220 	return blocking_notifier_chain_unregister(&vmap_notify_list, nb);
1221 }
1222 EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
1223 
1224 /*
1225  * Clear the pagetable entries of a given vmap_area
1226  */
1227 static void unmap_vmap_area(struct vmap_area *va)
1228 {
1229 	vunmap_page_range(va->va_start, va->va_end);
1230 }
1231 
1232 /*
1233  * lazy_max_pages is the maximum amount of virtual address space we gather up
1234  * before attempting to purge with a TLB flush.
1235  *
1236  * There is a tradeoff here: a larger number will cover more kernel page tables
1237  * and take slightly longer to purge, but it will linearly reduce the number of
1238  * global TLB flushes that must be performed. It would seem natural to scale
1239  * this number up linearly with the number of CPUs (because vmapping activity
1240  * could also scale linearly with the number of CPUs), however it is likely
1241  * that in practice, workloads might be constrained in other ways that mean
1242  * vmap activity will not scale linearly with CPUs. Also, I want to be
1243  * conservative and not introduce a big latency on huge systems, so go with
1244  * a less aggressive log scale. It will still be an improvement over the old
1245  * code, and it will be simple to change the scale factor if we find that it
1246  * becomes a problem on bigger systems.
1247  */
1248 static unsigned long lazy_max_pages(void)
1249 {
1250 	unsigned int log;
1251 
1252 	log = fls(num_online_cpus());
1253 
1254 	return log * (32UL * 1024 * 1024 / PAGE_SIZE);
1255 }
1256 
1257 static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0);
1258 
1259 /*
1260  * Serialize vmap purging.  There is no actual criticial section protected
1261  * by this look, but we want to avoid concurrent calls for performance
1262  * reasons and to make the pcpu_get_vm_areas more deterministic.
1263  */
1264 static DEFINE_MUTEX(vmap_purge_lock);
1265 
1266 /* for per-CPU blocks */
1267 static void purge_fragmented_blocks_allcpus(void);
1268 
1269 /*
1270  * called before a call to iounmap() if the caller wants vm_area_struct's
1271  * immediately freed.
1272  */
1273 void set_iounmap_nonlazy(void)
1274 {
1275 	atomic_long_set(&vmap_lazy_nr, lazy_max_pages()+1);
1276 }
1277 
1278 /*
1279  * Purges all lazily-freed vmap areas.
1280  */
1281 static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end)
1282 {
1283 	unsigned long resched_threshold;
1284 	struct llist_node *valist;
1285 	struct vmap_area *va;
1286 	struct vmap_area *n_va;
1287 
1288 	lockdep_assert_held(&vmap_purge_lock);
1289 
1290 	valist = llist_del_all(&vmap_purge_list);
1291 	if (unlikely(valist == NULL))
1292 		return false;
1293 
1294 	/*
1295 	 * First make sure the mappings are removed from all page-tables
1296 	 * before they are freed.
1297 	 */
1298 	vmalloc_sync_unmappings();
1299 
1300 	/*
1301 	 * TODO: to calculate a flush range without looping.
1302 	 * The list can be up to lazy_max_pages() elements.
1303 	 */
1304 	llist_for_each_entry(va, valist, purge_list) {
1305 		if (va->va_start < start)
1306 			start = va->va_start;
1307 		if (va->va_end > end)
1308 			end = va->va_end;
1309 	}
1310 
1311 	flush_tlb_kernel_range(start, end);
1312 	resched_threshold = lazy_max_pages() << 1;
1313 
1314 	spin_lock(&free_vmap_area_lock);
1315 	llist_for_each_entry_safe(va, n_va, valist, purge_list) {
1316 		unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT;
1317 		unsigned long orig_start = va->va_start;
1318 		unsigned long orig_end = va->va_end;
1319 
1320 		/*
1321 		 * Finally insert or merge lazily-freed area. It is
1322 		 * detached and there is no need to "unlink" it from
1323 		 * anything.
1324 		 */
1325 		va = merge_or_add_vmap_area(va, &free_vmap_area_root,
1326 					    &free_vmap_area_list);
1327 
1328 		if (is_vmalloc_or_module_addr((void *)orig_start))
1329 			kasan_release_vmalloc(orig_start, orig_end,
1330 					      va->va_start, va->va_end);
1331 
1332 		atomic_long_sub(nr, &vmap_lazy_nr);
1333 
1334 		if (atomic_long_read(&vmap_lazy_nr) < resched_threshold)
1335 			cond_resched_lock(&free_vmap_area_lock);
1336 	}
1337 	spin_unlock(&free_vmap_area_lock);
1338 	return true;
1339 }
1340 
1341 /*
1342  * Kick off a purge of the outstanding lazy areas. Don't bother if somebody
1343  * is already purging.
1344  */
1345 static void try_purge_vmap_area_lazy(void)
1346 {
1347 	if (mutex_trylock(&vmap_purge_lock)) {
1348 		__purge_vmap_area_lazy(ULONG_MAX, 0);
1349 		mutex_unlock(&vmap_purge_lock);
1350 	}
1351 }
1352 
1353 /*
1354  * Kick off a purge of the outstanding lazy areas.
1355  */
1356 static void purge_vmap_area_lazy(void)
1357 {
1358 	mutex_lock(&vmap_purge_lock);
1359 	purge_fragmented_blocks_allcpus();
1360 	__purge_vmap_area_lazy(ULONG_MAX, 0);
1361 	mutex_unlock(&vmap_purge_lock);
1362 }
1363 
1364 /*
1365  * Free a vmap area, caller ensuring that the area has been unmapped
1366  * and flush_cache_vunmap had been called for the correct range
1367  * previously.
1368  */
1369 static void free_vmap_area_noflush(struct vmap_area *va)
1370 {
1371 	unsigned long nr_lazy;
1372 
1373 	spin_lock(&vmap_area_lock);
1374 	unlink_va(va, &vmap_area_root);
1375 	spin_unlock(&vmap_area_lock);
1376 
1377 	nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >>
1378 				PAGE_SHIFT, &vmap_lazy_nr);
1379 
1380 	/* After this point, we may free va at any time */
1381 	llist_add(&va->purge_list, &vmap_purge_list);
1382 
1383 	if (unlikely(nr_lazy > lazy_max_pages()))
1384 		try_purge_vmap_area_lazy();
1385 }
1386 
1387 /*
1388  * Free and unmap a vmap area
1389  */
1390 static void free_unmap_vmap_area(struct vmap_area *va)
1391 {
1392 	flush_cache_vunmap(va->va_start, va->va_end);
1393 	unmap_vmap_area(va);
1394 	if (debug_pagealloc_enabled_static())
1395 		flush_tlb_kernel_range(va->va_start, va->va_end);
1396 
1397 	free_vmap_area_noflush(va);
1398 }
1399 
1400 static struct vmap_area *find_vmap_area(unsigned long addr)
1401 {
1402 	struct vmap_area *va;
1403 
1404 	spin_lock(&vmap_area_lock);
1405 	va = __find_vmap_area(addr);
1406 	spin_unlock(&vmap_area_lock);
1407 
1408 	return va;
1409 }
1410 
1411 /*** Per cpu kva allocator ***/
1412 
1413 /*
1414  * vmap space is limited especially on 32 bit architectures. Ensure there is
1415  * room for at least 16 percpu vmap blocks per CPU.
1416  */
1417 /*
1418  * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
1419  * to #define VMALLOC_SPACE		(VMALLOC_END-VMALLOC_START). Guess
1420  * instead (we just need a rough idea)
1421  */
1422 #if BITS_PER_LONG == 32
1423 #define VMALLOC_SPACE		(128UL*1024*1024)
1424 #else
1425 #define VMALLOC_SPACE		(128UL*1024*1024*1024)
1426 #endif
1427 
1428 #define VMALLOC_PAGES		(VMALLOC_SPACE / PAGE_SIZE)
1429 #define VMAP_MAX_ALLOC		BITS_PER_LONG	/* 256K with 4K pages */
1430 #define VMAP_BBMAP_BITS_MAX	1024	/* 4MB with 4K pages */
1431 #define VMAP_BBMAP_BITS_MIN	(VMAP_MAX_ALLOC*2)
1432 #define VMAP_MIN(x, y)		((x) < (y) ? (x) : (y)) /* can't use min() */
1433 #define VMAP_MAX(x, y)		((x) > (y) ? (x) : (y)) /* can't use max() */
1434 #define VMAP_BBMAP_BITS		\
1435 		VMAP_MIN(VMAP_BBMAP_BITS_MAX,	\
1436 		VMAP_MAX(VMAP_BBMAP_BITS_MIN,	\
1437 			VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
1438 
1439 #define VMAP_BLOCK_SIZE		(VMAP_BBMAP_BITS * PAGE_SIZE)
1440 
1441 struct vmap_block_queue {
1442 	spinlock_t lock;
1443 	struct list_head free;
1444 };
1445 
1446 struct vmap_block {
1447 	spinlock_t lock;
1448 	struct vmap_area *va;
1449 	unsigned long free, dirty;
1450 	unsigned long dirty_min, dirty_max; /*< dirty range */
1451 	struct list_head free_list;
1452 	struct rcu_head rcu_head;
1453 	struct list_head purge;
1454 };
1455 
1456 /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
1457 static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
1458 
1459 /*
1460  * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
1461  * in the free path. Could get rid of this if we change the API to return a
1462  * "cookie" from alloc, to be passed to free. But no big deal yet.
1463  */
1464 static DEFINE_SPINLOCK(vmap_block_tree_lock);
1465 static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
1466 
1467 /*
1468  * We should probably have a fallback mechanism to allocate virtual memory
1469  * out of partially filled vmap blocks. However vmap block sizing should be
1470  * fairly reasonable according to the vmalloc size, so it shouldn't be a
1471  * big problem.
1472  */
1473 
1474 static unsigned long addr_to_vb_idx(unsigned long addr)
1475 {
1476 	addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
1477 	addr /= VMAP_BLOCK_SIZE;
1478 	return addr;
1479 }
1480 
1481 static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off)
1482 {
1483 	unsigned long addr;
1484 
1485 	addr = va_start + (pages_off << PAGE_SHIFT);
1486 	BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start));
1487 	return (void *)addr;
1488 }
1489 
1490 /**
1491  * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
1492  *                  block. Of course pages number can't exceed VMAP_BBMAP_BITS
1493  * @order:    how many 2^order pages should be occupied in newly allocated block
1494  * @gfp_mask: flags for the page level allocator
1495  *
1496  * Return: virtual address in a newly allocated block or ERR_PTR(-errno)
1497  */
1498 static void *new_vmap_block(unsigned int order, gfp_t gfp_mask)
1499 {
1500 	struct vmap_block_queue *vbq;
1501 	struct vmap_block *vb;
1502 	struct vmap_area *va;
1503 	unsigned long vb_idx;
1504 	int node, err;
1505 	void *vaddr;
1506 
1507 	node = numa_node_id();
1508 
1509 	vb = kmalloc_node(sizeof(struct vmap_block),
1510 			gfp_mask & GFP_RECLAIM_MASK, node);
1511 	if (unlikely(!vb))
1512 		return ERR_PTR(-ENOMEM);
1513 
1514 	va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
1515 					VMALLOC_START, VMALLOC_END,
1516 					node, gfp_mask);
1517 	if (IS_ERR(va)) {
1518 		kfree(vb);
1519 		return ERR_CAST(va);
1520 	}
1521 
1522 	err = radix_tree_preload(gfp_mask);
1523 	if (unlikely(err)) {
1524 		kfree(vb);
1525 		free_vmap_area(va);
1526 		return ERR_PTR(err);
1527 	}
1528 
1529 	vaddr = vmap_block_vaddr(va->va_start, 0);
1530 	spin_lock_init(&vb->lock);
1531 	vb->va = va;
1532 	/* At least something should be left free */
1533 	BUG_ON(VMAP_BBMAP_BITS <= (1UL << order));
1534 	vb->free = VMAP_BBMAP_BITS - (1UL << order);
1535 	vb->dirty = 0;
1536 	vb->dirty_min = VMAP_BBMAP_BITS;
1537 	vb->dirty_max = 0;
1538 	INIT_LIST_HEAD(&vb->free_list);
1539 
1540 	vb_idx = addr_to_vb_idx(va->va_start);
1541 	spin_lock(&vmap_block_tree_lock);
1542 	err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
1543 	spin_unlock(&vmap_block_tree_lock);
1544 	BUG_ON(err);
1545 	radix_tree_preload_end();
1546 
1547 	vbq = &get_cpu_var(vmap_block_queue);
1548 	spin_lock(&vbq->lock);
1549 	list_add_tail_rcu(&vb->free_list, &vbq->free);
1550 	spin_unlock(&vbq->lock);
1551 	put_cpu_var(vmap_block_queue);
1552 
1553 	return vaddr;
1554 }
1555 
1556 static void free_vmap_block(struct vmap_block *vb)
1557 {
1558 	struct vmap_block *tmp;
1559 	unsigned long vb_idx;
1560 
1561 	vb_idx = addr_to_vb_idx(vb->va->va_start);
1562 	spin_lock(&vmap_block_tree_lock);
1563 	tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
1564 	spin_unlock(&vmap_block_tree_lock);
1565 	BUG_ON(tmp != vb);
1566 
1567 	free_vmap_area_noflush(vb->va);
1568 	kfree_rcu(vb, rcu_head);
1569 }
1570 
1571 static void purge_fragmented_blocks(int cpu)
1572 {
1573 	LIST_HEAD(purge);
1574 	struct vmap_block *vb;
1575 	struct vmap_block *n_vb;
1576 	struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
1577 
1578 	rcu_read_lock();
1579 	list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1580 
1581 		if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
1582 			continue;
1583 
1584 		spin_lock(&vb->lock);
1585 		if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
1586 			vb->free = 0; /* prevent further allocs after releasing lock */
1587 			vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
1588 			vb->dirty_min = 0;
1589 			vb->dirty_max = VMAP_BBMAP_BITS;
1590 			spin_lock(&vbq->lock);
1591 			list_del_rcu(&vb->free_list);
1592 			spin_unlock(&vbq->lock);
1593 			spin_unlock(&vb->lock);
1594 			list_add_tail(&vb->purge, &purge);
1595 		} else
1596 			spin_unlock(&vb->lock);
1597 	}
1598 	rcu_read_unlock();
1599 
1600 	list_for_each_entry_safe(vb, n_vb, &purge, purge) {
1601 		list_del(&vb->purge);
1602 		free_vmap_block(vb);
1603 	}
1604 }
1605 
1606 static void purge_fragmented_blocks_allcpus(void)
1607 {
1608 	int cpu;
1609 
1610 	for_each_possible_cpu(cpu)
1611 		purge_fragmented_blocks(cpu);
1612 }
1613 
1614 static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
1615 {
1616 	struct vmap_block_queue *vbq;
1617 	struct vmap_block *vb;
1618 	void *vaddr = NULL;
1619 	unsigned int order;
1620 
1621 	BUG_ON(offset_in_page(size));
1622 	BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
1623 	if (WARN_ON(size == 0)) {
1624 		/*
1625 		 * Allocating 0 bytes isn't what caller wants since
1626 		 * get_order(0) returns funny result. Just warn and terminate
1627 		 * early.
1628 		 */
1629 		return NULL;
1630 	}
1631 	order = get_order(size);
1632 
1633 	rcu_read_lock();
1634 	vbq = &get_cpu_var(vmap_block_queue);
1635 	list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1636 		unsigned long pages_off;
1637 
1638 		spin_lock(&vb->lock);
1639 		if (vb->free < (1UL << order)) {
1640 			spin_unlock(&vb->lock);
1641 			continue;
1642 		}
1643 
1644 		pages_off = VMAP_BBMAP_BITS - vb->free;
1645 		vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);
1646 		vb->free -= 1UL << order;
1647 		if (vb->free == 0) {
1648 			spin_lock(&vbq->lock);
1649 			list_del_rcu(&vb->free_list);
1650 			spin_unlock(&vbq->lock);
1651 		}
1652 
1653 		spin_unlock(&vb->lock);
1654 		break;
1655 	}
1656 
1657 	put_cpu_var(vmap_block_queue);
1658 	rcu_read_unlock();
1659 
1660 	/* Allocate new block if nothing was found */
1661 	if (!vaddr)
1662 		vaddr = new_vmap_block(order, gfp_mask);
1663 
1664 	return vaddr;
1665 }
1666 
1667 static void vb_free(const void *addr, unsigned long size)
1668 {
1669 	unsigned long offset;
1670 	unsigned long vb_idx;
1671 	unsigned int order;
1672 	struct vmap_block *vb;
1673 
1674 	BUG_ON(offset_in_page(size));
1675 	BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
1676 
1677 	flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size);
1678 
1679 	order = get_order(size);
1680 
1681 	offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
1682 	offset >>= PAGE_SHIFT;
1683 
1684 	vb_idx = addr_to_vb_idx((unsigned long)addr);
1685 	rcu_read_lock();
1686 	vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
1687 	rcu_read_unlock();
1688 	BUG_ON(!vb);
1689 
1690 	vunmap_page_range((unsigned long)addr, (unsigned long)addr + size);
1691 
1692 	if (debug_pagealloc_enabled_static())
1693 		flush_tlb_kernel_range((unsigned long)addr,
1694 					(unsigned long)addr + size);
1695 
1696 	spin_lock(&vb->lock);
1697 
1698 	/* Expand dirty range */
1699 	vb->dirty_min = min(vb->dirty_min, offset);
1700 	vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
1701 
1702 	vb->dirty += 1UL << order;
1703 	if (vb->dirty == VMAP_BBMAP_BITS) {
1704 		BUG_ON(vb->free);
1705 		spin_unlock(&vb->lock);
1706 		free_vmap_block(vb);
1707 	} else
1708 		spin_unlock(&vb->lock);
1709 }
1710 
1711 static void _vm_unmap_aliases(unsigned long start, unsigned long end, int flush)
1712 {
1713 	int cpu;
1714 
1715 	if (unlikely(!vmap_initialized))
1716 		return;
1717 
1718 	might_sleep();
1719 
1720 	for_each_possible_cpu(cpu) {
1721 		struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
1722 		struct vmap_block *vb;
1723 
1724 		rcu_read_lock();
1725 		list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1726 			spin_lock(&vb->lock);
1727 			if (vb->dirty) {
1728 				unsigned long va_start = vb->va->va_start;
1729 				unsigned long s, e;
1730 
1731 				s = va_start + (vb->dirty_min << PAGE_SHIFT);
1732 				e = va_start + (vb->dirty_max << PAGE_SHIFT);
1733 
1734 				start = min(s, start);
1735 				end   = max(e, end);
1736 
1737 				flush = 1;
1738 			}
1739 			spin_unlock(&vb->lock);
1740 		}
1741 		rcu_read_unlock();
1742 	}
1743 
1744 	mutex_lock(&vmap_purge_lock);
1745 	purge_fragmented_blocks_allcpus();
1746 	if (!__purge_vmap_area_lazy(start, end) && flush)
1747 		flush_tlb_kernel_range(start, end);
1748 	mutex_unlock(&vmap_purge_lock);
1749 }
1750 
1751 /**
1752  * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
1753  *
1754  * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
1755  * to amortize TLB flushing overheads. What this means is that any page you
1756  * have now, may, in a former life, have been mapped into kernel virtual
1757  * address by the vmap layer and so there might be some CPUs with TLB entries
1758  * still referencing that page (additional to the regular 1:1 kernel mapping).
1759  *
1760  * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
1761  * be sure that none of the pages we have control over will have any aliases
1762  * from the vmap layer.
1763  */
1764 void vm_unmap_aliases(void)
1765 {
1766 	unsigned long start = ULONG_MAX, end = 0;
1767 	int flush = 0;
1768 
1769 	_vm_unmap_aliases(start, end, flush);
1770 }
1771 EXPORT_SYMBOL_GPL(vm_unmap_aliases);
1772 
1773 /**
1774  * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
1775  * @mem: the pointer returned by vm_map_ram
1776  * @count: the count passed to that vm_map_ram call (cannot unmap partial)
1777  */
1778 void vm_unmap_ram(const void *mem, unsigned int count)
1779 {
1780 	unsigned long size = (unsigned long)count << PAGE_SHIFT;
1781 	unsigned long addr = (unsigned long)mem;
1782 	struct vmap_area *va;
1783 
1784 	might_sleep();
1785 	BUG_ON(!addr);
1786 	BUG_ON(addr < VMALLOC_START);
1787 	BUG_ON(addr > VMALLOC_END);
1788 	BUG_ON(!PAGE_ALIGNED(addr));
1789 
1790 	kasan_poison_vmalloc(mem, size);
1791 
1792 	if (likely(count <= VMAP_MAX_ALLOC)) {
1793 		debug_check_no_locks_freed(mem, size);
1794 		vb_free(mem, size);
1795 		return;
1796 	}
1797 
1798 	va = find_vmap_area(addr);
1799 	BUG_ON(!va);
1800 	debug_check_no_locks_freed((void *)va->va_start,
1801 				    (va->va_end - va->va_start));
1802 	free_unmap_vmap_area(va);
1803 }
1804 EXPORT_SYMBOL(vm_unmap_ram);
1805 
1806 /**
1807  * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
1808  * @pages: an array of pointers to the pages to be mapped
1809  * @count: number of pages
1810  * @node: prefer to allocate data structures on this node
1811  * @prot: memory protection to use. PAGE_KERNEL for regular RAM
1812  *
1813  * If you use this function for less than VMAP_MAX_ALLOC pages, it could be
1814  * faster than vmap so it's good.  But if you mix long-life and short-life
1815  * objects with vm_map_ram(), it could consume lots of address space through
1816  * fragmentation (especially on a 32bit machine).  You could see failures in
1817  * the end.  Please use this function for short-lived objects.
1818  *
1819  * Returns: a pointer to the address that has been mapped, or %NULL on failure
1820  */
1821 void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
1822 {
1823 	unsigned long size = (unsigned long)count << PAGE_SHIFT;
1824 	unsigned long addr;
1825 	void *mem;
1826 
1827 	if (likely(count <= VMAP_MAX_ALLOC)) {
1828 		mem = vb_alloc(size, GFP_KERNEL);
1829 		if (IS_ERR(mem))
1830 			return NULL;
1831 		addr = (unsigned long)mem;
1832 	} else {
1833 		struct vmap_area *va;
1834 		va = alloc_vmap_area(size, PAGE_SIZE,
1835 				VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
1836 		if (IS_ERR(va))
1837 			return NULL;
1838 
1839 		addr = va->va_start;
1840 		mem = (void *)addr;
1841 	}
1842 
1843 	kasan_unpoison_vmalloc(mem, size);
1844 
1845 	if (vmap_page_range(addr, addr + size, prot, pages) < 0) {
1846 		vm_unmap_ram(mem, count);
1847 		return NULL;
1848 	}
1849 	return mem;
1850 }
1851 EXPORT_SYMBOL(vm_map_ram);
1852 
1853 static struct vm_struct *vmlist __initdata;
1854 
1855 /**
1856  * vm_area_add_early - add vmap area early during boot
1857  * @vm: vm_struct to add
1858  *
1859  * This function is used to add fixed kernel vm area to vmlist before
1860  * vmalloc_init() is called.  @vm->addr, @vm->size, and @vm->flags
1861  * should contain proper values and the other fields should be zero.
1862  *
1863  * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1864  */
1865 void __init vm_area_add_early(struct vm_struct *vm)
1866 {
1867 	struct vm_struct *tmp, **p;
1868 
1869 	BUG_ON(vmap_initialized);
1870 	for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
1871 		if (tmp->addr >= vm->addr) {
1872 			BUG_ON(tmp->addr < vm->addr + vm->size);
1873 			break;
1874 		} else
1875 			BUG_ON(tmp->addr + tmp->size > vm->addr);
1876 	}
1877 	vm->next = *p;
1878 	*p = vm;
1879 }
1880 
1881 /**
1882  * vm_area_register_early - register vmap area early during boot
1883  * @vm: vm_struct to register
1884  * @align: requested alignment
1885  *
1886  * This function is used to register kernel vm area before
1887  * vmalloc_init() is called.  @vm->size and @vm->flags should contain
1888  * proper values on entry and other fields should be zero.  On return,
1889  * vm->addr contains the allocated address.
1890  *
1891  * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1892  */
1893 void __init vm_area_register_early(struct vm_struct *vm, size_t align)
1894 {
1895 	static size_t vm_init_off __initdata;
1896 	unsigned long addr;
1897 
1898 	addr = ALIGN(VMALLOC_START + vm_init_off, align);
1899 	vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
1900 
1901 	vm->addr = (void *)addr;
1902 
1903 	vm_area_add_early(vm);
1904 }
1905 
1906 static void vmap_init_free_space(void)
1907 {
1908 	unsigned long vmap_start = 1;
1909 	const unsigned long vmap_end = ULONG_MAX;
1910 	struct vmap_area *busy, *free;
1911 
1912 	/*
1913 	 *     B     F     B     B     B     F
1914 	 * -|-----|.....|-----|-----|-----|.....|-
1915 	 *  |           The KVA space           |
1916 	 *  |<--------------------------------->|
1917 	 */
1918 	list_for_each_entry(busy, &vmap_area_list, list) {
1919 		if (busy->va_start - vmap_start > 0) {
1920 			free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
1921 			if (!WARN_ON_ONCE(!free)) {
1922 				free->va_start = vmap_start;
1923 				free->va_end = busy->va_start;
1924 
1925 				insert_vmap_area_augment(free, NULL,
1926 					&free_vmap_area_root,
1927 						&free_vmap_area_list);
1928 			}
1929 		}
1930 
1931 		vmap_start = busy->va_end;
1932 	}
1933 
1934 	if (vmap_end - vmap_start > 0) {
1935 		free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
1936 		if (!WARN_ON_ONCE(!free)) {
1937 			free->va_start = vmap_start;
1938 			free->va_end = vmap_end;
1939 
1940 			insert_vmap_area_augment(free, NULL,
1941 				&free_vmap_area_root,
1942 					&free_vmap_area_list);
1943 		}
1944 	}
1945 }
1946 
1947 void __init vmalloc_init(void)
1948 {
1949 	struct vmap_area *va;
1950 	struct vm_struct *tmp;
1951 	int i;
1952 
1953 	/*
1954 	 * Create the cache for vmap_area objects.
1955 	 */
1956 	vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC);
1957 
1958 	for_each_possible_cpu(i) {
1959 		struct vmap_block_queue *vbq;
1960 		struct vfree_deferred *p;
1961 
1962 		vbq = &per_cpu(vmap_block_queue, i);
1963 		spin_lock_init(&vbq->lock);
1964 		INIT_LIST_HEAD(&vbq->free);
1965 		p = &per_cpu(vfree_deferred, i);
1966 		init_llist_head(&p->list);
1967 		INIT_WORK(&p->wq, free_work);
1968 	}
1969 
1970 	/* Import existing vmlist entries. */
1971 	for (tmp = vmlist; tmp; tmp = tmp->next) {
1972 		va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
1973 		if (WARN_ON_ONCE(!va))
1974 			continue;
1975 
1976 		va->va_start = (unsigned long)tmp->addr;
1977 		va->va_end = va->va_start + tmp->size;
1978 		va->vm = tmp;
1979 		insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
1980 	}
1981 
1982 	/*
1983 	 * Now we can initialize a free vmap space.
1984 	 */
1985 	vmap_init_free_space();
1986 	vmap_initialized = true;
1987 }
1988 
1989 /**
1990  * map_kernel_range_noflush - map kernel VM area with the specified pages
1991  * @addr: start of the VM area to map
1992  * @size: size of the VM area to map
1993  * @prot: page protection flags to use
1994  * @pages: pages to map
1995  *
1996  * Map PFN_UP(@size) pages at @addr.  The VM area @addr and @size
1997  * specify should have been allocated using get_vm_area() and its
1998  * friends.
1999  *
2000  * NOTE:
2001  * This function does NOT do any cache flushing.  The caller is
2002  * responsible for calling flush_cache_vmap() on to-be-mapped areas
2003  * before calling this function.
2004  *
2005  * RETURNS:
2006  * The number of pages mapped on success, -errno on failure.
2007  */
2008 int map_kernel_range_noflush(unsigned long addr, unsigned long size,
2009 			     pgprot_t prot, struct page **pages)
2010 {
2011 	return vmap_page_range_noflush(addr, addr + size, prot, pages);
2012 }
2013 
2014 /**
2015  * unmap_kernel_range_noflush - unmap kernel VM area
2016  * @addr: start of the VM area to unmap
2017  * @size: size of the VM area to unmap
2018  *
2019  * Unmap PFN_UP(@size) pages at @addr.  The VM area @addr and @size
2020  * specify should have been allocated using get_vm_area() and its
2021  * friends.
2022  *
2023  * NOTE:
2024  * This function does NOT do any cache flushing.  The caller is
2025  * responsible for calling flush_cache_vunmap() on to-be-mapped areas
2026  * before calling this function and flush_tlb_kernel_range() after.
2027  */
2028 void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
2029 {
2030 	vunmap_page_range(addr, addr + size);
2031 }
2032 EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush);
2033 
2034 /**
2035  * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
2036  * @addr: start of the VM area to unmap
2037  * @size: size of the VM area to unmap
2038  *
2039  * Similar to unmap_kernel_range_noflush() but flushes vcache before
2040  * the unmapping and tlb after.
2041  */
2042 void unmap_kernel_range(unsigned long addr, unsigned long size)
2043 {
2044 	unsigned long end = addr + size;
2045 
2046 	flush_cache_vunmap(addr, end);
2047 	vunmap_page_range(addr, end);
2048 	flush_tlb_kernel_range(addr, end);
2049 }
2050 EXPORT_SYMBOL_GPL(unmap_kernel_range);
2051 
2052 int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages)
2053 {
2054 	unsigned long addr = (unsigned long)area->addr;
2055 	unsigned long end = addr + get_vm_area_size(area);
2056 	int err;
2057 
2058 	err = vmap_page_range(addr, end, prot, pages);
2059 
2060 	return err > 0 ? 0 : err;
2061 }
2062 EXPORT_SYMBOL_GPL(map_vm_area);
2063 
2064 static inline void setup_vmalloc_vm_locked(struct vm_struct *vm,
2065 	struct vmap_area *va, unsigned long flags, const void *caller)
2066 {
2067 	vm->flags = flags;
2068 	vm->addr = (void *)va->va_start;
2069 	vm->size = va->va_end - va->va_start;
2070 	vm->caller = caller;
2071 	va->vm = vm;
2072 }
2073 
2074 static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
2075 			      unsigned long flags, const void *caller)
2076 {
2077 	spin_lock(&vmap_area_lock);
2078 	setup_vmalloc_vm_locked(vm, va, flags, caller);
2079 	spin_unlock(&vmap_area_lock);
2080 }
2081 
2082 static void clear_vm_uninitialized_flag(struct vm_struct *vm)
2083 {
2084 	/*
2085 	 * Before removing VM_UNINITIALIZED,
2086 	 * we should make sure that vm has proper values.
2087 	 * Pair with smp_rmb() in show_numa_info().
2088 	 */
2089 	smp_wmb();
2090 	vm->flags &= ~VM_UNINITIALIZED;
2091 }
2092 
2093 static struct vm_struct *__get_vm_area_node(unsigned long size,
2094 		unsigned long align, unsigned long flags, unsigned long start,
2095 		unsigned long end, int node, gfp_t gfp_mask, const void *caller)
2096 {
2097 	struct vmap_area *va;
2098 	struct vm_struct *area;
2099 	unsigned long requested_size = size;
2100 
2101 	BUG_ON(in_interrupt());
2102 	size = PAGE_ALIGN(size);
2103 	if (unlikely(!size))
2104 		return NULL;
2105 
2106 	if (flags & VM_IOREMAP)
2107 		align = 1ul << clamp_t(int, get_count_order_long(size),
2108 				       PAGE_SHIFT, IOREMAP_MAX_ORDER);
2109 
2110 	area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
2111 	if (unlikely(!area))
2112 		return NULL;
2113 
2114 	if (!(flags & VM_NO_GUARD))
2115 		size += PAGE_SIZE;
2116 
2117 	va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
2118 	if (IS_ERR(va)) {
2119 		kfree(area);
2120 		return NULL;
2121 	}
2122 
2123 	kasan_unpoison_vmalloc((void *)va->va_start, requested_size);
2124 
2125 	setup_vmalloc_vm(area, va, flags, caller);
2126 
2127 	return area;
2128 }
2129 
2130 struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
2131 				unsigned long start, unsigned long end)
2132 {
2133 	return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
2134 				  GFP_KERNEL, __builtin_return_address(0));
2135 }
2136 EXPORT_SYMBOL_GPL(__get_vm_area);
2137 
2138 struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
2139 				       unsigned long start, unsigned long end,
2140 				       const void *caller)
2141 {
2142 	return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
2143 				  GFP_KERNEL, caller);
2144 }
2145 
2146 /**
2147  * get_vm_area - reserve a contiguous kernel virtual area
2148  * @size:	 size of the area
2149  * @flags:	 %VM_IOREMAP for I/O mappings or VM_ALLOC
2150  *
2151  * Search an area of @size in the kernel virtual mapping area,
2152  * and reserved it for out purposes.  Returns the area descriptor
2153  * on success or %NULL on failure.
2154  *
2155  * Return: the area descriptor on success or %NULL on failure.
2156  */
2157 struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
2158 {
2159 	return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
2160 				  NUMA_NO_NODE, GFP_KERNEL,
2161 				  __builtin_return_address(0));
2162 }
2163 
2164 struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
2165 				const void *caller)
2166 {
2167 	return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
2168 				  NUMA_NO_NODE, GFP_KERNEL, caller);
2169 }
2170 
2171 /**
2172  * find_vm_area - find a continuous kernel virtual area
2173  * @addr:	  base address
2174  *
2175  * Search for the kernel VM area starting at @addr, and return it.
2176  * It is up to the caller to do all required locking to keep the returned
2177  * pointer valid.
2178  *
2179  * Return: pointer to the found area or %NULL on faulure
2180  */
2181 struct vm_struct *find_vm_area(const void *addr)
2182 {
2183 	struct vmap_area *va;
2184 
2185 	va = find_vmap_area((unsigned long)addr);
2186 	if (!va)
2187 		return NULL;
2188 
2189 	return va->vm;
2190 }
2191 
2192 /**
2193  * remove_vm_area - find and remove a continuous kernel virtual area
2194  * @addr:	    base address
2195  *
2196  * Search for the kernel VM area starting at @addr, and remove it.
2197  * This function returns the found VM area, but using it is NOT safe
2198  * on SMP machines, except for its size or flags.
2199  *
2200  * Return: pointer to the found area or %NULL on faulure
2201  */
2202 struct vm_struct *remove_vm_area(const void *addr)
2203 {
2204 	struct vmap_area *va;
2205 
2206 	might_sleep();
2207 
2208 	spin_lock(&vmap_area_lock);
2209 	va = __find_vmap_area((unsigned long)addr);
2210 	if (va && va->vm) {
2211 		struct vm_struct *vm = va->vm;
2212 
2213 		va->vm = NULL;
2214 		spin_unlock(&vmap_area_lock);
2215 
2216 		kasan_free_shadow(vm);
2217 		free_unmap_vmap_area(va);
2218 
2219 		return vm;
2220 	}
2221 
2222 	spin_unlock(&vmap_area_lock);
2223 	return NULL;
2224 }
2225 
2226 static inline void set_area_direct_map(const struct vm_struct *area,
2227 				       int (*set_direct_map)(struct page *page))
2228 {
2229 	int i;
2230 
2231 	for (i = 0; i < area->nr_pages; i++)
2232 		if (page_address(area->pages[i]))
2233 			set_direct_map(area->pages[i]);
2234 }
2235 
2236 /* Handle removing and resetting vm mappings related to the vm_struct. */
2237 static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)
2238 {
2239 	unsigned long start = ULONG_MAX, end = 0;
2240 	int flush_reset = area->flags & VM_FLUSH_RESET_PERMS;
2241 	int flush_dmap = 0;
2242 	int i;
2243 
2244 	remove_vm_area(area->addr);
2245 
2246 	/* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */
2247 	if (!flush_reset)
2248 		return;
2249 
2250 	/*
2251 	 * If not deallocating pages, just do the flush of the VM area and
2252 	 * return.
2253 	 */
2254 	if (!deallocate_pages) {
2255 		vm_unmap_aliases();
2256 		return;
2257 	}
2258 
2259 	/*
2260 	 * If execution gets here, flush the vm mapping and reset the direct
2261 	 * map. Find the start and end range of the direct mappings to make sure
2262 	 * the vm_unmap_aliases() flush includes the direct map.
2263 	 */
2264 	for (i = 0; i < area->nr_pages; i++) {
2265 		unsigned long addr = (unsigned long)page_address(area->pages[i]);
2266 		if (addr) {
2267 			start = min(addr, start);
2268 			end = max(addr + PAGE_SIZE, end);
2269 			flush_dmap = 1;
2270 		}
2271 	}
2272 
2273 	/*
2274 	 * Set direct map to something invalid so that it won't be cached if
2275 	 * there are any accesses after the TLB flush, then flush the TLB and
2276 	 * reset the direct map permissions to the default.
2277 	 */
2278 	set_area_direct_map(area, set_direct_map_invalid_noflush);
2279 	_vm_unmap_aliases(start, end, flush_dmap);
2280 	set_area_direct_map(area, set_direct_map_default_noflush);
2281 }
2282 
2283 static void __vunmap(const void *addr, int deallocate_pages)
2284 {
2285 	struct vm_struct *area;
2286 
2287 	if (!addr)
2288 		return;
2289 
2290 	if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n",
2291 			addr))
2292 		return;
2293 
2294 	area = find_vm_area(addr);
2295 	if (unlikely(!area)) {
2296 		WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
2297 				addr);
2298 		return;
2299 	}
2300 
2301 	debug_check_no_locks_freed(area->addr, get_vm_area_size(area));
2302 	debug_check_no_obj_freed(area->addr, get_vm_area_size(area));
2303 
2304 	kasan_poison_vmalloc(area->addr, area->size);
2305 
2306 	vm_remove_mappings(area, deallocate_pages);
2307 
2308 	if (deallocate_pages) {
2309 		int i;
2310 
2311 		for (i = 0; i < area->nr_pages; i++) {
2312 			struct page *page = area->pages[i];
2313 
2314 			BUG_ON(!page);
2315 			__free_pages(page, 0);
2316 		}
2317 		atomic_long_sub(area->nr_pages, &nr_vmalloc_pages);
2318 
2319 		kvfree(area->pages);
2320 	}
2321 
2322 	kfree(area);
2323 	return;
2324 }
2325 
2326 static inline void __vfree_deferred(const void *addr)
2327 {
2328 	/*
2329 	 * Use raw_cpu_ptr() because this can be called from preemptible
2330 	 * context. Preemption is absolutely fine here, because the llist_add()
2331 	 * implementation is lockless, so it works even if we are adding to
2332 	 * nother cpu's list.  schedule_work() should be fine with this too.
2333 	 */
2334 	struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred);
2335 
2336 	if (llist_add((struct llist_node *)addr, &p->list))
2337 		schedule_work(&p->wq);
2338 }
2339 
2340 /**
2341  * vfree_atomic - release memory allocated by vmalloc()
2342  * @addr:	  memory base address
2343  *
2344  * This one is just like vfree() but can be called in any atomic context
2345  * except NMIs.
2346  */
2347 void vfree_atomic(const void *addr)
2348 {
2349 	BUG_ON(in_nmi());
2350 
2351 	kmemleak_free(addr);
2352 
2353 	if (!addr)
2354 		return;
2355 	__vfree_deferred(addr);
2356 }
2357 
2358 static void __vfree(const void *addr)
2359 {
2360 	if (unlikely(in_interrupt()))
2361 		__vfree_deferred(addr);
2362 	else
2363 		__vunmap(addr, 1);
2364 }
2365 
2366 /**
2367  * vfree - release memory allocated by vmalloc()
2368  * @addr:  memory base address
2369  *
2370  * Free the virtually continuous memory area starting at @addr, as
2371  * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
2372  * NULL, no operation is performed.
2373  *
2374  * Must not be called in NMI context (strictly speaking, only if we don't
2375  * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
2376  * conventions for vfree() arch-depenedent would be a really bad idea)
2377  *
2378  * May sleep if called *not* from interrupt context.
2379  *
2380  * NOTE: assumes that the object at @addr has a size >= sizeof(llist_node)
2381  */
2382 void vfree(const void *addr)
2383 {
2384 	BUG_ON(in_nmi());
2385 
2386 	kmemleak_free(addr);
2387 
2388 	might_sleep_if(!in_interrupt());
2389 
2390 	if (!addr)
2391 		return;
2392 
2393 	__vfree(addr);
2394 }
2395 EXPORT_SYMBOL(vfree);
2396 
2397 /**
2398  * vunmap - release virtual mapping obtained by vmap()
2399  * @addr:   memory base address
2400  *
2401  * Free the virtually contiguous memory area starting at @addr,
2402  * which was created from the page array passed to vmap().
2403  *
2404  * Must not be called in interrupt context.
2405  */
2406 void vunmap(const void *addr)
2407 {
2408 	BUG_ON(in_interrupt());
2409 	might_sleep();
2410 	if (addr)
2411 		__vunmap(addr, 0);
2412 }
2413 EXPORT_SYMBOL(vunmap);
2414 
2415 /**
2416  * vmap - map an array of pages into virtually contiguous space
2417  * @pages: array of page pointers
2418  * @count: number of pages to map
2419  * @flags: vm_area->flags
2420  * @prot: page protection for the mapping
2421  *
2422  * Maps @count pages from @pages into contiguous kernel virtual
2423  * space.
2424  *
2425  * Return: the address of the area or %NULL on failure
2426  */
2427 void *vmap(struct page **pages, unsigned int count,
2428 	   unsigned long flags, pgprot_t prot)
2429 {
2430 	struct vm_struct *area;
2431 	unsigned long size;		/* In bytes */
2432 
2433 	might_sleep();
2434 
2435 	if (count > totalram_pages())
2436 		return NULL;
2437 
2438 	size = (unsigned long)count << PAGE_SHIFT;
2439 	area = get_vm_area_caller(size, flags, __builtin_return_address(0));
2440 	if (!area)
2441 		return NULL;
2442 
2443 	if (map_vm_area(area, prot, pages)) {
2444 		vunmap(area->addr);
2445 		return NULL;
2446 	}
2447 
2448 	return area->addr;
2449 }
2450 EXPORT_SYMBOL(vmap);
2451 
2452 static void *__vmalloc_node(unsigned long size, unsigned long align,
2453 			    gfp_t gfp_mask, pgprot_t prot,
2454 			    int node, const void *caller);
2455 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
2456 				 pgprot_t prot, int node)
2457 {
2458 	struct page **pages;
2459 	unsigned int nr_pages, array_size, i;
2460 	const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
2461 	const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN;
2462 	const gfp_t highmem_mask = (gfp_mask & (GFP_DMA | GFP_DMA32)) ?
2463 					0 :
2464 					__GFP_HIGHMEM;
2465 
2466 	nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
2467 	array_size = (nr_pages * sizeof(struct page *));
2468 
2469 	/* Please note that the recursion is strictly bounded. */
2470 	if (array_size > PAGE_SIZE) {
2471 		pages = __vmalloc_node(array_size, 1, nested_gfp|highmem_mask,
2472 				PAGE_KERNEL, node, area->caller);
2473 	} else {
2474 		pages = kmalloc_node(array_size, nested_gfp, node);
2475 	}
2476 
2477 	if (!pages) {
2478 		remove_vm_area(area->addr);
2479 		kfree(area);
2480 		return NULL;
2481 	}
2482 
2483 	area->pages = pages;
2484 	area->nr_pages = nr_pages;
2485 
2486 	for (i = 0; i < area->nr_pages; i++) {
2487 		struct page *page;
2488 
2489 		if (node == NUMA_NO_NODE)
2490 			page = alloc_page(alloc_mask|highmem_mask);
2491 		else
2492 			page = alloc_pages_node(node, alloc_mask|highmem_mask, 0);
2493 
2494 		if (unlikely(!page)) {
2495 			/* Successfully allocated i pages, free them in __vunmap() */
2496 			area->nr_pages = i;
2497 			atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
2498 			goto fail;
2499 		}
2500 		area->pages[i] = page;
2501 		if (gfpflags_allow_blocking(gfp_mask))
2502 			cond_resched();
2503 	}
2504 	atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
2505 
2506 	if (map_vm_area(area, prot, pages))
2507 		goto fail;
2508 	return area->addr;
2509 
2510 fail:
2511 	warn_alloc(gfp_mask, NULL,
2512 			  "vmalloc: allocation failure, allocated %ld of %ld bytes",
2513 			  (area->nr_pages*PAGE_SIZE), area->size);
2514 	__vfree(area->addr);
2515 	return NULL;
2516 }
2517 
2518 /**
2519  * __vmalloc_node_range - allocate virtually contiguous memory
2520  * @size:		  allocation size
2521  * @align:		  desired alignment
2522  * @start:		  vm area range start
2523  * @end:		  vm area range end
2524  * @gfp_mask:		  flags for the page level allocator
2525  * @prot:		  protection mask for the allocated pages
2526  * @vm_flags:		  additional vm area flags (e.g. %VM_NO_GUARD)
2527  * @node:		  node to use for allocation or NUMA_NO_NODE
2528  * @caller:		  caller's return address
2529  *
2530  * Allocate enough pages to cover @size from the page level
2531  * allocator with @gfp_mask flags.  Map them into contiguous
2532  * kernel virtual space, using a pagetable protection of @prot.
2533  *
2534  * Return: the address of the area or %NULL on failure
2535  */
2536 void *__vmalloc_node_range(unsigned long size, unsigned long align,
2537 			unsigned long start, unsigned long end, gfp_t gfp_mask,
2538 			pgprot_t prot, unsigned long vm_flags, int node,
2539 			const void *caller)
2540 {
2541 	struct vm_struct *area;
2542 	void *addr;
2543 	unsigned long real_size = size;
2544 
2545 	size = PAGE_ALIGN(size);
2546 	if (!size || (size >> PAGE_SHIFT) > totalram_pages())
2547 		goto fail;
2548 
2549 	area = __get_vm_area_node(real_size, align, VM_ALLOC | VM_UNINITIALIZED |
2550 				vm_flags, start, end, node, gfp_mask, caller);
2551 	if (!area)
2552 		goto fail;
2553 
2554 	addr = __vmalloc_area_node(area, gfp_mask, prot, node);
2555 	if (!addr)
2556 		return NULL;
2557 
2558 	/*
2559 	 * In this function, newly allocated vm_struct has VM_UNINITIALIZED
2560 	 * flag. It means that vm_struct is not fully initialized.
2561 	 * Now, it is fully initialized, so remove this flag here.
2562 	 */
2563 	clear_vm_uninitialized_flag(area);
2564 
2565 	kmemleak_vmalloc(area, size, gfp_mask);
2566 
2567 	return addr;
2568 
2569 fail:
2570 	warn_alloc(gfp_mask, NULL,
2571 			  "vmalloc: allocation failure: %lu bytes", real_size);
2572 	return NULL;
2573 }
2574 
2575 /*
2576  * This is only for performance analysis of vmalloc and stress purpose.
2577  * It is required by vmalloc test module, therefore do not use it other
2578  * than that.
2579  */
2580 #ifdef CONFIG_TEST_VMALLOC_MODULE
2581 EXPORT_SYMBOL_GPL(__vmalloc_node_range);
2582 #endif
2583 
2584 /**
2585  * __vmalloc_node - allocate virtually contiguous memory
2586  * @size:	    allocation size
2587  * @align:	    desired alignment
2588  * @gfp_mask:	    flags for the page level allocator
2589  * @prot:	    protection mask for the allocated pages
2590  * @node:	    node to use for allocation or NUMA_NO_NODE
2591  * @caller:	    caller's return address
2592  *
2593  * Allocate enough pages to cover @size from the page level
2594  * allocator with @gfp_mask flags.  Map them into contiguous
2595  * kernel virtual space, using a pagetable protection of @prot.
2596  *
2597  * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL
2598  * and __GFP_NOFAIL are not supported
2599  *
2600  * Any use of gfp flags outside of GFP_KERNEL should be consulted
2601  * with mm people.
2602  *
2603  * Return: pointer to the allocated memory or %NULL on error
2604  */
2605 static void *__vmalloc_node(unsigned long size, unsigned long align,
2606 			    gfp_t gfp_mask, pgprot_t prot,
2607 			    int node, const void *caller)
2608 {
2609 	return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
2610 				gfp_mask, prot, 0, node, caller);
2611 }
2612 
2613 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
2614 {
2615 	return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE,
2616 				__builtin_return_address(0));
2617 }
2618 EXPORT_SYMBOL(__vmalloc);
2619 
2620 static inline void *__vmalloc_node_flags(unsigned long size,
2621 					int node, gfp_t flags)
2622 {
2623 	return __vmalloc_node(size, 1, flags, PAGE_KERNEL,
2624 					node, __builtin_return_address(0));
2625 }
2626 
2627 
2628 void *__vmalloc_node_flags_caller(unsigned long size, int node, gfp_t flags,
2629 				  void *caller)
2630 {
2631 	return __vmalloc_node(size, 1, flags, PAGE_KERNEL, node, caller);
2632 }
2633 
2634 /**
2635  * vmalloc - allocate virtually contiguous memory
2636  * @size:    allocation size
2637  *
2638  * Allocate enough pages to cover @size from the page level
2639  * allocator and map them into contiguous kernel virtual space.
2640  *
2641  * For tight control over page level allocator and protection flags
2642  * use __vmalloc() instead.
2643  *
2644  * Return: pointer to the allocated memory or %NULL on error
2645  */
2646 void *vmalloc(unsigned long size)
2647 {
2648 	return __vmalloc_node_flags(size, NUMA_NO_NODE,
2649 				    GFP_KERNEL);
2650 }
2651 EXPORT_SYMBOL(vmalloc);
2652 
2653 /**
2654  * vzalloc - allocate virtually contiguous memory with zero fill
2655  * @size:    allocation size
2656  *
2657  * Allocate enough pages to cover @size from the page level
2658  * allocator and map them into contiguous kernel virtual space.
2659  * The memory allocated is set to zero.
2660  *
2661  * For tight control over page level allocator and protection flags
2662  * use __vmalloc() instead.
2663  *
2664  * Return: pointer to the allocated memory or %NULL on error
2665  */
2666 void *vzalloc(unsigned long size)
2667 {
2668 	return __vmalloc_node_flags(size, NUMA_NO_NODE,
2669 				GFP_KERNEL | __GFP_ZERO);
2670 }
2671 EXPORT_SYMBOL(vzalloc);
2672 
2673 /**
2674  * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
2675  * @size: allocation size
2676  *
2677  * The resulting memory area is zeroed so it can be mapped to userspace
2678  * without leaking data.
2679  *
2680  * Return: pointer to the allocated memory or %NULL on error
2681  */
2682 void *vmalloc_user(unsigned long size)
2683 {
2684 	return __vmalloc_node_range(size, SHMLBA,  VMALLOC_START, VMALLOC_END,
2685 				    GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL,
2686 				    VM_USERMAP, NUMA_NO_NODE,
2687 				    __builtin_return_address(0));
2688 }
2689 EXPORT_SYMBOL(vmalloc_user);
2690 
2691 /**
2692  * vmalloc_node - allocate memory on a specific node
2693  * @size:	  allocation size
2694  * @node:	  numa node
2695  *
2696  * Allocate enough pages to cover @size from the page level
2697  * allocator and map them into contiguous kernel virtual space.
2698  *
2699  * For tight control over page level allocator and protection flags
2700  * use __vmalloc() instead.
2701  *
2702  * Return: pointer to the allocated memory or %NULL on error
2703  */
2704 void *vmalloc_node(unsigned long size, int node)
2705 {
2706 	return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL,
2707 					node, __builtin_return_address(0));
2708 }
2709 EXPORT_SYMBOL(vmalloc_node);
2710 
2711 /**
2712  * vzalloc_node - allocate memory on a specific node with zero fill
2713  * @size:	allocation size
2714  * @node:	numa node
2715  *
2716  * Allocate enough pages to cover @size from the page level
2717  * allocator and map them into contiguous kernel virtual space.
2718  * The memory allocated is set to zero.
2719  *
2720  * For tight control over page level allocator and protection flags
2721  * use __vmalloc_node() instead.
2722  *
2723  * Return: pointer to the allocated memory or %NULL on error
2724  */
2725 void *vzalloc_node(unsigned long size, int node)
2726 {
2727 	return __vmalloc_node_flags(size, node,
2728 			 GFP_KERNEL | __GFP_ZERO);
2729 }
2730 EXPORT_SYMBOL(vzalloc_node);
2731 
2732 /**
2733  * vmalloc_user_node_flags - allocate memory for userspace on a specific node
2734  * @size: allocation size
2735  * @node: numa node
2736  * @flags: flags for the page level allocator
2737  *
2738  * The resulting memory area is zeroed so it can be mapped to userspace
2739  * without leaking data.
2740  *
2741  * Return: pointer to the allocated memory or %NULL on error
2742  */
2743 void *vmalloc_user_node_flags(unsigned long size, int node, gfp_t flags)
2744 {
2745 	return __vmalloc_node_range(size, SHMLBA,  VMALLOC_START, VMALLOC_END,
2746 				    flags | __GFP_ZERO, PAGE_KERNEL,
2747 				    VM_USERMAP, node,
2748 				    __builtin_return_address(0));
2749 }
2750 EXPORT_SYMBOL(vmalloc_user_node_flags);
2751 
2752 /**
2753  * vmalloc_exec - allocate virtually contiguous, executable memory
2754  * @size:	  allocation size
2755  *
2756  * Kernel-internal function to allocate enough pages to cover @size
2757  * the page level allocator and map them into contiguous and
2758  * executable kernel virtual space.
2759  *
2760  * For tight control over page level allocator and protection flags
2761  * use __vmalloc() instead.
2762  *
2763  * Return: pointer to the allocated memory or %NULL on error
2764  */
2765 void *vmalloc_exec(unsigned long size)
2766 {
2767 	return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,
2768 			GFP_KERNEL, PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS,
2769 			NUMA_NO_NODE, __builtin_return_address(0));
2770 }
2771 
2772 #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
2773 #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL)
2774 #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
2775 #define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL)
2776 #else
2777 /*
2778  * 64b systems should always have either DMA or DMA32 zones. For others
2779  * GFP_DMA32 should do the right thing and use the normal zone.
2780  */
2781 #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
2782 #endif
2783 
2784 /**
2785  * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
2786  * @size:	allocation size
2787  *
2788  * Allocate enough 32bit PA addressable pages to cover @size from the
2789  * page level allocator and map them into contiguous kernel virtual space.
2790  *
2791  * Return: pointer to the allocated memory or %NULL on error
2792  */
2793 void *vmalloc_32(unsigned long size)
2794 {
2795 	return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
2796 			      NUMA_NO_NODE, __builtin_return_address(0));
2797 }
2798 EXPORT_SYMBOL(vmalloc_32);
2799 
2800 /**
2801  * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
2802  * @size:	     allocation size
2803  *
2804  * The resulting memory area is 32bit addressable and zeroed so it can be
2805  * mapped to userspace without leaking data.
2806  *
2807  * Return: pointer to the allocated memory or %NULL on error
2808  */
2809 void *vmalloc_32_user(unsigned long size)
2810 {
2811 	return __vmalloc_node_range(size, SHMLBA,  VMALLOC_START, VMALLOC_END,
2812 				    GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
2813 				    VM_USERMAP, NUMA_NO_NODE,
2814 				    __builtin_return_address(0));
2815 }
2816 EXPORT_SYMBOL(vmalloc_32_user);
2817 
2818 /*
2819  * small helper routine , copy contents to buf from addr.
2820  * If the page is not present, fill zero.
2821  */
2822 
2823 static int aligned_vread(char *buf, char *addr, unsigned long count)
2824 {
2825 	struct page *p;
2826 	int copied = 0;
2827 
2828 	while (count) {
2829 		unsigned long offset, length;
2830 
2831 		offset = offset_in_page(addr);
2832 		length = PAGE_SIZE - offset;
2833 		if (length > count)
2834 			length = count;
2835 		p = vmalloc_to_page(addr);
2836 		/*
2837 		 * To do safe access to this _mapped_ area, we need
2838 		 * lock. But adding lock here means that we need to add
2839 		 * overhead of vmalloc()/vfree() calles for this _debug_
2840 		 * interface, rarely used. Instead of that, we'll use
2841 		 * kmap() and get small overhead in this access function.
2842 		 */
2843 		if (p) {
2844 			/*
2845 			 * we can expect USER0 is not used (see vread/vwrite's
2846 			 * function description)
2847 			 */
2848 			void *map = kmap_atomic(p);
2849 			memcpy(buf, map + offset, length);
2850 			kunmap_atomic(map);
2851 		} else
2852 			memset(buf, 0, length);
2853 
2854 		addr += length;
2855 		buf += length;
2856 		copied += length;
2857 		count -= length;
2858 	}
2859 	return copied;
2860 }
2861 
2862 static int aligned_vwrite(char *buf, char *addr, unsigned long count)
2863 {
2864 	struct page *p;
2865 	int copied = 0;
2866 
2867 	while (count) {
2868 		unsigned long offset, length;
2869 
2870 		offset = offset_in_page(addr);
2871 		length = PAGE_SIZE - offset;
2872 		if (length > count)
2873 			length = count;
2874 		p = vmalloc_to_page(addr);
2875 		/*
2876 		 * To do safe access to this _mapped_ area, we need
2877 		 * lock. But adding lock here means that we need to add
2878 		 * overhead of vmalloc()/vfree() calles for this _debug_
2879 		 * interface, rarely used. Instead of that, we'll use
2880 		 * kmap() and get small overhead in this access function.
2881 		 */
2882 		if (p) {
2883 			/*
2884 			 * we can expect USER0 is not used (see vread/vwrite's
2885 			 * function description)
2886 			 */
2887 			void *map = kmap_atomic(p);
2888 			memcpy(map + offset, buf, length);
2889 			kunmap_atomic(map);
2890 		}
2891 		addr += length;
2892 		buf += length;
2893 		copied += length;
2894 		count -= length;
2895 	}
2896 	return copied;
2897 }
2898 
2899 /**
2900  * vread() - read vmalloc area in a safe way.
2901  * @buf:     buffer for reading data
2902  * @addr:    vm address.
2903  * @count:   number of bytes to be read.
2904  *
2905  * This function checks that addr is a valid vmalloc'ed area, and
2906  * copy data from that area to a given buffer. If the given memory range
2907  * of [addr...addr+count) includes some valid address, data is copied to
2908  * proper area of @buf. If there are memory holes, they'll be zero-filled.
2909  * IOREMAP area is treated as memory hole and no copy is done.
2910  *
2911  * If [addr...addr+count) doesn't includes any intersects with alive
2912  * vm_struct area, returns 0. @buf should be kernel's buffer.
2913  *
2914  * Note: In usual ops, vread() is never necessary because the caller
2915  * should know vmalloc() area is valid and can use memcpy().
2916  * This is for routines which have to access vmalloc area without
2917  * any information, as /dev/kmem.
2918  *
2919  * Return: number of bytes for which addr and buf should be increased
2920  * (same number as @count) or %0 if [addr...addr+count) doesn't
2921  * include any intersection with valid vmalloc area
2922  */
2923 long vread(char *buf, char *addr, unsigned long count)
2924 {
2925 	struct vmap_area *va;
2926 	struct vm_struct *vm;
2927 	char *vaddr, *buf_start = buf;
2928 	unsigned long buflen = count;
2929 	unsigned long n;
2930 
2931 	/* Don't allow overflow */
2932 	if ((unsigned long) addr + count < count)
2933 		count = -(unsigned long) addr;
2934 
2935 	spin_lock(&vmap_area_lock);
2936 	list_for_each_entry(va, &vmap_area_list, list) {
2937 		if (!count)
2938 			break;
2939 
2940 		if (!va->vm)
2941 			continue;
2942 
2943 		vm = va->vm;
2944 		vaddr = (char *) vm->addr;
2945 		if (addr >= vaddr + get_vm_area_size(vm))
2946 			continue;
2947 		while (addr < vaddr) {
2948 			if (count == 0)
2949 				goto finished;
2950 			*buf = '\0';
2951 			buf++;
2952 			addr++;
2953 			count--;
2954 		}
2955 		n = vaddr + get_vm_area_size(vm) - addr;
2956 		if (n > count)
2957 			n = count;
2958 		if (!(vm->flags & VM_IOREMAP))
2959 			aligned_vread(buf, addr, n);
2960 		else /* IOREMAP area is treated as memory hole */
2961 			memset(buf, 0, n);
2962 		buf += n;
2963 		addr += n;
2964 		count -= n;
2965 	}
2966 finished:
2967 	spin_unlock(&vmap_area_lock);
2968 
2969 	if (buf == buf_start)
2970 		return 0;
2971 	/* zero-fill memory holes */
2972 	if (buf != buf_start + buflen)
2973 		memset(buf, 0, buflen - (buf - buf_start));
2974 
2975 	return buflen;
2976 }
2977 
2978 /**
2979  * vwrite() - write vmalloc area in a safe way.
2980  * @buf:      buffer for source data
2981  * @addr:     vm address.
2982  * @count:    number of bytes to be read.
2983  *
2984  * This function checks that addr is a valid vmalloc'ed area, and
2985  * copy data from a buffer to the given addr. If specified range of
2986  * [addr...addr+count) includes some valid address, data is copied from
2987  * proper area of @buf. If there are memory holes, no copy to hole.
2988  * IOREMAP area is treated as memory hole and no copy is done.
2989  *
2990  * If [addr...addr+count) doesn't includes any intersects with alive
2991  * vm_struct area, returns 0. @buf should be kernel's buffer.
2992  *
2993  * Note: In usual ops, vwrite() is never necessary because the caller
2994  * should know vmalloc() area is valid and can use memcpy().
2995  * This is for routines which have to access vmalloc area without
2996  * any information, as /dev/kmem.
2997  *
2998  * Return: number of bytes for which addr and buf should be
2999  * increased (same number as @count) or %0 if [addr...addr+count)
3000  * doesn't include any intersection with valid vmalloc area
3001  */
3002 long vwrite(char *buf, char *addr, unsigned long count)
3003 {
3004 	struct vmap_area *va;
3005 	struct vm_struct *vm;
3006 	char *vaddr;
3007 	unsigned long n, buflen;
3008 	int copied = 0;
3009 
3010 	/* Don't allow overflow */
3011 	if ((unsigned long) addr + count < count)
3012 		count = -(unsigned long) addr;
3013 	buflen = count;
3014 
3015 	spin_lock(&vmap_area_lock);
3016 	list_for_each_entry(va, &vmap_area_list, list) {
3017 		if (!count)
3018 			break;
3019 
3020 		if (!va->vm)
3021 			continue;
3022 
3023 		vm = va->vm;
3024 		vaddr = (char *) vm->addr;
3025 		if (addr >= vaddr + get_vm_area_size(vm))
3026 			continue;
3027 		while (addr < vaddr) {
3028 			if (count == 0)
3029 				goto finished;
3030 			buf++;
3031 			addr++;
3032 			count--;
3033 		}
3034 		n = vaddr + get_vm_area_size(vm) - addr;
3035 		if (n > count)
3036 			n = count;
3037 		if (!(vm->flags & VM_IOREMAP)) {
3038 			aligned_vwrite(buf, addr, n);
3039 			copied++;
3040 		}
3041 		buf += n;
3042 		addr += n;
3043 		count -= n;
3044 	}
3045 finished:
3046 	spin_unlock(&vmap_area_lock);
3047 	if (!copied)
3048 		return 0;
3049 	return buflen;
3050 }
3051 
3052 /**
3053  * remap_vmalloc_range_partial - map vmalloc pages to userspace
3054  * @vma:		vma to cover
3055  * @uaddr:		target user address to start at
3056  * @kaddr:		virtual address of vmalloc kernel memory
3057  * @size:		size of map area
3058  *
3059  * Returns:	0 for success, -Exxx on failure
3060  *
3061  * This function checks that @kaddr is a valid vmalloc'ed area,
3062  * and that it is big enough to cover the range starting at
3063  * @uaddr in @vma. Will return failure if that criteria isn't
3064  * met.
3065  *
3066  * Similar to remap_pfn_range() (see mm/memory.c)
3067  */
3068 int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr,
3069 				void *kaddr, unsigned long size)
3070 {
3071 	struct vm_struct *area;
3072 
3073 	size = PAGE_ALIGN(size);
3074 
3075 	if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
3076 		return -EINVAL;
3077 
3078 	area = find_vm_area(kaddr);
3079 	if (!area)
3080 		return -EINVAL;
3081 
3082 	if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT)))
3083 		return -EINVAL;
3084 
3085 	if (kaddr + size > area->addr + get_vm_area_size(area))
3086 		return -EINVAL;
3087 
3088 	do {
3089 		struct page *page = vmalloc_to_page(kaddr);
3090 		int ret;
3091 
3092 		ret = vm_insert_page(vma, uaddr, page);
3093 		if (ret)
3094 			return ret;
3095 
3096 		uaddr += PAGE_SIZE;
3097 		kaddr += PAGE_SIZE;
3098 		size -= PAGE_SIZE;
3099 	} while (size > 0);
3100 
3101 	vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
3102 
3103 	return 0;
3104 }
3105 EXPORT_SYMBOL(remap_vmalloc_range_partial);
3106 
3107 /**
3108  * remap_vmalloc_range - map vmalloc pages to userspace
3109  * @vma:		vma to cover (map full range of vma)
3110  * @addr:		vmalloc memory
3111  * @pgoff:		number of pages into addr before first page to map
3112  *
3113  * Returns:	0 for success, -Exxx on failure
3114  *
3115  * This function checks that addr is a valid vmalloc'ed area, and
3116  * that it is big enough to cover the vma. Will return failure if
3117  * that criteria isn't met.
3118  *
3119  * Similar to remap_pfn_range() (see mm/memory.c)
3120  */
3121 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
3122 						unsigned long pgoff)
3123 {
3124 	return remap_vmalloc_range_partial(vma, vma->vm_start,
3125 					   addr + (pgoff << PAGE_SHIFT),
3126 					   vma->vm_end - vma->vm_start);
3127 }
3128 EXPORT_SYMBOL(remap_vmalloc_range);
3129 
3130 /*
3131  * Implement stubs for vmalloc_sync_[un]mappings () if the architecture chose
3132  * not to have one.
3133  *
3134  * The purpose of this function is to make sure the vmalloc area
3135  * mappings are identical in all page-tables in the system.
3136  */
3137 void __weak vmalloc_sync_mappings(void)
3138 {
3139 }
3140 
3141 void __weak vmalloc_sync_unmappings(void)
3142 {
3143 }
3144 
3145 static int f(pte_t *pte, unsigned long addr, void *data)
3146 {
3147 	pte_t ***p = data;
3148 
3149 	if (p) {
3150 		*(*p) = pte;
3151 		(*p)++;
3152 	}
3153 	return 0;
3154 }
3155 
3156 /**
3157  * alloc_vm_area - allocate a range of kernel address space
3158  * @size:	   size of the area
3159  * @ptes:	   returns the PTEs for the address space
3160  *
3161  * Returns:	NULL on failure, vm_struct on success
3162  *
3163  * This function reserves a range of kernel address space, and
3164  * allocates pagetables to map that range.  No actual mappings
3165  * are created.
3166  *
3167  * If @ptes is non-NULL, pointers to the PTEs (in init_mm)
3168  * allocated for the VM area are returned.
3169  */
3170 struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
3171 {
3172 	struct vm_struct *area;
3173 
3174 	area = get_vm_area_caller(size, VM_IOREMAP,
3175 				__builtin_return_address(0));
3176 	if (area == NULL)
3177 		return NULL;
3178 
3179 	/*
3180 	 * This ensures that page tables are constructed for this region
3181 	 * of kernel virtual address space and mapped into init_mm.
3182 	 */
3183 	if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
3184 				size, f, ptes ? &ptes : NULL)) {
3185 		free_vm_area(area);
3186 		return NULL;
3187 	}
3188 
3189 	return area;
3190 }
3191 EXPORT_SYMBOL_GPL(alloc_vm_area);
3192 
3193 void free_vm_area(struct vm_struct *area)
3194 {
3195 	struct vm_struct *ret;
3196 	ret = remove_vm_area(area->addr);
3197 	BUG_ON(ret != area);
3198 	kfree(area);
3199 }
3200 EXPORT_SYMBOL_GPL(free_vm_area);
3201 
3202 #ifdef CONFIG_SMP
3203 static struct vmap_area *node_to_va(struct rb_node *n)
3204 {
3205 	return rb_entry_safe(n, struct vmap_area, rb_node);
3206 }
3207 
3208 /**
3209  * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to
3210  * @addr: target address
3211  *
3212  * Returns: vmap_area if it is found. If there is no such area
3213  *   the first highest(reverse order) vmap_area is returned
3214  *   i.e. va->va_start < addr && va->va_end < addr or NULL
3215  *   if there are no any areas before @addr.
3216  */
3217 static struct vmap_area *
3218 pvm_find_va_enclose_addr(unsigned long addr)
3219 {
3220 	struct vmap_area *va, *tmp;
3221 	struct rb_node *n;
3222 
3223 	n = free_vmap_area_root.rb_node;
3224 	va = NULL;
3225 
3226 	while (n) {
3227 		tmp = rb_entry(n, struct vmap_area, rb_node);
3228 		if (tmp->va_start <= addr) {
3229 			va = tmp;
3230 			if (tmp->va_end >= addr)
3231 				break;
3232 
3233 			n = n->rb_right;
3234 		} else {
3235 			n = n->rb_left;
3236 		}
3237 	}
3238 
3239 	return va;
3240 }
3241 
3242 /**
3243  * pvm_determine_end_from_reverse - find the highest aligned address
3244  * of free block below VMALLOC_END
3245  * @va:
3246  *   in - the VA we start the search(reverse order);
3247  *   out - the VA with the highest aligned end address.
3248  *
3249  * Returns: determined end address within vmap_area
3250  */
3251 static unsigned long
3252 pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align)
3253 {
3254 	unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
3255 	unsigned long addr;
3256 
3257 	if (likely(*va)) {
3258 		list_for_each_entry_from_reverse((*va),
3259 				&free_vmap_area_list, list) {
3260 			addr = min((*va)->va_end & ~(align - 1), vmalloc_end);
3261 			if ((*va)->va_start < addr)
3262 				return addr;
3263 		}
3264 	}
3265 
3266 	return 0;
3267 }
3268 
3269 /**
3270  * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
3271  * @offsets: array containing offset of each area
3272  * @sizes: array containing size of each area
3273  * @nr_vms: the number of areas to allocate
3274  * @align: alignment, all entries in @offsets and @sizes must be aligned to this
3275  *
3276  * Returns: kmalloc'd vm_struct pointer array pointing to allocated
3277  *	    vm_structs on success, %NULL on failure
3278  *
3279  * Percpu allocator wants to use congruent vm areas so that it can
3280  * maintain the offsets among percpu areas.  This function allocates
3281  * congruent vmalloc areas for it with GFP_KERNEL.  These areas tend to
3282  * be scattered pretty far, distance between two areas easily going up
3283  * to gigabytes.  To avoid interacting with regular vmallocs, these
3284  * areas are allocated from top.
3285  *
3286  * Despite its complicated look, this allocator is rather simple. It
3287  * does everything top-down and scans free blocks from the end looking
3288  * for matching base. While scanning, if any of the areas do not fit the
3289  * base address is pulled down to fit the area. Scanning is repeated till
3290  * all the areas fit and then all necessary data structures are inserted
3291  * and the result is returned.
3292  */
3293 struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
3294 				     const size_t *sizes, int nr_vms,
3295 				     size_t align)
3296 {
3297 	const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
3298 	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
3299 	struct vmap_area **vas, *va;
3300 	struct vm_struct **vms;
3301 	int area, area2, last_area, term_area;
3302 	unsigned long base, start, size, end, last_end, orig_start, orig_end;
3303 	bool purged = false;
3304 	enum fit_type type;
3305 
3306 	/* verify parameters and allocate data structures */
3307 	BUG_ON(offset_in_page(align) || !is_power_of_2(align));
3308 	for (last_area = 0, area = 0; area < nr_vms; area++) {
3309 		start = offsets[area];
3310 		end = start + sizes[area];
3311 
3312 		/* is everything aligned properly? */
3313 		BUG_ON(!IS_ALIGNED(offsets[area], align));
3314 		BUG_ON(!IS_ALIGNED(sizes[area], align));
3315 
3316 		/* detect the area with the highest address */
3317 		if (start > offsets[last_area])
3318 			last_area = area;
3319 
3320 		for (area2 = area + 1; area2 < nr_vms; area2++) {
3321 			unsigned long start2 = offsets[area2];
3322 			unsigned long end2 = start2 + sizes[area2];
3323 
3324 			BUG_ON(start2 < end && start < end2);
3325 		}
3326 	}
3327 	last_end = offsets[last_area] + sizes[last_area];
3328 
3329 	if (vmalloc_end - vmalloc_start < last_end) {
3330 		WARN_ON(true);
3331 		return NULL;
3332 	}
3333 
3334 	vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL);
3335 	vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL);
3336 	if (!vas || !vms)
3337 		goto err_free2;
3338 
3339 	for (area = 0; area < nr_vms; area++) {
3340 		vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);
3341 		vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
3342 		if (!vas[area] || !vms[area])
3343 			goto err_free;
3344 	}
3345 retry:
3346 	spin_lock(&free_vmap_area_lock);
3347 
3348 	/* start scanning - we scan from the top, begin with the last area */
3349 	area = term_area = last_area;
3350 	start = offsets[area];
3351 	end = start + sizes[area];
3352 
3353 	va = pvm_find_va_enclose_addr(vmalloc_end);
3354 	base = pvm_determine_end_from_reverse(&va, align) - end;
3355 
3356 	while (true) {
3357 		/*
3358 		 * base might have underflowed, add last_end before
3359 		 * comparing.
3360 		 */
3361 		if (base + last_end < vmalloc_start + last_end)
3362 			goto overflow;
3363 
3364 		/*
3365 		 * Fitting base has not been found.
3366 		 */
3367 		if (va == NULL)
3368 			goto overflow;
3369 
3370 		/*
3371 		 * If required width exceeds current VA block, move
3372 		 * base downwards and then recheck.
3373 		 */
3374 		if (base + end > va->va_end) {
3375 			base = pvm_determine_end_from_reverse(&va, align) - end;
3376 			term_area = area;
3377 			continue;
3378 		}
3379 
3380 		/*
3381 		 * If this VA does not fit, move base downwards and recheck.
3382 		 */
3383 		if (base + start < va->va_start) {
3384 			va = node_to_va(rb_prev(&va->rb_node));
3385 			base = pvm_determine_end_from_reverse(&va, align) - end;
3386 			term_area = area;
3387 			continue;
3388 		}
3389 
3390 		/*
3391 		 * This area fits, move on to the previous one.  If
3392 		 * the previous one is the terminal one, we're done.
3393 		 */
3394 		area = (area + nr_vms - 1) % nr_vms;
3395 		if (area == term_area)
3396 			break;
3397 
3398 		start = offsets[area];
3399 		end = start + sizes[area];
3400 		va = pvm_find_va_enclose_addr(base + end);
3401 	}
3402 
3403 	/* we've found a fitting base, insert all va's */
3404 	for (area = 0; area < nr_vms; area++) {
3405 		int ret;
3406 
3407 		start = base + offsets[area];
3408 		size = sizes[area];
3409 
3410 		va = pvm_find_va_enclose_addr(start);
3411 		if (WARN_ON_ONCE(va == NULL))
3412 			/* It is a BUG(), but trigger recovery instead. */
3413 			goto recovery;
3414 
3415 		type = classify_va_fit_type(va, start, size);
3416 		if (WARN_ON_ONCE(type == NOTHING_FIT))
3417 			/* It is a BUG(), but trigger recovery instead. */
3418 			goto recovery;
3419 
3420 		ret = adjust_va_to_fit_type(va, start, size, type);
3421 		if (unlikely(ret))
3422 			goto recovery;
3423 
3424 		/* Allocated area. */
3425 		va = vas[area];
3426 		va->va_start = start;
3427 		va->va_end = start + size;
3428 	}
3429 
3430 	spin_unlock(&free_vmap_area_lock);
3431 
3432 	/* populate the kasan shadow space */
3433 	for (area = 0; area < nr_vms; area++) {
3434 		if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area]))
3435 			goto err_free_shadow;
3436 
3437 		kasan_unpoison_vmalloc((void *)vas[area]->va_start,
3438 				       sizes[area]);
3439 	}
3440 
3441 	/* insert all vm's */
3442 	spin_lock(&vmap_area_lock);
3443 	for (area = 0; area < nr_vms; area++) {
3444 		insert_vmap_area(vas[area], &vmap_area_root, &vmap_area_list);
3445 
3446 		setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC,
3447 				 pcpu_get_vm_areas);
3448 	}
3449 	spin_unlock(&vmap_area_lock);
3450 
3451 	kfree(vas);
3452 	return vms;
3453 
3454 recovery:
3455 	/*
3456 	 * Remove previously allocated areas. There is no
3457 	 * need in removing these areas from the busy tree,
3458 	 * because they are inserted only on the final step
3459 	 * and when pcpu_get_vm_areas() is success.
3460 	 */
3461 	while (area--) {
3462 		orig_start = vas[area]->va_start;
3463 		orig_end = vas[area]->va_end;
3464 		va = merge_or_add_vmap_area(vas[area], &free_vmap_area_root,
3465 					    &free_vmap_area_list);
3466 		kasan_release_vmalloc(orig_start, orig_end,
3467 				      va->va_start, va->va_end);
3468 		vas[area] = NULL;
3469 	}
3470 
3471 overflow:
3472 	spin_unlock(&free_vmap_area_lock);
3473 	if (!purged) {
3474 		purge_vmap_area_lazy();
3475 		purged = true;
3476 
3477 		/* Before "retry", check if we recover. */
3478 		for (area = 0; area < nr_vms; area++) {
3479 			if (vas[area])
3480 				continue;
3481 
3482 			vas[area] = kmem_cache_zalloc(
3483 				vmap_area_cachep, GFP_KERNEL);
3484 			if (!vas[area])
3485 				goto err_free;
3486 		}
3487 
3488 		goto retry;
3489 	}
3490 
3491 err_free:
3492 	for (area = 0; area < nr_vms; area++) {
3493 		if (vas[area])
3494 			kmem_cache_free(vmap_area_cachep, vas[area]);
3495 
3496 		kfree(vms[area]);
3497 	}
3498 err_free2:
3499 	kfree(vas);
3500 	kfree(vms);
3501 	return NULL;
3502 
3503 err_free_shadow:
3504 	spin_lock(&free_vmap_area_lock);
3505 	/*
3506 	 * We release all the vmalloc shadows, even the ones for regions that
3507 	 * hadn't been successfully added. This relies on kasan_release_vmalloc
3508 	 * being able to tolerate this case.
3509 	 */
3510 	for (area = 0; area < nr_vms; area++) {
3511 		orig_start = vas[area]->va_start;
3512 		orig_end = vas[area]->va_end;
3513 		va = merge_or_add_vmap_area(vas[area], &free_vmap_area_root,
3514 					    &free_vmap_area_list);
3515 		kasan_release_vmalloc(orig_start, orig_end,
3516 				      va->va_start, va->va_end);
3517 		vas[area] = NULL;
3518 		kfree(vms[area]);
3519 	}
3520 	spin_unlock(&free_vmap_area_lock);
3521 	kfree(vas);
3522 	kfree(vms);
3523 	return NULL;
3524 }
3525 
3526 /**
3527  * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
3528  * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
3529  * @nr_vms: the number of allocated areas
3530  *
3531  * Free vm_structs and the array allocated by pcpu_get_vm_areas().
3532  */
3533 void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
3534 {
3535 	int i;
3536 
3537 	for (i = 0; i < nr_vms; i++)
3538 		free_vm_area(vms[i]);
3539 	kfree(vms);
3540 }
3541 #endif	/* CONFIG_SMP */
3542 
3543 #ifdef CONFIG_PROC_FS
3544 static void *s_start(struct seq_file *m, loff_t *pos)
3545 	__acquires(&vmap_purge_lock)
3546 	__acquires(&vmap_area_lock)
3547 {
3548 	mutex_lock(&vmap_purge_lock);
3549 	spin_lock(&vmap_area_lock);
3550 
3551 	return seq_list_start(&vmap_area_list, *pos);
3552 }
3553 
3554 static void *s_next(struct seq_file *m, void *p, loff_t *pos)
3555 {
3556 	return seq_list_next(p, &vmap_area_list, pos);
3557 }
3558 
3559 static void s_stop(struct seq_file *m, void *p)
3560 	__releases(&vmap_purge_lock)
3561 	__releases(&vmap_area_lock)
3562 {
3563 	mutex_unlock(&vmap_purge_lock);
3564 	spin_unlock(&vmap_area_lock);
3565 }
3566 
3567 static void show_numa_info(struct seq_file *m, struct vm_struct *v)
3568 {
3569 	if (IS_ENABLED(CONFIG_NUMA)) {
3570 		unsigned int nr, *counters = m->private;
3571 
3572 		if (!counters)
3573 			return;
3574 
3575 		if (v->flags & VM_UNINITIALIZED)
3576 			return;
3577 		/* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
3578 		smp_rmb();
3579 
3580 		memset(counters, 0, nr_node_ids * sizeof(unsigned int));
3581 
3582 		for (nr = 0; nr < v->nr_pages; nr++)
3583 			counters[page_to_nid(v->pages[nr])]++;
3584 
3585 		for_each_node_state(nr, N_HIGH_MEMORY)
3586 			if (counters[nr])
3587 				seq_printf(m, " N%u=%u", nr, counters[nr]);
3588 	}
3589 }
3590 
3591 static void show_purge_info(struct seq_file *m)
3592 {
3593 	struct llist_node *head;
3594 	struct vmap_area *va;
3595 
3596 	head = READ_ONCE(vmap_purge_list.first);
3597 	if (head == NULL)
3598 		return;
3599 
3600 	llist_for_each_entry(va, head, purge_list) {
3601 		seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n",
3602 			(void *)va->va_start, (void *)va->va_end,
3603 			va->va_end - va->va_start);
3604 	}
3605 }
3606 
3607 static int s_show(struct seq_file *m, void *p)
3608 {
3609 	struct vmap_area *va;
3610 	struct vm_struct *v;
3611 
3612 	va = list_entry(p, struct vmap_area, list);
3613 
3614 	/*
3615 	 * s_show can encounter race with remove_vm_area, !vm on behalf
3616 	 * of vmap area is being tear down or vm_map_ram allocation.
3617 	 */
3618 	if (!va->vm) {
3619 		seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n",
3620 			(void *)va->va_start, (void *)va->va_end,
3621 			va->va_end - va->va_start);
3622 
3623 		return 0;
3624 	}
3625 
3626 	v = va->vm;
3627 
3628 	seq_printf(m, "0x%pK-0x%pK %7ld",
3629 		v->addr, v->addr + v->size, v->size);
3630 
3631 	if (v->caller)
3632 		seq_printf(m, " %pS", v->caller);
3633 
3634 	if (v->nr_pages)
3635 		seq_printf(m, " pages=%d", v->nr_pages);
3636 
3637 	if (v->phys_addr)
3638 		seq_printf(m, " phys=%pa", &v->phys_addr);
3639 
3640 	if (v->flags & VM_IOREMAP)
3641 		seq_puts(m, " ioremap");
3642 
3643 	if (v->flags & VM_ALLOC)
3644 		seq_puts(m, " vmalloc");
3645 
3646 	if (v->flags & VM_MAP)
3647 		seq_puts(m, " vmap");
3648 
3649 	if (v->flags & VM_USERMAP)
3650 		seq_puts(m, " user");
3651 
3652 	if (v->flags & VM_DMA_COHERENT)
3653 		seq_puts(m, " dma-coherent");
3654 
3655 	if (is_vmalloc_addr(v->pages))
3656 		seq_puts(m, " vpages");
3657 
3658 	show_numa_info(m, v);
3659 	seq_putc(m, '\n');
3660 
3661 	/*
3662 	 * As a final step, dump "unpurged" areas. Note,
3663 	 * that entire "/proc/vmallocinfo" output will not
3664 	 * be address sorted, because the purge list is not
3665 	 * sorted.
3666 	 */
3667 	if (list_is_last(&va->list, &vmap_area_list))
3668 		show_purge_info(m);
3669 
3670 	return 0;
3671 }
3672 
3673 static const struct seq_operations vmalloc_op = {
3674 	.start = s_start,
3675 	.next = s_next,
3676 	.stop = s_stop,
3677 	.show = s_show,
3678 };
3679 
3680 static int __init proc_vmalloc_init(void)
3681 {
3682 	if (IS_ENABLED(CONFIG_NUMA))
3683 		proc_create_seq_private("vmallocinfo", 0400, NULL,
3684 				&vmalloc_op,
3685 				nr_node_ids * sizeof(unsigned int), NULL);
3686 	else
3687 		proc_create_seq("vmallocinfo", 0400, NULL, &vmalloc_op);
3688 	return 0;
3689 }
3690 module_init(proc_vmalloc_init);
3691 
3692 #endif
3693