1 // SPDX-License-Identifier: GPL-2.0
2
3 /*
4 * Handling Page Tables through page fragments
5 *
6 */
7
8 #include <linux/kernel.h>
9 #include <linux/gfp.h>
10 #include <linux/mm.h>
11 #include <linux/percpu.h>
12 #include <linux/hardirq.h>
13 #include <linux/hugetlb.h>
14 #include <asm/pgalloc.h>
15 #include <asm/tlbflush.h>
16 #include <asm/tlb.h>
17
pte_frag_destroy(void * pte_frag)18 void pte_frag_destroy(void *pte_frag)
19 {
20 int count;
21 struct ptdesc *ptdesc;
22
23 ptdesc = virt_to_ptdesc(pte_frag);
24 /* drop all the pending references */
25 count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
26 /* We allow PTE_FRAG_NR fragments from a PTE page */
27 if (atomic_sub_and_test(PTE_FRAG_NR - count, &ptdesc->pt_frag_refcount)) {
28 pagetable_dtor(ptdesc);
29 pagetable_free(ptdesc);
30 }
31 }
32
get_pte_from_cache(struct mm_struct * mm)33 static pte_t *get_pte_from_cache(struct mm_struct *mm)
34 {
35 void *pte_frag, *ret;
36
37 if (PTE_FRAG_NR == 1)
38 return NULL;
39
40 spin_lock(&mm->page_table_lock);
41 ret = pte_frag_get(&mm->context);
42 if (ret) {
43 pte_frag = ret + PTE_FRAG_SIZE;
44 /*
45 * If we have taken up all the fragments mark PTE page NULL
46 */
47 if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
48 pte_frag = NULL;
49 pte_frag_set(&mm->context, pte_frag);
50 }
51 spin_unlock(&mm->page_table_lock);
52 return (pte_t *)ret;
53 }
54
__alloc_for_ptecache(struct mm_struct * mm,int kernel)55 static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel)
56 {
57 void *ret = NULL;
58 struct ptdesc *ptdesc;
59 gfp_t gfp = PGALLOC_GFP;
60
61 if (!kernel)
62 gfp |= __GFP_ACCOUNT;
63
64 ptdesc = pagetable_alloc(gfp, 0);
65 if (!ptdesc)
66 return NULL;
67 if (!pagetable_pte_ctor(mm, ptdesc)) {
68 pagetable_free(ptdesc);
69 return NULL;
70 }
71
72 atomic_set(&ptdesc->pt_frag_refcount, 1);
73
74 ret = ptdesc_address(ptdesc);
75 /*
76 * if we support only one fragment just return the
77 * allocated page.
78 */
79 if (PTE_FRAG_NR == 1)
80 return ret;
81 spin_lock(&mm->page_table_lock);
82 /*
83 * If we find ptdesc_page set, we return
84 * the allocated page with single fragment
85 * count.
86 */
87 if (likely(!pte_frag_get(&mm->context))) {
88 atomic_set(&ptdesc->pt_frag_refcount, PTE_FRAG_NR);
89 pte_frag_set(&mm->context, ret + PTE_FRAG_SIZE);
90 }
91 spin_unlock(&mm->page_table_lock);
92
93 return (pte_t *)ret;
94 }
95
pte_fragment_alloc(struct mm_struct * mm,int kernel)96 pte_t *pte_fragment_alloc(struct mm_struct *mm, int kernel)
97 {
98 pte_t *pte;
99
100 pte = get_pte_from_cache(mm);
101 if (pte)
102 return pte;
103
104 return __alloc_for_ptecache(mm, kernel);
105 }
106
pte_free_now(struct rcu_head * head)107 static void pte_free_now(struct rcu_head *head)
108 {
109 struct ptdesc *ptdesc;
110
111 ptdesc = container_of(head, struct ptdesc, pt_rcu_head);
112 pagetable_dtor(ptdesc);
113 pagetable_free(ptdesc);
114 }
115
pte_fragment_free(unsigned long * table,int kernel)116 void pte_fragment_free(unsigned long *table, int kernel)
117 {
118 struct ptdesc *ptdesc = virt_to_ptdesc(table);
119
120 if (pagetable_is_reserved(ptdesc))
121 return free_reserved_ptdesc(ptdesc);
122
123 BUG_ON(atomic_read(&ptdesc->pt_frag_refcount) <= 0);
124 if (atomic_dec_and_test(&ptdesc->pt_frag_refcount)) {
125 if (kernel || !folio_test_clear_active(ptdesc_folio(ptdesc)))
126 pte_free_now(&ptdesc->pt_rcu_head);
127 else
128 call_rcu(&ptdesc->pt_rcu_head, pte_free_now);
129 }
130 }
131
132 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
pte_free_defer(struct mm_struct * mm,pgtable_t pgtable)133 void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
134 {
135 struct folio *folio;
136
137 folio = virt_to_folio(pgtable);
138 folio_set_active(folio);
139 pte_fragment_free((unsigned long *)pgtable, 0);
140 }
141 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
142