1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Handling Page Tables through page fragments 5 * 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/gfp.h> 10 #include <linux/mm.h> 11 #include <linux/percpu.h> 12 #include <linux/hardirq.h> 13 #include <linux/hugetlb.h> 14 #include <asm/pgalloc.h> 15 #include <asm/tlbflush.h> 16 #include <asm/tlb.h> 17 18 void pte_frag_destroy(void *pte_frag) 19 { 20 int count; 21 struct ptdesc *ptdesc; 22 23 ptdesc = virt_to_ptdesc(pte_frag); 24 /* drop all the pending references */ 25 count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT; 26 /* We allow PTE_FRAG_NR fragments from a PTE page */ 27 if (atomic_sub_and_test(PTE_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { 28 folio_clear_active(ptdesc_folio(ptdesc)); 29 pagetable_dtor(ptdesc); 30 pagetable_free(ptdesc); 31 } 32 } 33 34 static pte_t *get_pte_from_cache(struct mm_struct *mm) 35 { 36 void *pte_frag, *ret; 37 38 if (PTE_FRAG_NR == 1) 39 return NULL; 40 41 spin_lock(&mm->page_table_lock); 42 ret = pte_frag_get(&mm->context); 43 if (ret) { 44 pte_frag = ret + PTE_FRAG_SIZE; 45 /* 46 * If we have taken up all the fragments mark PTE page NULL 47 */ 48 if (((unsigned long)pte_frag & ~PAGE_MASK) == 0) 49 pte_frag = NULL; 50 pte_frag_set(&mm->context, pte_frag); 51 } 52 spin_unlock(&mm->page_table_lock); 53 return (pte_t *)ret; 54 } 55 56 static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel) 57 { 58 void *ret = NULL; 59 struct ptdesc *ptdesc; 60 gfp_t gfp = PGALLOC_GFP; 61 62 if (!kernel) 63 gfp |= __GFP_ACCOUNT; 64 65 ptdesc = pagetable_alloc(gfp, 0); 66 if (!ptdesc) 67 return NULL; 68 if (!pagetable_pte_ctor(mm, ptdesc)) { 69 pagetable_free(ptdesc); 70 return NULL; 71 } 72 73 atomic_set(&ptdesc->pt_frag_refcount, 1); 74 75 ret = ptdesc_address(ptdesc); 76 /* 77 * if we support only one fragment just return the 78 * allocated page. 79 */ 80 if (PTE_FRAG_NR == 1) 81 return ret; 82 spin_lock(&mm->page_table_lock); 83 /* 84 * If we find ptdesc_page set, we return 85 * the allocated page with single fragment 86 * count. 87 */ 88 if (likely(!pte_frag_get(&mm->context))) { 89 atomic_set(&ptdesc->pt_frag_refcount, PTE_FRAG_NR); 90 pte_frag_set(&mm->context, ret + PTE_FRAG_SIZE); 91 } 92 spin_unlock(&mm->page_table_lock); 93 94 return (pte_t *)ret; 95 } 96 97 pte_t *pte_fragment_alloc(struct mm_struct *mm, int kernel) 98 { 99 pte_t *pte; 100 101 pte = get_pte_from_cache(mm); 102 if (pte) 103 return pte; 104 105 return __alloc_for_ptecache(mm, kernel); 106 } 107 108 static void pte_free_now(struct rcu_head *head) 109 { 110 struct ptdesc *ptdesc; 111 112 ptdesc = container_of(head, struct ptdesc, pt_rcu_head); 113 pagetable_dtor(ptdesc); 114 pagetable_free(ptdesc); 115 } 116 117 void pte_fragment_free(unsigned long *table, int kernel) 118 { 119 struct ptdesc *ptdesc = virt_to_ptdesc(table); 120 121 if (pagetable_is_reserved(ptdesc)) 122 return free_reserved_ptdesc(ptdesc); 123 124 BUG_ON(atomic_read(&ptdesc->pt_frag_refcount) <= 0); 125 if (atomic_dec_and_test(&ptdesc->pt_frag_refcount)) { 126 if (kernel || !folio_test_clear_active(ptdesc_folio(ptdesc))) 127 pte_free_now(&ptdesc->pt_rcu_head); 128 else 129 call_rcu(&ptdesc->pt_rcu_head, pte_free_now); 130 } 131 } 132 133 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 134 void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable) 135 { 136 struct folio *folio; 137 138 folio = virt_to_folio(pgtable); 139 folio_set_active(folio); 140 pte_fragment_free((unsigned long *)pgtable, 0); 141 } 142 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 143