1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_PGALLOC_H 3 #define _ASM_X86_PGALLOC_H 4 5 #include <linux/threads.h> 6 #include <linux/mm.h> /* for struct page */ 7 #include <linux/pagemap.h> 8 9 #define __HAVE_ARCH_PTE_ALLOC_ONE 10 #define __HAVE_ARCH_PGD_FREE 11 #include <asm-generic/pgalloc.h> 12 13 static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; } 14 15 #ifdef CONFIG_PARAVIRT_XXL 16 #include <asm/paravirt.h> 17 #else 18 #define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm) 19 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) {} 20 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) {} 21 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) {} 22 static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn, 23 unsigned long start, unsigned long count) {} 24 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) {} 25 static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) {} 26 static inline void paravirt_release_pte(unsigned long pfn) {} 27 static inline void paravirt_release_pmd(unsigned long pfn) {} 28 static inline void paravirt_release_pud(unsigned long pfn) {} 29 static inline void paravirt_release_p4d(unsigned long pfn) {} 30 #endif 31 32 /* 33 * Flags to use when allocating a user page table page. 34 */ 35 extern gfp_t __userpte_alloc_gfp; 36 37 #ifdef CONFIG_PAGE_TABLE_ISOLATION 38 /* 39 * Instead of one PGD, we acquire two PGDs. Being order-1, it is 40 * both 8k in size and 8k-aligned. That lets us just flip bit 12 41 * in a pointer to swap between the two 4k halves. 42 */ 43 #define PGD_ALLOCATION_ORDER 1 44 #else 45 #define PGD_ALLOCATION_ORDER 0 46 #endif 47 48 /* 49 * Allocate and free page tables. 50 */ 51 extern pgd_t *pgd_alloc(struct mm_struct *); 52 extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); 53 54 extern pgtable_t pte_alloc_one(struct mm_struct *); 55 56 extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte); 57 58 static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte, 59 unsigned long address) 60 { 61 ___pte_free_tlb(tlb, pte); 62 } 63 64 static inline void pmd_populate_kernel(struct mm_struct *mm, 65 pmd_t *pmd, pte_t *pte) 66 { 67 paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); 68 set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); 69 } 70 71 static inline void pmd_populate_kernel_safe(struct mm_struct *mm, 72 pmd_t *pmd, pte_t *pte) 73 { 74 paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); 75 set_pmd_safe(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); 76 } 77 78 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, 79 struct page *pte) 80 { 81 unsigned long pfn = page_to_pfn(pte); 82 83 paravirt_alloc_pte(mm, pfn); 84 set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE)); 85 } 86 87 #define pmd_pgtable(pmd) pmd_page(pmd) 88 89 #if CONFIG_PGTABLE_LEVELS > 2 90 extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); 91 92 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, 93 unsigned long address) 94 { 95 ___pmd_free_tlb(tlb, pmd); 96 } 97 98 #ifdef CONFIG_X86_PAE 99 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd); 100 #else /* !CONFIG_X86_PAE */ 101 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 102 { 103 paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); 104 set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd))); 105 } 106 107 static inline void pud_populate_safe(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 108 { 109 paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); 110 set_pud_safe(pud, __pud(_PAGE_TABLE | __pa(pmd))); 111 } 112 #endif /* CONFIG_X86_PAE */ 113 114 #if CONFIG_PGTABLE_LEVELS > 3 115 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) 116 { 117 paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); 118 set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); 119 } 120 121 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) 122 { 123 paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); 124 set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud))); 125 } 126 127 extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud); 128 129 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, 130 unsigned long address) 131 { 132 ___pud_free_tlb(tlb, pud); 133 } 134 135 #if CONFIG_PGTABLE_LEVELS > 4 136 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) 137 { 138 if (!pgtable_l5_enabled()) 139 return; 140 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); 141 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); 142 } 143 144 static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) 145 { 146 if (!pgtable_l5_enabled()) 147 return; 148 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); 149 set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); 150 } 151 152 static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) 153 { 154 gfp_t gfp = GFP_KERNEL_ACCOUNT; 155 156 if (mm == &init_mm) 157 gfp &= ~__GFP_ACCOUNT; 158 return (p4d_t *)get_zeroed_page(gfp); 159 } 160 161 static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) 162 { 163 if (!pgtable_l5_enabled()) 164 return; 165 166 BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); 167 free_page((unsigned long)p4d); 168 } 169 170 extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); 171 172 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, 173 unsigned long address) 174 { 175 if (pgtable_l5_enabled()) 176 ___p4d_free_tlb(tlb, p4d); 177 } 178 179 #endif /* CONFIG_PGTABLE_LEVELS > 4 */ 180 #endif /* CONFIG_PGTABLE_LEVELS > 3 */ 181 #endif /* CONFIG_PGTABLE_LEVELS > 2 */ 182 183 #endif /* _ASM_X86_PGALLOC_H */ 184