1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_PGALLOC_H 3 #define _ASM_X86_PGALLOC_H 4 5 #include <linux/threads.h> 6 #include <linux/mm.h> /* for struct page */ 7 #include <linux/pagemap.h> 8 9 #define __HAVE_ARCH_PTE_ALLOC_ONE 10 #define __HAVE_ARCH_PGD_FREE 11 #include <asm-generic/pgalloc.h> 12 13 static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; } 14 15 #ifdef CONFIG_PARAVIRT_XXL 16 #include <asm/paravirt.h> 17 #else 18 #define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm) 19 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) {} 20 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) {} 21 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) {} 22 static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn, 23 unsigned long start, unsigned long count) {} 24 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) {} 25 static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) {} 26 static inline void paravirt_release_pte(unsigned long pfn) {} 27 static inline void paravirt_release_pmd(unsigned long pfn) {} 28 static inline void paravirt_release_pud(unsigned long pfn) {} 29 static inline void paravirt_release_p4d(unsigned long pfn) {} 30 #endif 31 32 #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION 33 /* 34 * Instead of one PGD, we acquire two PGDs. Being order-1, it is 35 * both 8k in size and 8k-aligned. That lets us just flip bit 12 36 * in a pointer to swap between the two 4k halves. 37 */ 38 #define PGD_ALLOCATION_ORDER 1 39 #else 40 #define PGD_ALLOCATION_ORDER 0 41 #endif 42 43 /* 44 * Allocate and free page tables. 45 */ 46 extern pgd_t *pgd_alloc(struct mm_struct *); 47 extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); 48 49 extern pgtable_t pte_alloc_one(struct mm_struct *); 50 51 extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte); 52 53 static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte, 54 unsigned long address) 55 { 56 ___pte_free_tlb(tlb, pte); 57 } 58 59 static inline void pmd_populate_kernel(struct mm_struct *mm, 60 pmd_t *pmd, pte_t *pte) 61 { 62 paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); 63 set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); 64 } 65 66 static inline void pmd_populate_kernel_safe(struct mm_struct *mm, 67 pmd_t *pmd, pte_t *pte) 68 { 69 paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); 70 set_pmd_safe(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); 71 } 72 73 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, 74 struct page *pte) 75 { 76 unsigned long pfn = page_to_pfn(pte); 77 78 paravirt_alloc_pte(mm, pfn); 79 set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE)); 80 } 81 82 #if CONFIG_PGTABLE_LEVELS > 2 83 extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); 84 85 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, 86 unsigned long address) 87 { 88 ___pmd_free_tlb(tlb, pmd); 89 } 90 91 #ifdef CONFIG_X86_PAE 92 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd); 93 #else /* !CONFIG_X86_PAE */ 94 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 95 { 96 paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); 97 set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd))); 98 } 99 100 static inline void pud_populate_safe(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 101 { 102 paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); 103 set_pud_safe(pud, __pud(_PAGE_TABLE | __pa(pmd))); 104 } 105 #endif /* CONFIG_X86_PAE */ 106 107 #if CONFIG_PGTABLE_LEVELS > 3 108 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) 109 { 110 paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); 111 set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); 112 } 113 114 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) 115 { 116 paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); 117 set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud))); 118 } 119 120 extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud); 121 122 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, 123 unsigned long address) 124 { 125 ___pud_free_tlb(tlb, pud); 126 } 127 128 #if CONFIG_PGTABLE_LEVELS > 4 129 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) 130 { 131 if (!pgtable_l5_enabled()) 132 return; 133 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); 134 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); 135 } 136 137 static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) 138 { 139 if (!pgtable_l5_enabled()) 140 return; 141 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); 142 set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); 143 } 144 145 extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); 146 147 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, 148 unsigned long address) 149 { 150 if (pgtable_l5_enabled()) 151 ___p4d_free_tlb(tlb, p4d); 152 } 153 154 #endif /* CONFIG_PGTABLE_LEVELS > 4 */ 155 #endif /* CONFIG_PGTABLE_LEVELS > 3 */ 156 #endif /* CONFIG_PGTABLE_LEVELS > 2 */ 157 158 #endif /* _ASM_X86_PGALLOC_H */ 159