1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Helper routines for building identity mapping page tables. This is 4 * included by both the compressed kernel and the regular kernel. 5 */ 6 7 static void free_pte(struct x86_mapping_info *info, pmd_t *pmd) 8 { 9 pte_t *pte = pte_offset_kernel(pmd, 0); 10 11 info->free_pgt_page(pte, info->context); 12 } 13 14 static void free_pmd(struct x86_mapping_info *info, pud_t *pud) 15 { 16 pmd_t *pmd = pmd_offset(pud, 0); 17 int i; 18 19 for (i = 0; i < PTRS_PER_PMD; i++) { 20 if (!pmd_present(pmd[i])) 21 continue; 22 23 if (pmd_leaf(pmd[i])) 24 continue; 25 26 free_pte(info, &pmd[i]); 27 } 28 29 info->free_pgt_page(pmd, info->context); 30 } 31 32 static void free_pud(struct x86_mapping_info *info, p4d_t *p4d) 33 { 34 pud_t *pud = pud_offset(p4d, 0); 35 int i; 36 37 for (i = 0; i < PTRS_PER_PUD; i++) { 38 if (!pud_present(pud[i])) 39 continue; 40 41 if (pud_leaf(pud[i])) 42 continue; 43 44 free_pmd(info, &pud[i]); 45 } 46 47 info->free_pgt_page(pud, info->context); 48 } 49 50 static void free_p4d(struct x86_mapping_info *info, pgd_t *pgd) 51 { 52 p4d_t *p4d = p4d_offset(pgd, 0); 53 int i; 54 55 for (i = 0; i < PTRS_PER_P4D; i++) { 56 if (!p4d_present(p4d[i])) 57 continue; 58 59 free_pud(info, &p4d[i]); 60 } 61 62 if (pgtable_l5_enabled()) 63 info->free_pgt_page(p4d, info->context); 64 } 65 66 void kernel_ident_mapping_free(struct x86_mapping_info *info, pgd_t *pgd) 67 { 68 int i; 69 70 for (i = 0; i < PTRS_PER_PGD; i++) { 71 if (!pgd_present(pgd[i])) 72 continue; 73 74 free_p4d(info, &pgd[i]); 75 } 76 77 info->free_pgt_page(pgd, info->context); 78 } 79 80 static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page, 81 unsigned long addr, unsigned long end) 82 { 83 addr &= PMD_MASK; 84 for (; addr < end; addr += PMD_SIZE) { 85 pmd_t *pmd = pmd_page + pmd_index(addr); 86 87 if (pmd_present(*pmd)) 88 continue; 89 90 set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag)); 91 } 92 } 93 94 static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, 95 unsigned long addr, unsigned long end) 96 { 97 unsigned long next; 98 99 for (; addr < end; addr = next) { 100 pud_t *pud = pud_page + pud_index(addr); 101 pmd_t *pmd; 102 bool use_gbpage; 103 104 next = (addr & PUD_MASK) + PUD_SIZE; 105 if (next > end) 106 next = end; 107 108 /* if this is already a gbpage, this portion is already mapped */ 109 if (pud_leaf(*pud)) 110 continue; 111 112 /* Is using a gbpage allowed? */ 113 use_gbpage = info->direct_gbpages; 114 115 /* Don't use gbpage if it maps more than the requested region. */ 116 /* at the begining: */ 117 use_gbpage &= ((addr & ~PUD_MASK) == 0); 118 /* ... or at the end: */ 119 use_gbpage &= ((next & ~PUD_MASK) == 0); 120 121 /* Never overwrite existing mappings */ 122 use_gbpage &= !pud_present(*pud); 123 124 if (use_gbpage) { 125 pud_t pudval; 126 127 pudval = __pud((addr - info->offset) | info->page_flag); 128 set_pud(pud, pudval); 129 continue; 130 } 131 132 if (pud_present(*pud)) { 133 pmd = pmd_offset(pud, 0); 134 ident_pmd_init(info, pmd, addr, next); 135 continue; 136 } 137 pmd = (pmd_t *)info->alloc_pgt_page(info->context); 138 if (!pmd) 139 return -ENOMEM; 140 ident_pmd_init(info, pmd, addr, next); 141 set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag)); 142 } 143 144 return 0; 145 } 146 147 static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page, 148 unsigned long addr, unsigned long end) 149 { 150 unsigned long next; 151 int result; 152 153 for (; addr < end; addr = next) { 154 p4d_t *p4d = p4d_page + p4d_index(addr); 155 pud_t *pud; 156 157 next = (addr & P4D_MASK) + P4D_SIZE; 158 if (next > end) 159 next = end; 160 161 if (p4d_present(*p4d)) { 162 pud = pud_offset(p4d, 0); 163 result = ident_pud_init(info, pud, addr, next); 164 if (result) 165 return result; 166 167 continue; 168 } 169 pud = (pud_t *)info->alloc_pgt_page(info->context); 170 if (!pud) 171 return -ENOMEM; 172 173 result = ident_pud_init(info, pud, addr, next); 174 if (result) 175 return result; 176 177 set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag)); 178 } 179 180 return 0; 181 } 182 183 int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page, 184 unsigned long pstart, unsigned long pend) 185 { 186 unsigned long addr = pstart + info->offset; 187 unsigned long end = pend + info->offset; 188 unsigned long next; 189 int result; 190 191 /* Set the default pagetable flags if not supplied */ 192 if (!info->kernpg_flag) 193 info->kernpg_flag = _KERNPG_TABLE; 194 195 /* Filter out unsupported __PAGE_KERNEL_* bits: */ 196 info->kernpg_flag &= __default_kernel_pte_mask; 197 198 for (; addr < end; addr = next) { 199 pgd_t *pgd = pgd_page + pgd_index(addr); 200 p4d_t *p4d; 201 202 next = (addr & PGDIR_MASK) + PGDIR_SIZE; 203 if (next > end) 204 next = end; 205 206 if (pgd_present(*pgd)) { 207 p4d = p4d_offset(pgd, 0); 208 result = ident_p4d_init(info, p4d, addr, next); 209 if (result) 210 return result; 211 continue; 212 } 213 214 p4d = (p4d_t *)info->alloc_pgt_page(info->context); 215 if (!p4d) 216 return -ENOMEM; 217 result = ident_p4d_init(info, p4d, addr, next); 218 if (result) 219 return result; 220 if (pgtable_l5_enabled()) { 221 set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag)); 222 } else { 223 /* 224 * With p4d folded, pgd is equal to p4d. 225 * The pgd entry has to point to the pud page table in this case. 226 */ 227 pud_t *pud = pud_offset(p4d, 0); 228 set_pgd(pgd, __pgd(__pa(pud) | info->kernpg_flag)); 229 } 230 } 231 232 return 0; 233 } 234