fixmap.c (b97547761b02cc95e0e6be827dc9ca9da8142761) | fixmap.c (414c109bdf496195269bc03d40841fe67fc2f839) |
---|---|
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Fixmap manipulation code 4 */ 5 6#include <linux/bug.h> 7#include <linux/init.h> 8#include <linux/kernel.h> 9#include <linux/libfdt.h> 10#include <linux/memory.h> 11#include <linux/mm.h> 12#include <linux/sizes.h> 13 14#include <asm/fixmap.h> 15#include <asm/kernel-pgtable.h> 16#include <asm/pgalloc.h> 17#include <asm/tlbflush.h> 18 | 1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Fixmap manipulation code 4 */ 5 6#include <linux/bug.h> 7#include <linux/init.h> 8#include <linux/kernel.h> 9#include <linux/libfdt.h> 10#include <linux/memory.h> 11#include <linux/mm.h> 12#include <linux/sizes.h> 13 14#include <asm/fixmap.h> 15#include <asm/kernel-pgtable.h> 16#include <asm/pgalloc.h> 17#include <asm/tlbflush.h> 18 |
19static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; | 19#define NR_BM_PTE_TABLES \ 20 SPAN_NR_ENTRIES(FIXADDR_TOT_START, FIXADDR_TOP, PMD_SHIFT) 21#define NR_BM_PMD_TABLES \ 22 SPAN_NR_ENTRIES(FIXADDR_TOT_START, FIXADDR_TOP, PUD_SHIFT) 23 24static_assert(NR_BM_PMD_TABLES == 1); 25 26#define __BM_TABLE_IDX(addr, shift) \ 27 (((addr) >> (shift)) - (FIXADDR_TOT_START >> (shift))) 28 29#define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) 30 31static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss; |
20static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; 21static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; 22 | 32static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; 33static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; 34 |
23static inline pud_t *fixmap_pud(unsigned long addr) | 35static inline pte_t *fixmap_pte(unsigned long addr) |
24{ | 36{ |
25 pgd_t *pgdp = pgd_offset_k(addr); 26 p4d_t *p4dp = p4d_offset(pgdp, addr); 27 p4d_t p4d = READ_ONCE(*p4dp); | 37 return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)]; 38} |
28 | 39 |
29 BUG_ON(p4d_none(p4d) || p4d_bad(p4d)); | 40static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr) 41{ 42 pmd_t pmd = READ_ONCE(*pmdp); 43 pte_t *ptep; |
30 | 44 |
31 return pud_offset_kimg(p4dp, addr); | 45 if (pmd_none(pmd)) { 46 ptep = bm_pte[BM_PTE_TABLE_IDX(addr)]; 47 __pmd_populate(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE); 48 } |
32} 33 | 49} 50 |
34static inline pmd_t *fixmap_pmd(unsigned long addr) | 51static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr, 52 unsigned long end) |
35{ | 53{ |
36 pud_t *pudp = fixmap_pud(addr); | 54 unsigned long next; |
37 pud_t pud = READ_ONCE(*pudp); | 55 pud_t pud = READ_ONCE(*pudp); |
56 pmd_t *pmdp; |
|
38 | 57 |
39 BUG_ON(pud_none(pud) || pud_bad(pud)); | 58 if (pud_none(pud)) 59 __pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE); |
40 | 60 |
41 return pmd_offset_kimg(pudp, addr); | 61 pmdp = pmd_offset_kimg(pudp, addr); 62 do { 63 next = pmd_addr_end(addr, end); 64 early_fixmap_init_pte(pmdp, addr); 65 } while (pmdp++, addr = next, addr != end); |
42} 43 | 66} 67 |
44static inline pte_t *fixmap_pte(unsigned long addr) 45{ 46 return &bm_pte[pte_index(addr)]; 47} | |
48 | 68 |
49/* 50 * The p*d_populate functions call virt_to_phys implicitly so they can't be used 51 * directly on kernel symbols (bm_p*d). This function is called too early to use 52 * lm_alias so __p*d_populate functions must be used to populate with the 53 * physical address from __pa_symbol. 54 */ 55void __init early_fixmap_init(void) | 69static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, 70 unsigned long end) |
56{ | 71{ |
57 pgd_t *pgdp; 58 p4d_t *p4dp, p4d; | 72 p4d_t p4d = READ_ONCE(*p4dp); |
59 pud_t *pudp; | 73 pud_t *pudp; |
60 pmd_t *pmdp; 61 unsigned long addr = FIXADDR_TOT_START; | |
62 | 74 |
63 pgdp = pgd_offset_k(addr); 64 p4dp = p4d_offset(pgdp, addr); 65 p4d = READ_ONCE(*p4dp); 66 if (CONFIG_PGTABLE_LEVELS > 3 && 67 !(p4d_none(p4d) || p4d_page_paddr(p4d) == __pa_symbol(bm_pud))) { | 75 if (CONFIG_PGTABLE_LEVELS > 3 && !p4d_none(p4d) && 76 p4d_page_paddr(p4d) != __pa_symbol(bm_pud)) { |
68 /* 69 * We only end up here if the kernel mapping and the fixmap 70 * share the top level pgd entry, which should only happen on 71 * 16k/4 levels configurations. 72 */ 73 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); | 77 /* 78 * We only end up here if the kernel mapping and the fixmap 79 * share the top level pgd entry, which should only happen on 80 * 16k/4 levels configurations. 81 */ 82 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); |
74 pudp = pud_offset_kimg(p4dp, addr); 75 } else { 76 if (p4d_none(p4d)) 77 __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE); 78 pudp = fixmap_pud(addr); | |
79 } | 83 } |
80 if (pud_none(READ_ONCE(*pudp))) 81 __pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE); 82 pmdp = fixmap_pmd(addr); 83 __pmd_populate(pmdp, __pa_symbol(bm_pte), PMD_TYPE_TABLE); | |
84 | 84 |
85 /* 86 * The boot-ioremap range spans multiple pmds, for which 87 * we are not prepared: 88 */ 89 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 90 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); | 85 if (p4d_none(p4d)) 86 __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE); |
91 | 87 |
92 if ((pmdp != fixmap_pmd(__fix_to_virt(FIX_BTMAP_BEGIN))) 93 || pmdp != fixmap_pmd(__fix_to_virt(FIX_BTMAP_END))) { 94 WARN_ON(1); 95 pr_warn("pmdp %p != %p, %p\n", 96 pmdp, fixmap_pmd(__fix_to_virt(FIX_BTMAP_BEGIN)), 97 fixmap_pmd(__fix_to_virt(FIX_BTMAP_END))); 98 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 99 __fix_to_virt(FIX_BTMAP_BEGIN)); 100 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 101 __fix_to_virt(FIX_BTMAP_END)); | 88 pudp = pud_offset_kimg(p4dp, addr); 89 early_fixmap_init_pmd(pudp, addr, end); 90} |
102 | 91 |
103 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 104 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 105 } | 92/* 93 * The p*d_populate functions call virt_to_phys implicitly so they can't be used 94 * directly on kernel symbols (bm_p*d). This function is called too early to use 95 * lm_alias so __p*d_populate functions must be used to populate with the 96 * physical address from __pa_symbol. 97 */ 98void __init early_fixmap_init(void) 99{ 100 unsigned long addr = FIXADDR_TOT_START; 101 unsigned long end = FIXADDR_TOP; 102 103 pgd_t *pgdp = pgd_offset_k(addr); 104 p4d_t *p4dp = p4d_offset(pgdp, addr); 105 106 early_fixmap_init_pud(p4dp, addr, end); |
106} 107 108/* 109 * Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we 110 * ever need to use IPIs for TLB broadcasting, then we're in trouble here. 111 */ 112void __set_fixmap(enum fixed_addresses idx, 113 phys_addr_t phys, pgprot_t flags) --- 11 unchanged lines hidden (view full) --- 125 pte_clear(&init_mm, addr, ptep); 126 flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 127 } 128} 129 130void *__init fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot) 131{ 132 const u64 dt_virt_base = __fix_to_virt(FIX_FDT); | 107} 108 109/* 110 * Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we 111 * ever need to use IPIs for TLB broadcasting, then we're in trouble here. 112 */ 113void __set_fixmap(enum fixed_addresses idx, 114 phys_addr_t phys, pgprot_t flags) --- 11 unchanged lines hidden (view full) --- 126 pte_clear(&init_mm, addr, ptep); 127 flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 128 } 129} 130 131void *__init fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot) 132{ 133 const u64 dt_virt_base = __fix_to_virt(FIX_FDT); |
134 phys_addr_t dt_phys_base; |
|
133 int offset; 134 void *dt_virt; 135 136 /* 137 * Check whether the physical FDT address is set and meets the minimum 138 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be 139 * at least 8 bytes so that we can always access the magic and size 140 * fields of the FDT header after mapping the first chunk, double check 141 * here if that is indeed the case. 142 */ 143 BUILD_BUG_ON(MIN_FDT_ALIGN < 8); 144 if (!dt_phys || dt_phys % MIN_FDT_ALIGN) 145 return NULL; 146 | 135 int offset; 136 void *dt_virt; 137 138 /* 139 * Check whether the physical FDT address is set and meets the minimum 140 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be 141 * at least 8 bytes so that we can always access the magic and size 142 * fields of the FDT header after mapping the first chunk, double check 143 * here if that is indeed the case. 144 */ 145 BUILD_BUG_ON(MIN_FDT_ALIGN < 8); 146 if (!dt_phys || dt_phys % MIN_FDT_ALIGN) 147 return NULL; 148 |
147 /* 148 * Make sure that the FDT region can be mapped without the need to 149 * allocate additional translation table pages, so that it is safe 150 * to call create_mapping_noalloc() this early. 151 * 152 * On 64k pages, the FDT will be mapped using PTEs, so we need to 153 * be in the same PMD as the rest of the fixmap. 154 * On 4k pages, we'll use section mappings for the FDT so we only 155 * have to be in the same PUD. 156 */ 157 BUILD_BUG_ON(dt_virt_base % SZ_2M); 158 159 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT != 160 __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT); 161 162 offset = dt_phys % SWAPPER_BLOCK_SIZE; | 149 dt_phys_base = round_down(dt_phys, PAGE_SIZE); 150 offset = dt_phys % PAGE_SIZE; |
163 dt_virt = (void *)dt_virt_base + offset; 164 165 /* map the first chunk so we can read the size from the header */ | 151 dt_virt = (void *)dt_virt_base + offset; 152 153 /* map the first chunk so we can read the size from the header */ |
166 create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), 167 dt_virt_base, SWAPPER_BLOCK_SIZE, prot); | 154 create_mapping_noalloc(dt_phys_base, dt_virt_base, PAGE_SIZE, prot); |
168 169 if (fdt_magic(dt_virt) != FDT_MAGIC) 170 return NULL; 171 172 *size = fdt_totalsize(dt_virt); 173 if (*size > MAX_FDT_SIZE) 174 return NULL; 175 | 155 156 if (fdt_magic(dt_virt) != FDT_MAGIC) 157 return NULL; 158 159 *size = fdt_totalsize(dt_virt); 160 if (*size > MAX_FDT_SIZE) 161 return NULL; 162 |
176 if (offset + *size > SWAPPER_BLOCK_SIZE) 177 create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, 178 round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot); | 163 if (offset + *size > PAGE_SIZE) { 164 create_mapping_noalloc(dt_phys_base, dt_virt_base, 165 offset + *size, prot); 166 } |
179 180 return dt_virt; 181} 182 183/* 184 * Copy the fixmap region into a new pgdir. 185 */ 186void __init fixmap_copy(pgd_t *pgdir) --- 29 unchanged lines hidden --- | 167 168 return dt_virt; 169} 170 171/* 172 * Copy the fixmap region into a new pgdir. 173 */ 174void __init fixmap_copy(pgd_t *pgdir) --- 29 unchanged lines hidden --- |