1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 #include <linux/init.h> 6 #include <linux/export.h> 7 #include <linux/mm.h> 8 #include <asm/fixmap.h> 9 #include <asm/pgalloc.h> 10 #include <asm/pgtable.h> 11 #include <asm/tlbflush.h> 12 13 struct page *dmw_virt_to_page(unsigned long kaddr) 14 { 15 return phys_to_page(__pa(kaddr)); 16 } 17 EXPORT_SYMBOL(dmw_virt_to_page); 18 19 struct page *tlb_virt_to_page(unsigned long kaddr) 20 { 21 return phys_to_page(pfn_to_phys(pte_pfn(*virt_to_kpte(kaddr)))); 22 } 23 EXPORT_SYMBOL(tlb_virt_to_page); 24 25 pgd_t *pgd_alloc(struct mm_struct *mm) 26 { 27 pgd_t *init, *ret; 28 29 ret = __pgd_alloc(mm, 0); 30 if (ret) { 31 init = pgd_offset(&init_mm, 0UL); 32 pgd_init(ret); 33 memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, 34 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); 35 } 36 37 return ret; 38 } 39 EXPORT_SYMBOL_GPL(pgd_alloc); 40 41 void pgd_init(void *addr) 42 { 43 unsigned long *p, *end; 44 unsigned long entry; 45 46 #if !defined(__PAGETABLE_PUD_FOLDED) 47 entry = (unsigned long)invalid_pud_table; 48 #elif !defined(__PAGETABLE_PMD_FOLDED) 49 entry = (unsigned long)invalid_pmd_table; 50 #else 51 entry = (unsigned long)invalid_pte_table; 52 #endif 53 54 p = (unsigned long *)addr; 55 end = p + PTRS_PER_PGD; 56 57 do { 58 p[0] = entry; 59 p[1] = entry; 60 p[2] = entry; 61 p[3] = entry; 62 p[4] = entry; 63 p += 8; 64 p[-3] = entry; 65 p[-2] = entry; 66 p[-1] = entry; 67 } while (p != end); 68 } 69 EXPORT_SYMBOL_GPL(pgd_init); 70 71 #ifndef __PAGETABLE_PMD_FOLDED 72 void pmd_init(void *addr) 73 { 74 unsigned long *p, *end; 75 unsigned long pagetable = (unsigned long)invalid_pte_table; 76 77 p = (unsigned long *)addr; 78 end = p + PTRS_PER_PMD; 79 80 do { 81 p[0] = pagetable; 82 p[1] = pagetable; 83 p[2] = pagetable; 84 p[3] = pagetable; 85 p[4] = pagetable; 86 p += 8; 87 p[-3] = pagetable; 88 p[-2] = pagetable; 89 p[-1] = pagetable; 90 } while (p != end); 91 } 92 EXPORT_SYMBOL_GPL(pmd_init); 93 #endif 94 95 #ifndef __PAGETABLE_PUD_FOLDED 96 void pud_init(void *addr) 97 { 98 unsigned long *p, *end; 99 unsigned long pagetable = (unsigned long)invalid_pmd_table; 100 101 p = (unsigned long *)addr; 102 end = p + PTRS_PER_PUD; 103 104 do { 105 p[0] = pagetable; 106 p[1] = pagetable; 107 p[2] = pagetable; 108 p[3] = pagetable; 109 p[4] = pagetable; 110 p += 8; 111 p[-3] = pagetable; 112 p[-2] = pagetable; 113 p[-1] = pagetable; 114 } while (p != end); 115 } 116 EXPORT_SYMBOL_GPL(pud_init); 117 #endif 118 119 void kernel_pte_init(void *addr) 120 { 121 unsigned long *p, *end; 122 123 p = (unsigned long *)addr; 124 end = p + PTRS_PER_PTE; 125 126 do { 127 p[0] = _PAGE_GLOBAL; 128 p[1] = _PAGE_GLOBAL; 129 p[2] = _PAGE_GLOBAL; 130 p[3] = _PAGE_GLOBAL; 131 p[4] = _PAGE_GLOBAL; 132 p += 8; 133 p[-3] = _PAGE_GLOBAL; 134 p[-2] = _PAGE_GLOBAL; 135 p[-1] = _PAGE_GLOBAL; 136 } while (p != end); 137 } 138 139 void set_pmd_at(struct mm_struct *mm, unsigned long addr, 140 pmd_t *pmdp, pmd_t pmd) 141 { 142 WRITE_ONCE(*pmdp, pmd); 143 flush_tlb_all(); 144 } 145 146 void __init pagetable_init(void) 147 { 148 #ifdef CONFIG_HIGHMEM 149 unsigned long vaddr; 150 pgd_t *pgd; 151 p4d_t *p4d; 152 pud_t *pud; 153 pmd_t *pmd; 154 pte_t *pte; 155 #endif 156 157 /* Initialize the entire pgd. */ 158 pgd_init(swapper_pg_dir); 159 pgd_init(invalid_pg_dir); 160 #ifndef __PAGETABLE_PUD_FOLDED 161 pud_init(invalid_pud_table); 162 #endif 163 #ifndef __PAGETABLE_PMD_FOLDED 164 pmd_init(invalid_pmd_table); 165 #endif 166 167 #ifdef CONFIG_HIGHMEM 168 /* Permanent kmaps */ 169 vaddr = PKMAP_BASE; 170 fixrange_init(vaddr & PMD_MASK, vaddr + PAGE_SIZE * LAST_PKMAP, swapper_pg_dir); 171 172 pgd = swapper_pg_dir + pgd_index(vaddr); 173 p4d = p4d_offset(pgd, vaddr); 174 pud = pud_offset(p4d, vaddr); 175 pmd = pmd_offset(pud, vaddr); 176 pte = pte_offset_kernel(pmd, vaddr); 177 pkmap_page_table = pte; 178 179 /* Fixed mappings */ 180 vaddr = __fix_to_virt(__end_of_fixed_addresses - 1); 181 fixrange_init(vaddr & PMD_MASK, vaddr + FIXADDR_SIZE, swapper_pg_dir); 182 #endif 183 } 184