1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 #include <linux/init.h> 6 #include <linux/export.h> 7 #include <linux/mm.h> 8 #include <asm/pgalloc.h> 9 #include <asm/pgtable.h> 10 #include <asm/tlbflush.h> 11 12 struct page *dmw_virt_to_page(unsigned long kaddr) 13 { 14 return phys_to_page(__pa(kaddr)); 15 } 16 EXPORT_SYMBOL(dmw_virt_to_page); 17 18 struct page *tlb_virt_to_page(unsigned long kaddr) 19 { 20 return phys_to_page(pfn_to_phys(pte_pfn(*virt_to_kpte(kaddr)))); 21 } 22 EXPORT_SYMBOL(tlb_virt_to_page); 23 24 pgd_t *pgd_alloc(struct mm_struct *mm) 25 { 26 pgd_t *init, *ret; 27 28 ret = __pgd_alloc(mm, 0); 29 if (ret) { 30 init = pgd_offset(&init_mm, 0UL); 31 pgd_init(ret); 32 memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, 33 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); 34 } 35 36 return ret; 37 } 38 EXPORT_SYMBOL_GPL(pgd_alloc); 39 40 void pgd_init(void *addr) 41 { 42 unsigned long *p, *end; 43 unsigned long entry; 44 45 #if !defined(__PAGETABLE_PUD_FOLDED) 46 entry = (unsigned long)invalid_pud_table; 47 #elif !defined(__PAGETABLE_PMD_FOLDED) 48 entry = (unsigned long)invalid_pmd_table; 49 #else 50 entry = (unsigned long)invalid_pte_table; 51 #endif 52 53 p = (unsigned long *)addr; 54 end = p + PTRS_PER_PGD; 55 56 do { 57 p[0] = entry; 58 p[1] = entry; 59 p[2] = entry; 60 p[3] = entry; 61 p[4] = entry; 62 p += 8; 63 p[-3] = entry; 64 p[-2] = entry; 65 p[-1] = entry; 66 } while (p != end); 67 } 68 EXPORT_SYMBOL_GPL(pgd_init); 69 70 #ifndef __PAGETABLE_PMD_FOLDED 71 void pmd_init(void *addr) 72 { 73 unsigned long *p, *end; 74 unsigned long pagetable = (unsigned long)invalid_pte_table; 75 76 p = (unsigned long *)addr; 77 end = p + PTRS_PER_PMD; 78 79 do { 80 p[0] = pagetable; 81 p[1] = pagetable; 82 p[2] = pagetable; 83 p[3] = pagetable; 84 p[4] = pagetable; 85 p += 8; 86 p[-3] = pagetable; 87 p[-2] = pagetable; 88 p[-1] = pagetable; 89 } while (p != end); 90 } 91 EXPORT_SYMBOL_GPL(pmd_init); 92 #endif 93 94 #ifndef __PAGETABLE_PUD_FOLDED 95 void pud_init(void *addr) 96 { 97 unsigned long *p, *end; 98 unsigned long pagetable = (unsigned long)invalid_pmd_table; 99 100 p = (unsigned long *)addr; 101 end = p + PTRS_PER_PUD; 102 103 do { 104 p[0] = pagetable; 105 p[1] = pagetable; 106 p[2] = pagetable; 107 p[3] = pagetable; 108 p[4] = pagetable; 109 p += 8; 110 p[-3] = pagetable; 111 p[-2] = pagetable; 112 p[-1] = pagetable; 113 } while (p != end); 114 } 115 EXPORT_SYMBOL_GPL(pud_init); 116 #endif 117 118 void kernel_pte_init(void *addr) 119 { 120 unsigned long *p, *end; 121 122 p = (unsigned long *)addr; 123 end = p + PTRS_PER_PTE; 124 125 do { 126 p[0] = _PAGE_GLOBAL; 127 p[1] = _PAGE_GLOBAL; 128 p[2] = _PAGE_GLOBAL; 129 p[3] = _PAGE_GLOBAL; 130 p[4] = _PAGE_GLOBAL; 131 p += 8; 132 p[-3] = _PAGE_GLOBAL; 133 p[-2] = _PAGE_GLOBAL; 134 p[-1] = _PAGE_GLOBAL; 135 } while (p != end); 136 } 137 138 pmd_t mk_pmd(struct page *page, pgprot_t prot) 139 { 140 pmd_t pmd; 141 142 pmd_val(pmd) = (page_to_pfn(page) << PFN_PTE_SHIFT) | pgprot_val(prot); 143 144 return pmd; 145 } 146 147 void set_pmd_at(struct mm_struct *mm, unsigned long addr, 148 pmd_t *pmdp, pmd_t pmd) 149 { 150 WRITE_ONCE(*pmdp, pmd); 151 flush_tlb_all(); 152 } 153 154 void __init pagetable_init(void) 155 { 156 /* Initialize the entire pgd. */ 157 pgd_init(swapper_pg_dir); 158 pgd_init(invalid_pg_dir); 159 #ifndef __PAGETABLE_PUD_FOLDED 160 pud_init(invalid_pud_table); 161 #endif 162 #ifndef __PAGETABLE_PMD_FOLDED 163 pmd_init(invalid_pmd_table); 164 #endif 165 } 166