1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5 #include <linux/init.h>
6 #include <linux/export.h>
7 #include <linux/mm.h>
8 #include <asm/fixmap.h>
9 #include <asm/pgalloc.h>
10 #include <asm/pgtable.h>
11 #include <asm/tlbflush.h>
12
dmw_virt_to_page(unsigned long kaddr)13 struct page *dmw_virt_to_page(unsigned long kaddr)
14 {
15 return phys_to_page(__pa(kaddr));
16 }
17 EXPORT_SYMBOL(dmw_virt_to_page);
18
tlb_virt_to_page(unsigned long kaddr)19 struct page *tlb_virt_to_page(unsigned long kaddr)
20 {
21 return phys_to_page(pfn_to_phys(pte_pfn(*virt_to_kpte(kaddr))));
22 }
23 EXPORT_SYMBOL(tlb_virt_to_page);
24
pgd_alloc(struct mm_struct * mm)25 pgd_t *pgd_alloc(struct mm_struct *mm)
26 {
27 pgd_t *init, *ret;
28
29 ret = __pgd_alloc(mm, 0);
30 if (ret) {
31 init = pgd_offset(&init_mm, 0UL);
32 pgd_init(ret);
33 memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
34 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
35 }
36
37 return ret;
38 }
39 EXPORT_SYMBOL_GPL(pgd_alloc);
40
pgd_init(void * addr)41 void pgd_init(void *addr)
42 {
43 unsigned long *p, *end;
44 unsigned long entry;
45
46 #if !defined(__PAGETABLE_PUD_FOLDED)
47 entry = (unsigned long)invalid_pud_table;
48 #elif !defined(__PAGETABLE_PMD_FOLDED)
49 entry = (unsigned long)invalid_pmd_table;
50 #else
51 entry = (unsigned long)invalid_pte_table;
52 #endif
53
54 p = (unsigned long *)addr;
55 end = p + PTRS_PER_PGD;
56
57 do {
58 p[0] = entry;
59 p[1] = entry;
60 p[2] = entry;
61 p[3] = entry;
62 p[4] = entry;
63 p += 8;
64 p[-3] = entry;
65 p[-2] = entry;
66 p[-1] = entry;
67 } while (p != end);
68 }
69 EXPORT_SYMBOL_GPL(pgd_init);
70
71 #ifndef __PAGETABLE_PMD_FOLDED
pmd_init(void * addr)72 void pmd_init(void *addr)
73 {
74 unsigned long *p, *end;
75 unsigned long pagetable = (unsigned long)invalid_pte_table;
76
77 p = (unsigned long *)addr;
78 end = p + PTRS_PER_PMD;
79
80 do {
81 p[0] = pagetable;
82 p[1] = pagetable;
83 p[2] = pagetable;
84 p[3] = pagetable;
85 p[4] = pagetable;
86 p += 8;
87 p[-3] = pagetable;
88 p[-2] = pagetable;
89 p[-1] = pagetable;
90 } while (p != end);
91 }
92 EXPORT_SYMBOL_GPL(pmd_init);
93 #endif
94
95 #ifndef __PAGETABLE_PUD_FOLDED
pud_init(void * addr)96 void pud_init(void *addr)
97 {
98 unsigned long *p, *end;
99 unsigned long pagetable = (unsigned long)invalid_pmd_table;
100
101 p = (unsigned long *)addr;
102 end = p + PTRS_PER_PUD;
103
104 do {
105 p[0] = pagetable;
106 p[1] = pagetable;
107 p[2] = pagetable;
108 p[3] = pagetable;
109 p[4] = pagetable;
110 p += 8;
111 p[-3] = pagetable;
112 p[-2] = pagetable;
113 p[-1] = pagetable;
114 } while (p != end);
115 }
116 EXPORT_SYMBOL_GPL(pud_init);
117 #endif
118
kernel_pte_init(void * addr)119 void kernel_pte_init(void *addr)
120 {
121 unsigned long *p, *end;
122
123 p = (unsigned long *)addr;
124 end = p + PTRS_PER_PTE;
125
126 do {
127 p[0] = _PAGE_GLOBAL;
128 p[1] = _PAGE_GLOBAL;
129 p[2] = _PAGE_GLOBAL;
130 p[3] = _PAGE_GLOBAL;
131 p[4] = _PAGE_GLOBAL;
132 p += 8;
133 p[-3] = _PAGE_GLOBAL;
134 p[-2] = _PAGE_GLOBAL;
135 p[-1] = _PAGE_GLOBAL;
136 } while (p != end);
137 }
138
set_pmd_at(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t pmd)139 void set_pmd_at(struct mm_struct *mm, unsigned long addr,
140 pmd_t *pmdp, pmd_t pmd)
141 {
142 WRITE_ONCE(*pmdp, pmd);
143 flush_tlb_all();
144 }
145
pagetable_init(void)146 void __init pagetable_init(void)
147 {
148 #ifdef CONFIG_HIGHMEM
149 unsigned long vaddr;
150 pgd_t *pgd;
151 p4d_t *p4d;
152 pud_t *pud;
153 pmd_t *pmd;
154 pte_t *pte;
155 #endif
156
157 /* Initialize the entire pgd. */
158 pgd_init(swapper_pg_dir);
159 pgd_init(invalid_pg_dir);
160 #ifndef __PAGETABLE_PUD_FOLDED
161 pud_init(invalid_pud_table);
162 #endif
163 #ifndef __PAGETABLE_PMD_FOLDED
164 pmd_init(invalid_pmd_table);
165 #endif
166
167 #ifdef CONFIG_HIGHMEM
168 /* Permanent kmaps */
169 vaddr = PKMAP_BASE;
170 fixrange_init(vaddr & PMD_MASK, vaddr + PAGE_SIZE * LAST_PKMAP, swapper_pg_dir);
171
172 pgd = swapper_pg_dir + pgd_index(vaddr);
173 p4d = p4d_offset(pgd, vaddr);
174 pud = pud_offset(p4d, vaddr);
175 pmd = pmd_offset(pud, vaddr);
176 pte = pte_offset_kernel(pmd, vaddr);
177 pkmap_page_table = pte;
178
179 /* Fixed mappings */
180 vaddr = __fix_to_virt(__end_of_fixed_addresses - 1);
181 fixrange_init(vaddr & PMD_MASK, vaddr + FIXADDR_SIZE, swapper_pg_dir);
182 #endif
183 }
184