1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2005 Peter Grehan
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 */
29
30 #include <sys/cdefs.h>
31 /*
32 * Dispatch MI pmap calls to the appropriate MMU implementation
33 * through a previously registered kernel object.
34 *
35 * Before pmap_bootstrap() can be called, a CPU module must have
36 * called pmap_mmu_install(). This may be called multiple times:
37 * the highest priority call will be installed as the default
38 * MMU handler when pmap_bootstrap() is called.
39 *
40 * It is required that mutex_init() be called before pmap_bootstrap(),
41 * as the PMAP layer makes extensive use of mutexes.
42 */
43
44 #include <sys/param.h>
45 #include <sys/kernel.h>
46 #include <sys/conf.h>
47 #include <sys/lock.h>
48 #include <sys/kerneldump.h>
49 #include <sys/ktr.h>
50 #include <sys/mutex.h>
51 #include <sys/sysctl.h>
52 #include <sys/systm.h>
53
54 #include <vm/vm.h>
55 #include <vm/vm_extern.h>
56 #include <vm/vm_page.h>
57
58 #include <machine/dump.h>
59 #include <machine/ifunc.h>
60 #include <machine/md_var.h>
61 #include <machine/mmuvar.h>
62 #include <machine/smp.h>
63
64 mmu_t mmu_obj;
65
66 /*
67 * pmap globals
68 */
69 struct pmap kernel_pmap_store;
70
71 vm_offset_t msgbuf_phys;
72
73 vm_offset_t kernel_vm_end;
74 vm_offset_t virtual_avail;
75 vm_offset_t virtual_end;
76 caddr_t crashdumpmap;
77
78 int pmap_bootstrapped;
79 /* Default level 0 reservations consist of 512 pages (2MB superpage). */
80 int vm_level_0_order = 9;
81
82 SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
83
84 int superpages_enabled = 1;
85 SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, CTLFLAG_RDTUN,
86 &superpages_enabled, 0, "Enable support for transparent superpages");
87
88 #ifdef AIM
89 int
pvo_vaddr_compare(struct pvo_entry * a,struct pvo_entry * b)90 pvo_vaddr_compare(struct pvo_entry *a, struct pvo_entry *b)
91 {
92 if (PVO_VADDR(a) < PVO_VADDR(b))
93 return (-1);
94 else if (PVO_VADDR(a) > PVO_VADDR(b))
95 return (1);
96 return (0);
97 }
98 RB_GENERATE(pvo_tree, pvo_entry, pvo_plink, pvo_vaddr_compare);
99 #endif
100
101 static int
pmap_nomethod(void)102 pmap_nomethod(void)
103 {
104 return (0);
105 }
106
107 #define DEFINE_PMAP_IFUNC(ret, func, args) \
108 DEFINE_IFUNC(, ret, pmap_##func, args) { \
109 pmap_##func##_t f; \
110 f = PMAP_RESOLVE_FUNC(func); \
111 return (f != NULL ? f : (pmap_##func##_t)pmap_nomethod);\
112 }
113 #define DEFINE_DUMPSYS_IFUNC(ret, func, args) \
114 DEFINE_IFUNC(, ret, dumpsys_##func, args) { \
115 pmap_dumpsys_##func##_t f; \
116 f = PMAP_RESOLVE_FUNC(dumpsys_##func); \
117 return (f != NULL ? f : (pmap_dumpsys_##func##_t)pmap_nomethod);\
118 }
119
120 DEFINE_PMAP_IFUNC(void, activate, (struct thread *));
121 DEFINE_PMAP_IFUNC(void, advise, (pmap_t, vm_offset_t, vm_offset_t, int));
122 DEFINE_PMAP_IFUNC(void, align_superpage, (vm_object_t, vm_ooffset_t,
123 vm_offset_t *, vm_size_t));
124 DEFINE_PMAP_IFUNC(void, clear_modify, (vm_page_t));
125 DEFINE_PMAP_IFUNC(void, copy, (pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t));
126 DEFINE_PMAP_IFUNC(int, enter, (pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t));
127 DEFINE_PMAP_IFUNC(void, enter_quick, (pmap_t, vm_offset_t, vm_page_t, vm_prot_t));
128 DEFINE_PMAP_IFUNC(void, enter_object, (pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
129 vm_prot_t));
130 DEFINE_PMAP_IFUNC(vm_paddr_t, extract, (pmap_t, vm_offset_t));
131 DEFINE_PMAP_IFUNC(vm_page_t, extract_and_hold, (pmap_t, vm_offset_t, vm_prot_t));
132 DEFINE_PMAP_IFUNC(void, kenter, (vm_offset_t, vm_paddr_t));
133 DEFINE_PMAP_IFUNC(void, kenter_attr, (vm_offset_t, vm_paddr_t, vm_memattr_t));
134 DEFINE_PMAP_IFUNC(vm_paddr_t, kextract, (vm_offset_t));
135 DEFINE_PMAP_IFUNC(void, kremove, (vm_offset_t));
136 DEFINE_PMAP_IFUNC(void, object_init_pt, (pmap_t, vm_offset_t, vm_object_t, vm_pindex_t,
137 vm_size_t));
138 DEFINE_PMAP_IFUNC(bool, is_modified, (vm_page_t));
139 DEFINE_PMAP_IFUNC(bool, is_prefaultable, (pmap_t, vm_offset_t));
140 DEFINE_PMAP_IFUNC(bool, is_referenced, (vm_page_t));
141 DEFINE_PMAP_IFUNC(bool, page_exists_quick, (pmap_t, vm_page_t));
142 DEFINE_PMAP_IFUNC(void, page_init, (vm_page_t));
143 DEFINE_PMAP_IFUNC(bool, page_is_mapped, (vm_page_t));
144 DEFINE_PMAP_IFUNC(int, page_wired_mappings, (vm_page_t));
145 DEFINE_PMAP_IFUNC(void, protect, (pmap_t, vm_offset_t, vm_offset_t, vm_prot_t));
146 DEFINE_PMAP_IFUNC(bool, ps_enabled, (pmap_t));
147 DEFINE_PMAP_IFUNC(void, qenter, (vm_offset_t, vm_page_t *, int));
148 DEFINE_PMAP_IFUNC(void, qremove, (vm_offset_t, int));
149 DEFINE_PMAP_IFUNC(vm_offset_t, quick_enter_page, (vm_page_t));
150 DEFINE_PMAP_IFUNC(void, quick_remove_page, (vm_offset_t));
151 DEFINE_PMAP_IFUNC(int, ts_referenced, (vm_page_t));
152 DEFINE_PMAP_IFUNC(void, release, (pmap_t));
153 DEFINE_PMAP_IFUNC(void, remove, (pmap_t, vm_offset_t, vm_offset_t));
154 DEFINE_PMAP_IFUNC(void, remove_all, (vm_page_t));
155 DEFINE_PMAP_IFUNC(void, remove_pages, (pmap_t));
156 DEFINE_PMAP_IFUNC(void, remove_write, (vm_page_t));
157 DEFINE_PMAP_IFUNC(void, unwire, (pmap_t, vm_offset_t, vm_offset_t));
158 DEFINE_PMAP_IFUNC(void, zero_page, (vm_page_t));
159 DEFINE_PMAP_IFUNC(void, zero_page_area, (vm_page_t, int, int));
160 DEFINE_PMAP_IFUNC(void, copy_page, (vm_page_t, vm_page_t));
161 DEFINE_PMAP_IFUNC(void, copy_pages,
162 (vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
163 vm_offset_t b_offset, int xfersize));
164 DEFINE_PMAP_IFUNC(void, growkernel, (vm_offset_t));
165 DEFINE_PMAP_IFUNC(void, init, (void));
166 DEFINE_PMAP_IFUNC(vm_offset_t, map, (vm_offset_t *, vm_paddr_t, vm_paddr_t, int));
167 DEFINE_PMAP_IFUNC(int, pinit, (pmap_t));
168 DEFINE_PMAP_IFUNC(void, pinit0, (pmap_t));
169 DEFINE_PMAP_IFUNC(int, mincore, (pmap_t, vm_offset_t, vm_paddr_t *));
170 DEFINE_PMAP_IFUNC(void, deactivate, (struct thread *));
171 DEFINE_PMAP_IFUNC(void, bootstrap, (vm_offset_t, vm_offset_t));
172 DEFINE_PMAP_IFUNC(void, cpu_bootstrap, (int));
173 DEFINE_PMAP_IFUNC(void *, mapdev, (vm_paddr_t, vm_size_t));
174 DEFINE_PMAP_IFUNC(void *, mapdev_attr, (vm_paddr_t, vm_size_t, vm_memattr_t));
175 DEFINE_PMAP_IFUNC(void, page_set_memattr, (vm_page_t, vm_memattr_t));
176 DEFINE_PMAP_IFUNC(void, unmapdev, (void *, vm_size_t));
177 DEFINE_PMAP_IFUNC(int, map_user_ptr,
178 (pmap_t, volatile const void *, void **, size_t, size_t *));
179 DEFINE_PMAP_IFUNC(int, decode_kernel_ptr, (vm_offset_t, int *, vm_offset_t *));
180 DEFINE_PMAP_IFUNC(int, dev_direct_mapped, (vm_paddr_t, vm_size_t));
181 DEFINE_PMAP_IFUNC(void, sync_icache, (pmap_t, vm_offset_t, vm_size_t));
182 DEFINE_PMAP_IFUNC(int, change_attr, (vm_offset_t, vm_size_t, vm_memattr_t));
183 DEFINE_PMAP_IFUNC(void, page_array_startup, (long));
184 DEFINE_PMAP_IFUNC(void, tlbie_all, (void));
185
186 DEFINE_DUMPSYS_IFUNC(void, map_chunk, (vm_paddr_t, size_t, void **));
187 DEFINE_DUMPSYS_IFUNC(void, unmap_chunk, (vm_paddr_t, size_t, void *));
188 DEFINE_DUMPSYS_IFUNC(void, pa_init, (void));
189 DEFINE_DUMPSYS_IFUNC(size_t, scan_pmap, (struct bitset *));
190 DEFINE_DUMPSYS_IFUNC(void *, dump_pmap_init, (unsigned));
191 DEFINE_DUMPSYS_IFUNC(void *, dump_pmap, (void *, void *, u_long *));
192
193 /*
194 * MMU install routines. Highest priority wins, equal priority also
195 * overrides allowing last-set to win.
196 */
197 SET_DECLARE(mmu_set, struct mmu_kobj);
198
199 bool
pmap_mmu_install(char * name,int prio)200 pmap_mmu_install(char *name, int prio)
201 {
202 mmu_t *mmupp, mmup;
203 static int curr_prio = 0;
204
205 /*
206 * Try and locate the MMU kobj corresponding to the name
207 */
208 SET_FOREACH(mmupp, mmu_set) {
209 mmup = *mmupp;
210
211 if (mmup->name &&
212 !strcmp(mmup->name, name) &&
213 (prio >= curr_prio || mmu_obj == NULL)) {
214 curr_prio = prio;
215 mmu_obj = mmup;
216 return (true);
217 }
218 }
219
220 return (false);
221 }
222
223 /* MMU "pre-bootstrap" init, used to install extra resolvers, etc. */
224 void
pmap_mmu_init(void)225 pmap_mmu_init(void)
226 {
227 if (mmu_obj->funcs->install != NULL)
228 (mmu_obj->funcs->install)();
229 }
230
231 const char *
pmap_mmu_name(void)232 pmap_mmu_name(void)
233 {
234 return (mmu_obj->name);
235 }
236
237 int unmapped_buf_allowed;
238
239 bool
pmap_is_valid_memattr(pmap_t pmap __unused,vm_memattr_t mode)240 pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)
241 {
242
243 switch (mode) {
244 case VM_MEMATTR_DEFAULT:
245 case VM_MEMATTR_UNCACHEABLE:
246 case VM_MEMATTR_CACHEABLE:
247 case VM_MEMATTR_WRITE_COMBINING:
248 case VM_MEMATTR_WRITE_BACK:
249 case VM_MEMATTR_WRITE_THROUGH:
250 case VM_MEMATTR_PREFETCHABLE:
251 return (true);
252 default:
253 return (false);
254 }
255 }
256
257 void
pmap_active_cpus(pmap_t pmap,cpuset_t * res)258 pmap_active_cpus(pmap_t pmap, cpuset_t *res)
259 {
260 *res = pmap->pm_active;
261 }
262