1 /*-
2 * Copyright (c) 2013 Ian Lepore <ian@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 /* Routines for mapping device memory. */
29
30 #include "opt_ddb.h"
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/devmap.h>
35 #include <vm/vm.h>
36 #include <vm/vm_extern.h>
37 #include <vm/pmap.h>
38 #include <machine/vmparam.h>
39
40 #ifdef __arm__
41 #include <machine/pte.h>
42 #endif
43
44 #ifdef __HAVE_STATIC_DEVMAP
45 #define DEVMAP_PADDR_NOTFOUND ((vm_paddr_t)(-1))
46
47 static const struct devmap_entry *devmap_table;
48 static boolean_t devmap_bootstrap_done = false;
49
50 /*
51 * The allocated-kva (akva) devmap table and metadata. Platforms can call
52 * devmap_add_entry() to add static device mappings to this table using
53 * automatically allocated virtual addresses carved out of the top of kva space.
54 * Allocation begins immediately below the max kernel virtual address.
55 */
56 #define AKVA_DEVMAP_MAX_ENTRIES 32
57 static struct devmap_entry akva_devmap_entries[AKVA_DEVMAP_MAX_ENTRIES];
58 static u_int akva_devmap_idx;
59 #endif
60 static vm_offset_t akva_devmap_vaddr = DEVMAP_MAX_VADDR;
61
62 #if defined(__aarch64__) || defined(__riscv)
63 extern int early_boot;
64 #endif
65
66 #ifdef __HAVE_STATIC_DEVMAP
67 /*
68 * Print the contents of the static mapping table using the provided printf-like
69 * output function (which will be either printf or db_printf).
70 */
71 static void
devmap_dump_table(int (* prfunc)(const char *,...))72 devmap_dump_table(int (*prfunc)(const char *, ...))
73 {
74 const struct devmap_entry *pd;
75
76 if (devmap_table == NULL || devmap_table[0].pd_size == 0) {
77 prfunc("No static device mappings.\n");
78 return;
79 }
80
81 prfunc("Static device mappings:\n");
82 for (pd = devmap_table; pd->pd_size != 0; ++pd) {
83 prfunc(" 0x%08jx - 0x%08jx mapped at VA 0x%08jx\n",
84 (uintmax_t)pd->pd_pa,
85 (uintmax_t)(pd->pd_pa + pd->pd_size - 1),
86 (uintmax_t)pd->pd_va);
87 }
88 }
89
90 /*
91 * Print the contents of the static mapping table. Used for bootverbose.
92 */
93 void
devmap_print_table(void)94 devmap_print_table(void)
95 {
96 devmap_dump_table(printf);
97 }
98
99 /*
100 * Return the "last" kva address used by the registered devmap table. It's
101 * actually the lowest address used by the static mappings, i.e., the address of
102 * the first unusable byte of KVA.
103 */
104 vm_offset_t
devmap_lastaddr(void)105 devmap_lastaddr(void)
106 {
107 const struct devmap_entry *pd;
108 vm_offset_t lowaddr;
109
110 if (akva_devmap_idx > 0)
111 return (akva_devmap_vaddr);
112
113 lowaddr = DEVMAP_MAX_VADDR;
114 for (pd = devmap_table; pd != NULL && pd->pd_size != 0; ++pd) {
115 if (lowaddr > pd->pd_va)
116 lowaddr = pd->pd_va;
117 }
118
119 return (lowaddr);
120 }
121
122 /*
123 * Add an entry to the internal "akva" static devmap table using the given
124 * physical address and size and a virtual address allocated from the top of
125 * kva. This automatically registers the akva table on the first call, so all a
126 * platform has to do is call this routine to install as many mappings as it
127 * needs and when the platform-specific init function calls devmap_bootstrap()
128 * it will pick up all the entries in the akva table automatically.
129 */
130 void
devmap_add_entry(vm_paddr_t pa,vm_size_t sz)131 devmap_add_entry(vm_paddr_t pa, vm_size_t sz)
132 {
133 struct devmap_entry *m;
134
135 if (devmap_bootstrap_done)
136 panic("devmap_add_entry() after devmap_bootstrap()");
137
138 if (akva_devmap_idx == (AKVA_DEVMAP_MAX_ENTRIES - 1))
139 panic("AKVA_DEVMAP_MAX_ENTRIES is too small");
140
141 if (akva_devmap_idx == 0)
142 devmap_register_table(akva_devmap_entries);
143
144 /* Allocate virtual address space from the top of kva downwards. */
145 /*
146 * If the range being mapped is aligned and sized to 1MB boundaries then
147 * also align the virtual address to the next-lower 1MB boundary so that
148 * we end with a nice efficient section mapping.
149 */
150 if ((pa & L1_S_OFFSET) == 0 && (sz & L1_S_OFFSET) == 0) {
151 akva_devmap_vaddr = trunc_1mpage(akva_devmap_vaddr - sz);
152 } else {
153 akva_devmap_vaddr = trunc_page(akva_devmap_vaddr - sz);
154 }
155 m = &akva_devmap_entries[akva_devmap_idx++];
156 m->pd_va = akva_devmap_vaddr;
157 m->pd_pa = pa;
158 m->pd_size = sz;
159 }
160
161 /*
162 * Register the given table as the one to use in devmap_bootstrap().
163 */
164 void
devmap_register_table(const struct devmap_entry * table)165 devmap_register_table(const struct devmap_entry *table)
166 {
167
168 devmap_table = table;
169 }
170
171 /*
172 * Map all of the static regions in the devmap table, and remember the devmap
173 * table so the mapdev, ptov, and vtop functions can do lookups later.
174 */
175 void
devmap_bootstrap(void)176 devmap_bootstrap(void)
177 {
178 const struct devmap_entry *pd;
179
180 devmap_bootstrap_done = true;
181
182 /*
183 * If a table was previously registered, use it. Otherwise, no work to
184 * do.
185 */
186 if (devmap_table == NULL)
187 return;
188
189 for (pd = devmap_table; pd->pd_size != 0; ++pd) {
190 pmap_preboot_map_attr(pd->pd_pa, pd->pd_va, pd->pd_size,
191 VM_PROT_READ | VM_PROT_WRITE, VM_MEMATTR_DEVICE);
192 }
193 }
194
195 /*
196 * Look up the given physical address in the static mapping data and return the
197 * corresponding virtual address, or NULL if not found.
198 */
199 static void *
devmap_ptov(vm_paddr_t pa,vm_size_t size)200 devmap_ptov(vm_paddr_t pa, vm_size_t size)
201 {
202 const struct devmap_entry *pd;
203
204 if (devmap_table == NULL)
205 return (NULL);
206
207 for (pd = devmap_table; pd->pd_size != 0; ++pd) {
208 if (pa >= pd->pd_pa && pa + size <= pd->pd_pa + pd->pd_size)
209 return ((void *)(pd->pd_va + (pa - pd->pd_pa)));
210 }
211
212 return (NULL);
213 }
214
215 /*
216 * Look up the given virtual address in the static mapping data and return the
217 * corresponding physical address, or DEVMAP_PADDR_NOTFOUND if not found.
218 */
219 static vm_paddr_t
devmap_vtop(void * vpva,vm_size_t size)220 devmap_vtop(void * vpva, vm_size_t size)
221 {
222 const struct devmap_entry *pd;
223 vm_offset_t va;
224
225 if (devmap_table == NULL)
226 return (DEVMAP_PADDR_NOTFOUND);
227
228 va = (vm_offset_t)vpva;
229 for (pd = devmap_table; pd->pd_size != 0; ++pd) {
230 if (va >= pd->pd_va && va + size <= pd->pd_va + pd->pd_size)
231 return ((vm_paddr_t)(pd->pd_pa + (va - pd->pd_va)));
232 }
233
234 return (DEVMAP_PADDR_NOTFOUND);
235 }
236 #endif
237
238 /*
239 * Map a set of physical memory pages into the kernel virtual address space.
240 * Return a pointer to where it is mapped.
241 *
242 * This uses a pre-established static mapping if one exists for the requested
243 * range, otherwise it allocates kva space and maps the physical pages into it.
244 *
245 * This routine is intended to be used for mapping device memory, NOT real
246 * memory; the mapping type is inherently VM_MEMATTR_DEVICE in
247 * pmap_kenter_device().
248 */
249 void *
pmap_mapdev(vm_paddr_t pa,vm_size_t size)250 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
251 {
252 return (pmap_mapdev_attr(pa, size, VM_MEMATTR_DEVICE));
253 }
254
255 void *
pmap_mapdev_attr(vm_paddr_t pa,vm_size_t size,vm_memattr_t ma)256 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma)
257 {
258 vm_offset_t va, offset;
259 #ifdef __HAVE_STATIC_DEVMAP
260 void * rva;
261
262 /*
263 * First look in the static mapping table. These are all mapped
264 * as device memory, so only use the devmap for VM_MEMATTR_DEVICE.
265 */
266 if ((rva = devmap_ptov(pa, size)) != NULL) {
267 KASSERT(ma == VM_MEMATTR_DEVICE,
268 ("%s: Non-device mapping for pa %jx (type %x)", __func__,
269 (uintmax_t)pa, ma));
270 return (rva);
271 }
272 #endif
273
274 offset = pa & PAGE_MASK;
275 pa = trunc_page(pa);
276 size = round_page(size + offset);
277
278 #ifdef PMAP_MAPDEV_EARLY_SIZE
279 if (early_boot) {
280 akva_devmap_vaddr = trunc_page(akva_devmap_vaddr - size);
281 va = akva_devmap_vaddr;
282 KASSERT(va >= (VM_MAX_KERNEL_ADDRESS - PMAP_MAPDEV_EARLY_SIZE),
283 ("%s: Too many early devmap mappings", __func__));
284 } else
285 #endif
286 #ifdef __aarch64__
287 if (size >= L2_SIZE && (pa & L2_OFFSET) == 0)
288 va = kva_alloc_aligned(size, L2_SIZE);
289 else if (size >= L3C_SIZE && (pa & L3C_OFFSET) == 0)
290 va = kva_alloc_aligned(size, L3C_SIZE);
291 else
292 #endif
293 va = kva_alloc(size);
294 if (!va)
295 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
296
297 pmap_kenter(va, size, pa, ma);
298
299 return ((void *)(va + offset));
300 }
301
302 /*
303 * Unmap device memory and free the kva space.
304 */
305 void
pmap_unmapdev(void * p,vm_size_t size)306 pmap_unmapdev(void *p, vm_size_t size)
307 {
308 vm_offset_t offset, va;
309
310 #ifdef __HAVE_STATIC_DEVMAP
311 /* Nothing to do if we find the mapping in the static table. */
312 if (devmap_vtop(p, size) != DEVMAP_PADDR_NOTFOUND)
313 return;
314 #endif
315
316 va = (vm_offset_t)p;
317 offset = va & PAGE_MASK;
318 va = trunc_page(va);
319 size = round_page(size + offset);
320
321 pmap_kremove_device(va, size);
322 kva_free(va, size);
323 }
324
325 #ifdef DDB
326 #ifdef __HAVE_STATIC_DEVMAP
327 #include <ddb/ddb.h>
328
DB_SHOW_COMMAND_FLAGS(devmap,db_show_devmap,DB_CMD_MEMSAFE)329 DB_SHOW_COMMAND_FLAGS(devmap, db_show_devmap, DB_CMD_MEMSAFE)
330 {
331 devmap_dump_table(db_printf);
332 }
333
334 #endif
335 #endif /* DDB */
336