1 /*- 2 * Copyright (c) 2013 Ian Lepore <ian@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 /* Routines for mapping device memory. */ 29 30 #include "opt_ddb.h" 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/devmap.h> 35 #include <vm/vm.h> 36 #include <vm/vm_extern.h> 37 #include <vm/pmap.h> 38 #include <machine/vmparam.h> 39 40 #ifdef __arm__ 41 #include <machine/pte.h> 42 #endif 43 44 #ifdef __HAVE_STATIC_DEVMAP 45 #define DEVMAP_PADDR_NOTFOUND ((vm_paddr_t)(-1)) 46 47 static const struct devmap_entry *devmap_table; 48 static boolean_t devmap_bootstrap_done = false; 49 50 /* 51 * The allocated-kva (akva) devmap table and metadata. Platforms can call 52 * devmap_add_entry() to add static device mappings to this table using 53 * automatically allocated virtual addresses carved out of the top of kva space. 54 * Allocation begins immediately below the max kernel virtual address. 55 */ 56 #define AKVA_DEVMAP_MAX_ENTRIES 32 57 static struct devmap_entry akva_devmap_entries[AKVA_DEVMAP_MAX_ENTRIES]; 58 static u_int akva_devmap_idx; 59 #endif 60 static vm_offset_t akva_devmap_vaddr = DEVMAP_MAX_VADDR; 61 62 #if defined(__aarch64__) || defined(__riscv) 63 extern int early_boot; 64 #endif 65 66 #ifdef __HAVE_STATIC_DEVMAP 67 /* 68 * Print the contents of the static mapping table using the provided printf-like 69 * output function (which will be either printf or db_printf). 70 */ 71 static void 72 devmap_dump_table(int (*prfunc)(const char *, ...)) 73 { 74 const struct devmap_entry *pd; 75 76 if (devmap_table == NULL || devmap_table[0].pd_size == 0) { 77 prfunc("No static device mappings.\n"); 78 return; 79 } 80 81 prfunc("Static device mappings:\n"); 82 for (pd = devmap_table; pd->pd_size != 0; ++pd) { 83 prfunc(" 0x%08jx - 0x%08jx mapped at VA 0x%08jx\n", 84 (uintmax_t)pd->pd_pa, 85 (uintmax_t)(pd->pd_pa + pd->pd_size - 1), 86 (uintmax_t)pd->pd_va); 87 } 88 } 89 90 /* 91 * Print the contents of the static mapping table. Used for bootverbose. 92 */ 93 void 94 devmap_print_table(void) 95 { 96 devmap_dump_table(printf); 97 } 98 99 /* 100 * Return the "last" kva address used by the registered devmap table. It's 101 * actually the lowest address used by the static mappings, i.e., the address of 102 * the first unusable byte of KVA. 103 */ 104 vm_offset_t 105 devmap_lastaddr(void) 106 { 107 const struct devmap_entry *pd; 108 vm_offset_t lowaddr; 109 110 if (akva_devmap_idx > 0) 111 return (akva_devmap_vaddr); 112 113 lowaddr = DEVMAP_MAX_VADDR; 114 for (pd = devmap_table; pd != NULL && pd->pd_size != 0; ++pd) { 115 if (lowaddr > pd->pd_va) 116 lowaddr = pd->pd_va; 117 } 118 119 return (lowaddr); 120 } 121 122 /* 123 * Add an entry to the internal "akva" static devmap table using the given 124 * physical address and size and a virtual address allocated from the top of 125 * kva. This automatically registers the akva table on the first call, so all a 126 * platform has to do is call this routine to install as many mappings as it 127 * needs and when the platform-specific init function calls devmap_bootstrap() 128 * it will pick up all the entries in the akva table automatically. 129 */ 130 void 131 devmap_add_entry(vm_paddr_t pa, vm_size_t sz) 132 { 133 struct devmap_entry *m; 134 135 if (devmap_bootstrap_done) 136 panic("devmap_add_entry() after devmap_bootstrap()"); 137 138 if (akva_devmap_idx == (AKVA_DEVMAP_MAX_ENTRIES - 1)) 139 panic("AKVA_DEVMAP_MAX_ENTRIES is too small"); 140 141 if (akva_devmap_idx == 0) 142 devmap_register_table(akva_devmap_entries); 143 144 /* Allocate virtual address space from the top of kva downwards. */ 145 /* 146 * If the range being mapped is aligned and sized to 1MB boundaries then 147 * also align the virtual address to the next-lower 1MB boundary so that 148 * we end with a nice efficient section mapping. 149 */ 150 if ((pa & L1_S_OFFSET) == 0 && (sz & L1_S_OFFSET) == 0) { 151 akva_devmap_vaddr = trunc_1mpage(akva_devmap_vaddr - sz); 152 } else { 153 akva_devmap_vaddr = trunc_page(akva_devmap_vaddr - sz); 154 } 155 m = &akva_devmap_entries[akva_devmap_idx++]; 156 m->pd_va = akva_devmap_vaddr; 157 m->pd_pa = pa; 158 m->pd_size = sz; 159 } 160 161 /* 162 * Register the given table as the one to use in devmap_bootstrap(). 163 */ 164 void 165 devmap_register_table(const struct devmap_entry *table) 166 { 167 168 devmap_table = table; 169 } 170 171 /* 172 * Map all of the static regions in the devmap table, and remember the devmap 173 * table so the mapdev, ptov, and vtop functions can do lookups later. 174 */ 175 void 176 devmap_bootstrap(void) 177 { 178 const struct devmap_entry *pd; 179 180 devmap_bootstrap_done = true; 181 182 /* 183 * If a table was previously registered, use it. Otherwise, no work to 184 * do. 185 */ 186 if (devmap_table == NULL) 187 return; 188 189 for (pd = devmap_table; pd->pd_size != 0; ++pd) { 190 pmap_preboot_map_attr(pd->pd_pa, pd->pd_va, pd->pd_size, 191 VM_PROT_READ | VM_PROT_WRITE, VM_MEMATTR_DEVICE); 192 } 193 } 194 195 /* 196 * Look up the given physical address in the static mapping data and return the 197 * corresponding virtual address, or NULL if not found. 198 */ 199 static void * 200 devmap_ptov(vm_paddr_t pa, vm_size_t size) 201 { 202 const struct devmap_entry *pd; 203 204 if (devmap_table == NULL) 205 return (NULL); 206 207 for (pd = devmap_table; pd->pd_size != 0; ++pd) { 208 if (pa >= pd->pd_pa && pa + size <= pd->pd_pa + pd->pd_size) 209 return ((void *)(pd->pd_va + (pa - pd->pd_pa))); 210 } 211 212 return (NULL); 213 } 214 215 /* 216 * Look up the given virtual address in the static mapping data and return the 217 * corresponding physical address, or DEVMAP_PADDR_NOTFOUND if not found. 218 */ 219 static vm_paddr_t 220 devmap_vtop(void * vpva, vm_size_t size) 221 { 222 const struct devmap_entry *pd; 223 vm_offset_t va; 224 225 if (devmap_table == NULL) 226 return (DEVMAP_PADDR_NOTFOUND); 227 228 va = (vm_offset_t)vpva; 229 for (pd = devmap_table; pd->pd_size != 0; ++pd) { 230 if (va >= pd->pd_va && va + size <= pd->pd_va + pd->pd_size) 231 return ((vm_paddr_t)(pd->pd_pa + (va - pd->pd_va))); 232 } 233 234 return (DEVMAP_PADDR_NOTFOUND); 235 } 236 #endif 237 238 /* 239 * Map a set of physical memory pages into the kernel virtual address space. 240 * Return a pointer to where it is mapped. 241 * 242 * This uses a pre-established static mapping if one exists for the requested 243 * range, otherwise it allocates kva space and maps the physical pages into it. 244 * 245 * This routine is intended to be used for mapping device memory, NOT real 246 * memory; the mapping type is inherently VM_MEMATTR_DEVICE in 247 * pmap_kenter_device(). 248 */ 249 void * 250 pmap_mapdev(vm_paddr_t pa, vm_size_t size) 251 { 252 return (pmap_mapdev_attr(pa, size, VM_MEMATTR_DEVICE)); 253 } 254 255 void * 256 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma) 257 { 258 vm_offset_t va, offset; 259 #ifdef __HAVE_STATIC_DEVMAP 260 void * rva; 261 262 /* 263 * First look in the static mapping table. These are all mapped 264 * as device memory, so only use the devmap for VM_MEMATTR_DEVICE. 265 */ 266 if ((rva = devmap_ptov(pa, size)) != NULL) { 267 KASSERT(ma == VM_MEMATTR_DEVICE, 268 ("%s: Non-device mapping for pa %jx (type %x)", __func__, 269 (uintmax_t)pa, ma)); 270 return (rva); 271 } 272 #endif 273 274 offset = pa & PAGE_MASK; 275 pa = trunc_page(pa); 276 size = round_page(size + offset); 277 278 #ifdef PMAP_MAPDEV_EARLY_SIZE 279 if (early_boot) { 280 akva_devmap_vaddr = trunc_page(akva_devmap_vaddr - size); 281 va = akva_devmap_vaddr; 282 KASSERT(va >= (VM_MAX_KERNEL_ADDRESS - PMAP_MAPDEV_EARLY_SIZE), 283 ("%s: Too many early devmap mappings", __func__)); 284 } else 285 #endif 286 #ifdef __aarch64__ 287 if (size >= L2_SIZE && (pa & L2_OFFSET) == 0) 288 va = kva_alloc_aligned(size, L2_SIZE); 289 else if (size >= L3C_SIZE && (pa & L3C_OFFSET) == 0) 290 va = kva_alloc_aligned(size, L3C_SIZE); 291 else 292 #endif 293 va = kva_alloc(size); 294 if (!va) 295 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 296 297 pmap_kenter(va, size, pa, ma); 298 299 return ((void *)(va + offset)); 300 } 301 302 /* 303 * Unmap device memory and free the kva space. 304 */ 305 void 306 pmap_unmapdev(void *p, vm_size_t size) 307 { 308 vm_offset_t offset, va; 309 310 #ifdef __HAVE_STATIC_DEVMAP 311 /* Nothing to do if we find the mapping in the static table. */ 312 if (devmap_vtop(p, size) != DEVMAP_PADDR_NOTFOUND) 313 return; 314 #endif 315 316 va = (vm_offset_t)p; 317 offset = va & PAGE_MASK; 318 va = trunc_page(va); 319 size = round_page(size + offset); 320 321 pmap_kremove_device(va, size); 322 kva_free(va, size); 323 } 324 325 #ifdef DDB 326 #ifdef __HAVE_STATIC_DEVMAP 327 #include <ddb/ddb.h> 328 329 DB_SHOW_COMMAND_FLAGS(devmap, db_show_devmap, DB_CMD_MEMSAFE) 330 { 331 devmap_dump_table(db_printf); 332 } 333 334 #endif 335 #endif /* DDB */ 336