1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com> 5 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 20 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 22 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 23 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 24 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 25 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * Some hw specific parts of this pmap were derived or influenced 29 * by NetBSD's ibm4xx pmap module. More generic code is shared with 30 * a few other pmap modules from the FreeBSD tree. 31 */ 32 33 /* 34 * VM layout notes: 35 * 36 * Kernel and user threads run within one common virtual address space 37 * defined by AS=0. 38 * 39 * 32-bit pmap: 40 * Virtual address space layout: 41 * ----------------------------- 42 * 0x0000_0000 - 0x7fff_ffff : user process 43 * 0x8000_0000 - 0xbfff_ffff : pmap_mapdev()-ed area (PCI/PCIE etc.) 44 * 0xc000_0000 - 0xc0ff_ffff : kernel reserved 45 * 0xc000_0000 - data_end : kernel code+data, env, metadata etc. 46 * 0xc100_0000 - 0xffff_ffff : KVA 47 * 0xc100_0000 - 0xc100_3fff : reserved for page zero/copy 48 * 0xc100_4000 - 0xc200_3fff : reserved for ptbl bufs 49 * 0xc200_4000 - 0xc200_8fff : guard page + kstack0 50 * 0xc200_9000 - 0xfeef_ffff : actual free KVA space 51 * 52 * 64-bit pmap: 53 * Virtual address space layout: 54 * ----------------------------- 55 * 0x0000_0000_0000_0000 - 0xbfff_ffff_ffff_ffff : user process 56 * 0x0000_0000_0000_0000 - 0x8fff_ffff_ffff_ffff : text, data, heap, maps, libraries 57 * 0x9000_0000_0000_0000 - 0xafff_ffff_ffff_ffff : mmio region 58 * 0xb000_0000_0000_0000 - 0xbfff_ffff_ffff_ffff : stack 59 * 0xc000_0000_0000_0000 - 0xcfff_ffff_ffff_ffff : kernel reserved 60 * 0xc000_0000_0000_0000 - endkernel-1 : kernel code & data 61 * endkernel - msgbufp-1 : flat device tree 62 * msgbufp - kernel_pdir-1 : message buffer 63 * kernel_pdir - kernel_pp2d-1 : kernel page directory 64 * kernel_pp2d - . : kernel pointers to page directory 65 * pmap_zero_copy_min - crashdumpmap-1 : reserved for page zero/copy 66 * crashdumpmap - ptbl_buf_pool_vabase-1 : reserved for ptbl bufs 67 * ptbl_buf_pool_vabase - virtual_avail-1 : user page directories and page tables 68 * virtual_avail - 0xcfff_ffff_ffff_ffff : actual free KVA space 69 * 0xd000_0000_0000_0000 - 0xdfff_ffff_ffff_ffff : coprocessor region 70 * 0xe000_0000_0000_0000 - 0xefff_ffff_ffff_ffff : mmio region 71 * 0xf000_0000_0000_0000 - 0xffff_ffff_ffff_ffff : direct map 72 * 0xf000_0000_0000_0000 - +Maxmem : physmem map 73 * - 0xffff_ffff_ffff_ffff : device direct map 74 */ 75 76 #include <sys/cdefs.h> 77 __FBSDID("$FreeBSD$"); 78 79 #include "opt_ddb.h" 80 #include "opt_kstack_pages.h" 81 82 #include <sys/param.h> 83 #include <sys/conf.h> 84 #include <sys/malloc.h> 85 #include <sys/ktr.h> 86 #include <sys/proc.h> 87 #include <sys/user.h> 88 #include <sys/queue.h> 89 #include <sys/systm.h> 90 #include <sys/kernel.h> 91 #include <sys/kerneldump.h> 92 #include <sys/linker.h> 93 #include <sys/msgbuf.h> 94 #include <sys/lock.h> 95 #include <sys/mutex.h> 96 #include <sys/rwlock.h> 97 #include <sys/sched.h> 98 #include <sys/smp.h> 99 #include <sys/vmmeter.h> 100 101 #include <vm/vm.h> 102 #include <vm/vm_page.h> 103 #include <vm/vm_kern.h> 104 #include <vm/vm_pageout.h> 105 #include <vm/vm_extern.h> 106 #include <vm/vm_object.h> 107 #include <vm/vm_param.h> 108 #include <vm/vm_map.h> 109 #include <vm/vm_pager.h> 110 #include <vm/vm_phys.h> 111 #include <vm/vm_pagequeue.h> 112 #include <vm/uma.h> 113 114 #include <machine/_inttypes.h> 115 #include <machine/cpu.h> 116 #include <machine/pcb.h> 117 #include <machine/platform.h> 118 119 #include <machine/tlb.h> 120 #include <machine/spr.h> 121 #include <machine/md_var.h> 122 #include <machine/mmuvar.h> 123 #include <machine/pmap.h> 124 #include <machine/pte.h> 125 126 #include <ddb/ddb.h> 127 128 #include "mmu_if.h" 129 130 #define SPARSE_MAPDEV 131 132 /* Use power-of-two mappings in mmu_booke_mapdev(), to save entries. */ 133 #define POW2_MAPPINGS 134 135 #ifdef DEBUG 136 #define debugf(fmt, args...) printf(fmt, ##args) 137 #else 138 #define debugf(fmt, args...) 139 #endif 140 141 #ifdef __powerpc64__ 142 #define PRI0ptrX "016lx" 143 #else 144 #define PRI0ptrX "08x" 145 #endif 146 147 #define TODO panic("%s: not implemented", __func__); 148 149 extern unsigned char _etext[]; 150 extern unsigned char _end[]; 151 152 extern uint32_t *bootinfo; 153 154 vm_paddr_t kernload; 155 vm_offset_t kernstart; 156 vm_size_t kernsize; 157 158 /* Message buffer and tables. */ 159 static vm_offset_t data_start; 160 static vm_size_t data_end; 161 162 /* Phys/avail memory regions. */ 163 static struct mem_region *availmem_regions; 164 static int availmem_regions_sz; 165 static struct mem_region *physmem_regions; 166 static int physmem_regions_sz; 167 168 #ifndef __powerpc64__ 169 /* Reserved KVA space and mutex for mmu_booke_zero_page. */ 170 static vm_offset_t zero_page_va; 171 static struct mtx zero_page_mutex; 172 173 /* Reserved KVA space and mutex for mmu_booke_copy_page. */ 174 static vm_offset_t copy_page_src_va; 175 static vm_offset_t copy_page_dst_va; 176 static struct mtx copy_page_mutex; 177 #endif 178 179 static struct mtx tlbivax_mutex; 180 181 /**************************************************************************/ 182 /* PMAP */ 183 /**************************************************************************/ 184 185 static int mmu_booke_enter_locked(mmu_t, pmap_t, vm_offset_t, vm_page_t, 186 vm_prot_t, u_int flags, int8_t psind); 187 188 unsigned int kptbl_min; /* Index of the first kernel ptbl. */ 189 static uma_zone_t ptbl_root_zone; 190 191 /* 192 * If user pmap is processed with mmu_booke_remove and the resident count 193 * drops to 0, there are no more pages to remove, so we need not continue. 194 */ 195 #define PMAP_REMOVE_DONE(pmap) \ 196 ((pmap) != kernel_pmap && (pmap)->pm_stats.resident_count == 0) 197 198 #if defined(COMPAT_FREEBSD32) || !defined(__powerpc64__) 199 extern int elf32_nxstack; 200 #endif 201 202 /**************************************************************************/ 203 /* TLB and TID handling */ 204 /**************************************************************************/ 205 206 /* Translation ID busy table */ 207 static volatile pmap_t tidbusy[MAXCPU][TID_MAX + 1]; 208 209 /* 210 * TLB0 capabilities (entry, way numbers etc.). These can vary between e500 211 * core revisions and should be read from h/w registers during early config. 212 */ 213 uint32_t tlb0_entries; 214 uint32_t tlb0_ways; 215 uint32_t tlb0_entries_per_way; 216 uint32_t tlb1_entries; 217 218 #define TLB0_ENTRIES (tlb0_entries) 219 #define TLB0_WAYS (tlb0_ways) 220 #define TLB0_ENTRIES_PER_WAY (tlb0_entries_per_way) 221 222 #define TLB1_ENTRIES (tlb1_entries) 223 224 static tlbtid_t tid_alloc(struct pmap *); 225 226 #ifdef DDB 227 #ifdef __powerpc64__ 228 static void tlb_print_entry(int, uint32_t, uint64_t, uint32_t, uint32_t); 229 #else 230 static void tlb_print_entry(int, uint32_t, uint32_t, uint32_t, uint32_t); 231 #endif 232 #endif 233 234 static void tlb1_read_entry(tlb_entry_t *, unsigned int); 235 static void tlb1_write_entry(tlb_entry_t *, unsigned int); 236 static int tlb1_iomapped(int, vm_paddr_t, vm_size_t, vm_offset_t *); 237 static vm_size_t tlb1_mapin_region(vm_offset_t, vm_paddr_t, vm_size_t, int); 238 239 static __inline uint32_t tlb_calc_wimg(vm_paddr_t pa, vm_memattr_t ma); 240 241 static vm_size_t tsize2size(unsigned int); 242 static unsigned int size2tsize(vm_size_t); 243 static unsigned long ilog2(unsigned long); 244 245 static void set_mas4_defaults(void); 246 247 static inline void tlb0_flush_entry(vm_offset_t); 248 static inline unsigned int tlb0_tableidx(vm_offset_t, unsigned int); 249 250 /**************************************************************************/ 251 /* Page table management */ 252 /**************************************************************************/ 253 254 static struct rwlock_padalign pvh_global_lock; 255 256 /* Data for the pv entry allocation mechanism */ 257 static uma_zone_t pvzone; 258 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; 259 260 #define PV_ENTRY_ZONE_MIN 2048 /* min pv entries in uma zone */ 261 262 #ifndef PMAP_SHPGPERPROC 263 #define PMAP_SHPGPERPROC 200 264 #endif 265 266 static vm_paddr_t pte_vatopa(mmu_t, pmap_t, vm_offset_t); 267 static int pte_enter(mmu_t, pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t); 268 static int pte_remove(mmu_t, pmap_t, vm_offset_t, uint8_t); 269 static pte_t *pte_find(mmu_t, pmap_t, vm_offset_t); 270 static void kernel_pte_alloc(vm_offset_t, vm_offset_t); 271 272 static pv_entry_t pv_alloc(void); 273 static void pv_free(pv_entry_t); 274 static void pv_insert(pmap_t, vm_offset_t, vm_page_t); 275 static void pv_remove(pmap_t, vm_offset_t, vm_page_t); 276 277 static void booke_pmap_init_qpages(void); 278 279 static inline void tlb_miss_lock(void); 280 static inline void tlb_miss_unlock(void); 281 282 #ifdef SMP 283 extern tlb_entry_t __boot_tlb1[]; 284 void pmap_bootstrap_ap(volatile uint32_t *); 285 #endif 286 287 /* 288 * Kernel MMU interface 289 */ 290 static void mmu_booke_clear_modify(mmu_t, vm_page_t); 291 static void mmu_booke_copy(mmu_t, pmap_t, pmap_t, vm_offset_t, 292 vm_size_t, vm_offset_t); 293 static void mmu_booke_copy_page(mmu_t, vm_page_t, vm_page_t); 294 static void mmu_booke_copy_pages(mmu_t, vm_page_t *, 295 vm_offset_t, vm_page_t *, vm_offset_t, int); 296 static int mmu_booke_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, 297 vm_prot_t, u_int flags, int8_t psind); 298 static void mmu_booke_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, 299 vm_page_t, vm_prot_t); 300 static void mmu_booke_enter_quick(mmu_t, pmap_t, vm_offset_t, vm_page_t, 301 vm_prot_t); 302 static vm_paddr_t mmu_booke_extract(mmu_t, pmap_t, vm_offset_t); 303 static vm_page_t mmu_booke_extract_and_hold(mmu_t, pmap_t, vm_offset_t, 304 vm_prot_t); 305 static void mmu_booke_init(mmu_t); 306 static boolean_t mmu_booke_is_modified(mmu_t, vm_page_t); 307 static boolean_t mmu_booke_is_prefaultable(mmu_t, pmap_t, vm_offset_t); 308 static boolean_t mmu_booke_is_referenced(mmu_t, vm_page_t); 309 static int mmu_booke_ts_referenced(mmu_t, vm_page_t); 310 static vm_offset_t mmu_booke_map(mmu_t, vm_offset_t *, vm_paddr_t, vm_paddr_t, 311 int); 312 static int mmu_booke_mincore(mmu_t, pmap_t, vm_offset_t, 313 vm_paddr_t *); 314 static void mmu_booke_object_init_pt(mmu_t, pmap_t, vm_offset_t, 315 vm_object_t, vm_pindex_t, vm_size_t); 316 static boolean_t mmu_booke_page_exists_quick(mmu_t, pmap_t, vm_page_t); 317 static void mmu_booke_page_init(mmu_t, vm_page_t); 318 static int mmu_booke_page_wired_mappings(mmu_t, vm_page_t); 319 static void mmu_booke_pinit(mmu_t, pmap_t); 320 static void mmu_booke_pinit0(mmu_t, pmap_t); 321 static void mmu_booke_protect(mmu_t, pmap_t, vm_offset_t, vm_offset_t, 322 vm_prot_t); 323 static void mmu_booke_qenter(mmu_t, vm_offset_t, vm_page_t *, int); 324 static void mmu_booke_qremove(mmu_t, vm_offset_t, int); 325 static void mmu_booke_release(mmu_t, pmap_t); 326 static void mmu_booke_remove(mmu_t, pmap_t, vm_offset_t, vm_offset_t); 327 static void mmu_booke_remove_all(mmu_t, vm_page_t); 328 static void mmu_booke_remove_write(mmu_t, vm_page_t); 329 static void mmu_booke_unwire(mmu_t, pmap_t, vm_offset_t, vm_offset_t); 330 static void mmu_booke_zero_page(mmu_t, vm_page_t); 331 static void mmu_booke_zero_page_area(mmu_t, vm_page_t, int, int); 332 static void mmu_booke_activate(mmu_t, struct thread *); 333 static void mmu_booke_deactivate(mmu_t, struct thread *); 334 static void mmu_booke_bootstrap(mmu_t, vm_offset_t, vm_offset_t); 335 static void *mmu_booke_mapdev(mmu_t, vm_paddr_t, vm_size_t); 336 static void *mmu_booke_mapdev_attr(mmu_t, vm_paddr_t, vm_size_t, vm_memattr_t); 337 static void mmu_booke_unmapdev(mmu_t, vm_offset_t, vm_size_t); 338 static vm_paddr_t mmu_booke_kextract(mmu_t, vm_offset_t); 339 static void mmu_booke_kenter(mmu_t, vm_offset_t, vm_paddr_t); 340 static void mmu_booke_kenter_attr(mmu_t, vm_offset_t, vm_paddr_t, vm_memattr_t); 341 static void mmu_booke_kremove(mmu_t, vm_offset_t); 342 static boolean_t mmu_booke_dev_direct_mapped(mmu_t, vm_paddr_t, vm_size_t); 343 static void mmu_booke_sync_icache(mmu_t, pmap_t, vm_offset_t, 344 vm_size_t); 345 static void mmu_booke_dumpsys_map(mmu_t, vm_paddr_t pa, size_t, 346 void **); 347 static void mmu_booke_dumpsys_unmap(mmu_t, vm_paddr_t pa, size_t, 348 void *); 349 static void mmu_booke_scan_init(mmu_t); 350 static vm_offset_t mmu_booke_quick_enter_page(mmu_t mmu, vm_page_t m); 351 static void mmu_booke_quick_remove_page(mmu_t mmu, vm_offset_t addr); 352 static int mmu_booke_change_attr(mmu_t mmu, vm_offset_t addr, 353 vm_size_t sz, vm_memattr_t mode); 354 static int mmu_booke_map_user_ptr(mmu_t mmu, pmap_t pm, 355 volatile const void *uaddr, void **kaddr, size_t ulen, size_t *klen); 356 static int mmu_booke_decode_kernel_ptr(mmu_t mmu, vm_offset_t addr, 357 int *is_user, vm_offset_t *decoded_addr); 358 static void mmu_booke_page_array_startup(mmu_t , long); 359 360 361 static mmu_method_t mmu_booke_methods[] = { 362 /* pmap dispatcher interface */ 363 MMUMETHOD(mmu_clear_modify, mmu_booke_clear_modify), 364 MMUMETHOD(mmu_copy, mmu_booke_copy), 365 MMUMETHOD(mmu_copy_page, mmu_booke_copy_page), 366 MMUMETHOD(mmu_copy_pages, mmu_booke_copy_pages), 367 MMUMETHOD(mmu_enter, mmu_booke_enter), 368 MMUMETHOD(mmu_enter_object, mmu_booke_enter_object), 369 MMUMETHOD(mmu_enter_quick, mmu_booke_enter_quick), 370 MMUMETHOD(mmu_extract, mmu_booke_extract), 371 MMUMETHOD(mmu_extract_and_hold, mmu_booke_extract_and_hold), 372 MMUMETHOD(mmu_init, mmu_booke_init), 373 MMUMETHOD(mmu_is_modified, mmu_booke_is_modified), 374 MMUMETHOD(mmu_is_prefaultable, mmu_booke_is_prefaultable), 375 MMUMETHOD(mmu_is_referenced, mmu_booke_is_referenced), 376 MMUMETHOD(mmu_ts_referenced, mmu_booke_ts_referenced), 377 MMUMETHOD(mmu_map, mmu_booke_map), 378 MMUMETHOD(mmu_mincore, mmu_booke_mincore), 379 MMUMETHOD(mmu_object_init_pt, mmu_booke_object_init_pt), 380 MMUMETHOD(mmu_page_exists_quick,mmu_booke_page_exists_quick), 381 MMUMETHOD(mmu_page_init, mmu_booke_page_init), 382 MMUMETHOD(mmu_page_wired_mappings, mmu_booke_page_wired_mappings), 383 MMUMETHOD(mmu_pinit, mmu_booke_pinit), 384 MMUMETHOD(mmu_pinit0, mmu_booke_pinit0), 385 MMUMETHOD(mmu_protect, mmu_booke_protect), 386 MMUMETHOD(mmu_qenter, mmu_booke_qenter), 387 MMUMETHOD(mmu_qremove, mmu_booke_qremove), 388 MMUMETHOD(mmu_release, mmu_booke_release), 389 MMUMETHOD(mmu_remove, mmu_booke_remove), 390 MMUMETHOD(mmu_remove_all, mmu_booke_remove_all), 391 MMUMETHOD(mmu_remove_write, mmu_booke_remove_write), 392 MMUMETHOD(mmu_sync_icache, mmu_booke_sync_icache), 393 MMUMETHOD(mmu_unwire, mmu_booke_unwire), 394 MMUMETHOD(mmu_zero_page, mmu_booke_zero_page), 395 MMUMETHOD(mmu_zero_page_area, mmu_booke_zero_page_area), 396 MMUMETHOD(mmu_activate, mmu_booke_activate), 397 MMUMETHOD(mmu_deactivate, mmu_booke_deactivate), 398 MMUMETHOD(mmu_quick_enter_page, mmu_booke_quick_enter_page), 399 MMUMETHOD(mmu_quick_remove_page, mmu_booke_quick_remove_page), 400 MMUMETHOD(mmu_page_array_startup, mmu_booke_page_array_startup), 401 402 /* Internal interfaces */ 403 MMUMETHOD(mmu_bootstrap, mmu_booke_bootstrap), 404 MMUMETHOD(mmu_dev_direct_mapped,mmu_booke_dev_direct_mapped), 405 MMUMETHOD(mmu_mapdev, mmu_booke_mapdev), 406 MMUMETHOD(mmu_mapdev_attr, mmu_booke_mapdev_attr), 407 MMUMETHOD(mmu_kenter, mmu_booke_kenter), 408 MMUMETHOD(mmu_kenter_attr, mmu_booke_kenter_attr), 409 MMUMETHOD(mmu_kextract, mmu_booke_kextract), 410 MMUMETHOD(mmu_kremove, mmu_booke_kremove), 411 MMUMETHOD(mmu_unmapdev, mmu_booke_unmapdev), 412 MMUMETHOD(mmu_change_attr, mmu_booke_change_attr), 413 MMUMETHOD(mmu_map_user_ptr, mmu_booke_map_user_ptr), 414 MMUMETHOD(mmu_decode_kernel_ptr, mmu_booke_decode_kernel_ptr), 415 416 /* dumpsys() support */ 417 MMUMETHOD(mmu_dumpsys_map, mmu_booke_dumpsys_map), 418 MMUMETHOD(mmu_dumpsys_unmap, mmu_booke_dumpsys_unmap), 419 MMUMETHOD(mmu_scan_init, mmu_booke_scan_init), 420 421 { 0, 0 } 422 }; 423 424 MMU_DEF(booke_mmu, MMU_TYPE_BOOKE, mmu_booke_methods, 0); 425 426 #ifdef __powerpc64__ 427 #include "pmap_64.c" 428 #else 429 #include "pmap_32.c" 430 #endif 431 432 static vm_offset_t tlb1_map_base = VM_MAPDEV_BASE; 433 434 static __inline uint32_t 435 tlb_calc_wimg(vm_paddr_t pa, vm_memattr_t ma) 436 { 437 uint32_t attrib; 438 int i; 439 440 if (ma != VM_MEMATTR_DEFAULT) { 441 switch (ma) { 442 case VM_MEMATTR_UNCACHEABLE: 443 return (MAS2_I | MAS2_G); 444 case VM_MEMATTR_WRITE_COMBINING: 445 case VM_MEMATTR_WRITE_BACK: 446 case VM_MEMATTR_PREFETCHABLE: 447 return (MAS2_I); 448 case VM_MEMATTR_WRITE_THROUGH: 449 return (MAS2_W | MAS2_M); 450 case VM_MEMATTR_CACHEABLE: 451 return (MAS2_M); 452 } 453 } 454 455 /* 456 * Assume the page is cache inhibited and access is guarded unless 457 * it's in our available memory array. 458 */ 459 attrib = _TLB_ENTRY_IO; 460 for (i = 0; i < physmem_regions_sz; i++) { 461 if ((pa >= physmem_regions[i].mr_start) && 462 (pa < (physmem_regions[i].mr_start + 463 physmem_regions[i].mr_size))) { 464 attrib = _TLB_ENTRY_MEM; 465 break; 466 } 467 } 468 469 return (attrib); 470 } 471 472 static inline void 473 tlb_miss_lock(void) 474 { 475 #ifdef SMP 476 struct pcpu *pc; 477 478 if (!smp_started) 479 return; 480 481 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 482 if (pc != pcpup) { 483 484 CTR3(KTR_PMAP, "%s: tlb miss LOCK of CPU=%d, " 485 "tlb_lock=%p", __func__, pc->pc_cpuid, pc->pc_booke.tlb_lock); 486 487 KASSERT((pc->pc_cpuid != PCPU_GET(cpuid)), 488 ("tlb_miss_lock: tried to lock self")); 489 490 tlb_lock(pc->pc_booke.tlb_lock); 491 492 CTR1(KTR_PMAP, "%s: locked", __func__); 493 } 494 } 495 #endif 496 } 497 498 static inline void 499 tlb_miss_unlock(void) 500 { 501 #ifdef SMP 502 struct pcpu *pc; 503 504 if (!smp_started) 505 return; 506 507 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 508 if (pc != pcpup) { 509 CTR2(KTR_PMAP, "%s: tlb miss UNLOCK of CPU=%d", 510 __func__, pc->pc_cpuid); 511 512 tlb_unlock(pc->pc_booke.tlb_lock); 513 514 CTR1(KTR_PMAP, "%s: unlocked", __func__); 515 } 516 } 517 #endif 518 } 519 520 /* Return number of entries in TLB0. */ 521 static __inline void 522 tlb0_get_tlbconf(void) 523 { 524 uint32_t tlb0_cfg; 525 526 tlb0_cfg = mfspr(SPR_TLB0CFG); 527 tlb0_entries = tlb0_cfg & TLBCFG_NENTRY_MASK; 528 tlb0_ways = (tlb0_cfg & TLBCFG_ASSOC_MASK) >> TLBCFG_ASSOC_SHIFT; 529 tlb0_entries_per_way = tlb0_entries / tlb0_ways; 530 } 531 532 /* Return number of entries in TLB1. */ 533 static __inline void 534 tlb1_get_tlbconf(void) 535 { 536 uint32_t tlb1_cfg; 537 538 tlb1_cfg = mfspr(SPR_TLB1CFG); 539 tlb1_entries = tlb1_cfg & TLBCFG_NENTRY_MASK; 540 } 541 542 /**************************************************************************/ 543 /* Page table related */ 544 /**************************************************************************/ 545 546 /* Allocate pv_entry structure. */ 547 pv_entry_t 548 pv_alloc(void) 549 { 550 pv_entry_t pv; 551 552 pv_entry_count++; 553 if (pv_entry_count > pv_entry_high_water) 554 pagedaemon_wakeup(0); /* XXX powerpc NUMA */ 555 pv = uma_zalloc(pvzone, M_NOWAIT); 556 557 return (pv); 558 } 559 560 /* Free pv_entry structure. */ 561 static __inline void 562 pv_free(pv_entry_t pve) 563 { 564 565 pv_entry_count--; 566 uma_zfree(pvzone, pve); 567 } 568 569 570 /* Allocate and initialize pv_entry structure. */ 571 static void 572 pv_insert(pmap_t pmap, vm_offset_t va, vm_page_t m) 573 { 574 pv_entry_t pve; 575 576 //int su = (pmap == kernel_pmap); 577 //debugf("pv_insert: s (su = %d pmap = 0x%08x va = 0x%08x m = 0x%08x)\n", su, 578 // (u_int32_t)pmap, va, (u_int32_t)m); 579 580 pve = pv_alloc(); 581 if (pve == NULL) 582 panic("pv_insert: no pv entries!"); 583 584 pve->pv_pmap = pmap; 585 pve->pv_va = va; 586 587 /* add to pv_list */ 588 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 589 rw_assert(&pvh_global_lock, RA_WLOCKED); 590 591 TAILQ_INSERT_TAIL(&m->md.pv_list, pve, pv_link); 592 593 //debugf("pv_insert: e\n"); 594 } 595 596 /* Destroy pv entry. */ 597 static void 598 pv_remove(pmap_t pmap, vm_offset_t va, vm_page_t m) 599 { 600 pv_entry_t pve; 601 602 //int su = (pmap == kernel_pmap); 603 //debugf("pv_remove: s (su = %d pmap = 0x%08x va = 0x%08x)\n", su, (u_int32_t)pmap, va); 604 605 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 606 rw_assert(&pvh_global_lock, RA_WLOCKED); 607 608 /* find pv entry */ 609 TAILQ_FOREACH(pve, &m->md.pv_list, pv_link) { 610 if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) { 611 /* remove from pv_list */ 612 TAILQ_REMOVE(&m->md.pv_list, pve, pv_link); 613 if (TAILQ_EMPTY(&m->md.pv_list)) 614 vm_page_aflag_clear(m, PGA_WRITEABLE); 615 616 /* free pv entry struct */ 617 pv_free(pve); 618 break; 619 } 620 } 621 622 //debugf("pv_remove: e\n"); 623 } 624 625 /**************************************************************************/ 626 /* PMAP related */ 627 /**************************************************************************/ 628 629 /* 630 * This is called during booke_init, before the system is really initialized. 631 */ 632 static void 633 mmu_booke_bootstrap(mmu_t mmu, vm_offset_t start, vm_offset_t kernelend) 634 { 635 vm_paddr_t phys_kernelend; 636 struct mem_region *mp, *mp1; 637 int cnt, i, j; 638 vm_paddr_t s, e, sz; 639 vm_paddr_t physsz, hwphyssz; 640 u_int phys_avail_count; 641 vm_size_t kstack0_sz; 642 vm_paddr_t kstack0_phys; 643 vm_offset_t kstack0; 644 void *dpcpu; 645 646 debugf("mmu_booke_bootstrap: entered\n"); 647 648 /* Set interesting system properties */ 649 #ifdef __powerpc64__ 650 hw_direct_map = 1; 651 #else 652 hw_direct_map = 0; 653 #endif 654 #if defined(COMPAT_FREEBSD32) || !defined(__powerpc64__) 655 elf32_nxstack = 1; 656 #endif 657 658 /* Initialize invalidation mutex */ 659 mtx_init(&tlbivax_mutex, "tlbivax", NULL, MTX_SPIN); 660 661 /* Read TLB0 size and associativity. */ 662 tlb0_get_tlbconf(); 663 664 /* 665 * Align kernel start and end address (kernel image). 666 * Note that kernel end does not necessarily relate to kernsize. 667 * kernsize is the size of the kernel that is actually mapped. 668 */ 669 data_start = round_page(kernelend); 670 data_end = data_start; 671 672 /* Allocate the dynamic per-cpu area. */ 673 dpcpu = (void *)data_end; 674 data_end += DPCPU_SIZE; 675 676 /* Allocate space for the message buffer. */ 677 msgbufp = (struct msgbuf *)data_end; 678 data_end += msgbufsize; 679 debugf(" msgbufp at 0x%"PRI0ptrX" end = 0x%"PRI0ptrX"\n", 680 (uintptr_t)msgbufp, data_end); 681 682 data_end = round_page(data_end); 683 data_end = round_page(mmu_booke_alloc_kernel_pgtables(data_end)); 684 685 /* Retrieve phys/avail mem regions */ 686 mem_regions(&physmem_regions, &physmem_regions_sz, 687 &availmem_regions, &availmem_regions_sz); 688 689 if (PHYS_AVAIL_ENTRIES < availmem_regions_sz) 690 panic("mmu_booke_bootstrap: phys_avail too small"); 691 692 data_end = round_page(data_end); 693 vm_page_array = (vm_page_t)data_end; 694 /* 695 * Get a rough idea (upper bound) on the size of the page array. The 696 * vm_page_array will not handle any more pages than we have in the 697 * avail_regions array, and most likely much less. 698 */ 699 sz = 0; 700 for (mp = availmem_regions; mp->mr_size; mp++) { 701 sz += mp->mr_size; 702 } 703 sz = (round_page(sz) / (PAGE_SIZE + sizeof(struct vm_page))); 704 data_end += round_page(sz * sizeof(struct vm_page)); 705 706 /* Pre-round up to 1MB. This wastes some space, but saves TLB entries */ 707 data_end = roundup2(data_end, 1 << 20); 708 709 debugf(" data_end: 0x%"PRI0ptrX"\n", data_end); 710 debugf(" kernstart: %#zx\n", kernstart); 711 debugf(" kernsize: %#zx\n", kernsize); 712 713 if (data_end - kernstart > kernsize) { 714 kernsize += tlb1_mapin_region(kernstart + kernsize, 715 kernload + kernsize, (data_end - kernstart) - kernsize, 716 _TLB_ENTRY_MEM); 717 } 718 data_end = kernstart + kernsize; 719 debugf(" updated data_end: 0x%"PRI0ptrX"\n", data_end); 720 721 /* 722 * Clear the structures - note we can only do it safely after the 723 * possible additional TLB1 translations are in place (above) so that 724 * all range up to the currently calculated 'data_end' is covered. 725 */ 726 bzero((void *)data_start, data_end - data_start); 727 dpcpu_init(dpcpu, 0); 728 729 /*******************************************************/ 730 /* Set the start and end of kva. */ 731 /*******************************************************/ 732 virtual_avail = round_page(data_end); 733 virtual_end = VM_MAX_KERNEL_ADDRESS; 734 735 #ifndef __powerpc64__ 736 /* Allocate KVA space for page zero/copy operations. */ 737 zero_page_va = virtual_avail; 738 virtual_avail += PAGE_SIZE; 739 copy_page_src_va = virtual_avail; 740 virtual_avail += PAGE_SIZE; 741 copy_page_dst_va = virtual_avail; 742 virtual_avail += PAGE_SIZE; 743 debugf("zero_page_va = 0x%"PRI0ptrX"\n", zero_page_va); 744 debugf("copy_page_src_va = 0x%"PRI0ptrX"\n", copy_page_src_va); 745 debugf("copy_page_dst_va = 0x%"PRI0ptrX"\n", copy_page_dst_va); 746 747 /* Initialize page zero/copy mutexes. */ 748 mtx_init(&zero_page_mutex, "mmu_booke_zero_page", NULL, MTX_DEF); 749 mtx_init(©_page_mutex, "mmu_booke_copy_page", NULL, MTX_DEF); 750 751 /* Allocate KVA space for ptbl bufs. */ 752 ptbl_buf_pool_vabase = virtual_avail; 753 virtual_avail += PTBL_BUFS * PTBL_PAGES * PAGE_SIZE; 754 debugf("ptbl_buf_pool_vabase = 0x%"PRI0ptrX" end = 0x%"PRI0ptrX"\n", 755 ptbl_buf_pool_vabase, virtual_avail); 756 #endif 757 758 /* Calculate corresponding physical addresses for the kernel region. */ 759 phys_kernelend = kernload + kernsize; 760 debugf("kernel image and allocated data:\n"); 761 debugf(" kernload = 0x%09jx\n", (uintmax_t)kernload); 762 debugf(" kernstart = 0x%"PRI0ptrX"\n", kernstart); 763 debugf(" kernsize = 0x%"PRI0ptrX"\n", kernsize); 764 765 /* 766 * Remove kernel physical address range from avail regions list. Page 767 * align all regions. Non-page aligned memory isn't very interesting 768 * to us. Also, sort the entries for ascending addresses. 769 */ 770 771 sz = 0; 772 cnt = availmem_regions_sz; 773 debugf("processing avail regions:\n"); 774 for (mp = availmem_regions; mp->mr_size; mp++) { 775 s = mp->mr_start; 776 e = mp->mr_start + mp->mr_size; 777 debugf(" %09jx-%09jx -> ", (uintmax_t)s, (uintmax_t)e); 778 /* Check whether this region holds all of the kernel. */ 779 if (s < kernload && e > phys_kernelend) { 780 availmem_regions[cnt].mr_start = phys_kernelend; 781 availmem_regions[cnt++].mr_size = e - phys_kernelend; 782 e = kernload; 783 } 784 /* Look whether this regions starts within the kernel. */ 785 if (s >= kernload && s < phys_kernelend) { 786 if (e <= phys_kernelend) 787 goto empty; 788 s = phys_kernelend; 789 } 790 /* Now look whether this region ends within the kernel. */ 791 if (e > kernload && e <= phys_kernelend) { 792 if (s >= kernload) 793 goto empty; 794 e = kernload; 795 } 796 /* Now page align the start and size of the region. */ 797 s = round_page(s); 798 e = trunc_page(e); 799 if (e < s) 800 e = s; 801 sz = e - s; 802 debugf("%09jx-%09jx = %jx\n", 803 (uintmax_t)s, (uintmax_t)e, (uintmax_t)sz); 804 805 /* Check whether some memory is left here. */ 806 if (sz == 0) { 807 empty: 808 memmove(mp, mp + 1, 809 (cnt - (mp - availmem_regions)) * sizeof(*mp)); 810 cnt--; 811 mp--; 812 continue; 813 } 814 815 /* Do an insertion sort. */ 816 for (mp1 = availmem_regions; mp1 < mp; mp1++) 817 if (s < mp1->mr_start) 818 break; 819 if (mp1 < mp) { 820 memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1); 821 mp1->mr_start = s; 822 mp1->mr_size = sz; 823 } else { 824 mp->mr_start = s; 825 mp->mr_size = sz; 826 } 827 } 828 availmem_regions_sz = cnt; 829 830 /*******************************************************/ 831 /* Steal physical memory for kernel stack from the end */ 832 /* of the first avail region */ 833 /*******************************************************/ 834 kstack0_sz = kstack_pages * PAGE_SIZE; 835 kstack0_phys = availmem_regions[0].mr_start + 836 availmem_regions[0].mr_size; 837 kstack0_phys -= kstack0_sz; 838 availmem_regions[0].mr_size -= kstack0_sz; 839 840 /*******************************************************/ 841 /* Fill in phys_avail table, based on availmem_regions */ 842 /*******************************************************/ 843 phys_avail_count = 0; 844 physsz = 0; 845 hwphyssz = 0; 846 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); 847 848 debugf("fill in phys_avail:\n"); 849 for (i = 0, j = 0; i < availmem_regions_sz; i++, j += 2) { 850 851 debugf(" region: 0x%jx - 0x%jx (0x%jx)\n", 852 (uintmax_t)availmem_regions[i].mr_start, 853 (uintmax_t)availmem_regions[i].mr_start + 854 availmem_regions[i].mr_size, 855 (uintmax_t)availmem_regions[i].mr_size); 856 857 if (hwphyssz != 0 && 858 (physsz + availmem_regions[i].mr_size) >= hwphyssz) { 859 debugf(" hw.physmem adjust\n"); 860 if (physsz < hwphyssz) { 861 phys_avail[j] = availmem_regions[i].mr_start; 862 phys_avail[j + 1] = 863 availmem_regions[i].mr_start + 864 hwphyssz - physsz; 865 physsz = hwphyssz; 866 phys_avail_count++; 867 dump_avail[j] = phys_avail[j]; 868 dump_avail[j + 1] = phys_avail[j + 1]; 869 } 870 break; 871 } 872 873 phys_avail[j] = availmem_regions[i].mr_start; 874 phys_avail[j + 1] = availmem_regions[i].mr_start + 875 availmem_regions[i].mr_size; 876 phys_avail_count++; 877 physsz += availmem_regions[i].mr_size; 878 dump_avail[j] = phys_avail[j]; 879 dump_avail[j + 1] = phys_avail[j + 1]; 880 } 881 physmem = btoc(physsz); 882 883 /* Calculate the last available physical address. */ 884 for (i = 0; phys_avail[i + 2] != 0; i += 2) 885 ; 886 Maxmem = powerpc_btop(phys_avail[i + 1]); 887 888 debugf("Maxmem = 0x%08lx\n", Maxmem); 889 debugf("phys_avail_count = %d\n", phys_avail_count); 890 debugf("physsz = 0x%09jx physmem = %jd (0x%09jx)\n", 891 (uintmax_t)physsz, (uintmax_t)physmem, (uintmax_t)physmem); 892 893 #ifdef __powerpc64__ 894 /* 895 * Map the physical memory contiguously in TLB1. 896 * Round so it fits into a single mapping. 897 */ 898 tlb1_mapin_region(DMAP_BASE_ADDRESS, 0, 899 phys_avail[i + 1], _TLB_ENTRY_MEM); 900 #endif 901 902 /*******************************************************/ 903 /* Initialize (statically allocated) kernel pmap. */ 904 /*******************************************************/ 905 PMAP_LOCK_INIT(kernel_pmap); 906 907 debugf("kernel_pmap = 0x%"PRI0ptrX"\n", (uintptr_t)kernel_pmap); 908 kernel_pte_alloc(virtual_avail, kernstart); 909 for (i = 0; i < MAXCPU; i++) { 910 kernel_pmap->pm_tid[i] = TID_KERNEL; 911 912 /* Initialize each CPU's tidbusy entry 0 with kernel_pmap */ 913 tidbusy[i][TID_KERNEL] = kernel_pmap; 914 } 915 916 /* Mark kernel_pmap active on all CPUs */ 917 CPU_FILL(&kernel_pmap->pm_active); 918 919 /* 920 * Initialize the global pv list lock. 921 */ 922 rw_init(&pvh_global_lock, "pmap pv global"); 923 924 /*******************************************************/ 925 /* Final setup */ 926 /*******************************************************/ 927 928 /* Enter kstack0 into kernel map, provide guard page */ 929 kstack0 = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; 930 thread0.td_kstack = kstack0; 931 thread0.td_kstack_pages = kstack_pages; 932 933 debugf("kstack_sz = 0x%08jx\n", (uintmax_t)kstack0_sz); 934 debugf("kstack0_phys at 0x%09jx - 0x%09jx\n", 935 (uintmax_t)kstack0_phys, (uintmax_t)kstack0_phys + kstack0_sz); 936 debugf("kstack0 at 0x%"PRI0ptrX" - 0x%"PRI0ptrX"\n", 937 kstack0, kstack0 + kstack0_sz); 938 939 virtual_avail += KSTACK_GUARD_PAGES * PAGE_SIZE + kstack0_sz; 940 for (i = 0; i < kstack_pages; i++) { 941 mmu_booke_kenter(mmu, kstack0, kstack0_phys); 942 kstack0 += PAGE_SIZE; 943 kstack0_phys += PAGE_SIZE; 944 } 945 946 pmap_bootstrapped = 1; 947 948 debugf("virtual_avail = %"PRI0ptrX"\n", virtual_avail); 949 debugf("virtual_end = %"PRI0ptrX"\n", virtual_end); 950 951 debugf("mmu_booke_bootstrap: exit\n"); 952 } 953 954 #ifdef SMP 955 void 956 tlb1_ap_prep(void) 957 { 958 tlb_entry_t *e, tmp; 959 unsigned int i; 960 961 /* Prepare TLB1 image for AP processors */ 962 e = __boot_tlb1; 963 for (i = 0; i < TLB1_ENTRIES; i++) { 964 tlb1_read_entry(&tmp, i); 965 966 if ((tmp.mas1 & MAS1_VALID) && (tmp.mas2 & _TLB_ENTRY_SHARED)) 967 memcpy(e++, &tmp, sizeof(tmp)); 968 } 969 } 970 971 void 972 pmap_bootstrap_ap(volatile uint32_t *trcp __unused) 973 { 974 int i; 975 976 /* 977 * Finish TLB1 configuration: the BSP already set up its TLB1 and we 978 * have the snapshot of its contents in the s/w __boot_tlb1[] table 979 * created by tlb1_ap_prep(), so use these values directly to 980 * (re)program AP's TLB1 hardware. 981 * 982 * Start at index 1 because index 0 has the kernel map. 983 */ 984 for (i = 1; i < TLB1_ENTRIES; i++) { 985 if (__boot_tlb1[i].mas1 & MAS1_VALID) 986 tlb1_write_entry(&__boot_tlb1[i], i); 987 } 988 989 set_mas4_defaults(); 990 } 991 #endif 992 993 static void 994 booke_pmap_init_qpages(void) 995 { 996 struct pcpu *pc; 997 int i; 998 999 CPU_FOREACH(i) { 1000 pc = pcpu_find(i); 1001 pc->pc_qmap_addr = kva_alloc(PAGE_SIZE); 1002 if (pc->pc_qmap_addr == 0) 1003 panic("pmap_init_qpages: unable to allocate KVA"); 1004 } 1005 } 1006 1007 SYSINIT(qpages_init, SI_SUB_CPU, SI_ORDER_ANY, booke_pmap_init_qpages, NULL); 1008 1009 /* 1010 * Get the physical page address for the given pmap/virtual address. 1011 */ 1012 static vm_paddr_t 1013 mmu_booke_extract(mmu_t mmu, pmap_t pmap, vm_offset_t va) 1014 { 1015 vm_paddr_t pa; 1016 1017 PMAP_LOCK(pmap); 1018 pa = pte_vatopa(mmu, pmap, va); 1019 PMAP_UNLOCK(pmap); 1020 1021 return (pa); 1022 } 1023 1024 /* 1025 * Extract the physical page address associated with the given 1026 * kernel virtual address. 1027 */ 1028 static vm_paddr_t 1029 mmu_booke_kextract(mmu_t mmu, vm_offset_t va) 1030 { 1031 tlb_entry_t e; 1032 vm_paddr_t p = 0; 1033 int i; 1034 1035 #ifdef __powerpc64__ 1036 if (va >= DMAP_BASE_ADDRESS && va <= DMAP_MAX_ADDRESS) 1037 return (DMAP_TO_PHYS(va)); 1038 #endif 1039 1040 if (va >= VM_MIN_KERNEL_ADDRESS && va <= VM_MAX_KERNEL_ADDRESS) 1041 p = pte_vatopa(mmu, kernel_pmap, va); 1042 1043 if (p == 0) { 1044 /* Check TLB1 mappings */ 1045 for (i = 0; i < TLB1_ENTRIES; i++) { 1046 tlb1_read_entry(&e, i); 1047 if (!(e.mas1 & MAS1_VALID)) 1048 continue; 1049 if (va >= e.virt && va < e.virt + e.size) 1050 return (e.phys + (va - e.virt)); 1051 } 1052 } 1053 1054 return (p); 1055 } 1056 1057 /* 1058 * Initialize the pmap module. 1059 * Called by vm_init, to initialize any structures that the pmap 1060 * system needs to map virtual memory. 1061 */ 1062 static void 1063 mmu_booke_init(mmu_t mmu) 1064 { 1065 int shpgperproc = PMAP_SHPGPERPROC; 1066 1067 /* 1068 * Initialize the address space (zone) for the pv entries. Set a 1069 * high water mark so that the system can recover from excessive 1070 * numbers of pv entries. 1071 */ 1072 pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL, 1073 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1074 1075 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); 1076 pv_entry_max = shpgperproc * maxproc + vm_cnt.v_page_count; 1077 1078 TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max); 1079 pv_entry_high_water = 9 * (pv_entry_max / 10); 1080 1081 uma_zone_reserve_kva(pvzone, pv_entry_max); 1082 1083 /* Pre-fill pvzone with initial number of pv entries. */ 1084 uma_prealloc(pvzone, PV_ENTRY_ZONE_MIN); 1085 1086 /* Create a UMA zone for page table roots. */ 1087 ptbl_root_zone = uma_zcreate("pmap root", PMAP_ROOT_SIZE, 1088 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, UMA_ZONE_VM); 1089 1090 /* Initialize ptbl allocation. */ 1091 ptbl_init(); 1092 } 1093 1094 /* 1095 * Map a list of wired pages into kernel virtual address space. This is 1096 * intended for temporary mappings which do not need page modification or 1097 * references recorded. Existing mappings in the region are overwritten. 1098 */ 1099 static void 1100 mmu_booke_qenter(mmu_t mmu, vm_offset_t sva, vm_page_t *m, int count) 1101 { 1102 vm_offset_t va; 1103 1104 va = sva; 1105 while (count-- > 0) { 1106 mmu_booke_kenter(mmu, va, VM_PAGE_TO_PHYS(*m)); 1107 va += PAGE_SIZE; 1108 m++; 1109 } 1110 } 1111 1112 /* 1113 * Remove page mappings from kernel virtual address space. Intended for 1114 * temporary mappings entered by mmu_booke_qenter. 1115 */ 1116 static void 1117 mmu_booke_qremove(mmu_t mmu, vm_offset_t sva, int count) 1118 { 1119 vm_offset_t va; 1120 1121 va = sva; 1122 while (count-- > 0) { 1123 mmu_booke_kremove(mmu, va); 1124 va += PAGE_SIZE; 1125 } 1126 } 1127 1128 /* 1129 * Map a wired page into kernel virtual address space. 1130 */ 1131 static void 1132 mmu_booke_kenter(mmu_t mmu, vm_offset_t va, vm_paddr_t pa) 1133 { 1134 1135 mmu_booke_kenter_attr(mmu, va, pa, VM_MEMATTR_DEFAULT); 1136 } 1137 1138 static void 1139 mmu_booke_kenter_attr(mmu_t mmu, vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) 1140 { 1141 uint32_t flags; 1142 pte_t *pte; 1143 1144 KASSERT(((va >= VM_MIN_KERNEL_ADDRESS) && 1145 (va <= VM_MAX_KERNEL_ADDRESS)), ("mmu_booke_kenter: invalid va")); 1146 1147 flags = PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | PTE_VALID; 1148 flags |= tlb_calc_wimg(pa, ma) << PTE_MAS2_SHIFT; 1149 flags |= PTE_PS_4KB; 1150 1151 pte = pte_find(mmu, kernel_pmap, va); 1152 KASSERT((pte != NULL), ("mmu_booke_kenter: invalid va. NULL PTE")); 1153 1154 mtx_lock_spin(&tlbivax_mutex); 1155 tlb_miss_lock(); 1156 1157 if (PTE_ISVALID(pte)) { 1158 1159 CTR1(KTR_PMAP, "%s: replacing entry!", __func__); 1160 1161 /* Flush entry from TLB0 */ 1162 tlb0_flush_entry(va); 1163 } 1164 1165 *pte = PTE_RPN_FROM_PA(pa) | flags; 1166 1167 //debugf("mmu_booke_kenter: pdir_idx = %d ptbl_idx = %d va=0x%08x " 1168 // "pa=0x%08x rpn=0x%08x flags=0x%08x\n", 1169 // pdir_idx, ptbl_idx, va, pa, pte->rpn, pte->flags); 1170 1171 /* Flush the real memory from the instruction cache. */ 1172 if ((flags & (PTE_I | PTE_G)) == 0) 1173 __syncicache((void *)va, PAGE_SIZE); 1174 1175 tlb_miss_unlock(); 1176 mtx_unlock_spin(&tlbivax_mutex); 1177 } 1178 1179 /* 1180 * Remove a page from kernel page table. 1181 */ 1182 static void 1183 mmu_booke_kremove(mmu_t mmu, vm_offset_t va) 1184 { 1185 pte_t *pte; 1186 1187 CTR2(KTR_PMAP,"%s: s (va = 0x%"PRI0ptrX")\n", __func__, va); 1188 1189 KASSERT(((va >= VM_MIN_KERNEL_ADDRESS) && 1190 (va <= VM_MAX_KERNEL_ADDRESS)), 1191 ("mmu_booke_kremove: invalid va")); 1192 1193 pte = pte_find(mmu, kernel_pmap, va); 1194 1195 if (!PTE_ISVALID(pte)) { 1196 1197 CTR1(KTR_PMAP, "%s: invalid pte", __func__); 1198 1199 return; 1200 } 1201 1202 mtx_lock_spin(&tlbivax_mutex); 1203 tlb_miss_lock(); 1204 1205 /* Invalidate entry in TLB0, update PTE. */ 1206 tlb0_flush_entry(va); 1207 *pte = 0; 1208 1209 tlb_miss_unlock(); 1210 mtx_unlock_spin(&tlbivax_mutex); 1211 } 1212 1213 /* 1214 * Provide a kernel pointer corresponding to a given userland pointer. 1215 * The returned pointer is valid until the next time this function is 1216 * called in this thread. This is used internally in copyin/copyout. 1217 */ 1218 int 1219 mmu_booke_map_user_ptr(mmu_t mmu, pmap_t pm, volatile const void *uaddr, 1220 void **kaddr, size_t ulen, size_t *klen) 1221 { 1222 1223 if (trunc_page((uintptr_t)uaddr + ulen) > VM_MAXUSER_ADDRESS) 1224 return (EFAULT); 1225 1226 *kaddr = (void *)(uintptr_t)uaddr; 1227 if (klen) 1228 *klen = ulen; 1229 1230 return (0); 1231 } 1232 1233 /* 1234 * Figure out where a given kernel pointer (usually in a fault) points 1235 * to from the VM's perspective, potentially remapping into userland's 1236 * address space. 1237 */ 1238 static int 1239 mmu_booke_decode_kernel_ptr(mmu_t mmu, vm_offset_t addr, int *is_user, 1240 vm_offset_t *decoded_addr) 1241 { 1242 1243 if (trunc_page(addr) <= VM_MAXUSER_ADDRESS) 1244 *is_user = 1; 1245 else 1246 *is_user = 0; 1247 1248 *decoded_addr = addr; 1249 return (0); 1250 } 1251 1252 /* 1253 * Initialize pmap associated with process 0. 1254 */ 1255 static void 1256 mmu_booke_pinit0(mmu_t mmu, pmap_t pmap) 1257 { 1258 1259 PMAP_LOCK_INIT(pmap); 1260 mmu_booke_pinit(mmu, pmap); 1261 PCPU_SET(curpmap, pmap); 1262 } 1263 1264 /* 1265 * Insert the given physical page at the specified virtual address in the 1266 * target physical map with the protection requested. If specified the page 1267 * will be wired down. 1268 */ 1269 static int 1270 mmu_booke_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, 1271 vm_prot_t prot, u_int flags, int8_t psind) 1272 { 1273 int error; 1274 1275 rw_wlock(&pvh_global_lock); 1276 PMAP_LOCK(pmap); 1277 error = mmu_booke_enter_locked(mmu, pmap, va, m, prot, flags, psind); 1278 PMAP_UNLOCK(pmap); 1279 rw_wunlock(&pvh_global_lock); 1280 return (error); 1281 } 1282 1283 static int 1284 mmu_booke_enter_locked(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, 1285 vm_prot_t prot, u_int pmap_flags, int8_t psind __unused) 1286 { 1287 pte_t *pte; 1288 vm_paddr_t pa; 1289 pte_t flags; 1290 int error, su, sync; 1291 1292 pa = VM_PAGE_TO_PHYS(m); 1293 su = (pmap == kernel_pmap); 1294 sync = 0; 1295 1296 //debugf("mmu_booke_enter_locked: s (pmap=0x%08x su=%d tid=%d m=0x%08x va=0x%08x " 1297 // "pa=0x%08x prot=0x%08x flags=%#x)\n", 1298 // (u_int32_t)pmap, su, pmap->pm_tid, 1299 // (u_int32_t)m, va, pa, prot, flags); 1300 1301 if (su) { 1302 KASSERT(((va >= virtual_avail) && 1303 (va <= VM_MAX_KERNEL_ADDRESS)), 1304 ("mmu_booke_enter_locked: kernel pmap, non kernel va")); 1305 } else { 1306 KASSERT((va <= VM_MAXUSER_ADDRESS), 1307 ("mmu_booke_enter_locked: user pmap, non user va")); 1308 } 1309 if ((m->oflags & VPO_UNMANAGED) == 0) { 1310 if ((pmap_flags & PMAP_ENTER_QUICK_LOCKED) == 0) 1311 VM_PAGE_OBJECT_BUSY_ASSERT(m); 1312 else 1313 VM_OBJECT_ASSERT_LOCKED(m->object); 1314 } 1315 1316 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1317 1318 /* 1319 * If there is an existing mapping, and the physical address has not 1320 * changed, must be protection or wiring change. 1321 */ 1322 if (((pte = pte_find(mmu, pmap, va)) != NULL) && 1323 (PTE_ISVALID(pte)) && (PTE_PA(pte) == pa)) { 1324 1325 /* 1326 * Before actually updating pte->flags we calculate and 1327 * prepare its new value in a helper var. 1328 */ 1329 flags = *pte; 1330 flags &= ~(PTE_UW | PTE_UX | PTE_SW | PTE_SX | PTE_MODIFIED); 1331 1332 /* Wiring change, just update stats. */ 1333 if ((pmap_flags & PMAP_ENTER_WIRED) != 0) { 1334 if (!PTE_ISWIRED(pte)) { 1335 flags |= PTE_WIRED; 1336 pmap->pm_stats.wired_count++; 1337 } 1338 } else { 1339 if (PTE_ISWIRED(pte)) { 1340 flags &= ~PTE_WIRED; 1341 pmap->pm_stats.wired_count--; 1342 } 1343 } 1344 1345 if (prot & VM_PROT_WRITE) { 1346 /* Add write permissions. */ 1347 flags |= PTE_SW; 1348 if (!su) 1349 flags |= PTE_UW; 1350 1351 if ((flags & PTE_MANAGED) != 0) 1352 vm_page_aflag_set(m, PGA_WRITEABLE); 1353 } else { 1354 /* Handle modified pages, sense modify status. */ 1355 1356 /* 1357 * The PTE_MODIFIED flag could be set by underlying 1358 * TLB misses since we last read it (above), possibly 1359 * other CPUs could update it so we check in the PTE 1360 * directly rather than rely on that saved local flags 1361 * copy. 1362 */ 1363 if (PTE_ISMODIFIED(pte)) 1364 vm_page_dirty(m); 1365 } 1366 1367 if (prot & VM_PROT_EXECUTE) { 1368 flags |= PTE_SX; 1369 if (!su) 1370 flags |= PTE_UX; 1371 1372 /* 1373 * Check existing flags for execute permissions: if we 1374 * are turning execute permissions on, icache should 1375 * be flushed. 1376 */ 1377 if ((*pte & (PTE_UX | PTE_SX)) == 0) 1378 sync++; 1379 } 1380 1381 flags &= ~PTE_REFERENCED; 1382 1383 /* 1384 * The new flags value is all calculated -- only now actually 1385 * update the PTE. 1386 */ 1387 mtx_lock_spin(&tlbivax_mutex); 1388 tlb_miss_lock(); 1389 1390 tlb0_flush_entry(va); 1391 *pte &= ~PTE_FLAGS_MASK; 1392 *pte |= flags; 1393 1394 tlb_miss_unlock(); 1395 mtx_unlock_spin(&tlbivax_mutex); 1396 1397 } else { 1398 /* 1399 * If there is an existing mapping, but it's for a different 1400 * physical address, pte_enter() will delete the old mapping. 1401 */ 1402 //if ((pte != NULL) && PTE_ISVALID(pte)) 1403 // debugf("mmu_booke_enter_locked: replace\n"); 1404 //else 1405 // debugf("mmu_booke_enter_locked: new\n"); 1406 1407 /* Now set up the flags and install the new mapping. */ 1408 flags = (PTE_SR | PTE_VALID); 1409 flags |= PTE_M; 1410 1411 if (!su) 1412 flags |= PTE_UR; 1413 1414 if (prot & VM_PROT_WRITE) { 1415 flags |= PTE_SW; 1416 if (!su) 1417 flags |= PTE_UW; 1418 1419 if ((m->oflags & VPO_UNMANAGED) == 0) 1420 vm_page_aflag_set(m, PGA_WRITEABLE); 1421 } 1422 1423 if (prot & VM_PROT_EXECUTE) { 1424 flags |= PTE_SX; 1425 if (!su) 1426 flags |= PTE_UX; 1427 } 1428 1429 /* If its wired update stats. */ 1430 if ((pmap_flags & PMAP_ENTER_WIRED) != 0) 1431 flags |= PTE_WIRED; 1432 1433 error = pte_enter(mmu, pmap, m, va, flags, 1434 (pmap_flags & PMAP_ENTER_NOSLEEP) != 0); 1435 if (error != 0) 1436 return (KERN_RESOURCE_SHORTAGE); 1437 1438 if ((flags & PMAP_ENTER_WIRED) != 0) 1439 pmap->pm_stats.wired_count++; 1440 1441 /* Flush the real memory from the instruction cache. */ 1442 if (prot & VM_PROT_EXECUTE) 1443 sync++; 1444 } 1445 1446 if (sync && (su || pmap == PCPU_GET(curpmap))) { 1447 __syncicache((void *)va, PAGE_SIZE); 1448 sync = 0; 1449 } 1450 1451 return (KERN_SUCCESS); 1452 } 1453 1454 /* 1455 * Maps a sequence of resident pages belonging to the same object. 1456 * The sequence begins with the given page m_start. This page is 1457 * mapped at the given virtual address start. Each subsequent page is 1458 * mapped at a virtual address that is offset from start by the same 1459 * amount as the page is offset from m_start within the object. The 1460 * last page in the sequence is the page with the largest offset from 1461 * m_start that can be mapped at a virtual address less than the given 1462 * virtual address end. Not every virtual page between start and end 1463 * is mapped; only those for which a resident page exists with the 1464 * corresponding offset from m_start are mapped. 1465 */ 1466 static void 1467 mmu_booke_enter_object(mmu_t mmu, pmap_t pmap, vm_offset_t start, 1468 vm_offset_t end, vm_page_t m_start, vm_prot_t prot) 1469 { 1470 vm_page_t m; 1471 vm_pindex_t diff, psize; 1472 1473 VM_OBJECT_ASSERT_LOCKED(m_start->object); 1474 1475 psize = atop(end - start); 1476 m = m_start; 1477 rw_wlock(&pvh_global_lock); 1478 PMAP_LOCK(pmap); 1479 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 1480 mmu_booke_enter_locked(mmu, pmap, start + ptoa(diff), m, 1481 prot & (VM_PROT_READ | VM_PROT_EXECUTE), 1482 PMAP_ENTER_NOSLEEP | PMAP_ENTER_QUICK_LOCKED, 0); 1483 m = TAILQ_NEXT(m, listq); 1484 } 1485 PMAP_UNLOCK(pmap); 1486 rw_wunlock(&pvh_global_lock); 1487 } 1488 1489 static void 1490 mmu_booke_enter_quick(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, 1491 vm_prot_t prot) 1492 { 1493 1494 rw_wlock(&pvh_global_lock); 1495 PMAP_LOCK(pmap); 1496 mmu_booke_enter_locked(mmu, pmap, va, m, 1497 prot & (VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP | 1498 PMAP_ENTER_QUICK_LOCKED, 0); 1499 PMAP_UNLOCK(pmap); 1500 rw_wunlock(&pvh_global_lock); 1501 } 1502 1503 /* 1504 * Remove the given range of addresses from the specified map. 1505 * 1506 * It is assumed that the start and end are properly rounded to the page size. 1507 */ 1508 static void 1509 mmu_booke_remove(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_offset_t endva) 1510 { 1511 pte_t *pte; 1512 uint8_t hold_flag; 1513 1514 int su = (pmap == kernel_pmap); 1515 1516 //debugf("mmu_booke_remove: s (su = %d pmap=0x%08x tid=%d va=0x%08x endva=0x%08x)\n", 1517 // su, (u_int32_t)pmap, pmap->pm_tid, va, endva); 1518 1519 if (su) { 1520 KASSERT(((va >= virtual_avail) && 1521 (va <= VM_MAX_KERNEL_ADDRESS)), 1522 ("mmu_booke_remove: kernel pmap, non kernel va")); 1523 } else { 1524 KASSERT((va <= VM_MAXUSER_ADDRESS), 1525 ("mmu_booke_remove: user pmap, non user va")); 1526 } 1527 1528 if (PMAP_REMOVE_DONE(pmap)) { 1529 //debugf("mmu_booke_remove: e (empty)\n"); 1530 return; 1531 } 1532 1533 hold_flag = PTBL_HOLD_FLAG(pmap); 1534 //debugf("mmu_booke_remove: hold_flag = %d\n", hold_flag); 1535 1536 rw_wlock(&pvh_global_lock); 1537 PMAP_LOCK(pmap); 1538 for (; va < endva; va += PAGE_SIZE) { 1539 pte = pte_find_next(mmu, pmap, &va); 1540 if ((pte == NULL) || !PTE_ISVALID(pte)) 1541 break; 1542 if (va >= endva) 1543 break; 1544 pte_remove(mmu, pmap, va, hold_flag); 1545 } 1546 PMAP_UNLOCK(pmap); 1547 rw_wunlock(&pvh_global_lock); 1548 1549 //debugf("mmu_booke_remove: e\n"); 1550 } 1551 1552 /* 1553 * Remove physical page from all pmaps in which it resides. 1554 */ 1555 static void 1556 mmu_booke_remove_all(mmu_t mmu, vm_page_t m) 1557 { 1558 pv_entry_t pv, pvn; 1559 uint8_t hold_flag; 1560 1561 rw_wlock(&pvh_global_lock); 1562 TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_link, pvn) { 1563 PMAP_LOCK(pv->pv_pmap); 1564 hold_flag = PTBL_HOLD_FLAG(pv->pv_pmap); 1565 pte_remove(mmu, pv->pv_pmap, pv->pv_va, hold_flag); 1566 PMAP_UNLOCK(pv->pv_pmap); 1567 } 1568 vm_page_aflag_clear(m, PGA_WRITEABLE); 1569 rw_wunlock(&pvh_global_lock); 1570 } 1571 1572 /* 1573 * Map a range of physical addresses into kernel virtual address space. 1574 */ 1575 static vm_offset_t 1576 mmu_booke_map(mmu_t mmu, vm_offset_t *virt, vm_paddr_t pa_start, 1577 vm_paddr_t pa_end, int prot) 1578 { 1579 vm_offset_t sva = *virt; 1580 vm_offset_t va = sva; 1581 1582 #ifdef __powerpc64__ 1583 /* XXX: Handle memory not starting at 0x0. */ 1584 if (pa_end < ctob(Maxmem)) 1585 return (PHYS_TO_DMAP(pa_start)); 1586 #endif 1587 1588 while (pa_start < pa_end) { 1589 mmu_booke_kenter(mmu, va, pa_start); 1590 va += PAGE_SIZE; 1591 pa_start += PAGE_SIZE; 1592 } 1593 *virt = va; 1594 1595 return (sva); 1596 } 1597 1598 /* 1599 * The pmap must be activated before it's address space can be accessed in any 1600 * way. 1601 */ 1602 static void 1603 mmu_booke_activate(mmu_t mmu, struct thread *td) 1604 { 1605 pmap_t pmap; 1606 u_int cpuid; 1607 1608 pmap = &td->td_proc->p_vmspace->vm_pmap; 1609 1610 CTR5(KTR_PMAP, "%s: s (td = %p, proc = '%s', id = %d, pmap = 0x%"PRI0ptrX")", 1611 __func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap); 1612 1613 KASSERT((pmap != kernel_pmap), ("mmu_booke_activate: kernel_pmap!")); 1614 1615 sched_pin(); 1616 1617 cpuid = PCPU_GET(cpuid); 1618 CPU_SET_ATOMIC(cpuid, &pmap->pm_active); 1619 PCPU_SET(curpmap, pmap); 1620 1621 if (pmap->pm_tid[cpuid] == TID_NONE) 1622 tid_alloc(pmap); 1623 1624 /* Load PID0 register with pmap tid value. */ 1625 mtspr(SPR_PID0, pmap->pm_tid[cpuid]); 1626 __asm __volatile("isync"); 1627 1628 mtspr(SPR_DBCR0, td->td_pcb->pcb_cpu.booke.dbcr0); 1629 1630 sched_unpin(); 1631 1632 CTR3(KTR_PMAP, "%s: e (tid = %d for '%s')", __func__, 1633 pmap->pm_tid[PCPU_GET(cpuid)], td->td_proc->p_comm); 1634 } 1635 1636 /* 1637 * Deactivate the specified process's address space. 1638 */ 1639 static void 1640 mmu_booke_deactivate(mmu_t mmu, struct thread *td) 1641 { 1642 pmap_t pmap; 1643 1644 pmap = &td->td_proc->p_vmspace->vm_pmap; 1645 1646 CTR5(KTR_PMAP, "%s: td=%p, proc = '%s', id = %d, pmap = 0x%"PRI0ptrX, 1647 __func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap); 1648 1649 td->td_pcb->pcb_cpu.booke.dbcr0 = mfspr(SPR_DBCR0); 1650 1651 CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmap->pm_active); 1652 PCPU_SET(curpmap, NULL); 1653 } 1654 1655 /* 1656 * Copy the range specified by src_addr/len 1657 * from the source map to the range dst_addr/len 1658 * in the destination map. 1659 * 1660 * This routine is only advisory and need not do anything. 1661 */ 1662 static void 1663 mmu_booke_copy(mmu_t mmu, pmap_t dst_pmap, pmap_t src_pmap, 1664 vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) 1665 { 1666 1667 } 1668 1669 /* 1670 * Set the physical protection on the specified range of this map as requested. 1671 */ 1672 static void 1673 mmu_booke_protect(mmu_t mmu, pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 1674 vm_prot_t prot) 1675 { 1676 vm_offset_t va; 1677 vm_page_t m; 1678 pte_t *pte; 1679 1680 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1681 mmu_booke_remove(mmu, pmap, sva, eva); 1682 return; 1683 } 1684 1685 if (prot & VM_PROT_WRITE) 1686 return; 1687 1688 PMAP_LOCK(pmap); 1689 for (va = sva; va < eva; va += PAGE_SIZE) { 1690 if ((pte = pte_find(mmu, pmap, va)) != NULL) { 1691 if (PTE_ISVALID(pte)) { 1692 m = PHYS_TO_VM_PAGE(PTE_PA(pte)); 1693 1694 mtx_lock_spin(&tlbivax_mutex); 1695 tlb_miss_lock(); 1696 1697 /* Handle modified pages. */ 1698 if (PTE_ISMODIFIED(pte) && PTE_ISMANAGED(pte)) 1699 vm_page_dirty(m); 1700 1701 tlb0_flush_entry(va); 1702 *pte &= ~(PTE_UW | PTE_SW | PTE_MODIFIED); 1703 1704 tlb_miss_unlock(); 1705 mtx_unlock_spin(&tlbivax_mutex); 1706 } 1707 } 1708 } 1709 PMAP_UNLOCK(pmap); 1710 } 1711 1712 /* 1713 * Clear the write and modified bits in each of the given page's mappings. 1714 */ 1715 static void 1716 mmu_booke_remove_write(mmu_t mmu, vm_page_t m) 1717 { 1718 pv_entry_t pv; 1719 pte_t *pte; 1720 1721 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1722 ("mmu_booke_remove_write: page %p is not managed", m)); 1723 vm_page_assert_busied(m); 1724 1725 if (!pmap_page_is_write_mapped(m)) 1726 return; 1727 rw_wlock(&pvh_global_lock); 1728 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 1729 PMAP_LOCK(pv->pv_pmap); 1730 if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL) { 1731 if (PTE_ISVALID(pte)) { 1732 m = PHYS_TO_VM_PAGE(PTE_PA(pte)); 1733 1734 mtx_lock_spin(&tlbivax_mutex); 1735 tlb_miss_lock(); 1736 1737 /* Handle modified pages. */ 1738 if (PTE_ISMODIFIED(pte)) 1739 vm_page_dirty(m); 1740 1741 /* Flush mapping from TLB0. */ 1742 *pte &= ~(PTE_UW | PTE_SW | PTE_MODIFIED); 1743 1744 tlb_miss_unlock(); 1745 mtx_unlock_spin(&tlbivax_mutex); 1746 } 1747 } 1748 PMAP_UNLOCK(pv->pv_pmap); 1749 } 1750 vm_page_aflag_clear(m, PGA_WRITEABLE); 1751 rw_wunlock(&pvh_global_lock); 1752 } 1753 1754 /* 1755 * Atomically extract and hold the physical page with the given 1756 * pmap and virtual address pair if that mapping permits the given 1757 * protection. 1758 */ 1759 static vm_page_t 1760 mmu_booke_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, 1761 vm_prot_t prot) 1762 { 1763 pte_t *pte; 1764 vm_page_t m; 1765 uint32_t pte_wbit; 1766 1767 m = NULL; 1768 PMAP_LOCK(pmap); 1769 pte = pte_find(mmu, pmap, va); 1770 if ((pte != NULL) && PTE_ISVALID(pte)) { 1771 if (pmap == kernel_pmap) 1772 pte_wbit = PTE_SW; 1773 else 1774 pte_wbit = PTE_UW; 1775 1776 if ((*pte & pte_wbit) != 0 || (prot & VM_PROT_WRITE) == 0) { 1777 m = PHYS_TO_VM_PAGE(PTE_PA(pte)); 1778 if (!vm_page_wire_mapped(m)) 1779 m = NULL; 1780 } 1781 } 1782 PMAP_UNLOCK(pmap); 1783 return (m); 1784 } 1785 1786 /* 1787 * Initialize a vm_page's machine-dependent fields. 1788 */ 1789 static void 1790 mmu_booke_page_init(mmu_t mmu, vm_page_t m) 1791 { 1792 1793 m->md.pv_tracked = 0; 1794 TAILQ_INIT(&m->md.pv_list); 1795 } 1796 1797 /* 1798 * Return whether or not the specified physical page was modified 1799 * in any of physical maps. 1800 */ 1801 static boolean_t 1802 mmu_booke_is_modified(mmu_t mmu, vm_page_t m) 1803 { 1804 pte_t *pte; 1805 pv_entry_t pv; 1806 boolean_t rv; 1807 1808 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1809 ("mmu_booke_is_modified: page %p is not managed", m)); 1810 rv = FALSE; 1811 1812 /* 1813 * If the page is not busied then this check is racy. 1814 */ 1815 if (!pmap_page_is_write_mapped(m)) 1816 return (FALSE); 1817 1818 rw_wlock(&pvh_global_lock); 1819 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 1820 PMAP_LOCK(pv->pv_pmap); 1821 if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL && 1822 PTE_ISVALID(pte)) { 1823 if (PTE_ISMODIFIED(pte)) 1824 rv = TRUE; 1825 } 1826 PMAP_UNLOCK(pv->pv_pmap); 1827 if (rv) 1828 break; 1829 } 1830 rw_wunlock(&pvh_global_lock); 1831 return (rv); 1832 } 1833 1834 /* 1835 * Return whether or not the specified virtual address is eligible 1836 * for prefault. 1837 */ 1838 static boolean_t 1839 mmu_booke_is_prefaultable(mmu_t mmu, pmap_t pmap, vm_offset_t addr) 1840 { 1841 1842 return (FALSE); 1843 } 1844 1845 /* 1846 * Return whether or not the specified physical page was referenced 1847 * in any physical maps. 1848 */ 1849 static boolean_t 1850 mmu_booke_is_referenced(mmu_t mmu, vm_page_t m) 1851 { 1852 pte_t *pte; 1853 pv_entry_t pv; 1854 boolean_t rv; 1855 1856 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1857 ("mmu_booke_is_referenced: page %p is not managed", m)); 1858 rv = FALSE; 1859 rw_wlock(&pvh_global_lock); 1860 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 1861 PMAP_LOCK(pv->pv_pmap); 1862 if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL && 1863 PTE_ISVALID(pte)) { 1864 if (PTE_ISREFERENCED(pte)) 1865 rv = TRUE; 1866 } 1867 PMAP_UNLOCK(pv->pv_pmap); 1868 if (rv) 1869 break; 1870 } 1871 rw_wunlock(&pvh_global_lock); 1872 return (rv); 1873 } 1874 1875 /* 1876 * Clear the modify bits on the specified physical page. 1877 */ 1878 static void 1879 mmu_booke_clear_modify(mmu_t mmu, vm_page_t m) 1880 { 1881 pte_t *pte; 1882 pv_entry_t pv; 1883 1884 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1885 ("mmu_booke_clear_modify: page %p is not managed", m)); 1886 vm_page_assert_busied(m); 1887 1888 if (!pmap_page_is_write_mapped(m)) 1889 return; 1890 1891 rw_wlock(&pvh_global_lock); 1892 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 1893 PMAP_LOCK(pv->pv_pmap); 1894 if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL && 1895 PTE_ISVALID(pte)) { 1896 mtx_lock_spin(&tlbivax_mutex); 1897 tlb_miss_lock(); 1898 1899 if (*pte & (PTE_SW | PTE_UW | PTE_MODIFIED)) { 1900 tlb0_flush_entry(pv->pv_va); 1901 *pte &= ~(PTE_SW | PTE_UW | PTE_MODIFIED | 1902 PTE_REFERENCED); 1903 } 1904 1905 tlb_miss_unlock(); 1906 mtx_unlock_spin(&tlbivax_mutex); 1907 } 1908 PMAP_UNLOCK(pv->pv_pmap); 1909 } 1910 rw_wunlock(&pvh_global_lock); 1911 } 1912 1913 /* 1914 * Return a count of reference bits for a page, clearing those bits. 1915 * It is not necessary for every reference bit to be cleared, but it 1916 * is necessary that 0 only be returned when there are truly no 1917 * reference bits set. 1918 * 1919 * As an optimization, update the page's dirty field if a modified bit is 1920 * found while counting reference bits. This opportunistic update can be 1921 * performed at low cost and can eliminate the need for some future calls 1922 * to pmap_is_modified(). However, since this function stops after 1923 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some 1924 * dirty pages. Those dirty pages will only be detected by a future call 1925 * to pmap_is_modified(). 1926 */ 1927 static int 1928 mmu_booke_ts_referenced(mmu_t mmu, vm_page_t m) 1929 { 1930 pte_t *pte; 1931 pv_entry_t pv; 1932 int count; 1933 1934 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1935 ("mmu_booke_ts_referenced: page %p is not managed", m)); 1936 count = 0; 1937 rw_wlock(&pvh_global_lock); 1938 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 1939 PMAP_LOCK(pv->pv_pmap); 1940 if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL && 1941 PTE_ISVALID(pte)) { 1942 if (PTE_ISMODIFIED(pte)) 1943 vm_page_dirty(m); 1944 if (PTE_ISREFERENCED(pte)) { 1945 mtx_lock_spin(&tlbivax_mutex); 1946 tlb_miss_lock(); 1947 1948 tlb0_flush_entry(pv->pv_va); 1949 *pte &= ~PTE_REFERENCED; 1950 1951 tlb_miss_unlock(); 1952 mtx_unlock_spin(&tlbivax_mutex); 1953 1954 if (++count >= PMAP_TS_REFERENCED_MAX) { 1955 PMAP_UNLOCK(pv->pv_pmap); 1956 break; 1957 } 1958 } 1959 } 1960 PMAP_UNLOCK(pv->pv_pmap); 1961 } 1962 rw_wunlock(&pvh_global_lock); 1963 return (count); 1964 } 1965 1966 /* 1967 * Clear the wired attribute from the mappings for the specified range of 1968 * addresses in the given pmap. Every valid mapping within that range must 1969 * have the wired attribute set. In contrast, invalid mappings cannot have 1970 * the wired attribute set, so they are ignored. 1971 * 1972 * The wired attribute of the page table entry is not a hardware feature, so 1973 * there is no need to invalidate any TLB entries. 1974 */ 1975 static void 1976 mmu_booke_unwire(mmu_t mmu, pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 1977 { 1978 vm_offset_t va; 1979 pte_t *pte; 1980 1981 PMAP_LOCK(pmap); 1982 for (va = sva; va < eva; va += PAGE_SIZE) { 1983 if ((pte = pte_find(mmu, pmap, va)) != NULL && 1984 PTE_ISVALID(pte)) { 1985 if (!PTE_ISWIRED(pte)) 1986 panic("mmu_booke_unwire: pte %p isn't wired", 1987 pte); 1988 *pte &= ~PTE_WIRED; 1989 pmap->pm_stats.wired_count--; 1990 } 1991 } 1992 PMAP_UNLOCK(pmap); 1993 1994 } 1995 1996 /* 1997 * Return true if the pmap's pv is one of the first 16 pvs linked to from this 1998 * page. This count may be changed upwards or downwards in the future; it is 1999 * only necessary that true be returned for a small subset of pmaps for proper 2000 * page aging. 2001 */ 2002 static boolean_t 2003 mmu_booke_page_exists_quick(mmu_t mmu, pmap_t pmap, vm_page_t m) 2004 { 2005 pv_entry_t pv; 2006 int loops; 2007 boolean_t rv; 2008 2009 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2010 ("mmu_booke_page_exists_quick: page %p is not managed", m)); 2011 loops = 0; 2012 rv = FALSE; 2013 rw_wlock(&pvh_global_lock); 2014 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 2015 if (pv->pv_pmap == pmap) { 2016 rv = TRUE; 2017 break; 2018 } 2019 if (++loops >= 16) 2020 break; 2021 } 2022 rw_wunlock(&pvh_global_lock); 2023 return (rv); 2024 } 2025 2026 /* 2027 * Return the number of managed mappings to the given physical page that are 2028 * wired. 2029 */ 2030 static int 2031 mmu_booke_page_wired_mappings(mmu_t mmu, vm_page_t m) 2032 { 2033 pv_entry_t pv; 2034 pte_t *pte; 2035 int count = 0; 2036 2037 if ((m->oflags & VPO_UNMANAGED) != 0) 2038 return (count); 2039 rw_wlock(&pvh_global_lock); 2040 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 2041 PMAP_LOCK(pv->pv_pmap); 2042 if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL) 2043 if (PTE_ISVALID(pte) && PTE_ISWIRED(pte)) 2044 count++; 2045 PMAP_UNLOCK(pv->pv_pmap); 2046 } 2047 rw_wunlock(&pvh_global_lock); 2048 return (count); 2049 } 2050 2051 static int 2052 mmu_booke_dev_direct_mapped(mmu_t mmu, vm_paddr_t pa, vm_size_t size) 2053 { 2054 int i; 2055 vm_offset_t va; 2056 2057 /* 2058 * This currently does not work for entries that 2059 * overlap TLB1 entries. 2060 */ 2061 for (i = 0; i < TLB1_ENTRIES; i ++) { 2062 if (tlb1_iomapped(i, pa, size, &va) == 0) 2063 return (0); 2064 } 2065 2066 return (EFAULT); 2067 } 2068 2069 void 2070 mmu_booke_dumpsys_map(mmu_t mmu, vm_paddr_t pa, size_t sz, void **va) 2071 { 2072 vm_paddr_t ppa; 2073 vm_offset_t ofs; 2074 vm_size_t gran; 2075 2076 /* Minidumps are based on virtual memory addresses. */ 2077 if (do_minidump) { 2078 *va = (void *)(vm_offset_t)pa; 2079 return; 2080 } 2081 2082 /* Raw physical memory dumps don't have a virtual address. */ 2083 /* We always map a 256MB page at 256M. */ 2084 gran = 256 * 1024 * 1024; 2085 ppa = rounddown2(pa, gran); 2086 ofs = pa - ppa; 2087 *va = (void *)gran; 2088 tlb1_set_entry((vm_offset_t)va, ppa, gran, _TLB_ENTRY_IO); 2089 2090 if (sz > (gran - ofs)) 2091 tlb1_set_entry((vm_offset_t)(va + gran), ppa + gran, gran, 2092 _TLB_ENTRY_IO); 2093 } 2094 2095 void 2096 mmu_booke_dumpsys_unmap(mmu_t mmu, vm_paddr_t pa, size_t sz, void *va) 2097 { 2098 vm_paddr_t ppa; 2099 vm_offset_t ofs; 2100 vm_size_t gran; 2101 tlb_entry_t e; 2102 int i; 2103 2104 /* Minidumps are based on virtual memory addresses. */ 2105 /* Nothing to do... */ 2106 if (do_minidump) 2107 return; 2108 2109 for (i = 0; i < TLB1_ENTRIES; i++) { 2110 tlb1_read_entry(&e, i); 2111 if (!(e.mas1 & MAS1_VALID)) 2112 break; 2113 } 2114 2115 /* Raw physical memory dumps don't have a virtual address. */ 2116 i--; 2117 e.mas1 = 0; 2118 e.mas2 = 0; 2119 e.mas3 = 0; 2120 tlb1_write_entry(&e, i); 2121 2122 gran = 256 * 1024 * 1024; 2123 ppa = rounddown2(pa, gran); 2124 ofs = pa - ppa; 2125 if (sz > (gran - ofs)) { 2126 i--; 2127 e.mas1 = 0; 2128 e.mas2 = 0; 2129 e.mas3 = 0; 2130 tlb1_write_entry(&e, i); 2131 } 2132 } 2133 2134 extern struct dump_pa dump_map[PHYS_AVAIL_SZ + 1]; 2135 2136 void 2137 mmu_booke_scan_init(mmu_t mmu) 2138 { 2139 vm_offset_t va; 2140 pte_t *pte; 2141 int i; 2142 2143 if (!do_minidump) { 2144 /* Initialize phys. segments for dumpsys(). */ 2145 memset(&dump_map, 0, sizeof(dump_map)); 2146 mem_regions(&physmem_regions, &physmem_regions_sz, &availmem_regions, 2147 &availmem_regions_sz); 2148 for (i = 0; i < physmem_regions_sz; i++) { 2149 dump_map[i].pa_start = physmem_regions[i].mr_start; 2150 dump_map[i].pa_size = physmem_regions[i].mr_size; 2151 } 2152 return; 2153 } 2154 2155 /* Virtual segments for minidumps: */ 2156 memset(&dump_map, 0, sizeof(dump_map)); 2157 2158 /* 1st: kernel .data and .bss. */ 2159 dump_map[0].pa_start = trunc_page((uintptr_t)_etext); 2160 dump_map[0].pa_size = 2161 round_page((uintptr_t)_end) - dump_map[0].pa_start; 2162 2163 /* 2nd: msgbuf and tables (see pmap_bootstrap()). */ 2164 dump_map[1].pa_start = data_start; 2165 dump_map[1].pa_size = data_end - data_start; 2166 2167 /* 3rd: kernel VM. */ 2168 va = dump_map[1].pa_start + dump_map[1].pa_size; 2169 /* Find start of next chunk (from va). */ 2170 while (va < virtual_end) { 2171 /* Don't dump the buffer cache. */ 2172 if (va >= kmi.buffer_sva && va < kmi.buffer_eva) { 2173 va = kmi.buffer_eva; 2174 continue; 2175 } 2176 pte = pte_find(mmu, kernel_pmap, va); 2177 if (pte != NULL && PTE_ISVALID(pte)) 2178 break; 2179 va += PAGE_SIZE; 2180 } 2181 if (va < virtual_end) { 2182 dump_map[2].pa_start = va; 2183 va += PAGE_SIZE; 2184 /* Find last page in chunk. */ 2185 while (va < virtual_end) { 2186 /* Don't run into the buffer cache. */ 2187 if (va == kmi.buffer_sva) 2188 break; 2189 pte = pte_find(mmu, kernel_pmap, va); 2190 if (pte == NULL || !PTE_ISVALID(pte)) 2191 break; 2192 va += PAGE_SIZE; 2193 } 2194 dump_map[2].pa_size = va - dump_map[2].pa_start; 2195 } 2196 } 2197 2198 /* 2199 * Map a set of physical memory pages into the kernel virtual address space. 2200 * Return a pointer to where it is mapped. This routine is intended to be used 2201 * for mapping device memory, NOT real memory. 2202 */ 2203 static void * 2204 mmu_booke_mapdev(mmu_t mmu, vm_paddr_t pa, vm_size_t size) 2205 { 2206 2207 return (mmu_booke_mapdev_attr(mmu, pa, size, VM_MEMATTR_DEFAULT)); 2208 } 2209 2210 static int 2211 tlb1_find_pa(vm_paddr_t pa, tlb_entry_t *e) 2212 { 2213 int i; 2214 2215 for (i = 0; i < TLB1_ENTRIES; i++) { 2216 tlb1_read_entry(e, i); 2217 if ((e->mas1 & MAS1_VALID) == 0) 2218 continue; 2219 if (e->phys == pa) 2220 return (i); 2221 } 2222 return (-1); 2223 } 2224 2225 static void * 2226 mmu_booke_mapdev_attr(mmu_t mmu, vm_paddr_t pa, vm_size_t size, vm_memattr_t ma) 2227 { 2228 tlb_entry_t e; 2229 vm_paddr_t tmppa; 2230 #ifndef __powerpc64__ 2231 uintptr_t tmpva; 2232 #endif 2233 uintptr_t va, retva; 2234 vm_size_t sz; 2235 int i; 2236 int wimge; 2237 2238 /* 2239 * Check if this is premapped in TLB1. 2240 */ 2241 sz = size; 2242 tmppa = pa; 2243 va = ~0; 2244 wimge = tlb_calc_wimg(pa, ma); 2245 for (i = 0; i < TLB1_ENTRIES; i++) { 2246 tlb1_read_entry(&e, i); 2247 if (!(e.mas1 & MAS1_VALID)) 2248 continue; 2249 if (wimge != (e.mas2 & (MAS2_WIMGE_MASK & ~_TLB_ENTRY_SHARED))) 2250 continue; 2251 if (tmppa >= e.phys && tmppa < e.phys + e.size) { 2252 va = e.virt + (pa - e.phys); 2253 tmppa = e.phys + e.size; 2254 sz -= MIN(sz, e.size - (pa - e.phys)); 2255 while (sz > 0 && (i = tlb1_find_pa(tmppa, &e)) != -1) { 2256 if (wimge != (e.mas2 & (MAS2_WIMGE_MASK & ~_TLB_ENTRY_SHARED))) 2257 break; 2258 sz -= MIN(sz, e.size); 2259 tmppa = e.phys + e.size; 2260 } 2261 if (sz != 0) 2262 break; 2263 return ((void *)va); 2264 } 2265 } 2266 2267 size = roundup(size, PAGE_SIZE); 2268 2269 #ifdef __powerpc64__ 2270 KASSERT(pa < VM_MAPDEV_PA_MAX, 2271 ("Unsupported physical address! %lx", pa)); 2272 va = VM_MAPDEV_BASE + pa; 2273 retva = va; 2274 #ifdef POW2_MAPPINGS 2275 /* 2276 * Align the mapping to a power of 2 size, taking into account that we 2277 * may need to increase the size multiple times to satisfy the size and 2278 * alignment requirements. 2279 * 2280 * This works in the general case because it's very rare (near never?) 2281 * to have different access properties (WIMG) within a single 2282 * power-of-two region. If a design does call for that, POW2_MAPPINGS 2283 * can be undefined, and exact mappings will be used instead. 2284 */ 2285 sz = size; 2286 size = roundup2(size, 1 << ilog2(size)); 2287 while (rounddown2(va, size) + size < va + sz) 2288 size <<= 1; 2289 va = rounddown2(va, size); 2290 pa = rounddown2(pa, size); 2291 #endif 2292 #else 2293 /* 2294 * The device mapping area is between VM_MAXUSER_ADDRESS and 2295 * VM_MIN_KERNEL_ADDRESS. This gives 1GB of device addressing. 2296 */ 2297 #ifdef SPARSE_MAPDEV 2298 /* 2299 * With a sparse mapdev, align to the largest starting region. This 2300 * could feasibly be optimized for a 'best-fit' alignment, but that 2301 * calculation could be very costly. 2302 * Align to the smaller of: 2303 * - first set bit in overlap of (pa & size mask) 2304 * - largest size envelope 2305 * 2306 * It's possible the device mapping may start at a PA that's not larger 2307 * than the size mask, so we need to offset in to maximize the TLB entry 2308 * range and minimize the number of used TLB entries. 2309 */ 2310 do { 2311 tmpva = tlb1_map_base; 2312 sz = ffsl((~((1 << flsl(size-1)) - 1)) & pa); 2313 sz = sz ? min(roundup(sz + 3, 4), flsl(size) - 1) : flsl(size) - 1; 2314 va = roundup(tlb1_map_base, 1 << sz) | (((1 << sz) - 1) & pa); 2315 } while (!atomic_cmpset_int(&tlb1_map_base, tmpva, va + size)); 2316 #endif 2317 va = atomic_fetchadd_int(&tlb1_map_base, size); 2318 retva = va; 2319 #endif 2320 2321 if (tlb1_mapin_region(va, pa, size, tlb_calc_wimg(pa, ma)) != size) 2322 return (NULL); 2323 2324 return ((void *)retva); 2325 } 2326 2327 /* 2328 * 'Unmap' a range mapped by mmu_booke_mapdev(). 2329 */ 2330 static void 2331 mmu_booke_unmapdev(mmu_t mmu, vm_offset_t va, vm_size_t size) 2332 { 2333 #ifdef SUPPORTS_SHRINKING_TLB1 2334 vm_offset_t base, offset; 2335 2336 /* 2337 * Unmap only if this is inside kernel virtual space. 2338 */ 2339 if ((va >= VM_MIN_KERNEL_ADDRESS) && (va <= VM_MAX_KERNEL_ADDRESS)) { 2340 base = trunc_page(va); 2341 offset = va & PAGE_MASK; 2342 size = roundup(offset + size, PAGE_SIZE); 2343 kva_free(base, size); 2344 } 2345 #endif 2346 } 2347 2348 /* 2349 * mmu_booke_object_init_pt preloads the ptes for a given object into the 2350 * specified pmap. This eliminates the blast of soft faults on process startup 2351 * and immediately after an mmap. 2352 */ 2353 static void 2354 mmu_booke_object_init_pt(mmu_t mmu, pmap_t pmap, vm_offset_t addr, 2355 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 2356 { 2357 2358 VM_OBJECT_ASSERT_WLOCKED(object); 2359 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 2360 ("mmu_booke_object_init_pt: non-device object")); 2361 } 2362 2363 /* 2364 * Perform the pmap work for mincore. 2365 */ 2366 static int 2367 mmu_booke_mincore(mmu_t mmu, pmap_t pmap, vm_offset_t addr, 2368 vm_paddr_t *pap) 2369 { 2370 2371 /* XXX: this should be implemented at some point */ 2372 return (0); 2373 } 2374 2375 static int 2376 mmu_booke_change_attr(mmu_t mmu, vm_offset_t addr, vm_size_t sz, 2377 vm_memattr_t mode) 2378 { 2379 vm_offset_t va; 2380 pte_t *pte; 2381 int i, j; 2382 tlb_entry_t e; 2383 2384 addr = trunc_page(addr); 2385 2386 /* Only allow changes to mapped kernel addresses. This includes: 2387 * - KVA 2388 * - DMAP (powerpc64) 2389 * - Device mappings 2390 */ 2391 if (addr <= VM_MAXUSER_ADDRESS || 2392 #ifdef __powerpc64__ 2393 (addr >= tlb1_map_base && addr < DMAP_BASE_ADDRESS) || 2394 (addr > DMAP_MAX_ADDRESS && addr < VM_MIN_KERNEL_ADDRESS) || 2395 #else 2396 (addr >= tlb1_map_base && addr < VM_MIN_KERNEL_ADDRESS) || 2397 #endif 2398 (addr > VM_MAX_KERNEL_ADDRESS)) 2399 return (EINVAL); 2400 2401 /* Check TLB1 mappings */ 2402 for (i = 0; i < TLB1_ENTRIES; i++) { 2403 tlb1_read_entry(&e, i); 2404 if (!(e.mas1 & MAS1_VALID)) 2405 continue; 2406 if (addr >= e.virt && addr < e.virt + e.size) 2407 break; 2408 } 2409 if (i < TLB1_ENTRIES) { 2410 /* Only allow full mappings to be modified for now. */ 2411 /* Validate the range. */ 2412 for (j = i, va = addr; va < addr + sz; va += e.size, j++) { 2413 tlb1_read_entry(&e, j); 2414 if (va != e.virt || (sz - (va - addr) < e.size)) 2415 return (EINVAL); 2416 } 2417 for (va = addr; va < addr + sz; va += e.size, i++) { 2418 tlb1_read_entry(&e, i); 2419 e.mas2 &= ~MAS2_WIMGE_MASK; 2420 e.mas2 |= tlb_calc_wimg(e.phys, mode); 2421 2422 /* 2423 * Write it out to the TLB. Should really re-sync with other 2424 * cores. 2425 */ 2426 tlb1_write_entry(&e, i); 2427 } 2428 return (0); 2429 } 2430 2431 /* Not in TLB1, try through pmap */ 2432 /* First validate the range. */ 2433 for (va = addr; va < addr + sz; va += PAGE_SIZE) { 2434 pte = pte_find(mmu, kernel_pmap, va); 2435 if (pte == NULL || !PTE_ISVALID(pte)) 2436 return (EINVAL); 2437 } 2438 2439 mtx_lock_spin(&tlbivax_mutex); 2440 tlb_miss_lock(); 2441 for (va = addr; va < addr + sz; va += PAGE_SIZE) { 2442 pte = pte_find(mmu, kernel_pmap, va); 2443 *pte &= ~(PTE_MAS2_MASK << PTE_MAS2_SHIFT); 2444 *pte |= tlb_calc_wimg(PTE_PA(pte), mode) << PTE_MAS2_SHIFT; 2445 tlb0_flush_entry(va); 2446 } 2447 tlb_miss_unlock(); 2448 mtx_unlock_spin(&tlbivax_mutex); 2449 2450 return (0); 2451 } 2452 2453 static void 2454 mmu_booke_page_array_startup(mmu_t mmu, long pages) 2455 { 2456 vm_page_array_size = pages; 2457 } 2458 2459 /**************************************************************************/ 2460 /* TID handling */ 2461 /**************************************************************************/ 2462 2463 /* 2464 * Allocate a TID. If necessary, steal one from someone else. 2465 * The new TID is flushed from the TLB before returning. 2466 */ 2467 static tlbtid_t 2468 tid_alloc(pmap_t pmap) 2469 { 2470 tlbtid_t tid; 2471 int thiscpu; 2472 2473 KASSERT((pmap != kernel_pmap), ("tid_alloc: kernel pmap")); 2474 2475 CTR2(KTR_PMAP, "%s: s (pmap = %p)", __func__, pmap); 2476 2477 thiscpu = PCPU_GET(cpuid); 2478 2479 tid = PCPU_GET(booke.tid_next); 2480 if (tid > TID_MAX) 2481 tid = TID_MIN; 2482 PCPU_SET(booke.tid_next, tid + 1); 2483 2484 /* If we are stealing TID then clear the relevant pmap's field */ 2485 if (tidbusy[thiscpu][tid] != NULL) { 2486 2487 CTR2(KTR_PMAP, "%s: warning: stealing tid %d", __func__, tid); 2488 2489 tidbusy[thiscpu][tid]->pm_tid[thiscpu] = TID_NONE; 2490 2491 /* Flush all entries from TLB0 matching this TID. */ 2492 tid_flush(tid); 2493 } 2494 2495 tidbusy[thiscpu][tid] = pmap; 2496 pmap->pm_tid[thiscpu] = tid; 2497 __asm __volatile("msync; isync"); 2498 2499 CTR3(KTR_PMAP, "%s: e (%02d next = %02d)", __func__, tid, 2500 PCPU_GET(booke.tid_next)); 2501 2502 return (tid); 2503 } 2504 2505 /**************************************************************************/ 2506 /* TLB0 handling */ 2507 /**************************************************************************/ 2508 2509 /* Convert TLB0 va and way number to tlb0[] table index. */ 2510 static inline unsigned int 2511 tlb0_tableidx(vm_offset_t va, unsigned int way) 2512 { 2513 unsigned int idx; 2514 2515 idx = (way * TLB0_ENTRIES_PER_WAY); 2516 idx += (va & MAS2_TLB0_ENTRY_IDX_MASK) >> MAS2_TLB0_ENTRY_IDX_SHIFT; 2517 return (idx); 2518 } 2519 2520 /* 2521 * Invalidate TLB0 entry. 2522 */ 2523 static inline void 2524 tlb0_flush_entry(vm_offset_t va) 2525 { 2526 2527 CTR2(KTR_PMAP, "%s: s va=0x%08x", __func__, va); 2528 2529 mtx_assert(&tlbivax_mutex, MA_OWNED); 2530 2531 __asm __volatile("tlbivax 0, %0" :: "r"(va & MAS2_EPN_MASK)); 2532 __asm __volatile("isync; msync"); 2533 __asm __volatile("tlbsync; msync"); 2534 2535 CTR1(KTR_PMAP, "%s: e", __func__); 2536 } 2537 2538 2539 /**************************************************************************/ 2540 /* TLB1 handling */ 2541 /**************************************************************************/ 2542 2543 /* 2544 * TLB1 mapping notes: 2545 * 2546 * TLB1[0] Kernel text and data. 2547 * TLB1[1-15] Additional kernel text and data mappings (if required), PCI 2548 * windows, other devices mappings. 2549 */ 2550 2551 /* 2552 * Read an entry from given TLB1 slot. 2553 */ 2554 void 2555 tlb1_read_entry(tlb_entry_t *entry, unsigned int slot) 2556 { 2557 register_t msr; 2558 uint32_t mas0; 2559 2560 KASSERT((entry != NULL), ("%s(): Entry is NULL!", __func__)); 2561 2562 msr = mfmsr(); 2563 __asm __volatile("wrteei 0"); 2564 2565 mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(slot); 2566 mtspr(SPR_MAS0, mas0); 2567 __asm __volatile("isync; tlbre"); 2568 2569 entry->mas1 = mfspr(SPR_MAS1); 2570 entry->mas2 = mfspr(SPR_MAS2); 2571 entry->mas3 = mfspr(SPR_MAS3); 2572 2573 switch ((mfpvr() >> 16) & 0xFFFF) { 2574 case FSL_E500v2: 2575 case FSL_E500mc: 2576 case FSL_E5500: 2577 case FSL_E6500: 2578 entry->mas7 = mfspr(SPR_MAS7); 2579 break; 2580 default: 2581 entry->mas7 = 0; 2582 break; 2583 } 2584 __asm __volatile("wrtee %0" :: "r"(msr)); 2585 2586 entry->virt = entry->mas2 & MAS2_EPN_MASK; 2587 entry->phys = ((vm_paddr_t)(entry->mas7 & MAS7_RPN) << 32) | 2588 (entry->mas3 & MAS3_RPN); 2589 entry->size = 2590 tsize2size((entry->mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT); 2591 } 2592 2593 struct tlbwrite_args { 2594 tlb_entry_t *e; 2595 unsigned int idx; 2596 }; 2597 2598 static uint32_t 2599 tlb1_find_free(void) 2600 { 2601 tlb_entry_t e; 2602 int i; 2603 2604 for (i = 0; i < TLB1_ENTRIES; i++) { 2605 tlb1_read_entry(&e, i); 2606 if ((e.mas1 & MAS1_VALID) == 0) 2607 return (i); 2608 } 2609 return (-1); 2610 } 2611 2612 static void 2613 tlb1_purge_va_range(vm_offset_t va, vm_size_t size) 2614 { 2615 tlb_entry_t e; 2616 int i; 2617 2618 for (i = 0; i < TLB1_ENTRIES; i++) { 2619 tlb1_read_entry(&e, i); 2620 if ((e.mas1 & MAS1_VALID) == 0) 2621 continue; 2622 if ((e.mas2 & MAS2_EPN_MASK) >= va && 2623 (e.mas2 & MAS2_EPN_MASK) < va + size) { 2624 mtspr(SPR_MAS1, e.mas1 & ~MAS1_VALID); 2625 __asm __volatile("isync; tlbwe; isync; msync"); 2626 } 2627 } 2628 } 2629 2630 static void 2631 tlb1_write_entry_int(void *arg) 2632 { 2633 struct tlbwrite_args *args = arg; 2634 uint32_t idx, mas0; 2635 2636 idx = args->idx; 2637 if (idx == -1) { 2638 tlb1_purge_va_range(args->e->virt, args->e->size); 2639 idx = tlb1_find_free(); 2640 if (idx == -1) 2641 panic("No free TLB1 entries!\n"); 2642 } 2643 /* Select entry */ 2644 mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(idx); 2645 2646 mtspr(SPR_MAS0, mas0); 2647 mtspr(SPR_MAS1, args->e->mas1); 2648 mtspr(SPR_MAS2, args->e->mas2); 2649 mtspr(SPR_MAS3, args->e->mas3); 2650 switch ((mfpvr() >> 16) & 0xFFFF) { 2651 case FSL_E500mc: 2652 case FSL_E5500: 2653 case FSL_E6500: 2654 mtspr(SPR_MAS8, 0); 2655 /* FALLTHROUGH */ 2656 case FSL_E500v2: 2657 mtspr(SPR_MAS7, args->e->mas7); 2658 break; 2659 default: 2660 break; 2661 } 2662 2663 __asm __volatile("isync; tlbwe; isync; msync"); 2664 2665 } 2666 2667 static void 2668 tlb1_write_entry_sync(void *arg) 2669 { 2670 /* Empty synchronization point for smp_rendezvous(). */ 2671 } 2672 2673 /* 2674 * Write given entry to TLB1 hardware. 2675 */ 2676 static void 2677 tlb1_write_entry(tlb_entry_t *e, unsigned int idx) 2678 { 2679 struct tlbwrite_args args; 2680 2681 args.e = e; 2682 args.idx = idx; 2683 2684 #ifdef SMP 2685 if ((e->mas2 & _TLB_ENTRY_SHARED) && smp_started) { 2686 mb(); 2687 smp_rendezvous(tlb1_write_entry_sync, 2688 tlb1_write_entry_int, 2689 tlb1_write_entry_sync, &args); 2690 } else 2691 #endif 2692 { 2693 register_t msr; 2694 2695 msr = mfmsr(); 2696 __asm __volatile("wrteei 0"); 2697 tlb1_write_entry_int(&args); 2698 __asm __volatile("wrtee %0" :: "r"(msr)); 2699 } 2700 } 2701 2702 /* 2703 * Convert TLB TSIZE value to mapped region size. 2704 */ 2705 static vm_size_t 2706 tsize2size(unsigned int tsize) 2707 { 2708 2709 /* 2710 * size = 4^tsize KB 2711 * size = 4^tsize * 2^10 = 2^(2 * tsize - 10) 2712 */ 2713 2714 return ((1 << (2 * tsize)) * 1024); 2715 } 2716 2717 /* 2718 * Convert region size (must be power of 4) to TLB TSIZE value. 2719 */ 2720 static unsigned int 2721 size2tsize(vm_size_t size) 2722 { 2723 2724 return (ilog2(size) / 2 - 5); 2725 } 2726 2727 /* 2728 * Register permanent kernel mapping in TLB1. 2729 * 2730 * Entries are created starting from index 0 (current free entry is 2731 * kept in tlb1_idx) and are not supposed to be invalidated. 2732 */ 2733 int 2734 tlb1_set_entry(vm_offset_t va, vm_paddr_t pa, vm_size_t size, 2735 uint32_t flags) 2736 { 2737 tlb_entry_t e; 2738 uint32_t ts, tid; 2739 int tsize, index; 2740 2741 /* First try to update an existing entry. */ 2742 for (index = 0; index < TLB1_ENTRIES; index++) { 2743 tlb1_read_entry(&e, index); 2744 /* Check if we're just updating the flags, and update them. */ 2745 if (e.phys == pa && e.virt == va && e.size == size) { 2746 e.mas2 = (va & MAS2_EPN_MASK) | flags; 2747 tlb1_write_entry(&e, index); 2748 return (0); 2749 } 2750 } 2751 2752 /* Convert size to TSIZE */ 2753 tsize = size2tsize(size); 2754 2755 tid = (TID_KERNEL << MAS1_TID_SHIFT) & MAS1_TID_MASK; 2756 /* XXX TS is hard coded to 0 for now as we only use single address space */ 2757 ts = (0 << MAS1_TS_SHIFT) & MAS1_TS_MASK; 2758 2759 e.phys = pa; 2760 e.virt = va; 2761 e.size = size; 2762 e.mas1 = MAS1_VALID | MAS1_IPROT | ts | tid; 2763 e.mas1 |= ((tsize << MAS1_TSIZE_SHIFT) & MAS1_TSIZE_MASK); 2764 e.mas2 = (va & MAS2_EPN_MASK) | flags; 2765 2766 /* Set supervisor RWX permission bits */ 2767 e.mas3 = (pa & MAS3_RPN) | MAS3_SR | MAS3_SW | MAS3_SX; 2768 e.mas7 = (pa >> 32) & MAS7_RPN; 2769 2770 tlb1_write_entry(&e, -1); 2771 2772 return (0); 2773 } 2774 2775 /* 2776 * Map in contiguous RAM region into the TLB1. 2777 */ 2778 static vm_size_t 2779 tlb1_mapin_region(vm_offset_t va, vm_paddr_t pa, vm_size_t size, int wimge) 2780 { 2781 vm_offset_t base; 2782 vm_size_t mapped, sz, ssize; 2783 2784 mapped = 0; 2785 base = va; 2786 ssize = size; 2787 2788 while (size > 0) { 2789 sz = 1UL << (ilog2(size) & ~1); 2790 /* Align size to PA */ 2791 if (pa % sz != 0) { 2792 do { 2793 sz >>= 2; 2794 } while (pa % sz != 0); 2795 } 2796 /* Now align from there to VA */ 2797 if (va % sz != 0) { 2798 do { 2799 sz >>= 2; 2800 } while (va % sz != 0); 2801 } 2802 #ifdef __powerpc64__ 2803 /* 2804 * Clamp TLB1 entries to 4G. 2805 * 2806 * While the e6500 supports up to 1TB mappings, the e5500 2807 * only supports up to 4G mappings. (0b1011) 2808 * 2809 * If any e6500 machines capable of supporting a very 2810 * large amount of memory appear in the future, we can 2811 * revisit this. 2812 * 2813 * For now, though, since we have plenty of space in TLB1, 2814 * always avoid creating entries larger than 4GB. 2815 */ 2816 sz = MIN(sz, 1UL << 32); 2817 #endif 2818 if (bootverbose) 2819 printf("Wiring VA=%p to PA=%jx (size=%lx)\n", 2820 (void *)va, (uintmax_t)pa, (long)sz); 2821 if (tlb1_set_entry(va, pa, sz, 2822 _TLB_ENTRY_SHARED | wimge) < 0) 2823 return (mapped); 2824 size -= sz; 2825 pa += sz; 2826 va += sz; 2827 } 2828 2829 mapped = (va - base); 2830 if (bootverbose) 2831 printf("mapped size 0x%"PRIxPTR" (wasted space 0x%"PRIxPTR")\n", 2832 mapped, mapped - ssize); 2833 2834 return (mapped); 2835 } 2836 2837 /* 2838 * TLB1 initialization routine, to be called after the very first 2839 * assembler level setup done in locore.S. 2840 */ 2841 void 2842 tlb1_init() 2843 { 2844 vm_offset_t mas2; 2845 uint32_t mas0, mas1, mas3, mas7; 2846 uint32_t tsz; 2847 2848 tlb1_get_tlbconf(); 2849 2850 mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(0); 2851 mtspr(SPR_MAS0, mas0); 2852 __asm __volatile("isync; tlbre"); 2853 2854 mas1 = mfspr(SPR_MAS1); 2855 mas2 = mfspr(SPR_MAS2); 2856 mas3 = mfspr(SPR_MAS3); 2857 mas7 = mfspr(SPR_MAS7); 2858 2859 kernload = ((vm_paddr_t)(mas7 & MAS7_RPN) << 32) | 2860 (mas3 & MAS3_RPN); 2861 2862 tsz = (mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT; 2863 kernsize += (tsz > 0) ? tsize2size(tsz) : 0; 2864 kernstart = trunc_page(mas2); 2865 2866 /* Setup TLB miss defaults */ 2867 set_mas4_defaults(); 2868 } 2869 2870 /* 2871 * pmap_early_io_unmap() should be used in short conjunction with 2872 * pmap_early_io_map(), as in the following snippet: 2873 * 2874 * x = pmap_early_io_map(...); 2875 * <do something with x> 2876 * pmap_early_io_unmap(x, size); 2877 * 2878 * And avoiding more allocations between. 2879 */ 2880 void 2881 pmap_early_io_unmap(vm_offset_t va, vm_size_t size) 2882 { 2883 int i; 2884 tlb_entry_t e; 2885 vm_size_t isize; 2886 2887 size = roundup(size, PAGE_SIZE); 2888 isize = size; 2889 for (i = 0; i < TLB1_ENTRIES && size > 0; i++) { 2890 tlb1_read_entry(&e, i); 2891 if (!(e.mas1 & MAS1_VALID)) 2892 continue; 2893 if (va <= e.virt && (va + isize) >= (e.virt + e.size)) { 2894 size -= e.size; 2895 e.mas1 &= ~MAS1_VALID; 2896 tlb1_write_entry(&e, i); 2897 } 2898 } 2899 if (tlb1_map_base == va + isize) 2900 tlb1_map_base -= isize; 2901 } 2902 2903 vm_offset_t 2904 pmap_early_io_map(vm_paddr_t pa, vm_size_t size) 2905 { 2906 vm_paddr_t pa_base; 2907 vm_offset_t va, sz; 2908 int i; 2909 tlb_entry_t e; 2910 2911 KASSERT(!pmap_bootstrapped, ("Do not use after PMAP is up!")); 2912 2913 for (i = 0; i < TLB1_ENTRIES; i++) { 2914 tlb1_read_entry(&e, i); 2915 if (!(e.mas1 & MAS1_VALID)) 2916 continue; 2917 if (pa >= e.phys && (pa + size) <= 2918 (e.phys + e.size)) 2919 return (e.virt + (pa - e.phys)); 2920 } 2921 2922 pa_base = rounddown(pa, PAGE_SIZE); 2923 size = roundup(size + (pa - pa_base), PAGE_SIZE); 2924 tlb1_map_base = roundup2(tlb1_map_base, 1 << (ilog2(size) & ~1)); 2925 va = tlb1_map_base + (pa - pa_base); 2926 2927 do { 2928 sz = 1 << (ilog2(size) & ~1); 2929 tlb1_set_entry(tlb1_map_base, pa_base, sz, 2930 _TLB_ENTRY_SHARED | _TLB_ENTRY_IO); 2931 size -= sz; 2932 pa_base += sz; 2933 tlb1_map_base += sz; 2934 } while (size > 0); 2935 2936 return (va); 2937 } 2938 2939 void 2940 pmap_track_page(pmap_t pmap, vm_offset_t va) 2941 { 2942 vm_paddr_t pa; 2943 vm_page_t page; 2944 struct pv_entry *pve; 2945 2946 va = trunc_page(va); 2947 pa = pmap_kextract(va); 2948 page = PHYS_TO_VM_PAGE(pa); 2949 2950 rw_wlock(&pvh_global_lock); 2951 PMAP_LOCK(pmap); 2952 2953 TAILQ_FOREACH(pve, &page->md.pv_list, pv_link) { 2954 if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) { 2955 goto out; 2956 } 2957 } 2958 page->md.pv_tracked = true; 2959 pv_insert(pmap, va, page); 2960 out: 2961 PMAP_UNLOCK(pmap); 2962 rw_wunlock(&pvh_global_lock); 2963 } 2964 2965 2966 /* 2967 * Setup MAS4 defaults. 2968 * These values are loaded to MAS0-2 on a TLB miss. 2969 */ 2970 static void 2971 set_mas4_defaults(void) 2972 { 2973 uint32_t mas4; 2974 2975 /* Defaults: TLB0, PID0, TSIZED=4K */ 2976 mas4 = MAS4_TLBSELD0; 2977 mas4 |= (TLB_SIZE_4K << MAS4_TSIZED_SHIFT) & MAS4_TSIZED_MASK; 2978 #ifdef SMP 2979 mas4 |= MAS4_MD; 2980 #endif 2981 mtspr(SPR_MAS4, mas4); 2982 __asm __volatile("isync"); 2983 } 2984 2985 2986 /* 2987 * Return 0 if the physical IO range is encompassed by one of the 2988 * the TLB1 entries, otherwise return related error code. 2989 */ 2990 static int 2991 tlb1_iomapped(int i, vm_paddr_t pa, vm_size_t size, vm_offset_t *va) 2992 { 2993 uint32_t prot; 2994 vm_paddr_t pa_start; 2995 vm_paddr_t pa_end; 2996 unsigned int entry_tsize; 2997 vm_size_t entry_size; 2998 tlb_entry_t e; 2999 3000 *va = (vm_offset_t)NULL; 3001 3002 tlb1_read_entry(&e, i); 3003 /* Skip invalid entries */ 3004 if (!(e.mas1 & MAS1_VALID)) 3005 return (EINVAL); 3006 3007 /* 3008 * The entry must be cache-inhibited, guarded, and r/w 3009 * so it can function as an i/o page 3010 */ 3011 prot = e.mas2 & (MAS2_I | MAS2_G); 3012 if (prot != (MAS2_I | MAS2_G)) 3013 return (EPERM); 3014 3015 prot = e.mas3 & (MAS3_SR | MAS3_SW); 3016 if (prot != (MAS3_SR | MAS3_SW)) 3017 return (EPERM); 3018 3019 /* The address should be within the entry range. */ 3020 entry_tsize = (e.mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT; 3021 KASSERT((entry_tsize), ("tlb1_iomapped: invalid entry tsize")); 3022 3023 entry_size = tsize2size(entry_tsize); 3024 pa_start = (((vm_paddr_t)e.mas7 & MAS7_RPN) << 32) | 3025 (e.mas3 & MAS3_RPN); 3026 pa_end = pa_start + entry_size; 3027 3028 if ((pa < pa_start) || ((pa + size) > pa_end)) 3029 return (ERANGE); 3030 3031 /* Return virtual address of this mapping. */ 3032 *va = (e.mas2 & MAS2_EPN_MASK) + (pa - pa_start); 3033 return (0); 3034 } 3035 3036 #ifdef DDB 3037 /* Print out contents of the MAS registers for each TLB0 entry */ 3038 static void 3039 #ifdef __powerpc64__ 3040 tlb_print_entry(int i, uint32_t mas1, uint64_t mas2, uint32_t mas3, 3041 #else 3042 tlb_print_entry(int i, uint32_t mas1, uint32_t mas2, uint32_t mas3, 3043 #endif 3044 uint32_t mas7) 3045 { 3046 int as; 3047 char desc[3]; 3048 tlbtid_t tid; 3049 vm_size_t size; 3050 unsigned int tsize; 3051 3052 desc[2] = '\0'; 3053 if (mas1 & MAS1_VALID) 3054 desc[0] = 'V'; 3055 else 3056 desc[0] = ' '; 3057 3058 if (mas1 & MAS1_IPROT) 3059 desc[1] = 'P'; 3060 else 3061 desc[1] = ' '; 3062 3063 as = (mas1 & MAS1_TS_MASK) ? 1 : 0; 3064 tid = MAS1_GETTID(mas1); 3065 3066 tsize = (mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT; 3067 size = 0; 3068 if (tsize) 3069 size = tsize2size(tsize); 3070 3071 printf("%3d: (%s) [AS=%d] " 3072 "sz = 0x%jx tsz = %d tid = %d mas1 = 0x%08x " 3073 "mas2(va) = 0x%"PRI0ptrX" mas3(pa) = 0x%08x mas7 = 0x%08x\n", 3074 i, desc, as, (uintmax_t)size, tsize, tid, mas1, mas2, mas3, mas7); 3075 } 3076 3077 DB_SHOW_COMMAND(tlb0, tlb0_print_tlbentries) 3078 { 3079 uint32_t mas0, mas1, mas3, mas7; 3080 #ifdef __powerpc64__ 3081 uint64_t mas2; 3082 #else 3083 uint32_t mas2; 3084 #endif 3085 int entryidx, way, idx; 3086 3087 printf("TLB0 entries:\n"); 3088 for (way = 0; way < TLB0_WAYS; way ++) 3089 for (entryidx = 0; entryidx < TLB0_ENTRIES_PER_WAY; entryidx++) { 3090 3091 mas0 = MAS0_TLBSEL(0) | MAS0_ESEL(way); 3092 mtspr(SPR_MAS0, mas0); 3093 3094 mas2 = entryidx << MAS2_TLB0_ENTRY_IDX_SHIFT; 3095 mtspr(SPR_MAS2, mas2); 3096 3097 __asm __volatile("isync; tlbre"); 3098 3099 mas1 = mfspr(SPR_MAS1); 3100 mas2 = mfspr(SPR_MAS2); 3101 mas3 = mfspr(SPR_MAS3); 3102 mas7 = mfspr(SPR_MAS7); 3103 3104 idx = tlb0_tableidx(mas2, way); 3105 tlb_print_entry(idx, mas1, mas2, mas3, mas7); 3106 } 3107 } 3108 3109 /* 3110 * Print out contents of the MAS registers for each TLB1 entry 3111 */ 3112 DB_SHOW_COMMAND(tlb1, tlb1_print_tlbentries) 3113 { 3114 uint32_t mas0, mas1, mas3, mas7; 3115 #ifdef __powerpc64__ 3116 uint64_t mas2; 3117 #else 3118 uint32_t mas2; 3119 #endif 3120 int i; 3121 3122 printf("TLB1 entries:\n"); 3123 for (i = 0; i < TLB1_ENTRIES; i++) { 3124 3125 mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(i); 3126 mtspr(SPR_MAS0, mas0); 3127 3128 __asm __volatile("isync; tlbre"); 3129 3130 mas1 = mfspr(SPR_MAS1); 3131 mas2 = mfspr(SPR_MAS2); 3132 mas3 = mfspr(SPR_MAS3); 3133 mas7 = mfspr(SPR_MAS7); 3134 3135 tlb_print_entry(i, mas1, mas2, mas3, mas7); 3136 } 3137 } 3138 #endif 3139