1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2020 Justin Hibbits 5 * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com> 6 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com> 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * Some hw specific parts of this pmap were derived or influenced 30 * by NetBSD's ibm4xx pmap module. More generic code is shared with 31 * a few other pmap modules from the FreeBSD tree. 32 */ 33 34 /* 35 * VM layout notes: 36 * 37 * Kernel and user threads run within one common virtual address space 38 * defined by AS=0. 39 * 40 * 64-bit pmap: 41 * Virtual address space layout: 42 * ----------------------------- 43 * 0x0000_0000_0000_0000 - 0x3fff_ffff_ffff_ffff : user process 44 * 0x4000_0000_0000_0000 - 0x7fff_ffff_ffff_ffff : unused 45 * 0x8000_0000_0000_0000 - 0xbfff_ffff_ffff_ffff : mmio region 46 * 0xc000_0000_0000_0000 - 0xdfff_ffff_ffff_ffff : direct map 47 * 0xe000_0000_0000_0000 - 0xffff_ffff_ffff_ffff : KVA 48 */ 49 50 #include <sys/cdefs.h> 51 __FBSDID("$FreeBSD$"); 52 53 #include "opt_ddb.h" 54 #include "opt_kstack_pages.h" 55 56 #include <sys/param.h> 57 #include <sys/conf.h> 58 #include <sys/malloc.h> 59 #include <sys/ktr.h> 60 #include <sys/proc.h> 61 #include <sys/user.h> 62 #include <sys/queue.h> 63 #include <sys/systm.h> 64 #include <sys/kernel.h> 65 #include <sys/kerneldump.h> 66 #include <sys/linker.h> 67 #include <sys/msgbuf.h> 68 #include <sys/lock.h> 69 #include <sys/mutex.h> 70 #include <sys/rwlock.h> 71 #include <sys/sched.h> 72 #include <sys/smp.h> 73 #include <sys/vmmeter.h> 74 75 #include <vm/vm.h> 76 #include <vm/vm_page.h> 77 #include <vm/vm_kern.h> 78 #include <vm/vm_pageout.h> 79 #include <vm/vm_extern.h> 80 #include <vm/vm_object.h> 81 #include <vm/vm_param.h> 82 #include <vm/vm_map.h> 83 #include <vm/vm_pager.h> 84 #include <vm/vm_phys.h> 85 #include <vm/vm_pagequeue.h> 86 #include <vm/uma.h> 87 88 #include <machine/_inttypes.h> 89 #include <machine/cpu.h> 90 #include <machine/pcb.h> 91 #include <machine/platform.h> 92 93 #include <machine/tlb.h> 94 #include <machine/spr.h> 95 #include <machine/md_var.h> 96 #include <machine/mmuvar.h> 97 #include <machine/pmap.h> 98 #include <machine/pte.h> 99 100 #include <ddb/ddb.h> 101 102 #ifdef DEBUG 103 #define debugf(fmt, args...) printf(fmt, ##args) 104 #else 105 #define debugf(fmt, args...) 106 #endif 107 108 #define PRI0ptrX "016lx" 109 110 /**************************************************************************/ 111 /* PMAP */ 112 /**************************************************************************/ 113 114 unsigned int kernel_pdirs; 115 static uma_zone_t ptbl_root_zone; 116 static pte_t ****kernel_ptbl_root; 117 118 /* 119 * Base of the pmap_mapdev() region. On 32-bit it immediately follows the 120 * userspace address range. On On 64-bit it's far above, at (1 << 63), and 121 * ranges up to the DMAP, giving 62 bits of PA allowed. This is far larger than 122 * the widest Book-E address bus, the e6500 has a 40-bit PA space. This allows 123 * us to map akin to the DMAP, with addresses identical to the PA, offset by the 124 * base. 125 */ 126 #define VM_MAPDEV_BASE 0x8000000000000000 127 #define VM_MAPDEV_PA_MAX 0x4000000000000000 /* Don't encroach on DMAP */ 128 129 static void tid_flush(tlbtid_t tid); 130 static unsigned long ilog2(unsigned long); 131 132 /**************************************************************************/ 133 /* Page table management */ 134 /**************************************************************************/ 135 136 #define PMAP_ROOT_SIZE (sizeof(pte_t****) * PG_ROOT_NENTRIES) 137 static pte_t *ptbl_alloc(pmap_t pmap, vm_offset_t va, 138 bool nosleep, bool *is_new); 139 static void ptbl_hold(pmap_t, pte_t *); 140 static int ptbl_unhold(pmap_t, vm_offset_t); 141 142 static vm_paddr_t pte_vatopa(pmap_t, vm_offset_t); 143 static int pte_enter(pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t); 144 static int pte_remove(pmap_t, vm_offset_t, uint8_t); 145 static pte_t *pte_find(pmap_t, vm_offset_t); 146 static pte_t *pte_find_next(pmap_t, vm_offset_t *); 147 static void kernel_pte_alloc(vm_offset_t, vm_offset_t); 148 149 /**************************************************************************/ 150 /* Page table related */ 151 /**************************************************************************/ 152 153 /* Allocate a page, to be used in a page table. */ 154 static vm_offset_t 155 mmu_booke_alloc_page(pmap_t pmap, unsigned int idx, bool nosleep) 156 { 157 vm_page_t m; 158 int req; 159 160 req = VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO; 161 while ((m = vm_page_alloc(NULL, idx, req)) == NULL) { 162 if (nosleep) 163 return (0); 164 165 PMAP_UNLOCK(pmap); 166 rw_wunlock(&pvh_global_lock); 167 vm_wait(NULL); 168 rw_wlock(&pvh_global_lock); 169 PMAP_LOCK(pmap); 170 } 171 172 if (!(m->flags & PG_ZERO)) 173 /* Zero whole ptbl. */ 174 mmu_booke_zero_page(m); 175 176 return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m))); 177 } 178 179 /* Initialize pool of kva ptbl buffers. */ 180 static void 181 ptbl_init(void) 182 { 183 } 184 185 /* Get a pointer to a PTE in a page table. */ 186 static __inline pte_t * 187 pte_find(pmap_t pmap, vm_offset_t va) 188 { 189 pte_t ***pdir_l1; 190 pte_t **pdir; 191 pte_t *ptbl; 192 193 KASSERT((pmap != NULL), ("pte_find: invalid pmap")); 194 195 pdir_l1 = pmap->pm_root[PG_ROOT_IDX(va)]; 196 if (pdir_l1 == NULL) 197 return (NULL); 198 pdir = pdir_l1[PDIR_L1_IDX(va)]; 199 if (pdir == NULL) 200 return (NULL); 201 ptbl = pdir[PDIR_IDX(va)]; 202 203 return ((ptbl != NULL) ? &ptbl[PTBL_IDX(va)] : NULL); 204 } 205 206 /* Get a pointer to a PTE in a page table, or the next closest (greater) one. */ 207 static __inline pte_t * 208 pte_find_next(pmap_t pmap, vm_offset_t *pva) 209 { 210 vm_offset_t va; 211 pte_t ****pm_root; 212 pte_t *pte; 213 unsigned long i, j, k, l; 214 215 KASSERT((pmap != NULL), ("pte_find: invalid pmap")); 216 217 va = *pva; 218 i = PG_ROOT_IDX(va); 219 j = PDIR_L1_IDX(va); 220 k = PDIR_IDX(va); 221 l = PTBL_IDX(va); 222 pm_root = pmap->pm_root; 223 224 /* truncate the VA for later. */ 225 va &= ~((1UL << (PG_ROOT_H + 1)) - 1); 226 for (; i < PG_ROOT_NENTRIES; i++, j = 0, k = 0, l = 0) { 227 if (pm_root[i] == 0) 228 continue; 229 for (; j < PDIR_L1_NENTRIES; j++, k = 0, l = 0) { 230 if (pm_root[i][j] == 0) 231 continue; 232 for (; k < PDIR_NENTRIES; k++, l = 0) { 233 if (pm_root[i][j][k] == NULL) 234 continue; 235 for (; l < PTBL_NENTRIES; l++) { 236 pte = &pm_root[i][j][k][l]; 237 if (!PTE_ISVALID(pte)) 238 continue; 239 *pva = va + PG_ROOT_SIZE * i + 240 PDIR_L1_SIZE * j + 241 PDIR_SIZE * k + 242 PAGE_SIZE * l; 243 return (pte); 244 } 245 } 246 } 247 } 248 return (NULL); 249 } 250 251 static bool 252 unhold_free_page(pmap_t pmap, vm_page_t m) 253 { 254 255 if (vm_page_unwire_noq(m)) { 256 vm_page_free_zero(m); 257 return (true); 258 } 259 260 return (false); 261 } 262 263 static vm_offset_t 264 get_pgtbl_page(pmap_t pmap, vm_offset_t *ptr_tbl, uint32_t index, 265 bool nosleep, bool hold_parent, bool *isnew) 266 { 267 vm_offset_t page; 268 vm_page_t m; 269 270 page = ptr_tbl[index]; 271 KASSERT(page != 0 || pmap != kernel_pmap, 272 ("NULL page table page found in kernel pmap!")); 273 if (page == 0) { 274 page = mmu_booke_alloc_page(pmap, index, nosleep); 275 if (ptr_tbl[index] == 0) { 276 *isnew = true; 277 ptr_tbl[index] = page; 278 if (hold_parent) { 279 m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)ptr_tbl)); 280 m->ref_count++; 281 } 282 return (page); 283 } 284 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS(page)); 285 page = ptr_tbl[index]; 286 vm_page_unwire_noq(m); 287 vm_page_free_zero(m); 288 } 289 290 *isnew = false; 291 292 return (page); 293 } 294 295 /* Allocate page table. */ 296 static pte_t* 297 ptbl_alloc(pmap_t pmap, vm_offset_t va, bool nosleep, bool *is_new) 298 { 299 unsigned int pg_root_idx = PG_ROOT_IDX(va); 300 unsigned int pdir_l1_idx = PDIR_L1_IDX(va); 301 unsigned int pdir_idx = PDIR_IDX(va); 302 vm_offset_t pdir_l1, pdir, ptbl; 303 304 /* When holding a parent, no need to hold the root index pages. */ 305 pdir_l1 = get_pgtbl_page(pmap, (vm_offset_t *)pmap->pm_root, 306 pg_root_idx, nosleep, false, is_new); 307 if (pdir_l1 == 0) 308 return (NULL); 309 pdir = get_pgtbl_page(pmap, (vm_offset_t *)pdir_l1, pdir_l1_idx, 310 nosleep, !*is_new, is_new); 311 if (pdir == 0) 312 return (NULL); 313 ptbl = get_pgtbl_page(pmap, (vm_offset_t *)pdir, pdir_idx, 314 nosleep, !*is_new, is_new); 315 316 return ((pte_t *)ptbl); 317 } 318 319 /* 320 * Decrement ptbl pages hold count and attempt to free ptbl pages. Called 321 * when removing pte entry from ptbl. 322 * 323 * Return 1 if ptbl pages were freed. 324 */ 325 static int 326 ptbl_unhold(pmap_t pmap, vm_offset_t va) 327 { 328 pte_t *ptbl; 329 vm_page_t m; 330 u_int pg_root_idx; 331 pte_t ***pdir_l1; 332 u_int pdir_l1_idx; 333 pte_t **pdir; 334 u_int pdir_idx; 335 336 pg_root_idx = PG_ROOT_IDX(va); 337 pdir_l1_idx = PDIR_L1_IDX(va); 338 pdir_idx = PDIR_IDX(va); 339 340 KASSERT((pmap != kernel_pmap), 341 ("ptbl_unhold: unholding kernel ptbl!")); 342 343 pdir_l1 = pmap->pm_root[pg_root_idx]; 344 pdir = pdir_l1[pdir_l1_idx]; 345 ptbl = pdir[pdir_idx]; 346 347 /* decrement hold count */ 348 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) ptbl)); 349 350 if (!unhold_free_page(pmap, m)) 351 return (0); 352 353 pdir[pdir_idx] = NULL; 354 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) pdir)); 355 356 if (!unhold_free_page(pmap, m)) 357 return (1); 358 359 pdir_l1[pdir_l1_idx] = NULL; 360 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) pdir_l1)); 361 362 if (!unhold_free_page(pmap, m)) 363 return (1); 364 pmap->pm_root[pg_root_idx] = NULL; 365 366 return (1); 367 } 368 369 /* 370 * Increment hold count for ptbl pages. This routine is used when new pte 371 * entry is being inserted into ptbl. 372 */ 373 static void 374 ptbl_hold(pmap_t pmap, pte_t *ptbl) 375 { 376 vm_page_t m; 377 378 KASSERT((pmap != kernel_pmap), 379 ("ptbl_hold: holding kernel ptbl!")); 380 381 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) ptbl)); 382 m->ref_count++; 383 } 384 385 /* 386 * Clean pte entry, try to free page table page if requested. 387 * 388 * Return 1 if ptbl pages were freed, otherwise return 0. 389 */ 390 static int 391 pte_remove(pmap_t pmap, vm_offset_t va, u_int8_t flags) 392 { 393 vm_page_t m; 394 pte_t *pte; 395 396 pte = pte_find(pmap, va); 397 KASSERT(pte != NULL, ("%s: NULL pte for va %#jx, pmap %p", 398 __func__, (uintmax_t)va, pmap)); 399 400 if (!PTE_ISVALID(pte)) 401 return (0); 402 403 /* Get vm_page_t for mapped pte. */ 404 m = PHYS_TO_VM_PAGE(PTE_PA(pte)); 405 406 if (PTE_ISWIRED(pte)) 407 pmap->pm_stats.wired_count--; 408 409 /* Handle managed entry. */ 410 if (PTE_ISMANAGED(pte)) { 411 412 /* Handle modified pages. */ 413 if (PTE_ISMODIFIED(pte)) 414 vm_page_dirty(m); 415 416 /* Referenced pages. */ 417 if (PTE_ISREFERENCED(pte)) 418 vm_page_aflag_set(m, PGA_REFERENCED); 419 420 /* Remove pv_entry from pv_list. */ 421 pv_remove(pmap, va, m); 422 } else if (pmap == kernel_pmap && m && m->md.pv_tracked) { 423 pv_remove(pmap, va, m); 424 if (TAILQ_EMPTY(&m->md.pv_list)) 425 m->md.pv_tracked = false; 426 } 427 mtx_lock_spin(&tlbivax_mutex); 428 tlb_miss_lock(); 429 430 tlb0_flush_entry(va); 431 *pte = 0; 432 433 tlb_miss_unlock(); 434 mtx_unlock_spin(&tlbivax_mutex); 435 436 pmap->pm_stats.resident_count--; 437 438 if (flags & PTBL_UNHOLD) { 439 return (ptbl_unhold(pmap, va)); 440 } 441 return (0); 442 } 443 444 /* 445 * Insert PTE for a given page and virtual address. 446 */ 447 static int 448 pte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags, 449 boolean_t nosleep) 450 { 451 unsigned int ptbl_idx = PTBL_IDX(va); 452 pte_t *ptbl, *pte, pte_tmp; 453 bool is_new; 454 455 /* Get the page directory pointer. */ 456 ptbl = ptbl_alloc(pmap, va, nosleep, &is_new); 457 if (ptbl == NULL) { 458 KASSERT(nosleep, ("nosleep and NULL ptbl")); 459 return (ENOMEM); 460 } 461 if (is_new) { 462 pte = &ptbl[ptbl_idx]; 463 } else { 464 /* 465 * Check if there is valid mapping for requested va, if there 466 * is, remove it. 467 */ 468 pte = &ptbl[ptbl_idx]; 469 if (PTE_ISVALID(pte)) { 470 pte_remove(pmap, va, PTBL_HOLD); 471 } else { 472 /* 473 * pte is not used, increment hold count for ptbl 474 * pages. 475 */ 476 if (pmap != kernel_pmap) 477 ptbl_hold(pmap, ptbl); 478 } 479 } 480 481 /* 482 * Insert pv_entry into pv_list for mapped page if part of managed 483 * memory. 484 */ 485 if ((m->oflags & VPO_UNMANAGED) == 0) { 486 flags |= PTE_MANAGED; 487 488 /* Create and insert pv entry. */ 489 pv_insert(pmap, va, m); 490 } 491 492 pmap->pm_stats.resident_count++; 493 494 pte_tmp = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m)); 495 pte_tmp |= (PTE_VALID | flags); 496 497 mtx_lock_spin(&tlbivax_mutex); 498 tlb_miss_lock(); 499 500 tlb0_flush_entry(va); 501 *pte = pte_tmp; 502 503 tlb_miss_unlock(); 504 mtx_unlock_spin(&tlbivax_mutex); 505 506 return (0); 507 } 508 509 /* Return the pa for the given pmap/va. */ 510 static vm_paddr_t 511 pte_vatopa(pmap_t pmap, vm_offset_t va) 512 { 513 vm_paddr_t pa = 0; 514 pte_t *pte; 515 516 pte = pte_find(pmap, va); 517 if ((pte != NULL) && PTE_ISVALID(pte)) 518 pa = (PTE_PA(pte) | (va & PTE_PA_MASK)); 519 return (pa); 520 } 521 522 523 /* allocate pte entries to manage (addr & mask) to (addr & mask) + size */ 524 static void 525 kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr) 526 { 527 pte_t *pte; 528 vm_size_t kva_size; 529 int kernel_pdirs, kernel_pgtbls, pdir_l1s; 530 vm_offset_t va, l1_va, pdir_va, ptbl_va; 531 int i, j, k; 532 533 kva_size = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS; 534 kernel_pmap->pm_root = kernel_ptbl_root; 535 pdir_l1s = howmany(kva_size, PG_ROOT_SIZE); 536 kernel_pdirs = howmany(kva_size, PDIR_L1_SIZE); 537 kernel_pgtbls = howmany(kva_size, PDIR_SIZE); 538 539 /* Initialize kernel pdir */ 540 l1_va = (vm_offset_t)kernel_ptbl_root + 541 round_page(PG_ROOT_NENTRIES * sizeof(pte_t ***)); 542 pdir_va = l1_va + pdir_l1s * PAGE_SIZE; 543 ptbl_va = pdir_va + kernel_pdirs * PAGE_SIZE; 544 if (bootverbose) { 545 printf("ptbl_root_va: %#lx\n", (vm_offset_t)kernel_ptbl_root); 546 printf("l1_va: %#lx (%d entries)\n", l1_va, pdir_l1s); 547 printf("pdir_va: %#lx(%d entries)\n", pdir_va, kernel_pdirs); 548 printf("ptbl_va: %#lx(%d entries)\n", ptbl_va, kernel_pgtbls); 549 } 550 551 va = VM_MIN_KERNEL_ADDRESS; 552 for (i = PG_ROOT_IDX(va); i < PG_ROOT_IDX(va) + pdir_l1s; 553 i++, l1_va += PAGE_SIZE) { 554 kernel_pmap->pm_root[i] = (pte_t ***)l1_va; 555 for (j = 0; 556 j < PDIR_L1_NENTRIES && va < VM_MAX_KERNEL_ADDRESS; 557 j++, pdir_va += PAGE_SIZE) { 558 kernel_pmap->pm_root[i][j] = (pte_t **)pdir_va; 559 for (k = 0; 560 k < PDIR_NENTRIES && va < VM_MAX_KERNEL_ADDRESS; 561 k++, va += PDIR_SIZE, ptbl_va += PAGE_SIZE) 562 kernel_pmap->pm_root[i][j][k] = (pte_t *)ptbl_va; 563 } 564 } 565 /* 566 * Fill in PTEs covering kernel code and data. They are not required 567 * for address translation, as this area is covered by static TLB1 568 * entries, but for pte_vatopa() to work correctly with kernel area 569 * addresses. 570 */ 571 for (va = addr; va < data_end; va += PAGE_SIZE) { 572 pte = &(kernel_pmap->pm_root[PG_ROOT_IDX(va)][PDIR_L1_IDX(va)][PDIR_IDX(va)][PTBL_IDX(va)]); 573 *pte = PTE_RPN_FROM_PA(kernload + (va - kernstart)); 574 *pte |= PTE_M | PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | 575 PTE_VALID | PTE_PS_4KB; 576 } 577 } 578 579 static vm_offset_t 580 mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end) 581 { 582 vm_size_t kva_size = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS; 583 kernel_ptbl_root = (pte_t ****)data_end; 584 585 data_end += round_page(PG_ROOT_NENTRIES * sizeof(pte_t ***)); 586 data_end += howmany(kva_size, PG_ROOT_SIZE) * PAGE_SIZE; 587 data_end += howmany(kva_size, PDIR_L1_SIZE) * PAGE_SIZE; 588 data_end += howmany(kva_size, PDIR_SIZE) * PAGE_SIZE; 589 590 return (data_end); 591 } 592 593 594 /* 595 * Initialize a preallocated and zeroed pmap structure, 596 * such as one in a vmspace structure. 597 */ 598 static int 599 mmu_booke_pinit(pmap_t pmap) 600 { 601 int i; 602 603 CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap, 604 curthread->td_proc->p_pid, curthread->td_proc->p_comm); 605 606 KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap")); 607 608 for (i = 0; i < MAXCPU; i++) 609 pmap->pm_tid[i] = TID_NONE; 610 CPU_ZERO(&kernel_pmap->pm_active); 611 bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); 612 pmap->pm_root = uma_zalloc(ptbl_root_zone, M_WAITOK); 613 bzero(pmap->pm_root, sizeof(pte_t **) * PG_ROOT_NENTRIES); 614 615 return (1); 616 } 617 618 /* 619 * Release any resources held by the given physical map. 620 * Called when a pmap initialized by mmu_booke_pinit is being released. 621 * Should only be called if the map contains no valid mappings. 622 */ 623 static void 624 mmu_booke_release(pmap_t pmap) 625 { 626 627 KASSERT(pmap->pm_stats.resident_count == 0, 628 ("pmap_release: pmap resident count %ld != 0", 629 pmap->pm_stats.resident_count)); 630 #ifdef INVARIANTS 631 /* 632 * Verify that all page directories are gone. 633 * Protects against reference count leakage. 634 */ 635 for (int i = 0; i < PG_ROOT_NENTRIES; i++) 636 KASSERT(pmap->pm_root[i] == 0, 637 ("Index %d on root page %p is non-zero!\n", i, pmap->pm_root)); 638 #endif 639 uma_zfree(ptbl_root_zone, pmap->pm_root); 640 } 641 642 static void 643 mmu_booke_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 644 { 645 pte_t *pte; 646 vm_paddr_t pa = 0; 647 int sync_sz, valid; 648 649 while (sz > 0) { 650 PMAP_LOCK(pm); 651 pte = pte_find(pm, va); 652 valid = (pte != NULL && PTE_ISVALID(pte)) ? 1 : 0; 653 if (valid) 654 pa = PTE_PA(pte); 655 PMAP_UNLOCK(pm); 656 sync_sz = PAGE_SIZE - (va & PAGE_MASK); 657 sync_sz = min(sync_sz, sz); 658 if (valid) { 659 pa += (va & PAGE_MASK); 660 __syncicache((void *)PHYS_TO_DMAP(pa), sync_sz); 661 } 662 va += sync_sz; 663 sz -= sync_sz; 664 } 665 } 666 667 /* 668 * mmu_booke_zero_page_area zeros the specified hardware page by 669 * mapping it into virtual memory and using bzero to clear 670 * its contents. 671 * 672 * off and size must reside within a single page. 673 */ 674 static void 675 mmu_booke_zero_page_area(vm_page_t m, int off, int size) 676 { 677 vm_offset_t va; 678 679 /* XXX KASSERT off and size are within a single page? */ 680 681 va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 682 bzero((caddr_t)va + off, size); 683 } 684 685 /* 686 * mmu_booke_zero_page zeros the specified hardware page. 687 */ 688 static void 689 mmu_booke_zero_page(vm_page_t m) 690 { 691 vm_offset_t off, va; 692 693 va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 694 695 for (off = 0; off < PAGE_SIZE; off += cacheline_size) 696 __asm __volatile("dcbz 0,%0" :: "r"(va + off)); 697 } 698 699 /* 700 * mmu_booke_copy_page copies the specified (machine independent) page by 701 * mapping the page into virtual memory and using memcopy to copy the page, 702 * one machine dependent page at a time. 703 */ 704 static void 705 mmu_booke_copy_page(vm_page_t sm, vm_page_t dm) 706 { 707 vm_offset_t sva, dva; 708 709 sva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(sm)); 710 dva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dm)); 711 memcpy((caddr_t)dva, (caddr_t)sva, PAGE_SIZE); 712 } 713 714 static inline void 715 mmu_booke_copy_pages(vm_page_t *ma, vm_offset_t a_offset, 716 vm_page_t *mb, vm_offset_t b_offset, int xfersize) 717 { 718 void *a_cp, *b_cp; 719 vm_offset_t a_pg_offset, b_pg_offset; 720 int cnt; 721 722 vm_page_t pa, pb; 723 724 while (xfersize > 0) { 725 a_pg_offset = a_offset & PAGE_MASK; 726 pa = ma[a_offset >> PAGE_SHIFT]; 727 b_pg_offset = b_offset & PAGE_MASK; 728 pb = mb[b_offset >> PAGE_SHIFT]; 729 cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 730 cnt = min(cnt, PAGE_SIZE - b_pg_offset); 731 a_cp = (caddr_t)((uintptr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pa)) + 732 a_pg_offset); 733 b_cp = (caddr_t)((uintptr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pb)) + 734 b_pg_offset); 735 bcopy(a_cp, b_cp, cnt); 736 a_offset += cnt; 737 b_offset += cnt; 738 xfersize -= cnt; 739 } 740 } 741 742 static vm_offset_t 743 mmu_booke_quick_enter_page(vm_page_t m) 744 { 745 return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m))); 746 } 747 748 static void 749 mmu_booke_quick_remove_page(vm_offset_t addr) 750 { 751 } 752 753 /**************************************************************************/ 754 /* TID handling */ 755 /**************************************************************************/ 756 757 /* 758 * Return the largest uint value log such that 2^log <= num. 759 */ 760 static unsigned long 761 ilog2(unsigned long num) 762 { 763 long lz; 764 765 __asm ("cntlzd %0, %1" : "=r" (lz) : "r" (num)); 766 return (63 - lz); 767 } 768 769 /* 770 * Invalidate all TLB0 entries which match the given TID. Note this is 771 * dedicated for cases when invalidations should NOT be propagated to other 772 * CPUs. 773 */ 774 static void 775 tid_flush(tlbtid_t tid) 776 { 777 register_t msr; 778 779 /* Don't evict kernel translations */ 780 if (tid == TID_KERNEL) 781 return; 782 783 msr = mfmsr(); 784 __asm __volatile("wrteei 0"); 785 786 /* 787 * Newer (e500mc and later) have tlbilx, which doesn't broadcast, so use 788 * it for PID invalidation. 789 */ 790 mtspr(SPR_MAS6, tid << MAS6_SPID0_SHIFT); 791 __asm __volatile("isync; .long 0x7c200024; isync; msync"); 792 793 __asm __volatile("wrtee %0" :: "r"(msr)); 794 } 795