1 /*- 2 * Copyright (c) 2001 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by the NetBSD 19 * Foundation, Inc. and its contributors. 20 * 4. Neither the name of The NetBSD Foundation nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 /*- 37 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 38 * Copyright (C) 1995, 1996 TooLs GmbH. 39 * All rights reserved. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. All advertising materials mentioning features or use of this software 50 * must display the following acknowledgement: 51 * This product includes software developed by TooLs GmbH. 52 * 4. The name of TooLs GmbH may not be used to endorse or promote products 53 * derived from this software without specific prior written permission. 54 * 55 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 56 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 57 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 58 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 59 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 60 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 61 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 62 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 63 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 64 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 65 * 66 * $NetBSD: pmap.c,v 1.28 2000/03/26 20:42:36 kleink Exp $ 67 */ 68 /*- 69 * Copyright (C) 2001 Benno Rice. 70 * All rights reserved. 71 * 72 * Redistribution and use in source and binary forms, with or without 73 * modification, are permitted provided that the following conditions 74 * are met: 75 * 1. Redistributions of source code must retain the above copyright 76 * notice, this list of conditions and the following disclaimer. 77 * 2. Redistributions in binary form must reproduce the above copyright 78 * notice, this list of conditions and the following disclaimer in the 79 * documentation and/or other materials provided with the distribution. 80 * 81 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR 82 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 83 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 84 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 85 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 86 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 87 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 88 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 89 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 90 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 91 */ 92 93 #include <sys/cdefs.h> 94 __FBSDID("$FreeBSD$"); 95 96 /* 97 * Native 64-bit page table operations for running without a hypervisor. 98 */ 99 100 #include <sys/param.h> 101 #include <sys/kernel.h> 102 #include <sys/ktr.h> 103 #include <sys/lock.h> 104 #include <sys/mutex.h> 105 #include <sys/proc.h> 106 #include <sys/sched.h> 107 #include <sys/sysctl.h> 108 #include <sys/systm.h> 109 110 #include <sys/kdb.h> 111 112 #include <vm/vm.h> 113 #include <vm/vm_param.h> 114 #include <vm/vm_kern.h> 115 #include <vm/vm_page.h> 116 #include <vm/vm_map.h> 117 #include <vm/vm_object.h> 118 #include <vm/vm_extern.h> 119 #include <vm/vm_pageout.h> 120 #include <vm/vm_pager.h> 121 122 #include <machine/md_var.h> 123 #include <machine/mmuvar.h> 124 125 #include "mmu_oea64.h" 126 #include "mmu_if.h" 127 #include "moea64_if.h" 128 129 #define PTESYNC() __asm __volatile("ptesync"); 130 #define TLBSYNC() __asm __volatile("tlbsync; ptesync"); 131 #define SYNC() __asm __volatile("sync"); 132 #define EIEIO() __asm __volatile("eieio"); 133 134 #define VSID_HASH_MASK 0x0000007fffffffffULL 135 136 /* 137 * The tlbie instruction must be executed in 64-bit mode 138 * so we have to twiddle MSR[SF] around every invocation. 139 * Just to add to the fun, exceptions must be off as well 140 * so that we can't trap in 64-bit mode. What a pain. 141 */ 142 static struct mtx tlbie_mutex; 143 144 static __inline void 145 TLBIE(uint64_t vpn) { 146 #ifndef __powerpc64__ 147 register_t vpn_hi, vpn_lo; 148 register_t msr; 149 register_t scratch; 150 #endif 151 152 vpn <<= ADDR_PIDX_SHFT; 153 vpn &= ~(0xffffULL << 48); 154 155 #ifdef __powerpc64__ 156 sched_pin(); 157 __asm __volatile("ptesync"); 158 mtx_lock(&tlbie_mutex); 159 __asm __volatile("tlbie %0" :: "r"(vpn) : "memory"); 160 mtx_unlock(&tlbie_mutex); 161 __asm __volatile("eieio; tlbsync; ptesync"); 162 sched_unpin(); 163 #else 164 vpn_hi = (uint32_t)(vpn >> 32); 165 vpn_lo = (uint32_t)vpn; 166 167 /* Note: spin mutex is to disable exceptions while fiddling MSR */ 168 mtx_lock_spin(&tlbie_mutex); 169 __asm __volatile("\ 170 mfmsr %0; \ 171 mr %1, %0; \ 172 insrdi %1,%5,1,0; \ 173 mtmsrd %1; isync; \ 174 ptesync; \ 175 \ 176 sld %1,%2,%4; \ 177 or %1,%1,%3; \ 178 tlbie %1; \ 179 \ 180 mtmsrd %0; isync; \ 181 eieio; \ 182 tlbsync; \ 183 ptesync;" 184 : "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1) 185 : "memory"); 186 mtx_unlock_spin(&tlbie_mutex); 187 #endif 188 } 189 190 #define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR) 191 #define ENABLE_TRANS(msr) mtmsr(msr) 192 193 /* 194 * PTEG data. 195 */ 196 static struct lpteg *moea64_pteg_table; 197 198 /* 199 * PTE calls. 200 */ 201 static int moea64_pte_insert_native(mmu_t, u_int, struct lpte *); 202 static uintptr_t moea64_pvo_to_pte_native(mmu_t, const struct pvo_entry *); 203 static void moea64_pte_synch_native(mmu_t, uintptr_t pt, 204 struct lpte *pvo_pt); 205 static void moea64_pte_clear_native(mmu_t, uintptr_t pt, 206 struct lpte *pvo_pt, uint64_t vpn, uint64_t ptebit); 207 static void moea64_pte_change_native(mmu_t, uintptr_t pt, 208 struct lpte *pvo_pt, uint64_t vpn); 209 static void moea64_pte_unset_native(mmu_t mmu, uintptr_t pt, 210 struct lpte *pvo_pt, uint64_t vpn); 211 212 /* 213 * Utility routines. 214 */ 215 static void moea64_bootstrap_native(mmu_t mmup, 216 vm_offset_t kernelstart, vm_offset_t kernelend); 217 static void moea64_cpu_bootstrap_native(mmu_t, int ap); 218 static void tlbia(void); 219 220 static mmu_method_t moea64_native_methods[] = { 221 /* Internal interfaces */ 222 MMUMETHOD(mmu_bootstrap, moea64_bootstrap_native), 223 MMUMETHOD(mmu_cpu_bootstrap, moea64_cpu_bootstrap_native), 224 225 MMUMETHOD(moea64_pte_synch, moea64_pte_synch_native), 226 MMUMETHOD(moea64_pte_clear, moea64_pte_clear_native), 227 MMUMETHOD(moea64_pte_unset, moea64_pte_unset_native), 228 MMUMETHOD(moea64_pte_change, moea64_pte_change_native), 229 MMUMETHOD(moea64_pte_insert, moea64_pte_insert_native), 230 MMUMETHOD(moea64_pvo_to_pte, moea64_pvo_to_pte_native), 231 232 { 0, 0 } 233 }; 234 235 MMU_DEF_INHERIT(oea64_mmu_native, MMU_TYPE_G5, moea64_native_methods, 236 0, oea64_mmu); 237 238 static __inline u_int 239 va_to_pteg(uint64_t vsid, vm_offset_t addr, int large) 240 { 241 uint64_t hash; 242 int shift; 243 244 shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT; 245 hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >> 246 shift); 247 return (hash & moea64_pteg_mask); 248 } 249 250 static void 251 moea64_pte_synch_native(mmu_t mmu, uintptr_t pt_cookie, struct lpte *pvo_pt) 252 { 253 struct lpte *pt = (struct lpte *)pt_cookie; 254 255 pvo_pt->pte_lo |= pt->pte_lo & (LPTE_REF | LPTE_CHG); 256 } 257 258 static void 259 moea64_pte_clear_native(mmu_t mmu, uintptr_t pt_cookie, struct lpte *pvo_pt, 260 uint64_t vpn, uint64_t ptebit) 261 { 262 struct lpte *pt = (struct lpte *)pt_cookie; 263 264 /* 265 * As shown in Section 7.6.3.2.3 266 */ 267 pt->pte_lo &= ~ptebit; 268 TLBIE(vpn); 269 } 270 271 static void 272 moea64_pte_set_native(struct lpte *pt, struct lpte *pvo_pt) 273 { 274 275 pvo_pt->pte_hi |= LPTE_VALID; 276 277 /* 278 * Update the PTE as defined in section 7.6.3.1. 279 * Note that the REF/CHG bits are from pvo_pt and thus should have 280 * been saved so this routine can restore them (if desired). 281 */ 282 pt->pte_lo = pvo_pt->pte_lo; 283 EIEIO(); 284 pt->pte_hi = pvo_pt->pte_hi; 285 PTESYNC(); 286 287 /* Keep statistics for unlocked pages */ 288 if (!(pvo_pt->pte_hi & LPTE_LOCKED)) 289 moea64_pte_valid++; 290 } 291 292 static void 293 moea64_pte_unset_native(mmu_t mmu, uintptr_t pt_cookie, struct lpte *pvo_pt, 294 uint64_t vpn) 295 { 296 struct lpte *pt = (struct lpte *)pt_cookie; 297 298 pvo_pt->pte_hi &= ~LPTE_VALID; 299 300 /* Finish all pending operations */ 301 isync(); 302 303 /* 304 * Force the reg & chg bits back into the PTEs. 305 */ 306 SYNC(); 307 308 /* 309 * Invalidate the pte. 310 */ 311 pt->pte_hi &= ~LPTE_VALID; 312 TLBIE(vpn); 313 314 /* 315 * Save the reg & chg bits. 316 */ 317 moea64_pte_synch_native(mmu, pt_cookie, pvo_pt); 318 319 /* Keep statistics for unlocked pages */ 320 if (!(pvo_pt->pte_hi & LPTE_LOCKED)) 321 moea64_pte_valid--; 322 } 323 324 static void 325 moea64_pte_change_native(mmu_t mmu, uintptr_t pt, struct lpte *pvo_pt, 326 uint64_t vpn) 327 { 328 329 /* 330 * Invalidate the PTE 331 */ 332 moea64_pte_unset_native(mmu, pt, pvo_pt, vpn); 333 moea64_pte_set_native((struct lpte *)pt, pvo_pt); 334 } 335 336 static void 337 moea64_cpu_bootstrap_native(mmu_t mmup, int ap) 338 { 339 int i = 0; 340 #ifdef __powerpc64__ 341 struct slb *slb = PCPU_GET(slb); 342 register_t seg0; 343 #endif 344 345 /* 346 * Initialize segment registers and MMU 347 */ 348 349 mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); 350 351 /* 352 * Install kernel SLB entries 353 */ 354 355 #ifdef __powerpc64__ 356 __asm __volatile ("slbia"); 357 __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : 358 "r"(0)); 359 360 for (i = 0; i < 64; i++) { 361 if (!(slb[i].slbe & SLBE_VALID)) 362 continue; 363 364 __asm __volatile ("slbmte %0, %1" :: 365 "r"(slb[i].slbv), "r"(slb[i].slbe)); 366 } 367 #else 368 for (i = 0; i < 16; i++) 369 mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]); 370 #endif 371 372 /* 373 * Install page table 374 */ 375 376 __asm __volatile ("ptesync; mtsdr1 %0; isync" 377 :: "r"((uintptr_t)moea64_pteg_table 378 | (uintptr_t)(flsl(moea64_pteg_mask >> 11)))); 379 tlbia(); 380 } 381 382 static void 383 moea64_bootstrap_native(mmu_t mmup, vm_offset_t kernelstart, 384 vm_offset_t kernelend) 385 { 386 vm_size_t size; 387 vm_offset_t off; 388 vm_paddr_t pa; 389 register_t msr; 390 391 moea64_early_bootstrap(mmup, kernelstart, kernelend); 392 393 /* 394 * Allocate PTEG table. 395 */ 396 397 size = moea64_pteg_count * sizeof(struct lpteg); 398 CTR2(KTR_PMAP, "moea64_bootstrap: %d PTEGs, %d bytes", 399 moea64_pteg_count, size); 400 401 /* 402 * We now need to allocate memory. This memory, to be allocated, 403 * has to reside in a page table. The page table we are about to 404 * allocate. We don't have BAT. So drop to data real mode for a minute 405 * as a measure of last resort. We do this a couple times. 406 */ 407 408 moea64_pteg_table = (struct lpteg *)moea64_bootstrap_alloc(size, size); 409 DISABLE_TRANS(msr); 410 bzero((void *)moea64_pteg_table, moea64_pteg_count * sizeof(struct lpteg)); 411 ENABLE_TRANS(msr); 412 413 CTR1(KTR_PMAP, "moea64_bootstrap: PTEG table at %p", moea64_pteg_table); 414 415 /* 416 * Initialize the TLBIE lock. TLBIE can only be executed by one CPU. 417 */ 418 #ifdef __powerpc64__ 419 mtx_init(&tlbie_mutex, "tlbie", NULL, MTX_DEF); 420 #else 421 mtx_init(&tlbie_mutex, "tlbie", NULL, MTX_SPIN); 422 #endif 423 424 moea64_mid_bootstrap(mmup, kernelstart, kernelend); 425 426 /* 427 * Add a mapping for the page table itself if there is no direct map. 428 */ 429 if (!hw_direct_map) { 430 size = moea64_pteg_count * sizeof(struct lpteg); 431 off = (vm_offset_t)(moea64_pteg_table); 432 DISABLE_TRANS(msr); 433 for (pa = off; pa < off + size; pa += PAGE_SIZE) 434 pmap_kenter(pa, pa); 435 ENABLE_TRANS(msr); 436 } 437 438 /* Bring up virtual memory */ 439 moea64_late_bootstrap(mmup, kernelstart, kernelend); 440 } 441 442 static void 443 tlbia(void) 444 { 445 vm_offset_t i; 446 #ifndef __powerpc64__ 447 register_t msr, scratch; 448 #endif 449 450 TLBSYNC(); 451 452 for (i = 0; i < 0xFF000; i += 0x00001000) { 453 #ifdef __powerpc64__ 454 __asm __volatile("tlbiel %0" :: "r"(i)); 455 #else 456 __asm __volatile("\ 457 mfmsr %0; \ 458 mr %1, %0; \ 459 insrdi %1,%3,1,0; \ 460 mtmsrd %1; \ 461 isync; \ 462 \ 463 tlbiel %2; \ 464 \ 465 mtmsrd %0; \ 466 isync;" 467 : "=r"(msr), "=r"(scratch) : "r"(i), "r"(1)); 468 #endif 469 } 470 471 EIEIO(); 472 TLBSYNC(); 473 } 474 475 static uintptr_t 476 moea64_pvo_to_pte_native(mmu_t mmu, const struct pvo_entry *pvo) 477 { 478 struct lpte *pt; 479 int pteidx, ptegidx; 480 uint64_t vsid; 481 482 /* If the PTEG index is not set, then there is no page table entry */ 483 if (!PVO_PTEGIDX_ISSET(pvo)) 484 return (-1); 485 486 /* 487 * Calculate the ptegidx 488 */ 489 vsid = PVO_VSID(pvo); 490 ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), 491 pvo->pvo_vaddr & PVO_LARGE); 492 493 /* 494 * We can find the actual pte entry without searching by grabbing 495 * the PTEG index from 3 unused bits in pvo_vaddr and by 496 * noticing the HID bit. 497 */ 498 if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID) 499 ptegidx ^= moea64_pteg_mask; 500 501 pteidx = (ptegidx << 3) | PVO_PTEGIDX_GET(pvo); 502 503 if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) && 504 !PVO_PTEGIDX_ISSET(pvo)) { 505 panic("moea64_pvo_to_pte: pvo %p has valid pte in pvo but no " 506 "valid pte index", pvo); 507 } 508 509 if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0 && 510 PVO_PTEGIDX_ISSET(pvo)) { 511 panic("moea64_pvo_to_pte: pvo %p has valid pte index in pvo " 512 "pvo but no valid pte", pvo); 513 } 514 515 pt = &moea64_pteg_table[pteidx >> 3].pt[pteidx & 7]; 516 if ((pt->pte_hi ^ (pvo->pvo_pte.lpte.pte_hi & ~LPTE_VALID)) == 517 LPTE_VALID) { 518 if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0) { 519 panic("moea64_pvo_to_pte: pvo %p has valid pte in " 520 "moea64_pteg_table %p but invalid in pvo", pvo, pt); 521 } 522 523 if (((pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo) & 524 ~(LPTE_M|LPTE_CHG|LPTE_REF)) != 0) { 525 panic("moea64_pvo_to_pte: pvo %p pte does not match " 526 "pte %p in moea64_pteg_table difference is %#x", 527 pvo, pt, 528 (uint32_t)(pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo)); 529 } 530 531 return ((uintptr_t)pt); 532 } 533 534 if (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) { 535 panic("moea64_pvo_to_pte: pvo %p has invalid pte %p in " 536 "moea64_pteg_table but valid in pvo", pvo, pt); 537 } 538 539 return (-1); 540 } 541 542 static __inline int 543 moea64_pte_spillable_ident(u_int ptegidx) 544 { 545 struct lpte *pt; 546 int i, j, k; 547 548 /* Start at a random slot */ 549 i = mftb() % 8; 550 k = -1; 551 for (j = 0; j < 8; j++) { 552 pt = &moea64_pteg_table[ptegidx].pt[(i + j) % 8]; 553 if (pt->pte_hi & (LPTE_LOCKED | LPTE_WIRED)) 554 continue; 555 556 /* This is a candidate, so remember it */ 557 k = (i + j) % 8; 558 559 /* Try to get a page that has not been used lately */ 560 if (!(pt->pte_lo & LPTE_REF)) 561 return (k); 562 } 563 564 return (k); 565 } 566 567 static int 568 moea64_pte_insert_native(mmu_t mmu, u_int ptegidx, struct lpte *pvo_pt) 569 { 570 struct lpte *pt; 571 struct pvo_entry *pvo; 572 u_int pteg_bktidx; 573 int i; 574 575 /* 576 * First try primary hash. 577 */ 578 pteg_bktidx = ptegidx; 579 for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) { 580 if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) { 581 pvo_pt->pte_hi &= ~LPTE_HID; 582 moea64_pte_set_native(pt, pvo_pt); 583 return (i); 584 } 585 } 586 587 /* 588 * Now try secondary hash. 589 */ 590 pteg_bktidx ^= moea64_pteg_mask; 591 for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) { 592 if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) { 593 pvo_pt->pte_hi |= LPTE_HID; 594 moea64_pte_set_native(pt, pvo_pt); 595 return (i); 596 } 597 } 598 599 /* 600 * Out of luck. Find a PTE to sacrifice. 601 */ 602 pteg_bktidx = ptegidx; 603 i = moea64_pte_spillable_ident(pteg_bktidx); 604 if (i < 0) { 605 pteg_bktidx ^= moea64_pteg_mask; 606 i = moea64_pte_spillable_ident(pteg_bktidx); 607 } 608 609 if (i < 0) { 610 /* No freeable slots in either PTEG? We're hosed. */ 611 panic("moea64_pte_insert: overflow"); 612 return (-1); 613 } 614 615 if (pteg_bktidx == ptegidx) 616 pvo_pt->pte_hi &= ~LPTE_HID; 617 else 618 pvo_pt->pte_hi |= LPTE_HID; 619 620 /* 621 * Synchronize the sacrifice PTE with its PVO, then mark both 622 * invalid. The PVO will be reused when/if the VM system comes 623 * here after a fault. 624 */ 625 pt = &moea64_pteg_table[pteg_bktidx].pt[i]; 626 627 if (pt->pte_hi & LPTE_HID) 628 pteg_bktidx ^= moea64_pteg_mask; /* PTEs indexed by primary */ 629 630 LIST_FOREACH(pvo, &moea64_pvo_table[pteg_bktidx], pvo_olink) { 631 if (pvo->pvo_pte.lpte.pte_hi == pt->pte_hi) { 632 KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, 633 ("Invalid PVO for valid PTE!")); 634 moea64_pte_unset_native(mmu, (uintptr_t)pt, 635 &pvo->pvo_pte.lpte, pvo->pvo_vpn); 636 PVO_PTEGIDX_CLR(pvo); 637 moea64_pte_overflow++; 638 break; 639 } 640 } 641 642 KASSERT(pvo->pvo_pte.lpte.pte_hi == pt->pte_hi, 643 ("Unable to find PVO for spilled PTE")); 644 645 /* 646 * Set the new PTE. 647 */ 648 moea64_pte_set_native(pt, pvo_pt); 649 650 return (i); 651 } 652 653