1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (C) 2010 Andreas Tobler 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 #include <sys/param.h> 30 #include <sys/kernel.h> 31 #include <sys/ktr.h> 32 #include <sys/lock.h> 33 #include <sys/rmlock.h> 34 #include <sys/mutex.h> 35 #include <sys/proc.h> 36 #include <sys/sysctl.h> 37 #include <sys/systm.h> 38 #include <sys/vmmeter.h> 39 40 #include <dev/ofw/openfirm.h> 41 #include <machine/ofw_machdep.h> 42 43 #include <vm/vm.h> 44 #include <vm/vm_param.h> 45 #include <vm/vm_kern.h> 46 #include <vm/vm_page.h> 47 #include <vm/vm_map.h> 48 #include <vm/vm_object.h> 49 #include <vm/vm_extern.h> 50 #include <vm/vm_pageout.h> 51 #include <vm/uma.h> 52 53 #include <powerpc/aim/mmu_oea64.h> 54 55 #include "phyp-hvcall.h" 56 57 #define MMU_PHYP_DEBUG 0 58 #define MMU_PHYP_ID "mmu_phyp: " 59 #if MMU_PHYP_DEBUG 60 #define dprintf(fmt, ...) printf(fmt, ## __VA_ARGS__) 61 #define dprintf0(fmt, ...) dprintf(MMU_PHYP_ID fmt, ## __VA_ARGS__) 62 #else 63 #define dprintf(fmt, args...) do { ; } while(0) 64 #define dprintf0(fmt, args...) do { ; } while(0) 65 #endif 66 67 static struct rmlock mphyp_eviction_lock; 68 69 /* 70 * Kernel MMU interface 71 */ 72 73 static void mphyp_install(void); 74 static void mphyp_bootstrap(vm_offset_t kernelstart, 75 vm_offset_t kernelend); 76 static void mphyp_cpu_bootstrap(int ap); 77 static void *mphyp_dump_pmap(void *ctx, void *buf, 78 u_long *nbytes); 79 static int64_t mphyp_pte_synch(struct pvo_entry *pvo); 80 static int64_t mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit); 81 static int64_t mphyp_pte_unset(struct pvo_entry *pvo); 82 static int64_t mphyp_pte_insert(struct pvo_entry *pvo); 83 static int64_t mphyp_pte_unset_sp(struct pvo_entry *pvo); 84 static int64_t mphyp_pte_insert_sp(struct pvo_entry *pvo); 85 static int64_t mphyp_pte_replace_sp(struct pvo_entry *pvo); 86 87 static struct pmap_funcs mphyp_methods = { 88 .install = mphyp_install, 89 .bootstrap = mphyp_bootstrap, 90 .cpu_bootstrap = mphyp_cpu_bootstrap, 91 .dumpsys_dump_pmap = mphyp_dump_pmap, 92 }; 93 94 static struct moea64_funcs mmu_phyp_funcs = { 95 .pte_synch = mphyp_pte_synch, 96 .pte_clear = mphyp_pte_clear, 97 .pte_unset = mphyp_pte_unset, 98 .pte_insert = mphyp_pte_insert, 99 .pte_unset_sp = mphyp_pte_unset_sp, 100 .pte_insert_sp = mphyp_pte_insert_sp, 101 .pte_replace_sp = mphyp_pte_replace_sp, 102 }; 103 104 MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, oea64_mmu); 105 106 static int brokenkvm = 0; 107 static uint64_t final_pteg_count = 0; 108 109 static void 110 print_kvm_bug_warning(void *data) 111 { 112 113 if (brokenkvm) 114 printf("WARNING: Running on a broken hypervisor that does " 115 "not support mandatory H_CLEAR_MOD and H_CLEAR_REF " 116 "hypercalls. Performance will be suboptimal.\n"); 117 } 118 119 SYSINIT(kvmbugwarn1, SI_SUB_COPYRIGHT, SI_ORDER_THIRD + 1, 120 print_kvm_bug_warning, NULL); 121 SYSINIT(kvmbugwarn2, SI_SUB_LAST, SI_ORDER_THIRD + 1, print_kvm_bug_warning, 122 NULL); 123 124 static void 125 mphyp_install(void) 126 { 127 char buf[8]; 128 uint32_t prop[2]; 129 uint32_t nptlp, shift = 0, slb_encoding = 0; 130 uint32_t lp_size, lp_encoding; 131 phandle_t dev, node, root; 132 int idx, len, res; 133 bool has_lp; 134 135 root = OF_peer(0); 136 137 dev = OF_child(root); 138 while (dev != 0) { 139 res = OF_getprop(dev, "name", buf, sizeof(buf)); 140 if (res > 0 && strcmp(buf, "cpus") == 0) 141 break; 142 dev = OF_peer(dev); 143 } 144 145 node = OF_child(dev); 146 147 while (node != 0) { 148 res = OF_getprop(node, "device_type", buf, sizeof(buf)); 149 if (res > 0 && strcmp(buf, "cpu") == 0) 150 break; 151 node = OF_peer(node); 152 } 153 154 res = OF_getencprop(node, "ibm,pft-size", prop, sizeof(prop)); 155 if (res <= 0) 156 panic("mmu_phyp: unknown PFT size"); 157 final_pteg_count = 1 << prop[1]; 158 res = OF_getencprop(node, "ibm,slb-size", prop, sizeof(prop[0])); 159 if (res > 0) 160 n_slbs = prop[0]; 161 dprintf0("slb-size=%i\n", n_slbs); 162 163 /* 164 * Scan the large page size property for PAPR compatible machines. 165 * See PAPR D.5 Changes to Section 5.1.4, 'CPU Node Properties' 166 * for the encoding of the property. 167 */ 168 169 len = OF_getproplen(node, "ibm,segment-page-sizes"); 170 if (len > 0) { 171 /* 172 * We have to use a variable length array on the stack 173 * since we have very limited stack space. 174 */ 175 pcell_t arr[len/sizeof(cell_t)]; 176 res = OF_getencprop(node, "ibm,segment-page-sizes", arr, 177 sizeof(arr)); 178 len /= 4; 179 idx = 0; 180 has_lp = false; 181 while (len > 0) { 182 shift = arr[idx]; 183 slb_encoding = arr[idx + 1]; 184 nptlp = arr[idx + 2]; 185 186 dprintf0("Segment Page Size: " 187 "%uKB, slb_enc=0x%X: {size, encoding}[%u] =", 188 shift > 10? 1 << (shift-10) : 0, 189 slb_encoding, nptlp); 190 191 idx += 3; 192 len -= 3; 193 while (len > 0 && nptlp) { 194 lp_size = arr[idx]; 195 lp_encoding = arr[idx+1]; 196 197 dprintf(" {%uKB, 0x%X}", 198 lp_size > 10? 1 << (lp_size-10) : 0, 199 lp_encoding); 200 201 if (slb_encoding == SLBV_L && lp_encoding == 0) 202 has_lp = true; 203 204 if (slb_encoding == SLB_PGSZ_4K_4K && 205 lp_encoding == LP_4K_16M) 206 moea64_has_lp_4k_16m = true; 207 208 idx += 2; 209 len -= 2; 210 nptlp--; 211 } 212 dprintf("\n"); 213 if (has_lp && moea64_has_lp_4k_16m) 214 break; 215 } 216 217 if (has_lp) { 218 moea64_large_page_shift = shift; 219 moea64_large_page_size = 1ULL << lp_size; 220 moea64_large_page_mask = moea64_large_page_size - 1; 221 hw_direct_map = 1; 222 printf(MMU_PHYP_ID 223 "Support for hugepages of %uKB detected\n", 224 moea64_large_page_shift > 10? 225 1 << (moea64_large_page_shift-10) : 0); 226 } else { 227 moea64_large_page_size = 0; 228 moea64_large_page_shift = 0; 229 moea64_large_page_mask = 0; 230 hw_direct_map = 0; 231 printf(MMU_PHYP_ID 232 "Support for hugepages not found\n"); 233 } 234 } 235 236 moea64_ops = &mmu_phyp_funcs; 237 238 moea64_install(); 239 } 240 241 static void 242 mphyp_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend) 243 { 244 struct lpte old; 245 uint64_t vsid; 246 int idx; 247 248 rm_init(&mphyp_eviction_lock, "pte eviction"); 249 250 moea64_early_bootstrap(kernelstart, kernelend); 251 252 moea64_pteg_count = final_pteg_count / sizeof(struct lpteg); 253 254 /* Clear any old page table entries */ 255 for (idx = 0; idx < moea64_pteg_count*8; idx++) { 256 phyp_pft_hcall(H_READ, 0, idx, 0, 0, &old.pte_hi, 257 &old.pte_lo, &old.pte_lo); 258 vsid = (old.pte_hi << (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) >> 28; 259 if (vsid == VSID_VRMA || vsid == 0 /* Older VRMA */) 260 continue; 261 262 if (old.pte_hi & LPTE_VALID) 263 phyp_hcall(H_REMOVE, 0, idx, 0); 264 } 265 266 moea64_mid_bootstrap(kernelstart, kernelend); 267 moea64_late_bootstrap(kernelstart, kernelend); 268 269 /* Test for broken versions of KVM that don't conform to the spec */ 270 if (phyp_hcall(H_CLEAR_MOD, 0, 0) == H_FUNCTION) 271 brokenkvm = 1; 272 } 273 274 static void 275 mphyp_cpu_bootstrap(int ap) 276 { 277 struct slb *slb = PCPU_GET(aim.slb); 278 register_t seg0; 279 int i; 280 281 /* 282 * Install kernel SLB entries 283 */ 284 285 __asm __volatile ("slbia"); 286 __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); 287 for (i = 0; i < 64; i++) { 288 if (!(slb[i].slbe & SLBE_VALID)) 289 continue; 290 291 __asm __volatile ("slbmte %0, %1" :: 292 "r"(slb[i].slbv), "r"(slb[i].slbe)); 293 } 294 } 295 296 static int64_t 297 mphyp_pte_synch(struct pvo_entry *pvo) 298 { 299 struct lpte pte; 300 uint64_t junk; 301 302 __asm __volatile("ptesync"); 303 phyp_pft_hcall(H_READ, 0, pvo->pvo_pte.slot, 0, 0, &pte.pte_hi, 304 &pte.pte_lo, &junk); 305 if ((pte.pte_hi & LPTE_AVPN_MASK) != 306 ((pvo->pvo_vpn >> (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) & 307 LPTE_AVPN_MASK)) 308 return (-1); 309 if (!(pte.pte_hi & LPTE_VALID)) 310 return (-1); 311 312 return (pte.pte_lo & (LPTE_CHG | LPTE_REF)); 313 } 314 315 static int64_t 316 mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit) 317 { 318 struct rm_priotracker track; 319 int64_t refchg; 320 uint64_t ptelo, junk; 321 int err __diagused; 322 323 /* 324 * This involves two steps (synch and clear) so we need the entry 325 * not to change in the middle. We are protected against deliberate 326 * unset by virtue of holding the pmap lock. Protection against 327 * incidental unset (page table eviction) comes from holding the 328 * shared eviction lock. 329 */ 330 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 331 rm_rlock(&mphyp_eviction_lock, &track); 332 333 refchg = mphyp_pte_synch(pvo); 334 if (refchg < 0) { 335 rm_runlock(&mphyp_eviction_lock, &track); 336 return (refchg); 337 } 338 339 if (brokenkvm) { 340 /* 341 * No way to clear either bit, which is total madness. 342 * Pessimistically claim that, once modified, it stays so 343 * forever and that it is never referenced. 344 */ 345 rm_runlock(&mphyp_eviction_lock, &track); 346 return (refchg & ~LPTE_REF); 347 } 348 349 if (ptebit & LPTE_CHG) { 350 err = phyp_pft_hcall(H_CLEAR_MOD, 0, pvo->pvo_pte.slot, 0, 0, 351 &ptelo, &junk, &junk); 352 KASSERT(err == H_SUCCESS, 353 ("Error clearing page change bit: %d", err)); 354 refchg |= (ptelo & LPTE_CHG); 355 } 356 if (ptebit & LPTE_REF) { 357 err = phyp_pft_hcall(H_CLEAR_REF, 0, pvo->pvo_pte.slot, 0, 0, 358 &ptelo, &junk, &junk); 359 KASSERT(err == H_SUCCESS, 360 ("Error clearing page reference bit: %d", err)); 361 refchg |= (ptelo & LPTE_REF); 362 } 363 364 rm_runlock(&mphyp_eviction_lock, &track); 365 366 return (refchg); 367 } 368 369 static int64_t 370 mphyp_pte_unset(struct pvo_entry *pvo) 371 { 372 struct lpte pte; 373 uint64_t junk; 374 int err; 375 376 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 377 378 moea64_pte_from_pvo(pvo, &pte); 379 380 err = phyp_pft_hcall(H_REMOVE, H_AVPN, pvo->pvo_pte.slot, 381 pte.pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo, 382 &junk); 383 KASSERT(err == H_SUCCESS || err == H_NOT_FOUND, 384 ("Error removing page: %d", err)); 385 386 if (err == H_NOT_FOUND) { 387 STAT_MOEA64(moea64_pte_overflow--); 388 return (-1); 389 } 390 391 return (pte.pte_lo & (LPTE_REF | LPTE_CHG)); 392 } 393 394 static uintptr_t 395 mphyp_pte_spillable_ident(uintptr_t ptegbase, struct lpte *to_evict) 396 { 397 uint64_t slot, junk, k; 398 struct lpte pt; 399 int i, j; 400 401 /* Start at a random slot */ 402 i = mftb() % 8; 403 k = -1; 404 for (j = 0; j < 8; j++) { 405 slot = ptegbase + (i + j) % 8; 406 phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi, 407 &pt.pte_lo, &junk); 408 409 if ((pt.pte_hi & (LPTE_WIRED | LPTE_BIG)) != 0) 410 continue; 411 412 /* This is a candidate, so remember it */ 413 k = slot; 414 415 /* Try to get a page that has not been used lately */ 416 if (!(pt.pte_hi & LPTE_VALID) || !(pt.pte_lo & LPTE_REF)) { 417 memcpy(to_evict, &pt, sizeof(struct lpte)); 418 return (k); 419 } 420 } 421 422 if (k == -1) 423 return (k); 424 425 phyp_pft_hcall(H_READ, 0, k, 0, 0, &to_evict->pte_hi, 426 &to_evict->pte_lo, &junk); 427 return (k); 428 } 429 430 static __inline int64_t 431 mphyp_pte_insert_locked(struct pvo_entry *pvo, struct lpte *pte) 432 { 433 struct lpte evicted; 434 uint64_t index, junk; 435 int64_t result; 436 437 /* 438 * First try primary hash. 439 */ 440 pvo->pvo_pte.slot &= ~7UL; /* Base slot address */ 441 result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte->pte_hi, 442 pte->pte_lo, &index, &evicted.pte_lo, &junk); 443 if (result == H_SUCCESS) { 444 pvo->pvo_pte.slot = index; 445 return (0); 446 } 447 KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld " 448 "(ptegidx: %#zx/%#lx, PTE %#lx/%#lx", result, pvo->pvo_pte.slot, 449 moea64_pteg_count, pte->pte_hi, pte->pte_lo)); 450 451 /* 452 * Next try secondary hash. 453 */ 454 pvo->pvo_vaddr ^= PVO_HID; 455 pte->pte_hi ^= LPTE_HID; 456 pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); 457 458 result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, 459 pte->pte_hi, pte->pte_lo, &index, &evicted.pte_lo, &junk); 460 if (result == H_SUCCESS) { 461 pvo->pvo_pte.slot = index; 462 return (0); 463 } 464 KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld", 465 result)); 466 467 return (-1); 468 } 469 470 471 static __inline int64_t 472 mphyp_pte_evict_and_insert_locked(struct pvo_entry *pvo, struct lpte *pte) 473 { 474 struct lpte evicted; 475 uint64_t index, junk, lastptelo; 476 int64_t result; 477 478 evicted.pte_hi = 0; 479 480 index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); 481 if (index == -1L) { 482 /* Try other hash table? */ 483 pvo->pvo_vaddr ^= PVO_HID; 484 pte->pte_hi ^= LPTE_HID; 485 pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); 486 index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); 487 } 488 489 if (index == -1L) { 490 /* No freeable slots in either PTEG? We're hosed. */ 491 rm_wunlock(&mphyp_eviction_lock); 492 panic("mphyp_pte_insert: overflow"); 493 return (-1); 494 } 495 496 /* Victim acquired: update page before waving goodbye */ 497 if (evicted.pte_hi & LPTE_VALID) { 498 result = phyp_pft_hcall(H_REMOVE, H_AVPN, index, 499 evicted.pte_hi & LPTE_AVPN_MASK, 0, &junk, &lastptelo, 500 &junk); 501 STAT_MOEA64(moea64_pte_overflow++); 502 KASSERT(result == H_SUCCESS || result == H_NOT_FOUND, 503 ("Error evicting page: %d", (int)result)); 504 } 505 506 /* 507 * Set the new PTE. 508 */ 509 result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pte->pte_hi, 510 pte->pte_lo, &index, &evicted.pte_lo, &junk); 511 512 pvo->pvo_pte.slot = index; 513 if (result == H_SUCCESS) 514 return (0); 515 516 rm_wunlock(&mphyp_eviction_lock); 517 panic("Page replacement error: %ld", result); 518 return (result); 519 } 520 521 static int64_t 522 mphyp_pte_insert(struct pvo_entry *pvo) 523 { 524 struct rm_priotracker track; 525 int64_t ret; 526 struct lpte pte; 527 528 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 529 530 /* Initialize PTE */ 531 moea64_pte_from_pvo(pvo, &pte); 532 533 /* Make sure further insertion is locked out during evictions */ 534 rm_rlock(&mphyp_eviction_lock, &track); 535 536 ret = mphyp_pte_insert_locked(pvo, &pte); 537 rm_runlock(&mphyp_eviction_lock, &track); 538 539 if (ret == -1) { 540 /* 541 * Out of luck. Find a PTE to sacrifice. 542 */ 543 544 /* Lock out all insertions for a bit */ 545 rm_wlock(&mphyp_eviction_lock); 546 ret = mphyp_pte_evict_and_insert_locked(pvo, &pte); 547 rm_wunlock(&mphyp_eviction_lock); /* All clear */ 548 } 549 550 return (ret); 551 } 552 553 static void * 554 mphyp_dump_pmap(void *ctx, void *buf, u_long *nbytes) 555 { 556 struct dump_context *dctx; 557 struct lpte p, *pbuf; 558 int bufidx; 559 uint64_t junk; 560 u_long ptex, ptex_end; 561 562 dctx = (struct dump_context *)ctx; 563 pbuf = (struct lpte *)buf; 564 bufidx = 0; 565 ptex = dctx->ptex; 566 ptex_end = ptex + dctx->blksz / sizeof(struct lpte); 567 ptex_end = MIN(ptex_end, dctx->ptex_end); 568 *nbytes = (ptex_end - ptex) * sizeof(struct lpte); 569 570 if (*nbytes == 0) 571 return (NULL); 572 573 for (; ptex < ptex_end; ptex++) { 574 phyp_pft_hcall(H_READ, 0, ptex, 0, 0, 575 &p.pte_hi, &p.pte_lo, &junk); 576 pbuf[bufidx++] = p; 577 } 578 579 dctx->ptex = ptex; 580 return (buf); 581 } 582 583 static int64_t 584 mphyp_pte_unset_sp(struct pvo_entry *pvo) 585 { 586 struct lpte pte; 587 uint64_t junk, refchg; 588 int err; 589 vm_offset_t eva; 590 pmap_t pm __diagused; 591 592 pm = pvo->pvo_pmap; 593 PMAP_LOCK_ASSERT(pm, MA_OWNED); 594 KASSERT((PVO_VADDR(pvo) & HPT_SP_MASK) == 0, 595 ("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo))); 596 597 refchg = 0; 598 eva = PVO_VADDR(pvo) + HPT_SP_SIZE; 599 600 for (; pvo != NULL && PVO_VADDR(pvo) < eva; 601 pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) { 602 moea64_pte_from_pvo(pvo, &pte); 603 604 err = phyp_pft_hcall(H_REMOVE, H_AVPN, pvo->pvo_pte.slot, 605 pte.pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo, 606 &junk); 607 KASSERT(err == H_SUCCESS || err == H_NOT_FOUND, 608 ("Error removing page: %d", err)); 609 610 if (err == H_NOT_FOUND) 611 STAT_MOEA64(moea64_pte_overflow--); 612 refchg |= pte.pte_lo & (LPTE_REF | LPTE_CHG); 613 } 614 615 return (refchg); 616 } 617 618 static int64_t 619 mphyp_pte_insert_sp(struct pvo_entry *pvo) 620 { 621 struct rm_priotracker track; 622 int64_t ret; 623 struct lpte pte; 624 vm_offset_t eva; 625 pmap_t pm __diagused; 626 627 pm = pvo->pvo_pmap; 628 PMAP_LOCK_ASSERT(pm, MA_OWNED); 629 KASSERT((PVO_VADDR(pvo) & HPT_SP_MASK) == 0, 630 ("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo))); 631 632 eva = PVO_VADDR(pvo) + HPT_SP_SIZE; 633 634 /* Make sure further insertion is locked out during evictions */ 635 rm_rlock(&mphyp_eviction_lock, &track); 636 637 for (; pvo != NULL && PVO_VADDR(pvo) < eva; 638 pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) { 639 /* Initialize PTE */ 640 moea64_pte_from_pvo(pvo, &pte); 641 642 ret = mphyp_pte_insert_locked(pvo, &pte); 643 if (ret == -1) { 644 /* 645 * Out of luck. Find a PTE to sacrifice. 646 */ 647 648 /* Lock out all insertions for a bit */ 649 rm_runlock(&mphyp_eviction_lock, &track); 650 rm_wlock(&mphyp_eviction_lock); 651 mphyp_pte_evict_and_insert_locked(pvo, &pte); 652 rm_wunlock(&mphyp_eviction_lock); /* All clear */ 653 rm_rlock(&mphyp_eviction_lock, &track); 654 } 655 } 656 657 rm_runlock(&mphyp_eviction_lock, &track); 658 return (0); 659 } 660 661 static int64_t 662 mphyp_pte_replace_sp(struct pvo_entry *pvo) 663 { 664 int64_t refchg; 665 666 refchg = mphyp_pte_unset_sp(pvo); 667 mphyp_pte_insert_sp(pvo); 668 return (refchg); 669 } 670