1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD AND 4-Clause-BSD 3 * 4 * Copyright (c) 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 /*- 32 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 33 * Copyright (C) 1995, 1996 TooLs GmbH. 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed by TooLs GmbH. 47 * 4. The name of TooLs GmbH may not be used to endorse or promote products 48 * derived from this software without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 53 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 55 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 56 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 57 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 58 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 59 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 60 * 61 * $NetBSD: pmap.c,v 1.28 2000/03/26 20:42:36 kleink Exp $ 62 */ 63 /*- 64 * Copyright (C) 2001 Benno Rice. 65 * All rights reserved. 66 * 67 * Redistribution and use in source and binary forms, with or without 68 * modification, are permitted provided that the following conditions 69 * are met: 70 * 1. Redistributions of source code must retain the above copyright 71 * notice, this list of conditions and the following disclaimer. 72 * 2. Redistributions in binary form must reproduce the above copyright 73 * notice, this list of conditions and the following disclaimer in the 74 * documentation and/or other materials provided with the distribution. 75 * 76 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR 77 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 78 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 79 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 80 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 81 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 82 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 83 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 84 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 85 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 86 */ 87 88 #include <sys/cdefs.h> 89 __FBSDID("$FreeBSD$"); 90 91 /* 92 * Native 64-bit page table operations for running without a hypervisor. 93 */ 94 95 #include <sys/param.h> 96 #include <sys/kernel.h> 97 #include <sys/ktr.h> 98 #include <sys/lock.h> 99 #include <sys/mutex.h> 100 #include <sys/proc.h> 101 #include <sys/sched.h> 102 #include <sys/sysctl.h> 103 #include <sys/systm.h> 104 #include <sys/rwlock.h> 105 #include <sys/endian.h> 106 107 #include <sys/kdb.h> 108 109 #include <vm/vm.h> 110 #include <vm/vm_param.h> 111 #include <vm/vm_kern.h> 112 #include <vm/vm_page.h> 113 #include <vm/vm_map.h> 114 #include <vm/vm_object.h> 115 #include <vm/vm_extern.h> 116 #include <vm/vm_pageout.h> 117 118 #include <machine/cpu.h> 119 #include <machine/hid.h> 120 #include <machine/md_var.h> 121 #include <machine/mmuvar.h> 122 123 #include "mmu_oea64.h" 124 #include "mmu_if.h" 125 #include "moea64_if.h" 126 127 #define PTESYNC() __asm __volatile("ptesync"); 128 #define TLBSYNC() __asm __volatile("tlbsync; ptesync"); 129 #define SYNC() __asm __volatile("sync"); 130 #define EIEIO() __asm __volatile("eieio"); 131 132 #define VSID_HASH_MASK 0x0000007fffffffffULL 133 134 /* POWER9 only permits a 64k partition table size. */ 135 #define PART_SIZE 0x10000 136 137 static int moea64_crop_tlbie; 138 139 static __inline void 140 TLBIE(uint64_t vpn) { 141 #ifndef __powerpc64__ 142 register_t vpn_hi, vpn_lo; 143 register_t msr; 144 register_t scratch, intr; 145 #endif 146 147 static volatile u_int tlbie_lock = 0; 148 149 vpn <<= ADDR_PIDX_SHFT; 150 151 /* Hobo spinlock: we need stronger guarantees than mutexes provide */ 152 while (!atomic_cmpset_int(&tlbie_lock, 0, 1)); 153 isync(); /* Flush instruction queue once lock acquired */ 154 155 if (moea64_crop_tlbie) 156 vpn &= ~(0xffffULL << 48); 157 158 #ifdef __powerpc64__ 159 __asm __volatile("tlbie %0" :: "r"(vpn) : "memory"); 160 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); 161 #else 162 vpn_hi = (uint32_t)(vpn >> 32); 163 vpn_lo = (uint32_t)vpn; 164 165 intr = intr_disable(); 166 __asm __volatile("\ 167 mfmsr %0; \ 168 mr %1, %0; \ 169 insrdi %1,%5,1,0; \ 170 mtmsrd %1; isync; \ 171 \ 172 sld %1,%2,%4; \ 173 or %1,%1,%3; \ 174 tlbie %1; \ 175 \ 176 mtmsrd %0; isync; \ 177 eieio; \ 178 tlbsync; \ 179 ptesync;" 180 : "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1) 181 : "memory"); 182 intr_restore(intr); 183 #endif 184 185 /* No barriers or special ops -- taken care of by ptesync above */ 186 tlbie_lock = 0; 187 } 188 189 #define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR) 190 #define ENABLE_TRANS(msr) mtmsr(msr) 191 192 /* 193 * PTEG data. 194 */ 195 static volatile struct lpte *moea64_pteg_table; 196 static struct rwlock moea64_eviction_lock; 197 198 /* 199 * PTE calls. 200 */ 201 static int moea64_pte_insert_native(mmu_t, struct pvo_entry *); 202 static int64_t moea64_pte_synch_native(mmu_t, struct pvo_entry *); 203 static int64_t moea64_pte_clear_native(mmu_t, struct pvo_entry *, uint64_t); 204 static int64_t moea64_pte_replace_native(mmu_t, struct pvo_entry *, int); 205 static int64_t moea64_pte_unset_native(mmu_t mmu, struct pvo_entry *); 206 207 /* 208 * Utility routines. 209 */ 210 static void moea64_bootstrap_native(mmu_t mmup, 211 vm_offset_t kernelstart, vm_offset_t kernelend); 212 static void moea64_cpu_bootstrap_native(mmu_t, int ap); 213 static void tlbia(void); 214 215 static mmu_method_t moea64_native_methods[] = { 216 /* Internal interfaces */ 217 MMUMETHOD(mmu_bootstrap, moea64_bootstrap_native), 218 MMUMETHOD(mmu_cpu_bootstrap, moea64_cpu_bootstrap_native), 219 220 MMUMETHOD(moea64_pte_synch, moea64_pte_synch_native), 221 MMUMETHOD(moea64_pte_clear, moea64_pte_clear_native), 222 MMUMETHOD(moea64_pte_unset, moea64_pte_unset_native), 223 MMUMETHOD(moea64_pte_replace, moea64_pte_replace_native), 224 MMUMETHOD(moea64_pte_insert, moea64_pte_insert_native), 225 226 { 0, 0 } 227 }; 228 229 MMU_DEF_INHERIT(oea64_mmu_native, MMU_TYPE_G5, moea64_native_methods, 230 0, oea64_mmu); 231 232 static int64_t 233 moea64_pte_synch_native(mmu_t mmu, struct pvo_entry *pvo) 234 { 235 volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot; 236 struct lpte properpt; 237 uint64_t ptelo; 238 239 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 240 241 moea64_pte_from_pvo(pvo, &properpt); 242 243 rw_rlock(&moea64_eviction_lock); 244 if ((be64toh(pt->pte_hi) & LPTE_AVPN_MASK) != 245 (properpt.pte_hi & LPTE_AVPN_MASK)) { 246 /* Evicted */ 247 rw_runlock(&moea64_eviction_lock); 248 return (-1); 249 } 250 251 PTESYNC(); 252 ptelo = be64toh(pt->pte_lo); 253 254 rw_runlock(&moea64_eviction_lock); 255 256 return (ptelo & (LPTE_REF | LPTE_CHG)); 257 } 258 259 static int64_t 260 moea64_pte_clear_native(mmu_t mmu, struct pvo_entry *pvo, uint64_t ptebit) 261 { 262 volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot; 263 struct lpte properpt; 264 uint64_t ptelo; 265 266 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 267 268 moea64_pte_from_pvo(pvo, &properpt); 269 270 rw_rlock(&moea64_eviction_lock); 271 if ((be64toh(pt->pte_hi) & LPTE_AVPN_MASK) != 272 (properpt.pte_hi & LPTE_AVPN_MASK)) { 273 /* Evicted */ 274 rw_runlock(&moea64_eviction_lock); 275 return (-1); 276 } 277 278 if (ptebit == LPTE_REF) { 279 /* See "Resetting the Reference Bit" in arch manual */ 280 PTESYNC(); 281 /* 2-step here safe: precision is not guaranteed */ 282 ptelo = be64toh(pt->pte_lo); 283 284 /* One-byte store to avoid touching the C bit */ 285 ((volatile uint8_t *)(&pt->pte_lo))[6] = 286 #if BYTE_ORDER == BIG_ENDIAN 287 ((uint8_t *)(&properpt.pte_lo))[6]; 288 #else 289 ((uint8_t *)(&properpt.pte_lo))[1]; 290 #endif 291 rw_runlock(&moea64_eviction_lock); 292 293 critical_enter(); 294 TLBIE(pvo->pvo_vpn); 295 critical_exit(); 296 } else { 297 rw_runlock(&moea64_eviction_lock); 298 ptelo = moea64_pte_unset_native(mmu, pvo); 299 moea64_pte_insert_native(mmu, pvo); 300 } 301 302 return (ptelo & (LPTE_REF | LPTE_CHG)); 303 } 304 305 static int64_t 306 moea64_pte_unset_native(mmu_t mmu, struct pvo_entry *pvo) 307 { 308 volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot; 309 struct lpte properpt; 310 uint64_t ptelo; 311 312 moea64_pte_from_pvo(pvo, &properpt); 313 314 rw_rlock(&moea64_eviction_lock); 315 if ((be64toh(pt->pte_hi & LPTE_AVPN_MASK)) != 316 (properpt.pte_hi & LPTE_AVPN_MASK)) { 317 /* Evicted */ 318 moea64_pte_overflow--; 319 rw_runlock(&moea64_eviction_lock); 320 return (-1); 321 } 322 323 /* 324 * Invalidate the pte, briefly locking it to collect RC bits. No 325 * atomics needed since this is protected against eviction by the lock. 326 */ 327 isync(); 328 critical_enter(); 329 pt->pte_hi = be64toh((pt->pte_hi & ~LPTE_VALID) | LPTE_LOCKED); 330 PTESYNC(); 331 TLBIE(pvo->pvo_vpn); 332 ptelo = be64toh(pt->pte_lo); 333 *((volatile int32_t *)(&pt->pte_hi) + 1) = 0; /* Release lock */ 334 critical_exit(); 335 rw_runlock(&moea64_eviction_lock); 336 337 /* Keep statistics */ 338 moea64_pte_valid--; 339 340 return (ptelo & (LPTE_CHG | LPTE_REF)); 341 } 342 343 static int64_t 344 moea64_pte_replace_native(mmu_t mmu, struct pvo_entry *pvo, int flags) 345 { 346 volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot; 347 struct lpte properpt; 348 int64_t ptelo; 349 350 if (flags == 0) { 351 /* Just some software bits changing. */ 352 moea64_pte_from_pvo(pvo, &properpt); 353 354 rw_rlock(&moea64_eviction_lock); 355 if ((be64toh(pt->pte_hi) & LPTE_AVPN_MASK) != 356 (properpt.pte_hi & LPTE_AVPN_MASK)) { 357 rw_runlock(&moea64_eviction_lock); 358 return (-1); 359 } 360 pt->pte_hi = htobe64(properpt.pte_hi); 361 ptelo = be64toh(pt->pte_lo); 362 rw_runlock(&moea64_eviction_lock); 363 } else { 364 /* Otherwise, need reinsertion and deletion */ 365 ptelo = moea64_pte_unset_native(mmu, pvo); 366 moea64_pte_insert_native(mmu, pvo); 367 } 368 369 return (ptelo); 370 } 371 372 static void 373 moea64_cpu_bootstrap_native(mmu_t mmup, int ap) 374 { 375 int i = 0; 376 #ifdef __powerpc64__ 377 struct slb *slb = PCPU_GET(aim.slb); 378 register_t seg0; 379 #endif 380 381 /* 382 * Initialize segment registers and MMU 383 */ 384 385 mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); 386 387 /* 388 * Install kernel SLB entries 389 */ 390 391 #ifdef __powerpc64__ 392 __asm __volatile ("slbia"); 393 __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : 394 "r"(0)); 395 396 for (i = 0; i < n_slbs; i++) { 397 if (!(slb[i].slbe & SLBE_VALID)) 398 continue; 399 400 __asm __volatile ("slbmte %0, %1" :: 401 "r"(slb[i].slbv), "r"(slb[i].slbe)); 402 } 403 #else 404 for (i = 0; i < 16; i++) 405 mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]); 406 #endif 407 408 /* 409 * Install page table 410 */ 411 412 __asm __volatile ("ptesync; mtsdr1 %0; isync" 413 :: "r"(((uintptr_t)moea64_pteg_table & ~DMAP_BASE_ADDRESS) 414 | (uintptr_t)(flsl(moea64_pteg_mask >> 11)))); 415 tlbia(); 416 } 417 418 static void 419 moea64_bootstrap_native(mmu_t mmup, vm_offset_t kernelstart, 420 vm_offset_t kernelend) 421 { 422 vm_size_t size; 423 vm_offset_t off; 424 vm_paddr_t pa; 425 register_t msr; 426 427 moea64_early_bootstrap(mmup, kernelstart, kernelend); 428 429 switch (mfpvr() >> 16) { 430 case IBMPOWER4: 431 case IBMPOWER4PLUS: 432 case IBM970: 433 case IBM970FX: 434 case IBM970GX: 435 case IBM970MP: 436 moea64_crop_tlbie = true; 437 } 438 /* 439 * Allocate PTEG table. 440 */ 441 442 size = moea64_pteg_count * sizeof(struct lpteg); 443 CTR2(KTR_PMAP, "moea64_bootstrap: %lu PTEGs, %lu bytes", 444 moea64_pteg_count, size); 445 rw_init(&moea64_eviction_lock, "pte eviction"); 446 447 /* 448 * We now need to allocate memory. This memory, to be allocated, 449 * has to reside in a page table. The page table we are about to 450 * allocate. We don't have BAT. So drop to data real mode for a minute 451 * as a measure of last resort. We do this a couple times. 452 */ 453 /* 454 * PTEG table must be aligned on a 256k boundary, but can be placed 455 * anywhere with that alignment on POWER ISA 3+ systems. On earlier 456 * systems, offset addition is done by the CPU with bitwise OR rather 457 * than addition, so the table must also be aligned on a boundary of 458 * its own size. Pick the larger of the two, which works on all 459 * systems. 460 */ 461 moea64_pteg_table = (struct lpte *)moea64_bootstrap_alloc(size, 462 MAX(256*1024, size)); 463 if (hw_direct_map) 464 moea64_pteg_table = 465 (struct lpte *)PHYS_TO_DMAP((vm_offset_t)moea64_pteg_table); 466 DISABLE_TRANS(msr); 467 bzero(__DEVOLATILE(void *, moea64_pteg_table), moea64_pteg_count * 468 sizeof(struct lpteg)); 469 ENABLE_TRANS(msr); 470 471 CTR1(KTR_PMAP, "moea64_bootstrap: PTEG table at %p", moea64_pteg_table); 472 473 moea64_mid_bootstrap(mmup, kernelstart, kernelend); 474 475 /* 476 * Add a mapping for the page table itself if there is no direct map. 477 */ 478 if (!hw_direct_map) { 479 size = moea64_pteg_count * sizeof(struct lpteg); 480 off = (vm_offset_t)(moea64_pteg_table); 481 DISABLE_TRANS(msr); 482 for (pa = off; pa < off + size; pa += PAGE_SIZE) 483 pmap_kenter(pa, pa); 484 ENABLE_TRANS(msr); 485 } 486 487 /* Bring up virtual memory */ 488 moea64_late_bootstrap(mmup, kernelstart, kernelend); 489 } 490 491 static void 492 tlbia(void) 493 { 494 vm_offset_t i; 495 #ifndef __powerpc64__ 496 register_t msr, scratch; 497 #endif 498 499 i = 0xc00; /* IS = 11 */ 500 switch (mfpvr() >> 16) { 501 case IBM970: 502 case IBM970FX: 503 case IBM970MP: 504 case IBM970GX: 505 case IBMPOWER4: 506 case IBMPOWER4PLUS: 507 case IBMPOWER5: 508 case IBMPOWER5PLUS: 509 i = 0; /* IS not supported */ 510 break; 511 } 512 513 TLBSYNC(); 514 515 for (; i < 0x200000; i += 0x00001000) { 516 #ifdef __powerpc64__ 517 __asm __volatile("tlbiel %0" :: "r"(i)); 518 #else 519 __asm __volatile("\ 520 mfmsr %0; \ 521 mr %1, %0; \ 522 insrdi %1,%3,1,0; \ 523 mtmsrd %1; \ 524 isync; \ 525 \ 526 tlbiel %2; \ 527 \ 528 mtmsrd %0; \ 529 isync;" 530 : "=r"(msr), "=r"(scratch) : "r"(i), "r"(1)); 531 #endif 532 } 533 534 EIEIO(); 535 TLBSYNC(); 536 } 537 538 static int 539 atomic_pte_lock(volatile struct lpte *pte, uint64_t bitmask, uint64_t *oldhi) 540 { 541 int ret; 542 uint32_t oldhihalf; 543 544 /* 545 * Note: in principle, if just the locked bit were set here, we 546 * could avoid needing the eviction lock. However, eviction occurs 547 * so rarely that it isn't worth bothering about in practice. 548 */ 549 550 __asm __volatile ( 551 "1:\tlwarx %1, 0, %3\n\t" /* load old value */ 552 "and. %0,%1,%4\n\t" /* check if any bits set */ 553 "bne 2f\n\t" /* exit if any set */ 554 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 555 "bne- 1b\n\t" /* spin if failed */ 556 "li %0, 1\n\t" /* success - retval = 1 */ 557 "b 3f\n\t" /* we've succeeded */ 558 "2:\n\t" 559 "stwcx. %1, 0, %3\n\t" /* clear reservation (74xx) */ 560 "li %0, 0\n\t" /* failure - retval = 0 */ 561 "3:\n\t" 562 : "=&r" (ret), "=&r"(oldhihalf), "=m" (pte->pte_hi) 563 : "r" ((volatile char *)&pte->pte_hi + 4), 564 "r" ((uint32_t)bitmask), "r" ((uint32_t)LPTE_LOCKED), 565 "m" (pte->pte_hi) 566 : "cr0", "cr1", "cr2", "memory"); 567 568 *oldhi = (pte->pte_hi & 0xffffffff00000000ULL) | oldhihalf; 569 570 return (ret); 571 } 572 573 static uintptr_t 574 moea64_insert_to_pteg_native(struct lpte *pvo_pt, uintptr_t slotbase, 575 uint64_t mask) 576 { 577 volatile struct lpte *pt; 578 uint64_t oldptehi, va; 579 uintptr_t k; 580 int i, j; 581 582 /* Start at a random slot */ 583 i = mftb() % 8; 584 for (j = 0; j < 8; j++) { 585 k = slotbase + (i + j) % 8; 586 pt = &moea64_pteg_table[k]; 587 /* Invalidate and seize lock only if no bits in mask set */ 588 if (atomic_pte_lock(pt, mask, &oldptehi)) /* Lock obtained */ 589 break; 590 } 591 592 if (j == 8) 593 return (-1); 594 595 if (oldptehi & LPTE_VALID) { 596 KASSERT(!(oldptehi & LPTE_WIRED), ("Unmapped wired entry")); 597 /* 598 * Need to invalidate old entry completely: see 599 * "Modifying a Page Table Entry". Need to reconstruct 600 * the virtual address for the outgoing entry to do that. 601 */ 602 if (oldptehi & LPTE_BIG) 603 va = oldptehi >> moea64_large_page_shift; 604 else 605 va = oldptehi >> ADDR_PIDX_SHFT; 606 if (oldptehi & LPTE_HID) 607 va = (((k >> 3) ^ moea64_pteg_mask) ^ va) & 608 VSID_HASH_MASK; 609 else 610 va = ((k >> 3) ^ va) & VSID_HASH_MASK; 611 va |= (oldptehi & LPTE_AVPN_MASK) << 612 (ADDR_API_SHFT64 - ADDR_PIDX_SHFT); 613 PTESYNC(); 614 TLBIE(va); 615 moea64_pte_valid--; 616 moea64_pte_overflow++; 617 } 618 619 /* 620 * Update the PTE as per "Adding a Page Table Entry". Lock is released 621 * by setting the high doubleworld. 622 */ 623 pt->pte_lo = htobe64(pvo_pt->pte_lo); 624 EIEIO(); 625 pt->pte_hi = htobe64(pvo_pt->pte_hi); 626 PTESYNC(); 627 628 /* Keep statistics */ 629 moea64_pte_valid++; 630 631 return (k); 632 } 633 634 static int 635 moea64_pte_insert_native(mmu_t mmu, struct pvo_entry *pvo) 636 { 637 struct lpte insertpt; 638 uintptr_t slot; 639 640 /* Initialize PTE */ 641 moea64_pte_from_pvo(pvo, &insertpt); 642 643 /* Make sure further insertion is locked out during evictions */ 644 rw_rlock(&moea64_eviction_lock); 645 646 /* 647 * First try primary hash. 648 */ 649 pvo->pvo_pte.slot &= ~7ULL; /* Base slot address */ 650 slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot, 651 LPTE_VALID | LPTE_WIRED | LPTE_LOCKED); 652 if (slot != -1) { 653 rw_runlock(&moea64_eviction_lock); 654 pvo->pvo_pte.slot = slot; 655 return (0); 656 } 657 658 /* 659 * Now try secondary hash. 660 */ 661 pvo->pvo_vaddr ^= PVO_HID; 662 insertpt.pte_hi ^= LPTE_HID; 663 pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); 664 slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot, 665 LPTE_VALID | LPTE_WIRED | LPTE_LOCKED); 666 if (slot != -1) { 667 rw_runlock(&moea64_eviction_lock); 668 pvo->pvo_pte.slot = slot; 669 return (0); 670 } 671 672 /* 673 * Out of luck. Find a PTE to sacrifice. 674 */ 675 676 /* Lock out all insertions for a bit */ 677 if (!rw_try_upgrade(&moea64_eviction_lock)) { 678 rw_runlock(&moea64_eviction_lock); 679 rw_wlock(&moea64_eviction_lock); 680 } 681 682 slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot, 683 LPTE_WIRED | LPTE_LOCKED); 684 if (slot != -1) { 685 rw_wunlock(&moea64_eviction_lock); 686 pvo->pvo_pte.slot = slot; 687 return (0); 688 } 689 690 /* Try other hash table. Now we're getting desperate... */ 691 pvo->pvo_vaddr ^= PVO_HID; 692 insertpt.pte_hi ^= LPTE_HID; 693 pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); 694 slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot, 695 LPTE_WIRED | LPTE_LOCKED); 696 if (slot != -1) { 697 rw_wunlock(&moea64_eviction_lock); 698 pvo->pvo_pte.slot = slot; 699 return (0); 700 } 701 702 /* No freeable slots in either PTEG? We're hosed. */ 703 rw_wunlock(&moea64_eviction_lock); 704 panic("moea64_pte_insert: overflow"); 705 return (-1); 706 } 707 708