1 /* 2 * Copyright (C) 2010 Andreas Tobler 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 22 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 23 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26 #include <sys/cdefs.h> 27 __FBSDID("$FreeBSD$"); 28 29 #include <sys/param.h> 30 #include <sys/kernel.h> 31 #include <sys/ktr.h> 32 #include <sys/lock.h> 33 #include <sys/msgbuf.h> 34 #include <sys/mutex.h> 35 #include <sys/proc.h> 36 #include <sys/sysctl.h> 37 #include <sys/systm.h> 38 #include <sys/vmmeter.h> 39 40 #include <dev/ofw/openfirm.h> 41 #include <machine/ofw_machdep.h> 42 43 #include <vm/vm.h> 44 #include <vm/vm_param.h> 45 #include <vm/vm_kern.h> 46 #include <vm/vm_page.h> 47 #include <vm/vm_map.h> 48 #include <vm/vm_object.h> 49 #include <vm/vm_extern.h> 50 #include <vm/vm_pageout.h> 51 #include <vm/uma.h> 52 53 #include <powerpc/aim/mmu_oea64.h> 54 55 #include "mmu_if.h" 56 #include "moea64_if.h" 57 58 #include "phyp-hvcall.h" 59 60 extern int n_slbs; 61 62 /* 63 * Kernel MMU interface 64 */ 65 66 static void mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, 67 vm_offset_t kernelend); 68 static void mphyp_cpu_bootstrap(mmu_t mmup, int ap); 69 static void mphyp_pte_synch(mmu_t, uintptr_t pt, struct lpte *pvo_pt); 70 static void mphyp_pte_clear(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 71 uint64_t vpn, u_int64_t ptebit); 72 static void mphyp_pte_unset(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 73 uint64_t vpn); 74 static void mphyp_pte_change(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 75 uint64_t vpn); 76 static int mphyp_pte_insert(mmu_t, u_int ptegidx, struct lpte *pvo_pt); 77 static uintptr_t mphyp_pvo_to_pte(mmu_t, const struct pvo_entry *pvo); 78 79 #define VSID_HASH_MASK 0x0000007fffffffffULL 80 81 82 static mmu_method_t mphyp_methods[] = { 83 MMUMETHOD(mmu_bootstrap, mphyp_bootstrap), 84 MMUMETHOD(mmu_cpu_bootstrap, mphyp_cpu_bootstrap), 85 86 MMUMETHOD(moea64_pte_synch, mphyp_pte_synch), 87 MMUMETHOD(moea64_pte_clear, mphyp_pte_clear), 88 MMUMETHOD(moea64_pte_unset, mphyp_pte_unset), 89 MMUMETHOD(moea64_pte_change, mphyp_pte_change), 90 MMUMETHOD(moea64_pte_insert, mphyp_pte_insert), 91 MMUMETHOD(moea64_pvo_to_pte, mphyp_pvo_to_pte), 92 93 { 0, 0 } 94 }; 95 96 MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, 0, oea64_mmu); 97 98 static void 99 mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 100 { 101 uint64_t final_pteg_count = 0; 102 char buf[8]; 103 uint32_t prop[2]; 104 uint32_t nptlp, shift = 0, slb_encoding = 0; 105 phandle_t dev, node, root; 106 int idx, len, res; 107 108 moea64_early_bootstrap(mmup, kernelstart, kernelend); 109 110 root = OF_peer(0); 111 112 dev = OF_child(root); 113 while (dev != 0) { 114 res = OF_getprop(dev, "name", buf, sizeof(buf)); 115 if (res > 0 && strcmp(buf, "cpus") == 0) 116 break; 117 dev = OF_peer(dev); 118 } 119 120 node = OF_child(dev); 121 122 while (node != 0) { 123 res = OF_getprop(node, "device_type", buf, sizeof(buf)); 124 if (res > 0 && strcmp(buf, "cpu") == 0) 125 break; 126 node = OF_peer(node); 127 } 128 129 res = OF_getprop(node, "ibm,pft-size", prop, sizeof(prop)); 130 if (res <= 0) 131 panic("mmu_phyp: unknown PFT size"); 132 final_pteg_count = 1 << prop[1]; 133 res = OF_getprop(node, "ibm,slb-size", prop, sizeof(prop[0])); 134 if (res > 0) 135 n_slbs = prop[0]; 136 137 moea64_pteg_count = final_pteg_count / sizeof(struct lpteg); 138 139 /* 140 * Scan the large page size property for PAPR compatible machines. 141 * See PAPR D.5 Changes to Section 5.1.4, 'CPU Node Properties' 142 * for the encoding of the property. 143 */ 144 145 len = OF_getproplen(node, "ibm,segment-page-sizes"); 146 if (len > 0) { 147 /* 148 * We have to use a variable length array on the stack 149 * since we have very limited stack space. 150 */ 151 cell_t arr[len/sizeof(cell_t)]; 152 res = OF_getprop(node, "ibm,segment-page-sizes", &arr, 153 sizeof(arr)); 154 len /= 4; 155 idx = 0; 156 while (len > 0) { 157 shift = arr[idx]; 158 slb_encoding = arr[idx + 1]; 159 nptlp = arr[idx + 2]; 160 idx += 3; 161 len -= 3; 162 while (len > 0 && nptlp) { 163 idx += 2; 164 len -= 2; 165 nptlp--; 166 } 167 } 168 169 /* For now we allow shift only to be <= 0x18. */ 170 if (shift >= 0x18) 171 shift = 0x18; 172 173 moea64_large_page_shift = shift; 174 moea64_large_page_size = 1ULL << shift; 175 } 176 177 moea64_mid_bootstrap(mmup, kernelstart, kernelend); 178 moea64_late_bootstrap(mmup, kernelstart, kernelend); 179 } 180 181 static void 182 mphyp_cpu_bootstrap(mmu_t mmup, int ap) 183 { 184 struct slb *slb = PCPU_GET(slb); 185 register_t seg0; 186 int i; 187 188 /* 189 * Install kernel SLB entries 190 */ 191 192 __asm __volatile ("slbia"); 193 __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); 194 for (i = 0; i < 64; i++) { 195 if (!(slb[i].slbe & SLBE_VALID)) 196 continue; 197 198 __asm __volatile ("slbmte %0, %1" :: 199 "r"(slb[i].slbv), "r"(slb[i].slbe)); 200 } 201 } 202 203 static void 204 mphyp_pte_synch(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt) 205 { 206 struct lpte pte; 207 uint64_t junk; 208 209 __asm __volatile("ptesync"); 210 phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pte.pte_hi, &pte.pte_lo, 211 &junk); 212 213 pvo_pt->pte_lo |= pte.pte_lo & (LPTE_CHG | LPTE_REF); 214 } 215 216 static void 217 mphyp_pte_clear(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn, 218 u_int64_t ptebit) 219 { 220 221 if (ptebit & LPTE_CHG) 222 phyp_hcall(H_CLEAR_MOD, 0, slot); 223 if (ptebit & LPTE_REF) 224 phyp_hcall(H_CLEAR_REF, 0, slot); 225 } 226 227 static void 228 mphyp_pte_unset(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 229 { 230 struct lpte pte; 231 uint64_t junk; 232 int err; 233 234 err = phyp_pft_hcall(H_REMOVE, 1UL << 31, slot, 235 pvo_pt->pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo, 236 &junk); 237 KASSERT(err == H_SUCCESS, ("Error removing page: %d", err)); 238 239 pvo_pt->pte_lo |= pte.pte_lo & (LPTE_CHG | LPTE_REF); 240 } 241 242 static void 243 mphyp_pte_change(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 244 { 245 struct lpte evicted; 246 uint64_t index, junk; 247 int64_t result; 248 249 /* 250 * NB: this is protected by the global table lock, so this two-step 251 * is safe, except for the scratch-page case. No CPUs on which we run 252 * this code should be using scratch pages. 253 */ 254 KASSERT(!(pvo_pt->pte_hi & LPTE_LOCKED), 255 ("Locked pages not supported on PHYP")); 256 257 /* XXX: optimization using H_PROTECT for common case? */ 258 mphyp_pte_unset(mmu, slot, pvo_pt, vpn); 259 result = phyp_pft_hcall(H_ENTER, H_EXACT, slot, pvo_pt->pte_hi, 260 pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 261 if (result != H_SUCCESS) 262 panic("mphyp_pte_change() insertion failure: %ld\n", result); 263 } 264 265 static __inline int 266 mphyp_pte_spillable_ident(u_int ptegidx, struct lpte *to_evict) 267 { 268 uint64_t slot, junk, k; 269 struct lpte pt; 270 int i, j; 271 272 /* Start at a random slot */ 273 i = mftb() % 8; 274 k = -1; 275 for (j = 0; j < 8; j++) { 276 slot = (ptegidx << 3) + (i + j) % 8; 277 phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi, &pt.pte_lo, 278 &junk); 279 280 if (pt.pte_hi & LPTE_SWBITS) 281 continue; 282 283 /* This is a candidate, so remember it */ 284 k = slot; 285 286 /* Try to get a page that has not been used lately */ 287 if (!(pt.pte_lo & LPTE_REF)) { 288 memcpy(to_evict, &pt, sizeof(struct lpte)); 289 return (k); 290 } 291 } 292 293 phyp_pft_hcall(H_READ, 0, slot, 0, 0, &to_evict->pte_hi, 294 &to_evict->pte_lo, &junk); 295 return (k); 296 } 297 298 static int 299 mphyp_pte_insert(mmu_t mmu, u_int ptegidx, struct lpte *pvo_pt) 300 { 301 int64_t result; 302 struct lpte evicted; 303 struct pvo_entry *pvo; 304 uint64_t index, junk; 305 u_int pteg_bktidx; 306 307 /* Check for locked pages, which we can't support on this system */ 308 KASSERT(!(pvo_pt->pte_hi & LPTE_LOCKED), 309 ("Locked pages not supported on PHYP")); 310 311 /* Initialize PTE */ 312 pvo_pt->pte_hi |= LPTE_VALID; 313 pvo_pt->pte_hi &= ~LPTE_HID; 314 evicted.pte_hi = 0; 315 316 /* 317 * First try primary hash. 318 */ 319 pteg_bktidx = ptegidx; 320 result = phyp_pft_hcall(H_ENTER, 0, pteg_bktidx << 3, pvo_pt->pte_hi, 321 pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 322 if (result == H_SUCCESS) 323 return (index & 0x07); 324 KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld " 325 "(ptegidx: %#x/%#x, PTE %#lx/%#lx", result, ptegidx, 326 moea64_pteg_count, pvo_pt->pte_hi, pvo_pt->pte_lo)); 327 328 /* 329 * Next try secondary hash. 330 */ 331 pteg_bktidx ^= moea64_pteg_mask; 332 pvo_pt->pte_hi |= LPTE_HID; 333 result = phyp_pft_hcall(H_ENTER, 0, pteg_bktidx << 3, 334 pvo_pt->pte_hi, pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 335 if (result == H_SUCCESS) 336 return (index & 0x07); 337 KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld", 338 result)); 339 340 /* 341 * Out of luck. Find a PTE to sacrifice. 342 */ 343 pteg_bktidx = ptegidx; 344 index = mphyp_pte_spillable_ident(pteg_bktidx, &evicted); 345 if (index == -1L) { 346 pteg_bktidx ^= moea64_pteg_mask; 347 index = mphyp_pte_spillable_ident(pteg_bktidx, &evicted); 348 } 349 350 if (index == -1L) { 351 /* No freeable slots in either PTEG? We're hosed. */ 352 panic("mphyp_pte_insert: overflow"); 353 return (-1); 354 } 355 356 if (pteg_bktidx == ptegidx) 357 pvo_pt->pte_hi &= ~LPTE_HID; 358 else 359 pvo_pt->pte_hi |= LPTE_HID; 360 361 /* 362 * Synchronize the sacrifice PTE with its PVO, then mark both 363 * invalid. The PVO will be reused when/if the VM system comes 364 * here after a fault. 365 */ 366 367 if (evicted.pte_hi & LPTE_HID) 368 pteg_bktidx ^= moea64_pteg_mask; /* PTEs indexed by primary */ 369 370 LIST_FOREACH(pvo, &moea64_pvo_table[pteg_bktidx], pvo_olink) { 371 if (pvo->pvo_pte.lpte.pte_hi == evicted.pte_hi) { 372 KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, 373 ("Invalid PVO for valid PTE!")); 374 mphyp_pte_unset(mmu, index, &pvo->pvo_pte.lpte, 375 pvo->pvo_vpn); 376 PVO_PTEGIDX_CLR(pvo); 377 moea64_pte_overflow++; 378 break; 379 } 380 } 381 382 KASSERT(pvo->pvo_pte.lpte.pte_hi == evicted.pte_hi, 383 ("Unable to find PVO for spilled PTE")); 384 385 /* 386 * Set the new PTE. 387 */ 388 result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pvo_pt->pte_hi, 389 pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 390 if (result == H_SUCCESS) 391 return (index & 0x07); 392 393 panic("Page replacement error: %ld", result); 394 return (-1); 395 } 396 397 static __inline u_int 398 va_to_pteg(uint64_t vsid, vm_offset_t addr, int large) 399 { 400 uint64_t hash; 401 int shift; 402 403 shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT; 404 hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >> 405 shift); 406 return (hash & moea64_pteg_mask); 407 } 408 409 static uintptr_t 410 mphyp_pvo_to_pte(mmu_t mmu, const struct pvo_entry *pvo) 411 { 412 uint64_t vsid; 413 u_int ptegidx; 414 415 /* If the PTEG index is not set, then there is no page table entry */ 416 if (!PVO_PTEGIDX_ISSET(pvo)) 417 return (-1); 418 419 vsid = PVO_VSID(pvo); 420 ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE); 421 422 /* 423 * We can find the actual pte entry without searching by grabbing 424 * the PTEG index from 3 unused bits in pvo_vaddr and by 425 * noticing the HID bit. 426 */ 427 if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID) 428 ptegidx ^= moea64_pteg_mask; 429 430 return ((ptegidx << 3) | PVO_PTEGIDX_GET(pvo)); 431 } 432 433