1 /*- 2 * Copyright (C) 2010 Nathan Whitehorn 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 22 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 23 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26 #include <sys/cdefs.h> 27 __FBSDID("$FreeBSD$"); 28 29 #include <sys/param.h> 30 #include <sys/kernel.h> 31 #include <sys/ktr.h> 32 #include <sys/lock.h> 33 #include <sys/msgbuf.h> 34 #include <sys/mutex.h> 35 #include <sys/proc.h> 36 #include <sys/sysctl.h> 37 #include <sys/systm.h> 38 #include <sys/vmmeter.h> 39 40 #include <vm/vm.h> 41 #include <vm/vm_param.h> 42 #include <vm/vm_kern.h> 43 #include <vm/vm_page.h> 44 #include <vm/vm_map.h> 45 #include <vm/vm_object.h> 46 #include <vm/vm_extern.h> 47 #include <vm/vm_pageout.h> 48 #include <vm/vm_pager.h> 49 #include <vm/uma.h> 50 51 #include <powerpc/aim/mmu_oea64.h> 52 53 #include "mmu_if.h" 54 #include "moea64_if.h" 55 #include "ps3-hvcall.h" 56 57 #define VSID_HASH_MASK 0x0000007fffffffffUL 58 #define PTESYNC() __asm __volatile("ptesync") 59 60 extern int ps3fb_remap(void); 61 62 static uint64_t mps3_vas_id; 63 64 /* 65 * Kernel MMU interface 66 */ 67 68 static void mps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart, 69 vm_offset_t kernelend); 70 static void mps3_cpu_bootstrap(mmu_t mmup, int ap); 71 static void mps3_pte_synch(mmu_t, uintptr_t pt, struct lpte *pvo_pt); 72 static void mps3_pte_clear(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 73 uint64_t vpn, uint64_t ptebit); 74 static void mps3_pte_unset(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 75 uint64_t vpn); 76 static void mps3_pte_change(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 77 uint64_t vpn); 78 static int mps3_pte_insert(mmu_t, u_int ptegidx, struct lpte *pvo_pt); 79 static uintptr_t mps3_pvo_to_pte(mmu_t, const struct pvo_entry *pvo); 80 81 82 static mmu_method_t mps3_methods[] = { 83 MMUMETHOD(mmu_bootstrap, mps3_bootstrap), 84 MMUMETHOD(mmu_cpu_bootstrap, mps3_cpu_bootstrap), 85 86 MMUMETHOD(moea64_pte_synch, mps3_pte_synch), 87 MMUMETHOD(moea64_pte_clear, mps3_pte_clear), 88 MMUMETHOD(moea64_pte_unset, mps3_pte_unset), 89 MMUMETHOD(moea64_pte_change, mps3_pte_change), 90 MMUMETHOD(moea64_pte_insert, mps3_pte_insert), 91 MMUMETHOD(moea64_pvo_to_pte, mps3_pvo_to_pte), 92 93 { 0, 0 } 94 }; 95 96 MMU_DEF_INHERIT(ps3_mmu, "mmu_ps3", mps3_methods, 0, oea64_mmu); 97 98 static void 99 mps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 100 { 101 uint64_t final_pteg_count; 102 103 moea64_early_bootstrap(mmup, kernelstart, kernelend); 104 105 lv1_construct_virtual_address_space( 106 20 /* log_2(moea64_pteg_count) */, 2 /* n page sizes */, 107 (24UL << 56) | (16UL << 48) /* page sizes 16 MB + 64 KB */, 108 &mps3_vas_id, &final_pteg_count 109 ); 110 111 moea64_pteg_count = final_pteg_count / sizeof(struct lpteg); 112 113 moea64_mid_bootstrap(mmup, kernelstart, kernelend); 114 moea64_late_bootstrap(mmup, kernelstart, kernelend); 115 } 116 117 static void 118 mps3_cpu_bootstrap(mmu_t mmup, int ap) 119 { 120 struct slb *slb = PCPU_GET(slb); 121 register_t seg0; 122 int i; 123 124 mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); 125 126 /* 127 * Destroy the loader's address space if we are coming up for 128 * the first time, and redo the FB mapping so we can continue 129 * having a console. 130 */ 131 132 if (!ap) 133 lv1_destruct_virtual_address_space(0); 134 135 lv1_select_virtual_address_space(mps3_vas_id); 136 137 if (!ap) 138 ps3fb_remap(); 139 140 /* 141 * Install kernel SLB entries 142 */ 143 144 __asm __volatile ("slbia"); 145 __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); 146 for (i = 0; i < 64; i++) { 147 if (!(slb[i].slbe & SLBE_VALID)) 148 continue; 149 150 __asm __volatile ("slbmte %0, %1" :: 151 "r"(slb[i].slbv), "r"(slb[i].slbe)); 152 } 153 } 154 155 static void 156 mps3_pte_synch(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt) 157 { 158 uint64_t halfbucket[4], rcbits; 159 160 PTESYNC(); 161 lv1_read_htab_entries(mps3_vas_id, slot & ~0x3UL, &halfbucket[0], 162 &halfbucket[1], &halfbucket[2], &halfbucket[3], &rcbits); 163 164 /* 165 * rcbits contains the low 12 bits of each PTEs 2nd part, 166 * spaced at 16-bit intervals 167 */ 168 169 KASSERT((halfbucket[slot & 0x3] & LPTE_AVPN_MASK) == 170 (pvo_pt->pte_hi & LPTE_AVPN_MASK), 171 ("PTE upper word %#lx != %#lx\n", 172 halfbucket[slot & 0x3], pvo_pt->pte_hi)); 173 174 pvo_pt->pte_lo |= (rcbits >> ((3 - (slot & 0x3))*16)) & 175 (LPTE_CHG | LPTE_REF); 176 } 177 178 static void 179 mps3_pte_clear(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn, 180 u_int64_t ptebit) 181 { 182 183 lv1_write_htab_entry(mps3_vas_id, slot, pvo_pt->pte_hi, 184 pvo_pt->pte_lo & ~ptebit); 185 } 186 187 static void 188 mps3_pte_unset(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 189 { 190 191 mps3_pte_synch(mmu, slot, pvo_pt); 192 pvo_pt->pte_hi &= ~LPTE_VALID; 193 lv1_write_htab_entry(mps3_vas_id, slot, 0, 0); 194 moea64_pte_valid--; 195 } 196 197 static void 198 mps3_pte_change(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 199 { 200 201 mps3_pte_synch(mmu, slot, pvo_pt); 202 lv1_write_htab_entry(mps3_vas_id, slot, pvo_pt->pte_hi, 203 pvo_pt->pte_lo); 204 } 205 206 static int 207 mps3_pte_insert(mmu_t mmu, u_int ptegidx, struct lpte *pvo_pt) 208 { 209 int result; 210 struct lpte evicted; 211 struct pvo_entry *pvo; 212 uint64_t index; 213 214 pvo_pt->pte_hi |= LPTE_VALID; 215 pvo_pt->pte_hi &= ~LPTE_HID; 216 evicted.pte_hi = 0; 217 PTESYNC(); 218 result = lv1_insert_htab_entry(mps3_vas_id, ptegidx << 3, 219 pvo_pt->pte_hi, pvo_pt->pte_lo, LPTE_LOCKED | LPTE_WIRED, 0, 220 &index, &evicted.pte_hi, &evicted.pte_lo); 221 222 if (result != 0) { 223 /* No freeable slots in either PTEG? We're hosed. */ 224 panic("mps3_pte_insert: overflow (%d)", result); 225 return (-1); 226 } 227 228 /* 229 * See where we ended up. 230 */ 231 if (index >> 3 != ptegidx) 232 pvo_pt->pte_hi |= LPTE_HID; 233 234 moea64_pte_valid++; 235 236 if (!evicted.pte_hi) 237 return (index & 0x7); 238 239 /* 240 * Synchronize the sacrifice PTE with its PVO, then mark both 241 * invalid. The PVO will be reused when/if the VM system comes 242 * here after a fault. 243 */ 244 245 ptegidx = index >> 3; /* Where the sacrifice PTE was found */ 246 if (evicted.pte_hi & LPTE_HID) 247 ptegidx ^= moea64_pteg_mask; /* PTEs indexed by primary */ 248 249 KASSERT((evicted.pte_hi & (LPTE_WIRED | LPTE_LOCKED)) == 0, 250 ("Evicted a wired PTE")); 251 252 result = 0; 253 LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) { 254 if (!PVO_PTEGIDX_ISSET(pvo)) 255 continue; 256 257 if (pvo->pvo_pte.lpte.pte_hi == (evicted.pte_hi | LPTE_VALID)) { 258 KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, 259 ("Invalid PVO for valid PTE!")); 260 pvo->pvo_pte.lpte.pte_hi &= ~LPTE_VALID; 261 pvo->pvo_pte.lpte.pte_lo |= 262 evicted.pte_lo & (LPTE_REF | LPTE_CHG); 263 PVO_PTEGIDX_CLR(pvo); 264 moea64_pte_valid--; 265 moea64_pte_overflow++; 266 result = 1; 267 break; 268 } 269 } 270 271 KASSERT(result == 1, ("PVO for sacrifice PTE not found")); 272 273 return (index & 0x7); 274 } 275 276 static __inline u_int 277 va_to_pteg(uint64_t vsid, vm_offset_t addr, int large) 278 { 279 uint64_t hash; 280 int shift; 281 282 shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT; 283 hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >> 284 shift); 285 return (hash & moea64_pteg_mask); 286 } 287 288 uintptr_t 289 mps3_pvo_to_pte(mmu_t mmu, const struct pvo_entry *pvo) 290 { 291 uint64_t vsid; 292 u_int ptegidx; 293 294 /* If the PTEG index is not set, then there is no page table entry */ 295 if (!PVO_PTEGIDX_ISSET(pvo)) 296 return (-1); 297 298 vsid = PVO_VSID(pvo); 299 ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE); 300 301 /* 302 * We can find the actual pte entry without searching by grabbing 303 * the PTEG index from 3 unused bits in pvo_vaddr and by 304 * noticing the HID bit. 305 */ 306 if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID) 307 ptegidx ^= moea64_pteg_mask; 308 309 return ((ptegidx << 3) | PVO_PTEGIDX_GET(pvo)); 310 } 311 312