1 /*- 2 * Copyright (c) 2010 Nathan Whitehorn 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #include <sys/param.h> 30 #include <sys/kernel.h> 31 #include <sys/lock.h> 32 #include <sys/malloc.h> 33 #include <sys/mutex.h> 34 #include <sys/proc.h> 35 #include <sys/systm.h> 36 37 #include <vm/vm.h> 38 #include <vm/pmap.h> 39 #include <vm/uma.h> 40 #include <vm/vm.h> 41 #include <vm/vm_map.h> 42 #include <vm/vm_page.h> 43 #include <vm/vm_pageout.h> 44 45 #include <machine/md_var.h> 46 #include <machine/platform.h> 47 #include <machine/vmparam.h> 48 49 uintptr_t moea64_get_unique_vsid(void); 50 void moea64_release_vsid(uint64_t vsid); 51 static void slb_zone_init(void *); 52 53 static uma_zone_t slbt_zone; 54 static uma_zone_t slb_cache_zone; 55 int n_slbs = 64; 56 57 SYSINIT(slb_zone_init, SI_SUB_KMEM, SI_ORDER_ANY, slb_zone_init, NULL); 58 59 struct slbtnode { 60 uint16_t ua_alloc; 61 uint8_t ua_level; 62 /* Only 36 bits needed for full 64-bit address space. */ 63 uint64_t ua_base; 64 union { 65 struct slbtnode *ua_child[16]; 66 struct slb slb_entries[16]; 67 } u; 68 }; 69 70 /* 71 * For a full 64-bit address space, there are 36 bits in play in an 72 * esid, so 8 levels, with the leaf being at level 0. 73 * 74 * |3333|3322|2222|2222|1111|1111|11 | | | esid 75 * |5432|1098|7654|3210|9876|5432|1098|7654|3210| bits 76 * +----+----+----+----+----+----+----+----+----+-------- 77 * | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | level 78 */ 79 #define UAD_ROOT_LEVEL 8 80 #define UAD_LEAF_LEVEL 0 81 82 static inline int 83 esid2idx(uint64_t esid, int level) 84 { 85 int shift; 86 87 shift = level * 4; 88 return ((esid >> shift) & 0xF); 89 } 90 91 /* 92 * The ua_base field should have 0 bits after the first 4*(level+1) 93 * bits; i.e. only 94 */ 95 #define uad_baseok(ua) \ 96 (esid2base(ua->ua_base, ua->ua_level) == ua->ua_base) 97 98 99 static inline uint64_t 100 esid2base(uint64_t esid, int level) 101 { 102 uint64_t mask; 103 int shift; 104 105 shift = (level + 1) * 4; 106 mask = ~((1ULL << shift) - 1); 107 return (esid & mask); 108 } 109 110 /* 111 * Allocate a new leaf node for the specified esid/vmhandle from the 112 * parent node. 113 */ 114 static struct slb * 115 make_new_leaf(uint64_t esid, uint64_t slbv, struct slbtnode *parent) 116 { 117 struct slbtnode *child; 118 struct slb *retval; 119 int idx; 120 121 idx = esid2idx(esid, parent->ua_level); 122 KASSERT(parent->u.ua_child[idx] == NULL, ("Child already exists!")); 123 124 /* unlock and M_WAITOK and loop? */ 125 child = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO); 126 KASSERT(child != NULL, ("unhandled NULL case")); 127 128 child->ua_level = UAD_LEAF_LEVEL; 129 child->ua_base = esid2base(esid, child->ua_level); 130 idx = esid2idx(esid, child->ua_level); 131 child->u.slb_entries[idx].slbv = slbv; 132 child->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID; 133 setbit(&child->ua_alloc, idx); 134 135 retval = &child->u.slb_entries[idx]; 136 137 /* 138 * The above stores must be visible before the next one, so 139 * that a lockless searcher always sees a valid path through 140 * the tree. 141 */ 142 powerpc_lwsync(); 143 144 idx = esid2idx(esid, parent->ua_level); 145 parent->u.ua_child[idx] = child; 146 setbit(&parent->ua_alloc, idx); 147 148 return (retval); 149 } 150 151 /* 152 * Allocate a new intermediate node to fit between the parent and 153 * esid. 154 */ 155 static struct slbtnode* 156 make_intermediate(uint64_t esid, struct slbtnode *parent) 157 { 158 struct slbtnode *child, *inter; 159 int idx, level; 160 161 idx = esid2idx(esid, parent->ua_level); 162 child = parent->u.ua_child[idx]; 163 KASSERT(esid2base(esid, child->ua_level) != child->ua_base, 164 ("No need for an intermediate node?")); 165 166 /* 167 * Find the level where the existing child and our new esid 168 * meet. It must be lower than parent->ua_level or we would 169 * have chosen a different index in parent. 170 */ 171 level = child->ua_level + 1; 172 while (esid2base(esid, level) != 173 esid2base(child->ua_base, level)) 174 level++; 175 KASSERT(level < parent->ua_level, 176 ("Found splitting level %d for %09jx and %09jx, " 177 "but it's the same as %p's", 178 level, esid, child->ua_base, parent)); 179 180 /* unlock and M_WAITOK and loop? */ 181 inter = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO); 182 KASSERT(inter != NULL, ("unhandled NULL case")); 183 184 /* Set up intermediate node to point to child ... */ 185 inter->ua_level = level; 186 inter->ua_base = esid2base(esid, inter->ua_level); 187 idx = esid2idx(child->ua_base, inter->ua_level); 188 inter->u.ua_child[idx] = child; 189 setbit(&inter->ua_alloc, idx); 190 powerpc_lwsync(); 191 192 /* Set up parent to point to intermediate node ... */ 193 idx = esid2idx(inter->ua_base, parent->ua_level); 194 parent->u.ua_child[idx] = inter; 195 setbit(&parent->ua_alloc, idx); 196 197 return (inter); 198 } 199 200 uint64_t 201 kernel_va_to_slbv(vm_offset_t va) 202 { 203 uint64_t slbv; 204 205 /* Set kernel VSID to deterministic value */ 206 slbv = (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT)) << SLBV_VSID_SHIFT; 207 208 /* Figure out if this is a large-page mapping */ 209 if (hw_direct_map && va < VM_MIN_KERNEL_ADDRESS) { 210 /* 211 * XXX: If we have set up a direct map, assumes 212 * all physical memory is mapped with large pages. 213 */ 214 if (mem_valid(va, 0) == 0) 215 slbv |= SLBV_L; 216 } 217 218 return (slbv); 219 } 220 221 struct slb * 222 user_va_to_slb_entry(pmap_t pm, vm_offset_t va) 223 { 224 uint64_t esid = va >> ADDR_SR_SHFT; 225 struct slbtnode *ua; 226 int idx; 227 228 ua = pm->pm_slb_tree_root; 229 230 for (;;) { 231 KASSERT(uad_baseok(ua), ("uad base %016jx level %d bad!", 232 ua->ua_base, ua->ua_level)); 233 idx = esid2idx(esid, ua->ua_level); 234 235 /* 236 * This code is specific to ppc64 where a load is 237 * atomic, so no need for atomic_load macro. 238 */ 239 if (ua->ua_level == UAD_LEAF_LEVEL) 240 return ((ua->u.slb_entries[idx].slbe & SLBE_VALID) ? 241 &ua->u.slb_entries[idx] : NULL); 242 243 /* 244 * The following accesses are implicitly ordered under the POWER 245 * ISA by load dependencies (the store ordering is provided by 246 * the powerpc_lwsync() calls elsewhere) and so are run without 247 * barriers. 248 */ 249 ua = ua->u.ua_child[idx]; 250 if (ua == NULL || 251 esid2base(esid, ua->ua_level) != ua->ua_base) 252 return (NULL); 253 } 254 255 return (NULL); 256 } 257 258 uint64_t 259 va_to_vsid(pmap_t pm, vm_offset_t va) 260 { 261 struct slb *entry; 262 263 /* Shortcut kernel case */ 264 if (pm == kernel_pmap) 265 return (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT)); 266 267 /* 268 * If there is no vsid for this VA, we need to add a new entry 269 * to the PMAP's segment table. 270 */ 271 272 entry = user_va_to_slb_entry(pm, va); 273 274 if (entry == NULL) 275 return (allocate_user_vsid(pm, 276 (uintptr_t)va >> ADDR_SR_SHFT, 0)); 277 278 return ((entry->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT); 279 } 280 281 uint64_t 282 allocate_user_vsid(pmap_t pm, uint64_t esid, int large) 283 { 284 uint64_t vsid, slbv; 285 struct slbtnode *ua, *next, *inter; 286 struct slb *slb; 287 int idx; 288 289 KASSERT(pm != kernel_pmap, ("Attempting to allocate a kernel VSID")); 290 291 PMAP_LOCK_ASSERT(pm, MA_OWNED); 292 vsid = moea64_get_unique_vsid(); 293 294 slbv = vsid << SLBV_VSID_SHIFT; 295 if (large) 296 slbv |= SLBV_L; 297 298 ua = pm->pm_slb_tree_root; 299 300 /* Descend to the correct leaf or NULL pointer. */ 301 for (;;) { 302 KASSERT(uad_baseok(ua), 303 ("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level)); 304 idx = esid2idx(esid, ua->ua_level); 305 306 if (ua->ua_level == UAD_LEAF_LEVEL) { 307 ua->u.slb_entries[idx].slbv = slbv; 308 eieio(); 309 ua->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT) 310 | SLBE_VALID; 311 setbit(&ua->ua_alloc, idx); 312 slb = &ua->u.slb_entries[idx]; 313 break; 314 } 315 316 next = ua->u.ua_child[idx]; 317 if (next == NULL) { 318 slb = make_new_leaf(esid, slbv, ua); 319 break; 320 } 321 322 /* 323 * Check if the next item down has an okay ua_base. 324 * If not, we need to allocate an intermediate node. 325 */ 326 if (esid2base(esid, next->ua_level) != next->ua_base) { 327 inter = make_intermediate(esid, ua); 328 slb = make_new_leaf(esid, slbv, inter); 329 break; 330 } 331 332 ua = next; 333 } 334 335 /* 336 * Someone probably wants this soon, and it may be a wired 337 * SLB mapping, so pre-spill this entry. 338 */ 339 eieio(); 340 slb_insert_user(pm, slb); 341 342 return (vsid); 343 } 344 345 void 346 free_vsid(pmap_t pm, uint64_t esid, int large) 347 { 348 struct slbtnode *ua; 349 int idx; 350 351 PMAP_LOCK_ASSERT(pm, MA_OWNED); 352 353 ua = pm->pm_slb_tree_root; 354 /* Descend to the correct leaf. */ 355 for (;;) { 356 KASSERT(uad_baseok(ua), 357 ("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level)); 358 359 idx = esid2idx(esid, ua->ua_level); 360 if (ua->ua_level == UAD_LEAF_LEVEL) { 361 ua->u.slb_entries[idx].slbv = 0; 362 eieio(); 363 ua->u.slb_entries[idx].slbe = 0; 364 clrbit(&ua->ua_alloc, idx); 365 return; 366 } 367 368 ua = ua->u.ua_child[idx]; 369 if (ua == NULL || 370 esid2base(esid, ua->ua_level) != ua->ua_base) { 371 /* Perhaps just return instead of assert? */ 372 KASSERT(0, 373 ("Asked to remove an entry that was never inserted!")); 374 return; 375 } 376 } 377 } 378 379 static void 380 free_slb_tree_node(struct slbtnode *ua) 381 { 382 int idx; 383 384 for (idx = 0; idx < 16; idx++) { 385 if (ua->ua_level != UAD_LEAF_LEVEL) { 386 if (ua->u.ua_child[idx] != NULL) 387 free_slb_tree_node(ua->u.ua_child[idx]); 388 } else { 389 if (ua->u.slb_entries[idx].slbv != 0) 390 moea64_release_vsid(ua->u.slb_entries[idx].slbv 391 >> SLBV_VSID_SHIFT); 392 } 393 } 394 395 uma_zfree(slbt_zone, ua); 396 } 397 398 void 399 slb_free_tree(pmap_t pm) 400 { 401 402 free_slb_tree_node(pm->pm_slb_tree_root); 403 } 404 405 struct slbtnode * 406 slb_alloc_tree(void) 407 { 408 struct slbtnode *root; 409 410 root = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO); 411 root->ua_level = UAD_ROOT_LEVEL; 412 413 return (root); 414 } 415 416 /* Lock entries mapping kernel text and stacks */ 417 418 void 419 slb_insert_kernel(uint64_t slbe, uint64_t slbv) 420 { 421 struct slb *slbcache; 422 int i; 423 424 /* We don't want to be preempted while modifying the kernel map */ 425 critical_enter(); 426 427 slbcache = PCPU_GET(slb); 428 429 /* Check for an unused slot, abusing the user slot as a full flag */ 430 if (slbcache[USER_SLB_SLOT].slbe == 0) { 431 for (i = 0; i < n_slbs; i++) { 432 if (i == USER_SLB_SLOT) 433 continue; 434 if (!(slbcache[i].slbe & SLBE_VALID)) 435 goto fillkernslb; 436 } 437 438 if (i == n_slbs) 439 slbcache[USER_SLB_SLOT].slbe = 1; 440 } 441 442 i = mftb() % n_slbs; 443 if (i == USER_SLB_SLOT) 444 i = (i+1) % n_slbs; 445 446 fillkernslb: 447 KASSERT(i != USER_SLB_SLOT, 448 ("Filling user SLB slot with a kernel mapping")); 449 slbcache[i].slbv = slbv; 450 slbcache[i].slbe = slbe | (uint64_t)i; 451 452 /* If it is for this CPU, put it in the SLB right away */ 453 if (pmap_bootstrapped) { 454 /* slbie not required */ 455 __asm __volatile ("slbmte %0, %1" :: 456 "r"(slbcache[i].slbv), "r"(slbcache[i].slbe)); 457 } 458 459 critical_exit(); 460 } 461 462 void 463 slb_insert_user(pmap_t pm, struct slb *slb) 464 { 465 int i; 466 467 PMAP_LOCK_ASSERT(pm, MA_OWNED); 468 469 if (pm->pm_slb_len < n_slbs) { 470 i = pm->pm_slb_len; 471 pm->pm_slb_len++; 472 } else { 473 i = mftb() % n_slbs; 474 } 475 476 /* Note that this replacement is atomic with respect to trap_subr */ 477 pm->pm_slb[i] = slb; 478 } 479 480 static void * 481 slb_uma_real_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait) 482 { 483 static vm_offset_t realmax = 0; 484 void *va; 485 vm_page_t m; 486 int pflags; 487 488 if (realmax == 0) 489 realmax = platform_real_maxaddr(); 490 491 *flags = UMA_SLAB_PRIV; 492 pflags = malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED; 493 494 for (;;) { 495 m = vm_page_alloc_contig(NULL, 0, pflags, 1, 0, realmax, 496 PAGE_SIZE, PAGE_SIZE, VM_MEMATTR_DEFAULT); 497 if (m == NULL) { 498 if (wait & M_NOWAIT) 499 return (NULL); 500 VM_WAIT; 501 } else 502 break; 503 } 504 505 va = (void *) VM_PAGE_TO_PHYS(m); 506 507 if (!hw_direct_map) 508 pmap_kenter((vm_offset_t)va, VM_PAGE_TO_PHYS(m)); 509 510 if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) 511 bzero(va, PAGE_SIZE); 512 513 return (va); 514 } 515 516 static void 517 slb_zone_init(void *dummy) 518 { 519 520 slbt_zone = uma_zcreate("SLB tree node", sizeof(struct slbtnode), 521 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM); 522 slb_cache_zone = uma_zcreate("SLB cache", 523 (n_slbs + 1)*sizeof(struct slb *), NULL, NULL, NULL, NULL, 524 UMA_ALIGN_PTR, UMA_ZONE_VM); 525 526 if (platform_real_maxaddr() != VM_MAX_ADDRESS) { 527 uma_zone_set_allocf(slb_cache_zone, slb_uma_real_alloc); 528 uma_zone_set_allocf(slbt_zone, slb_uma_real_alloc); 529 } 530 } 531 532 struct slb ** 533 slb_alloc_user_cache(void) 534 { 535 return (uma_zalloc(slb_cache_zone, M_ZERO)); 536 } 537 538 void 539 slb_free_user_cache(struct slb **slb) 540 { 541 uma_zfree(slb_cache_zone, slb); 542 } 543