1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26/* 27 * SFMMU primitives. These primitives should only be used by sfmmu 28 * routines. 29 */ 30 31#if defined(lint) 32#include <sys/types.h> 33#else /* lint */ 34#include "assym.h" 35#endif /* lint */ 36 37#include <sys/asm_linkage.h> 38#include <sys/machtrap.h> 39#include <sys/machasi.h> 40#include <sys/sun4asi.h> 41#include <sys/pte.h> 42#include <sys/mmu.h> 43#include <vm/hat_sfmmu.h> 44#include <vm/seg_spt.h> 45#include <sys/machparam.h> 46#include <sys/privregs.h> 47#include <sys/scb.h> 48#include <sys/intreg.h> 49#include <sys/machthread.h> 50#include <sys/intr.h> 51#include <sys/clock.h> 52#include <sys/trapstat.h> 53 54#ifdef TRAPTRACE 55#include <sys/traptrace.h> 56 57/* 58 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 59 */ 60#define TT_TRACE(label) \ 61 ba label ;\ 62 rd %pc, %g7 63#else 64 65#define TT_TRACE(label) 66 67#endif /* TRAPTRACE */ 68 69#ifndef lint 70 71#if (TTE_SUSPEND_SHIFT > 0) 72#define TTE_SUSPEND_INT_SHIFT(reg) \ 73 sllx reg, TTE_SUSPEND_SHIFT, reg 74#else 75#define TTE_SUSPEND_INT_SHIFT(reg) 76#endif 77 78#endif /* lint */ 79 80#ifndef lint 81 82/* 83 * Assumes TSBE_TAG is 0 84 * Assumes TSBE_INTHI is 0 85 * Assumes TSBREG.split is 0 86 */ 87 88#if TSBE_TAG != 0 89#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 90#endif 91 92#if TSBTAG_INTHI != 0 93#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 94#endif 95 96/* 97 * The following code assumes the tsb is not split. 98 * 99 * With TSBs no longer shared between processes, it's no longer 100 * necessary to hash the context bits into the tsb index to get 101 * tsb coloring; the new implementation treats the TSB as a 102 * direct-mapped, virtually-addressed cache. 103 * 104 * In: 105 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 106 * tsbbase = base address of TSB (clobbered) 107 * tagacc = tag access register (clobbered) 108 * szc = size code of TSB (ro) 109 * tmp = scratch reg 110 * Out: 111 * tsbbase = pointer to entry in TSB 112 */ 113#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 114 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 115 srlx tagacc, vpshift, tagacc ;\ 116 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 117 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 118 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 119 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 120 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 121 122/* 123 * When the kpm TSB is used it is assumed that it is direct mapped 124 * using (vaddr>>vpshift)%tsbsz as the index. 125 * 126 * Note that, for now, the kpm TSB and kernel TSB are the same for 127 * each mapping size. However that need not always be the case. If 128 * the trap handlers are updated to search a different TSB for kpm 129 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 130 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 131 * 132 * In: 133 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 134 * vaddr = virtual address (clobbered) 135 * tsbp, szc, tmp = scratch 136 * Out: 137 * tsbp = pointer to entry in TSB 138 */ 139#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 140 cmp vpshift, MMU_PAGESHIFT ;\ 141 bne,pn %icc, 1f /* branch if large case */ ;\ 142 sethi %hi(kpmsm_tsbsz), szc ;\ 143 sethi %hi(kpmsm_tsbbase), tsbp ;\ 144 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 145 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 146 ba,pt %icc, 2f ;\ 147 nop ;\ 1481: sethi %hi(kpm_tsbsz), szc ;\ 149 sethi %hi(kpm_tsbbase), tsbp ;\ 150 ld [szc + %lo(kpm_tsbsz)], szc ;\ 151 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1522: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 153 154/* 155 * Lock the TSBE at virtual address tsbep. 156 * 157 * tsbep = TSBE va (ro) 158 * tmp1, tmp2 = scratch registers (clobbered) 159 * label = label to use for branches (text) 160 * %asi = ASI to use for TSB access 161 * 162 * NOTE that we flush the TSB using fast VIS instructions that 163 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 164 * not be treated as a locked entry or we'll get stuck spinning on 165 * an entry that isn't locked but really invalid. 166 */ 167 168#if defined(UTSB_PHYS) 169 170#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 171 lda [tsbep]ASI_MEM, tmp1 ;\ 172label: ;\ 173 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 174 cmp tmp1, tmp2 ;\ 175 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 176 lda [tsbep]ASI_MEM, tmp1 ;\ 177 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 178 cmp tmp1, tmp2 ;\ 179 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 180 lda [tsbep]ASI_MEM, tmp1 ;\ 181 /* tsbe lock acquired */ ;\ 182 membar #StoreStore 183 184#else /* UTSB_PHYS */ 185 186#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 187 lda [tsbep]%asi, tmp1 ;\ 188label: ;\ 189 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 190 cmp tmp1, tmp2 ;\ 191 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 192 lda [tsbep]%asi, tmp1 ;\ 193 casa [tsbep]%asi, tmp1, tmp2 ;\ 194 cmp tmp1, tmp2 ;\ 195 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 196 lda [tsbep]%asi, tmp1 ;\ 197 /* tsbe lock acquired */ ;\ 198 membar #StoreStore 199 200#endif /* UTSB_PHYS */ 201 202/* 203 * Atomically write TSBE at virtual address tsbep. 204 * 205 * tsbep = TSBE va (ro) 206 * tte = TSBE TTE (ro) 207 * tagtarget = TSBE tag (ro) 208 * %asi = ASI to use for TSB access 209 */ 210 211#if defined(UTSB_PHYS) 212 213#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 214 add tsbep, TSBE_TTE, tmp1 ;\ 215 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 216 membar #StoreStore ;\ 217 add tsbep, TSBE_TAG, tmp1 ;\ 218 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 219 220#else /* UTSB_PHYS */ 221 222#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 223 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 224 membar #StoreStore ;\ 225 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 226 227#endif /* UTSB_PHYS */ 228 229/* 230 * Load an entry into the TSB at TL > 0. 231 * 232 * tsbep = pointer to the TSBE to load as va (ro) 233 * tte = value of the TTE retrieved and loaded (wo) 234 * tagtarget = tag target register. To get TSBE tag to load, 235 * we need to mask off the context and leave only the va (clobbered) 236 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 237 * tmp1, tmp2 = scratch registers 238 * label = label to use for branches (text) 239 * %asi = ASI to use for TSB access 240 */ 241 242#if defined(UTSB_PHYS) 243 244#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 245 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 246 /* ;\ 247 * I don't need to update the TSB then check for the valid tte. ;\ 248 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 249 * we always invalidate the hash table before we unload the TSB.;\ 250 */ ;\ 251 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 252 ldxa [ttepa]ASI_MEM, tte ;\ 253 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 254 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 255 add tsbep, TSBE_TAG, tmp1 ;\ 256 brgez,a,pn tte, label/**/f ;\ 257 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 258 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 259label: 260 261#else /* UTSB_PHYS */ 262 263#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 264 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 265 /* ;\ 266 * I don't need to update the TSB then check for the valid tte. ;\ 267 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 268 * we always invalidate the hash table before we unload the TSB.;\ 269 */ ;\ 270 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 271 ldxa [ttepa]ASI_MEM, tte ;\ 272 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 273 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 274 brgez,a,pn tte, label/**/f ;\ 275 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 276 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 277label: 278 279#endif /* UTSB_PHYS */ 280 281/* 282 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 283 * for ITLB synthesis. 284 * 285 * tsbep = pointer to the TSBE to load as va (ro) 286 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 287 * with exec_perm turned off and exec_synth turned on 288 * tagtarget = tag target register. To get TSBE tag to load, 289 * we need to mask off the context and leave only the va (clobbered) 290 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 291 * tmp1, tmp2 = scratch registers 292 * label = label to use for branch (text) 293 * %asi = ASI to use for TSB access 294 */ 295 296#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 297 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 298 /* ;\ 299 * I don't need to update the TSB then check for the valid tte. ;\ 300 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 301 * we always invalidate the hash table before we unload the TSB.;\ 302 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 303 * and exec_synth bit to 1. ;\ 304 */ ;\ 305 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 306 mov tte, tmp1 ;\ 307 ldxa [ttepa]ASI_MEM, tte ;\ 308 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 309 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 310 brgez,a,pn tte, label/**/f ;\ 311 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 312 or tte, tmp1, tte ;\ 313 andn tte, TTE_EXECPRM_INT, tte ;\ 314 or tte, TTE_E_SYNTH_INT, tte ;\ 315 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 316label: 317 318/* 319 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 320 * 321 * tte = value of the TTE, used to get tte_size bits (ro) 322 * tagaccess = tag access register, used to get 4M pfn bits (ro) 323 * pfn = 4M pfn bits shifted to offset for tte (out) 324 * tmp1 = scratch register 325 * label = label to use for branch (text) 326 */ 327 328#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 329 /* ;\ 330 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 331 * Return them, shifted, in pfn. ;\ 332 */ ;\ 333 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 334 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 335 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 336 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 337 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 338 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 339label: ;\ 340 sllx tagaccess, MMU_PAGESHIFT4M, pfn 341 342/* 343 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 344 * for ITLB synthesis. 345 * 346 * tte = value of the TTE, used to get tte_size bits (rw) 347 * tmp1 = scratch register 348 */ 349 350#define SET_TTE4M_PN(tte, tmp) \ 351 /* ;\ 352 * Set 4M pagesize tte bits. ;\ 353 */ ;\ 354 set TTE4M, tmp ;\ 355 sllx tmp, TTE_SZ_SHFT, tmp ;\ 356 or tte, tmp, tte 357 358/* 359 * Load an entry into the TSB at TL=0. 360 * 361 * tsbep = pointer to the TSBE to load as va (ro) 362 * tteva = pointer to the TTE to load as va (ro) 363 * tagtarget = TSBE tag to load (which contains no context), synthesized 364 * to match va of MMU tag target register only (ro) 365 * tmp1, tmp2 = scratch registers (clobbered) 366 * label = label to use for branches (text) 367 * %asi = ASI to use for TSB access 368 */ 369 370#if defined(UTSB_PHYS) 371 372#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 373 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 374 ldx [tteva], tteva /* load tte */ ;\ 375 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 376 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 377 add tsbep, TSBE_TAG, tmp1 ;\ 378 brgez,a,pn tteva, label/**/f ;\ 379 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 380 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 381label: 382 383#else /* UTSB_PHYS */ 384 385#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 386 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 387 ldx [tteva], tteva /* load tte */ ;\ 388 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 389 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 390 brgez,a,pn tteva, label/**/f ;\ 391 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 392 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 393label: 394 395#endif /* UTSB_PHYS */ 396 397/* 398 * Invalidate a TSB entry in the TSB. 399 * 400 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 401 * about this earlier to ensure this is true. Thus when we are 402 * directly referencing tsbep below, we are referencing the tte_tag 403 * field of the TSBE. If this offset ever changes, the code below 404 * will need to be modified. 405 * 406 * tsbep = pointer to TSBE as va (ro) 407 * tag = invalidation is done if this matches the TSBE tag (ro) 408 * tmp1 - tmp3 = scratch registers (clobbered) 409 * label = label name to use for branches (text) 410 * %asi = ASI to use for TSB access 411 */ 412 413#if defined(UTSB_PHYS) 414 415#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 416 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 417 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 418label/**/1: ;\ 419 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 420 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 421 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 422 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 423 cmp tag, tmp3 /* compare tags */ ;\ 424 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 425 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 426 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 427 cmp tmp1, tmp3 /* if not successful */ ;\ 428 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 429 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 430label/**/2: 431 432#else /* UTSB_PHYS */ 433 434#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 435 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 436 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 437label/**/1: ;\ 438 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 439 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 440 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 441 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 442 cmp tag, tmp3 /* compare tags */ ;\ 443 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 444 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 445 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 446 cmp tmp1, tmp3 /* if not successful */ ;\ 447 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 448 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 449label/**/2: 450 451#endif /* UTSB_PHYS */ 452 453#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 454#error - TSB_SOFTSZ_MASK too small 455#endif 456 457 458/* 459 * An implementation of setx which will be hot patched at run time. 460 * since it is being hot patched, there is no value passed in. 461 * Thus, essentially we are implementing 462 * setx value, tmp, dest 463 * where value is RUNTIME_PATCH (aka 0) in this case. 464 */ 465#define RUNTIME_PATCH_SETX(dest, tmp) \ 466 sethi %hh(RUNTIME_PATCH), tmp ;\ 467 sethi %lm(RUNTIME_PATCH), dest ;\ 468 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 469 or dest, %lo(RUNTIME_PATCH), dest ;\ 470 sllx tmp, 32, tmp ;\ 471 nop /* for perf reasons */ ;\ 472 or tmp, dest, dest /* contents of patched value */ 473 474 475#endif (lint) 476 477 478#if defined (lint) 479 480/* 481 * sfmmu related subroutines 482 */ 483uint_t 484sfmmu_disable_intrs() 485{ return(0); } 486 487/* ARGSUSED */ 488void 489sfmmu_enable_intrs(uint_t pstate_save) 490{} 491 492/* ARGSUSED */ 493int 494sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag) 495{ return(0); } 496 497/* 498 * Use cas, if tte has changed underneath us then reread and try again. 499 * In the case of a retry, it will update sttep with the new original. 500 */ 501/* ARGSUSED */ 502int 503sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 504{ return(0); } 505 506/* 507 * Use cas, if tte has changed underneath us then return 1, else return 0 508 */ 509/* ARGSUSED */ 510int 511sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 512{ return(0); } 513 514/* ARGSUSED */ 515void 516sfmmu_copytte(tte_t *sttep, tte_t *dttep) 517{} 518 519/*ARGSUSED*/ 520struct tsbe * 521sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 522{ return(0); } 523 524/*ARGSUSED*/ 525uint64_t 526sfmmu_make_tsbtag(caddr_t va) 527{ return(0); } 528 529#else /* lint */ 530 531 .seg ".data" 532 .global sfmmu_panic1 533sfmmu_panic1: 534 .asciz "sfmmu_asm: interrupts already disabled" 535 536 .global sfmmu_panic3 537sfmmu_panic3: 538 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 539 540 .global sfmmu_panic4 541sfmmu_panic4: 542 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 543 544 .global sfmmu_panic5 545sfmmu_panic5: 546 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 547 548 .global sfmmu_panic6 549sfmmu_panic6: 550 .asciz "sfmmu_asm: interrupts not disabled" 551 552 .global sfmmu_panic7 553sfmmu_panic7: 554 .asciz "sfmmu_asm: kernel as" 555 556 .global sfmmu_panic8 557sfmmu_panic8: 558 .asciz "sfmmu_asm: gnum is zero" 559 560 .global sfmmu_panic9 561sfmmu_panic9: 562 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 563 564 .global sfmmu_panic10 565sfmmu_panic10: 566 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 567 568 ENTRY(sfmmu_disable_intrs) 569 rdpr %pstate, %o0 570#ifdef DEBUG 571 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 572#endif /* DEBUG */ 573 retl 574 wrpr %o0, PSTATE_IE, %pstate 575 SET_SIZE(sfmmu_disable_intrs) 576 577 ENTRY(sfmmu_enable_intrs) 578 retl 579 wrpr %g0, %o0, %pstate 580 SET_SIZE(sfmmu_enable_intrs) 581 582/* 583 * This routine is called both by resume() and sfmmu_get_ctx() to 584 * allocate a new context for the process on a MMU. 585 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 586 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 587 * is the case when sfmmu_alloc_ctx is called from resume(). 588 * 589 * The caller must disable interrupts before entering this routine. 590 * To reduce ctx switch overhead, the code contains both 'fast path' and 591 * 'slow path' code. The fast path code covers the common case where only 592 * a quick check is needed and the real ctx allocation is not required. 593 * It can be done without holding the per-process (PP) lock. 594 * The 'slow path' code must be protected by the PP Lock and performs ctx 595 * allocation. 596 * Hardware context register and HAT mmu cnum are updated accordingly. 597 * 598 * %o0 - sfmmup 599 * %o1 - allocflag 600 * %o2 - CPU 601 * %o3 - sfmmu private/shared flag 602 * 603 * ret - 0: no ctx is allocated 604 * 1: a ctx is allocated 605 */ 606 ENTRY_NP(sfmmu_alloc_ctx) 607 608#ifdef DEBUG 609 sethi %hi(ksfmmup), %g1 610 ldx [%g1 + %lo(ksfmmup)], %g1 611 cmp %g1, %o0 612 bne,pt %xcc, 0f 613 nop 614 615 sethi %hi(panicstr), %g1 ! if kernel as, panic 616 ldx [%g1 + %lo(panicstr)], %g1 617 tst %g1 618 bnz,pn %icc, 7f 619 nop 620 621 sethi %hi(sfmmu_panic7), %o0 622 call panic 623 or %o0, %lo(sfmmu_panic7), %o0 624 6257: 626 retl 627 mov %g0, %o0 ! %o0 = ret = 0 628 6290: 630 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 631#endif /* DEBUG */ 632 633 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 634 635 ! load global mmu_ctxp info 636 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 637 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 638 639 ! load global mmu_ctxp gnum 640 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 641 642#ifdef DEBUG 643 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 644 bne,pt %xcc, 3f 645 nop 646 647 sethi %hi(panicstr), %g1 ! test if panicstr is already set 648 ldx [%g1 + %lo(panicstr)], %g1 649 tst %g1 650 bnz,pn %icc, 1f 651 nop 652 653 sethi %hi(sfmmu_panic8), %o0 654 call panic 655 or %o0, %lo(sfmmu_panic8), %o0 6561: 657 retl 658 mov %g0, %o0 ! %o0 = ret = 0 6593: 660#endif 661 662 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 663 664 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 665 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 666 667 /* 668 * %g5 = sfmmu gnum returned 669 * %g6 = sfmmu cnum returned 670 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 671 * %g4 = scratch 672 * 673 * Fast path code, do a quick check. 674 */ 675 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 676 677 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 678 bne,pt %icc, 1f ! valid hat cnum, check gnum 679 nop 680 681 ! cnum == INVALID, check allocflag 682 mov %g0, %g4 ! %g4 = ret = 0 683 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 684 mov %g6, %o1 685 686 ! (invalid HAT cnum) && (allocflag == 1) 687 ba,pt %icc, 2f 688 nop 6891: 690 ! valid HAT cnum, check gnum 691 cmp %g5, %o4 692 mov 1, %g4 !%g4 = ret = 1 693 be,a,pt %icc, 8f ! gnum unchanged, go to done 694 mov %g6, %o1 695 6962: 697 /* 698 * Grab per process (PP) sfmmu_ctx_lock spinlock, 699 * followed by the 'slow path' code. 700 */ 701 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 7023: 703 brz %g3, 5f 704 nop 7054: 706 brnz,a,pt %g3, 4b ! spin if lock is 1 707 ldub [%o0 + SFMMU_CTX_LOCK], %g3 708 ba %xcc, 3b ! retry the lock 709 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 710 7115: 712 membar #LoadLoad 713 /* 714 * %g5 = sfmmu gnum returned 715 * %g6 = sfmmu cnum returned 716 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 717 * %g4 = scratch 718 */ 719 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 720 721 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 722 bne,pt %icc, 1f ! valid hat cnum, check gnum 723 nop 724 725 ! cnum == INVALID, check allocflag 726 mov %g0, %g4 ! %g4 = ret = 0 727 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 728 mov %g6, %o1 729 730 ! (invalid HAT cnum) && (allocflag == 1) 731 ba,pt %icc, 6f 732 nop 7331: 734 ! valid HAT cnum, check gnum 735 cmp %g5, %o4 736 mov 1, %g4 ! %g4 = ret = 1 737 be,a,pt %icc, 2f ! gnum unchanged, go to done 738 mov %g6, %o1 739 740 ba,pt %icc, 6f 741 nop 7422: 743 membar #LoadStore|#StoreStore 744 ba,pt %icc, 8f 745 clrb [%o0 + SFMMU_CTX_LOCK] 7466: 747 /* 748 * We get here if we do not have a valid context, or 749 * the HAT gnum does not match global gnum. We hold 750 * sfmmu_ctx_lock spinlock. Allocate that context. 751 * 752 * %o3 = mmu_ctxp 753 */ 754 add %o3, MMU_CTX_CNUM, %g3 755 ld [%o3 + MMU_CTX_NCTXS], %g4 756 757 /* 758 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 759 * %g3 = mmu cnum address 760 * %g4 = mmu nctxs 761 * 762 * %o0 = sfmmup 763 * %o1 = mmu current cnum value (used as new cnum) 764 * %o4 = mmu gnum 765 * 766 * %o5 = scratch 767 */ 768 ld [%g3], %o1 7690: 770 cmp %o1, %g4 771 bl,a,pt %icc, 1f 772 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 773 774 /* 775 * cnum reachs max, bail, so wrap around can be performed later. 776 */ 777 set INVALID_CONTEXT, %o1 778 mov %g0, %g4 ! %g4 = ret = 0 779 780 membar #LoadStore|#StoreStore 781 ba,pt %icc, 8f 782 clrb [%o0 + SFMMU_CTX_LOCK] 7831: 784 ! %g3 = addr of mmu_ctxp->cnum 785 ! %o5 = mmu_ctxp->cnum + 1 786 cas [%g3], %o1, %o5 787 cmp %o1, %o5 788 bne,a,pn %xcc, 0b ! cas failed 789 ld [%g3], %o1 790 791#ifdef DEBUG 792 set MAX_SFMMU_CTX_VAL, %o5 793 cmp %o1, %o5 794 ble,pt %icc, 2f 795 nop 796 797 sethi %hi(sfmmu_panic9), %o0 798 call panic 799 or %o0, %lo(sfmmu_panic9), %o0 8002: 801#endif 802 ! update hat gnum and cnum 803 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 804 or %o4, %o1, %o4 805 stx %o4, [%g2 + SFMMU_CTXS] 806 807 membar #LoadStore|#StoreStore 808 clrb [%o0 + SFMMU_CTX_LOCK] 809 810 mov 1, %g4 ! %g4 = ret = 1 8118: 812 /* 813 * program the secondary context register 814 * 815 * %o1 = cnum 816 * %g1 = sfmmu private/shared flag (0:private, 1:shared) 817 */ 818 819 /* 820 * When we come here and context is invalid, we want to set both 821 * private and shared ctx regs to INVALID. In order to 822 * do so, we set the sfmmu priv/shared flag to 'private' regardless 823 * so that private ctx reg will be set to invalid. 824 * Note that on sun4v values written to private context register are 825 * automatically written to corresponding shared context register as 826 * well. On sun4u SET_SECCTX() will invalidate shared context register 827 * when it sets a private secondary context register. 828 */ 829 830 cmp %o1, INVALID_CONTEXT 831 be,a,pn %icc, 9f 832 clr %g1 8339: 834 835#ifdef sun4u 836 ldub [%o0 + SFMMU_CEXT], %o2 837 sll %o2, CTXREG_EXT_SHIFT, %o2 838 or %o1, %o2, %o1 839#endif /* sun4u */ 840 841 SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1) 842 843 retl 844 mov %g4, %o0 ! %o0 = ret 845 846 SET_SIZE(sfmmu_alloc_ctx) 847 848 ENTRY_NP(sfmmu_modifytte) 849 ldx [%o2], %g3 /* current */ 850 ldx [%o0], %g1 /* original */ 8512: 852 ldx [%o1], %g2 /* modified */ 853 cmp %g2, %g3 /* is modified = current? */ 854 be,a,pt %xcc,1f /* yes, don't write */ 855 stx %g3, [%o0] /* update new original */ 856 casx [%o2], %g1, %g2 857 cmp %g1, %g2 858 be,pt %xcc, 1f /* cas succeeded - return */ 859 nop 860 ldx [%o2], %g3 /* new current */ 861 stx %g3, [%o0] /* save as new original */ 862 ba,pt %xcc, 2b 863 mov %g3, %g1 8641: retl 865 membar #StoreLoad 866 SET_SIZE(sfmmu_modifytte) 867 868 ENTRY_NP(sfmmu_modifytte_try) 869 ldx [%o1], %g2 /* modified */ 870 ldx [%o2], %g3 /* current */ 871 ldx [%o0], %g1 /* original */ 872 cmp %g3, %g2 /* is modified = current? */ 873 be,a,pn %xcc,1f /* yes, don't write */ 874 mov 0, %o1 /* as if cas failed. */ 875 876 casx [%o2], %g1, %g2 877 membar #StoreLoad 878 cmp %g1, %g2 879 movne %xcc, -1, %o1 /* cas failed. */ 880 move %xcc, 1, %o1 /* cas succeeded. */ 8811: 882 stx %g2, [%o0] /* report "current" value */ 883 retl 884 mov %o1, %o0 885 SET_SIZE(sfmmu_modifytte_try) 886 887 ENTRY_NP(sfmmu_copytte) 888 ldx [%o0], %g1 889 retl 890 stx %g1, [%o1] 891 SET_SIZE(sfmmu_copytte) 892 893 894 /* 895 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 896 * %o0 = TSB base address (in), pointer to TSB entry (out) 897 * %o1 = vaddr (in) 898 * %o2 = vpshift (in) 899 * %o3 = tsb size code (in) 900 * %o4 = scratch register 901 */ 902 ENTRY_NP(sfmmu_get_tsbe) 903 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 904 retl 905 nop 906 SET_SIZE(sfmmu_get_tsbe) 907 908 /* 909 * Return a TSB tag for the given va. 910 * %o0 = va (in/clobbered) 911 * %o0 = va shifted to be in tsb tag format (with no context) (out) 912 */ 913 ENTRY_NP(sfmmu_make_tsbtag) 914 retl 915 srln %o0, TTARGET_VA_SHIFT, %o0 916 SET_SIZE(sfmmu_make_tsbtag) 917 918#endif /* lint */ 919 920/* 921 * Other sfmmu primitives 922 */ 923 924 925#if defined (lint) 926void 927sfmmu_patch_ktsb(void) 928{ 929} 930 931void 932sfmmu_kpm_patch_tlbm(void) 933{ 934} 935 936void 937sfmmu_kpm_patch_tsbm(void) 938{ 939} 940 941void 942sfmmu_patch_shctx(void) 943{ 944} 945 946/* ARGSUSED */ 947void 948sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 949{ 950} 951 952/* ARGSUSED */ 953void 954sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 955{ 956} 957 958/* ARGSUSED */ 959void 960sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 961{ 962} 963 964/* ARGSUSED */ 965void 966sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 967{ 968} 969 970#else /* lint */ 971 972#define I_SIZE 4 973 974 ENTRY_NP(sfmmu_fix_ktlb_traptable) 975 /* 976 * %o0 = start of patch area 977 * %o1 = size code of TSB to patch 978 * %o3 = scratch 979 */ 980 /* fix sll */ 981 ld [%o0], %o3 /* get sll */ 982 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 983 st %o3, [%o0] /* write sll */ 984 flush %o0 985 /* fix srl */ 986 add %o0, I_SIZE, %o0 /* goto next instr. */ 987 ld [%o0], %o3 /* get srl */ 988 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 989 st %o3, [%o0] /* write srl */ 990 retl 991 flush %o0 992 SET_SIZE(sfmmu_fix_ktlb_traptable) 993 994 ENTRY_NP(sfmmu_fixup_ktsbbase) 995 /* 996 * %o0 = start of patch area 997 * %o5 = kernel virtual or physical tsb base address 998 * %o2, %o3 are used as scratch registers. 999 */ 1000 /* fixup sethi instruction */ 1001 ld [%o0], %o3 1002 srl %o5, 10, %o2 ! offset is bits 32:10 1003 or %o3, %o2, %o3 ! set imm22 1004 st %o3, [%o0] 1005 /* fixup offset of lduw/ldx */ 1006 add %o0, I_SIZE, %o0 ! next instr 1007 ld [%o0], %o3 1008 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 1009 or %o3, %o2, %o3 1010 st %o3, [%o0] 1011 retl 1012 flush %o0 1013 SET_SIZE(sfmmu_fixup_ktsbbase) 1014 1015 ENTRY_NP(sfmmu_fixup_setx) 1016 /* 1017 * %o0 = start of patch area 1018 * %o4 = 64 bit value to patch 1019 * %o2, %o3 are used as scratch registers. 1020 * 1021 * Note: Assuming that all parts of the instructions which need to be 1022 * patched correspond to RUNTIME_PATCH (aka 0) 1023 * 1024 * Note the implementation of setx which is being patched is as follows: 1025 * 1026 * sethi %hh(RUNTIME_PATCH), tmp 1027 * sethi %lm(RUNTIME_PATCH), dest 1028 * or tmp, %hm(RUNTIME_PATCH), tmp 1029 * or dest, %lo(RUNTIME_PATCH), dest 1030 * sllx tmp, 32, tmp 1031 * nop 1032 * or tmp, dest, dest 1033 * 1034 * which differs from the implementation in the 1035 * "SPARC Architecture Manual" 1036 */ 1037 /* fixup sethi instruction */ 1038 ld [%o0], %o3 1039 srlx %o4, 42, %o2 ! bits [63:42] 1040 or %o3, %o2, %o3 ! set imm22 1041 st %o3, [%o0] 1042 /* fixup sethi instruction */ 1043 add %o0, I_SIZE, %o0 ! next instr 1044 ld [%o0], %o3 1045 sllx %o4, 32, %o2 ! clear upper bits 1046 srlx %o2, 42, %o2 ! bits [31:10] 1047 or %o3, %o2, %o3 ! set imm22 1048 st %o3, [%o0] 1049 /* fixup or instruction */ 1050 add %o0, I_SIZE, %o0 ! next instr 1051 ld [%o0], %o3 1052 srlx %o4, 32, %o2 ! bits [63:32] 1053 and %o2, 0x3ff, %o2 ! bits [41:32] 1054 or %o3, %o2, %o3 ! set imm 1055 st %o3, [%o0] 1056 /* fixup or instruction */ 1057 add %o0, I_SIZE, %o0 ! next instr 1058 ld [%o0], %o3 1059 and %o4, 0x3ff, %o2 ! bits [9:0] 1060 or %o3, %o2, %o3 ! set imm 1061 st %o3, [%o0] 1062 retl 1063 flush %o0 1064 SET_SIZE(sfmmu_fixup_setx) 1065 1066 ENTRY_NP(sfmmu_fixup_or) 1067 /* 1068 * %o0 = start of patch area 1069 * %o4 = 32 bit value to patch 1070 * %o2, %o3 are used as scratch registers. 1071 * Note: Assuming that all parts of the instructions which need to be 1072 * patched correspond to RUNTIME_PATCH (aka 0) 1073 */ 1074 ld [%o0], %o3 1075 and %o4, 0x3ff, %o2 ! bits [9:0] 1076 or %o3, %o2, %o3 ! set imm 1077 st %o3, [%o0] 1078 retl 1079 flush %o0 1080 SET_SIZE(sfmmu_fixup_or) 1081 1082 ENTRY_NP(sfmmu_fixup_shiftx) 1083 /* 1084 * %o0 = start of patch area 1085 * %o4 = signed int immediate value to add to sllx/srlx imm field 1086 * %o2, %o3 are used as scratch registers. 1087 * 1088 * sllx/srlx store the 6 bit immediate value in the lowest order bits 1089 * so we do a simple add. The caller must be careful to prevent 1090 * overflow, which could easily occur if the initial value is nonzero! 1091 */ 1092 ld [%o0], %o3 ! %o3 = instruction to patch 1093 and %o3, 0x3f, %o2 ! %o2 = existing imm value 1094 add %o2, %o4, %o2 ! %o2 = new imm value 1095 andn %o3, 0x3f, %o3 ! clear old imm value 1096 and %o2, 0x3f, %o2 ! truncate new imm value 1097 or %o3, %o2, %o3 ! set new imm value 1098 st %o3, [%o0] ! store updated instruction 1099 retl 1100 flush %o0 1101 SET_SIZE(sfmmu_fixup_shiftx) 1102 1103 ENTRY_NP(sfmmu_fixup_mmu_asi) 1104 /* 1105 * Patch imm_asi of all ldda instructions in the MMU 1106 * trap handlers. We search MMU_PATCH_INSTR instructions 1107 * starting from the itlb miss handler (trap 0x64). 1108 * %o0 = address of tt[0,1]_itlbmiss 1109 * %o1 = imm_asi to setup, shifted by appropriate offset. 1110 * %o3 = number of instructions to search 1111 * %o4 = reserved by caller: called from leaf routine 1112 */ 11131: ldsw [%o0], %o2 ! load instruction to %o2 1114 brgez,pt %o2, 2f 1115 srl %o2, 30, %o5 1116 btst 1, %o5 ! test bit 30; skip if not set 1117 bz,pt %icc, 2f 1118 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1119 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1120 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1121 brnz,pt %o5, 2f ! skip if not a match 1122 or %o2, %o1, %o2 ! or in imm_asi 1123 st %o2, [%o0] ! write patched instruction 11242: dec %o3 1125 brnz,a,pt %o3, 1b ! loop until we're done 1126 add %o0, I_SIZE, %o0 1127 retl 1128 flush %o0 1129 SET_SIZE(sfmmu_fixup_mmu_asi) 1130 1131 /* 1132 * Patch immediate ASI used to access the TSB in the 1133 * trap table. 1134 * inputs: %o0 = value of ktsb_phys 1135 */ 1136 ENTRY_NP(sfmmu_patch_mmu_asi) 1137 mov %o7, %o4 ! save return pc in %o4 1138 movrnz %o0, ASI_QUAD_LDD_PHYS, %o3 1139 movrz %o0, ASI_NQUAD_LD, %o3 1140 sll %o3, 5, %o1 ! imm_asi offset 1141 mov 6, %o3 ! number of instructions 1142 sethi %hi(dktsb), %o0 ! to search 1143 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1144 or %o0, %lo(dktsb), %o0 1145 mov 6, %o3 ! number of instructions 1146 sethi %hi(dktsb4m), %o0 ! to search 1147 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1148 or %o0, %lo(dktsb4m), %o0 1149 mov 6, %o3 ! number of instructions 1150 sethi %hi(iktsb), %o0 ! to search 1151 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1152 or %o0, %lo(iktsb), %o0 1153 mov 6, %o3 ! number of instructions 1154 sethi %hi(iktsb4m), %o0 ! to search 1155 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1156 or %o0, %lo(iktsb4m), %o0 1157 mov %o4, %o7 ! retore return pc -- leaf 1158 retl 1159 nop 1160 SET_SIZE(sfmmu_patch_mmu_asi) 1161 1162 ENTRY_NP(sfmmu_patch_ktsb) 1163 /* 1164 * We need to fix iktsb, dktsb, et. al. 1165 */ 1166 save %sp, -SA(MINFRAME), %sp 1167 set ktsb_phys, %o1 1168 ld [%o1], %o4 1169 set ktsb_base, %o5 1170 set ktsb4m_base, %l1 1171 brz,pt %o4, 1f 1172 nop 1173 set ktsb_pbase, %o5 1174 set ktsb4m_pbase, %l1 11751: 1176 sethi %hi(ktsb_szcode), %o1 1177 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1178 1179 sethi %hi(iktsb), %o0 1180 call sfmmu_fix_ktlb_traptable 1181 or %o0, %lo(iktsb), %o0 1182 1183 sethi %hi(dktsb), %o0 1184 call sfmmu_fix_ktlb_traptable 1185 or %o0, %lo(dktsb), %o0 1186 1187 sethi %hi(ktsb4m_szcode), %o1 1188 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1189 1190 sethi %hi(iktsb4m), %o0 1191 call sfmmu_fix_ktlb_traptable 1192 or %o0, %lo(iktsb4m), %o0 1193 1194 sethi %hi(dktsb4m), %o0 1195 call sfmmu_fix_ktlb_traptable 1196 or %o0, %lo(dktsb4m), %o0 1197 1198#ifndef sun4v 1199 mov ASI_N, %o2 1200 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1201 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1202 sethi %hi(tsb_kernel_patch_asi), %o0 1203 call sfmmu_fixup_or 1204 or %o0, %lo(tsb_kernel_patch_asi), %o0 1205#endif /* !sun4v */ 1206 1207 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1208 1209 sethi %hi(dktsbbase), %o0 1210 call sfmmu_fixup_setx ! patch value of ktsb base addr 1211 or %o0, %lo(dktsbbase), %o0 1212 1213 sethi %hi(iktsbbase), %o0 1214 call sfmmu_fixup_setx ! patch value of ktsb base addr 1215 or %o0, %lo(iktsbbase), %o0 1216 1217 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1218 call sfmmu_fixup_setx ! patch value of ktsb base addr 1219 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1220 1221#ifdef sun4v 1222 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1223 call sfmmu_fixup_setx ! patch value of ktsb base addr 1224 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1225#endif /* sun4v */ 1226 1227 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1228 1229 sethi %hi(dktsb4mbase), %o0 1230 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1231 or %o0, %lo(dktsb4mbase), %o0 1232 1233 sethi %hi(iktsb4mbase), %o0 1234 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1235 or %o0, %lo(iktsb4mbase), %o0 1236 1237 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1238 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1239 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1240 1241#ifdef sun4v 1242 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1243 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1244 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1245#endif /* sun4v */ 1246 1247 set ktsb_szcode, %o4 1248 ld [%o4], %o4 1249 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1250 call sfmmu_fixup_or ! patch value of ktsb_szcode 1251 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1252 1253#ifdef sun4v 1254 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1255 call sfmmu_fixup_or ! patch value of ktsb_szcode 1256 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1257#endif /* sun4v */ 1258 1259 set ktsb4m_szcode, %o4 1260 ld [%o4], %o4 1261 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1262 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1263 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1264 1265#ifdef sun4v 1266 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1267 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1268 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1269#endif /* sun4v */ 1270 1271 ret 1272 restore 1273 SET_SIZE(sfmmu_patch_ktsb) 1274 1275 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1276 /* 1277 * Fixup trap handlers in common segkpm case. This is reserved 1278 * for future use should kpm TSB be changed to be other than the 1279 * kernel TSB. 1280 */ 1281 retl 1282 nop 1283 SET_SIZE(sfmmu_kpm_patch_tlbm) 1284 1285 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1286 /* 1287 * nop the branch to sfmmu_kpm_dtsb_miss_small 1288 * in the case where we are using large pages for 1289 * seg_kpm (and hence must probe the second TSB for 1290 * seg_kpm VAs) 1291 */ 1292 set dktsb4m_kpmcheck_small, %o0 1293 MAKE_NOP_INSTR(%o1) 1294 st %o1, [%o0] 1295 flush %o0 1296 retl 1297 nop 1298 SET_SIZE(sfmmu_kpm_patch_tsbm) 1299 1300 ENTRY_NP(sfmmu_patch_utsb) 1301#ifdef UTSB_PHYS 1302 retl 1303 nop 1304#else /* UTSB_PHYS */ 1305 /* 1306 * We need to hot patch utsb_vabase and utsb4m_vabase 1307 */ 1308 save %sp, -SA(MINFRAME), %sp 1309 1310 /* patch value of utsb_vabase */ 1311 set utsb_vabase, %o1 1312 ldx [%o1], %o4 1313 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1314 call sfmmu_fixup_setx 1315 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1316 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1317 call sfmmu_fixup_setx 1318 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1319 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1320 call sfmmu_fixup_setx 1321 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1322 1323 /* patch value of utsb4m_vabase */ 1324 set utsb4m_vabase, %o1 1325 ldx [%o1], %o4 1326 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1327 call sfmmu_fixup_setx 1328 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1329 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1330 call sfmmu_fixup_setx 1331 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1332 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1333 call sfmmu_fixup_setx 1334 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1335 1336 /* 1337 * Patch TSB base register masks and shifts if needed. 1338 * By default the TSB base register contents are set up for 4M slab. 1339 * If we're using a smaller slab size and reserved VA range we need 1340 * to patch up those values here. 1341 */ 1342 set tsb_slab_shift, %o1 1343 set MMU_PAGESHIFT4M, %o4 1344 lduw [%o1], %o3 1345 subcc %o4, %o3, %o4 1346 bz,pt %icc, 1f 1347 /* delay slot safe */ 1348 1349 /* patch reserved VA range size if needed. */ 1350 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1351 call sfmmu_fixup_shiftx 1352 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1353 call sfmmu_fixup_shiftx 1354 add %o0, I_SIZE, %o0 1355 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1356 call sfmmu_fixup_shiftx 1357 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1358 call sfmmu_fixup_shiftx 1359 add %o0, I_SIZE, %o0 13601: 1361 /* patch TSBREG_VAMASK used to set up TSB base register */ 1362 set tsb_slab_mask, %o1 1363 ldx [%o1], %o4 1364 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1365 call sfmmu_fixup_or 1366 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1367 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1368 call sfmmu_fixup_or 1369 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1370 1371 ret 1372 restore 1373#endif /* UTSB_PHYS */ 1374 SET_SIZE(sfmmu_patch_utsb) 1375 1376 ENTRY_NP(sfmmu_patch_shctx) 1377#ifdef sun4u 1378 retl 1379 nop 1380#else /* sun4u */ 1381 set sfmmu_shctx_cpu_mondo_patch, %o0 1382 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1383 st %o1, [%o0] 1384 flush %o0 1385 MAKE_NOP_INSTR(%o1) 1386 add %o0, I_SIZE, %o0 ! next instr 1387 st %o1, [%o0] 1388 flush %o0 1389 1390 set sfmmu_shctx_user_rtt_patch, %o0 1391 st %o1, [%o0] ! nop 1st instruction 1392 flush %o0 1393 add %o0, I_SIZE, %o0 1394 st %o1, [%o0] ! nop 2nd instruction 1395 flush %o0 1396 add %o0, I_SIZE, %o0 1397 st %o1, [%o0] ! nop 3rd instruction 1398 flush %o0 1399 add %o0, I_SIZE, %o0 1400 st %o1, [%o0] ! nop 4th instruction 1401 flush %o0 1402 add %o0, I_SIZE, %o0 1403 st %o1, [%o0] ! nop 5th instruction 1404 retl 1405 flush %o0 1406#endif /* sun4u */ 1407 SET_SIZE(sfmmu_patch_shctx) 1408 1409 /* 1410 * Routine that loads an entry into a tsb using virtual addresses. 1411 * Locking is required since all cpus can use the same TSB. 1412 * Note that it is no longer required to have a valid context 1413 * when calling this function. 1414 */ 1415 ENTRY_NP(sfmmu_load_tsbe) 1416 /* 1417 * %o0 = pointer to tsbe to load 1418 * %o1 = tsb tag 1419 * %o2 = virtual pointer to TTE 1420 * %o3 = 1 if physical address in %o0 else 0 1421 */ 1422 rdpr %pstate, %o5 1423#ifdef DEBUG 1424 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1425#endif /* DEBUG */ 1426 1427 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1428 1429 SETUP_TSB_ASI(%o3, %g3) 1430 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1) 1431 1432 wrpr %g0, %o5, %pstate /* enable interrupts */ 1433 1434 retl 1435 membar #StoreStore|#StoreLoad 1436 SET_SIZE(sfmmu_load_tsbe) 1437 1438 /* 1439 * Flush TSB of a given entry if the tag matches. 1440 */ 1441 ENTRY(sfmmu_unload_tsbe) 1442 /* 1443 * %o0 = pointer to tsbe to be flushed 1444 * %o1 = tag to match 1445 * %o2 = 1 if physical address in %o0 else 0 1446 */ 1447 SETUP_TSB_ASI(%o2, %g1) 1448 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1449 retl 1450 membar #StoreStore|#StoreLoad 1451 SET_SIZE(sfmmu_unload_tsbe) 1452 1453 /* 1454 * Routine that loads a TTE into the kpm TSB from C code. 1455 * Locking is required since kpm TSB is shared among all CPUs. 1456 */ 1457 ENTRY_NP(sfmmu_kpm_load_tsb) 1458 /* 1459 * %o0 = vaddr 1460 * %o1 = ttep 1461 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1462 */ 1463 rdpr %pstate, %o5 ! %o5 = saved pstate 1464#ifdef DEBUG 1465 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1466#endif /* DEBUG */ 1467 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1468 1469#ifndef sun4v 1470 sethi %hi(ktsb_phys), %o4 1471 mov ASI_N, %o3 1472 ld [%o4 + %lo(ktsb_phys)], %o4 1473 movrnz %o4, ASI_MEM, %o3 1474 mov %o3, %asi 1475#endif /* !sun4v */ 1476 mov %o0, %g1 ! %g1 = vaddr 1477 1478 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1479 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1480 /* %g2 = tsbep, %g1 clobbered */ 1481 1482 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1483 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1484 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1) 1485 1486 wrpr %g0, %o5, %pstate ! enable interrupts 1487 retl 1488 membar #StoreStore|#StoreLoad 1489 SET_SIZE(sfmmu_kpm_load_tsb) 1490 1491 /* 1492 * Routine that shoots down a TTE in the kpm TSB or in the 1493 * kernel TSB depending on virtpg. Locking is required since 1494 * kpm/kernel TSB is shared among all CPUs. 1495 */ 1496 ENTRY_NP(sfmmu_kpm_unload_tsb) 1497 /* 1498 * %o0 = vaddr 1499 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1500 */ 1501#ifndef sun4v 1502 sethi %hi(ktsb_phys), %o4 1503 mov ASI_N, %o3 1504 ld [%o4 + %lo(ktsb_phys)], %o4 1505 movrnz %o4, ASI_MEM, %o3 1506 mov %o3, %asi 1507#endif /* !sun4v */ 1508 mov %o0, %g1 ! %g1 = vaddr 1509 1510 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1511 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1512 /* %g2 = tsbep, %g1 clobbered */ 1513 1514 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1515 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1516 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1517 1518 retl 1519 membar #StoreStore|#StoreLoad 1520 SET_SIZE(sfmmu_kpm_unload_tsb) 1521 1522#endif /* lint */ 1523 1524 1525#if defined (lint) 1526 1527/*ARGSUSED*/ 1528pfn_t 1529sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1530{ return(0); } 1531 1532#else /* lint */ 1533 1534 ENTRY_NP(sfmmu_ttetopfn) 1535 ldx [%o0], %g1 /* read tte */ 1536 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1537 /* 1538 * g1 = pfn 1539 */ 1540 retl 1541 mov %g1, %o0 1542 SET_SIZE(sfmmu_ttetopfn) 1543 1544#endif /* !lint */ 1545 1546 1547#if defined (lint) 1548/* 1549 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the 1550 * the hash list. 1551 */ 1552/* ARGSUSED */ 1553void 1554sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1555 uint64_t hblkpa) 1556{ 1557} 1558 1559/* 1560 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the 1561 * hash list. 1562 */ 1563/* ARGSUSED */ 1564void 1565sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1566 uint64_t hblkpa, struct hme_blk *prev_hblkp) 1567{ 1568} 1569#else /* lint */ 1570 1571/* 1572 * Functions to grab/release hme bucket list lock. I only use a byte 1573 * instead of the whole int because eventually we might want to 1574 * put some counters on the other bytes (of course, these routines would 1575 * have to change). The code that grab this lock should execute 1576 * with interrupts disabled and hold the lock for the least amount of time 1577 * possible. 1578 */ 1579 1580/* 1581 * Even though hmeh_listlock is updated using pa there's no need to flush 1582 * dcache since hmeh_listlock will be restored to the original value (0) 1583 * before interrupts are reenabled. 1584 */ 1585 1586/* 1587 * For sparcv9 hme hash buckets may not be in the nucleus. hme hash update 1588 * routines still use virtual addresses to update the bucket fields. But they 1589 * must not cause a TLB miss after grabbing the low level bucket lock. To 1590 * achieve this we must make sure the bucket structure is completely within an 1591 * 8K page. 1592 */ 1593 1594#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1)) 1595#error - the size of hmehash_bucket structure is not power of 2 1596#endif 1597 1598/* 1599 * Enable backoff to significantly reduce locking overhead and reduce a chance 1600 * of xcall timeout. This is only enabled for sun4v as a Makefile compile- 1601 * time option. 1602 * The rd %ccr is better for performance compared to a non pipeline releasing 1603 * tight spin on N2/VF. 1604 * Backoff based fix is a temporary solution and doesn't allow scaling above 1605 * lock saturation point. The final fix is to eliminate HMELOCK_ENTER() 1606 * to avoid xcall timeouts and improve GET_TTE() performance. 1607 */ 1608 1609#ifdef HMELOCK_BACKOFF_ENABLE 1610 1611#define HMELOCK_BACKOFF(reg, val) \ 1612 set val, reg ;\ 1613 rd %ccr, %g0 ;\ 1614 brnz reg, .-4 ;\ 1615 dec reg 1616 1617#define CAS_HME(tmp1, tmp2, exitlabel, asi) \ 1618 mov 0xff, tmp2 ;\ 1619 casa [tmp1]asi, %g0, tmp2 ;\ 1620 brz,a,pt tmp2, exitlabel ;\ 1621 membar #LoadLoad 1622 1623#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label, asi) \ 1624 mov 0xff, tmp2 ;\ 1625 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1626 casa [tmp1]asi, %g0, tmp2 ;\ 1627 brz,a,pt tmp2, label/**/2 ;\ 1628 membar #LoadLoad ;\ 1629 HMELOCK_BACKOFF(tmp2,0x8) ;\ 1630 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1631 HMELOCK_BACKOFF(tmp2,0x10) ;\ 1632 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1633 HMELOCK_BACKOFF(tmp2,0x20) ;\ 1634 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1635 HMELOCK_BACKOFF(tmp2,0x40) ;\ 1636 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1637 HMELOCK_BACKOFF(tmp2,0x80) ;\ 1638 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1639label/**/1: ;\ 1640 HMELOCK_BACKOFF(tmp2,0x100) ;\ 1641 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1642 HMELOCK_BACKOFF(tmp2,0x200) ;\ 1643 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1644 HMELOCK_BACKOFF(tmp2,0x400) ;\ 1645 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1646 HMELOCK_BACKOFF(tmp2,0x800) ;\ 1647 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1648 HMELOCK_BACKOFF(tmp2,0x1000) ;\ 1649 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1650 HMELOCK_BACKOFF(tmp2,0x2000) ;\ 1651 mov 0xff, tmp2 ;\ 1652 casa [tmp1]asi, %g0, tmp2 ;\ 1653 brnz,pn tmp2, label/**/1 /* reset backoff */ ;\ 1654 membar #LoadLoad ;\ 1655label/**/2: 1656 1657#else /* HMELOCK_BACKOFF_ENABLE */ 1658 1659#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi) \ 1660 mov 0xff, tmp2 ;\ 1661 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1662label1: ;\ 1663 casa [tmp1]asi, %g0, tmp2 ;\ 1664 brnz,pn tmp2, label1 ;\ 1665 mov 0xff, tmp2 ;\ 1666 membar #LoadLoad 1667 1668#endif /* HMELOCK_BACKOFF_ENABLE */ 1669 1670#define HMELOCK_EXIT(hmebp, tmp1, asi) \ 1671 membar #LoadStore|#StoreStore ;\ 1672 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1673 sta %g0, [tmp1]asi 1674 1675 .seg ".data" 1676hblk_add_panic1: 1677 .ascii "sfmmu_hblk_hash_add: interrupts disabled" 1678 .byte 0 1679hblk_add_panic2: 1680 .ascii "sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not" 1681 .byte 0 1682 .align 4 1683 .seg ".text" 1684 1685 ENTRY_NP(sfmmu_hblk_hash_add) 1686 /* 1687 * %o0 = hmebp 1688 * %o1 = hmeblkp 1689 * %o2 = hblkpa 1690 */ 1691 rdpr %pstate, %o5 1692#ifdef DEBUG 1693 andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ 1694 bnz,pt %icc, 3f /* disabled, panic */ 1695 nop 1696 save %sp, -SA(MINFRAME), %sp 1697 sethi %hi(hblk_add_panic1), %o0 1698 call panic 1699 or %o0, %lo(hblk_add_panic1), %o0 1700 ret 1701 restore 1702 17033: 1704#endif /* DEBUG */ 1705 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1706 mov %o2, %g1 1707 1708 /* 1709 * g1 = hblkpa 1710 */ 1711 ldn [%o0 + HMEBUCK_HBLK], %o4 /* next hmeblk */ 1712 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = next hblkpa */ 1713#ifdef DEBUG 1714 cmp %o4, %g0 1715 bne,pt %xcc, 1f 1716 nop 1717 brz,pt %g2, 1f 1718 nop 1719 wrpr %g0, %o5, %pstate /* enable interrupts */ 1720 save %sp, -SA(MINFRAME), %sp 1721 sethi %hi(hblk_add_panic2), %o0 1722 call panic 1723 or %o0, %lo(hblk_add_panic2), %o0 1724 ret 1725 restore 17261: 1727#endif /* DEBUG */ 1728 /* 1729 * We update hmeblks entries before grabbing lock because the stores 1730 * could take a tlb miss and require the hash lock. The buckets 1731 * are part of the nucleus so we are cool with those stores. 1732 * 1733 * if buckets are not part of the nucleus our game is to 1734 * not touch any other page via va until we drop the lock. 1735 * This guarantees we won't get a tlb miss before the lock release 1736 * since interrupts are disabled. 1737 */ 1738 stn %o4, [%o1 + HMEBLK_NEXT] /* update hmeblk's next */ 1739 stx %g2, [%o1 + HMEBLK_NEXTPA] /* update hmeblk's next pa */ 1740 HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N) 1741 stn %o1, [%o0 + HMEBUCK_HBLK] /* update bucket hblk next */ 1742 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* add hmeblk to list */ 1743 HMELOCK_EXIT(%o0, %g2, ASI_N) 1744 retl 1745 wrpr %g0, %o5, %pstate /* enable interrupts */ 1746 SET_SIZE(sfmmu_hblk_hash_add) 1747 1748 ENTRY_NP(sfmmu_hblk_hash_rm) 1749 /* 1750 * This function removes an hmeblk from the hash chain. 1751 * It is written to guarantee we don't take a tlb miss 1752 * by using physical addresses to update the list. 1753 * 1754 * %o0 = hmebp 1755 * %o1 = hmeblkp 1756 * %o2 = hmeblkp previous pa 1757 * %o3 = hmeblkp previous 1758 */ 1759 1760 mov %o3, %o4 /* o4 = hmeblkp previous */ 1761 1762 rdpr %pstate, %o5 1763#ifdef DEBUG 1764 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1) 1765#endif /* DEBUG */ 1766 /* 1767 * disable interrupts, clear Address Mask to access 64 bit physaddr 1768 */ 1769 andn %o5, PSTATE_IE, %g1 1770 wrpr %g1, 0, %pstate 1771 1772#ifndef sun4v 1773 sethi %hi(dcache_line_mask), %g4 1774 ld [%g4 + %lo(dcache_line_mask)], %g4 1775#endif /* sun4v */ 1776 1777 /* 1778 * if buckets are not part of the nucleus our game is to 1779 * not touch any other page via va until we drop the lock. 1780 * This guarantees we won't get a tlb miss before the lock release 1781 * since interrupts are disabled. 1782 */ 1783 HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N) 1784 ldn [%o0 + HMEBUCK_HBLK], %g2 /* first hmeblk in list */ 1785 cmp %g2, %o1 1786 bne,pt %ncc,1f 1787 mov ASI_MEM, %asi 1788 /* 1789 * hmeblk is first on list 1790 */ 1791 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = hmeblk pa */ 1792 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1793 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1794 stn %o3, [%o0 + HMEBUCK_HBLK] /* write va */ 1795 ba,pt %xcc, 2f 1796 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* write pa */ 17971: 1798 /* hmeblk is not first on list */ 1799 1800 mov %o2, %g3 1801#ifndef sun4v 1802 GET_CPU_IMPL(%g2) 1803 cmp %g2, CHEETAH_IMPL 1804 bge,a,pt %icc, hblk_hash_rm_1 1805 and %o4, %g4, %g2 1806 cmp %g2, SPITFIRE_IMPL 1807 blt %icc, hblk_hash_rm_2 /* no flushing needed for OPL */ 1808 and %o4, %g4, %g2 1809 stxa %g0, [%g2]ASI_DC_TAG /* flush prev pa from dcache */ 1810 add %o4, HMEBLK_NEXT, %o4 1811 and %o4, %g4, %g2 1812 ba hblk_hash_rm_2 1813 stxa %g0, [%g2]ASI_DC_TAG /* flush prev va from dcache */ 1814hblk_hash_rm_1: 1815 1816 stxa %g0, [%g3]ASI_DC_INVAL /* flush prev pa from dcache */ 1817 membar #Sync 1818 add %g3, HMEBLK_NEXT, %g2 1819 stxa %g0, [%g2]ASI_DC_INVAL /* flush prev va from dcache */ 1820hblk_hash_rm_2: 1821 membar #Sync 1822#endif /* sun4v */ 1823 ldxa [%g3 + HMEBLK_NEXTPA] %asi, %g2 /* g2 = hmeblk pa */ 1824 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1825 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1826 stna %o3, [%g3 + HMEBLK_NEXT] %asi /* write va */ 1827 stxa %g1, [%g3 + HMEBLK_NEXTPA] %asi /* write pa */ 18282: 1829 HMELOCK_EXIT(%o0, %g2, ASI_N) 1830 retl 1831 wrpr %g0, %o5, %pstate /* enable interrupts */ 1832 SET_SIZE(sfmmu_hblk_hash_rm) 1833 1834#endif /* lint */ 1835 1836/* 1837 * These macros are used to update global sfmmu hme hash statistics 1838 * in perf critical paths. It is only enabled in debug kernels or 1839 * if SFMMU_STAT_GATHER is defined 1840 */ 1841#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1842#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1843 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1844 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1845 cmp tmp1, hatid ;\ 1846 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1847 set sfmmu_global_stat, tmp1 ;\ 1848 add tmp1, tmp2, tmp1 ;\ 1849 ld [tmp1], tmp2 ;\ 1850 inc tmp2 ;\ 1851 st tmp2, [tmp1] 1852 1853#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1854 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1855 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1856 cmp tmp1, hatid ;\ 1857 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1858 set sfmmu_global_stat, tmp1 ;\ 1859 add tmp1, tmp2, tmp1 ;\ 1860 ld [tmp1], tmp2 ;\ 1861 inc tmp2 ;\ 1862 st tmp2, [tmp1] 1863 1864 1865#else /* DEBUG || SFMMU_STAT_GATHER */ 1866 1867#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1868 1869#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1870 1871#endif /* DEBUG || SFMMU_STAT_GATHER */ 1872 1873/* 1874 * This macro is used to update global sfmmu kstas in non 1875 * perf critical areas so they are enabled all the time 1876 */ 1877#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1878 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1879 add tmp1, statname, tmp1 ;\ 1880 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1881 inc tmp2 ;\ 1882 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1883 1884/* 1885 * These macros are used to update per cpu stats in non perf 1886 * critical areas so they are enabled all the time 1887 */ 1888#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1889 ld [tsbarea + stat], tmp1 ;\ 1890 inc tmp1 ;\ 1891 st tmp1, [tsbarea + stat] 1892 1893/* 1894 * These macros are used to update per cpu stats in non perf 1895 * critical areas so they are enabled all the time 1896 */ 1897#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1898 lduh [tsbarea + stat], tmp1 ;\ 1899 inc tmp1 ;\ 1900 stuh tmp1, [tsbarea + stat] 1901 1902#if defined(KPM_TLBMISS_STATS_GATHER) 1903 /* 1904 * Count kpm dtlb misses separately to allow a different 1905 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1906 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1907 */ 1908#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1909 brgez tagacc, label /* KPM VA? */ ;\ 1910 nop ;\ 1911 CPU_INDEX(tmp1, tsbma) ;\ 1912 sethi %hi(kpmtsbm_area), tsbma ;\ 1913 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1914 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1915 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1916 /* VA range check */ ;\ 1917 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1918 cmp tagacc, val ;\ 1919 blu,pn %xcc, label ;\ 1920 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1921 cmp tagacc, tmp1 ;\ 1922 bgeu,pn %xcc, label ;\ 1923 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1924 inc val ;\ 1925 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1926label: 1927#else 1928#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1929#endif /* KPM_TLBMISS_STATS_GATHER */ 1930 1931#if defined (lint) 1932/* 1933 * The following routines are jumped to from the mmu trap handlers to do 1934 * the setting up to call systrap. They are separate routines instead of 1935 * being part of the handlers because the handlers would exceed 32 1936 * instructions and since this is part of the slow path the jump 1937 * cost is irrelevant. 1938 */ 1939void 1940sfmmu_pagefault(void) 1941{ 1942} 1943 1944void 1945sfmmu_mmu_trap(void) 1946{ 1947} 1948 1949void 1950sfmmu_window_trap(void) 1951{ 1952} 1953 1954void 1955sfmmu_kpm_exception(void) 1956{ 1957} 1958 1959#else /* lint */ 1960 1961#ifdef PTL1_PANIC_DEBUG 1962 .seg ".data" 1963 .global test_ptl1_panic 1964test_ptl1_panic: 1965 .word 0 1966 .align 8 1967 1968 .seg ".text" 1969 .align 4 1970#endif /* PTL1_PANIC_DEBUG */ 1971 1972 1973 ENTRY_NP(sfmmu_pagefault) 1974 SET_GL_REG(1) 1975 USE_ALTERNATE_GLOBALS(%g5) 1976 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1977 rdpr %tt, %g6 1978 cmp %g6, FAST_IMMU_MISS_TT 1979 be,a,pn %icc, 1f 1980 mov T_INSTR_MMU_MISS, %g3 1981 cmp %g6, T_INSTR_MMU_MISS 1982 be,a,pn %icc, 1f 1983 mov T_INSTR_MMU_MISS, %g3 1984 mov %g5, %g2 1985 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1986 cmp %g6, FAST_DMMU_MISS_TT 1987 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1988 cmp %g6, T_DATA_MMU_MISS 1989 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1990 1991#ifdef PTL1_PANIC_DEBUG 1992 /* check if we want to test the tl1 panic */ 1993 sethi %hi(test_ptl1_panic), %g4 1994 ld [%g4 + %lo(test_ptl1_panic)], %g1 1995 st %g0, [%g4 + %lo(test_ptl1_panic)] 1996 cmp %g1, %g0 1997 bne,a,pn %icc, ptl1_panic 1998 or %g0, PTL1_BAD_DEBUG, %g1 1999#endif /* PTL1_PANIC_DEBUG */ 20001: 2001 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 2002 /* 2003 * g2 = tag access reg 2004 * g3.l = type 2005 * g3.h = 0 2006 */ 2007 sethi %hi(trap), %g1 2008 or %g1, %lo(trap), %g1 20092: 2010 ba,pt %xcc, sys_trap 2011 mov -1, %g4 2012 SET_SIZE(sfmmu_pagefault) 2013 2014 ENTRY_NP(sfmmu_mmu_trap) 2015 SET_GL_REG(1) 2016 USE_ALTERNATE_GLOBALS(%g5) 2017 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 2018 rdpr %tt, %g6 2019 cmp %g6, FAST_IMMU_MISS_TT 2020 be,a,pn %icc, 1f 2021 mov T_INSTR_MMU_MISS, %g3 2022 cmp %g6, T_INSTR_MMU_MISS 2023 be,a,pn %icc, 1f 2024 mov T_INSTR_MMU_MISS, %g3 2025 mov %g5, %g2 2026 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 2027 cmp %g6, FAST_DMMU_MISS_TT 2028 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 2029 cmp %g6, T_DATA_MMU_MISS 2030 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 20311: 2032 /* 2033 * g2 = tag access reg 2034 * g3 = type 2035 */ 2036 sethi %hi(sfmmu_tsbmiss_exception), %g1 2037 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 2038 ba,pt %xcc, sys_trap 2039 mov -1, %g4 2040 /*NOTREACHED*/ 2041 SET_SIZE(sfmmu_mmu_trap) 2042 2043 ENTRY_NP(sfmmu_suspend_tl) 2044 SET_GL_REG(1) 2045 USE_ALTERNATE_GLOBALS(%g5) 2046 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 2047 rdpr %tt, %g6 2048 cmp %g6, FAST_IMMU_MISS_TT 2049 be,a,pn %icc, 1f 2050 mov T_INSTR_MMU_MISS, %g3 2051 mov %g5, %g2 2052 cmp %g6, FAST_DMMU_MISS_TT 2053 move %icc, T_DATA_MMU_MISS, %g3 2054 movne %icc, T_DATA_PROT, %g3 20551: 2056 sethi %hi(sfmmu_tsbmiss_suspended), %g1 2057 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 2058 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 2059 ba,pt %xcc, sys_trap 2060 mov PIL_15, %g4 2061 /*NOTREACHED*/ 2062 SET_SIZE(sfmmu_suspend_tl) 2063 2064 /* 2065 * No %g registers in use at this point. 2066 */ 2067 ENTRY_NP(sfmmu_window_trap) 2068 rdpr %tpc, %g1 2069#ifdef sun4v 2070#ifdef DEBUG 2071 /* We assume previous %gl was 1 */ 2072 rdpr %tstate, %g4 2073 srlx %g4, TSTATE_GL_SHIFT, %g4 2074 and %g4, TSTATE_GL_MASK, %g4 2075 cmp %g4, 1 2076 bne,a,pn %icc, ptl1_panic 2077 mov PTL1_BAD_WTRAP, %g1 2078#endif /* DEBUG */ 2079 /* user miss at tl>1. better be the window handler or user_rtt */ 2080 /* in user_rtt? */ 2081 set rtt_fill_start, %g4 2082 cmp %g1, %g4 2083 blu,pn %xcc, 6f 2084 .empty 2085 set rtt_fill_end, %g4 2086 cmp %g1, %g4 2087 bgeu,pn %xcc, 6f 2088 nop 2089 set fault_rtt_fn1, %g1 2090 wrpr %g0, %g1, %tnpc 2091 ba,a 7f 20926: 2093 ! must save this trap level before descending trap stack 2094 ! no need to save %tnpc, either overwritten or discarded 2095 ! already got it: rdpr %tpc, %g1 2096 rdpr %tstate, %g6 2097 rdpr %tt, %g7 2098 ! trap level saved, go get underlying trap type 2099 rdpr %tl, %g5 2100 sub %g5, 1, %g3 2101 wrpr %g3, %tl 2102 rdpr %tt, %g2 2103 wrpr %g5, %tl 2104 ! restore saved trap level 2105 wrpr %g1, %tpc 2106 wrpr %g6, %tstate 2107 wrpr %g7, %tt 2108#else /* sun4v */ 2109 /* user miss at tl>1. better be the window handler */ 2110 rdpr %tl, %g5 2111 sub %g5, 1, %g3 2112 wrpr %g3, %tl 2113 rdpr %tt, %g2 2114 wrpr %g5, %tl 2115#endif /* sun4v */ 2116 and %g2, WTRAP_TTMASK, %g4 2117 cmp %g4, WTRAP_TYPE 2118 bne,pn %xcc, 1f 2119 nop 2120 /* tpc should be in the trap table */ 2121 set trap_table, %g4 2122 cmp %g1, %g4 2123 blt,pn %xcc, 1f 2124 .empty 2125 set etrap_table, %g4 2126 cmp %g1, %g4 2127 bge,pn %xcc, 1f 2128 .empty 2129 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 2130 add %g1, WTRAP_FAULTOFF, %g1 2131 wrpr %g0, %g1, %tnpc 21327: 2133 /* 2134 * some wbuf handlers will call systrap to resolve the fault 2135 * we pass the trap type so they figure out the correct parameters. 2136 * g5 = trap type, g6 = tag access reg 2137 */ 2138 2139 /* 2140 * only use g5, g6, g7 registers after we have switched to alternate 2141 * globals. 2142 */ 2143 SET_GL_REG(1) 2144 USE_ALTERNATE_GLOBALS(%g5) 2145 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 2146 rdpr %tt, %g7 2147 cmp %g7, FAST_IMMU_MISS_TT 2148 be,a,pn %icc, ptl1_panic 2149 mov PTL1_BAD_WTRAP, %g1 2150 cmp %g7, T_INSTR_MMU_MISS 2151 be,a,pn %icc, ptl1_panic 2152 mov PTL1_BAD_WTRAP, %g1 2153 mov T_DATA_PROT, %g5 2154 cmp %g7, FAST_DMMU_MISS_TT 2155 move %icc, T_DATA_MMU_MISS, %g5 2156 cmp %g7, T_DATA_MMU_MISS 2157 move %icc, T_DATA_MMU_MISS, %g5 2158 ! XXXQ AGS re-check out this one 2159 done 21601: 2161 CPU_PADDR(%g1, %g4) 2162 add %g1, CPU_TL1_HDLR, %g1 2163 lda [%g1]ASI_MEM, %g4 2164 brnz,a,pt %g4, sfmmu_mmu_trap 2165 sta %g0, [%g1]ASI_MEM 2166 ba,pt %icc, ptl1_panic 2167 mov PTL1_BAD_TRAP, %g1 2168 SET_SIZE(sfmmu_window_trap) 2169 2170 ENTRY_NP(sfmmu_kpm_exception) 2171 /* 2172 * We have accessed an unmapped segkpm address or a legal segkpm 2173 * address which is involved in a VAC alias conflict prevention. 2174 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 2175 * set. If it is, we will instead note that a fault has occurred 2176 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 2177 * a "retry"). This will step over the faulting instruction. 2178 * Note that this means that a legal segkpm address involved in 2179 * a VAC alias conflict prevention (a rare case to begin with) 2180 * cannot be used in DTrace. 2181 */ 2182 CPU_INDEX(%g1, %g2) 2183 set cpu_core, %g2 2184 sllx %g1, CPU_CORE_SHIFT, %g1 2185 add %g1, %g2, %g1 2186 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 2187 andcc %g2, CPU_DTRACE_NOFAULT, %g0 2188 bz 0f 2189 or %g2, CPU_DTRACE_BADADDR, %g2 2190 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 2191 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 2192 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 2193 done 21940: 2195 TSTAT_CHECK_TL1(1f, %g1, %g2) 21961: 2197 SET_GL_REG(1) 2198 USE_ALTERNATE_GLOBALS(%g5) 2199 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 2200 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 2201 /* 2202 * g2=tagacc g3.l=type g3.h=0 2203 */ 2204 sethi %hi(trap), %g1 2205 or %g1, %lo(trap), %g1 2206 ba,pt %xcc, sys_trap 2207 mov -1, %g4 2208 SET_SIZE(sfmmu_kpm_exception) 2209 2210#endif /* lint */ 2211 2212#if defined (lint) 2213 2214void 2215sfmmu_tsb_miss(void) 2216{ 2217} 2218 2219void 2220sfmmu_kpm_dtsb_miss(void) 2221{ 2222} 2223 2224void 2225sfmmu_kpm_dtsb_miss_small(void) 2226{ 2227} 2228 2229#else /* lint */ 2230 2231#if (IMAP_SEG != 0) 2232#error - ism_map->ism_seg offset is not zero 2233#endif 2234 2235/* 2236 * Copies ism mapping for this ctx in param "ism" if this is a ISM 2237 * tlb miss and branches to label "ismhit". If this is not an ISM 2238 * process or an ISM tlb miss it falls thru. 2239 * 2240 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 2241 * this process. 2242 * If so, it will branch to label "ismhit". If not, it will fall through. 2243 * 2244 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 2245 * so that any other threads of this process will not try and walk the ism 2246 * maps while they are being changed. 2247 * 2248 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 2249 * will make sure of that. This means we can terminate our search on 2250 * the first zero mapping we find. 2251 * 2252 * Parameters: 2253 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 2254 * tsbmiss = address of tsb miss area (in) 2255 * ismseg = contents of ism_seg for this ism map (out) 2256 * ismhat = physical address of imap_ismhat for this ism map (out) 2257 * tmp1 = scratch reg (CLOBBERED) 2258 * tmp2 = scratch reg (CLOBBERED) 2259 * tmp3 = scratch reg (CLOBBERED) 2260 * label: temporary labels 2261 * ismhit: label where to jump to if an ism dtlb miss 2262 * exitlabel:label where to jump if hat is busy due to hat_unshare. 2263 */ 2264#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 2265 label, ismhit) \ 2266 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 2267 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 2268 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 2269label/**/1: ;\ 2270 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 2271 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 2272label/**/2: ;\ 2273 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 2274 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 2275 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 2276 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 2277 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 2278 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 2279 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 2280 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 2281 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 2282 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 2283 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 2284 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 2285 ;\ 2286 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 2287 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 2288 cmp ismhat, tmp1 ;\ 2289 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 2290 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 2291 ;\ 2292 add tmp3, IBLK_NEXTPA, tmp1 ;\ 2293 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 2294 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 2295 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 2296label/**/3: 2297 2298/* 2299 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 2300 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 2301 * Parameters: 2302 * tagacc = reg containing virtual address 2303 * hatid = reg containing sfmmu pointer 2304 * hmeshift = constant/register to shift vaddr to obtain vapg 2305 * hmebp = register where bucket pointer will be stored 2306 * vapg = register where virtual page will be stored 2307 * tmp1, tmp2 = tmp registers 2308 */ 2309 2310 2311#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 2312 vapg, label, tmp1, tmp2) \ 2313 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 2314 brnz,a,pt tmp1, label/**/1 ;\ 2315 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 2316 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 2317 ba,pt %xcc, label/**/2 ;\ 2318 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 2319label/**/1: ;\ 2320 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 2321label/**/2: ;\ 2322 srlx tagacc, hmeshift, vapg ;\ 2323 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 2324 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 2325 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 2326 add hmebp, tmp1, hmebp 2327 2328/* 2329 * hashtag includes bspage + hashno (64 bits). 2330 */ 2331 2332#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 2333 sllx vapg, hmeshift, vapg ;\ 2334 mov hashno, hblktag ;\ 2335 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 2336 or vapg, hblktag, hblktag 2337 2338/* 2339 * Function to traverse hmeblk hash link list and find corresponding match. 2340 * The search is done using physical pointers. It returns the physical address 2341 * and virtual address pointers to the hmeblk that matches with the tag 2342 * provided. 2343 * Parameters: 2344 * hmebp = register that points to hme hash bucket, also used as 2345 * tmp reg (clobbered) 2346 * hmeblktag = register with hmeblk tag match 2347 * hatid = register with hatid 2348 * hmeblkpa = register where physical ptr will be stored 2349 * hmeblkva = register where virtual ptr will be stored 2350 * tmp1 = tmp reg 2351 * label: temporary label 2352 */ 2353 2354#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2355 tsbarea, tmp1, label) \ 2356 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 2357 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2358 add hmebp, HMEBUCK_HBLK, hmeblkva ;\ 2359 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2360 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2361label/**/1: ;\ 2362 brz,pn hmeblkva, label/**/2 ;\ 2363 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2364 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 2365 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2366 add hmebp, CLONGSIZE, hmebp ;\ 2367 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 2368 xor tmp1, hmeblktag, tmp1 ;\ 2369 xor hmebp, hatid, hmebp ;\ 2370 or hmebp, tmp1, hmebp ;\ 2371 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 2372 add hmeblkpa, HMEBLK_NEXT, hmebp ;\ 2373 ldna [hmebp]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2374 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 2375 ba,pt %xcc, label/**/1 ;\ 2376 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2377label/**/2: 2378 2379/* 2380 * Function to traverse hmeblk hash link list and find corresponding match. 2381 * The search is done using physical pointers. It returns the physical address 2382 * and virtual address pointers to the hmeblk that matches with the tag 2383 * provided. 2384 * Parameters: 2385 * hmeblktag = register with hmeblk tag match (rid field is 0) 2386 * hatid = register with hatid (pointer to SRD) 2387 * hmeblkpa = register where physical ptr will be stored 2388 * hmeblkva = register where virtual ptr will be stored 2389 * tmp1 = tmp reg 2390 * tmp2 = tmp reg 2391 * label: temporary label 2392 */ 2393 2394#define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2395 tsbarea, tmp1, tmp2, label) \ 2396label/**/1: ;\ 2397 brz,pn hmeblkva, label/**/4 ;\ 2398 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 2399 add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 2400 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2401 add tmp2, CLONGSIZE, tmp2 ;\ 2402 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 2403 xor tmp1, hmeblktag, tmp1 ;\ 2404 xor tmp2, hatid, tmp2 ;\ 2405 brz,pn tmp2, label/**/3 /* branch on hit */ ;\ 2406 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2407label/**/2: ;\ 2408 ldna [tmp2]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2409 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2410 ba,pt %xcc, label/**/1 ;\ 2411 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2412label/**/3: ;\ 2413 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 2414 bgeu,pt %xcc, label/**/2 ;\ 2415 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2416 and tmp1, BT_ULMASK, tmp2 ;\ 2417 srlx tmp1, BT_ULSHIFT, tmp1 ;\ 2418 sllx tmp1, CLONGSHIFT, tmp1 ;\ 2419 add tsbarea, tmp1, tmp1 ;\ 2420 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 2421 srlx tmp1, tmp2, tmp1 ;\ 2422 btst 0x1, tmp1 ;\ 2423 bz,pn %xcc, label/**/2 ;\ 2424 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2425label/**/4: 2426 2427#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2428#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2429#endif 2430 2431/* 2432 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2433 * he offset for the corresponding hment. 2434 * Parameters: 2435 * In: 2436 * vaddr = register with virtual address 2437 * hmeblkpa = physical pointer to hme_blk 2438 * Out: 2439 * hmentoff = register where hment offset will be stored 2440 * hmemisc = hblk_misc 2441 * Scratch: 2442 * tmp1 2443 */ 2444#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2445 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2446 lda [hmentoff]ASI_MEM, hmemisc ;\ 2447 andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2448 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2449 or %g0, HMEBLK_HME1, hmentoff ;\ 2450 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2451 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2452 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2453 add tmp1, HMEBLK_HME1, hmentoff ;\ 2454label1: 2455 2456/* 2457 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2458 * 2459 * tagacc = (pseudo-)tag access register (in) 2460 * hatid = sfmmu pointer for TSB miss (in) 2461 * tte = tte for TLB miss if found, otherwise clobbered (out) 2462 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2463 * hmeblkva = VA of hment if found, otherwise clobbered (out) 2464 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2465 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2466 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2467 * for this page size. 2468 * hashno = constant/register hash number 2469 * label = temporary label for branching within macro. 2470 * foundlabel = label to jump to when tte is found. 2471 * suspendlabel= label to jump to when tte is suspended. 2472 * exitlabel = label to jump to when tte is not found. 2473 * 2474 */ 2475#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmemisc, \ 2476 hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \ 2477 ;\ 2478 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2479 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2480 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2481 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2482 ;\ 2483 /* ;\ 2484 * tagacc = tagacc ;\ 2485 * hatid = hatid ;\ 2486 * tsbarea = tsbarea ;\ 2487 * tte = hmebp (hme bucket pointer) ;\ 2488 * hmeblkpa = vapg (virtual page) ;\ 2489 * hmemisc, hmeblkva = scratch ;\ 2490 */ ;\ 2491 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2492 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2493 ;\ 2494 /* ;\ 2495 * tagacc = tagacc ;\ 2496 * hatid = hatid ;\ 2497 * tte = hmebp ;\ 2498 * hmeblkpa = CLOBBERED ;\ 2499 * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2500 * hmeblkva = scratch ;\ 2501 */ ;\ 2502 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2503 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2504 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, hmeblkva, \ 2505 tsbarea, tagacc, label/**/1) ;\ 2506 /* ;\ 2507 * tagacc = CLOBBERED ;\ 2508 * tte = CLOBBERED ;\ 2509 * hmeblkpa = hmeblkpa ;\ 2510 * hmeblkva = hmeblkva ;\ 2511 */ ;\ 2512 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2513 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2514 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2515 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2516 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2517 nop ;\ 2518label/**/4: ;\ 2519 /* ;\ 2520 * We have found the hmeblk containing the hment. ;\ 2521 * Now we calculate the corresponding tte. ;\ 2522 * ;\ 2523 * tagacc = tagacc ;\ 2524 * hatid = hatid ;\ 2525 * tte = clobbered ;\ 2526 * hmeblkpa = hmeblkpa ;\ 2527 * hmemisc = hblktag ;\ 2528 * hmeblkva = hmeblkva ;\ 2529 */ ;\ 2530 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2531 label/**/2) ;\ 2532 ;\ 2533 /* ;\ 2534 * tagacc = tagacc ;\ 2535 * hatid = hmentoff ;\ 2536 * tte = clobbered ;\ 2537 * hmeblkpa = hmeblkpa ;\ 2538 * hmemisc = hblk_misc ;\ 2539 * hmeblkva = hmeblkva ;\ 2540 */ ;\ 2541 ;\ 2542 add hatid, SFHME_TTE, hatid ;\ 2543 add hmeblkpa, hatid, hmeblkpa ;\ 2544 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2545 add hmeblkva, hatid, hmeblkva ;\ 2546 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2547 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2548 set TTE_SUSPEND, hatid ;\ 2549 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2550 btst tte, hatid ;\ 2551 bz,pt %xcc, foundlabel ;\ 2552 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2553 ;\ 2554 /* ;\ 2555 * Mapping is suspended, so goto suspend label. ;\ 2556 */ ;\ 2557 ba,pt %xcc, suspendlabel ;\ 2558 nop 2559 2560/* 2561 * GET_SHME_TTE is similar to GET_TTE() except it searches 2562 * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2563 * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2564 * either 0 (not part of scd) or 1 (part of scd). 2565 */ 2566#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, \ 2567 hmemisc, hmeshift, hashno, label, foundlabel, \ 2568 suspendlabel, exitlabel) \ 2569 ;\ 2570 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2571 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2572 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2573 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2574 ;\ 2575 /* ;\ 2576 * tagacc = tagacc ;\ 2577 * hatid = hatid ;\ 2578 * tsbarea = tsbarea ;\ 2579 * tte = hmebp (hme bucket pointer) ;\ 2580 * hmeblkpa = vapg (virtual page) ;\ 2581 * hmemisc, hmeblkva = scratch ;\ 2582 */ ;\ 2583 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2584 ;\ 2585 /* ;\ 2586 * tagacc = tagacc ;\ 2587 * hatid = hatid ;\ 2588 * tsbarea = tsbarea ;\ 2589 * tte = hmebp ;\ 2590 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2591 * hmeblkpa = CLOBBERED ;\ 2592 * hmeblkva = scratch ;\ 2593 */ ;\ 2594 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2595 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2596 ;\ 2597 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2598 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2599 add tte, HMEBUCK_HBLK, hmeblkva ;\ 2600 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2601 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2602 ;\ 2603label/**/8: ;\ 2604 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, hmeblkva, \ 2605 tsbarea, tagacc, tte, label/**/1) ;\ 2606 /* ;\ 2607 * tagacc = CLOBBERED ;\ 2608 * tte = CLOBBERED ;\ 2609 * hmeblkpa = hmeblkpa ;\ 2610 * hmeblkva = hmeblkva ;\ 2611 */ ;\ 2612 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2613 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2614 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2615 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2616 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2617 nop ;\ 2618label/**/4: ;\ 2619 /* ;\ 2620 * We have found the hmeblk containing the hment. ;\ 2621 * Now we calculate the corresponding tte. ;\ 2622 * ;\ 2623 * tagacc = tagacc ;\ 2624 * hatid = hatid ;\ 2625 * tte = clobbered ;\ 2626 * hmeblkpa = hmeblkpa ;\ 2627 * hmemisc = hblktag ;\ 2628 * hmeblkva = hmeblkva ;\ 2629 * tsbarea = tsbmiss area ;\ 2630 */ ;\ 2631 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2632 label/**/2) ;\ 2633 ;\ 2634 /* ;\ 2635 * tagacc = tagacc ;\ 2636 * hatid = hmentoff ;\ 2637 * tte = clobbered ;\ 2638 * hmeblkpa = hmeblkpa ;\ 2639 * hmemisc = hblk_misc ;\ 2640 * hmeblkva = hmeblkva ;\ 2641 * tsbarea = tsbmiss area ;\ 2642 */ ;\ 2643 ;\ 2644 add hatid, SFHME_TTE, hatid ;\ 2645 add hmeblkpa, hatid, hmeblkpa ;\ 2646 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2647 brlz,pt tte, label/**/6 ;\ 2648 add hmeblkva, hatid, hmeblkva ;\ 2649 btst HBLK_SZMASK, hmemisc ;\ 2650 bnz,a,pt %icc, label/**/7 ;\ 2651 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2652 ;\ 2653 /* ;\ 2654 * We found an invalid 8K tte in shme. ;\ 2655 * it may not belong to shme's region since ;\ 2656 * region size/alignment granularity is 8K but different ;\ 2657 * regions don't share hmeblks. Continue the search. ;\ 2658 */ ;\ 2659 sub hmeblkpa, hatid, hmeblkpa ;\ 2660 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2661 srlx tagacc, hmeshift, tte ;\ 2662 add hmeblkpa, HMEBLK_NEXT, hmeblkva ;\ 2663 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2664 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2665 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2666 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2667 ba,a,pt %xcc, label/**/8 ;\ 2668label/**/6: ;\ 2669 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2670 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2671label/**/7: ;\ 2672 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2673 set TTE_SUSPEND, hatid ;\ 2674 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2675 btst tte, hatid ;\ 2676 bz,pt %xcc, foundlabel ;\ 2677 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2678 ;\ 2679 /* ;\ 2680 * Mapping is suspended, so goto suspend label. ;\ 2681 */ ;\ 2682 ba,pt %xcc, suspendlabel ;\ 2683 nop 2684 2685 /* 2686 * KERNEL PROTECTION HANDLER 2687 * 2688 * g1 = tsb8k pointer register (clobbered) 2689 * g2 = tag access register (ro) 2690 * g3 - g7 = scratch registers 2691 * 2692 * Note: This function is patched at runtime for performance reasons. 2693 * Any changes here require sfmmu_patch_ktsb fixed. 2694 */ 2695 ENTRY_NP(sfmmu_kprot_trap) 2696 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2697sfmmu_kprot_patch_ktsb_base: 2698 RUNTIME_PATCH_SETX(%g1, %g6) 2699 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2700sfmmu_kprot_patch_ktsb_szcode: 2701 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2702 2703 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2704 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2705 2706 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2707sfmmu_kprot_patch_ktsb4m_base: 2708 RUNTIME_PATCH_SETX(%g3, %g6) 2709 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2710sfmmu_kprot_patch_ktsb4m_szcode: 2711 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2712 2713 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2714 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2715 2716 CPU_TSBMISS_AREA(%g6, %g7) 2717 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2718 ba,pt %xcc, sfmmu_tsb_miss_tt 2719 nop 2720 2721 /* 2722 * USER PROTECTION HANDLER 2723 * 2724 * g1 = tsb8k pointer register (ro) 2725 * g2 = tag access register (ro) 2726 * g3 = faulting context (clobbered, currently not used) 2727 * g4 - g7 = scratch registers 2728 */ 2729 ALTENTRY(sfmmu_uprot_trap) 2730#ifdef sun4v 2731 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2732 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2733 2734 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2735 brlz,pt %g3, 9f /* check for 2nd TSB */ 2736 nop 2737 2738 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2739 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2740 2741#else /* sun4v */ 2742#ifdef UTSB_PHYS 2743 /* g1 = first TSB entry ptr */ 2744 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2745 brlz,pt %g3, 9f /* check for 2nd TSB */ 2746 nop 2747 2748 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2749 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2750#else /* UTSB_PHYS */ 2751 brgez,pt %g1, 9f /* check for 2nd TSB */ 2752 mov -1, %g3 /* set second tsbe ptr to -1 */ 2753 2754 mov %g2, %g7 2755 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2756 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2757 mov %g1, %g7 2758 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2759#endif /* UTSB_PHYS */ 2760#endif /* sun4v */ 27619: 2762 CPU_TSBMISS_AREA(%g6, %g7) 2763 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2764 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2765 nop 2766 2767 /* 2768 * Kernel 8K page iTLB miss. We also get here if we took a 2769 * fast instruction access mmu miss trap while running in 2770 * invalid context. 2771 * 2772 * %g1 = 8K TSB pointer register (not used, clobbered) 2773 * %g2 = tag access register (used) 2774 * %g3 = faulting context id (used) 2775 * %g7 = TSB tag to match (used) 2776 */ 2777 .align 64 2778 ALTENTRY(sfmmu_kitlb_miss) 2779 brnz,pn %g3, tsb_tl0_noctxt 2780 nop 2781 2782 /* kernel miss */ 2783 /* get kernel tsb pointer */ 2784 /* we patch the next set of instructions at run time */ 2785 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2786iktsbbase: 2787 RUNTIME_PATCH_SETX(%g4, %g5) 2788 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2789 2790iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2791 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2792 or %g4, %g1, %g1 ! form tsb ptr 2793 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2794 cmp %g4, %g7 2795 bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2796 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2797 2798 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2799 bz,pn %icc, exec_fault 2800 nop 2801 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2802 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2803 retry 2804 2805iktsb4mbase: 2806 RUNTIME_PATCH_SETX(%g4, %g6) 2807 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2808iktsb4m: 2809 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2810 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2811 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2812 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2813 cmp %g4, %g7 2814 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2815 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2816 bz,pn %icc, exec_fault 2817 nop 2818 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2819 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2820 retry 2821 2822 /* 2823 * Kernel dTLB miss. We also get here if we took a fast data 2824 * access mmu miss trap while running in invalid context. 2825 * 2826 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2827 * We select the TSB miss handler to branch to depending on 2828 * the virtual address of the access. In the future it may 2829 * be desirable to separate kpm TTEs into their own TSB, 2830 * in which case all that needs to be done is to set 2831 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2832 * early in the miss if we detect a kpm VA to a new handler. 2833 * 2834 * %g1 = 8K TSB pointer register (not used, clobbered) 2835 * %g2 = tag access register (used) 2836 * %g3 = faulting context id (used) 2837 */ 2838 .align 64 2839 ALTENTRY(sfmmu_kdtlb_miss) 2840 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2841 nop 2842 2843 /* Gather some stats for kpm misses in the TLB. */ 2844 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2845 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2846 2847 /* 2848 * Get first TSB offset and look for 8K/64K/512K mapping 2849 * using the 8K virtual page as the index. 2850 * 2851 * We patch the next set of instructions at run time; 2852 * any changes here require sfmmu_patch_ktsb changes too. 2853 */ 2854dktsbbase: 2855 RUNTIME_PATCH_SETX(%g7, %g6) 2856 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2857 2858dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2859 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2860 2861 /* 2862 * At this point %g1 is our index into the TSB. 2863 * We just masked off enough bits of the VA depending 2864 * on our TSB size code. 2865 */ 2866 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2867 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2868 cmp %g6, %g4 ! compare tag 2869 bne,pn %xcc, dktsb4m_kpmcheck_small 2870 add %g7, %g1, %g1 /* form tsb ptr */ 2871 TT_TRACE(trace_tsbhit) 2872 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2873 /* trapstat expects tte in %g5 */ 2874 retry 2875 2876 /* 2877 * If kpm is using large pages, the following instruction needs 2878 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2879 * so that we will probe the 4M TSB regardless of the VA. In 2880 * the case kpm is using small pages, we know no large kernel 2881 * mappings are located above 0x80000000.00000000 so we skip the 2882 * probe as an optimization. 2883 */ 2884dktsb4m_kpmcheck_small: 2885 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2886 /* delay slot safe, below */ 2887 2888 /* 2889 * Get second TSB offset and look for 4M mapping 2890 * using 4M virtual page as the TSB index. 2891 * 2892 * Here: 2893 * %g1 = 8K TSB pointer. Don't squash it. 2894 * %g2 = tag access register (we still need it) 2895 */ 2896 srlx %g2, MMU_PAGESHIFT4M, %g3 2897 2898 /* 2899 * We patch the next set of instructions at run time; 2900 * any changes here require sfmmu_patch_ktsb changes too. 2901 */ 2902dktsb4mbase: 2903 RUNTIME_PATCH_SETX(%g7, %g6) 2904 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2905dktsb4m: 2906 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2907 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2908 2909 /* 2910 * At this point %g3 is our index into the TSB. 2911 * We just masked off enough bits of the VA depending 2912 * on our TSB size code. 2913 */ 2914 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2915 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2916 cmp %g6, %g4 ! compare tag 2917 2918dktsb4m_tsbmiss: 2919 bne,pn %xcc, dktsb4m_kpmcheck 2920 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2921 TT_TRACE(trace_tsbhit) 2922 /* we don't check TTE size here since we assume 4M TSB is separate */ 2923 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2924 /* trapstat expects tte in %g5 */ 2925 retry 2926 2927 /* 2928 * So, we failed to find a valid TTE to match the faulting 2929 * address in either TSB. There are a few cases that could land 2930 * us here: 2931 * 2932 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2933 * to sfmmu_tsb_miss_tt to handle the miss. 2934 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2935 * 4M TSB. Let segkpm handle it. 2936 * 2937 * Note that we shouldn't land here in the case of a kpm VA when 2938 * kpm_smallpages is active -- we handled that case earlier at 2939 * dktsb4m_kpmcheck_small. 2940 * 2941 * At this point: 2942 * g1 = 8K-indexed primary TSB pointer 2943 * g2 = tag access register 2944 * g3 = 4M-indexed secondary TSB pointer 2945 */ 2946dktsb4m_kpmcheck: 2947 cmp %g2, %g0 2948 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2949 nop 2950 ba,a,pt %icc, sfmmu_tsb_miss_tt 2951 nop 2952 2953#ifdef sun4v 2954 /* 2955 * User instruction miss w/ single TSB. 2956 * The first probe covers 8K, 64K, and 512K page sizes, 2957 * because 64K and 512K mappings are replicated off 8K 2958 * pointer. 2959 * 2960 * g1 = tsb8k pointer register 2961 * g2 = tag access register 2962 * g3 - g6 = scratch registers 2963 * g7 = TSB tag to match 2964 */ 2965 .align 64 2966 ALTENTRY(sfmmu_uitlb_fastpath) 2967 2968 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2969 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2970 ba,pn %xcc, sfmmu_tsb_miss_tt 2971 mov -1, %g3 2972 2973 /* 2974 * User data miss w/ single TSB. 2975 * The first probe covers 8K, 64K, and 512K page sizes, 2976 * because 64K and 512K mappings are replicated off 8K 2977 * pointer. 2978 * 2979 * g1 = tsb8k pointer register 2980 * g2 = tag access register 2981 * g3 - g6 = scratch registers 2982 * g7 = TSB tag to match 2983 */ 2984 .align 64 2985 ALTENTRY(sfmmu_udtlb_fastpath) 2986 2987 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2988 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2989 ba,pn %xcc, sfmmu_tsb_miss_tt 2990 mov -1, %g3 2991 2992 /* 2993 * User instruction miss w/ multiple TSBs (sun4v). 2994 * The first probe covers 8K, 64K, and 512K page sizes, 2995 * because 64K and 512K mappings are replicated off 8K 2996 * pointer. Second probe covers 4M page size only. 2997 * 2998 * Just like sfmmu_udtlb_slowpath, except: 2999 * o Uses ASI_ITLB_IN 3000 * o checks for execute permission 3001 * o No ISM prediction. 3002 * 3003 * g1 = tsb8k pointer register 3004 * g2 = tag access register 3005 * g3 - g6 = scratch registers 3006 * g7 = TSB tag to match 3007 */ 3008 .align 64 3009 ALTENTRY(sfmmu_uitlb_slowpath) 3010 3011 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 3012 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 3013 /* g4 - g5 = clobbered here */ 3014 3015 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3016 /* g1 = first TSB pointer, g3 = second TSB pointer */ 3017 srlx %g2, TAG_VALO_SHIFT, %g7 3018 PROBE_2ND_ITSB(%g3, %g7) 3019 /* NOT REACHED */ 3020 3021#else /* sun4v */ 3022 3023 /* 3024 * User instruction miss w/ multiple TSBs (sun4u). 3025 * The first probe covers 8K, 64K, and 512K page sizes, 3026 * because 64K and 512K mappings are replicated off 8K 3027 * pointer. Probe of 1st TSB has already been done prior to entry 3028 * into this routine. For the UTSB_PHYS case we probe up to 3 3029 * valid other TSBs in the following order: 3030 * 1) shared TSB for 4M-256M pages 3031 * 2) private TSB for 4M-256M pages 3032 * 3) shared TSB for 8K-512K pages 3033 * 3034 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs 3035 * 4M-256M pages. 3036 * 3037 * Just like sfmmu_udtlb_slowpath, except: 3038 * o Uses ASI_ITLB_IN 3039 * o checks for execute permission 3040 * o No ISM prediction. 3041 * 3042 * g1 = tsb8k pointer register 3043 * g2 = tag access register 3044 * g4 - g6 = scratch registers 3045 * g7 = TSB tag to match 3046 */ 3047 .align 64 3048 ALTENTRY(sfmmu_uitlb_slowpath) 3049 3050#ifdef UTSB_PHYS 3051 3052 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3053 brlz,pt %g6, 1f 3054 nop 3055 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3056 PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail) 30571: 3058 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3059 brlz,pt %g3, 2f 3060 nop 3061 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3062 PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail) 30632: 3064 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3065 brlz,pt %g6, sfmmu_tsb_miss_tt 3066 nop 3067 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3068 PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail) 3069 ba,pn %xcc, sfmmu_tsb_miss_tt 3070 nop 3071 3072#else /* UTSB_PHYS */ 3073 mov %g1, %g3 /* save tsb8k reg in %g3 */ 3074 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 3075 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 3076 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 3077 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 3078 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 3079 /* g1 = first TSB pointer, g3 = second TSB pointer */ 3080 srlx %g2, TAG_VALO_SHIFT, %g7 3081 PROBE_2ND_ITSB(%g3, %g7, isynth) 3082 ba,pn %xcc, sfmmu_tsb_miss_tt 3083 nop 3084 3085#endif /* UTSB_PHYS */ 3086#endif /* sun4v */ 3087 3088#if defined(sun4u) && defined(UTSB_PHYS) 3089 3090 /* 3091 * We come here for ism predict DTLB_MISS case or if 3092 * if probe in first TSB failed. 3093 */ 3094 3095 .align 64 3096 ALTENTRY(sfmmu_udtlb_slowpath_noismpred) 3097 3098 /* 3099 * g1 = tsb8k pointer register 3100 * g2 = tag access register 3101 * g4 - %g6 = scratch registers 3102 * g7 = TSB tag to match 3103 */ 3104 3105 /* 3106 * ISM non-predict probe order 3107 * probe 1ST_TSB (8K index) 3108 * probe 2ND_TSB (4M index) 3109 * probe 4TH_TSB (4M index) 3110 * probe 3RD_TSB (8K index) 3111 * 3112 * We already probed first TSB in DTLB_MISS handler. 3113 */ 3114 3115 /* 3116 * Private 2ND TSB 4M-256 pages 3117 */ 3118 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3119 brlz,pt %g3, 1f 3120 nop 3121 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3122 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3123 3124 /* 3125 * Shared Context 4TH TSB 4M-256 pages 3126 */ 31271: 3128 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3129 brlz,pt %g6, 2f 3130 nop 3131 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3132 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail) 3133 3134 /* 3135 * Shared Context 3RD TSB 8K-512K pages 3136 */ 31372: 3138 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3139 brlz,pt %g6, sfmmu_tsb_miss_tt 3140 nop 3141 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3142 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail) 3143 ba,pn %xcc, sfmmu_tsb_miss_tt 3144 nop 3145 3146 .align 64 3147 ALTENTRY(sfmmu_udtlb_slowpath_ismpred) 3148 3149 /* 3150 * g1 = tsb8k pointer register 3151 * g2 = tag access register 3152 * g4 - g6 = scratch registers 3153 * g7 = TSB tag to match 3154 */ 3155 3156 /* 3157 * ISM predict probe order 3158 * probe 4TH_TSB (4M index) 3159 * probe 2ND_TSB (4M index) 3160 * probe 1ST_TSB (8K index) 3161 * probe 3RD_TSB (8K index) 3162 3163 /* 3164 * Shared Context 4TH TSB 4M-256 pages 3165 */ 3166 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3167 brlz,pt %g6, 4f 3168 nop 3169 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3170 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2) 3171 3172 /* 3173 * Private 2ND TSB 4M-256 pages 3174 */ 31754: 3176 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3177 brlz,pt %g3, 5f 3178 nop 3179 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3180 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2) 3181 31825: 3183 PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2) 3184 3185 /* 3186 * Shared Context 3RD TSB 8K-512K pages 3187 */ 3188 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3189 brlz,pt %g6, 6f 3190 nop 3191 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3192 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2) 31936: 3194 ba,pn %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */ 3195 nop 3196 3197#else /* sun4u && UTSB_PHYS */ 3198 3199 .align 64 3200 ALTENTRY(sfmmu_udtlb_slowpath) 3201 3202 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 3203 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 3204 mov %g1, %g3 3205 3206udtlb_miss_probefirst: 3207 /* 3208 * g1 = 8K TSB pointer register 3209 * g2 = tag access register 3210 * g3 = (potentially) second TSB entry ptr 3211 * g6 = ism pred. 3212 * g7 = vpg_4m 3213 */ 3214#ifdef sun4v 3215 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 3216 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3217 3218 /* 3219 * Here: 3220 * g1 = first TSB pointer 3221 * g2 = tag access reg 3222 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3223 */ 3224 brgz,pn %g6, sfmmu_tsb_miss_tt 3225 nop 3226#else /* sun4v */ 3227 mov %g1, %g4 3228 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 3229 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3230 3231 /* 3232 * Here: 3233 * g1 = first TSB pointer 3234 * g2 = tag access reg 3235 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3236 */ 3237 brgz,pn %g6, sfmmu_tsb_miss_tt 3238 nop 3239 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 3240 /* fall through in 8K->4M probe order */ 3241#endif /* sun4v */ 3242 3243udtlb_miss_probesecond: 3244 /* 3245 * Look in the second TSB for the TTE 3246 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 3247 * g2 = tag access reg 3248 * g3 = 8K TSB pointer register 3249 * g6 = ism pred. 3250 * g7 = vpg_4m 3251 */ 3252#ifdef sun4v 3253 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 3254 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3255 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 3256#else /* sun4v */ 3257 mov %g3, %g7 3258 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 3259 /* %g2 clobbered, %g3 =second tsbe ptr */ 3260 mov MMU_TAG_ACCESS, %g2 3261 ldxa [%g2]ASI_DMMU, %g2 3262#endif /* sun4v */ 3263 3264 srlx %g2, TAG_VALO_SHIFT, %g7 3265 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3266 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 3267 brgz,pn %g6, udtlb_miss_probefirst 3268 nop 3269 3270 /* fall through to sfmmu_tsb_miss_tt */ 3271#endif /* sun4u && UTSB_PHYS */ 3272 3273 3274 ALTENTRY(sfmmu_tsb_miss_tt) 3275 TT_TRACE(trace_tsbmiss) 3276 /* 3277 * We get here if there is a TSB miss OR a write protect trap. 3278 * 3279 * g1 = First TSB entry pointer 3280 * g2 = tag access register 3281 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 3282 * g4 - g7 = scratch registers 3283 */ 3284 3285 ALTENTRY(sfmmu_tsb_miss) 3286 3287 /* 3288 * If trapstat is running, we need to shift the %tpc and %tnpc to 3289 * point to trapstat's TSB miss return code (note that trapstat 3290 * itself will patch the correct offset to add). 3291 */ 3292 rdpr %tl, %g7 3293 cmp %g7, 1 3294 ble,pt %xcc, 0f 3295 sethi %hi(KERNELBASE), %g6 3296 rdpr %tpc, %g7 3297 or %g6, %lo(KERNELBASE), %g6 3298 cmp %g7, %g6 3299 bgeu,pt %xcc, 0f 3300 /* delay slot safe */ 3301 3302 ALTENTRY(tsbmiss_trapstat_patch_point) 3303 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 3304 wrpr %g7, %tpc 3305 add %g7, 4, %g7 3306 wrpr %g7, %tnpc 33070: 3308 CPU_TSBMISS_AREA(%g6, %g7) 3309 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save 1ST tsb pointer */ 3310 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save 2ND tsb pointer */ 3311 3312 sllx %g2, TAGACC_CTX_LSHIFT, %g3 3313 brz,a,pn %g3, 1f /* skip ahead if kernel */ 3314 ldn [%g6 + TSBMISS_KHATID], %g7 3315 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 3316 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 3317 3318 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 3319 3320 cmp %g3, INVALID_CONTEXT 3321 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 3322 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 3323 3324#if defined(sun4v) || defined(UTSB_PHYS) 3325 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 3326 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 3327 stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 3328#endif /* sun4v || UTSB_PHYS */ 3329 3330 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 3331 /* 3332 * The miss wasn't in an ISM segment. 3333 * 3334 * %g1 %g3, %g4, %g5, %g7 all clobbered 3335 * %g2 = (pseudo) tag access 3336 */ 3337 3338 ba,pt %icc, 2f 3339 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 3340 33411: 3342 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 3343 /* 3344 * 8K and 64K hash. 3345 */ 33462: 3347 3348 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3349 MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte, 3350 sfmmu_suspend_tl, tsb_512K) 3351 /* NOT REACHED */ 3352 3353tsb_512K: 3354 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3355 brz,pn %g5, 3f 3356 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3357 and %g4, HAT_512K_FLAG, %g5 3358 3359 /* 3360 * Note that there is a small window here where we may have 3361 * a 512k page in the hash list but have not set the HAT_512K_FLAG 3362 * flag yet, so we will skip searching the 512k hash list. 3363 * In this case we will end up in pagefault which will find 3364 * the mapping and return. So, in this instance we will end up 3365 * spending a bit more time resolving this TSB miss, but it can 3366 * only happen once per process and even then, the chances of that 3367 * are very small, so it's not worth the extra overhead it would 3368 * take to close this window. 3369 */ 3370 brz,pn %g5, tsb_4M 3371 nop 33723: 3373 /* 3374 * 512K hash 3375 */ 3376 3377 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3378 MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte, 3379 sfmmu_suspend_tl, tsb_4M) 3380 /* NOT REACHED */ 3381 3382tsb_4M: 3383 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3384 brz,pn %g5, 4f 3385 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3386 and %g4, HAT_4M_FLAG, %g5 3387 brz,pn %g5, tsb_32M 3388 nop 33894: 3390 /* 3391 * 4M hash 3392 */ 3393 3394 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3395 MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte, 3396 sfmmu_suspend_tl, tsb_32M) 3397 /* NOT REACHED */ 3398 3399tsb_32M: 3400 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3401#ifdef sun4v 3402 brz,pn %g5, 6f 3403#else 3404 brz,pn %g5, tsb_pagefault 3405#endif 3406 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3407 and %g4, HAT_32M_FLAG, %g5 3408 brz,pn %g5, tsb_256M 3409 nop 34105: 3411 /* 3412 * 32M hash 3413 */ 3414 3415 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3416 MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte, 3417 sfmmu_suspend_tl, tsb_256M) 3418 /* NOT REACHED */ 3419 3420#if defined(sun4u) && !defined(UTSB_PHYS) 3421#define tsb_shme tsb_pagefault 3422#endif 3423tsb_256M: 3424 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3425 and %g4, HAT_256M_FLAG, %g5 3426 brz,pn %g5, tsb_shme 3427 nop 34286: 3429 /* 3430 * 256M hash 3431 */ 3432 3433 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3434 MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte, 3435 sfmmu_suspend_tl, tsb_shme) 3436 /* NOT REACHED */ 3437 3438tsb_checktte: 3439 /* 3440 * g1 = hblk_misc 3441 * g2 = tagacc 3442 * g3 = tte 3443 * g4 = tte pa 3444 * g5 = tte va 3445 * g6 = tsbmiss area 3446 * g7 = hatid 3447 */ 3448 brlz,a,pt %g3, tsb_validtte 3449 rdpr %tt, %g7 3450 3451#if defined(sun4u) && !defined(UTSB_PHYS) 3452#undef tsb_shme 3453 ba tsb_pagefault 3454 nop 3455#else /* sun4u && !UTSB_PHYS */ 3456 3457tsb_shme: 3458 /* 3459 * g2 = tagacc 3460 * g6 = tsbmiss area 3461 */ 3462 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3463 brz,pn %g5, tsb_pagefault 3464 nop 3465 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3466 brz,pn %g7, tsb_pagefault 3467 nop 3468 3469 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3470 MMU_PAGESHIFT64K, TTE64K, tsb_shme_l8K, tsb_shme_checktte, 3471 sfmmu_suspend_tl, tsb_shme_512K) 3472 /* NOT REACHED */ 3473 3474tsb_shme_512K: 3475 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3476 and %g4, HAT_512K_FLAG, %g5 3477 brz,pn %g5, tsb_shme_4M 3478 nop 3479 3480 /* 3481 * 512K hash 3482 */ 3483 3484 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3485 MMU_PAGESHIFT512K, TTE512K, tsb_shme_l512K, tsb_shme_checktte, 3486 sfmmu_suspend_tl, tsb_shme_4M) 3487 /* NOT REACHED */ 3488 3489tsb_shme_4M: 3490 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3491 and %g4, HAT_4M_FLAG, %g5 3492 brz,pn %g5, tsb_shme_32M 3493 nop 34944: 3495 /* 3496 * 4M hash 3497 */ 3498 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3499 MMU_PAGESHIFT4M, TTE4M, tsb_shme_l4M, tsb_shme_checktte, 3500 sfmmu_suspend_tl, tsb_shme_32M) 3501 /* NOT REACHED */ 3502 3503tsb_shme_32M: 3504 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3505 and %g4, HAT_32M_FLAG, %g5 3506 brz,pn %g5, tsb_shme_256M 3507 nop 3508 3509 /* 3510 * 32M hash 3511 */ 3512 3513 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3514 MMU_PAGESHIFT32M, TTE32M, tsb_shme_l32M, tsb_shme_checktte, 3515 sfmmu_suspend_tl, tsb_shme_256M) 3516 /* NOT REACHED */ 3517 3518tsb_shme_256M: 3519 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3520 and %g4, HAT_256M_FLAG, %g5 3521 brz,pn %g5, tsb_pagefault 3522 nop 3523 3524 /* 3525 * 256M hash 3526 */ 3527 3528 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3529 MMU_PAGESHIFT256M, TTE256M, tsb_shme_l256M, tsb_shme_checktte, 3530 sfmmu_suspend_tl, tsb_pagefault) 3531 /* NOT REACHED */ 3532 3533tsb_shme_checktte: 3534 3535 brgez,pn %g3, tsb_pagefault 3536 rdpr %tt, %g7 3537 /* 3538 * g1 = ctx1 flag 3539 * g3 = tte 3540 * g4 = tte pa 3541 * g5 = tte va 3542 * g6 = tsbmiss area 3543 * g7 = tt 3544 */ 3545 3546 brz,pt %g1, tsb_validtte 3547 nop 3548 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3549 or %g1, HAT_CHKCTX1_FLAG, %g1 3550 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3551 3552 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3553#endif /* sun4u && !UTSB_PHYS */ 3554 3555tsb_validtte: 3556 /* 3557 * g3 = tte 3558 * g4 = tte pa 3559 * g5 = tte va 3560 * g6 = tsbmiss area 3561 * g7 = tt 3562 */ 3563 3564 /* 3565 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3566 */ 3567 cmp %g7, FAST_PROT_TT 3568 bne,pt %icc, 4f 3569 nop 3570 3571 TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod, 3572 tsb_protfault) 3573 3574 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3575#ifdef sun4v 3576 MMU_FAULT_STATUS_AREA(%g7) 3577 ldx [%g7 + MMFSA_D_ADDR], %g5 /* load fault addr for later */ 3578#else /* sun4v */ 3579 mov MMU_TAG_ACCESS, %g5 3580 ldxa [%g5]ASI_DMMU, %g5 3581#endif /* sun4v */ 3582 ba,pt %xcc, tsb_update_tl1 3583 nop 35844: 3585 /* 3586 * If ITLB miss check exec bit. 3587 * If not set treat as invalid TTE. 3588 */ 3589 cmp %g7, T_INSTR_MMU_MISS 3590 be,pn %icc, 5f 3591 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3592 cmp %g7, FAST_IMMU_MISS_TT 3593 bne,pt %icc, 3f 3594 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 35955: 3596 bz,pn %icc, tsb_protfault 3597 nop 3598 35993: 3600 /* 3601 * Set reference bit if not already set 3602 */ 3603 TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref) 3604 3605 /* 3606 * Now, load into TSB/TLB. At this point: 3607 * g3 = tte 3608 * g4 = patte 3609 * g6 = tsbmiss area 3610 */ 3611 rdpr %tt, %g7 3612#ifdef sun4v 3613 MMU_FAULT_STATUS_AREA(%g2) 3614 cmp %g7, T_INSTR_MMU_MISS 3615 be,a,pt %icc, 9f 3616 nop 3617 cmp %g7, FAST_IMMU_MISS_TT 3618 be,a,pt %icc, 9f 3619 nop 3620 add %g2, MMFSA_D_, %g2 36219: 3622 ldx [%g2 + MMFSA_CTX_], %g7 3623 sllx %g7, TTARGET_CTX_SHIFT, %g7 3624 ldx [%g2 + MMFSA_ADDR_], %g2 3625 mov %g2, %g5 ! load the fault addr for later use 3626 srlx %g2, TTARGET_VA_SHIFT, %g2 3627 or %g2, %g7, %g2 3628#else /* sun4v */ 3629 mov MMU_TAG_ACCESS, %g5 3630 cmp %g7, FAST_IMMU_MISS_TT 3631 be,a,pt %icc, 9f 3632 ldxa [%g0]ASI_IMMU, %g2 3633 ldxa [%g0]ASI_DMMU, %g2 3634 ba,pt %icc, tsb_update_tl1 3635 ldxa [%g5]ASI_DMMU, %g5 36369: 3637 ldxa [%g5]ASI_IMMU, %g5 3638#endif /* sun4v */ 3639 3640tsb_update_tl1: 3641 srlx %g2, TTARGET_CTX_SHIFT, %g7 3642 brz,pn %g7, tsb_kernel 3643#ifdef sun4v 3644 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3645#else /* sun4v */ 3646 srlx %g3, TTE_SZ_SHFT, %g7 3647#endif /* sun4v */ 3648 3649tsb_user: 3650#ifdef sun4v 3651 cmp %g7, TTE4M 3652 bge,pn %icc, tsb_user4m 3653 nop 3654#else /* sun4v */ 3655 cmp %g7, TTESZ_VALID | TTE4M 3656 be,pn %icc, tsb_user4m 3657 srlx %g3, TTE_SZ2_SHFT, %g7 3658 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3659#ifdef ITLB_32M_256M_SUPPORT 3660 bnz,pn %icc, tsb_user4m 3661 nop 3662#else /* ITLB_32M_256M_SUPPORT */ 3663 bnz,a,pn %icc, tsb_user_pn_synth 3664 nop 3665#endif /* ITLB_32M_256M_SUPPORT */ 3666#endif /* sun4v */ 3667 3668tsb_user8k: 3669#if defined(sun4v) || defined(UTSB_PHYS) 3670 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3671 and %g7, HAT_CHKCTX1_FLAG, %g1 3672 brz,a,pn %g1, 1f 3673 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3674 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3675 brlz,a,pn %g1, ptl1_panic ! if no shared 3RD tsb 3676 mov PTL1_NO_SCDTSB8K, %g1 ! panic 3677 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 36781: 3679#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3680 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3681#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3682 3683#ifndef UTSB_PHYS 3684 mov ASI_N, %g7 ! user TSBs accessed by VA 3685 mov %g7, %asi 3686#endif /* !UTSB_PHYS */ 3687 3688 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5) 3689 3690 rdpr %tt, %g5 3691#ifdef sun4v 3692 cmp %g5, T_INSTR_MMU_MISS 3693 be,a,pn %xcc, 9f 3694 mov %g3, %g5 3695#endif /* sun4v */ 3696 cmp %g5, FAST_IMMU_MISS_TT 3697 be,pn %xcc, 9f 3698 mov %g3, %g5 3699 3700 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3701 ! trapstat wants TTE in %g5 3702 retry 37039: 3704 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3705 ! trapstat wants TTE in %g5 3706 retry 3707 3708tsb_user4m: 3709#if defined(sun4v) || defined(UTSB_PHYS) 3710 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3711 and %g7, HAT_CHKCTX1_FLAG, %g1 3712 brz,a,pn %g1, 4f 3713 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3714 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr 3715 brlz,a,pn %g1, 5f ! if no shared 4TH TSB 3716 nop 3717 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 3718 3719#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3720 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3721#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 37224: 3723 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3724 nop 3725 3726#ifndef UTSB_PHYS 3727 mov ASI_N, %g7 ! user TSBs accessed by VA 3728 mov %g7, %asi 3729#endif /* UTSB_PHYS */ 3730 3731 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6) 3732 37335: 3734 rdpr %tt, %g5 3735#ifdef sun4v 3736 cmp %g5, T_INSTR_MMU_MISS 3737 be,a,pn %xcc, 9f 3738 mov %g3, %g5 3739#endif /* sun4v */ 3740 cmp %g5, FAST_IMMU_MISS_TT 3741 be,pn %xcc, 9f 3742 mov %g3, %g5 3743 3744 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3745 ! trapstat wants TTE in %g5 3746 retry 37479: 3748 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3749 ! trapstat wants TTE in %g5 3750 retry 3751 3752#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3753 /* 3754 * Panther ITLB synthesis. 3755 * The Panther 32M and 256M ITLB code simulates these two large page 3756 * sizes with 4M pages, to provide support for programs, for example 3757 * Java, that may copy instructions into a 32M or 256M data page and 3758 * then execute them. The code below generates the 4M pfn bits and 3759 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3760 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3761 * are ignored by the hardware. 3762 * 3763 * Now, load into TSB/TLB. At this point: 3764 * g2 = tagtarget 3765 * g3 = tte 3766 * g4 = patte 3767 * g5 = tt 3768 * g6 = tsbmiss area 3769 */ 3770tsb_user_pn_synth: 3771 rdpr %tt, %g5 3772 cmp %g5, FAST_IMMU_MISS_TT 3773 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3774 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3775 bz,pn %icc, 4b /* if not, been here before */ 3776 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3777 brlz,a,pn %g1, 5f /* no 2nd tsb */ 3778 mov %g3, %g5 3779 3780 mov MMU_TAG_ACCESS, %g7 3781 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3782 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3783 3784 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3785 mov %g7, %asi 3786 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */ 37875: 3788 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3789 retry 3790 3791tsb_user_itlb_synth: 3792 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = 2ND TSB */ 3793 3794 mov MMU_TAG_ACCESS, %g7 3795 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3796 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3797 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3798 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3799 3800 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3801 mov %g7, %asi 3802 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */ 38037: 3804 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3805 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3806 retry 3807#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3808 3809tsb_kernel: 3810 rdpr %tt, %g5 3811#ifdef sun4v 3812 cmp %g7, TTE4M 3813 bge,pn %icc, 5f 3814#else 3815 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3816 be,pn %icc, 5f 3817#endif /* sun4v */ 3818 nop 3819 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8K TSB ptr 3820 ba,pt %xcc, 6f 3821 nop 38225: 3823 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4M TSB ptr 3824 brlz,pn %g1, 3f /* skip programming if 4M TSB ptr is -1 */ 3825 nop 38266: 3827#ifndef sun4v 3828tsb_kernel_patch_asi: 3829 or %g0, RUNTIME_PATCH, %g6 3830 mov %g6, %asi ! XXX avoid writing to %asi !! 3831#endif 3832 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7) 38333: 3834#ifdef sun4v 3835 cmp %g5, T_INSTR_MMU_MISS 3836 be,a,pn %icc, 1f 3837 mov %g3, %g5 ! trapstat wants TTE in %g5 3838#endif /* sun4v */ 3839 cmp %g5, FAST_IMMU_MISS_TT 3840 be,pn %icc, 1f 3841 mov %g3, %g5 ! trapstat wants TTE in %g5 3842 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3843 ! trapstat wants TTE in %g5 3844 retry 38451: 3846 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3847 ! trapstat wants TTE in %g5 3848 retry 3849 3850tsb_ism: 3851 /* 3852 * This is an ISM [i|d]tlb miss. We optimize for largest 3853 * page size down to smallest. 3854 * 3855 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3856 * register 3857 * g3 = ismmap->ism_seg 3858 * g4 = physical address of ismmap->ism_sfmmu 3859 * g6 = tsbmiss area 3860 */ 3861 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3862 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3863 mov PTL1_BAD_ISM, %g1 3864 /* g5 = pa of imap_vb_shift */ 3865 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3866 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3867 srlx %g3, %g4, %g3 /* clr size field */ 3868 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3869 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3870 and %g2, %g1, %g4 /* g4 = ctx number */ 3871 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3872 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3873 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3874 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3875 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3876#if defined(sun4v) || defined(UTSB_PHYS) 3877 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3878 brz,pt %g5, tsb_chk4M_ism 3879 nop 3880 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3881 or %g5, HAT_CHKCTX1_FLAG, %g5 3882 stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3883 rdpr %tt, %g5 3884 SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl) 3885#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3886 3887 /* 3888 * ISM pages are always locked down. 3889 * If we can't find the tte then pagefault 3890 * and let the spt segment driver resolve it. 3891 * 3892 * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3893 * g4 = imap_hatflags 3894 * g6 = tsb miss area 3895 * g7 = ISM hatid 3896 */ 3897 3898tsb_chk4M_ism: 3899 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3900 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3901 nop 3902 3903tsb_ism_32M: 3904 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3905 brz,pn %g5, tsb_ism_256M 3906 nop 3907 3908 /* 3909 * 32M hash. 3910 */ 3911 3912 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M, 3913 TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3914 tsb_ism_4M) 3915 /* NOT REACHED */ 3916 3917tsb_ism_32M_found: 3918 brlz,a,pt %g3, tsb_validtte 3919 rdpr %tt, %g7 3920 ba,pt %xcc, tsb_ism_4M 3921 nop 3922 3923tsb_ism_256M: 3924 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3925 brz,a,pn %g5, ptl1_panic 3926 mov PTL1_BAD_ISM, %g1 3927 3928 /* 3929 * 256M hash. 3930 */ 3931 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M, 3932 TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3933 tsb_ism_4M) 3934 3935tsb_ism_256M_found: 3936 brlz,a,pt %g3, tsb_validtte 3937 rdpr %tt, %g7 3938 3939tsb_ism_4M: 3940 /* 3941 * 4M hash. 3942 */ 3943 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M, 3944 TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3945 tsb_ism_8K) 3946 /* NOT REACHED */ 3947 3948tsb_ism_4M_found: 3949 brlz,a,pt %g3, tsb_validtte 3950 rdpr %tt, %g7 3951 3952tsb_ism_8K: 3953 /* 3954 * 8K and 64K hash. 3955 */ 3956 3957 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K, 3958 TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3959 tsb_pagefault) 3960 /* NOT REACHED */ 3961 3962tsb_ism_8K_found: 3963 brlz,a,pt %g3, tsb_validtte 3964 rdpr %tt, %g7 3965 3966tsb_pagefault: 3967 rdpr %tt, %g7 3968 cmp %g7, FAST_PROT_TT 3969 be,a,pn %icc, tsb_protfault 3970 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3971 3972tsb_protfault: 3973 /* 3974 * we get here if we couldn't find a valid tte in the hash. 3975 * 3976 * If user and we are at tl>1 we go to window handling code. 3977 * 3978 * If kernel and the fault is on the same page as our stack 3979 * pointer, then we know the stack is bad and the trap handler 3980 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3981 * 3982 * If this is a kernel trap and tl>1, panic. 3983 * 3984 * Otherwise we call pagefault. 3985 */ 3986 cmp %g7, FAST_IMMU_MISS_TT 3987#ifdef sun4v 3988 MMU_FAULT_STATUS_AREA(%g4) 3989 ldx [%g4 + MMFSA_I_CTX], %g5 3990 ldx [%g4 + MMFSA_D_CTX], %g4 3991 move %icc, %g5, %g4 3992 cmp %g7, T_INSTR_MMU_MISS 3993 move %icc, %g5, %g4 3994#else 3995 mov MMU_TAG_ACCESS, %g4 3996 ldxa [%g4]ASI_DMMU, %g2 3997 ldxa [%g4]ASI_IMMU, %g5 3998 move %icc, %g5, %g2 3999 cmp %g7, T_INSTR_MMU_MISS 4000 move %icc, %g5, %g2 4001 sllx %g2, TAGACC_CTX_LSHIFT, %g4 4002#endif /* sun4v */ 4003 brnz,pn %g4, 3f /* skip if not kernel */ 4004 rdpr %tl, %g5 4005 4006 add %sp, STACK_BIAS, %g3 4007 srlx %g3, MMU_PAGESHIFT, %g3 4008 srlx %g2, MMU_PAGESHIFT, %g4 4009 cmp %g3, %g4 4010 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 4011 mov PTL1_BAD_STACK, %g1 4012 4013 cmp %g5, 1 4014 ble,pt %icc, 2f 4015 nop 4016 TSTAT_CHECK_TL1(2f, %g1, %g2) 4017 rdpr %tt, %g2 4018 cmp %g2, FAST_PROT_TT 4019 mov PTL1_BAD_KPROT_FAULT, %g1 4020 movne %icc, PTL1_BAD_KMISS, %g1 4021 ba,pt %icc, ptl1_panic 4022 nop 4023 40242: 4025 /* 4026 * We are taking a pagefault in the kernel on a kernel address. If 4027 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 4028 * want to call sfmmu_pagefault -- we will instead note that a fault 4029 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 4030 * (instead of a "retry"). This will step over the faulting 4031 * instruction. 4032 */ 4033 CPU_INDEX(%g1, %g2) 4034 set cpu_core, %g2 4035 sllx %g1, CPU_CORE_SHIFT, %g1 4036 add %g1, %g2, %g1 4037 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 4038 andcc %g2, CPU_DTRACE_NOFAULT, %g0 4039 bz sfmmu_pagefault 4040 or %g2, CPU_DTRACE_BADADDR, %g2 4041 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 4042 GET_MMU_D_ADDR(%g3, %g4) 4043 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 4044 done 4045 40463: 4047 cmp %g5, 1 4048 ble,pt %icc, 4f 4049 nop 4050 TSTAT_CHECK_TL1(4f, %g1, %g2) 4051 ba,pt %icc, sfmmu_window_trap 4052 nop 4053 40544: 4055 /* 4056 * We are taking a pagefault on a non-kernel address. If we are in 4057 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 4058 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 4059 */ 4060 CPU_INDEX(%g1, %g2) 4061 set cpu_core, %g2 4062 sllx %g1, CPU_CORE_SHIFT, %g1 4063 add %g1, %g2, %g1 4064 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 4065 andcc %g2, CPU_DTRACE_NOFAULT, %g0 4066 bz sfmmu_mmu_trap 4067 or %g2, CPU_DTRACE_BADADDR, %g2 4068 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 4069 GET_MMU_D_ADDR(%g3, %g4) 4070 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 4071 4072 /* 4073 * Be sure that we're actually taking this miss from the kernel -- 4074 * otherwise we have managed to return to user-level with 4075 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 4076 */ 4077 rdpr %tstate, %g2 4078 btst TSTATE_PRIV, %g2 4079 bz,a ptl1_panic 4080 mov PTL1_BAD_DTRACE_FLAGS, %g1 4081 done 4082 4083 ALTENTRY(tsb_tl0_noctxt) 4084 /* 4085 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 4086 * if it is, indicated that we have faulted and issue a done. 4087 */ 4088 CPU_INDEX(%g5, %g6) 4089 set cpu_core, %g6 4090 sllx %g5, CPU_CORE_SHIFT, %g5 4091 add %g5, %g6, %g5 4092 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 4093 andcc %g6, CPU_DTRACE_NOFAULT, %g0 4094 bz 1f 4095 or %g6, CPU_DTRACE_BADADDR, %g6 4096 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 4097 GET_MMU_D_ADDR(%g3, %g4) 4098 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 4099 4100 /* 4101 * Be sure that we're actually taking this miss from the kernel -- 4102 * otherwise we have managed to return to user-level with 4103 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 4104 */ 4105 rdpr %tstate, %g5 4106 btst TSTATE_PRIV, %g5 4107 bz,a ptl1_panic 4108 mov PTL1_BAD_DTRACE_FLAGS, %g1 4109 TSTAT_CHECK_TL1(2f, %g1, %g2); 41102: 4111 done 4112 41131: 4114 rdpr %tt, %g5 4115 cmp %g5, FAST_IMMU_MISS_TT 4116#ifdef sun4v 4117 MMU_FAULT_STATUS_AREA(%g2) 4118 be,a,pt %icc, 2f 4119 ldx [%g2 + MMFSA_I_CTX], %g3 4120 cmp %g5, T_INSTR_MMU_MISS 4121 be,a,pt %icc, 2f 4122 ldx [%g2 + MMFSA_I_CTX], %g3 4123 ldx [%g2 + MMFSA_D_CTX], %g3 41242: 4125#else 4126 mov MMU_TAG_ACCESS, %g2 4127 be,a,pt %icc, 2f 4128 ldxa [%g2]ASI_IMMU, %g3 4129 ldxa [%g2]ASI_DMMU, %g3 41302: sllx %g3, TAGACC_CTX_LSHIFT, %g3 4131#endif /* sun4v */ 4132 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 4133 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 4134 rdpr %tl, %g5 4135 cmp %g5, 1 4136 ble,pt %icc, sfmmu_mmu_trap 4137 nop 4138 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 4139 ba,pt %icc, sfmmu_window_trap 4140 nop 4141 SET_SIZE(sfmmu_tsb_miss) 4142#endif /* lint */ 4143 4144#if defined (lint) 4145/* 4146 * This routine will look for a user or kernel vaddr in the hash 4147 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 4148 * grab any locks. It should only be used by other sfmmu routines. 4149 */ 4150/* ARGSUSED */ 4151pfn_t 4152sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 4153{ 4154 return(0); 4155} 4156 4157/* ARGSUSED */ 4158pfn_t 4159sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno) 4160{ 4161 return(0); 4162} 4163 4164#else /* lint */ 4165 4166 ENTRY_NP(sfmmu_vatopfn) 4167 /* 4168 * disable interrupts 4169 */ 4170 rdpr %pstate, %o3 4171#ifdef DEBUG 4172 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 4173#endif 4174 /* 4175 * disable interrupts to protect the TSBMISS area 4176 */ 4177 andn %o3, PSTATE_IE, %o5 4178 wrpr %o5, 0, %pstate 4179 4180 /* 4181 * o0 = vaddr 4182 * o1 = sfmmup 4183 * o2 = ttep 4184 */ 4185 CPU_TSBMISS_AREA(%g1, %o5) 4186 ldn [%g1 + TSBMISS_KHATID], %o4 4187 cmp %o4, %o1 4188 bne,pn %ncc, vatopfn_nokernel 4189 mov TTE64K, %g5 /* g5 = rehash # */ 4190 mov %g1,%o5 /* o5 = tsbmiss_area */ 4191 /* 4192 * o0 = vaddr 4193 * o1 & o4 = hatid 4194 * o2 = ttep 4195 * o5 = tsbmiss area 4196 */ 4197 mov HBLK_RANGE_SHIFT, %g6 41981: 4199 4200 /* 4201 * o0 = vaddr 4202 * o1 = sfmmup 4203 * o2 = ttep 4204 * o3 = old %pstate 4205 * o4 = hatid 4206 * o5 = tsbmiss 4207 * g5 = rehash # 4208 * g6 = hmeshift 4209 * 4210 * The first arg to GET_TTE is actually tagaccess register 4211 * not just vaddr. Since this call is for kernel we need to clear 4212 * any lower vaddr bits that would be interpreted as ctx bits. 4213 */ 4214 set TAGACC_CTX_MASK, %g1 4215 andn %o0, %g1, %o0 4216 GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5, 4217 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 4218 4219kvtop_hblk_found: 4220 /* 4221 * o0 = vaddr 4222 * o1 = sfmmup 4223 * o2 = ttep 4224 * g1 = tte 4225 * g2 = tte pa 4226 * g3 = tte va 4227 * o2 = tsbmiss area 4228 * o1 = hat id 4229 */ 4230 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 4231 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4232 stx %g1,[%o2] /* put tte into *ttep */ 4233 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 4234 /* 4235 * o0 = vaddr 4236 * o1 = sfmmup 4237 * o2 = ttep 4238 * g1 = pfn 4239 */ 4240 ba,pt %xcc, 6f 4241 mov %g1, %o0 4242 4243kvtop_nohblk: 4244 /* 4245 * we get here if we couldn't find valid hblk in hash. We rehash 4246 * if neccesary. 4247 */ 4248 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 4249#ifdef sun4v 4250 cmp %g5, MAX_HASHCNT 4251#else 4252 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 4253#endif /* sun4v */ 4254 be,a,pn %icc, 6f 4255 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4256 mov %o1, %o4 /* restore hatid */ 4257#ifdef sun4v 4258 add %g5, 2, %g5 4259 cmp %g5, 3 4260 move %icc, MMU_PAGESHIFT4M, %g6 4261 ba,pt %icc, 1b 4262 movne %icc, MMU_PAGESHIFT256M, %g6 4263#else 4264 inc %g5 4265 cmp %g5, 2 4266 move %icc, MMU_PAGESHIFT512K, %g6 4267 ba,pt %icc, 1b 4268 movne %icc, MMU_PAGESHIFT4M, %g6 4269#endif /* sun4v */ 42706: 4271 retl 4272 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4273 4274tsb_suspend: 4275 /* 4276 * o0 = vaddr 4277 * o1 = sfmmup 4278 * o2 = ttep 4279 * g1 = tte 4280 * g2 = tte pa 4281 * g3 = tte va 4282 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 4283 */ 4284 stx %g1,[%o2] /* put tte into *ttep */ 4285 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 4286 sub %g0, 1, %o0 /* output = PFN_INVALID */ 4287 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 42888: 4289 retl 4290 wrpr %g0, %o3, %pstate /* enable interrupts */ 4291 4292vatopfn_nokernel: 4293 /* 4294 * This routine does NOT support user addresses 4295 * There is a routine in C that supports this. 4296 * The only reason why we don't have the C routine 4297 * support kernel addresses as well is because 4298 * we do va_to_pa while holding the hashlock. 4299 */ 4300 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4301 save %sp, -SA(MINFRAME), %sp 4302 sethi %hi(sfmmu_panic3), %o0 4303 call panic 4304 or %o0, %lo(sfmmu_panic3), %o0 4305 4306 SET_SIZE(sfmmu_vatopfn) 4307 4308 /* 4309 * %o0 = vaddr 4310 * %o1 = hashno (aka szc) 4311 * 4312 * 4313 * This routine is similar to sfmmu_vatopfn() but will only look for 4314 * a kernel vaddr in the hash structure for the specified rehash value. 4315 * It's just an optimization for the case when pagesize for a given 4316 * va range is already known (e.g. large page heap) and we don't want 4317 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 4318 * 4319 * Returns valid pfn or PFN_INVALID if 4320 * tte for specified rehash # is not found, invalid or suspended. 4321 */ 4322 ENTRY_NP(sfmmu_kvaszc2pfn) 4323 /* 4324 * disable interrupts 4325 */ 4326 rdpr %pstate, %o3 4327#ifdef DEBUG 4328 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 4329#endif 4330 /* 4331 * disable interrupts to protect the TSBMISS area 4332 */ 4333 andn %o3, PSTATE_IE, %o5 4334 wrpr %o5, 0, %pstate 4335 4336 CPU_TSBMISS_AREA(%g1, %o5) 4337 ldn [%g1 + TSBMISS_KHATID], %o4 4338 sll %o1, 1, %g6 4339 add %g6, %o1, %g6 4340 add %g6, MMU_PAGESHIFT, %g6 4341 /* 4342 * %o0 = vaddr 4343 * %o1 = hashno 4344 * %o3 = old %pstate 4345 * %o4 = ksfmmup 4346 * %g1 = tsbmiss area 4347 * %g6 = hmeshift 4348 */ 4349 4350 /* 4351 * The first arg to GET_TTE is actually tagaccess register 4352 * not just vaddr. Since this call is for kernel we need to clear 4353 * any lower vaddr bits that would be interpreted as ctx bits. 4354 */ 4355 srlx %o0, MMU_PAGESHIFT, %o0 4356 sllx %o0, MMU_PAGESHIFT, %o0 4357 GET_TTE(%o0, %o4, %g3, %g4, %g5, %g1, %o5, %g6, %o1, 4358 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 4359 kvaszc2pfn_nohblk) 4360 4361kvaszc2pfn_hblk_found: 4362 /* 4363 * %g3 = tte 4364 * %o0 = vaddr 4365 */ 4366 brgez,a,pn %g3, 1f /* check if tte is invalid */ 4367 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4368 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 4369 /* 4370 * g3 = pfn 4371 */ 4372 ba,pt %xcc, 1f 4373 mov %g3, %o0 4374 4375kvaszc2pfn_nohblk: 4376 mov -1, %o0 4377 43781: 4379 retl 4380 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4381 4382 SET_SIZE(sfmmu_kvaszc2pfn) 4383 4384#endif /* lint */ 4385 4386 4387 4388#if !defined(lint) 4389 4390/* 4391 * kpm lock used between trap level tsbmiss handler and kpm C level. 4392 */ 4393#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 4394 mov 0xff, tmp1 ;\ 4395label1: ;\ 4396 casa [kpmlckp]asi, %g0, tmp1 ;\ 4397 brnz,pn tmp1, label1 ;\ 4398 mov 0xff, tmp1 ;\ 4399 membar #LoadLoad 4400 4401#define KPMLOCK_EXIT(kpmlckp, asi) \ 4402 membar #LoadStore|#StoreStore ;\ 4403 sta %g0, [kpmlckp]asi 4404 4405/* 4406 * Lookup a memseg for a given pfn and if found, return the physical 4407 * address of the corresponding struct memseg in mseg, otherwise 4408 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 4409 * tsbmp, %asi is assumed to be ASI_MEM. 4410 * This lookup is done by strictly traversing only the physical memseg 4411 * linkage. The more generic approach, to check the virtual linkage 4412 * before using the physical (used e.g. with hmehash buckets), cannot 4413 * be used here. Memory DR operations can run in parallel to this 4414 * lookup w/o any locks and updates of the physical and virtual linkage 4415 * cannot be done atomically wrt. to each other. Because physical 4416 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 4417 * as "physical NULL" pointer. 4418 */ 4419#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 4420 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 4421 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 4422 udivx pfn, mseg, mseg ;\ 4423 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 4424 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 4425 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 4426 add tmp1, mseg, tmp1 ;\ 4427 ldxa [tmp1]%asi, mseg ;\ 4428 cmp mseg, MSEG_NULLPTR_PA ;\ 4429 be,pn %xcc, label/**/1 /* if not found */ ;\ 4430 nop ;\ 4431 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4432 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4433 blu,pn %xcc, label/**/1 ;\ 4434 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4435 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4436 bgeu,pn %xcc, label/**/1 ;\ 4437 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 4438 mulx tmp1, PAGE_SIZE, tmp1 ;\ 4439 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 4440 add tmp2, tmp1, tmp1 /* pp */ ;\ 4441 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 4442 cmp tmp2, pfn ;\ 4443 be,pt %xcc, label/**/_ok /* found */ ;\ 4444label/**/1: ;\ 4445 /* brute force lookup */ ;\ 4446 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 4447 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 4448label/**/2: ;\ 4449 cmp mseg, MSEG_NULLPTR_PA ;\ 4450 be,pn %xcc, label/**/_ok /* if not found */ ;\ 4451 nop ;\ 4452 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4453 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4454 blu,a,pt %xcc, label/**/2 ;\ 4455 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4456 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4457 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4458 bgeu,a,pt %xcc, label/**/2 ;\ 4459 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4460label/**/_ok: 4461 4462 /* 4463 * kpm tsb miss handler large pages 4464 * g1 = 8K kpm TSB entry pointer 4465 * g2 = tag access register 4466 * g3 = 4M kpm TSB entry pointer 4467 */ 4468 ALTENTRY(sfmmu_kpm_dtsb_miss) 4469 TT_TRACE(trace_tsbmiss) 4470 4471 CPU_INDEX(%g7, %g6) 4472 sethi %hi(kpmtsbm_area), %g6 4473 sllx %g7, KPMTSBM_SHIFT, %g7 4474 or %g6, %lo(kpmtsbm_area), %g6 4475 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4476 4477 /* check enable flag */ 4478 ldub [%g6 + KPMTSBM_FLAGS], %g4 4479 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4480 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4481 nop 4482 4483 /* VA range check */ 4484 ldx [%g6 + KPMTSBM_VBASE], %g7 4485 cmp %g2, %g7 4486 blu,pn %xcc, sfmmu_tsb_miss 4487 ldx [%g6 + KPMTSBM_VEND], %g5 4488 cmp %g2, %g5 4489 bgeu,pn %xcc, sfmmu_tsb_miss 4490 stx %g3, [%g6 + KPMTSBM_TSBPTR] 4491 4492 /* 4493 * check TL tsbmiss handling flag 4494 * bump tsbmiss counter 4495 */ 4496 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4497#ifdef DEBUG 4498 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4499 inc %g5 4500 brz,pn %g3, sfmmu_kpm_exception 4501 st %g5, [%g6 + KPMTSBM_TSBMISS] 4502#else 4503 inc %g5 4504 st %g5, [%g6 + KPMTSBM_TSBMISS] 4505#endif 4506 /* 4507 * At this point: 4508 * g1 = 8K kpm TSB pointer (not used) 4509 * g2 = tag access register 4510 * g3 = clobbered 4511 * g6 = per-CPU kpm tsbmiss area 4512 * g7 = kpm_vbase 4513 */ 4514 4515 /* vaddr2pfn */ 4516 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4517 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4518 srax %g4, %g3, %g2 /* which alias range (r) */ 4519 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4520 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4521 4522 /* 4523 * Setup %asi 4524 * mseg_pa = page_numtomemseg_nolock(pfn) 4525 * if (mseg_pa == NULL) sfmmu_kpm_exception 4526 * g2=pfn 4527 */ 4528 mov ASI_MEM, %asi 4529 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4530 cmp %g3, MSEG_NULLPTR_PA 4531 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4532 nop 4533 4534 /* 4535 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4536 * g2=pfn g3=mseg_pa 4537 */ 4538 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4539 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4540 srlx %g2, %g5, %g4 4541 sllx %g4, %g5, %g4 4542 sub %g4, %g7, %g4 4543 srlx %g4, %g5, %g4 4544 4545 /* 4546 * Validate inx value 4547 * g2=pfn g3=mseg_pa g4=inx 4548 */ 4549#ifdef DEBUG 4550 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4551 cmp %g4, %g5 /* inx - nkpmpgs */ 4552 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4553 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4554#else 4555 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4556#endif 4557 /* 4558 * kp = &mseg_pa->kpm_pages[inx] 4559 */ 4560 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4561 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4562 add %g5, %g4, %g5 /* kp */ 4563 4564 /* 4565 * KPMP_HASH(kp) 4566 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4567 */ 4568 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4569 sub %g7, 1, %g7 /* mask */ 4570 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4571 add %g5, %g1, %g5 /* y = ksp + x */ 4572 and %g5, %g7, %g5 /* hashinx = y & mask */ 4573 4574 /* 4575 * Calculate physical kpm_page pointer 4576 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4577 */ 4578 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4579 add %g1, %g4, %g1 /* kp_pa */ 4580 4581 /* 4582 * Calculate physical hash lock address 4583 * g1=kp_refcntc_pa g2=pfn g5=hashinx 4584 */ 4585 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4586 sllx %g5, KPMHLK_SHIFT, %g5 4587 add %g4, %g5, %g3 4588 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4589 4590 /* 4591 * Assemble tte 4592 * g1=kp_pa g2=pfn g3=hlck_pa 4593 */ 4594#ifdef sun4v 4595 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4596 sllx %g5, 32, %g5 4597 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4598 or %g4, TTE4M, %g4 4599 or %g5, %g4, %g5 4600#else 4601 sethi %hi(TTE_VALID_INT), %g4 4602 mov TTE4M, %g5 4603 sllx %g5, TTE_SZ_SHFT_INT, %g5 4604 or %g5, %g4, %g5 /* upper part */ 4605 sllx %g5, 32, %g5 4606 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4607 or %g5, %g4, %g5 4608#endif 4609 sllx %g2, MMU_PAGESHIFT, %g4 4610 or %g5, %g4, %g5 /* tte */ 4611 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4612 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4613 4614 /* 4615 * tsb dropin 4616 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4617 */ 4618 4619 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4620 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4621 4622 /* use C-handler if there's no go for dropin */ 4623 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4624 cmp %g7, -1 4625 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4626 nop 4627 4628#ifdef DEBUG 4629 /* double check refcnt */ 4630 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4631 brz,pn %g7, 5f /* let C-handler deal with this */ 4632 nop 4633#endif 4634 4635#ifndef sun4v 4636 ldub [%g6 + KPMTSBM_FLAGS], %g7 4637 mov ASI_N, %g1 4638 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4639 movnz %icc, ASI_MEM, %g1 4640 mov %g1, %asi 4641#endif 4642 4643 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4644 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4645 4646 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4647 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4648 4649 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4650 4651 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4652 KPMLOCK_EXIT(%g3, ASI_MEM) 4653 4654 /* 4655 * If trapstat is running, we need to shift the %tpc and %tnpc to 4656 * point to trapstat's TSB miss return code (note that trapstat 4657 * itself will patch the correct offset to add). 4658 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4659 */ 4660 rdpr %tl, %g7 4661 cmp %g7, 1 4662 ble %icc, 0f 4663 sethi %hi(KERNELBASE), %g6 4664 rdpr %tpc, %g7 4665 or %g6, %lo(KERNELBASE), %g6 4666 cmp %g7, %g6 4667 bgeu %xcc, 0f 4668 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4669 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4670 wrpr %g7, %tpc 4671 add %g7, 4, %g7 4672 wrpr %g7, %tnpc 46730: 4674 retry 46755: 4676 /* g3=hlck_pa */ 4677 KPMLOCK_EXIT(%g3, ASI_MEM) 4678 ba,pt %icc, sfmmu_kpm_exception 4679 nop 4680 SET_SIZE(sfmmu_kpm_dtsb_miss) 4681 4682 /* 4683 * kpm tsbmiss handler for smallpages 4684 * g1 = 8K kpm TSB pointer 4685 * g2 = tag access register 4686 * g3 = 4M kpm TSB pointer 4687 */ 4688 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4689 TT_TRACE(trace_tsbmiss) 4690 CPU_INDEX(%g7, %g6) 4691 sethi %hi(kpmtsbm_area), %g6 4692 sllx %g7, KPMTSBM_SHIFT, %g7 4693 or %g6, %lo(kpmtsbm_area), %g6 4694 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4695 4696 /* check enable flag */ 4697 ldub [%g6 + KPMTSBM_FLAGS], %g4 4698 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4699 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4700 nop 4701 4702 /* 4703 * VA range check 4704 * On fail: goto sfmmu_tsb_miss 4705 */ 4706 ldx [%g6 + KPMTSBM_VBASE], %g7 4707 cmp %g2, %g7 4708 blu,pn %xcc, sfmmu_tsb_miss 4709 ldx [%g6 + KPMTSBM_VEND], %g5 4710 cmp %g2, %g5 4711 bgeu,pn %xcc, sfmmu_tsb_miss 4712 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4713 4714 /* 4715 * check TL tsbmiss handling flag 4716 * bump tsbmiss counter 4717 */ 4718 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4719#ifdef DEBUG 4720 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4721 inc %g5 4722 brz,pn %g1, sfmmu_kpm_exception 4723 st %g5, [%g6 + KPMTSBM_TSBMISS] 4724#else 4725 inc %g5 4726 st %g5, [%g6 + KPMTSBM_TSBMISS] 4727#endif 4728 /* 4729 * At this point: 4730 * g1 = clobbered 4731 * g2 = tag access register 4732 * g3 = 4M kpm TSB pointer (not used) 4733 * g6 = per-CPU kpm tsbmiss area 4734 * g7 = kpm_vbase 4735 */ 4736 4737 /* 4738 * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr) 4739 * which is defined in mach_kpm.h. Any changes in that macro 4740 * should also be ported back to this assembly code. 4741 */ 4742 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 /* g3 = kpm_size_shift */ 4743 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4744 srax %g4, %g3, %g7 /* which alias range (r) */ 4745 brz,pt %g7, 2f 4746 sethi %hi(vac_colors_mask), %g5 4747 ld [%g5 + %lo(vac_colors_mask)], %g5 4748 4749 srlx %g2, MMU_PAGESHIFT, %g1 /* vaddr >> MMU_PAGESHIFT */ 4750 and %g1, %g5, %g1 /* g1 = v */ 4751 sllx %g7, %g3, %g5 /* g5 = r << kpm_size_shift */ 4752 cmp %g7, %g1 /* if (r > v) */ 4753 bleu,pn %xcc, 1f 4754 sub %g4, %g5, %g4 /* paddr -= r << kpm_size_shift */ 4755 sub %g7, %g1, %g5 /* g5 = r - v */ 4756 sllx %g5, MMU_PAGESHIFT, %g7 /* (r-v) << MMU_PAGESHIFT */ 4757 add %g4, %g7, %g4 /* paddr += (r-v)<<MMU_PAGESHIFT */ 4758 ba 2f 4759 nop 47601: 4761 sllx %g7, MMU_PAGESHIFT, %g5 /* else */ 4762 sub %g4, %g5, %g4 /* paddr -= r << MMU_PAGESHIFT */ 4763 4764 /* 4765 * paddr2pfn 4766 * g1 = vcolor (not used) 4767 * g2 = tag access register 4768 * g3 = clobbered 4769 * g4 = paddr 4770 * g5 = clobbered 4771 * g6 = per-CPU kpm tsbmiss area 4772 * g7 = clobbered 4773 */ 47742: 4775 srlx %g4, MMU_PAGESHIFT, %g2 /* g2 = pfn */ 4776 4777 /* 4778 * Setup %asi 4779 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4780 * if (mseg not found) sfmmu_kpm_exception 4781 * g2=pfn g6=per-CPU kpm tsbmiss area 4782 * g4 g5 g7 for scratch use. 4783 */ 4784 mov ASI_MEM, %asi 4785 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4786 cmp %g3, MSEG_NULLPTR_PA 4787 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4788 nop 4789 4790 /* 4791 * inx = pfn - mseg_pa->kpm_pbase 4792 * g2=pfn g3=mseg_pa g6=per-CPU kpm tsbmiss area 4793 */ 4794 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4795 sub %g2, %g7, %g4 4796 4797#ifdef DEBUG 4798 /* 4799 * Validate inx value 4800 * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area 4801 */ 4802 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4803 cmp %g4, %g5 /* inx - nkpmpgs */ 4804 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4805 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4806#else 4807 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4808#endif 4809 /* ksp = &mseg_pa->kpm_spages[inx] */ 4810 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4811 add %g5, %g4, %g5 /* ksp */ 4812 4813 /* 4814 * KPMP_SHASH(kp) 4815 * g2=pfn g3=mseg_pa g4=inx g5=ksp 4816 * g6=per-CPU kpm tsbmiss area g7=kpmp_stable_sz 4817 */ 4818 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4819 sub %g7, 1, %g7 /* mask */ 4820 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4821 add %g5, %g1, %g5 /* y = ksp + x */ 4822 and %g5, %g7, %g5 /* hashinx = y & mask */ 4823 4824 /* 4825 * Calculate physical kpm_spage pointer 4826 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4827 * g6=per-CPU kpm tsbmiss area 4828 */ 4829 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4830 add %g1, %g4, %g1 /* ksp_pa */ 4831 4832 /* 4833 * Calculate physical hash lock address. 4834 * Note: Changes in kpm_shlk_t must be reflected here. 4835 * g1=ksp_pa g2=pfn g5=hashinx 4836 * g6=per-CPU kpm tsbmiss area 4837 */ 4838 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4839 sllx %g5, KPMSHLK_SHIFT, %g5 4840 add %g4, %g5, %g3 /* hlck_pa */ 4841 4842 /* 4843 * Assemble non-cacheable tte initially 4844 * g1=ksp_pa g2=pfn g3=hlck_pa 4845 * g6=per-CPU kpm tsbmiss area 4846 */ 4847 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4848 sllx %g5, 32, %g5 4849 mov (TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4850 or %g5, %g4, %g5 4851 sllx %g2, MMU_PAGESHIFT, %g4 4852 or %g5, %g4, %g5 /* tte */ 4853 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4854 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4855 4856 /* 4857 * tsb dropin 4858 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable) 4859 * g6=per-CPU kpm tsbmiss area g7=scratch register 4860 */ 4861 4862 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4863 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4864 4865 /* use C-handler if there's no go for dropin */ 4866 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4867 andcc %g7, KPM_MAPPED_GO, %g0 /* go or no go ? */ 4868 bz,pt %icc, 5f /* no go */ 4869 nop 4870 and %g7, KPM_MAPPED_MASK, %g7 /* go */ 4871 cmp %g7, KPM_MAPPEDS /* cacheable ? */ 4872 be,a,pn %xcc, 3f 4873 or %g5, TTE_CV_INT, %g5 /* cacheable */ 48743: 4875#ifndef sun4v 4876 ldub [%g6 + KPMTSBM_FLAGS], %g7 4877 mov ASI_N, %g1 4878 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4879 movnz %icc, ASI_MEM, %g1 4880 mov %g1, %asi 4881#endif 4882 4883 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4884 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4885 4886 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4887 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4888 4889 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4890 4891 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4892 KPMLOCK_EXIT(%g3, ASI_MEM) 4893 4894 /* 4895 * If trapstat is running, we need to shift the %tpc and %tnpc to 4896 * point to trapstat's TSB miss return code (note that trapstat 4897 * itself will patch the correct offset to add). 4898 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4899 */ 4900 rdpr %tl, %g7 4901 cmp %g7, 1 4902 ble %icc, 0f 4903 sethi %hi(KERNELBASE), %g6 4904 rdpr %tpc, %g7 4905 or %g6, %lo(KERNELBASE), %g6 4906 cmp %g7, %g6 4907 bgeu %xcc, 0f 4908 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4909 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4910 wrpr %g7, %tpc 4911 add %g7, 4, %g7 4912 wrpr %g7, %tnpc 49130: 4914 retry 49155: 4916 /* g3=hlck_pa */ 4917 KPMLOCK_EXIT(%g3, ASI_MEM) 4918 ba,pt %icc, sfmmu_kpm_exception 4919 nop 4920 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4921 4922#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4923#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4924#endif 4925 4926#endif /* lint */ 4927 4928#ifdef lint 4929/* 4930 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4931 * Called from C-level, sets/clears "go" indication for trap level handler. 4932 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 4933 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 4934 * Assumes khl_mutex is held when called from C-level. 4935 */ 4936/* ARGSUSED */ 4937void 4938sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 4939{ 4940} 4941 4942/* 4943 * kpm_smallpages: stores val to byte at address mapped within 4944 * low level lock brackets. The old value is returned. 4945 * Called from C-level. 4946 */ 4947/* ARGSUSED */ 4948int 4949sfmmu_kpm_stsbmtl(uchar_t *mapped, uint_t *kshl_lock, int val) 4950{ 4951 return (0); 4952} 4953 4954#else /* lint */ 4955 4956 .seg ".data" 4957sfmmu_kpm_tsbmtl_panic: 4958 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4959 .byte 0 4960sfmmu_kpm_stsbmtl_panic: 4961 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4962 .byte 0 4963 .align 4 4964 .seg ".text" 4965 4966 ENTRY_NP(sfmmu_kpm_tsbmtl) 4967 rdpr %pstate, %o3 4968 /* 4969 * %o0 = &kp_refcntc 4970 * %o1 = &khl_lock 4971 * %o2 = 0/1 (off/on) 4972 * %o3 = pstate save 4973 */ 4974#ifdef DEBUG 4975 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4976 bnz,pt %icc, 1f /* disabled, panic */ 4977 nop 4978 save %sp, -SA(MINFRAME), %sp 4979 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4980 call panic 4981 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4982 ret 4983 restore 49841: 4985#endif /* DEBUG */ 4986 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4987 4988 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4989 mov -1, %o5 4990 brz,a %o2, 2f 4991 mov 0, %o5 49922: 4993 sth %o5, [%o0] 4994 KPMLOCK_EXIT(%o1, ASI_N) 4995 4996 retl 4997 wrpr %g0, %o3, %pstate /* enable interrupts */ 4998 SET_SIZE(sfmmu_kpm_tsbmtl) 4999 5000 ENTRY_NP(sfmmu_kpm_stsbmtl) 5001 rdpr %pstate, %o3 5002 /* 5003 * %o0 = &mapped 5004 * %o1 = &kshl_lock 5005 * %o2 = val 5006 * %o3 = pstate save 5007 */ 5008#ifdef DEBUG 5009 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 5010 bnz,pt %icc, 1f /* disabled, panic */ 5011 nop 5012 save %sp, -SA(MINFRAME), %sp 5013 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 5014 call panic 5015 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 5016 ret 5017 restore 50181: 5019#endif /* DEBUG */ 5020 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 5021 5022 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 5023 ldsb [%o0], %o5 5024 stb %o2, [%o0] 5025 KPMLOCK_EXIT(%o1, ASI_N) 5026 5027 and %o5, KPM_MAPPED_MASK, %o0 /* return old val */ 5028 retl 5029 wrpr %g0, %o3, %pstate /* enable interrupts */ 5030 SET_SIZE(sfmmu_kpm_stsbmtl) 5031 5032#endif /* lint */ 5033 5034#ifndef lint 5035#ifdef sun4v 5036 /* 5037 * User/kernel data miss w// multiple TSBs 5038 * The first probe covers 8K, 64K, and 512K page sizes, 5039 * because 64K and 512K mappings are replicated off 8K 5040 * pointer. Second probe covers 4M page size only. 5041 * 5042 * MMU fault area contains miss address and context. 5043 */ 5044 ALTENTRY(sfmmu_slow_dmmu_miss) 5045 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 5046 5047slow_miss_common: 5048 /* 5049 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 5050 * %g3 = ctx (cannot be INVALID_CONTEXT) 5051 */ 5052 brnz,pt %g3, 8f ! check for user context 5053 nop 5054 5055 /* 5056 * Kernel miss 5057 * Get 8K and 4M TSB pointers in %g1 and %g3 and 5058 * branch to sfmmu_tsb_miss_tt to handle it. 5059 */ 5060 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 5061sfmmu_dslow_patch_ktsb_base: 5062 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 5063sfmmu_dslow_patch_ktsb_szcode: 5064 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 5065 5066 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 5067 ! %g1 = First TSB entry pointer, as TSB miss handler expects 5068 5069 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 5070sfmmu_dslow_patch_ktsb4m_base: 5071 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 5072sfmmu_dslow_patch_ktsb4m_szcode: 5073 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 5074 5075 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 5076 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 5077 ba,a,pt %xcc, sfmmu_tsb_miss_tt 5078 .empty 5079 50808: 5081 /* 5082 * User miss 5083 * Get first TSB pointer in %g1 5084 * Get second TSB pointer (or NULL if no second TSB) in %g3 5085 * Branch to sfmmu_tsb_miss_tt to handle it 5086 */ 5087 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 5088 /* %g1 = first TSB entry ptr now, %g2 preserved */ 5089 5090 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 5091 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 5092 nop 5093 5094 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 5095 /* %g3 = second TSB entry ptr now, %g2 preserved */ 50969: 5097 ba,a,pt %xcc, sfmmu_tsb_miss_tt 5098 .empty 5099 SET_SIZE(sfmmu_slow_dmmu_miss) 5100 5101 5102 /* 5103 * User/kernel instruction miss w/ multiple TSBs 5104 * The first probe covers 8K, 64K, and 512K page sizes, 5105 * because 64K and 512K mappings are replicated off 8K 5106 * pointer. Second probe covers 4M page size only. 5107 * 5108 * MMU fault area contains miss address and context. 5109 */ 5110 ALTENTRY(sfmmu_slow_immu_miss) 5111 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 5112 ba,a,pt %xcc, slow_miss_common 5113 SET_SIZE(sfmmu_slow_immu_miss) 5114 5115#endif /* sun4v */ 5116#endif /* lint */ 5117 5118#ifndef lint 5119 5120/* 5121 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 5122 */ 5123 .seg ".data" 5124 .align 64 5125 .global tsbmiss_area 5126tsbmiss_area: 5127 .skip (TSBMISS_SIZE * NCPU) 5128 5129 .align 64 5130 .global kpmtsbm_area 5131kpmtsbm_area: 5132 .skip (KPMTSBM_SIZE * NCPU) 5133#endif /* lint */ 5134