1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26/* 27 * SFMMU primitives. These primitives should only be used by sfmmu 28 * routines. 29 */ 30 31#if defined(lint) 32#include <sys/types.h> 33#else /* lint */ 34#include "assym.h" 35#endif /* lint */ 36 37#include <sys/asm_linkage.h> 38#include <sys/machtrap.h> 39#include <sys/machasi.h> 40#include <sys/sun4asi.h> 41#include <sys/pte.h> 42#include <sys/mmu.h> 43#include <vm/hat_sfmmu.h> 44#include <vm/seg_spt.h> 45#include <sys/machparam.h> 46#include <sys/privregs.h> 47#include <sys/scb.h> 48#include <sys/intreg.h> 49#include <sys/machthread.h> 50#include <sys/intr.h> 51#include <sys/clock.h> 52#include <sys/trapstat.h> 53 54#ifdef TRAPTRACE 55#include <sys/traptrace.h> 56 57/* 58 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 59 */ 60#define TT_TRACE(label) \ 61 ba label ;\ 62 rd %pc, %g7 63#else 64 65#define TT_TRACE(label) 66 67#endif /* TRAPTRACE */ 68 69#ifndef lint 70 71#if (TTE_SUSPEND_SHIFT > 0) 72#define TTE_SUSPEND_INT_SHIFT(reg) \ 73 sllx reg, TTE_SUSPEND_SHIFT, reg 74#else 75#define TTE_SUSPEND_INT_SHIFT(reg) 76#endif 77 78#endif /* lint */ 79 80#ifndef lint 81 82/* 83 * Assumes TSBE_TAG is 0 84 * Assumes TSBE_INTHI is 0 85 * Assumes TSBREG.split is 0 86 */ 87 88#if TSBE_TAG != 0 89#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 90#endif 91 92#if TSBTAG_INTHI != 0 93#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 94#endif 95 96/* 97 * The following code assumes the tsb is not split. 98 * 99 * With TSBs no longer shared between processes, it's no longer 100 * necessary to hash the context bits into the tsb index to get 101 * tsb coloring; the new implementation treats the TSB as a 102 * direct-mapped, virtually-addressed cache. 103 * 104 * In: 105 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 106 * tsbbase = base address of TSB (clobbered) 107 * tagacc = tag access register (clobbered) 108 * szc = size code of TSB (ro) 109 * tmp = scratch reg 110 * Out: 111 * tsbbase = pointer to entry in TSB 112 */ 113#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 114 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 115 srlx tagacc, vpshift, tagacc ;\ 116 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 117 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 118 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 119 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 120 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 121 122/* 123 * When the kpm TSB is used it is assumed that it is direct mapped 124 * using (vaddr>>vpshift)%tsbsz as the index. 125 * 126 * Note that, for now, the kpm TSB and kernel TSB are the same for 127 * each mapping size. However that need not always be the case. If 128 * the trap handlers are updated to search a different TSB for kpm 129 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 130 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 131 * 132 * In: 133 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 134 * vaddr = virtual address (clobbered) 135 * tsbp, szc, tmp = scratch 136 * Out: 137 * tsbp = pointer to entry in TSB 138 */ 139#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 140 cmp vpshift, MMU_PAGESHIFT ;\ 141 bne,pn %icc, 1f /* branch if large case */ ;\ 142 sethi %hi(kpmsm_tsbsz), szc ;\ 143 sethi %hi(kpmsm_tsbbase), tsbp ;\ 144 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 145 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 146 ba,pt %icc, 2f ;\ 147 nop ;\ 1481: sethi %hi(kpm_tsbsz), szc ;\ 149 sethi %hi(kpm_tsbbase), tsbp ;\ 150 ld [szc + %lo(kpm_tsbsz)], szc ;\ 151 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1522: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 153 154/* 155 * Lock the TSBE at virtual address tsbep. 156 * 157 * tsbep = TSBE va (ro) 158 * tmp1, tmp2 = scratch registers (clobbered) 159 * label = label to jump to if we fail to lock the tsb entry 160 * %asi = ASI to use for TSB access 161 * 162 * NOTE that we flush the TSB using fast VIS instructions that 163 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 164 * not be treated as a locked entry or we'll get stuck spinning on 165 * an entry that isn't locked but really invalid. 166 */ 167 168#if defined(UTSB_PHYS) 169 170#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 171 lda [tsbep]ASI_MEM, tmp1 ;\ 172 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 173 cmp tmp1, tmp2 ;\ 174 be,a,pn %icc, label /* if locked ignore */ ;\ 175 nop ;\ 176 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 177 cmp tmp1, tmp2 ;\ 178 bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 179 nop ;\ 180 /* tsbe lock acquired */ ;\ 181 membar #StoreStore 182 183#else /* UTSB_PHYS */ 184 185#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 186 lda [tsbep]%asi, tmp1 ;\ 187 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 188 cmp tmp1, tmp2 ;\ 189 be,a,pn %icc, label /* if locked ignore */ ;\ 190 nop ;\ 191 casa [tsbep]%asi, tmp1, tmp2 ;\ 192 cmp tmp1, tmp2 ;\ 193 bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 194 nop ;\ 195 /* tsbe lock acquired */ ;\ 196 membar #StoreStore 197 198#endif /* UTSB_PHYS */ 199 200/* 201 * Atomically write TSBE at virtual address tsbep. 202 * 203 * tsbep = TSBE va (ro) 204 * tte = TSBE TTE (ro) 205 * tagtarget = TSBE tag (ro) 206 * %asi = ASI to use for TSB access 207 */ 208 209#if defined(UTSB_PHYS) 210 211#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 212 add tsbep, TSBE_TTE, tmp1 ;\ 213 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 214 membar #StoreStore ;\ 215 add tsbep, TSBE_TAG, tmp1 ;\ 216 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 217 218#else /* UTSB_PHYS */ 219 220#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 221 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 222 membar #StoreStore ;\ 223 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 224 225#endif /* UTSB_PHYS */ 226 227/* 228 * Load an entry into the TSB at TL > 0. 229 * 230 * tsbep = pointer to the TSBE to load as va (ro) 231 * tte = value of the TTE retrieved and loaded (wo) 232 * tagtarget = tag target register. To get TSBE tag to load, 233 * we need to mask off the context and leave only the va (clobbered) 234 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 235 * tmp1, tmp2 = scratch registers 236 * label = label to jump to if we fail to lock the tsb entry 237 * %asi = ASI to use for TSB access 238 */ 239 240#if defined(UTSB_PHYS) 241 242#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 243 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 244 /* ;\ 245 * I don't need to update the TSB then check for the valid tte. ;\ 246 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 247 * we always invalidate the hash table before we unload the TSB.;\ 248 */ ;\ 249 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 250 ldxa [ttepa]ASI_MEM, tte ;\ 251 TTE_CLR_SOFTEXEC_ML(tte) ;\ 252 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 253 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 254 add tsbep, TSBE_TAG, tmp1 ;\ 255 brgez,a,pn tte, label ;\ 256 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 257 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 258label: 259 260#else /* UTSB_PHYS */ 261 262#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 263 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 264 /* ;\ 265 * I don't need to update the TSB then check for the valid tte. ;\ 266 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 267 * we always invalidate the hash table before we unload the TSB.;\ 268 */ ;\ 269 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 270 ldxa [ttepa]ASI_MEM, tte ;\ 271 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 272 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 273 brgez,a,pn tte, label ;\ 274 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 275 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 276label: 277 278#endif /* UTSB_PHYS */ 279 280/* 281 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 282 * for ITLB synthesis. 283 * 284 * tsbep = pointer to the TSBE to load as va (ro) 285 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 286 * with exec_perm turned off and exec_synth turned on 287 * tagtarget = tag target register. To get TSBE tag to load, 288 * we need to mask off the context and leave only the va (clobbered) 289 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 290 * tmp1, tmp2 = scratch registers 291 * label = label to use for branch (text) 292 * %asi = ASI to use for TSB access 293 */ 294 295#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 296 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 297 /* ;\ 298 * I don't need to update the TSB then check for the valid tte. ;\ 299 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 300 * we always invalidate the hash table before we unload the TSB.;\ 301 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 302 * and exec_synth bit to 1. ;\ 303 */ ;\ 304 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 305 mov tte, tmp1 ;\ 306 ldxa [ttepa]ASI_MEM, tte ;\ 307 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 308 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 309 brgez,a,pn tte, label ;\ 310 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 311 or tte, tmp1, tte ;\ 312 andn tte, TTE_EXECPRM_INT, tte ;\ 313 or tte, TTE_E_SYNTH_INT, tte ;\ 314 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 315label: 316 317/* 318 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 319 * 320 * tte = value of the TTE, used to get tte_size bits (ro) 321 * tagaccess = tag access register, used to get 4M pfn bits (ro) 322 * pfn = 4M pfn bits shifted to offset for tte (out) 323 * tmp1 = scratch register 324 * label = label to use for branch (text) 325 */ 326 327#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 328 /* ;\ 329 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 330 * Return them, shifted, in pfn. ;\ 331 */ ;\ 332 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 333 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 334 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 335 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 336 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 337 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 338label: ;\ 339 sllx tagaccess, MMU_PAGESHIFT4M, pfn 340 341/* 342 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 343 * for ITLB synthesis. 344 * 345 * tte = value of the TTE, used to get tte_size bits (rw) 346 * tmp1 = scratch register 347 */ 348 349#define SET_TTE4M_PN(tte, tmp) \ 350 /* ;\ 351 * Set 4M pagesize tte bits. ;\ 352 */ ;\ 353 set TTE4M, tmp ;\ 354 sllx tmp, TTE_SZ_SHFT, tmp ;\ 355 or tte, tmp, tte 356 357/* 358 * Load an entry into the TSB at TL=0. 359 * 360 * tsbep = pointer to the TSBE to load as va (ro) 361 * tteva = pointer to the TTE to load as va (ro) 362 * tagtarget = TSBE tag to load (which contains no context), synthesized 363 * to match va of MMU tag target register only (ro) 364 * tmp1, tmp2 = scratch registers (clobbered) 365 * label = label to use for branches (text) 366 * %asi = ASI to use for TSB access 367 */ 368 369#if defined(UTSB_PHYS) 370 371#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 372 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 373 ldx [tteva], tteva /* load tte */ ;\ 374 TTE_CLR_SOFTEXEC_ML(tteva) ;\ 375 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 376 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 377 add tsbep, TSBE_TAG, tmp1 ;\ 378 brgez,a,pn tteva, label ;\ 379 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 380 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 381label: 382 383#else /* UTSB_PHYS */ 384 385#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 386 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 387 ldx [tteva], tteva /* load tte */ ;\ 388 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 389 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 390 brgez,a,pn tteva, label ;\ 391 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 392 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 393label: 394 395#endif /* UTSB_PHYS */ 396 397/* 398 * Invalidate a TSB entry in the TSB. 399 * 400 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 401 * about this earlier to ensure this is true. Thus when we are 402 * directly referencing tsbep below, we are referencing the tte_tag 403 * field of the TSBE. If this offset ever changes, the code below 404 * will need to be modified. 405 * 406 * tsbep = pointer to TSBE as va (ro) 407 * tag = invalidation is done if this matches the TSBE tag (ro) 408 * tmp1 - tmp3 = scratch registers (clobbered) 409 * label = label name to use for branches (text) 410 * %asi = ASI to use for TSB access 411 */ 412 413#if defined(UTSB_PHYS) 414 415#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 416 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 417 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 418label/**/1: ;\ 419 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 420 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 421 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 422 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 423 cmp tag, tmp3 /* compare tags */ ;\ 424 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 425 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 426 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 427 cmp tmp1, tmp3 /* if not successful */ ;\ 428 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 429 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 430label/**/2: 431 432#else /* UTSB_PHYS */ 433 434#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 435 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 436 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 437label/**/1: ;\ 438 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 439 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 440 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 441 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 442 cmp tag, tmp3 /* compare tags */ ;\ 443 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 444 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 445 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 446 cmp tmp1, tmp3 /* if not successful */ ;\ 447 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 448 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 449label/**/2: 450 451#endif /* UTSB_PHYS */ 452 453#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 454#error - TSB_SOFTSZ_MASK too small 455#endif 456 457 458/* 459 * An implementation of setx which will be hot patched at run time. 460 * since it is being hot patched, there is no value passed in. 461 * Thus, essentially we are implementing 462 * setx value, tmp, dest 463 * where value is RUNTIME_PATCH (aka 0) in this case. 464 */ 465#define RUNTIME_PATCH_SETX(dest, tmp) \ 466 sethi %hh(RUNTIME_PATCH), tmp ;\ 467 sethi %lm(RUNTIME_PATCH), dest ;\ 468 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 469 or dest, %lo(RUNTIME_PATCH), dest ;\ 470 sllx tmp, 32, tmp ;\ 471 nop /* for perf reasons */ ;\ 472 or tmp, dest, dest /* contents of patched value */ 473 474#endif /* lint */ 475 476 477#if defined (lint) 478 479/* 480 * sfmmu related subroutines 481 */ 482uint_t 483sfmmu_disable_intrs() 484{ return(0); } 485 486/* ARGSUSED */ 487void 488sfmmu_enable_intrs(uint_t pstate_save) 489{} 490 491/* ARGSUSED */ 492int 493sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag) 494{ return(0); } 495 496/* 497 * Use cas, if tte has changed underneath us then reread and try again. 498 * In the case of a retry, it will update sttep with the new original. 499 */ 500/* ARGSUSED */ 501int 502sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 503{ return(0); } 504 505/* 506 * Use cas, if tte has changed underneath us then return 1, else return 0 507 */ 508/* ARGSUSED */ 509int 510sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 511{ return(0); } 512 513/* ARGSUSED */ 514void 515sfmmu_copytte(tte_t *sttep, tte_t *dttep) 516{} 517 518/*ARGSUSED*/ 519struct tsbe * 520sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 521{ return(0); } 522 523/*ARGSUSED*/ 524uint64_t 525sfmmu_make_tsbtag(caddr_t va) 526{ return(0); } 527 528#else /* lint */ 529 530 .seg ".data" 531 .global sfmmu_panic1 532sfmmu_panic1: 533 .asciz "sfmmu_asm: interrupts already disabled" 534 535 .global sfmmu_panic3 536sfmmu_panic3: 537 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 538 539 .global sfmmu_panic4 540sfmmu_panic4: 541 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 542 543 .global sfmmu_panic5 544sfmmu_panic5: 545 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 546 547 .global sfmmu_panic6 548sfmmu_panic6: 549 .asciz "sfmmu_asm: interrupts not disabled" 550 551 .global sfmmu_panic7 552sfmmu_panic7: 553 .asciz "sfmmu_asm: kernel as" 554 555 .global sfmmu_panic8 556sfmmu_panic8: 557 .asciz "sfmmu_asm: gnum is zero" 558 559 .global sfmmu_panic9 560sfmmu_panic9: 561 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 562 563 .global sfmmu_panic10 564sfmmu_panic10: 565 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 566 567 ENTRY(sfmmu_disable_intrs) 568 rdpr %pstate, %o0 569#ifdef DEBUG 570 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 571#endif /* DEBUG */ 572 retl 573 wrpr %o0, PSTATE_IE, %pstate 574 SET_SIZE(sfmmu_disable_intrs) 575 576 ENTRY(sfmmu_enable_intrs) 577 retl 578 wrpr %g0, %o0, %pstate 579 SET_SIZE(sfmmu_enable_intrs) 580 581/* 582 * This routine is called both by resume() and sfmmu_get_ctx() to 583 * allocate a new context for the process on a MMU. 584 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 585 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 586 * is the case when sfmmu_alloc_ctx is called from resume(). 587 * 588 * The caller must disable interrupts before entering this routine. 589 * To reduce ctx switch overhead, the code contains both 'fast path' and 590 * 'slow path' code. The fast path code covers the common case where only 591 * a quick check is needed and the real ctx allocation is not required. 592 * It can be done without holding the per-process (PP) lock. 593 * The 'slow path' code must be protected by the PP Lock and performs ctx 594 * allocation. 595 * Hardware context register and HAT mmu cnum are updated accordingly. 596 * 597 * %o0 - sfmmup 598 * %o1 - allocflag 599 * %o2 - CPU 600 * %o3 - sfmmu private/shared flag 601 * 602 * ret - 0: no ctx is allocated 603 * 1: a ctx is allocated 604 */ 605 ENTRY_NP(sfmmu_alloc_ctx) 606 607#ifdef DEBUG 608 sethi %hi(ksfmmup), %g1 609 ldx [%g1 + %lo(ksfmmup)], %g1 610 cmp %g1, %o0 611 bne,pt %xcc, 0f 612 nop 613 614 sethi %hi(panicstr), %g1 ! if kernel as, panic 615 ldx [%g1 + %lo(panicstr)], %g1 616 tst %g1 617 bnz,pn %icc, 7f 618 nop 619 620 sethi %hi(sfmmu_panic7), %o0 621 call panic 622 or %o0, %lo(sfmmu_panic7), %o0 623 6247: 625 retl 626 mov %g0, %o0 ! %o0 = ret = 0 627 6280: 629 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 630#endif /* DEBUG */ 631 632 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 633 634 ! load global mmu_ctxp info 635 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 636 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 637 638 ! load global mmu_ctxp gnum 639 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 640 641#ifdef DEBUG 642 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 643 bne,pt %xcc, 3f 644 nop 645 646 sethi %hi(panicstr), %g1 ! test if panicstr is already set 647 ldx [%g1 + %lo(panicstr)], %g1 648 tst %g1 649 bnz,pn %icc, 1f 650 nop 651 652 sethi %hi(sfmmu_panic8), %o0 653 call panic 654 or %o0, %lo(sfmmu_panic8), %o0 6551: 656 retl 657 mov %g0, %o0 ! %o0 = ret = 0 6583: 659#endif 660 661 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 662 663 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 664 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 665 666 /* 667 * %g5 = sfmmu gnum returned 668 * %g6 = sfmmu cnum returned 669 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 670 * %g4 = scratch 671 * 672 * Fast path code, do a quick check. 673 */ 674 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 675 676 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 677 bne,pt %icc, 1f ! valid hat cnum, check gnum 678 nop 679 680 ! cnum == INVALID, check allocflag 681 mov %g0, %g4 ! %g4 = ret = 0 682 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 683 mov %g6, %o1 684 685 ! (invalid HAT cnum) && (allocflag == 1) 686 ba,pt %icc, 2f 687 nop 6881: 689 ! valid HAT cnum, check gnum 690 cmp %g5, %o4 691 mov 1, %g4 !%g4 = ret = 1 692 be,a,pt %icc, 8f ! gnum unchanged, go to done 693 mov %g6, %o1 694 6952: 696 /* 697 * Grab per process (PP) sfmmu_ctx_lock spinlock, 698 * followed by the 'slow path' code. 699 */ 700 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 7013: 702 brz %g3, 5f 703 nop 7044: 705 brnz,a,pt %g3, 4b ! spin if lock is 1 706 ldub [%o0 + SFMMU_CTX_LOCK], %g3 707 ba %xcc, 3b ! retry the lock 708 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 709 7105: 711 membar #LoadLoad 712 /* 713 * %g5 = sfmmu gnum returned 714 * %g6 = sfmmu cnum returned 715 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 716 * %g4 = scratch 717 */ 718 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 719 720 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 721 bne,pt %icc, 1f ! valid hat cnum, check gnum 722 nop 723 724 ! cnum == INVALID, check allocflag 725 mov %g0, %g4 ! %g4 = ret = 0 726 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 727 mov %g6, %o1 728 729 ! (invalid HAT cnum) && (allocflag == 1) 730 ba,pt %icc, 6f 731 nop 7321: 733 ! valid HAT cnum, check gnum 734 cmp %g5, %o4 735 mov 1, %g4 ! %g4 = ret = 1 736 be,a,pt %icc, 2f ! gnum unchanged, go to done 737 mov %g6, %o1 738 739 ba,pt %icc, 6f 740 nop 7412: 742 membar #LoadStore|#StoreStore 743 ba,pt %icc, 8f 744 clrb [%o0 + SFMMU_CTX_LOCK] 7456: 746 /* 747 * We get here if we do not have a valid context, or 748 * the HAT gnum does not match global gnum. We hold 749 * sfmmu_ctx_lock spinlock. Allocate that context. 750 * 751 * %o3 = mmu_ctxp 752 */ 753 add %o3, MMU_CTX_CNUM, %g3 754 ld [%o3 + MMU_CTX_NCTXS], %g4 755 756 /* 757 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 758 * %g3 = mmu cnum address 759 * %g4 = mmu nctxs 760 * 761 * %o0 = sfmmup 762 * %o1 = mmu current cnum value (used as new cnum) 763 * %o4 = mmu gnum 764 * 765 * %o5 = scratch 766 */ 767 ld [%g3], %o1 7680: 769 cmp %o1, %g4 770 bl,a,pt %icc, 1f 771 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 772 773 /* 774 * cnum reachs max, bail, so wrap around can be performed later. 775 */ 776 set INVALID_CONTEXT, %o1 777 mov %g0, %g4 ! %g4 = ret = 0 778 779 membar #LoadStore|#StoreStore 780 ba,pt %icc, 8f 781 clrb [%o0 + SFMMU_CTX_LOCK] 7821: 783 ! %g3 = addr of mmu_ctxp->cnum 784 ! %o5 = mmu_ctxp->cnum + 1 785 cas [%g3], %o1, %o5 786 cmp %o1, %o5 787 bne,a,pn %xcc, 0b ! cas failed 788 ld [%g3], %o1 789 790#ifdef DEBUG 791 set MAX_SFMMU_CTX_VAL, %o5 792 cmp %o1, %o5 793 ble,pt %icc, 2f 794 nop 795 796 sethi %hi(sfmmu_panic9), %o0 797 call panic 798 or %o0, %lo(sfmmu_panic9), %o0 7992: 800#endif 801 ! update hat gnum and cnum 802 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 803 or %o4, %o1, %o4 804 stx %o4, [%g2 + SFMMU_CTXS] 805 806 membar #LoadStore|#StoreStore 807 clrb [%o0 + SFMMU_CTX_LOCK] 808 809 mov 1, %g4 ! %g4 = ret = 1 8108: 811 /* 812 * program the secondary context register 813 * 814 * %o1 = cnum 815 * %g1 = sfmmu private/shared flag (0:private, 1:shared) 816 */ 817 818 /* 819 * When we come here and context is invalid, we want to set both 820 * private and shared ctx regs to INVALID. In order to 821 * do so, we set the sfmmu priv/shared flag to 'private' regardless 822 * so that private ctx reg will be set to invalid. 823 * Note that on sun4v values written to private context register are 824 * automatically written to corresponding shared context register as 825 * well. On sun4u SET_SECCTX() will invalidate shared context register 826 * when it sets a private secondary context register. 827 */ 828 829 cmp %o1, INVALID_CONTEXT 830 be,a,pn %icc, 9f 831 clr %g1 8329: 833 834#ifdef sun4u 835 ldub [%o0 + SFMMU_CEXT], %o2 836 sll %o2, CTXREG_EXT_SHIFT, %o2 837 or %o1, %o2, %o1 838#endif /* sun4u */ 839 840 SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1) 841 842 retl 843 mov %g4, %o0 ! %o0 = ret 844 845 SET_SIZE(sfmmu_alloc_ctx) 846 847 ENTRY_NP(sfmmu_modifytte) 848 ldx [%o2], %g3 /* current */ 849 ldx [%o0], %g1 /* original */ 8502: 851 ldx [%o1], %g2 /* modified */ 852 cmp %g2, %g3 /* is modified = current? */ 853 be,a,pt %xcc,1f /* yes, don't write */ 854 stx %g3, [%o0] /* update new original */ 855 casx [%o2], %g1, %g2 856 cmp %g1, %g2 857 be,pt %xcc, 1f /* cas succeeded - return */ 858 nop 859 ldx [%o2], %g3 /* new current */ 860 stx %g3, [%o0] /* save as new original */ 861 ba,pt %xcc, 2b 862 mov %g3, %g1 8631: retl 864 membar #StoreLoad 865 SET_SIZE(sfmmu_modifytte) 866 867 ENTRY_NP(sfmmu_modifytte_try) 868 ldx [%o1], %g2 /* modified */ 869 ldx [%o2], %g3 /* current */ 870 ldx [%o0], %g1 /* original */ 871 cmp %g3, %g2 /* is modified = current? */ 872 be,a,pn %xcc,1f /* yes, don't write */ 873 mov 0, %o1 /* as if cas failed. */ 874 875 casx [%o2], %g1, %g2 876 membar #StoreLoad 877 cmp %g1, %g2 878 movne %xcc, -1, %o1 /* cas failed. */ 879 move %xcc, 1, %o1 /* cas succeeded. */ 8801: 881 stx %g2, [%o0] /* report "current" value */ 882 retl 883 mov %o1, %o0 884 SET_SIZE(sfmmu_modifytte_try) 885 886 ENTRY_NP(sfmmu_copytte) 887 ldx [%o0], %g1 888 retl 889 stx %g1, [%o1] 890 SET_SIZE(sfmmu_copytte) 891 892 893 /* 894 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 895 * %o0 = TSB base address (in), pointer to TSB entry (out) 896 * %o1 = vaddr (in) 897 * %o2 = vpshift (in) 898 * %o3 = tsb size code (in) 899 * %o4 = scratch register 900 */ 901 ENTRY_NP(sfmmu_get_tsbe) 902 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 903 retl 904 nop 905 SET_SIZE(sfmmu_get_tsbe) 906 907 /* 908 * Return a TSB tag for the given va. 909 * %o0 = va (in/clobbered) 910 * %o0 = va shifted to be in tsb tag format (with no context) (out) 911 */ 912 ENTRY_NP(sfmmu_make_tsbtag) 913 retl 914 srln %o0, TTARGET_VA_SHIFT, %o0 915 SET_SIZE(sfmmu_make_tsbtag) 916 917#endif /* lint */ 918 919/* 920 * Other sfmmu primitives 921 */ 922 923 924#if defined (lint) 925void 926sfmmu_patch_ktsb(void) 927{ 928} 929 930void 931sfmmu_kpm_patch_tlbm(void) 932{ 933} 934 935void 936sfmmu_kpm_patch_tsbm(void) 937{ 938} 939 940void 941sfmmu_patch_shctx(void) 942{ 943} 944 945/* ARGSUSED */ 946void 947sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 948{ 949} 950 951/* ARGSUSED */ 952void 953sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 954{ 955} 956 957/* ARGSUSED */ 958void 959sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 960{ 961} 962 963/* ARGSUSED */ 964void 965sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 966{ 967} 968 969#else /* lint */ 970 971#define I_SIZE 4 972 973 ENTRY_NP(sfmmu_fix_ktlb_traptable) 974 /* 975 * %o0 = start of patch area 976 * %o1 = size code of TSB to patch 977 * %o3 = scratch 978 */ 979 /* fix sll */ 980 ld [%o0], %o3 /* get sll */ 981 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 982 st %o3, [%o0] /* write sll */ 983 flush %o0 984 /* fix srl */ 985 add %o0, I_SIZE, %o0 /* goto next instr. */ 986 ld [%o0], %o3 /* get srl */ 987 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 988 st %o3, [%o0] /* write srl */ 989 retl 990 flush %o0 991 SET_SIZE(sfmmu_fix_ktlb_traptable) 992 993 ENTRY_NP(sfmmu_fixup_ktsbbase) 994 /* 995 * %o0 = start of patch area 996 * %o5 = kernel virtual or physical tsb base address 997 * %o2, %o3 are used as scratch registers. 998 */ 999 /* fixup sethi instruction */ 1000 ld [%o0], %o3 1001 srl %o5, 10, %o2 ! offset is bits 32:10 1002 or %o3, %o2, %o3 ! set imm22 1003 st %o3, [%o0] 1004 /* fixup offset of lduw/ldx */ 1005 add %o0, I_SIZE, %o0 ! next instr 1006 ld [%o0], %o3 1007 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 1008 or %o3, %o2, %o3 1009 st %o3, [%o0] 1010 retl 1011 flush %o0 1012 SET_SIZE(sfmmu_fixup_ktsbbase) 1013 1014 ENTRY_NP(sfmmu_fixup_setx) 1015 /* 1016 * %o0 = start of patch area 1017 * %o4 = 64 bit value to patch 1018 * %o2, %o3 are used as scratch registers. 1019 * 1020 * Note: Assuming that all parts of the instructions which need to be 1021 * patched correspond to RUNTIME_PATCH (aka 0) 1022 * 1023 * Note the implementation of setx which is being patched is as follows: 1024 * 1025 * sethi %hh(RUNTIME_PATCH), tmp 1026 * sethi %lm(RUNTIME_PATCH), dest 1027 * or tmp, %hm(RUNTIME_PATCH), tmp 1028 * or dest, %lo(RUNTIME_PATCH), dest 1029 * sllx tmp, 32, tmp 1030 * nop 1031 * or tmp, dest, dest 1032 * 1033 * which differs from the implementation in the 1034 * "SPARC Architecture Manual" 1035 */ 1036 /* fixup sethi instruction */ 1037 ld [%o0], %o3 1038 srlx %o4, 42, %o2 ! bits [63:42] 1039 or %o3, %o2, %o3 ! set imm22 1040 st %o3, [%o0] 1041 /* fixup sethi instruction */ 1042 add %o0, I_SIZE, %o0 ! next instr 1043 ld [%o0], %o3 1044 sllx %o4, 32, %o2 ! clear upper bits 1045 srlx %o2, 42, %o2 ! bits [31:10] 1046 or %o3, %o2, %o3 ! set imm22 1047 st %o3, [%o0] 1048 /* fixup or instruction */ 1049 add %o0, I_SIZE, %o0 ! next instr 1050 ld [%o0], %o3 1051 srlx %o4, 32, %o2 ! bits [63:32] 1052 and %o2, 0x3ff, %o2 ! bits [41:32] 1053 or %o3, %o2, %o3 ! set imm 1054 st %o3, [%o0] 1055 /* fixup or instruction */ 1056 add %o0, I_SIZE, %o0 ! next instr 1057 ld [%o0], %o3 1058 and %o4, 0x3ff, %o2 ! bits [9:0] 1059 or %o3, %o2, %o3 ! set imm 1060 st %o3, [%o0] 1061 retl 1062 flush %o0 1063 SET_SIZE(sfmmu_fixup_setx) 1064 1065 ENTRY_NP(sfmmu_fixup_or) 1066 /* 1067 * %o0 = start of patch area 1068 * %o4 = 32 bit value to patch 1069 * %o2, %o3 are used as scratch registers. 1070 * Note: Assuming that all parts of the instructions which need to be 1071 * patched correspond to RUNTIME_PATCH (aka 0) 1072 */ 1073 ld [%o0], %o3 1074 and %o4, 0x3ff, %o2 ! bits [9:0] 1075 or %o3, %o2, %o3 ! set imm 1076 st %o3, [%o0] 1077 retl 1078 flush %o0 1079 SET_SIZE(sfmmu_fixup_or) 1080 1081 ENTRY_NP(sfmmu_fixup_shiftx) 1082 /* 1083 * %o0 = start of patch area 1084 * %o4 = signed int immediate value to add to sllx/srlx imm field 1085 * %o2, %o3 are used as scratch registers. 1086 * 1087 * sllx/srlx store the 6 bit immediate value in the lowest order bits 1088 * so we do a simple add. The caller must be careful to prevent 1089 * overflow, which could easily occur if the initial value is nonzero! 1090 */ 1091 ld [%o0], %o3 ! %o3 = instruction to patch 1092 and %o3, 0x3f, %o2 ! %o2 = existing imm value 1093 add %o2, %o4, %o2 ! %o2 = new imm value 1094 andn %o3, 0x3f, %o3 ! clear old imm value 1095 and %o2, 0x3f, %o2 ! truncate new imm value 1096 or %o3, %o2, %o3 ! set new imm value 1097 st %o3, [%o0] ! store updated instruction 1098 retl 1099 flush %o0 1100 SET_SIZE(sfmmu_fixup_shiftx) 1101 1102 ENTRY_NP(sfmmu_fixup_mmu_asi) 1103 /* 1104 * Patch imm_asi of all ldda instructions in the MMU 1105 * trap handlers. We search MMU_PATCH_INSTR instructions 1106 * starting from the itlb miss handler (trap 0x64). 1107 * %o0 = address of tt[0,1]_itlbmiss 1108 * %o1 = imm_asi to setup, shifted by appropriate offset. 1109 * %o3 = number of instructions to search 1110 * %o4 = reserved by caller: called from leaf routine 1111 */ 11121: ldsw [%o0], %o2 ! load instruction to %o2 1113 brgez,pt %o2, 2f 1114 srl %o2, 30, %o5 1115 btst 1, %o5 ! test bit 30; skip if not set 1116 bz,pt %icc, 2f 1117 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1118 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1119 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1120 brnz,pt %o5, 2f ! skip if not a match 1121 or %o2, %o1, %o2 ! or in imm_asi 1122 st %o2, [%o0] ! write patched instruction 11232: dec %o3 1124 brnz,a,pt %o3, 1b ! loop until we're done 1125 add %o0, I_SIZE, %o0 1126 retl 1127 flush %o0 1128 SET_SIZE(sfmmu_fixup_mmu_asi) 1129 1130 /* 1131 * Patch immediate ASI used to access the TSB in the 1132 * trap table. 1133 * inputs: %o0 = value of ktsb_phys 1134 */ 1135 ENTRY_NP(sfmmu_patch_mmu_asi) 1136 mov %o7, %o4 ! save return pc in %o4 1137 movrnz %o0, ASI_QUAD_LDD_PHYS, %o3 1138 movrz %o0, ASI_NQUAD_LD, %o3 1139 sll %o3, 5, %o1 ! imm_asi offset 1140 mov 6, %o3 ! number of instructions 1141 sethi %hi(dktsb), %o0 ! to search 1142 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1143 or %o0, %lo(dktsb), %o0 1144 mov 6, %o3 ! number of instructions 1145 sethi %hi(dktsb4m), %o0 ! to search 1146 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1147 or %o0, %lo(dktsb4m), %o0 1148 mov 6, %o3 ! number of instructions 1149 sethi %hi(iktsb), %o0 ! to search 1150 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1151 or %o0, %lo(iktsb), %o0 1152 mov 6, %o3 ! number of instructions 1153 sethi %hi(iktsb4m), %o0 ! to search 1154 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1155 or %o0, %lo(iktsb4m), %o0 1156 mov %o4, %o7 ! retore return pc -- leaf 1157 retl 1158 nop 1159 SET_SIZE(sfmmu_patch_mmu_asi) 1160 1161 ENTRY_NP(sfmmu_patch_ktsb) 1162 /* 1163 * We need to fix iktsb, dktsb, et. al. 1164 */ 1165 save %sp, -SA(MINFRAME), %sp 1166 set ktsb_phys, %o1 1167 ld [%o1], %o4 1168 set ktsb_base, %o5 1169 set ktsb4m_base, %l1 1170 brz,pt %o4, 1f 1171 nop 1172 set ktsb_pbase, %o5 1173 set ktsb4m_pbase, %l1 11741: 1175 sethi %hi(ktsb_szcode), %o1 1176 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1177 1178 sethi %hi(iktsb), %o0 1179 call sfmmu_fix_ktlb_traptable 1180 or %o0, %lo(iktsb), %o0 1181 1182 sethi %hi(dktsb), %o0 1183 call sfmmu_fix_ktlb_traptable 1184 or %o0, %lo(dktsb), %o0 1185 1186 sethi %hi(ktsb4m_szcode), %o1 1187 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1188 1189 sethi %hi(iktsb4m), %o0 1190 call sfmmu_fix_ktlb_traptable 1191 or %o0, %lo(iktsb4m), %o0 1192 1193 sethi %hi(dktsb4m), %o0 1194 call sfmmu_fix_ktlb_traptable 1195 or %o0, %lo(dktsb4m), %o0 1196 1197#ifndef sun4v 1198 mov ASI_N, %o2 1199 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1200 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1201 sethi %hi(tsb_kernel_patch_asi), %o0 1202 call sfmmu_fixup_or 1203 or %o0, %lo(tsb_kernel_patch_asi), %o0 1204#endif /* !sun4v */ 1205 1206 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1207 1208 sethi %hi(dktsbbase), %o0 1209 call sfmmu_fixup_setx ! patch value of ktsb base addr 1210 or %o0, %lo(dktsbbase), %o0 1211 1212 sethi %hi(iktsbbase), %o0 1213 call sfmmu_fixup_setx ! patch value of ktsb base addr 1214 or %o0, %lo(iktsbbase), %o0 1215 1216 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1217 call sfmmu_fixup_setx ! patch value of ktsb base addr 1218 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1219 1220#ifdef sun4v 1221 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1222 call sfmmu_fixup_setx ! patch value of ktsb base addr 1223 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1224#endif /* sun4v */ 1225 1226 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1227 1228 sethi %hi(dktsb4mbase), %o0 1229 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1230 or %o0, %lo(dktsb4mbase), %o0 1231 1232 sethi %hi(iktsb4mbase), %o0 1233 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1234 or %o0, %lo(iktsb4mbase), %o0 1235 1236 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1237 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1238 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1239 1240#ifdef sun4v 1241 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1242 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1243 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1244#endif /* sun4v */ 1245 1246 set ktsb_szcode, %o4 1247 ld [%o4], %o4 1248 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1249 call sfmmu_fixup_or ! patch value of ktsb_szcode 1250 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1251 1252#ifdef sun4v 1253 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1254 call sfmmu_fixup_or ! patch value of ktsb_szcode 1255 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1256#endif /* sun4v */ 1257 1258 set ktsb4m_szcode, %o4 1259 ld [%o4], %o4 1260 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1261 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1262 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1263 1264#ifdef sun4v 1265 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1266 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1267 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1268#endif /* sun4v */ 1269 1270 ret 1271 restore 1272 SET_SIZE(sfmmu_patch_ktsb) 1273 1274 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1275 /* 1276 * Fixup trap handlers in common segkpm case. This is reserved 1277 * for future use should kpm TSB be changed to be other than the 1278 * kernel TSB. 1279 */ 1280 retl 1281 nop 1282 SET_SIZE(sfmmu_kpm_patch_tlbm) 1283 1284 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1285 /* 1286 * nop the branch to sfmmu_kpm_dtsb_miss_small 1287 * in the case where we are using large pages for 1288 * seg_kpm (and hence must probe the second TSB for 1289 * seg_kpm VAs) 1290 */ 1291 set dktsb4m_kpmcheck_small, %o0 1292 MAKE_NOP_INSTR(%o1) 1293 st %o1, [%o0] 1294 flush %o0 1295 retl 1296 nop 1297 SET_SIZE(sfmmu_kpm_patch_tsbm) 1298 1299 ENTRY_NP(sfmmu_patch_utsb) 1300#ifdef UTSB_PHYS 1301 retl 1302 nop 1303#else /* UTSB_PHYS */ 1304 /* 1305 * We need to hot patch utsb_vabase and utsb4m_vabase 1306 */ 1307 save %sp, -SA(MINFRAME), %sp 1308 1309 /* patch value of utsb_vabase */ 1310 set utsb_vabase, %o1 1311 ldx [%o1], %o4 1312 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1313 call sfmmu_fixup_setx 1314 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1315 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1316 call sfmmu_fixup_setx 1317 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1318 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1319 call sfmmu_fixup_setx 1320 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1321 1322 /* patch value of utsb4m_vabase */ 1323 set utsb4m_vabase, %o1 1324 ldx [%o1], %o4 1325 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1326 call sfmmu_fixup_setx 1327 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1328 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1329 call sfmmu_fixup_setx 1330 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1331 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1332 call sfmmu_fixup_setx 1333 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1334 1335 /* 1336 * Patch TSB base register masks and shifts if needed. 1337 * By default the TSB base register contents are set up for 4M slab. 1338 * If we're using a smaller slab size and reserved VA range we need 1339 * to patch up those values here. 1340 */ 1341 set tsb_slab_shift, %o1 1342 set MMU_PAGESHIFT4M, %o4 1343 lduw [%o1], %o3 1344 subcc %o4, %o3, %o4 1345 bz,pt %icc, 1f 1346 /* delay slot safe */ 1347 1348 /* patch reserved VA range size if needed. */ 1349 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1350 call sfmmu_fixup_shiftx 1351 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1352 call sfmmu_fixup_shiftx 1353 add %o0, I_SIZE, %o0 1354 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1355 call sfmmu_fixup_shiftx 1356 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1357 call sfmmu_fixup_shiftx 1358 add %o0, I_SIZE, %o0 13591: 1360 /* patch TSBREG_VAMASK used to set up TSB base register */ 1361 set tsb_slab_mask, %o1 1362 ldx [%o1], %o4 1363 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1364 call sfmmu_fixup_or 1365 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1366 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1367 call sfmmu_fixup_or 1368 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1369 1370 ret 1371 restore 1372#endif /* UTSB_PHYS */ 1373 SET_SIZE(sfmmu_patch_utsb) 1374 1375 ENTRY_NP(sfmmu_patch_shctx) 1376#ifdef sun4u 1377 retl 1378 nop 1379#else /* sun4u */ 1380 set sfmmu_shctx_cpu_mondo_patch, %o0 1381 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1382 st %o1, [%o0] 1383 flush %o0 1384 MAKE_NOP_INSTR(%o1) 1385 add %o0, I_SIZE, %o0 ! next instr 1386 st %o1, [%o0] 1387 flush %o0 1388 1389 set sfmmu_shctx_user_rtt_patch, %o0 1390 st %o1, [%o0] ! nop 1st instruction 1391 flush %o0 1392 add %o0, I_SIZE, %o0 1393 st %o1, [%o0] ! nop 2nd instruction 1394 flush %o0 1395 add %o0, I_SIZE, %o0 1396 st %o1, [%o0] ! nop 3rd instruction 1397 flush %o0 1398 add %o0, I_SIZE, %o0 1399 st %o1, [%o0] ! nop 4th instruction 1400 retl 1401 flush %o0 1402#endif /* sun4u */ 1403 SET_SIZE(sfmmu_patch_shctx) 1404 1405 /* 1406 * Routine that loads an entry into a tsb using virtual addresses. 1407 * Locking is required since all cpus can use the same TSB. 1408 * Note that it is no longer required to have a valid context 1409 * when calling this function. 1410 */ 1411 ENTRY_NP(sfmmu_load_tsbe) 1412 /* 1413 * %o0 = pointer to tsbe to load 1414 * %o1 = tsb tag 1415 * %o2 = virtual pointer to TTE 1416 * %o3 = 1 if physical address in %o0 else 0 1417 */ 1418 rdpr %pstate, %o5 1419#ifdef DEBUG 1420 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1421#endif /* DEBUG */ 1422 1423 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1424 1425 SETUP_TSB_ASI(%o3, %g3) 1426 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, locked_tsb_l8) 1427 1428 wrpr %g0, %o5, %pstate /* enable interrupts */ 1429 1430 retl 1431 membar #StoreStore|#StoreLoad 1432 SET_SIZE(sfmmu_load_tsbe) 1433 1434 /* 1435 * Flush TSB of a given entry if the tag matches. 1436 */ 1437 ENTRY(sfmmu_unload_tsbe) 1438 /* 1439 * %o0 = pointer to tsbe to be flushed 1440 * %o1 = tag to match 1441 * %o2 = 1 if physical address in %o0 else 0 1442 */ 1443 SETUP_TSB_ASI(%o2, %g1) 1444 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1445 retl 1446 membar #StoreStore|#StoreLoad 1447 SET_SIZE(sfmmu_unload_tsbe) 1448 1449 /* 1450 * Routine that loads a TTE into the kpm TSB from C code. 1451 * Locking is required since kpm TSB is shared among all CPUs. 1452 */ 1453 ENTRY_NP(sfmmu_kpm_load_tsb) 1454 /* 1455 * %o0 = vaddr 1456 * %o1 = ttep 1457 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1458 */ 1459 rdpr %pstate, %o5 ! %o5 = saved pstate 1460#ifdef DEBUG 1461 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1462#endif /* DEBUG */ 1463 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1464 1465#ifndef sun4v 1466 sethi %hi(ktsb_phys), %o4 1467 mov ASI_N, %o3 1468 ld [%o4 + %lo(ktsb_phys)], %o4 1469 movrnz %o4, ASI_MEM, %o3 1470 mov %o3, %asi 1471#endif /* !sun4v */ 1472 mov %o0, %g1 ! %g1 = vaddr 1473 1474 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1475 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1476 /* %g2 = tsbep, %g1 clobbered */ 1477 1478 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1479 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1480 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, locked_tsb_l9) 1481 1482 wrpr %g0, %o5, %pstate ! enable interrupts 1483 retl 1484 membar #StoreStore|#StoreLoad 1485 SET_SIZE(sfmmu_kpm_load_tsb) 1486 1487 /* 1488 * Routine that shoots down a TTE in the kpm TSB or in the 1489 * kernel TSB depending on virtpg. Locking is required since 1490 * kpm/kernel TSB is shared among all CPUs. 1491 */ 1492 ENTRY_NP(sfmmu_kpm_unload_tsb) 1493 /* 1494 * %o0 = vaddr 1495 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1496 */ 1497#ifndef sun4v 1498 sethi %hi(ktsb_phys), %o4 1499 mov ASI_N, %o3 1500 ld [%o4 + %lo(ktsb_phys)], %o4 1501 movrnz %o4, ASI_MEM, %o3 1502 mov %o3, %asi 1503#endif /* !sun4v */ 1504 mov %o0, %g1 ! %g1 = vaddr 1505 1506 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1507 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1508 /* %g2 = tsbep, %g1 clobbered */ 1509 1510 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1511 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1512 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1513 1514 retl 1515 membar #StoreStore|#StoreLoad 1516 SET_SIZE(sfmmu_kpm_unload_tsb) 1517 1518#endif /* lint */ 1519 1520 1521#if defined (lint) 1522 1523/*ARGSUSED*/ 1524pfn_t 1525sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1526{ return(0); } 1527 1528#else /* lint */ 1529 1530 ENTRY_NP(sfmmu_ttetopfn) 1531 ldx [%o0], %g1 /* read tte */ 1532 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1533 /* 1534 * g1 = pfn 1535 */ 1536 retl 1537 mov %g1, %o0 1538 SET_SIZE(sfmmu_ttetopfn) 1539 1540#endif /* !lint */ 1541 1542/* 1543 * These macros are used to update global sfmmu hme hash statistics 1544 * in perf critical paths. It is only enabled in debug kernels or 1545 * if SFMMU_STAT_GATHER is defined 1546 */ 1547#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1548#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1549 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1550 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1551 cmp tmp1, hatid ;\ 1552 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1553 set sfmmu_global_stat, tmp1 ;\ 1554 add tmp1, tmp2, tmp1 ;\ 1555 ld [tmp1], tmp2 ;\ 1556 inc tmp2 ;\ 1557 st tmp2, [tmp1] 1558 1559#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1560 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1561 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1562 cmp tmp1, hatid ;\ 1563 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1564 set sfmmu_global_stat, tmp1 ;\ 1565 add tmp1, tmp2, tmp1 ;\ 1566 ld [tmp1], tmp2 ;\ 1567 inc tmp2 ;\ 1568 st tmp2, [tmp1] 1569 1570 1571#else /* DEBUG || SFMMU_STAT_GATHER */ 1572 1573#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1574 1575#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1576 1577#endif /* DEBUG || SFMMU_STAT_GATHER */ 1578 1579/* 1580 * This macro is used to update global sfmmu kstas in non 1581 * perf critical areas so they are enabled all the time 1582 */ 1583#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1584 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1585 add tmp1, statname, tmp1 ;\ 1586 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1587 inc tmp2 ;\ 1588 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1589 1590/* 1591 * These macros are used to update per cpu stats in non perf 1592 * critical areas so they are enabled all the time 1593 */ 1594#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1595 ld [tsbarea + stat], tmp1 ;\ 1596 inc tmp1 ;\ 1597 st tmp1, [tsbarea + stat] 1598 1599/* 1600 * These macros are used to update per cpu stats in non perf 1601 * critical areas so they are enabled all the time 1602 */ 1603#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1604 lduh [tsbarea + stat], tmp1 ;\ 1605 inc tmp1 ;\ 1606 stuh tmp1, [tsbarea + stat] 1607 1608#if defined(KPM_TLBMISS_STATS_GATHER) 1609 /* 1610 * Count kpm dtlb misses separately to allow a different 1611 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1612 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1613 */ 1614#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1615 brgez tagacc, label /* KPM VA? */ ;\ 1616 nop ;\ 1617 CPU_INDEX(tmp1, tsbma) ;\ 1618 sethi %hi(kpmtsbm_area), tsbma ;\ 1619 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1620 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1621 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1622 /* VA range check */ ;\ 1623 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1624 cmp tagacc, val ;\ 1625 blu,pn %xcc, label ;\ 1626 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1627 cmp tagacc, tmp1 ;\ 1628 bgeu,pn %xcc, label ;\ 1629 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1630 inc val ;\ 1631 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1632label: 1633#else 1634#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1635#endif /* KPM_TLBMISS_STATS_GATHER */ 1636 1637#if defined (lint) 1638/* 1639 * The following routines are jumped to from the mmu trap handlers to do 1640 * the setting up to call systrap. They are separate routines instead of 1641 * being part of the handlers because the handlers would exceed 32 1642 * instructions and since this is part of the slow path the jump 1643 * cost is irrelevant. 1644 */ 1645void 1646sfmmu_pagefault(void) 1647{ 1648} 1649 1650void 1651sfmmu_mmu_trap(void) 1652{ 1653} 1654 1655void 1656sfmmu_window_trap(void) 1657{ 1658} 1659 1660void 1661sfmmu_kpm_exception(void) 1662{ 1663} 1664 1665#else /* lint */ 1666 1667#ifdef PTL1_PANIC_DEBUG 1668 .seg ".data" 1669 .global test_ptl1_panic 1670test_ptl1_panic: 1671 .word 0 1672 .align 8 1673 1674 .seg ".text" 1675 .align 4 1676#endif /* PTL1_PANIC_DEBUG */ 1677 1678 1679 ENTRY_NP(sfmmu_pagefault) 1680 SET_GL_REG(1) 1681 USE_ALTERNATE_GLOBALS(%g5) 1682 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1683 rdpr %tt, %g6 1684 cmp %g6, FAST_IMMU_MISS_TT 1685 be,a,pn %icc, 1f 1686 mov T_INSTR_MMU_MISS, %g3 1687 cmp %g6, T_INSTR_MMU_MISS 1688 be,a,pn %icc, 1f 1689 mov T_INSTR_MMU_MISS, %g3 1690 mov %g5, %g2 1691 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1692 cmp %g6, FAST_DMMU_MISS_TT 1693 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1694 cmp %g6, T_DATA_MMU_MISS 1695 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1696 1697#ifdef PTL1_PANIC_DEBUG 1698 /* check if we want to test the tl1 panic */ 1699 sethi %hi(test_ptl1_panic), %g4 1700 ld [%g4 + %lo(test_ptl1_panic)], %g1 1701 st %g0, [%g4 + %lo(test_ptl1_panic)] 1702 cmp %g1, %g0 1703 bne,a,pn %icc, ptl1_panic 1704 or %g0, PTL1_BAD_DEBUG, %g1 1705#endif /* PTL1_PANIC_DEBUG */ 17061: 1707 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 1708 /* 1709 * g2 = tag access reg 1710 * g3.l = type 1711 * g3.h = 0 1712 */ 1713 sethi %hi(trap), %g1 1714 or %g1, %lo(trap), %g1 17152: 1716 ba,pt %xcc, sys_trap 1717 mov -1, %g4 1718 SET_SIZE(sfmmu_pagefault) 1719 1720 ENTRY_NP(sfmmu_mmu_trap) 1721 SET_GL_REG(1) 1722 USE_ALTERNATE_GLOBALS(%g5) 1723 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 1724 rdpr %tt, %g6 1725 cmp %g6, FAST_IMMU_MISS_TT 1726 be,a,pn %icc, 1f 1727 mov T_INSTR_MMU_MISS, %g3 1728 cmp %g6, T_INSTR_MMU_MISS 1729 be,a,pn %icc, 1f 1730 mov T_INSTR_MMU_MISS, %g3 1731 mov %g5, %g2 1732 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1733 cmp %g6, FAST_DMMU_MISS_TT 1734 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1735 cmp %g6, T_DATA_MMU_MISS 1736 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 17371: 1738 /* 1739 * g2 = tag access reg 1740 * g3 = type 1741 */ 1742 sethi %hi(sfmmu_tsbmiss_exception), %g1 1743 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 1744 ba,pt %xcc, sys_trap 1745 mov -1, %g4 1746 /*NOTREACHED*/ 1747 SET_SIZE(sfmmu_mmu_trap) 1748 1749 ENTRY_NP(sfmmu_suspend_tl) 1750 SET_GL_REG(1) 1751 USE_ALTERNATE_GLOBALS(%g5) 1752 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 1753 rdpr %tt, %g6 1754 cmp %g6, FAST_IMMU_MISS_TT 1755 be,a,pn %icc, 1f 1756 mov T_INSTR_MMU_MISS, %g3 1757 mov %g5, %g2 1758 cmp %g6, FAST_DMMU_MISS_TT 1759 move %icc, T_DATA_MMU_MISS, %g3 1760 movne %icc, T_DATA_PROT, %g3 17611: 1762 sethi %hi(sfmmu_tsbmiss_suspended), %g1 1763 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 1764 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 1765 ba,pt %xcc, sys_trap 1766 mov PIL_15, %g4 1767 /*NOTREACHED*/ 1768 SET_SIZE(sfmmu_suspend_tl) 1769 1770 /* 1771 * No %g registers in use at this point. 1772 */ 1773 ENTRY_NP(sfmmu_window_trap) 1774 rdpr %tpc, %g1 1775#ifdef sun4v 1776#ifdef DEBUG 1777 /* We assume previous %gl was 1 */ 1778 rdpr %tstate, %g4 1779 srlx %g4, TSTATE_GL_SHIFT, %g4 1780 and %g4, TSTATE_GL_MASK, %g4 1781 cmp %g4, 1 1782 bne,a,pn %icc, ptl1_panic 1783 mov PTL1_BAD_WTRAP, %g1 1784#endif /* DEBUG */ 1785 /* user miss at tl>1. better be the window handler or user_rtt */ 1786 /* in user_rtt? */ 1787 set rtt_fill_start, %g4 1788 cmp %g1, %g4 1789 blu,pn %xcc, 6f 1790 .empty 1791 set rtt_fill_end, %g4 1792 cmp %g1, %g4 1793 bgeu,pn %xcc, 6f 1794 nop 1795 set fault_rtt_fn1, %g1 1796 wrpr %g0, %g1, %tnpc 1797 ba,a 7f 17986: 1799 ! must save this trap level before descending trap stack 1800 ! no need to save %tnpc, either overwritten or discarded 1801 ! already got it: rdpr %tpc, %g1 1802 rdpr %tstate, %g6 1803 rdpr %tt, %g7 1804 ! trap level saved, go get underlying trap type 1805 rdpr %tl, %g5 1806 sub %g5, 1, %g3 1807 wrpr %g3, %tl 1808 rdpr %tt, %g2 1809 wrpr %g5, %tl 1810 ! restore saved trap level 1811 wrpr %g1, %tpc 1812 wrpr %g6, %tstate 1813 wrpr %g7, %tt 1814#else /* sun4v */ 1815 /* user miss at tl>1. better be the window handler */ 1816 rdpr %tl, %g5 1817 sub %g5, 1, %g3 1818 wrpr %g3, %tl 1819 rdpr %tt, %g2 1820 wrpr %g5, %tl 1821#endif /* sun4v */ 1822 and %g2, WTRAP_TTMASK, %g4 1823 cmp %g4, WTRAP_TYPE 1824 bne,pn %xcc, 1f 1825 nop 1826 /* tpc should be in the trap table */ 1827 set trap_table, %g4 1828 cmp %g1, %g4 1829 blt,pn %xcc, 1f 1830 .empty 1831 set etrap_table, %g4 1832 cmp %g1, %g4 1833 bge,pn %xcc, 1f 1834 .empty 1835 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 1836 add %g1, WTRAP_FAULTOFF, %g1 1837 wrpr %g0, %g1, %tnpc 18387: 1839 /* 1840 * some wbuf handlers will call systrap to resolve the fault 1841 * we pass the trap type so they figure out the correct parameters. 1842 * g5 = trap type, g6 = tag access reg 1843 */ 1844 1845 /* 1846 * only use g5, g6, g7 registers after we have switched to alternate 1847 * globals. 1848 */ 1849 SET_GL_REG(1) 1850 USE_ALTERNATE_GLOBALS(%g5) 1851 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 1852 rdpr %tt, %g7 1853 cmp %g7, FAST_IMMU_MISS_TT 1854 be,a,pn %icc, ptl1_panic 1855 mov PTL1_BAD_WTRAP, %g1 1856 cmp %g7, T_INSTR_MMU_MISS 1857 be,a,pn %icc, ptl1_panic 1858 mov PTL1_BAD_WTRAP, %g1 1859 mov T_DATA_PROT, %g5 1860 cmp %g7, FAST_DMMU_MISS_TT 1861 move %icc, T_DATA_MMU_MISS, %g5 1862 cmp %g7, T_DATA_MMU_MISS 1863 move %icc, T_DATA_MMU_MISS, %g5 1864 ! XXXQ AGS re-check out this one 1865 done 18661: 1867 CPU_PADDR(%g1, %g4) 1868 add %g1, CPU_TL1_HDLR, %g1 1869 lda [%g1]ASI_MEM, %g4 1870 brnz,a,pt %g4, sfmmu_mmu_trap 1871 sta %g0, [%g1]ASI_MEM 1872 ba,pt %icc, ptl1_panic 1873 mov PTL1_BAD_TRAP, %g1 1874 SET_SIZE(sfmmu_window_trap) 1875 1876 ENTRY_NP(sfmmu_kpm_exception) 1877 /* 1878 * We have accessed an unmapped segkpm address or a legal segkpm 1879 * address which is involved in a VAC alias conflict prevention. 1880 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 1881 * set. If it is, we will instead note that a fault has occurred 1882 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 1883 * a "retry"). This will step over the faulting instruction. 1884 * Note that this means that a legal segkpm address involved in 1885 * a VAC alias conflict prevention (a rare case to begin with) 1886 * cannot be used in DTrace. 1887 */ 1888 CPU_INDEX(%g1, %g2) 1889 set cpu_core, %g2 1890 sllx %g1, CPU_CORE_SHIFT, %g1 1891 add %g1, %g2, %g1 1892 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 1893 andcc %g2, CPU_DTRACE_NOFAULT, %g0 1894 bz 0f 1895 or %g2, CPU_DTRACE_BADADDR, %g2 1896 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 1897 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 1898 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 1899 done 19000: 1901 TSTAT_CHECK_TL1(1f, %g1, %g2) 19021: 1903 SET_GL_REG(1) 1904 USE_ALTERNATE_GLOBALS(%g5) 1905 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 1906 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1907 /* 1908 * g2=tagacc g3.l=type g3.h=0 1909 */ 1910 sethi %hi(trap), %g1 1911 or %g1, %lo(trap), %g1 1912 ba,pt %xcc, sys_trap 1913 mov -1, %g4 1914 SET_SIZE(sfmmu_kpm_exception) 1915 1916#endif /* lint */ 1917 1918#if defined (lint) 1919 1920void 1921sfmmu_tsb_miss(void) 1922{ 1923} 1924 1925void 1926sfmmu_kpm_dtsb_miss(void) 1927{ 1928} 1929 1930void 1931sfmmu_kpm_dtsb_miss_small(void) 1932{ 1933} 1934 1935#else /* lint */ 1936 1937#if (IMAP_SEG != 0) 1938#error - ism_map->ism_seg offset is not zero 1939#endif 1940 1941/* 1942 * Copies ism mapping for this ctx in param "ism" if this is a ISM 1943 * tlb miss and branches to label "ismhit". If this is not an ISM 1944 * process or an ISM tlb miss it falls thru. 1945 * 1946 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 1947 * this process. 1948 * If so, it will branch to label "ismhit". If not, it will fall through. 1949 * 1950 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 1951 * so that any other threads of this process will not try and walk the ism 1952 * maps while they are being changed. 1953 * 1954 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 1955 * will make sure of that. This means we can terminate our search on 1956 * the first zero mapping we find. 1957 * 1958 * Parameters: 1959 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 1960 * tsbmiss = address of tsb miss area (in) 1961 * ismseg = contents of ism_seg for this ism map (out) 1962 * ismhat = physical address of imap_ismhat for this ism map (out) 1963 * tmp1 = scratch reg (CLOBBERED) 1964 * tmp2 = scratch reg (CLOBBERED) 1965 * tmp3 = scratch reg (CLOBBERED) 1966 * label: temporary labels 1967 * ismhit: label where to jump to if an ism dtlb miss 1968 * exitlabel:label where to jump if hat is busy due to hat_unshare. 1969 */ 1970#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 1971 label, ismhit) \ 1972 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 1973 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 1974 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 1975label/**/1: ;\ 1976 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 1977 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 1978label/**/2: ;\ 1979 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 1980 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 1981 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 1982 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 1983 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 1984 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 1985 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 1986 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 1987 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 1988 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 1989 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 1990 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 1991 ;\ 1992 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 1993 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 1994 cmp ismhat, tmp1 ;\ 1995 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 1996 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 1997 ;\ 1998 add tmp3, IBLK_NEXTPA, tmp1 ;\ 1999 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 2000 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 2001 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 2002label/**/3: 2003 2004/* 2005 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 2006 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 2007 * Parameters: 2008 * tagacc = reg containing virtual address 2009 * hatid = reg containing sfmmu pointer 2010 * hmeshift = constant/register to shift vaddr to obtain vapg 2011 * hmebp = register where bucket pointer will be stored 2012 * vapg = register where virtual page will be stored 2013 * tmp1, tmp2 = tmp registers 2014 */ 2015 2016 2017#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 2018 vapg, label, tmp1, tmp2) \ 2019 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 2020 brnz,a,pt tmp1, label/**/1 ;\ 2021 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 2022 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 2023 ba,pt %xcc, label/**/2 ;\ 2024 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 2025label/**/1: ;\ 2026 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 2027label/**/2: ;\ 2028 srlx tagacc, hmeshift, vapg ;\ 2029 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 2030 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 2031 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 2032 add hmebp, tmp1, hmebp 2033 2034/* 2035 * hashtag includes bspage + hashno (64 bits). 2036 */ 2037 2038#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 2039 sllx vapg, hmeshift, vapg ;\ 2040 mov hashno, hblktag ;\ 2041 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 2042 or vapg, hblktag, hblktag 2043 2044/* 2045 * Function to traverse hmeblk hash link list and find corresponding match. 2046 * The search is done using physical pointers. It returns the physical address 2047 * pointer to the hmeblk that matches with the tag provided. 2048 * Parameters: 2049 * hmebp = register that points to hme hash bucket, also used as 2050 * tmp reg (clobbered) 2051 * hmeblktag = register with hmeblk tag match 2052 * hatid = register with hatid 2053 * hmeblkpa = register where physical ptr will be stored 2054 * tmp1 = tmp reg 2055 * label: temporary label 2056 */ 2057 2058#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, tsbarea, \ 2059 tmp1, label) \ 2060 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 2061 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2062 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2063label/**/1: ;\ 2064 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2065 be,pn %xcc, label/**/2 ;\ 2066 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2067 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 2068 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2069 add hmebp, CLONGSIZE, hmebp ;\ 2070 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 2071 xor tmp1, hmeblktag, tmp1 ;\ 2072 xor hmebp, hatid, hmebp ;\ 2073 or hmebp, tmp1, hmebp ;\ 2074 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 2075 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 2076 ba,pt %xcc, label/**/1 ;\ 2077 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2078label/**/2: 2079 2080/* 2081 * Function to traverse hmeblk hash link list and find corresponding match. 2082 * The search is done using physical pointers. It returns the physical address 2083 * pointer to the hmeblk that matches with the tag 2084 * provided. 2085 * Parameters: 2086 * hmeblktag = register with hmeblk tag match (rid field is 0) 2087 * hatid = register with hatid (pointer to SRD) 2088 * hmeblkpa = register where physical ptr will be stored 2089 * tmp1 = tmp reg 2090 * tmp2 = tmp reg 2091 * label: temporary label 2092 */ 2093 2094#define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, tsbarea, \ 2095 tmp1, tmp2, label) \ 2096label/**/1: ;\ 2097 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2098 be,pn %xcc, label/**/4 ;\ 2099 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 2100 add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 2101 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2102 add tmp2, CLONGSIZE, tmp2 ;\ 2103 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 2104 xor tmp1, hmeblktag, tmp1 ;\ 2105 xor tmp2, hatid, tmp2 ;\ 2106 brz,pn tmp2, label/**/3 /* branch on hit */ ;\ 2107 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2108label/**/2: ;\ 2109 ba,pt %xcc, label/**/1 ;\ 2110 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2111label/**/3: ;\ 2112 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 2113 bgeu,pt %xcc, label/**/2 ;\ 2114 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2115 and tmp1, BT_ULMASK, tmp2 ;\ 2116 srlx tmp1, BT_ULSHIFT, tmp1 ;\ 2117 sllx tmp1, CLONGSHIFT, tmp1 ;\ 2118 add tsbarea, tmp1, tmp1 ;\ 2119 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 2120 srlx tmp1, tmp2, tmp1 ;\ 2121 btst 0x1, tmp1 ;\ 2122 bz,pn %xcc, label/**/2 ;\ 2123 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2124label/**/4: 2125 2126#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2127#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2128#endif 2129 2130/* 2131 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2132 * he offset for the corresponding hment. 2133 * Parameters: 2134 * In: 2135 * vaddr = register with virtual address 2136 * hmeblkpa = physical pointer to hme_blk 2137 * Out: 2138 * hmentoff = register where hment offset will be stored 2139 * hmemisc = hblk_misc 2140 * Scratch: 2141 * tmp1 2142 */ 2143#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2144 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2145 lda [hmentoff]ASI_MEM, hmemisc ;\ 2146 andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2147 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2148 or %g0, HMEBLK_HME1, hmentoff ;\ 2149 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2150 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2151 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2152 add tmp1, HMEBLK_HME1, hmentoff ;\ 2153label1: 2154 2155/* 2156 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2157 * 2158 * tagacc = (pseudo-)tag access register (in) 2159 * hatid = sfmmu pointer for TSB miss (in) 2160 * tte = tte for TLB miss if found, otherwise clobbered (out) 2161 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2162 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2163 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2164 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2165 * for this page size. 2166 * hashno = constant/register hash number 2167 * tmp = temp value - clobbered 2168 * label = temporary label for branching within macro. 2169 * foundlabel = label to jump to when tte is found. 2170 * suspendlabel= label to jump to when tte is suspended. 2171 * exitlabel = label to jump to when tte is not found. 2172 * 2173 */ 2174#define GET_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, hmeshift, \ 2175 hashno, tmp, label, foundlabel, suspendlabel, exitlabel) \ 2176 ;\ 2177 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2178 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2179 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2180 hmeblkpa, label/**/5, hmemisc, tmp) ;\ 2181 ;\ 2182 /* ;\ 2183 * tagacc = tagacc ;\ 2184 * hatid = hatid ;\ 2185 * tsbarea = tsbarea ;\ 2186 * tte = hmebp (hme bucket pointer) ;\ 2187 * hmeblkpa = vapg (virtual page) ;\ 2188 * hmemisc, tmp = scratch ;\ 2189 */ ;\ 2190 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2191 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2192 ;\ 2193 /* ;\ 2194 * tagacc = tagacc ;\ 2195 * hatid = hatid ;\ 2196 * tte = hmebp ;\ 2197 * hmeblkpa = CLOBBERED ;\ 2198 * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2199 * tmp = scratch ;\ 2200 */ ;\ 2201 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2202 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, \ 2203 tsbarea, tagacc, label/**/1) ;\ 2204 /* ;\ 2205 * tagacc = CLOBBERED ;\ 2206 * tte = CLOBBERED ;\ 2207 * hmeblkpa = hmeblkpa ;\ 2208 * tmp = scratch ;\ 2209 */ ;\ 2210 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2211 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\ 2212 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2213 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2214 nop ;\ 2215label/**/4: ;\ 2216 /* ;\ 2217 * We have found the hmeblk containing the hment. ;\ 2218 * Now we calculate the corresponding tte. ;\ 2219 * ;\ 2220 * tagacc = tagacc ;\ 2221 * hatid = hatid ;\ 2222 * tte = clobbered ;\ 2223 * hmeblkpa = hmeblkpa ;\ 2224 * hmemisc = hblktag ;\ 2225 * tmp = scratch ;\ 2226 */ ;\ 2227 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2228 label/**/2) ;\ 2229 ;\ 2230 /* ;\ 2231 * tagacc = tagacc ;\ 2232 * hatid = hmentoff ;\ 2233 * tte = clobbered ;\ 2234 * hmeblkpa = hmeblkpa ;\ 2235 * hmemisc = hblk_misc ;\ 2236 * tmp = scratch ;\ 2237 */ ;\ 2238 ;\ 2239 add hatid, SFHME_TTE, hatid ;\ 2240 add hmeblkpa, hatid, hmeblkpa ;\ 2241 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2242 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2243 set TTE_SUSPEND, hatid ;\ 2244 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2245 btst tte, hatid ;\ 2246 bz,pt %xcc, foundlabel ;\ 2247 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2248 ;\ 2249 /* ;\ 2250 * Mapping is suspended, so goto suspend label. ;\ 2251 */ ;\ 2252 ba,pt %xcc, suspendlabel ;\ 2253 nop 2254 2255/* 2256 * GET_SHME_TTE is similar to GET_TTE() except it searches 2257 * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2258 * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2259 * either 0 (not part of scd) or 1 (part of scd). 2260 */ 2261#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, \ 2262 hmeshift, hashno, tmp, label, foundlabel, \ 2263 suspendlabel, exitlabel) \ 2264 ;\ 2265 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2266 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2267 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2268 hmeblkpa, label/**/5, hmemisc, tmp) ;\ 2269 ;\ 2270 /* ;\ 2271 * tagacc = tagacc ;\ 2272 * hatid = hatid ;\ 2273 * tsbarea = tsbarea ;\ 2274 * tte = hmebp (hme bucket pointer) ;\ 2275 * hmeblkpa = vapg (virtual page) ;\ 2276 * hmemisc, tmp = scratch ;\ 2277 */ ;\ 2278 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2279 ;\ 2280 /* ;\ 2281 * tagacc = tagacc ;\ 2282 * hatid = hatid ;\ 2283 * tsbarea = tsbarea ;\ 2284 * tte = hmebp ;\ 2285 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2286 * hmeblkpa = CLOBBERED ;\ 2287 * tmp = scratch ;\ 2288 */ ;\ 2289 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2290 ;\ 2291 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2292 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2293 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2294 ;\ 2295label/**/8: ;\ 2296 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, \ 2297 tsbarea, tagacc, tte, label/**/1) ;\ 2298 /* ;\ 2299 * tagacc = CLOBBERED ;\ 2300 * tte = CLOBBERED ;\ 2301 * hmeblkpa = hmeblkpa ;\ 2302 * tmp = scratch ;\ 2303 */ ;\ 2304 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2305 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\ 2306 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2307 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2308 nop ;\ 2309label/**/4: ;\ 2310 /* ;\ 2311 * We have found the hmeblk containing the hment. ;\ 2312 * Now we calculate the corresponding tte. ;\ 2313 * ;\ 2314 * tagacc = tagacc ;\ 2315 * hatid = hatid ;\ 2316 * tte = clobbered ;\ 2317 * hmeblkpa = hmeblkpa ;\ 2318 * hmemisc = hblktag ;\ 2319 * tsbarea = tsbmiss area ;\ 2320 * tmp = scratch ;\ 2321 */ ;\ 2322 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2323 label/**/2) ;\ 2324 ;\ 2325 /* ;\ 2326 * tagacc = tagacc ;\ 2327 * hatid = hmentoff ;\ 2328 * tte = clobbered ;\ 2329 * hmeblkpa = hmeblkpa ;\ 2330 * hmemisc = hblk_misc ;\ 2331 * tsbarea = tsbmiss area ;\ 2332 * tmp = scratch ;\ 2333 */ ;\ 2334 ;\ 2335 add hatid, SFHME_TTE, hatid ;\ 2336 add hmeblkpa, hatid, hmeblkpa ;\ 2337 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2338 brlz,pt tte, label/**/6 ;\ 2339 nop ;\ 2340 btst HBLK_SZMASK, hmemisc ;\ 2341 bnz,a,pt %icc, label/**/7 ;\ 2342 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2343 ;\ 2344 /* ;\ 2345 * We found an invalid 8K tte in shme. ;\ 2346 * it may not belong to shme's region since ;\ 2347 * region size/alignment granularity is 8K but different ;\ 2348 * regions don't share hmeblks. Continue the search. ;\ 2349 */ ;\ 2350 sub hmeblkpa, hatid, hmeblkpa ;\ 2351 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2352 srlx tagacc, hmeshift, tte ;\ 2353 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2354 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2355 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2356 ba,a,pt %xcc, label/**/8 ;\ 2357label/**/6: ;\ 2358 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2359 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2360label/**/7: ;\ 2361 set TTE_SUSPEND, hatid ;\ 2362 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2363 btst tte, hatid ;\ 2364 bz,pt %xcc, foundlabel ;\ 2365 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2366 ;\ 2367 /* ;\ 2368 * Mapping is suspended, so goto suspend label. ;\ 2369 */ ;\ 2370 ba,pt %xcc, suspendlabel ;\ 2371 nop 2372 2373 /* 2374 * KERNEL PROTECTION HANDLER 2375 * 2376 * g1 = tsb8k pointer register (clobbered) 2377 * g2 = tag access register (ro) 2378 * g3 - g7 = scratch registers 2379 * 2380 * Note: This function is patched at runtime for performance reasons. 2381 * Any changes here require sfmmu_patch_ktsb fixed. 2382 */ 2383 ENTRY_NP(sfmmu_kprot_trap) 2384 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2385sfmmu_kprot_patch_ktsb_base: 2386 RUNTIME_PATCH_SETX(%g1, %g6) 2387 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2388sfmmu_kprot_patch_ktsb_szcode: 2389 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2390 2391 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2392 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2393 2394 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2395sfmmu_kprot_patch_ktsb4m_base: 2396 RUNTIME_PATCH_SETX(%g3, %g6) 2397 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2398sfmmu_kprot_patch_ktsb4m_szcode: 2399 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2400 2401 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2402 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2403 2404 CPU_TSBMISS_AREA(%g6, %g7) 2405 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2406 ba,pt %xcc, sfmmu_tsb_miss_tt 2407 nop 2408 2409 /* 2410 * USER PROTECTION HANDLER 2411 * 2412 * g1 = tsb8k pointer register (ro) 2413 * g2 = tag access register (ro) 2414 * g3 = faulting context (clobbered, currently not used) 2415 * g4 - g7 = scratch registers 2416 */ 2417 ALTENTRY(sfmmu_uprot_trap) 2418#ifdef sun4v 2419 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2420 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2421 2422 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2423 brlz,pt %g3, 9f /* check for 2nd TSB */ 2424 nop 2425 2426 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2427 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2428 2429#else /* sun4v */ 2430#ifdef UTSB_PHYS 2431 /* g1 = first TSB entry ptr */ 2432 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2433 brlz,pt %g3, 9f /* check for 2nd TSB */ 2434 nop 2435 2436 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2437 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2438#else /* UTSB_PHYS */ 2439 brgez,pt %g1, 9f /* check for 2nd TSB */ 2440 mov -1, %g3 /* set second tsbe ptr to -1 */ 2441 2442 mov %g2, %g7 2443 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2444 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2445 mov %g1, %g7 2446 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2447#endif /* UTSB_PHYS */ 2448#endif /* sun4v */ 24499: 2450 CPU_TSBMISS_AREA(%g6, %g7) 2451 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2452 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2453 nop 2454 2455 /* 2456 * Kernel 8K page iTLB miss. We also get here if we took a 2457 * fast instruction access mmu miss trap while running in 2458 * invalid context. 2459 * 2460 * %g1 = 8K TSB pointer register (not used, clobbered) 2461 * %g2 = tag access register (used) 2462 * %g3 = faulting context id (used) 2463 * %g7 = TSB tag to match (used) 2464 */ 2465 .align 64 2466 ALTENTRY(sfmmu_kitlb_miss) 2467 brnz,pn %g3, tsb_tl0_noctxt 2468 nop 2469 2470 /* kernel miss */ 2471 /* get kernel tsb pointer */ 2472 /* we patch the next set of instructions at run time */ 2473 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2474iktsbbase: 2475 RUNTIME_PATCH_SETX(%g4, %g5) 2476 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2477 2478iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2479 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2480 or %g4, %g1, %g1 ! form tsb ptr 2481 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2482 cmp %g4, %g7 2483 bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2484 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2485 2486 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2487 bz,pn %icc, exec_fault 2488 nop 2489 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2490 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2491 retry 2492 2493iktsb4mbase: 2494 RUNTIME_PATCH_SETX(%g4, %g6) 2495 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2496iktsb4m: 2497 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2498 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2499 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2500 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2501 cmp %g4, %g7 2502 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2503 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2504 bz,pn %icc, exec_fault 2505 nop 2506 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2507 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2508 retry 2509 2510 /* 2511 * Kernel dTLB miss. We also get here if we took a fast data 2512 * access mmu miss trap while running in invalid context. 2513 * 2514 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2515 * We select the TSB miss handler to branch to depending on 2516 * the virtual address of the access. In the future it may 2517 * be desirable to separate kpm TTEs into their own TSB, 2518 * in which case all that needs to be done is to set 2519 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2520 * early in the miss if we detect a kpm VA to a new handler. 2521 * 2522 * %g1 = 8K TSB pointer register (not used, clobbered) 2523 * %g2 = tag access register (used) 2524 * %g3 = faulting context id (used) 2525 */ 2526 .align 64 2527 ALTENTRY(sfmmu_kdtlb_miss) 2528 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2529 nop 2530 2531 /* Gather some stats for kpm misses in the TLB. */ 2532 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2533 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2534 2535 /* 2536 * Get first TSB offset and look for 8K/64K/512K mapping 2537 * using the 8K virtual page as the index. 2538 * 2539 * We patch the next set of instructions at run time; 2540 * any changes here require sfmmu_patch_ktsb changes too. 2541 */ 2542dktsbbase: 2543 RUNTIME_PATCH_SETX(%g7, %g6) 2544 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2545 2546dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2547 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2548 2549 /* 2550 * At this point %g1 is our index into the TSB. 2551 * We just masked off enough bits of the VA depending 2552 * on our TSB size code. 2553 */ 2554 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2555 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2556 cmp %g6, %g4 ! compare tag 2557 bne,pn %xcc, dktsb4m_kpmcheck_small 2558 add %g7, %g1, %g1 /* form tsb ptr */ 2559 TT_TRACE(trace_tsbhit) 2560 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2561 /* trapstat expects tte in %g5 */ 2562 retry 2563 2564 /* 2565 * If kpm is using large pages, the following instruction needs 2566 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2567 * so that we will probe the 4M TSB regardless of the VA. In 2568 * the case kpm is using small pages, we know no large kernel 2569 * mappings are located above 0x80000000.00000000 so we skip the 2570 * probe as an optimization. 2571 */ 2572dktsb4m_kpmcheck_small: 2573 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2574 /* delay slot safe, below */ 2575 2576 /* 2577 * Get second TSB offset and look for 4M mapping 2578 * using 4M virtual page as the TSB index. 2579 * 2580 * Here: 2581 * %g1 = 8K TSB pointer. Don't squash it. 2582 * %g2 = tag access register (we still need it) 2583 */ 2584 srlx %g2, MMU_PAGESHIFT4M, %g3 2585 2586 /* 2587 * We patch the next set of instructions at run time; 2588 * any changes here require sfmmu_patch_ktsb changes too. 2589 */ 2590dktsb4mbase: 2591 RUNTIME_PATCH_SETX(%g7, %g6) 2592 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2593dktsb4m: 2594 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2595 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2596 2597 /* 2598 * At this point %g3 is our index into the TSB. 2599 * We just masked off enough bits of the VA depending 2600 * on our TSB size code. 2601 */ 2602 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2603 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2604 cmp %g6, %g4 ! compare tag 2605 2606dktsb4m_tsbmiss: 2607 bne,pn %xcc, dktsb4m_kpmcheck 2608 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2609 TT_TRACE(trace_tsbhit) 2610 /* we don't check TTE size here since we assume 4M TSB is separate */ 2611 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2612 /* trapstat expects tte in %g5 */ 2613 retry 2614 2615 /* 2616 * So, we failed to find a valid TTE to match the faulting 2617 * address in either TSB. There are a few cases that could land 2618 * us here: 2619 * 2620 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2621 * to sfmmu_tsb_miss_tt to handle the miss. 2622 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2623 * 4M TSB. Let segkpm handle it. 2624 * 2625 * Note that we shouldn't land here in the case of a kpm VA when 2626 * kpm_smallpages is active -- we handled that case earlier at 2627 * dktsb4m_kpmcheck_small. 2628 * 2629 * At this point: 2630 * g1 = 8K-indexed primary TSB pointer 2631 * g2 = tag access register 2632 * g3 = 4M-indexed secondary TSB pointer 2633 */ 2634dktsb4m_kpmcheck: 2635 cmp %g2, %g0 2636 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2637 nop 2638 ba,a,pt %icc, sfmmu_tsb_miss_tt 2639 nop 2640 2641#ifdef sun4v 2642 /* 2643 * User instruction miss w/ single TSB. 2644 * The first probe covers 8K, 64K, and 512K page sizes, 2645 * because 64K and 512K mappings are replicated off 8K 2646 * pointer. 2647 * 2648 * g1 = tsb8k pointer register 2649 * g2 = tag access register 2650 * g3 - g6 = scratch registers 2651 * g7 = TSB tag to match 2652 */ 2653 .align 64 2654 ALTENTRY(sfmmu_uitlb_fastpath) 2655 2656 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2657 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2658 ba,pn %xcc, sfmmu_tsb_miss_tt 2659 mov -1, %g3 2660 2661 /* 2662 * User data miss w/ single TSB. 2663 * The first probe covers 8K, 64K, and 512K page sizes, 2664 * because 64K and 512K mappings are replicated off 8K 2665 * pointer. 2666 * 2667 * g1 = tsb8k pointer register 2668 * g2 = tag access register 2669 * g3 - g6 = scratch registers 2670 * g7 = TSB tag to match 2671 */ 2672 .align 64 2673 ALTENTRY(sfmmu_udtlb_fastpath) 2674 2675 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2676 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2677 ba,pn %xcc, sfmmu_tsb_miss_tt 2678 mov -1, %g3 2679 2680 /* 2681 * User instruction miss w/ multiple TSBs (sun4v). 2682 * The first probe covers 8K, 64K, and 512K page sizes, 2683 * because 64K and 512K mappings are replicated off 8K 2684 * pointer. Second probe covers 4M page size only. 2685 * 2686 * Just like sfmmu_udtlb_slowpath, except: 2687 * o Uses ASI_ITLB_IN 2688 * o checks for execute permission 2689 * o No ISM prediction. 2690 * 2691 * g1 = tsb8k pointer register 2692 * g2 = tag access register 2693 * g3 - g6 = scratch registers 2694 * g7 = TSB tag to match 2695 */ 2696 .align 64 2697 ALTENTRY(sfmmu_uitlb_slowpath) 2698 2699 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2700 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2701 /* g4 - g5 = clobbered here */ 2702 2703 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2704 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2705 srlx %g2, TAG_VALO_SHIFT, %g7 2706 PROBE_2ND_ITSB(%g3, %g7) 2707 /* NOT REACHED */ 2708 2709#else /* sun4v */ 2710 2711 /* 2712 * User instruction miss w/ multiple TSBs (sun4u). 2713 * The first probe covers 8K, 64K, and 512K page sizes, 2714 * because 64K and 512K mappings are replicated off 8K 2715 * pointer. Probe of 1st TSB has already been done prior to entry 2716 * into this routine. For the UTSB_PHYS case we probe up to 3 2717 * valid other TSBs in the following order: 2718 * 1) shared TSB for 4M-256M pages 2719 * 2) private TSB for 4M-256M pages 2720 * 3) shared TSB for 8K-512K pages 2721 * 2722 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs 2723 * 4M-256M pages. 2724 * 2725 * Just like sfmmu_udtlb_slowpath, except: 2726 * o Uses ASI_ITLB_IN 2727 * o checks for execute permission 2728 * o No ISM prediction. 2729 * 2730 * g1 = tsb8k pointer register 2731 * g2 = tag access register 2732 * g4 - g6 = scratch registers 2733 * g7 = TSB tag to match 2734 */ 2735 .align 64 2736 ALTENTRY(sfmmu_uitlb_slowpath) 2737 2738#ifdef UTSB_PHYS 2739 2740 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2741 brlz,pt %g6, 1f 2742 nop 2743 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2744 PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail) 27451: 2746 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2747 brlz,pt %g3, 2f 2748 nop 2749 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2750 PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail) 27512: 2752 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2753 brlz,pt %g6, sfmmu_tsb_miss_tt 2754 nop 2755 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2756 PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail) 2757 ba,pn %xcc, sfmmu_tsb_miss_tt 2758 nop 2759 2760#else /* UTSB_PHYS */ 2761 mov %g1, %g3 /* save tsb8k reg in %g3 */ 2762 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 2763 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2764 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 2765 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 2766 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 2767 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2768 srlx %g2, TAG_VALO_SHIFT, %g7 2769 PROBE_2ND_ITSB(%g3, %g7, isynth) 2770 ba,pn %xcc, sfmmu_tsb_miss_tt 2771 nop 2772 2773#endif /* UTSB_PHYS */ 2774#endif /* sun4v */ 2775 2776#if defined(sun4u) && defined(UTSB_PHYS) 2777 2778 /* 2779 * We come here for ism predict DTLB_MISS case or if 2780 * if probe in first TSB failed. 2781 */ 2782 2783 .align 64 2784 ALTENTRY(sfmmu_udtlb_slowpath_noismpred) 2785 2786 /* 2787 * g1 = tsb8k pointer register 2788 * g2 = tag access register 2789 * g4 - %g6 = scratch registers 2790 * g7 = TSB tag to match 2791 */ 2792 2793 /* 2794 * ISM non-predict probe order 2795 * probe 1ST_TSB (8K index) 2796 * probe 2ND_TSB (4M index) 2797 * probe 4TH_TSB (4M index) 2798 * probe 3RD_TSB (8K index) 2799 * 2800 * We already probed first TSB in DTLB_MISS handler. 2801 */ 2802 2803 /* 2804 * Private 2ND TSB 4M-256 pages 2805 */ 2806 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2807 brlz,pt %g3, 1f 2808 nop 2809 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2810 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2811 2812 /* 2813 * Shared Context 4TH TSB 4M-256 pages 2814 */ 28151: 2816 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2817 brlz,pt %g6, 2f 2818 nop 2819 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2820 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail) 2821 2822 /* 2823 * Shared Context 3RD TSB 8K-512K pages 2824 */ 28252: 2826 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2827 brlz,pt %g6, sfmmu_tsb_miss_tt 2828 nop 2829 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2830 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail) 2831 ba,pn %xcc, sfmmu_tsb_miss_tt 2832 nop 2833 2834 .align 64 2835 ALTENTRY(sfmmu_udtlb_slowpath_ismpred) 2836 2837 /* 2838 * g1 = tsb8k pointer register 2839 * g2 = tag access register 2840 * g4 - g6 = scratch registers 2841 * g7 = TSB tag to match 2842 */ 2843 2844 /* 2845 * ISM predict probe order 2846 * probe 4TH_TSB (4M index) 2847 * probe 2ND_TSB (4M index) 2848 * probe 1ST_TSB (8K index) 2849 * probe 3RD_TSB (8K index) 2850 2851 /* 2852 * Shared Context 4TH TSB 4M-256 pages 2853 */ 2854 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2855 brlz,pt %g6, 4f 2856 nop 2857 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2858 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2) 2859 2860 /* 2861 * Private 2ND TSB 4M-256 pages 2862 */ 28634: 2864 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2865 brlz,pt %g3, 5f 2866 nop 2867 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2868 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2) 2869 28705: 2871 PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2) 2872 2873 /* 2874 * Shared Context 3RD TSB 8K-512K pages 2875 */ 2876 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2877 brlz,pt %g6, 6f 2878 nop 2879 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2880 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2) 28816: 2882 ba,pn %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */ 2883 nop 2884 2885#else /* sun4u && UTSB_PHYS */ 2886 2887 .align 64 2888 ALTENTRY(sfmmu_udtlb_slowpath) 2889 2890 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 2891 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 2892 mov %g1, %g3 2893 2894udtlb_miss_probefirst: 2895 /* 2896 * g1 = 8K TSB pointer register 2897 * g2 = tag access register 2898 * g3 = (potentially) second TSB entry ptr 2899 * g6 = ism pred. 2900 * g7 = vpg_4m 2901 */ 2902#ifdef sun4v 2903 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2904 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2905 2906 /* 2907 * Here: 2908 * g1 = first TSB pointer 2909 * g2 = tag access reg 2910 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2911 */ 2912 brgz,pn %g6, sfmmu_tsb_miss_tt 2913 nop 2914#else /* sun4v */ 2915 mov %g1, %g4 2916 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 2917 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2918 2919 /* 2920 * Here: 2921 * g1 = first TSB pointer 2922 * g2 = tag access reg 2923 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2924 */ 2925 brgz,pn %g6, sfmmu_tsb_miss_tt 2926 nop 2927 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 2928 /* fall through in 8K->4M probe order */ 2929#endif /* sun4v */ 2930 2931udtlb_miss_probesecond: 2932 /* 2933 * Look in the second TSB for the TTE 2934 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 2935 * g2 = tag access reg 2936 * g3 = 8K TSB pointer register 2937 * g6 = ism pred. 2938 * g7 = vpg_4m 2939 */ 2940#ifdef sun4v 2941 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 2942 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2943 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 2944#else /* sun4v */ 2945 mov %g3, %g7 2946 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 2947 /* %g2 clobbered, %g3 =second tsbe ptr */ 2948 mov MMU_TAG_ACCESS, %g2 2949 ldxa [%g2]ASI_DMMU, %g2 2950#endif /* sun4v */ 2951 2952 srlx %g2, TAG_VALO_SHIFT, %g7 2953 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2954 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 2955 brgz,pn %g6, udtlb_miss_probefirst 2956 nop 2957 2958 /* fall through to sfmmu_tsb_miss_tt */ 2959#endif /* sun4u && UTSB_PHYS */ 2960 2961 2962 ALTENTRY(sfmmu_tsb_miss_tt) 2963 TT_TRACE(trace_tsbmiss) 2964 /* 2965 * We get here if there is a TSB miss OR a write protect trap. 2966 * 2967 * g1 = First TSB entry pointer 2968 * g2 = tag access register 2969 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 2970 * g4 - g7 = scratch registers 2971 */ 2972 2973 ALTENTRY(sfmmu_tsb_miss) 2974 2975 /* 2976 * If trapstat is running, we need to shift the %tpc and %tnpc to 2977 * point to trapstat's TSB miss return code (note that trapstat 2978 * itself will patch the correct offset to add). 2979 */ 2980 rdpr %tl, %g7 2981 cmp %g7, 1 2982 ble,pt %xcc, 0f 2983 sethi %hi(KERNELBASE), %g6 2984 rdpr %tpc, %g7 2985 or %g6, %lo(KERNELBASE), %g6 2986 cmp %g7, %g6 2987 bgeu,pt %xcc, 0f 2988 /* delay slot safe */ 2989 2990 ALTENTRY(tsbmiss_trapstat_patch_point) 2991 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 2992 wrpr %g7, %tpc 2993 add %g7, 4, %g7 2994 wrpr %g7, %tnpc 29950: 2996 CPU_TSBMISS_AREA(%g6, %g7) 2997 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save 1ST tsb pointer */ 2998 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save 2ND tsb pointer */ 2999 3000 sllx %g2, TAGACC_CTX_LSHIFT, %g3 3001 brz,a,pn %g3, 1f /* skip ahead if kernel */ 3002 ldn [%g6 + TSBMISS_KHATID], %g7 3003 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 3004 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 3005 3006 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 3007 3008 cmp %g3, INVALID_CONTEXT 3009 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 3010 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 3011 3012#if defined(sun4v) || defined(UTSB_PHYS) 3013 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 3014 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 3015 stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 3016#endif /* sun4v || UTSB_PHYS */ 3017 3018 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 3019 /* 3020 * The miss wasn't in an ISM segment. 3021 * 3022 * %g1 %g3, %g4, %g5, %g7 all clobbered 3023 * %g2 = (pseudo) tag access 3024 */ 3025 3026 ba,pt %icc, 2f 3027 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 3028 30291: 3030 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 3031 /* 3032 * 8K and 64K hash. 3033 */ 30342: 3035 3036 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3037 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_l8K, tsb_checktte, 3038 sfmmu_suspend_tl, tsb_512K) 3039 /* NOT REACHED */ 3040 3041tsb_512K: 3042 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3043 brz,pn %g5, 3f 3044 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3045 and %g4, HAT_512K_FLAG, %g5 3046 3047 /* 3048 * Note that there is a small window here where we may have 3049 * a 512k page in the hash list but have not set the HAT_512K_FLAG 3050 * flag yet, so we will skip searching the 512k hash list. 3051 * In this case we will end up in pagefault which will find 3052 * the mapping and return. So, in this instance we will end up 3053 * spending a bit more time resolving this TSB miss, but it can 3054 * only happen once per process and even then, the chances of that 3055 * are very small, so it's not worth the extra overhead it would 3056 * take to close this window. 3057 */ 3058 brz,pn %g5, tsb_4M 3059 nop 30603: 3061 /* 3062 * 512K hash 3063 */ 3064 3065 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3066 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_l512K, tsb_checktte, 3067 sfmmu_suspend_tl, tsb_4M) 3068 /* NOT REACHED */ 3069 3070tsb_4M: 3071 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3072 brz,pn %g5, 4f 3073 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3074 and %g4, HAT_4M_FLAG, %g5 3075 brz,pn %g5, tsb_32M 3076 nop 30774: 3078 /* 3079 * 4M hash 3080 */ 3081 3082 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3083 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_l4M, tsb_checktte, 3084 sfmmu_suspend_tl, tsb_32M) 3085 /* NOT REACHED */ 3086 3087tsb_32M: 3088 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3089#ifdef sun4v 3090 brz,pn %g5, 6f 3091#else 3092 brz,pn %g5, tsb_pagefault 3093#endif 3094 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3095 and %g4, HAT_32M_FLAG, %g5 3096 brz,pn %g5, tsb_256M 3097 nop 30985: 3099 /* 3100 * 32M hash 3101 */ 3102 3103 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3104 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_l32M, tsb_checktte, 3105 sfmmu_suspend_tl, tsb_256M) 3106 /* NOT REACHED */ 3107 3108#if defined(sun4u) && !defined(UTSB_PHYS) 3109#define tsb_shme tsb_pagefault 3110#endif 3111tsb_256M: 3112 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3113 and %g4, HAT_256M_FLAG, %g5 3114 brz,pn %g5, tsb_shme 3115 nop 31166: 3117 /* 3118 * 256M hash 3119 */ 3120 3121 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3122 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_l256M, tsb_checktte, 3123 sfmmu_suspend_tl, tsb_shme) 3124 /* NOT REACHED */ 3125 3126tsb_checktte: 3127 /* 3128 * g1 = hblk_misc 3129 * g2 = tagacc 3130 * g3 = tte 3131 * g4 = tte pa 3132 * g6 = tsbmiss area 3133 * g7 = hatid 3134 */ 3135 brlz,a,pt %g3, tsb_validtte 3136 rdpr %tt, %g7 3137 3138#if defined(sun4u) && !defined(UTSB_PHYS) 3139#undef tsb_shme 3140 ba tsb_pagefault 3141 nop 3142#else /* sun4u && !UTSB_PHYS */ 3143 3144tsb_shme: 3145 /* 3146 * g2 = tagacc 3147 * g6 = tsbmiss area 3148 */ 3149 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3150 brz,pn %g5, tsb_pagefault 3151 nop 3152 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3153 brz,pn %g7, tsb_pagefault 3154 nop 3155 3156 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3157 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_shme_l8K, tsb_shme_checktte, 3158 sfmmu_suspend_tl, tsb_shme_512K) 3159 /* NOT REACHED */ 3160 3161tsb_shme_512K: 3162 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3163 and %g4, HAT_512K_FLAG, %g5 3164 brz,pn %g5, tsb_shme_4M 3165 nop 3166 3167 /* 3168 * 512K hash 3169 */ 3170 3171 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3172 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_shme_l512K, tsb_shme_checktte, 3173 sfmmu_suspend_tl, tsb_shme_4M) 3174 /* NOT REACHED */ 3175 3176tsb_shme_4M: 3177 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3178 and %g4, HAT_4M_FLAG, %g5 3179 brz,pn %g5, tsb_shme_32M 3180 nop 31814: 3182 /* 3183 * 4M hash 3184 */ 3185 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3186 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_shme_l4M, tsb_shme_checktte, 3187 sfmmu_suspend_tl, tsb_shme_32M) 3188 /* NOT REACHED */ 3189 3190tsb_shme_32M: 3191 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3192 and %g4, HAT_32M_FLAG, %g5 3193 brz,pn %g5, tsb_shme_256M 3194 nop 3195 3196 /* 3197 * 32M hash 3198 */ 3199 3200 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3201 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_shme_l32M, tsb_shme_checktte, 3202 sfmmu_suspend_tl, tsb_shme_256M) 3203 /* NOT REACHED */ 3204 3205tsb_shme_256M: 3206 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3207 and %g4, HAT_256M_FLAG, %g5 3208 brz,pn %g5, tsb_pagefault 3209 nop 3210 3211 /* 3212 * 256M hash 3213 */ 3214 3215 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3216 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_shme_l256M, tsb_shme_checktte, 3217 sfmmu_suspend_tl, tsb_pagefault) 3218 /* NOT REACHED */ 3219 3220tsb_shme_checktte: 3221 3222 brgez,pn %g3, tsb_pagefault 3223 rdpr %tt, %g7 3224 /* 3225 * g1 = ctx1 flag 3226 * g3 = tte 3227 * g4 = tte pa 3228 * g6 = tsbmiss area 3229 * g7 = tt 3230 */ 3231 3232 brz,pt %g1, tsb_validtte 3233 nop 3234 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3235 or %g1, HAT_CHKCTX1_FLAG, %g1 3236 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3237 3238 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3239#endif /* sun4u && !UTSB_PHYS */ 3240 3241tsb_validtte: 3242 /* 3243 * g3 = tte 3244 * g4 = tte pa 3245 * g6 = tsbmiss area 3246 * g7 = tt 3247 */ 3248 3249 /* 3250 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3251 */ 3252 cmp %g7, FAST_PROT_TT 3253 bne,pt %icc, 4f 3254 nop 3255 3256 TTE_SET_REFMOD_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_refmod, 3257 tsb_protfault) 3258 3259 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3260#ifdef sun4v 3261 MMU_FAULT_STATUS_AREA(%g7) 3262 ldx [%g7 + MMFSA_D_ADDR], %g5 /* load fault addr for later */ 3263#else /* sun4v */ 3264 mov MMU_TAG_ACCESS, %g5 3265 ldxa [%g5]ASI_DMMU, %g5 3266#endif /* sun4v */ 3267 ba,pt %xcc, tsb_update_tl1 3268 nop 32694: 3270 /* 3271 * ITLB translation was found but execute permission is 3272 * disabled. If we have software execute permission (soft exec 3273 * bit is set), then enable hardware execute permission. 3274 * Otherwise continue with a protection violation. 3275 */ 3276 cmp %g7, T_INSTR_MMU_MISS 3277 be,pn %icc, 5f 3278 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3279 cmp %g7, FAST_IMMU_MISS_TT 3280 bne,pt %icc, 3f 3281 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 32825: 3283 bnz,pn %icc, 3f 3284 TTE_CHK_SOFTEXEC_ML(%g3) /* check soft execute */ 3285 bz,pn %icc, tsb_protfault 3286 nop 3287 TTE_SET_EXEC_ML(%g3, %g4, %g7, tsb_lset_exec) 32883: 3289 /* 3290 * Set reference bit if not already set 3291 */ 3292 TTE_SET_REF_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_ref) 3293 3294 /* 3295 * Now, load into TSB/TLB. At this point: 3296 * g3 = tte 3297 * g4 = patte 3298 * g6 = tsbmiss area 3299 */ 3300 rdpr %tt, %g7 3301#ifdef sun4v 3302 MMU_FAULT_STATUS_AREA(%g2) 3303 cmp %g7, T_INSTR_MMU_MISS 3304 be,a,pt %icc, 9f 3305 nop 3306 cmp %g7, FAST_IMMU_MISS_TT 3307 be,a,pt %icc, 9f 3308 nop 3309 add %g2, MMFSA_D_, %g2 33109: 3311 ldx [%g2 + MMFSA_CTX_], %g7 3312 sllx %g7, TTARGET_CTX_SHIFT, %g7 3313 ldx [%g2 + MMFSA_ADDR_], %g2 3314 mov %g2, %g5 ! load the fault addr for later use 3315 srlx %g2, TTARGET_VA_SHIFT, %g2 3316 or %g2, %g7, %g2 3317#else /* sun4v */ 3318 mov MMU_TAG_ACCESS, %g5 3319 cmp %g7, FAST_IMMU_MISS_TT 3320 be,a,pt %icc, 9f 3321 ldxa [%g0]ASI_IMMU, %g2 3322 ldxa [%g0]ASI_DMMU, %g2 3323 ba,pt %icc, tsb_update_tl1 3324 ldxa [%g5]ASI_DMMU, %g5 33259: 3326 ldxa [%g5]ASI_IMMU, %g5 3327#endif /* sun4v */ 3328 3329tsb_update_tl1: 3330 TTE_CLR_SOFTEXEC_ML(%g3) 3331 srlx %g2, TTARGET_CTX_SHIFT, %g7 3332 brz,pn %g7, tsb_kernel 3333#ifdef sun4v 3334 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3335#else /* sun4v */ 3336 srlx %g3, TTE_SZ_SHFT, %g7 3337#endif /* sun4v */ 3338 3339tsb_user: 3340#ifdef sun4v 3341 cmp %g7, TTE4M 3342 bge,pn %icc, tsb_user4m 3343 nop 3344#else /* sun4v */ 3345 cmp %g7, TTESZ_VALID | TTE4M 3346 be,pn %icc, tsb_user4m 3347 srlx %g3, TTE_SZ2_SHFT, %g7 3348 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3349#ifdef ITLB_32M_256M_SUPPORT 3350 bnz,pn %icc, tsb_user4m 3351 nop 3352#else /* ITLB_32M_256M_SUPPORT */ 3353 bnz,a,pn %icc, tsb_user_pn_synth 3354 nop 3355#endif /* ITLB_32M_256M_SUPPORT */ 3356#endif /* sun4v */ 3357 3358tsb_user8k: 3359#if defined(sun4v) || defined(UTSB_PHYS) 3360 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3361 and %g7, HAT_CHKCTX1_FLAG, %g1 3362 brz,a,pn %g1, 1f 3363 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3364 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3365 brlz,a,pn %g1, ptl1_panic ! if no shared 3RD tsb 3366 mov PTL1_NO_SCDTSB8K, %g1 ! panic 3367 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 33681: 3369#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3370 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3371#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3372 3373#ifndef UTSB_PHYS 3374 mov ASI_N, %g7 ! user TSBs accessed by VA 3375 mov %g7, %asi 3376#endif /* !UTSB_PHYS */ 3377 3378 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l3) 3379 3380 rdpr %tt, %g5 3381#ifdef sun4v 3382 cmp %g5, T_INSTR_MMU_MISS 3383 be,a,pn %xcc, 9f 3384 mov %g3, %g5 3385#endif /* sun4v */ 3386 cmp %g5, FAST_IMMU_MISS_TT 3387 be,pn %xcc, 9f 3388 mov %g3, %g5 3389 3390 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3391 ! trapstat wants TTE in %g5 3392 retry 33939: 3394 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3395 ! trapstat wants TTE in %g5 3396 retry 3397 3398tsb_user4m: 3399#if defined(sun4v) || defined(UTSB_PHYS) 3400 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3401 and %g7, HAT_CHKCTX1_FLAG, %g1 3402 brz,a,pn %g1, 4f 3403 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3404 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr 3405 brlz,a,pn %g1, 5f ! if no shared 4TH TSB 3406 nop 3407 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 3408 3409#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3410 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3411#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 34124: 3413 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3414 nop 3415 3416#ifndef UTSB_PHYS 3417 mov ASI_N, %g7 ! user TSBs accessed by VA 3418 mov %g7, %asi 3419#endif /* UTSB_PHYS */ 3420 3421 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l4) 3422 34235: 3424 rdpr %tt, %g5 3425#ifdef sun4v 3426 cmp %g5, T_INSTR_MMU_MISS 3427 be,a,pn %xcc, 9f 3428 mov %g3, %g5 3429#endif /* sun4v */ 3430 cmp %g5, FAST_IMMU_MISS_TT 3431 be,pn %xcc, 9f 3432 mov %g3, %g5 3433 3434 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3435 ! trapstat wants TTE in %g5 3436 retry 34379: 3438 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3439 ! trapstat wants TTE in %g5 3440 retry 3441 3442#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3443 /* 3444 * Panther ITLB synthesis. 3445 * The Panther 32M and 256M ITLB code simulates these two large page 3446 * sizes with 4M pages, to provide support for programs, for example 3447 * Java, that may copy instructions into a 32M or 256M data page and 3448 * then execute them. The code below generates the 4M pfn bits and 3449 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3450 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3451 * are ignored by the hardware. 3452 * 3453 * Now, load into TSB/TLB. At this point: 3454 * g2 = tagtarget 3455 * g3 = tte 3456 * g4 = patte 3457 * g5 = tt 3458 * g6 = tsbmiss area 3459 */ 3460tsb_user_pn_synth: 3461 rdpr %tt, %g5 3462 cmp %g5, FAST_IMMU_MISS_TT 3463 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3464 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3465 bz,pn %icc, 4b /* if not, been here before */ 3466 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3467 brlz,a,pn %g1, 5f /* no 2nd tsb */ 3468 mov %g3, %g5 3469 3470 mov MMU_TAG_ACCESS, %g7 3471 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3472 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3473 3474 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3475 mov %g7, %asi 3476 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l5) /* update TSB */ 34775: 3478 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3479 retry 3480 3481tsb_user_itlb_synth: 3482 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = 2ND TSB */ 3483 3484 mov MMU_TAG_ACCESS, %g7 3485 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3486 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3487 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3488 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3489 3490 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3491 mov %g7, %asi 3492 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l6) /* update TSB */ 34937: 3494 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3495 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3496 retry 3497#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3498 3499tsb_kernel: 3500 rdpr %tt, %g5 3501#ifdef sun4v 3502 cmp %g7, TTE4M 3503 bge,pn %icc, 5f 3504#else 3505 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3506 be,pn %icc, 5f 3507#endif /* sun4v */ 3508 nop 3509 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8K TSB ptr 3510 ba,pt %xcc, 6f 3511 nop 35125: 3513 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4M TSB ptr 3514 brlz,pn %g1, 3f /* skip programming if 4M TSB ptr is -1 */ 3515 nop 35166: 3517#ifndef sun4v 3518tsb_kernel_patch_asi: 3519 or %g0, RUNTIME_PATCH, %g6 3520 mov %g6, %asi ! XXX avoid writing to %asi !! 3521#endif 3522 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l7) 35233: 3524#ifdef sun4v 3525 cmp %g5, T_INSTR_MMU_MISS 3526 be,a,pn %icc, 1f 3527 mov %g3, %g5 ! trapstat wants TTE in %g5 3528#endif /* sun4v */ 3529 cmp %g5, FAST_IMMU_MISS_TT 3530 be,pn %icc, 1f 3531 mov %g3, %g5 ! trapstat wants TTE in %g5 3532 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3533 ! trapstat wants TTE in %g5 3534 retry 35351: 3536 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3537 ! trapstat wants TTE in %g5 3538 retry 3539 3540tsb_ism: 3541 /* 3542 * This is an ISM [i|d]tlb miss. We optimize for largest 3543 * page size down to smallest. 3544 * 3545 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3546 * register 3547 * g3 = ismmap->ism_seg 3548 * g4 = physical address of ismmap->ism_sfmmu 3549 * g6 = tsbmiss area 3550 */ 3551 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3552 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3553 mov PTL1_BAD_ISM, %g1 3554 /* g5 = pa of imap_vb_shift */ 3555 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3556 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3557 srlx %g3, %g4, %g3 /* clr size field */ 3558 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3559 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3560 and %g2, %g1, %g4 /* g4 = ctx number */ 3561 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3562 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3563 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3564 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3565 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3566#if defined(sun4v) || defined(UTSB_PHYS) 3567 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3568 brz,pt %g5, tsb_chk4M_ism 3569 nop 3570 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3571 or %g5, HAT_CHKCTX1_FLAG, %g5 3572 stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3573 rdpr %tt, %g5 3574 SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl) 3575#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3576 3577 /* 3578 * ISM pages are always locked down. 3579 * If we can't find the tte then pagefault 3580 * and let the spt segment driver resolve it. 3581 * 3582 * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3583 * g4 = imap_hatflags 3584 * g6 = tsb miss area 3585 * g7 = ISM hatid 3586 */ 3587 3588tsb_chk4M_ism: 3589 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3590 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3591 nop 3592 3593tsb_ism_32M: 3594 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3595 brz,pn %g5, tsb_ism_256M 3596 nop 3597 3598 /* 3599 * 32M hash. 3600 */ 3601 3602 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT32M, 3603 TTE32M, %g5, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3604 tsb_ism_4M) 3605 /* NOT REACHED */ 3606 3607tsb_ism_32M_found: 3608 brlz,a,pt %g3, tsb_validtte 3609 rdpr %tt, %g7 3610 ba,pt %xcc, tsb_ism_4M 3611 nop 3612 3613tsb_ism_256M: 3614 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3615 brz,a,pn %g5, ptl1_panic 3616 mov PTL1_BAD_ISM, %g1 3617 3618 /* 3619 * 256M hash. 3620 */ 3621 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT256M, 3622 TTE256M, %g5, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3623 tsb_ism_4M) 3624 3625tsb_ism_256M_found: 3626 brlz,a,pt %g3, tsb_validtte 3627 rdpr %tt, %g7 3628 3629tsb_ism_4M: 3630 /* 3631 * 4M hash. 3632 */ 3633 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT4M, 3634 TTE4M, %g5, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3635 tsb_ism_8K) 3636 /* NOT REACHED */ 3637 3638tsb_ism_4M_found: 3639 brlz,a,pt %g3, tsb_validtte 3640 rdpr %tt, %g7 3641 3642tsb_ism_8K: 3643 /* 3644 * 8K and 64K hash. 3645 */ 3646 3647 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT64K, 3648 TTE64K, %g5, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3649 tsb_pagefault) 3650 /* NOT REACHED */ 3651 3652tsb_ism_8K_found: 3653 brlz,a,pt %g3, tsb_validtte 3654 rdpr %tt, %g7 3655 3656tsb_pagefault: 3657 rdpr %tt, %g7 3658 cmp %g7, FAST_PROT_TT 3659 be,a,pn %icc, tsb_protfault 3660 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3661 3662tsb_protfault: 3663 /* 3664 * we get here if we couldn't find a valid tte in the hash. 3665 * 3666 * If user and we are at tl>1 we go to window handling code. 3667 * 3668 * If kernel and the fault is on the same page as our stack 3669 * pointer, then we know the stack is bad and the trap handler 3670 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3671 * 3672 * If this is a kernel trap and tl>1, panic. 3673 * 3674 * Otherwise we call pagefault. 3675 */ 3676 cmp %g7, FAST_IMMU_MISS_TT 3677#ifdef sun4v 3678 MMU_FAULT_STATUS_AREA(%g4) 3679 ldx [%g4 + MMFSA_I_CTX], %g5 3680 ldx [%g4 + MMFSA_D_CTX], %g4 3681 move %icc, %g5, %g4 3682 cmp %g7, T_INSTR_MMU_MISS 3683 move %icc, %g5, %g4 3684#else 3685 mov MMU_TAG_ACCESS, %g4 3686 ldxa [%g4]ASI_DMMU, %g2 3687 ldxa [%g4]ASI_IMMU, %g5 3688 move %icc, %g5, %g2 3689 cmp %g7, T_INSTR_MMU_MISS 3690 move %icc, %g5, %g2 3691 sllx %g2, TAGACC_CTX_LSHIFT, %g4 3692#endif /* sun4v */ 3693 brnz,pn %g4, 3f /* skip if not kernel */ 3694 rdpr %tl, %g5 3695 3696 add %sp, STACK_BIAS, %g3 3697 srlx %g3, MMU_PAGESHIFT, %g3 3698 srlx %g2, MMU_PAGESHIFT, %g4 3699 cmp %g3, %g4 3700 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 3701 mov PTL1_BAD_STACK, %g1 3702 3703 cmp %g5, 1 3704 ble,pt %icc, 2f 3705 nop 3706 TSTAT_CHECK_TL1(2f, %g1, %g2) 3707 rdpr %tt, %g2 3708 cmp %g2, FAST_PROT_TT 3709 mov PTL1_BAD_KPROT_FAULT, %g1 3710 movne %icc, PTL1_BAD_KMISS, %g1 3711 ba,pt %icc, ptl1_panic 3712 nop 3713 37142: 3715 /* 3716 * We are taking a pagefault in the kernel on a kernel address. If 3717 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 3718 * want to call sfmmu_pagefault -- we will instead note that a fault 3719 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 3720 * (instead of a "retry"). This will step over the faulting 3721 * instruction. 3722 */ 3723 CPU_INDEX(%g1, %g2) 3724 set cpu_core, %g2 3725 sllx %g1, CPU_CORE_SHIFT, %g1 3726 add %g1, %g2, %g1 3727 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3728 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3729 bz sfmmu_pagefault 3730 or %g2, CPU_DTRACE_BADADDR, %g2 3731 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3732 GET_MMU_D_ADDR(%g3, %g4) 3733 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3734 done 3735 37363: 3737 cmp %g5, 1 3738 ble,pt %icc, 4f 3739 nop 3740 TSTAT_CHECK_TL1(4f, %g1, %g2) 3741 ba,pt %icc, sfmmu_window_trap 3742 nop 3743 37444: 3745 /* 3746 * We are taking a pagefault on a non-kernel address. If we are in 3747 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 3748 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 3749 */ 3750 CPU_INDEX(%g1, %g2) 3751 set cpu_core, %g2 3752 sllx %g1, CPU_CORE_SHIFT, %g1 3753 add %g1, %g2, %g1 3754 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3755 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3756 bz sfmmu_mmu_trap 3757 or %g2, CPU_DTRACE_BADADDR, %g2 3758 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3759 GET_MMU_D_ADDR(%g3, %g4) 3760 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3761 3762 /* 3763 * Be sure that we're actually taking this miss from the kernel -- 3764 * otherwise we have managed to return to user-level with 3765 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3766 */ 3767 rdpr %tstate, %g2 3768 btst TSTATE_PRIV, %g2 3769 bz,a ptl1_panic 3770 mov PTL1_BAD_DTRACE_FLAGS, %g1 3771 done 3772 3773 ALTENTRY(tsb_tl0_noctxt) 3774 /* 3775 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 3776 * if it is, indicated that we have faulted and issue a done. 3777 */ 3778 CPU_INDEX(%g5, %g6) 3779 set cpu_core, %g6 3780 sllx %g5, CPU_CORE_SHIFT, %g5 3781 add %g5, %g6, %g5 3782 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 3783 andcc %g6, CPU_DTRACE_NOFAULT, %g0 3784 bz 1f 3785 or %g6, CPU_DTRACE_BADADDR, %g6 3786 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 3787 GET_MMU_D_ADDR(%g3, %g4) 3788 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 3789 3790 /* 3791 * Be sure that we're actually taking this miss from the kernel -- 3792 * otherwise we have managed to return to user-level with 3793 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3794 */ 3795 rdpr %tstate, %g5 3796 btst TSTATE_PRIV, %g5 3797 bz,a ptl1_panic 3798 mov PTL1_BAD_DTRACE_FLAGS, %g1 3799 TSTAT_CHECK_TL1(2f, %g1, %g2); 38002: 3801 done 3802 38031: 3804 rdpr %tt, %g5 3805 cmp %g5, FAST_IMMU_MISS_TT 3806#ifdef sun4v 3807 MMU_FAULT_STATUS_AREA(%g2) 3808 be,a,pt %icc, 2f 3809 ldx [%g2 + MMFSA_I_CTX], %g3 3810 cmp %g5, T_INSTR_MMU_MISS 3811 be,a,pt %icc, 2f 3812 ldx [%g2 + MMFSA_I_CTX], %g3 3813 ldx [%g2 + MMFSA_D_CTX], %g3 38142: 3815#else 3816 mov MMU_TAG_ACCESS, %g2 3817 be,a,pt %icc, 2f 3818 ldxa [%g2]ASI_IMMU, %g3 3819 ldxa [%g2]ASI_DMMU, %g3 38202: sllx %g3, TAGACC_CTX_LSHIFT, %g3 3821#endif /* sun4v */ 3822 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 3823 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 3824 rdpr %tl, %g5 3825 cmp %g5, 1 3826 ble,pt %icc, sfmmu_mmu_trap 3827 nop 3828 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 3829 ba,pt %icc, sfmmu_window_trap 3830 nop 3831 SET_SIZE(sfmmu_tsb_miss) 3832#endif /* lint */ 3833 3834#if defined (lint) 3835/* 3836 * This routine will look for a user or kernel vaddr in the hash 3837 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 3838 * grab any locks. It should only be used by other sfmmu routines. 3839 */ 3840/* ARGSUSED */ 3841pfn_t 3842sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 3843{ 3844 return(0); 3845} 3846 3847/* ARGSUSED */ 3848pfn_t 3849sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno) 3850{ 3851 return(0); 3852} 3853 3854#else /* lint */ 3855 3856 ENTRY_NP(sfmmu_vatopfn) 3857 /* 3858 * disable interrupts 3859 */ 3860 rdpr %pstate, %o3 3861#ifdef DEBUG 3862 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 3863#endif 3864 /* 3865 * disable interrupts to protect the TSBMISS area 3866 */ 3867 andn %o3, PSTATE_IE, %o5 3868 wrpr %o5, 0, %pstate 3869 3870 /* 3871 * o0 = vaddr 3872 * o1 = sfmmup 3873 * o2 = ttep 3874 */ 3875 CPU_TSBMISS_AREA(%g1, %o5) 3876 ldn [%g1 + TSBMISS_KHATID], %o4 3877 cmp %o4, %o1 3878 bne,pn %ncc, vatopfn_nokernel 3879 mov TTE64K, %g5 /* g5 = rehash # */ 3880 mov %g1,%o5 /* o5 = tsbmiss_area */ 3881 /* 3882 * o0 = vaddr 3883 * o1 & o4 = hatid 3884 * o2 = ttep 3885 * o5 = tsbmiss area 3886 */ 3887 mov HBLK_RANGE_SHIFT, %g6 38881: 3889 3890 /* 3891 * o0 = vaddr 3892 * o1 = sfmmup 3893 * o2 = ttep 3894 * o3 = old %pstate 3895 * o4 = hatid 3896 * o5 = tsbmiss 3897 * g5 = rehash # 3898 * g6 = hmeshift 3899 * 3900 * The first arg to GET_TTE is actually tagaccess register 3901 * not just vaddr. Since this call is for kernel we need to clear 3902 * any lower vaddr bits that would be interpreted as ctx bits. 3903 */ 3904 set TAGACC_CTX_MASK, %g1 3905 andn %o0, %g1, %o0 3906 GET_TTE(%o0, %o4, %g1, %g2, %o5, %g4, %g6, %g5, %g3, 3907 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 3908 3909kvtop_hblk_found: 3910 /* 3911 * o0 = vaddr 3912 * o1 = sfmmup 3913 * o2 = ttep 3914 * g1 = tte 3915 * g2 = tte pa 3916 * g3 = scratch 3917 * o2 = tsbmiss area 3918 * o1 = hat id 3919 */ 3920 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 3921 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3922 stx %g1,[%o2] /* put tte into *ttep */ 3923 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 3924 /* 3925 * o0 = vaddr 3926 * o1 = sfmmup 3927 * o2 = ttep 3928 * g1 = pfn 3929 */ 3930 ba,pt %xcc, 6f 3931 mov %g1, %o0 3932 3933kvtop_nohblk: 3934 /* 3935 * we get here if we couldn't find valid hblk in hash. We rehash 3936 * if neccesary. 3937 */ 3938 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 3939#ifdef sun4v 3940 cmp %g5, MAX_HASHCNT 3941#else 3942 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 3943#endif /* sun4v */ 3944 be,a,pn %icc, 6f 3945 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3946 mov %o1, %o4 /* restore hatid */ 3947#ifdef sun4v 3948 add %g5, 2, %g5 3949 cmp %g5, 3 3950 move %icc, MMU_PAGESHIFT4M, %g6 3951 ba,pt %icc, 1b 3952 movne %icc, MMU_PAGESHIFT256M, %g6 3953#else 3954 inc %g5 3955 cmp %g5, 2 3956 move %icc, MMU_PAGESHIFT512K, %g6 3957 ba,pt %icc, 1b 3958 movne %icc, MMU_PAGESHIFT4M, %g6 3959#endif /* sun4v */ 39606: 3961 retl 3962 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3963 3964tsb_suspend: 3965 /* 3966 * o0 = vaddr 3967 * o1 = sfmmup 3968 * o2 = ttep 3969 * g1 = tte 3970 * g2 = tte pa 3971 * g3 = tte va 3972 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 3973 */ 3974 stx %g1,[%o2] /* put tte into *ttep */ 3975 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 3976 sub %g0, 1, %o0 /* output = PFN_INVALID */ 3977 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 39788: 3979 retl 3980 wrpr %g0, %o3, %pstate /* enable interrupts */ 3981 3982vatopfn_nokernel: 3983 /* 3984 * This routine does NOT support user addresses 3985 * There is a routine in C that supports this. 3986 * The only reason why we don't have the C routine 3987 * support kernel addresses as well is because 3988 * we do va_to_pa while holding the hashlock. 3989 */ 3990 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3991 save %sp, -SA(MINFRAME), %sp 3992 sethi %hi(sfmmu_panic3), %o0 3993 call panic 3994 or %o0, %lo(sfmmu_panic3), %o0 3995 3996 SET_SIZE(sfmmu_vatopfn) 3997 3998 /* 3999 * %o0 = vaddr 4000 * %o1 = hashno (aka szc) 4001 * 4002 * 4003 * This routine is similar to sfmmu_vatopfn() but will only look for 4004 * a kernel vaddr in the hash structure for the specified rehash value. 4005 * It's just an optimization for the case when pagesize for a given 4006 * va range is already known (e.g. large page heap) and we don't want 4007 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 4008 * 4009 * Returns valid pfn or PFN_INVALID if 4010 * tte for specified rehash # is not found, invalid or suspended. 4011 */ 4012 ENTRY_NP(sfmmu_kvaszc2pfn) 4013 /* 4014 * disable interrupts 4015 */ 4016 rdpr %pstate, %o3 4017#ifdef DEBUG 4018 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 4019#endif 4020 /* 4021 * disable interrupts to protect the TSBMISS area 4022 */ 4023 andn %o3, PSTATE_IE, %o5 4024 wrpr %o5, 0, %pstate 4025 4026 CPU_TSBMISS_AREA(%g1, %o5) 4027 ldn [%g1 + TSBMISS_KHATID], %o4 4028 sll %o1, 1, %g6 4029 add %g6, %o1, %g6 4030 add %g6, MMU_PAGESHIFT, %g6 4031 /* 4032 * %o0 = vaddr 4033 * %o1 = hashno 4034 * %o3 = old %pstate 4035 * %o4 = ksfmmup 4036 * %g1 = tsbmiss area 4037 * %g6 = hmeshift 4038 */ 4039 4040 /* 4041 * The first arg to GET_TTE is actually tagaccess register 4042 * not just vaddr. Since this call is for kernel we need to clear 4043 * any lower vaddr bits that would be interpreted as ctx bits. 4044 */ 4045 srlx %o0, MMU_PAGESHIFT, %o0 4046 sllx %o0, MMU_PAGESHIFT, %o0 4047 GET_TTE(%o0, %o4, %g3, %g4, %g1, %o5, %g6, %o1, %g5, 4048 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 4049 kvaszc2pfn_nohblk) 4050 4051kvaszc2pfn_hblk_found: 4052 /* 4053 * %g3 = tte 4054 * %o0 = vaddr 4055 */ 4056 brgez,a,pn %g3, 1f /* check if tte is invalid */ 4057 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4058 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 4059 /* 4060 * g3 = pfn 4061 */ 4062 ba,pt %xcc, 1f 4063 mov %g3, %o0 4064 4065kvaszc2pfn_nohblk: 4066 mov -1, %o0 4067 40681: 4069 retl 4070 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4071 4072 SET_SIZE(sfmmu_kvaszc2pfn) 4073 4074#endif /* lint */ 4075 4076 4077 4078#if !defined(lint) 4079 4080/* 4081 * kpm lock used between trap level tsbmiss handler and kpm C level. 4082 */ 4083#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 4084 mov 0xff, tmp1 ;\ 4085label1: ;\ 4086 casa [kpmlckp]asi, %g0, tmp1 ;\ 4087 brnz,pn tmp1, label1 ;\ 4088 mov 0xff, tmp1 ;\ 4089 membar #LoadLoad 4090 4091#define KPMLOCK_EXIT(kpmlckp, asi) \ 4092 membar #LoadStore|#StoreStore ;\ 4093 sta %g0, [kpmlckp]asi 4094 4095/* 4096 * Lookup a memseg for a given pfn and if found, return the physical 4097 * address of the corresponding struct memseg in mseg, otherwise 4098 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 4099 * tsbmp, %asi is assumed to be ASI_MEM. 4100 * This lookup is done by strictly traversing only the physical memseg 4101 * linkage. The more generic approach, to check the virtual linkage 4102 * before using the physical (used e.g. with hmehash buckets), cannot 4103 * be used here. Memory DR operations can run in parallel to this 4104 * lookup w/o any locks and updates of the physical and virtual linkage 4105 * cannot be done atomically wrt. to each other. Because physical 4106 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 4107 * as "physical NULL" pointer. 4108 */ 4109#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 4110 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 4111 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 4112 udivx pfn, mseg, mseg ;\ 4113 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 4114 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 4115 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 4116 add tmp1, mseg, tmp1 ;\ 4117 ldxa [tmp1]%asi, mseg ;\ 4118 cmp mseg, MSEG_NULLPTR_PA ;\ 4119 be,pn %xcc, label/**/1 /* if not found */ ;\ 4120 nop ;\ 4121 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4122 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4123 blu,pn %xcc, label/**/1 ;\ 4124 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4125 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4126 bgeu,pn %xcc, label/**/1 ;\ 4127 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 4128 mulx tmp1, PAGE_SIZE, tmp1 ;\ 4129 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 4130 add tmp2, tmp1, tmp1 /* pp */ ;\ 4131 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 4132 cmp tmp2, pfn ;\ 4133 be,pt %xcc, label/**/_ok /* found */ ;\ 4134label/**/1: ;\ 4135 /* brute force lookup */ ;\ 4136 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 4137 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 4138label/**/2: ;\ 4139 cmp mseg, MSEG_NULLPTR_PA ;\ 4140 be,pn %xcc, label/**/_ok /* if not found */ ;\ 4141 nop ;\ 4142 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4143 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4144 blu,a,pt %xcc, label/**/2 ;\ 4145 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4146 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4147 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4148 bgeu,a,pt %xcc, label/**/2 ;\ 4149 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4150label/**/_ok: 4151 4152 /* 4153 * kpm tsb miss handler large pages 4154 * g1 = 8K kpm TSB entry pointer 4155 * g2 = tag access register 4156 * g3 = 4M kpm TSB entry pointer 4157 */ 4158 ALTENTRY(sfmmu_kpm_dtsb_miss) 4159 TT_TRACE(trace_tsbmiss) 4160 4161 CPU_INDEX(%g7, %g6) 4162 sethi %hi(kpmtsbm_area), %g6 4163 sllx %g7, KPMTSBM_SHIFT, %g7 4164 or %g6, %lo(kpmtsbm_area), %g6 4165 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4166 4167 /* check enable flag */ 4168 ldub [%g6 + KPMTSBM_FLAGS], %g4 4169 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4170 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4171 nop 4172 4173 /* VA range check */ 4174 ldx [%g6 + KPMTSBM_VBASE], %g7 4175 cmp %g2, %g7 4176 blu,pn %xcc, sfmmu_tsb_miss 4177 ldx [%g6 + KPMTSBM_VEND], %g5 4178 cmp %g2, %g5 4179 bgeu,pn %xcc, sfmmu_tsb_miss 4180 stx %g3, [%g6 + KPMTSBM_TSBPTR] 4181 4182 /* 4183 * check TL tsbmiss handling flag 4184 * bump tsbmiss counter 4185 */ 4186 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4187#ifdef DEBUG 4188 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4189 inc %g5 4190 brz,pn %g3, sfmmu_kpm_exception 4191 st %g5, [%g6 + KPMTSBM_TSBMISS] 4192#else 4193 inc %g5 4194 st %g5, [%g6 + KPMTSBM_TSBMISS] 4195#endif 4196 /* 4197 * At this point: 4198 * g1 = 8K kpm TSB pointer (not used) 4199 * g2 = tag access register 4200 * g3 = clobbered 4201 * g6 = per-CPU kpm tsbmiss area 4202 * g7 = kpm_vbase 4203 */ 4204 4205 /* vaddr2pfn */ 4206 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4207 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4208 srax %g4, %g3, %g2 /* which alias range (r) */ 4209 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4210 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4211 4212 /* 4213 * Setup %asi 4214 * mseg_pa = page_numtomemseg_nolock(pfn) 4215 * if (mseg_pa == NULL) sfmmu_kpm_exception 4216 * g2=pfn 4217 */ 4218 mov ASI_MEM, %asi 4219 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4220 cmp %g3, MSEG_NULLPTR_PA 4221 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4222 nop 4223 4224 /* 4225 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4226 * g2=pfn g3=mseg_pa 4227 */ 4228 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4229 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4230 srlx %g2, %g5, %g4 4231 sllx %g4, %g5, %g4 4232 sub %g4, %g7, %g4 4233 srlx %g4, %g5, %g4 4234 4235 /* 4236 * Validate inx value 4237 * g2=pfn g3=mseg_pa g4=inx 4238 */ 4239#ifdef DEBUG 4240 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4241 cmp %g4, %g5 /* inx - nkpmpgs */ 4242 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4243 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4244#else 4245 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4246#endif 4247 /* 4248 * kp = &mseg_pa->kpm_pages[inx] 4249 */ 4250 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4251 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4252 add %g5, %g4, %g5 /* kp */ 4253 4254 /* 4255 * KPMP_HASH(kp) 4256 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4257 */ 4258 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4259 sub %g7, 1, %g7 /* mask */ 4260 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4261 add %g5, %g1, %g5 /* y = ksp + x */ 4262 and %g5, %g7, %g5 /* hashinx = y & mask */ 4263 4264 /* 4265 * Calculate physical kpm_page pointer 4266 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4267 */ 4268 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4269 add %g1, %g4, %g1 /* kp_pa */ 4270 4271 /* 4272 * Calculate physical hash lock address 4273 * g1=kp_refcntc_pa g2=pfn g5=hashinx 4274 */ 4275 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4276 sllx %g5, KPMHLK_SHIFT, %g5 4277 add %g4, %g5, %g3 4278 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4279 4280 /* 4281 * Assemble tte 4282 * g1=kp_pa g2=pfn g3=hlck_pa 4283 */ 4284#ifdef sun4v 4285 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4286 sllx %g5, 32, %g5 4287 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4288 or %g4, TTE4M, %g4 4289 or %g5, %g4, %g5 4290#else 4291 sethi %hi(TTE_VALID_INT), %g4 4292 mov TTE4M, %g5 4293 sllx %g5, TTE_SZ_SHFT_INT, %g5 4294 or %g5, %g4, %g5 /* upper part */ 4295 sllx %g5, 32, %g5 4296 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4297 or %g5, %g4, %g5 4298#endif 4299 sllx %g2, MMU_PAGESHIFT, %g4 4300 or %g5, %g4, %g5 /* tte */ 4301 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4302 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4303 4304 /* 4305 * tsb dropin 4306 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4307 */ 4308 4309 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4310 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4311 4312 /* use C-handler if there's no go for dropin */ 4313 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4314 cmp %g7, -1 4315 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4316 nop 4317 4318#ifdef DEBUG 4319 /* double check refcnt */ 4320 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4321 brz,pn %g7, 5f /* let C-handler deal with this */ 4322 nop 4323#endif 4324 4325#ifndef sun4v 4326 ldub [%g6 + KPMTSBM_FLAGS], %g7 4327 mov ASI_N, %g1 4328 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4329 movnz %icc, ASI_MEM, %g1 4330 mov %g1, %asi 4331#endif 4332 4333 /* 4334 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4335 * If we fail to lock the TSB entry then just load the tte into the 4336 * TLB. 4337 */ 4338 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l1) 4339 4340 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4341 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4342locked_tsb_l1: 4343 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4344 4345 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4346 KPMLOCK_EXIT(%g3, ASI_MEM) 4347 4348 /* 4349 * If trapstat is running, we need to shift the %tpc and %tnpc to 4350 * point to trapstat's TSB miss return code (note that trapstat 4351 * itself will patch the correct offset to add). 4352 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4353 */ 4354 rdpr %tl, %g7 4355 cmp %g7, 1 4356 ble %icc, 0f 4357 sethi %hi(KERNELBASE), %g6 4358 rdpr %tpc, %g7 4359 or %g6, %lo(KERNELBASE), %g6 4360 cmp %g7, %g6 4361 bgeu %xcc, 0f 4362 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4363 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4364 wrpr %g7, %tpc 4365 add %g7, 4, %g7 4366 wrpr %g7, %tnpc 43670: 4368 retry 43695: 4370 /* g3=hlck_pa */ 4371 KPMLOCK_EXIT(%g3, ASI_MEM) 4372 ba,pt %icc, sfmmu_kpm_exception 4373 nop 4374 SET_SIZE(sfmmu_kpm_dtsb_miss) 4375 4376 /* 4377 * kpm tsbmiss handler for smallpages 4378 * g1 = 8K kpm TSB pointer 4379 * g2 = tag access register 4380 * g3 = 4M kpm TSB pointer 4381 */ 4382 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4383 TT_TRACE(trace_tsbmiss) 4384 CPU_INDEX(%g7, %g6) 4385 sethi %hi(kpmtsbm_area), %g6 4386 sllx %g7, KPMTSBM_SHIFT, %g7 4387 or %g6, %lo(kpmtsbm_area), %g6 4388 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4389 4390 /* check enable flag */ 4391 ldub [%g6 + KPMTSBM_FLAGS], %g4 4392 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4393 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4394 nop 4395 4396 /* 4397 * VA range check 4398 * On fail: goto sfmmu_tsb_miss 4399 */ 4400 ldx [%g6 + KPMTSBM_VBASE], %g7 4401 cmp %g2, %g7 4402 blu,pn %xcc, sfmmu_tsb_miss 4403 ldx [%g6 + KPMTSBM_VEND], %g5 4404 cmp %g2, %g5 4405 bgeu,pn %xcc, sfmmu_tsb_miss 4406 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4407 4408 /* 4409 * check TL tsbmiss handling flag 4410 * bump tsbmiss counter 4411 */ 4412 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4413#ifdef DEBUG 4414 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4415 inc %g5 4416 brz,pn %g1, sfmmu_kpm_exception 4417 st %g5, [%g6 + KPMTSBM_TSBMISS] 4418#else 4419 inc %g5 4420 st %g5, [%g6 + KPMTSBM_TSBMISS] 4421#endif 4422 /* 4423 * At this point: 4424 * g1 = clobbered 4425 * g2 = tag access register 4426 * g3 = 4M kpm TSB pointer (not used) 4427 * g6 = per-CPU kpm tsbmiss area 4428 * g7 = kpm_vbase 4429 */ 4430 4431 /* 4432 * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr) 4433 * which is defined in mach_kpm.h. Any changes in that macro 4434 * should also be ported back to this assembly code. 4435 */ 4436 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 /* g3 = kpm_size_shift */ 4437 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4438 srax %g4, %g3, %g7 /* which alias range (r) */ 4439 brz,pt %g7, 2f 4440 sethi %hi(vac_colors_mask), %g5 4441 ld [%g5 + %lo(vac_colors_mask)], %g5 4442 4443 srlx %g2, MMU_PAGESHIFT, %g1 /* vaddr >> MMU_PAGESHIFT */ 4444 and %g1, %g5, %g1 /* g1 = v */ 4445 sllx %g7, %g3, %g5 /* g5 = r << kpm_size_shift */ 4446 cmp %g7, %g1 /* if (r > v) */ 4447 bleu,pn %xcc, 1f 4448 sub %g4, %g5, %g4 /* paddr -= r << kpm_size_shift */ 4449 sub %g7, %g1, %g5 /* g5 = r - v */ 4450 sllx %g5, MMU_PAGESHIFT, %g7 /* (r-v) << MMU_PAGESHIFT */ 4451 add %g4, %g7, %g4 /* paddr += (r-v)<<MMU_PAGESHIFT */ 4452 ba 2f 4453 nop 44541: 4455 sllx %g7, MMU_PAGESHIFT, %g5 /* else */ 4456 sub %g4, %g5, %g4 /* paddr -= r << MMU_PAGESHIFT */ 4457 4458 /* 4459 * paddr2pfn 4460 * g1 = vcolor (not used) 4461 * g2 = tag access register 4462 * g3 = clobbered 4463 * g4 = paddr 4464 * g5 = clobbered 4465 * g6 = per-CPU kpm tsbmiss area 4466 * g7 = clobbered 4467 */ 44682: 4469 srlx %g4, MMU_PAGESHIFT, %g2 /* g2 = pfn */ 4470 4471 /* 4472 * Setup %asi 4473 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4474 * if (mseg not found) sfmmu_kpm_exception 4475 * g2=pfn g6=per-CPU kpm tsbmiss area 4476 * g4 g5 g7 for scratch use. 4477 */ 4478 mov ASI_MEM, %asi 4479 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4480 cmp %g3, MSEG_NULLPTR_PA 4481 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4482 nop 4483 4484 /* 4485 * inx = pfn - mseg_pa->kpm_pbase 4486 * g2=pfn g3=mseg_pa g6=per-CPU kpm tsbmiss area 4487 */ 4488 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4489 sub %g2, %g7, %g4 4490 4491#ifdef DEBUG 4492 /* 4493 * Validate inx value 4494 * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area 4495 */ 4496 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4497 cmp %g4, %g5 /* inx - nkpmpgs */ 4498 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4499 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4500#else 4501 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4502#endif 4503 /* ksp = &mseg_pa->kpm_spages[inx] */ 4504 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4505 add %g5, %g4, %g5 /* ksp */ 4506 4507 /* 4508 * KPMP_SHASH(kp) 4509 * g2=pfn g3=mseg_pa g4=inx g5=ksp 4510 * g6=per-CPU kpm tsbmiss area g7=kpmp_stable_sz 4511 */ 4512 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4513 sub %g7, 1, %g7 /* mask */ 4514 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4515 add %g5, %g1, %g5 /* y = ksp + x */ 4516 and %g5, %g7, %g5 /* hashinx = y & mask */ 4517 4518 /* 4519 * Calculate physical kpm_spage pointer 4520 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4521 * g6=per-CPU kpm tsbmiss area 4522 */ 4523 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4524 add %g1, %g4, %g1 /* ksp_pa */ 4525 4526 /* 4527 * Calculate physical hash lock address. 4528 * Note: Changes in kpm_shlk_t must be reflected here. 4529 * g1=ksp_pa g2=pfn g5=hashinx 4530 * g6=per-CPU kpm tsbmiss area 4531 */ 4532 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4533 sllx %g5, KPMSHLK_SHIFT, %g5 4534 add %g4, %g5, %g3 /* hlck_pa */ 4535 4536 /* 4537 * Assemble non-cacheable tte initially 4538 * g1=ksp_pa g2=pfn g3=hlck_pa 4539 * g6=per-CPU kpm tsbmiss area 4540 */ 4541 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4542 sllx %g5, 32, %g5 4543 mov (TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4544 or %g5, %g4, %g5 4545 sllx %g2, MMU_PAGESHIFT, %g4 4546 or %g5, %g4, %g5 /* tte */ 4547 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4548 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4549 4550 /* 4551 * tsb dropin 4552 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable) 4553 * g6=per-CPU kpm tsbmiss area g7=scratch register 4554 */ 4555 4556 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4557 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4558 4559 /* use C-handler if there's no go for dropin */ 4560 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4561 andcc %g7, KPM_MAPPED_GO, %g0 /* go or no go ? */ 4562 bz,pt %icc, 5f /* no go */ 4563 nop 4564 and %g7, KPM_MAPPED_MASK, %g7 /* go */ 4565 cmp %g7, KPM_MAPPEDS /* cacheable ? */ 4566 be,a,pn %xcc, 3f 4567 or %g5, TTE_CV_INT, %g5 /* cacheable */ 45683: 4569#ifndef sun4v 4570 ldub [%g6 + KPMTSBM_FLAGS], %g7 4571 mov ASI_N, %g1 4572 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4573 movnz %icc, ASI_MEM, %g1 4574 mov %g1, %asi 4575#endif 4576 4577 /* 4578 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4579 * If we fail to lock the TSB entry then just load the tte into the 4580 * TLB. 4581 */ 4582 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l2) 4583 4584 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4585 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4586locked_tsb_l2: 4587 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4588 4589 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4590 KPMLOCK_EXIT(%g3, ASI_MEM) 4591 4592 /* 4593 * If trapstat is running, we need to shift the %tpc and %tnpc to 4594 * point to trapstat's TSB miss return code (note that trapstat 4595 * itself will patch the correct offset to add). 4596 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4597 */ 4598 rdpr %tl, %g7 4599 cmp %g7, 1 4600 ble %icc, 0f 4601 sethi %hi(KERNELBASE), %g6 4602 rdpr %tpc, %g7 4603 or %g6, %lo(KERNELBASE), %g6 4604 cmp %g7, %g6 4605 bgeu %xcc, 0f 4606 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4607 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4608 wrpr %g7, %tpc 4609 add %g7, 4, %g7 4610 wrpr %g7, %tnpc 46110: 4612 retry 46135: 4614 /* g3=hlck_pa */ 4615 KPMLOCK_EXIT(%g3, ASI_MEM) 4616 ba,pt %icc, sfmmu_kpm_exception 4617 nop 4618 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4619 4620#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4621#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4622#endif 4623 4624#endif /* lint */ 4625 4626#ifdef lint 4627/* 4628 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4629 * Called from C-level, sets/clears "go" indication for trap level handler. 4630 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 4631 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 4632 * Assumes khl_mutex is held when called from C-level. 4633 */ 4634/* ARGSUSED */ 4635void 4636sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 4637{ 4638} 4639 4640/* 4641 * kpm_smallpages: stores val to byte at address mapped within 4642 * low level lock brackets. The old value is returned. 4643 * Called from C-level. 4644 */ 4645/* ARGSUSED */ 4646int 4647sfmmu_kpm_stsbmtl(uchar_t *mapped, uint_t *kshl_lock, int val) 4648{ 4649 return (0); 4650} 4651 4652#else /* lint */ 4653 4654 .seg ".data" 4655sfmmu_kpm_tsbmtl_panic: 4656 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4657 .byte 0 4658sfmmu_kpm_stsbmtl_panic: 4659 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4660 .byte 0 4661 .align 4 4662 .seg ".text" 4663 4664 ENTRY_NP(sfmmu_kpm_tsbmtl) 4665 rdpr %pstate, %o3 4666 /* 4667 * %o0 = &kp_refcntc 4668 * %o1 = &khl_lock 4669 * %o2 = 0/1 (off/on) 4670 * %o3 = pstate save 4671 */ 4672#ifdef DEBUG 4673 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4674 bnz,pt %icc, 1f /* disabled, panic */ 4675 nop 4676 save %sp, -SA(MINFRAME), %sp 4677 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4678 call panic 4679 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4680 ret 4681 restore 46821: 4683#endif /* DEBUG */ 4684 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4685 4686 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4687 mov -1, %o5 4688 brz,a %o2, 2f 4689 mov 0, %o5 46902: 4691 sth %o5, [%o0] 4692 KPMLOCK_EXIT(%o1, ASI_N) 4693 4694 retl 4695 wrpr %g0, %o3, %pstate /* enable interrupts */ 4696 SET_SIZE(sfmmu_kpm_tsbmtl) 4697 4698 ENTRY_NP(sfmmu_kpm_stsbmtl) 4699 rdpr %pstate, %o3 4700 /* 4701 * %o0 = &mapped 4702 * %o1 = &kshl_lock 4703 * %o2 = val 4704 * %o3 = pstate save 4705 */ 4706#ifdef DEBUG 4707 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4708 bnz,pt %icc, 1f /* disabled, panic */ 4709 nop 4710 save %sp, -SA(MINFRAME), %sp 4711 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4712 call panic 4713 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4714 ret 4715 restore 47161: 4717#endif /* DEBUG */ 4718 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4719 4720 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4721 ldsb [%o0], %o5 4722 stb %o2, [%o0] 4723 KPMLOCK_EXIT(%o1, ASI_N) 4724 4725 and %o5, KPM_MAPPED_MASK, %o0 /* return old val */ 4726 retl 4727 wrpr %g0, %o3, %pstate /* enable interrupts */ 4728 SET_SIZE(sfmmu_kpm_stsbmtl) 4729 4730#endif /* lint */ 4731 4732#ifndef lint 4733#ifdef sun4v 4734 /* 4735 * User/kernel data miss w// multiple TSBs 4736 * The first probe covers 8K, 64K, and 512K page sizes, 4737 * because 64K and 512K mappings are replicated off 8K 4738 * pointer. Second probe covers 4M page size only. 4739 * 4740 * MMU fault area contains miss address and context. 4741 */ 4742 ALTENTRY(sfmmu_slow_dmmu_miss) 4743 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 4744 4745slow_miss_common: 4746 /* 4747 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 4748 * %g3 = ctx (cannot be INVALID_CONTEXT) 4749 */ 4750 brnz,pt %g3, 8f ! check for user context 4751 nop 4752 4753 /* 4754 * Kernel miss 4755 * Get 8K and 4M TSB pointers in %g1 and %g3 and 4756 * branch to sfmmu_tsb_miss_tt to handle it. 4757 */ 4758 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4759sfmmu_dslow_patch_ktsb_base: 4760 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 4761sfmmu_dslow_patch_ktsb_szcode: 4762 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 4763 4764 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 4765 ! %g1 = First TSB entry pointer, as TSB miss handler expects 4766 4767 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4768sfmmu_dslow_patch_ktsb4m_base: 4769 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 4770sfmmu_dslow_patch_ktsb4m_szcode: 4771 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 4772 4773 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 4774 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 4775 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4776 .empty 4777 47788: 4779 /* 4780 * User miss 4781 * Get first TSB pointer in %g1 4782 * Get second TSB pointer (or NULL if no second TSB) in %g3 4783 * Branch to sfmmu_tsb_miss_tt to handle it 4784 */ 4785 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 4786 /* %g1 = first TSB entry ptr now, %g2 preserved */ 4787 4788 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 4789 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 4790 nop 4791 4792 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 4793 /* %g3 = second TSB entry ptr now, %g2 preserved */ 47949: 4795 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4796 .empty 4797 SET_SIZE(sfmmu_slow_dmmu_miss) 4798 4799 4800 /* 4801 * User/kernel instruction miss w/ multiple TSBs 4802 * The first probe covers 8K, 64K, and 512K page sizes, 4803 * because 64K and 512K mappings are replicated off 8K 4804 * pointer. Second probe covers 4M page size only. 4805 * 4806 * MMU fault area contains miss address and context. 4807 */ 4808 ALTENTRY(sfmmu_slow_immu_miss) 4809 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 4810 ba,a,pt %xcc, slow_miss_common 4811 SET_SIZE(sfmmu_slow_immu_miss) 4812 4813#endif /* sun4v */ 4814#endif /* lint */ 4815 4816#ifndef lint 4817 4818/* 4819 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 4820 */ 4821 .seg ".data" 4822 .align 64 4823 .global tsbmiss_area 4824tsbmiss_area: 4825 .skip (TSBMISS_SIZE * NCPU) 4826 4827 .align 64 4828 .global kpmtsbm_area 4829kpmtsbm_area: 4830 .skip (KPMTSBM_SIZE * NCPU) 4831#endif /* lint */ 4832