1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26/* 27 * SFMMU primitives. These primitives should only be used by sfmmu 28 * routines. 29 */ 30 31#if defined(lint) 32#include <sys/types.h> 33#else /* lint */ 34#include "assym.h" 35#endif /* lint */ 36 37#include <sys/asm_linkage.h> 38#include <sys/machtrap.h> 39#include <sys/machasi.h> 40#include <sys/sun4asi.h> 41#include <sys/pte.h> 42#include <sys/mmu.h> 43#include <vm/hat_sfmmu.h> 44#include <vm/seg_spt.h> 45#include <sys/machparam.h> 46#include <sys/privregs.h> 47#include <sys/scb.h> 48#include <sys/intreg.h> 49#include <sys/machthread.h> 50#include <sys/intr.h> 51#include <sys/clock.h> 52#include <sys/trapstat.h> 53 54#ifdef TRAPTRACE 55#include <sys/traptrace.h> 56 57/* 58 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 59 */ 60#define TT_TRACE(label) \ 61 ba label ;\ 62 rd %pc, %g7 63#else 64 65#define TT_TRACE(label) 66 67#endif /* TRAPTRACE */ 68 69#ifndef lint 70 71#if (TTE_SUSPEND_SHIFT > 0) 72#define TTE_SUSPEND_INT_SHIFT(reg) \ 73 sllx reg, TTE_SUSPEND_SHIFT, reg 74#else 75#define TTE_SUSPEND_INT_SHIFT(reg) 76#endif 77 78#endif /* lint */ 79 80#ifndef lint 81 82/* 83 * Assumes TSBE_TAG is 0 84 * Assumes TSBE_INTHI is 0 85 * Assumes TSBREG.split is 0 86 */ 87 88#if TSBE_TAG != 0 89#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 90#endif 91 92#if TSBTAG_INTHI != 0 93#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 94#endif 95 96/* 97 * The following code assumes the tsb is not split. 98 * 99 * With TSBs no longer shared between processes, it's no longer 100 * necessary to hash the context bits into the tsb index to get 101 * tsb coloring; the new implementation treats the TSB as a 102 * direct-mapped, virtually-addressed cache. 103 * 104 * In: 105 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 106 * tsbbase = base address of TSB (clobbered) 107 * tagacc = tag access register (clobbered) 108 * szc = size code of TSB (ro) 109 * tmp = scratch reg 110 * Out: 111 * tsbbase = pointer to entry in TSB 112 */ 113#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 114 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 115 srlx tagacc, vpshift, tagacc ;\ 116 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 117 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 118 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 119 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 120 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 121 122/* 123 * When the kpm TSB is used it is assumed that it is direct mapped 124 * using (vaddr>>vpshift)%tsbsz as the index. 125 * 126 * Note that, for now, the kpm TSB and kernel TSB are the same for 127 * each mapping size. However that need not always be the case. If 128 * the trap handlers are updated to search a different TSB for kpm 129 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 130 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 131 * 132 * In: 133 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 134 * vaddr = virtual address (clobbered) 135 * tsbp, szc, tmp = scratch 136 * Out: 137 * tsbp = pointer to entry in TSB 138 */ 139#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 140 cmp vpshift, MMU_PAGESHIFT ;\ 141 bne,pn %icc, 1f /* branch if large case */ ;\ 142 sethi %hi(kpmsm_tsbsz), szc ;\ 143 sethi %hi(kpmsm_tsbbase), tsbp ;\ 144 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 145 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 146 ba,pt %icc, 2f ;\ 147 nop ;\ 1481: sethi %hi(kpm_tsbsz), szc ;\ 149 sethi %hi(kpm_tsbbase), tsbp ;\ 150 ld [szc + %lo(kpm_tsbsz)], szc ;\ 151 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1522: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 153 154/* 155 * Lock the TSBE at virtual address tsbep. 156 * 157 * tsbep = TSBE va (ro) 158 * tmp1, tmp2 = scratch registers (clobbered) 159 * label = label to jump to if we fail to lock the tsb entry 160 * %asi = ASI to use for TSB access 161 * 162 * NOTE that we flush the TSB using fast VIS instructions that 163 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 164 * not be treated as a locked entry or we'll get stuck spinning on 165 * an entry that isn't locked but really invalid. 166 */ 167 168#if defined(UTSB_PHYS) 169 170#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 171 lda [tsbep]ASI_MEM, tmp1 ;\ 172 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 173 cmp tmp1, tmp2 ;\ 174 be,a,pn %icc, label /* if locked ignore */ ;\ 175 nop ;\ 176 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 177 cmp tmp1, tmp2 ;\ 178 bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 179 nop ;\ 180 /* tsbe lock acquired */ ;\ 181 membar #StoreStore 182 183#else /* UTSB_PHYS */ 184 185#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 186 lda [tsbep]%asi, tmp1 ;\ 187 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 188 cmp tmp1, tmp2 ;\ 189 be,a,pn %icc, label /* if locked ignore */ ;\ 190 nop ;\ 191 casa [tsbep]%asi, tmp1, tmp2 ;\ 192 cmp tmp1, tmp2 ;\ 193 bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 194 nop ;\ 195 /* tsbe lock acquired */ ;\ 196 membar #StoreStore 197 198#endif /* UTSB_PHYS */ 199 200/* 201 * Atomically write TSBE at virtual address tsbep. 202 * 203 * tsbep = TSBE va (ro) 204 * tte = TSBE TTE (ro) 205 * tagtarget = TSBE tag (ro) 206 * %asi = ASI to use for TSB access 207 */ 208 209#if defined(UTSB_PHYS) 210 211#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 212 add tsbep, TSBE_TTE, tmp1 ;\ 213 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 214 membar #StoreStore ;\ 215 add tsbep, TSBE_TAG, tmp1 ;\ 216 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 217 218#else /* UTSB_PHYS */ 219 220#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 221 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 222 membar #StoreStore ;\ 223 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 224 225#endif /* UTSB_PHYS */ 226 227/* 228 * Load an entry into the TSB at TL > 0. 229 * 230 * tsbep = pointer to the TSBE to load as va (ro) 231 * tte = value of the TTE retrieved and loaded (wo) 232 * tagtarget = tag target register. To get TSBE tag to load, 233 * we need to mask off the context and leave only the va (clobbered) 234 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 235 * tmp1, tmp2 = scratch registers 236 * label = label to jump to if we fail to lock the tsb entry 237 * %asi = ASI to use for TSB access 238 */ 239 240#if defined(UTSB_PHYS) 241 242#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 243 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 244 /* ;\ 245 * I don't need to update the TSB then check for the valid tte. ;\ 246 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 247 * we always invalidate the hash table before we unload the TSB.;\ 248 */ ;\ 249 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 250 ldxa [ttepa]ASI_MEM, tte ;\ 251 TTE_CLR_SOFTEXEC_ML(tte) ;\ 252 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 253 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 254 add tsbep, TSBE_TAG, tmp1 ;\ 255 brgez,a,pn tte, label ;\ 256 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 257 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 258label: 259 260#else /* UTSB_PHYS */ 261 262#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 263 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 264 /* ;\ 265 * I don't need to update the TSB then check for the valid tte. ;\ 266 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 267 * we always invalidate the hash table before we unload the TSB.;\ 268 */ ;\ 269 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 270 ldxa [ttepa]ASI_MEM, tte ;\ 271 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 272 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 273 brgez,a,pn tte, label ;\ 274 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 275 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 276label: 277 278#endif /* UTSB_PHYS */ 279 280/* 281 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 282 * for ITLB synthesis. 283 * 284 * tsbep = pointer to the TSBE to load as va (ro) 285 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 286 * with exec_perm turned off and exec_synth turned on 287 * tagtarget = tag target register. To get TSBE tag to load, 288 * we need to mask off the context and leave only the va (clobbered) 289 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 290 * tmp1, tmp2 = scratch registers 291 * label = label to use for branch (text) 292 * %asi = ASI to use for TSB access 293 */ 294 295#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 296 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 297 /* ;\ 298 * I don't need to update the TSB then check for the valid tte. ;\ 299 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 300 * we always invalidate the hash table before we unload the TSB.;\ 301 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 302 * and exec_synth bit to 1. ;\ 303 */ ;\ 304 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 305 mov tte, tmp1 ;\ 306 ldxa [ttepa]ASI_MEM, tte ;\ 307 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 308 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 309 brgez,a,pn tte, label ;\ 310 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 311 or tte, tmp1, tte ;\ 312 andn tte, TTE_EXECPRM_INT, tte ;\ 313 or tte, TTE_E_SYNTH_INT, tte ;\ 314 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 315label: 316 317/* 318 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 319 * 320 * tte = value of the TTE, used to get tte_size bits (ro) 321 * tagaccess = tag access register, used to get 4M pfn bits (ro) 322 * pfn = 4M pfn bits shifted to offset for tte (out) 323 * tmp1 = scratch register 324 * label = label to use for branch (text) 325 */ 326 327#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 328 /* ;\ 329 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 330 * Return them, shifted, in pfn. ;\ 331 */ ;\ 332 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 333 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 334 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 335 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 336 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 337 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 338label: ;\ 339 sllx tagaccess, MMU_PAGESHIFT4M, pfn 340 341/* 342 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 343 * for ITLB synthesis. 344 * 345 * tte = value of the TTE, used to get tte_size bits (rw) 346 * tmp1 = scratch register 347 */ 348 349#define SET_TTE4M_PN(tte, tmp) \ 350 /* ;\ 351 * Set 4M pagesize tte bits. ;\ 352 */ ;\ 353 set TTE4M, tmp ;\ 354 sllx tmp, TTE_SZ_SHFT, tmp ;\ 355 or tte, tmp, tte 356 357/* 358 * Load an entry into the TSB at TL=0. 359 * 360 * tsbep = pointer to the TSBE to load as va (ro) 361 * tteva = pointer to the TTE to load as va (ro) 362 * tagtarget = TSBE tag to load (which contains no context), synthesized 363 * to match va of MMU tag target register only (ro) 364 * tmp1, tmp2 = scratch registers (clobbered) 365 * label = label to use for branches (text) 366 * %asi = ASI to use for TSB access 367 */ 368 369#if defined(UTSB_PHYS) 370 371#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 372 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 373 ldx [tteva], tteva /* load tte */ ;\ 374 TTE_CLR_SOFTEXEC_ML(tteva) ;\ 375 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 376 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 377 add tsbep, TSBE_TAG, tmp1 ;\ 378 brgez,a,pn tteva, label ;\ 379 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 380 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 381label: 382 383#else /* UTSB_PHYS */ 384 385#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 386 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 387 ldx [tteva], tteva /* load tte */ ;\ 388 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 389 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 390 brgez,a,pn tteva, label ;\ 391 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 392 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 393label: 394 395#endif /* UTSB_PHYS */ 396 397/* 398 * Invalidate a TSB entry in the TSB. 399 * 400 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 401 * about this earlier to ensure this is true. Thus when we are 402 * directly referencing tsbep below, we are referencing the tte_tag 403 * field of the TSBE. If this offset ever changes, the code below 404 * will need to be modified. 405 * 406 * tsbep = pointer to TSBE as va (ro) 407 * tag = invalidation is done if this matches the TSBE tag (ro) 408 * tmp1 - tmp3 = scratch registers (clobbered) 409 * label = label name to use for branches (text) 410 * %asi = ASI to use for TSB access 411 */ 412 413#if defined(UTSB_PHYS) 414 415#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 416 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 417 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 418label/**/1: ;\ 419 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 420 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 421 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 422 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 423 cmp tag, tmp3 /* compare tags */ ;\ 424 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 425 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 426 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 427 cmp tmp1, tmp3 /* if not successful */ ;\ 428 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 429 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 430label/**/2: 431 432#else /* UTSB_PHYS */ 433 434#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 435 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 436 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 437label/**/1: ;\ 438 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 439 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 440 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 441 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 442 cmp tag, tmp3 /* compare tags */ ;\ 443 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 444 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 445 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 446 cmp tmp1, tmp3 /* if not successful */ ;\ 447 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 448 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 449label/**/2: 450 451#endif /* UTSB_PHYS */ 452 453#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 454#error - TSB_SOFTSZ_MASK too small 455#endif 456 457 458/* 459 * An implementation of setx which will be hot patched at run time. 460 * since it is being hot patched, there is no value passed in. 461 * Thus, essentially we are implementing 462 * setx value, tmp, dest 463 * where value is RUNTIME_PATCH (aka 0) in this case. 464 */ 465#define RUNTIME_PATCH_SETX(dest, tmp) \ 466 sethi %hh(RUNTIME_PATCH), tmp ;\ 467 sethi %lm(RUNTIME_PATCH), dest ;\ 468 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 469 or dest, %lo(RUNTIME_PATCH), dest ;\ 470 sllx tmp, 32, tmp ;\ 471 nop /* for perf reasons */ ;\ 472 or tmp, dest, dest /* contents of patched value */ 473 474#endif /* lint */ 475 476 477#if defined (lint) 478 479/* 480 * sfmmu related subroutines 481 */ 482uint_t 483sfmmu_disable_intrs() 484{ return(0); } 485 486/* ARGSUSED */ 487void 488sfmmu_enable_intrs(uint_t pstate_save) 489{} 490 491/* ARGSUSED */ 492int 493sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag) 494{ return(0); } 495 496/* 497 * Use cas, if tte has changed underneath us then reread and try again. 498 * In the case of a retry, it will update sttep with the new original. 499 */ 500/* ARGSUSED */ 501int 502sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 503{ return(0); } 504 505/* 506 * Use cas, if tte has changed underneath us then return 1, else return 0 507 */ 508/* ARGSUSED */ 509int 510sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 511{ return(0); } 512 513/* ARGSUSED */ 514void 515sfmmu_copytte(tte_t *sttep, tte_t *dttep) 516{} 517 518/*ARGSUSED*/ 519struct tsbe * 520sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 521{ return(0); } 522 523/*ARGSUSED*/ 524uint64_t 525sfmmu_make_tsbtag(caddr_t va) 526{ return(0); } 527 528#else /* lint */ 529 530 .seg ".data" 531 .global sfmmu_panic1 532sfmmu_panic1: 533 .asciz "sfmmu_asm: interrupts already disabled" 534 535 .global sfmmu_panic3 536sfmmu_panic3: 537 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 538 539 .global sfmmu_panic4 540sfmmu_panic4: 541 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 542 543 .global sfmmu_panic5 544sfmmu_panic5: 545 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 546 547 .global sfmmu_panic6 548sfmmu_panic6: 549 .asciz "sfmmu_asm: interrupts not disabled" 550 551 .global sfmmu_panic7 552sfmmu_panic7: 553 .asciz "sfmmu_asm: kernel as" 554 555 .global sfmmu_panic8 556sfmmu_panic8: 557 .asciz "sfmmu_asm: gnum is zero" 558 559 .global sfmmu_panic9 560sfmmu_panic9: 561 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 562 563 .global sfmmu_panic10 564sfmmu_panic10: 565 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 566 567 .global sfmmu_panic11 568sfmmu_panic11: 569 .asciz "sfmmu_asm: ktsb_phys must not be 0 on a sun4v platform" 570 571 ENTRY(sfmmu_disable_intrs) 572 rdpr %pstate, %o0 573#ifdef DEBUG 574 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 575#endif /* DEBUG */ 576 retl 577 wrpr %o0, PSTATE_IE, %pstate 578 SET_SIZE(sfmmu_disable_intrs) 579 580 ENTRY(sfmmu_enable_intrs) 581 retl 582 wrpr %g0, %o0, %pstate 583 SET_SIZE(sfmmu_enable_intrs) 584 585/* 586 * This routine is called both by resume() and sfmmu_get_ctx() to 587 * allocate a new context for the process on a MMU. 588 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 589 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 590 * is the case when sfmmu_alloc_ctx is called from resume(). 591 * 592 * The caller must disable interrupts before entering this routine. 593 * To reduce ctx switch overhead, the code contains both 'fast path' and 594 * 'slow path' code. The fast path code covers the common case where only 595 * a quick check is needed and the real ctx allocation is not required. 596 * It can be done without holding the per-process (PP) lock. 597 * The 'slow path' code must be protected by the PP Lock and performs ctx 598 * allocation. 599 * Hardware context register and HAT mmu cnum are updated accordingly. 600 * 601 * %o0 - sfmmup 602 * %o1 - allocflag 603 * %o2 - CPU 604 * %o3 - sfmmu private/shared flag 605 * 606 * ret - 0: no ctx is allocated 607 * 1: a ctx is allocated 608 */ 609 ENTRY_NP(sfmmu_alloc_ctx) 610 611#ifdef DEBUG 612 sethi %hi(ksfmmup), %g1 613 ldx [%g1 + %lo(ksfmmup)], %g1 614 cmp %g1, %o0 615 bne,pt %xcc, 0f 616 nop 617 618 sethi %hi(panicstr), %g1 ! if kernel as, panic 619 ldx [%g1 + %lo(panicstr)], %g1 620 tst %g1 621 bnz,pn %icc, 7f 622 nop 623 624 sethi %hi(sfmmu_panic7), %o0 625 call panic 626 or %o0, %lo(sfmmu_panic7), %o0 627 6287: 629 retl 630 mov %g0, %o0 ! %o0 = ret = 0 631 6320: 633 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 634#endif /* DEBUG */ 635 636 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 637 638 ! load global mmu_ctxp info 639 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 640 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 641 642 ! load global mmu_ctxp gnum 643 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 644 645#ifdef DEBUG 646 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 647 bne,pt %xcc, 3f 648 nop 649 650 sethi %hi(panicstr), %g1 ! test if panicstr is already set 651 ldx [%g1 + %lo(panicstr)], %g1 652 tst %g1 653 bnz,pn %icc, 1f 654 nop 655 656 sethi %hi(sfmmu_panic8), %o0 657 call panic 658 or %o0, %lo(sfmmu_panic8), %o0 6591: 660 retl 661 mov %g0, %o0 ! %o0 = ret = 0 6623: 663#endif 664 665 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 666 667 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 668 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 669 670 /* 671 * %g5 = sfmmu gnum returned 672 * %g6 = sfmmu cnum returned 673 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 674 * %g4 = scratch 675 * 676 * Fast path code, do a quick check. 677 */ 678 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 679 680 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 681 bne,pt %icc, 1f ! valid hat cnum, check gnum 682 nop 683 684 ! cnum == INVALID, check allocflag 685 mov %g0, %g4 ! %g4 = ret = 0 686 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 687 mov %g6, %o1 688 689 ! (invalid HAT cnum) && (allocflag == 1) 690 ba,pt %icc, 2f 691 nop 6921: 693 ! valid HAT cnum, check gnum 694 cmp %g5, %o4 695 mov 1, %g4 !%g4 = ret = 1 696 be,a,pt %icc, 8f ! gnum unchanged, go to done 697 mov %g6, %o1 698 6992: 700 /* 701 * Grab per process (PP) sfmmu_ctx_lock spinlock, 702 * followed by the 'slow path' code. 703 */ 704 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 7053: 706 brz %g3, 5f 707 nop 7084: 709 brnz,a,pt %g3, 4b ! spin if lock is 1 710 ldub [%o0 + SFMMU_CTX_LOCK], %g3 711 ba %xcc, 3b ! retry the lock 712 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 713 7145: 715 membar #LoadLoad 716 /* 717 * %g5 = sfmmu gnum returned 718 * %g6 = sfmmu cnum returned 719 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 720 * %g4 = scratch 721 */ 722 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 723 724 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 725 bne,pt %icc, 1f ! valid hat cnum, check gnum 726 nop 727 728 ! cnum == INVALID, check allocflag 729 mov %g0, %g4 ! %g4 = ret = 0 730 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 731 mov %g6, %o1 732 733 ! (invalid HAT cnum) && (allocflag == 1) 734 ba,pt %icc, 6f 735 nop 7361: 737 ! valid HAT cnum, check gnum 738 cmp %g5, %o4 739 mov 1, %g4 ! %g4 = ret = 1 740 be,a,pt %icc, 2f ! gnum unchanged, go to done 741 mov %g6, %o1 742 743 ba,pt %icc, 6f 744 nop 7452: 746 membar #LoadStore|#StoreStore 747 ba,pt %icc, 8f 748 clrb [%o0 + SFMMU_CTX_LOCK] 7496: 750 /* 751 * We get here if we do not have a valid context, or 752 * the HAT gnum does not match global gnum. We hold 753 * sfmmu_ctx_lock spinlock. Allocate that context. 754 * 755 * %o3 = mmu_ctxp 756 */ 757 add %o3, MMU_CTX_CNUM, %g3 758 ld [%o3 + MMU_CTX_NCTXS], %g4 759 760 /* 761 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 762 * %g3 = mmu cnum address 763 * %g4 = mmu nctxs 764 * 765 * %o0 = sfmmup 766 * %o1 = mmu current cnum value (used as new cnum) 767 * %o4 = mmu gnum 768 * 769 * %o5 = scratch 770 */ 771 ld [%g3], %o1 7720: 773 cmp %o1, %g4 774 bl,a,pt %icc, 1f 775 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 776 777 /* 778 * cnum reachs max, bail, so wrap around can be performed later. 779 */ 780 set INVALID_CONTEXT, %o1 781 mov %g0, %g4 ! %g4 = ret = 0 782 783 membar #LoadStore|#StoreStore 784 ba,pt %icc, 8f 785 clrb [%o0 + SFMMU_CTX_LOCK] 7861: 787 ! %g3 = addr of mmu_ctxp->cnum 788 ! %o5 = mmu_ctxp->cnum + 1 789 cas [%g3], %o1, %o5 790 cmp %o1, %o5 791 bne,a,pn %xcc, 0b ! cas failed 792 ld [%g3], %o1 793 794#ifdef DEBUG 795 set MAX_SFMMU_CTX_VAL, %o5 796 cmp %o1, %o5 797 ble,pt %icc, 2f 798 nop 799 800 sethi %hi(sfmmu_panic9), %o0 801 call panic 802 or %o0, %lo(sfmmu_panic9), %o0 8032: 804#endif 805 ! update hat gnum and cnum 806 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 807 or %o4, %o1, %o4 808 stx %o4, [%g2 + SFMMU_CTXS] 809 810 membar #LoadStore|#StoreStore 811 clrb [%o0 + SFMMU_CTX_LOCK] 812 813 mov 1, %g4 ! %g4 = ret = 1 8148: 815 /* 816 * program the secondary context register 817 * 818 * %o1 = cnum 819 * %g1 = sfmmu private/shared flag (0:private, 1:shared) 820 */ 821 822 /* 823 * When we come here and context is invalid, we want to set both 824 * private and shared ctx regs to INVALID. In order to 825 * do so, we set the sfmmu priv/shared flag to 'private' regardless 826 * so that private ctx reg will be set to invalid. 827 * Note that on sun4v values written to private context register are 828 * automatically written to corresponding shared context register as 829 * well. On sun4u SET_SECCTX() will invalidate shared context register 830 * when it sets a private secondary context register. 831 */ 832 833 cmp %o1, INVALID_CONTEXT 834 be,a,pn %icc, 9f 835 clr %g1 8369: 837 838#ifdef sun4u 839 ldub [%o0 + SFMMU_CEXT], %o2 840 sll %o2, CTXREG_EXT_SHIFT, %o2 841 or %o1, %o2, %o1 842#endif /* sun4u */ 843 844 SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1) 845 846 retl 847 mov %g4, %o0 ! %o0 = ret 848 849 SET_SIZE(sfmmu_alloc_ctx) 850 851 ENTRY_NP(sfmmu_modifytte) 852 ldx [%o2], %g3 /* current */ 853 ldx [%o0], %g1 /* original */ 8542: 855 ldx [%o1], %g2 /* modified */ 856 cmp %g2, %g3 /* is modified = current? */ 857 be,a,pt %xcc,1f /* yes, don't write */ 858 stx %g3, [%o0] /* update new original */ 859 casx [%o2], %g1, %g2 860 cmp %g1, %g2 861 be,pt %xcc, 1f /* cas succeeded - return */ 862 nop 863 ldx [%o2], %g3 /* new current */ 864 stx %g3, [%o0] /* save as new original */ 865 ba,pt %xcc, 2b 866 mov %g3, %g1 8671: retl 868 membar #StoreLoad 869 SET_SIZE(sfmmu_modifytte) 870 871 ENTRY_NP(sfmmu_modifytte_try) 872 ldx [%o1], %g2 /* modified */ 873 ldx [%o2], %g3 /* current */ 874 ldx [%o0], %g1 /* original */ 875 cmp %g3, %g2 /* is modified = current? */ 876 be,a,pn %xcc,1f /* yes, don't write */ 877 mov 0, %o1 /* as if cas failed. */ 878 879 casx [%o2], %g1, %g2 880 membar #StoreLoad 881 cmp %g1, %g2 882 movne %xcc, -1, %o1 /* cas failed. */ 883 move %xcc, 1, %o1 /* cas succeeded. */ 8841: 885 stx %g2, [%o0] /* report "current" value */ 886 retl 887 mov %o1, %o0 888 SET_SIZE(sfmmu_modifytte_try) 889 890 ENTRY_NP(sfmmu_copytte) 891 ldx [%o0], %g1 892 retl 893 stx %g1, [%o1] 894 SET_SIZE(sfmmu_copytte) 895 896 897 /* 898 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 899 * %o0 = TSB base address (in), pointer to TSB entry (out) 900 * %o1 = vaddr (in) 901 * %o2 = vpshift (in) 902 * %o3 = tsb size code (in) 903 * %o4 = scratch register 904 */ 905 ENTRY_NP(sfmmu_get_tsbe) 906 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 907 retl 908 nop 909 SET_SIZE(sfmmu_get_tsbe) 910 911 /* 912 * Return a TSB tag for the given va. 913 * %o0 = va (in/clobbered) 914 * %o0 = va shifted to be in tsb tag format (with no context) (out) 915 */ 916 ENTRY_NP(sfmmu_make_tsbtag) 917 retl 918 srln %o0, TTARGET_VA_SHIFT, %o0 919 SET_SIZE(sfmmu_make_tsbtag) 920 921#endif /* lint */ 922 923/* 924 * Other sfmmu primitives 925 */ 926 927 928#if defined (lint) 929void 930sfmmu_patch_ktsb(void) 931{ 932} 933 934void 935sfmmu_kpm_patch_tlbm(void) 936{ 937} 938 939void 940sfmmu_kpm_patch_tsbm(void) 941{ 942} 943 944void 945sfmmu_patch_shctx(void) 946{ 947} 948 949void 950sfmmu_patch_pgsz_reg(void) 951{ 952} 953 954/* ARGSUSED */ 955void 956sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 957{ 958} 959 960/* ARGSUSED */ 961void 962sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 963{ 964} 965 966/* ARGSUSED */ 967void 968sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 969{ 970} 971 972/* ARGSUSED */ 973void 974sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 975{ 976} 977 978#else /* lint */ 979 980#define I_SIZE 4 981 982 ENTRY_NP(sfmmu_fix_ktlb_traptable) 983 /* 984 * %o0 = start of patch area 985 * %o1 = size code of TSB to patch 986 * %o3 = scratch 987 */ 988 /* fix sll */ 989 ld [%o0], %o3 /* get sll */ 990 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 991 st %o3, [%o0] /* write sll */ 992 flush %o0 993 /* fix srl */ 994 add %o0, I_SIZE, %o0 /* goto next instr. */ 995 ld [%o0], %o3 /* get srl */ 996 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 997 st %o3, [%o0] /* write srl */ 998 retl 999 flush %o0 1000 SET_SIZE(sfmmu_fix_ktlb_traptable) 1001 1002 ENTRY_NP(sfmmu_fixup_ktsbbase) 1003 /* 1004 * %o0 = start of patch area 1005 * %o5 = kernel virtual or physical tsb base address 1006 * %o2, %o3 are used as scratch registers. 1007 */ 1008 /* fixup sethi instruction */ 1009 ld [%o0], %o3 1010 srl %o5, 10, %o2 ! offset is bits 32:10 1011 or %o3, %o2, %o3 ! set imm22 1012 st %o3, [%o0] 1013 /* fixup offset of lduw/ldx */ 1014 add %o0, I_SIZE, %o0 ! next instr 1015 ld [%o0], %o3 1016 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 1017 or %o3, %o2, %o3 1018 st %o3, [%o0] 1019 retl 1020 flush %o0 1021 SET_SIZE(sfmmu_fixup_ktsbbase) 1022 1023 ENTRY_NP(sfmmu_fixup_setx) 1024 /* 1025 * %o0 = start of patch area 1026 * %o4 = 64 bit value to patch 1027 * %o2, %o3 are used as scratch registers. 1028 * 1029 * Note: Assuming that all parts of the instructions which need to be 1030 * patched correspond to RUNTIME_PATCH (aka 0) 1031 * 1032 * Note the implementation of setx which is being patched is as follows: 1033 * 1034 * sethi %hh(RUNTIME_PATCH), tmp 1035 * sethi %lm(RUNTIME_PATCH), dest 1036 * or tmp, %hm(RUNTIME_PATCH), tmp 1037 * or dest, %lo(RUNTIME_PATCH), dest 1038 * sllx tmp, 32, tmp 1039 * nop 1040 * or tmp, dest, dest 1041 * 1042 * which differs from the implementation in the 1043 * "SPARC Architecture Manual" 1044 */ 1045 /* fixup sethi instruction */ 1046 ld [%o0], %o3 1047 srlx %o4, 42, %o2 ! bits [63:42] 1048 or %o3, %o2, %o3 ! set imm22 1049 st %o3, [%o0] 1050 /* fixup sethi instruction */ 1051 add %o0, I_SIZE, %o0 ! next instr 1052 ld [%o0], %o3 1053 sllx %o4, 32, %o2 ! clear upper bits 1054 srlx %o2, 42, %o2 ! bits [31:10] 1055 or %o3, %o2, %o3 ! set imm22 1056 st %o3, [%o0] 1057 /* fixup or instruction */ 1058 add %o0, I_SIZE, %o0 ! next instr 1059 ld [%o0], %o3 1060 srlx %o4, 32, %o2 ! bits [63:32] 1061 and %o2, 0x3ff, %o2 ! bits [41:32] 1062 or %o3, %o2, %o3 ! set imm 1063 st %o3, [%o0] 1064 /* fixup or instruction */ 1065 add %o0, I_SIZE, %o0 ! next instr 1066 ld [%o0], %o3 1067 and %o4, 0x3ff, %o2 ! bits [9:0] 1068 or %o3, %o2, %o3 ! set imm 1069 st %o3, [%o0] 1070 retl 1071 flush %o0 1072 SET_SIZE(sfmmu_fixup_setx) 1073 1074 ENTRY_NP(sfmmu_fixup_or) 1075 /* 1076 * %o0 = start of patch area 1077 * %o4 = 32 bit value to patch 1078 * %o2, %o3 are used as scratch registers. 1079 * Note: Assuming that all parts of the instructions which need to be 1080 * patched correspond to RUNTIME_PATCH (aka 0) 1081 */ 1082 ld [%o0], %o3 1083 and %o4, 0x3ff, %o2 ! bits [9:0] 1084 or %o3, %o2, %o3 ! set imm 1085 st %o3, [%o0] 1086 retl 1087 flush %o0 1088 SET_SIZE(sfmmu_fixup_or) 1089 1090 ENTRY_NP(sfmmu_fixup_shiftx) 1091 /* 1092 * %o0 = start of patch area 1093 * %o4 = signed int immediate value to add to sllx/srlx imm field 1094 * %o2, %o3 are used as scratch registers. 1095 * 1096 * sllx/srlx store the 6 bit immediate value in the lowest order bits 1097 * so we do a simple add. The caller must be careful to prevent 1098 * overflow, which could easily occur if the initial value is nonzero! 1099 */ 1100 ld [%o0], %o3 ! %o3 = instruction to patch 1101 and %o3, 0x3f, %o2 ! %o2 = existing imm value 1102 add %o2, %o4, %o2 ! %o2 = new imm value 1103 andn %o3, 0x3f, %o3 ! clear old imm value 1104 and %o2, 0x3f, %o2 ! truncate new imm value 1105 or %o3, %o2, %o3 ! set new imm value 1106 st %o3, [%o0] ! store updated instruction 1107 retl 1108 flush %o0 1109 SET_SIZE(sfmmu_fixup_shiftx) 1110 1111 ENTRY_NP(sfmmu_fixup_mmu_asi) 1112 /* 1113 * Patch imm_asi of all ldda instructions in the MMU 1114 * trap handlers. We search MMU_PATCH_INSTR instructions 1115 * starting from the itlb miss handler (trap 0x64). 1116 * %o0 = address of tt[0,1]_itlbmiss 1117 * %o1 = imm_asi to setup, shifted by appropriate offset. 1118 * %o3 = number of instructions to search 1119 * %o4 = reserved by caller: called from leaf routine 1120 */ 11211: ldsw [%o0], %o2 ! load instruction to %o2 1122 brgez,pt %o2, 2f 1123 srl %o2, 30, %o5 1124 btst 1, %o5 ! test bit 30; skip if not set 1125 bz,pt %icc, 2f 1126 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1127 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1128 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1129 brnz,pt %o5, 2f ! skip if not a match 1130 or %o2, %o1, %o2 ! or in imm_asi 1131 st %o2, [%o0] ! write patched instruction 11322: dec %o3 1133 brnz,a,pt %o3, 1b ! loop until we're done 1134 add %o0, I_SIZE, %o0 1135 retl 1136 flush %o0 1137 SET_SIZE(sfmmu_fixup_mmu_asi) 1138 1139 /* 1140 * Patch immediate ASI used to access the TSB in the 1141 * trap table. 1142 * inputs: %o0 = value of ktsb_phys 1143 */ 1144 ENTRY_NP(sfmmu_patch_mmu_asi) 1145 mov %o7, %o4 ! save return pc in %o4 1146 mov ASI_QUAD_LDD_PHYS, %o3 ! set QUAD_LDD_PHYS by default 1147 1148#ifdef sun4v 1149 1150 /* 1151 * Check ktsb_phys. It must be non-zero for sun4v, panic if not. 1152 */ 1153 1154 brnz,pt %o0, do_patch 1155 nop 1156 1157 sethi %hi(sfmmu_panic11), %o0 1158 call panic 1159 or %o0, %lo(sfmmu_panic11), %o0 1160do_patch: 1161 1162#else /* sun4v */ 1163 /* 1164 * Some non-sun4v platforms deploy virtual ktsb (ktsb_phys==0). 1165 * Note that ASI_NQUAD_LD is not defined/used for sun4v 1166 */ 1167 movrz %o0, ASI_NQUAD_LD, %o3 1168 1169#endif /* sun4v */ 1170 1171 sll %o3, 5, %o1 ! imm_asi offset 1172 mov 6, %o3 ! number of instructions 1173 sethi %hi(dktsb), %o0 ! to search 1174 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1175 or %o0, %lo(dktsb), %o0 1176 mov 6, %o3 ! number of instructions 1177 sethi %hi(dktsb4m), %o0 ! to search 1178 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1179 or %o0, %lo(dktsb4m), %o0 1180 mov 6, %o3 ! number of instructions 1181 sethi %hi(iktsb), %o0 ! to search 1182 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1183 or %o0, %lo(iktsb), %o0 1184 mov 6, %o3 ! number of instructions 1185 sethi %hi(iktsb4m), %o0 ! to search 1186 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1187 or %o0, %lo(iktsb4m), %o0 1188 mov %o4, %o7 ! retore return pc -- leaf 1189 retl 1190 nop 1191 SET_SIZE(sfmmu_patch_mmu_asi) 1192 1193 1194 ENTRY_NP(sfmmu_patch_ktsb) 1195 /* 1196 * We need to fix iktsb, dktsb, et. al. 1197 */ 1198 save %sp, -SA(MINFRAME), %sp 1199 set ktsb_phys, %o1 1200 ld [%o1], %o4 1201 set ktsb_base, %o5 1202 set ktsb4m_base, %l1 1203 brz,pt %o4, 1f 1204 nop 1205 set ktsb_pbase, %o5 1206 set ktsb4m_pbase, %l1 12071: 1208 sethi %hi(ktsb_szcode), %o1 1209 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1210 1211 sethi %hi(iktsb), %o0 1212 call sfmmu_fix_ktlb_traptable 1213 or %o0, %lo(iktsb), %o0 1214 1215 sethi %hi(dktsb), %o0 1216 call sfmmu_fix_ktlb_traptable 1217 or %o0, %lo(dktsb), %o0 1218 1219 sethi %hi(ktsb4m_szcode), %o1 1220 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1221 1222 sethi %hi(iktsb4m), %o0 1223 call sfmmu_fix_ktlb_traptable 1224 or %o0, %lo(iktsb4m), %o0 1225 1226 sethi %hi(dktsb4m), %o0 1227 call sfmmu_fix_ktlb_traptable 1228 or %o0, %lo(dktsb4m), %o0 1229 1230#ifndef sun4v 1231 mov ASI_N, %o2 1232 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1233 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1234 sethi %hi(tsb_kernel_patch_asi), %o0 1235 call sfmmu_fixup_or 1236 or %o0, %lo(tsb_kernel_patch_asi), %o0 1237#endif /* !sun4v */ 1238 1239 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1240 1241 sethi %hi(dktsbbase), %o0 1242 call sfmmu_fixup_setx ! patch value of ktsb base addr 1243 or %o0, %lo(dktsbbase), %o0 1244 1245 sethi %hi(iktsbbase), %o0 1246 call sfmmu_fixup_setx ! patch value of ktsb base addr 1247 or %o0, %lo(iktsbbase), %o0 1248 1249 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1250 call sfmmu_fixup_setx ! patch value of ktsb base addr 1251 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1252 1253#ifdef sun4v 1254 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1255 call sfmmu_fixup_setx ! patch value of ktsb base addr 1256 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1257#endif /* sun4v */ 1258 1259 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1260 1261 sethi %hi(dktsb4mbase), %o0 1262 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1263 or %o0, %lo(dktsb4mbase), %o0 1264 1265 sethi %hi(iktsb4mbase), %o0 1266 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1267 or %o0, %lo(iktsb4mbase), %o0 1268 1269 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1270 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1271 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1272 1273#ifdef sun4v 1274 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1275 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1276 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1277#endif /* sun4v */ 1278 1279 set ktsb_szcode, %o4 1280 ld [%o4], %o4 1281 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1282 call sfmmu_fixup_or ! patch value of ktsb_szcode 1283 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1284 1285#ifdef sun4v 1286 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1287 call sfmmu_fixup_or ! patch value of ktsb_szcode 1288 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1289#endif /* sun4v */ 1290 1291 set ktsb4m_szcode, %o4 1292 ld [%o4], %o4 1293 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1294 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1295 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1296 1297#ifdef sun4v 1298 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1299 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1300 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1301#endif /* sun4v */ 1302 1303 ret 1304 restore 1305 SET_SIZE(sfmmu_patch_ktsb) 1306 1307 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1308 /* 1309 * Fixup trap handlers in common segkpm case. This is reserved 1310 * for future use should kpm TSB be changed to be other than the 1311 * kernel TSB. 1312 */ 1313 retl 1314 nop 1315 SET_SIZE(sfmmu_kpm_patch_tlbm) 1316 1317 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1318 /* 1319 * nop the branch to sfmmu_kpm_dtsb_miss_small 1320 * in the case where we are using large pages for 1321 * seg_kpm (and hence must probe the second TSB for 1322 * seg_kpm VAs) 1323 */ 1324 set dktsb4m_kpmcheck_small, %o0 1325 MAKE_NOP_INSTR(%o1) 1326 st %o1, [%o0] 1327 flush %o0 1328 retl 1329 nop 1330 SET_SIZE(sfmmu_kpm_patch_tsbm) 1331 1332 ENTRY_NP(sfmmu_patch_utsb) 1333#ifdef UTSB_PHYS 1334 retl 1335 nop 1336#else /* UTSB_PHYS */ 1337 /* 1338 * We need to hot patch utsb_vabase and utsb4m_vabase 1339 */ 1340 save %sp, -SA(MINFRAME), %sp 1341 1342 /* patch value of utsb_vabase */ 1343 set utsb_vabase, %o1 1344 ldx [%o1], %o4 1345 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1346 call sfmmu_fixup_setx 1347 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1348 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1349 call sfmmu_fixup_setx 1350 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1351 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1352 call sfmmu_fixup_setx 1353 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1354 1355 /* patch value of utsb4m_vabase */ 1356 set utsb4m_vabase, %o1 1357 ldx [%o1], %o4 1358 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1359 call sfmmu_fixup_setx 1360 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1361 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1362 call sfmmu_fixup_setx 1363 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1364 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1365 call sfmmu_fixup_setx 1366 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1367 1368 /* 1369 * Patch TSB base register masks and shifts if needed. 1370 * By default the TSB base register contents are set up for 4M slab. 1371 * If we're using a smaller slab size and reserved VA range we need 1372 * to patch up those values here. 1373 */ 1374 set tsb_slab_shift, %o1 1375 set MMU_PAGESHIFT4M, %o4 1376 lduw [%o1], %o3 1377 subcc %o4, %o3, %o4 1378 bz,pt %icc, 1f 1379 /* delay slot safe */ 1380 1381 /* patch reserved VA range size if needed. */ 1382 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1383 call sfmmu_fixup_shiftx 1384 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1385 call sfmmu_fixup_shiftx 1386 add %o0, I_SIZE, %o0 1387 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1388 call sfmmu_fixup_shiftx 1389 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1390 call sfmmu_fixup_shiftx 1391 add %o0, I_SIZE, %o0 13921: 1393 /* patch TSBREG_VAMASK used to set up TSB base register */ 1394 set tsb_slab_mask, %o1 1395 ldx [%o1], %o4 1396 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1397 call sfmmu_fixup_or 1398 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1399 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1400 call sfmmu_fixup_or 1401 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1402 1403 ret 1404 restore 1405#endif /* UTSB_PHYS */ 1406 SET_SIZE(sfmmu_patch_utsb) 1407 1408 ENTRY_NP(sfmmu_patch_shctx) 1409#ifdef sun4u 1410 retl 1411 nop 1412#else /* sun4u */ 1413 set sfmmu_shctx_cpu_mondo_patch, %o0 1414 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1415 st %o1, [%o0] 1416 flush %o0 1417 MAKE_NOP_INSTR(%o1) 1418 add %o0, I_SIZE, %o0 ! next instr 1419 st %o1, [%o0] 1420 flush %o0 1421 1422 set sfmmu_shctx_user_rtt_patch, %o0 1423 st %o1, [%o0] ! nop 1st instruction 1424 flush %o0 1425 add %o0, I_SIZE, %o0 1426 st %o1, [%o0] ! nop 2nd instruction 1427 flush %o0 1428 add %o0, I_SIZE, %o0 1429 st %o1, [%o0] ! nop 3rd instruction 1430 flush %o0 1431 add %o0, I_SIZE, %o0 1432 st %o1, [%o0] ! nop 4th instruction 1433 flush %o0 1434 add %o0, I_SIZE, %o0 1435 st %o1, [%o0] ! nop 5th instruction 1436 flush %o0 1437 add %o0, I_SIZE, %o0 1438 st %o1, [%o0] ! nop 6th instruction 1439 retl 1440 flush %o0 1441#endif /* sun4u */ 1442 SET_SIZE(sfmmu_patch_shctx) 1443 1444 ENTRY_NP(sfmmu_patch_pgsz_reg) 1445#ifdef sun4u 1446 retl 1447 nop 1448#else /* sun4u */ 1449 set sfmmu_pgsz_load_mmustate_patch, %o0 1450 MAKE_NOP_INSTR(%o1) 1451 st %o1, [%o0] 1452 retl 1453 flush %o0 1454#endif /* sun4u */ 1455 SET_SIZE(sfmmu_patch_pgsz_reg) 1456 1457 /* 1458 * Routine that loads an entry into a tsb using virtual addresses. 1459 * Locking is required since all cpus can use the same TSB. 1460 * Note that it is no longer required to have a valid context 1461 * when calling this function. 1462 */ 1463 ENTRY_NP(sfmmu_load_tsbe) 1464 /* 1465 * %o0 = pointer to tsbe to load 1466 * %o1 = tsb tag 1467 * %o2 = virtual pointer to TTE 1468 * %o3 = 1 if physical address in %o0 else 0 1469 */ 1470 rdpr %pstate, %o5 1471#ifdef DEBUG 1472 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1473#endif /* DEBUG */ 1474 1475 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1476 1477 SETUP_TSB_ASI(%o3, %g3) 1478 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, locked_tsb_l8) 1479 1480 wrpr %g0, %o5, %pstate /* enable interrupts */ 1481 1482 retl 1483 membar #StoreStore|#StoreLoad 1484 SET_SIZE(sfmmu_load_tsbe) 1485 1486 /* 1487 * Flush TSB of a given entry if the tag matches. 1488 */ 1489 ENTRY(sfmmu_unload_tsbe) 1490 /* 1491 * %o0 = pointer to tsbe to be flushed 1492 * %o1 = tag to match 1493 * %o2 = 1 if physical address in %o0 else 0 1494 */ 1495 SETUP_TSB_ASI(%o2, %g1) 1496 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1497 retl 1498 membar #StoreStore|#StoreLoad 1499 SET_SIZE(sfmmu_unload_tsbe) 1500 1501 /* 1502 * Routine that loads a TTE into the kpm TSB from C code. 1503 * Locking is required since kpm TSB is shared among all CPUs. 1504 */ 1505 ENTRY_NP(sfmmu_kpm_load_tsb) 1506 /* 1507 * %o0 = vaddr 1508 * %o1 = ttep 1509 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1510 */ 1511 rdpr %pstate, %o5 ! %o5 = saved pstate 1512#ifdef DEBUG 1513 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1514#endif /* DEBUG */ 1515 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1516 1517#ifndef sun4v 1518 sethi %hi(ktsb_phys), %o4 1519 mov ASI_N, %o3 1520 ld [%o4 + %lo(ktsb_phys)], %o4 1521 movrnz %o4, ASI_MEM, %o3 1522 mov %o3, %asi 1523#endif /* !sun4v */ 1524 mov %o0, %g1 ! %g1 = vaddr 1525 1526 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1527 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1528 /* %g2 = tsbep, %g1 clobbered */ 1529 1530 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1531 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1532 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, locked_tsb_l9) 1533 1534 wrpr %g0, %o5, %pstate ! enable interrupts 1535 retl 1536 membar #StoreStore|#StoreLoad 1537 SET_SIZE(sfmmu_kpm_load_tsb) 1538 1539 /* 1540 * Routine that shoots down a TTE in the kpm TSB or in the 1541 * kernel TSB depending on virtpg. Locking is required since 1542 * kpm/kernel TSB is shared among all CPUs. 1543 */ 1544 ENTRY_NP(sfmmu_kpm_unload_tsb) 1545 /* 1546 * %o0 = vaddr 1547 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1548 */ 1549#ifndef sun4v 1550 sethi %hi(ktsb_phys), %o4 1551 mov ASI_N, %o3 1552 ld [%o4 + %lo(ktsb_phys)], %o4 1553 movrnz %o4, ASI_MEM, %o3 1554 mov %o3, %asi 1555#endif /* !sun4v */ 1556 mov %o0, %g1 ! %g1 = vaddr 1557 1558 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1559 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1560 /* %g2 = tsbep, %g1 clobbered */ 1561 1562 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1563 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1564 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1565 1566 retl 1567 membar #StoreStore|#StoreLoad 1568 SET_SIZE(sfmmu_kpm_unload_tsb) 1569 1570#endif /* lint */ 1571 1572 1573#if defined (lint) 1574 1575/*ARGSUSED*/ 1576pfn_t 1577sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1578{ return(0); } 1579 1580#else /* lint */ 1581 1582 ENTRY_NP(sfmmu_ttetopfn) 1583 ldx [%o0], %g1 /* read tte */ 1584 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1585 /* 1586 * g1 = pfn 1587 */ 1588 retl 1589 mov %g1, %o0 1590 SET_SIZE(sfmmu_ttetopfn) 1591 1592#endif /* !lint */ 1593 1594/* 1595 * These macros are used to update global sfmmu hme hash statistics 1596 * in perf critical paths. It is only enabled in debug kernels or 1597 * if SFMMU_STAT_GATHER is defined 1598 */ 1599#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1600#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1601 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1602 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1603 cmp tmp1, hatid ;\ 1604 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1605 set sfmmu_global_stat, tmp1 ;\ 1606 add tmp1, tmp2, tmp1 ;\ 1607 ld [tmp1], tmp2 ;\ 1608 inc tmp2 ;\ 1609 st tmp2, [tmp1] 1610 1611#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1612 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1613 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1614 cmp tmp1, hatid ;\ 1615 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1616 set sfmmu_global_stat, tmp1 ;\ 1617 add tmp1, tmp2, tmp1 ;\ 1618 ld [tmp1], tmp2 ;\ 1619 inc tmp2 ;\ 1620 st tmp2, [tmp1] 1621 1622 1623#else /* DEBUG || SFMMU_STAT_GATHER */ 1624 1625#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1626 1627#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1628 1629#endif /* DEBUG || SFMMU_STAT_GATHER */ 1630 1631/* 1632 * This macro is used to update global sfmmu kstas in non 1633 * perf critical areas so they are enabled all the time 1634 */ 1635#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1636 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1637 add tmp1, statname, tmp1 ;\ 1638 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1639 inc tmp2 ;\ 1640 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1641 1642/* 1643 * These macros are used to update per cpu stats in non perf 1644 * critical areas so they are enabled all the time 1645 */ 1646#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1647 ld [tsbarea + stat], tmp1 ;\ 1648 inc tmp1 ;\ 1649 st tmp1, [tsbarea + stat] 1650 1651/* 1652 * These macros are used to update per cpu stats in non perf 1653 * critical areas so they are enabled all the time 1654 */ 1655#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1656 lduh [tsbarea + stat], tmp1 ;\ 1657 inc tmp1 ;\ 1658 stuh tmp1, [tsbarea + stat] 1659 1660#if defined(KPM_TLBMISS_STATS_GATHER) 1661 /* 1662 * Count kpm dtlb misses separately to allow a different 1663 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1664 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1665 */ 1666#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1667 brgez tagacc, label /* KPM VA? */ ;\ 1668 nop ;\ 1669 CPU_INDEX(tmp1, tsbma) ;\ 1670 sethi %hi(kpmtsbm_area), tsbma ;\ 1671 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1672 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1673 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1674 /* VA range check */ ;\ 1675 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1676 cmp tagacc, val ;\ 1677 blu,pn %xcc, label ;\ 1678 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1679 cmp tagacc, tmp1 ;\ 1680 bgeu,pn %xcc, label ;\ 1681 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1682 inc val ;\ 1683 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1684label: 1685#else 1686#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1687#endif /* KPM_TLBMISS_STATS_GATHER */ 1688 1689#if defined (lint) 1690/* 1691 * The following routines are jumped to from the mmu trap handlers to do 1692 * the setting up to call systrap. They are separate routines instead of 1693 * being part of the handlers because the handlers would exceed 32 1694 * instructions and since this is part of the slow path the jump 1695 * cost is irrelevant. 1696 */ 1697void 1698sfmmu_pagefault(void) 1699{ 1700} 1701 1702void 1703sfmmu_mmu_trap(void) 1704{ 1705} 1706 1707void 1708sfmmu_window_trap(void) 1709{ 1710} 1711 1712void 1713sfmmu_kpm_exception(void) 1714{ 1715} 1716 1717#else /* lint */ 1718 1719#ifdef PTL1_PANIC_DEBUG 1720 .seg ".data" 1721 .global test_ptl1_panic 1722test_ptl1_panic: 1723 .word 0 1724 .align 8 1725 1726 .seg ".text" 1727 .align 4 1728#endif /* PTL1_PANIC_DEBUG */ 1729 1730 1731 ENTRY_NP(sfmmu_pagefault) 1732 SET_GL_REG(1) 1733 USE_ALTERNATE_GLOBALS(%g5) 1734 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1735 rdpr %tt, %g6 1736 cmp %g6, FAST_IMMU_MISS_TT 1737 be,a,pn %icc, 1f 1738 mov T_INSTR_MMU_MISS, %g3 1739 cmp %g6, T_INSTR_MMU_MISS 1740 be,a,pn %icc, 1f 1741 mov T_INSTR_MMU_MISS, %g3 1742 mov %g5, %g2 1743 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1744 cmp %g6, FAST_DMMU_MISS_TT 1745 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1746 cmp %g6, T_DATA_MMU_MISS 1747 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1748 1749#ifdef PTL1_PANIC_DEBUG 1750 /* check if we want to test the tl1 panic */ 1751 sethi %hi(test_ptl1_panic), %g4 1752 ld [%g4 + %lo(test_ptl1_panic)], %g1 1753 st %g0, [%g4 + %lo(test_ptl1_panic)] 1754 cmp %g1, %g0 1755 bne,a,pn %icc, ptl1_panic 1756 or %g0, PTL1_BAD_DEBUG, %g1 1757#endif /* PTL1_PANIC_DEBUG */ 17581: 1759 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 1760 /* 1761 * g2 = tag access reg 1762 * g3.l = type 1763 * g3.h = 0 1764 */ 1765 sethi %hi(trap), %g1 1766 or %g1, %lo(trap), %g1 17672: 1768 ba,pt %xcc, sys_trap 1769 mov -1, %g4 1770 SET_SIZE(sfmmu_pagefault) 1771 1772 ENTRY_NP(sfmmu_mmu_trap) 1773 SET_GL_REG(1) 1774 USE_ALTERNATE_GLOBALS(%g5) 1775 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 1776 rdpr %tt, %g6 1777 cmp %g6, FAST_IMMU_MISS_TT 1778 be,a,pn %icc, 1f 1779 mov T_INSTR_MMU_MISS, %g3 1780 cmp %g6, T_INSTR_MMU_MISS 1781 be,a,pn %icc, 1f 1782 mov T_INSTR_MMU_MISS, %g3 1783 mov %g5, %g2 1784 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1785 cmp %g6, FAST_DMMU_MISS_TT 1786 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1787 cmp %g6, T_DATA_MMU_MISS 1788 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 17891: 1790 /* 1791 * g2 = tag access reg 1792 * g3 = type 1793 */ 1794 sethi %hi(sfmmu_tsbmiss_exception), %g1 1795 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 1796 ba,pt %xcc, sys_trap 1797 mov -1, %g4 1798 /*NOTREACHED*/ 1799 SET_SIZE(sfmmu_mmu_trap) 1800 1801 ENTRY_NP(sfmmu_suspend_tl) 1802 SET_GL_REG(1) 1803 USE_ALTERNATE_GLOBALS(%g5) 1804 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 1805 rdpr %tt, %g6 1806 cmp %g6, FAST_IMMU_MISS_TT 1807 be,a,pn %icc, 1f 1808 mov T_INSTR_MMU_MISS, %g3 1809 mov %g5, %g2 1810 cmp %g6, FAST_DMMU_MISS_TT 1811 move %icc, T_DATA_MMU_MISS, %g3 1812 movne %icc, T_DATA_PROT, %g3 18131: 1814 sethi %hi(sfmmu_tsbmiss_suspended), %g1 1815 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 1816 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 1817 ba,pt %xcc, sys_trap 1818 mov PIL_15, %g4 1819 /*NOTREACHED*/ 1820 SET_SIZE(sfmmu_suspend_tl) 1821 1822 /* 1823 * No %g registers in use at this point. 1824 */ 1825 ENTRY_NP(sfmmu_window_trap) 1826 rdpr %tpc, %g1 1827#ifdef sun4v 1828#ifdef DEBUG 1829 /* We assume previous %gl was 1 */ 1830 rdpr %tstate, %g4 1831 srlx %g4, TSTATE_GL_SHIFT, %g4 1832 and %g4, TSTATE_GL_MASK, %g4 1833 cmp %g4, 1 1834 bne,a,pn %icc, ptl1_panic 1835 mov PTL1_BAD_WTRAP, %g1 1836#endif /* DEBUG */ 1837 /* user miss at tl>1. better be the window handler or user_rtt */ 1838 /* in user_rtt? */ 1839 set rtt_fill_start, %g4 1840 cmp %g1, %g4 1841 blu,pn %xcc, 6f 1842 .empty 1843 set rtt_fill_end, %g4 1844 cmp %g1, %g4 1845 bgeu,pn %xcc, 6f 1846 nop 1847 set fault_rtt_fn1, %g1 1848 wrpr %g0, %g1, %tnpc 1849 ba,a 7f 18506: 1851 ! must save this trap level before descending trap stack 1852 ! no need to save %tnpc, either overwritten or discarded 1853 ! already got it: rdpr %tpc, %g1 1854 rdpr %tstate, %g6 1855 rdpr %tt, %g7 1856 ! trap level saved, go get underlying trap type 1857 rdpr %tl, %g5 1858 sub %g5, 1, %g3 1859 wrpr %g3, %tl 1860 rdpr %tt, %g2 1861 wrpr %g5, %tl 1862 ! restore saved trap level 1863 wrpr %g1, %tpc 1864 wrpr %g6, %tstate 1865 wrpr %g7, %tt 1866#else /* sun4v */ 1867 /* user miss at tl>1. better be the window handler */ 1868 rdpr %tl, %g5 1869 sub %g5, 1, %g3 1870 wrpr %g3, %tl 1871 rdpr %tt, %g2 1872 wrpr %g5, %tl 1873#endif /* sun4v */ 1874 and %g2, WTRAP_TTMASK, %g4 1875 cmp %g4, WTRAP_TYPE 1876 bne,pn %xcc, 1f 1877 nop 1878 /* tpc should be in the trap table */ 1879 set trap_table, %g4 1880 cmp %g1, %g4 1881 blt,pn %xcc, 1f 1882 .empty 1883 set etrap_table, %g4 1884 cmp %g1, %g4 1885 bge,pn %xcc, 1f 1886 .empty 1887 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 1888 add %g1, WTRAP_FAULTOFF, %g1 1889 wrpr %g0, %g1, %tnpc 18907: 1891 /* 1892 * some wbuf handlers will call systrap to resolve the fault 1893 * we pass the trap type so they figure out the correct parameters. 1894 * g5 = trap type, g6 = tag access reg 1895 */ 1896 1897 /* 1898 * only use g5, g6, g7 registers after we have switched to alternate 1899 * globals. 1900 */ 1901 SET_GL_REG(1) 1902 USE_ALTERNATE_GLOBALS(%g5) 1903 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 1904 rdpr %tt, %g7 1905 cmp %g7, FAST_IMMU_MISS_TT 1906 be,a,pn %icc, ptl1_panic 1907 mov PTL1_BAD_WTRAP, %g1 1908 cmp %g7, T_INSTR_MMU_MISS 1909 be,a,pn %icc, ptl1_panic 1910 mov PTL1_BAD_WTRAP, %g1 1911 mov T_DATA_PROT, %g5 1912 cmp %g7, FAST_DMMU_MISS_TT 1913 move %icc, T_DATA_MMU_MISS, %g5 1914 cmp %g7, T_DATA_MMU_MISS 1915 move %icc, T_DATA_MMU_MISS, %g5 1916 ! XXXQ AGS re-check out this one 1917 done 19181: 1919 CPU_PADDR(%g1, %g4) 1920 add %g1, CPU_TL1_HDLR, %g1 1921 lda [%g1]ASI_MEM, %g4 1922 brnz,a,pt %g4, sfmmu_mmu_trap 1923 sta %g0, [%g1]ASI_MEM 1924 ba,pt %icc, ptl1_panic 1925 mov PTL1_BAD_TRAP, %g1 1926 SET_SIZE(sfmmu_window_trap) 1927 1928 ENTRY_NP(sfmmu_kpm_exception) 1929 /* 1930 * We have accessed an unmapped segkpm address or a legal segkpm 1931 * address which is involved in a VAC alias conflict prevention. 1932 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 1933 * set. If it is, we will instead note that a fault has occurred 1934 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 1935 * a "retry"). This will step over the faulting instruction. 1936 * Note that this means that a legal segkpm address involved in 1937 * a VAC alias conflict prevention (a rare case to begin with) 1938 * cannot be used in DTrace. 1939 */ 1940 CPU_INDEX(%g1, %g2) 1941 set cpu_core, %g2 1942 sllx %g1, CPU_CORE_SHIFT, %g1 1943 add %g1, %g2, %g1 1944 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 1945 andcc %g2, CPU_DTRACE_NOFAULT, %g0 1946 bz 0f 1947 or %g2, CPU_DTRACE_BADADDR, %g2 1948 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 1949 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 1950 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 1951 done 19520: 1953 TSTAT_CHECK_TL1(1f, %g1, %g2) 19541: 1955 SET_GL_REG(1) 1956 USE_ALTERNATE_GLOBALS(%g5) 1957 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 1958 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1959 /* 1960 * g2=tagacc g3.l=type g3.h=0 1961 */ 1962 sethi %hi(trap), %g1 1963 or %g1, %lo(trap), %g1 1964 ba,pt %xcc, sys_trap 1965 mov -1, %g4 1966 SET_SIZE(sfmmu_kpm_exception) 1967 1968#endif /* lint */ 1969 1970#if defined (lint) 1971 1972void 1973sfmmu_tsb_miss(void) 1974{ 1975} 1976 1977void 1978sfmmu_kpm_dtsb_miss(void) 1979{ 1980} 1981 1982void 1983sfmmu_kpm_dtsb_miss_small(void) 1984{ 1985} 1986 1987#else /* lint */ 1988 1989#if (IMAP_SEG != 0) 1990#error - ism_map->ism_seg offset is not zero 1991#endif 1992 1993/* 1994 * Copies ism mapping for this ctx in param "ism" if this is a ISM 1995 * tlb miss and branches to label "ismhit". If this is not an ISM 1996 * process or an ISM tlb miss it falls thru. 1997 * 1998 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 1999 * this process. 2000 * If so, it will branch to label "ismhit". If not, it will fall through. 2001 * 2002 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 2003 * so that any other threads of this process will not try and walk the ism 2004 * maps while they are being changed. 2005 * 2006 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 2007 * will make sure of that. This means we can terminate our search on 2008 * the first zero mapping we find. 2009 * 2010 * Parameters: 2011 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 2012 * tsbmiss = address of tsb miss area (in) 2013 * ismseg = contents of ism_seg for this ism map (out) 2014 * ismhat = physical address of imap_ismhat for this ism map (out) 2015 * tmp1 = scratch reg (CLOBBERED) 2016 * tmp2 = scratch reg (CLOBBERED) 2017 * tmp3 = scratch reg (CLOBBERED) 2018 * label: temporary labels 2019 * ismhit: label where to jump to if an ism dtlb miss 2020 * exitlabel:label where to jump if hat is busy due to hat_unshare. 2021 */ 2022#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 2023 label, ismhit) \ 2024 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 2025 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 2026 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 2027label/**/1: ;\ 2028 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 2029 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 2030label/**/2: ;\ 2031 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 2032 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 2033 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 2034 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 2035 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 2036 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 2037 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 2038 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 2039 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 2040 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 2041 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 2042 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 2043 ;\ 2044 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 2045 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 2046 cmp ismhat, tmp1 ;\ 2047 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 2048 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 2049 ;\ 2050 add tmp3, IBLK_NEXTPA, tmp1 ;\ 2051 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 2052 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 2053 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 2054label/**/3: 2055 2056/* 2057 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 2058 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 2059 * Parameters: 2060 * tagacc = reg containing virtual address 2061 * hatid = reg containing sfmmu pointer 2062 * hmeshift = constant/register to shift vaddr to obtain vapg 2063 * hmebp = register where bucket pointer will be stored 2064 * vapg = register where virtual page will be stored 2065 * tmp1, tmp2 = tmp registers 2066 */ 2067 2068 2069#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 2070 vapg, label, tmp1, tmp2) \ 2071 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 2072 brnz,a,pt tmp1, label/**/1 ;\ 2073 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 2074 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 2075 ba,pt %xcc, label/**/2 ;\ 2076 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 2077label/**/1: ;\ 2078 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 2079label/**/2: ;\ 2080 srlx tagacc, hmeshift, vapg ;\ 2081 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 2082 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 2083 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 2084 add hmebp, tmp1, hmebp 2085 2086/* 2087 * hashtag includes bspage + hashno (64 bits). 2088 */ 2089 2090#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 2091 sllx vapg, hmeshift, vapg ;\ 2092 mov hashno, hblktag ;\ 2093 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 2094 or vapg, hblktag, hblktag 2095 2096/* 2097 * Function to traverse hmeblk hash link list and find corresponding match. 2098 * The search is done using physical pointers. It returns the physical address 2099 * pointer to the hmeblk that matches with the tag provided. 2100 * Parameters: 2101 * hmebp = register that points to hme hash bucket, also used as 2102 * tmp reg (clobbered) 2103 * hmeblktag = register with hmeblk tag match 2104 * hatid = register with hatid 2105 * hmeblkpa = register where physical ptr will be stored 2106 * tmp1 = tmp reg 2107 * label: temporary label 2108 */ 2109 2110#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, tsbarea, \ 2111 tmp1, label) \ 2112 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 2113 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2114 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2115label/**/1: ;\ 2116 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2117 be,pn %xcc, label/**/2 ;\ 2118 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2119 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 2120 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2121 add hmebp, CLONGSIZE, hmebp ;\ 2122 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 2123 xor tmp1, hmeblktag, tmp1 ;\ 2124 xor hmebp, hatid, hmebp ;\ 2125 or hmebp, tmp1, hmebp ;\ 2126 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 2127 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 2128 ba,pt %xcc, label/**/1 ;\ 2129 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2130label/**/2: 2131 2132/* 2133 * Function to traverse hmeblk hash link list and find corresponding match. 2134 * The search is done using physical pointers. It returns the physical address 2135 * pointer to the hmeblk that matches with the tag 2136 * provided. 2137 * Parameters: 2138 * hmeblktag = register with hmeblk tag match (rid field is 0) 2139 * hatid = register with hatid (pointer to SRD) 2140 * hmeblkpa = register where physical ptr will be stored 2141 * tmp1 = tmp reg 2142 * tmp2 = tmp reg 2143 * label: temporary label 2144 */ 2145 2146#define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, tsbarea, \ 2147 tmp1, tmp2, label) \ 2148label/**/1: ;\ 2149 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2150 be,pn %xcc, label/**/4 ;\ 2151 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 2152 add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 2153 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2154 add tmp2, CLONGSIZE, tmp2 ;\ 2155 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 2156 xor tmp1, hmeblktag, tmp1 ;\ 2157 xor tmp2, hatid, tmp2 ;\ 2158 brz,pn tmp2, label/**/3 /* branch on hit */ ;\ 2159 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2160label/**/2: ;\ 2161 ba,pt %xcc, label/**/1 ;\ 2162 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2163label/**/3: ;\ 2164 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 2165 bgeu,pt %xcc, label/**/2 ;\ 2166 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2167 and tmp1, BT_ULMASK, tmp2 ;\ 2168 srlx tmp1, BT_ULSHIFT, tmp1 ;\ 2169 sllx tmp1, CLONGSHIFT, tmp1 ;\ 2170 add tsbarea, tmp1, tmp1 ;\ 2171 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 2172 srlx tmp1, tmp2, tmp1 ;\ 2173 btst 0x1, tmp1 ;\ 2174 bz,pn %xcc, label/**/2 ;\ 2175 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2176label/**/4: 2177 2178#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2179#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2180#endif 2181 2182/* 2183 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2184 * he offset for the corresponding hment. 2185 * Parameters: 2186 * In: 2187 * vaddr = register with virtual address 2188 * hmeblkpa = physical pointer to hme_blk 2189 * Out: 2190 * hmentoff = register where hment offset will be stored 2191 * hmemisc = hblk_misc 2192 * Scratch: 2193 * tmp1 2194 */ 2195#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2196 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2197 lda [hmentoff]ASI_MEM, hmemisc ;\ 2198 andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2199 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2200 or %g0, HMEBLK_HME1, hmentoff ;\ 2201 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2202 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2203 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2204 add tmp1, HMEBLK_HME1, hmentoff ;\ 2205label1: 2206 2207/* 2208 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2209 * 2210 * tagacc = (pseudo-)tag access register (in) 2211 * hatid = sfmmu pointer for TSB miss (in) 2212 * tte = tte for TLB miss if found, otherwise clobbered (out) 2213 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2214 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2215 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2216 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2217 * for this page size. 2218 * hashno = constant/register hash number 2219 * tmp = temp value - clobbered 2220 * label = temporary label for branching within macro. 2221 * foundlabel = label to jump to when tte is found. 2222 * suspendlabel= label to jump to when tte is suspended. 2223 * exitlabel = label to jump to when tte is not found. 2224 * 2225 */ 2226#define GET_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, hmeshift, \ 2227 hashno, tmp, label, foundlabel, suspendlabel, exitlabel) \ 2228 ;\ 2229 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2230 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2231 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2232 hmeblkpa, label/**/5, hmemisc, tmp) ;\ 2233 ;\ 2234 /* ;\ 2235 * tagacc = tagacc ;\ 2236 * hatid = hatid ;\ 2237 * tsbarea = tsbarea ;\ 2238 * tte = hmebp (hme bucket pointer) ;\ 2239 * hmeblkpa = vapg (virtual page) ;\ 2240 * hmemisc, tmp = scratch ;\ 2241 */ ;\ 2242 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2243 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2244 ;\ 2245 /* ;\ 2246 * tagacc = tagacc ;\ 2247 * hatid = hatid ;\ 2248 * tte = hmebp ;\ 2249 * hmeblkpa = CLOBBERED ;\ 2250 * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2251 * tmp = scratch ;\ 2252 */ ;\ 2253 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2254 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, \ 2255 tsbarea, tagacc, label/**/1) ;\ 2256 /* ;\ 2257 * tagacc = CLOBBERED ;\ 2258 * tte = CLOBBERED ;\ 2259 * hmeblkpa = hmeblkpa ;\ 2260 * tmp = scratch ;\ 2261 */ ;\ 2262 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2263 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\ 2264 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2265 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2266 nop ;\ 2267label/**/4: ;\ 2268 /* ;\ 2269 * We have found the hmeblk containing the hment. ;\ 2270 * Now we calculate the corresponding tte. ;\ 2271 * ;\ 2272 * tagacc = tagacc ;\ 2273 * hatid = hatid ;\ 2274 * tte = clobbered ;\ 2275 * hmeblkpa = hmeblkpa ;\ 2276 * hmemisc = hblktag ;\ 2277 * tmp = scratch ;\ 2278 */ ;\ 2279 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2280 label/**/2) ;\ 2281 ;\ 2282 /* ;\ 2283 * tagacc = tagacc ;\ 2284 * hatid = hmentoff ;\ 2285 * tte = clobbered ;\ 2286 * hmeblkpa = hmeblkpa ;\ 2287 * hmemisc = hblk_misc ;\ 2288 * tmp = scratch ;\ 2289 */ ;\ 2290 ;\ 2291 add hatid, SFHME_TTE, hatid ;\ 2292 add hmeblkpa, hatid, hmeblkpa ;\ 2293 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2294 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2295 set TTE_SUSPEND, hatid ;\ 2296 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2297 btst tte, hatid ;\ 2298 bz,pt %xcc, foundlabel ;\ 2299 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2300 ;\ 2301 /* ;\ 2302 * Mapping is suspended, so goto suspend label. ;\ 2303 */ ;\ 2304 ba,pt %xcc, suspendlabel ;\ 2305 nop 2306 2307/* 2308 * GET_SHME_TTE is similar to GET_TTE() except it searches 2309 * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2310 * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2311 * either 0 (not part of scd) or 1 (part of scd). 2312 */ 2313#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, \ 2314 hmeshift, hashno, tmp, label, foundlabel, \ 2315 suspendlabel, exitlabel) \ 2316 ;\ 2317 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2318 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2319 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2320 hmeblkpa, label/**/5, hmemisc, tmp) ;\ 2321 ;\ 2322 /* ;\ 2323 * tagacc = tagacc ;\ 2324 * hatid = hatid ;\ 2325 * tsbarea = tsbarea ;\ 2326 * tte = hmebp (hme bucket pointer) ;\ 2327 * hmeblkpa = vapg (virtual page) ;\ 2328 * hmemisc, tmp = scratch ;\ 2329 */ ;\ 2330 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2331 ;\ 2332 /* ;\ 2333 * tagacc = tagacc ;\ 2334 * hatid = hatid ;\ 2335 * tsbarea = tsbarea ;\ 2336 * tte = hmebp ;\ 2337 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2338 * hmeblkpa = CLOBBERED ;\ 2339 * tmp = scratch ;\ 2340 */ ;\ 2341 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2342 ;\ 2343 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2344 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2345 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2346 ;\ 2347label/**/8: ;\ 2348 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, \ 2349 tsbarea, tagacc, tte, label/**/1) ;\ 2350 /* ;\ 2351 * tagacc = CLOBBERED ;\ 2352 * tte = CLOBBERED ;\ 2353 * hmeblkpa = hmeblkpa ;\ 2354 * tmp = scratch ;\ 2355 */ ;\ 2356 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2357 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\ 2358 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2359 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2360 nop ;\ 2361label/**/4: ;\ 2362 /* ;\ 2363 * We have found the hmeblk containing the hment. ;\ 2364 * Now we calculate the corresponding tte. ;\ 2365 * ;\ 2366 * tagacc = tagacc ;\ 2367 * hatid = hatid ;\ 2368 * tte = clobbered ;\ 2369 * hmeblkpa = hmeblkpa ;\ 2370 * hmemisc = hblktag ;\ 2371 * tsbarea = tsbmiss area ;\ 2372 * tmp = scratch ;\ 2373 */ ;\ 2374 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2375 label/**/2) ;\ 2376 ;\ 2377 /* ;\ 2378 * tagacc = tagacc ;\ 2379 * hatid = hmentoff ;\ 2380 * tte = clobbered ;\ 2381 * hmeblkpa = hmeblkpa ;\ 2382 * hmemisc = hblk_misc ;\ 2383 * tsbarea = tsbmiss area ;\ 2384 * tmp = scratch ;\ 2385 */ ;\ 2386 ;\ 2387 add hatid, SFHME_TTE, hatid ;\ 2388 add hmeblkpa, hatid, hmeblkpa ;\ 2389 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2390 brlz,pt tte, label/**/6 ;\ 2391 nop ;\ 2392 btst HBLK_SZMASK, hmemisc ;\ 2393 bnz,a,pt %icc, label/**/7 ;\ 2394 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2395 ;\ 2396 /* ;\ 2397 * We found an invalid 8K tte in shme. ;\ 2398 * it may not belong to shme's region since ;\ 2399 * region size/alignment granularity is 8K but different ;\ 2400 * regions don't share hmeblks. Continue the search. ;\ 2401 */ ;\ 2402 sub hmeblkpa, hatid, hmeblkpa ;\ 2403 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2404 srlx tagacc, hmeshift, tte ;\ 2405 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2406 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2407 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2408 ba,a,pt %xcc, label/**/8 ;\ 2409label/**/6: ;\ 2410 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2411 /* ;\ 2412 * hmemisc is set to 1 if this is a shared mapping. It will ;\ 2413 * be cleared by CHECK_SHARED_PGSZ if this pagesize is not ;\ 2414 * allowed, in order to limit the number of entries in the ;\ 2415 * pagesize register. ;\ 2416 */ ;\ 2417 CHECK_SHARED_PGSZ(tsbarea, tte, hatid, hmemisc, label/**/9) ;\ 2418 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2419label/**/7: ;\ 2420 set TTE_SUSPEND, hatid ;\ 2421 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2422 btst tte, hatid ;\ 2423 bz,pt %xcc, foundlabel ;\ 2424 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2425 ;\ 2426 /* ;\ 2427 * Mapping is suspended, so goto suspend label. ;\ 2428 */ ;\ 2429 ba,pt %xcc, suspendlabel ;\ 2430 nop 2431 2432 /* 2433 * KERNEL PROTECTION HANDLER 2434 * 2435 * g1 = tsb8k pointer register (clobbered) 2436 * g2 = tag access register (ro) 2437 * g3 - g7 = scratch registers 2438 * 2439 * Note: This function is patched at runtime for performance reasons. 2440 * Any changes here require sfmmu_patch_ktsb fixed. 2441 */ 2442 ENTRY_NP(sfmmu_kprot_trap) 2443 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2444sfmmu_kprot_patch_ktsb_base: 2445 RUNTIME_PATCH_SETX(%g1, %g6) 2446 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2447sfmmu_kprot_patch_ktsb_szcode: 2448 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2449 2450 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2451 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2452 2453 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2454sfmmu_kprot_patch_ktsb4m_base: 2455 RUNTIME_PATCH_SETX(%g3, %g6) 2456 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2457sfmmu_kprot_patch_ktsb4m_szcode: 2458 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2459 2460 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2461 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2462 2463 CPU_TSBMISS_AREA(%g6, %g7) 2464 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2465 ba,pt %xcc, sfmmu_tsb_miss_tt 2466 nop 2467 2468 /* 2469 * USER PROTECTION HANDLER 2470 * 2471 * g1 = tsb8k pointer register (ro) 2472 * g2 = tag access register (ro) 2473 * g3 = faulting context (clobbered, currently not used) 2474 * g4 - g7 = scratch registers 2475 */ 2476 ALTENTRY(sfmmu_uprot_trap) 2477#ifdef sun4v 2478 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2479 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2480 2481 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2482 brlz,pt %g3, 9f /* check for 2nd TSB */ 2483 nop 2484 2485 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2486 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2487 2488#else /* sun4v */ 2489#ifdef UTSB_PHYS 2490 /* g1 = first TSB entry ptr */ 2491 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2492 brlz,pt %g3, 9f /* check for 2nd TSB */ 2493 nop 2494 2495 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2496 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2497#else /* UTSB_PHYS */ 2498 brgez,pt %g1, 9f /* check for 2nd TSB */ 2499 mov -1, %g3 /* set second tsbe ptr to -1 */ 2500 2501 mov %g2, %g7 2502 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2503 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2504 mov %g1, %g7 2505 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2506#endif /* UTSB_PHYS */ 2507#endif /* sun4v */ 25089: 2509 CPU_TSBMISS_AREA(%g6, %g7) 2510 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2511 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2512 nop 2513 2514 /* 2515 * Kernel 8K page iTLB miss. We also get here if we took a 2516 * fast instruction access mmu miss trap while running in 2517 * invalid context. 2518 * 2519 * %g1 = 8K TSB pointer register (not used, clobbered) 2520 * %g2 = tag access register (used) 2521 * %g3 = faulting context id (used) 2522 * %g7 = TSB tag to match (used) 2523 */ 2524 .align 64 2525 ALTENTRY(sfmmu_kitlb_miss) 2526 brnz,pn %g3, tsb_tl0_noctxt 2527 nop 2528 2529 /* kernel miss */ 2530 /* get kernel tsb pointer */ 2531 /* we patch the next set of instructions at run time */ 2532 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2533iktsbbase: 2534 RUNTIME_PATCH_SETX(%g4, %g5) 2535 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2536 2537iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2538 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2539 or %g4, %g1, %g1 ! form tsb ptr 2540 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2541 cmp %g4, %g7 2542 bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2543 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2544 2545 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2546 bz,pn %icc, exec_fault 2547 nop 2548 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2549 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2550 retry 2551 2552iktsb4mbase: 2553 RUNTIME_PATCH_SETX(%g4, %g6) 2554 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2555iktsb4m: 2556 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2557 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2558 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2559 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2560 cmp %g4, %g7 2561 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2562 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2563 bz,pn %icc, exec_fault 2564 nop 2565 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2566 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2567 retry 2568 2569 /* 2570 * Kernel dTLB miss. We also get here if we took a fast data 2571 * access mmu miss trap while running in invalid context. 2572 * 2573 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2574 * We select the TSB miss handler to branch to depending on 2575 * the virtual address of the access. In the future it may 2576 * be desirable to separate kpm TTEs into their own TSB, 2577 * in which case all that needs to be done is to set 2578 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2579 * early in the miss if we detect a kpm VA to a new handler. 2580 * 2581 * %g1 = 8K TSB pointer register (not used, clobbered) 2582 * %g2 = tag access register (used) 2583 * %g3 = faulting context id (used) 2584 */ 2585 .align 64 2586 ALTENTRY(sfmmu_kdtlb_miss) 2587 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2588 nop 2589 2590 /* Gather some stats for kpm misses in the TLB. */ 2591 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2592 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2593 2594 /* 2595 * Get first TSB offset and look for 8K/64K/512K mapping 2596 * using the 8K virtual page as the index. 2597 * 2598 * We patch the next set of instructions at run time; 2599 * any changes here require sfmmu_patch_ktsb changes too. 2600 */ 2601dktsbbase: 2602 RUNTIME_PATCH_SETX(%g7, %g6) 2603 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2604 2605dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2606 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2607 2608 /* 2609 * At this point %g1 is our index into the TSB. 2610 * We just masked off enough bits of the VA depending 2611 * on our TSB size code. 2612 */ 2613 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2614 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2615 cmp %g6, %g4 ! compare tag 2616 bne,pn %xcc, dktsb4m_kpmcheck_small 2617 add %g7, %g1, %g1 /* form tsb ptr */ 2618 TT_TRACE(trace_tsbhit) 2619 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2620 /* trapstat expects tte in %g5 */ 2621 retry 2622 2623 /* 2624 * If kpm is using large pages, the following instruction needs 2625 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2626 * so that we will probe the 4M TSB regardless of the VA. In 2627 * the case kpm is using small pages, we know no large kernel 2628 * mappings are located above 0x80000000.00000000 so we skip the 2629 * probe as an optimization. 2630 */ 2631dktsb4m_kpmcheck_small: 2632 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2633 /* delay slot safe, below */ 2634 2635 /* 2636 * Get second TSB offset and look for 4M mapping 2637 * using 4M virtual page as the TSB index. 2638 * 2639 * Here: 2640 * %g1 = 8K TSB pointer. Don't squash it. 2641 * %g2 = tag access register (we still need it) 2642 */ 2643 srlx %g2, MMU_PAGESHIFT4M, %g3 2644 2645 /* 2646 * We patch the next set of instructions at run time; 2647 * any changes here require sfmmu_patch_ktsb changes too. 2648 */ 2649dktsb4mbase: 2650 RUNTIME_PATCH_SETX(%g7, %g6) 2651 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2652dktsb4m: 2653 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2654 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2655 2656 /* 2657 * At this point %g3 is our index into the TSB. 2658 * We just masked off enough bits of the VA depending 2659 * on our TSB size code. 2660 */ 2661 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2662 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2663 cmp %g6, %g4 ! compare tag 2664 2665dktsb4m_tsbmiss: 2666 bne,pn %xcc, dktsb4m_kpmcheck 2667 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2668 TT_TRACE(trace_tsbhit) 2669 /* we don't check TTE size here since we assume 4M TSB is separate */ 2670 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2671 /* trapstat expects tte in %g5 */ 2672 retry 2673 2674 /* 2675 * So, we failed to find a valid TTE to match the faulting 2676 * address in either TSB. There are a few cases that could land 2677 * us here: 2678 * 2679 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2680 * to sfmmu_tsb_miss_tt to handle the miss. 2681 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2682 * 4M TSB. Let segkpm handle it. 2683 * 2684 * Note that we shouldn't land here in the case of a kpm VA when 2685 * kpm_smallpages is active -- we handled that case earlier at 2686 * dktsb4m_kpmcheck_small. 2687 * 2688 * At this point: 2689 * g1 = 8K-indexed primary TSB pointer 2690 * g2 = tag access register 2691 * g3 = 4M-indexed secondary TSB pointer 2692 */ 2693dktsb4m_kpmcheck: 2694 cmp %g2, %g0 2695 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2696 nop 2697 ba,a,pt %icc, sfmmu_tsb_miss_tt 2698 nop 2699 2700#ifdef sun4v 2701 /* 2702 * User instruction miss w/ single TSB. 2703 * The first probe covers 8K, 64K, and 512K page sizes, 2704 * because 64K and 512K mappings are replicated off 8K 2705 * pointer. 2706 * 2707 * g1 = tsb8k pointer register 2708 * g2 = tag access register 2709 * g3 - g6 = scratch registers 2710 * g7 = TSB tag to match 2711 */ 2712 .align 64 2713 ALTENTRY(sfmmu_uitlb_fastpath) 2714 2715 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2716 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2717 ba,pn %xcc, sfmmu_tsb_miss_tt 2718 mov -1, %g3 2719 2720 /* 2721 * User data miss w/ single TSB. 2722 * The first probe covers 8K, 64K, and 512K page sizes, 2723 * because 64K and 512K mappings are replicated off 8K 2724 * pointer. 2725 * 2726 * g1 = tsb8k pointer register 2727 * g2 = tag access register 2728 * g3 - g6 = scratch registers 2729 * g7 = TSB tag to match 2730 */ 2731 .align 64 2732 ALTENTRY(sfmmu_udtlb_fastpath) 2733 2734 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2735 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2736 ba,pn %xcc, sfmmu_tsb_miss_tt 2737 mov -1, %g3 2738 2739 /* 2740 * User instruction miss w/ multiple TSBs (sun4v). 2741 * The first probe covers 8K, 64K, and 512K page sizes, 2742 * because 64K and 512K mappings are replicated off 8K 2743 * pointer. Second probe covers 4M page size only. 2744 * 2745 * Just like sfmmu_udtlb_slowpath, except: 2746 * o Uses ASI_ITLB_IN 2747 * o checks for execute permission 2748 * o No ISM prediction. 2749 * 2750 * g1 = tsb8k pointer register 2751 * g2 = tag access register 2752 * g3 - g6 = scratch registers 2753 * g7 = TSB tag to match 2754 */ 2755 .align 64 2756 ALTENTRY(sfmmu_uitlb_slowpath) 2757 2758 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2759 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2760 /* g4 - g5 = clobbered here */ 2761 2762 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2763 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2764 srlx %g2, TAG_VALO_SHIFT, %g7 2765 PROBE_2ND_ITSB(%g3, %g7) 2766 /* NOT REACHED */ 2767 2768#else /* sun4v */ 2769 2770 /* 2771 * User instruction miss w/ multiple TSBs (sun4u). 2772 * The first probe covers 8K, 64K, and 512K page sizes, 2773 * because 64K and 512K mappings are replicated off 8K 2774 * pointer. Probe of 1st TSB has already been done prior to entry 2775 * into this routine. For the UTSB_PHYS case we probe up to 3 2776 * valid other TSBs in the following order: 2777 * 1) shared TSB for 4M-256M pages 2778 * 2) private TSB for 4M-256M pages 2779 * 3) shared TSB for 8K-512K pages 2780 * 2781 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs 2782 * 4M-256M pages. 2783 * 2784 * Just like sfmmu_udtlb_slowpath, except: 2785 * o Uses ASI_ITLB_IN 2786 * o checks for execute permission 2787 * o No ISM prediction. 2788 * 2789 * g1 = tsb8k pointer register 2790 * g2 = tag access register 2791 * g4 - g6 = scratch registers 2792 * g7 = TSB tag to match 2793 */ 2794 .align 64 2795 ALTENTRY(sfmmu_uitlb_slowpath) 2796 2797#ifdef UTSB_PHYS 2798 2799 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2800 brlz,pt %g6, 1f 2801 nop 2802 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2803 PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail) 28041: 2805 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2806 brlz,pt %g3, 2f 2807 nop 2808 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2809 PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail) 28102: 2811 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2812 brlz,pt %g6, sfmmu_tsb_miss_tt 2813 nop 2814 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2815 PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail) 2816 ba,pn %xcc, sfmmu_tsb_miss_tt 2817 nop 2818 2819#else /* UTSB_PHYS */ 2820 mov %g1, %g3 /* save tsb8k reg in %g3 */ 2821 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 2822 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2823 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 2824 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 2825 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 2826 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2827 srlx %g2, TAG_VALO_SHIFT, %g7 2828 PROBE_2ND_ITSB(%g3, %g7, isynth) 2829 ba,pn %xcc, sfmmu_tsb_miss_tt 2830 nop 2831 2832#endif /* UTSB_PHYS */ 2833#endif /* sun4v */ 2834 2835#if defined(sun4u) && defined(UTSB_PHYS) 2836 2837 /* 2838 * We come here for ism predict DTLB_MISS case or if 2839 * if probe in first TSB failed. 2840 */ 2841 2842 .align 64 2843 ALTENTRY(sfmmu_udtlb_slowpath_noismpred) 2844 2845 /* 2846 * g1 = tsb8k pointer register 2847 * g2 = tag access register 2848 * g4 - %g6 = scratch registers 2849 * g7 = TSB tag to match 2850 */ 2851 2852 /* 2853 * ISM non-predict probe order 2854 * probe 1ST_TSB (8K index) 2855 * probe 2ND_TSB (4M index) 2856 * probe 4TH_TSB (4M index) 2857 * probe 3RD_TSB (8K index) 2858 * 2859 * We already probed first TSB in DTLB_MISS handler. 2860 */ 2861 2862 /* 2863 * Private 2ND TSB 4M-256 pages 2864 */ 2865 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2866 brlz,pt %g3, 1f 2867 nop 2868 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2869 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2870 2871 /* 2872 * Shared Context 4TH TSB 4M-256 pages 2873 */ 28741: 2875 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2876 brlz,pt %g6, 2f 2877 nop 2878 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2879 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail) 2880 2881 /* 2882 * Shared Context 3RD TSB 8K-512K pages 2883 */ 28842: 2885 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2886 brlz,pt %g6, sfmmu_tsb_miss_tt 2887 nop 2888 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2889 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail) 2890 ba,pn %xcc, sfmmu_tsb_miss_tt 2891 nop 2892 2893 .align 64 2894 ALTENTRY(sfmmu_udtlb_slowpath_ismpred) 2895 2896 /* 2897 * g1 = tsb8k pointer register 2898 * g2 = tag access register 2899 * g4 - g6 = scratch registers 2900 * g7 = TSB tag to match 2901 */ 2902 2903 /* 2904 * ISM predict probe order 2905 * probe 4TH_TSB (4M index) 2906 * probe 2ND_TSB (4M index) 2907 * probe 1ST_TSB (8K index) 2908 * probe 3RD_TSB (8K index) 2909 2910 /* 2911 * Shared Context 4TH TSB 4M-256 pages 2912 */ 2913 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2914 brlz,pt %g6, 4f 2915 nop 2916 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2917 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2) 2918 2919 /* 2920 * Private 2ND TSB 4M-256 pages 2921 */ 29224: 2923 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2924 brlz,pt %g3, 5f 2925 nop 2926 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2927 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2) 2928 29295: 2930 PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2) 2931 2932 /* 2933 * Shared Context 3RD TSB 8K-512K pages 2934 */ 2935 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2936 brlz,pt %g6, 6f 2937 nop 2938 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2939 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2) 29406: 2941 ba,pn %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */ 2942 nop 2943 2944#else /* sun4u && UTSB_PHYS */ 2945 2946 .align 64 2947 ALTENTRY(sfmmu_udtlb_slowpath) 2948 2949 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 2950 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 2951 mov %g1, %g3 2952 2953udtlb_miss_probefirst: 2954 /* 2955 * g1 = 8K TSB pointer register 2956 * g2 = tag access register 2957 * g3 = (potentially) second TSB entry ptr 2958 * g6 = ism pred. 2959 * g7 = vpg_4m 2960 */ 2961#ifdef sun4v 2962 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2963 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2964 2965 /* 2966 * Here: 2967 * g1 = first TSB pointer 2968 * g2 = tag access reg 2969 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2970 */ 2971 brgz,pn %g6, sfmmu_tsb_miss_tt 2972 nop 2973#else /* sun4v */ 2974 mov %g1, %g4 2975 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 2976 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2977 2978 /* 2979 * Here: 2980 * g1 = first TSB pointer 2981 * g2 = tag access reg 2982 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2983 */ 2984 brgz,pn %g6, sfmmu_tsb_miss_tt 2985 nop 2986 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 2987 /* fall through in 8K->4M probe order */ 2988#endif /* sun4v */ 2989 2990udtlb_miss_probesecond: 2991 /* 2992 * Look in the second TSB for the TTE 2993 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 2994 * g2 = tag access reg 2995 * g3 = 8K TSB pointer register 2996 * g6 = ism pred. 2997 * g7 = vpg_4m 2998 */ 2999#ifdef sun4v 3000 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 3001 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3002 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 3003#else /* sun4v */ 3004 mov %g3, %g7 3005 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 3006 /* %g2 clobbered, %g3 =second tsbe ptr */ 3007 mov MMU_TAG_ACCESS, %g2 3008 ldxa [%g2]ASI_DMMU, %g2 3009#endif /* sun4v */ 3010 3011 srlx %g2, TAG_VALO_SHIFT, %g7 3012 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3013 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 3014 brgz,pn %g6, udtlb_miss_probefirst 3015 nop 3016 3017 /* fall through to sfmmu_tsb_miss_tt */ 3018#endif /* sun4u && UTSB_PHYS */ 3019 3020 3021 ALTENTRY(sfmmu_tsb_miss_tt) 3022 TT_TRACE(trace_tsbmiss) 3023 /* 3024 * We get here if there is a TSB miss OR a write protect trap. 3025 * 3026 * g1 = First TSB entry pointer 3027 * g2 = tag access register 3028 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 3029 * g4 - g7 = scratch registers 3030 */ 3031 3032 ALTENTRY(sfmmu_tsb_miss) 3033 3034 /* 3035 * If trapstat is running, we need to shift the %tpc and %tnpc to 3036 * point to trapstat's TSB miss return code (note that trapstat 3037 * itself will patch the correct offset to add). 3038 */ 3039 rdpr %tl, %g7 3040 cmp %g7, 1 3041 ble,pt %xcc, 0f 3042 sethi %hi(KERNELBASE), %g6 3043 rdpr %tpc, %g7 3044 or %g6, %lo(KERNELBASE), %g6 3045 cmp %g7, %g6 3046 bgeu,pt %xcc, 0f 3047 /* delay slot safe */ 3048 3049 ALTENTRY(tsbmiss_trapstat_patch_point) 3050 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 3051 wrpr %g7, %tpc 3052 add %g7, 4, %g7 3053 wrpr %g7, %tnpc 30540: 3055 CPU_TSBMISS_AREA(%g6, %g7) 3056 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save 1ST tsb pointer */ 3057 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save 2ND tsb pointer */ 3058 3059 sllx %g2, TAGACC_CTX_LSHIFT, %g3 3060 brz,a,pn %g3, 1f /* skip ahead if kernel */ 3061 ldn [%g6 + TSBMISS_KHATID], %g7 3062 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 3063 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 3064 3065 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 3066 3067 cmp %g3, INVALID_CONTEXT 3068 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 3069 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 3070 3071#if defined(sun4v) || defined(UTSB_PHYS) 3072 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 3073 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 3074 stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 3075#endif /* sun4v || UTSB_PHYS */ 3076 3077 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 3078 /* 3079 * The miss wasn't in an ISM segment. 3080 * 3081 * %g1 %g3, %g4, %g5, %g7 all clobbered 3082 * %g2 = (pseudo) tag access 3083 */ 3084 3085 ba,pt %icc, 2f 3086 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 3087 30881: 3089 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 3090 /* 3091 * 8K and 64K hash. 3092 */ 30932: 3094 3095 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3096 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_l8K, tsb_checktte, 3097 sfmmu_suspend_tl, tsb_512K) 3098 /* NOT REACHED */ 3099 3100tsb_512K: 3101 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3102 brz,pn %g5, 3f 3103 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3104 and %g4, HAT_512K_FLAG, %g5 3105 3106 /* 3107 * Note that there is a small window here where we may have 3108 * a 512k page in the hash list but have not set the HAT_512K_FLAG 3109 * flag yet, so we will skip searching the 512k hash list. 3110 * In this case we will end up in pagefault which will find 3111 * the mapping and return. So, in this instance we will end up 3112 * spending a bit more time resolving this TSB miss, but it can 3113 * only happen once per process and even then, the chances of that 3114 * are very small, so it's not worth the extra overhead it would 3115 * take to close this window. 3116 */ 3117 brz,pn %g5, tsb_4M 3118 nop 31193: 3120 /* 3121 * 512K hash 3122 */ 3123 3124 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3125 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_l512K, tsb_checktte, 3126 sfmmu_suspend_tl, tsb_4M) 3127 /* NOT REACHED */ 3128 3129tsb_4M: 3130 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3131 brz,pn %g5, 4f 3132 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3133 and %g4, HAT_4M_FLAG, %g5 3134 brz,pn %g5, tsb_32M 3135 nop 31364: 3137 /* 3138 * 4M hash 3139 */ 3140 3141 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3142 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_l4M, tsb_checktte, 3143 sfmmu_suspend_tl, tsb_32M) 3144 /* NOT REACHED */ 3145 3146tsb_32M: 3147 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3148#ifdef sun4v 3149 brz,pn %g5, 6f 3150#else 3151 brz,pn %g5, tsb_pagefault 3152#endif 3153 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3154 and %g4, HAT_32M_FLAG, %g5 3155 brz,pn %g5, tsb_256M 3156 nop 31575: 3158 /* 3159 * 32M hash 3160 */ 3161 3162 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3163 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_l32M, tsb_checktte, 3164 sfmmu_suspend_tl, tsb_256M) 3165 /* NOT REACHED */ 3166 3167#if defined(sun4u) && !defined(UTSB_PHYS) 3168#define tsb_shme tsb_pagefault 3169#endif 3170tsb_256M: 3171 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3172 and %g4, HAT_256M_FLAG, %g5 3173 brz,pn %g5, tsb_shme 3174 nop 31756: 3176 /* 3177 * 256M hash 3178 */ 3179 3180 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3181 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_l256M, tsb_checktte, 3182 sfmmu_suspend_tl, tsb_shme) 3183 /* NOT REACHED */ 3184 3185tsb_checktte: 3186 /* 3187 * g1 = hblk_misc 3188 * g2 = tagacc 3189 * g3 = tte 3190 * g4 = tte pa 3191 * g6 = tsbmiss area 3192 * g7 = hatid 3193 */ 3194 brlz,a,pt %g3, tsb_validtte 3195 rdpr %tt, %g7 3196 3197#if defined(sun4u) && !defined(UTSB_PHYS) 3198#undef tsb_shme 3199 ba tsb_pagefault 3200 nop 3201#else /* sun4u && !UTSB_PHYS */ 3202 3203tsb_shme: 3204 /* 3205 * g2 = tagacc 3206 * g6 = tsbmiss area 3207 */ 3208 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3209 brz,pn %g5, tsb_pagefault 3210 nop 3211 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3212 brz,pn %g7, tsb_pagefault 3213 nop 3214 3215 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3216 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_shme_l8K, tsb_shme_checktte, 3217 sfmmu_suspend_tl, tsb_shme_512K) 3218 /* NOT REACHED */ 3219 3220tsb_shme_512K: 3221 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3222 and %g4, HAT_512K_FLAG, %g5 3223 brz,pn %g5, tsb_shme_4M 3224 nop 3225 3226 /* 3227 * 512K hash 3228 */ 3229 3230 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3231 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_shme_l512K, tsb_shme_checktte, 3232 sfmmu_suspend_tl, tsb_shme_4M) 3233 /* NOT REACHED */ 3234 3235tsb_shme_4M: 3236 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3237 and %g4, HAT_4M_FLAG, %g5 3238 brz,pn %g5, tsb_shme_32M 3239 nop 32404: 3241 /* 3242 * 4M hash 3243 */ 3244 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3245 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_shme_l4M, tsb_shme_checktte, 3246 sfmmu_suspend_tl, tsb_shme_32M) 3247 /* NOT REACHED */ 3248 3249tsb_shme_32M: 3250 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3251 and %g4, HAT_32M_FLAG, %g5 3252 brz,pn %g5, tsb_shme_256M 3253 nop 3254 3255 /* 3256 * 32M hash 3257 */ 3258 3259 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3260 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_shme_l32M, tsb_shme_checktte, 3261 sfmmu_suspend_tl, tsb_shme_256M) 3262 /* NOT REACHED */ 3263 3264tsb_shme_256M: 3265 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3266 and %g4, HAT_256M_FLAG, %g5 3267 brz,pn %g5, tsb_pagefault 3268 nop 3269 3270 /* 3271 * 256M hash 3272 */ 3273 3274 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3275 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_shme_l256M, tsb_shme_checktte, 3276 sfmmu_suspend_tl, tsb_pagefault) 3277 /* NOT REACHED */ 3278 3279tsb_shme_checktte: 3280 3281 brgez,pn %g3, tsb_pagefault 3282 rdpr %tt, %g7 3283 /* 3284 * g1 = ctx1 flag 3285 * g3 = tte 3286 * g4 = tte pa 3287 * g6 = tsbmiss area 3288 * g7 = tt 3289 */ 3290 3291 brz,pt %g1, tsb_validtte 3292 nop 3293 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3294 or %g1, HAT_CHKCTX1_FLAG, %g1 3295 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3296 3297 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3298 ba tsb_validtte 3299#endif /* sun4u && !UTSB_PHYS */ 3300 3301tsb_ism_validtte: 3302#ifdef sun4v 3303 /* 3304 * Check pagesize against bitmap for Rock page size register, 3305 * for ism mappings. 3306 * 3307 * %g1, %g2 = scratch 3308 * %g3 = tte 3309 * g4 = tte pa 3310 * g5 = tte va 3311 * g6 = tsbmiss area 3312 * %g7 = tt 3313 */ 3314 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3315 and %g1, HAT_CHKCTX1_FLAG, %g2 3316 /* 3317 * Clear the HAT_CHKCTX1_FLAG in %g2 if this shared pagesize is not allowed 3318 * to limit the number of entries in the pagesize search register. 3319 */ 3320 CHECK_SHARED_PGSZ(%g6, %g3, %g7, %g2, ism_chk_pgsz) 3321 andn %g1, HAT_CHKCTX1_FLAG, %g1 3322 or %g1, %g2, %g1 3323 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3324 brz %g2, tsb_validtte 3325 rdpr %tt, %g7 3326 SAVE_CTX1(%g7, %g1, %g2, tsb_shctxl) 3327#endif /* sun4v */ 3328 3329tsb_validtte: 3330 /* 3331 * g3 = tte 3332 * g4 = tte pa 3333 * g6 = tsbmiss area 3334 * g7 = tt 3335 */ 3336 3337 /* 3338 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3339 */ 3340 cmp %g7, FAST_PROT_TT 3341 bne,pt %icc, 4f 3342 nop 3343 3344 TTE_SET_REFMOD_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_refmod, 3345 tsb_protfault) 3346 3347 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3348#ifdef sun4v 3349 MMU_FAULT_STATUS_AREA(%g7) 3350 ldx [%g7 + MMFSA_D_ADDR], %g5 /* load fault addr for later */ 3351#else /* sun4v */ 3352 mov MMU_TAG_ACCESS, %g5 3353 ldxa [%g5]ASI_DMMU, %g5 3354#endif /* sun4v */ 3355 ba,pt %xcc, tsb_update_tl1 3356 nop 33574: 3358 /* 3359 * ITLB translation was found but execute permission is 3360 * disabled. If we have software execute permission (soft exec 3361 * bit is set), then enable hardware execute permission. 3362 * Otherwise continue with a protection violation. 3363 */ 3364 cmp %g7, T_INSTR_MMU_MISS 3365 be,pn %icc, 5f 3366 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3367 cmp %g7, FAST_IMMU_MISS_TT 3368 bne,pt %icc, 3f 3369 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 33705: 3371 bnz,pn %icc, 3f 3372 TTE_CHK_SOFTEXEC_ML(%g3) /* check soft execute */ 3373 bz,pn %icc, tsb_protfault 3374 nop 3375 TTE_SET_EXEC_ML(%g3, %g4, %g7, tsb_lset_exec) 33763: 3377 /* 3378 * Set reference bit if not already set 3379 */ 3380 TTE_SET_REF_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_ref) 3381 3382 /* 3383 * Now, load into TSB/TLB. At this point: 3384 * g3 = tte 3385 * g4 = patte 3386 * g6 = tsbmiss area 3387 */ 3388 rdpr %tt, %g7 3389#ifdef sun4v 3390 MMU_FAULT_STATUS_AREA(%g2) 3391 cmp %g7, T_INSTR_MMU_MISS 3392 be,a,pt %icc, 9f 3393 nop 3394 cmp %g7, FAST_IMMU_MISS_TT 3395 be,a,pt %icc, 9f 3396 nop 3397 add %g2, MMFSA_D_, %g2 33989: 3399 ldx [%g2 + MMFSA_CTX_], %g7 3400 sllx %g7, TTARGET_CTX_SHIFT, %g7 3401 ldx [%g2 + MMFSA_ADDR_], %g2 3402 mov %g2, %g5 ! load the fault addr for later use 3403 srlx %g2, TTARGET_VA_SHIFT, %g2 3404 or %g2, %g7, %g2 3405#else /* sun4v */ 3406 mov MMU_TAG_ACCESS, %g5 3407 cmp %g7, FAST_IMMU_MISS_TT 3408 be,a,pt %icc, 9f 3409 ldxa [%g0]ASI_IMMU, %g2 3410 ldxa [%g0]ASI_DMMU, %g2 3411 ba,pt %icc, tsb_update_tl1 3412 ldxa [%g5]ASI_DMMU, %g5 34139: 3414 ldxa [%g5]ASI_IMMU, %g5 3415#endif /* sun4v */ 3416 3417tsb_update_tl1: 3418 TTE_CLR_SOFTEXEC_ML(%g3) 3419 srlx %g2, TTARGET_CTX_SHIFT, %g7 3420 brz,pn %g7, tsb_kernel 3421#ifdef sun4v 3422 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3423#else /* sun4v */ 3424 srlx %g3, TTE_SZ_SHFT, %g7 3425#endif /* sun4v */ 3426 3427tsb_user: 3428#ifdef sun4v 3429 cmp %g7, TTE4M 3430 bge,pn %icc, tsb_user4m 3431 nop 3432#else /* sun4v */ 3433 cmp %g7, TTESZ_VALID | TTE4M 3434 be,pn %icc, tsb_user4m 3435 srlx %g3, TTE_SZ2_SHFT, %g7 3436 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3437#ifdef ITLB_32M_256M_SUPPORT 3438 bnz,pn %icc, tsb_user4m 3439 nop 3440#else /* ITLB_32M_256M_SUPPORT */ 3441 bnz,a,pn %icc, tsb_user_pn_synth 3442 nop 3443#endif /* ITLB_32M_256M_SUPPORT */ 3444#endif /* sun4v */ 3445 3446tsb_user8k: 3447#if defined(sun4v) || defined(UTSB_PHYS) 3448 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3449 and %g7, HAT_CHKCTX1_FLAG, %g1 3450 brz,a,pn %g1, 1f 3451 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3452 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3453 brlz,a,pn %g1, ptl1_panic ! if no shared 3RD tsb 3454 mov PTL1_NO_SCDTSB8K, %g1 ! panic 3455 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 34561: 3457#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3458 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3459#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3460 3461#ifndef UTSB_PHYS 3462 mov ASI_N, %g7 ! user TSBs accessed by VA 3463 mov %g7, %asi 3464#endif /* !UTSB_PHYS */ 3465 3466 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l3) 3467 3468 rdpr %tt, %g5 3469#ifdef sun4v 3470 cmp %g5, T_INSTR_MMU_MISS 3471 be,a,pn %xcc, 9f 3472 mov %g3, %g5 3473#endif /* sun4v */ 3474 cmp %g5, FAST_IMMU_MISS_TT 3475 be,pn %xcc, 9f 3476 mov %g3, %g5 3477 3478 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3479 ! trapstat wants TTE in %g5 3480 retry 34819: 3482 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3483 ! trapstat wants TTE in %g5 3484 retry 3485 3486tsb_user4m: 3487#if defined(sun4v) || defined(UTSB_PHYS) 3488 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3489 and %g7, HAT_CHKCTX1_FLAG, %g1 3490 brz,a,pn %g1, 4f 3491 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3492 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr 3493 brlz,a,pn %g1, 5f ! if no shared 4TH TSB 3494 nop 3495 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 3496 3497#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3498 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3499#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 35004: 3501 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3502 nop 3503 3504#ifndef UTSB_PHYS 3505 mov ASI_N, %g7 ! user TSBs accessed by VA 3506 mov %g7, %asi 3507#endif /* UTSB_PHYS */ 3508 3509 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l4) 3510 35115: 3512 rdpr %tt, %g5 3513#ifdef sun4v 3514 cmp %g5, T_INSTR_MMU_MISS 3515 be,a,pn %xcc, 9f 3516 mov %g3, %g5 3517#endif /* sun4v */ 3518 cmp %g5, FAST_IMMU_MISS_TT 3519 be,pn %xcc, 9f 3520 mov %g3, %g5 3521 3522 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3523 ! trapstat wants TTE in %g5 3524 retry 35259: 3526 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3527 ! trapstat wants TTE in %g5 3528 retry 3529 3530#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3531 /* 3532 * Panther ITLB synthesis. 3533 * The Panther 32M and 256M ITLB code simulates these two large page 3534 * sizes with 4M pages, to provide support for programs, for example 3535 * Java, that may copy instructions into a 32M or 256M data page and 3536 * then execute them. The code below generates the 4M pfn bits and 3537 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3538 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3539 * are ignored by the hardware. 3540 * 3541 * Now, load into TSB/TLB. At this point: 3542 * g2 = tagtarget 3543 * g3 = tte 3544 * g4 = patte 3545 * g5 = tt 3546 * g6 = tsbmiss area 3547 */ 3548tsb_user_pn_synth: 3549 rdpr %tt, %g5 3550 cmp %g5, FAST_IMMU_MISS_TT 3551 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3552 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3553 bz,pn %icc, 4b /* if not, been here before */ 3554 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3555 brlz,a,pn %g1, 5f /* no 2nd tsb */ 3556 mov %g3, %g5 3557 3558 mov MMU_TAG_ACCESS, %g7 3559 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3560 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3561 3562 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3563 mov %g7, %asi 3564 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l5) /* update TSB */ 35655: 3566 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3567 retry 3568 3569tsb_user_itlb_synth: 3570 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = 2ND TSB */ 3571 3572 mov MMU_TAG_ACCESS, %g7 3573 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3574 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3575 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3576 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3577 3578 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3579 mov %g7, %asi 3580 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l6) /* update TSB */ 35817: 3582 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3583 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3584 retry 3585#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3586 3587tsb_kernel: 3588 rdpr %tt, %g5 3589#ifdef sun4v 3590 cmp %g7, TTE4M 3591 bge,pn %icc, 5f 3592#else 3593 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3594 be,pn %icc, 5f 3595#endif /* sun4v */ 3596 nop 3597 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8K TSB ptr 3598 ba,pt %xcc, 6f 3599 nop 36005: 3601 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4M TSB ptr 3602 brlz,pn %g1, 3f /* skip programming if 4M TSB ptr is -1 */ 3603 nop 36046: 3605#ifndef sun4v 3606tsb_kernel_patch_asi: 3607 or %g0, RUNTIME_PATCH, %g6 3608 mov %g6, %asi ! XXX avoid writing to %asi !! 3609#endif 3610 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l7) 36113: 3612#ifdef sun4v 3613 cmp %g5, T_INSTR_MMU_MISS 3614 be,a,pn %icc, 1f 3615 mov %g3, %g5 ! trapstat wants TTE in %g5 3616#endif /* sun4v */ 3617 cmp %g5, FAST_IMMU_MISS_TT 3618 be,pn %icc, 1f 3619 mov %g3, %g5 ! trapstat wants TTE in %g5 3620 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3621 ! trapstat wants TTE in %g5 3622 retry 36231: 3624 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3625 ! trapstat wants TTE in %g5 3626 retry 3627 3628tsb_ism: 3629 /* 3630 * This is an ISM [i|d]tlb miss. We optimize for largest 3631 * page size down to smallest. 3632 * 3633 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3634 * register 3635 * g3 = ismmap->ism_seg 3636 * g4 = physical address of ismmap->ism_sfmmu 3637 * g6 = tsbmiss area 3638 */ 3639 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3640 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3641 mov PTL1_BAD_ISM, %g1 3642 /* g5 = pa of imap_vb_shift */ 3643 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3644 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3645 srlx %g3, %g4, %g3 /* clr size field */ 3646 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3647 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3648 and %g2, %g1, %g4 /* g4 = ctx number */ 3649 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3650 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3651 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3652 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3653 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3654#if defined(sun4v) || defined(UTSB_PHYS) 3655 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3656 brz,pt %g5, tsb_chk4M_ism 3657 nop 3658 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3659 or %g5, HAT_CHKCTX1_FLAG, %g5 3660 stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3661#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3662 /* 3663 * ISM pages are always locked down. 3664 * If we can't find the tte then pagefault 3665 * and let the spt segment driver resolve it. 3666 * 3667 * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3668 * g4 = imap_hatflags 3669 * g6 = tsb miss area 3670 * g7 = ISM hatid 3671 */ 3672 3673tsb_chk4M_ism: 3674 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3675 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3676 nop 3677 3678tsb_ism_32M: 3679 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3680 brz,pn %g5, tsb_ism_256M 3681 nop 3682 3683 /* 3684 * 32M hash. 3685 */ 3686 3687 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT32M, 3688 TTE32M, %g5, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3689 tsb_ism_4M) 3690 /* NOT REACHED */ 3691 3692tsb_ism_32M_found: 3693 brlz,a,pt %g3, tsb_ism_validtte 3694 rdpr %tt, %g7 3695 ba,pt %xcc, tsb_ism_4M 3696 nop 3697 3698tsb_ism_256M: 3699 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3700 brz,a,pn %g5, ptl1_panic 3701 mov PTL1_BAD_ISM, %g1 3702 3703 /* 3704 * 256M hash. 3705 */ 3706 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT256M, 3707 TTE256M, %g5, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3708 tsb_ism_4M) 3709 3710tsb_ism_256M_found: 3711 brlz,a,pt %g3, tsb_ism_validtte 3712 rdpr %tt, %g7 3713 3714tsb_ism_4M: 3715 /* 3716 * 4M hash. 3717 */ 3718 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT4M, 3719 TTE4M, %g5, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3720 tsb_ism_8K) 3721 /* NOT REACHED */ 3722 3723tsb_ism_4M_found: 3724 brlz,a,pt %g3, tsb_ism_validtte 3725 rdpr %tt, %g7 3726 3727tsb_ism_8K: 3728 /* 3729 * 8K and 64K hash. 3730 */ 3731 3732 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT64K, 3733 TTE64K, %g5, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3734 tsb_pagefault) 3735 /* NOT REACHED */ 3736 3737tsb_ism_8K_found: 3738 brlz,a,pt %g3, tsb_ism_validtte 3739 rdpr %tt, %g7 3740 3741tsb_pagefault: 3742 rdpr %tt, %g7 3743 cmp %g7, FAST_PROT_TT 3744 be,a,pn %icc, tsb_protfault 3745 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3746 3747tsb_protfault: 3748 /* 3749 * we get here if we couldn't find a valid tte in the hash. 3750 * 3751 * If user and we are at tl>1 we go to window handling code. 3752 * 3753 * If kernel and the fault is on the same page as our stack 3754 * pointer, then we know the stack is bad and the trap handler 3755 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3756 * 3757 * If this is a kernel trap and tl>1, panic. 3758 * 3759 * Otherwise we call pagefault. 3760 */ 3761 cmp %g7, FAST_IMMU_MISS_TT 3762#ifdef sun4v 3763 MMU_FAULT_STATUS_AREA(%g4) 3764 ldx [%g4 + MMFSA_I_CTX], %g5 3765 ldx [%g4 + MMFSA_D_CTX], %g4 3766 move %icc, %g5, %g4 3767 cmp %g7, T_INSTR_MMU_MISS 3768 move %icc, %g5, %g4 3769#else 3770 mov MMU_TAG_ACCESS, %g4 3771 ldxa [%g4]ASI_DMMU, %g2 3772 ldxa [%g4]ASI_IMMU, %g5 3773 move %icc, %g5, %g2 3774 cmp %g7, T_INSTR_MMU_MISS 3775 move %icc, %g5, %g2 3776 sllx %g2, TAGACC_CTX_LSHIFT, %g4 3777#endif /* sun4v */ 3778 brnz,pn %g4, 3f /* skip if not kernel */ 3779 rdpr %tl, %g5 3780 3781 add %sp, STACK_BIAS, %g3 3782 srlx %g3, MMU_PAGESHIFT, %g3 3783 srlx %g2, MMU_PAGESHIFT, %g4 3784 cmp %g3, %g4 3785 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 3786 mov PTL1_BAD_STACK, %g1 3787 3788 cmp %g5, 1 3789 ble,pt %icc, 2f 3790 nop 3791 TSTAT_CHECK_TL1(2f, %g1, %g2) 3792 rdpr %tt, %g2 3793 cmp %g2, FAST_PROT_TT 3794 mov PTL1_BAD_KPROT_FAULT, %g1 3795 movne %icc, PTL1_BAD_KMISS, %g1 3796 ba,pt %icc, ptl1_panic 3797 nop 3798 37992: 3800 /* 3801 * We are taking a pagefault in the kernel on a kernel address. If 3802 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 3803 * want to call sfmmu_pagefault -- we will instead note that a fault 3804 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 3805 * (instead of a "retry"). This will step over the faulting 3806 * instruction. 3807 */ 3808 CPU_INDEX(%g1, %g2) 3809 set cpu_core, %g2 3810 sllx %g1, CPU_CORE_SHIFT, %g1 3811 add %g1, %g2, %g1 3812 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3813 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3814 bz sfmmu_pagefault 3815 or %g2, CPU_DTRACE_BADADDR, %g2 3816 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3817 GET_MMU_D_ADDR(%g3, %g4) 3818 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3819 done 3820 38213: 3822 cmp %g5, 1 3823 ble,pt %icc, 4f 3824 nop 3825 TSTAT_CHECK_TL1(4f, %g1, %g2) 3826 ba,pt %icc, sfmmu_window_trap 3827 nop 3828 38294: 3830 /* 3831 * We are taking a pagefault on a non-kernel address. If we are in 3832 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 3833 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 3834 */ 3835 CPU_INDEX(%g1, %g2) 3836 set cpu_core, %g2 3837 sllx %g1, CPU_CORE_SHIFT, %g1 3838 add %g1, %g2, %g1 3839 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3840 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3841 bz sfmmu_mmu_trap 3842 or %g2, CPU_DTRACE_BADADDR, %g2 3843 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3844 GET_MMU_D_ADDR(%g3, %g4) 3845 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3846 3847 /* 3848 * Be sure that we're actually taking this miss from the kernel -- 3849 * otherwise we have managed to return to user-level with 3850 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3851 */ 3852 rdpr %tstate, %g2 3853 btst TSTATE_PRIV, %g2 3854 bz,a ptl1_panic 3855 mov PTL1_BAD_DTRACE_FLAGS, %g1 3856 done 3857 3858 ALTENTRY(tsb_tl0_noctxt) 3859 /* 3860 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 3861 * if it is, indicated that we have faulted and issue a done. 3862 */ 3863 CPU_INDEX(%g5, %g6) 3864 set cpu_core, %g6 3865 sllx %g5, CPU_CORE_SHIFT, %g5 3866 add %g5, %g6, %g5 3867 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 3868 andcc %g6, CPU_DTRACE_NOFAULT, %g0 3869 bz 1f 3870 or %g6, CPU_DTRACE_BADADDR, %g6 3871 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 3872 GET_MMU_D_ADDR(%g3, %g4) 3873 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 3874 3875 /* 3876 * Be sure that we're actually taking this miss from the kernel -- 3877 * otherwise we have managed to return to user-level with 3878 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3879 */ 3880 rdpr %tstate, %g5 3881 btst TSTATE_PRIV, %g5 3882 bz,a ptl1_panic 3883 mov PTL1_BAD_DTRACE_FLAGS, %g1 3884 TSTAT_CHECK_TL1(2f, %g1, %g2); 38852: 3886 done 3887 38881: 3889 rdpr %tt, %g5 3890 cmp %g5, FAST_IMMU_MISS_TT 3891#ifdef sun4v 3892 MMU_FAULT_STATUS_AREA(%g2) 3893 be,a,pt %icc, 2f 3894 ldx [%g2 + MMFSA_I_CTX], %g3 3895 cmp %g5, T_INSTR_MMU_MISS 3896 be,a,pt %icc, 2f 3897 ldx [%g2 + MMFSA_I_CTX], %g3 3898 ldx [%g2 + MMFSA_D_CTX], %g3 38992: 3900#else 3901 mov MMU_TAG_ACCESS, %g2 3902 be,a,pt %icc, 2f 3903 ldxa [%g2]ASI_IMMU, %g3 3904 ldxa [%g2]ASI_DMMU, %g3 39052: sllx %g3, TAGACC_CTX_LSHIFT, %g3 3906#endif /* sun4v */ 3907 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 3908 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 3909 rdpr %tl, %g5 3910 cmp %g5, 1 3911 ble,pt %icc, sfmmu_mmu_trap 3912 nop 3913 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 3914 ba,pt %icc, sfmmu_window_trap 3915 nop 3916 SET_SIZE(sfmmu_tsb_miss) 3917#endif /* lint */ 3918 3919#if defined (lint) 3920/* 3921 * This routine will look for a user or kernel vaddr in the hash 3922 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 3923 * grab any locks. It should only be used by other sfmmu routines. 3924 */ 3925/* ARGSUSED */ 3926pfn_t 3927sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 3928{ 3929 return(0); 3930} 3931 3932/* ARGSUSED */ 3933pfn_t 3934sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno) 3935{ 3936 return(0); 3937} 3938 3939#else /* lint */ 3940 3941 ENTRY_NP(sfmmu_vatopfn) 3942 /* 3943 * disable interrupts 3944 */ 3945 rdpr %pstate, %o3 3946#ifdef DEBUG 3947 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 3948#endif 3949 /* 3950 * disable interrupts to protect the TSBMISS area 3951 */ 3952 andn %o3, PSTATE_IE, %o5 3953 wrpr %o5, 0, %pstate 3954 3955 /* 3956 * o0 = vaddr 3957 * o1 = sfmmup 3958 * o2 = ttep 3959 */ 3960 CPU_TSBMISS_AREA(%g1, %o5) 3961 ldn [%g1 + TSBMISS_KHATID], %o4 3962 cmp %o4, %o1 3963 bne,pn %ncc, vatopfn_nokernel 3964 mov TTE64K, %g5 /* g5 = rehash # */ 3965 mov %g1,%o5 /* o5 = tsbmiss_area */ 3966 /* 3967 * o0 = vaddr 3968 * o1 & o4 = hatid 3969 * o2 = ttep 3970 * o5 = tsbmiss area 3971 */ 3972 mov HBLK_RANGE_SHIFT, %g6 39731: 3974 3975 /* 3976 * o0 = vaddr 3977 * o1 = sfmmup 3978 * o2 = ttep 3979 * o3 = old %pstate 3980 * o4 = hatid 3981 * o5 = tsbmiss 3982 * g5 = rehash # 3983 * g6 = hmeshift 3984 * 3985 * The first arg to GET_TTE is actually tagaccess register 3986 * not just vaddr. Since this call is for kernel we need to clear 3987 * any lower vaddr bits that would be interpreted as ctx bits. 3988 */ 3989 set TAGACC_CTX_MASK, %g1 3990 andn %o0, %g1, %o0 3991 GET_TTE(%o0, %o4, %g1, %g2, %o5, %g4, %g6, %g5, %g3, 3992 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 3993 3994kvtop_hblk_found: 3995 /* 3996 * o0 = vaddr 3997 * o1 = sfmmup 3998 * o2 = ttep 3999 * g1 = tte 4000 * g2 = tte pa 4001 * g3 = scratch 4002 * o2 = tsbmiss area 4003 * o1 = hat id 4004 */ 4005 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 4006 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4007 stx %g1,[%o2] /* put tte into *ttep */ 4008 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 4009 /* 4010 * o0 = vaddr 4011 * o1 = sfmmup 4012 * o2 = ttep 4013 * g1 = pfn 4014 */ 4015 ba,pt %xcc, 6f 4016 mov %g1, %o0 4017 4018kvtop_nohblk: 4019 /* 4020 * we get here if we couldn't find valid hblk in hash. We rehash 4021 * if neccesary. 4022 */ 4023 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 4024#ifdef sun4v 4025 cmp %g5, MAX_HASHCNT 4026#else 4027 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 4028#endif /* sun4v */ 4029 be,a,pn %icc, 6f 4030 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4031 mov %o1, %o4 /* restore hatid */ 4032#ifdef sun4v 4033 add %g5, 2, %g5 4034 cmp %g5, 3 4035 move %icc, MMU_PAGESHIFT4M, %g6 4036 ba,pt %icc, 1b 4037 movne %icc, MMU_PAGESHIFT256M, %g6 4038#else 4039 inc %g5 4040 cmp %g5, 2 4041 move %icc, MMU_PAGESHIFT512K, %g6 4042 ba,pt %icc, 1b 4043 movne %icc, MMU_PAGESHIFT4M, %g6 4044#endif /* sun4v */ 40456: 4046 retl 4047 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4048 4049tsb_suspend: 4050 /* 4051 * o0 = vaddr 4052 * o1 = sfmmup 4053 * o2 = ttep 4054 * g1 = tte 4055 * g2 = tte pa 4056 * g3 = tte va 4057 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 4058 */ 4059 stx %g1,[%o2] /* put tte into *ttep */ 4060 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 4061 sub %g0, 1, %o0 /* output = PFN_INVALID */ 4062 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 40638: 4064 retl 4065 wrpr %g0, %o3, %pstate /* enable interrupts */ 4066 4067vatopfn_nokernel: 4068 /* 4069 * This routine does NOT support user addresses 4070 * There is a routine in C that supports this. 4071 * The only reason why we don't have the C routine 4072 * support kernel addresses as well is because 4073 * we do va_to_pa while holding the hashlock. 4074 */ 4075 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4076 save %sp, -SA(MINFRAME), %sp 4077 sethi %hi(sfmmu_panic3), %o0 4078 call panic 4079 or %o0, %lo(sfmmu_panic3), %o0 4080 4081 SET_SIZE(sfmmu_vatopfn) 4082 4083 /* 4084 * %o0 = vaddr 4085 * %o1 = hashno (aka szc) 4086 * 4087 * 4088 * This routine is similar to sfmmu_vatopfn() but will only look for 4089 * a kernel vaddr in the hash structure for the specified rehash value. 4090 * It's just an optimization for the case when pagesize for a given 4091 * va range is already known (e.g. large page heap) and we don't want 4092 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 4093 * 4094 * Returns valid pfn or PFN_INVALID if 4095 * tte for specified rehash # is not found, invalid or suspended. 4096 */ 4097 ENTRY_NP(sfmmu_kvaszc2pfn) 4098 /* 4099 * disable interrupts 4100 */ 4101 rdpr %pstate, %o3 4102#ifdef DEBUG 4103 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 4104#endif 4105 /* 4106 * disable interrupts to protect the TSBMISS area 4107 */ 4108 andn %o3, PSTATE_IE, %o5 4109 wrpr %o5, 0, %pstate 4110 4111 CPU_TSBMISS_AREA(%g1, %o5) 4112 ldn [%g1 + TSBMISS_KHATID], %o4 4113 sll %o1, 1, %g6 4114 add %g6, %o1, %g6 4115 add %g6, MMU_PAGESHIFT, %g6 4116 /* 4117 * %o0 = vaddr 4118 * %o1 = hashno 4119 * %o3 = old %pstate 4120 * %o4 = ksfmmup 4121 * %g1 = tsbmiss area 4122 * %g6 = hmeshift 4123 */ 4124 4125 /* 4126 * The first arg to GET_TTE is actually tagaccess register 4127 * not just vaddr. Since this call is for kernel we need to clear 4128 * any lower vaddr bits that would be interpreted as ctx bits. 4129 */ 4130 srlx %o0, MMU_PAGESHIFT, %o0 4131 sllx %o0, MMU_PAGESHIFT, %o0 4132 GET_TTE(%o0, %o4, %g3, %g4, %g1, %o5, %g6, %o1, %g5, 4133 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 4134 kvaszc2pfn_nohblk) 4135 4136kvaszc2pfn_hblk_found: 4137 /* 4138 * %g3 = tte 4139 * %o0 = vaddr 4140 */ 4141 brgez,a,pn %g3, 1f /* check if tte is invalid */ 4142 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4143 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 4144 /* 4145 * g3 = pfn 4146 */ 4147 ba,pt %xcc, 1f 4148 mov %g3, %o0 4149 4150kvaszc2pfn_nohblk: 4151 mov -1, %o0 4152 41531: 4154 retl 4155 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4156 4157 SET_SIZE(sfmmu_kvaszc2pfn) 4158 4159#endif /* lint */ 4160 4161 4162 4163#if !defined(lint) 4164 4165/* 4166 * kpm lock used between trap level tsbmiss handler and kpm C level. 4167 */ 4168#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 4169 mov 0xff, tmp1 ;\ 4170label1: ;\ 4171 casa [kpmlckp]asi, %g0, tmp1 ;\ 4172 brnz,pn tmp1, label1 ;\ 4173 mov 0xff, tmp1 ;\ 4174 membar #LoadLoad 4175 4176#define KPMLOCK_EXIT(kpmlckp, asi) \ 4177 membar #LoadStore|#StoreStore ;\ 4178 sta %g0, [kpmlckp]asi 4179 4180/* 4181 * Lookup a memseg for a given pfn and if found, return the physical 4182 * address of the corresponding struct memseg in mseg, otherwise 4183 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 4184 * tsbmp, %asi is assumed to be ASI_MEM. 4185 * This lookup is done by strictly traversing only the physical memseg 4186 * linkage. The more generic approach, to check the virtual linkage 4187 * before using the physical (used e.g. with hmehash buckets), cannot 4188 * be used here. Memory DR operations can run in parallel to this 4189 * lookup w/o any locks and updates of the physical and virtual linkage 4190 * cannot be done atomically wrt. to each other. Because physical 4191 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 4192 * as "physical NULL" pointer. 4193 */ 4194#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 4195 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 4196 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 4197 udivx pfn, mseg, mseg ;\ 4198 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 4199 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 4200 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 4201 add tmp1, mseg, tmp1 ;\ 4202 ldxa [tmp1]%asi, mseg ;\ 4203 cmp mseg, MSEG_NULLPTR_PA ;\ 4204 be,pn %xcc, label/**/1 /* if not found */ ;\ 4205 nop ;\ 4206 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4207 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4208 blu,pn %xcc, label/**/1 ;\ 4209 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4210 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4211 bgeu,pn %xcc, label/**/1 ;\ 4212 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 4213 mulx tmp1, PAGE_SIZE, tmp1 ;\ 4214 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 4215 add tmp2, tmp1, tmp1 /* pp */ ;\ 4216 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 4217 cmp tmp2, pfn ;\ 4218 be,pt %xcc, label/**/_ok /* found */ ;\ 4219label/**/1: ;\ 4220 /* brute force lookup */ ;\ 4221 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 4222 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 4223label/**/2: ;\ 4224 cmp mseg, MSEG_NULLPTR_PA ;\ 4225 be,pn %xcc, label/**/_ok /* if not found */ ;\ 4226 nop ;\ 4227 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4228 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4229 blu,a,pt %xcc, label/**/2 ;\ 4230 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4231 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4232 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4233 bgeu,a,pt %xcc, label/**/2 ;\ 4234 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4235label/**/_ok: 4236 4237 /* 4238 * kpm tsb miss handler large pages 4239 * g1 = 8K kpm TSB entry pointer 4240 * g2 = tag access register 4241 * g3 = 4M kpm TSB entry pointer 4242 */ 4243 ALTENTRY(sfmmu_kpm_dtsb_miss) 4244 TT_TRACE(trace_tsbmiss) 4245 4246 CPU_INDEX(%g7, %g6) 4247 sethi %hi(kpmtsbm_area), %g6 4248 sllx %g7, KPMTSBM_SHIFT, %g7 4249 or %g6, %lo(kpmtsbm_area), %g6 4250 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4251 4252 /* check enable flag */ 4253 ldub [%g6 + KPMTSBM_FLAGS], %g4 4254 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4255 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4256 nop 4257 4258 /* VA range check */ 4259 ldx [%g6 + KPMTSBM_VBASE], %g7 4260 cmp %g2, %g7 4261 blu,pn %xcc, sfmmu_tsb_miss 4262 ldx [%g6 + KPMTSBM_VEND], %g5 4263 cmp %g2, %g5 4264 bgeu,pn %xcc, sfmmu_tsb_miss 4265 stx %g3, [%g6 + KPMTSBM_TSBPTR] 4266 4267 /* 4268 * check TL tsbmiss handling flag 4269 * bump tsbmiss counter 4270 */ 4271 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4272#ifdef DEBUG 4273 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4274 inc %g5 4275 brz,pn %g3, sfmmu_kpm_exception 4276 st %g5, [%g6 + KPMTSBM_TSBMISS] 4277#else 4278 inc %g5 4279 st %g5, [%g6 + KPMTSBM_TSBMISS] 4280#endif 4281 /* 4282 * At this point: 4283 * g1 = 8K kpm TSB pointer (not used) 4284 * g2 = tag access register 4285 * g3 = clobbered 4286 * g6 = per-CPU kpm tsbmiss area 4287 * g7 = kpm_vbase 4288 */ 4289 4290 /* vaddr2pfn */ 4291 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4292 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4293 srax %g4, %g3, %g2 /* which alias range (r) */ 4294 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4295 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4296 4297 /* 4298 * Setup %asi 4299 * mseg_pa = page_numtomemseg_nolock(pfn) 4300 * if (mseg_pa == NULL) sfmmu_kpm_exception 4301 * g2=pfn 4302 */ 4303 mov ASI_MEM, %asi 4304 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4305 cmp %g3, MSEG_NULLPTR_PA 4306 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4307 nop 4308 4309 /* 4310 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4311 * g2=pfn g3=mseg_pa 4312 */ 4313 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4314 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4315 srlx %g2, %g5, %g4 4316 sllx %g4, %g5, %g4 4317 sub %g4, %g7, %g4 4318 srlx %g4, %g5, %g4 4319 4320 /* 4321 * Validate inx value 4322 * g2=pfn g3=mseg_pa g4=inx 4323 */ 4324#ifdef DEBUG 4325 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4326 cmp %g4, %g5 /* inx - nkpmpgs */ 4327 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4328 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4329#else 4330 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4331#endif 4332 /* 4333 * kp = &mseg_pa->kpm_pages[inx] 4334 */ 4335 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4336 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4337 add %g5, %g4, %g5 /* kp */ 4338 4339 /* 4340 * KPMP_HASH(kp) 4341 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4342 */ 4343 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4344 sub %g7, 1, %g7 /* mask */ 4345 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4346 add %g5, %g1, %g5 /* y = ksp + x */ 4347 and %g5, %g7, %g5 /* hashinx = y & mask */ 4348 4349 /* 4350 * Calculate physical kpm_page pointer 4351 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4352 */ 4353 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4354 add %g1, %g4, %g1 /* kp_pa */ 4355 4356 /* 4357 * Calculate physical hash lock address 4358 * g1=kp_refcntc_pa g2=pfn g5=hashinx 4359 */ 4360 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4361 sllx %g5, KPMHLK_SHIFT, %g5 4362 add %g4, %g5, %g3 4363 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4364 4365 /* 4366 * Assemble tte 4367 * g1=kp_pa g2=pfn g3=hlck_pa 4368 */ 4369#ifdef sun4v 4370 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4371 sllx %g5, 32, %g5 4372 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4373 or %g4, TTE4M, %g4 4374 or %g5, %g4, %g5 4375#else 4376 sethi %hi(TTE_VALID_INT), %g4 4377 mov TTE4M, %g5 4378 sllx %g5, TTE_SZ_SHFT_INT, %g5 4379 or %g5, %g4, %g5 /* upper part */ 4380 sllx %g5, 32, %g5 4381 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4382 or %g5, %g4, %g5 4383#endif 4384 sllx %g2, MMU_PAGESHIFT, %g4 4385 or %g5, %g4, %g5 /* tte */ 4386 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4387 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4388 4389 /* 4390 * tsb dropin 4391 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4392 */ 4393 4394 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4395 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4396 4397 /* use C-handler if there's no go for dropin */ 4398 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4399 cmp %g7, -1 4400 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4401 nop 4402 4403#ifdef DEBUG 4404 /* double check refcnt */ 4405 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4406 brz,pn %g7, 5f /* let C-handler deal with this */ 4407 nop 4408#endif 4409 4410#ifndef sun4v 4411 ldub [%g6 + KPMTSBM_FLAGS], %g7 4412 mov ASI_N, %g1 4413 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4414 movnz %icc, ASI_MEM, %g1 4415 mov %g1, %asi 4416#endif 4417 4418 /* 4419 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4420 * If we fail to lock the TSB entry then just load the tte into the 4421 * TLB. 4422 */ 4423 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l1) 4424 4425 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4426 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4427locked_tsb_l1: 4428 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4429 4430 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4431 KPMLOCK_EXIT(%g3, ASI_MEM) 4432 4433 /* 4434 * If trapstat is running, we need to shift the %tpc and %tnpc to 4435 * point to trapstat's TSB miss return code (note that trapstat 4436 * itself will patch the correct offset to add). 4437 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4438 */ 4439 rdpr %tl, %g7 4440 cmp %g7, 1 4441 ble %icc, 0f 4442 sethi %hi(KERNELBASE), %g6 4443 rdpr %tpc, %g7 4444 or %g6, %lo(KERNELBASE), %g6 4445 cmp %g7, %g6 4446 bgeu %xcc, 0f 4447 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4448 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4449 wrpr %g7, %tpc 4450 add %g7, 4, %g7 4451 wrpr %g7, %tnpc 44520: 4453 retry 44545: 4455 /* g3=hlck_pa */ 4456 KPMLOCK_EXIT(%g3, ASI_MEM) 4457 ba,pt %icc, sfmmu_kpm_exception 4458 nop 4459 SET_SIZE(sfmmu_kpm_dtsb_miss) 4460 4461 /* 4462 * kpm tsbmiss handler for smallpages 4463 * g1 = 8K kpm TSB pointer 4464 * g2 = tag access register 4465 * g3 = 4M kpm TSB pointer 4466 */ 4467 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4468 TT_TRACE(trace_tsbmiss) 4469 CPU_INDEX(%g7, %g6) 4470 sethi %hi(kpmtsbm_area), %g6 4471 sllx %g7, KPMTSBM_SHIFT, %g7 4472 or %g6, %lo(kpmtsbm_area), %g6 4473 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4474 4475 /* check enable flag */ 4476 ldub [%g6 + KPMTSBM_FLAGS], %g4 4477 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4478 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4479 nop 4480 4481 /* 4482 * VA range check 4483 * On fail: goto sfmmu_tsb_miss 4484 */ 4485 ldx [%g6 + KPMTSBM_VBASE], %g7 4486 cmp %g2, %g7 4487 blu,pn %xcc, sfmmu_tsb_miss 4488 ldx [%g6 + KPMTSBM_VEND], %g5 4489 cmp %g2, %g5 4490 bgeu,pn %xcc, sfmmu_tsb_miss 4491 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4492 4493 /* 4494 * check TL tsbmiss handling flag 4495 * bump tsbmiss counter 4496 */ 4497 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4498#ifdef DEBUG 4499 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4500 inc %g5 4501 brz,pn %g1, sfmmu_kpm_exception 4502 st %g5, [%g6 + KPMTSBM_TSBMISS] 4503#else 4504 inc %g5 4505 st %g5, [%g6 + KPMTSBM_TSBMISS] 4506#endif 4507 /* 4508 * At this point: 4509 * g1 = clobbered 4510 * g2 = tag access register 4511 * g3 = 4M kpm TSB pointer (not used) 4512 * g6 = per-CPU kpm tsbmiss area 4513 * g7 = kpm_vbase 4514 */ 4515 4516 /* 4517 * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr) 4518 * which is defined in mach_kpm.h. Any changes in that macro 4519 * should also be ported back to this assembly code. 4520 */ 4521 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 /* g3 = kpm_size_shift */ 4522 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4523 srax %g4, %g3, %g7 /* which alias range (r) */ 4524 brz,pt %g7, 2f 4525 sethi %hi(vac_colors_mask), %g5 4526 ld [%g5 + %lo(vac_colors_mask)], %g5 4527 4528 srlx %g2, MMU_PAGESHIFT, %g1 /* vaddr >> MMU_PAGESHIFT */ 4529 and %g1, %g5, %g1 /* g1 = v */ 4530 sllx %g7, %g3, %g5 /* g5 = r << kpm_size_shift */ 4531 cmp %g7, %g1 /* if (r > v) */ 4532 bleu,pn %xcc, 1f 4533 sub %g4, %g5, %g4 /* paddr -= r << kpm_size_shift */ 4534 sub %g7, %g1, %g5 /* g5 = r - v */ 4535 sllx %g5, MMU_PAGESHIFT, %g7 /* (r-v) << MMU_PAGESHIFT */ 4536 add %g4, %g7, %g4 /* paddr += (r-v)<<MMU_PAGESHIFT */ 4537 ba 2f 4538 nop 45391: 4540 sllx %g7, MMU_PAGESHIFT, %g5 /* else */ 4541 sub %g4, %g5, %g4 /* paddr -= r << MMU_PAGESHIFT */ 4542 4543 /* 4544 * paddr2pfn 4545 * g1 = vcolor (not used) 4546 * g2 = tag access register 4547 * g3 = clobbered 4548 * g4 = paddr 4549 * g5 = clobbered 4550 * g6 = per-CPU kpm tsbmiss area 4551 * g7 = clobbered 4552 */ 45532: 4554 srlx %g4, MMU_PAGESHIFT, %g2 /* g2 = pfn */ 4555 4556 /* 4557 * Setup %asi 4558 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4559 * if (mseg not found) sfmmu_kpm_exception 4560 * g2=pfn g6=per-CPU kpm tsbmiss area 4561 * g4 g5 g7 for scratch use. 4562 */ 4563 mov ASI_MEM, %asi 4564 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4565 cmp %g3, MSEG_NULLPTR_PA 4566 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4567 nop 4568 4569 /* 4570 * inx = pfn - mseg_pa->kpm_pbase 4571 * g2=pfn g3=mseg_pa g6=per-CPU kpm tsbmiss area 4572 */ 4573 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4574 sub %g2, %g7, %g4 4575 4576#ifdef DEBUG 4577 /* 4578 * Validate inx value 4579 * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area 4580 */ 4581 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4582 cmp %g4, %g5 /* inx - nkpmpgs */ 4583 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4584 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4585#else 4586 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4587#endif 4588 /* ksp = &mseg_pa->kpm_spages[inx] */ 4589 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4590 add %g5, %g4, %g5 /* ksp */ 4591 4592 /* 4593 * KPMP_SHASH(kp) 4594 * g2=pfn g3=mseg_pa g4=inx g5=ksp 4595 * g6=per-CPU kpm tsbmiss area g7=kpmp_stable_sz 4596 */ 4597 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4598 sub %g7, 1, %g7 /* mask */ 4599 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4600 add %g5, %g1, %g5 /* y = ksp + x */ 4601 and %g5, %g7, %g5 /* hashinx = y & mask */ 4602 4603 /* 4604 * Calculate physical kpm_spage pointer 4605 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4606 * g6=per-CPU kpm tsbmiss area 4607 */ 4608 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4609 add %g1, %g4, %g1 /* ksp_pa */ 4610 4611 /* 4612 * Calculate physical hash lock address. 4613 * Note: Changes in kpm_shlk_t must be reflected here. 4614 * g1=ksp_pa g2=pfn g5=hashinx 4615 * g6=per-CPU kpm tsbmiss area 4616 */ 4617 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4618 sllx %g5, KPMSHLK_SHIFT, %g5 4619 add %g4, %g5, %g3 /* hlck_pa */ 4620 4621 /* 4622 * Assemble non-cacheable tte initially 4623 * g1=ksp_pa g2=pfn g3=hlck_pa 4624 * g6=per-CPU kpm tsbmiss area 4625 */ 4626 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4627 sllx %g5, 32, %g5 4628 mov (TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4629 or %g5, %g4, %g5 4630 sllx %g2, MMU_PAGESHIFT, %g4 4631 or %g5, %g4, %g5 /* tte */ 4632 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4633 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4634 4635 /* 4636 * tsb dropin 4637 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable) 4638 * g6=per-CPU kpm tsbmiss area g7=scratch register 4639 */ 4640 4641 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4642 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4643 4644 /* use C-handler if there's no go for dropin */ 4645 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4646 andcc %g7, KPM_MAPPED_GO, %g0 /* go or no go ? */ 4647 bz,pt %icc, 5f /* no go */ 4648 nop 4649 and %g7, KPM_MAPPED_MASK, %g7 /* go */ 4650 cmp %g7, KPM_MAPPEDS /* cacheable ? */ 4651 be,a,pn %xcc, 3f 4652 or %g5, TTE_CV_INT, %g5 /* cacheable */ 46533: 4654#ifndef sun4v 4655 ldub [%g6 + KPMTSBM_FLAGS], %g7 4656 mov ASI_N, %g1 4657 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4658 movnz %icc, ASI_MEM, %g1 4659 mov %g1, %asi 4660#endif 4661 4662 /* 4663 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4664 * If we fail to lock the TSB entry then just load the tte into the 4665 * TLB. 4666 */ 4667 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l2) 4668 4669 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4670 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4671locked_tsb_l2: 4672 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4673 4674 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4675 KPMLOCK_EXIT(%g3, ASI_MEM) 4676 4677 /* 4678 * If trapstat is running, we need to shift the %tpc and %tnpc to 4679 * point to trapstat's TSB miss return code (note that trapstat 4680 * itself will patch the correct offset to add). 4681 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4682 */ 4683 rdpr %tl, %g7 4684 cmp %g7, 1 4685 ble %icc, 0f 4686 sethi %hi(KERNELBASE), %g6 4687 rdpr %tpc, %g7 4688 or %g6, %lo(KERNELBASE), %g6 4689 cmp %g7, %g6 4690 bgeu %xcc, 0f 4691 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4692 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4693 wrpr %g7, %tpc 4694 add %g7, 4, %g7 4695 wrpr %g7, %tnpc 46960: 4697 retry 46985: 4699 /* g3=hlck_pa */ 4700 KPMLOCK_EXIT(%g3, ASI_MEM) 4701 ba,pt %icc, sfmmu_kpm_exception 4702 nop 4703 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4704 4705#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4706#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4707#endif 4708 4709#endif /* lint */ 4710 4711#ifdef lint 4712/* 4713 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4714 * Called from C-level, sets/clears "go" indication for trap level handler. 4715 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 4716 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 4717 * Assumes khl_mutex is held when called from C-level. 4718 */ 4719/* ARGSUSED */ 4720void 4721sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 4722{ 4723} 4724 4725/* 4726 * kpm_smallpages: stores val to byte at address mapped within 4727 * low level lock brackets. The old value is returned. 4728 * Called from C-level. 4729 */ 4730/* ARGSUSED */ 4731int 4732sfmmu_kpm_stsbmtl(uchar_t *mapped, uint_t *kshl_lock, int val) 4733{ 4734 return (0); 4735} 4736 4737#else /* lint */ 4738 4739 .seg ".data" 4740sfmmu_kpm_tsbmtl_panic: 4741 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4742 .byte 0 4743sfmmu_kpm_stsbmtl_panic: 4744 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4745 .byte 0 4746 .align 4 4747 .seg ".text" 4748 4749 ENTRY_NP(sfmmu_kpm_tsbmtl) 4750 rdpr %pstate, %o3 4751 /* 4752 * %o0 = &kp_refcntc 4753 * %o1 = &khl_lock 4754 * %o2 = 0/1 (off/on) 4755 * %o3 = pstate save 4756 */ 4757#ifdef DEBUG 4758 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4759 bnz,pt %icc, 1f /* disabled, panic */ 4760 nop 4761 save %sp, -SA(MINFRAME), %sp 4762 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4763 call panic 4764 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4765 ret 4766 restore 47671: 4768#endif /* DEBUG */ 4769 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4770 4771 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4772 mov -1, %o5 4773 brz,a %o2, 2f 4774 mov 0, %o5 47752: 4776 sth %o5, [%o0] 4777 KPMLOCK_EXIT(%o1, ASI_N) 4778 4779 retl 4780 wrpr %g0, %o3, %pstate /* enable interrupts */ 4781 SET_SIZE(sfmmu_kpm_tsbmtl) 4782 4783 ENTRY_NP(sfmmu_kpm_stsbmtl) 4784 rdpr %pstate, %o3 4785 /* 4786 * %o0 = &mapped 4787 * %o1 = &kshl_lock 4788 * %o2 = val 4789 * %o3 = pstate save 4790 */ 4791#ifdef DEBUG 4792 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4793 bnz,pt %icc, 1f /* disabled, panic */ 4794 nop 4795 save %sp, -SA(MINFRAME), %sp 4796 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4797 call panic 4798 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4799 ret 4800 restore 48011: 4802#endif /* DEBUG */ 4803 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4804 4805 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4806 ldsb [%o0], %o5 4807 stb %o2, [%o0] 4808 KPMLOCK_EXIT(%o1, ASI_N) 4809 4810 and %o5, KPM_MAPPED_MASK, %o0 /* return old val */ 4811 retl 4812 wrpr %g0, %o3, %pstate /* enable interrupts */ 4813 SET_SIZE(sfmmu_kpm_stsbmtl) 4814 4815#endif /* lint */ 4816 4817#ifndef lint 4818#ifdef sun4v 4819 /* 4820 * User/kernel data miss w// multiple TSBs 4821 * The first probe covers 8K, 64K, and 512K page sizes, 4822 * because 64K and 512K mappings are replicated off 8K 4823 * pointer. Second probe covers 4M page size only. 4824 * 4825 * MMU fault area contains miss address and context. 4826 */ 4827 ALTENTRY(sfmmu_slow_dmmu_miss) 4828 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 4829 4830slow_miss_common: 4831 /* 4832 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 4833 * %g3 = ctx (cannot be INVALID_CONTEXT) 4834 */ 4835 brnz,pt %g3, 8f ! check for user context 4836 nop 4837 4838 /* 4839 * Kernel miss 4840 * Get 8K and 4M TSB pointers in %g1 and %g3 and 4841 * branch to sfmmu_tsb_miss_tt to handle it. 4842 */ 4843 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4844sfmmu_dslow_patch_ktsb_base: 4845 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 4846sfmmu_dslow_patch_ktsb_szcode: 4847 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 4848 4849 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 4850 ! %g1 = First TSB entry pointer, as TSB miss handler expects 4851 4852 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4853sfmmu_dslow_patch_ktsb4m_base: 4854 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 4855sfmmu_dslow_patch_ktsb4m_szcode: 4856 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 4857 4858 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 4859 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 4860 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4861 .empty 4862 48638: 4864 /* 4865 * User miss 4866 * Get first TSB pointer in %g1 4867 * Get second TSB pointer (or NULL if no second TSB) in %g3 4868 * Branch to sfmmu_tsb_miss_tt to handle it 4869 */ 4870 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 4871 /* %g1 = first TSB entry ptr now, %g2 preserved */ 4872 4873 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 4874 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 4875 nop 4876 4877 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 4878 /* %g3 = second TSB entry ptr now, %g2 preserved */ 48799: 4880 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4881 .empty 4882 SET_SIZE(sfmmu_slow_dmmu_miss) 4883 4884 4885 /* 4886 * User/kernel instruction miss w/ multiple TSBs 4887 * The first probe covers 8K, 64K, and 512K page sizes, 4888 * because 64K and 512K mappings are replicated off 8K 4889 * pointer. Second probe covers 4M page size only. 4890 * 4891 * MMU fault area contains miss address and context. 4892 */ 4893 ALTENTRY(sfmmu_slow_immu_miss) 4894 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 4895 ba,a,pt %xcc, slow_miss_common 4896 SET_SIZE(sfmmu_slow_immu_miss) 4897 4898#endif /* sun4v */ 4899#endif /* lint */ 4900 4901#ifndef lint 4902 4903/* 4904 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 4905 */ 4906 .seg ".data" 4907 .align 64 4908 .global tsbmiss_area 4909tsbmiss_area: 4910 .skip (TSBMISS_SIZE * NCPU) 4911 4912 .align 64 4913 .global kpmtsbm_area 4914kpmtsbm_area: 4915 .skip (KPMTSBM_SIZE * NCPU) 4916#endif /* lint */ 4917