1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#pragma ident "%Z%%M% %I% %E% SMI" 27 28/* 29 * SFMMU primitives. These primitives should only be used by sfmmu 30 * routines. 31 */ 32 33#if defined(lint) 34#include <sys/types.h> 35#else /* lint */ 36#include "assym.h" 37#endif /* lint */ 38 39#include <sys/asm_linkage.h> 40#include <sys/machtrap.h> 41#include <sys/machasi.h> 42#include <sys/sun4asi.h> 43#include <sys/pte.h> 44#include <sys/mmu.h> 45#include <vm/hat_sfmmu.h> 46#include <vm/seg_spt.h> 47#include <sys/machparam.h> 48#include <sys/privregs.h> 49#include <sys/scb.h> 50#include <sys/intreg.h> 51#include <sys/machthread.h> 52#include <sys/intr.h> 53#include <sys/clock.h> 54#include <sys/trapstat.h> 55 56#ifdef TRAPTRACE 57#include <sys/traptrace.h> 58 59/* 60 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 61 */ 62#define TT_TRACE(label) \ 63 ba label ;\ 64 rd %pc, %g7 65#else 66 67#define TT_TRACE(label) 68 69#endif /* TRAPTRACE */ 70 71#ifndef lint 72 73#if (TTE_SUSPEND_SHIFT > 0) 74#define TTE_SUSPEND_INT_SHIFT(reg) \ 75 sllx reg, TTE_SUSPEND_SHIFT, reg 76#else 77#define TTE_SUSPEND_INT_SHIFT(reg) 78#endif 79 80#endif /* lint */ 81 82#ifndef lint 83 84/* 85 * Assumes TSBE_TAG is 0 86 * Assumes TSBE_INTHI is 0 87 * Assumes TSBREG.split is 0 88 */ 89 90#if TSBE_TAG != 0 91#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 92#endif 93 94#if TSBTAG_INTHI != 0 95#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 96#endif 97 98/* 99 * The following code assumes the tsb is not split. 100 * 101 * With TSBs no longer shared between processes, it's no longer 102 * necessary to hash the context bits into the tsb index to get 103 * tsb coloring; the new implementation treats the TSB as a 104 * direct-mapped, virtually-addressed cache. 105 * 106 * In: 107 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 108 * tsbbase = base address of TSB (clobbered) 109 * tagacc = tag access register (clobbered) 110 * szc = size code of TSB (ro) 111 * tmp = scratch reg 112 * Out: 113 * tsbbase = pointer to entry in TSB 114 */ 115#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 116 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 117 srlx tagacc, vpshift, tagacc ;\ 118 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 119 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 120 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 121 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 122 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 123 124/* 125 * When the kpm TSB is used it is assumed that it is direct mapped 126 * using (vaddr>>vpshift)%tsbsz as the index. 127 * 128 * Note that, for now, the kpm TSB and kernel TSB are the same for 129 * each mapping size. However that need not always be the case. If 130 * the trap handlers are updated to search a different TSB for kpm 131 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 132 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 133 * 134 * In: 135 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 136 * vaddr = virtual address (clobbered) 137 * tsbp, szc, tmp = scratch 138 * Out: 139 * tsbp = pointer to entry in TSB 140 */ 141#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 142 cmp vpshift, MMU_PAGESHIFT ;\ 143 bne,pn %icc, 1f /* branch if large case */ ;\ 144 sethi %hi(kpmsm_tsbsz), szc ;\ 145 sethi %hi(kpmsm_tsbbase), tsbp ;\ 146 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 147 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 148 ba,pt %icc, 2f ;\ 149 nop ;\ 1501: sethi %hi(kpm_tsbsz), szc ;\ 151 sethi %hi(kpm_tsbbase), tsbp ;\ 152 ld [szc + %lo(kpm_tsbsz)], szc ;\ 153 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1542: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 155 156/* 157 * Lock the TSBE at virtual address tsbep. 158 * 159 * tsbep = TSBE va (ro) 160 * tmp1, tmp2 = scratch registers (clobbered) 161 * label = label to use for branches (text) 162 * %asi = ASI to use for TSB access 163 * 164 * NOTE that we flush the TSB using fast VIS instructions that 165 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 166 * not be treated as a locked entry or we'll get stuck spinning on 167 * an entry that isn't locked but really invalid. 168 */ 169 170#if defined(UTSB_PHYS) 171 172#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 173 lda [tsbep]ASI_MEM, tmp1 ;\ 174label: ;\ 175 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 176 cmp tmp1, tmp2 ;\ 177 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 178 lda [tsbep]ASI_MEM, tmp1 ;\ 179 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 180 cmp tmp1, tmp2 ;\ 181 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 182 lda [tsbep]ASI_MEM, tmp1 ;\ 183 /* tsbe lock acquired */ ;\ 184 membar #StoreStore 185 186#else /* UTSB_PHYS */ 187 188#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 189 lda [tsbep]%asi, tmp1 ;\ 190label: ;\ 191 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 192 cmp tmp1, tmp2 ;\ 193 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 194 lda [tsbep]%asi, tmp1 ;\ 195 casa [tsbep]%asi, tmp1, tmp2 ;\ 196 cmp tmp1, tmp2 ;\ 197 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 198 lda [tsbep]%asi, tmp1 ;\ 199 /* tsbe lock acquired */ ;\ 200 membar #StoreStore 201 202#endif /* UTSB_PHYS */ 203 204/* 205 * Atomically write TSBE at virtual address tsbep. 206 * 207 * tsbep = TSBE va (ro) 208 * tte = TSBE TTE (ro) 209 * tagtarget = TSBE tag (ro) 210 * %asi = ASI to use for TSB access 211 */ 212 213#if defined(UTSB_PHYS) 214 215#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 216 add tsbep, TSBE_TTE, tmp1 ;\ 217 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 218 membar #StoreStore ;\ 219 add tsbep, TSBE_TAG, tmp1 ;\ 220 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 221 222#else /* UTSB_PHYS */ 223 224#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 225 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 226 membar #StoreStore ;\ 227 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 228 229#endif /* UTSB_PHYS */ 230 231/* 232 * Load an entry into the TSB at TL > 0. 233 * 234 * tsbep = pointer to the TSBE to load as va (ro) 235 * tte = value of the TTE retrieved and loaded (wo) 236 * tagtarget = tag target register. To get TSBE tag to load, 237 * we need to mask off the context and leave only the va (clobbered) 238 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 239 * tmp1, tmp2 = scratch registers 240 * label = label to use for branches (text) 241 * %asi = ASI to use for TSB access 242 */ 243 244#if defined(UTSB_PHYS) 245 246#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 247 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 248 /* ;\ 249 * I don't need to update the TSB then check for the valid tte. ;\ 250 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 251 * we always invalidate the hash table before we unload the TSB.;\ 252 */ ;\ 253 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 254 ldxa [ttepa]ASI_MEM, tte ;\ 255 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 256 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 257 add tsbep, TSBE_TAG, tmp1 ;\ 258 brgez,a,pn tte, label/**/f ;\ 259 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 260 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 261label: 262 263#else /* UTSB_PHYS */ 264 265#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 266 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 267 /* ;\ 268 * I don't need to update the TSB then check for the valid tte. ;\ 269 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 270 * we always invalidate the hash table before we unload the TSB.;\ 271 */ ;\ 272 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 273 ldxa [ttepa]ASI_MEM, tte ;\ 274 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 275 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 276 brgez,a,pn tte, label/**/f ;\ 277 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 278 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 279label: 280 281#endif /* UTSB_PHYS */ 282 283/* 284 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 285 * for ITLB synthesis. 286 * 287 * tsbep = pointer to the TSBE to load as va (ro) 288 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 289 * with exec_perm turned off and exec_synth turned on 290 * tagtarget = tag target register. To get TSBE tag to load, 291 * we need to mask off the context and leave only the va (clobbered) 292 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 293 * tmp1, tmp2 = scratch registers 294 * label = label to use for branch (text) 295 * %asi = ASI to use for TSB access 296 */ 297 298#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 299 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 300 /* ;\ 301 * I don't need to update the TSB then check for the valid tte. ;\ 302 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 303 * we always invalidate the hash table before we unload the TSB.;\ 304 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 305 * and exec_synth bit to 1. ;\ 306 */ ;\ 307 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 308 mov tte, tmp1 ;\ 309 ldxa [ttepa]ASI_MEM, tte ;\ 310 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 311 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 312 brgez,a,pn tte, label/**/f ;\ 313 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 314 or tte, tmp1, tte ;\ 315 andn tte, TTE_EXECPRM_INT, tte ;\ 316 or tte, TTE_E_SYNTH_INT, tte ;\ 317 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 318label: 319 320/* 321 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 322 * 323 * tte = value of the TTE, used to get tte_size bits (ro) 324 * tagaccess = tag access register, used to get 4M pfn bits (ro) 325 * pfn = 4M pfn bits shifted to offset for tte (out) 326 * tmp1 = scratch register 327 * label = label to use for branch (text) 328 */ 329 330#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 331 /* ;\ 332 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 333 * Return them, shifted, in pfn. ;\ 334 */ ;\ 335 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 336 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 337 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 338 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 339 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 340 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 341label: ;\ 342 sllx tagaccess, MMU_PAGESHIFT4M, pfn 343 344/* 345 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 346 * for ITLB synthesis. 347 * 348 * tte = value of the TTE, used to get tte_size bits (rw) 349 * tmp1 = scratch register 350 */ 351 352#define SET_TTE4M_PN(tte, tmp) \ 353 /* ;\ 354 * Set 4M pagesize tte bits. ;\ 355 */ ;\ 356 set TTE4M, tmp ;\ 357 sllx tmp, TTE_SZ_SHFT, tmp ;\ 358 or tte, tmp, tte 359 360/* 361 * Load an entry into the TSB at TL=0. 362 * 363 * tsbep = pointer to the TSBE to load as va (ro) 364 * tteva = pointer to the TTE to load as va (ro) 365 * tagtarget = TSBE tag to load (which contains no context), synthesized 366 * to match va of MMU tag target register only (ro) 367 * tmp1, tmp2 = scratch registers (clobbered) 368 * label = label to use for branches (text) 369 * %asi = ASI to use for TSB access 370 */ 371 372#if defined(UTSB_PHYS) 373 374#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 375 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 376 ldx [tteva], tteva /* load tte */ ;\ 377 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 378 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 379 add tsbep, TSBE_TAG, tmp1 ;\ 380 brgez,a,pn tteva, label/**/f ;\ 381 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 382 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 383label: 384 385#else /* UTSB_PHYS */ 386 387#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 388 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 389 ldx [tteva], tteva /* load tte */ ;\ 390 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 391 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 392 brgez,a,pn tteva, label/**/f ;\ 393 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 394 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 395label: 396 397#endif /* UTSB_PHYS */ 398 399/* 400 * Invalidate a TSB entry in the TSB. 401 * 402 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 403 * about this earlier to ensure this is true. Thus when we are 404 * directly referencing tsbep below, we are referencing the tte_tag 405 * field of the TSBE. If this offset ever changes, the code below 406 * will need to be modified. 407 * 408 * tsbep = pointer to TSBE as va (ro) 409 * tag = invalidation is done if this matches the TSBE tag (ro) 410 * tmp1 - tmp3 = scratch registers (clobbered) 411 * label = label name to use for branches (text) 412 * %asi = ASI to use for TSB access 413 */ 414 415#if defined(UTSB_PHYS) 416 417#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 418 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 419 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 420label/**/1: ;\ 421 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 422 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 423 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 424 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 425 cmp tag, tmp3 /* compare tags */ ;\ 426 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 427 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 428 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 429 cmp tmp1, tmp3 /* if not successful */ ;\ 430 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 431 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 432label/**/2: 433 434#else /* UTSB_PHYS */ 435 436#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 437 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 438 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 439label/**/1: ;\ 440 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 441 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 442 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 443 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 444 cmp tag, tmp3 /* compare tags */ ;\ 445 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 446 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 447 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 448 cmp tmp1, tmp3 /* if not successful */ ;\ 449 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 450 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 451label/**/2: 452 453#endif /* UTSB_PHYS */ 454 455#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 456#error - TSB_SOFTSZ_MASK too small 457#endif 458 459 460/* 461 * An implementation of setx which will be hot patched at run time. 462 * since it is being hot patched, there is no value passed in. 463 * Thus, essentially we are implementing 464 * setx value, tmp, dest 465 * where value is RUNTIME_PATCH (aka 0) in this case. 466 */ 467#define RUNTIME_PATCH_SETX(dest, tmp) \ 468 sethi %hh(RUNTIME_PATCH), tmp ;\ 469 sethi %lm(RUNTIME_PATCH), dest ;\ 470 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 471 or dest, %lo(RUNTIME_PATCH), dest ;\ 472 sllx tmp, 32, tmp ;\ 473 nop /* for perf reasons */ ;\ 474 or tmp, dest, dest /* contents of patched value */ 475 476 477#endif (lint) 478 479 480#if defined (lint) 481 482/* 483 * sfmmu related subroutines 484 */ 485uint_t 486sfmmu_disable_intrs() 487{ return(0); } 488 489/* ARGSUSED */ 490void 491sfmmu_enable_intrs(uint_t pstate_save) 492{} 493 494/* ARGSUSED */ 495int 496sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag) 497{ return(0); } 498 499/* 500 * Use cas, if tte has changed underneath us then reread and try again. 501 * In the case of a retry, it will update sttep with the new original. 502 */ 503/* ARGSUSED */ 504int 505sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 506{ return(0); } 507 508/* 509 * Use cas, if tte has changed underneath us then return 1, else return 0 510 */ 511/* ARGSUSED */ 512int 513sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 514{ return(0); } 515 516/* ARGSUSED */ 517void 518sfmmu_copytte(tte_t *sttep, tte_t *dttep) 519{} 520 521/*ARGSUSED*/ 522struct tsbe * 523sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 524{ return(0); } 525 526/*ARGSUSED*/ 527uint64_t 528sfmmu_make_tsbtag(caddr_t va) 529{ return(0); } 530 531#else /* lint */ 532 533 .seg ".data" 534 .global sfmmu_panic1 535sfmmu_panic1: 536 .asciz "sfmmu_asm: interrupts already disabled" 537 538 .global sfmmu_panic3 539sfmmu_panic3: 540 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 541 542 .global sfmmu_panic4 543sfmmu_panic4: 544 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 545 546 .global sfmmu_panic5 547sfmmu_panic5: 548 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 549 550 .global sfmmu_panic6 551sfmmu_panic6: 552 .asciz "sfmmu_asm: interrupts not disabled" 553 554 .global sfmmu_panic7 555sfmmu_panic7: 556 .asciz "sfmmu_asm: kernel as" 557 558 .global sfmmu_panic8 559sfmmu_panic8: 560 .asciz "sfmmu_asm: gnum is zero" 561 562 .global sfmmu_panic9 563sfmmu_panic9: 564 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 565 566 .global sfmmu_panic10 567sfmmu_panic10: 568 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 569 570 ENTRY(sfmmu_disable_intrs) 571 rdpr %pstate, %o0 572#ifdef DEBUG 573 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 574#endif /* DEBUG */ 575 retl 576 wrpr %o0, PSTATE_IE, %pstate 577 SET_SIZE(sfmmu_disable_intrs) 578 579 ENTRY(sfmmu_enable_intrs) 580 retl 581 wrpr %g0, %o0, %pstate 582 SET_SIZE(sfmmu_enable_intrs) 583 584/* 585 * This routine is called both by resume() and sfmmu_get_ctx() to 586 * allocate a new context for the process on a MMU. 587 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 588 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 589 * is the case when sfmmu_alloc_ctx is called from resume(). 590 * 591 * The caller must disable interrupts before entering this routine. 592 * To reduce ctx switch overhead, the code contains both 'fast path' and 593 * 'slow path' code. The fast path code covers the common case where only 594 * a quick check is needed and the real ctx allocation is not required. 595 * It can be done without holding the per-process (PP) lock. 596 * The 'slow path' code must be protected by the PP Lock and performs ctx 597 * allocation. 598 * Hardware context register and HAT mmu cnum are updated accordingly. 599 * 600 * %o0 - sfmmup 601 * %o1 - allocflag 602 * %o2 - CPU 603 * %o3 - sfmmu private/shared flag 604 * 605 * ret - 0: no ctx is allocated 606 * 1: a ctx is allocated 607 */ 608 ENTRY_NP(sfmmu_alloc_ctx) 609 610#ifdef DEBUG 611 sethi %hi(ksfmmup), %g1 612 ldx [%g1 + %lo(ksfmmup)], %g1 613 cmp %g1, %o0 614 bne,pt %xcc, 0f 615 nop 616 617 sethi %hi(panicstr), %g1 ! if kernel as, panic 618 ldx [%g1 + %lo(panicstr)], %g1 619 tst %g1 620 bnz,pn %icc, 7f 621 nop 622 623 sethi %hi(sfmmu_panic7), %o0 624 call panic 625 or %o0, %lo(sfmmu_panic7), %o0 626 6277: 628 retl 629 mov %g0, %o0 ! %o0 = ret = 0 630 6310: 632 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 633#endif /* DEBUG */ 634 635 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 636 637 ! load global mmu_ctxp info 638 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 639 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 640 641 ! load global mmu_ctxp gnum 642 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 643 644#ifdef DEBUG 645 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 646 bne,pt %xcc, 3f 647 nop 648 649 sethi %hi(panicstr), %g1 ! test if panicstr is already set 650 ldx [%g1 + %lo(panicstr)], %g1 651 tst %g1 652 bnz,pn %icc, 1f 653 nop 654 655 sethi %hi(sfmmu_panic8), %o0 656 call panic 657 or %o0, %lo(sfmmu_panic8), %o0 6581: 659 retl 660 mov %g0, %o0 ! %o0 = ret = 0 6613: 662#endif 663 664 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 665 666 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 667 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 668 669 /* 670 * %g5 = sfmmu gnum returned 671 * %g6 = sfmmu cnum returned 672 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 673 * %g4 = scratch 674 * 675 * Fast path code, do a quick check. 676 */ 677 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 678 679 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 680 bne,pt %icc, 1f ! valid hat cnum, check gnum 681 nop 682 683 ! cnum == INVALID, check allocflag 684 mov %g0, %g4 ! %g4 = ret = 0 685 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 686 mov %g6, %o1 687 688 ! (invalid HAT cnum) && (allocflag == 1) 689 ba,pt %icc, 2f 690 nop 6911: 692 ! valid HAT cnum, check gnum 693 cmp %g5, %o4 694 mov 1, %g4 !%g4 = ret = 1 695 be,a,pt %icc, 8f ! gnum unchanged, go to done 696 mov %g6, %o1 697 6982: 699 /* 700 * Grab per process (PP) sfmmu_ctx_lock spinlock, 701 * followed by the 'slow path' code. 702 */ 703 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 7043: 705 brz %g3, 5f 706 nop 7074: 708 brnz,a,pt %g3, 4b ! spin if lock is 1 709 ldub [%o0 + SFMMU_CTX_LOCK], %g3 710 ba %xcc, 3b ! retry the lock 711 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 712 7135: 714 membar #LoadLoad 715 /* 716 * %g5 = sfmmu gnum returned 717 * %g6 = sfmmu cnum returned 718 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 719 * %g4 = scratch 720 */ 721 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 722 723 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 724 bne,pt %icc, 1f ! valid hat cnum, check gnum 725 nop 726 727 ! cnum == INVALID, check allocflag 728 mov %g0, %g4 ! %g4 = ret = 0 729 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 730 mov %g6, %o1 731 732 ! (invalid HAT cnum) && (allocflag == 1) 733 ba,pt %icc, 6f 734 nop 7351: 736 ! valid HAT cnum, check gnum 737 cmp %g5, %o4 738 mov 1, %g4 ! %g4 = ret = 1 739 be,a,pt %icc, 2f ! gnum unchanged, go to done 740 mov %g6, %o1 741 742 ba,pt %icc, 6f 743 nop 7442: 745 membar #LoadStore|#StoreStore 746 ba,pt %icc, 8f 747 clrb [%o0 + SFMMU_CTX_LOCK] 7486: 749 /* 750 * We get here if we do not have a valid context, or 751 * the HAT gnum does not match global gnum. We hold 752 * sfmmu_ctx_lock spinlock. Allocate that context. 753 * 754 * %o3 = mmu_ctxp 755 */ 756 add %o3, MMU_CTX_CNUM, %g3 757 ld [%o3 + MMU_CTX_NCTXS], %g4 758 759 /* 760 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 761 * %g3 = mmu cnum address 762 * %g4 = mmu nctxs 763 * 764 * %o0 = sfmmup 765 * %o1 = mmu current cnum value (used as new cnum) 766 * %o4 = mmu gnum 767 * 768 * %o5 = scratch 769 */ 770 ld [%g3], %o1 7710: 772 cmp %o1, %g4 773 bl,a,pt %icc, 1f 774 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 775 776 /* 777 * cnum reachs max, bail, so wrap around can be performed later. 778 */ 779 set INVALID_CONTEXT, %o1 780 /* 781 * When the routine is called by shared ctx, we want to set 782 * both private and shared ctx regs to INVALID. In order to 783 * do so, we set the sfmmu priv/shared flag to 'private' regardless. 784 * so that private ctx reg will be set to invalid. 785 * Note that values written to private context register are 786 * automatically written to shared context register as well. 787 */ 788 mov %g0, %g1 ! %g1 = sfmmu private/shared flag 789 mov %g0, %g4 ! %g4 = ret = 0 790 791 membar #LoadStore|#StoreStore 792 ba,pt %icc, 8f 793 clrb [%o0 + SFMMU_CTX_LOCK] 7941: 795 ! %g3 = addr of mmu_ctxp->cnum 796 ! %o5 = mmu_ctxp->cnum + 1 797 cas [%g3], %o1, %o5 798 cmp %o1, %o5 799 bne,a,pn %xcc, 0b ! cas failed 800 ld [%g3], %o1 801 802#ifdef DEBUG 803 set MAX_SFMMU_CTX_VAL, %o5 804 cmp %o1, %o5 805 ble,pt %icc, 2f 806 nop 807 808 sethi %hi(sfmmu_panic9), %o0 809 call panic 810 or %o0, %lo(sfmmu_panic9), %o0 8112: 812#endif 813 ! update hat gnum and cnum 814 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 815 or %o4, %o1, %o4 816 stx %o4, [%g2 + SFMMU_CTXS] 817 818 membar #LoadStore|#StoreStore 819 clrb [%o0 + SFMMU_CTX_LOCK] 820 821 mov 1, %g4 ! %g4 = ret = 1 8228: 823 /* 824 * program the secondary context register 825 * 826 * %o1 = cnum 827 * %g1 = sfmmu private/shared flag (0:private, 1:shared) 828 */ 829 830#ifdef sun4u 831 ldub [%o0 + SFMMU_CEXT], %o2 832 sll %o2, CTXREG_EXT_SHIFT, %o2 833 or %o1, %o2, %o1 834#endif 835 SET_SECCTX(%o1, %g1, %o4, %o5) 836 837 retl 838 mov %g4, %o0 ! %o0 = ret 839 840 SET_SIZE(sfmmu_alloc_ctx) 841 842 ENTRY_NP(sfmmu_modifytte) 843 ldx [%o2], %g3 /* current */ 844 ldx [%o0], %g1 /* original */ 8452: 846 ldx [%o1], %g2 /* modified */ 847 cmp %g2, %g3 /* is modified = current? */ 848 be,a,pt %xcc,1f /* yes, don't write */ 849 stx %g3, [%o0] /* update new original */ 850 casx [%o2], %g1, %g2 851 cmp %g1, %g2 852 be,pt %xcc, 1f /* cas succeeded - return */ 853 nop 854 ldx [%o2], %g3 /* new current */ 855 stx %g3, [%o0] /* save as new original */ 856 ba,pt %xcc, 2b 857 mov %g3, %g1 8581: retl 859 membar #StoreLoad 860 SET_SIZE(sfmmu_modifytte) 861 862 ENTRY_NP(sfmmu_modifytte_try) 863 ldx [%o1], %g2 /* modified */ 864 ldx [%o2], %g3 /* current */ 865 ldx [%o0], %g1 /* original */ 866 cmp %g3, %g2 /* is modified = current? */ 867 be,a,pn %xcc,1f /* yes, don't write */ 868 mov 0, %o1 /* as if cas failed. */ 869 870 casx [%o2], %g1, %g2 871 membar #StoreLoad 872 cmp %g1, %g2 873 movne %xcc, -1, %o1 /* cas failed. */ 874 move %xcc, 1, %o1 /* cas succeeded. */ 8751: 876 stx %g2, [%o0] /* report "current" value */ 877 retl 878 mov %o1, %o0 879 SET_SIZE(sfmmu_modifytte_try) 880 881 ENTRY_NP(sfmmu_copytte) 882 ldx [%o0], %g1 883 retl 884 stx %g1, [%o1] 885 SET_SIZE(sfmmu_copytte) 886 887 888 /* 889 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 890 * %o0 = TSB base address (in), pointer to TSB entry (out) 891 * %o1 = vaddr (in) 892 * %o2 = vpshift (in) 893 * %o3 = tsb size code (in) 894 * %o4 = scratch register 895 */ 896 ENTRY_NP(sfmmu_get_tsbe) 897 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 898 retl 899 nop 900 SET_SIZE(sfmmu_get_tsbe) 901 902 /* 903 * Return a TSB tag for the given va. 904 * %o0 = va (in/clobbered) 905 * %o0 = va shifted to be in tsb tag format (with no context) (out) 906 */ 907 ENTRY_NP(sfmmu_make_tsbtag) 908 retl 909 srln %o0, TTARGET_VA_SHIFT, %o0 910 SET_SIZE(sfmmu_make_tsbtag) 911 912#endif /* lint */ 913 914/* 915 * Other sfmmu primitives 916 */ 917 918 919#if defined (lint) 920void 921sfmmu_patch_ktsb(void) 922{ 923} 924 925void 926sfmmu_kpm_patch_tlbm(void) 927{ 928} 929 930void 931sfmmu_kpm_patch_tsbm(void) 932{ 933} 934 935void 936sfmmu_patch_shctx(void) 937{ 938} 939 940/* ARGSUSED */ 941void 942sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 943{ 944} 945 946/* ARGSUSED */ 947void 948sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 949{ 950} 951 952/* ARGSUSED */ 953void 954sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 955{ 956} 957 958/* ARGSUSED */ 959void 960sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 961{ 962} 963 964#else /* lint */ 965 966#define I_SIZE 4 967 968 ENTRY_NP(sfmmu_fix_ktlb_traptable) 969 /* 970 * %o0 = start of patch area 971 * %o1 = size code of TSB to patch 972 * %o3 = scratch 973 */ 974 /* fix sll */ 975 ld [%o0], %o3 /* get sll */ 976 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 977 st %o3, [%o0] /* write sll */ 978 flush %o0 979 /* fix srl */ 980 add %o0, I_SIZE, %o0 /* goto next instr. */ 981 ld [%o0], %o3 /* get srl */ 982 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 983 st %o3, [%o0] /* write srl */ 984 retl 985 flush %o0 986 SET_SIZE(sfmmu_fix_ktlb_traptable) 987 988 ENTRY_NP(sfmmu_fixup_ktsbbase) 989 /* 990 * %o0 = start of patch area 991 * %o5 = kernel virtual or physical tsb base address 992 * %o2, %o3 are used as scratch registers. 993 */ 994 /* fixup sethi instruction */ 995 ld [%o0], %o3 996 srl %o5, 10, %o2 ! offset is bits 32:10 997 or %o3, %o2, %o3 ! set imm22 998 st %o3, [%o0] 999 /* fixup offset of lduw/ldx */ 1000 add %o0, I_SIZE, %o0 ! next instr 1001 ld [%o0], %o3 1002 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 1003 or %o3, %o2, %o3 1004 st %o3, [%o0] 1005 retl 1006 flush %o0 1007 SET_SIZE(sfmmu_fixup_ktsbbase) 1008 1009 ENTRY_NP(sfmmu_fixup_setx) 1010 /* 1011 * %o0 = start of patch area 1012 * %o4 = 64 bit value to patch 1013 * %o2, %o3 are used as scratch registers. 1014 * 1015 * Note: Assuming that all parts of the instructions which need to be 1016 * patched correspond to RUNTIME_PATCH (aka 0) 1017 * 1018 * Note the implementation of setx which is being patched is as follows: 1019 * 1020 * sethi %hh(RUNTIME_PATCH), tmp 1021 * sethi %lm(RUNTIME_PATCH), dest 1022 * or tmp, %hm(RUNTIME_PATCH), tmp 1023 * or dest, %lo(RUNTIME_PATCH), dest 1024 * sllx tmp, 32, tmp 1025 * nop 1026 * or tmp, dest, dest 1027 * 1028 * which differs from the implementation in the 1029 * "SPARC Architecture Manual" 1030 */ 1031 /* fixup sethi instruction */ 1032 ld [%o0], %o3 1033 srlx %o4, 42, %o2 ! bits [63:42] 1034 or %o3, %o2, %o3 ! set imm22 1035 st %o3, [%o0] 1036 /* fixup sethi instruction */ 1037 add %o0, I_SIZE, %o0 ! next instr 1038 ld [%o0], %o3 1039 sllx %o4, 32, %o2 ! clear upper bits 1040 srlx %o2, 42, %o2 ! bits [31:10] 1041 or %o3, %o2, %o3 ! set imm22 1042 st %o3, [%o0] 1043 /* fixup or instruction */ 1044 add %o0, I_SIZE, %o0 ! next instr 1045 ld [%o0], %o3 1046 srlx %o4, 32, %o2 ! bits [63:32] 1047 and %o2, 0x3ff, %o2 ! bits [41:32] 1048 or %o3, %o2, %o3 ! set imm 1049 st %o3, [%o0] 1050 /* fixup or instruction */ 1051 add %o0, I_SIZE, %o0 ! next instr 1052 ld [%o0], %o3 1053 and %o4, 0x3ff, %o2 ! bits [9:0] 1054 or %o3, %o2, %o3 ! set imm 1055 st %o3, [%o0] 1056 retl 1057 flush %o0 1058 SET_SIZE(sfmmu_fixup_setx) 1059 1060 ENTRY_NP(sfmmu_fixup_or) 1061 /* 1062 * %o0 = start of patch area 1063 * %o4 = 32 bit value to patch 1064 * %o2, %o3 are used as scratch registers. 1065 * Note: Assuming that all parts of the instructions which need to be 1066 * patched correspond to RUNTIME_PATCH (aka 0) 1067 */ 1068 ld [%o0], %o3 1069 and %o4, 0x3ff, %o2 ! bits [9:0] 1070 or %o3, %o2, %o3 ! set imm 1071 st %o3, [%o0] 1072 retl 1073 flush %o0 1074 SET_SIZE(sfmmu_fixup_or) 1075 1076 ENTRY_NP(sfmmu_fixup_shiftx) 1077 /* 1078 * %o0 = start of patch area 1079 * %o4 = signed int immediate value to add to sllx/srlx imm field 1080 * %o2, %o3 are used as scratch registers. 1081 * 1082 * sllx/srlx store the 6 bit immediate value in the lowest order bits 1083 * so we do a simple add. The caller must be careful to prevent 1084 * overflow, which could easily occur if the initial value is nonzero! 1085 */ 1086 ld [%o0], %o3 ! %o3 = instruction to patch 1087 and %o3, 0x3f, %o2 ! %o2 = existing imm value 1088 add %o2, %o4, %o2 ! %o2 = new imm value 1089 andn %o3, 0x3f, %o3 ! clear old imm value 1090 and %o2, 0x3f, %o2 ! truncate new imm value 1091 or %o3, %o2, %o3 ! set new imm value 1092 st %o3, [%o0] ! store updated instruction 1093 retl 1094 flush %o0 1095 SET_SIZE(sfmmu_fixup_shiftx) 1096 1097 ENTRY_NP(sfmmu_fixup_mmu_asi) 1098 /* 1099 * Patch imm_asi of all ldda instructions in the MMU 1100 * trap handlers. We search MMU_PATCH_INSTR instructions 1101 * starting from the itlb miss handler (trap 0x64). 1102 * %o0 = address of tt[0,1]_itlbmiss 1103 * %o1 = imm_asi to setup, shifted by appropriate offset. 1104 * %o3 = number of instructions to search 1105 * %o4 = reserved by caller: called from leaf routine 1106 */ 11071: ldsw [%o0], %o2 ! load instruction to %o2 1108 brgez,pt %o2, 2f 1109 srl %o2, 30, %o5 1110 btst 1, %o5 ! test bit 30; skip if not set 1111 bz,pt %icc, 2f 1112 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1113 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1114 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1115 brnz,pt %o5, 2f ! skip if not a match 1116 or %o2, %o1, %o2 ! or in imm_asi 1117 st %o2, [%o0] ! write patched instruction 11182: dec %o3 1119 brnz,a,pt %o3, 1b ! loop until we're done 1120 add %o0, I_SIZE, %o0 1121 retl 1122 flush %o0 1123 SET_SIZE(sfmmu_fixup_mmu_asi) 1124 1125 /* 1126 * Patch immediate ASI used to access the TSB in the 1127 * trap table. 1128 * inputs: %o0 = value of ktsb_phys 1129 */ 1130 ENTRY_NP(sfmmu_patch_mmu_asi) 1131 mov %o7, %o4 ! save return pc in %o4 1132 movrnz %o0, ASI_QUAD_LDD_PHYS, %o3 1133 movrz %o0, ASI_NQUAD_LD, %o3 1134 sll %o3, 5, %o1 ! imm_asi offset 1135 mov 6, %o3 ! number of instructions 1136 sethi %hi(dktsb), %o0 ! to search 1137 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1138 or %o0, %lo(dktsb), %o0 1139 mov 6, %o3 ! number of instructions 1140 sethi %hi(dktsb4m), %o0 ! to search 1141 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1142 or %o0, %lo(dktsb4m), %o0 1143 mov 6, %o3 ! number of instructions 1144 sethi %hi(iktsb), %o0 ! to search 1145 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1146 or %o0, %lo(iktsb), %o0 1147 mov 6, %o3 ! number of instructions 1148 sethi %hi(iktsb4m), %o0 ! to search 1149 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1150 or %o0, %lo(iktsb4m), %o0 1151 mov %o4, %o7 ! retore return pc -- leaf 1152 retl 1153 nop 1154 SET_SIZE(sfmmu_patch_mmu_asi) 1155 1156 ENTRY_NP(sfmmu_patch_ktsb) 1157 /* 1158 * We need to fix iktsb, dktsb, et. al. 1159 */ 1160 save %sp, -SA(MINFRAME), %sp 1161 set ktsb_phys, %o1 1162 ld [%o1], %o4 1163 set ktsb_base, %o5 1164 set ktsb4m_base, %l1 1165 brz,pt %o4, 1f 1166 nop 1167 set ktsb_pbase, %o5 1168 set ktsb4m_pbase, %l1 11691: 1170 sethi %hi(ktsb_szcode), %o1 1171 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1172 1173 sethi %hi(iktsb), %o0 1174 call sfmmu_fix_ktlb_traptable 1175 or %o0, %lo(iktsb), %o0 1176 1177 sethi %hi(dktsb), %o0 1178 call sfmmu_fix_ktlb_traptable 1179 or %o0, %lo(dktsb), %o0 1180 1181 sethi %hi(ktsb4m_szcode), %o1 1182 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1183 1184 sethi %hi(iktsb4m), %o0 1185 call sfmmu_fix_ktlb_traptable 1186 or %o0, %lo(iktsb4m), %o0 1187 1188 sethi %hi(dktsb4m), %o0 1189 call sfmmu_fix_ktlb_traptable 1190 or %o0, %lo(dktsb4m), %o0 1191 1192#ifndef sun4v 1193 mov ASI_N, %o2 1194 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1195 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1196 sethi %hi(tsb_kernel_patch_asi), %o0 1197 call sfmmu_fixup_or 1198 or %o0, %lo(tsb_kernel_patch_asi), %o0 1199#endif 1200 1201 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1202 1203 sethi %hi(dktsbbase), %o0 1204 call sfmmu_fixup_setx ! patch value of ktsb base addr 1205 or %o0, %lo(dktsbbase), %o0 1206 1207 sethi %hi(iktsbbase), %o0 1208 call sfmmu_fixup_setx ! patch value of ktsb base addr 1209 or %o0, %lo(iktsbbase), %o0 1210 1211 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1212 call sfmmu_fixup_setx ! patch value of ktsb base addr 1213 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1214 1215#ifdef sun4v 1216 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1217 call sfmmu_fixup_setx ! patch value of ktsb base addr 1218 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1219#endif /* sun4v */ 1220 1221 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1222 1223 sethi %hi(dktsb4mbase), %o0 1224 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1225 or %o0, %lo(dktsb4mbase), %o0 1226 1227 sethi %hi(iktsb4mbase), %o0 1228 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1229 or %o0, %lo(iktsb4mbase), %o0 1230 1231 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1232 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1233 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1234 1235#ifdef sun4v 1236 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1237 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1238 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1239#endif /* sun4v */ 1240 1241 set ktsb_szcode, %o4 1242 ld [%o4], %o4 1243 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1244 call sfmmu_fixup_or ! patch value of ktsb_szcode 1245 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1246 1247#ifdef sun4v 1248 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1249 call sfmmu_fixup_or ! patch value of ktsb_szcode 1250 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1251#endif /* sun4v */ 1252 1253 set ktsb4m_szcode, %o4 1254 ld [%o4], %o4 1255 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1256 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1257 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1258 1259#ifdef sun4v 1260 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1261 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1262 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1263#endif /* sun4v */ 1264 1265 ret 1266 restore 1267 SET_SIZE(sfmmu_patch_ktsb) 1268 1269 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1270 /* 1271 * Fixup trap handlers in common segkpm case. This is reserved 1272 * for future use should kpm TSB be changed to be other than the 1273 * kernel TSB. 1274 */ 1275 retl 1276 nop 1277 SET_SIZE(sfmmu_kpm_patch_tlbm) 1278 1279 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1280 /* 1281 * nop the branch to sfmmu_kpm_dtsb_miss_small 1282 * in the case where we are using large pages for 1283 * seg_kpm (and hence must probe the second TSB for 1284 * seg_kpm VAs) 1285 */ 1286 set dktsb4m_kpmcheck_small, %o0 1287 MAKE_NOP_INSTR(%o1) 1288 st %o1, [%o0] 1289 flush %o0 1290 retl 1291 nop 1292 SET_SIZE(sfmmu_kpm_patch_tsbm) 1293 1294 ENTRY_NP(sfmmu_patch_utsb) 1295#ifdef UTSB_PHYS 1296 retl 1297 nop 1298#else /* UTSB_PHYS */ 1299 /* 1300 * We need to hot patch utsb_vabase and utsb4m_vabase 1301 */ 1302 save %sp, -SA(MINFRAME), %sp 1303 1304 /* patch value of utsb_vabase */ 1305 set utsb_vabase, %o1 1306 ldx [%o1], %o4 1307 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1308 call sfmmu_fixup_setx 1309 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1310 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1311 call sfmmu_fixup_setx 1312 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1313 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1314 call sfmmu_fixup_setx 1315 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1316 1317 /* patch value of utsb4m_vabase */ 1318 set utsb4m_vabase, %o1 1319 ldx [%o1], %o4 1320 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1321 call sfmmu_fixup_setx 1322 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1323 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1324 call sfmmu_fixup_setx 1325 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1326 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1327 call sfmmu_fixup_setx 1328 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1329 1330 /* 1331 * Patch TSB base register masks and shifts if needed. 1332 * By default the TSB base register contents are set up for 4M slab. 1333 * If we're using a smaller slab size and reserved VA range we need 1334 * to patch up those values here. 1335 */ 1336 set tsb_slab_shift, %o1 1337 set MMU_PAGESHIFT4M, %o4 1338 lduw [%o1], %o3 1339 subcc %o4, %o3, %o4 1340 bz,pt %icc, 1f 1341 /* delay slot safe */ 1342 1343 /* patch reserved VA range size if needed. */ 1344 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1345 call sfmmu_fixup_shiftx 1346 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1347 call sfmmu_fixup_shiftx 1348 add %o0, I_SIZE, %o0 1349 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1350 call sfmmu_fixup_shiftx 1351 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1352 call sfmmu_fixup_shiftx 1353 add %o0, I_SIZE, %o0 13541: 1355 /* patch TSBREG_VAMASK used to set up TSB base register */ 1356 set tsb_slab_mask, %o1 1357 ldx [%o1], %o4 1358 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1359 call sfmmu_fixup_or 1360 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1361 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1362 call sfmmu_fixup_or 1363 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1364 1365 ret 1366 restore 1367#endif /* UTSB_PHYS */ 1368 SET_SIZE(sfmmu_patch_utsb) 1369 1370 ENTRY_NP(sfmmu_patch_shctx) 1371#ifdef sun4u 1372 retl 1373 nop 1374#else /* sun4u */ 1375 set sfmmu_shctx_cpu_mondo_patch, %o0 1376 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1377 st %o1, [%o0] 1378 flush %o0 1379 MAKE_NOP_INSTR(%o1) 1380 add %o0, I_SIZE, %o0 ! next instr 1381 st %o1, [%o0] 1382 flush %o0 1383 1384 set sfmmu_shctx_user_rtt_patch, %o0 1385 st %o1, [%o0] ! nop 1st instruction 1386 flush %o0 1387 add %o0, I_SIZE, %o0 1388 st %o1, [%o0] ! nop 2nd instruction 1389 flush %o0 1390 add %o0, I_SIZE, %o0 1391 st %o1, [%o0] ! nop 3rd instruction 1392 flush %o0 1393 add %o0, I_SIZE, %o0 1394 st %o1, [%o0] ! nop 4th instruction 1395 flush %o0 1396 add %o0, I_SIZE, %o0 1397 st %o1, [%o0] ! nop 5th instruction 1398 retl 1399 flush %o0 1400#endif /* sun4u */ 1401 SET_SIZE(sfmmu_patch_shctx) 1402 1403 /* 1404 * Routine that loads an entry into a tsb using virtual addresses. 1405 * Locking is required since all cpus can use the same TSB. 1406 * Note that it is no longer required to have a valid context 1407 * when calling this function. 1408 */ 1409 ENTRY_NP(sfmmu_load_tsbe) 1410 /* 1411 * %o0 = pointer to tsbe to load 1412 * %o1 = tsb tag 1413 * %o2 = virtual pointer to TTE 1414 * %o3 = 1 if physical address in %o0 else 0 1415 */ 1416 rdpr %pstate, %o5 1417#ifdef DEBUG 1418 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1419#endif /* DEBUG */ 1420 1421 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1422 1423 SETUP_TSB_ASI(%o3, %g3) 1424 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1) 1425 1426 wrpr %g0, %o5, %pstate /* enable interrupts */ 1427 1428 retl 1429 membar #StoreStore|#StoreLoad 1430 SET_SIZE(sfmmu_load_tsbe) 1431 1432 /* 1433 * Flush TSB of a given entry if the tag matches. 1434 */ 1435 ENTRY(sfmmu_unload_tsbe) 1436 /* 1437 * %o0 = pointer to tsbe to be flushed 1438 * %o1 = tag to match 1439 * %o2 = 1 if physical address in %o0 else 0 1440 */ 1441 SETUP_TSB_ASI(%o2, %g1) 1442 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1443 retl 1444 membar #StoreStore|#StoreLoad 1445 SET_SIZE(sfmmu_unload_tsbe) 1446 1447 /* 1448 * Routine that loads a TTE into the kpm TSB from C code. 1449 * Locking is required since kpm TSB is shared among all CPUs. 1450 */ 1451 ENTRY_NP(sfmmu_kpm_load_tsb) 1452 /* 1453 * %o0 = vaddr 1454 * %o1 = ttep 1455 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1456 */ 1457 rdpr %pstate, %o5 ! %o5 = saved pstate 1458#ifdef DEBUG 1459 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1460#endif /* DEBUG */ 1461 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1462 1463#ifndef sun4v 1464 sethi %hi(ktsb_phys), %o4 1465 mov ASI_N, %o3 1466 ld [%o4 + %lo(ktsb_phys)], %o4 1467 movrnz %o4, ASI_MEM, %o3 1468 mov %o3, %asi 1469#endif 1470 mov %o0, %g1 ! %g1 = vaddr 1471 1472 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1473 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1474 /* %g2 = tsbep, %g1 clobbered */ 1475 1476 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1477 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1478 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1) 1479 1480 wrpr %g0, %o5, %pstate ! enable interrupts 1481 retl 1482 membar #StoreStore|#StoreLoad 1483 SET_SIZE(sfmmu_kpm_load_tsb) 1484 1485 /* 1486 * Routine that shoots down a TTE in the kpm TSB or in the 1487 * kernel TSB depending on virtpg. Locking is required since 1488 * kpm/kernel TSB is shared among all CPUs. 1489 */ 1490 ENTRY_NP(sfmmu_kpm_unload_tsb) 1491 /* 1492 * %o0 = vaddr 1493 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1494 */ 1495#ifndef sun4v 1496 sethi %hi(ktsb_phys), %o4 1497 mov ASI_N, %o3 1498 ld [%o4 + %lo(ktsb_phys)], %o4 1499 movrnz %o4, ASI_MEM, %o3 1500 mov %o3, %asi 1501#endif 1502 mov %o0, %g1 ! %g1 = vaddr 1503 1504 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1505 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1506 /* %g2 = tsbep, %g1 clobbered */ 1507 1508 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1509 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1510 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1511 1512 retl 1513 membar #StoreStore|#StoreLoad 1514 SET_SIZE(sfmmu_kpm_unload_tsb) 1515 1516#endif /* lint */ 1517 1518 1519#if defined (lint) 1520 1521/*ARGSUSED*/ 1522pfn_t 1523sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1524{ return(0); } 1525 1526#else /* lint */ 1527 1528 ENTRY_NP(sfmmu_ttetopfn) 1529 ldx [%o0], %g1 /* read tte */ 1530 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1531 /* 1532 * g1 = pfn 1533 */ 1534 retl 1535 mov %g1, %o0 1536 SET_SIZE(sfmmu_ttetopfn) 1537 1538#endif /* !lint */ 1539 1540 1541#if defined (lint) 1542/* 1543 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the 1544 * the hash list. 1545 */ 1546/* ARGSUSED */ 1547void 1548sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1549 uint64_t hblkpa) 1550{ 1551} 1552 1553/* 1554 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the 1555 * hash list. 1556 */ 1557/* ARGSUSED */ 1558void 1559sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1560 uint64_t hblkpa, struct hme_blk *prev_hblkp) 1561{ 1562} 1563#else /* lint */ 1564 1565/* 1566 * Functions to grab/release hme bucket list lock. I only use a byte 1567 * instead of the whole int because eventually we might want to 1568 * put some counters on the other bytes (of course, these routines would 1569 * have to change). The code that grab this lock should execute 1570 * with interrupts disabled and hold the lock for the least amount of time 1571 * possible. 1572 */ 1573 1574/* 1575 * Even though hmeh_listlock is updated using pa there's no need to flush 1576 * dcache since hmeh_listlock will be restored to the original value (0) 1577 * before interrupts are reenabled. 1578 */ 1579 1580/* 1581 * For sparcv9 hme hash buckets may not be in the nucleus. hme hash update 1582 * routines still use virtual addresses to update the bucket fields. But they 1583 * must not cause a TLB miss after grabbing the low level bucket lock. To 1584 * achieve this we must make sure the bucket structure is completely within an 1585 * 8K page. 1586 */ 1587 1588#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1)) 1589#error - the size of hmehash_bucket structure is not power of 2 1590#endif 1591 1592#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi) \ 1593 mov 0xff, tmp2 ;\ 1594 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1595label1: ;\ 1596 casa [tmp1]asi, %g0, tmp2 ;\ 1597 brnz,pn tmp2, label1 ;\ 1598 mov 0xff, tmp2 ;\ 1599 membar #LoadLoad 1600 1601#define HMELOCK_EXIT(hmebp, tmp1, asi) \ 1602 membar #LoadStore|#StoreStore ;\ 1603 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1604 sta %g0, [tmp1]asi 1605 1606 .seg ".data" 1607hblk_add_panic1: 1608 .ascii "sfmmu_hblk_hash_add: interrupts disabled" 1609 .byte 0 1610hblk_add_panic2: 1611 .ascii "sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not" 1612 .byte 0 1613 .align 4 1614 .seg ".text" 1615 1616 ENTRY_NP(sfmmu_hblk_hash_add) 1617 /* 1618 * %o0 = hmebp 1619 * %o1 = hmeblkp 1620 * %o2 = hblkpa 1621 */ 1622 rdpr %pstate, %o5 1623#ifdef DEBUG 1624 andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ 1625 bnz,pt %icc, 3f /* disabled, panic */ 1626 nop 1627 save %sp, -SA(MINFRAME), %sp 1628 sethi %hi(hblk_add_panic1), %o0 1629 call panic 1630 or %o0, %lo(hblk_add_panic1), %o0 1631 ret 1632 restore 1633 16343: 1635#endif /* DEBUG */ 1636 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1637 mov %o2, %g1 1638 1639 /* 1640 * g1 = hblkpa 1641 */ 1642 ldn [%o0 + HMEBUCK_HBLK], %o4 /* next hmeblk */ 1643 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = next hblkpa */ 1644#ifdef DEBUG 1645 cmp %o4, %g0 1646 bne,pt %xcc, 1f 1647 nop 1648 brz,pt %g2, 1f 1649 nop 1650 wrpr %g0, %o5, %pstate /* enable interrupts */ 1651 save %sp, -SA(MINFRAME), %sp 1652 sethi %hi(hblk_add_panic2), %o0 1653 call panic 1654 or %o0, %lo(hblk_add_panic2), %o0 1655 ret 1656 restore 16571: 1658#endif /* DEBUG */ 1659 /* 1660 * We update hmeblks entries before grabbing lock because the stores 1661 * could take a tlb miss and require the hash lock. The buckets 1662 * are part of the nucleus so we are cool with those stores. 1663 * 1664 * if buckets are not part of the nucleus our game is to 1665 * not touch any other page via va until we drop the lock. 1666 * This guarantees we won't get a tlb miss before the lock release 1667 * since interrupts are disabled. 1668 */ 1669 stn %o4, [%o1 + HMEBLK_NEXT] /* update hmeblk's next */ 1670 stx %g2, [%o1 + HMEBLK_NEXTPA] /* update hmeblk's next pa */ 1671 HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N) 1672 stn %o1, [%o0 + HMEBUCK_HBLK] /* update bucket hblk next */ 1673 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* add hmeblk to list */ 1674 HMELOCK_EXIT(%o0, %g2, ASI_N) 1675 retl 1676 wrpr %g0, %o5, %pstate /* enable interrupts */ 1677 SET_SIZE(sfmmu_hblk_hash_add) 1678 1679 ENTRY_NP(sfmmu_hblk_hash_rm) 1680 /* 1681 * This function removes an hmeblk from the hash chain. 1682 * It is written to guarantee we don't take a tlb miss 1683 * by using physical addresses to update the list. 1684 * 1685 * %o0 = hmebp 1686 * %o1 = hmeblkp 1687 * %o2 = hmeblkp previous pa 1688 * %o3 = hmeblkp previous 1689 */ 1690 1691 mov %o3, %o4 /* o4 = hmeblkp previous */ 1692 1693 rdpr %pstate, %o5 1694#ifdef DEBUG 1695 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1) 1696#endif /* DEBUG */ 1697 /* 1698 * disable interrupts, clear Address Mask to access 64 bit physaddr 1699 */ 1700 andn %o5, PSTATE_IE, %g1 1701 wrpr %g1, 0, %pstate 1702 1703#ifndef sun4v 1704 sethi %hi(dcache_line_mask), %g4 1705 ld [%g4 + %lo(dcache_line_mask)], %g4 1706#endif /* sun4v */ 1707 1708 /* 1709 * if buckets are not part of the nucleus our game is to 1710 * not touch any other page via va until we drop the lock. 1711 * This guarantees we won't get a tlb miss before the lock release 1712 * since interrupts are disabled. 1713 */ 1714 HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N) 1715 ldn [%o0 + HMEBUCK_HBLK], %g2 /* first hmeblk in list */ 1716 cmp %g2, %o1 1717 bne,pt %ncc,1f 1718 mov ASI_MEM, %asi 1719 /* 1720 * hmeblk is first on list 1721 */ 1722 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = hmeblk pa */ 1723 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1724 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1725 stn %o3, [%o0 + HMEBUCK_HBLK] /* write va */ 1726 ba,pt %xcc, 2f 1727 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* write pa */ 17281: 1729 /* hmeblk is not first on list */ 1730 1731 mov %o2, %g3 1732#ifndef sun4v 1733 GET_CPU_IMPL(%g2) 1734 cmp %g2, CHEETAH_IMPL 1735 bge,a,pt %icc, hblk_hash_rm_1 1736 and %o4, %g4, %g2 1737 cmp %g2, SPITFIRE_IMPL 1738 blt %icc, hblk_hash_rm_2 /* no flushing needed for OPL */ 1739 and %o4, %g4, %g2 1740 stxa %g0, [%g2]ASI_DC_TAG /* flush prev pa from dcache */ 1741 add %o4, HMEBLK_NEXT, %o4 1742 and %o4, %g4, %g2 1743 ba hblk_hash_rm_2 1744 stxa %g0, [%g2]ASI_DC_TAG /* flush prev va from dcache */ 1745hblk_hash_rm_1: 1746 1747 stxa %g0, [%g3]ASI_DC_INVAL /* flush prev pa from dcache */ 1748 membar #Sync 1749 add %g3, HMEBLK_NEXT, %g2 1750 stxa %g0, [%g2]ASI_DC_INVAL /* flush prev va from dcache */ 1751hblk_hash_rm_2: 1752 membar #Sync 1753#endif /* sun4v */ 1754 ldxa [%g3 + HMEBLK_NEXTPA] %asi, %g2 /* g2 = hmeblk pa */ 1755 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1756 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1757 stna %o3, [%g3 + HMEBLK_NEXT] %asi /* write va */ 1758 stxa %g1, [%g3 + HMEBLK_NEXTPA] %asi /* write pa */ 17592: 1760 HMELOCK_EXIT(%o0, %g2, ASI_N) 1761 retl 1762 wrpr %g0, %o5, %pstate /* enable interrupts */ 1763 SET_SIZE(sfmmu_hblk_hash_rm) 1764 1765#endif /* lint */ 1766 1767/* 1768 * These macros are used to update global sfmmu hme hash statistics 1769 * in perf critical paths. It is only enabled in debug kernels or 1770 * if SFMMU_STAT_GATHER is defined 1771 */ 1772#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1773#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1774 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1775 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1776 cmp tmp1, hatid ;\ 1777 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1778 set sfmmu_global_stat, tmp1 ;\ 1779 add tmp1, tmp2, tmp1 ;\ 1780 ld [tmp1], tmp2 ;\ 1781 inc tmp2 ;\ 1782 st tmp2, [tmp1] 1783 1784#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1785 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1786 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1787 cmp tmp1, hatid ;\ 1788 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1789 set sfmmu_global_stat, tmp1 ;\ 1790 add tmp1, tmp2, tmp1 ;\ 1791 ld [tmp1], tmp2 ;\ 1792 inc tmp2 ;\ 1793 st tmp2, [tmp1] 1794 1795 1796#else /* DEBUG || SFMMU_STAT_GATHER */ 1797 1798#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1799 1800#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1801 1802#endif /* DEBUG || SFMMU_STAT_GATHER */ 1803 1804/* 1805 * This macro is used to update global sfmmu kstas in non 1806 * perf critical areas so they are enabled all the time 1807 */ 1808#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1809 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1810 add tmp1, statname, tmp1 ;\ 1811 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1812 inc tmp2 ;\ 1813 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1814 1815/* 1816 * These macros are used to update per cpu stats in non perf 1817 * critical areas so they are enabled all the time 1818 */ 1819#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1820 ld [tsbarea + stat], tmp1 ;\ 1821 inc tmp1 ;\ 1822 st tmp1, [tsbarea + stat] 1823 1824/* 1825 * These macros are used to update per cpu stats in non perf 1826 * critical areas so they are enabled all the time 1827 */ 1828#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1829 lduh [tsbarea + stat], tmp1 ;\ 1830 inc tmp1 ;\ 1831 stuh tmp1, [tsbarea + stat] 1832 1833#if defined(KPM_TLBMISS_STATS_GATHER) 1834 /* 1835 * Count kpm dtlb misses separately to allow a different 1836 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1837 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1838 */ 1839#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1840 brgez tagacc, label /* KPM VA? */ ;\ 1841 nop ;\ 1842 CPU_INDEX(tmp1, tsbma) ;\ 1843 sethi %hi(kpmtsbm_area), tsbma ;\ 1844 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1845 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1846 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1847 /* VA range check */ ;\ 1848 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1849 cmp tagacc, val ;\ 1850 blu,pn %xcc, label ;\ 1851 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1852 cmp tagacc, tmp1 ;\ 1853 bgeu,pn %xcc, label ;\ 1854 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1855 inc val ;\ 1856 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1857label: 1858#else 1859#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1860#endif /* KPM_TLBMISS_STATS_GATHER */ 1861 1862#if defined (lint) 1863/* 1864 * The following routines are jumped to from the mmu trap handlers to do 1865 * the setting up to call systrap. They are separate routines instead of 1866 * being part of the handlers because the handlers would exceed 32 1867 * instructions and since this is part of the slow path the jump 1868 * cost is irrelevant. 1869 */ 1870void 1871sfmmu_pagefault(void) 1872{ 1873} 1874 1875void 1876sfmmu_mmu_trap(void) 1877{ 1878} 1879 1880void 1881sfmmu_window_trap(void) 1882{ 1883} 1884 1885void 1886sfmmu_kpm_exception(void) 1887{ 1888} 1889 1890#else /* lint */ 1891 1892#ifdef PTL1_PANIC_DEBUG 1893 .seg ".data" 1894 .global test_ptl1_panic 1895test_ptl1_panic: 1896 .word 0 1897 .align 8 1898 1899 .seg ".text" 1900 .align 4 1901#endif /* PTL1_PANIC_DEBUG */ 1902 1903 1904 ENTRY_NP(sfmmu_pagefault) 1905 SET_GL_REG(1) 1906 USE_ALTERNATE_GLOBALS(%g5) 1907 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1908 rdpr %tt, %g6 1909 cmp %g6, FAST_IMMU_MISS_TT 1910 be,a,pn %icc, 1f 1911 mov T_INSTR_MMU_MISS, %g3 1912 cmp %g6, T_INSTR_MMU_MISS 1913 be,a,pn %icc, 1f 1914 mov T_INSTR_MMU_MISS, %g3 1915 mov %g5, %g2 1916 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1917 cmp %g6, FAST_DMMU_MISS_TT 1918 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1919 cmp %g6, T_DATA_MMU_MISS 1920 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1921 1922#ifdef PTL1_PANIC_DEBUG 1923 /* check if we want to test the tl1 panic */ 1924 sethi %hi(test_ptl1_panic), %g4 1925 ld [%g4 + %lo(test_ptl1_panic)], %g1 1926 st %g0, [%g4 + %lo(test_ptl1_panic)] 1927 cmp %g1, %g0 1928 bne,a,pn %icc, ptl1_panic 1929 or %g0, PTL1_BAD_DEBUG, %g1 1930#endif /* PTL1_PANIC_DEBUG */ 19311: 1932 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 1933 /* 1934 * g2 = tag access reg 1935 * g3.l = type 1936 * g3.h = 0 1937 */ 1938 sethi %hi(trap), %g1 1939 or %g1, %lo(trap), %g1 19402: 1941 ba,pt %xcc, sys_trap 1942 mov -1, %g4 1943 SET_SIZE(sfmmu_pagefault) 1944 1945 ENTRY_NP(sfmmu_mmu_trap) 1946 SET_GL_REG(1) 1947 USE_ALTERNATE_GLOBALS(%g5) 1948 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 1949 rdpr %tt, %g6 1950 cmp %g6, FAST_IMMU_MISS_TT 1951 be,a,pn %icc, 1f 1952 mov T_INSTR_MMU_MISS, %g3 1953 cmp %g6, T_INSTR_MMU_MISS 1954 be,a,pn %icc, 1f 1955 mov T_INSTR_MMU_MISS, %g3 1956 mov %g5, %g2 1957 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1958 cmp %g6, FAST_DMMU_MISS_TT 1959 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1960 cmp %g6, T_DATA_MMU_MISS 1961 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 19621: 1963 /* 1964 * g2 = tag access reg 1965 * g3 = type 1966 */ 1967 sethi %hi(sfmmu_tsbmiss_exception), %g1 1968 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 1969 ba,pt %xcc, sys_trap 1970 mov -1, %g4 1971 /*NOTREACHED*/ 1972 SET_SIZE(sfmmu_mmu_trap) 1973 1974 ENTRY_NP(sfmmu_suspend_tl) 1975 SET_GL_REG(1) 1976 USE_ALTERNATE_GLOBALS(%g5) 1977 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 1978 rdpr %tt, %g6 1979 cmp %g6, FAST_IMMU_MISS_TT 1980 be,a,pn %icc, 1f 1981 mov T_INSTR_MMU_MISS, %g3 1982 mov %g5, %g2 1983 cmp %g6, FAST_DMMU_MISS_TT 1984 move %icc, T_DATA_MMU_MISS, %g3 1985 movne %icc, T_DATA_PROT, %g3 19861: 1987 sethi %hi(sfmmu_tsbmiss_suspended), %g1 1988 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 1989 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 1990 ba,pt %xcc, sys_trap 1991 mov PIL_15, %g4 1992 /*NOTREACHED*/ 1993 SET_SIZE(sfmmu_suspend_tl) 1994 1995 /* 1996 * No %g registers in use at this point. 1997 */ 1998 ENTRY_NP(sfmmu_window_trap) 1999 rdpr %tpc, %g1 2000#ifdef sun4v 2001#ifdef DEBUG 2002 /* We assume previous %gl was 1 */ 2003 rdpr %tstate, %g4 2004 srlx %g4, TSTATE_GL_SHIFT, %g4 2005 and %g4, TSTATE_GL_MASK, %g4 2006 cmp %g4, 1 2007 bne,a,pn %icc, ptl1_panic 2008 mov PTL1_BAD_WTRAP, %g1 2009#endif /* DEBUG */ 2010 /* user miss at tl>1. better be the window handler or user_rtt */ 2011 /* in user_rtt? */ 2012 set rtt_fill_start, %g4 2013 cmp %g1, %g4 2014 blu,pn %xcc, 6f 2015 .empty 2016 set rtt_fill_end, %g4 2017 cmp %g1, %g4 2018 bgeu,pn %xcc, 6f 2019 nop 2020 set fault_rtt_fn1, %g1 2021 wrpr %g0, %g1, %tnpc 2022 ba,a 7f 20236: 2024 ! must save this trap level before descending trap stack 2025 ! no need to save %tnpc, either overwritten or discarded 2026 ! already got it: rdpr %tpc, %g1 2027 rdpr %tstate, %g6 2028 rdpr %tt, %g7 2029 ! trap level saved, go get underlying trap type 2030 rdpr %tl, %g5 2031 sub %g5, 1, %g3 2032 wrpr %g3, %tl 2033 rdpr %tt, %g2 2034 wrpr %g5, %tl 2035 ! restore saved trap level 2036 wrpr %g1, %tpc 2037 wrpr %g6, %tstate 2038 wrpr %g7, %tt 2039#else /* sun4v */ 2040 /* user miss at tl>1. better be the window handler */ 2041 rdpr %tl, %g5 2042 sub %g5, 1, %g3 2043 wrpr %g3, %tl 2044 rdpr %tt, %g2 2045 wrpr %g5, %tl 2046#endif /* sun4v */ 2047 and %g2, WTRAP_TTMASK, %g4 2048 cmp %g4, WTRAP_TYPE 2049 bne,pn %xcc, 1f 2050 nop 2051 /* tpc should be in the trap table */ 2052 set trap_table, %g4 2053 cmp %g1, %g4 2054 blt,pn %xcc, 1f 2055 .empty 2056 set etrap_table, %g4 2057 cmp %g1, %g4 2058 bge,pn %xcc, 1f 2059 .empty 2060 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 2061 add %g1, WTRAP_FAULTOFF, %g1 2062 wrpr %g0, %g1, %tnpc 20637: 2064 /* 2065 * some wbuf handlers will call systrap to resolve the fault 2066 * we pass the trap type so they figure out the correct parameters. 2067 * g5 = trap type, g6 = tag access reg 2068 */ 2069 2070 /* 2071 * only use g5, g6, g7 registers after we have switched to alternate 2072 * globals. 2073 */ 2074 SET_GL_REG(1) 2075 USE_ALTERNATE_GLOBALS(%g5) 2076 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 2077 rdpr %tt, %g7 2078 cmp %g7, FAST_IMMU_MISS_TT 2079 be,a,pn %icc, ptl1_panic 2080 mov PTL1_BAD_WTRAP, %g1 2081 cmp %g7, T_INSTR_MMU_MISS 2082 be,a,pn %icc, ptl1_panic 2083 mov PTL1_BAD_WTRAP, %g1 2084 mov T_DATA_PROT, %g5 2085 cmp %g7, FAST_DMMU_MISS_TT 2086 move %icc, T_DATA_MMU_MISS, %g5 2087 cmp %g7, T_DATA_MMU_MISS 2088 move %icc, T_DATA_MMU_MISS, %g5 2089 ! XXXQ AGS re-check out this one 2090 done 20911: 2092 CPU_ADDR(%g1, %g4) 2093 ld [%g1 + CPU_TL1_HDLR], %g4 2094 brnz,a,pt %g4, sfmmu_mmu_trap 2095 st %g0, [%g1 + CPU_TL1_HDLR] 2096 ba,pt %icc, ptl1_panic 2097 mov PTL1_BAD_TRAP, %g1 2098 SET_SIZE(sfmmu_window_trap) 2099 2100 ENTRY_NP(sfmmu_kpm_exception) 2101 /* 2102 * We have accessed an unmapped segkpm address or a legal segkpm 2103 * address which is involved in a VAC alias conflict prevention. 2104 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 2105 * set. If it is, we will instead note that a fault has occurred 2106 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 2107 * a "retry"). This will step over the faulting instruction. 2108 * Note that this means that a legal segkpm address involved in 2109 * a VAC alias conflict prevention (a rare case to begin with) 2110 * cannot be used in DTrace. 2111 */ 2112 CPU_INDEX(%g1, %g2) 2113 set cpu_core, %g2 2114 sllx %g1, CPU_CORE_SHIFT, %g1 2115 add %g1, %g2, %g1 2116 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 2117 andcc %g2, CPU_DTRACE_NOFAULT, %g0 2118 bz 0f 2119 or %g2, CPU_DTRACE_BADADDR, %g2 2120 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 2121 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 2122 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 2123 done 21240: 2125 TSTAT_CHECK_TL1(1f, %g1, %g2) 21261: 2127 SET_GL_REG(1) 2128 USE_ALTERNATE_GLOBALS(%g5) 2129 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 2130 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 2131 /* 2132 * g2=tagacc g3.l=type g3.h=0 2133 */ 2134 sethi %hi(trap), %g1 2135 or %g1, %lo(trap), %g1 2136 ba,pt %xcc, sys_trap 2137 mov -1, %g4 2138 SET_SIZE(sfmmu_kpm_exception) 2139 2140#endif /* lint */ 2141 2142#if defined (lint) 2143 2144void 2145sfmmu_tsb_miss(void) 2146{ 2147} 2148 2149void 2150sfmmu_kpm_dtsb_miss(void) 2151{ 2152} 2153 2154void 2155sfmmu_kpm_dtsb_miss_small(void) 2156{ 2157} 2158 2159#else /* lint */ 2160 2161#if (IMAP_SEG != 0) 2162#error - ism_map->ism_seg offset is not zero 2163#endif 2164 2165/* 2166 * Copies ism mapping for this ctx in param "ism" if this is a ISM 2167 * tlb miss and branches to label "ismhit". If this is not an ISM 2168 * process or an ISM tlb miss it falls thru. 2169 * 2170 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 2171 * this process. 2172 * If so, it will branch to label "ismhit". If not, it will fall through. 2173 * 2174 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 2175 * so that any other threads of this process will not try and walk the ism 2176 * maps while they are being changed. 2177 * 2178 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 2179 * will make sure of that. This means we can terminate our search on 2180 * the first zero mapping we find. 2181 * 2182 * Parameters: 2183 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 2184 * tsbmiss = address of tsb miss area (in) 2185 * ismseg = contents of ism_seg for this ism map (out) 2186 * ismhat = physical address of imap_ismhat for this ism map (out) 2187 * tmp1 = scratch reg (CLOBBERED) 2188 * tmp2 = scratch reg (CLOBBERED) 2189 * tmp3 = scratch reg (CLOBBERED) 2190 * label: temporary labels 2191 * ismhit: label where to jump to if an ism dtlb miss 2192 * exitlabel:label where to jump if hat is busy due to hat_unshare. 2193 */ 2194#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 2195 label, ismhit) \ 2196 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 2197 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 2198 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 2199label/**/1: ;\ 2200 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 2201 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 2202label/**/2: ;\ 2203 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 2204 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 2205 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 2206 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 2207 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 2208 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 2209 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 2210 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 2211 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 2212 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 2213 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 2214 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 2215 ;\ 2216 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 2217 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 2218 cmp ismhat, tmp1 ;\ 2219 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 2220 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 2221 ;\ 2222 add tmp3, IBLK_NEXTPA, tmp1 ;\ 2223 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 2224 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 2225 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 2226label/**/3: 2227 2228/* 2229 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 2230 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 2231 * Parameters: 2232 * tagacc = reg containing virtual address 2233 * hatid = reg containing sfmmu pointer 2234 * hmeshift = constant/register to shift vaddr to obtain vapg 2235 * hmebp = register where bucket pointer will be stored 2236 * vapg = register where virtual page will be stored 2237 * tmp1, tmp2 = tmp registers 2238 */ 2239 2240 2241#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 2242 vapg, label, tmp1, tmp2) \ 2243 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 2244 brnz,a,pt tmp1, label/**/1 ;\ 2245 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 2246 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 2247 ba,pt %xcc, label/**/2 ;\ 2248 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 2249label/**/1: ;\ 2250 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 2251label/**/2: ;\ 2252 srlx tagacc, hmeshift, vapg ;\ 2253 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 2254 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 2255 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 2256 add hmebp, tmp1, hmebp 2257 2258/* 2259 * hashtag includes bspage + hashno (64 bits). 2260 */ 2261 2262#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 2263 sllx vapg, hmeshift, vapg ;\ 2264 mov hashno, hblktag ;\ 2265 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 2266 or vapg, hblktag, hblktag 2267 2268/* 2269 * Function to traverse hmeblk hash link list and find corresponding match. 2270 * The search is done using physical pointers. It returns the physical address 2271 * and virtual address pointers to the hmeblk that matches with the tag 2272 * provided. 2273 * Parameters: 2274 * hmebp = register that points to hme hash bucket, also used as 2275 * tmp reg (clobbered) 2276 * hmeblktag = register with hmeblk tag match 2277 * hatid = register with hatid 2278 * hmeblkpa = register where physical ptr will be stored 2279 * hmeblkva = register where virtual ptr will be stored 2280 * tmp1 = tmp reg 2281 * label: temporary label 2282 */ 2283 2284#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2285 tsbarea, tmp1, label) \ 2286 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 2287 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2288 add hmebp, HMEBUCK_HBLK, hmeblkva ;\ 2289 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2290 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2291label/**/1: ;\ 2292 brz,pn hmeblkva, label/**/2 ;\ 2293 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2294 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 2295 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2296 add hmebp, CLONGSIZE, hmebp ;\ 2297 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 2298 xor tmp1, hmeblktag, tmp1 ;\ 2299 xor hmebp, hatid, hmebp ;\ 2300 or hmebp, tmp1, hmebp ;\ 2301 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 2302 add hmeblkpa, HMEBLK_NEXT, hmebp ;\ 2303 ldna [hmebp]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2304 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 2305 ba,pt %xcc, label/**/1 ;\ 2306 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2307label/**/2: 2308 2309/* 2310 * Function to traverse hmeblk hash link list and find corresponding match. 2311 * The search is done using physical pointers. It returns the physical address 2312 * and virtual address pointers to the hmeblk that matches with the tag 2313 * provided. 2314 * Parameters: 2315 * hmeblktag = register with hmeblk tag match (rid field is 0) 2316 * hatid = register with hatid (pointer to SRD) 2317 * hmeblkpa = register where physical ptr will be stored 2318 * hmeblkva = register where virtual ptr will be stored 2319 * tmp1 = tmp reg 2320 * tmp2 = tmp reg 2321 * label: temporary label 2322 */ 2323 2324#define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2325 tsbarea, tmp1, tmp2, label) \ 2326label/**/1: ;\ 2327 brz,pn hmeblkva, label/**/4 ;\ 2328 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 2329 add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 2330 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2331 add tmp2, CLONGSIZE, tmp2 ;\ 2332 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 2333 xor tmp1, hmeblktag, tmp1 ;\ 2334 xor tmp2, hatid, tmp2 ;\ 2335 brz,pn tmp2, label/**/3 /* branch on hit */ ;\ 2336 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2337label/**/2: ;\ 2338 ldna [tmp2]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2339 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2340 ba,pt %xcc, label/**/1 ;\ 2341 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2342label/**/3: ;\ 2343 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 2344 bgeu,pt %xcc, label/**/2 ;\ 2345 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2346 and tmp1, BT_ULMASK, tmp2 ;\ 2347 srlx tmp1, BT_ULSHIFT, tmp1 ;\ 2348 sllx tmp1, CLONGSHIFT, tmp1 ;\ 2349 add tsbarea, tmp1, tmp1 ;\ 2350 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 2351 srlx tmp1, tmp2, tmp1 ;\ 2352 btst 0x1, tmp1 ;\ 2353 bz,pn %xcc, label/**/2 ;\ 2354 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2355label/**/4: 2356 2357#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2358#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2359#endif 2360 2361/* 2362 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2363 * he offset for the corresponding hment. 2364 * Parameters: 2365 * In: 2366 * vaddr = register with virtual address 2367 * hmeblkpa = physical pointer to hme_blk 2368 * Out: 2369 * hmentoff = register where hment offset will be stored 2370 * hmemisc = hblk_misc 2371 * Scratch: 2372 * tmp1 2373 */ 2374#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2375 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2376 lda [hmentoff]ASI_MEM, hmemisc ;\ 2377 andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2378 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2379 or %g0, HMEBLK_HME1, hmentoff ;\ 2380 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2381 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2382 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2383 add tmp1, HMEBLK_HME1, hmentoff ;\ 2384label1: 2385 2386/* 2387 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2388 * 2389 * tagacc = (pseudo-)tag access register (in) 2390 * hatid = sfmmu pointer for TSB miss (in) 2391 * tte = tte for TLB miss if found, otherwise clobbered (out) 2392 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2393 * hmeblkva = VA of hment if found, otherwise clobbered (out) 2394 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2395 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2396 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2397 * for this page size. 2398 * hashno = constant/register hash number 2399 * label = temporary label for branching within macro. 2400 * foundlabel = label to jump to when tte is found. 2401 * suspendlabel= label to jump to when tte is suspended. 2402 * exitlabel = label to jump to when tte is not found. 2403 * 2404 */ 2405#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmemisc, \ 2406 hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \ 2407 ;\ 2408 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2409 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2410 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2411 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2412 ;\ 2413 /* ;\ 2414 * tagacc = tagacc ;\ 2415 * hatid = hatid ;\ 2416 * tsbarea = tsbarea ;\ 2417 * tte = hmebp (hme bucket pointer) ;\ 2418 * hmeblkpa = vapg (virtual page) ;\ 2419 * hmemisc, hmeblkva = scratch ;\ 2420 */ ;\ 2421 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2422 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2423 ;\ 2424 /* ;\ 2425 * tagacc = tagacc ;\ 2426 * hatid = hatid ;\ 2427 * tte = hmebp ;\ 2428 * hmeblkpa = CLOBBERED ;\ 2429 * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2430 * hmeblkva = scratch ;\ 2431 */ ;\ 2432 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2433 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2434 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, hmeblkva, \ 2435 tsbarea, tagacc, label/**/1) ;\ 2436 /* ;\ 2437 * tagacc = CLOBBERED ;\ 2438 * tte = CLOBBERED ;\ 2439 * hmeblkpa = hmeblkpa ;\ 2440 * hmeblkva = hmeblkva ;\ 2441 */ ;\ 2442 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2443 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2444 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2445 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2446 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2447 nop ;\ 2448label/**/4: ;\ 2449 /* ;\ 2450 * We have found the hmeblk containing the hment. ;\ 2451 * Now we calculate the corresponding tte. ;\ 2452 * ;\ 2453 * tagacc = tagacc ;\ 2454 * hatid = hatid ;\ 2455 * tte = clobbered ;\ 2456 * hmeblkpa = hmeblkpa ;\ 2457 * hmemisc = hblktag ;\ 2458 * hmeblkva = hmeblkva ;\ 2459 */ ;\ 2460 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2461 label/**/2) ;\ 2462 ;\ 2463 /* ;\ 2464 * tagacc = tagacc ;\ 2465 * hatid = hmentoff ;\ 2466 * tte = clobbered ;\ 2467 * hmeblkpa = hmeblkpa ;\ 2468 * hmemisc = hblk_misc ;\ 2469 * hmeblkva = hmeblkva ;\ 2470 */ ;\ 2471 ;\ 2472 add hatid, SFHME_TTE, hatid ;\ 2473 add hmeblkpa, hatid, hmeblkpa ;\ 2474 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2475 add hmeblkva, hatid, hmeblkva ;\ 2476 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2477 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2478 set TTE_SUSPEND, hatid ;\ 2479 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2480 btst tte, hatid ;\ 2481 bz,pt %xcc, foundlabel ;\ 2482 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2483 ;\ 2484 /* ;\ 2485 * Mapping is suspended, so goto suspend label. ;\ 2486 */ ;\ 2487 ba,pt %xcc, suspendlabel ;\ 2488 nop 2489 2490/* 2491 * GET_SHME_TTE is similar to GET_TTE() except it searches 2492 * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2493 * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2494 * either 0 (not part of scd) or 1 (part of scd). 2495 */ 2496#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, \ 2497 hmemisc, hmeshift, hashno, label, foundlabel, \ 2498 suspendlabel, exitlabel) \ 2499 ;\ 2500 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2501 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2502 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2503 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2504 ;\ 2505 /* ;\ 2506 * tagacc = tagacc ;\ 2507 * hatid = hatid ;\ 2508 * tsbarea = tsbarea ;\ 2509 * tte = hmebp (hme bucket pointer) ;\ 2510 * hmeblkpa = vapg (virtual page) ;\ 2511 * hmemisc, hmeblkva = scratch ;\ 2512 */ ;\ 2513 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2514 ;\ 2515 /* ;\ 2516 * tagacc = tagacc ;\ 2517 * hatid = hatid ;\ 2518 * tsbarea = tsbarea ;\ 2519 * tte = hmebp ;\ 2520 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2521 * hmeblkpa = CLOBBERED ;\ 2522 * hmeblkva = scratch ;\ 2523 */ ;\ 2524 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2525 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2526 ;\ 2527 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2528 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2529 add tte, HMEBUCK_HBLK, hmeblkva ;\ 2530 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2531 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2532 ;\ 2533label/**/8: ;\ 2534 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, hmeblkva, \ 2535 tsbarea, tagacc, tte, label/**/1) ;\ 2536 /* ;\ 2537 * tagacc = CLOBBERED ;\ 2538 * tte = CLOBBERED ;\ 2539 * hmeblkpa = hmeblkpa ;\ 2540 * hmeblkva = hmeblkva ;\ 2541 */ ;\ 2542 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2543 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2544 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2545 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2546 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2547 nop ;\ 2548label/**/4: ;\ 2549 /* ;\ 2550 * We have found the hmeblk containing the hment. ;\ 2551 * Now we calculate the corresponding tte. ;\ 2552 * ;\ 2553 * tagacc = tagacc ;\ 2554 * hatid = hatid ;\ 2555 * tte = clobbered ;\ 2556 * hmeblkpa = hmeblkpa ;\ 2557 * hmemisc = hblktag ;\ 2558 * hmeblkva = hmeblkva ;\ 2559 * tsbarea = tsbmiss area ;\ 2560 */ ;\ 2561 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2562 label/**/2) ;\ 2563 ;\ 2564 /* ;\ 2565 * tagacc = tagacc ;\ 2566 * hatid = hmentoff ;\ 2567 * tte = clobbered ;\ 2568 * hmeblkpa = hmeblkpa ;\ 2569 * hmemisc = hblk_misc ;\ 2570 * hmeblkva = hmeblkva ;\ 2571 * tsbarea = tsbmiss area ;\ 2572 */ ;\ 2573 ;\ 2574 add hatid, SFHME_TTE, hatid ;\ 2575 add hmeblkpa, hatid, hmeblkpa ;\ 2576 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2577 brlz,pt tte, label/**/6 ;\ 2578 add hmeblkva, hatid, hmeblkva ;\ 2579 btst HBLK_SZMASK, hmemisc ;\ 2580 bnz,a,pt %icc, label/**/7 ;\ 2581 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2582 ;\ 2583 /* ;\ 2584 * We found an invalid 8K tte in shme. ;\ 2585 * it may not belong to shme's region since ;\ 2586 * region size/alignment granularity is 8K but different ;\ 2587 * regions don't share hmeblks. Continue the search. ;\ 2588 */ ;\ 2589 sub hmeblkpa, hatid, hmeblkpa ;\ 2590 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2591 srlx tagacc, hmeshift, tte ;\ 2592 add hmeblkpa, HMEBLK_NEXT, hmeblkva ;\ 2593 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2594 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2595 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2596 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2597 ba,a,pt %xcc, label/**/8 ;\ 2598label/**/6: ;\ 2599 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2600 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2601label/**/7: ;\ 2602 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2603 set TTE_SUSPEND, hatid ;\ 2604 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2605 btst tte, hatid ;\ 2606 bz,pt %xcc, foundlabel ;\ 2607 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2608 ;\ 2609 /* ;\ 2610 * Mapping is suspended, so goto suspend label. ;\ 2611 */ ;\ 2612 ba,pt %xcc, suspendlabel ;\ 2613 nop 2614 2615 /* 2616 * KERNEL PROTECTION HANDLER 2617 * 2618 * g1 = tsb8k pointer register (clobbered) 2619 * g2 = tag access register (ro) 2620 * g3 - g7 = scratch registers 2621 * 2622 * Note: This function is patched at runtime for performance reasons. 2623 * Any changes here require sfmmu_patch_ktsb fixed. 2624 */ 2625 ENTRY_NP(sfmmu_kprot_trap) 2626 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2627sfmmu_kprot_patch_ktsb_base: 2628 RUNTIME_PATCH_SETX(%g1, %g6) 2629 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2630sfmmu_kprot_patch_ktsb_szcode: 2631 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2632 2633 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2634 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2635 2636 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2637sfmmu_kprot_patch_ktsb4m_base: 2638 RUNTIME_PATCH_SETX(%g3, %g6) 2639 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2640sfmmu_kprot_patch_ktsb4m_szcode: 2641 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2642 2643 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2644 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2645 2646 CPU_TSBMISS_AREA(%g6, %g7) 2647 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2648 ba,pt %xcc, sfmmu_tsb_miss_tt 2649 nop 2650 2651 /* 2652 * USER PROTECTION HANDLER 2653 * 2654 * g1 = tsb8k pointer register (ro) 2655 * g2 = tag access register (ro) 2656 * g3 = faulting context (clobbered, currently not used) 2657 * g4 - g7 = scratch registers 2658 */ 2659 ALTENTRY(sfmmu_uprot_trap) 2660#ifdef sun4v 2661 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2662 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2663 2664 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2665 brlz,pt %g3, 9f /* check for 2nd TSB */ 2666 nop 2667 2668 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2669 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2670 2671#else /* sun4v */ 2672#ifdef UTSB_PHYS 2673 /* g1 = first TSB entry ptr */ 2674 GET_2ND_TSBREG(%g3) 2675 brlz,pt %g3, 9f /* check for 2nd TSB */ 2676 nop 2677 2678 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2679 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2680#else /* UTSB_PHYS */ 2681 brgez,pt %g1, 9f /* check for 2nd TSB */ 2682 mov -1, %g3 /* set second tsbe ptr to -1 */ 2683 2684 mov %g2, %g7 2685 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2686 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2687 mov %g1, %g7 2688 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2689#endif /* UTSB_PHYS */ 2690#endif /* sun4v */ 26919: 2692 CPU_TSBMISS_AREA(%g6, %g7) 2693 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2694 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2695 nop 2696 2697 /* 2698 * Kernel 8K page iTLB miss. We also get here if we took a 2699 * fast instruction access mmu miss trap while running in 2700 * invalid context. 2701 * 2702 * %g1 = 8K TSB pointer register (not used, clobbered) 2703 * %g2 = tag access register (used) 2704 * %g3 = faulting context id (used) 2705 * %g7 = TSB tag to match (used) 2706 */ 2707 .align 64 2708 ALTENTRY(sfmmu_kitlb_miss) 2709 brnz,pn %g3, tsb_tl0_noctxt 2710 nop 2711 2712 /* kernel miss */ 2713 /* get kernel tsb pointer */ 2714 /* we patch the next set of instructions at run time */ 2715 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2716iktsbbase: 2717 RUNTIME_PATCH_SETX(%g4, %g5) 2718 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2719 2720iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2721 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2722 or %g4, %g1, %g1 ! form tsb ptr 2723 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2724 cmp %g4, %g7 2725 bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2726 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2727 2728 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2729 bz,pn %icc, exec_fault 2730 nop 2731 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2732 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2733 retry 2734 2735iktsb4mbase: 2736 RUNTIME_PATCH_SETX(%g4, %g6) 2737 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2738iktsb4m: 2739 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2740 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2741 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2742 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2743 cmp %g4, %g7 2744 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2745 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2746 bz,pn %icc, exec_fault 2747 nop 2748 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2749 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2750 retry 2751 2752 /* 2753 * Kernel dTLB miss. We also get here if we took a fast data 2754 * access mmu miss trap while running in invalid context. 2755 * 2756 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2757 * We select the TSB miss handler to branch to depending on 2758 * the virtual address of the access. In the future it may 2759 * be desirable to separate kpm TTEs into their own TSB, 2760 * in which case all that needs to be done is to set 2761 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2762 * early in the miss if we detect a kpm VA to a new handler. 2763 * 2764 * %g1 = 8K TSB pointer register (not used, clobbered) 2765 * %g2 = tag access register (used) 2766 * %g3 = faulting context id (used) 2767 */ 2768 .align 64 2769 ALTENTRY(sfmmu_kdtlb_miss) 2770 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2771 nop 2772 2773 /* Gather some stats for kpm misses in the TLB. */ 2774 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2775 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2776 2777 /* 2778 * Get first TSB offset and look for 8K/64K/512K mapping 2779 * using the 8K virtual page as the index. 2780 * 2781 * We patch the next set of instructions at run time; 2782 * any changes here require sfmmu_patch_ktsb changes too. 2783 */ 2784dktsbbase: 2785 RUNTIME_PATCH_SETX(%g7, %g6) 2786 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2787 2788dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2789 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2790 2791 /* 2792 * At this point %g1 is our index into the TSB. 2793 * We just masked off enough bits of the VA depending 2794 * on our TSB size code. 2795 */ 2796 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2797 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2798 cmp %g6, %g4 ! compare tag 2799 bne,pn %xcc, dktsb4m_kpmcheck_small 2800 add %g7, %g1, %g1 /* form tsb ptr */ 2801 TT_TRACE(trace_tsbhit) 2802 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2803 /* trapstat expects tte in %g5 */ 2804 retry 2805 2806 /* 2807 * If kpm is using large pages, the following instruction needs 2808 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2809 * so that we will probe the 4M TSB regardless of the VA. In 2810 * the case kpm is using small pages, we know no large kernel 2811 * mappings are located above 0x80000000.00000000 so we skip the 2812 * probe as an optimization. 2813 */ 2814dktsb4m_kpmcheck_small: 2815 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2816 /* delay slot safe, below */ 2817 2818 /* 2819 * Get second TSB offset and look for 4M mapping 2820 * using 4M virtual page as the TSB index. 2821 * 2822 * Here: 2823 * %g1 = 8K TSB pointer. Don't squash it. 2824 * %g2 = tag access register (we still need it) 2825 */ 2826 srlx %g2, MMU_PAGESHIFT4M, %g3 2827 2828 /* 2829 * We patch the next set of instructions at run time; 2830 * any changes here require sfmmu_patch_ktsb changes too. 2831 */ 2832dktsb4mbase: 2833 RUNTIME_PATCH_SETX(%g7, %g6) 2834 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2835dktsb4m: 2836 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2837 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2838 2839 /* 2840 * At this point %g3 is our index into the TSB. 2841 * We just masked off enough bits of the VA depending 2842 * on our TSB size code. 2843 */ 2844 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2845 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2846 cmp %g6, %g4 ! compare tag 2847 2848dktsb4m_tsbmiss: 2849 bne,pn %xcc, dktsb4m_kpmcheck 2850 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2851 TT_TRACE(trace_tsbhit) 2852 /* we don't check TTE size here since we assume 4M TSB is separate */ 2853 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2854 /* trapstat expects tte in %g5 */ 2855 retry 2856 2857 /* 2858 * So, we failed to find a valid TTE to match the faulting 2859 * address in either TSB. There are a few cases that could land 2860 * us here: 2861 * 2862 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2863 * to sfmmu_tsb_miss_tt to handle the miss. 2864 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2865 * 4M TSB. Let segkpm handle it. 2866 * 2867 * Note that we shouldn't land here in the case of a kpm VA when 2868 * kpm_smallpages is active -- we handled that case earlier at 2869 * dktsb4m_kpmcheck_small. 2870 * 2871 * At this point: 2872 * g1 = 8K-indexed primary TSB pointer 2873 * g2 = tag access register 2874 * g3 = 4M-indexed secondary TSB pointer 2875 */ 2876dktsb4m_kpmcheck: 2877 cmp %g2, %g0 2878 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2879 nop 2880 ba,a,pt %icc, sfmmu_tsb_miss_tt 2881 nop 2882 2883#ifdef sun4v 2884 /* 2885 * User instruction miss w/ single TSB. 2886 * The first probe covers 8K, 64K, and 512K page sizes, 2887 * because 64K and 512K mappings are replicated off 8K 2888 * pointer. 2889 * 2890 * g1 = tsb8k pointer register 2891 * g2 = tag access register 2892 * g3 - g6 = scratch registers 2893 * g7 = TSB tag to match 2894 */ 2895 .align 64 2896 ALTENTRY(sfmmu_uitlb_fastpath) 2897 2898 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2899 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2900 ba,pn %xcc, sfmmu_tsb_miss_tt 2901 mov -1, %g3 2902 2903 /* 2904 * User data miss w/ single TSB. 2905 * The first probe covers 8K, 64K, and 512K page sizes, 2906 * because 64K and 512K mappings are replicated off 8K 2907 * pointer. 2908 * 2909 * g1 = tsb8k pointer register 2910 * g2 = tag access register 2911 * g3 - g6 = scratch registers 2912 * g7 = TSB tag to match 2913 */ 2914 .align 64 2915 ALTENTRY(sfmmu_udtlb_fastpath) 2916 2917 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2918 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2919 ba,pn %xcc, sfmmu_tsb_miss_tt 2920 mov -1, %g3 2921 2922 /* 2923 * User instruction miss w/ multiple TSBs (sun4v). 2924 * The first probe covers 8K, 64K, and 512K page sizes, 2925 * because 64K and 512K mappings are replicated off 8K 2926 * pointer. Second probe covers 4M page size only. 2927 * 2928 * Just like sfmmu_udtlb_slowpath, except: 2929 * o Uses ASI_ITLB_IN 2930 * o checks for execute permission 2931 * o No ISM prediction. 2932 * 2933 * g1 = tsb8k pointer register 2934 * g2 = tag access register 2935 * g3 - g6 = scratch registers 2936 * g7 = TSB tag to match 2937 */ 2938 .align 64 2939 ALTENTRY(sfmmu_uitlb_slowpath) 2940 2941 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2942 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2943 /* g4 - g5 = clobbered here */ 2944 2945 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2946 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2947 srlx %g2, TAG_VALO_SHIFT, %g7 2948 PROBE_2ND_ITSB(%g3, %g7) 2949 /* NOT REACHED */ 2950 2951#else /* sun4v */ 2952 2953 /* 2954 * User instruction miss w/ multiple TSBs (sun4u). 2955 * The first probe covers 8K, 64K, and 512K page sizes, 2956 * because 64K and 512K mappings are replicated off 8K 2957 * pointer. Second probe covers 4M page size only. 2958 * 2959 * Just like sfmmu_udtlb_slowpath, except: 2960 * o Uses ASI_ITLB_IN 2961 * o checks for execute permission 2962 * o No ISM prediction. 2963 * 2964 * g1 = tsb8k pointer register 2965 * g2 = tag access register 2966 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch 2967 * g4 - g6 = scratch registers 2968 * g7 = TSB tag to match 2969 */ 2970 .align 64 2971 ALTENTRY(sfmmu_uitlb_slowpath) 2972 2973#ifdef UTSB_PHYS 2974 /* 2975 * g1 = 1st TSB entry pointer 2976 * g3 = 2nd TSB base register 2977 * Need 2nd TSB entry pointer for 2nd probe. 2978 */ 2979 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2980 2981 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2982#else /* UTSB_PHYS */ 2983 mov %g1, %g3 /* save tsb8k reg in %g3 */ 2984 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 2985 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2986 2987 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 2988 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 2989 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 2990#endif /* UTSB_PHYS */ 2991 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2992 srlx %g2, TAG_VALO_SHIFT, %g7 2993 PROBE_2ND_ITSB(%g3, %g7, isynth) 2994 /* NOT REACHED */ 2995#endif /* sun4v */ 2996 2997 /* 2998 * User data miss w/ multiple TSBs. 2999 * The first probe covers 8K, 64K, and 512K page sizes, 3000 * because 64K and 512K mappings are replicated off 8K 3001 * pointer. Second probe covers 4M page size only. 3002 * 3003 * We consider probing for 4M pages first if the VA falls 3004 * in a range that's likely to be ISM. 3005 * 3006 * g1 = tsb8k pointer register 3007 * g2 = tag access register 3008 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch 3009 * g4 - g6 = scratch registers 3010 * g7 = TSB tag to match 3011 */ 3012 .align 64 3013 ALTENTRY(sfmmu_udtlb_slowpath) 3014 3015 /* 3016 * Check for ISM. If it exists, look for 4M mappings in the second TSB 3017 * first, then probe for other mappings in the first TSB if that fails. 3018 */ 3019 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 3020 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 3021 mov %g1, %g3 3022 3023udtlb_miss_probefirst: 3024 /* 3025 * g1 = 8K TSB pointer register 3026 * g2 = tag access register 3027 * g3 = (potentially) second TSB entry ptr 3028 * g6 = ism pred. 3029 * g7 = vpg_4m 3030 */ 3031#ifdef sun4v 3032 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 3033 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3034 3035 /* 3036 * Here: 3037 * g1 = first TSB pointer 3038 * g2 = tag access reg 3039 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3040 */ 3041 brgz,pn %g6, sfmmu_tsb_miss_tt 3042 nop 3043#else /* sun4v */ 3044#ifndef UTSB_PHYS 3045 mov %g1, %g4 3046 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 3047#endif UTSB_PHYS 3048 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3049 3050 /* 3051 * Here: 3052 * g1 = first TSB pointer 3053 * g2 = tag access reg 3054 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3055 */ 3056 brgz,pn %g6, sfmmu_tsb_miss_tt 3057 nop 3058#ifndef UTSB_PHYS 3059 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 3060#endif UTSB_PHYS 3061 /* fall through in 8K->4M probe order */ 3062#endif /* sun4v */ 3063 3064udtlb_miss_probesecond: 3065 /* 3066 * Look in the second TSB for the TTE 3067 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 3068 * g2 = tag access reg 3069 * g3 = 8K TSB pointer register 3070 * g6 = ism pred. 3071 * g7 = vpg_4m 3072 */ 3073#ifdef sun4v 3074 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 3075 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3076 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 3077#else /* sun4v */ 3078#ifdef UTSB_PHYS 3079 GET_2ND_TSBREG(%g3) 3080 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3081 /* tagacc (%g2) is okay, no need to reload, %g3 = second tsbe ptr */ 3082#else /* UTSB_PHYS */ 3083 mov %g3, %g7 3084 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 3085 /* %g2 clobbered, %g3 =second tsbe ptr */ 3086 mov MMU_TAG_ACCESS, %g2 3087 ldxa [%g2]ASI_DMMU, %g2 3088#endif /* UTSB_PHYS */ 3089#endif /* sun4v */ 3090 3091 srlx %g2, TAG_VALO_SHIFT, %g7 3092 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3093 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 3094 brgz,pn %g6, udtlb_miss_probefirst 3095 nop 3096 3097 /* fall through to sfmmu_tsb_miss_tt */ 3098 3099 ALTENTRY(sfmmu_tsb_miss_tt) 3100 TT_TRACE(trace_tsbmiss) 3101 /* 3102 * We get here if there is a TSB miss OR a write protect trap. 3103 * 3104 * g1 = First TSB entry pointer 3105 * g2 = tag access register 3106 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 3107 * g4 - g7 = scratch registers 3108 */ 3109 3110 ALTENTRY(sfmmu_tsb_miss) 3111 3112 /* 3113 * If trapstat is running, we need to shift the %tpc and %tnpc to 3114 * point to trapstat's TSB miss return code (note that trapstat 3115 * itself will patch the correct offset to add). 3116 */ 3117 rdpr %tl, %g7 3118 cmp %g7, 1 3119 ble,pt %xcc, 0f 3120 sethi %hi(KERNELBASE), %g6 3121 rdpr %tpc, %g7 3122 or %g6, %lo(KERNELBASE), %g6 3123 cmp %g7, %g6 3124 bgeu,pt %xcc, 0f 3125 /* delay slot safe */ 3126 3127 ALTENTRY(tsbmiss_trapstat_patch_point) 3128 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 3129 wrpr %g7, %tpc 3130 add %g7, 4, %g7 3131 wrpr %g7, %tnpc 31320: 3133 CPU_TSBMISS_AREA(%g6, %g7) 3134 3135 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save first tsb pointer */ 3136 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save second tsb pointer */ 3137 3138 sllx %g2, TAGACC_CTX_LSHIFT, %g3 3139 brz,a,pn %g3, 1f /* skip ahead if kernel */ 3140 ldn [%g6 + TSBMISS_KHATID], %g7 3141 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 3142 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 3143 3144 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 3145 3146 cmp %g3, INVALID_CONTEXT 3147 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 3148 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 3149 3150#ifdef sun4v 3151 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 3152 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 3153 stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 3154#endif 3155 3156 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 3157 /* 3158 * The miss wasn't in an ISM segment. 3159 * 3160 * %g1 %g3, %g4, %g5, %g7 all clobbered 3161 * %g2 = (pseudo) tag access 3162 */ 3163 3164 ba,pt %icc, 2f 3165 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 3166 31671: 3168 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 3169 /* 3170 * 8K and 64K hash. 3171 */ 31722: 3173 3174 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3175 MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte, 3176 sfmmu_suspend_tl, tsb_512K) 3177 /* NOT REACHED */ 3178 3179tsb_512K: 3180 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3181 brz,pn %g5, 3f 3182 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3183 and %g4, HAT_512K_FLAG, %g5 3184 3185 /* 3186 * Note that there is a small window here where we may have 3187 * a 512k page in the hash list but have not set the HAT_512K_FLAG 3188 * flag yet, so we will skip searching the 512k hash list. 3189 * In this case we will end up in pagefault which will find 3190 * the mapping and return. So, in this instance we will end up 3191 * spending a bit more time resolving this TSB miss, but it can 3192 * only happen once per process and even then, the chances of that 3193 * are very small, so it's not worth the extra overhead it would 3194 * take to close this window. 3195 */ 3196 brz,pn %g5, tsb_4M 3197 nop 31983: 3199 /* 3200 * 512K hash 3201 */ 3202 3203 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3204 MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte, 3205 sfmmu_suspend_tl, tsb_4M) 3206 /* NOT REACHED */ 3207 3208tsb_4M: 3209 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3210 brz,pn %g5, 4f 3211 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3212 and %g4, HAT_4M_FLAG, %g5 3213 brz,pn %g5, tsb_32M 3214 nop 32154: 3216 /* 3217 * 4M hash 3218 */ 3219 3220 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3221 MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte, 3222 sfmmu_suspend_tl, tsb_32M) 3223 /* NOT REACHED */ 3224 3225tsb_32M: 3226 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3227#ifdef sun4v 3228 brz,pn %g5, 6f 3229#else 3230 brz,pn %g5, tsb_pagefault 3231#endif 3232 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3233 and %g4, HAT_32M_FLAG, %g5 3234 brz,pn %g5, tsb_256M 3235 nop 32365: 3237 /* 3238 * 32M hash 3239 */ 3240 3241 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3242 MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte, 3243 sfmmu_suspend_tl, tsb_256M) 3244 /* NOT REACHED */ 3245 3246#ifdef sun4u 3247#define tsb_shme tsb_pagefault 3248#endif 3249tsb_256M: 3250 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3251 and %g4, HAT_256M_FLAG, %g5 3252 brz,pn %g5, tsb_shme 3253 nop 32546: 3255 /* 3256 * 256M hash 3257 */ 3258 3259 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3260 MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte, 3261 sfmmu_suspend_tl, tsb_shme) 3262 /* NOT REACHED */ 3263 3264tsb_checktte: 3265 /* 3266 * g1 = hblk_misc 3267 * g2 = tagacc 3268 * g3 = tte 3269 * g4 = tte pa 3270 * g5 = tte va 3271 * g6 = tsbmiss area 3272 * g7 = hatid 3273 */ 3274 brlz,a,pt %g3, tsb_validtte 3275 rdpr %tt, %g7 3276 3277#ifdef sun4u 3278#undef tsb_shme 3279 ba tsb_pagefault 3280 nop 3281#else 3282 3283tsb_shme: 3284 /* 3285 * g2 = tagacc 3286 * g6 = tsbmiss area 3287 */ 3288 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3289 brz,pn %g5, tsb_pagefault 3290 nop 3291 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3292 brz,pn %g7, tsb_pagefault 3293 nop 3294 3295 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3296 MMU_PAGESHIFT64K, TTE64K, tsb_shme_l8K, tsb_shme_checktte, 3297 sfmmu_suspend_tl, tsb_shme_512K) 3298 /* NOT REACHED */ 3299 3300tsb_shme_512K: 3301 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3302 and %g4, HAT_512K_FLAG, %g5 3303 brz,pn %g5, tsb_shme_4M 3304 nop 3305 3306 /* 3307 * 512K hash 3308 */ 3309 3310 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3311 MMU_PAGESHIFT512K, TTE512K, tsb_shme_l512K, tsb_shme_checktte, 3312 sfmmu_suspend_tl, tsb_shme_4M) 3313 /* NOT REACHED */ 3314 3315tsb_shme_4M: 3316 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3317 and %g4, HAT_4M_FLAG, %g5 3318 brz,pn %g5, tsb_shme_32M 3319 nop 33204: 3321 /* 3322 * 4M hash 3323 */ 3324 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3325 MMU_PAGESHIFT4M, TTE4M, tsb_shme_l4M, tsb_shme_checktte, 3326 sfmmu_suspend_tl, tsb_shme_32M) 3327 /* NOT REACHED */ 3328 3329tsb_shme_32M: 3330 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3331 and %g4, HAT_32M_FLAG, %g5 3332 brz,pn %g5, tsb_shme_256M 3333 nop 3334 3335 /* 3336 * 32M hash 3337 */ 3338 3339 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3340 MMU_PAGESHIFT32M, TTE32M, tsb_shme_l32M, tsb_shme_checktte, 3341 sfmmu_suspend_tl, tsb_shme_256M) 3342 /* NOT REACHED */ 3343 3344tsb_shme_256M: 3345 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3346 and %g4, HAT_256M_FLAG, %g5 3347 brz,pn %g5, tsb_pagefault 3348 nop 3349 3350 /* 3351 * 256M hash 3352 */ 3353 3354 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3355 MMU_PAGESHIFT256M, TTE256M, tsb_shme_l256M, tsb_shme_checktte, 3356 sfmmu_suspend_tl, tsb_pagefault) 3357 /* NOT REACHED */ 3358 3359tsb_shme_checktte: 3360 3361 brgez,pn %g3, tsb_pagefault 3362 rdpr %tt, %g7 3363 /* 3364 * g1 = ctx1 flag 3365 * g3 = tte 3366 * g4 = tte pa 3367 * g5 = tte va 3368 * g6 = tsbmiss area 3369 * g7 = tt 3370 */ 3371 3372 brz,pt %g1, tsb_validtte 3373 nop 3374 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3375 or %g1, HAT_CHKCTX1_FLAG, %g1 3376 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3377 3378 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3379#endif /* sun4u */ 3380 3381tsb_validtte: 3382 /* 3383 * g3 = tte 3384 * g4 = tte pa 3385 * g5 = tte va 3386 * g6 = tsbmiss area 3387 * g7 = tt 3388 */ 3389 3390 /* 3391 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3392 */ 3393 cmp %g7, FAST_PROT_TT 3394 bne,pt %icc, 4f 3395 nop 3396 3397 TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod, 3398 tsb_protfault) 3399 3400 rdpr %tt, %g5 3401 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3402#ifdef sun4v 3403 MMU_FAULT_STATUS_AREA(%g7) 3404 ldx [%g7 + MMFSA_D_ADDR], %g5 /* save fault addr for later */ 3405#endif 3406 ba,pt %xcc, tsb_update_tl1 3407 nop 3408 34094: 3410 /* 3411 * If ITLB miss check exec bit. 3412 * If not set treat as invalid TTE. 3413 */ 3414 cmp %g7, T_INSTR_MMU_MISS 3415 be,pn %icc, 5f 3416 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3417 cmp %g7, FAST_IMMU_MISS_TT 3418 bne,pt %icc, 3f 3419 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 34205: 3421 bz,pn %icc, tsb_protfault 3422 nop 3423 34243: 3425 /* 3426 * Set reference bit if not already set 3427 */ 3428 TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref) 3429 3430 /* 3431 * Now, load into TSB/TLB. At this point: 3432 * g3 = tte 3433 * g4 = patte 3434 * g6 = tsbmiss area 3435 */ 3436 rdpr %tt, %g5 3437#ifdef sun4v 3438 MMU_FAULT_STATUS_AREA(%g2) 3439 cmp %g5, T_INSTR_MMU_MISS 3440 be,a,pt %icc, 9f 3441 nop 3442 cmp %g5, FAST_IMMU_MISS_TT 3443 be,a,pt %icc, 9f 3444 nop 3445 add %g2, MMFSA_D_, %g2 34469: 3447 ldx [%g2 + MMFSA_CTX_], %g7 3448 sllx %g7, TTARGET_CTX_SHIFT, %g7 3449 ldx [%g2 + MMFSA_ADDR_], %g2 3450 mov %g2, %g5 ! save the fault addr for later use 3451 srlx %g2, TTARGET_VA_SHIFT, %g2 3452 or %g2, %g7, %g2 3453#else 3454 cmp %g5, FAST_IMMU_MISS_TT 3455 be,a,pt %icc, tsb_update_tl1 3456 ldxa [%g0]ASI_IMMU, %g2 3457 ldxa [%g0]ASI_DMMU, %g2 3458#endif 3459tsb_update_tl1: 3460 srlx %g2, TTARGET_CTX_SHIFT, %g7 3461 brz,pn %g7, tsb_kernel 3462#ifdef sun4v 3463 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3464#else 3465 srlx %g3, TTE_SZ_SHFT, %g7 3466#endif 3467 3468tsb_user: 3469#ifdef sun4v 3470 cmp %g7, TTE4M 3471 bge,pn %icc, tsb_user4m 3472 nop 3473#else /* sun4v */ 3474 cmp %g7, TTESZ_VALID | TTE4M 3475 be,pn %icc, tsb_user4m 3476 srlx %g3, TTE_SZ2_SHFT, %g7 3477 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3478#ifdef ITLB_32M_256M_SUPPORT 3479 bnz,pn %icc, tsb_user4m 3480 nop 3481#else /* ITLB_32M_256M_SUPPORT */ 3482 bnz,a,pn %icc, tsb_user_pn_synth 3483 cmp %g5, FAST_IMMU_MISS_TT 3484#endif /* ITLB_32M_256M_SUPPORT */ 3485#endif /* sun4v */ 3486 3487tsb_user8k: 3488#ifdef sun4v 3489 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3490 and %g7, HAT_CHKCTX1_FLAG, %g1 3491 brz,a,pn %g1, 1f 3492 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = first TSB ptr 3493 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3494 brlz,a,pn %g1, ptl1_panic ! if no shared tsb 3495 mov PTL1_NO_SCDTSB8K, %g1 ! panic 3496 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 34971: 3498#else 3499 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = first TSB ptr 3500 3501#ifndef UTSB_PHYS 3502 mov ASI_N, %g7 ! user TSBs accessed by VA 3503 mov %g7, %asi 3504#endif /* UTSB_PHYS */ 3505 3506#endif /* sun4v */ 3507 3508 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5) 3509 3510#ifdef sun4v 3511 rdpr %tt, %g5 3512 cmp %g5, T_INSTR_MMU_MISS 3513 be,a,pn %xcc, 9f 3514 mov %g3, %g5 3515#endif /* sun4v */ 3516 cmp %g5, FAST_IMMU_MISS_TT 3517 be,pn %xcc, 9f 3518 mov %g3, %g5 3519 3520 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3521 ! trapstat wants TTE in %g5 3522 retry 35239: 3524 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3525 ! trapstat wants TTE in %g5 3526 retry 3527 3528tsb_user4m: 3529#ifdef sun4v 3530 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3531 and %g7, HAT_CHKCTX1_FLAG, %g1 3532 brz,a,pn %g1, 4f 3533 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = TSB ptr 3534 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1) 3535 brlz,a,pn %g1, 5f ! if no shared 2nd tsb 3536 nop 3537 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 3538#else 3539 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = TSB ptr 3540#endif 35414: 3542 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3543 nop 3544 3545#ifndef UTSB_PHYS 3546 mov ASI_N, %g7 ! user TSBs accessed by VA 3547 mov %g7, %asi 3548#endif /* UTSB_PHYS */ 3549 3550 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6) 3551 35525: 3553#ifdef sun4v 3554 rdpr %tt, %g5 3555 cmp %g5, T_INSTR_MMU_MISS 3556 be,a,pn %xcc, 9f 3557 mov %g3, %g5 3558#endif /* sun4v */ 3559 cmp %g5, FAST_IMMU_MISS_TT 3560 be,pn %xcc, 9f 3561 mov %g3, %g5 3562 3563 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3564 ! trapstat wants TTE in %g5 3565 retry 35669: 3567 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3568 ! trapstat wants TTE in %g5 3569 retry 3570 3571#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3572 /* 3573 * Panther ITLB synthesis. 3574 * The Panther 32M and 256M ITLB code simulates these two large page 3575 * sizes with 4M pages, to provide support for programs, for example 3576 * Java, that may copy instructions into a 32M or 256M data page and 3577 * then execute them. The code below generates the 4M pfn bits and 3578 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3579 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3580 * are ignored by the hardware. 3581 * 3582 * Now, load into TSB/TLB. At this point: 3583 * g2 = tagtarget 3584 * g3 = tte 3585 * g4 = patte 3586 * g5 = tt 3587 * g6 = tsbmiss area 3588 */ 3589tsb_user_pn_synth: 3590 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3591 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3592 bz,pn %icc, 4b /* if not, been here before */ 3593 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3594 brlz,a,pn %g1, 5f /* no 2nd tsb */ 3595 mov %g3, %g5 3596 3597 mov MMU_TAG_ACCESS, %g7 3598 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3599 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3600 3601 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3602 mov %g7, %asi 3603 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */ 36045: 3605 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3606 retry 3607 3608tsb_user_itlb_synth: 3609 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3610 3611 mov MMU_TAG_ACCESS, %g7 3612 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3613 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3614 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3615 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3616 3617 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3618 mov %g7, %asi 3619 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */ 36207: 3621 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3622 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3623 retry 3624#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3625 3626tsb_kernel: 3627#ifdef sun4v 3628 rdpr %tt, %g5 3629 cmp %g7, TTE4M 3630 bge,pn %icc, 5f 3631#else 3632 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3633 be,pn %icc, 5f 3634#endif 3635 nop 3636 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8k tsbptr 3637 ba,pt %xcc, 6f 3638 nop 36395: 3640 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4m tsbptr 3641 brlz,pn %g1, 3f /* skip programming if 4m TSB ptr is -1 */ 3642 nop 36436: 3644#ifndef sun4v 3645tsb_kernel_patch_asi: 3646 or %g0, RUNTIME_PATCH, %g6 3647 mov %g6, %asi ! XXX avoid writing to %asi !! 3648#endif 3649 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7) 36503: 3651#ifdef sun4v 3652 cmp %g5, T_INSTR_MMU_MISS 3653 be,a,pn %icc, 1f 3654 mov %g3, %g5 ! trapstat wants TTE in %g5 3655#endif /* sun4v */ 3656 cmp %g5, FAST_IMMU_MISS_TT 3657 be,pn %icc, 1f 3658 mov %g3, %g5 ! trapstat wants TTE in %g5 3659 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3660 ! trapstat wants TTE in %g5 3661 retry 36621: 3663 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3664 ! trapstat wants TTE in %g5 3665 retry 3666 3667tsb_ism: 3668 /* 3669 * This is an ISM [i|d]tlb miss. We optimize for largest 3670 * page size down to smallest. 3671 * 3672 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3673 * register 3674 * g3 = ismmap->ism_seg 3675 * g4 = physical address of ismmap->ism_sfmmu 3676 * g6 = tsbmiss area 3677 */ 3678 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3679 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3680 mov PTL1_BAD_ISM, %g1 3681 /* g5 = pa of imap_vb_shift */ 3682 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3683 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3684 srlx %g3, %g4, %g3 /* clr size field */ 3685 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3686 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3687 and %g2, %g1, %g4 /* g4 = ctx number */ 3688 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3689 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3690 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3691 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3692 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3693#ifdef sun4v 3694 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3695 brz,pt %g5, tsb_chk4M_ism 3696 nop 3697 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3698 or %g5, HAT_CHKCTX1_FLAG, %g5 3699 stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3700 rdpr %tt, %g5 3701 SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl) 3702#endif 3703 /* 3704 * ISM pages are always locked down. 3705 * If we can't find the tte then pagefault 3706 * and let the spt segment driver resolve it. 3707 * 3708 * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3709 * g4 = imap_hatflags 3710 * g6 = tsb miss area 3711 * g7 = ISM hatid 3712 */ 3713 3714tsb_chk4M_ism: 3715 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3716 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3717 nop 3718 3719tsb_ism_32M: 3720 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3721 brz,pn %g5, tsb_ism_256M 3722 nop 3723 3724 /* 3725 * 32M hash. 3726 */ 3727 3728 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M, 3729 TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3730 tsb_ism_4M) 3731 /* NOT REACHED */ 3732 3733tsb_ism_32M_found: 3734 brlz,a,pt %g3, tsb_validtte 3735 rdpr %tt, %g7 3736 ba,pt %xcc, tsb_ism_4M 3737 nop 3738 3739tsb_ism_256M: 3740 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3741 brz,a,pn %g5, ptl1_panic 3742 mov PTL1_BAD_ISM, %g1 3743 3744 /* 3745 * 256M hash. 3746 */ 3747 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M, 3748 TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3749 tsb_ism_4M) 3750 3751tsb_ism_256M_found: 3752 brlz,a,pt %g3, tsb_validtte 3753 rdpr %tt, %g7 3754 3755tsb_ism_4M: 3756 /* 3757 * 4M hash. 3758 */ 3759 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M, 3760 TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3761 tsb_ism_8K) 3762 /* NOT REACHED */ 3763 3764tsb_ism_4M_found: 3765 brlz,a,pt %g3, tsb_validtte 3766 rdpr %tt, %g7 3767 3768tsb_ism_8K: 3769 /* 3770 * 8K and 64K hash. 3771 */ 3772 3773 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K, 3774 TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3775 tsb_pagefault) 3776 /* NOT REACHED */ 3777 3778tsb_ism_8K_found: 3779 brlz,a,pt %g3, tsb_validtte 3780 rdpr %tt, %g7 3781 3782tsb_pagefault: 3783 rdpr %tt, %g7 3784 cmp %g7, FAST_PROT_TT 3785 be,a,pn %icc, tsb_protfault 3786 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3787 3788tsb_protfault: 3789 /* 3790 * we get here if we couldn't find a valid tte in the hash. 3791 * 3792 * If user and we are at tl>1 we go to window handling code. 3793 * 3794 * If kernel and the fault is on the same page as our stack 3795 * pointer, then we know the stack is bad and the trap handler 3796 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3797 * 3798 * If this is a kernel trap and tl>1, panic. 3799 * 3800 * Otherwise we call pagefault. 3801 */ 3802 cmp %g7, FAST_IMMU_MISS_TT 3803#ifdef sun4v 3804 MMU_FAULT_STATUS_AREA(%g4) 3805 ldx [%g4 + MMFSA_I_CTX], %g5 3806 ldx [%g4 + MMFSA_D_CTX], %g4 3807 move %icc, %g5, %g4 3808 cmp %g7, T_INSTR_MMU_MISS 3809 move %icc, %g5, %g4 3810#else 3811 mov MMU_TAG_ACCESS, %g4 3812 ldxa [%g4]ASI_DMMU, %g2 3813 ldxa [%g4]ASI_IMMU, %g5 3814 move %icc, %g5, %g2 3815 cmp %g7, T_INSTR_MMU_MISS 3816 move %icc, %g5, %g2 3817 sllx %g2, TAGACC_CTX_LSHIFT, %g4 3818#endif 3819 brnz,pn %g4, 3f /* skip if not kernel */ 3820 rdpr %tl, %g5 3821 3822 add %sp, STACK_BIAS, %g3 3823 srlx %g3, MMU_PAGESHIFT, %g3 3824 srlx %g2, MMU_PAGESHIFT, %g4 3825 cmp %g3, %g4 3826 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 3827 mov PTL1_BAD_STACK, %g1 3828 3829 cmp %g5, 1 3830 ble,pt %icc, 2f 3831 nop 3832 TSTAT_CHECK_TL1(2f, %g1, %g2) 3833 rdpr %tt, %g2 3834 cmp %g2, FAST_PROT_TT 3835 mov PTL1_BAD_KPROT_FAULT, %g1 3836 movne %icc, PTL1_BAD_KMISS, %g1 3837 ba,pt %icc, ptl1_panic 3838 nop 3839 38402: 3841 /* 3842 * We are taking a pagefault in the kernel on a kernel address. If 3843 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 3844 * want to call sfmmu_pagefault -- we will instead note that a fault 3845 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 3846 * (instead of a "retry"). This will step over the faulting 3847 * instruction. 3848 */ 3849 CPU_INDEX(%g1, %g2) 3850 set cpu_core, %g2 3851 sllx %g1, CPU_CORE_SHIFT, %g1 3852 add %g1, %g2, %g1 3853 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3854 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3855 bz sfmmu_pagefault 3856 or %g2, CPU_DTRACE_BADADDR, %g2 3857 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3858 GET_MMU_D_ADDR(%g3, %g4) 3859 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3860 done 3861 38623: 3863 cmp %g5, 1 3864 ble,pt %icc, 4f 3865 nop 3866 TSTAT_CHECK_TL1(4f, %g1, %g2) 3867 ba,pt %icc, sfmmu_window_trap 3868 nop 3869 38704: 3871 /* 3872 * We are taking a pagefault on a non-kernel address. If we are in 3873 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 3874 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 3875 */ 3876 CPU_INDEX(%g1, %g2) 3877 set cpu_core, %g2 3878 sllx %g1, CPU_CORE_SHIFT, %g1 3879 add %g1, %g2, %g1 3880 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3881 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3882 bz sfmmu_mmu_trap 3883 or %g2, CPU_DTRACE_BADADDR, %g2 3884 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3885 GET_MMU_D_ADDR(%g3, %g4) 3886 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3887 3888 /* 3889 * Be sure that we're actually taking this miss from the kernel -- 3890 * otherwise we have managed to return to user-level with 3891 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3892 */ 3893 rdpr %tstate, %g2 3894 btst TSTATE_PRIV, %g2 3895 bz,a ptl1_panic 3896 mov PTL1_BAD_DTRACE_FLAGS, %g1 3897 done 3898 3899 ALTENTRY(tsb_tl0_noctxt) 3900 /* 3901 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 3902 * if it is, indicated that we have faulted and issue a done. 3903 */ 3904 CPU_INDEX(%g5, %g6) 3905 set cpu_core, %g6 3906 sllx %g5, CPU_CORE_SHIFT, %g5 3907 add %g5, %g6, %g5 3908 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 3909 andcc %g6, CPU_DTRACE_NOFAULT, %g0 3910 bz 1f 3911 or %g6, CPU_DTRACE_BADADDR, %g6 3912 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 3913 GET_MMU_D_ADDR(%g3, %g4) 3914 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 3915 3916 /* 3917 * Be sure that we're actually taking this miss from the kernel -- 3918 * otherwise we have managed to return to user-level with 3919 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3920 */ 3921 rdpr %tstate, %g5 3922 btst TSTATE_PRIV, %g5 3923 bz,a ptl1_panic 3924 mov PTL1_BAD_DTRACE_FLAGS, %g1 3925 TSTAT_CHECK_TL1(2f, %g1, %g2); 39262: 3927 done 3928 39291: 3930 rdpr %tt, %g5 3931 cmp %g5, FAST_IMMU_MISS_TT 3932#ifdef sun4v 3933 MMU_FAULT_STATUS_AREA(%g2) 3934 be,a,pt %icc, 2f 3935 ldx [%g2 + MMFSA_I_CTX], %g3 3936 cmp %g5, T_INSTR_MMU_MISS 3937 be,a,pt %icc, 2f 3938 ldx [%g2 + MMFSA_I_CTX], %g3 3939 ldx [%g2 + MMFSA_D_CTX], %g3 39402: 3941#else 3942 mov MMU_TAG_ACCESS, %g2 3943 be,a,pt %icc, 2f 3944 ldxa [%g2]ASI_IMMU, %g3 3945 ldxa [%g2]ASI_DMMU, %g3 39462: sllx %g3, TAGACC_CTX_LSHIFT, %g3 3947#endif 3948 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 3949 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 3950 rdpr %tl, %g5 3951 cmp %g5, 1 3952 ble,pt %icc, sfmmu_mmu_trap 3953 nop 3954 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 3955 ba,pt %icc, sfmmu_window_trap 3956 nop 3957 SET_SIZE(sfmmu_tsb_miss) 3958#endif /* lint */ 3959 3960#if defined (lint) 3961/* 3962 * This routine will look for a user or kernel vaddr in the hash 3963 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 3964 * grab any locks. It should only be used by other sfmmu routines. 3965 */ 3966/* ARGSUSED */ 3967pfn_t 3968sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 3969{ 3970 return(0); 3971} 3972 3973/* ARGSUSED */ 3974pfn_t 3975sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno) 3976{ 3977 return(0); 3978} 3979 3980#else /* lint */ 3981 3982 ENTRY_NP(sfmmu_vatopfn) 3983 /* 3984 * disable interrupts 3985 */ 3986 rdpr %pstate, %o3 3987#ifdef DEBUG 3988 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 3989#endif 3990 /* 3991 * disable interrupts to protect the TSBMISS area 3992 */ 3993 andn %o3, PSTATE_IE, %o5 3994 wrpr %o5, 0, %pstate 3995 3996 /* 3997 * o0 = vaddr 3998 * o1 = sfmmup 3999 * o2 = ttep 4000 */ 4001 CPU_TSBMISS_AREA(%g1, %o5) 4002 ldn [%g1 + TSBMISS_KHATID], %o4 4003 cmp %o4, %o1 4004 bne,pn %ncc, vatopfn_nokernel 4005 mov TTE64K, %g5 /* g5 = rehash # */ 4006 mov %g1,%o5 /* o5 = tsbmiss_area */ 4007 /* 4008 * o0 = vaddr 4009 * o1 & o4 = hatid 4010 * o2 = ttep 4011 * o5 = tsbmiss area 4012 */ 4013 mov HBLK_RANGE_SHIFT, %g6 40141: 4015 4016 /* 4017 * o0 = vaddr 4018 * o1 = sfmmup 4019 * o2 = ttep 4020 * o3 = old %pstate 4021 * o4 = hatid 4022 * o5 = tsbmiss 4023 * g5 = rehash # 4024 * g6 = hmeshift 4025 * 4026 * The first arg to GET_TTE is actually tagaccess register 4027 * not just vaddr. Since this call is for kernel we need to clear 4028 * any lower vaddr bits that would be interpreted as ctx bits. 4029 */ 4030 set TAGACC_CTX_MASK, %g1 4031 andn %o0, %g1, %o0 4032 GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5, 4033 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 4034 4035kvtop_hblk_found: 4036 /* 4037 * o0 = vaddr 4038 * o1 = sfmmup 4039 * o2 = ttep 4040 * g1 = tte 4041 * g2 = tte pa 4042 * g3 = tte va 4043 * o2 = tsbmiss area 4044 * o1 = hat id 4045 */ 4046 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 4047 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4048 stx %g1,[%o2] /* put tte into *ttep */ 4049 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 4050 /* 4051 * o0 = vaddr 4052 * o1 = sfmmup 4053 * o2 = ttep 4054 * g1 = pfn 4055 */ 4056 ba,pt %xcc, 6f 4057 mov %g1, %o0 4058 4059kvtop_nohblk: 4060 /* 4061 * we get here if we couldn't find valid hblk in hash. We rehash 4062 * if neccesary. 4063 */ 4064 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 4065#ifdef sun4v 4066 cmp %g5, MAX_HASHCNT 4067#else 4068 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 4069#endif 4070 be,a,pn %icc, 6f 4071 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4072 mov %o1, %o4 /* restore hatid */ 4073#ifdef sun4v 4074 add %g5, 2, %g5 4075 cmp %g5, 3 4076 move %icc, MMU_PAGESHIFT4M, %g6 4077 ba,pt %icc, 1b 4078 movne %icc, MMU_PAGESHIFT256M, %g6 4079#else 4080 inc %g5 4081 cmp %g5, 2 4082 move %icc, MMU_PAGESHIFT512K, %g6 4083 ba,pt %icc, 1b 4084 movne %icc, MMU_PAGESHIFT4M, %g6 4085#endif 40866: 4087 retl 4088 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4089 4090tsb_suspend: 4091 /* 4092 * o0 = vaddr 4093 * o1 = sfmmup 4094 * o2 = ttep 4095 * g1 = tte 4096 * g2 = tte pa 4097 * g3 = tte va 4098 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 4099 */ 4100 stx %g1,[%o2] /* put tte into *ttep */ 4101 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 4102 sub %g0, 1, %o0 /* output = PFN_INVALID */ 4103 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 41048: 4105 retl 4106 wrpr %g0, %o3, %pstate /* enable interrupts */ 4107 4108vatopfn_nokernel: 4109 /* 4110 * This routine does NOT support user addresses 4111 * There is a routine in C that supports this. 4112 * The only reason why we don't have the C routine 4113 * support kernel addresses as well is because 4114 * we do va_to_pa while holding the hashlock. 4115 */ 4116 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4117 save %sp, -SA(MINFRAME), %sp 4118 sethi %hi(sfmmu_panic3), %o0 4119 call panic 4120 or %o0, %lo(sfmmu_panic3), %o0 4121 4122 SET_SIZE(sfmmu_vatopfn) 4123 4124 /* 4125 * %o0 = vaddr 4126 * %o1 = hashno (aka szc) 4127 * 4128 * 4129 * This routine is similar to sfmmu_vatopfn() but will only look for 4130 * a kernel vaddr in the hash structure for the specified rehash value. 4131 * It's just an optimization for the case when pagesize for a given 4132 * va range is already known (e.g. large page heap) and we don't want 4133 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 4134 * 4135 * Returns valid pfn or PFN_INVALID if 4136 * tte for specified rehash # is not found, invalid or suspended. 4137 */ 4138 ENTRY_NP(sfmmu_kvaszc2pfn) 4139 /* 4140 * disable interrupts 4141 */ 4142 rdpr %pstate, %o3 4143#ifdef DEBUG 4144 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 4145#endif 4146 /* 4147 * disable interrupts to protect the TSBMISS area 4148 */ 4149 andn %o3, PSTATE_IE, %o5 4150 wrpr %o5, 0, %pstate 4151 4152 CPU_TSBMISS_AREA(%g1, %o5) 4153 ldn [%g1 + TSBMISS_KHATID], %o4 4154 sll %o1, 1, %g6 4155 add %g6, %o1, %g6 4156 add %g6, MMU_PAGESHIFT, %g6 4157 /* 4158 * %o0 = vaddr 4159 * %o1 = hashno 4160 * %o3 = old %pstate 4161 * %o4 = ksfmmup 4162 * %g1 = tsbmiss area 4163 * %g6 = hmeshift 4164 */ 4165 4166 /* 4167 * The first arg to GET_TTE is actually tagaccess register 4168 * not just vaddr. Since this call is for kernel we need to clear 4169 * any lower vaddr bits that would be interpreted as ctx bits. 4170 */ 4171 srlx %o0, MMU_PAGESHIFT, %o0 4172 sllx %o0, MMU_PAGESHIFT, %o0 4173 GET_TTE(%o0, %o4, %g3, %g4, %g5, %g1, %o5, %g6, %o1, 4174 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 4175 kvaszc2pfn_nohblk) 4176 4177kvaszc2pfn_hblk_found: 4178 /* 4179 * %g3 = tte 4180 * %o0 = vaddr 4181 */ 4182 brgez,a,pn %g3, 1f /* check if tte is invalid */ 4183 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4184 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 4185 /* 4186 * g3 = pfn 4187 */ 4188 ba,pt %xcc, 1f 4189 mov %g3, %o0 4190 4191kvaszc2pfn_nohblk: 4192 mov -1, %o0 4193 41941: 4195 retl 4196 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4197 4198 SET_SIZE(sfmmu_kvaszc2pfn) 4199 4200#endif /* lint */ 4201 4202 4203 4204#if !defined(lint) 4205 4206/* 4207 * kpm lock used between trap level tsbmiss handler and kpm C level. 4208 */ 4209#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 4210 mov 0xff, tmp1 ;\ 4211label1: ;\ 4212 casa [kpmlckp]asi, %g0, tmp1 ;\ 4213 brnz,pn tmp1, label1 ;\ 4214 mov 0xff, tmp1 ;\ 4215 membar #LoadLoad 4216 4217#define KPMLOCK_EXIT(kpmlckp, asi) \ 4218 membar #LoadStore|#StoreStore ;\ 4219 sta %g0, [kpmlckp]asi 4220 4221/* 4222 * Lookup a memseg for a given pfn and if found, return the physical 4223 * address of the corresponding struct memseg in mseg, otherwise 4224 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 4225 * tsbmp, %asi is assumed to be ASI_MEM. 4226 * This lookup is done by strictly traversing only the physical memseg 4227 * linkage. The more generic approach, to check the virtual linkage 4228 * before using the physical (used e.g. with hmehash buckets), cannot 4229 * be used here. Memory DR operations can run in parallel to this 4230 * lookup w/o any locks and updates of the physical and virtual linkage 4231 * cannot be done atomically wrt. to each other. Because physical 4232 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 4233 * as "physical NULL" pointer. 4234 */ 4235#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 4236 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 4237 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 4238 udivx pfn, mseg, mseg ;\ 4239 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 4240 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 4241 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 4242 add tmp1, mseg, tmp1 ;\ 4243 ldxa [tmp1]%asi, mseg ;\ 4244 cmp mseg, MSEG_NULLPTR_PA ;\ 4245 be,pn %xcc, label/**/1 /* if not found */ ;\ 4246 nop ;\ 4247 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4248 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4249 blu,pn %xcc, label/**/1 ;\ 4250 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4251 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4252 bgeu,pn %xcc, label/**/1 ;\ 4253 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 4254 mulx tmp1, PAGE_SIZE, tmp1 ;\ 4255 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 4256 add tmp2, tmp1, tmp1 /* pp */ ;\ 4257 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 4258 cmp tmp2, pfn ;\ 4259 be,pt %xcc, label/**/_ok /* found */ ;\ 4260label/**/1: ;\ 4261 /* brute force lookup */ ;\ 4262 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 4263 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 4264label/**/2: ;\ 4265 cmp mseg, MSEG_NULLPTR_PA ;\ 4266 be,pn %xcc, label/**/_ok /* if not found */ ;\ 4267 nop ;\ 4268 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4269 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4270 blu,a,pt %xcc, label/**/2 ;\ 4271 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4272 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4273 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4274 bgeu,a,pt %xcc, label/**/2 ;\ 4275 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4276label/**/_ok: 4277 4278 /* 4279 * kpm tsb miss handler large pages 4280 * g1 = 8K kpm TSB entry pointer 4281 * g2 = tag access register 4282 * g3 = 4M kpm TSB entry pointer 4283 */ 4284 ALTENTRY(sfmmu_kpm_dtsb_miss) 4285 TT_TRACE(trace_tsbmiss) 4286 4287 CPU_INDEX(%g7, %g6) 4288 sethi %hi(kpmtsbm_area), %g6 4289 sllx %g7, KPMTSBM_SHIFT, %g7 4290 or %g6, %lo(kpmtsbm_area), %g6 4291 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4292 4293 /* check enable flag */ 4294 ldub [%g6 + KPMTSBM_FLAGS], %g4 4295 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4296 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4297 nop 4298 4299 /* VA range check */ 4300 ldx [%g6 + KPMTSBM_VBASE], %g7 4301 cmp %g2, %g7 4302 blu,pn %xcc, sfmmu_tsb_miss 4303 ldx [%g6 + KPMTSBM_VEND], %g5 4304 cmp %g2, %g5 4305 bgeu,pn %xcc, sfmmu_tsb_miss 4306 stx %g3, [%g6 + KPMTSBM_TSBPTR] 4307 4308 /* 4309 * check TL tsbmiss handling flag 4310 * bump tsbmiss counter 4311 */ 4312 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4313#ifdef DEBUG 4314 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4315 inc %g5 4316 brz,pn %g3, sfmmu_kpm_exception 4317 st %g5, [%g6 + KPMTSBM_TSBMISS] 4318#else 4319 inc %g5 4320 st %g5, [%g6 + KPMTSBM_TSBMISS] 4321#endif 4322 /* 4323 * At this point: 4324 * g1 = 8K kpm TSB pointer (not used) 4325 * g2 = tag access register 4326 * g3 = clobbered 4327 * g6 = per-CPU kpm tsbmiss area 4328 * g7 = kpm_vbase 4329 */ 4330 4331 /* vaddr2pfn */ 4332 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4333 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4334 srax %g4, %g3, %g2 /* which alias range (r) */ 4335 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4336 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4337 4338 /* 4339 * Setup %asi 4340 * mseg_pa = page_numtomemseg_nolock(pfn) 4341 * if (mseg_pa == NULL) sfmmu_kpm_exception 4342 * g2=pfn 4343 */ 4344 mov ASI_MEM, %asi 4345 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4346 cmp %g3, MSEG_NULLPTR_PA 4347 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4348 nop 4349 4350 /* 4351 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4352 * g2=pfn g3=mseg_pa 4353 */ 4354 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4355 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4356 srlx %g2, %g5, %g4 4357 sllx %g4, %g5, %g4 4358 sub %g4, %g7, %g4 4359 srlx %g4, %g5, %g4 4360 4361 /* 4362 * Validate inx value 4363 * g2=pfn g3=mseg_pa g4=inx 4364 */ 4365#ifdef DEBUG 4366 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4367 cmp %g4, %g5 /* inx - nkpmpgs */ 4368 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4369 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4370#else 4371 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4372#endif 4373 /* 4374 * kp = &mseg_pa->kpm_pages[inx] 4375 */ 4376 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4377 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4378 add %g5, %g4, %g5 /* kp */ 4379 4380 /* 4381 * KPMP_HASH(kp) 4382 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4383 */ 4384 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4385 sub %g7, 1, %g7 /* mask */ 4386 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4387 add %g5, %g1, %g5 /* y = ksp + x */ 4388 and %g5, %g7, %g5 /* hashinx = y & mask */ 4389 4390 /* 4391 * Calculate physical kpm_page pointer 4392 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4393 */ 4394 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4395 add %g1, %g4, %g1 /* kp_pa */ 4396 4397 /* 4398 * Calculate physical hash lock address 4399 * g1=kp_refcntc_pa g2=pfn g5=hashinx 4400 */ 4401 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4402 sllx %g5, KPMHLK_SHIFT, %g5 4403 add %g4, %g5, %g3 4404 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4405 4406 /* 4407 * Assemble tte 4408 * g1=kp_pa g2=pfn g3=hlck_pa 4409 */ 4410#ifdef sun4v 4411 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4412 sllx %g5, 32, %g5 4413 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4414 or %g4, TTE4M, %g4 4415 or %g5, %g4, %g5 4416#else 4417 sethi %hi(TTE_VALID_INT), %g4 4418 mov TTE4M, %g5 4419 sllx %g5, TTE_SZ_SHFT_INT, %g5 4420 or %g5, %g4, %g5 /* upper part */ 4421 sllx %g5, 32, %g5 4422 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4423 or %g5, %g4, %g5 4424#endif 4425 sllx %g2, MMU_PAGESHIFT, %g4 4426 or %g5, %g4, %g5 /* tte */ 4427 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4428 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4429 4430 /* 4431 * tsb dropin 4432 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4433 */ 4434 4435 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4436 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4437 4438 /* use C-handler if there's no go for dropin */ 4439 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4440 cmp %g7, -1 4441 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4442 nop 4443 4444#ifdef DEBUG 4445 /* double check refcnt */ 4446 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4447 brz,pn %g7, 5f /* let C-handler deal with this */ 4448 nop 4449#endif 4450 4451#ifndef sun4v 4452 ldub [%g6 + KPMTSBM_FLAGS], %g7 4453 mov ASI_N, %g1 4454 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4455 movnz %icc, ASI_MEM, %g1 4456 mov %g1, %asi 4457#endif 4458 4459 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4460 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4461 4462 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4463 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4464 4465 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4466 4467 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4468 KPMLOCK_EXIT(%g3, ASI_MEM) 4469 4470 /* 4471 * If trapstat is running, we need to shift the %tpc and %tnpc to 4472 * point to trapstat's TSB miss return code (note that trapstat 4473 * itself will patch the correct offset to add). 4474 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4475 */ 4476 rdpr %tl, %g7 4477 cmp %g7, 1 4478 ble %icc, 0f 4479 sethi %hi(KERNELBASE), %g6 4480 rdpr %tpc, %g7 4481 or %g6, %lo(KERNELBASE), %g6 4482 cmp %g7, %g6 4483 bgeu %xcc, 0f 4484 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4485 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4486 wrpr %g7, %tpc 4487 add %g7, 4, %g7 4488 wrpr %g7, %tnpc 44890: 4490 retry 44915: 4492 /* g3=hlck_pa */ 4493 KPMLOCK_EXIT(%g3, ASI_MEM) 4494 ba,pt %icc, sfmmu_kpm_exception 4495 nop 4496 SET_SIZE(sfmmu_kpm_dtsb_miss) 4497 4498 /* 4499 * kpm tsbmiss handler for smallpages 4500 * g1 = 8K kpm TSB pointer 4501 * g2 = tag access register 4502 * g3 = 4M kpm TSB pointer 4503 */ 4504 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4505 TT_TRACE(trace_tsbmiss) 4506 CPU_INDEX(%g7, %g6) 4507 sethi %hi(kpmtsbm_area), %g6 4508 sllx %g7, KPMTSBM_SHIFT, %g7 4509 or %g6, %lo(kpmtsbm_area), %g6 4510 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4511 4512 /* check enable flag */ 4513 ldub [%g6 + KPMTSBM_FLAGS], %g4 4514 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4515 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4516 nop 4517 4518 /* 4519 * VA range check 4520 * On fail: goto sfmmu_tsb_miss 4521 */ 4522 ldx [%g6 + KPMTSBM_VBASE], %g7 4523 cmp %g2, %g7 4524 blu,pn %xcc, sfmmu_tsb_miss 4525 ldx [%g6 + KPMTSBM_VEND], %g5 4526 cmp %g2, %g5 4527 bgeu,pn %xcc, sfmmu_tsb_miss 4528 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4529 4530 /* 4531 * check TL tsbmiss handling flag 4532 * bump tsbmiss counter 4533 */ 4534 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4535#ifdef DEBUG 4536 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4537 inc %g5 4538 brz,pn %g1, sfmmu_kpm_exception 4539 st %g5, [%g6 + KPMTSBM_TSBMISS] 4540#else 4541 inc %g5 4542 st %g5, [%g6 + KPMTSBM_TSBMISS] 4543#endif 4544 /* 4545 * At this point: 4546 * g1 = clobbered 4547 * g2 = tag access register 4548 * g3 = 4M kpm TSB pointer (not used) 4549 * g6 = per-CPU kpm tsbmiss area 4550 * g7 = kpm_vbase 4551 */ 4552 4553 /* vaddr2pfn */ 4554 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4555 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4556 srax %g4, %g3, %g2 /* which alias range (r) */ 4557 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4558 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4559 4560 /* 4561 * Setup %asi 4562 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4563 * if (mseg not found) sfmmu_kpm_exception 4564 * g2=pfn 4565 */ 4566 mov ASI_MEM, %asi 4567 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4568 cmp %g3, MSEG_NULLPTR_PA 4569 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4570 nop 4571 4572 /* 4573 * inx = pfn - mseg_pa->kpm_pbase 4574 * g2=pfn g3=mseg_pa 4575 */ 4576 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4577 sub %g2, %g7, %g4 4578 4579#ifdef DEBUG 4580 /* 4581 * Validate inx value 4582 * g2=pfn g3=mseg_pa g4=inx 4583 */ 4584 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4585 cmp %g4, %g5 /* inx - nkpmpgs */ 4586 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4587 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4588#else 4589 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4590#endif 4591 /* ksp = &mseg_pa->kpm_spages[inx] */ 4592 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4593 add %g5, %g4, %g5 /* ksp */ 4594 4595 /* 4596 * KPMP_SHASH(kp) 4597 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz 4598 */ 4599 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4600 sub %g7, 1, %g7 /* mask */ 4601 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4602 add %g5, %g1, %g5 /* y = ksp + x */ 4603 and %g5, %g7, %g5 /* hashinx = y & mask */ 4604 4605 /* 4606 * Calculate physical kpm_spage pointer 4607 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4608 */ 4609 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4610 add %g1, %g4, %g1 /* ksp_pa */ 4611 4612 /* 4613 * Calculate physical hash lock address. 4614 * Note: Changes in kpm_shlk_t must be reflected here. 4615 * g1=ksp_pa g2=pfn g5=hashinx 4616 */ 4617 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4618 sllx %g5, KPMSHLK_SHIFT, %g5 4619 add %g4, %g5, %g3 /* hlck_pa */ 4620 4621 /* 4622 * Assemble tte 4623 * g1=ksp_pa g2=pfn g3=hlck_pa 4624 */ 4625 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4626 sllx %g5, 32, %g5 4627 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4628 or %g5, %g4, %g5 4629 sllx %g2, MMU_PAGESHIFT, %g4 4630 or %g5, %g4, %g5 /* tte */ 4631 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4632 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4633 4634 /* 4635 * tsb dropin 4636 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte 4637 */ 4638 4639 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4640 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4641 4642 /* use C-handler if there's no go for dropin */ 4643 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4644 cmp %g7, -1 4645 bne,pn %xcc, 5f 4646 nop 4647 4648#ifndef sun4v 4649 ldub [%g6 + KPMTSBM_FLAGS], %g7 4650 mov ASI_N, %g1 4651 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4652 movnz %icc, ASI_MEM, %g1 4653 mov %g1, %asi 4654#endif 4655 4656 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4657 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4658 4659 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4660 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4661 4662 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4663 4664 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4665 KPMLOCK_EXIT(%g3, ASI_MEM) 4666 4667 /* 4668 * If trapstat is running, we need to shift the %tpc and %tnpc to 4669 * point to trapstat's TSB miss return code (note that trapstat 4670 * itself will patch the correct offset to add). 4671 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4672 */ 4673 rdpr %tl, %g7 4674 cmp %g7, 1 4675 ble %icc, 0f 4676 sethi %hi(KERNELBASE), %g6 4677 rdpr %tpc, %g7 4678 or %g6, %lo(KERNELBASE), %g6 4679 cmp %g7, %g6 4680 bgeu %xcc, 0f 4681 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4682 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4683 wrpr %g7, %tpc 4684 add %g7, 4, %g7 4685 wrpr %g7, %tnpc 46860: 4687 retry 46885: 4689 /* g3=hlck_pa */ 4690 KPMLOCK_EXIT(%g3, ASI_MEM) 4691 ba,pt %icc, sfmmu_kpm_exception 4692 nop 4693 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4694 4695#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4696#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4697#endif 4698 4699#endif /* lint */ 4700 4701#ifdef lint 4702/* 4703 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4704 * Called from C-level, sets/clears "go" indication for trap level handler. 4705 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 4706 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 4707 * Assumes khl_mutex is held when called from C-level. 4708 */ 4709/* ARGSUSED */ 4710void 4711sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 4712{ 4713} 4714 4715/* 4716 * kpm_smallpages: stores val to byte at address mapped within 4717 * low level lock brackets. The old value is returned. 4718 * Called from C-level. 4719 */ 4720/* ARGSUSED */ 4721int 4722sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val) 4723{ 4724 return (0); 4725} 4726 4727#else /* lint */ 4728 4729 .seg ".data" 4730sfmmu_kpm_tsbmtl_panic: 4731 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4732 .byte 0 4733sfmmu_kpm_stsbmtl_panic: 4734 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4735 .byte 0 4736 .align 4 4737 .seg ".text" 4738 4739 ENTRY_NP(sfmmu_kpm_tsbmtl) 4740 rdpr %pstate, %o3 4741 /* 4742 * %o0 = &kp_refcntc 4743 * %o1 = &khl_lock 4744 * %o2 = 0/1 (off/on) 4745 * %o3 = pstate save 4746 */ 4747#ifdef DEBUG 4748 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4749 bnz,pt %icc, 1f /* disabled, panic */ 4750 nop 4751 save %sp, -SA(MINFRAME), %sp 4752 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4753 call panic 4754 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4755 ret 4756 restore 47571: 4758#endif /* DEBUG */ 4759 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4760 4761 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4762 mov -1, %o5 4763 brz,a %o2, 2f 4764 mov 0, %o5 47652: 4766 sth %o5, [%o0] 4767 KPMLOCK_EXIT(%o1, ASI_N) 4768 4769 retl 4770 wrpr %g0, %o3, %pstate /* enable interrupts */ 4771 SET_SIZE(sfmmu_kpm_tsbmtl) 4772 4773 ENTRY_NP(sfmmu_kpm_stsbmtl) 4774 rdpr %pstate, %o3 4775 /* 4776 * %o0 = &mapped 4777 * %o1 = &kshl_lock 4778 * %o2 = val 4779 * %o3 = pstate save 4780 */ 4781#ifdef DEBUG 4782 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4783 bnz,pt %icc, 1f /* disabled, panic */ 4784 nop 4785 save %sp, -SA(MINFRAME), %sp 4786 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4787 call panic 4788 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4789 ret 4790 restore 47911: 4792#endif /* DEBUG */ 4793 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4794 4795 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4796 ldsb [%o0], %o5 4797 stb %o2, [%o0] 4798 KPMLOCK_EXIT(%o1, ASI_N) 4799 4800 mov %o5, %o0 /* return old val */ 4801 retl 4802 wrpr %g0, %o3, %pstate /* enable interrupts */ 4803 SET_SIZE(sfmmu_kpm_stsbmtl) 4804 4805#endif /* lint */ 4806 4807#ifndef lint 4808#ifdef sun4v 4809 /* 4810 * User/kernel data miss w// multiple TSBs 4811 * The first probe covers 8K, 64K, and 512K page sizes, 4812 * because 64K and 512K mappings are replicated off 8K 4813 * pointer. Second probe covers 4M page size only. 4814 * 4815 * MMU fault area contains miss address and context. 4816 */ 4817 ALTENTRY(sfmmu_slow_dmmu_miss) 4818 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 4819 4820slow_miss_common: 4821 /* 4822 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 4823 * %g3 = ctx (cannot be INVALID_CONTEXT) 4824 */ 4825 brnz,pt %g3, 8f ! check for user context 4826 nop 4827 4828 /* 4829 * Kernel miss 4830 * Get 8K and 4M TSB pointers in %g1 and %g3 and 4831 * branch to sfmmu_tsb_miss_tt to handle it. 4832 */ 4833 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4834sfmmu_dslow_patch_ktsb_base: 4835 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 4836sfmmu_dslow_patch_ktsb_szcode: 4837 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 4838 4839 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 4840 ! %g1 = First TSB entry pointer, as TSB miss handler expects 4841 4842 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4843sfmmu_dslow_patch_ktsb4m_base: 4844 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 4845sfmmu_dslow_patch_ktsb4m_szcode: 4846 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 4847 4848 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 4849 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 4850 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4851 .empty 4852 48538: 4854 /* 4855 * User miss 4856 * Get first TSB pointer in %g1 4857 * Get second TSB pointer (or NULL if no second TSB) in %g3 4858 * Branch to sfmmu_tsb_miss_tt to handle it 4859 */ 4860 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 4861 /* %g1 = first TSB entry ptr now, %g2 preserved */ 4862 4863 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 4864 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 4865 nop 4866 4867 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 4868 /* %g3 = second TSB entry ptr now, %g2 preserved */ 48699: 4870 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4871 .empty 4872 SET_SIZE(sfmmu_slow_dmmu_miss) 4873 4874 4875 /* 4876 * User/kernel instruction miss w/ multiple TSBs 4877 * The first probe covers 8K, 64K, and 512K page sizes, 4878 * because 64K and 512K mappings are replicated off 8K 4879 * pointer. Second probe covers 4M page size only. 4880 * 4881 * MMU fault area contains miss address and context. 4882 */ 4883 ALTENTRY(sfmmu_slow_immu_miss) 4884 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 4885 ba,a,pt %xcc, slow_miss_common 4886 SET_SIZE(sfmmu_slow_immu_miss) 4887 4888#endif /* sun4v */ 4889#endif /* lint */ 4890 4891#ifndef lint 4892 4893/* 4894 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 4895 */ 4896 .seg ".data" 4897 .align 64 4898 .global tsbmiss_area 4899tsbmiss_area: 4900 .skip (TSBMISS_SIZE * NCPU) 4901 4902 .align 64 4903 .global kpmtsbm_area 4904kpmtsbm_area: 4905 .skip (KPMTSBM_SIZE * NCPU) 4906#endif /* lint */ 4907