1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#pragma ident "%Z%%M% %I% %E% SMI" 27 28/* 29 * SFMMU primitives. These primitives should only be used by sfmmu 30 * routines. 31 */ 32 33#if defined(lint) 34#include <sys/types.h> 35#else /* lint */ 36#include "assym.h" 37#endif /* lint */ 38 39#include <sys/asm_linkage.h> 40#include <sys/machtrap.h> 41#include <sys/machasi.h> 42#include <sys/sun4asi.h> 43#include <sys/pte.h> 44#include <sys/mmu.h> 45#include <vm/hat_sfmmu.h> 46#include <vm/seg_spt.h> 47#include <sys/machparam.h> 48#include <sys/privregs.h> 49#include <sys/scb.h> 50#include <sys/intreg.h> 51#include <sys/machthread.h> 52#include <sys/intr.h> 53#include <sys/clock.h> 54#include <sys/trapstat.h> 55 56#ifdef TRAPTRACE 57#include <sys/traptrace.h> 58 59/* 60 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 61 */ 62#define TT_TRACE(label) \ 63 ba label ;\ 64 rd %pc, %g7 65#else 66 67#define TT_TRACE(label) 68 69#endif /* TRAPTRACE */ 70 71#ifndef lint 72 73#if (TTE_SUSPEND_SHIFT > 0) 74#define TTE_SUSPEND_INT_SHIFT(reg) \ 75 sllx reg, TTE_SUSPEND_SHIFT, reg 76#else 77#define TTE_SUSPEND_INT_SHIFT(reg) 78#endif 79 80#endif /* lint */ 81 82#ifndef lint 83 84/* 85 * Assumes TSBE_TAG is 0 86 * Assumes TSBE_INTHI is 0 87 * Assumes TSBREG.split is 0 88 */ 89 90#if TSBE_TAG != 0 91#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 92#endif 93 94#if TSBTAG_INTHI != 0 95#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 96#endif 97 98/* 99 * The following code assumes the tsb is not split. 100 * 101 * With TSBs no longer shared between processes, it's no longer 102 * necessary to hash the context bits into the tsb index to get 103 * tsb coloring; the new implementation treats the TSB as a 104 * direct-mapped, virtually-addressed cache. 105 * 106 * In: 107 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 108 * tsbbase = base address of TSB (clobbered) 109 * tagacc = tag access register (clobbered) 110 * szc = size code of TSB (ro) 111 * tmp = scratch reg 112 * Out: 113 * tsbbase = pointer to entry in TSB 114 */ 115#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 116 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 117 srlx tagacc, vpshift, tagacc ;\ 118 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 119 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 120 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 121 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 122 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 123 124/* 125 * When the kpm TSB is used it is assumed that it is direct mapped 126 * using (vaddr>>vpshift)%tsbsz as the index. 127 * 128 * Note that, for now, the kpm TSB and kernel TSB are the same for 129 * each mapping size. However that need not always be the case. If 130 * the trap handlers are updated to search a different TSB for kpm 131 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 132 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 133 * 134 * In: 135 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 136 * vaddr = virtual address (clobbered) 137 * tsbp, szc, tmp = scratch 138 * Out: 139 * tsbp = pointer to entry in TSB 140 */ 141#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 142 cmp vpshift, MMU_PAGESHIFT ;\ 143 bne,pn %icc, 1f /* branch if large case */ ;\ 144 sethi %hi(kpmsm_tsbsz), szc ;\ 145 sethi %hi(kpmsm_tsbbase), tsbp ;\ 146 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 147 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 148 ba,pt %icc, 2f ;\ 149 nop ;\ 1501: sethi %hi(kpm_tsbsz), szc ;\ 151 sethi %hi(kpm_tsbbase), tsbp ;\ 152 ld [szc + %lo(kpm_tsbsz)], szc ;\ 153 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1542: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 155 156/* 157 * Lock the TSBE at virtual address tsbep. 158 * 159 * tsbep = TSBE va (ro) 160 * tmp1, tmp2 = scratch registers (clobbered) 161 * label = label to use for branches (text) 162 * %asi = ASI to use for TSB access 163 * 164 * NOTE that we flush the TSB using fast VIS instructions that 165 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 166 * not be treated as a locked entry or we'll get stuck spinning on 167 * an entry that isn't locked but really invalid. 168 */ 169 170#if defined(UTSB_PHYS) 171 172#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 173 lda [tsbep]ASI_MEM, tmp1 ;\ 174label: ;\ 175 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 176 cmp tmp1, tmp2 ;\ 177 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 178 lda [tsbep]ASI_MEM, tmp1 ;\ 179 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 180 cmp tmp1, tmp2 ;\ 181 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 182 lda [tsbep]ASI_MEM, tmp1 ;\ 183 /* tsbe lock acquired */ ;\ 184 membar #StoreStore 185 186#else /* UTSB_PHYS */ 187 188#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 189 lda [tsbep]%asi, tmp1 ;\ 190label: ;\ 191 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 192 cmp tmp1, tmp2 ;\ 193 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 194 lda [tsbep]%asi, tmp1 ;\ 195 casa [tsbep]%asi, tmp1, tmp2 ;\ 196 cmp tmp1, tmp2 ;\ 197 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 198 lda [tsbep]%asi, tmp1 ;\ 199 /* tsbe lock acquired */ ;\ 200 membar #StoreStore 201 202#endif /* UTSB_PHYS */ 203 204/* 205 * Atomically write TSBE at virtual address tsbep. 206 * 207 * tsbep = TSBE va (ro) 208 * tte = TSBE TTE (ro) 209 * tagtarget = TSBE tag (ro) 210 * %asi = ASI to use for TSB access 211 */ 212 213#if defined(UTSB_PHYS) 214 215#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 216 add tsbep, TSBE_TTE, tmp1 ;\ 217 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 218 membar #StoreStore ;\ 219 add tsbep, TSBE_TAG, tmp1 ;\ 220 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 221 222#else /* UTSB_PHYS */ 223 224#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 225 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 226 membar #StoreStore ;\ 227 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 228 229#endif /* UTSB_PHYS */ 230 231/* 232 * Load an entry into the TSB at TL > 0. 233 * 234 * tsbep = pointer to the TSBE to load as va (ro) 235 * tte = value of the TTE retrieved and loaded (wo) 236 * tagtarget = tag target register. To get TSBE tag to load, 237 * we need to mask off the context and leave only the va (clobbered) 238 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 239 * tmp1, tmp2 = scratch registers 240 * label = label to use for branches (text) 241 * %asi = ASI to use for TSB access 242 */ 243 244#if defined(UTSB_PHYS) 245 246#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 247 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 248 /* ;\ 249 * I don't need to update the TSB then check for the valid tte. ;\ 250 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 251 * we always invalidate the hash table before we unload the TSB.;\ 252 */ ;\ 253 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 254 ldxa [ttepa]ASI_MEM, tte ;\ 255 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 256 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 257 add tsbep, TSBE_TAG, tmp1 ;\ 258 brgez,a,pn tte, label/**/f ;\ 259 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 260 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 261label: 262 263#else /* UTSB_PHYS */ 264 265#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 266 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 267 /* ;\ 268 * I don't need to update the TSB then check for the valid tte. ;\ 269 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 270 * we always invalidate the hash table before we unload the TSB.;\ 271 */ ;\ 272 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 273 ldxa [ttepa]ASI_MEM, tte ;\ 274 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 275 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 276 brgez,a,pn tte, label/**/f ;\ 277 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 278 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 279label: 280 281#endif /* UTSB_PHYS */ 282 283/* 284 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 285 * for ITLB synthesis. 286 * 287 * tsbep = pointer to the TSBE to load as va (ro) 288 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 289 * with exec_perm turned off and exec_synth turned on 290 * tagtarget = tag target register. To get TSBE tag to load, 291 * we need to mask off the context and leave only the va (clobbered) 292 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 293 * tmp1, tmp2 = scratch registers 294 * label = label to use for branch (text) 295 * %asi = ASI to use for TSB access 296 */ 297 298#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 299 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 300 /* ;\ 301 * I don't need to update the TSB then check for the valid tte. ;\ 302 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 303 * we always invalidate the hash table before we unload the TSB.;\ 304 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 305 * and exec_synth bit to 1. ;\ 306 */ ;\ 307 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 308 mov tte, tmp1 ;\ 309 ldxa [ttepa]ASI_MEM, tte ;\ 310 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 311 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 312 brgez,a,pn tte, label/**/f ;\ 313 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 314 or tte, tmp1, tte ;\ 315 andn tte, TTE_EXECPRM_INT, tte ;\ 316 or tte, TTE_E_SYNTH_INT, tte ;\ 317 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 318label: 319 320/* 321 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 322 * 323 * tte = value of the TTE, used to get tte_size bits (ro) 324 * tagaccess = tag access register, used to get 4M pfn bits (ro) 325 * pfn = 4M pfn bits shifted to offset for tte (out) 326 * tmp1 = scratch register 327 * label = label to use for branch (text) 328 */ 329 330#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 331 /* ;\ 332 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 333 * Return them, shifted, in pfn. ;\ 334 */ ;\ 335 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 336 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 337 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 338 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 339 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 340 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 341label: ;\ 342 sllx tagaccess, MMU_PAGESHIFT4M, pfn 343 344/* 345 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 346 * for ITLB synthesis. 347 * 348 * tte = value of the TTE, used to get tte_size bits (rw) 349 * tmp1 = scratch register 350 */ 351 352#define SET_TTE4M_PN(tte, tmp) \ 353 /* ;\ 354 * Set 4M pagesize tte bits. ;\ 355 */ ;\ 356 set TTE4M, tmp ;\ 357 sllx tmp, TTE_SZ_SHFT, tmp ;\ 358 or tte, tmp, tte 359 360/* 361 * Load an entry into the TSB at TL=0. 362 * 363 * tsbep = pointer to the TSBE to load as va (ro) 364 * tteva = pointer to the TTE to load as va (ro) 365 * tagtarget = TSBE tag to load (which contains no context), synthesized 366 * to match va of MMU tag target register only (ro) 367 * tmp1, tmp2 = scratch registers (clobbered) 368 * label = label to use for branches (text) 369 * %asi = ASI to use for TSB access 370 */ 371 372#if defined(UTSB_PHYS) 373 374#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 375 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 376 ldx [tteva], tteva /* load tte */ ;\ 377 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 378 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 379 add tsbep, TSBE_TAG, tmp1 ;\ 380 brgez,a,pn tteva, label/**/f ;\ 381 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 382 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 383label: 384 385#else /* UTSB_PHYS */ 386 387#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 388 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 389 ldx [tteva], tteva /* load tte */ ;\ 390 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 391 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 392 brgez,a,pn tteva, label/**/f ;\ 393 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 394 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 395label: 396 397#endif /* UTSB_PHYS */ 398 399/* 400 * Invalidate a TSB entry in the TSB. 401 * 402 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 403 * about this earlier to ensure this is true. Thus when we are 404 * directly referencing tsbep below, we are referencing the tte_tag 405 * field of the TSBE. If this offset ever changes, the code below 406 * will need to be modified. 407 * 408 * tsbep = pointer to TSBE as va (ro) 409 * tag = invalidation is done if this matches the TSBE tag (ro) 410 * tmp1 - tmp3 = scratch registers (clobbered) 411 * label = label name to use for branches (text) 412 * %asi = ASI to use for TSB access 413 */ 414 415#if defined(UTSB_PHYS) 416 417#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 418 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 419 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 420label/**/1: ;\ 421 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 422 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 423 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 424 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 425 cmp tag, tmp3 /* compare tags */ ;\ 426 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 427 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 428 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 429 cmp tmp1, tmp3 /* if not successful */ ;\ 430 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 431 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 432label/**/2: 433 434#else /* UTSB_PHYS */ 435 436#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 437 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 438 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 439label/**/1: ;\ 440 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 441 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 442 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 443 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 444 cmp tag, tmp3 /* compare tags */ ;\ 445 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 446 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 447 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 448 cmp tmp1, tmp3 /* if not successful */ ;\ 449 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 450 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 451label/**/2: 452 453#endif /* UTSB_PHYS */ 454 455#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 456#error - TSB_SOFTSZ_MASK too small 457#endif 458 459 460/* 461 * An implementation of setx which will be hot patched at run time. 462 * since it is being hot patched, there is no value passed in. 463 * Thus, essentially we are implementing 464 * setx value, tmp, dest 465 * where value is RUNTIME_PATCH (aka 0) in this case. 466 */ 467#define RUNTIME_PATCH_SETX(dest, tmp) \ 468 sethi %hh(RUNTIME_PATCH), tmp ;\ 469 sethi %lm(RUNTIME_PATCH), dest ;\ 470 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 471 or dest, %lo(RUNTIME_PATCH), dest ;\ 472 sllx tmp, 32, tmp ;\ 473 nop /* for perf reasons */ ;\ 474 or tmp, dest, dest /* contents of patched value */ 475 476 477#endif (lint) 478 479 480#if defined (lint) 481 482/* 483 * sfmmu related subroutines 484 */ 485uint_t 486sfmmu_disable_intrs() 487{ return(0); } 488 489/* ARGSUSED */ 490void 491sfmmu_enable_intrs(uint_t pstate_save) 492{} 493 494/* ARGSUSED */ 495int 496sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag) 497{ return(0); } 498 499/* 500 * Use cas, if tte has changed underneath us then reread and try again. 501 * In the case of a retry, it will update sttep with the new original. 502 */ 503/* ARGSUSED */ 504int 505sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 506{ return(0); } 507 508/* 509 * Use cas, if tte has changed underneath us then return 1, else return 0 510 */ 511/* ARGSUSED */ 512int 513sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 514{ return(0); } 515 516/* ARGSUSED */ 517void 518sfmmu_copytte(tte_t *sttep, tte_t *dttep) 519{} 520 521/*ARGSUSED*/ 522struct tsbe * 523sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 524{ return(0); } 525 526/*ARGSUSED*/ 527uint64_t 528sfmmu_make_tsbtag(caddr_t va) 529{ return(0); } 530 531#else /* lint */ 532 533 .seg ".data" 534 .global sfmmu_panic1 535sfmmu_panic1: 536 .asciz "sfmmu_asm: interrupts already disabled" 537 538 .global sfmmu_panic3 539sfmmu_panic3: 540 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 541 542 .global sfmmu_panic4 543sfmmu_panic4: 544 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 545 546 .global sfmmu_panic5 547sfmmu_panic5: 548 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 549 550 .global sfmmu_panic6 551sfmmu_panic6: 552 .asciz "sfmmu_asm: interrupts not disabled" 553 554 .global sfmmu_panic7 555sfmmu_panic7: 556 .asciz "sfmmu_asm: kernel as" 557 558 .global sfmmu_panic8 559sfmmu_panic8: 560 .asciz "sfmmu_asm: gnum is zero" 561 562 .global sfmmu_panic9 563sfmmu_panic9: 564 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 565 566 .global sfmmu_panic10 567sfmmu_panic10: 568 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 569 570 ENTRY(sfmmu_disable_intrs) 571 rdpr %pstate, %o0 572#ifdef DEBUG 573 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 574#endif /* DEBUG */ 575 retl 576 wrpr %o0, PSTATE_IE, %pstate 577 SET_SIZE(sfmmu_disable_intrs) 578 579 ENTRY(sfmmu_enable_intrs) 580 retl 581 wrpr %g0, %o0, %pstate 582 SET_SIZE(sfmmu_enable_intrs) 583 584/* 585 * This routine is called both by resume() and sfmmu_get_ctx() to 586 * allocate a new context for the process on a MMU. 587 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 588 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 589 * is the case when sfmmu_alloc_ctx is called from resume(). 590 * 591 * The caller must disable interrupts before entering this routine. 592 * To reduce ctx switch overhead, the code contains both 'fast path' and 593 * 'slow path' code. The fast path code covers the common case where only 594 * a quick check is needed and the real ctx allocation is not required. 595 * It can be done without holding the per-process (PP) lock. 596 * The 'slow path' code must be protected by the PP Lock and performs ctx 597 * allocation. 598 * Hardware context register and HAT mmu cnum are updated accordingly. 599 * 600 * %o0 - sfmmup 601 * %o1 - allocflag 602 * %o2 - CPU 603 * %o3 - sfmmu private/shared flag 604 * 605 * ret - 0: no ctx is allocated 606 * 1: a ctx is allocated 607 */ 608 ENTRY_NP(sfmmu_alloc_ctx) 609 610#ifdef DEBUG 611 sethi %hi(ksfmmup), %g1 612 ldx [%g1 + %lo(ksfmmup)], %g1 613 cmp %g1, %o0 614 bne,pt %xcc, 0f 615 nop 616 617 sethi %hi(panicstr), %g1 ! if kernel as, panic 618 ldx [%g1 + %lo(panicstr)], %g1 619 tst %g1 620 bnz,pn %icc, 7f 621 nop 622 623 sethi %hi(sfmmu_panic7), %o0 624 call panic 625 or %o0, %lo(sfmmu_panic7), %o0 626 6277: 628 retl 629 mov %g0, %o0 ! %o0 = ret = 0 630 6310: 632 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 633#endif /* DEBUG */ 634 635 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 636 637 ! load global mmu_ctxp info 638 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 639 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 640 641 ! load global mmu_ctxp gnum 642 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 643 644#ifdef DEBUG 645 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 646 bne,pt %xcc, 3f 647 nop 648 649 sethi %hi(panicstr), %g1 ! test if panicstr is already set 650 ldx [%g1 + %lo(panicstr)], %g1 651 tst %g1 652 bnz,pn %icc, 1f 653 nop 654 655 sethi %hi(sfmmu_panic8), %o0 656 call panic 657 or %o0, %lo(sfmmu_panic8), %o0 6581: 659 retl 660 mov %g0, %o0 ! %o0 = ret = 0 6613: 662#endif 663 664 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 665 666 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 667 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 668 669 /* 670 * %g5 = sfmmu gnum returned 671 * %g6 = sfmmu cnum returned 672 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 673 * %g4 = scratch 674 * 675 * Fast path code, do a quick check. 676 */ 677 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 678 679 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 680 bne,pt %icc, 1f ! valid hat cnum, check gnum 681 nop 682 683 ! cnum == INVALID, check allocflag 684 mov %g0, %g4 ! %g4 = ret = 0 685 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 686 mov %g6, %o1 687 688 ! (invalid HAT cnum) && (allocflag == 1) 689 ba,pt %icc, 2f 690 nop 6911: 692 ! valid HAT cnum, check gnum 693 cmp %g5, %o4 694 mov 1, %g4 !%g4 = ret = 1 695 be,a,pt %icc, 8f ! gnum unchanged, go to done 696 mov %g6, %o1 697 6982: 699 /* 700 * Grab per process (PP) sfmmu_ctx_lock spinlock, 701 * followed by the 'slow path' code. 702 */ 703 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 7043: 705 brz %g3, 5f 706 nop 7074: 708 brnz,a,pt %g3, 4b ! spin if lock is 1 709 ldub [%o0 + SFMMU_CTX_LOCK], %g3 710 ba %xcc, 3b ! retry the lock 711 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 712 7135: 714 membar #LoadLoad 715 /* 716 * %g5 = sfmmu gnum returned 717 * %g6 = sfmmu cnum returned 718 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 719 * %g4 = scratch 720 */ 721 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 722 723 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 724 bne,pt %icc, 1f ! valid hat cnum, check gnum 725 nop 726 727 ! cnum == INVALID, check allocflag 728 mov %g0, %g4 ! %g4 = ret = 0 729 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 730 mov %g6, %o1 731 732 ! (invalid HAT cnum) && (allocflag == 1) 733 ba,pt %icc, 6f 734 nop 7351: 736 ! valid HAT cnum, check gnum 737 cmp %g5, %o4 738 mov 1, %g4 ! %g4 = ret = 1 739 be,a,pt %icc, 2f ! gnum unchanged, go to done 740 mov %g6, %o1 741 742 ba,pt %icc, 6f 743 nop 7442: 745 membar #LoadStore|#StoreStore 746 ba,pt %icc, 8f 747 clrb [%o0 + SFMMU_CTX_LOCK] 7486: 749 /* 750 * We get here if we do not have a valid context, or 751 * the HAT gnum does not match global gnum. We hold 752 * sfmmu_ctx_lock spinlock. Allocate that context. 753 * 754 * %o3 = mmu_ctxp 755 */ 756 add %o3, MMU_CTX_CNUM, %g3 757 ld [%o3 + MMU_CTX_NCTXS], %g4 758 759 /* 760 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 761 * %g3 = mmu cnum address 762 * %g4 = mmu nctxs 763 * 764 * %o0 = sfmmup 765 * %o1 = mmu current cnum value (used as new cnum) 766 * %o4 = mmu gnum 767 * 768 * %o5 = scratch 769 */ 770 ld [%g3], %o1 7710: 772 cmp %o1, %g4 773 bl,a,pt %icc, 1f 774 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 775 776 /* 777 * cnum reachs max, bail, so wrap around can be performed later. 778 */ 779 set INVALID_CONTEXT, %o1 780 /* 781 * When the routine is called by shared ctx, we want to set 782 * both private and shared ctx regs to INVALID. In order to 783 * do so, we set the sfmmu priv/shared flag to 'private' regardless. 784 * so that private ctx reg will be set to invalid. 785 * Note that values written to private context register are 786 * automatically written to shared context register as well. 787 */ 788 mov %g0, %g1 ! %g1 = sfmmu private/shared flag 789 mov %g0, %g4 ! %g4 = ret = 0 790 791 membar #LoadStore|#StoreStore 792 ba,pt %icc, 8f 793 clrb [%o0 + SFMMU_CTX_LOCK] 7941: 795 ! %g3 = addr of mmu_ctxp->cnum 796 ! %o5 = mmu_ctxp->cnum + 1 797 cas [%g3], %o1, %o5 798 cmp %o1, %o5 799 bne,a,pn %xcc, 0b ! cas failed 800 ld [%g3], %o1 801 802#ifdef DEBUG 803 set MAX_SFMMU_CTX_VAL, %o5 804 cmp %o1, %o5 805 ble,pt %icc, 2f 806 nop 807 808 sethi %hi(sfmmu_panic9), %o0 809 call panic 810 or %o0, %lo(sfmmu_panic9), %o0 8112: 812#endif 813 ! update hat gnum and cnum 814 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 815 or %o4, %o1, %o4 816 stx %o4, [%g2 + SFMMU_CTXS] 817 818 membar #LoadStore|#StoreStore 819 clrb [%o0 + SFMMU_CTX_LOCK] 820 821 mov 1, %g4 ! %g4 = ret = 1 8228: 823 /* 824 * program the secondary context register 825 * 826 * %o1 = cnum 827 * %g1 = sfmmu private/shared flag (0:private, 1:shared) 828 */ 829 830#ifdef sun4u 831 ldub [%o0 + SFMMU_CEXT], %o2 832 sll %o2, CTXREG_EXT_SHIFT, %o2 833 or %o1, %o2, %o1 834#endif 835 SET_SECCTX(%o1, %g1, %o4, %o5) 836 837 retl 838 mov %g4, %o0 ! %o0 = ret 839 840 SET_SIZE(sfmmu_alloc_ctx) 841 842 ENTRY_NP(sfmmu_modifytte) 843 ldx [%o2], %g3 /* current */ 844 ldx [%o0], %g1 /* original */ 8452: 846 ldx [%o1], %g2 /* modified */ 847 cmp %g2, %g3 /* is modified = current? */ 848 be,a,pt %xcc,1f /* yes, don't write */ 849 stx %g3, [%o0] /* update new original */ 850 casx [%o2], %g1, %g2 851 cmp %g1, %g2 852 be,pt %xcc, 1f /* cas succeeded - return */ 853 nop 854 ldx [%o2], %g3 /* new current */ 855 stx %g3, [%o0] /* save as new original */ 856 ba,pt %xcc, 2b 857 mov %g3, %g1 8581: retl 859 membar #StoreLoad 860 SET_SIZE(sfmmu_modifytte) 861 862 ENTRY_NP(sfmmu_modifytte_try) 863 ldx [%o1], %g2 /* modified */ 864 ldx [%o2], %g3 /* current */ 865 ldx [%o0], %g1 /* original */ 866 cmp %g3, %g2 /* is modified = current? */ 867 be,a,pn %xcc,1f /* yes, don't write */ 868 mov 0, %o1 /* as if cas failed. */ 869 870 casx [%o2], %g1, %g2 871 membar #StoreLoad 872 cmp %g1, %g2 873 movne %xcc, -1, %o1 /* cas failed. */ 874 move %xcc, 1, %o1 /* cas succeeded. */ 8751: 876 stx %g2, [%o0] /* report "current" value */ 877 retl 878 mov %o1, %o0 879 SET_SIZE(sfmmu_modifytte_try) 880 881 ENTRY_NP(sfmmu_copytte) 882 ldx [%o0], %g1 883 retl 884 stx %g1, [%o1] 885 SET_SIZE(sfmmu_copytte) 886 887 888 /* 889 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 890 * %o0 = TSB base address (in), pointer to TSB entry (out) 891 * %o1 = vaddr (in) 892 * %o2 = vpshift (in) 893 * %o3 = tsb size code (in) 894 * %o4 = scratch register 895 */ 896 ENTRY_NP(sfmmu_get_tsbe) 897 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 898 retl 899 nop 900 SET_SIZE(sfmmu_get_tsbe) 901 902 /* 903 * Return a TSB tag for the given va. 904 * %o0 = va (in/clobbered) 905 * %o0 = va shifted to be in tsb tag format (with no context) (out) 906 */ 907 ENTRY_NP(sfmmu_make_tsbtag) 908 retl 909 srln %o0, TTARGET_VA_SHIFT, %o0 910 SET_SIZE(sfmmu_make_tsbtag) 911 912#endif /* lint */ 913 914/* 915 * Other sfmmu primitives 916 */ 917 918 919#if defined (lint) 920void 921sfmmu_patch_ktsb(void) 922{ 923} 924 925void 926sfmmu_kpm_patch_tlbm(void) 927{ 928} 929 930void 931sfmmu_kpm_patch_tsbm(void) 932{ 933} 934 935void 936sfmmu_patch_shctx(void) 937{ 938} 939 940/* ARGSUSED */ 941void 942sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 943{ 944} 945 946/* ARGSUSED */ 947void 948sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 949{ 950} 951 952/* ARGSUSED */ 953void 954sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 955{ 956} 957 958/* ARGSUSED */ 959void 960sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 961{ 962} 963 964#else /* lint */ 965 966#define I_SIZE 4 967 968 ENTRY_NP(sfmmu_fix_ktlb_traptable) 969 /* 970 * %o0 = start of patch area 971 * %o1 = size code of TSB to patch 972 * %o3 = scratch 973 */ 974 /* fix sll */ 975 ld [%o0], %o3 /* get sll */ 976 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 977 st %o3, [%o0] /* write sll */ 978 flush %o0 979 /* fix srl */ 980 add %o0, I_SIZE, %o0 /* goto next instr. */ 981 ld [%o0], %o3 /* get srl */ 982 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 983 st %o3, [%o0] /* write srl */ 984 retl 985 flush %o0 986 SET_SIZE(sfmmu_fix_ktlb_traptable) 987 988 ENTRY_NP(sfmmu_fixup_ktsbbase) 989 /* 990 * %o0 = start of patch area 991 * %o5 = kernel virtual or physical tsb base address 992 * %o2, %o3 are used as scratch registers. 993 */ 994 /* fixup sethi instruction */ 995 ld [%o0], %o3 996 srl %o5, 10, %o2 ! offset is bits 32:10 997 or %o3, %o2, %o3 ! set imm22 998 st %o3, [%o0] 999 /* fixup offset of lduw/ldx */ 1000 add %o0, I_SIZE, %o0 ! next instr 1001 ld [%o0], %o3 1002 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 1003 or %o3, %o2, %o3 1004 st %o3, [%o0] 1005 retl 1006 flush %o0 1007 SET_SIZE(sfmmu_fixup_ktsbbase) 1008 1009 ENTRY_NP(sfmmu_fixup_setx) 1010 /* 1011 * %o0 = start of patch area 1012 * %o4 = 64 bit value to patch 1013 * %o2, %o3 are used as scratch registers. 1014 * 1015 * Note: Assuming that all parts of the instructions which need to be 1016 * patched correspond to RUNTIME_PATCH (aka 0) 1017 * 1018 * Note the implementation of setx which is being patched is as follows: 1019 * 1020 * sethi %hh(RUNTIME_PATCH), tmp 1021 * sethi %lm(RUNTIME_PATCH), dest 1022 * or tmp, %hm(RUNTIME_PATCH), tmp 1023 * or dest, %lo(RUNTIME_PATCH), dest 1024 * sllx tmp, 32, tmp 1025 * nop 1026 * or tmp, dest, dest 1027 * 1028 * which differs from the implementation in the 1029 * "SPARC Architecture Manual" 1030 */ 1031 /* fixup sethi instruction */ 1032 ld [%o0], %o3 1033 srlx %o4, 42, %o2 ! bits [63:42] 1034 or %o3, %o2, %o3 ! set imm22 1035 st %o3, [%o0] 1036 /* fixup sethi instruction */ 1037 add %o0, I_SIZE, %o0 ! next instr 1038 ld [%o0], %o3 1039 sllx %o4, 32, %o2 ! clear upper bits 1040 srlx %o2, 42, %o2 ! bits [31:10] 1041 or %o3, %o2, %o3 ! set imm22 1042 st %o3, [%o0] 1043 /* fixup or instruction */ 1044 add %o0, I_SIZE, %o0 ! next instr 1045 ld [%o0], %o3 1046 srlx %o4, 32, %o2 ! bits [63:32] 1047 and %o2, 0x3ff, %o2 ! bits [41:32] 1048 or %o3, %o2, %o3 ! set imm 1049 st %o3, [%o0] 1050 /* fixup or instruction */ 1051 add %o0, I_SIZE, %o0 ! next instr 1052 ld [%o0], %o3 1053 and %o4, 0x3ff, %o2 ! bits [9:0] 1054 or %o3, %o2, %o3 ! set imm 1055 st %o3, [%o0] 1056 retl 1057 flush %o0 1058 SET_SIZE(sfmmu_fixup_setx) 1059 1060 ENTRY_NP(sfmmu_fixup_or) 1061 /* 1062 * %o0 = start of patch area 1063 * %o4 = 32 bit value to patch 1064 * %o2, %o3 are used as scratch registers. 1065 * Note: Assuming that all parts of the instructions which need to be 1066 * patched correspond to RUNTIME_PATCH (aka 0) 1067 */ 1068 ld [%o0], %o3 1069 and %o4, 0x3ff, %o2 ! bits [9:0] 1070 or %o3, %o2, %o3 ! set imm 1071 st %o3, [%o0] 1072 retl 1073 flush %o0 1074 SET_SIZE(sfmmu_fixup_or) 1075 1076 ENTRY_NP(sfmmu_fixup_shiftx) 1077 /* 1078 * %o0 = start of patch area 1079 * %o4 = signed int immediate value to add to sllx/srlx imm field 1080 * %o2, %o3 are used as scratch registers. 1081 * 1082 * sllx/srlx store the 6 bit immediate value in the lowest order bits 1083 * so we do a simple add. The caller must be careful to prevent 1084 * overflow, which could easily occur if the initial value is nonzero! 1085 */ 1086 ld [%o0], %o3 ! %o3 = instruction to patch 1087 and %o3, 0x3f, %o2 ! %o2 = existing imm value 1088 add %o2, %o4, %o2 ! %o2 = new imm value 1089 andn %o3, 0x3f, %o3 ! clear old imm value 1090 and %o2, 0x3f, %o2 ! truncate new imm value 1091 or %o3, %o2, %o3 ! set new imm value 1092 st %o3, [%o0] ! store updated instruction 1093 retl 1094 flush %o0 1095 SET_SIZE(sfmmu_fixup_shiftx) 1096 1097 ENTRY_NP(sfmmu_fixup_mmu_asi) 1098 /* 1099 * Patch imm_asi of all ldda instructions in the MMU 1100 * trap handlers. We search MMU_PATCH_INSTR instructions 1101 * starting from the itlb miss handler (trap 0x64). 1102 * %o0 = address of tt[0,1]_itlbmiss 1103 * %o1 = imm_asi to setup, shifted by appropriate offset. 1104 * %o3 = number of instructions to search 1105 * %o4 = reserved by caller: called from leaf routine 1106 */ 11071: ldsw [%o0], %o2 ! load instruction to %o2 1108 brgez,pt %o2, 2f 1109 srl %o2, 30, %o5 1110 btst 1, %o5 ! test bit 30; skip if not set 1111 bz,pt %icc, 2f 1112 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1113 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1114 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1115 brnz,pt %o5, 2f ! skip if not a match 1116 or %o2, %o1, %o2 ! or in imm_asi 1117 st %o2, [%o0] ! write patched instruction 11182: dec %o3 1119 brnz,a,pt %o3, 1b ! loop until we're done 1120 add %o0, I_SIZE, %o0 1121 retl 1122 flush %o0 1123 SET_SIZE(sfmmu_fixup_mmu_asi) 1124 1125 /* 1126 * Patch immediate ASI used to access the TSB in the 1127 * trap table. 1128 * inputs: %o0 = value of ktsb_phys 1129 */ 1130 ENTRY_NP(sfmmu_patch_mmu_asi) 1131 mov %o7, %o4 ! save return pc in %o4 1132 movrnz %o0, ASI_QUAD_LDD_PHYS, %o3 1133 movrz %o0, ASI_NQUAD_LD, %o3 1134 sll %o3, 5, %o1 ! imm_asi offset 1135 mov 6, %o3 ! number of instructions 1136 sethi %hi(dktsb), %o0 ! to search 1137 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1138 or %o0, %lo(dktsb), %o0 1139 mov 6, %o3 ! number of instructions 1140 sethi %hi(dktsb4m), %o0 ! to search 1141 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1142 or %o0, %lo(dktsb4m), %o0 1143 mov 6, %o3 ! number of instructions 1144 sethi %hi(iktsb), %o0 ! to search 1145 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1146 or %o0, %lo(iktsb), %o0 1147 mov 6, %o3 ! number of instructions 1148 sethi %hi(iktsb4m), %o0 ! to search 1149 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1150 or %o0, %lo(iktsb4m), %o0 1151 mov %o4, %o7 ! retore return pc -- leaf 1152 retl 1153 nop 1154 SET_SIZE(sfmmu_patch_mmu_asi) 1155 1156 ENTRY_NP(sfmmu_patch_ktsb) 1157 /* 1158 * We need to fix iktsb, dktsb, et. al. 1159 */ 1160 save %sp, -SA(MINFRAME), %sp 1161 set ktsb_phys, %o1 1162 ld [%o1], %o4 1163 set ktsb_base, %o5 1164 set ktsb4m_base, %l1 1165 brz,pt %o4, 1f 1166 nop 1167 set ktsb_pbase, %o5 1168 set ktsb4m_pbase, %l1 11691: 1170 sethi %hi(ktsb_szcode), %o1 1171 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1172 1173 sethi %hi(iktsb), %o0 1174 call sfmmu_fix_ktlb_traptable 1175 or %o0, %lo(iktsb), %o0 1176 1177 sethi %hi(dktsb), %o0 1178 call sfmmu_fix_ktlb_traptable 1179 or %o0, %lo(dktsb), %o0 1180 1181 sethi %hi(ktsb4m_szcode), %o1 1182 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1183 1184 sethi %hi(iktsb4m), %o0 1185 call sfmmu_fix_ktlb_traptable 1186 or %o0, %lo(iktsb4m), %o0 1187 1188 sethi %hi(dktsb4m), %o0 1189 call sfmmu_fix_ktlb_traptable 1190 or %o0, %lo(dktsb4m), %o0 1191 1192#ifndef sun4v 1193 mov ASI_N, %o2 1194 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1195 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1196 sethi %hi(tsb_kernel_patch_asi), %o0 1197 call sfmmu_fixup_or 1198 or %o0, %lo(tsb_kernel_patch_asi), %o0 1199#endif 1200 1201 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1202 1203 sethi %hi(dktsbbase), %o0 1204 call sfmmu_fixup_setx ! patch value of ktsb base addr 1205 or %o0, %lo(dktsbbase), %o0 1206 1207 sethi %hi(iktsbbase), %o0 1208 call sfmmu_fixup_setx ! patch value of ktsb base addr 1209 or %o0, %lo(iktsbbase), %o0 1210 1211 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1212 call sfmmu_fixup_setx ! patch value of ktsb base addr 1213 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1214 1215#ifdef sun4v 1216 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1217 call sfmmu_fixup_setx ! patch value of ktsb base addr 1218 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1219#endif /* sun4v */ 1220 1221 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1222 1223 sethi %hi(dktsb4mbase), %o0 1224 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1225 or %o0, %lo(dktsb4mbase), %o0 1226 1227 sethi %hi(iktsb4mbase), %o0 1228 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1229 or %o0, %lo(iktsb4mbase), %o0 1230 1231 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1232 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1233 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1234 1235#ifdef sun4v 1236 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1237 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1238 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1239#endif /* sun4v */ 1240 1241 set ktsb_szcode, %o4 1242 ld [%o4], %o4 1243 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1244 call sfmmu_fixup_or ! patch value of ktsb_szcode 1245 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1246 1247#ifdef sun4v 1248 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1249 call sfmmu_fixup_or ! patch value of ktsb_szcode 1250 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1251#endif /* sun4v */ 1252 1253 set ktsb4m_szcode, %o4 1254 ld [%o4], %o4 1255 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1256 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1257 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1258 1259#ifdef sun4v 1260 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1261 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1262 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1263#endif /* sun4v */ 1264 1265 ret 1266 restore 1267 SET_SIZE(sfmmu_patch_ktsb) 1268 1269 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1270 /* 1271 * Fixup trap handlers in common segkpm case. This is reserved 1272 * for future use should kpm TSB be changed to be other than the 1273 * kernel TSB. 1274 */ 1275 retl 1276 nop 1277 SET_SIZE(sfmmu_kpm_patch_tlbm) 1278 1279 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1280 /* 1281 * nop the branch to sfmmu_kpm_dtsb_miss_small 1282 * in the case where we are using large pages for 1283 * seg_kpm (and hence must probe the second TSB for 1284 * seg_kpm VAs) 1285 */ 1286 set dktsb4m_kpmcheck_small, %o0 1287 MAKE_NOP_INSTR(%o1) 1288 st %o1, [%o0] 1289 flush %o0 1290 retl 1291 nop 1292 SET_SIZE(sfmmu_kpm_patch_tsbm) 1293 1294 ENTRY_NP(sfmmu_patch_utsb) 1295#ifdef UTSB_PHYS 1296 retl 1297 nop 1298#else /* UTSB_PHYS */ 1299 /* 1300 * We need to hot patch utsb_vabase and utsb4m_vabase 1301 */ 1302 save %sp, -SA(MINFRAME), %sp 1303 1304 /* patch value of utsb_vabase */ 1305 set utsb_vabase, %o1 1306 ldx [%o1], %o4 1307 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1308 call sfmmu_fixup_setx 1309 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1310 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1311 call sfmmu_fixup_setx 1312 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1313 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1314 call sfmmu_fixup_setx 1315 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1316 1317 /* patch value of utsb4m_vabase */ 1318 set utsb4m_vabase, %o1 1319 ldx [%o1], %o4 1320 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1321 call sfmmu_fixup_setx 1322 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1323 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1324 call sfmmu_fixup_setx 1325 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1326 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1327 call sfmmu_fixup_setx 1328 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1329 1330 /* 1331 * Patch TSB base register masks and shifts if needed. 1332 * By default the TSB base register contents are set up for 4M slab. 1333 * If we're using a smaller slab size and reserved VA range we need 1334 * to patch up those values here. 1335 */ 1336 set tsb_slab_shift, %o1 1337 set MMU_PAGESHIFT4M, %o4 1338 lduw [%o1], %o3 1339 subcc %o4, %o3, %o4 1340 bz,pt %icc, 1f 1341 /* delay slot safe */ 1342 1343 /* patch reserved VA range size if needed. */ 1344 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1345 call sfmmu_fixup_shiftx 1346 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1347 call sfmmu_fixup_shiftx 1348 add %o0, I_SIZE, %o0 1349 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1350 call sfmmu_fixup_shiftx 1351 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1352 call sfmmu_fixup_shiftx 1353 add %o0, I_SIZE, %o0 13541: 1355 /* patch TSBREG_VAMASK used to set up TSB base register */ 1356 set tsb_slab_mask, %o1 1357 ldx [%o1], %o4 1358 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1359 call sfmmu_fixup_or 1360 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1361 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1362 call sfmmu_fixup_or 1363 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1364 1365 ret 1366 restore 1367#endif /* UTSB_PHYS */ 1368 SET_SIZE(sfmmu_patch_utsb) 1369 1370 ENTRY_NP(sfmmu_patch_shctx) 1371#ifdef sun4u 1372 retl 1373 nop 1374#else /* sun4u */ 1375 set sfmmu_shctx_cpu_mondo_patch, %o0 1376 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1377 st %o1, [%o0] 1378 flush %o0 1379 MAKE_NOP_INSTR(%o1) 1380 add %o0, I_SIZE, %o0 ! next instr 1381 st %o1, [%o0] 1382 flush %o0 1383 1384 set sfmmu_shctx_user_rtt_patch, %o0 1385 st %o1, [%o0] ! nop 1st instruction 1386 flush %o0 1387 add %o0, I_SIZE, %o0 1388 st %o1, [%o0] ! nop 2nd instruction 1389 flush %o0 1390 add %o0, I_SIZE, %o0 1391 st %o1, [%o0] ! nop 3rd instruction 1392 flush %o0 1393 add %o0, I_SIZE, %o0 1394 st %o1, [%o0] ! nop 4th instruction 1395 flush %o0 1396 add %o0, I_SIZE, %o0 1397 st %o1, [%o0] ! nop 5th instruction 1398 retl 1399 flush %o0 1400#endif /* sun4u */ 1401 SET_SIZE(sfmmu_patch_shctx) 1402 1403 /* 1404 * Routine that loads an entry into a tsb using virtual addresses. 1405 * Locking is required since all cpus can use the same TSB. 1406 * Note that it is no longer required to have a valid context 1407 * when calling this function. 1408 */ 1409 ENTRY_NP(sfmmu_load_tsbe) 1410 /* 1411 * %o0 = pointer to tsbe to load 1412 * %o1 = tsb tag 1413 * %o2 = virtual pointer to TTE 1414 * %o3 = 1 if physical address in %o0 else 0 1415 */ 1416 rdpr %pstate, %o5 1417#ifdef DEBUG 1418 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1419#endif /* DEBUG */ 1420 1421 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1422 1423 SETUP_TSB_ASI(%o3, %g3) 1424 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1) 1425 1426 wrpr %g0, %o5, %pstate /* enable interrupts */ 1427 1428 retl 1429 membar #StoreStore|#StoreLoad 1430 SET_SIZE(sfmmu_load_tsbe) 1431 1432 /* 1433 * Flush TSB of a given entry if the tag matches. 1434 */ 1435 ENTRY(sfmmu_unload_tsbe) 1436 /* 1437 * %o0 = pointer to tsbe to be flushed 1438 * %o1 = tag to match 1439 * %o2 = 1 if physical address in %o0 else 0 1440 */ 1441 SETUP_TSB_ASI(%o2, %g1) 1442 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1443 retl 1444 membar #StoreStore|#StoreLoad 1445 SET_SIZE(sfmmu_unload_tsbe) 1446 1447 /* 1448 * Routine that loads a TTE into the kpm TSB from C code. 1449 * Locking is required since kpm TSB is shared among all CPUs. 1450 */ 1451 ENTRY_NP(sfmmu_kpm_load_tsb) 1452 /* 1453 * %o0 = vaddr 1454 * %o1 = ttep 1455 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1456 */ 1457 rdpr %pstate, %o5 ! %o5 = saved pstate 1458#ifdef DEBUG 1459 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1460#endif /* DEBUG */ 1461 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1462 1463#ifndef sun4v 1464 sethi %hi(ktsb_phys), %o4 1465 mov ASI_N, %o3 1466 ld [%o4 + %lo(ktsb_phys)], %o4 1467 movrnz %o4, ASI_MEM, %o3 1468 mov %o3, %asi 1469#endif 1470 mov %o0, %g1 ! %g1 = vaddr 1471 1472 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1473 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1474 /* %g2 = tsbep, %g1 clobbered */ 1475 1476 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1477 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1478 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1) 1479 1480 wrpr %g0, %o5, %pstate ! enable interrupts 1481 retl 1482 membar #StoreStore|#StoreLoad 1483 SET_SIZE(sfmmu_kpm_load_tsb) 1484 1485 /* 1486 * Routine that shoots down a TTE in the kpm TSB or in the 1487 * kernel TSB depending on virtpg. Locking is required since 1488 * kpm/kernel TSB is shared among all CPUs. 1489 */ 1490 ENTRY_NP(sfmmu_kpm_unload_tsb) 1491 /* 1492 * %o0 = vaddr 1493 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1494 */ 1495#ifndef sun4v 1496 sethi %hi(ktsb_phys), %o4 1497 mov ASI_N, %o3 1498 ld [%o4 + %lo(ktsb_phys)], %o4 1499 movrnz %o4, ASI_MEM, %o3 1500 mov %o3, %asi 1501#endif 1502 mov %o0, %g1 ! %g1 = vaddr 1503 1504 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1505 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1506 /* %g2 = tsbep, %g1 clobbered */ 1507 1508 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1509 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1510 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1511 1512 retl 1513 membar #StoreStore|#StoreLoad 1514 SET_SIZE(sfmmu_kpm_unload_tsb) 1515 1516#endif /* lint */ 1517 1518 1519#if defined (lint) 1520 1521/*ARGSUSED*/ 1522pfn_t 1523sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1524{ return(0); } 1525 1526#else /* lint */ 1527 1528 ENTRY_NP(sfmmu_ttetopfn) 1529 ldx [%o0], %g1 /* read tte */ 1530 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1531 /* 1532 * g1 = pfn 1533 */ 1534 retl 1535 mov %g1, %o0 1536 SET_SIZE(sfmmu_ttetopfn) 1537 1538#endif /* !lint */ 1539 1540 1541#if defined (lint) 1542/* 1543 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the 1544 * the hash list. 1545 */ 1546/* ARGSUSED */ 1547void 1548sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1549 uint64_t hblkpa) 1550{ 1551} 1552 1553/* 1554 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the 1555 * hash list. 1556 */ 1557/* ARGSUSED */ 1558void 1559sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1560 uint64_t hblkpa, struct hme_blk *prev_hblkp) 1561{ 1562} 1563#else /* lint */ 1564 1565/* 1566 * Functions to grab/release hme bucket list lock. I only use a byte 1567 * instead of the whole int because eventually we might want to 1568 * put some counters on the other bytes (of course, these routines would 1569 * have to change). The code that grab this lock should execute 1570 * with interrupts disabled and hold the lock for the least amount of time 1571 * possible. 1572 */ 1573 1574/* 1575 * Even though hmeh_listlock is updated using pa there's no need to flush 1576 * dcache since hmeh_listlock will be restored to the original value (0) 1577 * before interrupts are reenabled. 1578 */ 1579 1580/* 1581 * For sparcv9 hme hash buckets may not be in the nucleus. hme hash update 1582 * routines still use virtual addresses to update the bucket fields. But they 1583 * must not cause a TLB miss after grabbing the low level bucket lock. To 1584 * achieve this we must make sure the bucket structure is completely within an 1585 * 8K page. 1586 */ 1587 1588#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1)) 1589#error - the size of hmehash_bucket structure is not power of 2 1590#endif 1591 1592#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi) \ 1593 mov 0xff, tmp2 ;\ 1594 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1595label1: ;\ 1596 casa [tmp1]asi, %g0, tmp2 ;\ 1597 brnz,pn tmp2, label1 ;\ 1598 mov 0xff, tmp2 ;\ 1599 membar #LoadLoad 1600 1601#define HMELOCK_EXIT(hmebp, tmp1, asi) \ 1602 membar #LoadStore|#StoreStore ;\ 1603 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1604 sta %g0, [tmp1]asi 1605 1606 .seg ".data" 1607hblk_add_panic1: 1608 .ascii "sfmmu_hblk_hash_add: interrupts disabled" 1609 .byte 0 1610hblk_add_panic2: 1611 .ascii "sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not" 1612 .byte 0 1613 .align 4 1614 .seg ".text" 1615 1616 ENTRY_NP(sfmmu_hblk_hash_add) 1617 /* 1618 * %o0 = hmebp 1619 * %o1 = hmeblkp 1620 * %o2 = hblkpa 1621 */ 1622 rdpr %pstate, %o5 1623#ifdef DEBUG 1624 andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ 1625 bnz,pt %icc, 3f /* disabled, panic */ 1626 nop 1627 save %sp, -SA(MINFRAME), %sp 1628 sethi %hi(hblk_add_panic1), %o0 1629 call panic 1630 or %o0, %lo(hblk_add_panic1), %o0 1631 ret 1632 restore 1633 16343: 1635#endif /* DEBUG */ 1636 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1637 mov %o2, %g1 1638 1639 /* 1640 * g1 = hblkpa 1641 */ 1642 ldn [%o0 + HMEBUCK_HBLK], %o4 /* next hmeblk */ 1643 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = next hblkpa */ 1644#ifdef DEBUG 1645 cmp %o4, %g0 1646 bne,pt %xcc, 1f 1647 nop 1648 brz,pt %g2, 1f 1649 nop 1650 wrpr %g0, %o5, %pstate /* enable interrupts */ 1651 save %sp, -SA(MINFRAME), %sp 1652 sethi %hi(hblk_add_panic2), %o0 1653 call panic 1654 or %o0, %lo(hblk_add_panic2), %o0 1655 ret 1656 restore 16571: 1658#endif /* DEBUG */ 1659 /* 1660 * We update hmeblks entries before grabbing lock because the stores 1661 * could take a tlb miss and require the hash lock. The buckets 1662 * are part of the nucleus so we are cool with those stores. 1663 * 1664 * if buckets are not part of the nucleus our game is to 1665 * not touch any other page via va until we drop the lock. 1666 * This guarantees we won't get a tlb miss before the lock release 1667 * since interrupts are disabled. 1668 */ 1669 stn %o4, [%o1 + HMEBLK_NEXT] /* update hmeblk's next */ 1670 stx %g2, [%o1 + HMEBLK_NEXTPA] /* update hmeblk's next pa */ 1671 HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N) 1672 stn %o1, [%o0 + HMEBUCK_HBLK] /* update bucket hblk next */ 1673 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* add hmeblk to list */ 1674 HMELOCK_EXIT(%o0, %g2, ASI_N) 1675 retl 1676 wrpr %g0, %o5, %pstate /* enable interrupts */ 1677 SET_SIZE(sfmmu_hblk_hash_add) 1678 1679 ENTRY_NP(sfmmu_hblk_hash_rm) 1680 /* 1681 * This function removes an hmeblk from the hash chain. 1682 * It is written to guarantee we don't take a tlb miss 1683 * by using physical addresses to update the list. 1684 * 1685 * %o0 = hmebp 1686 * %o1 = hmeblkp 1687 * %o2 = hmeblkp previous pa 1688 * %o3 = hmeblkp previous 1689 */ 1690 1691 mov %o3, %o4 /* o4 = hmeblkp previous */ 1692 1693 rdpr %pstate, %o5 1694#ifdef DEBUG 1695 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1) 1696#endif /* DEBUG */ 1697 /* 1698 * disable interrupts, clear Address Mask to access 64 bit physaddr 1699 */ 1700 andn %o5, PSTATE_IE, %g1 1701 wrpr %g1, 0, %pstate 1702 1703#ifndef sun4v 1704 sethi %hi(dcache_line_mask), %g4 1705 ld [%g4 + %lo(dcache_line_mask)], %g4 1706#endif /* sun4v */ 1707 1708 /* 1709 * if buckets are not part of the nucleus our game is to 1710 * not touch any other page via va until we drop the lock. 1711 * This guarantees we won't get a tlb miss before the lock release 1712 * since interrupts are disabled. 1713 */ 1714 HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N) 1715 ldn [%o0 + HMEBUCK_HBLK], %g2 /* first hmeblk in list */ 1716 cmp %g2, %o1 1717 bne,pt %ncc,1f 1718 mov ASI_MEM, %asi 1719 /* 1720 * hmeblk is first on list 1721 */ 1722 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = hmeblk pa */ 1723 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1724 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1725 stn %o3, [%o0 + HMEBUCK_HBLK] /* write va */ 1726 ba,pt %xcc, 2f 1727 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* write pa */ 17281: 1729 /* hmeblk is not first on list */ 1730 1731 mov %o2, %g3 1732#ifndef sun4v 1733 GET_CPU_IMPL(%g2) 1734 cmp %g2, CHEETAH_IMPL 1735 bge,a,pt %icc, hblk_hash_rm_1 1736 and %o4, %g4, %g2 1737 cmp %g2, SPITFIRE_IMPL 1738 blt %icc, hblk_hash_rm_2 /* no flushing needed for OPL */ 1739 and %o4, %g4, %g2 1740 stxa %g0, [%g2]ASI_DC_TAG /* flush prev pa from dcache */ 1741 add %o4, HMEBLK_NEXT, %o4 1742 and %o4, %g4, %g2 1743 ba hblk_hash_rm_2 1744 stxa %g0, [%g2]ASI_DC_TAG /* flush prev va from dcache */ 1745hblk_hash_rm_1: 1746 1747 stxa %g0, [%g3]ASI_DC_INVAL /* flush prev pa from dcache */ 1748 membar #Sync 1749 add %g3, HMEBLK_NEXT, %g2 1750 stxa %g0, [%g2]ASI_DC_INVAL /* flush prev va from dcache */ 1751hblk_hash_rm_2: 1752 membar #Sync 1753#endif /* sun4v */ 1754 ldxa [%g3 + HMEBLK_NEXTPA] %asi, %g2 /* g2 = hmeblk pa */ 1755 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1756 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1757 stna %o3, [%g3 + HMEBLK_NEXT] %asi /* write va */ 1758 stxa %g1, [%g3 + HMEBLK_NEXTPA] %asi /* write pa */ 17592: 1760 HMELOCK_EXIT(%o0, %g2, ASI_N) 1761 retl 1762 wrpr %g0, %o5, %pstate /* enable interrupts */ 1763 SET_SIZE(sfmmu_hblk_hash_rm) 1764 1765#endif /* lint */ 1766 1767/* 1768 * These macros are used to update global sfmmu hme hash statistics 1769 * in perf critical paths. It is only enabled in debug kernels or 1770 * if SFMMU_STAT_GATHER is defined 1771 */ 1772#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1773#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1774 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1775 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1776 cmp tmp1, hatid ;\ 1777 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1778 set sfmmu_global_stat, tmp1 ;\ 1779 add tmp1, tmp2, tmp1 ;\ 1780 ld [tmp1], tmp2 ;\ 1781 inc tmp2 ;\ 1782 st tmp2, [tmp1] 1783 1784#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1785 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1786 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1787 cmp tmp1, hatid ;\ 1788 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1789 set sfmmu_global_stat, tmp1 ;\ 1790 add tmp1, tmp2, tmp1 ;\ 1791 ld [tmp1], tmp2 ;\ 1792 inc tmp2 ;\ 1793 st tmp2, [tmp1] 1794 1795 1796#else /* DEBUG || SFMMU_STAT_GATHER */ 1797 1798#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1799 1800#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1801 1802#endif /* DEBUG || SFMMU_STAT_GATHER */ 1803 1804/* 1805 * This macro is used to update global sfmmu kstas in non 1806 * perf critical areas so they are enabled all the time 1807 */ 1808#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1809 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1810 add tmp1, statname, tmp1 ;\ 1811 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1812 inc tmp2 ;\ 1813 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1814 1815/* 1816 * These macros are used to update per cpu stats in non perf 1817 * critical areas so they are enabled all the time 1818 */ 1819#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1820 ld [tsbarea + stat], tmp1 ;\ 1821 inc tmp1 ;\ 1822 st tmp1, [tsbarea + stat] 1823 1824/* 1825 * These macros are used to update per cpu stats in non perf 1826 * critical areas so they are enabled all the time 1827 */ 1828#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1829 lduh [tsbarea + stat], tmp1 ;\ 1830 inc tmp1 ;\ 1831 stuh tmp1, [tsbarea + stat] 1832 1833#if defined(KPM_TLBMISS_STATS_GATHER) 1834 /* 1835 * Count kpm dtlb misses separately to allow a different 1836 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1837 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1838 */ 1839#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1840 brgez tagacc, label /* KPM VA? */ ;\ 1841 nop ;\ 1842 CPU_INDEX(tmp1, tsbma) ;\ 1843 sethi %hi(kpmtsbm_area), tsbma ;\ 1844 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1845 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1846 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1847 /* VA range check */ ;\ 1848 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1849 cmp tagacc, val ;\ 1850 blu,pn %xcc, label ;\ 1851 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1852 cmp tagacc, tmp1 ;\ 1853 bgeu,pn %xcc, label ;\ 1854 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1855 inc val ;\ 1856 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1857label: 1858#else 1859#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1860#endif /* KPM_TLBMISS_STATS_GATHER */ 1861 1862#if defined (lint) 1863/* 1864 * The following routines are jumped to from the mmu trap handlers to do 1865 * the setting up to call systrap. They are separate routines instead of 1866 * being part of the handlers because the handlers would exceed 32 1867 * instructions and since this is part of the slow path the jump 1868 * cost is irrelevant. 1869 */ 1870void 1871sfmmu_pagefault(void) 1872{ 1873} 1874 1875void 1876sfmmu_mmu_trap(void) 1877{ 1878} 1879 1880void 1881sfmmu_window_trap(void) 1882{ 1883} 1884 1885void 1886sfmmu_kpm_exception(void) 1887{ 1888} 1889 1890#else /* lint */ 1891 1892#ifdef PTL1_PANIC_DEBUG 1893 .seg ".data" 1894 .global test_ptl1_panic 1895test_ptl1_panic: 1896 .word 0 1897 .align 8 1898 1899 .seg ".text" 1900 .align 4 1901#endif /* PTL1_PANIC_DEBUG */ 1902 1903 1904 ENTRY_NP(sfmmu_pagefault) 1905 SET_GL_REG(1) 1906 USE_ALTERNATE_GLOBALS(%g5) 1907 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1908 rdpr %tt, %g6 1909 cmp %g6, FAST_IMMU_MISS_TT 1910 be,a,pn %icc, 1f 1911 mov T_INSTR_MMU_MISS, %g3 1912 cmp %g6, T_INSTR_MMU_MISS 1913 be,a,pn %icc, 1f 1914 mov T_INSTR_MMU_MISS, %g3 1915 mov %g5, %g2 1916 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1917 cmp %g6, FAST_DMMU_MISS_TT 1918 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1919 cmp %g6, T_DATA_MMU_MISS 1920 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1921 1922#ifdef PTL1_PANIC_DEBUG 1923 /* check if we want to test the tl1 panic */ 1924 sethi %hi(test_ptl1_panic), %g4 1925 ld [%g4 + %lo(test_ptl1_panic)], %g1 1926 st %g0, [%g4 + %lo(test_ptl1_panic)] 1927 cmp %g1, %g0 1928 bne,a,pn %icc, ptl1_panic 1929 or %g0, PTL1_BAD_DEBUG, %g1 1930#endif /* PTL1_PANIC_DEBUG */ 19311: 1932 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 1933 /* 1934 * g2 = tag access reg 1935 * g3.l = type 1936 * g3.h = 0 1937 */ 1938 sethi %hi(trap), %g1 1939 or %g1, %lo(trap), %g1 19402: 1941 ba,pt %xcc, sys_trap 1942 mov -1, %g4 1943 SET_SIZE(sfmmu_pagefault) 1944 1945 ENTRY_NP(sfmmu_mmu_trap) 1946 SET_GL_REG(1) 1947 USE_ALTERNATE_GLOBALS(%g5) 1948 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 1949 rdpr %tt, %g6 1950 cmp %g6, FAST_IMMU_MISS_TT 1951 be,a,pn %icc, 1f 1952 mov T_INSTR_MMU_MISS, %g3 1953 cmp %g6, T_INSTR_MMU_MISS 1954 be,a,pn %icc, 1f 1955 mov T_INSTR_MMU_MISS, %g3 1956 mov %g5, %g2 1957 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1958 cmp %g6, FAST_DMMU_MISS_TT 1959 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1960 cmp %g6, T_DATA_MMU_MISS 1961 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 19621: 1963 /* 1964 * g2 = tag access reg 1965 * g3 = type 1966 */ 1967 sethi %hi(sfmmu_tsbmiss_exception), %g1 1968 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 1969 ba,pt %xcc, sys_trap 1970 mov -1, %g4 1971 /*NOTREACHED*/ 1972 SET_SIZE(sfmmu_mmu_trap) 1973 1974 ENTRY_NP(sfmmu_suspend_tl) 1975 SET_GL_REG(1) 1976 USE_ALTERNATE_GLOBALS(%g5) 1977 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 1978 rdpr %tt, %g6 1979 cmp %g6, FAST_IMMU_MISS_TT 1980 be,a,pn %icc, 1f 1981 mov T_INSTR_MMU_MISS, %g3 1982 mov %g5, %g2 1983 cmp %g6, FAST_DMMU_MISS_TT 1984 move %icc, T_DATA_MMU_MISS, %g3 1985 movne %icc, T_DATA_PROT, %g3 19861: 1987 sethi %hi(sfmmu_tsbmiss_suspended), %g1 1988 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 1989 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 1990 ba,pt %xcc, sys_trap 1991 mov PIL_15, %g4 1992 /*NOTREACHED*/ 1993 SET_SIZE(sfmmu_suspend_tl) 1994 1995 /* 1996 * No %g registers in use at this point. 1997 */ 1998 ENTRY_NP(sfmmu_window_trap) 1999 rdpr %tpc, %g1 2000#ifdef sun4v 2001#ifdef DEBUG 2002 /* We assume previous %gl was 1 */ 2003 rdpr %tstate, %g4 2004 srlx %g4, TSTATE_GL_SHIFT, %g4 2005 and %g4, TSTATE_GL_MASK, %g4 2006 cmp %g4, 1 2007 bne,a,pn %icc, ptl1_panic 2008 mov PTL1_BAD_WTRAP, %g1 2009#endif /* DEBUG */ 2010 /* user miss at tl>1. better be the window handler or user_rtt */ 2011 /* in user_rtt? */ 2012 set rtt_fill_start, %g4 2013 cmp %g1, %g4 2014 blu,pn %xcc, 6f 2015 .empty 2016 set rtt_fill_end, %g4 2017 cmp %g1, %g4 2018 bgeu,pn %xcc, 6f 2019 nop 2020 set fault_rtt_fn1, %g1 2021 wrpr %g0, %g1, %tnpc 2022 ba,a 7f 20236: 2024 ! must save this trap level before descending trap stack 2025 ! no need to save %tnpc, either overwritten or discarded 2026 ! already got it: rdpr %tpc, %g1 2027 rdpr %tstate, %g6 2028 rdpr %tt, %g7 2029 ! trap level saved, go get underlying trap type 2030 rdpr %tl, %g5 2031 sub %g5, 1, %g3 2032 wrpr %g3, %tl 2033 rdpr %tt, %g2 2034 wrpr %g5, %tl 2035 ! restore saved trap level 2036 wrpr %g1, %tpc 2037 wrpr %g6, %tstate 2038 wrpr %g7, %tt 2039#else /* sun4v */ 2040 /* user miss at tl>1. better be the window handler */ 2041 rdpr %tl, %g5 2042 sub %g5, 1, %g3 2043 wrpr %g3, %tl 2044 rdpr %tt, %g2 2045 wrpr %g5, %tl 2046#endif /* sun4v */ 2047 and %g2, WTRAP_TTMASK, %g4 2048 cmp %g4, WTRAP_TYPE 2049 bne,pn %xcc, 1f 2050 nop 2051 /* tpc should be in the trap table */ 2052 set trap_table, %g4 2053 cmp %g1, %g4 2054 blt,pn %xcc, 1f 2055 .empty 2056 set etrap_table, %g4 2057 cmp %g1, %g4 2058 bge,pn %xcc, 1f 2059 .empty 2060 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 2061 add %g1, WTRAP_FAULTOFF, %g1 2062 wrpr %g0, %g1, %tnpc 20637: 2064 /* 2065 * some wbuf handlers will call systrap to resolve the fault 2066 * we pass the trap type so they figure out the correct parameters. 2067 * g5 = trap type, g6 = tag access reg 2068 */ 2069 2070 /* 2071 * only use g5, g6, g7 registers after we have switched to alternate 2072 * globals. 2073 */ 2074 SET_GL_REG(1) 2075 USE_ALTERNATE_GLOBALS(%g5) 2076 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 2077 rdpr %tt, %g7 2078 cmp %g7, FAST_IMMU_MISS_TT 2079 be,a,pn %icc, ptl1_panic 2080 mov PTL1_BAD_WTRAP, %g1 2081 cmp %g7, T_INSTR_MMU_MISS 2082 be,a,pn %icc, ptl1_panic 2083 mov PTL1_BAD_WTRAP, %g1 2084 mov T_DATA_PROT, %g5 2085 cmp %g7, FAST_DMMU_MISS_TT 2086 move %icc, T_DATA_MMU_MISS, %g5 2087 cmp %g7, T_DATA_MMU_MISS 2088 move %icc, T_DATA_MMU_MISS, %g5 2089 ! XXXQ AGS re-check out this one 2090 done 20911: 2092 CPU_PADDR(%g1, %g4) 2093 add %g1, CPU_TL1_HDLR, %g1 2094 lda [%g1]ASI_MEM, %g4 2095 brnz,a,pt %g4, sfmmu_mmu_trap 2096 sta %g0, [%g1]ASI_MEM 2097 ba,pt %icc, ptl1_panic 2098 mov PTL1_BAD_TRAP, %g1 2099 SET_SIZE(sfmmu_window_trap) 2100 2101 ENTRY_NP(sfmmu_kpm_exception) 2102 /* 2103 * We have accessed an unmapped segkpm address or a legal segkpm 2104 * address which is involved in a VAC alias conflict prevention. 2105 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 2106 * set. If it is, we will instead note that a fault has occurred 2107 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 2108 * a "retry"). This will step over the faulting instruction. 2109 * Note that this means that a legal segkpm address involved in 2110 * a VAC alias conflict prevention (a rare case to begin with) 2111 * cannot be used in DTrace. 2112 */ 2113 CPU_INDEX(%g1, %g2) 2114 set cpu_core, %g2 2115 sllx %g1, CPU_CORE_SHIFT, %g1 2116 add %g1, %g2, %g1 2117 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 2118 andcc %g2, CPU_DTRACE_NOFAULT, %g0 2119 bz 0f 2120 or %g2, CPU_DTRACE_BADADDR, %g2 2121 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 2122 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 2123 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 2124 done 21250: 2126 TSTAT_CHECK_TL1(1f, %g1, %g2) 21271: 2128 SET_GL_REG(1) 2129 USE_ALTERNATE_GLOBALS(%g5) 2130 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 2131 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 2132 /* 2133 * g2=tagacc g3.l=type g3.h=0 2134 */ 2135 sethi %hi(trap), %g1 2136 or %g1, %lo(trap), %g1 2137 ba,pt %xcc, sys_trap 2138 mov -1, %g4 2139 SET_SIZE(sfmmu_kpm_exception) 2140 2141#endif /* lint */ 2142 2143#if defined (lint) 2144 2145void 2146sfmmu_tsb_miss(void) 2147{ 2148} 2149 2150void 2151sfmmu_kpm_dtsb_miss(void) 2152{ 2153} 2154 2155void 2156sfmmu_kpm_dtsb_miss_small(void) 2157{ 2158} 2159 2160#else /* lint */ 2161 2162#if (IMAP_SEG != 0) 2163#error - ism_map->ism_seg offset is not zero 2164#endif 2165 2166/* 2167 * Copies ism mapping for this ctx in param "ism" if this is a ISM 2168 * tlb miss and branches to label "ismhit". If this is not an ISM 2169 * process or an ISM tlb miss it falls thru. 2170 * 2171 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 2172 * this process. 2173 * If so, it will branch to label "ismhit". If not, it will fall through. 2174 * 2175 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 2176 * so that any other threads of this process will not try and walk the ism 2177 * maps while they are being changed. 2178 * 2179 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 2180 * will make sure of that. This means we can terminate our search on 2181 * the first zero mapping we find. 2182 * 2183 * Parameters: 2184 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 2185 * tsbmiss = address of tsb miss area (in) 2186 * ismseg = contents of ism_seg for this ism map (out) 2187 * ismhat = physical address of imap_ismhat for this ism map (out) 2188 * tmp1 = scratch reg (CLOBBERED) 2189 * tmp2 = scratch reg (CLOBBERED) 2190 * tmp3 = scratch reg (CLOBBERED) 2191 * label: temporary labels 2192 * ismhit: label where to jump to if an ism dtlb miss 2193 * exitlabel:label where to jump if hat is busy due to hat_unshare. 2194 */ 2195#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 2196 label, ismhit) \ 2197 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 2198 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 2199 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 2200label/**/1: ;\ 2201 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 2202 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 2203label/**/2: ;\ 2204 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 2205 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 2206 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 2207 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 2208 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 2209 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 2210 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 2211 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 2212 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 2213 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 2214 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 2215 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 2216 ;\ 2217 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 2218 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 2219 cmp ismhat, tmp1 ;\ 2220 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 2221 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 2222 ;\ 2223 add tmp3, IBLK_NEXTPA, tmp1 ;\ 2224 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 2225 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 2226 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 2227label/**/3: 2228 2229/* 2230 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 2231 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 2232 * Parameters: 2233 * tagacc = reg containing virtual address 2234 * hatid = reg containing sfmmu pointer 2235 * hmeshift = constant/register to shift vaddr to obtain vapg 2236 * hmebp = register where bucket pointer will be stored 2237 * vapg = register where virtual page will be stored 2238 * tmp1, tmp2 = tmp registers 2239 */ 2240 2241 2242#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 2243 vapg, label, tmp1, tmp2) \ 2244 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 2245 brnz,a,pt tmp1, label/**/1 ;\ 2246 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 2247 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 2248 ba,pt %xcc, label/**/2 ;\ 2249 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 2250label/**/1: ;\ 2251 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 2252label/**/2: ;\ 2253 srlx tagacc, hmeshift, vapg ;\ 2254 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 2255 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 2256 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 2257 add hmebp, tmp1, hmebp 2258 2259/* 2260 * hashtag includes bspage + hashno (64 bits). 2261 */ 2262 2263#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 2264 sllx vapg, hmeshift, vapg ;\ 2265 mov hashno, hblktag ;\ 2266 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 2267 or vapg, hblktag, hblktag 2268 2269/* 2270 * Function to traverse hmeblk hash link list and find corresponding match. 2271 * The search is done using physical pointers. It returns the physical address 2272 * and virtual address pointers to the hmeblk that matches with the tag 2273 * provided. 2274 * Parameters: 2275 * hmebp = register that points to hme hash bucket, also used as 2276 * tmp reg (clobbered) 2277 * hmeblktag = register with hmeblk tag match 2278 * hatid = register with hatid 2279 * hmeblkpa = register where physical ptr will be stored 2280 * hmeblkva = register where virtual ptr will be stored 2281 * tmp1 = tmp reg 2282 * label: temporary label 2283 */ 2284 2285#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2286 tsbarea, tmp1, label) \ 2287 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 2288 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2289 add hmebp, HMEBUCK_HBLK, hmeblkva ;\ 2290 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2291 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2292label/**/1: ;\ 2293 brz,pn hmeblkva, label/**/2 ;\ 2294 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2295 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 2296 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2297 add hmebp, CLONGSIZE, hmebp ;\ 2298 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 2299 xor tmp1, hmeblktag, tmp1 ;\ 2300 xor hmebp, hatid, hmebp ;\ 2301 or hmebp, tmp1, hmebp ;\ 2302 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 2303 add hmeblkpa, HMEBLK_NEXT, hmebp ;\ 2304 ldna [hmebp]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2305 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 2306 ba,pt %xcc, label/**/1 ;\ 2307 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2308label/**/2: 2309 2310/* 2311 * Function to traverse hmeblk hash link list and find corresponding match. 2312 * The search is done using physical pointers. It returns the physical address 2313 * and virtual address pointers to the hmeblk that matches with the tag 2314 * provided. 2315 * Parameters: 2316 * hmeblktag = register with hmeblk tag match (rid field is 0) 2317 * hatid = register with hatid (pointer to SRD) 2318 * hmeblkpa = register where physical ptr will be stored 2319 * hmeblkva = register where virtual ptr will be stored 2320 * tmp1 = tmp reg 2321 * tmp2 = tmp reg 2322 * label: temporary label 2323 */ 2324 2325#define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2326 tsbarea, tmp1, tmp2, label) \ 2327label/**/1: ;\ 2328 brz,pn hmeblkva, label/**/4 ;\ 2329 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 2330 add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 2331 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2332 add tmp2, CLONGSIZE, tmp2 ;\ 2333 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 2334 xor tmp1, hmeblktag, tmp1 ;\ 2335 xor tmp2, hatid, tmp2 ;\ 2336 brz,pn tmp2, label/**/3 /* branch on hit */ ;\ 2337 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2338label/**/2: ;\ 2339 ldna [tmp2]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2340 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2341 ba,pt %xcc, label/**/1 ;\ 2342 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2343label/**/3: ;\ 2344 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 2345 bgeu,pt %xcc, label/**/2 ;\ 2346 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2347 and tmp1, BT_ULMASK, tmp2 ;\ 2348 srlx tmp1, BT_ULSHIFT, tmp1 ;\ 2349 sllx tmp1, CLONGSHIFT, tmp1 ;\ 2350 add tsbarea, tmp1, tmp1 ;\ 2351 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 2352 srlx tmp1, tmp2, tmp1 ;\ 2353 btst 0x1, tmp1 ;\ 2354 bz,pn %xcc, label/**/2 ;\ 2355 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2356label/**/4: 2357 2358#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2359#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2360#endif 2361 2362/* 2363 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2364 * he offset for the corresponding hment. 2365 * Parameters: 2366 * In: 2367 * vaddr = register with virtual address 2368 * hmeblkpa = physical pointer to hme_blk 2369 * Out: 2370 * hmentoff = register where hment offset will be stored 2371 * hmemisc = hblk_misc 2372 * Scratch: 2373 * tmp1 2374 */ 2375#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2376 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2377 lda [hmentoff]ASI_MEM, hmemisc ;\ 2378 andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2379 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2380 or %g0, HMEBLK_HME1, hmentoff ;\ 2381 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2382 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2383 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2384 add tmp1, HMEBLK_HME1, hmentoff ;\ 2385label1: 2386 2387/* 2388 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2389 * 2390 * tagacc = (pseudo-)tag access register (in) 2391 * hatid = sfmmu pointer for TSB miss (in) 2392 * tte = tte for TLB miss if found, otherwise clobbered (out) 2393 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2394 * hmeblkva = VA of hment if found, otherwise clobbered (out) 2395 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2396 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2397 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2398 * for this page size. 2399 * hashno = constant/register hash number 2400 * label = temporary label for branching within macro. 2401 * foundlabel = label to jump to when tte is found. 2402 * suspendlabel= label to jump to when tte is suspended. 2403 * exitlabel = label to jump to when tte is not found. 2404 * 2405 */ 2406#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmemisc, \ 2407 hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \ 2408 ;\ 2409 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2410 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2411 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2412 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2413 ;\ 2414 /* ;\ 2415 * tagacc = tagacc ;\ 2416 * hatid = hatid ;\ 2417 * tsbarea = tsbarea ;\ 2418 * tte = hmebp (hme bucket pointer) ;\ 2419 * hmeblkpa = vapg (virtual page) ;\ 2420 * hmemisc, hmeblkva = scratch ;\ 2421 */ ;\ 2422 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2423 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2424 ;\ 2425 /* ;\ 2426 * tagacc = tagacc ;\ 2427 * hatid = hatid ;\ 2428 * tte = hmebp ;\ 2429 * hmeblkpa = CLOBBERED ;\ 2430 * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2431 * hmeblkva = scratch ;\ 2432 */ ;\ 2433 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2434 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2435 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, hmeblkva, \ 2436 tsbarea, tagacc, label/**/1) ;\ 2437 /* ;\ 2438 * tagacc = CLOBBERED ;\ 2439 * tte = CLOBBERED ;\ 2440 * hmeblkpa = hmeblkpa ;\ 2441 * hmeblkva = hmeblkva ;\ 2442 */ ;\ 2443 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2444 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2445 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2446 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2447 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2448 nop ;\ 2449label/**/4: ;\ 2450 /* ;\ 2451 * We have found the hmeblk containing the hment. ;\ 2452 * Now we calculate the corresponding tte. ;\ 2453 * ;\ 2454 * tagacc = tagacc ;\ 2455 * hatid = hatid ;\ 2456 * tte = clobbered ;\ 2457 * hmeblkpa = hmeblkpa ;\ 2458 * hmemisc = hblktag ;\ 2459 * hmeblkva = hmeblkva ;\ 2460 */ ;\ 2461 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2462 label/**/2) ;\ 2463 ;\ 2464 /* ;\ 2465 * tagacc = tagacc ;\ 2466 * hatid = hmentoff ;\ 2467 * tte = clobbered ;\ 2468 * hmeblkpa = hmeblkpa ;\ 2469 * hmemisc = hblk_misc ;\ 2470 * hmeblkva = hmeblkva ;\ 2471 */ ;\ 2472 ;\ 2473 add hatid, SFHME_TTE, hatid ;\ 2474 add hmeblkpa, hatid, hmeblkpa ;\ 2475 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2476 add hmeblkva, hatid, hmeblkva ;\ 2477 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2478 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2479 set TTE_SUSPEND, hatid ;\ 2480 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2481 btst tte, hatid ;\ 2482 bz,pt %xcc, foundlabel ;\ 2483 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2484 ;\ 2485 /* ;\ 2486 * Mapping is suspended, so goto suspend label. ;\ 2487 */ ;\ 2488 ba,pt %xcc, suspendlabel ;\ 2489 nop 2490 2491/* 2492 * GET_SHME_TTE is similar to GET_TTE() except it searches 2493 * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2494 * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2495 * either 0 (not part of scd) or 1 (part of scd). 2496 */ 2497#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, \ 2498 hmemisc, hmeshift, hashno, label, foundlabel, \ 2499 suspendlabel, exitlabel) \ 2500 ;\ 2501 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2502 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2503 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2504 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2505 ;\ 2506 /* ;\ 2507 * tagacc = tagacc ;\ 2508 * hatid = hatid ;\ 2509 * tsbarea = tsbarea ;\ 2510 * tte = hmebp (hme bucket pointer) ;\ 2511 * hmeblkpa = vapg (virtual page) ;\ 2512 * hmemisc, hmeblkva = scratch ;\ 2513 */ ;\ 2514 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2515 ;\ 2516 /* ;\ 2517 * tagacc = tagacc ;\ 2518 * hatid = hatid ;\ 2519 * tsbarea = tsbarea ;\ 2520 * tte = hmebp ;\ 2521 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2522 * hmeblkpa = CLOBBERED ;\ 2523 * hmeblkva = scratch ;\ 2524 */ ;\ 2525 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2526 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2527 ;\ 2528 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2529 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2530 add tte, HMEBUCK_HBLK, hmeblkva ;\ 2531 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2532 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2533 ;\ 2534label/**/8: ;\ 2535 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, hmeblkva, \ 2536 tsbarea, tagacc, tte, label/**/1) ;\ 2537 /* ;\ 2538 * tagacc = CLOBBERED ;\ 2539 * tte = CLOBBERED ;\ 2540 * hmeblkpa = hmeblkpa ;\ 2541 * hmeblkva = hmeblkva ;\ 2542 */ ;\ 2543 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2544 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2545 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2546 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2547 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2548 nop ;\ 2549label/**/4: ;\ 2550 /* ;\ 2551 * We have found the hmeblk containing the hment. ;\ 2552 * Now we calculate the corresponding tte. ;\ 2553 * ;\ 2554 * tagacc = tagacc ;\ 2555 * hatid = hatid ;\ 2556 * tte = clobbered ;\ 2557 * hmeblkpa = hmeblkpa ;\ 2558 * hmemisc = hblktag ;\ 2559 * hmeblkva = hmeblkva ;\ 2560 * tsbarea = tsbmiss area ;\ 2561 */ ;\ 2562 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2563 label/**/2) ;\ 2564 ;\ 2565 /* ;\ 2566 * tagacc = tagacc ;\ 2567 * hatid = hmentoff ;\ 2568 * tte = clobbered ;\ 2569 * hmeblkpa = hmeblkpa ;\ 2570 * hmemisc = hblk_misc ;\ 2571 * hmeblkva = hmeblkva ;\ 2572 * tsbarea = tsbmiss area ;\ 2573 */ ;\ 2574 ;\ 2575 add hatid, SFHME_TTE, hatid ;\ 2576 add hmeblkpa, hatid, hmeblkpa ;\ 2577 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2578 brlz,pt tte, label/**/6 ;\ 2579 add hmeblkva, hatid, hmeblkva ;\ 2580 btst HBLK_SZMASK, hmemisc ;\ 2581 bnz,a,pt %icc, label/**/7 ;\ 2582 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2583 ;\ 2584 /* ;\ 2585 * We found an invalid 8K tte in shme. ;\ 2586 * it may not belong to shme's region since ;\ 2587 * region size/alignment granularity is 8K but different ;\ 2588 * regions don't share hmeblks. Continue the search. ;\ 2589 */ ;\ 2590 sub hmeblkpa, hatid, hmeblkpa ;\ 2591 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2592 srlx tagacc, hmeshift, tte ;\ 2593 add hmeblkpa, HMEBLK_NEXT, hmeblkva ;\ 2594 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2595 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2596 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2597 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2598 ba,a,pt %xcc, label/**/8 ;\ 2599label/**/6: ;\ 2600 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2601 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2602label/**/7: ;\ 2603 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2604 set TTE_SUSPEND, hatid ;\ 2605 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2606 btst tte, hatid ;\ 2607 bz,pt %xcc, foundlabel ;\ 2608 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2609 ;\ 2610 /* ;\ 2611 * Mapping is suspended, so goto suspend label. ;\ 2612 */ ;\ 2613 ba,pt %xcc, suspendlabel ;\ 2614 nop 2615 2616 /* 2617 * KERNEL PROTECTION HANDLER 2618 * 2619 * g1 = tsb8k pointer register (clobbered) 2620 * g2 = tag access register (ro) 2621 * g3 - g7 = scratch registers 2622 * 2623 * Note: This function is patched at runtime for performance reasons. 2624 * Any changes here require sfmmu_patch_ktsb fixed. 2625 */ 2626 ENTRY_NP(sfmmu_kprot_trap) 2627 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2628sfmmu_kprot_patch_ktsb_base: 2629 RUNTIME_PATCH_SETX(%g1, %g6) 2630 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2631sfmmu_kprot_patch_ktsb_szcode: 2632 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2633 2634 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2635 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2636 2637 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2638sfmmu_kprot_patch_ktsb4m_base: 2639 RUNTIME_PATCH_SETX(%g3, %g6) 2640 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2641sfmmu_kprot_patch_ktsb4m_szcode: 2642 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2643 2644 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2645 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2646 2647 CPU_TSBMISS_AREA(%g6, %g7) 2648 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2649 ba,pt %xcc, sfmmu_tsb_miss_tt 2650 nop 2651 2652 /* 2653 * USER PROTECTION HANDLER 2654 * 2655 * g1 = tsb8k pointer register (ro) 2656 * g2 = tag access register (ro) 2657 * g3 = faulting context (clobbered, currently not used) 2658 * g4 - g7 = scratch registers 2659 */ 2660 ALTENTRY(sfmmu_uprot_trap) 2661#ifdef sun4v 2662 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2663 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2664 2665 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2666 brlz,pt %g3, 9f /* check for 2nd TSB */ 2667 nop 2668 2669 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2670 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2671 2672#else /* sun4v */ 2673#ifdef UTSB_PHYS 2674 /* g1 = first TSB entry ptr */ 2675 GET_2ND_TSBREG(%g3) 2676 brlz,pt %g3, 9f /* check for 2nd TSB */ 2677 nop 2678 2679 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2680 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2681#else /* UTSB_PHYS */ 2682 brgez,pt %g1, 9f /* check for 2nd TSB */ 2683 mov -1, %g3 /* set second tsbe ptr to -1 */ 2684 2685 mov %g2, %g7 2686 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2687 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2688 mov %g1, %g7 2689 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2690#endif /* UTSB_PHYS */ 2691#endif /* sun4v */ 26929: 2693 CPU_TSBMISS_AREA(%g6, %g7) 2694 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2695 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2696 nop 2697 2698 /* 2699 * Kernel 8K page iTLB miss. We also get here if we took a 2700 * fast instruction access mmu miss trap while running in 2701 * invalid context. 2702 * 2703 * %g1 = 8K TSB pointer register (not used, clobbered) 2704 * %g2 = tag access register (used) 2705 * %g3 = faulting context id (used) 2706 * %g7 = TSB tag to match (used) 2707 */ 2708 .align 64 2709 ALTENTRY(sfmmu_kitlb_miss) 2710 brnz,pn %g3, tsb_tl0_noctxt 2711 nop 2712 2713 /* kernel miss */ 2714 /* get kernel tsb pointer */ 2715 /* we patch the next set of instructions at run time */ 2716 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2717iktsbbase: 2718 RUNTIME_PATCH_SETX(%g4, %g5) 2719 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2720 2721iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2722 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2723 or %g4, %g1, %g1 ! form tsb ptr 2724 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2725 cmp %g4, %g7 2726 bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2727 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2728 2729 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2730 bz,pn %icc, exec_fault 2731 nop 2732 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2733 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2734 retry 2735 2736iktsb4mbase: 2737 RUNTIME_PATCH_SETX(%g4, %g6) 2738 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2739iktsb4m: 2740 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2741 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2742 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2743 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2744 cmp %g4, %g7 2745 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2746 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2747 bz,pn %icc, exec_fault 2748 nop 2749 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2750 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2751 retry 2752 2753 /* 2754 * Kernel dTLB miss. We also get here if we took a fast data 2755 * access mmu miss trap while running in invalid context. 2756 * 2757 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2758 * We select the TSB miss handler to branch to depending on 2759 * the virtual address of the access. In the future it may 2760 * be desirable to separate kpm TTEs into their own TSB, 2761 * in which case all that needs to be done is to set 2762 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2763 * early in the miss if we detect a kpm VA to a new handler. 2764 * 2765 * %g1 = 8K TSB pointer register (not used, clobbered) 2766 * %g2 = tag access register (used) 2767 * %g3 = faulting context id (used) 2768 */ 2769 .align 64 2770 ALTENTRY(sfmmu_kdtlb_miss) 2771 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2772 nop 2773 2774 /* Gather some stats for kpm misses in the TLB. */ 2775 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2776 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2777 2778 /* 2779 * Get first TSB offset and look for 8K/64K/512K mapping 2780 * using the 8K virtual page as the index. 2781 * 2782 * We patch the next set of instructions at run time; 2783 * any changes here require sfmmu_patch_ktsb changes too. 2784 */ 2785dktsbbase: 2786 RUNTIME_PATCH_SETX(%g7, %g6) 2787 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2788 2789dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2790 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2791 2792 /* 2793 * At this point %g1 is our index into the TSB. 2794 * We just masked off enough bits of the VA depending 2795 * on our TSB size code. 2796 */ 2797 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2798 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2799 cmp %g6, %g4 ! compare tag 2800 bne,pn %xcc, dktsb4m_kpmcheck_small 2801 add %g7, %g1, %g1 /* form tsb ptr */ 2802 TT_TRACE(trace_tsbhit) 2803 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2804 /* trapstat expects tte in %g5 */ 2805 retry 2806 2807 /* 2808 * If kpm is using large pages, the following instruction needs 2809 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2810 * so that we will probe the 4M TSB regardless of the VA. In 2811 * the case kpm is using small pages, we know no large kernel 2812 * mappings are located above 0x80000000.00000000 so we skip the 2813 * probe as an optimization. 2814 */ 2815dktsb4m_kpmcheck_small: 2816 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2817 /* delay slot safe, below */ 2818 2819 /* 2820 * Get second TSB offset and look for 4M mapping 2821 * using 4M virtual page as the TSB index. 2822 * 2823 * Here: 2824 * %g1 = 8K TSB pointer. Don't squash it. 2825 * %g2 = tag access register (we still need it) 2826 */ 2827 srlx %g2, MMU_PAGESHIFT4M, %g3 2828 2829 /* 2830 * We patch the next set of instructions at run time; 2831 * any changes here require sfmmu_patch_ktsb changes too. 2832 */ 2833dktsb4mbase: 2834 RUNTIME_PATCH_SETX(%g7, %g6) 2835 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2836dktsb4m: 2837 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2838 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2839 2840 /* 2841 * At this point %g3 is our index into the TSB. 2842 * We just masked off enough bits of the VA depending 2843 * on our TSB size code. 2844 */ 2845 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2846 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2847 cmp %g6, %g4 ! compare tag 2848 2849dktsb4m_tsbmiss: 2850 bne,pn %xcc, dktsb4m_kpmcheck 2851 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2852 TT_TRACE(trace_tsbhit) 2853 /* we don't check TTE size here since we assume 4M TSB is separate */ 2854 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2855 /* trapstat expects tte in %g5 */ 2856 retry 2857 2858 /* 2859 * So, we failed to find a valid TTE to match the faulting 2860 * address in either TSB. There are a few cases that could land 2861 * us here: 2862 * 2863 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2864 * to sfmmu_tsb_miss_tt to handle the miss. 2865 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2866 * 4M TSB. Let segkpm handle it. 2867 * 2868 * Note that we shouldn't land here in the case of a kpm VA when 2869 * kpm_smallpages is active -- we handled that case earlier at 2870 * dktsb4m_kpmcheck_small. 2871 * 2872 * At this point: 2873 * g1 = 8K-indexed primary TSB pointer 2874 * g2 = tag access register 2875 * g3 = 4M-indexed secondary TSB pointer 2876 */ 2877dktsb4m_kpmcheck: 2878 cmp %g2, %g0 2879 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2880 nop 2881 ba,a,pt %icc, sfmmu_tsb_miss_tt 2882 nop 2883 2884#ifdef sun4v 2885 /* 2886 * User instruction miss w/ single TSB. 2887 * The first probe covers 8K, 64K, and 512K page sizes, 2888 * because 64K and 512K mappings are replicated off 8K 2889 * pointer. 2890 * 2891 * g1 = tsb8k pointer register 2892 * g2 = tag access register 2893 * g3 - g6 = scratch registers 2894 * g7 = TSB tag to match 2895 */ 2896 .align 64 2897 ALTENTRY(sfmmu_uitlb_fastpath) 2898 2899 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2900 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2901 ba,pn %xcc, sfmmu_tsb_miss_tt 2902 mov -1, %g3 2903 2904 /* 2905 * User data miss w/ single TSB. 2906 * The first probe covers 8K, 64K, and 512K page sizes, 2907 * because 64K and 512K mappings are replicated off 8K 2908 * pointer. 2909 * 2910 * g1 = tsb8k pointer register 2911 * g2 = tag access register 2912 * g3 - g6 = scratch registers 2913 * g7 = TSB tag to match 2914 */ 2915 .align 64 2916 ALTENTRY(sfmmu_udtlb_fastpath) 2917 2918 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2919 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2920 ba,pn %xcc, sfmmu_tsb_miss_tt 2921 mov -1, %g3 2922 2923 /* 2924 * User instruction miss w/ multiple TSBs (sun4v). 2925 * The first probe covers 8K, 64K, and 512K page sizes, 2926 * because 64K and 512K mappings are replicated off 8K 2927 * pointer. Second probe covers 4M page size only. 2928 * 2929 * Just like sfmmu_udtlb_slowpath, except: 2930 * o Uses ASI_ITLB_IN 2931 * o checks for execute permission 2932 * o No ISM prediction. 2933 * 2934 * g1 = tsb8k pointer register 2935 * g2 = tag access register 2936 * g3 - g6 = scratch registers 2937 * g7 = TSB tag to match 2938 */ 2939 .align 64 2940 ALTENTRY(sfmmu_uitlb_slowpath) 2941 2942 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2943 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2944 /* g4 - g5 = clobbered here */ 2945 2946 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2947 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2948 srlx %g2, TAG_VALO_SHIFT, %g7 2949 PROBE_2ND_ITSB(%g3, %g7) 2950 /* NOT REACHED */ 2951 2952#else /* sun4v */ 2953 2954 /* 2955 * User instruction miss w/ multiple TSBs (sun4u). 2956 * The first probe covers 8K, 64K, and 512K page sizes, 2957 * because 64K and 512K mappings are replicated off 8K 2958 * pointer. Second probe covers 4M page size only. 2959 * 2960 * Just like sfmmu_udtlb_slowpath, except: 2961 * o Uses ASI_ITLB_IN 2962 * o checks for execute permission 2963 * o No ISM prediction. 2964 * 2965 * g1 = tsb8k pointer register 2966 * g2 = tag access register 2967 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch 2968 * g4 - g6 = scratch registers 2969 * g7 = TSB tag to match 2970 */ 2971 .align 64 2972 ALTENTRY(sfmmu_uitlb_slowpath) 2973 2974#ifdef UTSB_PHYS 2975 /* 2976 * g1 = 1st TSB entry pointer 2977 * g3 = 2nd TSB base register 2978 * Need 2nd TSB entry pointer for 2nd probe. 2979 */ 2980 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2981 2982 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2983#else /* UTSB_PHYS */ 2984 mov %g1, %g3 /* save tsb8k reg in %g3 */ 2985 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 2986 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2987 2988 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 2989 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 2990 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 2991#endif /* UTSB_PHYS */ 2992 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2993 srlx %g2, TAG_VALO_SHIFT, %g7 2994 PROBE_2ND_ITSB(%g3, %g7, isynth) 2995 /* NOT REACHED */ 2996#endif /* sun4v */ 2997 2998 /* 2999 * User data miss w/ multiple TSBs. 3000 * The first probe covers 8K, 64K, and 512K page sizes, 3001 * because 64K and 512K mappings are replicated off 8K 3002 * pointer. Second probe covers 4M page size only. 3003 * 3004 * We consider probing for 4M pages first if the VA falls 3005 * in a range that's likely to be ISM. 3006 * 3007 * g1 = tsb8k pointer register 3008 * g2 = tag access register 3009 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch 3010 * g4 - g6 = scratch registers 3011 * g7 = TSB tag to match 3012 */ 3013 .align 64 3014 ALTENTRY(sfmmu_udtlb_slowpath) 3015 3016 /* 3017 * Check for ISM. If it exists, look for 4M mappings in the second TSB 3018 * first, then probe for other mappings in the first TSB if that fails. 3019 */ 3020 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 3021 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 3022 mov %g1, %g3 3023 3024udtlb_miss_probefirst: 3025 /* 3026 * g1 = 8K TSB pointer register 3027 * g2 = tag access register 3028 * g3 = (potentially) second TSB entry ptr 3029 * g6 = ism pred. 3030 * g7 = vpg_4m 3031 */ 3032#ifdef sun4v 3033 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 3034 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3035 3036 /* 3037 * Here: 3038 * g1 = first TSB pointer 3039 * g2 = tag access reg 3040 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3041 */ 3042 brgz,pn %g6, sfmmu_tsb_miss_tt 3043 nop 3044#else /* sun4v */ 3045#ifndef UTSB_PHYS 3046 mov %g1, %g4 3047 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 3048#endif UTSB_PHYS 3049 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3050 3051 /* 3052 * Here: 3053 * g1 = first TSB pointer 3054 * g2 = tag access reg 3055 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3056 */ 3057 brgz,pn %g6, sfmmu_tsb_miss_tt 3058 nop 3059#ifndef UTSB_PHYS 3060 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 3061#endif UTSB_PHYS 3062 /* fall through in 8K->4M probe order */ 3063#endif /* sun4v */ 3064 3065udtlb_miss_probesecond: 3066 /* 3067 * Look in the second TSB for the TTE 3068 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 3069 * g2 = tag access reg 3070 * g3 = 8K TSB pointer register 3071 * g6 = ism pred. 3072 * g7 = vpg_4m 3073 */ 3074#ifdef sun4v 3075 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 3076 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3077 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 3078#else /* sun4v */ 3079#ifdef UTSB_PHYS 3080 GET_2ND_TSBREG(%g3) 3081 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3082 /* tagacc (%g2) is okay, no need to reload, %g3 = second tsbe ptr */ 3083#else /* UTSB_PHYS */ 3084 mov %g3, %g7 3085 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 3086 /* %g2 clobbered, %g3 =second tsbe ptr */ 3087 mov MMU_TAG_ACCESS, %g2 3088 ldxa [%g2]ASI_DMMU, %g2 3089#endif /* UTSB_PHYS */ 3090#endif /* sun4v */ 3091 3092 srlx %g2, TAG_VALO_SHIFT, %g7 3093 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3094 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 3095 brgz,pn %g6, udtlb_miss_probefirst 3096 nop 3097 3098 /* fall through to sfmmu_tsb_miss_tt */ 3099 3100 ALTENTRY(sfmmu_tsb_miss_tt) 3101 TT_TRACE(trace_tsbmiss) 3102 /* 3103 * We get here if there is a TSB miss OR a write protect trap. 3104 * 3105 * g1 = First TSB entry pointer 3106 * g2 = tag access register 3107 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 3108 * g4 - g7 = scratch registers 3109 */ 3110 3111 ALTENTRY(sfmmu_tsb_miss) 3112 3113 /* 3114 * If trapstat is running, we need to shift the %tpc and %tnpc to 3115 * point to trapstat's TSB miss return code (note that trapstat 3116 * itself will patch the correct offset to add). 3117 */ 3118 rdpr %tl, %g7 3119 cmp %g7, 1 3120 ble,pt %xcc, 0f 3121 sethi %hi(KERNELBASE), %g6 3122 rdpr %tpc, %g7 3123 or %g6, %lo(KERNELBASE), %g6 3124 cmp %g7, %g6 3125 bgeu,pt %xcc, 0f 3126 /* delay slot safe */ 3127 3128 ALTENTRY(tsbmiss_trapstat_patch_point) 3129 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 3130 wrpr %g7, %tpc 3131 add %g7, 4, %g7 3132 wrpr %g7, %tnpc 31330: 3134 CPU_TSBMISS_AREA(%g6, %g7) 3135 3136 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save first tsb pointer */ 3137 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save second tsb pointer */ 3138 3139 sllx %g2, TAGACC_CTX_LSHIFT, %g3 3140 brz,a,pn %g3, 1f /* skip ahead if kernel */ 3141 ldn [%g6 + TSBMISS_KHATID], %g7 3142 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 3143 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 3144 3145 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 3146 3147 cmp %g3, INVALID_CONTEXT 3148 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 3149 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 3150 3151#ifdef sun4v 3152 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 3153 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 3154 stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 3155#endif 3156 3157 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 3158 /* 3159 * The miss wasn't in an ISM segment. 3160 * 3161 * %g1 %g3, %g4, %g5, %g7 all clobbered 3162 * %g2 = (pseudo) tag access 3163 */ 3164 3165 ba,pt %icc, 2f 3166 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 3167 31681: 3169 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 3170 /* 3171 * 8K and 64K hash. 3172 */ 31732: 3174 3175 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3176 MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte, 3177 sfmmu_suspend_tl, tsb_512K) 3178 /* NOT REACHED */ 3179 3180tsb_512K: 3181 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3182 brz,pn %g5, 3f 3183 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3184 and %g4, HAT_512K_FLAG, %g5 3185 3186 /* 3187 * Note that there is a small window here where we may have 3188 * a 512k page in the hash list but have not set the HAT_512K_FLAG 3189 * flag yet, so we will skip searching the 512k hash list. 3190 * In this case we will end up in pagefault which will find 3191 * the mapping and return. So, in this instance we will end up 3192 * spending a bit more time resolving this TSB miss, but it can 3193 * only happen once per process and even then, the chances of that 3194 * are very small, so it's not worth the extra overhead it would 3195 * take to close this window. 3196 */ 3197 brz,pn %g5, tsb_4M 3198 nop 31993: 3200 /* 3201 * 512K hash 3202 */ 3203 3204 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3205 MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte, 3206 sfmmu_suspend_tl, tsb_4M) 3207 /* NOT REACHED */ 3208 3209tsb_4M: 3210 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3211 brz,pn %g5, 4f 3212 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3213 and %g4, HAT_4M_FLAG, %g5 3214 brz,pn %g5, tsb_32M 3215 nop 32164: 3217 /* 3218 * 4M hash 3219 */ 3220 3221 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3222 MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte, 3223 sfmmu_suspend_tl, tsb_32M) 3224 /* NOT REACHED */ 3225 3226tsb_32M: 3227 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3228#ifdef sun4v 3229 brz,pn %g5, 6f 3230#else 3231 brz,pn %g5, tsb_pagefault 3232#endif 3233 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3234 and %g4, HAT_32M_FLAG, %g5 3235 brz,pn %g5, tsb_256M 3236 nop 32375: 3238 /* 3239 * 32M hash 3240 */ 3241 3242 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3243 MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte, 3244 sfmmu_suspend_tl, tsb_256M) 3245 /* NOT REACHED */ 3246 3247#ifdef sun4u 3248#define tsb_shme tsb_pagefault 3249#endif 3250tsb_256M: 3251 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3252 and %g4, HAT_256M_FLAG, %g5 3253 brz,pn %g5, tsb_shme 3254 nop 32556: 3256 /* 3257 * 256M hash 3258 */ 3259 3260 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3261 MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte, 3262 sfmmu_suspend_tl, tsb_shme) 3263 /* NOT REACHED */ 3264 3265tsb_checktte: 3266 /* 3267 * g1 = hblk_misc 3268 * g2 = tagacc 3269 * g3 = tte 3270 * g4 = tte pa 3271 * g5 = tte va 3272 * g6 = tsbmiss area 3273 * g7 = hatid 3274 */ 3275 brlz,a,pt %g3, tsb_validtte 3276 rdpr %tt, %g7 3277 3278#ifdef sun4u 3279#undef tsb_shme 3280 ba tsb_pagefault 3281 nop 3282#else 3283 3284tsb_shme: 3285 /* 3286 * g2 = tagacc 3287 * g6 = tsbmiss area 3288 */ 3289 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3290 brz,pn %g5, tsb_pagefault 3291 nop 3292 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3293 brz,pn %g7, tsb_pagefault 3294 nop 3295 3296 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3297 MMU_PAGESHIFT64K, TTE64K, tsb_shme_l8K, tsb_shme_checktte, 3298 sfmmu_suspend_tl, tsb_shme_512K) 3299 /* NOT REACHED */ 3300 3301tsb_shme_512K: 3302 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3303 and %g4, HAT_512K_FLAG, %g5 3304 brz,pn %g5, tsb_shme_4M 3305 nop 3306 3307 /* 3308 * 512K hash 3309 */ 3310 3311 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3312 MMU_PAGESHIFT512K, TTE512K, tsb_shme_l512K, tsb_shme_checktte, 3313 sfmmu_suspend_tl, tsb_shme_4M) 3314 /* NOT REACHED */ 3315 3316tsb_shme_4M: 3317 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3318 and %g4, HAT_4M_FLAG, %g5 3319 brz,pn %g5, tsb_shme_32M 3320 nop 33214: 3322 /* 3323 * 4M hash 3324 */ 3325 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3326 MMU_PAGESHIFT4M, TTE4M, tsb_shme_l4M, tsb_shme_checktte, 3327 sfmmu_suspend_tl, tsb_shme_32M) 3328 /* NOT REACHED */ 3329 3330tsb_shme_32M: 3331 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3332 and %g4, HAT_32M_FLAG, %g5 3333 brz,pn %g5, tsb_shme_256M 3334 nop 3335 3336 /* 3337 * 32M hash 3338 */ 3339 3340 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3341 MMU_PAGESHIFT32M, TTE32M, tsb_shme_l32M, tsb_shme_checktte, 3342 sfmmu_suspend_tl, tsb_shme_256M) 3343 /* NOT REACHED */ 3344 3345tsb_shme_256M: 3346 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3347 and %g4, HAT_256M_FLAG, %g5 3348 brz,pn %g5, tsb_pagefault 3349 nop 3350 3351 /* 3352 * 256M hash 3353 */ 3354 3355 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3356 MMU_PAGESHIFT256M, TTE256M, tsb_shme_l256M, tsb_shme_checktte, 3357 sfmmu_suspend_tl, tsb_pagefault) 3358 /* NOT REACHED */ 3359 3360tsb_shme_checktte: 3361 3362 brgez,pn %g3, tsb_pagefault 3363 rdpr %tt, %g7 3364 /* 3365 * g1 = ctx1 flag 3366 * g3 = tte 3367 * g4 = tte pa 3368 * g5 = tte va 3369 * g6 = tsbmiss area 3370 * g7 = tt 3371 */ 3372 3373 brz,pt %g1, tsb_validtte 3374 nop 3375 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3376 or %g1, HAT_CHKCTX1_FLAG, %g1 3377 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3378 3379 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3380#endif /* sun4u */ 3381 3382tsb_validtte: 3383 /* 3384 * g3 = tte 3385 * g4 = tte pa 3386 * g5 = tte va 3387 * g6 = tsbmiss area 3388 * g7 = tt 3389 */ 3390 3391 /* 3392 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3393 */ 3394 cmp %g7, FAST_PROT_TT 3395 bne,pt %icc, 4f 3396 nop 3397 3398 TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod, 3399 tsb_protfault) 3400 3401 rdpr %tt, %g5 3402 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3403#ifdef sun4v 3404 MMU_FAULT_STATUS_AREA(%g7) 3405 ldx [%g7 + MMFSA_D_ADDR], %g5 /* save fault addr for later */ 3406#endif 3407 ba,pt %xcc, tsb_update_tl1 3408 nop 3409 34104: 3411 /* 3412 * If ITLB miss check exec bit. 3413 * If not set treat as invalid TTE. 3414 */ 3415 cmp %g7, T_INSTR_MMU_MISS 3416 be,pn %icc, 5f 3417 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3418 cmp %g7, FAST_IMMU_MISS_TT 3419 bne,pt %icc, 3f 3420 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 34215: 3422 bz,pn %icc, tsb_protfault 3423 nop 3424 34253: 3426 /* 3427 * Set reference bit if not already set 3428 */ 3429 TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref) 3430 3431 /* 3432 * Now, load into TSB/TLB. At this point: 3433 * g3 = tte 3434 * g4 = patte 3435 * g6 = tsbmiss area 3436 */ 3437 rdpr %tt, %g5 3438#ifdef sun4v 3439 MMU_FAULT_STATUS_AREA(%g2) 3440 cmp %g5, T_INSTR_MMU_MISS 3441 be,a,pt %icc, 9f 3442 nop 3443 cmp %g5, FAST_IMMU_MISS_TT 3444 be,a,pt %icc, 9f 3445 nop 3446 add %g2, MMFSA_D_, %g2 34479: 3448 ldx [%g2 + MMFSA_CTX_], %g7 3449 sllx %g7, TTARGET_CTX_SHIFT, %g7 3450 ldx [%g2 + MMFSA_ADDR_], %g2 3451 mov %g2, %g5 ! save the fault addr for later use 3452 srlx %g2, TTARGET_VA_SHIFT, %g2 3453 or %g2, %g7, %g2 3454#else 3455 cmp %g5, FAST_IMMU_MISS_TT 3456 be,a,pt %icc, tsb_update_tl1 3457 ldxa [%g0]ASI_IMMU, %g2 3458 ldxa [%g0]ASI_DMMU, %g2 3459#endif 3460tsb_update_tl1: 3461 srlx %g2, TTARGET_CTX_SHIFT, %g7 3462 brz,pn %g7, tsb_kernel 3463#ifdef sun4v 3464 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3465#else 3466 srlx %g3, TTE_SZ_SHFT, %g7 3467#endif 3468 3469tsb_user: 3470#ifdef sun4v 3471 cmp %g7, TTE4M 3472 bge,pn %icc, tsb_user4m 3473 nop 3474#else /* sun4v */ 3475 cmp %g7, TTESZ_VALID | TTE4M 3476 be,pn %icc, tsb_user4m 3477 srlx %g3, TTE_SZ2_SHFT, %g7 3478 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3479#ifdef ITLB_32M_256M_SUPPORT 3480 bnz,pn %icc, tsb_user4m 3481 nop 3482#else /* ITLB_32M_256M_SUPPORT */ 3483 bnz,a,pn %icc, tsb_user_pn_synth 3484 cmp %g5, FAST_IMMU_MISS_TT 3485#endif /* ITLB_32M_256M_SUPPORT */ 3486#endif /* sun4v */ 3487 3488tsb_user8k: 3489#ifdef sun4v 3490 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3491 and %g7, HAT_CHKCTX1_FLAG, %g1 3492 brz,a,pn %g1, 1f 3493 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = first TSB ptr 3494 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3495 brlz,a,pn %g1, ptl1_panic ! if no shared tsb 3496 mov PTL1_NO_SCDTSB8K, %g1 ! panic 3497 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 34981: 3499#else 3500 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = first TSB ptr 3501 3502#ifndef UTSB_PHYS 3503 mov ASI_N, %g7 ! user TSBs accessed by VA 3504 mov %g7, %asi 3505#endif /* UTSB_PHYS */ 3506 3507#endif /* sun4v */ 3508 3509 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5) 3510 3511#ifdef sun4v 3512 rdpr %tt, %g5 3513 cmp %g5, T_INSTR_MMU_MISS 3514 be,a,pn %xcc, 9f 3515 mov %g3, %g5 3516#endif /* sun4v */ 3517 cmp %g5, FAST_IMMU_MISS_TT 3518 be,pn %xcc, 9f 3519 mov %g3, %g5 3520 3521 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3522 ! trapstat wants TTE in %g5 3523 retry 35249: 3525 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3526 ! trapstat wants TTE in %g5 3527 retry 3528 3529tsb_user4m: 3530#ifdef sun4v 3531 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3532 and %g7, HAT_CHKCTX1_FLAG, %g1 3533 brz,a,pn %g1, 4f 3534 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = TSB ptr 3535 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1) 3536 brlz,a,pn %g1, 5f ! if no shared 2nd tsb 3537 nop 3538 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 3539#else 3540 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = TSB ptr 3541#endif 35424: 3543 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3544 nop 3545 3546#ifndef UTSB_PHYS 3547 mov ASI_N, %g7 ! user TSBs accessed by VA 3548 mov %g7, %asi 3549#endif /* UTSB_PHYS */ 3550 3551 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6) 3552 35535: 3554#ifdef sun4v 3555 rdpr %tt, %g5 3556 cmp %g5, T_INSTR_MMU_MISS 3557 be,a,pn %xcc, 9f 3558 mov %g3, %g5 3559#endif /* sun4v */ 3560 cmp %g5, FAST_IMMU_MISS_TT 3561 be,pn %xcc, 9f 3562 mov %g3, %g5 3563 3564 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3565 ! trapstat wants TTE in %g5 3566 retry 35679: 3568 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3569 ! trapstat wants TTE in %g5 3570 retry 3571 3572#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3573 /* 3574 * Panther ITLB synthesis. 3575 * The Panther 32M and 256M ITLB code simulates these two large page 3576 * sizes with 4M pages, to provide support for programs, for example 3577 * Java, that may copy instructions into a 32M or 256M data page and 3578 * then execute them. The code below generates the 4M pfn bits and 3579 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3580 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3581 * are ignored by the hardware. 3582 * 3583 * Now, load into TSB/TLB. At this point: 3584 * g2 = tagtarget 3585 * g3 = tte 3586 * g4 = patte 3587 * g5 = tt 3588 * g6 = tsbmiss area 3589 */ 3590tsb_user_pn_synth: 3591 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3592 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3593 bz,pn %icc, 4b /* if not, been here before */ 3594 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3595 brlz,a,pn %g1, 5f /* no 2nd tsb */ 3596 mov %g3, %g5 3597 3598 mov MMU_TAG_ACCESS, %g7 3599 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3600 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3601 3602 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3603 mov %g7, %asi 3604 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */ 36055: 3606 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3607 retry 3608 3609tsb_user_itlb_synth: 3610 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3611 3612 mov MMU_TAG_ACCESS, %g7 3613 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3614 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3615 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3616 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3617 3618 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3619 mov %g7, %asi 3620 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */ 36217: 3622 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3623 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3624 retry 3625#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3626 3627tsb_kernel: 3628#ifdef sun4v 3629 rdpr %tt, %g5 3630 cmp %g7, TTE4M 3631 bge,pn %icc, 5f 3632#else 3633 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3634 be,pn %icc, 5f 3635#endif 3636 nop 3637 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8k tsbptr 3638 ba,pt %xcc, 6f 3639 nop 36405: 3641 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4m tsbptr 3642 brlz,pn %g1, 3f /* skip programming if 4m TSB ptr is -1 */ 3643 nop 36446: 3645#ifndef sun4v 3646tsb_kernel_patch_asi: 3647 or %g0, RUNTIME_PATCH, %g6 3648 mov %g6, %asi ! XXX avoid writing to %asi !! 3649#endif 3650 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7) 36513: 3652#ifdef sun4v 3653 cmp %g5, T_INSTR_MMU_MISS 3654 be,a,pn %icc, 1f 3655 mov %g3, %g5 ! trapstat wants TTE in %g5 3656#endif /* sun4v */ 3657 cmp %g5, FAST_IMMU_MISS_TT 3658 be,pn %icc, 1f 3659 mov %g3, %g5 ! trapstat wants TTE in %g5 3660 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3661 ! trapstat wants TTE in %g5 3662 retry 36631: 3664 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3665 ! trapstat wants TTE in %g5 3666 retry 3667 3668tsb_ism: 3669 /* 3670 * This is an ISM [i|d]tlb miss. We optimize for largest 3671 * page size down to smallest. 3672 * 3673 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3674 * register 3675 * g3 = ismmap->ism_seg 3676 * g4 = physical address of ismmap->ism_sfmmu 3677 * g6 = tsbmiss area 3678 */ 3679 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3680 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3681 mov PTL1_BAD_ISM, %g1 3682 /* g5 = pa of imap_vb_shift */ 3683 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3684 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3685 srlx %g3, %g4, %g3 /* clr size field */ 3686 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3687 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3688 and %g2, %g1, %g4 /* g4 = ctx number */ 3689 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3690 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3691 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3692 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3693 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3694#ifdef sun4v 3695 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3696 brz,pt %g5, tsb_chk4M_ism 3697 nop 3698 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3699 or %g5, HAT_CHKCTX1_FLAG, %g5 3700 stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3701 rdpr %tt, %g5 3702 SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl) 3703#endif 3704 /* 3705 * ISM pages are always locked down. 3706 * If we can't find the tte then pagefault 3707 * and let the spt segment driver resolve it. 3708 * 3709 * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3710 * g4 = imap_hatflags 3711 * g6 = tsb miss area 3712 * g7 = ISM hatid 3713 */ 3714 3715tsb_chk4M_ism: 3716 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3717 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3718 nop 3719 3720tsb_ism_32M: 3721 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3722 brz,pn %g5, tsb_ism_256M 3723 nop 3724 3725 /* 3726 * 32M hash. 3727 */ 3728 3729 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M, 3730 TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3731 tsb_ism_4M) 3732 /* NOT REACHED */ 3733 3734tsb_ism_32M_found: 3735 brlz,a,pt %g3, tsb_validtte 3736 rdpr %tt, %g7 3737 ba,pt %xcc, tsb_ism_4M 3738 nop 3739 3740tsb_ism_256M: 3741 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3742 brz,a,pn %g5, ptl1_panic 3743 mov PTL1_BAD_ISM, %g1 3744 3745 /* 3746 * 256M hash. 3747 */ 3748 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M, 3749 TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3750 tsb_ism_4M) 3751 3752tsb_ism_256M_found: 3753 brlz,a,pt %g3, tsb_validtte 3754 rdpr %tt, %g7 3755 3756tsb_ism_4M: 3757 /* 3758 * 4M hash. 3759 */ 3760 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M, 3761 TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3762 tsb_ism_8K) 3763 /* NOT REACHED */ 3764 3765tsb_ism_4M_found: 3766 brlz,a,pt %g3, tsb_validtte 3767 rdpr %tt, %g7 3768 3769tsb_ism_8K: 3770 /* 3771 * 8K and 64K hash. 3772 */ 3773 3774 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K, 3775 TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3776 tsb_pagefault) 3777 /* NOT REACHED */ 3778 3779tsb_ism_8K_found: 3780 brlz,a,pt %g3, tsb_validtte 3781 rdpr %tt, %g7 3782 3783tsb_pagefault: 3784 rdpr %tt, %g7 3785 cmp %g7, FAST_PROT_TT 3786 be,a,pn %icc, tsb_protfault 3787 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3788 3789tsb_protfault: 3790 /* 3791 * we get here if we couldn't find a valid tte in the hash. 3792 * 3793 * If user and we are at tl>1 we go to window handling code. 3794 * 3795 * If kernel and the fault is on the same page as our stack 3796 * pointer, then we know the stack is bad and the trap handler 3797 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3798 * 3799 * If this is a kernel trap and tl>1, panic. 3800 * 3801 * Otherwise we call pagefault. 3802 */ 3803 cmp %g7, FAST_IMMU_MISS_TT 3804#ifdef sun4v 3805 MMU_FAULT_STATUS_AREA(%g4) 3806 ldx [%g4 + MMFSA_I_CTX], %g5 3807 ldx [%g4 + MMFSA_D_CTX], %g4 3808 move %icc, %g5, %g4 3809 cmp %g7, T_INSTR_MMU_MISS 3810 move %icc, %g5, %g4 3811#else 3812 mov MMU_TAG_ACCESS, %g4 3813 ldxa [%g4]ASI_DMMU, %g2 3814 ldxa [%g4]ASI_IMMU, %g5 3815 move %icc, %g5, %g2 3816 cmp %g7, T_INSTR_MMU_MISS 3817 move %icc, %g5, %g2 3818 sllx %g2, TAGACC_CTX_LSHIFT, %g4 3819#endif 3820 brnz,pn %g4, 3f /* skip if not kernel */ 3821 rdpr %tl, %g5 3822 3823 add %sp, STACK_BIAS, %g3 3824 srlx %g3, MMU_PAGESHIFT, %g3 3825 srlx %g2, MMU_PAGESHIFT, %g4 3826 cmp %g3, %g4 3827 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 3828 mov PTL1_BAD_STACK, %g1 3829 3830 cmp %g5, 1 3831 ble,pt %icc, 2f 3832 nop 3833 TSTAT_CHECK_TL1(2f, %g1, %g2) 3834 rdpr %tt, %g2 3835 cmp %g2, FAST_PROT_TT 3836 mov PTL1_BAD_KPROT_FAULT, %g1 3837 movne %icc, PTL1_BAD_KMISS, %g1 3838 ba,pt %icc, ptl1_panic 3839 nop 3840 38412: 3842 /* 3843 * We are taking a pagefault in the kernel on a kernel address. If 3844 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 3845 * want to call sfmmu_pagefault -- we will instead note that a fault 3846 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 3847 * (instead of a "retry"). This will step over the faulting 3848 * instruction. 3849 */ 3850 CPU_INDEX(%g1, %g2) 3851 set cpu_core, %g2 3852 sllx %g1, CPU_CORE_SHIFT, %g1 3853 add %g1, %g2, %g1 3854 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3855 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3856 bz sfmmu_pagefault 3857 or %g2, CPU_DTRACE_BADADDR, %g2 3858 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3859 GET_MMU_D_ADDR(%g3, %g4) 3860 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3861 done 3862 38633: 3864 cmp %g5, 1 3865 ble,pt %icc, 4f 3866 nop 3867 TSTAT_CHECK_TL1(4f, %g1, %g2) 3868 ba,pt %icc, sfmmu_window_trap 3869 nop 3870 38714: 3872 /* 3873 * We are taking a pagefault on a non-kernel address. If we are in 3874 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 3875 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 3876 */ 3877 CPU_INDEX(%g1, %g2) 3878 set cpu_core, %g2 3879 sllx %g1, CPU_CORE_SHIFT, %g1 3880 add %g1, %g2, %g1 3881 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3882 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3883 bz sfmmu_mmu_trap 3884 or %g2, CPU_DTRACE_BADADDR, %g2 3885 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3886 GET_MMU_D_ADDR(%g3, %g4) 3887 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3888 3889 /* 3890 * Be sure that we're actually taking this miss from the kernel -- 3891 * otherwise we have managed to return to user-level with 3892 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3893 */ 3894 rdpr %tstate, %g2 3895 btst TSTATE_PRIV, %g2 3896 bz,a ptl1_panic 3897 mov PTL1_BAD_DTRACE_FLAGS, %g1 3898 done 3899 3900 ALTENTRY(tsb_tl0_noctxt) 3901 /* 3902 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 3903 * if it is, indicated that we have faulted and issue a done. 3904 */ 3905 CPU_INDEX(%g5, %g6) 3906 set cpu_core, %g6 3907 sllx %g5, CPU_CORE_SHIFT, %g5 3908 add %g5, %g6, %g5 3909 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 3910 andcc %g6, CPU_DTRACE_NOFAULT, %g0 3911 bz 1f 3912 or %g6, CPU_DTRACE_BADADDR, %g6 3913 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 3914 GET_MMU_D_ADDR(%g3, %g4) 3915 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 3916 3917 /* 3918 * Be sure that we're actually taking this miss from the kernel -- 3919 * otherwise we have managed to return to user-level with 3920 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3921 */ 3922 rdpr %tstate, %g5 3923 btst TSTATE_PRIV, %g5 3924 bz,a ptl1_panic 3925 mov PTL1_BAD_DTRACE_FLAGS, %g1 3926 TSTAT_CHECK_TL1(2f, %g1, %g2); 39272: 3928 done 3929 39301: 3931 rdpr %tt, %g5 3932 cmp %g5, FAST_IMMU_MISS_TT 3933#ifdef sun4v 3934 MMU_FAULT_STATUS_AREA(%g2) 3935 be,a,pt %icc, 2f 3936 ldx [%g2 + MMFSA_I_CTX], %g3 3937 cmp %g5, T_INSTR_MMU_MISS 3938 be,a,pt %icc, 2f 3939 ldx [%g2 + MMFSA_I_CTX], %g3 3940 ldx [%g2 + MMFSA_D_CTX], %g3 39412: 3942#else 3943 mov MMU_TAG_ACCESS, %g2 3944 be,a,pt %icc, 2f 3945 ldxa [%g2]ASI_IMMU, %g3 3946 ldxa [%g2]ASI_DMMU, %g3 39472: sllx %g3, TAGACC_CTX_LSHIFT, %g3 3948#endif 3949 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 3950 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 3951 rdpr %tl, %g5 3952 cmp %g5, 1 3953 ble,pt %icc, sfmmu_mmu_trap 3954 nop 3955 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 3956 ba,pt %icc, sfmmu_window_trap 3957 nop 3958 SET_SIZE(sfmmu_tsb_miss) 3959#endif /* lint */ 3960 3961#if defined (lint) 3962/* 3963 * This routine will look for a user or kernel vaddr in the hash 3964 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 3965 * grab any locks. It should only be used by other sfmmu routines. 3966 */ 3967/* ARGSUSED */ 3968pfn_t 3969sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 3970{ 3971 return(0); 3972} 3973 3974/* ARGSUSED */ 3975pfn_t 3976sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno) 3977{ 3978 return(0); 3979} 3980 3981#else /* lint */ 3982 3983 ENTRY_NP(sfmmu_vatopfn) 3984 /* 3985 * disable interrupts 3986 */ 3987 rdpr %pstate, %o3 3988#ifdef DEBUG 3989 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 3990#endif 3991 /* 3992 * disable interrupts to protect the TSBMISS area 3993 */ 3994 andn %o3, PSTATE_IE, %o5 3995 wrpr %o5, 0, %pstate 3996 3997 /* 3998 * o0 = vaddr 3999 * o1 = sfmmup 4000 * o2 = ttep 4001 */ 4002 CPU_TSBMISS_AREA(%g1, %o5) 4003 ldn [%g1 + TSBMISS_KHATID], %o4 4004 cmp %o4, %o1 4005 bne,pn %ncc, vatopfn_nokernel 4006 mov TTE64K, %g5 /* g5 = rehash # */ 4007 mov %g1,%o5 /* o5 = tsbmiss_area */ 4008 /* 4009 * o0 = vaddr 4010 * o1 & o4 = hatid 4011 * o2 = ttep 4012 * o5 = tsbmiss area 4013 */ 4014 mov HBLK_RANGE_SHIFT, %g6 40151: 4016 4017 /* 4018 * o0 = vaddr 4019 * o1 = sfmmup 4020 * o2 = ttep 4021 * o3 = old %pstate 4022 * o4 = hatid 4023 * o5 = tsbmiss 4024 * g5 = rehash # 4025 * g6 = hmeshift 4026 * 4027 * The first arg to GET_TTE is actually tagaccess register 4028 * not just vaddr. Since this call is for kernel we need to clear 4029 * any lower vaddr bits that would be interpreted as ctx bits. 4030 */ 4031 set TAGACC_CTX_MASK, %g1 4032 andn %o0, %g1, %o0 4033 GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5, 4034 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 4035 4036kvtop_hblk_found: 4037 /* 4038 * o0 = vaddr 4039 * o1 = sfmmup 4040 * o2 = ttep 4041 * g1 = tte 4042 * g2 = tte pa 4043 * g3 = tte va 4044 * o2 = tsbmiss area 4045 * o1 = hat id 4046 */ 4047 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 4048 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4049 stx %g1,[%o2] /* put tte into *ttep */ 4050 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 4051 /* 4052 * o0 = vaddr 4053 * o1 = sfmmup 4054 * o2 = ttep 4055 * g1 = pfn 4056 */ 4057 ba,pt %xcc, 6f 4058 mov %g1, %o0 4059 4060kvtop_nohblk: 4061 /* 4062 * we get here if we couldn't find valid hblk in hash. We rehash 4063 * if neccesary. 4064 */ 4065 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 4066#ifdef sun4v 4067 cmp %g5, MAX_HASHCNT 4068#else 4069 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 4070#endif 4071 be,a,pn %icc, 6f 4072 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4073 mov %o1, %o4 /* restore hatid */ 4074#ifdef sun4v 4075 add %g5, 2, %g5 4076 cmp %g5, 3 4077 move %icc, MMU_PAGESHIFT4M, %g6 4078 ba,pt %icc, 1b 4079 movne %icc, MMU_PAGESHIFT256M, %g6 4080#else 4081 inc %g5 4082 cmp %g5, 2 4083 move %icc, MMU_PAGESHIFT512K, %g6 4084 ba,pt %icc, 1b 4085 movne %icc, MMU_PAGESHIFT4M, %g6 4086#endif 40876: 4088 retl 4089 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4090 4091tsb_suspend: 4092 /* 4093 * o0 = vaddr 4094 * o1 = sfmmup 4095 * o2 = ttep 4096 * g1 = tte 4097 * g2 = tte pa 4098 * g3 = tte va 4099 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 4100 */ 4101 stx %g1,[%o2] /* put tte into *ttep */ 4102 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 4103 sub %g0, 1, %o0 /* output = PFN_INVALID */ 4104 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 41058: 4106 retl 4107 wrpr %g0, %o3, %pstate /* enable interrupts */ 4108 4109vatopfn_nokernel: 4110 /* 4111 * This routine does NOT support user addresses 4112 * There is a routine in C that supports this. 4113 * The only reason why we don't have the C routine 4114 * support kernel addresses as well is because 4115 * we do va_to_pa while holding the hashlock. 4116 */ 4117 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4118 save %sp, -SA(MINFRAME), %sp 4119 sethi %hi(sfmmu_panic3), %o0 4120 call panic 4121 or %o0, %lo(sfmmu_panic3), %o0 4122 4123 SET_SIZE(sfmmu_vatopfn) 4124 4125 /* 4126 * %o0 = vaddr 4127 * %o1 = hashno (aka szc) 4128 * 4129 * 4130 * This routine is similar to sfmmu_vatopfn() but will only look for 4131 * a kernel vaddr in the hash structure for the specified rehash value. 4132 * It's just an optimization for the case when pagesize for a given 4133 * va range is already known (e.g. large page heap) and we don't want 4134 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 4135 * 4136 * Returns valid pfn or PFN_INVALID if 4137 * tte for specified rehash # is not found, invalid or suspended. 4138 */ 4139 ENTRY_NP(sfmmu_kvaszc2pfn) 4140 /* 4141 * disable interrupts 4142 */ 4143 rdpr %pstate, %o3 4144#ifdef DEBUG 4145 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 4146#endif 4147 /* 4148 * disable interrupts to protect the TSBMISS area 4149 */ 4150 andn %o3, PSTATE_IE, %o5 4151 wrpr %o5, 0, %pstate 4152 4153 CPU_TSBMISS_AREA(%g1, %o5) 4154 ldn [%g1 + TSBMISS_KHATID], %o4 4155 sll %o1, 1, %g6 4156 add %g6, %o1, %g6 4157 add %g6, MMU_PAGESHIFT, %g6 4158 /* 4159 * %o0 = vaddr 4160 * %o1 = hashno 4161 * %o3 = old %pstate 4162 * %o4 = ksfmmup 4163 * %g1 = tsbmiss area 4164 * %g6 = hmeshift 4165 */ 4166 4167 /* 4168 * The first arg to GET_TTE is actually tagaccess register 4169 * not just vaddr. Since this call is for kernel we need to clear 4170 * any lower vaddr bits that would be interpreted as ctx bits. 4171 */ 4172 srlx %o0, MMU_PAGESHIFT, %o0 4173 sllx %o0, MMU_PAGESHIFT, %o0 4174 GET_TTE(%o0, %o4, %g3, %g4, %g5, %g1, %o5, %g6, %o1, 4175 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 4176 kvaszc2pfn_nohblk) 4177 4178kvaszc2pfn_hblk_found: 4179 /* 4180 * %g3 = tte 4181 * %o0 = vaddr 4182 */ 4183 brgez,a,pn %g3, 1f /* check if tte is invalid */ 4184 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4185 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 4186 /* 4187 * g3 = pfn 4188 */ 4189 ba,pt %xcc, 1f 4190 mov %g3, %o0 4191 4192kvaszc2pfn_nohblk: 4193 mov -1, %o0 4194 41951: 4196 retl 4197 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4198 4199 SET_SIZE(sfmmu_kvaszc2pfn) 4200 4201#endif /* lint */ 4202 4203 4204 4205#if !defined(lint) 4206 4207/* 4208 * kpm lock used between trap level tsbmiss handler and kpm C level. 4209 */ 4210#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 4211 mov 0xff, tmp1 ;\ 4212label1: ;\ 4213 casa [kpmlckp]asi, %g0, tmp1 ;\ 4214 brnz,pn tmp1, label1 ;\ 4215 mov 0xff, tmp1 ;\ 4216 membar #LoadLoad 4217 4218#define KPMLOCK_EXIT(kpmlckp, asi) \ 4219 membar #LoadStore|#StoreStore ;\ 4220 sta %g0, [kpmlckp]asi 4221 4222/* 4223 * Lookup a memseg for a given pfn and if found, return the physical 4224 * address of the corresponding struct memseg in mseg, otherwise 4225 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 4226 * tsbmp, %asi is assumed to be ASI_MEM. 4227 * This lookup is done by strictly traversing only the physical memseg 4228 * linkage. The more generic approach, to check the virtual linkage 4229 * before using the physical (used e.g. with hmehash buckets), cannot 4230 * be used here. Memory DR operations can run in parallel to this 4231 * lookup w/o any locks and updates of the physical and virtual linkage 4232 * cannot be done atomically wrt. to each other. Because physical 4233 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 4234 * as "physical NULL" pointer. 4235 */ 4236#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 4237 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 4238 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 4239 udivx pfn, mseg, mseg ;\ 4240 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 4241 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 4242 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 4243 add tmp1, mseg, tmp1 ;\ 4244 ldxa [tmp1]%asi, mseg ;\ 4245 cmp mseg, MSEG_NULLPTR_PA ;\ 4246 be,pn %xcc, label/**/1 /* if not found */ ;\ 4247 nop ;\ 4248 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4249 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4250 blu,pn %xcc, label/**/1 ;\ 4251 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4252 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4253 bgeu,pn %xcc, label/**/1 ;\ 4254 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 4255 mulx tmp1, PAGE_SIZE, tmp1 ;\ 4256 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 4257 add tmp2, tmp1, tmp1 /* pp */ ;\ 4258 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 4259 cmp tmp2, pfn ;\ 4260 be,pt %xcc, label/**/_ok /* found */ ;\ 4261label/**/1: ;\ 4262 /* brute force lookup */ ;\ 4263 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 4264 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 4265label/**/2: ;\ 4266 cmp mseg, MSEG_NULLPTR_PA ;\ 4267 be,pn %xcc, label/**/_ok /* if not found */ ;\ 4268 nop ;\ 4269 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4270 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4271 blu,a,pt %xcc, label/**/2 ;\ 4272 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4273 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4274 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4275 bgeu,a,pt %xcc, label/**/2 ;\ 4276 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4277label/**/_ok: 4278 4279 /* 4280 * kpm tsb miss handler large pages 4281 * g1 = 8K kpm TSB entry pointer 4282 * g2 = tag access register 4283 * g3 = 4M kpm TSB entry pointer 4284 */ 4285 ALTENTRY(sfmmu_kpm_dtsb_miss) 4286 TT_TRACE(trace_tsbmiss) 4287 4288 CPU_INDEX(%g7, %g6) 4289 sethi %hi(kpmtsbm_area), %g6 4290 sllx %g7, KPMTSBM_SHIFT, %g7 4291 or %g6, %lo(kpmtsbm_area), %g6 4292 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4293 4294 /* check enable flag */ 4295 ldub [%g6 + KPMTSBM_FLAGS], %g4 4296 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4297 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4298 nop 4299 4300 /* VA range check */ 4301 ldx [%g6 + KPMTSBM_VBASE], %g7 4302 cmp %g2, %g7 4303 blu,pn %xcc, sfmmu_tsb_miss 4304 ldx [%g6 + KPMTSBM_VEND], %g5 4305 cmp %g2, %g5 4306 bgeu,pn %xcc, sfmmu_tsb_miss 4307 stx %g3, [%g6 + KPMTSBM_TSBPTR] 4308 4309 /* 4310 * check TL tsbmiss handling flag 4311 * bump tsbmiss counter 4312 */ 4313 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4314#ifdef DEBUG 4315 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4316 inc %g5 4317 brz,pn %g3, sfmmu_kpm_exception 4318 st %g5, [%g6 + KPMTSBM_TSBMISS] 4319#else 4320 inc %g5 4321 st %g5, [%g6 + KPMTSBM_TSBMISS] 4322#endif 4323 /* 4324 * At this point: 4325 * g1 = 8K kpm TSB pointer (not used) 4326 * g2 = tag access register 4327 * g3 = clobbered 4328 * g6 = per-CPU kpm tsbmiss area 4329 * g7 = kpm_vbase 4330 */ 4331 4332 /* vaddr2pfn */ 4333 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4334 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4335 srax %g4, %g3, %g2 /* which alias range (r) */ 4336 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4337 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4338 4339 /* 4340 * Setup %asi 4341 * mseg_pa = page_numtomemseg_nolock(pfn) 4342 * if (mseg_pa == NULL) sfmmu_kpm_exception 4343 * g2=pfn 4344 */ 4345 mov ASI_MEM, %asi 4346 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4347 cmp %g3, MSEG_NULLPTR_PA 4348 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4349 nop 4350 4351 /* 4352 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4353 * g2=pfn g3=mseg_pa 4354 */ 4355 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4356 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4357 srlx %g2, %g5, %g4 4358 sllx %g4, %g5, %g4 4359 sub %g4, %g7, %g4 4360 srlx %g4, %g5, %g4 4361 4362 /* 4363 * Validate inx value 4364 * g2=pfn g3=mseg_pa g4=inx 4365 */ 4366#ifdef DEBUG 4367 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4368 cmp %g4, %g5 /* inx - nkpmpgs */ 4369 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4370 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4371#else 4372 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4373#endif 4374 /* 4375 * kp = &mseg_pa->kpm_pages[inx] 4376 */ 4377 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4378 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4379 add %g5, %g4, %g5 /* kp */ 4380 4381 /* 4382 * KPMP_HASH(kp) 4383 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4384 */ 4385 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4386 sub %g7, 1, %g7 /* mask */ 4387 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4388 add %g5, %g1, %g5 /* y = ksp + x */ 4389 and %g5, %g7, %g5 /* hashinx = y & mask */ 4390 4391 /* 4392 * Calculate physical kpm_page pointer 4393 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4394 */ 4395 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4396 add %g1, %g4, %g1 /* kp_pa */ 4397 4398 /* 4399 * Calculate physical hash lock address 4400 * g1=kp_refcntc_pa g2=pfn g5=hashinx 4401 */ 4402 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4403 sllx %g5, KPMHLK_SHIFT, %g5 4404 add %g4, %g5, %g3 4405 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4406 4407 /* 4408 * Assemble tte 4409 * g1=kp_pa g2=pfn g3=hlck_pa 4410 */ 4411#ifdef sun4v 4412 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4413 sllx %g5, 32, %g5 4414 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4415 or %g4, TTE4M, %g4 4416 or %g5, %g4, %g5 4417#else 4418 sethi %hi(TTE_VALID_INT), %g4 4419 mov TTE4M, %g5 4420 sllx %g5, TTE_SZ_SHFT_INT, %g5 4421 or %g5, %g4, %g5 /* upper part */ 4422 sllx %g5, 32, %g5 4423 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4424 or %g5, %g4, %g5 4425#endif 4426 sllx %g2, MMU_PAGESHIFT, %g4 4427 or %g5, %g4, %g5 /* tte */ 4428 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4429 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4430 4431 /* 4432 * tsb dropin 4433 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4434 */ 4435 4436 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4437 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4438 4439 /* use C-handler if there's no go for dropin */ 4440 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4441 cmp %g7, -1 4442 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4443 nop 4444 4445#ifdef DEBUG 4446 /* double check refcnt */ 4447 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4448 brz,pn %g7, 5f /* let C-handler deal with this */ 4449 nop 4450#endif 4451 4452#ifndef sun4v 4453 ldub [%g6 + KPMTSBM_FLAGS], %g7 4454 mov ASI_N, %g1 4455 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4456 movnz %icc, ASI_MEM, %g1 4457 mov %g1, %asi 4458#endif 4459 4460 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4461 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4462 4463 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4464 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4465 4466 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4467 4468 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4469 KPMLOCK_EXIT(%g3, ASI_MEM) 4470 4471 /* 4472 * If trapstat is running, we need to shift the %tpc and %tnpc to 4473 * point to trapstat's TSB miss return code (note that trapstat 4474 * itself will patch the correct offset to add). 4475 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4476 */ 4477 rdpr %tl, %g7 4478 cmp %g7, 1 4479 ble %icc, 0f 4480 sethi %hi(KERNELBASE), %g6 4481 rdpr %tpc, %g7 4482 or %g6, %lo(KERNELBASE), %g6 4483 cmp %g7, %g6 4484 bgeu %xcc, 0f 4485 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4486 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4487 wrpr %g7, %tpc 4488 add %g7, 4, %g7 4489 wrpr %g7, %tnpc 44900: 4491 retry 44925: 4493 /* g3=hlck_pa */ 4494 KPMLOCK_EXIT(%g3, ASI_MEM) 4495 ba,pt %icc, sfmmu_kpm_exception 4496 nop 4497 SET_SIZE(sfmmu_kpm_dtsb_miss) 4498 4499 /* 4500 * kpm tsbmiss handler for smallpages 4501 * g1 = 8K kpm TSB pointer 4502 * g2 = tag access register 4503 * g3 = 4M kpm TSB pointer 4504 */ 4505 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4506 TT_TRACE(trace_tsbmiss) 4507 CPU_INDEX(%g7, %g6) 4508 sethi %hi(kpmtsbm_area), %g6 4509 sllx %g7, KPMTSBM_SHIFT, %g7 4510 or %g6, %lo(kpmtsbm_area), %g6 4511 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4512 4513 /* check enable flag */ 4514 ldub [%g6 + KPMTSBM_FLAGS], %g4 4515 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4516 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4517 nop 4518 4519 /* 4520 * VA range check 4521 * On fail: goto sfmmu_tsb_miss 4522 */ 4523 ldx [%g6 + KPMTSBM_VBASE], %g7 4524 cmp %g2, %g7 4525 blu,pn %xcc, sfmmu_tsb_miss 4526 ldx [%g6 + KPMTSBM_VEND], %g5 4527 cmp %g2, %g5 4528 bgeu,pn %xcc, sfmmu_tsb_miss 4529 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4530 4531 /* 4532 * check TL tsbmiss handling flag 4533 * bump tsbmiss counter 4534 */ 4535 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4536#ifdef DEBUG 4537 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4538 inc %g5 4539 brz,pn %g1, sfmmu_kpm_exception 4540 st %g5, [%g6 + KPMTSBM_TSBMISS] 4541#else 4542 inc %g5 4543 st %g5, [%g6 + KPMTSBM_TSBMISS] 4544#endif 4545 /* 4546 * At this point: 4547 * g1 = clobbered 4548 * g2 = tag access register 4549 * g3 = 4M kpm TSB pointer (not used) 4550 * g6 = per-CPU kpm tsbmiss area 4551 * g7 = kpm_vbase 4552 */ 4553 4554 /* vaddr2pfn */ 4555 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4556 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4557 srax %g4, %g3, %g2 /* which alias range (r) */ 4558 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4559 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4560 4561 /* 4562 * Setup %asi 4563 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4564 * if (mseg not found) sfmmu_kpm_exception 4565 * g2=pfn 4566 */ 4567 mov ASI_MEM, %asi 4568 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4569 cmp %g3, MSEG_NULLPTR_PA 4570 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4571 nop 4572 4573 /* 4574 * inx = pfn - mseg_pa->kpm_pbase 4575 * g2=pfn g3=mseg_pa 4576 */ 4577 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4578 sub %g2, %g7, %g4 4579 4580#ifdef DEBUG 4581 /* 4582 * Validate inx value 4583 * g2=pfn g3=mseg_pa g4=inx 4584 */ 4585 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4586 cmp %g4, %g5 /* inx - nkpmpgs */ 4587 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4588 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4589#else 4590 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4591#endif 4592 /* ksp = &mseg_pa->kpm_spages[inx] */ 4593 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4594 add %g5, %g4, %g5 /* ksp */ 4595 4596 /* 4597 * KPMP_SHASH(kp) 4598 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz 4599 */ 4600 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4601 sub %g7, 1, %g7 /* mask */ 4602 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4603 add %g5, %g1, %g5 /* y = ksp + x */ 4604 and %g5, %g7, %g5 /* hashinx = y & mask */ 4605 4606 /* 4607 * Calculate physical kpm_spage pointer 4608 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4609 */ 4610 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4611 add %g1, %g4, %g1 /* ksp_pa */ 4612 4613 /* 4614 * Calculate physical hash lock address. 4615 * Note: Changes in kpm_shlk_t must be reflected here. 4616 * g1=ksp_pa g2=pfn g5=hashinx 4617 */ 4618 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4619 sllx %g5, KPMSHLK_SHIFT, %g5 4620 add %g4, %g5, %g3 /* hlck_pa */ 4621 4622 /* 4623 * Assemble tte 4624 * g1=ksp_pa g2=pfn g3=hlck_pa 4625 */ 4626 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4627 sllx %g5, 32, %g5 4628 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4629 or %g5, %g4, %g5 4630 sllx %g2, MMU_PAGESHIFT, %g4 4631 or %g5, %g4, %g5 /* tte */ 4632 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4633 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4634 4635 /* 4636 * tsb dropin 4637 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte 4638 */ 4639 4640 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4641 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4642 4643 /* use C-handler if there's no go for dropin */ 4644 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4645 cmp %g7, -1 4646 bne,pn %xcc, 5f 4647 nop 4648 4649#ifndef sun4v 4650 ldub [%g6 + KPMTSBM_FLAGS], %g7 4651 mov ASI_N, %g1 4652 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4653 movnz %icc, ASI_MEM, %g1 4654 mov %g1, %asi 4655#endif 4656 4657 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4658 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4659 4660 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4661 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4662 4663 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4664 4665 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4666 KPMLOCK_EXIT(%g3, ASI_MEM) 4667 4668 /* 4669 * If trapstat is running, we need to shift the %tpc and %tnpc to 4670 * point to trapstat's TSB miss return code (note that trapstat 4671 * itself will patch the correct offset to add). 4672 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4673 */ 4674 rdpr %tl, %g7 4675 cmp %g7, 1 4676 ble %icc, 0f 4677 sethi %hi(KERNELBASE), %g6 4678 rdpr %tpc, %g7 4679 or %g6, %lo(KERNELBASE), %g6 4680 cmp %g7, %g6 4681 bgeu %xcc, 0f 4682 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4683 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4684 wrpr %g7, %tpc 4685 add %g7, 4, %g7 4686 wrpr %g7, %tnpc 46870: 4688 retry 46895: 4690 /* g3=hlck_pa */ 4691 KPMLOCK_EXIT(%g3, ASI_MEM) 4692 ba,pt %icc, sfmmu_kpm_exception 4693 nop 4694 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4695 4696#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4697#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4698#endif 4699 4700#endif /* lint */ 4701 4702#ifdef lint 4703/* 4704 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4705 * Called from C-level, sets/clears "go" indication for trap level handler. 4706 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 4707 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 4708 * Assumes khl_mutex is held when called from C-level. 4709 */ 4710/* ARGSUSED */ 4711void 4712sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 4713{ 4714} 4715 4716/* 4717 * kpm_smallpages: stores val to byte at address mapped within 4718 * low level lock brackets. The old value is returned. 4719 * Called from C-level. 4720 */ 4721/* ARGSUSED */ 4722int 4723sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val) 4724{ 4725 return (0); 4726} 4727 4728#else /* lint */ 4729 4730 .seg ".data" 4731sfmmu_kpm_tsbmtl_panic: 4732 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4733 .byte 0 4734sfmmu_kpm_stsbmtl_panic: 4735 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4736 .byte 0 4737 .align 4 4738 .seg ".text" 4739 4740 ENTRY_NP(sfmmu_kpm_tsbmtl) 4741 rdpr %pstate, %o3 4742 /* 4743 * %o0 = &kp_refcntc 4744 * %o1 = &khl_lock 4745 * %o2 = 0/1 (off/on) 4746 * %o3 = pstate save 4747 */ 4748#ifdef DEBUG 4749 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4750 bnz,pt %icc, 1f /* disabled, panic */ 4751 nop 4752 save %sp, -SA(MINFRAME), %sp 4753 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4754 call panic 4755 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4756 ret 4757 restore 47581: 4759#endif /* DEBUG */ 4760 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4761 4762 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4763 mov -1, %o5 4764 brz,a %o2, 2f 4765 mov 0, %o5 47662: 4767 sth %o5, [%o0] 4768 KPMLOCK_EXIT(%o1, ASI_N) 4769 4770 retl 4771 wrpr %g0, %o3, %pstate /* enable interrupts */ 4772 SET_SIZE(sfmmu_kpm_tsbmtl) 4773 4774 ENTRY_NP(sfmmu_kpm_stsbmtl) 4775 rdpr %pstate, %o3 4776 /* 4777 * %o0 = &mapped 4778 * %o1 = &kshl_lock 4779 * %o2 = val 4780 * %o3 = pstate save 4781 */ 4782#ifdef DEBUG 4783 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4784 bnz,pt %icc, 1f /* disabled, panic */ 4785 nop 4786 save %sp, -SA(MINFRAME), %sp 4787 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4788 call panic 4789 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4790 ret 4791 restore 47921: 4793#endif /* DEBUG */ 4794 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4795 4796 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4797 ldsb [%o0], %o5 4798 stb %o2, [%o0] 4799 KPMLOCK_EXIT(%o1, ASI_N) 4800 4801 mov %o5, %o0 /* return old val */ 4802 retl 4803 wrpr %g0, %o3, %pstate /* enable interrupts */ 4804 SET_SIZE(sfmmu_kpm_stsbmtl) 4805 4806#endif /* lint */ 4807 4808#ifndef lint 4809#ifdef sun4v 4810 /* 4811 * User/kernel data miss w// multiple TSBs 4812 * The first probe covers 8K, 64K, and 512K page sizes, 4813 * because 64K and 512K mappings are replicated off 8K 4814 * pointer. Second probe covers 4M page size only. 4815 * 4816 * MMU fault area contains miss address and context. 4817 */ 4818 ALTENTRY(sfmmu_slow_dmmu_miss) 4819 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 4820 4821slow_miss_common: 4822 /* 4823 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 4824 * %g3 = ctx (cannot be INVALID_CONTEXT) 4825 */ 4826 brnz,pt %g3, 8f ! check for user context 4827 nop 4828 4829 /* 4830 * Kernel miss 4831 * Get 8K and 4M TSB pointers in %g1 and %g3 and 4832 * branch to sfmmu_tsb_miss_tt to handle it. 4833 */ 4834 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4835sfmmu_dslow_patch_ktsb_base: 4836 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 4837sfmmu_dslow_patch_ktsb_szcode: 4838 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 4839 4840 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 4841 ! %g1 = First TSB entry pointer, as TSB miss handler expects 4842 4843 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4844sfmmu_dslow_patch_ktsb4m_base: 4845 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 4846sfmmu_dslow_patch_ktsb4m_szcode: 4847 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 4848 4849 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 4850 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 4851 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4852 .empty 4853 48548: 4855 /* 4856 * User miss 4857 * Get first TSB pointer in %g1 4858 * Get second TSB pointer (or NULL if no second TSB) in %g3 4859 * Branch to sfmmu_tsb_miss_tt to handle it 4860 */ 4861 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 4862 /* %g1 = first TSB entry ptr now, %g2 preserved */ 4863 4864 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 4865 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 4866 nop 4867 4868 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 4869 /* %g3 = second TSB entry ptr now, %g2 preserved */ 48709: 4871 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4872 .empty 4873 SET_SIZE(sfmmu_slow_dmmu_miss) 4874 4875 4876 /* 4877 * User/kernel instruction miss w/ multiple TSBs 4878 * The first probe covers 8K, 64K, and 512K page sizes, 4879 * because 64K and 512K mappings are replicated off 8K 4880 * pointer. Second probe covers 4M page size only. 4881 * 4882 * MMU fault area contains miss address and context. 4883 */ 4884 ALTENTRY(sfmmu_slow_immu_miss) 4885 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 4886 ba,a,pt %xcc, slow_miss_common 4887 SET_SIZE(sfmmu_slow_immu_miss) 4888 4889#endif /* sun4v */ 4890#endif /* lint */ 4891 4892#ifndef lint 4893 4894/* 4895 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 4896 */ 4897 .seg ".data" 4898 .align 64 4899 .global tsbmiss_area 4900tsbmiss_area: 4901 .skip (TSBMISS_SIZE * NCPU) 4902 4903 .align 64 4904 .global kpmtsbm_area 4905kpmtsbm_area: 4906 .skip (KPMTSBM_SIZE * NCPU) 4907#endif /* lint */ 4908