1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#pragma ident "%Z%%M% %I% %E% SMI" 27 28/* 29 * SFMMU primitives. These primitives should only be used by sfmmu 30 * routines. 31 */ 32 33#if defined(lint) 34#include <sys/types.h> 35#else /* lint */ 36#include "assym.h" 37#endif /* lint */ 38 39#include <sys/asm_linkage.h> 40#include <sys/machtrap.h> 41#include <sys/machasi.h> 42#include <sys/sun4asi.h> 43#include <sys/pte.h> 44#include <sys/mmu.h> 45#include <vm/hat_sfmmu.h> 46#include <vm/seg_spt.h> 47#include <sys/machparam.h> 48#include <sys/privregs.h> 49#include <sys/scb.h> 50#include <sys/intreg.h> 51#include <sys/machthread.h> 52#include <sys/intr.h> 53#include <sys/clock.h> 54#include <sys/trapstat.h> 55 56#ifdef TRAPTRACE 57#include <sys/traptrace.h> 58 59/* 60 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 61 */ 62#define TT_TRACE(label) \ 63 ba label ;\ 64 rd %pc, %g7 65#else 66 67#define TT_TRACE(label) 68 69#endif /* TRAPTRACE */ 70 71#ifndef lint 72 73#if (TTE_SUSPEND_SHIFT > 0) 74#define TTE_SUSPEND_INT_SHIFT(reg) \ 75 sllx reg, TTE_SUSPEND_SHIFT, reg 76#else 77#define TTE_SUSPEND_INT_SHIFT(reg) 78#endif 79 80#endif /* lint */ 81 82#ifndef lint 83 84/* 85 * Assumes TSBE_TAG is 0 86 * Assumes TSBE_INTHI is 0 87 * Assumes TSBREG.split is 0 88 */ 89 90#if TSBE_TAG != 0 91#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 92#endif 93 94#if TSBTAG_INTHI != 0 95#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 96#endif 97 98/* 99 * The following code assumes the tsb is not split. 100 * 101 * With TSBs no longer shared between processes, it's no longer 102 * necessary to hash the context bits into the tsb index to get 103 * tsb coloring; the new implementation treats the TSB as a 104 * direct-mapped, virtually-addressed cache. 105 * 106 * In: 107 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 108 * tsbbase = base address of TSB (clobbered) 109 * tagacc = tag access register (clobbered) 110 * szc = size code of TSB (ro) 111 * tmp = scratch reg 112 * Out: 113 * tsbbase = pointer to entry in TSB 114 */ 115#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 116 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 117 srlx tagacc, vpshift, tagacc ;\ 118 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 119 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 120 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 121 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 122 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 123 124/* 125 * When the kpm TSB is used it is assumed that it is direct mapped 126 * using (vaddr>>vpshift)%tsbsz as the index. 127 * 128 * Note that, for now, the kpm TSB and kernel TSB are the same for 129 * each mapping size. However that need not always be the case. If 130 * the trap handlers are updated to search a different TSB for kpm 131 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 132 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 133 * 134 * In: 135 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 136 * vaddr = virtual address (clobbered) 137 * tsbp, szc, tmp = scratch 138 * Out: 139 * tsbp = pointer to entry in TSB 140 */ 141#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 142 cmp vpshift, MMU_PAGESHIFT ;\ 143 bne,pn %icc, 1f /* branch if large case */ ;\ 144 sethi %hi(kpmsm_tsbsz), szc ;\ 145 sethi %hi(kpmsm_tsbbase), tsbp ;\ 146 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 147 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 148 ba,pt %icc, 2f ;\ 149 nop ;\ 1501: sethi %hi(kpm_tsbsz), szc ;\ 151 sethi %hi(kpm_tsbbase), tsbp ;\ 152 ld [szc + %lo(kpm_tsbsz)], szc ;\ 153 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1542: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 155 156/* 157 * Lock the TSBE at virtual address tsbep. 158 * 159 * tsbep = TSBE va (ro) 160 * tmp1, tmp2 = scratch registers (clobbered) 161 * label = label to use for branches (text) 162 * %asi = ASI to use for TSB access 163 * 164 * NOTE that we flush the TSB using fast VIS instructions that 165 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 166 * not be treated as a locked entry or we'll get stuck spinning on 167 * an entry that isn't locked but really invalid. 168 */ 169 170#if defined(UTSB_PHYS) 171 172#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 173 lda [tsbep]ASI_MEM, tmp1 ;\ 174label: ;\ 175 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 176 cmp tmp1, tmp2 ;\ 177 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 178 lda [tsbep]ASI_MEM, tmp1 ;\ 179 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 180 cmp tmp1, tmp2 ;\ 181 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 182 lda [tsbep]ASI_MEM, tmp1 ;\ 183 /* tsbe lock acquired */ ;\ 184 membar #StoreStore 185 186#else /* UTSB_PHYS */ 187 188#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 189 lda [tsbep]%asi, tmp1 ;\ 190label: ;\ 191 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 192 cmp tmp1, tmp2 ;\ 193 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 194 lda [tsbep]%asi, tmp1 ;\ 195 casa [tsbep]%asi, tmp1, tmp2 ;\ 196 cmp tmp1, tmp2 ;\ 197 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 198 lda [tsbep]%asi, tmp1 ;\ 199 /* tsbe lock acquired */ ;\ 200 membar #StoreStore 201 202#endif /* UTSB_PHYS */ 203 204/* 205 * Atomically write TSBE at virtual address tsbep. 206 * 207 * tsbep = TSBE va (ro) 208 * tte = TSBE TTE (ro) 209 * tagtarget = TSBE tag (ro) 210 * %asi = ASI to use for TSB access 211 */ 212 213#if defined(UTSB_PHYS) 214 215#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 216 add tsbep, TSBE_TTE, tmp1 ;\ 217 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 218 membar #StoreStore ;\ 219 add tsbep, TSBE_TAG, tmp1 ;\ 220 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 221 222#else /* UTSB_PHYS */ 223 224#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 225 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 226 membar #StoreStore ;\ 227 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 228 229#endif /* UTSB_PHYS */ 230 231/* 232 * Load an entry into the TSB at TL > 0. 233 * 234 * tsbep = pointer to the TSBE to load as va (ro) 235 * tte = value of the TTE retrieved and loaded (wo) 236 * tagtarget = tag target register. To get TSBE tag to load, 237 * we need to mask off the context and leave only the va (clobbered) 238 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 239 * tmp1, tmp2 = scratch registers 240 * label = label to use for branches (text) 241 * %asi = ASI to use for TSB access 242 */ 243 244#if defined(UTSB_PHYS) 245 246#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 247 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 248 /* ;\ 249 * I don't need to update the TSB then check for the valid tte. ;\ 250 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 251 * we always invalidate the hash table before we unload the TSB.;\ 252 */ ;\ 253 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 254 ldxa [ttepa]ASI_MEM, tte ;\ 255 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 256 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 257 add tsbep, TSBE_TAG, tmp1 ;\ 258 brgez,a,pn tte, label/**/f ;\ 259 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 260 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 261label: 262 263#else /* UTSB_PHYS */ 264 265#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 266 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 267 /* ;\ 268 * I don't need to update the TSB then check for the valid tte. ;\ 269 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 270 * we always invalidate the hash table before we unload the TSB.;\ 271 */ ;\ 272 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 273 ldxa [ttepa]ASI_MEM, tte ;\ 274 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 275 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 276 brgez,a,pn tte, label/**/f ;\ 277 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 278 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 279label: 280 281#endif /* UTSB_PHYS */ 282 283/* 284 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 285 * for ITLB synthesis. 286 * 287 * tsbep = pointer to the TSBE to load as va (ro) 288 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 289 * with exec_perm turned off and exec_synth turned on 290 * tagtarget = tag target register. To get TSBE tag to load, 291 * we need to mask off the context and leave only the va (clobbered) 292 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 293 * tmp1, tmp2 = scratch registers 294 * label = label to use for branch (text) 295 * %asi = ASI to use for TSB access 296 */ 297 298#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 299 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 300 /* ;\ 301 * I don't need to update the TSB then check for the valid tte. ;\ 302 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 303 * we always invalidate the hash table before we unload the TSB.;\ 304 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 305 * and exec_synth bit to 1. ;\ 306 */ ;\ 307 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 308 mov tte, tmp1 ;\ 309 ldxa [ttepa]ASI_MEM, tte ;\ 310 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 311 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 312 brgez,a,pn tte, label/**/f ;\ 313 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 314 or tte, tmp1, tte ;\ 315 andn tte, TTE_EXECPRM_INT, tte ;\ 316 or tte, TTE_E_SYNTH_INT, tte ;\ 317 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 318label: 319 320/* 321 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 322 * 323 * tte = value of the TTE, used to get tte_size bits (ro) 324 * tagaccess = tag access register, used to get 4M pfn bits (ro) 325 * pfn = 4M pfn bits shifted to offset for tte (out) 326 * tmp1 = scratch register 327 * label = label to use for branch (text) 328 */ 329 330#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 331 /* ;\ 332 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 333 * Return them, shifted, in pfn. ;\ 334 */ ;\ 335 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 336 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 337 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 338 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 339 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 340 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 341label: ;\ 342 sllx tagaccess, MMU_PAGESHIFT4M, pfn 343 344/* 345 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 346 * for ITLB synthesis. 347 * 348 * tte = value of the TTE, used to get tte_size bits (rw) 349 * tmp1 = scratch register 350 */ 351 352#define SET_TTE4M_PN(tte, tmp) \ 353 /* ;\ 354 * Set 4M pagesize tte bits. ;\ 355 */ ;\ 356 set TTE4M, tmp ;\ 357 sllx tmp, TTE_SZ_SHFT, tmp ;\ 358 or tte, tmp, tte 359 360/* 361 * Load an entry into the TSB at TL=0. 362 * 363 * tsbep = pointer to the TSBE to load as va (ro) 364 * tteva = pointer to the TTE to load as va (ro) 365 * tagtarget = TSBE tag to load (which contains no context), synthesized 366 * to match va of MMU tag target register only (ro) 367 * tmp1, tmp2 = scratch registers (clobbered) 368 * label = label to use for branches (text) 369 * %asi = ASI to use for TSB access 370 */ 371 372#if defined(UTSB_PHYS) 373 374#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 375 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 376 ldx [tteva], tteva /* load tte */ ;\ 377 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 378 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 379 add tsbep, TSBE_TAG, tmp1 ;\ 380 brgez,a,pn tteva, label/**/f ;\ 381 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 382 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 383label: 384 385#else /* UTSB_PHYS */ 386 387#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 388 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 389 ldx [tteva], tteva /* load tte */ ;\ 390 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 391 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 392 brgez,a,pn tteva, label/**/f ;\ 393 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 394 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 395label: 396 397#endif /* UTSB_PHYS */ 398 399/* 400 * Invalidate a TSB entry in the TSB. 401 * 402 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 403 * about this earlier to ensure this is true. Thus when we are 404 * directly referencing tsbep below, we are referencing the tte_tag 405 * field of the TSBE. If this offset ever changes, the code below 406 * will need to be modified. 407 * 408 * tsbep = pointer to TSBE as va (ro) 409 * tag = invalidation is done if this matches the TSBE tag (ro) 410 * tmp1 - tmp3 = scratch registers (clobbered) 411 * label = label name to use for branches (text) 412 * %asi = ASI to use for TSB access 413 */ 414 415#if defined(UTSB_PHYS) 416 417#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 418 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 419 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 420label/**/1: ;\ 421 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 422 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 423 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 424 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 425 cmp tag, tmp3 /* compare tags */ ;\ 426 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 427 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 428 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 429 cmp tmp1, tmp3 /* if not successful */ ;\ 430 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 431 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 432label/**/2: 433 434#else /* UTSB_PHYS */ 435 436#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 437 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 438 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 439label/**/1: ;\ 440 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 441 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 442 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 443 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 444 cmp tag, tmp3 /* compare tags */ ;\ 445 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 446 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 447 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 448 cmp tmp1, tmp3 /* if not successful */ ;\ 449 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 450 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 451label/**/2: 452 453#endif /* UTSB_PHYS */ 454 455#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 456#error - TSB_SOFTSZ_MASK too small 457#endif 458 459 460/* 461 * An implementation of setx which will be hot patched at run time. 462 * since it is being hot patched, there is no value passed in. 463 * Thus, essentially we are implementing 464 * setx value, tmp, dest 465 * where value is RUNTIME_PATCH (aka 0) in this case. 466 */ 467#define RUNTIME_PATCH_SETX(dest, tmp) \ 468 sethi %hh(RUNTIME_PATCH), tmp ;\ 469 sethi %lm(RUNTIME_PATCH), dest ;\ 470 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 471 or dest, %lo(RUNTIME_PATCH), dest ;\ 472 sllx tmp, 32, tmp ;\ 473 nop /* for perf reasons */ ;\ 474 or tmp, dest, dest /* contents of patched value */ 475 476 477#endif (lint) 478 479 480#if defined (lint) 481 482/* 483 * sfmmu related subroutines 484 */ 485uint_t 486sfmmu_disable_intrs() 487{ return(0); } 488 489/* ARGSUSED */ 490void 491sfmmu_enable_intrs(uint_t pstate_save) 492{} 493 494/* ARGSUSED */ 495int 496sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag) 497{ return(0); } 498 499/* 500 * Use cas, if tte has changed underneath us then reread and try again. 501 * In the case of a retry, it will update sttep with the new original. 502 */ 503/* ARGSUSED */ 504int 505sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 506{ return(0); } 507 508/* 509 * Use cas, if tte has changed underneath us then return 1, else return 0 510 */ 511/* ARGSUSED */ 512int 513sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 514{ return(0); } 515 516/* ARGSUSED */ 517void 518sfmmu_copytte(tte_t *sttep, tte_t *dttep) 519{} 520 521/*ARGSUSED*/ 522struct tsbe * 523sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 524{ return(0); } 525 526/*ARGSUSED*/ 527uint64_t 528sfmmu_make_tsbtag(caddr_t va) 529{ return(0); } 530 531#else /* lint */ 532 533 .seg ".data" 534 .global sfmmu_panic1 535sfmmu_panic1: 536 .asciz "sfmmu_asm: interrupts already disabled" 537 538 .global sfmmu_panic3 539sfmmu_panic3: 540 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 541 542 .global sfmmu_panic4 543sfmmu_panic4: 544 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 545 546 .global sfmmu_panic5 547sfmmu_panic5: 548 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 549 550 .global sfmmu_panic6 551sfmmu_panic6: 552 .asciz "sfmmu_asm: interrupts not disabled" 553 554 .global sfmmu_panic7 555sfmmu_panic7: 556 .asciz "sfmmu_asm: kernel as" 557 558 .global sfmmu_panic8 559sfmmu_panic8: 560 .asciz "sfmmu_asm: gnum is zero" 561 562 .global sfmmu_panic9 563sfmmu_panic9: 564 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 565 566 .global sfmmu_panic10 567sfmmu_panic10: 568 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 569 570 ENTRY(sfmmu_disable_intrs) 571 rdpr %pstate, %o0 572#ifdef DEBUG 573 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 574#endif /* DEBUG */ 575 retl 576 wrpr %o0, PSTATE_IE, %pstate 577 SET_SIZE(sfmmu_disable_intrs) 578 579 ENTRY(sfmmu_enable_intrs) 580 retl 581 wrpr %g0, %o0, %pstate 582 SET_SIZE(sfmmu_enable_intrs) 583 584/* 585 * This routine is called both by resume() and sfmmu_get_ctx() to 586 * allocate a new context for the process on a MMU. 587 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 588 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 589 * is the case when sfmmu_alloc_ctx is called from resume(). 590 * 591 * The caller must disable interrupts before entering this routine. 592 * To reduce ctx switch overhead, the code contains both 'fast path' and 593 * 'slow path' code. The fast path code covers the common case where only 594 * a quick check is needed and the real ctx allocation is not required. 595 * It can be done without holding the per-process (PP) lock. 596 * The 'slow path' code must be protected by the PP Lock and performs ctx 597 * allocation. 598 * Hardware context register and HAT mmu cnum are updated accordingly. 599 * 600 * %o0 - sfmmup 601 * %o1 - allocflag 602 * %o2 - CPU 603 * %o3 - sfmmu private/shared flag 604 * 605 * ret - 0: no ctx is allocated 606 * 1: a ctx is allocated 607 */ 608 ENTRY_NP(sfmmu_alloc_ctx) 609 610#ifdef DEBUG 611 sethi %hi(ksfmmup), %g1 612 ldx [%g1 + %lo(ksfmmup)], %g1 613 cmp %g1, %o0 614 bne,pt %xcc, 0f 615 nop 616 617 sethi %hi(panicstr), %g1 ! if kernel as, panic 618 ldx [%g1 + %lo(panicstr)], %g1 619 tst %g1 620 bnz,pn %icc, 7f 621 nop 622 623 sethi %hi(sfmmu_panic7), %o0 624 call panic 625 or %o0, %lo(sfmmu_panic7), %o0 626 6277: 628 retl 629 mov %g0, %o0 ! %o0 = ret = 0 630 6310: 632 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 633#endif /* DEBUG */ 634 635 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 636 637 ! load global mmu_ctxp info 638 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 639 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 640 641 ! load global mmu_ctxp gnum 642 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 643 644#ifdef DEBUG 645 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 646 bne,pt %xcc, 3f 647 nop 648 649 sethi %hi(panicstr), %g1 ! test if panicstr is already set 650 ldx [%g1 + %lo(panicstr)], %g1 651 tst %g1 652 bnz,pn %icc, 1f 653 nop 654 655 sethi %hi(sfmmu_panic8), %o0 656 call panic 657 or %o0, %lo(sfmmu_panic8), %o0 6581: 659 retl 660 mov %g0, %o0 ! %o0 = ret = 0 6613: 662#endif 663 664 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 665 666 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 667 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 668 669 /* 670 * %g5 = sfmmu gnum returned 671 * %g6 = sfmmu cnum returned 672 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 673 * %g4 = scratch 674 * 675 * Fast path code, do a quick check. 676 */ 677 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 678 679 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 680 bne,pt %icc, 1f ! valid hat cnum, check gnum 681 nop 682 683 ! cnum == INVALID, check allocflag 684 mov %g0, %g4 ! %g4 = ret = 0 685 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 686 mov %g6, %o1 687 688 ! (invalid HAT cnum) && (allocflag == 1) 689 ba,pt %icc, 2f 690 nop 6911: 692 ! valid HAT cnum, check gnum 693 cmp %g5, %o4 694 mov 1, %g4 !%g4 = ret = 1 695 be,a,pt %icc, 8f ! gnum unchanged, go to done 696 mov %g6, %o1 697 6982: 699 /* 700 * Grab per process (PP) sfmmu_ctx_lock spinlock, 701 * followed by the 'slow path' code. 702 */ 703 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 7043: 705 brz %g3, 5f 706 nop 7074: 708 brnz,a,pt %g3, 4b ! spin if lock is 1 709 ldub [%o0 + SFMMU_CTX_LOCK], %g3 710 ba %xcc, 3b ! retry the lock 711 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 712 7135: 714 membar #LoadLoad 715 /* 716 * %g5 = sfmmu gnum returned 717 * %g6 = sfmmu cnum returned 718 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 719 * %g4 = scratch 720 */ 721 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 722 723 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 724 bne,pt %icc, 1f ! valid hat cnum, check gnum 725 nop 726 727 ! cnum == INVALID, check allocflag 728 mov %g0, %g4 ! %g4 = ret = 0 729 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 730 mov %g6, %o1 731 732 ! (invalid HAT cnum) && (allocflag == 1) 733 ba,pt %icc, 6f 734 nop 7351: 736 ! valid HAT cnum, check gnum 737 cmp %g5, %o4 738 mov 1, %g4 ! %g4 = ret = 1 739 be,a,pt %icc, 2f ! gnum unchanged, go to done 740 mov %g6, %o1 741 742 ba,pt %icc, 6f 743 nop 7442: 745 membar #LoadStore|#StoreStore 746 ba,pt %icc, 8f 747 clrb [%o0 + SFMMU_CTX_LOCK] 7486: 749 /* 750 * We get here if we do not have a valid context, or 751 * the HAT gnum does not match global gnum. We hold 752 * sfmmu_ctx_lock spinlock. Allocate that context. 753 * 754 * %o3 = mmu_ctxp 755 */ 756 add %o3, MMU_CTX_CNUM, %g3 757 ld [%o3 + MMU_CTX_NCTXS], %g4 758 759 /* 760 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 761 * %g3 = mmu cnum address 762 * %g4 = mmu nctxs 763 * 764 * %o0 = sfmmup 765 * %o1 = mmu current cnum value (used as new cnum) 766 * %o4 = mmu gnum 767 * 768 * %o5 = scratch 769 */ 770 ld [%g3], %o1 7710: 772 cmp %o1, %g4 773 bl,a,pt %icc, 1f 774 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 775 776 /* 777 * cnum reachs max, bail, so wrap around can be performed later. 778 */ 779 set INVALID_CONTEXT, %o1 780 mov %g0, %g4 ! %g4 = ret = 0 781 782 membar #LoadStore|#StoreStore 783 ba,pt %icc, 8f 784 clrb [%o0 + SFMMU_CTX_LOCK] 7851: 786 ! %g3 = addr of mmu_ctxp->cnum 787 ! %o5 = mmu_ctxp->cnum + 1 788 cas [%g3], %o1, %o5 789 cmp %o1, %o5 790 bne,a,pn %xcc, 0b ! cas failed 791 ld [%g3], %o1 792 793#ifdef DEBUG 794 set MAX_SFMMU_CTX_VAL, %o5 795 cmp %o1, %o5 796 ble,pt %icc, 2f 797 nop 798 799 sethi %hi(sfmmu_panic9), %o0 800 call panic 801 or %o0, %lo(sfmmu_panic9), %o0 8022: 803#endif 804 ! update hat gnum and cnum 805 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 806 or %o4, %o1, %o4 807 stx %o4, [%g2 + SFMMU_CTXS] 808 809 membar #LoadStore|#StoreStore 810 clrb [%o0 + SFMMU_CTX_LOCK] 811 812 mov 1, %g4 ! %g4 = ret = 1 8138: 814 /* 815 * program the secondary context register 816 * 817 * %o1 = cnum 818 * %g1 = sfmmu private/shared flag (0:private, 1:shared) 819 */ 820 821 /* 822 * When we come here and context is invalid, we want to set both 823 * private and shared ctx regs to INVALID. In order to 824 * do so, we set the sfmmu priv/shared flag to 'private' regardless 825 * so that private ctx reg will be set to invalid. 826 * Note that on sun4v values written to private context register are 827 * automatically written to corresponding shared context register as 828 * well. On sun4u SET_SECCTX() will invalidate shared context register 829 * when it sets a private secondary context register. 830 */ 831 832 cmp %o1, INVALID_CONTEXT 833 be,a,pn %icc, 9f 834 clr %g1 8359: 836 837#ifdef sun4u 838 ldub [%o0 + SFMMU_CEXT], %o2 839 sll %o2, CTXREG_EXT_SHIFT, %o2 840 or %o1, %o2, %o1 841#endif /* sun4u */ 842 843 SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1) 844 845 retl 846 mov %g4, %o0 ! %o0 = ret 847 848 SET_SIZE(sfmmu_alloc_ctx) 849 850 ENTRY_NP(sfmmu_modifytte) 851 ldx [%o2], %g3 /* current */ 852 ldx [%o0], %g1 /* original */ 8532: 854 ldx [%o1], %g2 /* modified */ 855 cmp %g2, %g3 /* is modified = current? */ 856 be,a,pt %xcc,1f /* yes, don't write */ 857 stx %g3, [%o0] /* update new original */ 858 casx [%o2], %g1, %g2 859 cmp %g1, %g2 860 be,pt %xcc, 1f /* cas succeeded - return */ 861 nop 862 ldx [%o2], %g3 /* new current */ 863 stx %g3, [%o0] /* save as new original */ 864 ba,pt %xcc, 2b 865 mov %g3, %g1 8661: retl 867 membar #StoreLoad 868 SET_SIZE(sfmmu_modifytte) 869 870 ENTRY_NP(sfmmu_modifytte_try) 871 ldx [%o1], %g2 /* modified */ 872 ldx [%o2], %g3 /* current */ 873 ldx [%o0], %g1 /* original */ 874 cmp %g3, %g2 /* is modified = current? */ 875 be,a,pn %xcc,1f /* yes, don't write */ 876 mov 0, %o1 /* as if cas failed. */ 877 878 casx [%o2], %g1, %g2 879 membar #StoreLoad 880 cmp %g1, %g2 881 movne %xcc, -1, %o1 /* cas failed. */ 882 move %xcc, 1, %o1 /* cas succeeded. */ 8831: 884 stx %g2, [%o0] /* report "current" value */ 885 retl 886 mov %o1, %o0 887 SET_SIZE(sfmmu_modifytte_try) 888 889 ENTRY_NP(sfmmu_copytte) 890 ldx [%o0], %g1 891 retl 892 stx %g1, [%o1] 893 SET_SIZE(sfmmu_copytte) 894 895 896 /* 897 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 898 * %o0 = TSB base address (in), pointer to TSB entry (out) 899 * %o1 = vaddr (in) 900 * %o2 = vpshift (in) 901 * %o3 = tsb size code (in) 902 * %o4 = scratch register 903 */ 904 ENTRY_NP(sfmmu_get_tsbe) 905 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 906 retl 907 nop 908 SET_SIZE(sfmmu_get_tsbe) 909 910 /* 911 * Return a TSB tag for the given va. 912 * %o0 = va (in/clobbered) 913 * %o0 = va shifted to be in tsb tag format (with no context) (out) 914 */ 915 ENTRY_NP(sfmmu_make_tsbtag) 916 retl 917 srln %o0, TTARGET_VA_SHIFT, %o0 918 SET_SIZE(sfmmu_make_tsbtag) 919 920#endif /* lint */ 921 922/* 923 * Other sfmmu primitives 924 */ 925 926 927#if defined (lint) 928void 929sfmmu_patch_ktsb(void) 930{ 931} 932 933void 934sfmmu_kpm_patch_tlbm(void) 935{ 936} 937 938void 939sfmmu_kpm_patch_tsbm(void) 940{ 941} 942 943void 944sfmmu_patch_shctx(void) 945{ 946} 947 948/* ARGSUSED */ 949void 950sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 951{ 952} 953 954/* ARGSUSED */ 955void 956sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 957{ 958} 959 960/* ARGSUSED */ 961void 962sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 963{ 964} 965 966/* ARGSUSED */ 967void 968sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 969{ 970} 971 972#else /* lint */ 973 974#define I_SIZE 4 975 976 ENTRY_NP(sfmmu_fix_ktlb_traptable) 977 /* 978 * %o0 = start of patch area 979 * %o1 = size code of TSB to patch 980 * %o3 = scratch 981 */ 982 /* fix sll */ 983 ld [%o0], %o3 /* get sll */ 984 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 985 st %o3, [%o0] /* write sll */ 986 flush %o0 987 /* fix srl */ 988 add %o0, I_SIZE, %o0 /* goto next instr. */ 989 ld [%o0], %o3 /* get srl */ 990 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 991 st %o3, [%o0] /* write srl */ 992 retl 993 flush %o0 994 SET_SIZE(sfmmu_fix_ktlb_traptable) 995 996 ENTRY_NP(sfmmu_fixup_ktsbbase) 997 /* 998 * %o0 = start of patch area 999 * %o5 = kernel virtual or physical tsb base address 1000 * %o2, %o3 are used as scratch registers. 1001 */ 1002 /* fixup sethi instruction */ 1003 ld [%o0], %o3 1004 srl %o5, 10, %o2 ! offset is bits 32:10 1005 or %o3, %o2, %o3 ! set imm22 1006 st %o3, [%o0] 1007 /* fixup offset of lduw/ldx */ 1008 add %o0, I_SIZE, %o0 ! next instr 1009 ld [%o0], %o3 1010 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 1011 or %o3, %o2, %o3 1012 st %o3, [%o0] 1013 retl 1014 flush %o0 1015 SET_SIZE(sfmmu_fixup_ktsbbase) 1016 1017 ENTRY_NP(sfmmu_fixup_setx) 1018 /* 1019 * %o0 = start of patch area 1020 * %o4 = 64 bit value to patch 1021 * %o2, %o3 are used as scratch registers. 1022 * 1023 * Note: Assuming that all parts of the instructions which need to be 1024 * patched correspond to RUNTIME_PATCH (aka 0) 1025 * 1026 * Note the implementation of setx which is being patched is as follows: 1027 * 1028 * sethi %hh(RUNTIME_PATCH), tmp 1029 * sethi %lm(RUNTIME_PATCH), dest 1030 * or tmp, %hm(RUNTIME_PATCH), tmp 1031 * or dest, %lo(RUNTIME_PATCH), dest 1032 * sllx tmp, 32, tmp 1033 * nop 1034 * or tmp, dest, dest 1035 * 1036 * which differs from the implementation in the 1037 * "SPARC Architecture Manual" 1038 */ 1039 /* fixup sethi instruction */ 1040 ld [%o0], %o3 1041 srlx %o4, 42, %o2 ! bits [63:42] 1042 or %o3, %o2, %o3 ! set imm22 1043 st %o3, [%o0] 1044 /* fixup sethi instruction */ 1045 add %o0, I_SIZE, %o0 ! next instr 1046 ld [%o0], %o3 1047 sllx %o4, 32, %o2 ! clear upper bits 1048 srlx %o2, 42, %o2 ! bits [31:10] 1049 or %o3, %o2, %o3 ! set imm22 1050 st %o3, [%o0] 1051 /* fixup or instruction */ 1052 add %o0, I_SIZE, %o0 ! next instr 1053 ld [%o0], %o3 1054 srlx %o4, 32, %o2 ! bits [63:32] 1055 and %o2, 0x3ff, %o2 ! bits [41:32] 1056 or %o3, %o2, %o3 ! set imm 1057 st %o3, [%o0] 1058 /* fixup or instruction */ 1059 add %o0, I_SIZE, %o0 ! next instr 1060 ld [%o0], %o3 1061 and %o4, 0x3ff, %o2 ! bits [9:0] 1062 or %o3, %o2, %o3 ! set imm 1063 st %o3, [%o0] 1064 retl 1065 flush %o0 1066 SET_SIZE(sfmmu_fixup_setx) 1067 1068 ENTRY_NP(sfmmu_fixup_or) 1069 /* 1070 * %o0 = start of patch area 1071 * %o4 = 32 bit value to patch 1072 * %o2, %o3 are used as scratch registers. 1073 * Note: Assuming that all parts of the instructions which need to be 1074 * patched correspond to RUNTIME_PATCH (aka 0) 1075 */ 1076 ld [%o0], %o3 1077 and %o4, 0x3ff, %o2 ! bits [9:0] 1078 or %o3, %o2, %o3 ! set imm 1079 st %o3, [%o0] 1080 retl 1081 flush %o0 1082 SET_SIZE(sfmmu_fixup_or) 1083 1084 ENTRY_NP(sfmmu_fixup_shiftx) 1085 /* 1086 * %o0 = start of patch area 1087 * %o4 = signed int immediate value to add to sllx/srlx imm field 1088 * %o2, %o3 are used as scratch registers. 1089 * 1090 * sllx/srlx store the 6 bit immediate value in the lowest order bits 1091 * so we do a simple add. The caller must be careful to prevent 1092 * overflow, which could easily occur if the initial value is nonzero! 1093 */ 1094 ld [%o0], %o3 ! %o3 = instruction to patch 1095 and %o3, 0x3f, %o2 ! %o2 = existing imm value 1096 add %o2, %o4, %o2 ! %o2 = new imm value 1097 andn %o3, 0x3f, %o3 ! clear old imm value 1098 and %o2, 0x3f, %o2 ! truncate new imm value 1099 or %o3, %o2, %o3 ! set new imm value 1100 st %o3, [%o0] ! store updated instruction 1101 retl 1102 flush %o0 1103 SET_SIZE(sfmmu_fixup_shiftx) 1104 1105 ENTRY_NP(sfmmu_fixup_mmu_asi) 1106 /* 1107 * Patch imm_asi of all ldda instructions in the MMU 1108 * trap handlers. We search MMU_PATCH_INSTR instructions 1109 * starting from the itlb miss handler (trap 0x64). 1110 * %o0 = address of tt[0,1]_itlbmiss 1111 * %o1 = imm_asi to setup, shifted by appropriate offset. 1112 * %o3 = number of instructions to search 1113 * %o4 = reserved by caller: called from leaf routine 1114 */ 11151: ldsw [%o0], %o2 ! load instruction to %o2 1116 brgez,pt %o2, 2f 1117 srl %o2, 30, %o5 1118 btst 1, %o5 ! test bit 30; skip if not set 1119 bz,pt %icc, 2f 1120 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1121 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1122 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1123 brnz,pt %o5, 2f ! skip if not a match 1124 or %o2, %o1, %o2 ! or in imm_asi 1125 st %o2, [%o0] ! write patched instruction 11262: dec %o3 1127 brnz,a,pt %o3, 1b ! loop until we're done 1128 add %o0, I_SIZE, %o0 1129 retl 1130 flush %o0 1131 SET_SIZE(sfmmu_fixup_mmu_asi) 1132 1133 /* 1134 * Patch immediate ASI used to access the TSB in the 1135 * trap table. 1136 * inputs: %o0 = value of ktsb_phys 1137 */ 1138 ENTRY_NP(sfmmu_patch_mmu_asi) 1139 mov %o7, %o4 ! save return pc in %o4 1140 movrnz %o0, ASI_QUAD_LDD_PHYS, %o3 1141 movrz %o0, ASI_NQUAD_LD, %o3 1142 sll %o3, 5, %o1 ! imm_asi offset 1143 mov 6, %o3 ! number of instructions 1144 sethi %hi(dktsb), %o0 ! to search 1145 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1146 or %o0, %lo(dktsb), %o0 1147 mov 6, %o3 ! number of instructions 1148 sethi %hi(dktsb4m), %o0 ! to search 1149 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1150 or %o0, %lo(dktsb4m), %o0 1151 mov 6, %o3 ! number of instructions 1152 sethi %hi(iktsb), %o0 ! to search 1153 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1154 or %o0, %lo(iktsb), %o0 1155 mov 6, %o3 ! number of instructions 1156 sethi %hi(iktsb4m), %o0 ! to search 1157 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1158 or %o0, %lo(iktsb4m), %o0 1159 mov %o4, %o7 ! retore return pc -- leaf 1160 retl 1161 nop 1162 SET_SIZE(sfmmu_patch_mmu_asi) 1163 1164 ENTRY_NP(sfmmu_patch_ktsb) 1165 /* 1166 * We need to fix iktsb, dktsb, et. al. 1167 */ 1168 save %sp, -SA(MINFRAME), %sp 1169 set ktsb_phys, %o1 1170 ld [%o1], %o4 1171 set ktsb_base, %o5 1172 set ktsb4m_base, %l1 1173 brz,pt %o4, 1f 1174 nop 1175 set ktsb_pbase, %o5 1176 set ktsb4m_pbase, %l1 11771: 1178 sethi %hi(ktsb_szcode), %o1 1179 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1180 1181 sethi %hi(iktsb), %o0 1182 call sfmmu_fix_ktlb_traptable 1183 or %o0, %lo(iktsb), %o0 1184 1185 sethi %hi(dktsb), %o0 1186 call sfmmu_fix_ktlb_traptable 1187 or %o0, %lo(dktsb), %o0 1188 1189 sethi %hi(ktsb4m_szcode), %o1 1190 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1191 1192 sethi %hi(iktsb4m), %o0 1193 call sfmmu_fix_ktlb_traptable 1194 or %o0, %lo(iktsb4m), %o0 1195 1196 sethi %hi(dktsb4m), %o0 1197 call sfmmu_fix_ktlb_traptable 1198 or %o0, %lo(dktsb4m), %o0 1199 1200#ifndef sun4v 1201 mov ASI_N, %o2 1202 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1203 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1204 sethi %hi(tsb_kernel_patch_asi), %o0 1205 call sfmmu_fixup_or 1206 or %o0, %lo(tsb_kernel_patch_asi), %o0 1207#endif /* !sun4v */ 1208 1209 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1210 1211 sethi %hi(dktsbbase), %o0 1212 call sfmmu_fixup_setx ! patch value of ktsb base addr 1213 or %o0, %lo(dktsbbase), %o0 1214 1215 sethi %hi(iktsbbase), %o0 1216 call sfmmu_fixup_setx ! patch value of ktsb base addr 1217 or %o0, %lo(iktsbbase), %o0 1218 1219 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1220 call sfmmu_fixup_setx ! patch value of ktsb base addr 1221 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1222 1223#ifdef sun4v 1224 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1225 call sfmmu_fixup_setx ! patch value of ktsb base addr 1226 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1227#endif /* sun4v */ 1228 1229 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1230 1231 sethi %hi(dktsb4mbase), %o0 1232 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1233 or %o0, %lo(dktsb4mbase), %o0 1234 1235 sethi %hi(iktsb4mbase), %o0 1236 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1237 or %o0, %lo(iktsb4mbase), %o0 1238 1239 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1240 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1241 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1242 1243#ifdef sun4v 1244 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1245 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1246 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1247#endif /* sun4v */ 1248 1249 set ktsb_szcode, %o4 1250 ld [%o4], %o4 1251 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1252 call sfmmu_fixup_or ! patch value of ktsb_szcode 1253 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1254 1255#ifdef sun4v 1256 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1257 call sfmmu_fixup_or ! patch value of ktsb_szcode 1258 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1259#endif /* sun4v */ 1260 1261 set ktsb4m_szcode, %o4 1262 ld [%o4], %o4 1263 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1264 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1265 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1266 1267#ifdef sun4v 1268 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1269 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1270 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1271#endif /* sun4v */ 1272 1273 ret 1274 restore 1275 SET_SIZE(sfmmu_patch_ktsb) 1276 1277 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1278 /* 1279 * Fixup trap handlers in common segkpm case. This is reserved 1280 * for future use should kpm TSB be changed to be other than the 1281 * kernel TSB. 1282 */ 1283 retl 1284 nop 1285 SET_SIZE(sfmmu_kpm_patch_tlbm) 1286 1287 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1288 /* 1289 * nop the branch to sfmmu_kpm_dtsb_miss_small 1290 * in the case where we are using large pages for 1291 * seg_kpm (and hence must probe the second TSB for 1292 * seg_kpm VAs) 1293 */ 1294 set dktsb4m_kpmcheck_small, %o0 1295 MAKE_NOP_INSTR(%o1) 1296 st %o1, [%o0] 1297 flush %o0 1298 retl 1299 nop 1300 SET_SIZE(sfmmu_kpm_patch_tsbm) 1301 1302 ENTRY_NP(sfmmu_patch_utsb) 1303#ifdef UTSB_PHYS 1304 retl 1305 nop 1306#else /* UTSB_PHYS */ 1307 /* 1308 * We need to hot patch utsb_vabase and utsb4m_vabase 1309 */ 1310 save %sp, -SA(MINFRAME), %sp 1311 1312 /* patch value of utsb_vabase */ 1313 set utsb_vabase, %o1 1314 ldx [%o1], %o4 1315 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1316 call sfmmu_fixup_setx 1317 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1318 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1319 call sfmmu_fixup_setx 1320 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1321 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1322 call sfmmu_fixup_setx 1323 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1324 1325 /* patch value of utsb4m_vabase */ 1326 set utsb4m_vabase, %o1 1327 ldx [%o1], %o4 1328 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1329 call sfmmu_fixup_setx 1330 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1331 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1332 call sfmmu_fixup_setx 1333 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1334 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1335 call sfmmu_fixup_setx 1336 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1337 1338 /* 1339 * Patch TSB base register masks and shifts if needed. 1340 * By default the TSB base register contents are set up for 4M slab. 1341 * If we're using a smaller slab size and reserved VA range we need 1342 * to patch up those values here. 1343 */ 1344 set tsb_slab_shift, %o1 1345 set MMU_PAGESHIFT4M, %o4 1346 lduw [%o1], %o3 1347 subcc %o4, %o3, %o4 1348 bz,pt %icc, 1f 1349 /* delay slot safe */ 1350 1351 /* patch reserved VA range size if needed. */ 1352 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1353 call sfmmu_fixup_shiftx 1354 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1355 call sfmmu_fixup_shiftx 1356 add %o0, I_SIZE, %o0 1357 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1358 call sfmmu_fixup_shiftx 1359 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1360 call sfmmu_fixup_shiftx 1361 add %o0, I_SIZE, %o0 13621: 1363 /* patch TSBREG_VAMASK used to set up TSB base register */ 1364 set tsb_slab_mask, %o1 1365 ldx [%o1], %o4 1366 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1367 call sfmmu_fixup_or 1368 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1369 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1370 call sfmmu_fixup_or 1371 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1372 1373 ret 1374 restore 1375#endif /* UTSB_PHYS */ 1376 SET_SIZE(sfmmu_patch_utsb) 1377 1378 ENTRY_NP(sfmmu_patch_shctx) 1379#ifdef sun4u 1380 retl 1381 nop 1382#else /* sun4u */ 1383 set sfmmu_shctx_cpu_mondo_patch, %o0 1384 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1385 st %o1, [%o0] 1386 flush %o0 1387 MAKE_NOP_INSTR(%o1) 1388 add %o0, I_SIZE, %o0 ! next instr 1389 st %o1, [%o0] 1390 flush %o0 1391 1392 set sfmmu_shctx_user_rtt_patch, %o0 1393 st %o1, [%o0] ! nop 1st instruction 1394 flush %o0 1395 add %o0, I_SIZE, %o0 1396 st %o1, [%o0] ! nop 2nd instruction 1397 flush %o0 1398 add %o0, I_SIZE, %o0 1399 st %o1, [%o0] ! nop 3rd instruction 1400 flush %o0 1401 add %o0, I_SIZE, %o0 1402 st %o1, [%o0] ! nop 4th instruction 1403 flush %o0 1404 add %o0, I_SIZE, %o0 1405 st %o1, [%o0] ! nop 5th instruction 1406 retl 1407 flush %o0 1408#endif /* sun4u */ 1409 SET_SIZE(sfmmu_patch_shctx) 1410 1411 /* 1412 * Routine that loads an entry into a tsb using virtual addresses. 1413 * Locking is required since all cpus can use the same TSB. 1414 * Note that it is no longer required to have a valid context 1415 * when calling this function. 1416 */ 1417 ENTRY_NP(sfmmu_load_tsbe) 1418 /* 1419 * %o0 = pointer to tsbe to load 1420 * %o1 = tsb tag 1421 * %o2 = virtual pointer to TTE 1422 * %o3 = 1 if physical address in %o0 else 0 1423 */ 1424 rdpr %pstate, %o5 1425#ifdef DEBUG 1426 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1427#endif /* DEBUG */ 1428 1429 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1430 1431 SETUP_TSB_ASI(%o3, %g3) 1432 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1) 1433 1434 wrpr %g0, %o5, %pstate /* enable interrupts */ 1435 1436 retl 1437 membar #StoreStore|#StoreLoad 1438 SET_SIZE(sfmmu_load_tsbe) 1439 1440 /* 1441 * Flush TSB of a given entry if the tag matches. 1442 */ 1443 ENTRY(sfmmu_unload_tsbe) 1444 /* 1445 * %o0 = pointer to tsbe to be flushed 1446 * %o1 = tag to match 1447 * %o2 = 1 if physical address in %o0 else 0 1448 */ 1449 SETUP_TSB_ASI(%o2, %g1) 1450 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1451 retl 1452 membar #StoreStore|#StoreLoad 1453 SET_SIZE(sfmmu_unload_tsbe) 1454 1455 /* 1456 * Routine that loads a TTE into the kpm TSB from C code. 1457 * Locking is required since kpm TSB is shared among all CPUs. 1458 */ 1459 ENTRY_NP(sfmmu_kpm_load_tsb) 1460 /* 1461 * %o0 = vaddr 1462 * %o1 = ttep 1463 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1464 */ 1465 rdpr %pstate, %o5 ! %o5 = saved pstate 1466#ifdef DEBUG 1467 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1468#endif /* DEBUG */ 1469 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1470 1471#ifndef sun4v 1472 sethi %hi(ktsb_phys), %o4 1473 mov ASI_N, %o3 1474 ld [%o4 + %lo(ktsb_phys)], %o4 1475 movrnz %o4, ASI_MEM, %o3 1476 mov %o3, %asi 1477#endif /* !sun4v */ 1478 mov %o0, %g1 ! %g1 = vaddr 1479 1480 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1481 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1482 /* %g2 = tsbep, %g1 clobbered */ 1483 1484 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1485 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1486 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1) 1487 1488 wrpr %g0, %o5, %pstate ! enable interrupts 1489 retl 1490 membar #StoreStore|#StoreLoad 1491 SET_SIZE(sfmmu_kpm_load_tsb) 1492 1493 /* 1494 * Routine that shoots down a TTE in the kpm TSB or in the 1495 * kernel TSB depending on virtpg. Locking is required since 1496 * kpm/kernel TSB is shared among all CPUs. 1497 */ 1498 ENTRY_NP(sfmmu_kpm_unload_tsb) 1499 /* 1500 * %o0 = vaddr 1501 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1502 */ 1503#ifndef sun4v 1504 sethi %hi(ktsb_phys), %o4 1505 mov ASI_N, %o3 1506 ld [%o4 + %lo(ktsb_phys)], %o4 1507 movrnz %o4, ASI_MEM, %o3 1508 mov %o3, %asi 1509#endif /* !sun4v */ 1510 mov %o0, %g1 ! %g1 = vaddr 1511 1512 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1513 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1514 /* %g2 = tsbep, %g1 clobbered */ 1515 1516 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1517 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1518 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1519 1520 retl 1521 membar #StoreStore|#StoreLoad 1522 SET_SIZE(sfmmu_kpm_unload_tsb) 1523 1524#endif /* lint */ 1525 1526 1527#if defined (lint) 1528 1529/*ARGSUSED*/ 1530pfn_t 1531sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1532{ return(0); } 1533 1534#else /* lint */ 1535 1536 ENTRY_NP(sfmmu_ttetopfn) 1537 ldx [%o0], %g1 /* read tte */ 1538 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1539 /* 1540 * g1 = pfn 1541 */ 1542 retl 1543 mov %g1, %o0 1544 SET_SIZE(sfmmu_ttetopfn) 1545 1546#endif /* !lint */ 1547 1548 1549#if defined (lint) 1550/* 1551 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the 1552 * the hash list. 1553 */ 1554/* ARGSUSED */ 1555void 1556sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1557 uint64_t hblkpa) 1558{ 1559} 1560 1561/* 1562 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the 1563 * hash list. 1564 */ 1565/* ARGSUSED */ 1566void 1567sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1568 uint64_t hblkpa, struct hme_blk *prev_hblkp) 1569{ 1570} 1571#else /* lint */ 1572 1573/* 1574 * Functions to grab/release hme bucket list lock. I only use a byte 1575 * instead of the whole int because eventually we might want to 1576 * put some counters on the other bytes (of course, these routines would 1577 * have to change). The code that grab this lock should execute 1578 * with interrupts disabled and hold the lock for the least amount of time 1579 * possible. 1580 */ 1581 1582/* 1583 * Even though hmeh_listlock is updated using pa there's no need to flush 1584 * dcache since hmeh_listlock will be restored to the original value (0) 1585 * before interrupts are reenabled. 1586 */ 1587 1588/* 1589 * For sparcv9 hme hash buckets may not be in the nucleus. hme hash update 1590 * routines still use virtual addresses to update the bucket fields. But they 1591 * must not cause a TLB miss after grabbing the low level bucket lock. To 1592 * achieve this we must make sure the bucket structure is completely within an 1593 * 8K page. 1594 */ 1595 1596#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1)) 1597#error - the size of hmehash_bucket structure is not power of 2 1598#endif 1599 1600/* 1601 * Enable backoff to significantly reduce locking overhead and reduce a chance 1602 * of xcall timeout. This is only enabled for sun4v as a Makefile compile- 1603 * time option. 1604 * The rd %ccr is better for performance compared to a non pipeline releasing 1605 * tight spin on N2/VF. 1606 * Backoff based fix is a temporary solution and doesn't allow scaling above 1607 * lock saturation point. The final fix is to eliminate HMELOCK_ENTER() 1608 * to avoid xcall timeouts and improve GET_TTE() performance. 1609 */ 1610 1611#ifdef HMELOCK_BACKOFF_ENABLE 1612 1613#define HMELOCK_BACKOFF(reg, val) \ 1614 set val, reg ;\ 1615 rd %ccr, %g0 ;\ 1616 brnz reg, .-4 ;\ 1617 dec reg 1618 1619#define CAS_HME(tmp1, tmp2, exitlabel, asi) \ 1620 mov 0xff, tmp2 ;\ 1621 casa [tmp1]asi, %g0, tmp2 ;\ 1622 brz,a,pt tmp2, exitlabel ;\ 1623 membar #LoadLoad 1624 1625#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label, asi) \ 1626 mov 0xff, tmp2 ;\ 1627 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1628 casa [tmp1]asi, %g0, tmp2 ;\ 1629 brz,a,pt tmp2, label/**/2 ;\ 1630 membar #LoadLoad ;\ 1631 HMELOCK_BACKOFF(tmp2,0x8) ;\ 1632 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1633 HMELOCK_BACKOFF(tmp2,0x10) ;\ 1634 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1635 HMELOCK_BACKOFF(tmp2,0x20) ;\ 1636 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1637 HMELOCK_BACKOFF(tmp2,0x40) ;\ 1638 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1639 HMELOCK_BACKOFF(tmp2,0x80) ;\ 1640 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1641label/**/1: ;\ 1642 HMELOCK_BACKOFF(tmp2,0x100) ;\ 1643 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1644 HMELOCK_BACKOFF(tmp2,0x200) ;\ 1645 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1646 HMELOCK_BACKOFF(tmp2,0x400) ;\ 1647 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1648 HMELOCK_BACKOFF(tmp2,0x800) ;\ 1649 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1650 HMELOCK_BACKOFF(tmp2,0x1000) ;\ 1651 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1652 HMELOCK_BACKOFF(tmp2,0x2000) ;\ 1653 mov 0xff, tmp2 ;\ 1654 casa [tmp1]asi, %g0, tmp2 ;\ 1655 brnz,pn tmp2, label/**/1 /* reset backoff */ ;\ 1656 membar #LoadLoad ;\ 1657label/**/2: 1658 1659#else /* HMELOCK_BACKOFF_ENABLE */ 1660 1661#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi) \ 1662 mov 0xff, tmp2 ;\ 1663 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1664label1: ;\ 1665 casa [tmp1]asi, %g0, tmp2 ;\ 1666 brnz,pn tmp2, label1 ;\ 1667 mov 0xff, tmp2 ;\ 1668 membar #LoadLoad 1669 1670#endif /* HMELOCK_BACKOFF_ENABLE */ 1671 1672#define HMELOCK_EXIT(hmebp, tmp1, asi) \ 1673 membar #LoadStore|#StoreStore ;\ 1674 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1675 sta %g0, [tmp1]asi 1676 1677 .seg ".data" 1678hblk_add_panic1: 1679 .ascii "sfmmu_hblk_hash_add: interrupts disabled" 1680 .byte 0 1681hblk_add_panic2: 1682 .ascii "sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not" 1683 .byte 0 1684 .align 4 1685 .seg ".text" 1686 1687 ENTRY_NP(sfmmu_hblk_hash_add) 1688 /* 1689 * %o0 = hmebp 1690 * %o1 = hmeblkp 1691 * %o2 = hblkpa 1692 */ 1693 rdpr %pstate, %o5 1694#ifdef DEBUG 1695 andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ 1696 bnz,pt %icc, 3f /* disabled, panic */ 1697 nop 1698 save %sp, -SA(MINFRAME), %sp 1699 sethi %hi(hblk_add_panic1), %o0 1700 call panic 1701 or %o0, %lo(hblk_add_panic1), %o0 1702 ret 1703 restore 1704 17053: 1706#endif /* DEBUG */ 1707 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1708 mov %o2, %g1 1709 1710 /* 1711 * g1 = hblkpa 1712 */ 1713 ldn [%o0 + HMEBUCK_HBLK], %o4 /* next hmeblk */ 1714 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = next hblkpa */ 1715#ifdef DEBUG 1716 cmp %o4, %g0 1717 bne,pt %xcc, 1f 1718 nop 1719 brz,pt %g2, 1f 1720 nop 1721 wrpr %g0, %o5, %pstate /* enable interrupts */ 1722 save %sp, -SA(MINFRAME), %sp 1723 sethi %hi(hblk_add_panic2), %o0 1724 call panic 1725 or %o0, %lo(hblk_add_panic2), %o0 1726 ret 1727 restore 17281: 1729#endif /* DEBUG */ 1730 /* 1731 * We update hmeblks entries before grabbing lock because the stores 1732 * could take a tlb miss and require the hash lock. The buckets 1733 * are part of the nucleus so we are cool with those stores. 1734 * 1735 * if buckets are not part of the nucleus our game is to 1736 * not touch any other page via va until we drop the lock. 1737 * This guarantees we won't get a tlb miss before the lock release 1738 * since interrupts are disabled. 1739 */ 1740 stn %o4, [%o1 + HMEBLK_NEXT] /* update hmeblk's next */ 1741 stx %g2, [%o1 + HMEBLK_NEXTPA] /* update hmeblk's next pa */ 1742 HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N) 1743 stn %o1, [%o0 + HMEBUCK_HBLK] /* update bucket hblk next */ 1744 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* add hmeblk to list */ 1745 HMELOCK_EXIT(%o0, %g2, ASI_N) 1746 retl 1747 wrpr %g0, %o5, %pstate /* enable interrupts */ 1748 SET_SIZE(sfmmu_hblk_hash_add) 1749 1750 ENTRY_NP(sfmmu_hblk_hash_rm) 1751 /* 1752 * This function removes an hmeblk from the hash chain. 1753 * It is written to guarantee we don't take a tlb miss 1754 * by using physical addresses to update the list. 1755 * 1756 * %o0 = hmebp 1757 * %o1 = hmeblkp 1758 * %o2 = hmeblkp previous pa 1759 * %o3 = hmeblkp previous 1760 */ 1761 1762 mov %o3, %o4 /* o4 = hmeblkp previous */ 1763 1764 rdpr %pstate, %o5 1765#ifdef DEBUG 1766 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1) 1767#endif /* DEBUG */ 1768 /* 1769 * disable interrupts, clear Address Mask to access 64 bit physaddr 1770 */ 1771 andn %o5, PSTATE_IE, %g1 1772 wrpr %g1, 0, %pstate 1773 1774#ifndef sun4v 1775 sethi %hi(dcache_line_mask), %g4 1776 ld [%g4 + %lo(dcache_line_mask)], %g4 1777#endif /* sun4v */ 1778 1779 /* 1780 * if buckets are not part of the nucleus our game is to 1781 * not touch any other page via va until we drop the lock. 1782 * This guarantees we won't get a tlb miss before the lock release 1783 * since interrupts are disabled. 1784 */ 1785 HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N) 1786 ldn [%o0 + HMEBUCK_HBLK], %g2 /* first hmeblk in list */ 1787 cmp %g2, %o1 1788 bne,pt %ncc,1f 1789 mov ASI_MEM, %asi 1790 /* 1791 * hmeblk is first on list 1792 */ 1793 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = hmeblk pa */ 1794 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1795 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1796 stn %o3, [%o0 + HMEBUCK_HBLK] /* write va */ 1797 ba,pt %xcc, 2f 1798 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* write pa */ 17991: 1800 /* hmeblk is not first on list */ 1801 1802 mov %o2, %g3 1803#ifndef sun4v 1804 GET_CPU_IMPL(%g2) 1805 cmp %g2, CHEETAH_IMPL 1806 bge,a,pt %icc, hblk_hash_rm_1 1807 and %o4, %g4, %g2 1808 cmp %g2, SPITFIRE_IMPL 1809 blt %icc, hblk_hash_rm_2 /* no flushing needed for OPL */ 1810 and %o4, %g4, %g2 1811 stxa %g0, [%g2]ASI_DC_TAG /* flush prev pa from dcache */ 1812 add %o4, HMEBLK_NEXT, %o4 1813 and %o4, %g4, %g2 1814 ba hblk_hash_rm_2 1815 stxa %g0, [%g2]ASI_DC_TAG /* flush prev va from dcache */ 1816hblk_hash_rm_1: 1817 1818 stxa %g0, [%g3]ASI_DC_INVAL /* flush prev pa from dcache */ 1819 membar #Sync 1820 add %g3, HMEBLK_NEXT, %g2 1821 stxa %g0, [%g2]ASI_DC_INVAL /* flush prev va from dcache */ 1822hblk_hash_rm_2: 1823 membar #Sync 1824#endif /* sun4v */ 1825 ldxa [%g3 + HMEBLK_NEXTPA] %asi, %g2 /* g2 = hmeblk pa */ 1826 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1827 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1828 stna %o3, [%g3 + HMEBLK_NEXT] %asi /* write va */ 1829 stxa %g1, [%g3 + HMEBLK_NEXTPA] %asi /* write pa */ 18302: 1831 HMELOCK_EXIT(%o0, %g2, ASI_N) 1832 retl 1833 wrpr %g0, %o5, %pstate /* enable interrupts */ 1834 SET_SIZE(sfmmu_hblk_hash_rm) 1835 1836#endif /* lint */ 1837 1838/* 1839 * These macros are used to update global sfmmu hme hash statistics 1840 * in perf critical paths. It is only enabled in debug kernels or 1841 * if SFMMU_STAT_GATHER is defined 1842 */ 1843#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1844#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1845 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1846 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1847 cmp tmp1, hatid ;\ 1848 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1849 set sfmmu_global_stat, tmp1 ;\ 1850 add tmp1, tmp2, tmp1 ;\ 1851 ld [tmp1], tmp2 ;\ 1852 inc tmp2 ;\ 1853 st tmp2, [tmp1] 1854 1855#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1856 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1857 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1858 cmp tmp1, hatid ;\ 1859 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1860 set sfmmu_global_stat, tmp1 ;\ 1861 add tmp1, tmp2, tmp1 ;\ 1862 ld [tmp1], tmp2 ;\ 1863 inc tmp2 ;\ 1864 st tmp2, [tmp1] 1865 1866 1867#else /* DEBUG || SFMMU_STAT_GATHER */ 1868 1869#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1870 1871#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1872 1873#endif /* DEBUG || SFMMU_STAT_GATHER */ 1874 1875/* 1876 * This macro is used to update global sfmmu kstas in non 1877 * perf critical areas so they are enabled all the time 1878 */ 1879#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1880 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1881 add tmp1, statname, tmp1 ;\ 1882 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1883 inc tmp2 ;\ 1884 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1885 1886/* 1887 * These macros are used to update per cpu stats in non perf 1888 * critical areas so they are enabled all the time 1889 */ 1890#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1891 ld [tsbarea + stat], tmp1 ;\ 1892 inc tmp1 ;\ 1893 st tmp1, [tsbarea + stat] 1894 1895/* 1896 * These macros are used to update per cpu stats in non perf 1897 * critical areas so they are enabled all the time 1898 */ 1899#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1900 lduh [tsbarea + stat], tmp1 ;\ 1901 inc tmp1 ;\ 1902 stuh tmp1, [tsbarea + stat] 1903 1904#if defined(KPM_TLBMISS_STATS_GATHER) 1905 /* 1906 * Count kpm dtlb misses separately to allow a different 1907 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1908 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1909 */ 1910#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1911 brgez tagacc, label /* KPM VA? */ ;\ 1912 nop ;\ 1913 CPU_INDEX(tmp1, tsbma) ;\ 1914 sethi %hi(kpmtsbm_area), tsbma ;\ 1915 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1916 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1917 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1918 /* VA range check */ ;\ 1919 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1920 cmp tagacc, val ;\ 1921 blu,pn %xcc, label ;\ 1922 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1923 cmp tagacc, tmp1 ;\ 1924 bgeu,pn %xcc, label ;\ 1925 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1926 inc val ;\ 1927 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1928label: 1929#else 1930#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1931#endif /* KPM_TLBMISS_STATS_GATHER */ 1932 1933#if defined (lint) 1934/* 1935 * The following routines are jumped to from the mmu trap handlers to do 1936 * the setting up to call systrap. They are separate routines instead of 1937 * being part of the handlers because the handlers would exceed 32 1938 * instructions and since this is part of the slow path the jump 1939 * cost is irrelevant. 1940 */ 1941void 1942sfmmu_pagefault(void) 1943{ 1944} 1945 1946void 1947sfmmu_mmu_trap(void) 1948{ 1949} 1950 1951void 1952sfmmu_window_trap(void) 1953{ 1954} 1955 1956void 1957sfmmu_kpm_exception(void) 1958{ 1959} 1960 1961#else /* lint */ 1962 1963#ifdef PTL1_PANIC_DEBUG 1964 .seg ".data" 1965 .global test_ptl1_panic 1966test_ptl1_panic: 1967 .word 0 1968 .align 8 1969 1970 .seg ".text" 1971 .align 4 1972#endif /* PTL1_PANIC_DEBUG */ 1973 1974 1975 ENTRY_NP(sfmmu_pagefault) 1976 SET_GL_REG(1) 1977 USE_ALTERNATE_GLOBALS(%g5) 1978 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1979 rdpr %tt, %g6 1980 cmp %g6, FAST_IMMU_MISS_TT 1981 be,a,pn %icc, 1f 1982 mov T_INSTR_MMU_MISS, %g3 1983 cmp %g6, T_INSTR_MMU_MISS 1984 be,a,pn %icc, 1f 1985 mov T_INSTR_MMU_MISS, %g3 1986 mov %g5, %g2 1987 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1988 cmp %g6, FAST_DMMU_MISS_TT 1989 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1990 cmp %g6, T_DATA_MMU_MISS 1991 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1992 1993#ifdef PTL1_PANIC_DEBUG 1994 /* check if we want to test the tl1 panic */ 1995 sethi %hi(test_ptl1_panic), %g4 1996 ld [%g4 + %lo(test_ptl1_panic)], %g1 1997 st %g0, [%g4 + %lo(test_ptl1_panic)] 1998 cmp %g1, %g0 1999 bne,a,pn %icc, ptl1_panic 2000 or %g0, PTL1_BAD_DEBUG, %g1 2001#endif /* PTL1_PANIC_DEBUG */ 20021: 2003 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 2004 /* 2005 * g2 = tag access reg 2006 * g3.l = type 2007 * g3.h = 0 2008 */ 2009 sethi %hi(trap), %g1 2010 or %g1, %lo(trap), %g1 20112: 2012 ba,pt %xcc, sys_trap 2013 mov -1, %g4 2014 SET_SIZE(sfmmu_pagefault) 2015 2016 ENTRY_NP(sfmmu_mmu_trap) 2017 SET_GL_REG(1) 2018 USE_ALTERNATE_GLOBALS(%g5) 2019 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 2020 rdpr %tt, %g6 2021 cmp %g6, FAST_IMMU_MISS_TT 2022 be,a,pn %icc, 1f 2023 mov T_INSTR_MMU_MISS, %g3 2024 cmp %g6, T_INSTR_MMU_MISS 2025 be,a,pn %icc, 1f 2026 mov T_INSTR_MMU_MISS, %g3 2027 mov %g5, %g2 2028 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 2029 cmp %g6, FAST_DMMU_MISS_TT 2030 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 2031 cmp %g6, T_DATA_MMU_MISS 2032 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 20331: 2034 /* 2035 * g2 = tag access reg 2036 * g3 = type 2037 */ 2038 sethi %hi(sfmmu_tsbmiss_exception), %g1 2039 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 2040 ba,pt %xcc, sys_trap 2041 mov -1, %g4 2042 /*NOTREACHED*/ 2043 SET_SIZE(sfmmu_mmu_trap) 2044 2045 ENTRY_NP(sfmmu_suspend_tl) 2046 SET_GL_REG(1) 2047 USE_ALTERNATE_GLOBALS(%g5) 2048 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 2049 rdpr %tt, %g6 2050 cmp %g6, FAST_IMMU_MISS_TT 2051 be,a,pn %icc, 1f 2052 mov T_INSTR_MMU_MISS, %g3 2053 mov %g5, %g2 2054 cmp %g6, FAST_DMMU_MISS_TT 2055 move %icc, T_DATA_MMU_MISS, %g3 2056 movne %icc, T_DATA_PROT, %g3 20571: 2058 sethi %hi(sfmmu_tsbmiss_suspended), %g1 2059 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 2060 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 2061 ba,pt %xcc, sys_trap 2062 mov PIL_15, %g4 2063 /*NOTREACHED*/ 2064 SET_SIZE(sfmmu_suspend_tl) 2065 2066 /* 2067 * No %g registers in use at this point. 2068 */ 2069 ENTRY_NP(sfmmu_window_trap) 2070 rdpr %tpc, %g1 2071#ifdef sun4v 2072#ifdef DEBUG 2073 /* We assume previous %gl was 1 */ 2074 rdpr %tstate, %g4 2075 srlx %g4, TSTATE_GL_SHIFT, %g4 2076 and %g4, TSTATE_GL_MASK, %g4 2077 cmp %g4, 1 2078 bne,a,pn %icc, ptl1_panic 2079 mov PTL1_BAD_WTRAP, %g1 2080#endif /* DEBUG */ 2081 /* user miss at tl>1. better be the window handler or user_rtt */ 2082 /* in user_rtt? */ 2083 set rtt_fill_start, %g4 2084 cmp %g1, %g4 2085 blu,pn %xcc, 6f 2086 .empty 2087 set rtt_fill_end, %g4 2088 cmp %g1, %g4 2089 bgeu,pn %xcc, 6f 2090 nop 2091 set fault_rtt_fn1, %g1 2092 wrpr %g0, %g1, %tnpc 2093 ba,a 7f 20946: 2095 ! must save this trap level before descending trap stack 2096 ! no need to save %tnpc, either overwritten or discarded 2097 ! already got it: rdpr %tpc, %g1 2098 rdpr %tstate, %g6 2099 rdpr %tt, %g7 2100 ! trap level saved, go get underlying trap type 2101 rdpr %tl, %g5 2102 sub %g5, 1, %g3 2103 wrpr %g3, %tl 2104 rdpr %tt, %g2 2105 wrpr %g5, %tl 2106 ! restore saved trap level 2107 wrpr %g1, %tpc 2108 wrpr %g6, %tstate 2109 wrpr %g7, %tt 2110#else /* sun4v */ 2111 /* user miss at tl>1. better be the window handler */ 2112 rdpr %tl, %g5 2113 sub %g5, 1, %g3 2114 wrpr %g3, %tl 2115 rdpr %tt, %g2 2116 wrpr %g5, %tl 2117#endif /* sun4v */ 2118 and %g2, WTRAP_TTMASK, %g4 2119 cmp %g4, WTRAP_TYPE 2120 bne,pn %xcc, 1f 2121 nop 2122 /* tpc should be in the trap table */ 2123 set trap_table, %g4 2124 cmp %g1, %g4 2125 blt,pn %xcc, 1f 2126 .empty 2127 set etrap_table, %g4 2128 cmp %g1, %g4 2129 bge,pn %xcc, 1f 2130 .empty 2131 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 2132 add %g1, WTRAP_FAULTOFF, %g1 2133 wrpr %g0, %g1, %tnpc 21347: 2135 /* 2136 * some wbuf handlers will call systrap to resolve the fault 2137 * we pass the trap type so they figure out the correct parameters. 2138 * g5 = trap type, g6 = tag access reg 2139 */ 2140 2141 /* 2142 * only use g5, g6, g7 registers after we have switched to alternate 2143 * globals. 2144 */ 2145 SET_GL_REG(1) 2146 USE_ALTERNATE_GLOBALS(%g5) 2147 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 2148 rdpr %tt, %g7 2149 cmp %g7, FAST_IMMU_MISS_TT 2150 be,a,pn %icc, ptl1_panic 2151 mov PTL1_BAD_WTRAP, %g1 2152 cmp %g7, T_INSTR_MMU_MISS 2153 be,a,pn %icc, ptl1_panic 2154 mov PTL1_BAD_WTRAP, %g1 2155 mov T_DATA_PROT, %g5 2156 cmp %g7, FAST_DMMU_MISS_TT 2157 move %icc, T_DATA_MMU_MISS, %g5 2158 cmp %g7, T_DATA_MMU_MISS 2159 move %icc, T_DATA_MMU_MISS, %g5 2160 ! XXXQ AGS re-check out this one 2161 done 21621: 2163 CPU_PADDR(%g1, %g4) 2164 add %g1, CPU_TL1_HDLR, %g1 2165 lda [%g1]ASI_MEM, %g4 2166 brnz,a,pt %g4, sfmmu_mmu_trap 2167 sta %g0, [%g1]ASI_MEM 2168 ba,pt %icc, ptl1_panic 2169 mov PTL1_BAD_TRAP, %g1 2170 SET_SIZE(sfmmu_window_trap) 2171 2172 ENTRY_NP(sfmmu_kpm_exception) 2173 /* 2174 * We have accessed an unmapped segkpm address or a legal segkpm 2175 * address which is involved in a VAC alias conflict prevention. 2176 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 2177 * set. If it is, we will instead note that a fault has occurred 2178 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 2179 * a "retry"). This will step over the faulting instruction. 2180 * Note that this means that a legal segkpm address involved in 2181 * a VAC alias conflict prevention (a rare case to begin with) 2182 * cannot be used in DTrace. 2183 */ 2184 CPU_INDEX(%g1, %g2) 2185 set cpu_core, %g2 2186 sllx %g1, CPU_CORE_SHIFT, %g1 2187 add %g1, %g2, %g1 2188 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 2189 andcc %g2, CPU_DTRACE_NOFAULT, %g0 2190 bz 0f 2191 or %g2, CPU_DTRACE_BADADDR, %g2 2192 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 2193 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 2194 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 2195 done 21960: 2197 TSTAT_CHECK_TL1(1f, %g1, %g2) 21981: 2199 SET_GL_REG(1) 2200 USE_ALTERNATE_GLOBALS(%g5) 2201 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 2202 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 2203 /* 2204 * g2=tagacc g3.l=type g3.h=0 2205 */ 2206 sethi %hi(trap), %g1 2207 or %g1, %lo(trap), %g1 2208 ba,pt %xcc, sys_trap 2209 mov -1, %g4 2210 SET_SIZE(sfmmu_kpm_exception) 2211 2212#endif /* lint */ 2213 2214#if defined (lint) 2215 2216void 2217sfmmu_tsb_miss(void) 2218{ 2219} 2220 2221void 2222sfmmu_kpm_dtsb_miss(void) 2223{ 2224} 2225 2226void 2227sfmmu_kpm_dtsb_miss_small(void) 2228{ 2229} 2230 2231#else /* lint */ 2232 2233#if (IMAP_SEG != 0) 2234#error - ism_map->ism_seg offset is not zero 2235#endif 2236 2237/* 2238 * Copies ism mapping for this ctx in param "ism" if this is a ISM 2239 * tlb miss and branches to label "ismhit". If this is not an ISM 2240 * process or an ISM tlb miss it falls thru. 2241 * 2242 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 2243 * this process. 2244 * If so, it will branch to label "ismhit". If not, it will fall through. 2245 * 2246 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 2247 * so that any other threads of this process will not try and walk the ism 2248 * maps while they are being changed. 2249 * 2250 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 2251 * will make sure of that. This means we can terminate our search on 2252 * the first zero mapping we find. 2253 * 2254 * Parameters: 2255 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 2256 * tsbmiss = address of tsb miss area (in) 2257 * ismseg = contents of ism_seg for this ism map (out) 2258 * ismhat = physical address of imap_ismhat for this ism map (out) 2259 * tmp1 = scratch reg (CLOBBERED) 2260 * tmp2 = scratch reg (CLOBBERED) 2261 * tmp3 = scratch reg (CLOBBERED) 2262 * label: temporary labels 2263 * ismhit: label where to jump to if an ism dtlb miss 2264 * exitlabel:label where to jump if hat is busy due to hat_unshare. 2265 */ 2266#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 2267 label, ismhit) \ 2268 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 2269 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 2270 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 2271label/**/1: ;\ 2272 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 2273 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 2274label/**/2: ;\ 2275 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 2276 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 2277 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 2278 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 2279 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 2280 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 2281 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 2282 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 2283 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 2284 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 2285 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 2286 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 2287 ;\ 2288 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 2289 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 2290 cmp ismhat, tmp1 ;\ 2291 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 2292 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 2293 ;\ 2294 add tmp3, IBLK_NEXTPA, tmp1 ;\ 2295 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 2296 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 2297 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 2298label/**/3: 2299 2300/* 2301 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 2302 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 2303 * Parameters: 2304 * tagacc = reg containing virtual address 2305 * hatid = reg containing sfmmu pointer 2306 * hmeshift = constant/register to shift vaddr to obtain vapg 2307 * hmebp = register where bucket pointer will be stored 2308 * vapg = register where virtual page will be stored 2309 * tmp1, tmp2 = tmp registers 2310 */ 2311 2312 2313#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 2314 vapg, label, tmp1, tmp2) \ 2315 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 2316 brnz,a,pt tmp1, label/**/1 ;\ 2317 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 2318 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 2319 ba,pt %xcc, label/**/2 ;\ 2320 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 2321label/**/1: ;\ 2322 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 2323label/**/2: ;\ 2324 srlx tagacc, hmeshift, vapg ;\ 2325 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 2326 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 2327 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 2328 add hmebp, tmp1, hmebp 2329 2330/* 2331 * hashtag includes bspage + hashno (64 bits). 2332 */ 2333 2334#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 2335 sllx vapg, hmeshift, vapg ;\ 2336 mov hashno, hblktag ;\ 2337 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 2338 or vapg, hblktag, hblktag 2339 2340/* 2341 * Function to traverse hmeblk hash link list and find corresponding match. 2342 * The search is done using physical pointers. It returns the physical address 2343 * and virtual address pointers to the hmeblk that matches with the tag 2344 * provided. 2345 * Parameters: 2346 * hmebp = register that points to hme hash bucket, also used as 2347 * tmp reg (clobbered) 2348 * hmeblktag = register with hmeblk tag match 2349 * hatid = register with hatid 2350 * hmeblkpa = register where physical ptr will be stored 2351 * hmeblkva = register where virtual ptr will be stored 2352 * tmp1 = tmp reg 2353 * label: temporary label 2354 */ 2355 2356#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2357 tsbarea, tmp1, label) \ 2358 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 2359 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2360 add hmebp, HMEBUCK_HBLK, hmeblkva ;\ 2361 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2362 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2363label/**/1: ;\ 2364 brz,pn hmeblkva, label/**/2 ;\ 2365 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2366 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 2367 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2368 add hmebp, CLONGSIZE, hmebp ;\ 2369 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 2370 xor tmp1, hmeblktag, tmp1 ;\ 2371 xor hmebp, hatid, hmebp ;\ 2372 or hmebp, tmp1, hmebp ;\ 2373 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 2374 add hmeblkpa, HMEBLK_NEXT, hmebp ;\ 2375 ldna [hmebp]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2376 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 2377 ba,pt %xcc, label/**/1 ;\ 2378 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2379label/**/2: 2380 2381/* 2382 * Function to traverse hmeblk hash link list and find corresponding match. 2383 * The search is done using physical pointers. It returns the physical address 2384 * and virtual address pointers to the hmeblk that matches with the tag 2385 * provided. 2386 * Parameters: 2387 * hmeblktag = register with hmeblk tag match (rid field is 0) 2388 * hatid = register with hatid (pointer to SRD) 2389 * hmeblkpa = register where physical ptr will be stored 2390 * hmeblkva = register where virtual ptr will be stored 2391 * tmp1 = tmp reg 2392 * tmp2 = tmp reg 2393 * label: temporary label 2394 */ 2395 2396#define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2397 tsbarea, tmp1, tmp2, label) \ 2398label/**/1: ;\ 2399 brz,pn hmeblkva, label/**/4 ;\ 2400 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 2401 add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 2402 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2403 add tmp2, CLONGSIZE, tmp2 ;\ 2404 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 2405 xor tmp1, hmeblktag, tmp1 ;\ 2406 xor tmp2, hatid, tmp2 ;\ 2407 brz,pn tmp2, label/**/3 /* branch on hit */ ;\ 2408 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2409label/**/2: ;\ 2410 ldna [tmp2]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2411 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2412 ba,pt %xcc, label/**/1 ;\ 2413 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2414label/**/3: ;\ 2415 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 2416 bgeu,pt %xcc, label/**/2 ;\ 2417 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2418 and tmp1, BT_ULMASK, tmp2 ;\ 2419 srlx tmp1, BT_ULSHIFT, tmp1 ;\ 2420 sllx tmp1, CLONGSHIFT, tmp1 ;\ 2421 add tsbarea, tmp1, tmp1 ;\ 2422 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 2423 srlx tmp1, tmp2, tmp1 ;\ 2424 btst 0x1, tmp1 ;\ 2425 bz,pn %xcc, label/**/2 ;\ 2426 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2427label/**/4: 2428 2429#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2430#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2431#endif 2432 2433/* 2434 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2435 * he offset for the corresponding hment. 2436 * Parameters: 2437 * In: 2438 * vaddr = register with virtual address 2439 * hmeblkpa = physical pointer to hme_blk 2440 * Out: 2441 * hmentoff = register where hment offset will be stored 2442 * hmemisc = hblk_misc 2443 * Scratch: 2444 * tmp1 2445 */ 2446#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2447 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2448 lda [hmentoff]ASI_MEM, hmemisc ;\ 2449 andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2450 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2451 or %g0, HMEBLK_HME1, hmentoff ;\ 2452 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2453 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2454 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2455 add tmp1, HMEBLK_HME1, hmentoff ;\ 2456label1: 2457 2458/* 2459 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2460 * 2461 * tagacc = (pseudo-)tag access register (in) 2462 * hatid = sfmmu pointer for TSB miss (in) 2463 * tte = tte for TLB miss if found, otherwise clobbered (out) 2464 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2465 * hmeblkva = VA of hment if found, otherwise clobbered (out) 2466 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2467 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2468 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2469 * for this page size. 2470 * hashno = constant/register hash number 2471 * label = temporary label for branching within macro. 2472 * foundlabel = label to jump to when tte is found. 2473 * suspendlabel= label to jump to when tte is suspended. 2474 * exitlabel = label to jump to when tte is not found. 2475 * 2476 */ 2477#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmemisc, \ 2478 hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \ 2479 ;\ 2480 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2481 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2482 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2483 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2484 ;\ 2485 /* ;\ 2486 * tagacc = tagacc ;\ 2487 * hatid = hatid ;\ 2488 * tsbarea = tsbarea ;\ 2489 * tte = hmebp (hme bucket pointer) ;\ 2490 * hmeblkpa = vapg (virtual page) ;\ 2491 * hmemisc, hmeblkva = scratch ;\ 2492 */ ;\ 2493 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2494 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2495 ;\ 2496 /* ;\ 2497 * tagacc = tagacc ;\ 2498 * hatid = hatid ;\ 2499 * tte = hmebp ;\ 2500 * hmeblkpa = CLOBBERED ;\ 2501 * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2502 * hmeblkva = scratch ;\ 2503 */ ;\ 2504 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2505 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2506 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, hmeblkva, \ 2507 tsbarea, tagacc, label/**/1) ;\ 2508 /* ;\ 2509 * tagacc = CLOBBERED ;\ 2510 * tte = CLOBBERED ;\ 2511 * hmeblkpa = hmeblkpa ;\ 2512 * hmeblkva = hmeblkva ;\ 2513 */ ;\ 2514 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2515 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2516 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2517 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2518 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2519 nop ;\ 2520label/**/4: ;\ 2521 /* ;\ 2522 * We have found the hmeblk containing the hment. ;\ 2523 * Now we calculate the corresponding tte. ;\ 2524 * ;\ 2525 * tagacc = tagacc ;\ 2526 * hatid = hatid ;\ 2527 * tte = clobbered ;\ 2528 * hmeblkpa = hmeblkpa ;\ 2529 * hmemisc = hblktag ;\ 2530 * hmeblkva = hmeblkva ;\ 2531 */ ;\ 2532 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2533 label/**/2) ;\ 2534 ;\ 2535 /* ;\ 2536 * tagacc = tagacc ;\ 2537 * hatid = hmentoff ;\ 2538 * tte = clobbered ;\ 2539 * hmeblkpa = hmeblkpa ;\ 2540 * hmemisc = hblk_misc ;\ 2541 * hmeblkva = hmeblkva ;\ 2542 */ ;\ 2543 ;\ 2544 add hatid, SFHME_TTE, hatid ;\ 2545 add hmeblkpa, hatid, hmeblkpa ;\ 2546 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2547 add hmeblkva, hatid, hmeblkva ;\ 2548 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2549 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2550 set TTE_SUSPEND, hatid ;\ 2551 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2552 btst tte, hatid ;\ 2553 bz,pt %xcc, foundlabel ;\ 2554 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2555 ;\ 2556 /* ;\ 2557 * Mapping is suspended, so goto suspend label. ;\ 2558 */ ;\ 2559 ba,pt %xcc, suspendlabel ;\ 2560 nop 2561 2562/* 2563 * GET_SHME_TTE is similar to GET_TTE() except it searches 2564 * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2565 * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2566 * either 0 (not part of scd) or 1 (part of scd). 2567 */ 2568#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, \ 2569 hmemisc, hmeshift, hashno, label, foundlabel, \ 2570 suspendlabel, exitlabel) \ 2571 ;\ 2572 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2573 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2574 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2575 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2576 ;\ 2577 /* ;\ 2578 * tagacc = tagacc ;\ 2579 * hatid = hatid ;\ 2580 * tsbarea = tsbarea ;\ 2581 * tte = hmebp (hme bucket pointer) ;\ 2582 * hmeblkpa = vapg (virtual page) ;\ 2583 * hmemisc, hmeblkva = scratch ;\ 2584 */ ;\ 2585 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2586 ;\ 2587 /* ;\ 2588 * tagacc = tagacc ;\ 2589 * hatid = hatid ;\ 2590 * tsbarea = tsbarea ;\ 2591 * tte = hmebp ;\ 2592 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2593 * hmeblkpa = CLOBBERED ;\ 2594 * hmeblkva = scratch ;\ 2595 */ ;\ 2596 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2597 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2598 ;\ 2599 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2600 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2601 add tte, HMEBUCK_HBLK, hmeblkva ;\ 2602 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2603 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2604 ;\ 2605label/**/8: ;\ 2606 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, hmeblkva, \ 2607 tsbarea, tagacc, tte, label/**/1) ;\ 2608 /* ;\ 2609 * tagacc = CLOBBERED ;\ 2610 * tte = CLOBBERED ;\ 2611 * hmeblkpa = hmeblkpa ;\ 2612 * hmeblkva = hmeblkva ;\ 2613 */ ;\ 2614 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2615 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2616 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2617 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2618 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2619 nop ;\ 2620label/**/4: ;\ 2621 /* ;\ 2622 * We have found the hmeblk containing the hment. ;\ 2623 * Now we calculate the corresponding tte. ;\ 2624 * ;\ 2625 * tagacc = tagacc ;\ 2626 * hatid = hatid ;\ 2627 * tte = clobbered ;\ 2628 * hmeblkpa = hmeblkpa ;\ 2629 * hmemisc = hblktag ;\ 2630 * hmeblkva = hmeblkva ;\ 2631 * tsbarea = tsbmiss area ;\ 2632 */ ;\ 2633 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2634 label/**/2) ;\ 2635 ;\ 2636 /* ;\ 2637 * tagacc = tagacc ;\ 2638 * hatid = hmentoff ;\ 2639 * tte = clobbered ;\ 2640 * hmeblkpa = hmeblkpa ;\ 2641 * hmemisc = hblk_misc ;\ 2642 * hmeblkva = hmeblkva ;\ 2643 * tsbarea = tsbmiss area ;\ 2644 */ ;\ 2645 ;\ 2646 add hatid, SFHME_TTE, hatid ;\ 2647 add hmeblkpa, hatid, hmeblkpa ;\ 2648 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2649 brlz,pt tte, label/**/6 ;\ 2650 add hmeblkva, hatid, hmeblkva ;\ 2651 btst HBLK_SZMASK, hmemisc ;\ 2652 bnz,a,pt %icc, label/**/7 ;\ 2653 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2654 ;\ 2655 /* ;\ 2656 * We found an invalid 8K tte in shme. ;\ 2657 * it may not belong to shme's region since ;\ 2658 * region size/alignment granularity is 8K but different ;\ 2659 * regions don't share hmeblks. Continue the search. ;\ 2660 */ ;\ 2661 sub hmeblkpa, hatid, hmeblkpa ;\ 2662 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2663 srlx tagacc, hmeshift, tte ;\ 2664 add hmeblkpa, HMEBLK_NEXT, hmeblkva ;\ 2665 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2666 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2667 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2668 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2669 ba,a,pt %xcc, label/**/8 ;\ 2670label/**/6: ;\ 2671 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2672 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2673label/**/7: ;\ 2674 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2675 set TTE_SUSPEND, hatid ;\ 2676 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2677 btst tte, hatid ;\ 2678 bz,pt %xcc, foundlabel ;\ 2679 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2680 ;\ 2681 /* ;\ 2682 * Mapping is suspended, so goto suspend label. ;\ 2683 */ ;\ 2684 ba,pt %xcc, suspendlabel ;\ 2685 nop 2686 2687 /* 2688 * KERNEL PROTECTION HANDLER 2689 * 2690 * g1 = tsb8k pointer register (clobbered) 2691 * g2 = tag access register (ro) 2692 * g3 - g7 = scratch registers 2693 * 2694 * Note: This function is patched at runtime for performance reasons. 2695 * Any changes here require sfmmu_patch_ktsb fixed. 2696 */ 2697 ENTRY_NP(sfmmu_kprot_trap) 2698 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2699sfmmu_kprot_patch_ktsb_base: 2700 RUNTIME_PATCH_SETX(%g1, %g6) 2701 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2702sfmmu_kprot_patch_ktsb_szcode: 2703 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2704 2705 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2706 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2707 2708 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2709sfmmu_kprot_patch_ktsb4m_base: 2710 RUNTIME_PATCH_SETX(%g3, %g6) 2711 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2712sfmmu_kprot_patch_ktsb4m_szcode: 2713 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2714 2715 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2716 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2717 2718 CPU_TSBMISS_AREA(%g6, %g7) 2719 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2720 ba,pt %xcc, sfmmu_tsb_miss_tt 2721 nop 2722 2723 /* 2724 * USER PROTECTION HANDLER 2725 * 2726 * g1 = tsb8k pointer register (ro) 2727 * g2 = tag access register (ro) 2728 * g3 = faulting context (clobbered, currently not used) 2729 * g4 - g7 = scratch registers 2730 */ 2731 ALTENTRY(sfmmu_uprot_trap) 2732#ifdef sun4v 2733 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2734 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2735 2736 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2737 brlz,pt %g3, 9f /* check for 2nd TSB */ 2738 nop 2739 2740 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2741 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2742 2743#else /* sun4v */ 2744#ifdef UTSB_PHYS 2745 /* g1 = first TSB entry ptr */ 2746 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2747 brlz,pt %g3, 9f /* check for 2nd TSB */ 2748 nop 2749 2750 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2751 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2752#else /* UTSB_PHYS */ 2753 brgez,pt %g1, 9f /* check for 2nd TSB */ 2754 mov -1, %g3 /* set second tsbe ptr to -1 */ 2755 2756 mov %g2, %g7 2757 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2758 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2759 mov %g1, %g7 2760 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2761#endif /* UTSB_PHYS */ 2762#endif /* sun4v */ 27639: 2764 CPU_TSBMISS_AREA(%g6, %g7) 2765 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2766 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2767 nop 2768 2769 /* 2770 * Kernel 8K page iTLB miss. We also get here if we took a 2771 * fast instruction access mmu miss trap while running in 2772 * invalid context. 2773 * 2774 * %g1 = 8K TSB pointer register (not used, clobbered) 2775 * %g2 = tag access register (used) 2776 * %g3 = faulting context id (used) 2777 * %g7 = TSB tag to match (used) 2778 */ 2779 .align 64 2780 ALTENTRY(sfmmu_kitlb_miss) 2781 brnz,pn %g3, tsb_tl0_noctxt 2782 nop 2783 2784 /* kernel miss */ 2785 /* get kernel tsb pointer */ 2786 /* we patch the next set of instructions at run time */ 2787 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2788iktsbbase: 2789 RUNTIME_PATCH_SETX(%g4, %g5) 2790 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2791 2792iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2793 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2794 or %g4, %g1, %g1 ! form tsb ptr 2795 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2796 cmp %g4, %g7 2797 bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2798 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2799 2800 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2801 bz,pn %icc, exec_fault 2802 nop 2803 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2804 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2805 retry 2806 2807iktsb4mbase: 2808 RUNTIME_PATCH_SETX(%g4, %g6) 2809 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2810iktsb4m: 2811 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2812 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2813 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2814 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2815 cmp %g4, %g7 2816 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2817 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2818 bz,pn %icc, exec_fault 2819 nop 2820 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2821 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2822 retry 2823 2824 /* 2825 * Kernel dTLB miss. We also get here if we took a fast data 2826 * access mmu miss trap while running in invalid context. 2827 * 2828 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2829 * We select the TSB miss handler to branch to depending on 2830 * the virtual address of the access. In the future it may 2831 * be desirable to separate kpm TTEs into their own TSB, 2832 * in which case all that needs to be done is to set 2833 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2834 * early in the miss if we detect a kpm VA to a new handler. 2835 * 2836 * %g1 = 8K TSB pointer register (not used, clobbered) 2837 * %g2 = tag access register (used) 2838 * %g3 = faulting context id (used) 2839 */ 2840 .align 64 2841 ALTENTRY(sfmmu_kdtlb_miss) 2842 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2843 nop 2844 2845 /* Gather some stats for kpm misses in the TLB. */ 2846 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2847 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2848 2849 /* 2850 * Get first TSB offset and look for 8K/64K/512K mapping 2851 * using the 8K virtual page as the index. 2852 * 2853 * We patch the next set of instructions at run time; 2854 * any changes here require sfmmu_patch_ktsb changes too. 2855 */ 2856dktsbbase: 2857 RUNTIME_PATCH_SETX(%g7, %g6) 2858 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2859 2860dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2861 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2862 2863 /* 2864 * At this point %g1 is our index into the TSB. 2865 * We just masked off enough bits of the VA depending 2866 * on our TSB size code. 2867 */ 2868 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2869 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2870 cmp %g6, %g4 ! compare tag 2871 bne,pn %xcc, dktsb4m_kpmcheck_small 2872 add %g7, %g1, %g1 /* form tsb ptr */ 2873 TT_TRACE(trace_tsbhit) 2874 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2875 /* trapstat expects tte in %g5 */ 2876 retry 2877 2878 /* 2879 * If kpm is using large pages, the following instruction needs 2880 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2881 * so that we will probe the 4M TSB regardless of the VA. In 2882 * the case kpm is using small pages, we know no large kernel 2883 * mappings are located above 0x80000000.00000000 so we skip the 2884 * probe as an optimization. 2885 */ 2886dktsb4m_kpmcheck_small: 2887 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2888 /* delay slot safe, below */ 2889 2890 /* 2891 * Get second TSB offset and look for 4M mapping 2892 * using 4M virtual page as the TSB index. 2893 * 2894 * Here: 2895 * %g1 = 8K TSB pointer. Don't squash it. 2896 * %g2 = tag access register (we still need it) 2897 */ 2898 srlx %g2, MMU_PAGESHIFT4M, %g3 2899 2900 /* 2901 * We patch the next set of instructions at run time; 2902 * any changes here require sfmmu_patch_ktsb changes too. 2903 */ 2904dktsb4mbase: 2905 RUNTIME_PATCH_SETX(%g7, %g6) 2906 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2907dktsb4m: 2908 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2909 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2910 2911 /* 2912 * At this point %g3 is our index into the TSB. 2913 * We just masked off enough bits of the VA depending 2914 * on our TSB size code. 2915 */ 2916 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2917 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2918 cmp %g6, %g4 ! compare tag 2919 2920dktsb4m_tsbmiss: 2921 bne,pn %xcc, dktsb4m_kpmcheck 2922 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2923 TT_TRACE(trace_tsbhit) 2924 /* we don't check TTE size here since we assume 4M TSB is separate */ 2925 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2926 /* trapstat expects tte in %g5 */ 2927 retry 2928 2929 /* 2930 * So, we failed to find a valid TTE to match the faulting 2931 * address in either TSB. There are a few cases that could land 2932 * us here: 2933 * 2934 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2935 * to sfmmu_tsb_miss_tt to handle the miss. 2936 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2937 * 4M TSB. Let segkpm handle it. 2938 * 2939 * Note that we shouldn't land here in the case of a kpm VA when 2940 * kpm_smallpages is active -- we handled that case earlier at 2941 * dktsb4m_kpmcheck_small. 2942 * 2943 * At this point: 2944 * g1 = 8K-indexed primary TSB pointer 2945 * g2 = tag access register 2946 * g3 = 4M-indexed secondary TSB pointer 2947 */ 2948dktsb4m_kpmcheck: 2949 cmp %g2, %g0 2950 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2951 nop 2952 ba,a,pt %icc, sfmmu_tsb_miss_tt 2953 nop 2954 2955#ifdef sun4v 2956 /* 2957 * User instruction miss w/ single TSB. 2958 * The first probe covers 8K, 64K, and 512K page sizes, 2959 * because 64K and 512K mappings are replicated off 8K 2960 * pointer. 2961 * 2962 * g1 = tsb8k pointer register 2963 * g2 = tag access register 2964 * g3 - g6 = scratch registers 2965 * g7 = TSB tag to match 2966 */ 2967 .align 64 2968 ALTENTRY(sfmmu_uitlb_fastpath) 2969 2970 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2971 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2972 ba,pn %xcc, sfmmu_tsb_miss_tt 2973 mov -1, %g3 2974 2975 /* 2976 * User data miss w/ single TSB. 2977 * The first probe covers 8K, 64K, and 512K page sizes, 2978 * because 64K and 512K mappings are replicated off 8K 2979 * pointer. 2980 * 2981 * g1 = tsb8k pointer register 2982 * g2 = tag access register 2983 * g3 - g6 = scratch registers 2984 * g7 = TSB tag to match 2985 */ 2986 .align 64 2987 ALTENTRY(sfmmu_udtlb_fastpath) 2988 2989 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2990 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2991 ba,pn %xcc, sfmmu_tsb_miss_tt 2992 mov -1, %g3 2993 2994 /* 2995 * User instruction miss w/ multiple TSBs (sun4v). 2996 * The first probe covers 8K, 64K, and 512K page sizes, 2997 * because 64K and 512K mappings are replicated off 8K 2998 * pointer. Second probe covers 4M page size only. 2999 * 3000 * Just like sfmmu_udtlb_slowpath, except: 3001 * o Uses ASI_ITLB_IN 3002 * o checks for execute permission 3003 * o No ISM prediction. 3004 * 3005 * g1 = tsb8k pointer register 3006 * g2 = tag access register 3007 * g3 - g6 = scratch registers 3008 * g7 = TSB tag to match 3009 */ 3010 .align 64 3011 ALTENTRY(sfmmu_uitlb_slowpath) 3012 3013 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 3014 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 3015 /* g4 - g5 = clobbered here */ 3016 3017 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3018 /* g1 = first TSB pointer, g3 = second TSB pointer */ 3019 srlx %g2, TAG_VALO_SHIFT, %g7 3020 PROBE_2ND_ITSB(%g3, %g7) 3021 /* NOT REACHED */ 3022 3023#else /* sun4v */ 3024 3025 /* 3026 * User instruction miss w/ multiple TSBs (sun4u). 3027 * The first probe covers 8K, 64K, and 512K page sizes, 3028 * because 64K and 512K mappings are replicated off 8K 3029 * pointer. Probe of 1st TSB has already been done prior to entry 3030 * into this routine. For the UTSB_PHYS case we probe up to 3 3031 * valid other TSBs in the following order: 3032 * 1) shared TSB for 4M-256M pages 3033 * 2) private TSB for 4M-256M pages 3034 * 3) shared TSB for 8K-512K pages 3035 * 3036 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs 3037 * 4M-256M pages. 3038 * 3039 * Just like sfmmu_udtlb_slowpath, except: 3040 * o Uses ASI_ITLB_IN 3041 * o checks for execute permission 3042 * o No ISM prediction. 3043 * 3044 * g1 = tsb8k pointer register 3045 * g2 = tag access register 3046 * g4 - g6 = scratch registers 3047 * g7 = TSB tag to match 3048 */ 3049 .align 64 3050 ALTENTRY(sfmmu_uitlb_slowpath) 3051 3052#ifdef UTSB_PHYS 3053 3054 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3055 brlz,pt %g6, 1f 3056 nop 3057 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3058 PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail) 30591: 3060 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3061 brlz,pt %g3, 2f 3062 nop 3063 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3064 PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail) 30652: 3066 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3067 brlz,pt %g6, sfmmu_tsb_miss_tt 3068 nop 3069 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3070 PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail) 3071 ba,pn %xcc, sfmmu_tsb_miss_tt 3072 nop 3073 3074#else /* UTSB_PHYS */ 3075 mov %g1, %g3 /* save tsb8k reg in %g3 */ 3076 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 3077 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 3078 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 3079 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 3080 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 3081 /* g1 = first TSB pointer, g3 = second TSB pointer */ 3082 srlx %g2, TAG_VALO_SHIFT, %g7 3083 PROBE_2ND_ITSB(%g3, %g7, isynth) 3084 ba,pn %xcc, sfmmu_tsb_miss_tt 3085 nop 3086 3087#endif /* UTSB_PHYS */ 3088#endif /* sun4v */ 3089 3090#if defined(sun4u) && defined(UTSB_PHYS) 3091 3092 /* 3093 * We come here for ism predict DTLB_MISS case or if 3094 * if probe in first TSB failed. 3095 */ 3096 3097 .align 64 3098 ALTENTRY(sfmmu_udtlb_slowpath_noismpred) 3099 3100 /* 3101 * g1 = tsb8k pointer register 3102 * g2 = tag access register 3103 * g4 - %g6 = scratch registers 3104 * g7 = TSB tag to match 3105 */ 3106 3107 /* 3108 * ISM non-predict probe order 3109 * probe 1ST_TSB (8K index) 3110 * probe 2ND_TSB (4M index) 3111 * probe 4TH_TSB (4M index) 3112 * probe 3RD_TSB (8K index) 3113 * 3114 * We already probed first TSB in DTLB_MISS handler. 3115 */ 3116 3117 /* 3118 * Private 2ND TSB 4M-256 pages 3119 */ 3120 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3121 brlz,pt %g3, 1f 3122 nop 3123 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3124 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3125 3126 /* 3127 * Shared Context 4TH TSB 4M-256 pages 3128 */ 31291: 3130 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3131 brlz,pt %g6, 2f 3132 nop 3133 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3134 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail) 3135 3136 /* 3137 * Shared Context 3RD TSB 8K-512K pages 3138 */ 31392: 3140 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3141 brlz,pt %g6, sfmmu_tsb_miss_tt 3142 nop 3143 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3144 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail) 3145 ba,pn %xcc, sfmmu_tsb_miss_tt 3146 nop 3147 3148 .align 64 3149 ALTENTRY(sfmmu_udtlb_slowpath_ismpred) 3150 3151 /* 3152 * g1 = tsb8k pointer register 3153 * g2 = tag access register 3154 * g4 - g6 = scratch registers 3155 * g7 = TSB tag to match 3156 */ 3157 3158 /* 3159 * ISM predict probe order 3160 * probe 4TH_TSB (4M index) 3161 * probe 2ND_TSB (4M index) 3162 * probe 1ST_TSB (8K index) 3163 * probe 3RD_TSB (8K index) 3164 3165 /* 3166 * Shared Context 4TH TSB 4M-256 pages 3167 */ 3168 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3169 brlz,pt %g6, 4f 3170 nop 3171 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3172 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2) 3173 3174 /* 3175 * Private 2ND TSB 4M-256 pages 3176 */ 31774: 3178 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3179 brlz,pt %g3, 5f 3180 nop 3181 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3182 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2) 3183 31845: 3185 PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2) 3186 3187 /* 3188 * Shared Context 3RD TSB 8K-512K pages 3189 */ 3190 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3191 brlz,pt %g6, 6f 3192 nop 3193 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3194 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2) 31956: 3196 ba,pn %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */ 3197 nop 3198 3199#else /* sun4u && UTSB_PHYS */ 3200 3201 .align 64 3202 ALTENTRY(sfmmu_udtlb_slowpath) 3203 3204 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 3205 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 3206 mov %g1, %g3 3207 3208udtlb_miss_probefirst: 3209 /* 3210 * g1 = 8K TSB pointer register 3211 * g2 = tag access register 3212 * g3 = (potentially) second TSB entry ptr 3213 * g6 = ism pred. 3214 * g7 = vpg_4m 3215 */ 3216#ifdef sun4v 3217 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 3218 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3219 3220 /* 3221 * Here: 3222 * g1 = first TSB pointer 3223 * g2 = tag access reg 3224 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3225 */ 3226 brgz,pn %g6, sfmmu_tsb_miss_tt 3227 nop 3228#else /* sun4v */ 3229 mov %g1, %g4 3230 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 3231 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3232 3233 /* 3234 * Here: 3235 * g1 = first TSB pointer 3236 * g2 = tag access reg 3237 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3238 */ 3239 brgz,pn %g6, sfmmu_tsb_miss_tt 3240 nop 3241 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 3242 /* fall through in 8K->4M probe order */ 3243#endif /* sun4v */ 3244 3245udtlb_miss_probesecond: 3246 /* 3247 * Look in the second TSB for the TTE 3248 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 3249 * g2 = tag access reg 3250 * g3 = 8K TSB pointer register 3251 * g6 = ism pred. 3252 * g7 = vpg_4m 3253 */ 3254#ifdef sun4v 3255 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 3256 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3257 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 3258#else /* sun4v */ 3259 mov %g3, %g7 3260 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 3261 /* %g2 clobbered, %g3 =second tsbe ptr */ 3262 mov MMU_TAG_ACCESS, %g2 3263 ldxa [%g2]ASI_DMMU, %g2 3264#endif /* sun4v */ 3265 3266 srlx %g2, TAG_VALO_SHIFT, %g7 3267 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3268 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 3269 brgz,pn %g6, udtlb_miss_probefirst 3270 nop 3271 3272 /* fall through to sfmmu_tsb_miss_tt */ 3273#endif /* sun4u && UTSB_PHYS */ 3274 3275 3276 ALTENTRY(sfmmu_tsb_miss_tt) 3277 TT_TRACE(trace_tsbmiss) 3278 /* 3279 * We get here if there is a TSB miss OR a write protect trap. 3280 * 3281 * g1 = First TSB entry pointer 3282 * g2 = tag access register 3283 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 3284 * g4 - g7 = scratch registers 3285 */ 3286 3287 ALTENTRY(sfmmu_tsb_miss) 3288 3289 /* 3290 * If trapstat is running, we need to shift the %tpc and %tnpc to 3291 * point to trapstat's TSB miss return code (note that trapstat 3292 * itself will patch the correct offset to add). 3293 */ 3294 rdpr %tl, %g7 3295 cmp %g7, 1 3296 ble,pt %xcc, 0f 3297 sethi %hi(KERNELBASE), %g6 3298 rdpr %tpc, %g7 3299 or %g6, %lo(KERNELBASE), %g6 3300 cmp %g7, %g6 3301 bgeu,pt %xcc, 0f 3302 /* delay slot safe */ 3303 3304 ALTENTRY(tsbmiss_trapstat_patch_point) 3305 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 3306 wrpr %g7, %tpc 3307 add %g7, 4, %g7 3308 wrpr %g7, %tnpc 33090: 3310 CPU_TSBMISS_AREA(%g6, %g7) 3311 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save 1ST tsb pointer */ 3312 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save 2ND tsb pointer */ 3313 3314 sllx %g2, TAGACC_CTX_LSHIFT, %g3 3315 brz,a,pn %g3, 1f /* skip ahead if kernel */ 3316 ldn [%g6 + TSBMISS_KHATID], %g7 3317 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 3318 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 3319 3320 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 3321 3322 cmp %g3, INVALID_CONTEXT 3323 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 3324 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 3325 3326#if defined(sun4v) || defined(UTSB_PHYS) 3327 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 3328 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 3329 stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 3330#endif /* sun4v || UTSB_PHYS */ 3331 3332 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 3333 /* 3334 * The miss wasn't in an ISM segment. 3335 * 3336 * %g1 %g3, %g4, %g5, %g7 all clobbered 3337 * %g2 = (pseudo) tag access 3338 */ 3339 3340 ba,pt %icc, 2f 3341 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 3342 33431: 3344 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 3345 /* 3346 * 8K and 64K hash. 3347 */ 33482: 3349 3350 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3351 MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte, 3352 sfmmu_suspend_tl, tsb_512K) 3353 /* NOT REACHED */ 3354 3355tsb_512K: 3356 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3357 brz,pn %g5, 3f 3358 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3359 and %g4, HAT_512K_FLAG, %g5 3360 3361 /* 3362 * Note that there is a small window here where we may have 3363 * a 512k page in the hash list but have not set the HAT_512K_FLAG 3364 * flag yet, so we will skip searching the 512k hash list. 3365 * In this case we will end up in pagefault which will find 3366 * the mapping and return. So, in this instance we will end up 3367 * spending a bit more time resolving this TSB miss, but it can 3368 * only happen once per process and even then, the chances of that 3369 * are very small, so it's not worth the extra overhead it would 3370 * take to close this window. 3371 */ 3372 brz,pn %g5, tsb_4M 3373 nop 33743: 3375 /* 3376 * 512K hash 3377 */ 3378 3379 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3380 MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte, 3381 sfmmu_suspend_tl, tsb_4M) 3382 /* NOT REACHED */ 3383 3384tsb_4M: 3385 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3386 brz,pn %g5, 4f 3387 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3388 and %g4, HAT_4M_FLAG, %g5 3389 brz,pn %g5, tsb_32M 3390 nop 33914: 3392 /* 3393 * 4M hash 3394 */ 3395 3396 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3397 MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte, 3398 sfmmu_suspend_tl, tsb_32M) 3399 /* NOT REACHED */ 3400 3401tsb_32M: 3402 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3403#ifdef sun4v 3404 brz,pn %g5, 6f 3405#else 3406 brz,pn %g5, tsb_pagefault 3407#endif 3408 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3409 and %g4, HAT_32M_FLAG, %g5 3410 brz,pn %g5, tsb_256M 3411 nop 34125: 3413 /* 3414 * 32M hash 3415 */ 3416 3417 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3418 MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte, 3419 sfmmu_suspend_tl, tsb_256M) 3420 /* NOT REACHED */ 3421 3422#if defined(sun4u) && !defined(UTSB_PHYS) 3423#define tsb_shme tsb_pagefault 3424#endif 3425tsb_256M: 3426 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3427 and %g4, HAT_256M_FLAG, %g5 3428 brz,pn %g5, tsb_shme 3429 nop 34306: 3431 /* 3432 * 256M hash 3433 */ 3434 3435 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3436 MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte, 3437 sfmmu_suspend_tl, tsb_shme) 3438 /* NOT REACHED */ 3439 3440tsb_checktte: 3441 /* 3442 * g1 = hblk_misc 3443 * g2 = tagacc 3444 * g3 = tte 3445 * g4 = tte pa 3446 * g5 = tte va 3447 * g6 = tsbmiss area 3448 * g7 = hatid 3449 */ 3450 brlz,a,pt %g3, tsb_validtte 3451 rdpr %tt, %g7 3452 3453#if defined(sun4u) && !defined(UTSB_PHYS) 3454#undef tsb_shme 3455 ba tsb_pagefault 3456 nop 3457#else /* sun4u && !UTSB_PHYS */ 3458 3459tsb_shme: 3460 /* 3461 * g2 = tagacc 3462 * g6 = tsbmiss area 3463 */ 3464 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3465 brz,pn %g5, tsb_pagefault 3466 nop 3467 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3468 brz,pn %g7, tsb_pagefault 3469 nop 3470 3471 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3472 MMU_PAGESHIFT64K, TTE64K, tsb_shme_l8K, tsb_shme_checktte, 3473 sfmmu_suspend_tl, tsb_shme_512K) 3474 /* NOT REACHED */ 3475 3476tsb_shme_512K: 3477 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3478 and %g4, HAT_512K_FLAG, %g5 3479 brz,pn %g5, tsb_shme_4M 3480 nop 3481 3482 /* 3483 * 512K hash 3484 */ 3485 3486 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3487 MMU_PAGESHIFT512K, TTE512K, tsb_shme_l512K, tsb_shme_checktte, 3488 sfmmu_suspend_tl, tsb_shme_4M) 3489 /* NOT REACHED */ 3490 3491tsb_shme_4M: 3492 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3493 and %g4, HAT_4M_FLAG, %g5 3494 brz,pn %g5, tsb_shme_32M 3495 nop 34964: 3497 /* 3498 * 4M hash 3499 */ 3500 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3501 MMU_PAGESHIFT4M, TTE4M, tsb_shme_l4M, tsb_shme_checktte, 3502 sfmmu_suspend_tl, tsb_shme_32M) 3503 /* NOT REACHED */ 3504 3505tsb_shme_32M: 3506 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3507 and %g4, HAT_32M_FLAG, %g5 3508 brz,pn %g5, tsb_shme_256M 3509 nop 3510 3511 /* 3512 * 32M hash 3513 */ 3514 3515 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3516 MMU_PAGESHIFT32M, TTE32M, tsb_shme_l32M, tsb_shme_checktte, 3517 sfmmu_suspend_tl, tsb_shme_256M) 3518 /* NOT REACHED */ 3519 3520tsb_shme_256M: 3521 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3522 and %g4, HAT_256M_FLAG, %g5 3523 brz,pn %g5, tsb_pagefault 3524 nop 3525 3526 /* 3527 * 256M hash 3528 */ 3529 3530 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3531 MMU_PAGESHIFT256M, TTE256M, tsb_shme_l256M, tsb_shme_checktte, 3532 sfmmu_suspend_tl, tsb_pagefault) 3533 /* NOT REACHED */ 3534 3535tsb_shme_checktte: 3536 3537 brgez,pn %g3, tsb_pagefault 3538 rdpr %tt, %g7 3539 /* 3540 * g1 = ctx1 flag 3541 * g3 = tte 3542 * g4 = tte pa 3543 * g5 = tte va 3544 * g6 = tsbmiss area 3545 * g7 = tt 3546 */ 3547 3548 brz,pt %g1, tsb_validtte 3549 nop 3550 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3551 or %g1, HAT_CHKCTX1_FLAG, %g1 3552 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3553 3554 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3555#endif /* sun4u && !UTSB_PHYS */ 3556 3557tsb_validtte: 3558 /* 3559 * g3 = tte 3560 * g4 = tte pa 3561 * g5 = tte va 3562 * g6 = tsbmiss area 3563 * g7 = tt 3564 */ 3565 3566 /* 3567 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3568 */ 3569 cmp %g7, FAST_PROT_TT 3570 bne,pt %icc, 4f 3571 nop 3572 3573 TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod, 3574 tsb_protfault) 3575 3576 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3577#ifdef sun4v 3578 MMU_FAULT_STATUS_AREA(%g7) 3579 ldx [%g7 + MMFSA_D_ADDR], %g5 /* load fault addr for later */ 3580#else /* sun4v */ 3581 mov MMU_TAG_ACCESS, %g5 3582 ldxa [%g5]ASI_DMMU, %g5 3583#endif /* sun4v */ 3584 ba,pt %xcc, tsb_update_tl1 3585 nop 35864: 3587 /* 3588 * If ITLB miss check exec bit. 3589 * If not set treat as invalid TTE. 3590 */ 3591 cmp %g7, T_INSTR_MMU_MISS 3592 be,pn %icc, 5f 3593 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3594 cmp %g7, FAST_IMMU_MISS_TT 3595 bne,pt %icc, 3f 3596 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 35975: 3598 bz,pn %icc, tsb_protfault 3599 nop 3600 36013: 3602 /* 3603 * Set reference bit if not already set 3604 */ 3605 TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref) 3606 3607 /* 3608 * Now, load into TSB/TLB. At this point: 3609 * g3 = tte 3610 * g4 = patte 3611 * g6 = tsbmiss area 3612 */ 3613 rdpr %tt, %g7 3614#ifdef sun4v 3615 MMU_FAULT_STATUS_AREA(%g2) 3616 cmp %g7, T_INSTR_MMU_MISS 3617 be,a,pt %icc, 9f 3618 nop 3619 cmp %g7, FAST_IMMU_MISS_TT 3620 be,a,pt %icc, 9f 3621 nop 3622 add %g2, MMFSA_D_, %g2 36239: 3624 ldx [%g2 + MMFSA_CTX_], %g7 3625 sllx %g7, TTARGET_CTX_SHIFT, %g7 3626 ldx [%g2 + MMFSA_ADDR_], %g2 3627 mov %g2, %g5 ! load the fault addr for later use 3628 srlx %g2, TTARGET_VA_SHIFT, %g2 3629 or %g2, %g7, %g2 3630#else /* sun4v */ 3631 mov MMU_TAG_ACCESS, %g5 3632 cmp %g7, FAST_IMMU_MISS_TT 3633 be,a,pt %icc, 9f 3634 ldxa [%g0]ASI_IMMU, %g2 3635 ldxa [%g0]ASI_DMMU, %g2 3636 ba,pt %icc, tsb_update_tl1 3637 ldxa [%g5]ASI_DMMU, %g5 36389: 3639 ldxa [%g5]ASI_IMMU, %g5 3640#endif /* sun4v */ 3641 3642tsb_update_tl1: 3643 srlx %g2, TTARGET_CTX_SHIFT, %g7 3644 brz,pn %g7, tsb_kernel 3645#ifdef sun4v 3646 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3647#else /* sun4v */ 3648 srlx %g3, TTE_SZ_SHFT, %g7 3649#endif /* sun4v */ 3650 3651tsb_user: 3652#ifdef sun4v 3653 cmp %g7, TTE4M 3654 bge,pn %icc, tsb_user4m 3655 nop 3656#else /* sun4v */ 3657 cmp %g7, TTESZ_VALID | TTE4M 3658 be,pn %icc, tsb_user4m 3659 srlx %g3, TTE_SZ2_SHFT, %g7 3660 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3661#ifdef ITLB_32M_256M_SUPPORT 3662 bnz,pn %icc, tsb_user4m 3663 nop 3664#else /* ITLB_32M_256M_SUPPORT */ 3665 bnz,a,pn %icc, tsb_user_pn_synth 3666 nop 3667#endif /* ITLB_32M_256M_SUPPORT */ 3668#endif /* sun4v */ 3669 3670tsb_user8k: 3671#if defined(sun4v) || defined(UTSB_PHYS) 3672 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3673 and %g7, HAT_CHKCTX1_FLAG, %g1 3674 brz,a,pn %g1, 1f 3675 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3676 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3677 brlz,a,pn %g1, ptl1_panic ! if no shared 3RD tsb 3678 mov PTL1_NO_SCDTSB8K, %g1 ! panic 3679 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 36801: 3681#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3682 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3683#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3684 3685#ifndef UTSB_PHYS 3686 mov ASI_N, %g7 ! user TSBs accessed by VA 3687 mov %g7, %asi 3688#endif /* !UTSB_PHYS */ 3689 3690 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5) 3691 3692 rdpr %tt, %g5 3693#ifdef sun4v 3694 cmp %g5, T_INSTR_MMU_MISS 3695 be,a,pn %xcc, 9f 3696 mov %g3, %g5 3697#endif /* sun4v */ 3698 cmp %g5, FAST_IMMU_MISS_TT 3699 be,pn %xcc, 9f 3700 mov %g3, %g5 3701 3702 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3703 ! trapstat wants TTE in %g5 3704 retry 37059: 3706 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3707 ! trapstat wants TTE in %g5 3708 retry 3709 3710tsb_user4m: 3711#if defined(sun4v) || defined(UTSB_PHYS) 3712 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3713 and %g7, HAT_CHKCTX1_FLAG, %g1 3714 brz,a,pn %g1, 4f 3715 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3716 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr 3717 brlz,a,pn %g1, 5f ! if no shared 4TH TSB 3718 nop 3719 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 3720 3721#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3722 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3723#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 37244: 3725 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3726 nop 3727 3728#ifndef UTSB_PHYS 3729 mov ASI_N, %g7 ! user TSBs accessed by VA 3730 mov %g7, %asi 3731#endif /* UTSB_PHYS */ 3732 3733 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6) 3734 37355: 3736 rdpr %tt, %g5 3737#ifdef sun4v 3738 cmp %g5, T_INSTR_MMU_MISS 3739 be,a,pn %xcc, 9f 3740 mov %g3, %g5 3741#endif /* sun4v */ 3742 cmp %g5, FAST_IMMU_MISS_TT 3743 be,pn %xcc, 9f 3744 mov %g3, %g5 3745 3746 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3747 ! trapstat wants TTE in %g5 3748 retry 37499: 3750 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3751 ! trapstat wants TTE in %g5 3752 retry 3753 3754#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3755 /* 3756 * Panther ITLB synthesis. 3757 * The Panther 32M and 256M ITLB code simulates these two large page 3758 * sizes with 4M pages, to provide support for programs, for example 3759 * Java, that may copy instructions into a 32M or 256M data page and 3760 * then execute them. The code below generates the 4M pfn bits and 3761 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3762 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3763 * are ignored by the hardware. 3764 * 3765 * Now, load into TSB/TLB. At this point: 3766 * g2 = tagtarget 3767 * g3 = tte 3768 * g4 = patte 3769 * g5 = tt 3770 * g6 = tsbmiss area 3771 */ 3772tsb_user_pn_synth: 3773 rdpr %tt, %g5 3774 cmp %g5, FAST_IMMU_MISS_TT 3775 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3776 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3777 bz,pn %icc, 4b /* if not, been here before */ 3778 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3779 brlz,a,pn %g1, 5f /* no 2nd tsb */ 3780 mov %g3, %g5 3781 3782 mov MMU_TAG_ACCESS, %g7 3783 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3784 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3785 3786 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3787 mov %g7, %asi 3788 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */ 37895: 3790 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3791 retry 3792 3793tsb_user_itlb_synth: 3794 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = 2ND TSB */ 3795 3796 mov MMU_TAG_ACCESS, %g7 3797 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3798 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3799 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3800 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3801 3802 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3803 mov %g7, %asi 3804 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */ 38057: 3806 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3807 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3808 retry 3809#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3810 3811tsb_kernel: 3812 rdpr %tt, %g5 3813#ifdef sun4v 3814 cmp %g7, TTE4M 3815 bge,pn %icc, 5f 3816#else 3817 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3818 be,pn %icc, 5f 3819#endif /* sun4v */ 3820 nop 3821 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8K TSB ptr 3822 ba,pt %xcc, 6f 3823 nop 38245: 3825 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4M TSB ptr 3826 brlz,pn %g1, 3f /* skip programming if 4M TSB ptr is -1 */ 3827 nop 38286: 3829#ifndef sun4v 3830tsb_kernel_patch_asi: 3831 or %g0, RUNTIME_PATCH, %g6 3832 mov %g6, %asi ! XXX avoid writing to %asi !! 3833#endif 3834 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7) 38353: 3836#ifdef sun4v 3837 cmp %g5, T_INSTR_MMU_MISS 3838 be,a,pn %icc, 1f 3839 mov %g3, %g5 ! trapstat wants TTE in %g5 3840#endif /* sun4v */ 3841 cmp %g5, FAST_IMMU_MISS_TT 3842 be,pn %icc, 1f 3843 mov %g3, %g5 ! trapstat wants TTE in %g5 3844 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3845 ! trapstat wants TTE in %g5 3846 retry 38471: 3848 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3849 ! trapstat wants TTE in %g5 3850 retry 3851 3852tsb_ism: 3853 /* 3854 * This is an ISM [i|d]tlb miss. We optimize for largest 3855 * page size down to smallest. 3856 * 3857 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3858 * register 3859 * g3 = ismmap->ism_seg 3860 * g4 = physical address of ismmap->ism_sfmmu 3861 * g6 = tsbmiss area 3862 */ 3863 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3864 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3865 mov PTL1_BAD_ISM, %g1 3866 /* g5 = pa of imap_vb_shift */ 3867 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3868 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3869 srlx %g3, %g4, %g3 /* clr size field */ 3870 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3871 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3872 and %g2, %g1, %g4 /* g4 = ctx number */ 3873 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3874 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3875 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3876 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3877 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3878#if defined(sun4v) || defined(UTSB_PHYS) 3879 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3880 brz,pt %g5, tsb_chk4M_ism 3881 nop 3882 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3883 or %g5, HAT_CHKCTX1_FLAG, %g5 3884 stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3885 rdpr %tt, %g5 3886 SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl) 3887#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3888 3889 /* 3890 * ISM pages are always locked down. 3891 * If we can't find the tte then pagefault 3892 * and let the spt segment driver resolve it. 3893 * 3894 * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3895 * g4 = imap_hatflags 3896 * g6 = tsb miss area 3897 * g7 = ISM hatid 3898 */ 3899 3900tsb_chk4M_ism: 3901 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3902 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3903 nop 3904 3905tsb_ism_32M: 3906 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3907 brz,pn %g5, tsb_ism_256M 3908 nop 3909 3910 /* 3911 * 32M hash. 3912 */ 3913 3914 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M, 3915 TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3916 tsb_ism_4M) 3917 /* NOT REACHED */ 3918 3919tsb_ism_32M_found: 3920 brlz,a,pt %g3, tsb_validtte 3921 rdpr %tt, %g7 3922 ba,pt %xcc, tsb_ism_4M 3923 nop 3924 3925tsb_ism_256M: 3926 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3927 brz,a,pn %g5, ptl1_panic 3928 mov PTL1_BAD_ISM, %g1 3929 3930 /* 3931 * 256M hash. 3932 */ 3933 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M, 3934 TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3935 tsb_ism_4M) 3936 3937tsb_ism_256M_found: 3938 brlz,a,pt %g3, tsb_validtte 3939 rdpr %tt, %g7 3940 3941tsb_ism_4M: 3942 /* 3943 * 4M hash. 3944 */ 3945 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M, 3946 TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3947 tsb_ism_8K) 3948 /* NOT REACHED */ 3949 3950tsb_ism_4M_found: 3951 brlz,a,pt %g3, tsb_validtte 3952 rdpr %tt, %g7 3953 3954tsb_ism_8K: 3955 /* 3956 * 8K and 64K hash. 3957 */ 3958 3959 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K, 3960 TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3961 tsb_pagefault) 3962 /* NOT REACHED */ 3963 3964tsb_ism_8K_found: 3965 brlz,a,pt %g3, tsb_validtte 3966 rdpr %tt, %g7 3967 3968tsb_pagefault: 3969 rdpr %tt, %g7 3970 cmp %g7, FAST_PROT_TT 3971 be,a,pn %icc, tsb_protfault 3972 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3973 3974tsb_protfault: 3975 /* 3976 * we get here if we couldn't find a valid tte in the hash. 3977 * 3978 * If user and we are at tl>1 we go to window handling code. 3979 * 3980 * If kernel and the fault is on the same page as our stack 3981 * pointer, then we know the stack is bad and the trap handler 3982 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3983 * 3984 * If this is a kernel trap and tl>1, panic. 3985 * 3986 * Otherwise we call pagefault. 3987 */ 3988 cmp %g7, FAST_IMMU_MISS_TT 3989#ifdef sun4v 3990 MMU_FAULT_STATUS_AREA(%g4) 3991 ldx [%g4 + MMFSA_I_CTX], %g5 3992 ldx [%g4 + MMFSA_D_CTX], %g4 3993 move %icc, %g5, %g4 3994 cmp %g7, T_INSTR_MMU_MISS 3995 move %icc, %g5, %g4 3996#else 3997 mov MMU_TAG_ACCESS, %g4 3998 ldxa [%g4]ASI_DMMU, %g2 3999 ldxa [%g4]ASI_IMMU, %g5 4000 move %icc, %g5, %g2 4001 cmp %g7, T_INSTR_MMU_MISS 4002 move %icc, %g5, %g2 4003 sllx %g2, TAGACC_CTX_LSHIFT, %g4 4004#endif /* sun4v */ 4005 brnz,pn %g4, 3f /* skip if not kernel */ 4006 rdpr %tl, %g5 4007 4008 add %sp, STACK_BIAS, %g3 4009 srlx %g3, MMU_PAGESHIFT, %g3 4010 srlx %g2, MMU_PAGESHIFT, %g4 4011 cmp %g3, %g4 4012 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 4013 mov PTL1_BAD_STACK, %g1 4014 4015 cmp %g5, 1 4016 ble,pt %icc, 2f 4017 nop 4018 TSTAT_CHECK_TL1(2f, %g1, %g2) 4019 rdpr %tt, %g2 4020 cmp %g2, FAST_PROT_TT 4021 mov PTL1_BAD_KPROT_FAULT, %g1 4022 movne %icc, PTL1_BAD_KMISS, %g1 4023 ba,pt %icc, ptl1_panic 4024 nop 4025 40262: 4027 /* 4028 * We are taking a pagefault in the kernel on a kernel address. If 4029 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 4030 * want to call sfmmu_pagefault -- we will instead note that a fault 4031 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 4032 * (instead of a "retry"). This will step over the faulting 4033 * instruction. 4034 */ 4035 CPU_INDEX(%g1, %g2) 4036 set cpu_core, %g2 4037 sllx %g1, CPU_CORE_SHIFT, %g1 4038 add %g1, %g2, %g1 4039 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 4040 andcc %g2, CPU_DTRACE_NOFAULT, %g0 4041 bz sfmmu_pagefault 4042 or %g2, CPU_DTRACE_BADADDR, %g2 4043 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 4044 GET_MMU_D_ADDR(%g3, %g4) 4045 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 4046 done 4047 40483: 4049 cmp %g5, 1 4050 ble,pt %icc, 4f 4051 nop 4052 TSTAT_CHECK_TL1(4f, %g1, %g2) 4053 ba,pt %icc, sfmmu_window_trap 4054 nop 4055 40564: 4057 /* 4058 * We are taking a pagefault on a non-kernel address. If we are in 4059 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 4060 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 4061 */ 4062 CPU_INDEX(%g1, %g2) 4063 set cpu_core, %g2 4064 sllx %g1, CPU_CORE_SHIFT, %g1 4065 add %g1, %g2, %g1 4066 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 4067 andcc %g2, CPU_DTRACE_NOFAULT, %g0 4068 bz sfmmu_mmu_trap 4069 or %g2, CPU_DTRACE_BADADDR, %g2 4070 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 4071 GET_MMU_D_ADDR(%g3, %g4) 4072 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 4073 4074 /* 4075 * Be sure that we're actually taking this miss from the kernel -- 4076 * otherwise we have managed to return to user-level with 4077 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 4078 */ 4079 rdpr %tstate, %g2 4080 btst TSTATE_PRIV, %g2 4081 bz,a ptl1_panic 4082 mov PTL1_BAD_DTRACE_FLAGS, %g1 4083 done 4084 4085 ALTENTRY(tsb_tl0_noctxt) 4086 /* 4087 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 4088 * if it is, indicated that we have faulted and issue a done. 4089 */ 4090 CPU_INDEX(%g5, %g6) 4091 set cpu_core, %g6 4092 sllx %g5, CPU_CORE_SHIFT, %g5 4093 add %g5, %g6, %g5 4094 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 4095 andcc %g6, CPU_DTRACE_NOFAULT, %g0 4096 bz 1f 4097 or %g6, CPU_DTRACE_BADADDR, %g6 4098 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 4099 GET_MMU_D_ADDR(%g3, %g4) 4100 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 4101 4102 /* 4103 * Be sure that we're actually taking this miss from the kernel -- 4104 * otherwise we have managed to return to user-level with 4105 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 4106 */ 4107 rdpr %tstate, %g5 4108 btst TSTATE_PRIV, %g5 4109 bz,a ptl1_panic 4110 mov PTL1_BAD_DTRACE_FLAGS, %g1 4111 TSTAT_CHECK_TL1(2f, %g1, %g2); 41122: 4113 done 4114 41151: 4116 rdpr %tt, %g5 4117 cmp %g5, FAST_IMMU_MISS_TT 4118#ifdef sun4v 4119 MMU_FAULT_STATUS_AREA(%g2) 4120 be,a,pt %icc, 2f 4121 ldx [%g2 + MMFSA_I_CTX], %g3 4122 cmp %g5, T_INSTR_MMU_MISS 4123 be,a,pt %icc, 2f 4124 ldx [%g2 + MMFSA_I_CTX], %g3 4125 ldx [%g2 + MMFSA_D_CTX], %g3 41262: 4127#else 4128 mov MMU_TAG_ACCESS, %g2 4129 be,a,pt %icc, 2f 4130 ldxa [%g2]ASI_IMMU, %g3 4131 ldxa [%g2]ASI_DMMU, %g3 41322: sllx %g3, TAGACC_CTX_LSHIFT, %g3 4133#endif /* sun4v */ 4134 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 4135 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 4136 rdpr %tl, %g5 4137 cmp %g5, 1 4138 ble,pt %icc, sfmmu_mmu_trap 4139 nop 4140 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 4141 ba,pt %icc, sfmmu_window_trap 4142 nop 4143 SET_SIZE(sfmmu_tsb_miss) 4144#endif /* lint */ 4145 4146#if defined (lint) 4147/* 4148 * This routine will look for a user or kernel vaddr in the hash 4149 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 4150 * grab any locks. It should only be used by other sfmmu routines. 4151 */ 4152/* ARGSUSED */ 4153pfn_t 4154sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 4155{ 4156 return(0); 4157} 4158 4159/* ARGSUSED */ 4160pfn_t 4161sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno) 4162{ 4163 return(0); 4164} 4165 4166#else /* lint */ 4167 4168 ENTRY_NP(sfmmu_vatopfn) 4169 /* 4170 * disable interrupts 4171 */ 4172 rdpr %pstate, %o3 4173#ifdef DEBUG 4174 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 4175#endif 4176 /* 4177 * disable interrupts to protect the TSBMISS area 4178 */ 4179 andn %o3, PSTATE_IE, %o5 4180 wrpr %o5, 0, %pstate 4181 4182 /* 4183 * o0 = vaddr 4184 * o1 = sfmmup 4185 * o2 = ttep 4186 */ 4187 CPU_TSBMISS_AREA(%g1, %o5) 4188 ldn [%g1 + TSBMISS_KHATID], %o4 4189 cmp %o4, %o1 4190 bne,pn %ncc, vatopfn_nokernel 4191 mov TTE64K, %g5 /* g5 = rehash # */ 4192 mov %g1,%o5 /* o5 = tsbmiss_area */ 4193 /* 4194 * o0 = vaddr 4195 * o1 & o4 = hatid 4196 * o2 = ttep 4197 * o5 = tsbmiss area 4198 */ 4199 mov HBLK_RANGE_SHIFT, %g6 42001: 4201 4202 /* 4203 * o0 = vaddr 4204 * o1 = sfmmup 4205 * o2 = ttep 4206 * o3 = old %pstate 4207 * o4 = hatid 4208 * o5 = tsbmiss 4209 * g5 = rehash # 4210 * g6 = hmeshift 4211 * 4212 * The first arg to GET_TTE is actually tagaccess register 4213 * not just vaddr. Since this call is for kernel we need to clear 4214 * any lower vaddr bits that would be interpreted as ctx bits. 4215 */ 4216 set TAGACC_CTX_MASK, %g1 4217 andn %o0, %g1, %o0 4218 GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5, 4219 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 4220 4221kvtop_hblk_found: 4222 /* 4223 * o0 = vaddr 4224 * o1 = sfmmup 4225 * o2 = ttep 4226 * g1 = tte 4227 * g2 = tte pa 4228 * g3 = tte va 4229 * o2 = tsbmiss area 4230 * o1 = hat id 4231 */ 4232 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 4233 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4234 stx %g1,[%o2] /* put tte into *ttep */ 4235 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 4236 /* 4237 * o0 = vaddr 4238 * o1 = sfmmup 4239 * o2 = ttep 4240 * g1 = pfn 4241 */ 4242 ba,pt %xcc, 6f 4243 mov %g1, %o0 4244 4245kvtop_nohblk: 4246 /* 4247 * we get here if we couldn't find valid hblk in hash. We rehash 4248 * if neccesary. 4249 */ 4250 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 4251#ifdef sun4v 4252 cmp %g5, MAX_HASHCNT 4253#else 4254 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 4255#endif /* sun4v */ 4256 be,a,pn %icc, 6f 4257 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4258 mov %o1, %o4 /* restore hatid */ 4259#ifdef sun4v 4260 add %g5, 2, %g5 4261 cmp %g5, 3 4262 move %icc, MMU_PAGESHIFT4M, %g6 4263 ba,pt %icc, 1b 4264 movne %icc, MMU_PAGESHIFT256M, %g6 4265#else 4266 inc %g5 4267 cmp %g5, 2 4268 move %icc, MMU_PAGESHIFT512K, %g6 4269 ba,pt %icc, 1b 4270 movne %icc, MMU_PAGESHIFT4M, %g6 4271#endif /* sun4v */ 42726: 4273 retl 4274 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4275 4276tsb_suspend: 4277 /* 4278 * o0 = vaddr 4279 * o1 = sfmmup 4280 * o2 = ttep 4281 * g1 = tte 4282 * g2 = tte pa 4283 * g3 = tte va 4284 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 4285 */ 4286 stx %g1,[%o2] /* put tte into *ttep */ 4287 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 4288 sub %g0, 1, %o0 /* output = PFN_INVALID */ 4289 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 42908: 4291 retl 4292 wrpr %g0, %o3, %pstate /* enable interrupts */ 4293 4294vatopfn_nokernel: 4295 /* 4296 * This routine does NOT support user addresses 4297 * There is a routine in C that supports this. 4298 * The only reason why we don't have the C routine 4299 * support kernel addresses as well is because 4300 * we do va_to_pa while holding the hashlock. 4301 */ 4302 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4303 save %sp, -SA(MINFRAME), %sp 4304 sethi %hi(sfmmu_panic3), %o0 4305 call panic 4306 or %o0, %lo(sfmmu_panic3), %o0 4307 4308 SET_SIZE(sfmmu_vatopfn) 4309 4310 /* 4311 * %o0 = vaddr 4312 * %o1 = hashno (aka szc) 4313 * 4314 * 4315 * This routine is similar to sfmmu_vatopfn() but will only look for 4316 * a kernel vaddr in the hash structure for the specified rehash value. 4317 * It's just an optimization for the case when pagesize for a given 4318 * va range is already known (e.g. large page heap) and we don't want 4319 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 4320 * 4321 * Returns valid pfn or PFN_INVALID if 4322 * tte for specified rehash # is not found, invalid or suspended. 4323 */ 4324 ENTRY_NP(sfmmu_kvaszc2pfn) 4325 /* 4326 * disable interrupts 4327 */ 4328 rdpr %pstate, %o3 4329#ifdef DEBUG 4330 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 4331#endif 4332 /* 4333 * disable interrupts to protect the TSBMISS area 4334 */ 4335 andn %o3, PSTATE_IE, %o5 4336 wrpr %o5, 0, %pstate 4337 4338 CPU_TSBMISS_AREA(%g1, %o5) 4339 ldn [%g1 + TSBMISS_KHATID], %o4 4340 sll %o1, 1, %g6 4341 add %g6, %o1, %g6 4342 add %g6, MMU_PAGESHIFT, %g6 4343 /* 4344 * %o0 = vaddr 4345 * %o1 = hashno 4346 * %o3 = old %pstate 4347 * %o4 = ksfmmup 4348 * %g1 = tsbmiss area 4349 * %g6 = hmeshift 4350 */ 4351 4352 /* 4353 * The first arg to GET_TTE is actually tagaccess register 4354 * not just vaddr. Since this call is for kernel we need to clear 4355 * any lower vaddr bits that would be interpreted as ctx bits. 4356 */ 4357 srlx %o0, MMU_PAGESHIFT, %o0 4358 sllx %o0, MMU_PAGESHIFT, %o0 4359 GET_TTE(%o0, %o4, %g3, %g4, %g5, %g1, %o5, %g6, %o1, 4360 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 4361 kvaszc2pfn_nohblk) 4362 4363kvaszc2pfn_hblk_found: 4364 /* 4365 * %g3 = tte 4366 * %o0 = vaddr 4367 */ 4368 brgez,a,pn %g3, 1f /* check if tte is invalid */ 4369 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4370 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 4371 /* 4372 * g3 = pfn 4373 */ 4374 ba,pt %xcc, 1f 4375 mov %g3, %o0 4376 4377kvaszc2pfn_nohblk: 4378 mov -1, %o0 4379 43801: 4381 retl 4382 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4383 4384 SET_SIZE(sfmmu_kvaszc2pfn) 4385 4386#endif /* lint */ 4387 4388 4389 4390#if !defined(lint) 4391 4392/* 4393 * kpm lock used between trap level tsbmiss handler and kpm C level. 4394 */ 4395#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 4396 mov 0xff, tmp1 ;\ 4397label1: ;\ 4398 casa [kpmlckp]asi, %g0, tmp1 ;\ 4399 brnz,pn tmp1, label1 ;\ 4400 mov 0xff, tmp1 ;\ 4401 membar #LoadLoad 4402 4403#define KPMLOCK_EXIT(kpmlckp, asi) \ 4404 membar #LoadStore|#StoreStore ;\ 4405 sta %g0, [kpmlckp]asi 4406 4407/* 4408 * Lookup a memseg for a given pfn and if found, return the physical 4409 * address of the corresponding struct memseg in mseg, otherwise 4410 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 4411 * tsbmp, %asi is assumed to be ASI_MEM. 4412 * This lookup is done by strictly traversing only the physical memseg 4413 * linkage. The more generic approach, to check the virtual linkage 4414 * before using the physical (used e.g. with hmehash buckets), cannot 4415 * be used here. Memory DR operations can run in parallel to this 4416 * lookup w/o any locks and updates of the physical and virtual linkage 4417 * cannot be done atomically wrt. to each other. Because physical 4418 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 4419 * as "physical NULL" pointer. 4420 */ 4421#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 4422 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 4423 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 4424 udivx pfn, mseg, mseg ;\ 4425 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 4426 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 4427 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 4428 add tmp1, mseg, tmp1 ;\ 4429 ldxa [tmp1]%asi, mseg ;\ 4430 cmp mseg, MSEG_NULLPTR_PA ;\ 4431 be,pn %xcc, label/**/1 /* if not found */ ;\ 4432 nop ;\ 4433 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4434 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4435 blu,pn %xcc, label/**/1 ;\ 4436 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4437 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4438 bgeu,pn %xcc, label/**/1 ;\ 4439 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 4440 mulx tmp1, PAGE_SIZE, tmp1 ;\ 4441 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 4442 add tmp2, tmp1, tmp1 /* pp */ ;\ 4443 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 4444 cmp tmp2, pfn ;\ 4445 be,pt %xcc, label/**/_ok /* found */ ;\ 4446label/**/1: ;\ 4447 /* brute force lookup */ ;\ 4448 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 4449 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 4450label/**/2: ;\ 4451 cmp mseg, MSEG_NULLPTR_PA ;\ 4452 be,pn %xcc, label/**/_ok /* if not found */ ;\ 4453 nop ;\ 4454 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4455 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4456 blu,a,pt %xcc, label/**/2 ;\ 4457 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4458 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4459 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4460 bgeu,a,pt %xcc, label/**/2 ;\ 4461 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4462label/**/_ok: 4463 4464 /* 4465 * kpm tsb miss handler large pages 4466 * g1 = 8K kpm TSB entry pointer 4467 * g2 = tag access register 4468 * g3 = 4M kpm TSB entry pointer 4469 */ 4470 ALTENTRY(sfmmu_kpm_dtsb_miss) 4471 TT_TRACE(trace_tsbmiss) 4472 4473 CPU_INDEX(%g7, %g6) 4474 sethi %hi(kpmtsbm_area), %g6 4475 sllx %g7, KPMTSBM_SHIFT, %g7 4476 or %g6, %lo(kpmtsbm_area), %g6 4477 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4478 4479 /* check enable flag */ 4480 ldub [%g6 + KPMTSBM_FLAGS], %g4 4481 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4482 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4483 nop 4484 4485 /* VA range check */ 4486 ldx [%g6 + KPMTSBM_VBASE], %g7 4487 cmp %g2, %g7 4488 blu,pn %xcc, sfmmu_tsb_miss 4489 ldx [%g6 + KPMTSBM_VEND], %g5 4490 cmp %g2, %g5 4491 bgeu,pn %xcc, sfmmu_tsb_miss 4492 stx %g3, [%g6 + KPMTSBM_TSBPTR] 4493 4494 /* 4495 * check TL tsbmiss handling flag 4496 * bump tsbmiss counter 4497 */ 4498 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4499#ifdef DEBUG 4500 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4501 inc %g5 4502 brz,pn %g3, sfmmu_kpm_exception 4503 st %g5, [%g6 + KPMTSBM_TSBMISS] 4504#else 4505 inc %g5 4506 st %g5, [%g6 + KPMTSBM_TSBMISS] 4507#endif 4508 /* 4509 * At this point: 4510 * g1 = 8K kpm TSB pointer (not used) 4511 * g2 = tag access register 4512 * g3 = clobbered 4513 * g6 = per-CPU kpm tsbmiss area 4514 * g7 = kpm_vbase 4515 */ 4516 4517 /* vaddr2pfn */ 4518 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4519 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4520 srax %g4, %g3, %g2 /* which alias range (r) */ 4521 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4522 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4523 4524 /* 4525 * Setup %asi 4526 * mseg_pa = page_numtomemseg_nolock(pfn) 4527 * if (mseg_pa == NULL) sfmmu_kpm_exception 4528 * g2=pfn 4529 */ 4530 mov ASI_MEM, %asi 4531 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4532 cmp %g3, MSEG_NULLPTR_PA 4533 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4534 nop 4535 4536 /* 4537 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4538 * g2=pfn g3=mseg_pa 4539 */ 4540 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4541 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4542 srlx %g2, %g5, %g4 4543 sllx %g4, %g5, %g4 4544 sub %g4, %g7, %g4 4545 srlx %g4, %g5, %g4 4546 4547 /* 4548 * Validate inx value 4549 * g2=pfn g3=mseg_pa g4=inx 4550 */ 4551#ifdef DEBUG 4552 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4553 cmp %g4, %g5 /* inx - nkpmpgs */ 4554 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4555 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4556#else 4557 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4558#endif 4559 /* 4560 * kp = &mseg_pa->kpm_pages[inx] 4561 */ 4562 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4563 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4564 add %g5, %g4, %g5 /* kp */ 4565 4566 /* 4567 * KPMP_HASH(kp) 4568 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4569 */ 4570 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4571 sub %g7, 1, %g7 /* mask */ 4572 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4573 add %g5, %g1, %g5 /* y = ksp + x */ 4574 and %g5, %g7, %g5 /* hashinx = y & mask */ 4575 4576 /* 4577 * Calculate physical kpm_page pointer 4578 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4579 */ 4580 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4581 add %g1, %g4, %g1 /* kp_pa */ 4582 4583 /* 4584 * Calculate physical hash lock address 4585 * g1=kp_refcntc_pa g2=pfn g5=hashinx 4586 */ 4587 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4588 sllx %g5, KPMHLK_SHIFT, %g5 4589 add %g4, %g5, %g3 4590 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4591 4592 /* 4593 * Assemble tte 4594 * g1=kp_pa g2=pfn g3=hlck_pa 4595 */ 4596#ifdef sun4v 4597 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4598 sllx %g5, 32, %g5 4599 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4600 or %g4, TTE4M, %g4 4601 or %g5, %g4, %g5 4602#else 4603 sethi %hi(TTE_VALID_INT), %g4 4604 mov TTE4M, %g5 4605 sllx %g5, TTE_SZ_SHFT_INT, %g5 4606 or %g5, %g4, %g5 /* upper part */ 4607 sllx %g5, 32, %g5 4608 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4609 or %g5, %g4, %g5 4610#endif 4611 sllx %g2, MMU_PAGESHIFT, %g4 4612 or %g5, %g4, %g5 /* tte */ 4613 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4614 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4615 4616 /* 4617 * tsb dropin 4618 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4619 */ 4620 4621 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4622 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4623 4624 /* use C-handler if there's no go for dropin */ 4625 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4626 cmp %g7, -1 4627 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4628 nop 4629 4630#ifdef DEBUG 4631 /* double check refcnt */ 4632 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4633 brz,pn %g7, 5f /* let C-handler deal with this */ 4634 nop 4635#endif 4636 4637#ifndef sun4v 4638 ldub [%g6 + KPMTSBM_FLAGS], %g7 4639 mov ASI_N, %g1 4640 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4641 movnz %icc, ASI_MEM, %g1 4642 mov %g1, %asi 4643#endif 4644 4645 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4646 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4647 4648 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4649 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4650 4651 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4652 4653 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4654 KPMLOCK_EXIT(%g3, ASI_MEM) 4655 4656 /* 4657 * If trapstat is running, we need to shift the %tpc and %tnpc to 4658 * point to trapstat's TSB miss return code (note that trapstat 4659 * itself will patch the correct offset to add). 4660 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4661 */ 4662 rdpr %tl, %g7 4663 cmp %g7, 1 4664 ble %icc, 0f 4665 sethi %hi(KERNELBASE), %g6 4666 rdpr %tpc, %g7 4667 or %g6, %lo(KERNELBASE), %g6 4668 cmp %g7, %g6 4669 bgeu %xcc, 0f 4670 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4671 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4672 wrpr %g7, %tpc 4673 add %g7, 4, %g7 4674 wrpr %g7, %tnpc 46750: 4676 retry 46775: 4678 /* g3=hlck_pa */ 4679 KPMLOCK_EXIT(%g3, ASI_MEM) 4680 ba,pt %icc, sfmmu_kpm_exception 4681 nop 4682 SET_SIZE(sfmmu_kpm_dtsb_miss) 4683 4684 /* 4685 * kpm tsbmiss handler for smallpages 4686 * g1 = 8K kpm TSB pointer 4687 * g2 = tag access register 4688 * g3 = 4M kpm TSB pointer 4689 */ 4690 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4691 TT_TRACE(trace_tsbmiss) 4692 CPU_INDEX(%g7, %g6) 4693 sethi %hi(kpmtsbm_area), %g6 4694 sllx %g7, KPMTSBM_SHIFT, %g7 4695 or %g6, %lo(kpmtsbm_area), %g6 4696 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4697 4698 /* check enable flag */ 4699 ldub [%g6 + KPMTSBM_FLAGS], %g4 4700 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4701 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4702 nop 4703 4704 /* 4705 * VA range check 4706 * On fail: goto sfmmu_tsb_miss 4707 */ 4708 ldx [%g6 + KPMTSBM_VBASE], %g7 4709 cmp %g2, %g7 4710 blu,pn %xcc, sfmmu_tsb_miss 4711 ldx [%g6 + KPMTSBM_VEND], %g5 4712 cmp %g2, %g5 4713 bgeu,pn %xcc, sfmmu_tsb_miss 4714 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4715 4716 /* 4717 * check TL tsbmiss handling flag 4718 * bump tsbmiss counter 4719 */ 4720 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4721#ifdef DEBUG 4722 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4723 inc %g5 4724 brz,pn %g1, sfmmu_kpm_exception 4725 st %g5, [%g6 + KPMTSBM_TSBMISS] 4726#else 4727 inc %g5 4728 st %g5, [%g6 + KPMTSBM_TSBMISS] 4729#endif 4730 /* 4731 * At this point: 4732 * g1 = clobbered 4733 * g2 = tag access register 4734 * g3 = 4M kpm TSB pointer (not used) 4735 * g6 = per-CPU kpm tsbmiss area 4736 * g7 = kpm_vbase 4737 */ 4738 4739 /* vaddr2pfn */ 4740 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4741 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4742 srax %g4, %g3, %g2 /* which alias range (r) */ 4743 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4744 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4745 4746 /* 4747 * Setup %asi 4748 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4749 * if (mseg not found) sfmmu_kpm_exception 4750 * g2=pfn 4751 */ 4752 mov ASI_MEM, %asi 4753 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4754 cmp %g3, MSEG_NULLPTR_PA 4755 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4756 nop 4757 4758 /* 4759 * inx = pfn - mseg_pa->kpm_pbase 4760 * g2=pfn g3=mseg_pa 4761 */ 4762 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4763 sub %g2, %g7, %g4 4764 4765#ifdef DEBUG 4766 /* 4767 * Validate inx value 4768 * g2=pfn g3=mseg_pa g4=inx 4769 */ 4770 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4771 cmp %g4, %g5 /* inx - nkpmpgs */ 4772 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4773 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4774#else 4775 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4776#endif 4777 /* ksp = &mseg_pa->kpm_spages[inx] */ 4778 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4779 add %g5, %g4, %g5 /* ksp */ 4780 4781 /* 4782 * KPMP_SHASH(kp) 4783 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz 4784 */ 4785 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4786 sub %g7, 1, %g7 /* mask */ 4787 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4788 add %g5, %g1, %g5 /* y = ksp + x */ 4789 and %g5, %g7, %g5 /* hashinx = y & mask */ 4790 4791 /* 4792 * Calculate physical kpm_spage pointer 4793 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4794 */ 4795 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4796 add %g1, %g4, %g1 /* ksp_pa */ 4797 4798 /* 4799 * Calculate physical hash lock address. 4800 * Note: Changes in kpm_shlk_t must be reflected here. 4801 * g1=ksp_pa g2=pfn g5=hashinx 4802 */ 4803 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4804 sllx %g5, KPMSHLK_SHIFT, %g5 4805 add %g4, %g5, %g3 /* hlck_pa */ 4806 4807 /* 4808 * Assemble tte 4809 * g1=ksp_pa g2=pfn g3=hlck_pa 4810 */ 4811 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4812 sllx %g5, 32, %g5 4813 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4814 or %g5, %g4, %g5 4815 sllx %g2, MMU_PAGESHIFT, %g4 4816 or %g5, %g4, %g5 /* tte */ 4817 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4818 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4819 4820 /* 4821 * tsb dropin 4822 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte 4823 */ 4824 4825 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4826 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4827 4828 /* use C-handler if there's no go for dropin */ 4829 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4830 cmp %g7, -1 4831 bne,pn %xcc, 5f 4832 nop 4833 4834#ifndef sun4v 4835 ldub [%g6 + KPMTSBM_FLAGS], %g7 4836 mov ASI_N, %g1 4837 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4838 movnz %icc, ASI_MEM, %g1 4839 mov %g1, %asi 4840#endif 4841 4842 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4843 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4844 4845 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4846 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4847 4848 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4849 4850 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4851 KPMLOCK_EXIT(%g3, ASI_MEM) 4852 4853 /* 4854 * If trapstat is running, we need to shift the %tpc and %tnpc to 4855 * point to trapstat's TSB miss return code (note that trapstat 4856 * itself will patch the correct offset to add). 4857 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4858 */ 4859 rdpr %tl, %g7 4860 cmp %g7, 1 4861 ble %icc, 0f 4862 sethi %hi(KERNELBASE), %g6 4863 rdpr %tpc, %g7 4864 or %g6, %lo(KERNELBASE), %g6 4865 cmp %g7, %g6 4866 bgeu %xcc, 0f 4867 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4868 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4869 wrpr %g7, %tpc 4870 add %g7, 4, %g7 4871 wrpr %g7, %tnpc 48720: 4873 retry 48745: 4875 /* g3=hlck_pa */ 4876 KPMLOCK_EXIT(%g3, ASI_MEM) 4877 ba,pt %icc, sfmmu_kpm_exception 4878 nop 4879 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4880 4881#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4882#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4883#endif 4884 4885#endif /* lint */ 4886 4887#ifdef lint 4888/* 4889 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4890 * Called from C-level, sets/clears "go" indication for trap level handler. 4891 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 4892 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 4893 * Assumes khl_mutex is held when called from C-level. 4894 */ 4895/* ARGSUSED */ 4896void 4897sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 4898{ 4899} 4900 4901/* 4902 * kpm_smallpages: stores val to byte at address mapped within 4903 * low level lock brackets. The old value is returned. 4904 * Called from C-level. 4905 */ 4906/* ARGSUSED */ 4907int 4908sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val) 4909{ 4910 return (0); 4911} 4912 4913#else /* lint */ 4914 4915 .seg ".data" 4916sfmmu_kpm_tsbmtl_panic: 4917 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4918 .byte 0 4919sfmmu_kpm_stsbmtl_panic: 4920 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4921 .byte 0 4922 .align 4 4923 .seg ".text" 4924 4925 ENTRY_NP(sfmmu_kpm_tsbmtl) 4926 rdpr %pstate, %o3 4927 /* 4928 * %o0 = &kp_refcntc 4929 * %o1 = &khl_lock 4930 * %o2 = 0/1 (off/on) 4931 * %o3 = pstate save 4932 */ 4933#ifdef DEBUG 4934 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4935 bnz,pt %icc, 1f /* disabled, panic */ 4936 nop 4937 save %sp, -SA(MINFRAME), %sp 4938 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4939 call panic 4940 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4941 ret 4942 restore 49431: 4944#endif /* DEBUG */ 4945 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4946 4947 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4948 mov -1, %o5 4949 brz,a %o2, 2f 4950 mov 0, %o5 49512: 4952 sth %o5, [%o0] 4953 KPMLOCK_EXIT(%o1, ASI_N) 4954 4955 retl 4956 wrpr %g0, %o3, %pstate /* enable interrupts */ 4957 SET_SIZE(sfmmu_kpm_tsbmtl) 4958 4959 ENTRY_NP(sfmmu_kpm_stsbmtl) 4960 rdpr %pstate, %o3 4961 /* 4962 * %o0 = &mapped 4963 * %o1 = &kshl_lock 4964 * %o2 = val 4965 * %o3 = pstate save 4966 */ 4967#ifdef DEBUG 4968 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4969 bnz,pt %icc, 1f /* disabled, panic */ 4970 nop 4971 save %sp, -SA(MINFRAME), %sp 4972 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4973 call panic 4974 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4975 ret 4976 restore 49771: 4978#endif /* DEBUG */ 4979 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4980 4981 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4982 ldsb [%o0], %o5 4983 stb %o2, [%o0] 4984 KPMLOCK_EXIT(%o1, ASI_N) 4985 4986 mov %o5, %o0 /* return old val */ 4987 retl 4988 wrpr %g0, %o3, %pstate /* enable interrupts */ 4989 SET_SIZE(sfmmu_kpm_stsbmtl) 4990 4991#endif /* lint */ 4992 4993#ifndef lint 4994#ifdef sun4v 4995 /* 4996 * User/kernel data miss w// multiple TSBs 4997 * The first probe covers 8K, 64K, and 512K page sizes, 4998 * because 64K and 512K mappings are replicated off 8K 4999 * pointer. Second probe covers 4M page size only. 5000 * 5001 * MMU fault area contains miss address and context. 5002 */ 5003 ALTENTRY(sfmmu_slow_dmmu_miss) 5004 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 5005 5006slow_miss_common: 5007 /* 5008 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 5009 * %g3 = ctx (cannot be INVALID_CONTEXT) 5010 */ 5011 brnz,pt %g3, 8f ! check for user context 5012 nop 5013 5014 /* 5015 * Kernel miss 5016 * Get 8K and 4M TSB pointers in %g1 and %g3 and 5017 * branch to sfmmu_tsb_miss_tt to handle it. 5018 */ 5019 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 5020sfmmu_dslow_patch_ktsb_base: 5021 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 5022sfmmu_dslow_patch_ktsb_szcode: 5023 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 5024 5025 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 5026 ! %g1 = First TSB entry pointer, as TSB miss handler expects 5027 5028 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 5029sfmmu_dslow_patch_ktsb4m_base: 5030 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 5031sfmmu_dslow_patch_ktsb4m_szcode: 5032 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 5033 5034 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 5035 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 5036 ba,a,pt %xcc, sfmmu_tsb_miss_tt 5037 .empty 5038 50398: 5040 /* 5041 * User miss 5042 * Get first TSB pointer in %g1 5043 * Get second TSB pointer (or NULL if no second TSB) in %g3 5044 * Branch to sfmmu_tsb_miss_tt to handle it 5045 */ 5046 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 5047 /* %g1 = first TSB entry ptr now, %g2 preserved */ 5048 5049 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 5050 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 5051 nop 5052 5053 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 5054 /* %g3 = second TSB entry ptr now, %g2 preserved */ 50559: 5056 ba,a,pt %xcc, sfmmu_tsb_miss_tt 5057 .empty 5058 SET_SIZE(sfmmu_slow_dmmu_miss) 5059 5060 5061 /* 5062 * User/kernel instruction miss w/ multiple TSBs 5063 * The first probe covers 8K, 64K, and 512K page sizes, 5064 * because 64K and 512K mappings are replicated off 8K 5065 * pointer. Second probe covers 4M page size only. 5066 * 5067 * MMU fault area contains miss address and context. 5068 */ 5069 ALTENTRY(sfmmu_slow_immu_miss) 5070 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 5071 ba,a,pt %xcc, slow_miss_common 5072 SET_SIZE(sfmmu_slow_immu_miss) 5073 5074#endif /* sun4v */ 5075#endif /* lint */ 5076 5077#ifndef lint 5078 5079/* 5080 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 5081 */ 5082 .seg ".data" 5083 .align 64 5084 .global tsbmiss_area 5085tsbmiss_area: 5086 .skip (TSBMISS_SIZE * NCPU) 5087 5088 .align 64 5089 .global kpmtsbm_area 5090kpmtsbm_area: 5091 .skip (KPMTSBM_SIZE * NCPU) 5092#endif /* lint */ 5093