1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#pragma ident "%Z%%M% %I% %E% SMI" 27 28/* 29 * SFMMU primitives. These primitives should only be used by sfmmu 30 * routines. 31 */ 32 33#if defined(lint) 34#include <sys/types.h> 35#else /* lint */ 36#include "assym.h" 37#endif /* lint */ 38 39#include <sys/asm_linkage.h> 40#include <sys/machtrap.h> 41#include <sys/machasi.h> 42#include <sys/sun4asi.h> 43#include <sys/pte.h> 44#include <sys/mmu.h> 45#include <vm/hat_sfmmu.h> 46#include <vm/seg_spt.h> 47#include <sys/machparam.h> 48#include <sys/privregs.h> 49#include <sys/scb.h> 50#include <sys/intreg.h> 51#include <sys/machthread.h> 52#include <sys/intr.h> 53#include <sys/clock.h> 54#include <sys/trapstat.h> 55 56#ifdef TRAPTRACE 57#include <sys/traptrace.h> 58 59/* 60 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 61 */ 62#define TT_TRACE(label) \ 63 ba label ;\ 64 rd %pc, %g7 65#else 66 67#define TT_TRACE(label) 68 69#endif /* TRAPTRACE */ 70 71#ifndef lint 72 73#if (TTE_SUSPEND_SHIFT > 0) 74#define TTE_SUSPEND_INT_SHIFT(reg) \ 75 sllx reg, TTE_SUSPEND_SHIFT, reg 76#else 77#define TTE_SUSPEND_INT_SHIFT(reg) 78#endif 79 80#endif /* lint */ 81 82#ifndef lint 83 84/* 85 * Assumes TSBE_TAG is 0 86 * Assumes TSBE_INTHI is 0 87 * Assumes TSBREG.split is 0 88 */ 89 90#if TSBE_TAG != 0 91#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 92#endif 93 94#if TSBTAG_INTHI != 0 95#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 96#endif 97 98/* 99 * The following code assumes the tsb is not split. 100 * 101 * With TSBs no longer shared between processes, it's no longer 102 * necessary to hash the context bits into the tsb index to get 103 * tsb coloring; the new implementation treats the TSB as a 104 * direct-mapped, virtually-addressed cache. 105 * 106 * In: 107 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 108 * tsbbase = base address of TSB (clobbered) 109 * tagacc = tag access register (clobbered) 110 * szc = size code of TSB (ro) 111 * tmp = scratch reg 112 * Out: 113 * tsbbase = pointer to entry in TSB 114 */ 115#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 116 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 117 srlx tagacc, vpshift, tagacc ;\ 118 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 119 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 120 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 121 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 122 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 123 124/* 125 * When the kpm TSB is used it is assumed that it is direct mapped 126 * using (vaddr>>vpshift)%tsbsz as the index. 127 * 128 * Note that, for now, the kpm TSB and kernel TSB are the same for 129 * each mapping size. However that need not always be the case. If 130 * the trap handlers are updated to search a different TSB for kpm 131 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 132 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 133 * 134 * In: 135 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 136 * vaddr = virtual address (clobbered) 137 * tsbp, szc, tmp = scratch 138 * Out: 139 * tsbp = pointer to entry in TSB 140 */ 141#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 142 cmp vpshift, MMU_PAGESHIFT ;\ 143 bne,pn %icc, 1f /* branch if large case */ ;\ 144 sethi %hi(kpmsm_tsbsz), szc ;\ 145 sethi %hi(kpmsm_tsbbase), tsbp ;\ 146 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 147 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 148 ba,pt %icc, 2f ;\ 149 nop ;\ 1501: sethi %hi(kpm_tsbsz), szc ;\ 151 sethi %hi(kpm_tsbbase), tsbp ;\ 152 ld [szc + %lo(kpm_tsbsz)], szc ;\ 153 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1542: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 155 156/* 157 * Lock the TSBE at virtual address tsbep. 158 * 159 * tsbep = TSBE va (ro) 160 * tmp1, tmp2 = scratch registers (clobbered) 161 * label = label to use for branches (text) 162 * %asi = ASI to use for TSB access 163 * 164 * NOTE that we flush the TSB using fast VIS instructions that 165 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 166 * not be treated as a locked entry or we'll get stuck spinning on 167 * an entry that isn't locked but really invalid. 168 */ 169 170#if defined(UTSB_PHYS) 171 172#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 173 lda [tsbep]ASI_MEM, tmp1 ;\ 174label: ;\ 175 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 176 cmp tmp1, tmp2 ;\ 177 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 178 lda [tsbep]ASI_MEM, tmp1 ;\ 179 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 180 cmp tmp1, tmp2 ;\ 181 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 182 lda [tsbep]ASI_MEM, tmp1 ;\ 183 /* tsbe lock acquired */ ;\ 184 membar #StoreStore 185 186#else /* UTSB_PHYS */ 187 188#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 189 lda [tsbep]%asi, tmp1 ;\ 190label: ;\ 191 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 192 cmp tmp1, tmp2 ;\ 193 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 194 lda [tsbep]%asi, tmp1 ;\ 195 casa [tsbep]%asi, tmp1, tmp2 ;\ 196 cmp tmp1, tmp2 ;\ 197 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 198 lda [tsbep]%asi, tmp1 ;\ 199 /* tsbe lock acquired */ ;\ 200 membar #StoreStore 201 202#endif /* UTSB_PHYS */ 203 204/* 205 * Atomically write TSBE at virtual address tsbep. 206 * 207 * tsbep = TSBE va (ro) 208 * tte = TSBE TTE (ro) 209 * tagtarget = TSBE tag (ro) 210 * %asi = ASI to use for TSB access 211 */ 212 213#if defined(UTSB_PHYS) 214 215#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 216 add tsbep, TSBE_TTE, tmp1 ;\ 217 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 218 membar #StoreStore ;\ 219 add tsbep, TSBE_TAG, tmp1 ;\ 220 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 221 222#else /* UTSB_PHYS */ 223 224#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 225 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 226 membar #StoreStore ;\ 227 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 228 229#endif /* UTSB_PHYS */ 230 231/* 232 * Load an entry into the TSB at TL > 0. 233 * 234 * tsbep = pointer to the TSBE to load as va (ro) 235 * tte = value of the TTE retrieved and loaded (wo) 236 * tagtarget = tag target register. To get TSBE tag to load, 237 * we need to mask off the context and leave only the va (clobbered) 238 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 239 * tmp1, tmp2 = scratch registers 240 * label = label to use for branches (text) 241 * %asi = ASI to use for TSB access 242 */ 243 244#if defined(UTSB_PHYS) 245 246#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 247 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 248 /* ;\ 249 * I don't need to update the TSB then check for the valid tte. ;\ 250 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 251 * we always invalidate the hash table before we unload the TSB.;\ 252 */ ;\ 253 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 254 ldxa [ttepa]ASI_MEM, tte ;\ 255 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 256 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 257 add tsbep, TSBE_TAG, tmp1 ;\ 258 brgez,a,pn tte, label/**/f ;\ 259 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 260 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 261label: 262 263#else /* UTSB_PHYS */ 264 265#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 266 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 267 /* ;\ 268 * I don't need to update the TSB then check for the valid tte. ;\ 269 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 270 * we always invalidate the hash table before we unload the TSB.;\ 271 */ ;\ 272 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 273 ldxa [ttepa]ASI_MEM, tte ;\ 274 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 275 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 276 brgez,a,pn tte, label/**/f ;\ 277 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 278 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 279label: 280 281#endif /* UTSB_PHYS */ 282 283/* 284 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 285 * for ITLB synthesis. 286 * 287 * tsbep = pointer to the TSBE to load as va (ro) 288 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 289 * with exec_perm turned off and exec_synth turned on 290 * tagtarget = tag target register. To get TSBE tag to load, 291 * we need to mask off the context and leave only the va (clobbered) 292 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 293 * tmp1, tmp2 = scratch registers 294 * label = label to use for branch (text) 295 * %asi = ASI to use for TSB access 296 */ 297 298#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 299 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 300 /* ;\ 301 * I don't need to update the TSB then check for the valid tte. ;\ 302 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 303 * we always invalidate the hash table before we unload the TSB.;\ 304 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 305 * and exec_synth bit to 1. ;\ 306 */ ;\ 307 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 308 mov tte, tmp1 ;\ 309 ldxa [ttepa]ASI_MEM, tte ;\ 310 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 311 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 312 brgez,a,pn tte, label/**/f ;\ 313 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 314 or tte, tmp1, tte ;\ 315 andn tte, TTE_EXECPRM_INT, tte ;\ 316 or tte, TTE_E_SYNTH_INT, tte ;\ 317 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 318label: 319 320/* 321 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 322 * 323 * tte = value of the TTE, used to get tte_size bits (ro) 324 * tagaccess = tag access register, used to get 4M pfn bits (ro) 325 * pfn = 4M pfn bits shifted to offset for tte (out) 326 * tmp1 = scratch register 327 * label = label to use for branch (text) 328 */ 329 330#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 331 /* ;\ 332 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 333 * Return them, shifted, in pfn. ;\ 334 */ ;\ 335 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 336 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 337 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 338 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 339 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 340 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 341label: ;\ 342 sllx tagaccess, MMU_PAGESHIFT4M, pfn 343 344/* 345 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 346 * for ITLB synthesis. 347 * 348 * tte = value of the TTE, used to get tte_size bits (rw) 349 * tmp1 = scratch register 350 */ 351 352#define SET_TTE4M_PN(tte, tmp) \ 353 /* ;\ 354 * Set 4M pagesize tte bits. ;\ 355 */ ;\ 356 set TTE4M, tmp ;\ 357 sllx tmp, TTE_SZ_SHFT, tmp ;\ 358 or tte, tmp, tte 359 360/* 361 * Load an entry into the TSB at TL=0. 362 * 363 * tsbep = pointer to the TSBE to load as va (ro) 364 * tteva = pointer to the TTE to load as va (ro) 365 * tagtarget = TSBE tag to load (which contains no context), synthesized 366 * to match va of MMU tag target register only (ro) 367 * tmp1, tmp2 = scratch registers (clobbered) 368 * label = label to use for branches (text) 369 * %asi = ASI to use for TSB access 370 */ 371 372#if defined(UTSB_PHYS) 373 374#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 375 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 376 ldx [tteva], tteva /* load tte */ ;\ 377 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 378 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 379 add tsbep, TSBE_TAG, tmp1 ;\ 380 brgez,a,pn tteva, label/**/f ;\ 381 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 382 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 383label: 384 385#else /* UTSB_PHYS */ 386 387#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 388 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 389 ldx [tteva], tteva /* load tte */ ;\ 390 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 391 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 392 brgez,a,pn tteva, label/**/f ;\ 393 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 394 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 395label: 396 397#endif /* UTSB_PHYS */ 398 399/* 400 * Invalidate a TSB entry in the TSB. 401 * 402 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 403 * about this earlier to ensure this is true. Thus when we are 404 * directly referencing tsbep below, we are referencing the tte_tag 405 * field of the TSBE. If this offset ever changes, the code below 406 * will need to be modified. 407 * 408 * tsbep = pointer to TSBE as va (ro) 409 * tag = invalidation is done if this matches the TSBE tag (ro) 410 * tmp1 - tmp3 = scratch registers (clobbered) 411 * label = label name to use for branches (text) 412 * %asi = ASI to use for TSB access 413 */ 414 415#if defined(UTSB_PHYS) 416 417#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 418 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 419 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 420label/**/1: ;\ 421 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 422 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 423 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 424 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 425 cmp tag, tmp3 /* compare tags */ ;\ 426 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 427 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 428 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 429 cmp tmp1, tmp3 /* if not successful */ ;\ 430 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 431 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 432label/**/2: 433 434#else /* UTSB_PHYS */ 435 436#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 437 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 438 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 439label/**/1: ;\ 440 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 441 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 442 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 443 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 444 cmp tag, tmp3 /* compare tags */ ;\ 445 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 446 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 447 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 448 cmp tmp1, tmp3 /* if not successful */ ;\ 449 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 450 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 451label/**/2: 452 453#endif /* UTSB_PHYS */ 454 455#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 456#error - TSB_SOFTSZ_MASK too small 457#endif 458 459 460/* 461 * An implementation of setx which will be hot patched at run time. 462 * since it is being hot patched, there is no value passed in. 463 * Thus, essentially we are implementing 464 * setx value, tmp, dest 465 * where value is RUNTIME_PATCH (aka 0) in this case. 466 */ 467#define RUNTIME_PATCH_SETX(dest, tmp) \ 468 sethi %hh(RUNTIME_PATCH), tmp ;\ 469 sethi %lm(RUNTIME_PATCH), dest ;\ 470 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 471 or dest, %lo(RUNTIME_PATCH), dest ;\ 472 sllx tmp, 32, tmp ;\ 473 nop /* for perf reasons */ ;\ 474 or tmp, dest, dest /* contents of patched value */ 475 476 477#endif (lint) 478 479 480#if defined (lint) 481 482/* 483 * sfmmu related subroutines 484 */ 485uint_t 486sfmmu_disable_intrs() 487{ return(0); } 488 489/* ARGSUSED */ 490void 491sfmmu_enable_intrs(uint_t pstate_save) 492{} 493 494/* ARGSUSED */ 495void 496sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp) 497{} 498 499/* 500 * Use cas, if tte has changed underneath us then reread and try again. 501 * In the case of a retry, it will update sttep with the new original. 502 */ 503/* ARGSUSED */ 504int 505sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 506{ return(0); } 507 508/* 509 * Use cas, if tte has changed underneath us then return 1, else return 0 510 */ 511/* ARGSUSED */ 512int 513sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 514{ return(0); } 515 516/* ARGSUSED */ 517void 518sfmmu_copytte(tte_t *sttep, tte_t *dttep) 519{} 520 521/*ARGSUSED*/ 522struct tsbe * 523sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 524{ return(0); } 525 526/*ARGSUSED*/ 527uint64_t 528sfmmu_make_tsbtag(caddr_t va) 529{ return(0); } 530 531#else /* lint */ 532 533 .seg ".data" 534 .global sfmmu_panic1 535sfmmu_panic1: 536 .asciz "sfmmu_asm: interrupts already disabled" 537 538 .global sfmmu_panic3 539sfmmu_panic3: 540 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 541 542 .global sfmmu_panic4 543sfmmu_panic4: 544 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 545 546 .global sfmmu_panic5 547sfmmu_panic5: 548 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 549 550 .global sfmmu_panic6 551sfmmu_panic6: 552 .asciz "sfmmu_asm: interrupts not disabled" 553 554 .global sfmmu_panic7 555sfmmu_panic7: 556 .asciz "sfmmu_asm: kernel as" 557 558 .global sfmmu_panic8 559sfmmu_panic8: 560 .asciz "sfmmu_asm: gnum is zero" 561 562 .global sfmmu_panic9 563sfmmu_panic9: 564 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 565 566 ENTRY(sfmmu_disable_intrs) 567 rdpr %pstate, %o0 568#ifdef DEBUG 569 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 570#endif /* DEBUG */ 571 retl 572 wrpr %o0, PSTATE_IE, %pstate 573 SET_SIZE(sfmmu_disable_intrs) 574 575 ENTRY(sfmmu_enable_intrs) 576 retl 577 wrpr %g0, %o0, %pstate 578 SET_SIZE(sfmmu_enable_intrs) 579 580/* 581 * This routine is called both by resume() and sfmmu_get_ctx() to 582 * allocate a new context for the process on a MMU. 583 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 584 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 585 * is the case when sfmmu_alloc_ctx is called from resume(). 586 * 587 * The caller must disable interrupts before entering this routine. 588 * To reduce ctx switch overhead, the code contains both 'fast path' and 589 * 'slow path' code. The fast path code covers the common case where only 590 * a quick check is needed and the real ctx allocation is not required. 591 * It can be done without holding the per-process (PP) lock. 592 * The 'slow path' code must be protected by the PP Lock and performs ctx 593 * allocation. 594 * Hardware context register and HAT mmu cnum are updated accordingly. 595 * 596 * %o0 - sfmmup 597 * %o1 - allocflag 598 * %o2 - CPU 599 */ 600 ENTRY_NP(sfmmu_alloc_ctx) 601 602#ifdef DEBUG 603 sethi %hi(ksfmmup), %o3 604 ldx [%o3 + %lo(ksfmmup)], %o3 605 cmp %o3, %o0 606 bne,pt %xcc, 0f 607 nop 608 609 sethi %hi(panicstr), %g1 ! if kernel as, panic 610 ldx [%g1 + %lo(panicstr)], %g1 611 tst %g1 612 bnz,pn %icc, 7f 613 nop 614 615 sethi %hi(sfmmu_panic7), %o0 616 call panic 617 or %o0, %lo(sfmmu_panic7), %o0 618 6197: 620 retl 621 nop 622 6230: 624 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 625#endif /* DEBUG */ 626 627 ! load global mmu_ctxp info 628 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 629 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 630 631 ! load global mmu_ctxp gnum 632 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 633 634#ifdef DEBUG 635 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 636 bne,pt %xcc, 3f 637 nop 638 639 sethi %hi(panicstr), %g1 ! test if panicstr is already set 640 ldx [%g1 + %lo(panicstr)], %g1 641 tst %g1 642 bnz,pn %icc, 3f 643 nop 644 645 sethi %hi(sfmmu_panic8), %o0 646 call panic 647 or %o0, %lo(sfmmu_panic8), %o0 6483: 649#endif 650 651 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 652 653 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 654 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 655 656 /* 657 * %g5 = sfmmu gnum returned 658 * %g6 = sfmmu cnum returned 659 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 660 * %g4 = scratch 661 * 662 * Fast path code, do a quick check. 663 */ 664 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 665 666 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 667 bne,pt %icc, 1f ! valid hat cnum, check gnum 668 nop 669 670 ! cnum == INVALID, check allocflag 671 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 672 mov %g6, %o1 673 674 ! (invalid HAT cnum) && (allocflag == 1) 675 ba,pt %icc, 2f 676 nop 6771: 678 ! valid HAT cnum, check gnum 679 cmp %g5, %o4 680 be,a,pt %icc, 8f ! gnum unchanged, go to done 681 mov %g6, %o1 682 6832: 684 /* 685 * Grab per process (PP) sfmmu_ctx_lock spinlock, 686 * followed by the 'slow path' code. 687 */ 688 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 6893: 690 brz %g3, 5f 691 nop 6924: 693 brnz,a,pt %g3, 4b ! spin if lock is 1 694 ldub [%o0 + SFMMU_CTX_LOCK], %g3 695 ba %xcc, 3b ! retry the lock 696 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 697 6985: 699 membar #LoadLoad 700 /* 701 * %g5 = sfmmu gnum returned 702 * %g6 = sfmmu cnum returned 703 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 704 * %g4 = scratch 705 */ 706 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 707 708 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 709 bne,pt %icc, 1f ! valid hat cnum, check gnum 710 nop 711 712 ! cnum == INVALID, check allocflag 713 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 714 mov %g6, %o1 715 716 ! (invalid HAT cnum) && (allocflag == 1) 717 ba,pt %icc, 6f 718 nop 7191: 720 ! valid HAT cnum, check gnum 721 cmp %g5, %o4 722 be,a,pt %icc, 2f ! gnum unchanged, go to done 723 mov %g6, %o1 724 725 ba,pt %icc, 6f 726 nop 7272: 728 membar #LoadStore|#StoreStore 729 ba,pt %icc, 8f 730 clrb [%o0 + SFMMU_CTX_LOCK] 7316: 732 /* 733 * We get here if we do not have a valid context, or 734 * the HAT gnum does not match global gnum. We hold 735 * sfmmu_ctx_lock spinlock. Allocate that context. 736 * 737 * %o3 = mmu_ctxp 738 */ 739 add %o3, MMU_CTX_CNUM, %g3 740 ld [%o3 + MMU_CTX_NCTXS], %g4 741 742 /* 743 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 744 * %g3 = mmu cnum address 745 * %g4 = mmu nctxs 746 * 747 * %o0 = sfmmup 748 * %o1 = mmu current cnum value (used as new cnum) 749 * %o4 = mmu gnum 750 * 751 * %o5 = scratch 752 */ 753 ld [%g3], %o1 7540: 755 cmp %o1, %g4 756 bl,a,pt %icc, 1f 757 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 758 759 /* 760 * cnum reachs max, update HAT with INVALID 761 */ 762 set INVALID_CONTEXT, %o1 763 764 /* 765 * update hat cnum to INVALID, sun4v sfmmu_load_mmustate checks 766 * hat cnum to determine if set the number of TSBs to 0. 767 */ 768 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 769 or %o4, %o1, %o4 770 stx %o4, [%g2 + SFMMU_CTXS] 771 772 membar #LoadStore|#StoreStore 773 ba,pt %icc, 8f 774 clrb [%o0 + SFMMU_CTX_LOCK] 7751: 776 ! %g3 = addr of mmu_ctxp->cnum 777 ! %o5 = mmu_ctxp->cnum + 1 778 cas [%g3], %o1, %o5 779 cmp %o1, %o5 780 bne,a,pn %xcc, 0b ! cas failed 781 ld [%g3], %o1 782 783#ifdef DEBUG 784 set MAX_SFMMU_CTX_VAL, %o5 785 cmp %o1, %o5 786 ble,pt %icc, 2f 787 nop 788 789 sethi %hi(sfmmu_panic9), %o0 790 call panic 791 or %o0, %lo(sfmmu_panic9), %o0 7922: 793#endif 794 ! update hat gnum and cnum 795 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 796 or %o4, %o1, %o4 797 stx %o4, [%g2 + SFMMU_CTXS] 798 799 membar #LoadStore|#StoreStore 800 clrb [%o0 + SFMMU_CTX_LOCK] 801 8028: 803 /* 804 * program the secondary context register 805 * 806 * %o1 = cnum 807 */ 808#ifdef sun4u 809 ldub [%o0 + SFMMU_CEXT], %o2 810 sll %o2, CTXREG_EXT_SHIFT, %o2 811 or %o1, %o2, %o1 812#endif 813 814 mov MMU_SCONTEXT, %o4 815 sethi %hi(FLUSH_ADDR), %o5 816 stxa %o1, [%o4]ASI_MMU_CTX ! set 2nd context reg. 817 flush %o5 818 819 retl 820 nop 821 822 SET_SIZE(sfmmu_alloc_ctx) 823 824 825 ENTRY_NP(sfmmu_modifytte) 826 ldx [%o2], %g3 /* current */ 827 ldx [%o0], %g1 /* original */ 8282: 829 ldx [%o1], %g2 /* modified */ 830 cmp %g2, %g3 /* is modified = current? */ 831 be,a,pt %xcc,1f /* yes, don't write */ 832 stx %g3, [%o0] /* update new original */ 833 casx [%o2], %g1, %g2 834 cmp %g1, %g2 835 be,pt %xcc, 1f /* cas succeeded - return */ 836 nop 837 ldx [%o2], %g3 /* new current */ 838 stx %g3, [%o0] /* save as new original */ 839 ba,pt %xcc, 2b 840 mov %g3, %g1 8411: retl 842 membar #StoreLoad 843 SET_SIZE(sfmmu_modifytte) 844 845 ENTRY_NP(sfmmu_modifytte_try) 846 ldx [%o1], %g2 /* modified */ 847 ldx [%o2], %g3 /* current */ 848 ldx [%o0], %g1 /* original */ 849 cmp %g3, %g2 /* is modified = current? */ 850 be,a,pn %xcc,1f /* yes, don't write */ 851 mov 0, %o1 /* as if cas failed. */ 852 853 casx [%o2], %g1, %g2 854 membar #StoreLoad 855 cmp %g1, %g2 856 movne %xcc, -1, %o1 /* cas failed. */ 857 move %xcc, 1, %o1 /* cas succeeded. */ 8581: 859 stx %g2, [%o0] /* report "current" value */ 860 retl 861 mov %o1, %o0 862 SET_SIZE(sfmmu_modifytte_try) 863 864 ENTRY_NP(sfmmu_copytte) 865 ldx [%o0], %g1 866 retl 867 stx %g1, [%o1] 868 SET_SIZE(sfmmu_copytte) 869 870 871 /* 872 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 873 * %o0 = TSB base address (in), pointer to TSB entry (out) 874 * %o1 = vaddr (in) 875 * %o2 = vpshift (in) 876 * %o3 = tsb size code (in) 877 * %o4 = scratch register 878 */ 879 ENTRY_NP(sfmmu_get_tsbe) 880 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 881 retl 882 nop 883 SET_SIZE(sfmmu_get_tsbe) 884 885 /* 886 * Return a TSB tag for the given va. 887 * %o0 = va (in/clobbered) 888 * %o0 = va shifted to be in tsb tag format (with no context) (out) 889 */ 890 ENTRY_NP(sfmmu_make_tsbtag) 891 retl 892 srln %o0, TTARGET_VA_SHIFT, %o0 893 SET_SIZE(sfmmu_make_tsbtag) 894 895#endif /* lint */ 896 897/* 898 * Other sfmmu primitives 899 */ 900 901 902#if defined (lint) 903void 904sfmmu_patch_ktsb(void) 905{ 906} 907 908void 909sfmmu_kpm_patch_tlbm(void) 910{ 911} 912 913void 914sfmmu_kpm_patch_tsbm(void) 915{ 916} 917 918/* ARGSUSED */ 919void 920sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 921{ 922} 923 924/* ARGSUSED */ 925void 926sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 927{ 928} 929 930/* ARGSUSED */ 931void 932sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 933{ 934} 935 936/* ARGSUSED */ 937void 938sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 939{ 940} 941 942#else /* lint */ 943 944#define I_SIZE 4 945 946 ENTRY_NP(sfmmu_fix_ktlb_traptable) 947 /* 948 * %o0 = start of patch area 949 * %o1 = size code of TSB to patch 950 * %o3 = scratch 951 */ 952 /* fix sll */ 953 ld [%o0], %o3 /* get sll */ 954 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 955 st %o3, [%o0] /* write sll */ 956 flush %o0 957 /* fix srl */ 958 add %o0, I_SIZE, %o0 /* goto next instr. */ 959 ld [%o0], %o3 /* get srl */ 960 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 961 st %o3, [%o0] /* write srl */ 962 retl 963 flush %o0 964 SET_SIZE(sfmmu_fix_ktlb_traptable) 965 966 ENTRY_NP(sfmmu_fixup_ktsbbase) 967 /* 968 * %o0 = start of patch area 969 * %o5 = kernel virtual or physical tsb base address 970 * %o2, %o3 are used as scratch registers. 971 */ 972 /* fixup sethi instruction */ 973 ld [%o0], %o3 974 srl %o5, 10, %o2 ! offset is bits 32:10 975 or %o3, %o2, %o3 ! set imm22 976 st %o3, [%o0] 977 /* fixup offset of lduw/ldx */ 978 add %o0, I_SIZE, %o0 ! next instr 979 ld [%o0], %o3 980 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 981 or %o3, %o2, %o3 982 st %o3, [%o0] 983 retl 984 flush %o0 985 SET_SIZE(sfmmu_fixup_ktsbbase) 986 987 ENTRY_NP(sfmmu_fixup_setx) 988 /* 989 * %o0 = start of patch area 990 * %o4 = 64 bit value to patch 991 * %o2, %o3 are used as scratch registers. 992 * 993 * Note: Assuming that all parts of the instructions which need to be 994 * patched correspond to RUNTIME_PATCH (aka 0) 995 * 996 * Note the implementation of setx which is being patched is as follows: 997 * 998 * sethi %hh(RUNTIME_PATCH), tmp 999 * sethi %lm(RUNTIME_PATCH), dest 1000 * or tmp, %hm(RUNTIME_PATCH), tmp 1001 * or dest, %lo(RUNTIME_PATCH), dest 1002 * sllx tmp, 32, tmp 1003 * nop 1004 * or tmp, dest, dest 1005 * 1006 * which differs from the implementation in the 1007 * "SPARC Architecture Manual" 1008 */ 1009 /* fixup sethi instruction */ 1010 ld [%o0], %o3 1011 srlx %o4, 42, %o2 ! bits [63:42] 1012 or %o3, %o2, %o3 ! set imm22 1013 st %o3, [%o0] 1014 /* fixup sethi instruction */ 1015 add %o0, I_SIZE, %o0 ! next instr 1016 ld [%o0], %o3 1017 sllx %o4, 32, %o2 ! clear upper bits 1018 srlx %o2, 42, %o2 ! bits [31:10] 1019 or %o3, %o2, %o3 ! set imm22 1020 st %o3, [%o0] 1021 /* fixup or instruction */ 1022 add %o0, I_SIZE, %o0 ! next instr 1023 ld [%o0], %o3 1024 srlx %o4, 32, %o2 ! bits [63:32] 1025 and %o2, 0x3ff, %o2 ! bits [41:32] 1026 or %o3, %o2, %o3 ! set imm 1027 st %o3, [%o0] 1028 /* fixup or instruction */ 1029 add %o0, I_SIZE, %o0 ! next instr 1030 ld [%o0], %o3 1031 and %o4, 0x3ff, %o2 ! bits [9:0] 1032 or %o3, %o2, %o3 ! set imm 1033 st %o3, [%o0] 1034 retl 1035 flush %o0 1036 SET_SIZE(sfmmu_fixup_setx) 1037 1038 ENTRY_NP(sfmmu_fixup_or) 1039 /* 1040 * %o0 = start of patch area 1041 * %o4 = 32 bit value to patch 1042 * %o2, %o3 are used as scratch registers. 1043 * Note: Assuming that all parts of the instructions which need to be 1044 * patched correspond to RUNTIME_PATCH (aka 0) 1045 */ 1046 ld [%o0], %o3 1047 and %o4, 0x3ff, %o2 ! bits [9:0] 1048 or %o3, %o2, %o3 ! set imm 1049 st %o3, [%o0] 1050 retl 1051 flush %o0 1052 SET_SIZE(sfmmu_fixup_or) 1053 1054 ENTRY_NP(sfmmu_fixup_shiftx) 1055 /* 1056 * %o0 = start of patch area 1057 * %o4 = signed int immediate value to add to sllx/srlx imm field 1058 * %o2, %o3 are used as scratch registers. 1059 * 1060 * sllx/srlx store the 6 bit immediate value in the lowest order bits 1061 * so we do a simple add. The caller must be careful to prevent 1062 * overflow, which could easily occur if the initial value is nonzero! 1063 */ 1064 ld [%o0], %o3 ! %o3 = instruction to patch 1065 and %o3, 0x3f, %o2 ! %o2 = existing imm value 1066 add %o2, %o4, %o2 ! %o2 = new imm value 1067 andn %o3, 0x3f, %o3 ! clear old imm value 1068 and %o2, 0x3f, %o2 ! truncate new imm value 1069 or %o3, %o2, %o3 ! set new imm value 1070 st %o3, [%o0] ! store updated instruction 1071 retl 1072 flush %o0 1073 SET_SIZE(sfmmu_fixup_shiftx) 1074 1075 ENTRY_NP(sfmmu_fixup_mmu_asi) 1076 /* 1077 * Patch imm_asi of all ldda instructions in the MMU 1078 * trap handlers. We search MMU_PATCH_INSTR instructions 1079 * starting from the itlb miss handler (trap 0x64). 1080 * %o0 = address of tt[0,1]_itlbmiss 1081 * %o1 = imm_asi to setup, shifted by appropriate offset. 1082 * %o3 = number of instructions to search 1083 * %o4 = reserved by caller: called from leaf routine 1084 */ 10851: ldsw [%o0], %o2 ! load instruction to %o2 1086 brgez,pt %o2, 2f 1087 srl %o2, 30, %o5 1088 btst 1, %o5 ! test bit 30; skip if not set 1089 bz,pt %icc, 2f 1090 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1091 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1092 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1093 brnz,pt %o5, 2f ! skip if not a match 1094 or %o2, %o1, %o2 ! or in imm_asi 1095 st %o2, [%o0] ! write patched instruction 10962: dec %o3 1097 brnz,a,pt %o3, 1b ! loop until we're done 1098 add %o0, I_SIZE, %o0 1099 retl 1100 flush %o0 1101 SET_SIZE(sfmmu_fixup_mmu_asi) 1102 1103 /* 1104 * Patch immediate ASI used to access the TSB in the 1105 * trap table. 1106 * inputs: %o0 = value of ktsb_phys 1107 */ 1108 ENTRY_NP(sfmmu_patch_mmu_asi) 1109 mov %o7, %o4 ! save return pc in %o4 1110 movrnz %o0, ASI_QUAD_LDD_PHYS, %o3 1111 movrz %o0, ASI_NQUAD_LD, %o3 1112 sll %o3, 5, %o1 ! imm_asi offset 1113 mov 6, %o3 ! number of instructions 1114 sethi %hi(dktsb), %o0 ! to search 1115 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1116 or %o0, %lo(dktsb), %o0 1117 mov 6, %o3 ! number of instructions 1118 sethi %hi(dktsb4m), %o0 ! to search 1119 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1120 or %o0, %lo(dktsb4m), %o0 1121 mov 6, %o3 ! number of instructions 1122 sethi %hi(iktsb), %o0 ! to search 1123 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1124 or %o0, %lo(iktsb), %o0 1125 mov %o4, %o7 ! retore return pc -- leaf 1126 retl 1127 nop 1128 SET_SIZE(sfmmu_patch_mmu_asi) 1129 1130 ENTRY_NP(sfmmu_patch_ktsb) 1131 /* 1132 * We need to fix iktsb, dktsb, et. al. 1133 */ 1134 save %sp, -SA(MINFRAME), %sp 1135 set ktsb_phys, %o1 1136 ld [%o1], %o4 1137 set ktsb_base, %o5 1138 set ktsb4m_base, %l1 1139 brz,pt %o4, 1f 1140 nop 1141 set ktsb_pbase, %o5 1142 set ktsb4m_pbase, %l1 11431: 1144 sethi %hi(ktsb_szcode), %o1 1145 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1146 1147 sethi %hi(iktsb), %o0 1148 call sfmmu_fix_ktlb_traptable 1149 or %o0, %lo(iktsb), %o0 1150 1151 sethi %hi(dktsb), %o0 1152 call sfmmu_fix_ktlb_traptable 1153 or %o0, %lo(dktsb), %o0 1154 1155 sethi %hi(ktsb4m_szcode), %o1 1156 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1157 1158 sethi %hi(dktsb4m), %o0 1159 call sfmmu_fix_ktlb_traptable 1160 or %o0, %lo(dktsb4m), %o0 1161 1162#ifndef sun4v 1163 mov ASI_N, %o2 1164 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1165 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1166 sethi %hi(tsb_kernel_patch_asi), %o0 1167 call sfmmu_fixup_or 1168 or %o0, %lo(tsb_kernel_patch_asi), %o0 1169#endif 1170 1171 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1172 1173 sethi %hi(dktsbbase), %o0 1174 call sfmmu_fixup_setx ! patch value of ktsb base addr 1175 or %o0, %lo(dktsbbase), %o0 1176 1177 sethi %hi(iktsbbase), %o0 1178 call sfmmu_fixup_setx ! patch value of ktsb base addr 1179 or %o0, %lo(iktsbbase), %o0 1180 1181 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1182 call sfmmu_fixup_setx ! patch value of ktsb base addr 1183 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1184 1185#ifdef sun4v 1186 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1187 call sfmmu_fixup_setx ! patch value of ktsb base addr 1188 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1189#endif /* sun4v */ 1190 1191 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1192 1193 sethi %hi(dktsb4mbase), %o0 1194 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1195 or %o0, %lo(dktsb4mbase), %o0 1196 1197 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1198 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1199 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1200 1201#ifdef sun4v 1202 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1203 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1204 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1205#endif /* sun4v */ 1206 1207 set ktsb_szcode, %o4 1208 ld [%o4], %o4 1209 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1210 call sfmmu_fixup_or ! patch value of ktsb_szcode 1211 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1212 1213#ifdef sun4v 1214 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1215 call sfmmu_fixup_or ! patch value of ktsb_szcode 1216 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1217#endif /* sun4v */ 1218 1219 set ktsb4m_szcode, %o4 1220 ld [%o4], %o4 1221 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1222 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1223 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1224 1225#ifdef sun4v 1226 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1227 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1228 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1229#endif /* sun4v */ 1230 1231 ret 1232 restore 1233 SET_SIZE(sfmmu_patch_ktsb) 1234 1235 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1236 /* 1237 * Fixup trap handlers in common segkpm case. This is reserved 1238 * for future use should kpm TSB be changed to be other than the 1239 * kernel TSB. 1240 */ 1241 retl 1242 nop 1243 SET_SIZE(sfmmu_kpm_patch_tlbm) 1244 1245 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1246 /* 1247 * nop the branch to sfmmu_kpm_dtsb_miss_small 1248 * in the case where we are using large pages for 1249 * seg_kpm (and hence must probe the second TSB for 1250 * seg_kpm VAs) 1251 */ 1252 set dktsb4m_kpmcheck_small, %o0 1253 MAKE_NOP_INSTR(%o1) 1254 st %o1, [%o0] 1255 flush %o0 1256 retl 1257 nop 1258 SET_SIZE(sfmmu_kpm_patch_tsbm) 1259 1260 ENTRY_NP(sfmmu_patch_utsb) 1261#ifdef UTSB_PHYS 1262 retl 1263 nop 1264#else /* UTSB_PHYS */ 1265 /* 1266 * We need to hot patch utsb_vabase and utsb4m_vabase 1267 */ 1268 save %sp, -SA(MINFRAME), %sp 1269 1270 /* patch value of utsb_vabase */ 1271 set utsb_vabase, %o1 1272 ldx [%o1], %o4 1273 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1274 call sfmmu_fixup_setx 1275 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1276 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1277 call sfmmu_fixup_setx 1278 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1279 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1280 call sfmmu_fixup_setx 1281 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1282 1283 /* patch value of utsb4m_vabase */ 1284 set utsb4m_vabase, %o1 1285 ldx [%o1], %o4 1286 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1287 call sfmmu_fixup_setx 1288 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1289 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1290 call sfmmu_fixup_setx 1291 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1292 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1293 call sfmmu_fixup_setx 1294 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1295 1296 /* 1297 * Patch TSB base register masks and shifts if needed. 1298 * By default the TSB base register contents are set up for 4M slab. 1299 * If we're using a smaller slab size and reserved VA range we need 1300 * to patch up those values here. 1301 */ 1302 set tsb_slab_shift, %o1 1303 set MMU_PAGESHIFT4M, %o4 1304 ldsw [%o1], %o3 1305 subcc %o4, %o3, %o4 1306 bz,pt %icc, 1f 1307 /* delay slot safe */ 1308 1309 /* patch reserved VA range size if needed. */ 1310 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1311 call sfmmu_fixup_shiftx 1312 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1313 call sfmmu_fixup_shiftx 1314 add %o0, I_SIZE, %o0 1315 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1316 call sfmmu_fixup_shiftx 1317 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1318 call sfmmu_fixup_shiftx 1319 add %o0, I_SIZE, %o0 13201: 1321 /* patch TSBREG_VAMASK used to set up TSB base register */ 1322 set tsb_slab_mask, %o1 1323 lduw [%o1], %o4 1324 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1325 call sfmmu_fixup_or 1326 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1327 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1328 call sfmmu_fixup_or 1329 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1330 1331 ret 1332 restore 1333#endif /* UTSB_PHYS */ 1334 SET_SIZE(sfmmu_patch_utsb) 1335 1336 1337 /* 1338 * Routine that loads an entry into a tsb using virtual addresses. 1339 * Locking is required since all cpus can use the same TSB. 1340 * Note that it is no longer required to have a valid context 1341 * when calling this function. 1342 */ 1343 ENTRY_NP(sfmmu_load_tsbe) 1344 /* 1345 * %o0 = pointer to tsbe to load 1346 * %o1 = tsb tag 1347 * %o2 = virtual pointer to TTE 1348 * %o3 = 1 if physical address in %o0 else 0 1349 */ 1350 rdpr %pstate, %o5 1351#ifdef DEBUG 1352 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1353#endif /* DEBUG */ 1354 1355 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1356 1357 SETUP_TSB_ASI(%o3, %g3) 1358 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1) 1359 1360 wrpr %g0, %o5, %pstate /* enable interrupts */ 1361 1362 retl 1363 membar #StoreStore|#StoreLoad 1364 SET_SIZE(sfmmu_load_tsbe) 1365 1366 /* 1367 * Flush TSB of a given entry if the tag matches. 1368 */ 1369 ENTRY(sfmmu_unload_tsbe) 1370 /* 1371 * %o0 = pointer to tsbe to be flushed 1372 * %o1 = tag to match 1373 * %o2 = 1 if physical address in %o0 else 0 1374 */ 1375 SETUP_TSB_ASI(%o2, %g1) 1376 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1377 retl 1378 membar #StoreStore|#StoreLoad 1379 SET_SIZE(sfmmu_unload_tsbe) 1380 1381 /* 1382 * Routine that loads a TTE into the kpm TSB from C code. 1383 * Locking is required since kpm TSB is shared among all CPUs. 1384 */ 1385 ENTRY_NP(sfmmu_kpm_load_tsb) 1386 /* 1387 * %o0 = vaddr 1388 * %o1 = ttep 1389 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1390 */ 1391 rdpr %pstate, %o5 ! %o5 = saved pstate 1392#ifdef DEBUG 1393 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1394#endif /* DEBUG */ 1395 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1396 1397#ifndef sun4v 1398 sethi %hi(ktsb_phys), %o4 1399 mov ASI_N, %o3 1400 ld [%o4 + %lo(ktsb_phys)], %o4 1401 movrnz %o4, ASI_MEM, %o3 1402 mov %o3, %asi 1403#endif 1404 mov %o0, %g1 ! %g1 = vaddr 1405 1406 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1407 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1408 /* %g2 = tsbep, %g1 clobbered */ 1409 1410 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1411 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1412 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1) 1413 1414 wrpr %g0, %o5, %pstate ! enable interrupts 1415 retl 1416 membar #StoreStore|#StoreLoad 1417 SET_SIZE(sfmmu_kpm_load_tsb) 1418 1419 /* 1420 * Routine that shoots down a TTE in the kpm TSB or in the 1421 * kernel TSB depending on virtpg. Locking is required since 1422 * kpm/kernel TSB is shared among all CPUs. 1423 */ 1424 ENTRY_NP(sfmmu_kpm_unload_tsb) 1425 /* 1426 * %o0 = vaddr 1427 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1428 */ 1429#ifndef sun4v 1430 sethi %hi(ktsb_phys), %o4 1431 mov ASI_N, %o3 1432 ld [%o4 + %lo(ktsb_phys)], %o4 1433 movrnz %o4, ASI_MEM, %o3 1434 mov %o3, %asi 1435#endif 1436 mov %o0, %g1 ! %g1 = vaddr 1437 1438 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1439 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1440 /* %g2 = tsbep, %g1 clobbered */ 1441 1442 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1443 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1444 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1445 1446 retl 1447 membar #StoreStore|#StoreLoad 1448 SET_SIZE(sfmmu_kpm_unload_tsb) 1449 1450#endif /* lint */ 1451 1452 1453#if defined (lint) 1454 1455/*ARGSUSED*/ 1456pfn_t 1457sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1458{ return(0); } 1459 1460#else /* lint */ 1461 1462 ENTRY_NP(sfmmu_ttetopfn) 1463 ldx [%o0], %g1 /* read tte */ 1464 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1465 /* 1466 * g1 = pfn 1467 */ 1468 retl 1469 mov %g1, %o0 1470 SET_SIZE(sfmmu_ttetopfn) 1471 1472#endif /* !lint */ 1473 1474 1475#if defined (lint) 1476/* 1477 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the 1478 * the hash list. 1479 */ 1480/* ARGSUSED */ 1481void 1482sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1483 uint64_t hblkpa) 1484{ 1485} 1486 1487/* 1488 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the 1489 * hash list. 1490 */ 1491/* ARGSUSED */ 1492void 1493sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1494 uint64_t hblkpa, struct hme_blk *prev_hblkp) 1495{ 1496} 1497#else /* lint */ 1498 1499/* 1500 * Functions to grab/release hme bucket list lock. I only use a byte 1501 * instead of the whole int because eventually we might want to 1502 * put some counters on the other bytes (of course, these routines would 1503 * have to change). The code that grab this lock should execute 1504 * with interrupts disabled and hold the lock for the least amount of time 1505 * possible. 1506 */ 1507 1508/* 1509 * Even though hmeh_listlock is updated using pa there's no need to flush 1510 * dcache since hmeh_listlock will be restored to the original value (0) 1511 * before interrupts are reenabled. 1512 */ 1513 1514/* 1515 * For sparcv9 hme hash buckets may not be in the nucleus. hme hash update 1516 * routines still use virtual addresses to update the bucket fields. But they 1517 * must not cause a TLB miss after grabbing the low level bucket lock. To 1518 * achieve this we must make sure the bucket structure is completely within an 1519 * 8K page. 1520 */ 1521 1522#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1)) 1523#error - the size of hmehash_bucket structure is not power of 2 1524#endif 1525 1526#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi) \ 1527 mov 0xff, tmp2 ;\ 1528 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1529label1: ;\ 1530 casa [tmp1]asi, %g0, tmp2 ;\ 1531 brnz,pn tmp2, label1 ;\ 1532 mov 0xff, tmp2 ;\ 1533 membar #LoadLoad 1534 1535#define HMELOCK_EXIT(hmebp, tmp1, asi) \ 1536 membar #LoadStore|#StoreStore ;\ 1537 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1538 sta %g0, [tmp1]asi 1539 1540 .seg ".data" 1541hblk_add_panic1: 1542 .ascii "sfmmu_hblk_hash_add: interrupts disabled" 1543 .byte 0 1544hblk_add_panic2: 1545 .ascii "sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not" 1546 .byte 0 1547 .align 4 1548 .seg ".text" 1549 1550 ENTRY_NP(sfmmu_hblk_hash_add) 1551 /* 1552 * %o0 = hmebp 1553 * %o1 = hmeblkp 1554 * %o2 = hblkpa 1555 */ 1556 rdpr %pstate, %o5 1557#ifdef DEBUG 1558 andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ 1559 bnz,pt %icc, 3f /* disabled, panic */ 1560 nop 1561 save %sp, -SA(MINFRAME), %sp 1562 sethi %hi(hblk_add_panic1), %o0 1563 call panic 1564 or %o0, %lo(hblk_add_panic1), %o0 1565 ret 1566 restore 1567 15683: 1569#endif /* DEBUG */ 1570 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1571 mov %o2, %g1 1572 1573 /* 1574 * g1 = hblkpa 1575 */ 1576 ldn [%o0 + HMEBUCK_HBLK], %o4 /* next hmeblk */ 1577 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = next hblkpa */ 1578#ifdef DEBUG 1579 cmp %o4, %g0 1580 bne,pt %xcc, 1f 1581 nop 1582 brz,pt %g2, 1f 1583 nop 1584 wrpr %g0, %o5, %pstate /* enable interrupts */ 1585 save %sp, -SA(MINFRAME), %sp 1586 sethi %hi(hblk_add_panic2), %o0 1587 call panic 1588 or %o0, %lo(hblk_add_panic2), %o0 1589 ret 1590 restore 15911: 1592#endif /* DEBUG */ 1593 /* 1594 * We update hmeblks entries before grabbing lock because the stores 1595 * could take a tlb miss and require the hash lock. The buckets 1596 * are part of the nucleus so we are cool with those stores. 1597 * 1598 * if buckets are not part of the nucleus our game is to 1599 * not touch any other page via va until we drop the lock. 1600 * This guarantees we won't get a tlb miss before the lock release 1601 * since interrupts are disabled. 1602 */ 1603 stn %o4, [%o1 + HMEBLK_NEXT] /* update hmeblk's next */ 1604 stx %g2, [%o1 + HMEBLK_NEXTPA] /* update hmeblk's next pa */ 1605 HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N) 1606 stn %o1, [%o0 + HMEBUCK_HBLK] /* update bucket hblk next */ 1607 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* add hmeblk to list */ 1608 HMELOCK_EXIT(%o0, %g2, ASI_N) 1609 retl 1610 wrpr %g0, %o5, %pstate /* enable interrupts */ 1611 SET_SIZE(sfmmu_hblk_hash_add) 1612 1613 ENTRY_NP(sfmmu_hblk_hash_rm) 1614 /* 1615 * This function removes an hmeblk from the hash chain. 1616 * It is written to guarantee we don't take a tlb miss 1617 * by using physical addresses to update the list. 1618 * 1619 * %o0 = hmebp 1620 * %o1 = hmeblkp 1621 * %o2 = hmeblkp previous pa 1622 * %o3 = hmeblkp previous 1623 */ 1624 1625 mov %o3, %o4 /* o4 = hmeblkp previous */ 1626 1627 rdpr %pstate, %o5 1628#ifdef DEBUG 1629 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1) 1630#endif /* DEBUG */ 1631 /* 1632 * disable interrupts, clear Address Mask to access 64 bit physaddr 1633 */ 1634 andn %o5, PSTATE_IE, %g1 1635 wrpr %g1, 0, %pstate 1636 1637#ifndef sun4v 1638 sethi %hi(dcache_line_mask), %g4 1639 ld [%g4 + %lo(dcache_line_mask)], %g4 1640#endif /* sun4v */ 1641 1642 /* 1643 * if buckets are not part of the nucleus our game is to 1644 * not touch any other page via va until we drop the lock. 1645 * This guarantees we won't get a tlb miss before the lock release 1646 * since interrupts are disabled. 1647 */ 1648 HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N) 1649 ldn [%o0 + HMEBUCK_HBLK], %g2 /* first hmeblk in list */ 1650 cmp %g2, %o1 1651 bne,pt %ncc,1f 1652 mov ASI_MEM, %asi 1653 /* 1654 * hmeblk is first on list 1655 */ 1656 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = hmeblk pa */ 1657 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1658 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1659 stn %o3, [%o0 + HMEBUCK_HBLK] /* write va */ 1660 ba,pt %xcc, 2f 1661 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* write pa */ 16621: 1663 /* hmeblk is not first on list */ 1664 1665 mov %o2, %g3 1666#ifndef sun4v 1667 GET_CPU_IMPL(%g2) 1668 cmp %g2, CHEETAH_IMPL 1669 bge,a,pt %icc, hblk_hash_rm_1 1670 and %o4, %g4, %g2 1671 cmp %g2, SPITFIRE_IMPL 1672 blt %icc, hblk_hash_rm_2 /* no flushing needed for OPL */ 1673 and %o4, %g4, %g2 1674 stxa %g0, [%g2]ASI_DC_TAG /* flush prev pa from dcache */ 1675 add %o4, HMEBLK_NEXT, %o4 1676 and %o4, %g4, %g2 1677 ba hblk_hash_rm_2 1678 stxa %g0, [%g2]ASI_DC_TAG /* flush prev va from dcache */ 1679hblk_hash_rm_1: 1680 1681 stxa %g0, [%g3]ASI_DC_INVAL /* flush prev pa from dcache */ 1682 membar #Sync 1683 add %g3, HMEBLK_NEXT, %g2 1684 stxa %g0, [%g2]ASI_DC_INVAL /* flush prev va from dcache */ 1685hblk_hash_rm_2: 1686 membar #Sync 1687#endif /* sun4v */ 1688 ldxa [%g3 + HMEBLK_NEXTPA] %asi, %g2 /* g2 = hmeblk pa */ 1689 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1690 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1691 stna %o3, [%g3 + HMEBLK_NEXT] %asi /* write va */ 1692 stxa %g1, [%g3 + HMEBLK_NEXTPA] %asi /* write pa */ 16932: 1694 HMELOCK_EXIT(%o0, %g2, ASI_N) 1695 retl 1696 wrpr %g0, %o5, %pstate /* enable interrupts */ 1697 SET_SIZE(sfmmu_hblk_hash_rm) 1698 1699#endif /* lint */ 1700 1701/* 1702 * These macros are used to update global sfmmu hme hash statistics 1703 * in perf critical paths. It is only enabled in debug kernels or 1704 * if SFMMU_STAT_GATHER is defined 1705 */ 1706#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1707#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1708 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1709 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1710 cmp tmp1, hatid ;\ 1711 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1712 set sfmmu_global_stat, tmp1 ;\ 1713 add tmp1, tmp2, tmp1 ;\ 1714 ld [tmp1], tmp2 ;\ 1715 inc tmp2 ;\ 1716 st tmp2, [tmp1] 1717 1718#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1719 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1720 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1721 cmp tmp1, hatid ;\ 1722 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1723 set sfmmu_global_stat, tmp1 ;\ 1724 add tmp1, tmp2, tmp1 ;\ 1725 ld [tmp1], tmp2 ;\ 1726 inc tmp2 ;\ 1727 st tmp2, [tmp1] 1728 1729 1730#else /* DEBUG || SFMMU_STAT_GATHER */ 1731 1732#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1733 1734#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1735 1736#endif /* DEBUG || SFMMU_STAT_GATHER */ 1737 1738/* 1739 * This macro is used to update global sfmmu kstas in non 1740 * perf critical areas so they are enabled all the time 1741 */ 1742#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1743 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1744 add tmp1, statname, tmp1 ;\ 1745 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1746 inc tmp2 ;\ 1747 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1748 1749/* 1750 * These macros are used to update per cpu stats in non perf 1751 * critical areas so they are enabled all the time 1752 */ 1753#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1754 ld [tsbarea + stat], tmp1 ;\ 1755 inc tmp1 ;\ 1756 st tmp1, [tsbarea + stat] 1757 1758/* 1759 * These macros are used to update per cpu stats in non perf 1760 * critical areas so they are enabled all the time 1761 */ 1762#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1763 lduh [tsbarea + stat], tmp1 ;\ 1764 inc tmp1 ;\ 1765 stuh tmp1, [tsbarea + stat] 1766 1767#if defined(KPM_TLBMISS_STATS_GATHER) 1768 /* 1769 * Count kpm dtlb misses separately to allow a different 1770 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1771 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1772 */ 1773#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1774 brgez tagacc, label /* KPM VA? */ ;\ 1775 nop ;\ 1776 CPU_INDEX(tmp1, tsbma) ;\ 1777 sethi %hi(kpmtsbm_area), tsbma ;\ 1778 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1779 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1780 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1781 /* VA range check */ ;\ 1782 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1783 cmp tagacc, val ;\ 1784 blu,pn %xcc, label ;\ 1785 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1786 cmp tagacc, tmp1 ;\ 1787 bgeu,pn %xcc, label ;\ 1788 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1789 inc val ;\ 1790 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1791label: 1792#else 1793#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1794#endif /* KPM_TLBMISS_STATS_GATHER */ 1795 1796#if defined (lint) 1797/* 1798 * The following routines are jumped to from the mmu trap handlers to do 1799 * the setting up to call systrap. They are separate routines instead of 1800 * being part of the handlers because the handlers would exceed 32 1801 * instructions and since this is part of the slow path the jump 1802 * cost is irrelevant. 1803 */ 1804void 1805sfmmu_pagefault(void) 1806{ 1807} 1808 1809void 1810sfmmu_mmu_trap(void) 1811{ 1812} 1813 1814void 1815sfmmu_window_trap(void) 1816{ 1817} 1818 1819void 1820sfmmu_kpm_exception(void) 1821{ 1822} 1823 1824#else /* lint */ 1825 1826#ifdef PTL1_PANIC_DEBUG 1827 .seg ".data" 1828 .global test_ptl1_panic 1829test_ptl1_panic: 1830 .word 0 1831 .align 8 1832 1833 .seg ".text" 1834 .align 4 1835#endif /* PTL1_PANIC_DEBUG */ 1836 1837 1838 ENTRY_NP(sfmmu_pagefault) 1839 SET_GL_REG(1) 1840 USE_ALTERNATE_GLOBALS(%g5) 1841 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1842 rdpr %tt, %g6 1843 cmp %g6, FAST_IMMU_MISS_TT 1844 be,a,pn %icc, 1f 1845 mov T_INSTR_MMU_MISS, %g3 1846 cmp %g6, T_INSTR_MMU_MISS 1847 be,a,pn %icc, 1f 1848 mov T_INSTR_MMU_MISS, %g3 1849 mov %g5, %g2 1850 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1851 cmp %g6, FAST_DMMU_MISS_TT 1852 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1853 cmp %g6, T_DATA_MMU_MISS 1854 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1855 1856#ifdef PTL1_PANIC_DEBUG 1857 /* check if we want to test the tl1 panic */ 1858 sethi %hi(test_ptl1_panic), %g4 1859 ld [%g4 + %lo(test_ptl1_panic)], %g1 1860 st %g0, [%g4 + %lo(test_ptl1_panic)] 1861 cmp %g1, %g0 1862 bne,a,pn %icc, ptl1_panic 1863 or %g0, PTL1_BAD_DEBUG, %g1 1864#endif /* PTL1_PANIC_DEBUG */ 18651: 1866 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 1867 /* 1868 * g2 = tag access reg 1869 * g3.l = type 1870 * g3.h = 0 1871 */ 1872 sethi %hi(trap), %g1 1873 or %g1, %lo(trap), %g1 18742: 1875 ba,pt %xcc, sys_trap 1876 mov -1, %g4 1877 SET_SIZE(sfmmu_pagefault) 1878 1879 ENTRY_NP(sfmmu_mmu_trap) 1880 SET_GL_REG(1) 1881 USE_ALTERNATE_GLOBALS(%g5) 1882 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 1883 rdpr %tt, %g6 1884 cmp %g6, FAST_IMMU_MISS_TT 1885 be,a,pn %icc, 1f 1886 mov T_INSTR_MMU_MISS, %g3 1887 cmp %g6, T_INSTR_MMU_MISS 1888 be,a,pn %icc, 1f 1889 mov T_INSTR_MMU_MISS, %g3 1890 mov %g5, %g2 1891 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1892 cmp %g6, FAST_DMMU_MISS_TT 1893 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1894 cmp %g6, T_DATA_MMU_MISS 1895 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 18961: 1897 /* 1898 * g2 = tag access reg 1899 * g3 = type 1900 */ 1901 sethi %hi(sfmmu_tsbmiss_exception), %g1 1902 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 1903 ba,pt %xcc, sys_trap 1904 mov -1, %g4 1905 /*NOTREACHED*/ 1906 SET_SIZE(sfmmu_mmu_trap) 1907 1908 ENTRY_NP(sfmmu_suspend_tl) 1909 SET_GL_REG(1) 1910 USE_ALTERNATE_GLOBALS(%g5) 1911 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 1912 rdpr %tt, %g6 1913 cmp %g6, FAST_IMMU_MISS_TT 1914 be,a,pn %icc, 1f 1915 mov T_INSTR_MMU_MISS, %g3 1916 mov %g5, %g2 1917 cmp %g6, FAST_DMMU_MISS_TT 1918 move %icc, T_DATA_MMU_MISS, %g3 1919 movne %icc, T_DATA_PROT, %g3 19201: 1921 sethi %hi(sfmmu_tsbmiss_suspended), %g1 1922 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 1923 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 1924 ba,pt %xcc, sys_trap 1925 mov PIL_15, %g4 1926 /*NOTREACHED*/ 1927 SET_SIZE(sfmmu_suspend_tl) 1928 1929 /* 1930 * No %g registers in use at this point. 1931 */ 1932 ENTRY_NP(sfmmu_window_trap) 1933 rdpr %tpc, %g1 1934#ifdef sun4v 1935#ifdef DEBUG 1936 /* We assume previous %gl was 1 */ 1937 rdpr %tstate, %g4 1938 srlx %g4, TSTATE_GL_SHIFT, %g4 1939 and %g4, TSTATE_GL_MASK, %g4 1940 cmp %g4, 1 1941 bne,a,pn %icc, ptl1_panic 1942 mov PTL1_BAD_WTRAP, %g1 1943#endif /* DEBUG */ 1944 /* user miss at tl>1. better be the window handler or user_rtt */ 1945 /* in user_rtt? */ 1946 set rtt_fill_start, %g4 1947 cmp %g1, %g4 1948 blu,pn %xcc, 6f 1949 .empty 1950 set rtt_fill_end, %g4 1951 cmp %g1, %g4 1952 bgeu,pn %xcc, 6f 1953 nop 1954 set fault_rtt_fn1, %g1 1955 wrpr %g0, %g1, %tnpc 1956 ba,a 7f 19576: 1958 ! must save this trap level before descending trap stack 1959 ! no need to save %tnpc, either overwritten or discarded 1960 ! already got it: rdpr %tpc, %g1 1961 rdpr %tstate, %g6 1962 rdpr %tt, %g7 1963 ! trap level saved, go get underlying trap type 1964 rdpr %tl, %g5 1965 sub %g5, 1, %g3 1966 wrpr %g3, %tl 1967 rdpr %tt, %g2 1968 wrpr %g5, %tl 1969 ! restore saved trap level 1970 wrpr %g1, %tpc 1971 wrpr %g6, %tstate 1972 wrpr %g7, %tt 1973#else /* sun4v */ 1974 /* user miss at tl>1. better be the window handler */ 1975 rdpr %tl, %g5 1976 sub %g5, 1, %g3 1977 wrpr %g3, %tl 1978 rdpr %tt, %g2 1979 wrpr %g5, %tl 1980#endif /* sun4v */ 1981 and %g2, WTRAP_TTMASK, %g4 1982 cmp %g4, WTRAP_TYPE 1983 bne,pn %xcc, 1f 1984 nop 1985 /* tpc should be in the trap table */ 1986 set trap_table, %g4 1987 cmp %g1, %g4 1988 blt,pn %xcc, 1f 1989 .empty 1990 set etrap_table, %g4 1991 cmp %g1, %g4 1992 bge,pn %xcc, 1f 1993 .empty 1994 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 1995 add %g1, WTRAP_FAULTOFF, %g1 1996 wrpr %g0, %g1, %tnpc 19977: 1998 /* 1999 * some wbuf handlers will call systrap to resolve the fault 2000 * we pass the trap type so they figure out the correct parameters. 2001 * g5 = trap type, g6 = tag access reg 2002 */ 2003 2004 /* 2005 * only use g5, g6, g7 registers after we have switched to alternate 2006 * globals. 2007 */ 2008 SET_GL_REG(1) 2009 USE_ALTERNATE_GLOBALS(%g5) 2010 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 2011 rdpr %tt, %g7 2012 cmp %g7, FAST_IMMU_MISS_TT 2013 be,a,pn %icc, ptl1_panic 2014 mov PTL1_BAD_WTRAP, %g1 2015 cmp %g7, T_INSTR_MMU_MISS 2016 be,a,pn %icc, ptl1_panic 2017 mov PTL1_BAD_WTRAP, %g1 2018 mov T_DATA_PROT, %g5 2019 cmp %g7, FAST_DMMU_MISS_TT 2020 move %icc, T_DATA_MMU_MISS, %g5 2021 cmp %g7, T_DATA_MMU_MISS 2022 move %icc, T_DATA_MMU_MISS, %g5 2023 ! XXXQ AGS re-check out this one 2024 done 20251: 2026 CPU_ADDR(%g1, %g4) 2027 ld [%g1 + CPU_TL1_HDLR], %g4 2028 brnz,a,pt %g4, sfmmu_mmu_trap 2029 st %g0, [%g1 + CPU_TL1_HDLR] 2030 ba,pt %icc, ptl1_panic 2031 mov PTL1_BAD_TRAP, %g1 2032 SET_SIZE(sfmmu_window_trap) 2033 2034 ENTRY_NP(sfmmu_kpm_exception) 2035 /* 2036 * We have accessed an unmapped segkpm address or a legal segkpm 2037 * address which is involved in a VAC alias conflict prevention. 2038 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 2039 * set. If it is, we will instead note that a fault has occurred 2040 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 2041 * a "retry"). This will step over the faulting instruction. 2042 * Note that this means that a legal segkpm address involved in 2043 * a VAC alias conflict prevention (a rare case to begin with) 2044 * cannot be used in DTrace. 2045 */ 2046 CPU_INDEX(%g1, %g2) 2047 set cpu_core, %g2 2048 sllx %g1, CPU_CORE_SHIFT, %g1 2049 add %g1, %g2, %g1 2050 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 2051 andcc %g2, CPU_DTRACE_NOFAULT, %g0 2052 bz 0f 2053 or %g2, CPU_DTRACE_BADADDR, %g2 2054 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 2055 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 2056 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 2057 done 20580: 2059 TSTAT_CHECK_TL1(1f, %g1, %g2) 20601: 2061 SET_GL_REG(1) 2062 USE_ALTERNATE_GLOBALS(%g5) 2063 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 2064 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 2065 /* 2066 * g2=tagacc g3.l=type g3.h=0 2067 */ 2068 sethi %hi(trap), %g1 2069 or %g1, %lo(trap), %g1 2070 ba,pt %xcc, sys_trap 2071 mov -1, %g4 2072 SET_SIZE(sfmmu_kpm_exception) 2073 2074#endif /* lint */ 2075 2076#if defined (lint) 2077 2078void 2079sfmmu_tsb_miss(void) 2080{ 2081} 2082 2083void 2084sfmmu_kpm_dtsb_miss(void) 2085{ 2086} 2087 2088void 2089sfmmu_kpm_dtsb_miss_small(void) 2090{ 2091} 2092 2093#else /* lint */ 2094 2095#if (IMAP_SEG != 0) 2096#error - ism_map->ism_seg offset is not zero 2097#endif 2098 2099/* 2100 * Copies ism mapping for this ctx in param "ism" if this is a ISM 2101 * tlb miss and branches to label "ismhit". If this is not an ISM 2102 * process or an ISM tlb miss it falls thru. 2103 * 2104 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 2105 * this process. 2106 * If so, it will branch to label "ismhit". If not, it will fall through. 2107 * 2108 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 2109 * so that any other threads of this process will not try and walk the ism 2110 * maps while they are being changed. 2111 * 2112 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 2113 * will make sure of that. This means we can terminate our search on 2114 * the first zero mapping we find. 2115 * 2116 * Parameters: 2117 * tagacc = tag access register (vaddr + ctx) (in) 2118 * tsbmiss = address of tsb miss area (in) 2119 * ismseg = contents of ism_seg for this ism map (out) 2120 * ismhat = physical address of imap_ismhat for this ism map (out) 2121 * tmp1 = scratch reg (CLOBBERED) 2122 * tmp2 = scratch reg (CLOBBERED) 2123 * tmp3 = scratch reg (CLOBBERED) 2124 * label: temporary labels 2125 * ismhit: label where to jump to if an ism dtlb miss 2126 * exitlabel:label where to jump if hat is busy due to hat_unshare. 2127 */ 2128#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 2129 label, ismhit) \ 2130 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 2131 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 2132 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 2133label/**/1: ;\ 2134 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 2135 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 2136label/**/2: ;\ 2137 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 2138 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 2139 lduha [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 2140 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 2141 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 2142 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 2143 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 2144 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 2145 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 2146 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 2147 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 2148 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 2149 ;\ 2150 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 2151 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 2152 cmp ismhat, tmp1 ;\ 2153 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 2154 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 2155 ;\ 2156 add tmp3, IBLK_NEXTPA, tmp1 ;\ 2157 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 2158 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 2159 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 2160label/**/3: 2161 2162/* 2163 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 2164 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 2165 * Parameters: 2166 * vaddr = reg containing virtual address 2167 * hatid = reg containing sfmmu pointer 2168 * hmeshift = constant/register to shift vaddr to obtain vapg 2169 * hmebp = register where bucket pointer will be stored 2170 * vapg = register where virtual page will be stored 2171 * tmp1, tmp2 = tmp registers 2172 */ 2173 2174 2175#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 2176 vapg, label, tmp1, tmp2) \ 2177 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 2178 brnz,a,pt tmp1, label/**/1 ;\ 2179 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 2180 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 2181 ba,pt %xcc, label/**/2 ;\ 2182 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 2183label/**/1: ;\ 2184 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 2185label/**/2: ;\ 2186 srlx tagacc, hmeshift, vapg ;\ 2187 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 2188 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 2189 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 2190 add hmebp, tmp1, hmebp 2191 2192/* 2193 * hashtag includes bspage + hashno (64 bits). 2194 */ 2195 2196#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 2197 sllx vapg, hmeshift, vapg ;\ 2198 or vapg, hashno, hblktag 2199 2200/* 2201 * Function to traverse hmeblk hash link list and find corresponding match. 2202 * The search is done using physical pointers. It returns the physical address 2203 * and virtual address pointers to the hmeblk that matches with the tag 2204 * provided. 2205 * Parameters: 2206 * hmebp = register that points to hme hash bucket, also used as 2207 * tmp reg (clobbered) 2208 * hmeblktag = register with hmeblk tag match 2209 * hatid = register with hatid 2210 * hmeblkpa = register where physical ptr will be stored 2211 * hmeblkva = register where virtual ptr will be stored 2212 * tmp1 = tmp reg 2213 * label: temporary label 2214 */ 2215 2216#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2217 tsbarea, tmp1, label) \ 2218 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 2219 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2220 add hmebp, HMEBUCK_HBLK, hmeblkva ;\ 2221 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2222 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2223label/**/1: ;\ 2224 brz,pn hmeblkva, label/**/2 ;\ 2225 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2226 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 2227 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2228 add hmebp, CLONGSIZE, hmebp ;\ 2229 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 2230 xor tmp1, hmeblktag, tmp1 ;\ 2231 xor hmebp, hatid, hmebp ;\ 2232 or hmebp, tmp1, hmebp ;\ 2233 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 2234 add hmeblkpa, HMEBLK_NEXT, hmebp ;\ 2235 ldna [hmebp]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2236 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 2237 ba,pt %xcc, label/**/1 ;\ 2238 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2239label/**/2: 2240 2241 2242#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2243#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2244#endif 2245 2246/* 2247 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2248 * he offset for the corresponding hment. 2249 * Parameters: 2250 * vaddr = register with virtual address 2251 * hmeblkpa = physical pointer to hme_blk 2252 * hment = register where address of hment will be stored 2253 * hmentoff = register where hment offset will be stored 2254 * label1 = temporary label 2255 */ 2256#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, tmp1, label1) \ 2257 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2258 lda [hmentoff]ASI_MEM, tmp1 ;\ 2259 andcc tmp1, HBLK_SZMASK, %g0 /* tmp1 = get_hblk_sz(%g5) */ ;\ 2260 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2261 or %g0, HMEBLK_HME1, hmentoff ;\ 2262 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2263 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2264 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2265 add tmp1, HMEBLK_HME1, hmentoff ;\ 2266label1: 2267 2268/* 2269 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2270 * 2271 * tagacc = tag access register (vaddr + ctx) (in) 2272 * hatid = sfmmu pointer for TSB miss (in) 2273 * tte = tte for TLB miss if found, otherwise clobbered (out) 2274 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2275 * hmeblkva = VA of hment if found, otherwise clobbered (out) 2276 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2277 * hmentoff = temporarily stores hment offset (clobbered) 2278 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2279 * for this page size. 2280 * hashno = constant/register hash number 2281 * label = temporary label for branching within macro. 2282 * foundlabel = label to jump to when tte is found. 2283 * suspendlabel= label to jump to when tte is suspended. 2284 * exitlabel = label to jump to when tte is not found. The hmebp lock 2285 * is still held at this time. 2286 * 2287 * The caller should set up the tsbmiss->scratch[2] field correctly before 2288 * calling this funciton (aka TSBMISS_SCRATCH + TSBMISS_HATID) 2289 */ 2290#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmentoff, \ 2291 hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \ 2292 ;\ 2293 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2294 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2295 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2296 hmeblkpa, label/**/5, hmentoff, hmeblkva) ;\ 2297 ;\ 2298 /* ;\ 2299 * tagacc = tagacc ;\ 2300 * hatid = hatid ;\ 2301 * tsbarea = tsbarea ;\ 2302 * tte = hmebp (hme bucket pointer) ;\ 2303 * hmeblkpa = vapg (virtual page) ;\ 2304 * hmentoff, hmeblkva = scratch ;\ 2305 */ ;\ 2306 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmentoff) ;\ 2307 ;\ 2308 /* ;\ 2309 * tagacc = tagacc ;\ 2310 * hatid = hatid ;\ 2311 * tte = hmebp ;\ 2312 * hmeblkpa = CLOBBERED ;\ 2313 * hmentoff = htag_bspage & hashno ;\ 2314 * hmeblkva = scratch ;\ 2315 */ ;\ 2316 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2317 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2318 HMEHASH_SEARCH(tte, hmentoff, hatid, hmeblkpa, hmeblkva, \ 2319 tsbarea, tagacc, label/**/1) ;\ 2320 /* ;\ 2321 * tagacc = CLOBBERED ;\ 2322 * tte = CLOBBERED ;\ 2323 * hmeblkpa = hmeblkpa ;\ 2324 * hmeblkva = hmeblkva ;\ 2325 */ ;\ 2326 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2327 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2328 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2329 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2330 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2331 nop ;\ 2332label/**/4: ;\ 2333 /* ;\ 2334 * We have found the hmeblk containing the hment. ;\ 2335 * Now we calculate the corresponding tte. ;\ 2336 * ;\ 2337 * tagacc = tagacc ;\ 2338 * hatid = clobbered ;\ 2339 * tte = hmebp ;\ 2340 * hmeblkpa = hmeblkpa ;\ 2341 * hmentoff = hblktag ;\ 2342 * hmeblkva = hmeblkva ;\ 2343 */ ;\ 2344 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hmentoff, hatid, label/**/2) ;\ 2345 ;\ 2346 add hmentoff, SFHME_TTE, hmentoff ;\ 2347 add hmeblkpa, hmentoff, hmeblkpa ;\ 2348 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2349 add hmeblkva, hmentoff, hmeblkva ;\ 2350 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2351 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmentoff ;\ 2352 HMELOCK_EXIT(hmentoff, hmentoff, ASI_MEM) /* drop lock */ ;\ 2353 set TTE_SUSPEND, hmentoff ;\ 2354 TTE_SUSPEND_INT_SHIFT(hmentoff) ;\ 2355 btst tte, hmentoff ;\ 2356 bz,pt %xcc, foundlabel ;\ 2357 nop ;\ 2358 ;\ 2359 /* ;\ 2360 * Mapping is suspended, so goto suspend label. ;\ 2361 */ ;\ 2362 ba,pt %xcc, suspendlabel ;\ 2363 nop 2364 2365 /* 2366 * KERNEL PROTECTION HANDLER 2367 * 2368 * g1 = tsb8k pointer register (clobbered) 2369 * g2 = tag access register (ro) 2370 * g3 - g7 = scratch registers 2371 * 2372 * Note: This function is patched at runtime for performance reasons. 2373 * Any changes here require sfmmu_patch_ktsb fixed. 2374 */ 2375 ENTRY_NP(sfmmu_kprot_trap) 2376 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2377sfmmu_kprot_patch_ktsb_base: 2378 RUNTIME_PATCH_SETX(%g1, %g6) 2379 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2380sfmmu_kprot_patch_ktsb_szcode: 2381 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2382 2383 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2384 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2385 2386 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2387sfmmu_kprot_patch_ktsb4m_base: 2388 RUNTIME_PATCH_SETX(%g3, %g6) 2389 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2390sfmmu_kprot_patch_ktsb4m_szcode: 2391 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2392 2393 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2394 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2395 2396 CPU_TSBMISS_AREA(%g6, %g7) 2397 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2398 ba,pt %xcc, sfmmu_tsb_miss_tt 2399 nop 2400 2401 /* 2402 * USER PROTECTION HANDLER 2403 * 2404 * g1 = tsb8k pointer register (ro) 2405 * g2 = tag access register (ro) 2406 * g3 = faulting context (clobbered, currently not used) 2407 * g4 - g7 = scratch registers 2408 */ 2409 ALTENTRY(sfmmu_uprot_trap) 2410#ifdef sun4v 2411 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2412 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2413 2414 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2415 brlz,pt %g3, 9f /* check for 2nd TSB */ 2416 mov %g0, %g3 /* clear second tsbe ptr */ 2417 2418 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2419 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2420 2421#else /* sun4v */ 2422#ifdef UTSB_PHYS 2423 /* g1 = first TSB entry ptr */ 2424 GET_2ND_TSBREG(%g3) 2425 brlz,a,pt %g3, 9f /* check for 2nd TSB */ 2426 mov %g0, %g3 /* clear second tsbe ptr */ 2427 2428 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2429 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2430#else /* UTSB_PHYS */ 2431 brgez,pt %g1, 9f /* check for 2nd TSB */ 2432 mov %g0, %g3 /* clear second tsbe ptr */ 2433 2434 mov %g2, %g7 2435 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2436 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2437 mov %g1, %g7 2438 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2439#endif /* UTSB_PHYS */ 2440#endif /* sun4v */ 24419: 2442 CPU_TSBMISS_AREA(%g6, %g7) 2443 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2444 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2445 nop 2446 2447 /* 2448 * Kernel 8K page iTLB miss. We also get here if we took a 2449 * fast instruction access mmu miss trap while running in 2450 * invalid context. 2451 * 2452 * %g1 = 8K TSB pointer register (not used, clobbered) 2453 * %g2 = tag access register (used) 2454 * %g3 = faulting context id (used) 2455 * %g7 = 4M virtual page number for tag matching (used) 2456 */ 2457 .align 64 2458 ALTENTRY(sfmmu_kitlb_miss) 2459 brnz,pn %g3, tsb_tl0_noctxt 2460 nop 2461 2462 /* kernel miss */ 2463 /* get kernel tsb pointer */ 2464 /* we patch the next set of instructions at run time */ 2465 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2466iktsbbase: 2467 RUNTIME_PATCH_SETX(%g4, %g5) 2468 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2469 2470iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2471 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2472 or %g4, %g1, %g1 ! form tsb ptr 2473 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2474 cmp %g4, %g7 2475 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2476 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2477 bz,pn %icc, exec_fault 2478 nop 2479 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2480 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2481 retry 2482 2483 /* 2484 * Kernel dTLB miss. We also get here if we took a fast data 2485 * access mmu miss trap while running in invalid context. 2486 * 2487 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2488 * We select the TSB miss handler to branch to depending on 2489 * the virtual address of the access. In the future it may 2490 * be desirable to separate kpm TTEs into their own TSB, 2491 * in which case all that needs to be done is to set 2492 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2493 * early in the miss if we detect a kpm VA to a new handler. 2494 * 2495 * %g1 = 8K TSB pointer register (not used, clobbered) 2496 * %g2 = tag access register (used) 2497 * %g3 = faulting context id (used) 2498 */ 2499 .align 64 2500 ALTENTRY(sfmmu_kdtlb_miss) 2501 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2502 nop 2503 2504 /* Gather some stats for kpm misses in the TLB. */ 2505 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2506 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2507 2508 /* 2509 * Get first TSB offset and look for 8K/64K/512K mapping 2510 * using the 8K virtual page as the index. 2511 * 2512 * We patch the next set of instructions at run time; 2513 * any changes here require sfmmu_patch_ktsb changes too. 2514 */ 2515dktsbbase: 2516 RUNTIME_PATCH_SETX(%g7, %g6) 2517 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2518 2519dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2520 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2521 2522 /* 2523 * At this point %g1 is our index into the TSB. 2524 * We just masked off enough bits of the VA depending 2525 * on our TSB size code. 2526 */ 2527 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2528 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2529 cmp %g6, %g4 ! compare tag 2530 bne,pn %xcc, dktsb4m_kpmcheck_small 2531 add %g7, %g1, %g1 /* form tsb ptr */ 2532 TT_TRACE(trace_tsbhit) 2533 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2534 /* trapstat expects tte in %g5 */ 2535 retry 2536 2537 /* 2538 * If kpm is using large pages, the following instruction needs 2539 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2540 * so that we will probe the 4M TSB regardless of the VA. In 2541 * the case kpm is using small pages, we know no large kernel 2542 * mappings are located above 0x80000000.00000000 so we skip the 2543 * probe as an optimization. 2544 */ 2545dktsb4m_kpmcheck_small: 2546 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2547 /* delay slot safe, below */ 2548 2549 /* 2550 * Get second TSB offset and look for 4M mapping 2551 * using 4M virtual page as the TSB index. 2552 * 2553 * Here: 2554 * %g1 = 8K TSB pointer. Don't squash it. 2555 * %g2 = tag access register (we still need it) 2556 */ 2557 srlx %g2, MMU_PAGESHIFT4M, %g3 2558 2559 /* 2560 * We patch the next set of instructions at run time; 2561 * any changes here require sfmmu_patch_ktsb changes too. 2562 */ 2563dktsb4mbase: 2564 RUNTIME_PATCH_SETX(%g7, %g6) 2565 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2566dktsb4m: 2567 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2568 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2569 2570 /* 2571 * At this point %g3 is our index into the TSB. 2572 * We just masked off enough bits of the VA depending 2573 * on our TSB size code. 2574 */ 2575 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2576 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2577 cmp %g6, %g4 ! compare tag 2578 2579dktsb4m_tsbmiss: 2580 bne,pn %xcc, dktsb4m_kpmcheck 2581 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2582 TT_TRACE(trace_tsbhit) 2583 /* we don't check TTE size here since we assume 4M TSB is separate */ 2584 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2585 /* trapstat expects tte in %g5 */ 2586 retry 2587 2588 /* 2589 * So, we failed to find a valid TTE to match the faulting 2590 * address in either TSB. There are a few cases that could land 2591 * us here: 2592 * 2593 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2594 * to sfmmu_tsb_miss_tt to handle the miss. 2595 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2596 * 4M TSB. Let segkpm handle it. 2597 * 2598 * Note that we shouldn't land here in the case of a kpm VA when 2599 * kpm_smallpages is active -- we handled that case earlier at 2600 * dktsb4m_kpmcheck_small. 2601 * 2602 * At this point: 2603 * g1 = 8K-indexed primary TSB pointer 2604 * g2 = tag access register 2605 * g3 = 4M-indexed secondary TSB pointer 2606 */ 2607dktsb4m_kpmcheck: 2608 cmp %g2, %g0 2609 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2610 nop 2611 ba,a,pt %icc, sfmmu_tsb_miss_tt 2612 nop 2613 2614#ifdef sun4v 2615 /* 2616 * User instruction miss w/ single TSB. 2617 * The first probe covers 8K, 64K, and 512K page sizes, 2618 * because 64K and 512K mappings are replicated off 8K 2619 * pointer. 2620 * 2621 * g1 = tsb8k pointer register 2622 * g2 = tag access register 2623 * g3 - g6 = scratch registers 2624 * g7 = TSB tag to match 2625 */ 2626 .align 64 2627 ALTENTRY(sfmmu_uitlb_fastpath) 2628 2629 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2630 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2631 ba,pn %xcc, sfmmu_tsb_miss_tt 2632 mov %g0, %g3 2633 2634 /* 2635 * User data miss w/ single TSB. 2636 * The first probe covers 8K, 64K, and 512K page sizes, 2637 * because 64K and 512K mappings are replicated off 8K 2638 * pointer. 2639 * 2640 * g1 = tsb8k pointer register 2641 * g2 = tag access register 2642 * g3 - g6 = scratch registers 2643 * g7 = TSB tag to match 2644 */ 2645 .align 64 2646 ALTENTRY(sfmmu_udtlb_fastpath) 2647 2648 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2649 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2650 ba,pn %xcc, sfmmu_tsb_miss_tt 2651 mov %g0, %g3 2652 2653 /* 2654 * User instruction miss w/ multiple TSBs (sun4v). 2655 * The first probe covers 8K, 64K, and 512K page sizes, 2656 * because 64K and 512K mappings are replicated off 8K 2657 * pointer. Second probe covers 4M page size only. 2658 * 2659 * Just like sfmmu_udtlb_slowpath, except: 2660 * o Uses ASI_ITLB_IN 2661 * o checks for execute permission 2662 * o No ISM prediction. 2663 * 2664 * g1 = tsb8k pointer register 2665 * g2 = tag access register 2666 * g3 - g6 = scratch registers 2667 * g7 = TSB tag to match 2668 */ 2669 .align 64 2670 ALTENTRY(sfmmu_uitlb_slowpath) 2671 2672 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2673 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2674 /* g4 - g5 = clobbered here */ 2675 2676 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2677 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2678 srlx %g2, TAG_VALO_SHIFT, %g7 2679 PROBE_2ND_ITSB(%g3, %g7) 2680 /* NOT REACHED */ 2681 2682#else /* sun4v */ 2683 2684 /* 2685 * User instruction miss w/ multiple TSBs (sun4u). 2686 * The first probe covers 8K, 64K, and 512K page sizes, 2687 * because 64K and 512K mappings are replicated off 8K 2688 * pointer. Second probe covers 4M page size only. 2689 * 2690 * Just like sfmmu_udtlb_slowpath, except: 2691 * o Uses ASI_ITLB_IN 2692 * o checks for execute permission 2693 * o No ISM prediction. 2694 * 2695 * g1 = tsb8k pointer register 2696 * g2 = tag access register 2697 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch 2698 * g4 - g6 = scratch registers 2699 * g7 = TSB tag to match 2700 */ 2701 .align 64 2702 ALTENTRY(sfmmu_uitlb_slowpath) 2703 2704#ifdef UTSB_PHYS 2705 /* 2706 * g1 = 1st TSB entry pointer 2707 * g3 = 2nd TSB base register 2708 * Need 2nd TSB entry pointer for 2nd probe. 2709 */ 2710 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2711 2712 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2713#else /* UTSB_PHYS */ 2714 mov %g1, %g3 /* save tsb8k reg in %g3 */ 2715 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 2716 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2717 2718 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 2719 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 2720 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 2721#endif /* UTSB_PHYS */ 2722 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2723 srlx %g2, TAG_VALO_SHIFT, %g7 2724 PROBE_2ND_ITSB(%g3, %g7, isynth) 2725 /* NOT REACHED */ 2726#endif /* sun4v */ 2727 2728 /* 2729 * User data miss w/ multiple TSBs. 2730 * The first probe covers 8K, 64K, and 512K page sizes, 2731 * because 64K and 512K mappings are replicated off 8K 2732 * pointer. Second probe covers 4M page size only. 2733 * 2734 * We consider probing for 4M pages first if the VA falls 2735 * in a range that's likely to be ISM. 2736 * 2737 * g1 = tsb8k pointer register 2738 * g2 = tag access register 2739 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch 2740 * g4 - g6 = scratch registers 2741 * g7 = TSB tag to match 2742 */ 2743 .align 64 2744 ALTENTRY(sfmmu_udtlb_slowpath) 2745 2746 /* 2747 * Check for ISM. If it exists, look for 4M mappings in the second TSB 2748 * first, then probe for other mappings in the first TSB if that fails. 2749 */ 2750 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 2751 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 2752 mov %g1, %g3 2753 2754udtlb_miss_probefirst: 2755 /* 2756 * g1 = 8K TSB pointer register 2757 * g2 = tag access register 2758 * g3 = (potentially) second TSB entry ptr 2759 * g6 = ism pred. 2760 * g7 = vpg_4m 2761 */ 2762#ifdef sun4v 2763 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2764 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2765 2766 /* 2767 * Here: 2768 * g1 = first TSB pointer 2769 * g2 = tag access reg 2770 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2771 */ 2772 brgz,pn %g6, sfmmu_tsb_miss_tt 2773 nop 2774#else /* sun4v */ 2775#ifndef UTSB_PHYS 2776 mov %g1, %g4 2777 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 2778#endif UTSB_PHYS 2779 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2780 2781 /* 2782 * Here: 2783 * g1 = first TSB pointer 2784 * g2 = tag access reg 2785 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2786 */ 2787 brgz,pn %g6, sfmmu_tsb_miss_tt 2788 nop 2789#ifndef UTSB_PHYS 2790 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 2791#endif UTSB_PHYS 2792 /* fall through in 8K->4M probe order */ 2793#endif /* sun4v */ 2794 2795udtlb_miss_probesecond: 2796 /* 2797 * Look in the second TSB for the TTE 2798 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 2799 * g2 = tag access reg 2800 * g3 = 8K TSB pointer register 2801 * g6 = ism pred. 2802 * g7 = vpg_4m 2803 */ 2804#ifdef sun4v 2805 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 2806 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2807 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 2808#else /* sun4v */ 2809#ifdef UTSB_PHYS 2810 GET_2ND_TSBREG(%g3) 2811 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2812 /* tagacc (%g2) is okay, no need to reload, %g3 = second tsbe ptr */ 2813#else /* UTSB_PHYS */ 2814 mov %g3, %g7 2815 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 2816 /* %g2 clobbered, %g3 =second tsbe ptr */ 2817 mov MMU_TAG_ACCESS, %g2 2818 ldxa [%g2]ASI_DMMU, %g2 2819#endif /* UTSB_PHYS */ 2820#endif /* sun4v */ 2821 2822 srlx %g2, TAG_VALO_SHIFT, %g7 2823 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2824 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 2825 brgz,pn %g6, udtlb_miss_probefirst 2826 nop 2827 2828 /* fall through to sfmmu_tsb_miss_tt */ 2829 2830 ALTENTRY(sfmmu_tsb_miss_tt) 2831 TT_TRACE(trace_tsbmiss) 2832 /* 2833 * We get here if there is a TSB miss OR a write protect trap. 2834 * 2835 * g1 = First TSB entry pointer 2836 * g2 = tag access register 2837 * g3 = 4M TSB entry pointer; NULL if no 2nd TSB 2838 * g4 - g7 = scratch registers 2839 */ 2840 2841 ALTENTRY(sfmmu_tsb_miss) 2842 2843 /* 2844 * If trapstat is running, we need to shift the %tpc and %tnpc to 2845 * point to trapstat's TSB miss return code (note that trapstat 2846 * itself will patch the correct offset to add). 2847 */ 2848 rdpr %tl, %g7 2849 cmp %g7, 1 2850 ble,pt %xcc, 0f 2851 sethi %hi(KERNELBASE), %g6 2852 rdpr %tpc, %g7 2853 or %g6, %lo(KERNELBASE), %g6 2854 cmp %g7, %g6 2855 bgeu,pt %xcc, 0f 2856 /* delay slot safe */ 2857 2858 ALTENTRY(tsbmiss_trapstat_patch_point) 2859 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 2860 wrpr %g7, %tpc 2861 add %g7, 4, %g7 2862 wrpr %g7, %tnpc 28630: 2864 CPU_TSBMISS_AREA(%g6, %g7) 2865 2866 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save first tsb pointer */ 2867 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save second tsb pointer */ 2868 2869 sllx %g2, TAGACC_CTX_LSHIFT, %g3 2870 brz,a,pn %g3, 1f /* skip ahead if kernel */ 2871 ldn [%g6 + TSBMISS_KHATID], %g7 2872 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 2873 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 2874 2875 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 2876 2877 cmp %g3, INVALID_CONTEXT 2878 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 2879 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 2880 2881 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 2882 /* 2883 * The miss wasn't in an ISM segment. 2884 * 2885 * %g1 %g3, %g4, %g5, %g7 all clobbered 2886 * %g2 = tag access (vaddr + ctx) 2887 */ 2888 2889 ba,pt %icc, 2f 2890 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 2891 28921: 2893 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 2894 /* 2895 * 8K and 64K hash. 2896 */ 28972: 2898 2899 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 2900 MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte, 2901 sfmmu_suspend_tl, tsb_512K) 2902 /* NOT REACHED */ 2903 2904tsb_512K: 2905 ldn [%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3 2906 sllx %g3, TAGACC_CTX_LSHIFT, %g5 2907 brz,pn %g5, 3f 2908 lduh [%g6 + TSBMISS_HATFLAGS], %g4 2909 and %g4, HAT_512K_FLAG, %g5 2910 2911 /* 2912 * Note that there is a small window here where we may have 2913 * a 512k page in the hash list but have not set the HAT_512K_FLAG 2914 * flag yet, so we will skip searching the 512k hash list. 2915 * In this case we will end up in pagefault which will find 2916 * the mapping and return. So, in this instance we will end up 2917 * spending a bit more time resolving this TSB miss, but it can 2918 * only happen once per process and even then, the chances of that 2919 * are very small, so it's not worth the extra overhead it would 2920 * take to close this window. 2921 */ 2922 brz,pn %g5, tsb_4M 2923 nop 29243: 2925 /* 2926 * 512K hash 2927 */ 2928 2929 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 2930 MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte, 2931 sfmmu_suspend_tl, tsb_4M) 2932 /* NOT REACHED */ 2933 2934tsb_4M: 2935 ldn [%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3 2936 sllx %g3, TAGACC_CTX_LSHIFT, %g5 2937 brz,pn %g5, 4f 2938 lduh [%g6 + TSBMISS_HATFLAGS], %g4 2939 and %g4, HAT_4M_FLAG, %g5 2940 brz,pn %g5, tsb_32M 2941 nop 29424: 2943 /* 2944 * 4M hash 2945 */ 2946 2947 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 2948 MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte, 2949 sfmmu_suspend_tl, tsb_32M) 2950 /* NOT REACHED */ 2951 2952tsb_32M: 2953#ifndef sun4v 2954 GET_CPU_IMPL(%g5) 2955 cmp %g5, OLYMPUS_C_IMPL 2956 be,pn %xcc, 0f 2957 nop 2958 cmp %g5, PANTHER_IMPL 2959 bne,pt %xcc, tsb_pagefault 2960 nop 2961#endif 2962 29630: 2964 ldn [%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3 2965 sllx %g3, TAGACC_CTX_LSHIFT, %g5 2966#ifdef sun4v 2967 brz,pn %g5, 6f 2968#else 2969 brz,pn %g5, tsb_pagefault 2970#endif 2971 lduh [%g6 + TSBMISS_HATFLAGS], %g4 2972 and %g4, HAT_32M_FLAG, %g5 2973 brz,pn %g5, tsb_256M 2974 nop 29755: 2976 /* 2977 * 32M hash 2978 */ 2979 2980 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 2981 MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte, 2982 sfmmu_suspend_tl, tsb_256M) 2983 /* NOT REACHED */ 2984 2985tsb_256M: 2986 lduh [%g6 + TSBMISS_HATFLAGS], %g4 2987 and %g4, HAT_256M_FLAG, %g5 2988 brz,pn %g5, tsb_pagefault 2989 nop 29906: 2991 /* 2992 * 256M hash 2993 */ 2994 2995 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 2996 MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte, 2997 sfmmu_suspend_tl, tsb_pagefault) 2998 /* NOT REACHED */ 2999 3000tsb_checktte: 3001 /* 3002 * g3 = tte 3003 * g4 = tte pa 3004 * g5 = tte va 3005 * g6 = tsbmiss area 3006 */ 3007 brgez,pn %g3, tsb_pagefault /* if tte invalid branch */ 3008 nop 3009 3010tsb_validtte: 3011 /* 3012 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3013 */ 3014 rdpr %tt, %g7 3015 cmp %g7, FAST_PROT_TT 3016 bne,pt %icc, 4f 3017 nop 3018 3019 TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod, 3020 tsb_protfault) 3021 3022 rdpr %tt, %g5 3023 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3024 ba,pt %xcc, tsb_update_tl1 3025 nop 3026 30274: 3028 /* 3029 * If ITLB miss check exec bit. 3030 * If not set treat as invalid TTE. 3031 */ 3032 cmp %g7, T_INSTR_MMU_MISS 3033 be,pn %icc, 5f 3034 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3035 cmp %g7, FAST_IMMU_MISS_TT 3036 bne,pt %icc, 3f 3037 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 30385: 3039 bz,pn %icc, tsb_protfault 3040 nop 3041 30423: 3043 /* 3044 * Set reference bit if not already set 3045 */ 3046 TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref) 3047 3048 /* 3049 * Now, load into TSB/TLB. At this point: 3050 * g3 = tte 3051 * g4 = patte 3052 * g6 = tsbmiss area 3053 */ 3054 rdpr %tt, %g5 3055#ifdef sun4v 3056 MMU_FAULT_STATUS_AREA(%g2) 3057 cmp %g5, T_INSTR_MMU_MISS 3058 be,a,pt %icc, 9f 3059 nop 3060 cmp %g5, FAST_IMMU_MISS_TT 3061 be,a,pt %icc, 9f 3062 nop 3063 add %g2, MMFSA_D_, %g2 30649: 3065 ldx [%g2 + MMFSA_CTX_], %g7 3066 sllx %g7, TTARGET_CTX_SHIFT, %g7 3067 ldx [%g2 + MMFSA_ADDR_], %g2 3068 srlx %g2, TTARGET_VA_SHIFT, %g2 3069 or %g2, %g7, %g2 3070#else 3071 cmp %g5, FAST_IMMU_MISS_TT 3072 be,a,pt %icc, tsb_update_tl1 3073 ldxa [%g0]ASI_IMMU, %g2 3074 ldxa [%g0]ASI_DMMU, %g2 3075#endif 3076tsb_update_tl1: 3077 srlx %g2, TTARGET_CTX_SHIFT, %g7 3078 brz,pn %g7, tsb_kernel 3079#ifdef sun4v 3080 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3081#else 3082 srlx %g3, TTE_SZ_SHFT, %g7 3083#endif 3084 3085tsb_user: 3086#ifdef sun4v 3087 cmp %g7, TTE4M 3088 bge,pn %icc, tsb_user4m 3089 nop 3090#else /* sun4v */ 3091 cmp %g7, TTESZ_VALID | TTE4M 3092 be,pn %icc, tsb_user4m 3093 srlx %g3, TTE_SZ2_SHFT, %g7 3094 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3095#ifdef ITLB_32M_256M_SUPPORT 3096 bnz,pn %icc, tsb_user4m 3097 nop 3098#else /* ITLB_32M_256M_SUPPORT */ 3099 bnz,a,pn %icc, tsb_user_pn_synth 3100 cmp %g5, FAST_IMMU_MISS_TT 3101#endif /* ITLB_32M_256M_SUPPORT */ 3102#endif /* sun4v */ 3103 3104tsb_user8k: 3105 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = first TSB ptr 3106 3107#ifndef UTSB_PHYS 3108 mov ASI_N, %g7 ! user TSBs accessed by VA 3109 mov %g7, %asi 3110#endif /* UTSB_PHYS */ 3111 3112 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5) 3113 3114#ifdef sun4v 3115 cmp %g5, T_INSTR_MMU_MISS 3116 be,a,pn %xcc, 9f 3117 mov %g3, %g5 3118#endif /* sun4v */ 3119 cmp %g5, FAST_IMMU_MISS_TT 3120 be,pn %xcc, 9f 3121 mov %g3, %g5 3122 3123 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3124 ! trapstat wants TTE in %g5 3125 retry 31269: 3127 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3128 ! trapstat wants TTE in %g5 3129 retry 3130 3131tsb_user4m: 3132 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 31334: 3134 brz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3135 nop 3136 3137#ifndef UTSB_PHYS 3138 mov ASI_N, %g7 ! user TSBs accessed by VA 3139 mov %g7, %asi 3140#endif /* UTSB_PHYS */ 3141 3142 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6) 3143 31445: 3145#ifdef sun4v 3146 cmp %g5, T_INSTR_MMU_MISS 3147 be,a,pn %xcc, 9f 3148 mov %g3, %g5 3149#endif /* sun4v */ 3150 cmp %g5, FAST_IMMU_MISS_TT 3151 be,pn %xcc, 9f 3152 mov %g3, %g5 3153 3154 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3155 ! trapstat wants TTE in %g5 3156 retry 31579: 3158 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3159 ! trapstat wants TTE in %g5 3160 retry 3161 3162#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3163 /* 3164 * Panther ITLB synthesis. 3165 * The Panther 32M and 256M ITLB code simulates these two large page 3166 * sizes with 4M pages, to provide support for programs, for example 3167 * Java, that may copy instructions into a 32M or 256M data page and 3168 * then execute them. The code below generates the 4M pfn bits and 3169 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3170 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3171 * are ignored by the hardware. 3172 * 3173 * Now, load into TSB/TLB. At this point: 3174 * g2 = tagtarget 3175 * g3 = tte 3176 * g4 = patte 3177 * g5 = tt 3178 * g6 = tsbmiss area 3179 */ 3180tsb_user_pn_synth: 3181 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3182 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3183 bz,pn %icc, 4b /* if not, been here before */ 3184 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3185 brz,a,pn %g1, 5f /* no 2nd tsb */ 3186 mov %g3, %g5 3187 3188 mov MMU_TAG_ACCESS, %g7 3189 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3190 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3191 3192 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3193 mov %g7, %asi 3194 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */ 31955: 3196 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3197 retry 3198 3199tsb_user_itlb_synth: 3200 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3201 3202 mov MMU_TAG_ACCESS, %g7 3203 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3204 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3205 brz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3206 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3207 3208 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3209 mov %g7, %asi 3210 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */ 32117: 3212 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3213 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3214 retry 3215#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3216 3217tsb_kernel: 3218#ifdef sun4v 3219 cmp %g7, TTE4M 3220 bge,pn %icc, 5f 3221#else 3222 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3223 be,pn %icc, 5f 3224#endif 3225 nop 3226 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8k tsbptr 3227 ba,pt %xcc, 6f 3228 nop 32295: 3230 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4m tsbptr 3231 brz,pn %g1, 3f /* skip programming if 4m TSB ptr is NULL */ 3232 nop 32336: 3234#ifndef sun4v 3235tsb_kernel_patch_asi: 3236 or %g0, RUNTIME_PATCH, %g6 3237 mov %g6, %asi ! XXX avoid writing to %asi !! 3238#endif 3239 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7) 32403: 3241#ifdef sun4v 3242 cmp %g5, T_INSTR_MMU_MISS 3243 be,a,pn %icc, 1f 3244 mov %g3, %g5 ! trapstat wants TTE in %g5 3245#endif /* sun4v */ 3246 cmp %g5, FAST_IMMU_MISS_TT 3247 be,pn %icc, 1f 3248 mov %g3, %g5 ! trapstat wants TTE in %g5 3249 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3250 ! trapstat wants TTE in %g5 3251 retry 32521: 3253 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3254 ! trapstat wants TTE in %g5 3255 retry 3256 3257tsb_ism: 3258 /* 3259 * This is an ISM [i|d]tlb miss. We optimize for largest 3260 * page size down to smallest. 3261 * 3262 * g2 = vaddr + ctx aka tag access register 3263 * g3 = ismmap->ism_seg 3264 * g4 = physical address of ismmap->ism_sfmmu 3265 * g6 = tsbmiss area 3266 */ 3267 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3268 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3269 mov PTL1_BAD_ISM, %g1 3270 /* g5 = pa of imap_vb_shift */ 3271 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3272 lduha [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3273 srlx %g3, %g4, %g3 /* clr size field */ 3274 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3275 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3276 and %g2, %g1, %g4 /* g4 = ctx number */ 3277 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3278 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3279 or %g2, %g4, %g2 /* g2 = tagacc (vaddr + ctx) */ 3280 3281 /* 3282 * ISM pages are always locked down. 3283 * If we can't find the tte then pagefault 3284 * and let the spt segment driver resovle it. 3285 * 3286 * g2 = ISM vaddr (offset in ISM seg) 3287 * g6 = tsb miss area 3288 * g7 = ISM hatid 3289 */ 3290 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3291 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3292 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3293 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3294 nop 3295 3296tsb_ism_32M: 3297 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3298 brz,pn %g5, tsb_ism_256M 3299 nop 3300 3301 /* 3302 * 32M hash. 3303 */ 3304 3305 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M, 3306 TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3307 tsb_ism_4M) 3308 /* NOT REACHED */ 3309 3310tsb_ism_32M_found: 3311 brlz,pt %g3, tsb_validtte 3312 nop 3313 ba,pt %xcc, tsb_ism_4M 3314 nop 3315 3316tsb_ism_256M: 3317 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3318 brz,a,pn %g5, ptl1_panic 3319 mov PTL1_BAD_ISM, %g1 3320 3321 /* 3322 * 256M hash. 3323 */ 3324 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M, 3325 TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3326 tsb_ism_4M) 3327 3328tsb_ism_256M_found: 3329 brlz,pt %g3, tsb_validtte 3330 nop 3331 3332tsb_ism_4M: 3333 /* 3334 * 4M hash. 3335 */ 3336 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M, 3337 TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3338 tsb_ism_8K) 3339 /* NOT REACHED */ 3340 3341tsb_ism_4M_found: 3342 brlz,pt %g3, tsb_validtte 3343 nop 3344 3345tsb_ism_8K: 3346 /* 3347 * 8K and 64K hash. 3348 */ 3349 3350 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K, 3351 TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3352 tsb_pagefault) 3353 /* NOT REACHED */ 3354 3355tsb_ism_8K_found: 3356 brlz,pt %g3, tsb_validtte 3357 nop 3358 3359tsb_pagefault: 3360 rdpr %tt, %g7 3361 cmp %g7, FAST_PROT_TT 3362 be,a,pn %icc, tsb_protfault 3363 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3364 3365tsb_protfault: 3366 /* 3367 * we get here if we couldn't find a valid tte in the hash. 3368 * 3369 * If user and we are at tl>1 we go to window handling code. 3370 * 3371 * If kernel and the fault is on the same page as our stack 3372 * pointer, then we know the stack is bad and the trap handler 3373 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3374 * 3375 * If this is a kernel trap and tl>1, panic. 3376 * 3377 * Otherwise we call pagefault. 3378 */ 3379 cmp %g7, FAST_IMMU_MISS_TT 3380#ifdef sun4v 3381 MMU_FAULT_STATUS_AREA(%g4) 3382 ldx [%g4 + MMFSA_I_CTX], %g5 3383 ldx [%g4 + MMFSA_D_CTX], %g4 3384 move %icc, %g5, %g4 3385 cmp %g7, T_INSTR_MMU_MISS 3386 move %icc, %g5, %g4 3387#else 3388 mov MMU_TAG_ACCESS, %g4 3389 ldxa [%g4]ASI_DMMU, %g2 3390 ldxa [%g4]ASI_IMMU, %g5 3391 move %icc, %g5, %g2 3392 cmp %g7, T_INSTR_MMU_MISS 3393 move %icc, %g5, %g2 3394 sllx %g2, TAGACC_CTX_LSHIFT, %g4 3395#endif 3396 brnz,pn %g4, 3f /* skip if not kernel */ 3397 rdpr %tl, %g5 3398 3399 add %sp, STACK_BIAS, %g3 3400 srlx %g3, MMU_PAGESHIFT, %g3 3401 srlx %g2, MMU_PAGESHIFT, %g4 3402 cmp %g3, %g4 3403 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 3404 mov PTL1_BAD_STACK, %g1 3405 3406 cmp %g5, 1 3407 ble,pt %icc, 2f 3408 nop 3409 TSTAT_CHECK_TL1(2f, %g1, %g2) 3410 rdpr %tt, %g2 3411 cmp %g2, FAST_PROT_TT 3412 mov PTL1_BAD_KPROT_FAULT, %g1 3413 movne %icc, PTL1_BAD_KMISS, %g1 3414 ba,pt %icc, ptl1_panic 3415 nop 3416 34172: 3418 /* 3419 * We are taking a pagefault in the kernel on a kernel address. If 3420 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 3421 * want to call sfmmu_pagefault -- we will instead note that a fault 3422 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 3423 * (instead of a "retry"). This will step over the faulting 3424 * instruction. 3425 */ 3426 CPU_INDEX(%g1, %g2) 3427 set cpu_core, %g2 3428 sllx %g1, CPU_CORE_SHIFT, %g1 3429 add %g1, %g2, %g1 3430 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3431 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3432 bz sfmmu_pagefault 3433 or %g2, CPU_DTRACE_BADADDR, %g2 3434 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3435 GET_MMU_D_ADDR(%g3, %g4) 3436 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3437 done 3438 34393: 3440 cmp %g5, 1 3441 ble,pt %icc, 4f 3442 nop 3443 TSTAT_CHECK_TL1(4f, %g1, %g2) 3444 ba,pt %icc, sfmmu_window_trap 3445 nop 3446 34474: 3448 /* 3449 * We are taking a pagefault on a non-kernel address. If we are in 3450 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 3451 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 3452 */ 3453 CPU_INDEX(%g1, %g2) 3454 set cpu_core, %g2 3455 sllx %g1, CPU_CORE_SHIFT, %g1 3456 add %g1, %g2, %g1 3457 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3458 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3459 bz sfmmu_pagefault 3460 or %g2, CPU_DTRACE_BADADDR, %g2 3461 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3462 GET_MMU_D_ADDR(%g3, %g4) 3463 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3464 3465 /* 3466 * Be sure that we're actually taking this miss from the kernel -- 3467 * otherwise we have managed to return to user-level with 3468 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3469 */ 3470 rdpr %tstate, %g2 3471 btst TSTATE_PRIV, %g2 3472 bz,a ptl1_panic 3473 mov PTL1_BAD_DTRACE_FLAGS, %g1 3474 done 3475 3476 ALTENTRY(tsb_tl0_noctxt) 3477 /* 3478 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 3479 * if it is, indicated that we have faulted and issue a done. 3480 */ 3481 CPU_INDEX(%g5, %g6) 3482 set cpu_core, %g6 3483 sllx %g5, CPU_CORE_SHIFT, %g5 3484 add %g5, %g6, %g5 3485 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 3486 andcc %g6, CPU_DTRACE_NOFAULT, %g0 3487 bz 1f 3488 or %g6, CPU_DTRACE_BADADDR, %g6 3489 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 3490 GET_MMU_D_ADDR(%g3, %g4) 3491 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 3492 3493 /* 3494 * Be sure that we're actually taking this miss from the kernel -- 3495 * otherwise we have managed to return to user-level with 3496 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3497 */ 3498 rdpr %tstate, %g5 3499 btst TSTATE_PRIV, %g5 3500 bz,a ptl1_panic 3501 mov PTL1_BAD_DTRACE_FLAGS, %g1 3502 done 3503 35041: 3505 rdpr %tt, %g5 3506 cmp %g5, FAST_IMMU_MISS_TT 3507#ifdef sun4v 3508 MMU_FAULT_STATUS_AREA(%g2) 3509 be,a,pt %icc, 2f 3510 ldx [%g2 + MMFSA_I_CTX], %g3 3511 cmp %g5, T_INSTR_MMU_MISS 3512 be,a,pt %icc, 2f 3513 ldx [%g2 + MMFSA_I_CTX], %g3 3514 ldx [%g2 + MMFSA_D_CTX], %g3 35152: 3516#else 3517 mov MMU_TAG_ACCESS, %g2 3518 be,a,pt %icc, 2f 3519 ldxa [%g2]ASI_IMMU, %g3 3520 ldxa [%g2]ASI_DMMU, %g3 35212: sllx %g3, TAGACC_CTX_LSHIFT, %g3 3522#endif 3523 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 3524 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 3525 rdpr %tl, %g5 3526 cmp %g5, 1 3527 ble,pt %icc, sfmmu_mmu_trap 3528 nop 3529 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 3530 ba,pt %icc, sfmmu_window_trap 3531 nop 3532 SET_SIZE(sfmmu_tsb_miss) 3533 3534#if (1<< TSBMISS_SHIFT) != TSBMISS_SIZE 3535#error - TSBMISS_SHIFT does not correspond to size of tsbmiss struct 3536#endif 3537 3538#endif /* lint */ 3539 3540#if defined (lint) 3541/* 3542 * This routine will look for a user or kernel vaddr in the hash 3543 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 3544 * grab any locks. It should only be used by other sfmmu routines. 3545 */ 3546/* ARGSUSED */ 3547pfn_t 3548sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 3549{ 3550 return(0); 3551} 3552 3553/* ARGSUSED */ 3554pfn_t 3555sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno) 3556{ 3557 return(0); 3558} 3559 3560#else /* lint */ 3561 3562 ENTRY_NP(sfmmu_vatopfn) 3563 /* 3564 * disable interrupts 3565 */ 3566 rdpr %pstate, %o3 3567#ifdef DEBUG 3568 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 3569#endif 3570 /* 3571 * disable interrupts to protect the TSBMISS area 3572 */ 3573 andn %o3, PSTATE_IE, %o5 3574 wrpr %o5, 0, %pstate 3575 3576 /* 3577 * o0 = vaddr 3578 * o1 = sfmmup 3579 * o2 = ttep 3580 */ 3581 CPU_TSBMISS_AREA(%g1, %o5) 3582 ldn [%g1 + TSBMISS_KHATID], %o4 3583 cmp %o4, %o1 3584 bne,pn %ncc, vatopfn_nokernel 3585 mov TTE64K, %g5 /* g5 = rehash # */ 3586 mov %g1,%o5 /* o5 = tsbmiss_area */ 3587 /* 3588 * o0 = vaddr 3589 * o1 & o4 = hatid 3590 * o2 = ttep 3591 * o5 = tsbmiss area 3592 */ 3593 mov HBLK_RANGE_SHIFT, %g6 35941: 3595 3596 /* 3597 * o0 = vaddr 3598 * o1 = sfmmup 3599 * o2 = ttep 3600 * o3 = old %pstate 3601 * o4 = hatid 3602 * o5 = tsbmiss 3603 * g5 = rehash # 3604 * g6 = hmeshift 3605 * 3606 * The first arg to GET_TTE is actually tagaccess register 3607 * not just vaddr. Since this call is for kernel we need to clear 3608 * any lower vaddr bits that would be interpreted as ctx bits. 3609 */ 3610 set TAGACC_CTX_MASK, %g1 3611 andn %o0, %g1, %o0 3612 GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5, 3613 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 3614 3615kvtop_hblk_found: 3616 /* 3617 * o0 = vaddr 3618 * o1 = sfmmup 3619 * o2 = ttep 3620 * g1 = tte 3621 * g2 = tte pa 3622 * g3 = tte va 3623 * o2 = tsbmiss area 3624 * o1 = hat id 3625 */ 3626 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 3627 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3628 stx %g1,[%o2] /* put tte into *ttep */ 3629 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 3630 /* 3631 * o0 = vaddr 3632 * o1 = sfmmup 3633 * o2 = ttep 3634 * g1 = pfn 3635 */ 3636 ba,pt %xcc, 6f 3637 mov %g1, %o0 3638 3639kvtop_nohblk: 3640 /* 3641 * we get here if we couldn't find valid hblk in hash. We rehash 3642 * if neccesary. 3643 */ 3644 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 3645#ifdef sun4v 3646 cmp %g5, MAX_HASHCNT 3647#else 3648 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 3649#endif 3650 be,a,pn %icc, 6f 3651 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3652 mov %o1, %o4 /* restore hatid */ 3653#ifdef sun4v 3654 add %g5, 2, %g5 3655 cmp %g5, 3 3656 move %icc, MMU_PAGESHIFT4M, %g6 3657 ba,pt %icc, 1b 3658 movne %icc, MMU_PAGESHIFT256M, %g6 3659#else 3660 inc %g5 3661 cmp %g5, 2 3662 move %icc, MMU_PAGESHIFT512K, %g6 3663 ba,pt %icc, 1b 3664 movne %icc, MMU_PAGESHIFT4M, %g6 3665#endif 36666: 3667 retl 3668 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3669 3670tsb_suspend: 3671 /* 3672 * o0 = vaddr 3673 * o1 = sfmmup 3674 * o2 = ttep 3675 * g1 = tte 3676 * g2 = tte pa 3677 * g3 = tte va 3678 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 3679 */ 3680 stx %g1,[%o2] /* put tte into *ttep */ 3681 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 3682 sub %g0, 1, %o0 /* output = -1 (PFN_INVALID) */ 3683 TTETOPFN(%g1, %o0, vatopfn_l3, %g2, %g3, %g4) 3684 /* 3685 * o0 = PFN return value PFN_INVALID, PFN_SUSPENDED, or pfn# 3686 * o1 = sfmmup 3687 * o2 = ttep 3688 * g1 = pfn 3689 */ 3690 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 36918: 3692 retl 3693 wrpr %g0, %o3, %pstate /* enable interrupts */ 3694 3695vatopfn_nokernel: 3696 /* 3697 * This routine does NOT support user addresses 3698 * There is a routine in C that supports this. 3699 * The only reason why we don't have the C routine 3700 * support kernel addresses as well is because 3701 * we do va_to_pa while holding the hashlock. 3702 */ 3703 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3704 save %sp, -SA(MINFRAME), %sp 3705 sethi %hi(sfmmu_panic3), %o0 3706 call panic 3707 or %o0, %lo(sfmmu_panic3), %o0 3708 3709 SET_SIZE(sfmmu_vatopfn) 3710 3711 /* 3712 * %o0 = vaddr 3713 * %o1 = hashno (aka szc) 3714 * 3715 * 3716 * This routine is similar to sfmmu_vatopfn() but will only look for 3717 * a kernel vaddr in the hash structure for the specified rehash value. 3718 * It's just an optimization for the case when pagesize for a given 3719 * va range is already known (e.g. large page heap) and we don't want 3720 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 3721 * 3722 * Returns valid pfn or PFN_INVALID if 3723 * tte for specified rehash # is not found, invalid or suspended. 3724 */ 3725 ENTRY_NP(sfmmu_kvaszc2pfn) 3726 /* 3727 * disable interrupts 3728 */ 3729 rdpr %pstate, %o3 3730#ifdef DEBUG 3731 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 3732#endif 3733 /* 3734 * disable interrupts to protect the TSBMISS area 3735 */ 3736 andn %o3, PSTATE_IE, %o5 3737 wrpr %o5, 0, %pstate 3738 3739 CPU_TSBMISS_AREA(%g1, %o5) 3740 ldn [%g1 + TSBMISS_KHATID], %o4 3741 sll %o1, 1, %g6 3742 add %g6, %o1, %g6 3743 add %g6, MMU_PAGESHIFT, %g6 3744 /* 3745 * %o0 = vaddr 3746 * %o1 = hashno 3747 * %o3 = old %pstate 3748 * %o4 = ksfmmup 3749 * %g1 = tsbmiss area 3750 * %g6 = hmeshift 3751 */ 3752 3753 /* 3754 * The first arg to GET_TTE is actually tagaccess register 3755 * not just vaddr. Since this call is for kernel we need to clear 3756 * any lower vaddr bits that would be interpreted as ctx bits. 3757 */ 3758 srlx %o0, MMU_PAGESHIFT, %o0 3759 sllx %o0, MMU_PAGESHIFT, %o0 3760 GET_TTE(%o0, %o4, %g3, %g4, %g5, %g1, %o5, %g6, %o1, 3761 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 3762 kvaszc2pfn_nohblk) 3763 3764kvaszc2pfn_hblk_found: 3765 /* 3766 * %g3 = tte 3767 * %o0 = vaddr 3768 */ 3769 brgez,a,pn %g3, 1f /* check if tte is invalid */ 3770 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3771 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 3772 /* 3773 * g3 = pfn 3774 */ 3775 ba,pt %xcc, 1f 3776 mov %g3, %o0 3777 3778kvaszc2pfn_nohblk: 3779 mov -1, %o0 3780 37811: 3782 retl 3783 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3784 3785 SET_SIZE(sfmmu_kvaszc2pfn) 3786 3787#endif /* lint */ 3788 3789 3790 3791#if !defined(lint) 3792 3793/* 3794 * kpm lock used between trap level tsbmiss handler and kpm C level. 3795 */ 3796#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 3797 mov 0xff, tmp1 ;\ 3798label1: ;\ 3799 casa [kpmlckp]asi, %g0, tmp1 ;\ 3800 brnz,pn tmp1, label1 ;\ 3801 mov 0xff, tmp1 ;\ 3802 membar #LoadLoad 3803 3804#define KPMLOCK_EXIT(kpmlckp, asi) \ 3805 membar #LoadStore|#StoreStore ;\ 3806 sta %g0, [kpmlckp]asi 3807 3808/* 3809 * Lookup a memseg for a given pfn and if found, return the physical 3810 * address of the corresponding struct memseg in mseg, otherwise 3811 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 3812 * tsbmp, %asi is assumed to be ASI_MEM. 3813 * This lookup is done by strictly traversing only the physical memseg 3814 * linkage. The more generic approach, to check the virtual linkage 3815 * before using the physical (used e.g. with hmehash buckets), cannot 3816 * be used here. Memory DR operations can run in parallel to this 3817 * lookup w/o any locks and updates of the physical and virtual linkage 3818 * cannot be done atomically wrt. to each other. Because physical 3819 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 3820 * as "physical NULL" pointer. 3821 */ 3822#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 3823 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 3824 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 3825 udivx pfn, mseg, mseg ;\ 3826 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 3827 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 3828 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 3829 add tmp1, mseg, tmp1 ;\ 3830 ldxa [tmp1]%asi, mseg ;\ 3831 cmp mseg, MSEG_NULLPTR_PA ;\ 3832 be,pn %xcc, label/**/1 /* if not found */ ;\ 3833 nop ;\ 3834 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 3835 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 3836 blu,pn %xcc, label/**/1 ;\ 3837 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 3838 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 3839 bgeu,pn %xcc, label/**/1 ;\ 3840 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 3841 mulx tmp1, PAGE_SIZE, tmp1 ;\ 3842 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 3843 add tmp2, tmp1, tmp1 /* pp */ ;\ 3844 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 3845 cmp tmp2, pfn ;\ 3846 be,pt %xcc, label/**/_ok /* found */ ;\ 3847label/**/1: ;\ 3848 /* brute force lookup */ ;\ 3849 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 3850 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 3851label/**/2: ;\ 3852 cmp mseg, MSEG_NULLPTR_PA ;\ 3853 be,pn %xcc, label/**/_ok /* if not found */ ;\ 3854 nop ;\ 3855 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 3856 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 3857 blu,a,pt %xcc, label/**/2 ;\ 3858 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 3859 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 3860 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 3861 bgeu,a,pt %xcc, label/**/2 ;\ 3862 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 3863label/**/_ok: 3864 3865 /* 3866 * kpm tsb miss handler large pages 3867 * g1 = 8K kpm TSB entry pointer 3868 * g2 = tag access register 3869 * g3 = 4M kpm TSB entry pointer 3870 */ 3871 ALTENTRY(sfmmu_kpm_dtsb_miss) 3872 TT_TRACE(trace_tsbmiss) 3873 3874 CPU_INDEX(%g7, %g6) 3875 sethi %hi(kpmtsbm_area), %g6 3876 sllx %g7, KPMTSBM_SHIFT, %g7 3877 or %g6, %lo(kpmtsbm_area), %g6 3878 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 3879 3880 /* check enable flag */ 3881 ldub [%g6 + KPMTSBM_FLAGS], %g4 3882 and %g4, KPMTSBM_ENABLE_FLAG, %g5 3883 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 3884 nop 3885 3886 /* VA range check */ 3887 ldx [%g6 + KPMTSBM_VBASE], %g7 3888 cmp %g2, %g7 3889 blu,pn %xcc, sfmmu_tsb_miss 3890 ldx [%g6 + KPMTSBM_VEND], %g5 3891 cmp %g2, %g5 3892 bgeu,pn %xcc, sfmmu_tsb_miss 3893 stx %g3, [%g6 + KPMTSBM_TSBPTR] 3894 3895 /* 3896 * check TL tsbmiss handling flag 3897 * bump tsbmiss counter 3898 */ 3899 lduw [%g6 + KPMTSBM_TSBMISS], %g5 3900#ifdef DEBUG 3901 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 3902 inc %g5 3903 brz,pn %g3, sfmmu_kpm_exception 3904 st %g5, [%g6 + KPMTSBM_TSBMISS] 3905#else 3906 inc %g5 3907 st %g5, [%g6 + KPMTSBM_TSBMISS] 3908#endif 3909 /* 3910 * At this point: 3911 * g1 = 8K kpm TSB pointer (not used) 3912 * g2 = tag access register 3913 * g3 = clobbered 3914 * g6 = per-CPU kpm tsbmiss area 3915 * g7 = kpm_vbase 3916 */ 3917 3918 /* vaddr2pfn */ 3919 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 3920 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 3921 srax %g4, %g3, %g2 /* which alias range (r) */ 3922 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 3923 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 3924 3925 /* 3926 * Setup %asi 3927 * mseg_pa = page_numtomemseg_nolock(pfn) 3928 * if (mseg_pa == NULL) sfmmu_kpm_exception 3929 * g2=pfn 3930 */ 3931 mov ASI_MEM, %asi 3932 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 3933 cmp %g3, MSEG_NULLPTR_PA 3934 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 3935 nop 3936 3937 /* 3938 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 3939 * g2=pfn g3=mseg_pa 3940 */ 3941 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 3942 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 3943 srlx %g2, %g5, %g4 3944 sllx %g4, %g5, %g4 3945 sub %g4, %g7, %g4 3946 srlx %g4, %g5, %g4 3947 3948 /* 3949 * Validate inx value 3950 * g2=pfn g3=mseg_pa g4=inx 3951 */ 3952#ifdef DEBUG 3953 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 3954 cmp %g4, %g5 /* inx - nkpmpgs */ 3955 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 3956 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 3957#else 3958 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 3959#endif 3960 /* 3961 * kp = &mseg_pa->kpm_pages[inx] 3962 */ 3963 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 3964 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 3965 add %g5, %g4, %g5 /* kp */ 3966 3967 /* 3968 * KPMP_HASH(kp) 3969 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 3970 */ 3971 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 3972 sub %g7, 1, %g7 /* mask */ 3973 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 3974 add %g5, %g1, %g5 /* y = ksp + x */ 3975 and %g5, %g7, %g5 /* hashinx = y & mask */ 3976 3977 /* 3978 * Calculate physical kpm_page pointer 3979 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 3980 */ 3981 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 3982 add %g1, %g4, %g1 /* kp_pa */ 3983 3984 /* 3985 * Calculate physical hash lock address 3986 * g1=kp_refcntc_pa g2=pfn g5=hashinx 3987 */ 3988 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 3989 sllx %g5, KPMHLK_SHIFT, %g5 3990 add %g4, %g5, %g3 3991 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 3992 3993 /* 3994 * Assemble tte 3995 * g1=kp_pa g2=pfn g3=hlck_pa 3996 */ 3997#ifdef sun4v 3998 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 3999 sllx %g5, 32, %g5 4000 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4001 or %g4, TTE4M, %g4 4002 or %g5, %g4, %g5 4003#else 4004 sethi %hi(TTE_VALID_INT), %g4 4005 mov TTE4M, %g5 4006 sllx %g5, TTE_SZ_SHFT_INT, %g5 4007 or %g5, %g4, %g5 /* upper part */ 4008 sllx %g5, 32, %g5 4009 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4010 or %g5, %g4, %g5 4011#endif 4012 sllx %g2, MMU_PAGESHIFT, %g4 4013 or %g5, %g4, %g5 /* tte */ 4014 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4015 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4016 4017 /* 4018 * tsb dropin 4019 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4020 */ 4021 4022 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4023 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4024 4025 /* use C-handler if there's no go for dropin */ 4026 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4027 cmp %g7, -1 4028 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4029 nop 4030 4031#ifdef DEBUG 4032 /* double check refcnt */ 4033 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4034 brz,pn %g7, 5f /* let C-handler deal with this */ 4035 nop 4036#endif 4037 4038#ifndef sun4v 4039 ldub [%g6 + KPMTSBM_FLAGS], %g7 4040 mov ASI_N, %g1 4041 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4042 movnz %icc, ASI_MEM, %g1 4043 mov %g1, %asi 4044#endif 4045 4046 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4047 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4048 4049 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4050 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4051 4052 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4053 4054 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4055 KPMLOCK_EXIT(%g3, ASI_MEM) 4056 4057 /* 4058 * If trapstat is running, we need to shift the %tpc and %tnpc to 4059 * point to trapstat's TSB miss return code (note that trapstat 4060 * itself will patch the correct offset to add). 4061 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4062 */ 4063 rdpr %tl, %g7 4064 cmp %g7, 1 4065 ble %icc, 0f 4066 sethi %hi(KERNELBASE), %g6 4067 rdpr %tpc, %g7 4068 or %g6, %lo(KERNELBASE), %g6 4069 cmp %g7, %g6 4070 bgeu %xcc, 0f 4071 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4072 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4073 wrpr %g7, %tpc 4074 add %g7, 4, %g7 4075 wrpr %g7, %tnpc 40760: 4077 retry 40785: 4079 /* g3=hlck_pa */ 4080 KPMLOCK_EXIT(%g3, ASI_MEM) 4081 ba,pt %icc, sfmmu_kpm_exception 4082 nop 4083 SET_SIZE(sfmmu_kpm_dtsb_miss) 4084 4085 /* 4086 * kpm tsbmiss handler for smallpages 4087 * g1 = 8K kpm TSB pointer 4088 * g2 = tag access register 4089 * g3 = 4M kpm TSB pointer 4090 */ 4091 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4092 TT_TRACE(trace_tsbmiss) 4093 CPU_INDEX(%g7, %g6) 4094 sethi %hi(kpmtsbm_area), %g6 4095 sllx %g7, KPMTSBM_SHIFT, %g7 4096 or %g6, %lo(kpmtsbm_area), %g6 4097 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4098 4099 /* check enable flag */ 4100 ldub [%g6 + KPMTSBM_FLAGS], %g4 4101 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4102 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4103 nop 4104 4105 /* 4106 * VA range check 4107 * On fail: goto sfmmu_tsb_miss 4108 */ 4109 ldx [%g6 + KPMTSBM_VBASE], %g7 4110 cmp %g2, %g7 4111 blu,pn %xcc, sfmmu_tsb_miss 4112 ldx [%g6 + KPMTSBM_VEND], %g5 4113 cmp %g2, %g5 4114 bgeu,pn %xcc, sfmmu_tsb_miss 4115 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4116 4117 /* 4118 * check TL tsbmiss handling flag 4119 * bump tsbmiss counter 4120 */ 4121 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4122#ifdef DEBUG 4123 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4124 inc %g5 4125 brz,pn %g1, sfmmu_kpm_exception 4126 st %g5, [%g6 + KPMTSBM_TSBMISS] 4127#else 4128 inc %g5 4129 st %g5, [%g6 + KPMTSBM_TSBMISS] 4130#endif 4131 /* 4132 * At this point: 4133 * g1 = clobbered 4134 * g2 = tag access register 4135 * g3 = 4M kpm TSB pointer (not used) 4136 * g6 = per-CPU kpm tsbmiss area 4137 * g7 = kpm_vbase 4138 */ 4139 4140 /* vaddr2pfn */ 4141 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4142 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4143 srax %g4, %g3, %g2 /* which alias range (r) */ 4144 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4145 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4146 4147 /* 4148 * Setup %asi 4149 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4150 * if (mseg not found) sfmmu_kpm_exception 4151 * g2=pfn 4152 */ 4153 mov ASI_MEM, %asi 4154 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4155 cmp %g3, MSEG_NULLPTR_PA 4156 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4157 nop 4158 4159 /* 4160 * inx = pfn - mseg_pa->kpm_pbase 4161 * g2=pfn g3=mseg_pa 4162 */ 4163 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4164 sub %g2, %g7, %g4 4165 4166#ifdef DEBUG 4167 /* 4168 * Validate inx value 4169 * g2=pfn g3=mseg_pa g4=inx 4170 */ 4171 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4172 cmp %g4, %g5 /* inx - nkpmpgs */ 4173 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4174 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4175#else 4176 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4177#endif 4178 /* ksp = &mseg_pa->kpm_spages[inx] */ 4179 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4180 add %g5, %g4, %g5 /* ksp */ 4181 4182 /* 4183 * KPMP_SHASH(kp) 4184 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz 4185 */ 4186 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4187 sub %g7, 1, %g7 /* mask */ 4188 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4189 add %g5, %g1, %g5 /* y = ksp + x */ 4190 and %g5, %g7, %g5 /* hashinx = y & mask */ 4191 4192 /* 4193 * Calculate physical kpm_spage pointer 4194 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4195 */ 4196 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4197 add %g1, %g4, %g1 /* ksp_pa */ 4198 4199 /* 4200 * Calculate physical hash lock address. 4201 * Note: Changes in kpm_shlk_t must be reflected here. 4202 * g1=ksp_pa g2=pfn g5=hashinx 4203 */ 4204 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4205 sllx %g5, KPMSHLK_SHIFT, %g5 4206 add %g4, %g5, %g3 /* hlck_pa */ 4207 4208 /* 4209 * Assemble tte 4210 * g1=ksp_pa g2=pfn g3=hlck_pa 4211 */ 4212 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4213 sllx %g5, 32, %g5 4214 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4215 or %g5, %g4, %g5 4216 sllx %g2, MMU_PAGESHIFT, %g4 4217 or %g5, %g4, %g5 /* tte */ 4218 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4219 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4220 4221 /* 4222 * tsb dropin 4223 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte 4224 */ 4225 4226 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4227 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4228 4229 /* use C-handler if there's no go for dropin */ 4230 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4231 cmp %g7, -1 4232 bne,pn %xcc, 5f 4233 nop 4234 4235#ifndef sun4v 4236 ldub [%g6 + KPMTSBM_FLAGS], %g7 4237 mov ASI_N, %g1 4238 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4239 movnz %icc, ASI_MEM, %g1 4240 mov %g1, %asi 4241#endif 4242 4243 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4244 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4245 4246 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4247 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4248 4249 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4250 4251 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4252 KPMLOCK_EXIT(%g3, ASI_MEM) 4253 4254 /* 4255 * If trapstat is running, we need to shift the %tpc and %tnpc to 4256 * point to trapstat's TSB miss return code (note that trapstat 4257 * itself will patch the correct offset to add). 4258 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4259 */ 4260 rdpr %tl, %g7 4261 cmp %g7, 1 4262 ble %icc, 0f 4263 sethi %hi(KERNELBASE), %g6 4264 rdpr %tpc, %g7 4265 or %g6, %lo(KERNELBASE), %g6 4266 cmp %g7, %g6 4267 bgeu %xcc, 0f 4268 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4269 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4270 wrpr %g7, %tpc 4271 add %g7, 4, %g7 4272 wrpr %g7, %tnpc 42730: 4274 retry 42755: 4276 /* g3=hlck_pa */ 4277 KPMLOCK_EXIT(%g3, ASI_MEM) 4278 ba,pt %icc, sfmmu_kpm_exception 4279 nop 4280 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4281 4282#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4283#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4284#endif 4285 4286#endif /* lint */ 4287 4288#ifdef lint 4289/* 4290 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4291 * Called from C-level, sets/clears "go" indication for trap level handler. 4292 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 4293 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 4294 * Assumes khl_mutex is held when called from C-level. 4295 */ 4296/* ARGSUSED */ 4297void 4298sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 4299{ 4300} 4301 4302/* 4303 * kpm_smallpages: stores val to byte at address mapped within 4304 * low level lock brackets. The old value is returned. 4305 * Called from C-level. 4306 */ 4307/* ARGSUSED */ 4308int 4309sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val) 4310{ 4311 return (0); 4312} 4313 4314#else /* lint */ 4315 4316 .seg ".data" 4317sfmmu_kpm_tsbmtl_panic: 4318 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4319 .byte 0 4320sfmmu_kpm_stsbmtl_panic: 4321 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4322 .byte 0 4323 .align 4 4324 .seg ".text" 4325 4326 ENTRY_NP(sfmmu_kpm_tsbmtl) 4327 rdpr %pstate, %o3 4328 /* 4329 * %o0 = &kp_refcntc 4330 * %o1 = &khl_lock 4331 * %o2 = 0/1 (off/on) 4332 * %o3 = pstate save 4333 */ 4334#ifdef DEBUG 4335 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4336 bnz,pt %icc, 1f /* disabled, panic */ 4337 nop 4338 save %sp, -SA(MINFRAME), %sp 4339 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4340 call panic 4341 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4342 ret 4343 restore 43441: 4345#endif /* DEBUG */ 4346 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4347 4348 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4349 mov -1, %o5 4350 brz,a %o2, 2f 4351 mov 0, %o5 43522: 4353 sth %o5, [%o0] 4354 KPMLOCK_EXIT(%o1, ASI_N) 4355 4356 retl 4357 wrpr %g0, %o3, %pstate /* enable interrupts */ 4358 SET_SIZE(sfmmu_kpm_tsbmtl) 4359 4360 ENTRY_NP(sfmmu_kpm_stsbmtl) 4361 rdpr %pstate, %o3 4362 /* 4363 * %o0 = &mapped 4364 * %o1 = &kshl_lock 4365 * %o2 = val 4366 * %o3 = pstate save 4367 */ 4368#ifdef DEBUG 4369 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4370 bnz,pt %icc, 1f /* disabled, panic */ 4371 nop 4372 save %sp, -SA(MINFRAME), %sp 4373 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4374 call panic 4375 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4376 ret 4377 restore 43781: 4379#endif /* DEBUG */ 4380 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4381 4382 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4383 ldsb [%o0], %o5 4384 stb %o2, [%o0] 4385 KPMLOCK_EXIT(%o1, ASI_N) 4386 4387 mov %o5, %o0 /* return old val */ 4388 retl 4389 wrpr %g0, %o3, %pstate /* enable interrupts */ 4390 SET_SIZE(sfmmu_kpm_stsbmtl) 4391 4392#endif /* lint */ 4393 4394#ifndef lint 4395#ifdef sun4v 4396 /* 4397 * User/kernel data miss w// multiple TSBs 4398 * The first probe covers 8K, 64K, and 512K page sizes, 4399 * because 64K and 512K mappings are replicated off 8K 4400 * pointer. Second probe covers 4M page size only. 4401 * 4402 * MMU fault area contains miss address and context. 4403 */ 4404 ALTENTRY(sfmmu_slow_dmmu_miss) 4405 GET_MMU_D_TAGACC_CTX(%g2, %g3) ! %g2 = tagacc, %g3 = ctx 4406 4407slow_miss_common: 4408 /* 4409 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 4410 * %g3 = ctx (cannot be INVALID_CONTEXT) 4411 */ 4412 brnz,pt %g3, 8f ! check for user context 4413 nop 4414 4415 /* 4416 * Kernel miss 4417 * Get 8K and 4M TSB pointers in %g1 and %g3 and 4418 * branch to sfmmu_tsb_miss_tt to handle it. 4419 */ 4420 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4421sfmmu_dslow_patch_ktsb_base: 4422 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 4423sfmmu_dslow_patch_ktsb_szcode: 4424 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 4425 4426 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 4427 ! %g1 = First TSB entry pointer, as TSB miss handler expects 4428 4429 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4430sfmmu_dslow_patch_ktsb4m_base: 4431 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 4432sfmmu_dslow_patch_ktsb4m_szcode: 4433 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 4434 4435 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 4436 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 4437 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4438 .empty 4439 44408: 4441 /* 4442 * User miss 4443 * Get first TSB pointer in %g1 4444 * Get second TSB pointer (or NULL if no second TSB) in %g3 4445 * Branch to sfmmu_tsb_miss_tt to handle it 4446 */ 4447 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 4448 /* %g1 = first TSB entry ptr now, %g2 preserved */ 4449 4450 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 4451 brlz,a,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 4452 mov %g0, %g3 4453 4454 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 4455 /* %g3 = second TSB entry ptr now, %g2 preserved */ 44569: 4457 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4458 .empty 4459 SET_SIZE(sfmmu_slow_dmmu_miss) 4460 4461 4462 /* 4463 * User/kernel instruction miss w/ multiple TSBs 4464 * The first probe covers 8K, 64K, and 512K page sizes, 4465 * because 64K and 512K mappings are replicated off 8K 4466 * pointer. Second probe covers 4M page size only. 4467 * 4468 * MMU fault area contains miss address and context. 4469 */ 4470 ALTENTRY(sfmmu_slow_immu_miss) 4471 MMU_FAULT_STATUS_AREA(%g2) 4472 ldx [%g2 + MMFSA_I_CTX], %g3 4473 ldx [%g2 + MMFSA_I_ADDR], %g2 4474 srlx %g2, MMU_PAGESHIFT, %g2 ! align address to page boundry 4475 sllx %g2, MMU_PAGESHIFT, %g2 4476 ba,pt %xcc, slow_miss_common 4477 or %g2, %g3, %g2 4478 SET_SIZE(sfmmu_slow_immu_miss) 4479 4480#endif /* sun4v */ 4481#endif /* lint */ 4482 4483#ifndef lint 4484 4485/* 4486 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 4487 */ 4488 .seg ".data" 4489 .align 64 4490 .global tsbmiss_area 4491tsbmiss_area: 4492 .skip (TSBMISS_SIZE * NCPU) 4493 4494 .align 64 4495 .global kpmtsbm_area 4496kpmtsbm_area: 4497 .skip (KPMTSBM_SIZE * NCPU) 4498#endif /* lint */ 4499