1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26/* 27 * SFMMU primitives. These primitives should only be used by sfmmu 28 * routines. 29 */ 30 31#if defined(lint) 32#include <sys/types.h> 33#else /* lint */ 34#include "assym.h" 35#endif /* lint */ 36 37#include <sys/asm_linkage.h> 38#include <sys/machtrap.h> 39#include <sys/machasi.h> 40#include <sys/sun4asi.h> 41#include <sys/pte.h> 42#include <sys/mmu.h> 43#include <vm/hat_sfmmu.h> 44#include <vm/seg_spt.h> 45#include <sys/machparam.h> 46#include <sys/privregs.h> 47#include <sys/scb.h> 48#include <sys/intreg.h> 49#include <sys/machthread.h> 50#include <sys/intr.h> 51#include <sys/clock.h> 52#include <sys/trapstat.h> 53 54#ifdef TRAPTRACE 55#include <sys/traptrace.h> 56 57/* 58 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 59 */ 60#define TT_TRACE(label) \ 61 ba label ;\ 62 rd %pc, %g7 63#else 64 65#define TT_TRACE(label) 66 67#endif /* TRAPTRACE */ 68 69#ifndef lint 70 71#if (TTE_SUSPEND_SHIFT > 0) 72#define TTE_SUSPEND_INT_SHIFT(reg) \ 73 sllx reg, TTE_SUSPEND_SHIFT, reg 74#else 75#define TTE_SUSPEND_INT_SHIFT(reg) 76#endif 77 78#endif /* lint */ 79 80#ifndef lint 81 82/* 83 * Assumes TSBE_TAG is 0 84 * Assumes TSBE_INTHI is 0 85 * Assumes TSBREG.split is 0 86 */ 87 88#if TSBE_TAG != 0 89#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 90#endif 91 92#if TSBTAG_INTHI != 0 93#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 94#endif 95 96/* 97 * The following code assumes the tsb is not split. 98 * 99 * With TSBs no longer shared between processes, it's no longer 100 * necessary to hash the context bits into the tsb index to get 101 * tsb coloring; the new implementation treats the TSB as a 102 * direct-mapped, virtually-addressed cache. 103 * 104 * In: 105 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 106 * tsbbase = base address of TSB (clobbered) 107 * tagacc = tag access register (clobbered) 108 * szc = size code of TSB (ro) 109 * tmp = scratch reg 110 * Out: 111 * tsbbase = pointer to entry in TSB 112 */ 113#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 114 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 115 srlx tagacc, vpshift, tagacc ;\ 116 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 117 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 118 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 119 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 120 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 121 122/* 123 * When the kpm TSB is used it is assumed that it is direct mapped 124 * using (vaddr>>vpshift)%tsbsz as the index. 125 * 126 * Note that, for now, the kpm TSB and kernel TSB are the same for 127 * each mapping size. However that need not always be the case. If 128 * the trap handlers are updated to search a different TSB for kpm 129 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 130 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 131 * 132 * In: 133 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 134 * vaddr = virtual address (clobbered) 135 * tsbp, szc, tmp = scratch 136 * Out: 137 * tsbp = pointer to entry in TSB 138 */ 139#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 140 cmp vpshift, MMU_PAGESHIFT ;\ 141 bne,pn %icc, 1f /* branch if large case */ ;\ 142 sethi %hi(kpmsm_tsbsz), szc ;\ 143 sethi %hi(kpmsm_tsbbase), tsbp ;\ 144 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 145 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 146 ba,pt %icc, 2f ;\ 147 nop ;\ 1481: sethi %hi(kpm_tsbsz), szc ;\ 149 sethi %hi(kpm_tsbbase), tsbp ;\ 150 ld [szc + %lo(kpm_tsbsz)], szc ;\ 151 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1522: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 153 154/* 155 * Lock the TSBE at virtual address tsbep. 156 * 157 * tsbep = TSBE va (ro) 158 * tmp1, tmp2 = scratch registers (clobbered) 159 * label = label to jump to if we fail to lock the tsb entry 160 * %asi = ASI to use for TSB access 161 * 162 * NOTE that we flush the TSB using fast VIS instructions that 163 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 164 * not be treated as a locked entry or we'll get stuck spinning on 165 * an entry that isn't locked but really invalid. 166 */ 167 168#if defined(UTSB_PHYS) 169 170#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 171 lda [tsbep]ASI_MEM, tmp1 ;\ 172 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 173 cmp tmp1, tmp2 ;\ 174 be,a,pn %icc, label /* if locked ignore */ ;\ 175 nop ;\ 176 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 177 cmp tmp1, tmp2 ;\ 178 bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 179 nop ;\ 180 /* tsbe lock acquired */ ;\ 181 membar #StoreStore 182 183#else /* UTSB_PHYS */ 184 185#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 186 lda [tsbep]%asi, tmp1 ;\ 187 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 188 cmp tmp1, tmp2 ;\ 189 be,a,pn %icc, label /* if locked ignore */ ;\ 190 nop ;\ 191 casa [tsbep]%asi, tmp1, tmp2 ;\ 192 cmp tmp1, tmp2 ;\ 193 bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 194 nop ;\ 195 /* tsbe lock acquired */ ;\ 196 membar #StoreStore 197 198#endif /* UTSB_PHYS */ 199 200/* 201 * Atomically write TSBE at virtual address tsbep. 202 * 203 * tsbep = TSBE va (ro) 204 * tte = TSBE TTE (ro) 205 * tagtarget = TSBE tag (ro) 206 * %asi = ASI to use for TSB access 207 */ 208 209#if defined(UTSB_PHYS) 210 211#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 212 add tsbep, TSBE_TTE, tmp1 ;\ 213 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 214 membar #StoreStore ;\ 215 add tsbep, TSBE_TAG, tmp1 ;\ 216 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 217 218#else /* UTSB_PHYS */ 219 220#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 221 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 222 membar #StoreStore ;\ 223 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 224 225#endif /* UTSB_PHYS */ 226 227/* 228 * Load an entry into the TSB at TL > 0. 229 * 230 * tsbep = pointer to the TSBE to load as va (ro) 231 * tte = value of the TTE retrieved and loaded (wo) 232 * tagtarget = tag target register. To get TSBE tag to load, 233 * we need to mask off the context and leave only the va (clobbered) 234 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 235 * tmp1, tmp2 = scratch registers 236 * label = label to jump to if we fail to lock the tsb entry 237 * %asi = ASI to use for TSB access 238 */ 239 240#if defined(UTSB_PHYS) 241 242#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 243 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 244 /* ;\ 245 * I don't need to update the TSB then check for the valid tte. ;\ 246 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 247 * we always invalidate the hash table before we unload the TSB.;\ 248 */ ;\ 249 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 250 ldxa [ttepa]ASI_MEM, tte ;\ 251 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 252 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 253 add tsbep, TSBE_TAG, tmp1 ;\ 254 brgez,a,pn tte, label ;\ 255 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 256 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 257label: 258 259#else /* UTSB_PHYS */ 260 261#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 262 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 263 /* ;\ 264 * I don't need to update the TSB then check for the valid tte. ;\ 265 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 266 * we always invalidate the hash table before we unload the TSB.;\ 267 */ ;\ 268 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 269 ldxa [ttepa]ASI_MEM, tte ;\ 270 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 271 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 272 brgez,a,pn tte, label ;\ 273 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 274 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 275label: 276 277#endif /* UTSB_PHYS */ 278 279/* 280 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 281 * for ITLB synthesis. 282 * 283 * tsbep = pointer to the TSBE to load as va (ro) 284 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 285 * with exec_perm turned off and exec_synth turned on 286 * tagtarget = tag target register. To get TSBE tag to load, 287 * we need to mask off the context and leave only the va (clobbered) 288 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 289 * tmp1, tmp2 = scratch registers 290 * label = label to use for branch (text) 291 * %asi = ASI to use for TSB access 292 */ 293 294#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 295 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 296 /* ;\ 297 * I don't need to update the TSB then check for the valid tte. ;\ 298 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 299 * we always invalidate the hash table before we unload the TSB.;\ 300 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 301 * and exec_synth bit to 1. ;\ 302 */ ;\ 303 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 304 mov tte, tmp1 ;\ 305 ldxa [ttepa]ASI_MEM, tte ;\ 306 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 307 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 308 brgez,a,pn tte, label ;\ 309 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 310 or tte, tmp1, tte ;\ 311 andn tte, TTE_EXECPRM_INT, tte ;\ 312 or tte, TTE_E_SYNTH_INT, tte ;\ 313 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 314label: 315 316/* 317 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 318 * 319 * tte = value of the TTE, used to get tte_size bits (ro) 320 * tagaccess = tag access register, used to get 4M pfn bits (ro) 321 * pfn = 4M pfn bits shifted to offset for tte (out) 322 * tmp1 = scratch register 323 * label = label to use for branch (text) 324 */ 325 326#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 327 /* ;\ 328 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 329 * Return them, shifted, in pfn. ;\ 330 */ ;\ 331 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 332 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 333 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 334 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 335 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 336 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 337label: ;\ 338 sllx tagaccess, MMU_PAGESHIFT4M, pfn 339 340/* 341 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 342 * for ITLB synthesis. 343 * 344 * tte = value of the TTE, used to get tte_size bits (rw) 345 * tmp1 = scratch register 346 */ 347 348#define SET_TTE4M_PN(tte, tmp) \ 349 /* ;\ 350 * Set 4M pagesize tte bits. ;\ 351 */ ;\ 352 set TTE4M, tmp ;\ 353 sllx tmp, TTE_SZ_SHFT, tmp ;\ 354 or tte, tmp, tte 355 356/* 357 * Load an entry into the TSB at TL=0. 358 * 359 * tsbep = pointer to the TSBE to load as va (ro) 360 * tteva = pointer to the TTE to load as va (ro) 361 * tagtarget = TSBE tag to load (which contains no context), synthesized 362 * to match va of MMU tag target register only (ro) 363 * tmp1, tmp2 = scratch registers (clobbered) 364 * label = label to use for branches (text) 365 * %asi = ASI to use for TSB access 366 */ 367 368#if defined(UTSB_PHYS) 369 370#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 371 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 372 ldx [tteva], tteva /* load tte */ ;\ 373 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 374 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 375 add tsbep, TSBE_TAG, tmp1 ;\ 376 brgez,a,pn tteva, label ;\ 377 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 378 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 379label: 380 381#else /* UTSB_PHYS */ 382 383#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 384 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 385 ldx [tteva], tteva /* load tte */ ;\ 386 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 387 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 388 brgez,a,pn tteva, label ;\ 389 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 390 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 391label: 392 393#endif /* UTSB_PHYS */ 394 395/* 396 * Invalidate a TSB entry in the TSB. 397 * 398 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 399 * about this earlier to ensure this is true. Thus when we are 400 * directly referencing tsbep below, we are referencing the tte_tag 401 * field of the TSBE. If this offset ever changes, the code below 402 * will need to be modified. 403 * 404 * tsbep = pointer to TSBE as va (ro) 405 * tag = invalidation is done if this matches the TSBE tag (ro) 406 * tmp1 - tmp3 = scratch registers (clobbered) 407 * label = label name to use for branches (text) 408 * %asi = ASI to use for TSB access 409 */ 410 411#if defined(UTSB_PHYS) 412 413#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 414 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 415 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 416label/**/1: ;\ 417 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 418 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 419 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 420 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 421 cmp tag, tmp3 /* compare tags */ ;\ 422 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 423 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 424 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 425 cmp tmp1, tmp3 /* if not successful */ ;\ 426 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 427 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 428label/**/2: 429 430#else /* UTSB_PHYS */ 431 432#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 433 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 434 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 435label/**/1: ;\ 436 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 437 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 438 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 439 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 440 cmp tag, tmp3 /* compare tags */ ;\ 441 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 442 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 443 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 444 cmp tmp1, tmp3 /* if not successful */ ;\ 445 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 446 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 447label/**/2: 448 449#endif /* UTSB_PHYS */ 450 451#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 452#error - TSB_SOFTSZ_MASK too small 453#endif 454 455 456/* 457 * An implementation of setx which will be hot patched at run time. 458 * since it is being hot patched, there is no value passed in. 459 * Thus, essentially we are implementing 460 * setx value, tmp, dest 461 * where value is RUNTIME_PATCH (aka 0) in this case. 462 */ 463#define RUNTIME_PATCH_SETX(dest, tmp) \ 464 sethi %hh(RUNTIME_PATCH), tmp ;\ 465 sethi %lm(RUNTIME_PATCH), dest ;\ 466 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 467 or dest, %lo(RUNTIME_PATCH), dest ;\ 468 sllx tmp, 32, tmp ;\ 469 nop /* for perf reasons */ ;\ 470 or tmp, dest, dest /* contents of patched value */ 471 472#endif /* lint */ 473 474 475#if defined (lint) 476 477/* 478 * sfmmu related subroutines 479 */ 480uint_t 481sfmmu_disable_intrs() 482{ return(0); } 483 484/* ARGSUSED */ 485void 486sfmmu_enable_intrs(uint_t pstate_save) 487{} 488 489/* ARGSUSED */ 490int 491sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag) 492{ return(0); } 493 494/* 495 * Use cas, if tte has changed underneath us then reread and try again. 496 * In the case of a retry, it will update sttep with the new original. 497 */ 498/* ARGSUSED */ 499int 500sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 501{ return(0); } 502 503/* 504 * Use cas, if tte has changed underneath us then return 1, else return 0 505 */ 506/* ARGSUSED */ 507int 508sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 509{ return(0); } 510 511/* ARGSUSED */ 512void 513sfmmu_copytte(tte_t *sttep, tte_t *dttep) 514{} 515 516/*ARGSUSED*/ 517struct tsbe * 518sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 519{ return(0); } 520 521/*ARGSUSED*/ 522uint64_t 523sfmmu_make_tsbtag(caddr_t va) 524{ return(0); } 525 526#else /* lint */ 527 528 .seg ".data" 529 .global sfmmu_panic1 530sfmmu_panic1: 531 .asciz "sfmmu_asm: interrupts already disabled" 532 533 .global sfmmu_panic3 534sfmmu_panic3: 535 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 536 537 .global sfmmu_panic4 538sfmmu_panic4: 539 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 540 541 .global sfmmu_panic5 542sfmmu_panic5: 543 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 544 545 .global sfmmu_panic6 546sfmmu_panic6: 547 .asciz "sfmmu_asm: interrupts not disabled" 548 549 .global sfmmu_panic7 550sfmmu_panic7: 551 .asciz "sfmmu_asm: kernel as" 552 553 .global sfmmu_panic8 554sfmmu_panic8: 555 .asciz "sfmmu_asm: gnum is zero" 556 557 .global sfmmu_panic9 558sfmmu_panic9: 559 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 560 561 .global sfmmu_panic10 562sfmmu_panic10: 563 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 564 565 .global sfmmu_panic11 566sfmmu_panic11: 567 .asciz "sfmmu_asm: ktsb_phys must not be 0 on a sun4v platform" 568 569 ENTRY(sfmmu_disable_intrs) 570 rdpr %pstate, %o0 571#ifdef DEBUG 572 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 573#endif /* DEBUG */ 574 retl 575 wrpr %o0, PSTATE_IE, %pstate 576 SET_SIZE(sfmmu_disable_intrs) 577 578 ENTRY(sfmmu_enable_intrs) 579 retl 580 wrpr %g0, %o0, %pstate 581 SET_SIZE(sfmmu_enable_intrs) 582 583/* 584 * This routine is called both by resume() and sfmmu_get_ctx() to 585 * allocate a new context for the process on a MMU. 586 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 587 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 588 * is the case when sfmmu_alloc_ctx is called from resume(). 589 * 590 * The caller must disable interrupts before entering this routine. 591 * To reduce ctx switch overhead, the code contains both 'fast path' and 592 * 'slow path' code. The fast path code covers the common case where only 593 * a quick check is needed and the real ctx allocation is not required. 594 * It can be done without holding the per-process (PP) lock. 595 * The 'slow path' code must be protected by the PP Lock and performs ctx 596 * allocation. 597 * Hardware context register and HAT mmu cnum are updated accordingly. 598 * 599 * %o0 - sfmmup 600 * %o1 - allocflag 601 * %o2 - CPU 602 * %o3 - sfmmu private/shared flag 603 * 604 * ret - 0: no ctx is allocated 605 * 1: a ctx is allocated 606 */ 607 ENTRY_NP(sfmmu_alloc_ctx) 608 609#ifdef DEBUG 610 sethi %hi(ksfmmup), %g1 611 ldx [%g1 + %lo(ksfmmup)], %g1 612 cmp %g1, %o0 613 bne,pt %xcc, 0f 614 nop 615 616 sethi %hi(panicstr), %g1 ! if kernel as, panic 617 ldx [%g1 + %lo(panicstr)], %g1 618 tst %g1 619 bnz,pn %icc, 7f 620 nop 621 622 sethi %hi(sfmmu_panic7), %o0 623 call panic 624 or %o0, %lo(sfmmu_panic7), %o0 625 6267: 627 retl 628 mov %g0, %o0 ! %o0 = ret = 0 629 6300: 631 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 632#endif /* DEBUG */ 633 634 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 635 636 ! load global mmu_ctxp info 637 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 638 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 639 640 ! load global mmu_ctxp gnum 641 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 642 643#ifdef DEBUG 644 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 645 bne,pt %xcc, 3f 646 nop 647 648 sethi %hi(panicstr), %g1 ! test if panicstr is already set 649 ldx [%g1 + %lo(panicstr)], %g1 650 tst %g1 651 bnz,pn %icc, 1f 652 nop 653 654 sethi %hi(sfmmu_panic8), %o0 655 call panic 656 or %o0, %lo(sfmmu_panic8), %o0 6571: 658 retl 659 mov %g0, %o0 ! %o0 = ret = 0 6603: 661#endif 662 663 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 664 665 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 666 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 667 668 /* 669 * %g5 = sfmmu gnum returned 670 * %g6 = sfmmu cnum returned 671 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 672 * %g4 = scratch 673 * 674 * Fast path code, do a quick check. 675 */ 676 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 677 678 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 679 bne,pt %icc, 1f ! valid hat cnum, check gnum 680 nop 681 682 ! cnum == INVALID, check allocflag 683 mov %g0, %g4 ! %g4 = ret = 0 684 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 685 mov %g6, %o1 686 687 ! (invalid HAT cnum) && (allocflag == 1) 688 ba,pt %icc, 2f 689 nop 6901: 691 ! valid HAT cnum, check gnum 692 cmp %g5, %o4 693 mov 1, %g4 !%g4 = ret = 1 694 be,a,pt %icc, 8f ! gnum unchanged, go to done 695 mov %g6, %o1 696 6972: 698 /* 699 * Grab per process (PP) sfmmu_ctx_lock spinlock, 700 * followed by the 'slow path' code. 701 */ 702 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 7033: 704 brz %g3, 5f 705 nop 7064: 707 brnz,a,pt %g3, 4b ! spin if lock is 1 708 ldub [%o0 + SFMMU_CTX_LOCK], %g3 709 ba %xcc, 3b ! retry the lock 710 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 711 7125: 713 membar #LoadLoad 714 /* 715 * %g5 = sfmmu gnum returned 716 * %g6 = sfmmu cnum returned 717 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 718 * %g4 = scratch 719 */ 720 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 721 722 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 723 bne,pt %icc, 1f ! valid hat cnum, check gnum 724 nop 725 726 ! cnum == INVALID, check allocflag 727 mov %g0, %g4 ! %g4 = ret = 0 728 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 729 mov %g6, %o1 730 731 ! (invalid HAT cnum) && (allocflag == 1) 732 ba,pt %icc, 6f 733 nop 7341: 735 ! valid HAT cnum, check gnum 736 cmp %g5, %o4 737 mov 1, %g4 ! %g4 = ret = 1 738 be,a,pt %icc, 2f ! gnum unchanged, go to done 739 mov %g6, %o1 740 741 ba,pt %icc, 6f 742 nop 7432: 744 membar #LoadStore|#StoreStore 745 ba,pt %icc, 8f 746 clrb [%o0 + SFMMU_CTX_LOCK] 7476: 748 /* 749 * We get here if we do not have a valid context, or 750 * the HAT gnum does not match global gnum. We hold 751 * sfmmu_ctx_lock spinlock. Allocate that context. 752 * 753 * %o3 = mmu_ctxp 754 */ 755 add %o3, MMU_CTX_CNUM, %g3 756 ld [%o3 + MMU_CTX_NCTXS], %g4 757 758 /* 759 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 760 * %g3 = mmu cnum address 761 * %g4 = mmu nctxs 762 * 763 * %o0 = sfmmup 764 * %o1 = mmu current cnum value (used as new cnum) 765 * %o4 = mmu gnum 766 * 767 * %o5 = scratch 768 */ 769 ld [%g3], %o1 7700: 771 cmp %o1, %g4 772 bl,a,pt %icc, 1f 773 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 774 775 /* 776 * cnum reachs max, bail, so wrap around can be performed later. 777 */ 778 set INVALID_CONTEXT, %o1 779 mov %g0, %g4 ! %g4 = ret = 0 780 781 membar #LoadStore|#StoreStore 782 ba,pt %icc, 8f 783 clrb [%o0 + SFMMU_CTX_LOCK] 7841: 785 ! %g3 = addr of mmu_ctxp->cnum 786 ! %o5 = mmu_ctxp->cnum + 1 787 cas [%g3], %o1, %o5 788 cmp %o1, %o5 789 bne,a,pn %xcc, 0b ! cas failed 790 ld [%g3], %o1 791 792#ifdef DEBUG 793 set MAX_SFMMU_CTX_VAL, %o5 794 cmp %o1, %o5 795 ble,pt %icc, 2f 796 nop 797 798 sethi %hi(sfmmu_panic9), %o0 799 call panic 800 or %o0, %lo(sfmmu_panic9), %o0 8012: 802#endif 803 ! update hat gnum and cnum 804 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 805 or %o4, %o1, %o4 806 stx %o4, [%g2 + SFMMU_CTXS] 807 808 membar #LoadStore|#StoreStore 809 clrb [%o0 + SFMMU_CTX_LOCK] 810 811 mov 1, %g4 ! %g4 = ret = 1 8128: 813 /* 814 * program the secondary context register 815 * 816 * %o1 = cnum 817 * %g1 = sfmmu private/shared flag (0:private, 1:shared) 818 */ 819 820 /* 821 * When we come here and context is invalid, we want to set both 822 * private and shared ctx regs to INVALID. In order to 823 * do so, we set the sfmmu priv/shared flag to 'private' regardless 824 * so that private ctx reg will be set to invalid. 825 * Note that on sun4v values written to private context register are 826 * automatically written to corresponding shared context register as 827 * well. On sun4u SET_SECCTX() will invalidate shared context register 828 * when it sets a private secondary context register. 829 */ 830 831 cmp %o1, INVALID_CONTEXT 832 be,a,pn %icc, 9f 833 clr %g1 8349: 835 836#ifdef sun4u 837 ldub [%o0 + SFMMU_CEXT], %o2 838 sll %o2, CTXREG_EXT_SHIFT, %o2 839 or %o1, %o2, %o1 840#endif /* sun4u */ 841 842 SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1) 843 844 retl 845 mov %g4, %o0 ! %o0 = ret 846 847 SET_SIZE(sfmmu_alloc_ctx) 848 849 ENTRY_NP(sfmmu_modifytte) 850 ldx [%o2], %g3 /* current */ 851 ldx [%o0], %g1 /* original */ 8522: 853 ldx [%o1], %g2 /* modified */ 854 cmp %g2, %g3 /* is modified = current? */ 855 be,a,pt %xcc,1f /* yes, don't write */ 856 stx %g3, [%o0] /* update new original */ 857 casx [%o2], %g1, %g2 858 cmp %g1, %g2 859 be,pt %xcc, 1f /* cas succeeded - return */ 860 nop 861 ldx [%o2], %g3 /* new current */ 862 stx %g3, [%o0] /* save as new original */ 863 ba,pt %xcc, 2b 864 mov %g3, %g1 8651: retl 866 membar #StoreLoad 867 SET_SIZE(sfmmu_modifytte) 868 869 ENTRY_NP(sfmmu_modifytte_try) 870 ldx [%o1], %g2 /* modified */ 871 ldx [%o2], %g3 /* current */ 872 ldx [%o0], %g1 /* original */ 873 cmp %g3, %g2 /* is modified = current? */ 874 be,a,pn %xcc,1f /* yes, don't write */ 875 mov 0, %o1 /* as if cas failed. */ 876 877 casx [%o2], %g1, %g2 878 membar #StoreLoad 879 cmp %g1, %g2 880 movne %xcc, -1, %o1 /* cas failed. */ 881 move %xcc, 1, %o1 /* cas succeeded. */ 8821: 883 stx %g2, [%o0] /* report "current" value */ 884 retl 885 mov %o1, %o0 886 SET_SIZE(sfmmu_modifytte_try) 887 888 ENTRY_NP(sfmmu_copytte) 889 ldx [%o0], %g1 890 retl 891 stx %g1, [%o1] 892 SET_SIZE(sfmmu_copytte) 893 894 895 /* 896 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 897 * %o0 = TSB base address (in), pointer to TSB entry (out) 898 * %o1 = vaddr (in) 899 * %o2 = vpshift (in) 900 * %o3 = tsb size code (in) 901 * %o4 = scratch register 902 */ 903 ENTRY_NP(sfmmu_get_tsbe) 904 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 905 retl 906 nop 907 SET_SIZE(sfmmu_get_tsbe) 908 909 /* 910 * Return a TSB tag for the given va. 911 * %o0 = va (in/clobbered) 912 * %o0 = va shifted to be in tsb tag format (with no context) (out) 913 */ 914 ENTRY_NP(sfmmu_make_tsbtag) 915 retl 916 srln %o0, TTARGET_VA_SHIFT, %o0 917 SET_SIZE(sfmmu_make_tsbtag) 918 919#endif /* lint */ 920 921/* 922 * Other sfmmu primitives 923 */ 924 925 926#if defined (lint) 927void 928sfmmu_patch_ktsb(void) 929{ 930} 931 932void 933sfmmu_kpm_patch_tlbm(void) 934{ 935} 936 937void 938sfmmu_kpm_patch_tsbm(void) 939{ 940} 941 942void 943sfmmu_patch_shctx(void) 944{ 945} 946 947/* ARGSUSED */ 948void 949sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 950{ 951} 952 953/* ARGSUSED */ 954void 955sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 956{ 957} 958 959/* ARGSUSED */ 960void 961sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 962{ 963} 964 965/* ARGSUSED */ 966void 967sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 968{ 969} 970 971#else /* lint */ 972 973#define I_SIZE 4 974 975 ENTRY_NP(sfmmu_fix_ktlb_traptable) 976 /* 977 * %o0 = start of patch area 978 * %o1 = size code of TSB to patch 979 * %o3 = scratch 980 */ 981 /* fix sll */ 982 ld [%o0], %o3 /* get sll */ 983 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 984 st %o3, [%o0] /* write sll */ 985 flush %o0 986 /* fix srl */ 987 add %o0, I_SIZE, %o0 /* goto next instr. */ 988 ld [%o0], %o3 /* get srl */ 989 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 990 st %o3, [%o0] /* write srl */ 991 retl 992 flush %o0 993 SET_SIZE(sfmmu_fix_ktlb_traptable) 994 995 ENTRY_NP(sfmmu_fixup_ktsbbase) 996 /* 997 * %o0 = start of patch area 998 * %o5 = kernel virtual or physical tsb base address 999 * %o2, %o3 are used as scratch registers. 1000 */ 1001 /* fixup sethi instruction */ 1002 ld [%o0], %o3 1003 srl %o5, 10, %o2 ! offset is bits 32:10 1004 or %o3, %o2, %o3 ! set imm22 1005 st %o3, [%o0] 1006 /* fixup offset of lduw/ldx */ 1007 add %o0, I_SIZE, %o0 ! next instr 1008 ld [%o0], %o3 1009 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 1010 or %o3, %o2, %o3 1011 st %o3, [%o0] 1012 retl 1013 flush %o0 1014 SET_SIZE(sfmmu_fixup_ktsbbase) 1015 1016 ENTRY_NP(sfmmu_fixup_setx) 1017 /* 1018 * %o0 = start of patch area 1019 * %o4 = 64 bit value to patch 1020 * %o2, %o3 are used as scratch registers. 1021 * 1022 * Note: Assuming that all parts of the instructions which need to be 1023 * patched correspond to RUNTIME_PATCH (aka 0) 1024 * 1025 * Note the implementation of setx which is being patched is as follows: 1026 * 1027 * sethi %hh(RUNTIME_PATCH), tmp 1028 * sethi %lm(RUNTIME_PATCH), dest 1029 * or tmp, %hm(RUNTIME_PATCH), tmp 1030 * or dest, %lo(RUNTIME_PATCH), dest 1031 * sllx tmp, 32, tmp 1032 * nop 1033 * or tmp, dest, dest 1034 * 1035 * which differs from the implementation in the 1036 * "SPARC Architecture Manual" 1037 */ 1038 /* fixup sethi instruction */ 1039 ld [%o0], %o3 1040 srlx %o4, 42, %o2 ! bits [63:42] 1041 or %o3, %o2, %o3 ! set imm22 1042 st %o3, [%o0] 1043 /* fixup sethi instruction */ 1044 add %o0, I_SIZE, %o0 ! next instr 1045 ld [%o0], %o3 1046 sllx %o4, 32, %o2 ! clear upper bits 1047 srlx %o2, 42, %o2 ! bits [31:10] 1048 or %o3, %o2, %o3 ! set imm22 1049 st %o3, [%o0] 1050 /* fixup or instruction */ 1051 add %o0, I_SIZE, %o0 ! next instr 1052 ld [%o0], %o3 1053 srlx %o4, 32, %o2 ! bits [63:32] 1054 and %o2, 0x3ff, %o2 ! bits [41:32] 1055 or %o3, %o2, %o3 ! set imm 1056 st %o3, [%o0] 1057 /* fixup or instruction */ 1058 add %o0, I_SIZE, %o0 ! next instr 1059 ld [%o0], %o3 1060 and %o4, 0x3ff, %o2 ! bits [9:0] 1061 or %o3, %o2, %o3 ! set imm 1062 st %o3, [%o0] 1063 retl 1064 flush %o0 1065 SET_SIZE(sfmmu_fixup_setx) 1066 1067 ENTRY_NP(sfmmu_fixup_or) 1068 /* 1069 * %o0 = start of patch area 1070 * %o4 = 32 bit value to patch 1071 * %o2, %o3 are used as scratch registers. 1072 * Note: Assuming that all parts of the instructions which need to be 1073 * patched correspond to RUNTIME_PATCH (aka 0) 1074 */ 1075 ld [%o0], %o3 1076 and %o4, 0x3ff, %o2 ! bits [9:0] 1077 or %o3, %o2, %o3 ! set imm 1078 st %o3, [%o0] 1079 retl 1080 flush %o0 1081 SET_SIZE(sfmmu_fixup_or) 1082 1083 ENTRY_NP(sfmmu_fixup_shiftx) 1084 /* 1085 * %o0 = start of patch area 1086 * %o4 = signed int immediate value to add to sllx/srlx imm field 1087 * %o2, %o3 are used as scratch registers. 1088 * 1089 * sllx/srlx store the 6 bit immediate value in the lowest order bits 1090 * so we do a simple add. The caller must be careful to prevent 1091 * overflow, which could easily occur if the initial value is nonzero! 1092 */ 1093 ld [%o0], %o3 ! %o3 = instruction to patch 1094 and %o3, 0x3f, %o2 ! %o2 = existing imm value 1095 add %o2, %o4, %o2 ! %o2 = new imm value 1096 andn %o3, 0x3f, %o3 ! clear old imm value 1097 and %o2, 0x3f, %o2 ! truncate new imm value 1098 or %o3, %o2, %o3 ! set new imm value 1099 st %o3, [%o0] ! store updated instruction 1100 retl 1101 flush %o0 1102 SET_SIZE(sfmmu_fixup_shiftx) 1103 1104 ENTRY_NP(sfmmu_fixup_mmu_asi) 1105 /* 1106 * Patch imm_asi of all ldda instructions in the MMU 1107 * trap handlers. We search MMU_PATCH_INSTR instructions 1108 * starting from the itlb miss handler (trap 0x64). 1109 * %o0 = address of tt[0,1]_itlbmiss 1110 * %o1 = imm_asi to setup, shifted by appropriate offset. 1111 * %o3 = number of instructions to search 1112 * %o4 = reserved by caller: called from leaf routine 1113 */ 11141: ldsw [%o0], %o2 ! load instruction to %o2 1115 brgez,pt %o2, 2f 1116 srl %o2, 30, %o5 1117 btst 1, %o5 ! test bit 30; skip if not set 1118 bz,pt %icc, 2f 1119 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1120 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1121 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1122 brnz,pt %o5, 2f ! skip if not a match 1123 or %o2, %o1, %o2 ! or in imm_asi 1124 st %o2, [%o0] ! write patched instruction 11252: dec %o3 1126 brnz,a,pt %o3, 1b ! loop until we're done 1127 add %o0, I_SIZE, %o0 1128 retl 1129 flush %o0 1130 SET_SIZE(sfmmu_fixup_mmu_asi) 1131 1132 /* 1133 * Patch immediate ASI used to access the TSB in the 1134 * trap table. 1135 * inputs: %o0 = value of ktsb_phys 1136 */ 1137 ENTRY_NP(sfmmu_patch_mmu_asi) 1138 mov %o7, %o4 ! save return pc in %o4 1139 mov ASI_QUAD_LDD_PHYS, %o3 ! set QUAD_LDD_PHYS by default 1140 1141#ifdef sun4v 1142 1143 /* 1144 * Check ktsb_phys. It must be non-zero for sun4v, panic if not. 1145 */ 1146 1147 brnz,pt %o0, do_patch 1148 nop 1149 1150 sethi %hi(sfmmu_panic11), %o0 1151 call panic 1152 or %o0, %lo(sfmmu_panic11), %o0 1153do_patch: 1154 1155#else /* sun4v */ 1156 /* 1157 * Some non-sun4v platforms deploy virtual ktsb (ktsb_phys==0). 1158 * Note that ASI_NQUAD_LD is not defined/used for sun4v 1159 */ 1160 movrz %o0, ASI_NQUAD_LD, %o3 1161 1162#endif /* sun4v */ 1163 1164 sll %o3, 5, %o1 ! imm_asi offset 1165 mov 6, %o3 ! number of instructions 1166 sethi %hi(dktsb), %o0 ! to search 1167 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1168 or %o0, %lo(dktsb), %o0 1169 mov 6, %o3 ! number of instructions 1170 sethi %hi(dktsb4m), %o0 ! to search 1171 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1172 or %o0, %lo(dktsb4m), %o0 1173 mov 6, %o3 ! number of instructions 1174 sethi %hi(iktsb), %o0 ! to search 1175 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1176 or %o0, %lo(iktsb), %o0 1177 mov 6, %o3 ! number of instructions 1178 sethi %hi(iktsb4m), %o0 ! to search 1179 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1180 or %o0, %lo(iktsb4m), %o0 1181 mov %o4, %o7 ! retore return pc -- leaf 1182 retl 1183 nop 1184 SET_SIZE(sfmmu_patch_mmu_asi) 1185 1186 1187 ENTRY_NP(sfmmu_patch_ktsb) 1188 /* 1189 * We need to fix iktsb, dktsb, et. al. 1190 */ 1191 save %sp, -SA(MINFRAME), %sp 1192 set ktsb_phys, %o1 1193 ld [%o1], %o4 1194 set ktsb_base, %o5 1195 set ktsb4m_base, %l1 1196 brz,pt %o4, 1f 1197 nop 1198 set ktsb_pbase, %o5 1199 set ktsb4m_pbase, %l1 12001: 1201 sethi %hi(ktsb_szcode), %o1 1202 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1203 1204 sethi %hi(iktsb), %o0 1205 call sfmmu_fix_ktlb_traptable 1206 or %o0, %lo(iktsb), %o0 1207 1208 sethi %hi(dktsb), %o0 1209 call sfmmu_fix_ktlb_traptable 1210 or %o0, %lo(dktsb), %o0 1211 1212 sethi %hi(ktsb4m_szcode), %o1 1213 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1214 1215 sethi %hi(iktsb4m), %o0 1216 call sfmmu_fix_ktlb_traptable 1217 or %o0, %lo(iktsb4m), %o0 1218 1219 sethi %hi(dktsb4m), %o0 1220 call sfmmu_fix_ktlb_traptable 1221 or %o0, %lo(dktsb4m), %o0 1222 1223#ifndef sun4v 1224 mov ASI_N, %o2 1225 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1226 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1227 sethi %hi(tsb_kernel_patch_asi), %o0 1228 call sfmmu_fixup_or 1229 or %o0, %lo(tsb_kernel_patch_asi), %o0 1230#endif /* !sun4v */ 1231 1232 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1233 1234 sethi %hi(dktsbbase), %o0 1235 call sfmmu_fixup_setx ! patch value of ktsb base addr 1236 or %o0, %lo(dktsbbase), %o0 1237 1238 sethi %hi(iktsbbase), %o0 1239 call sfmmu_fixup_setx ! patch value of ktsb base addr 1240 or %o0, %lo(iktsbbase), %o0 1241 1242 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1243 call sfmmu_fixup_setx ! patch value of ktsb base addr 1244 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1245 1246#ifdef sun4v 1247 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1248 call sfmmu_fixup_setx ! patch value of ktsb base addr 1249 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1250#endif /* sun4v */ 1251 1252 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1253 1254 sethi %hi(dktsb4mbase), %o0 1255 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1256 or %o0, %lo(dktsb4mbase), %o0 1257 1258 sethi %hi(iktsb4mbase), %o0 1259 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1260 or %o0, %lo(iktsb4mbase), %o0 1261 1262 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1263 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1264 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1265 1266#ifdef sun4v 1267 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1268 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1269 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1270#endif /* sun4v */ 1271 1272 set ktsb_szcode, %o4 1273 ld [%o4], %o4 1274 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1275 call sfmmu_fixup_or ! patch value of ktsb_szcode 1276 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1277 1278#ifdef sun4v 1279 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1280 call sfmmu_fixup_or ! patch value of ktsb_szcode 1281 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1282#endif /* sun4v */ 1283 1284 set ktsb4m_szcode, %o4 1285 ld [%o4], %o4 1286 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1287 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1288 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1289 1290#ifdef sun4v 1291 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1292 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1293 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1294#endif /* sun4v */ 1295 1296 ret 1297 restore 1298 SET_SIZE(sfmmu_patch_ktsb) 1299 1300 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1301 /* 1302 * Fixup trap handlers in common segkpm case. This is reserved 1303 * for future use should kpm TSB be changed to be other than the 1304 * kernel TSB. 1305 */ 1306 retl 1307 nop 1308 SET_SIZE(sfmmu_kpm_patch_tlbm) 1309 1310 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1311 /* 1312 * nop the branch to sfmmu_kpm_dtsb_miss_small 1313 * in the case where we are using large pages for 1314 * seg_kpm (and hence must probe the second TSB for 1315 * seg_kpm VAs) 1316 */ 1317 set dktsb4m_kpmcheck_small, %o0 1318 MAKE_NOP_INSTR(%o1) 1319 st %o1, [%o0] 1320 flush %o0 1321 retl 1322 nop 1323 SET_SIZE(sfmmu_kpm_patch_tsbm) 1324 1325 ENTRY_NP(sfmmu_patch_utsb) 1326#ifdef UTSB_PHYS 1327 retl 1328 nop 1329#else /* UTSB_PHYS */ 1330 /* 1331 * We need to hot patch utsb_vabase and utsb4m_vabase 1332 */ 1333 save %sp, -SA(MINFRAME), %sp 1334 1335 /* patch value of utsb_vabase */ 1336 set utsb_vabase, %o1 1337 ldx [%o1], %o4 1338 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1339 call sfmmu_fixup_setx 1340 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1341 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1342 call sfmmu_fixup_setx 1343 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1344 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1345 call sfmmu_fixup_setx 1346 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1347 1348 /* patch value of utsb4m_vabase */ 1349 set utsb4m_vabase, %o1 1350 ldx [%o1], %o4 1351 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1352 call sfmmu_fixup_setx 1353 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1354 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1355 call sfmmu_fixup_setx 1356 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1357 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1358 call sfmmu_fixup_setx 1359 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1360 1361 /* 1362 * Patch TSB base register masks and shifts if needed. 1363 * By default the TSB base register contents are set up for 4M slab. 1364 * If we're using a smaller slab size and reserved VA range we need 1365 * to patch up those values here. 1366 */ 1367 set tsb_slab_shift, %o1 1368 set MMU_PAGESHIFT4M, %o4 1369 lduw [%o1], %o3 1370 subcc %o4, %o3, %o4 1371 bz,pt %icc, 1f 1372 /* delay slot safe */ 1373 1374 /* patch reserved VA range size if needed. */ 1375 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1376 call sfmmu_fixup_shiftx 1377 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1378 call sfmmu_fixup_shiftx 1379 add %o0, I_SIZE, %o0 1380 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1381 call sfmmu_fixup_shiftx 1382 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1383 call sfmmu_fixup_shiftx 1384 add %o0, I_SIZE, %o0 13851: 1386 /* patch TSBREG_VAMASK used to set up TSB base register */ 1387 set tsb_slab_mask, %o1 1388 ldx [%o1], %o4 1389 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1390 call sfmmu_fixup_or 1391 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1392 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1393 call sfmmu_fixup_or 1394 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1395 1396 ret 1397 restore 1398#endif /* UTSB_PHYS */ 1399 SET_SIZE(sfmmu_patch_utsb) 1400 1401 ENTRY_NP(sfmmu_patch_shctx) 1402#ifdef sun4u 1403 retl 1404 nop 1405#else /* sun4u */ 1406 set sfmmu_shctx_cpu_mondo_patch, %o0 1407 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1408 st %o1, [%o0] 1409 flush %o0 1410 MAKE_NOP_INSTR(%o1) 1411 add %o0, I_SIZE, %o0 ! next instr 1412 st %o1, [%o0] 1413 flush %o0 1414 1415 set sfmmu_shctx_user_rtt_patch, %o0 1416 st %o1, [%o0] ! nop 1st instruction 1417 flush %o0 1418 add %o0, I_SIZE, %o0 1419 st %o1, [%o0] ! nop 2nd instruction 1420 flush %o0 1421 add %o0, I_SIZE, %o0 1422 st %o1, [%o0] ! nop 3rd instruction 1423 flush %o0 1424 add %o0, I_SIZE, %o0 1425 st %o1, [%o0] ! nop 4th instruction 1426 flush %o0 1427 add %o0, I_SIZE, %o0 1428 st %o1, [%o0] ! nop 5th instruction 1429 flush %o0 1430 add %o0, I_SIZE, %o0 1431 st %o1, [%o0] ! nop 6th instruction 1432 retl 1433 flush %o0 1434#endif /* sun4u */ 1435 SET_SIZE(sfmmu_patch_shctx) 1436 1437 /* 1438 * Routine that loads an entry into a tsb using virtual addresses. 1439 * Locking is required since all cpus can use the same TSB. 1440 * Note that it is no longer required to have a valid context 1441 * when calling this function. 1442 */ 1443 ENTRY_NP(sfmmu_load_tsbe) 1444 /* 1445 * %o0 = pointer to tsbe to load 1446 * %o1 = tsb tag 1447 * %o2 = virtual pointer to TTE 1448 * %o3 = 1 if physical address in %o0 else 0 1449 */ 1450 rdpr %pstate, %o5 1451#ifdef DEBUG 1452 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1453#endif /* DEBUG */ 1454 1455 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1456 1457 SETUP_TSB_ASI(%o3, %g3) 1458 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, locked_tsb_l8) 1459 1460 wrpr %g0, %o5, %pstate /* enable interrupts */ 1461 1462 retl 1463 membar #StoreStore|#StoreLoad 1464 SET_SIZE(sfmmu_load_tsbe) 1465 1466 /* 1467 * Flush TSB of a given entry if the tag matches. 1468 */ 1469 ENTRY(sfmmu_unload_tsbe) 1470 /* 1471 * %o0 = pointer to tsbe to be flushed 1472 * %o1 = tag to match 1473 * %o2 = 1 if physical address in %o0 else 0 1474 */ 1475 SETUP_TSB_ASI(%o2, %g1) 1476 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1477 retl 1478 membar #StoreStore|#StoreLoad 1479 SET_SIZE(sfmmu_unload_tsbe) 1480 1481 /* 1482 * Routine that loads a TTE into the kpm TSB from C code. 1483 * Locking is required since kpm TSB is shared among all CPUs. 1484 */ 1485 ENTRY_NP(sfmmu_kpm_load_tsb) 1486 /* 1487 * %o0 = vaddr 1488 * %o1 = ttep 1489 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1490 */ 1491 rdpr %pstate, %o5 ! %o5 = saved pstate 1492#ifdef DEBUG 1493 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1494#endif /* DEBUG */ 1495 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1496 1497#ifndef sun4v 1498 sethi %hi(ktsb_phys), %o4 1499 mov ASI_N, %o3 1500 ld [%o4 + %lo(ktsb_phys)], %o4 1501 movrnz %o4, ASI_MEM, %o3 1502 mov %o3, %asi 1503#endif /* !sun4v */ 1504 mov %o0, %g1 ! %g1 = vaddr 1505 1506 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1507 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1508 /* %g2 = tsbep, %g1 clobbered */ 1509 1510 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1511 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1512 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, locked_tsb_l9) 1513 1514 wrpr %g0, %o5, %pstate ! enable interrupts 1515 retl 1516 membar #StoreStore|#StoreLoad 1517 SET_SIZE(sfmmu_kpm_load_tsb) 1518 1519 /* 1520 * Routine that shoots down a TTE in the kpm TSB or in the 1521 * kernel TSB depending on virtpg. Locking is required since 1522 * kpm/kernel TSB is shared among all CPUs. 1523 */ 1524 ENTRY_NP(sfmmu_kpm_unload_tsb) 1525 /* 1526 * %o0 = vaddr 1527 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1528 */ 1529#ifndef sun4v 1530 sethi %hi(ktsb_phys), %o4 1531 mov ASI_N, %o3 1532 ld [%o4 + %lo(ktsb_phys)], %o4 1533 movrnz %o4, ASI_MEM, %o3 1534 mov %o3, %asi 1535#endif /* !sun4v */ 1536 mov %o0, %g1 ! %g1 = vaddr 1537 1538 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1539 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1540 /* %g2 = tsbep, %g1 clobbered */ 1541 1542 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1543 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1544 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1545 1546 retl 1547 membar #StoreStore|#StoreLoad 1548 SET_SIZE(sfmmu_kpm_unload_tsb) 1549 1550#endif /* lint */ 1551 1552 1553#if defined (lint) 1554 1555/*ARGSUSED*/ 1556pfn_t 1557sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1558{ return(0); } 1559 1560#else /* lint */ 1561 1562 ENTRY_NP(sfmmu_ttetopfn) 1563 ldx [%o0], %g1 /* read tte */ 1564 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1565 /* 1566 * g1 = pfn 1567 */ 1568 retl 1569 mov %g1, %o0 1570 SET_SIZE(sfmmu_ttetopfn) 1571 1572#endif /* !lint */ 1573 1574/* 1575 * These macros are used to update global sfmmu hme hash statistics 1576 * in perf critical paths. It is only enabled in debug kernels or 1577 * if SFMMU_STAT_GATHER is defined 1578 */ 1579#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1580#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1581 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1582 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1583 cmp tmp1, hatid ;\ 1584 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1585 set sfmmu_global_stat, tmp1 ;\ 1586 add tmp1, tmp2, tmp1 ;\ 1587 ld [tmp1], tmp2 ;\ 1588 inc tmp2 ;\ 1589 st tmp2, [tmp1] 1590 1591#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1592 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1593 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1594 cmp tmp1, hatid ;\ 1595 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1596 set sfmmu_global_stat, tmp1 ;\ 1597 add tmp1, tmp2, tmp1 ;\ 1598 ld [tmp1], tmp2 ;\ 1599 inc tmp2 ;\ 1600 st tmp2, [tmp1] 1601 1602 1603#else /* DEBUG || SFMMU_STAT_GATHER */ 1604 1605#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1606 1607#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1608 1609#endif /* DEBUG || SFMMU_STAT_GATHER */ 1610 1611/* 1612 * This macro is used to update global sfmmu kstas in non 1613 * perf critical areas so they are enabled all the time 1614 */ 1615#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1616 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1617 add tmp1, statname, tmp1 ;\ 1618 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1619 inc tmp2 ;\ 1620 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1621 1622/* 1623 * These macros are used to update per cpu stats in non perf 1624 * critical areas so they are enabled all the time 1625 */ 1626#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1627 ld [tsbarea + stat], tmp1 ;\ 1628 inc tmp1 ;\ 1629 st tmp1, [tsbarea + stat] 1630 1631/* 1632 * These macros are used to update per cpu stats in non perf 1633 * critical areas so they are enabled all the time 1634 */ 1635#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1636 lduh [tsbarea + stat], tmp1 ;\ 1637 inc tmp1 ;\ 1638 stuh tmp1, [tsbarea + stat] 1639 1640#if defined(KPM_TLBMISS_STATS_GATHER) 1641 /* 1642 * Count kpm dtlb misses separately to allow a different 1643 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1644 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1645 */ 1646#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1647 brgez tagacc, label /* KPM VA? */ ;\ 1648 nop ;\ 1649 CPU_INDEX(tmp1, tsbma) ;\ 1650 sethi %hi(kpmtsbm_area), tsbma ;\ 1651 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1652 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1653 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1654 /* VA range check */ ;\ 1655 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1656 cmp tagacc, val ;\ 1657 blu,pn %xcc, label ;\ 1658 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1659 cmp tagacc, tmp1 ;\ 1660 bgeu,pn %xcc, label ;\ 1661 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1662 inc val ;\ 1663 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1664label: 1665#else 1666#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1667#endif /* KPM_TLBMISS_STATS_GATHER */ 1668 1669#if defined (lint) 1670/* 1671 * The following routines are jumped to from the mmu trap handlers to do 1672 * the setting up to call systrap. They are separate routines instead of 1673 * being part of the handlers because the handlers would exceed 32 1674 * instructions and since this is part of the slow path the jump 1675 * cost is irrelevant. 1676 */ 1677void 1678sfmmu_pagefault(void) 1679{ 1680} 1681 1682void 1683sfmmu_mmu_trap(void) 1684{ 1685} 1686 1687void 1688sfmmu_window_trap(void) 1689{ 1690} 1691 1692void 1693sfmmu_kpm_exception(void) 1694{ 1695} 1696 1697#else /* lint */ 1698 1699#ifdef PTL1_PANIC_DEBUG 1700 .seg ".data" 1701 .global test_ptl1_panic 1702test_ptl1_panic: 1703 .word 0 1704 .align 8 1705 1706 .seg ".text" 1707 .align 4 1708#endif /* PTL1_PANIC_DEBUG */ 1709 1710 1711 ENTRY_NP(sfmmu_pagefault) 1712 SET_GL_REG(1) 1713 USE_ALTERNATE_GLOBALS(%g5) 1714 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1715 rdpr %tt, %g6 1716 cmp %g6, FAST_IMMU_MISS_TT 1717 be,a,pn %icc, 1f 1718 mov T_INSTR_MMU_MISS, %g3 1719 cmp %g6, T_INSTR_MMU_MISS 1720 be,a,pn %icc, 1f 1721 mov T_INSTR_MMU_MISS, %g3 1722 mov %g5, %g2 1723 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1724 cmp %g6, FAST_DMMU_MISS_TT 1725 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1726 cmp %g6, T_DATA_MMU_MISS 1727 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1728 1729#ifdef PTL1_PANIC_DEBUG 1730 /* check if we want to test the tl1 panic */ 1731 sethi %hi(test_ptl1_panic), %g4 1732 ld [%g4 + %lo(test_ptl1_panic)], %g1 1733 st %g0, [%g4 + %lo(test_ptl1_panic)] 1734 cmp %g1, %g0 1735 bne,a,pn %icc, ptl1_panic 1736 or %g0, PTL1_BAD_DEBUG, %g1 1737#endif /* PTL1_PANIC_DEBUG */ 17381: 1739 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 1740 /* 1741 * g2 = tag access reg 1742 * g3.l = type 1743 * g3.h = 0 1744 */ 1745 sethi %hi(trap), %g1 1746 or %g1, %lo(trap), %g1 17472: 1748 ba,pt %xcc, sys_trap 1749 mov -1, %g4 1750 SET_SIZE(sfmmu_pagefault) 1751 1752 ENTRY_NP(sfmmu_mmu_trap) 1753 SET_GL_REG(1) 1754 USE_ALTERNATE_GLOBALS(%g5) 1755 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 1756 rdpr %tt, %g6 1757 cmp %g6, FAST_IMMU_MISS_TT 1758 be,a,pn %icc, 1f 1759 mov T_INSTR_MMU_MISS, %g3 1760 cmp %g6, T_INSTR_MMU_MISS 1761 be,a,pn %icc, 1f 1762 mov T_INSTR_MMU_MISS, %g3 1763 mov %g5, %g2 1764 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1765 cmp %g6, FAST_DMMU_MISS_TT 1766 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1767 cmp %g6, T_DATA_MMU_MISS 1768 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 17691: 1770 /* 1771 * g2 = tag access reg 1772 * g3 = type 1773 */ 1774 sethi %hi(sfmmu_tsbmiss_exception), %g1 1775 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 1776 ba,pt %xcc, sys_trap 1777 mov -1, %g4 1778 /*NOTREACHED*/ 1779 SET_SIZE(sfmmu_mmu_trap) 1780 1781 ENTRY_NP(sfmmu_suspend_tl) 1782 SET_GL_REG(1) 1783 USE_ALTERNATE_GLOBALS(%g5) 1784 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 1785 rdpr %tt, %g6 1786 cmp %g6, FAST_IMMU_MISS_TT 1787 be,a,pn %icc, 1f 1788 mov T_INSTR_MMU_MISS, %g3 1789 mov %g5, %g2 1790 cmp %g6, FAST_DMMU_MISS_TT 1791 move %icc, T_DATA_MMU_MISS, %g3 1792 movne %icc, T_DATA_PROT, %g3 17931: 1794 sethi %hi(sfmmu_tsbmiss_suspended), %g1 1795 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 1796 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 1797 ba,pt %xcc, sys_trap 1798 mov PIL_15, %g4 1799 /*NOTREACHED*/ 1800 SET_SIZE(sfmmu_suspend_tl) 1801 1802 /* 1803 * No %g registers in use at this point. 1804 */ 1805 ENTRY_NP(sfmmu_window_trap) 1806 rdpr %tpc, %g1 1807#ifdef sun4v 1808#ifdef DEBUG 1809 /* We assume previous %gl was 1 */ 1810 rdpr %tstate, %g4 1811 srlx %g4, TSTATE_GL_SHIFT, %g4 1812 and %g4, TSTATE_GL_MASK, %g4 1813 cmp %g4, 1 1814 bne,a,pn %icc, ptl1_panic 1815 mov PTL1_BAD_WTRAP, %g1 1816#endif /* DEBUG */ 1817 /* user miss at tl>1. better be the window handler or user_rtt */ 1818 /* in user_rtt? */ 1819 set rtt_fill_start, %g4 1820 cmp %g1, %g4 1821 blu,pn %xcc, 6f 1822 .empty 1823 set rtt_fill_end, %g4 1824 cmp %g1, %g4 1825 bgeu,pn %xcc, 6f 1826 nop 1827 set fault_rtt_fn1, %g1 1828 wrpr %g0, %g1, %tnpc 1829 ba,a 7f 18306: 1831 ! must save this trap level before descending trap stack 1832 ! no need to save %tnpc, either overwritten or discarded 1833 ! already got it: rdpr %tpc, %g1 1834 rdpr %tstate, %g6 1835 rdpr %tt, %g7 1836 ! trap level saved, go get underlying trap type 1837 rdpr %tl, %g5 1838 sub %g5, 1, %g3 1839 wrpr %g3, %tl 1840 rdpr %tt, %g2 1841 wrpr %g5, %tl 1842 ! restore saved trap level 1843 wrpr %g1, %tpc 1844 wrpr %g6, %tstate 1845 wrpr %g7, %tt 1846#else /* sun4v */ 1847 /* user miss at tl>1. better be the window handler */ 1848 rdpr %tl, %g5 1849 sub %g5, 1, %g3 1850 wrpr %g3, %tl 1851 rdpr %tt, %g2 1852 wrpr %g5, %tl 1853#endif /* sun4v */ 1854 and %g2, WTRAP_TTMASK, %g4 1855 cmp %g4, WTRAP_TYPE 1856 bne,pn %xcc, 1f 1857 nop 1858 /* tpc should be in the trap table */ 1859 set trap_table, %g4 1860 cmp %g1, %g4 1861 blt,pn %xcc, 1f 1862 .empty 1863 set etrap_table, %g4 1864 cmp %g1, %g4 1865 bge,pn %xcc, 1f 1866 .empty 1867 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 1868 add %g1, WTRAP_FAULTOFF, %g1 1869 wrpr %g0, %g1, %tnpc 18707: 1871 /* 1872 * some wbuf handlers will call systrap to resolve the fault 1873 * we pass the trap type so they figure out the correct parameters. 1874 * g5 = trap type, g6 = tag access reg 1875 */ 1876 1877 /* 1878 * only use g5, g6, g7 registers after we have switched to alternate 1879 * globals. 1880 */ 1881 SET_GL_REG(1) 1882 USE_ALTERNATE_GLOBALS(%g5) 1883 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 1884 rdpr %tt, %g7 1885 cmp %g7, FAST_IMMU_MISS_TT 1886 be,a,pn %icc, ptl1_panic 1887 mov PTL1_BAD_WTRAP, %g1 1888 cmp %g7, T_INSTR_MMU_MISS 1889 be,a,pn %icc, ptl1_panic 1890 mov PTL1_BAD_WTRAP, %g1 1891 mov T_DATA_PROT, %g5 1892 cmp %g7, FAST_DMMU_MISS_TT 1893 move %icc, T_DATA_MMU_MISS, %g5 1894 cmp %g7, T_DATA_MMU_MISS 1895 move %icc, T_DATA_MMU_MISS, %g5 1896 ! XXXQ AGS re-check out this one 1897 done 18981: 1899 CPU_PADDR(%g1, %g4) 1900 add %g1, CPU_TL1_HDLR, %g1 1901 lda [%g1]ASI_MEM, %g4 1902 brnz,a,pt %g4, sfmmu_mmu_trap 1903 sta %g0, [%g1]ASI_MEM 1904 ba,pt %icc, ptl1_panic 1905 mov PTL1_BAD_TRAP, %g1 1906 SET_SIZE(sfmmu_window_trap) 1907 1908 ENTRY_NP(sfmmu_kpm_exception) 1909 /* 1910 * We have accessed an unmapped segkpm address or a legal segkpm 1911 * address which is involved in a VAC alias conflict prevention. 1912 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 1913 * set. If it is, we will instead note that a fault has occurred 1914 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 1915 * a "retry"). This will step over the faulting instruction. 1916 * Note that this means that a legal segkpm address involved in 1917 * a VAC alias conflict prevention (a rare case to begin with) 1918 * cannot be used in DTrace. 1919 */ 1920 CPU_INDEX(%g1, %g2) 1921 set cpu_core, %g2 1922 sllx %g1, CPU_CORE_SHIFT, %g1 1923 add %g1, %g2, %g1 1924 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 1925 andcc %g2, CPU_DTRACE_NOFAULT, %g0 1926 bz 0f 1927 or %g2, CPU_DTRACE_BADADDR, %g2 1928 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 1929 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 1930 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 1931 done 19320: 1933 TSTAT_CHECK_TL1(1f, %g1, %g2) 19341: 1935 SET_GL_REG(1) 1936 USE_ALTERNATE_GLOBALS(%g5) 1937 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 1938 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1939 /* 1940 * g2=tagacc g3.l=type g3.h=0 1941 */ 1942 sethi %hi(trap), %g1 1943 or %g1, %lo(trap), %g1 1944 ba,pt %xcc, sys_trap 1945 mov -1, %g4 1946 SET_SIZE(sfmmu_kpm_exception) 1947 1948#endif /* lint */ 1949 1950#if defined (lint) 1951 1952void 1953sfmmu_tsb_miss(void) 1954{ 1955} 1956 1957void 1958sfmmu_kpm_dtsb_miss(void) 1959{ 1960} 1961 1962void 1963sfmmu_kpm_dtsb_miss_small(void) 1964{ 1965} 1966 1967#else /* lint */ 1968 1969#if (IMAP_SEG != 0) 1970#error - ism_map->ism_seg offset is not zero 1971#endif 1972 1973/* 1974 * Copies ism mapping for this ctx in param "ism" if this is a ISM 1975 * tlb miss and branches to label "ismhit". If this is not an ISM 1976 * process or an ISM tlb miss it falls thru. 1977 * 1978 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 1979 * this process. 1980 * If so, it will branch to label "ismhit". If not, it will fall through. 1981 * 1982 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 1983 * so that any other threads of this process will not try and walk the ism 1984 * maps while they are being changed. 1985 * 1986 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 1987 * will make sure of that. This means we can terminate our search on 1988 * the first zero mapping we find. 1989 * 1990 * Parameters: 1991 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 1992 * tsbmiss = address of tsb miss area (in) 1993 * ismseg = contents of ism_seg for this ism map (out) 1994 * ismhat = physical address of imap_ismhat for this ism map (out) 1995 * tmp1 = scratch reg (CLOBBERED) 1996 * tmp2 = scratch reg (CLOBBERED) 1997 * tmp3 = scratch reg (CLOBBERED) 1998 * label: temporary labels 1999 * ismhit: label where to jump to if an ism dtlb miss 2000 * exitlabel:label where to jump if hat is busy due to hat_unshare. 2001 */ 2002#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 2003 label, ismhit) \ 2004 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 2005 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 2006 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 2007label/**/1: ;\ 2008 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 2009 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 2010label/**/2: ;\ 2011 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 2012 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 2013 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 2014 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 2015 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 2016 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 2017 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 2018 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 2019 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 2020 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 2021 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 2022 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 2023 ;\ 2024 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 2025 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 2026 cmp ismhat, tmp1 ;\ 2027 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 2028 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 2029 ;\ 2030 add tmp3, IBLK_NEXTPA, tmp1 ;\ 2031 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 2032 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 2033 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 2034label/**/3: 2035 2036/* 2037 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 2038 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 2039 * Parameters: 2040 * tagacc = reg containing virtual address 2041 * hatid = reg containing sfmmu pointer 2042 * hmeshift = constant/register to shift vaddr to obtain vapg 2043 * hmebp = register where bucket pointer will be stored 2044 * vapg = register where virtual page will be stored 2045 * tmp1, tmp2 = tmp registers 2046 */ 2047 2048 2049#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 2050 vapg, label, tmp1, tmp2) \ 2051 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 2052 brnz,a,pt tmp1, label/**/1 ;\ 2053 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 2054 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 2055 ba,pt %xcc, label/**/2 ;\ 2056 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 2057label/**/1: ;\ 2058 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 2059label/**/2: ;\ 2060 srlx tagacc, hmeshift, vapg ;\ 2061 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 2062 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 2063 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 2064 add hmebp, tmp1, hmebp 2065 2066/* 2067 * hashtag includes bspage + hashno (64 bits). 2068 */ 2069 2070#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 2071 sllx vapg, hmeshift, vapg ;\ 2072 mov hashno, hblktag ;\ 2073 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 2074 or vapg, hblktag, hblktag 2075 2076/* 2077 * Function to traverse hmeblk hash link list and find corresponding match. 2078 * The search is done using physical pointers. It returns the physical address 2079 * pointer to the hmeblk that matches with the tag provided. 2080 * Parameters: 2081 * hmebp = register that points to hme hash bucket, also used as 2082 * tmp reg (clobbered) 2083 * hmeblktag = register with hmeblk tag match 2084 * hatid = register with hatid 2085 * hmeblkpa = register where physical ptr will be stored 2086 * tmp1 = tmp reg 2087 * label: temporary label 2088 */ 2089 2090#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, tsbarea, \ 2091 tmp1, label) \ 2092 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 2093 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2094 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2095label/**/1: ;\ 2096 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2097 be,pn %xcc, label/**/2 ;\ 2098 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2099 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 2100 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2101 add hmebp, CLONGSIZE, hmebp ;\ 2102 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 2103 xor tmp1, hmeblktag, tmp1 ;\ 2104 xor hmebp, hatid, hmebp ;\ 2105 or hmebp, tmp1, hmebp ;\ 2106 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 2107 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 2108 ba,pt %xcc, label/**/1 ;\ 2109 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2110label/**/2: 2111 2112/* 2113 * Function to traverse hmeblk hash link list and find corresponding match. 2114 * The search is done using physical pointers. It returns the physical address 2115 * pointer to the hmeblk that matches with the tag 2116 * provided. 2117 * Parameters: 2118 * hmeblktag = register with hmeblk tag match (rid field is 0) 2119 * hatid = register with hatid (pointer to SRD) 2120 * hmeblkpa = register where physical ptr will be stored 2121 * tmp1 = tmp reg 2122 * tmp2 = tmp reg 2123 * label: temporary label 2124 */ 2125 2126#define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, tsbarea, \ 2127 tmp1, tmp2, label) \ 2128label/**/1: ;\ 2129 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2130 be,pn %xcc, label/**/4 ;\ 2131 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 2132 add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 2133 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2134 add tmp2, CLONGSIZE, tmp2 ;\ 2135 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 2136 xor tmp1, hmeblktag, tmp1 ;\ 2137 xor tmp2, hatid, tmp2 ;\ 2138 brz,pn tmp2, label/**/3 /* branch on hit */ ;\ 2139 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2140label/**/2: ;\ 2141 ba,pt %xcc, label/**/1 ;\ 2142 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2143label/**/3: ;\ 2144 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 2145 bgeu,pt %xcc, label/**/2 ;\ 2146 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2147 and tmp1, BT_ULMASK, tmp2 ;\ 2148 srlx tmp1, BT_ULSHIFT, tmp1 ;\ 2149 sllx tmp1, CLONGSHIFT, tmp1 ;\ 2150 add tsbarea, tmp1, tmp1 ;\ 2151 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 2152 srlx tmp1, tmp2, tmp1 ;\ 2153 btst 0x1, tmp1 ;\ 2154 bz,pn %xcc, label/**/2 ;\ 2155 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2156label/**/4: 2157 2158#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2159#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2160#endif 2161 2162/* 2163 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2164 * he offset for the corresponding hment. 2165 * Parameters: 2166 * In: 2167 * vaddr = register with virtual address 2168 * hmeblkpa = physical pointer to hme_blk 2169 * Out: 2170 * hmentoff = register where hment offset will be stored 2171 * hmemisc = hblk_misc 2172 * Scratch: 2173 * tmp1 2174 */ 2175#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2176 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2177 lda [hmentoff]ASI_MEM, hmemisc ;\ 2178 andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2179 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2180 or %g0, HMEBLK_HME1, hmentoff ;\ 2181 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2182 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2183 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2184 add tmp1, HMEBLK_HME1, hmentoff ;\ 2185label1: 2186 2187/* 2188 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2189 * 2190 * tagacc = (pseudo-)tag access register (in) 2191 * hatid = sfmmu pointer for TSB miss (in) 2192 * tte = tte for TLB miss if found, otherwise clobbered (out) 2193 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2194 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2195 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2196 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2197 * for this page size. 2198 * hashno = constant/register hash number 2199 * tmp = temp value - clobbered 2200 * label = temporary label for branching within macro. 2201 * foundlabel = label to jump to when tte is found. 2202 * suspendlabel= label to jump to when tte is suspended. 2203 * exitlabel = label to jump to when tte is not found. 2204 * 2205 */ 2206#define GET_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, hmeshift, \ 2207 hashno, tmp, label, foundlabel, suspendlabel, exitlabel) \ 2208 ;\ 2209 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2210 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2211 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2212 hmeblkpa, label/**/5, hmemisc, tmp) ;\ 2213 ;\ 2214 /* ;\ 2215 * tagacc = tagacc ;\ 2216 * hatid = hatid ;\ 2217 * tsbarea = tsbarea ;\ 2218 * tte = hmebp (hme bucket pointer) ;\ 2219 * hmeblkpa = vapg (virtual page) ;\ 2220 * hmemisc, tmp = scratch ;\ 2221 */ ;\ 2222 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2223 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2224 ;\ 2225 /* ;\ 2226 * tagacc = tagacc ;\ 2227 * hatid = hatid ;\ 2228 * tte = hmebp ;\ 2229 * hmeblkpa = CLOBBERED ;\ 2230 * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2231 * tmp = scratch ;\ 2232 */ ;\ 2233 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2234 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, \ 2235 tsbarea, tagacc, label/**/1) ;\ 2236 /* ;\ 2237 * tagacc = CLOBBERED ;\ 2238 * tte = CLOBBERED ;\ 2239 * hmeblkpa = hmeblkpa ;\ 2240 * tmp = scratch ;\ 2241 */ ;\ 2242 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2243 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\ 2244 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2245 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2246 nop ;\ 2247label/**/4: ;\ 2248 /* ;\ 2249 * We have found the hmeblk containing the hment. ;\ 2250 * Now we calculate the corresponding tte. ;\ 2251 * ;\ 2252 * tagacc = tagacc ;\ 2253 * hatid = hatid ;\ 2254 * tte = clobbered ;\ 2255 * hmeblkpa = hmeblkpa ;\ 2256 * hmemisc = hblktag ;\ 2257 * tmp = scratch ;\ 2258 */ ;\ 2259 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2260 label/**/2) ;\ 2261 ;\ 2262 /* ;\ 2263 * tagacc = tagacc ;\ 2264 * hatid = hmentoff ;\ 2265 * tte = clobbered ;\ 2266 * hmeblkpa = hmeblkpa ;\ 2267 * hmemisc = hblk_misc ;\ 2268 * tmp = scratch ;\ 2269 */ ;\ 2270 ;\ 2271 add hatid, SFHME_TTE, hatid ;\ 2272 add hmeblkpa, hatid, hmeblkpa ;\ 2273 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2274 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2275 set TTE_SUSPEND, hatid ;\ 2276 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2277 btst tte, hatid ;\ 2278 bz,pt %xcc, foundlabel ;\ 2279 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2280 ;\ 2281 /* ;\ 2282 * Mapping is suspended, so goto suspend label. ;\ 2283 */ ;\ 2284 ba,pt %xcc, suspendlabel ;\ 2285 nop 2286 2287/* 2288 * GET_SHME_TTE is similar to GET_TTE() except it searches 2289 * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2290 * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2291 * either 0 (not part of scd) or 1 (part of scd). 2292 */ 2293#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, \ 2294 hmeshift, hashno, tmp, label, foundlabel, \ 2295 suspendlabel, exitlabel) \ 2296 ;\ 2297 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2298 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2299 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2300 hmeblkpa, label/**/5, hmemisc, tmp) ;\ 2301 ;\ 2302 /* ;\ 2303 * tagacc = tagacc ;\ 2304 * hatid = hatid ;\ 2305 * tsbarea = tsbarea ;\ 2306 * tte = hmebp (hme bucket pointer) ;\ 2307 * hmeblkpa = vapg (virtual page) ;\ 2308 * hmemisc, tmp = scratch ;\ 2309 */ ;\ 2310 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2311 ;\ 2312 /* ;\ 2313 * tagacc = tagacc ;\ 2314 * hatid = hatid ;\ 2315 * tsbarea = tsbarea ;\ 2316 * tte = hmebp ;\ 2317 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2318 * hmeblkpa = CLOBBERED ;\ 2319 * tmp = scratch ;\ 2320 */ ;\ 2321 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2322 ;\ 2323 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2324 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2325 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2326 ;\ 2327label/**/8: ;\ 2328 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, \ 2329 tsbarea, tagacc, tte, label/**/1) ;\ 2330 /* ;\ 2331 * tagacc = CLOBBERED ;\ 2332 * tte = CLOBBERED ;\ 2333 * hmeblkpa = hmeblkpa ;\ 2334 * tmp = scratch ;\ 2335 */ ;\ 2336 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2337 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\ 2338 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2339 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2340 nop ;\ 2341label/**/4: ;\ 2342 /* ;\ 2343 * We have found the hmeblk containing the hment. ;\ 2344 * Now we calculate the corresponding tte. ;\ 2345 * ;\ 2346 * tagacc = tagacc ;\ 2347 * hatid = hatid ;\ 2348 * tte = clobbered ;\ 2349 * hmeblkpa = hmeblkpa ;\ 2350 * hmemisc = hblktag ;\ 2351 * tsbarea = tsbmiss area ;\ 2352 * tmp = scratch ;\ 2353 */ ;\ 2354 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2355 label/**/2) ;\ 2356 ;\ 2357 /* ;\ 2358 * tagacc = tagacc ;\ 2359 * hatid = hmentoff ;\ 2360 * tte = clobbered ;\ 2361 * hmeblkpa = hmeblkpa ;\ 2362 * hmemisc = hblk_misc ;\ 2363 * tsbarea = tsbmiss area ;\ 2364 * tmp = scratch ;\ 2365 */ ;\ 2366 ;\ 2367 add hatid, SFHME_TTE, hatid ;\ 2368 add hmeblkpa, hatid, hmeblkpa ;\ 2369 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2370 brlz,pt tte, label/**/6 ;\ 2371 nop ;\ 2372 btst HBLK_SZMASK, hmemisc ;\ 2373 bnz,a,pt %icc, label/**/7 ;\ 2374 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2375 ;\ 2376 /* ;\ 2377 * We found an invalid 8K tte in shme. ;\ 2378 * it may not belong to shme's region since ;\ 2379 * region size/alignment granularity is 8K but different ;\ 2380 * regions don't share hmeblks. Continue the search. ;\ 2381 */ ;\ 2382 sub hmeblkpa, hatid, hmeblkpa ;\ 2383 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2384 srlx tagacc, hmeshift, tte ;\ 2385 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2386 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2387 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2388 ba,a,pt %xcc, label/**/8 ;\ 2389label/**/6: ;\ 2390 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2391 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2392label/**/7: ;\ 2393 set TTE_SUSPEND, hatid ;\ 2394 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2395 btst tte, hatid ;\ 2396 bz,pt %xcc, foundlabel ;\ 2397 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2398 ;\ 2399 /* ;\ 2400 * Mapping is suspended, so goto suspend label. ;\ 2401 */ ;\ 2402 ba,pt %xcc, suspendlabel ;\ 2403 nop 2404 2405 /* 2406 * KERNEL PROTECTION HANDLER 2407 * 2408 * g1 = tsb8k pointer register (clobbered) 2409 * g2 = tag access register (ro) 2410 * g3 - g7 = scratch registers 2411 * 2412 * Note: This function is patched at runtime for performance reasons. 2413 * Any changes here require sfmmu_patch_ktsb fixed. 2414 */ 2415 ENTRY_NP(sfmmu_kprot_trap) 2416 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2417sfmmu_kprot_patch_ktsb_base: 2418 RUNTIME_PATCH_SETX(%g1, %g6) 2419 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2420sfmmu_kprot_patch_ktsb_szcode: 2421 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2422 2423 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2424 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2425 2426 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2427sfmmu_kprot_patch_ktsb4m_base: 2428 RUNTIME_PATCH_SETX(%g3, %g6) 2429 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2430sfmmu_kprot_patch_ktsb4m_szcode: 2431 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2432 2433 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2434 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2435 2436 CPU_TSBMISS_AREA(%g6, %g7) 2437 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2438 ba,pt %xcc, sfmmu_tsb_miss_tt 2439 nop 2440 2441 /* 2442 * USER PROTECTION HANDLER 2443 * 2444 * g1 = tsb8k pointer register (ro) 2445 * g2 = tag access register (ro) 2446 * g3 = faulting context (clobbered, currently not used) 2447 * g4 - g7 = scratch registers 2448 */ 2449 ALTENTRY(sfmmu_uprot_trap) 2450#ifdef sun4v 2451 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2452 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2453 2454 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2455 brlz,pt %g3, 9f /* check for 2nd TSB */ 2456 nop 2457 2458 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2459 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2460 2461#else /* sun4v */ 2462#ifdef UTSB_PHYS 2463 /* g1 = first TSB entry ptr */ 2464 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2465 brlz,pt %g3, 9f /* check for 2nd TSB */ 2466 nop 2467 2468 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2469 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2470#else /* UTSB_PHYS */ 2471 brgez,pt %g1, 9f /* check for 2nd TSB */ 2472 mov -1, %g3 /* set second tsbe ptr to -1 */ 2473 2474 mov %g2, %g7 2475 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2476 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2477 mov %g1, %g7 2478 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2479#endif /* UTSB_PHYS */ 2480#endif /* sun4v */ 24819: 2482 CPU_TSBMISS_AREA(%g6, %g7) 2483 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2484 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2485 nop 2486 2487 /* 2488 * Kernel 8K page iTLB miss. We also get here if we took a 2489 * fast instruction access mmu miss trap while running in 2490 * invalid context. 2491 * 2492 * %g1 = 8K TSB pointer register (not used, clobbered) 2493 * %g2 = tag access register (used) 2494 * %g3 = faulting context id (used) 2495 * %g7 = TSB tag to match (used) 2496 */ 2497 .align 64 2498 ALTENTRY(sfmmu_kitlb_miss) 2499 brnz,pn %g3, tsb_tl0_noctxt 2500 nop 2501 2502 /* kernel miss */ 2503 /* get kernel tsb pointer */ 2504 /* we patch the next set of instructions at run time */ 2505 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2506iktsbbase: 2507 RUNTIME_PATCH_SETX(%g4, %g5) 2508 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2509 2510iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2511 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2512 or %g4, %g1, %g1 ! form tsb ptr 2513 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2514 cmp %g4, %g7 2515 bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2516 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2517 2518 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2519 bz,pn %icc, exec_fault 2520 nop 2521 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2522 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2523 retry 2524 2525iktsb4mbase: 2526 RUNTIME_PATCH_SETX(%g4, %g6) 2527 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2528iktsb4m: 2529 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2530 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2531 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2532 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2533 cmp %g4, %g7 2534 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2535 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2536 bz,pn %icc, exec_fault 2537 nop 2538 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2539 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2540 retry 2541 2542 /* 2543 * Kernel dTLB miss. We also get here if we took a fast data 2544 * access mmu miss trap while running in invalid context. 2545 * 2546 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2547 * We select the TSB miss handler to branch to depending on 2548 * the virtual address of the access. In the future it may 2549 * be desirable to separate kpm TTEs into their own TSB, 2550 * in which case all that needs to be done is to set 2551 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2552 * early in the miss if we detect a kpm VA to a new handler. 2553 * 2554 * %g1 = 8K TSB pointer register (not used, clobbered) 2555 * %g2 = tag access register (used) 2556 * %g3 = faulting context id (used) 2557 */ 2558 .align 64 2559 ALTENTRY(sfmmu_kdtlb_miss) 2560 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2561 nop 2562 2563 /* Gather some stats for kpm misses in the TLB. */ 2564 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2565 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2566 2567 /* 2568 * Get first TSB offset and look for 8K/64K/512K mapping 2569 * using the 8K virtual page as the index. 2570 * 2571 * We patch the next set of instructions at run time; 2572 * any changes here require sfmmu_patch_ktsb changes too. 2573 */ 2574dktsbbase: 2575 RUNTIME_PATCH_SETX(%g7, %g6) 2576 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2577 2578dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2579 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2580 2581 /* 2582 * At this point %g1 is our index into the TSB. 2583 * We just masked off enough bits of the VA depending 2584 * on our TSB size code. 2585 */ 2586 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2587 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2588 cmp %g6, %g4 ! compare tag 2589 bne,pn %xcc, dktsb4m_kpmcheck_small 2590 add %g7, %g1, %g1 /* form tsb ptr */ 2591 TT_TRACE(trace_tsbhit) 2592 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2593 /* trapstat expects tte in %g5 */ 2594 retry 2595 2596 /* 2597 * If kpm is using large pages, the following instruction needs 2598 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2599 * so that we will probe the 4M TSB regardless of the VA. In 2600 * the case kpm is using small pages, we know no large kernel 2601 * mappings are located above 0x80000000.00000000 so we skip the 2602 * probe as an optimization. 2603 */ 2604dktsb4m_kpmcheck_small: 2605 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2606 /* delay slot safe, below */ 2607 2608 /* 2609 * Get second TSB offset and look for 4M mapping 2610 * using 4M virtual page as the TSB index. 2611 * 2612 * Here: 2613 * %g1 = 8K TSB pointer. Don't squash it. 2614 * %g2 = tag access register (we still need it) 2615 */ 2616 srlx %g2, MMU_PAGESHIFT4M, %g3 2617 2618 /* 2619 * We patch the next set of instructions at run time; 2620 * any changes here require sfmmu_patch_ktsb changes too. 2621 */ 2622dktsb4mbase: 2623 RUNTIME_PATCH_SETX(%g7, %g6) 2624 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2625dktsb4m: 2626 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2627 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2628 2629 /* 2630 * At this point %g3 is our index into the TSB. 2631 * We just masked off enough bits of the VA depending 2632 * on our TSB size code. 2633 */ 2634 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2635 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2636 cmp %g6, %g4 ! compare tag 2637 2638dktsb4m_tsbmiss: 2639 bne,pn %xcc, dktsb4m_kpmcheck 2640 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2641 TT_TRACE(trace_tsbhit) 2642 /* we don't check TTE size here since we assume 4M TSB is separate */ 2643 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2644 /* trapstat expects tte in %g5 */ 2645 retry 2646 2647 /* 2648 * So, we failed to find a valid TTE to match the faulting 2649 * address in either TSB. There are a few cases that could land 2650 * us here: 2651 * 2652 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2653 * to sfmmu_tsb_miss_tt to handle the miss. 2654 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2655 * 4M TSB. Let segkpm handle it. 2656 * 2657 * Note that we shouldn't land here in the case of a kpm VA when 2658 * kpm_smallpages is active -- we handled that case earlier at 2659 * dktsb4m_kpmcheck_small. 2660 * 2661 * At this point: 2662 * g1 = 8K-indexed primary TSB pointer 2663 * g2 = tag access register 2664 * g3 = 4M-indexed secondary TSB pointer 2665 */ 2666dktsb4m_kpmcheck: 2667 cmp %g2, %g0 2668 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2669 nop 2670 ba,a,pt %icc, sfmmu_tsb_miss_tt 2671 nop 2672 2673#ifdef sun4v 2674 /* 2675 * User instruction miss w/ single TSB. 2676 * The first probe covers 8K, 64K, and 512K page sizes, 2677 * because 64K and 512K mappings are replicated off 8K 2678 * pointer. 2679 * 2680 * g1 = tsb8k pointer register 2681 * g2 = tag access register 2682 * g3 - g6 = scratch registers 2683 * g7 = TSB tag to match 2684 */ 2685 .align 64 2686 ALTENTRY(sfmmu_uitlb_fastpath) 2687 2688 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2689 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2690 ba,pn %xcc, sfmmu_tsb_miss_tt 2691 mov -1, %g3 2692 2693 /* 2694 * User data miss w/ single TSB. 2695 * The first probe covers 8K, 64K, and 512K page sizes, 2696 * because 64K and 512K mappings are replicated off 8K 2697 * pointer. 2698 * 2699 * g1 = tsb8k pointer register 2700 * g2 = tag access register 2701 * g3 - g6 = scratch registers 2702 * g7 = TSB tag to match 2703 */ 2704 .align 64 2705 ALTENTRY(sfmmu_udtlb_fastpath) 2706 2707 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2708 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2709 ba,pn %xcc, sfmmu_tsb_miss_tt 2710 mov -1, %g3 2711 2712 /* 2713 * User instruction miss w/ multiple TSBs (sun4v). 2714 * The first probe covers 8K, 64K, and 512K page sizes, 2715 * because 64K and 512K mappings are replicated off 8K 2716 * pointer. Second probe covers 4M page size only. 2717 * 2718 * Just like sfmmu_udtlb_slowpath, except: 2719 * o Uses ASI_ITLB_IN 2720 * o checks for execute permission 2721 * o No ISM prediction. 2722 * 2723 * g1 = tsb8k pointer register 2724 * g2 = tag access register 2725 * g3 - g6 = scratch registers 2726 * g7 = TSB tag to match 2727 */ 2728 .align 64 2729 ALTENTRY(sfmmu_uitlb_slowpath) 2730 2731 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2732 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2733 /* g4 - g5 = clobbered here */ 2734 2735 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2736 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2737 srlx %g2, TAG_VALO_SHIFT, %g7 2738 PROBE_2ND_ITSB(%g3, %g7) 2739 /* NOT REACHED */ 2740 2741#else /* sun4v */ 2742 2743 /* 2744 * User instruction miss w/ multiple TSBs (sun4u). 2745 * The first probe covers 8K, 64K, and 512K page sizes, 2746 * because 64K and 512K mappings are replicated off 8K 2747 * pointer. Probe of 1st TSB has already been done prior to entry 2748 * into this routine. For the UTSB_PHYS case we probe up to 3 2749 * valid other TSBs in the following order: 2750 * 1) shared TSB for 4M-256M pages 2751 * 2) private TSB for 4M-256M pages 2752 * 3) shared TSB for 8K-512K pages 2753 * 2754 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs 2755 * 4M-256M pages. 2756 * 2757 * Just like sfmmu_udtlb_slowpath, except: 2758 * o Uses ASI_ITLB_IN 2759 * o checks for execute permission 2760 * o No ISM prediction. 2761 * 2762 * g1 = tsb8k pointer register 2763 * g2 = tag access register 2764 * g4 - g6 = scratch registers 2765 * g7 = TSB tag to match 2766 */ 2767 .align 64 2768 ALTENTRY(sfmmu_uitlb_slowpath) 2769 2770#ifdef UTSB_PHYS 2771 2772 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2773 brlz,pt %g6, 1f 2774 nop 2775 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2776 PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail) 27771: 2778 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2779 brlz,pt %g3, 2f 2780 nop 2781 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2782 PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail) 27832: 2784 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2785 brlz,pt %g6, sfmmu_tsb_miss_tt 2786 nop 2787 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2788 PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail) 2789 ba,pn %xcc, sfmmu_tsb_miss_tt 2790 nop 2791 2792#else /* UTSB_PHYS */ 2793 mov %g1, %g3 /* save tsb8k reg in %g3 */ 2794 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 2795 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2796 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 2797 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 2798 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 2799 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2800 srlx %g2, TAG_VALO_SHIFT, %g7 2801 PROBE_2ND_ITSB(%g3, %g7, isynth) 2802 ba,pn %xcc, sfmmu_tsb_miss_tt 2803 nop 2804 2805#endif /* UTSB_PHYS */ 2806#endif /* sun4v */ 2807 2808#if defined(sun4u) && defined(UTSB_PHYS) 2809 2810 /* 2811 * We come here for ism predict DTLB_MISS case or if 2812 * if probe in first TSB failed. 2813 */ 2814 2815 .align 64 2816 ALTENTRY(sfmmu_udtlb_slowpath_noismpred) 2817 2818 /* 2819 * g1 = tsb8k pointer register 2820 * g2 = tag access register 2821 * g4 - %g6 = scratch registers 2822 * g7 = TSB tag to match 2823 */ 2824 2825 /* 2826 * ISM non-predict probe order 2827 * probe 1ST_TSB (8K index) 2828 * probe 2ND_TSB (4M index) 2829 * probe 4TH_TSB (4M index) 2830 * probe 3RD_TSB (8K index) 2831 * 2832 * We already probed first TSB in DTLB_MISS handler. 2833 */ 2834 2835 /* 2836 * Private 2ND TSB 4M-256 pages 2837 */ 2838 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2839 brlz,pt %g3, 1f 2840 nop 2841 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2842 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2843 2844 /* 2845 * Shared Context 4TH TSB 4M-256 pages 2846 */ 28471: 2848 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2849 brlz,pt %g6, 2f 2850 nop 2851 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2852 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail) 2853 2854 /* 2855 * Shared Context 3RD TSB 8K-512K pages 2856 */ 28572: 2858 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2859 brlz,pt %g6, sfmmu_tsb_miss_tt 2860 nop 2861 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2862 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail) 2863 ba,pn %xcc, sfmmu_tsb_miss_tt 2864 nop 2865 2866 .align 64 2867 ALTENTRY(sfmmu_udtlb_slowpath_ismpred) 2868 2869 /* 2870 * g1 = tsb8k pointer register 2871 * g2 = tag access register 2872 * g4 - g6 = scratch registers 2873 * g7 = TSB tag to match 2874 */ 2875 2876 /* 2877 * ISM predict probe order 2878 * probe 4TH_TSB (4M index) 2879 * probe 2ND_TSB (4M index) 2880 * probe 1ST_TSB (8K index) 2881 * probe 3RD_TSB (8K index) 2882 2883 /* 2884 * Shared Context 4TH TSB 4M-256 pages 2885 */ 2886 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2887 brlz,pt %g6, 4f 2888 nop 2889 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2890 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2) 2891 2892 /* 2893 * Private 2ND TSB 4M-256 pages 2894 */ 28954: 2896 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2897 brlz,pt %g3, 5f 2898 nop 2899 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2900 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2) 2901 29025: 2903 PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2) 2904 2905 /* 2906 * Shared Context 3RD TSB 8K-512K pages 2907 */ 2908 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2909 brlz,pt %g6, 6f 2910 nop 2911 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2912 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2) 29136: 2914 ba,pn %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */ 2915 nop 2916 2917#else /* sun4u && UTSB_PHYS */ 2918 2919 .align 64 2920 ALTENTRY(sfmmu_udtlb_slowpath) 2921 2922 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 2923 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 2924 mov %g1, %g3 2925 2926udtlb_miss_probefirst: 2927 /* 2928 * g1 = 8K TSB pointer register 2929 * g2 = tag access register 2930 * g3 = (potentially) second TSB entry ptr 2931 * g6 = ism pred. 2932 * g7 = vpg_4m 2933 */ 2934#ifdef sun4v 2935 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2936 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2937 2938 /* 2939 * Here: 2940 * g1 = first TSB pointer 2941 * g2 = tag access reg 2942 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2943 */ 2944 brgz,pn %g6, sfmmu_tsb_miss_tt 2945 nop 2946#else /* sun4v */ 2947 mov %g1, %g4 2948 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 2949 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2950 2951 /* 2952 * Here: 2953 * g1 = first TSB pointer 2954 * g2 = tag access reg 2955 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2956 */ 2957 brgz,pn %g6, sfmmu_tsb_miss_tt 2958 nop 2959 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 2960 /* fall through in 8K->4M probe order */ 2961#endif /* sun4v */ 2962 2963udtlb_miss_probesecond: 2964 /* 2965 * Look in the second TSB for the TTE 2966 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 2967 * g2 = tag access reg 2968 * g3 = 8K TSB pointer register 2969 * g6 = ism pred. 2970 * g7 = vpg_4m 2971 */ 2972#ifdef sun4v 2973 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 2974 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2975 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 2976#else /* sun4v */ 2977 mov %g3, %g7 2978 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 2979 /* %g2 clobbered, %g3 =second tsbe ptr */ 2980 mov MMU_TAG_ACCESS, %g2 2981 ldxa [%g2]ASI_DMMU, %g2 2982#endif /* sun4v */ 2983 2984 srlx %g2, TAG_VALO_SHIFT, %g7 2985 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2986 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 2987 brgz,pn %g6, udtlb_miss_probefirst 2988 nop 2989 2990 /* fall through to sfmmu_tsb_miss_tt */ 2991#endif /* sun4u && UTSB_PHYS */ 2992 2993 2994 ALTENTRY(sfmmu_tsb_miss_tt) 2995 TT_TRACE(trace_tsbmiss) 2996 /* 2997 * We get here if there is a TSB miss OR a write protect trap. 2998 * 2999 * g1 = First TSB entry pointer 3000 * g2 = tag access register 3001 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 3002 * g4 - g7 = scratch registers 3003 */ 3004 3005 ALTENTRY(sfmmu_tsb_miss) 3006 3007 /* 3008 * If trapstat is running, we need to shift the %tpc and %tnpc to 3009 * point to trapstat's TSB miss return code (note that trapstat 3010 * itself will patch the correct offset to add). 3011 */ 3012 rdpr %tl, %g7 3013 cmp %g7, 1 3014 ble,pt %xcc, 0f 3015 sethi %hi(KERNELBASE), %g6 3016 rdpr %tpc, %g7 3017 or %g6, %lo(KERNELBASE), %g6 3018 cmp %g7, %g6 3019 bgeu,pt %xcc, 0f 3020 /* delay slot safe */ 3021 3022 ALTENTRY(tsbmiss_trapstat_patch_point) 3023 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 3024 wrpr %g7, %tpc 3025 add %g7, 4, %g7 3026 wrpr %g7, %tnpc 30270: 3028 CPU_TSBMISS_AREA(%g6, %g7) 3029 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save 1ST tsb pointer */ 3030 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save 2ND tsb pointer */ 3031 3032 sllx %g2, TAGACC_CTX_LSHIFT, %g3 3033 brz,a,pn %g3, 1f /* skip ahead if kernel */ 3034 ldn [%g6 + TSBMISS_KHATID], %g7 3035 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 3036 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 3037 3038 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 3039 3040 cmp %g3, INVALID_CONTEXT 3041 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 3042 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 3043 3044#if defined(sun4v) || defined(UTSB_PHYS) 3045 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 3046 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 3047 stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 3048#endif /* sun4v || UTSB_PHYS */ 3049 3050 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 3051 /* 3052 * The miss wasn't in an ISM segment. 3053 * 3054 * %g1 %g3, %g4, %g5, %g7 all clobbered 3055 * %g2 = (pseudo) tag access 3056 */ 3057 3058 ba,pt %icc, 2f 3059 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 3060 30611: 3062 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 3063 /* 3064 * 8K and 64K hash. 3065 */ 30662: 3067 3068 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3069 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_l8K, tsb_checktte, 3070 sfmmu_suspend_tl, tsb_512K) 3071 /* NOT REACHED */ 3072 3073tsb_512K: 3074 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3075 brz,pn %g5, 3f 3076 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3077 and %g4, HAT_512K_FLAG, %g5 3078 3079 /* 3080 * Note that there is a small window here where we may have 3081 * a 512k page in the hash list but have not set the HAT_512K_FLAG 3082 * flag yet, so we will skip searching the 512k hash list. 3083 * In this case we will end up in pagefault which will find 3084 * the mapping and return. So, in this instance we will end up 3085 * spending a bit more time resolving this TSB miss, but it can 3086 * only happen once per process and even then, the chances of that 3087 * are very small, so it's not worth the extra overhead it would 3088 * take to close this window. 3089 */ 3090 brz,pn %g5, tsb_4M 3091 nop 30923: 3093 /* 3094 * 512K hash 3095 */ 3096 3097 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3098 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_l512K, tsb_checktte, 3099 sfmmu_suspend_tl, tsb_4M) 3100 /* NOT REACHED */ 3101 3102tsb_4M: 3103 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3104 brz,pn %g5, 4f 3105 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3106 and %g4, HAT_4M_FLAG, %g5 3107 brz,pn %g5, tsb_32M 3108 nop 31094: 3110 /* 3111 * 4M hash 3112 */ 3113 3114 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3115 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_l4M, tsb_checktte, 3116 sfmmu_suspend_tl, tsb_32M) 3117 /* NOT REACHED */ 3118 3119tsb_32M: 3120 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3121#ifdef sun4v 3122 brz,pn %g5, 6f 3123#else 3124 brz,pn %g5, tsb_pagefault 3125#endif 3126 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3127 and %g4, HAT_32M_FLAG, %g5 3128 brz,pn %g5, tsb_256M 3129 nop 31305: 3131 /* 3132 * 32M hash 3133 */ 3134 3135 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3136 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_l32M, tsb_checktte, 3137 sfmmu_suspend_tl, tsb_256M) 3138 /* NOT REACHED */ 3139 3140#if defined(sun4u) && !defined(UTSB_PHYS) 3141#define tsb_shme tsb_pagefault 3142#endif 3143tsb_256M: 3144 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3145 and %g4, HAT_256M_FLAG, %g5 3146 brz,pn %g5, tsb_shme 3147 nop 31486: 3149 /* 3150 * 256M hash 3151 */ 3152 3153 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3154 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_l256M, tsb_checktte, 3155 sfmmu_suspend_tl, tsb_shme) 3156 /* NOT REACHED */ 3157 3158tsb_checktte: 3159 /* 3160 * g1 = hblk_misc 3161 * g2 = tagacc 3162 * g3 = tte 3163 * g4 = tte pa 3164 * g6 = tsbmiss area 3165 * g7 = hatid 3166 */ 3167 brlz,a,pt %g3, tsb_validtte 3168 rdpr %tt, %g7 3169 3170#if defined(sun4u) && !defined(UTSB_PHYS) 3171#undef tsb_shme 3172 ba tsb_pagefault 3173 nop 3174#else /* sun4u && !UTSB_PHYS */ 3175 3176tsb_shme: 3177 /* 3178 * g2 = tagacc 3179 * g6 = tsbmiss area 3180 */ 3181 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3182 brz,pn %g5, tsb_pagefault 3183 nop 3184 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3185 brz,pn %g7, tsb_pagefault 3186 nop 3187 3188 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3189 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_shme_l8K, tsb_shme_checktte, 3190 sfmmu_suspend_tl, tsb_shme_512K) 3191 /* NOT REACHED */ 3192 3193tsb_shme_512K: 3194 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3195 and %g4, HAT_512K_FLAG, %g5 3196 brz,pn %g5, tsb_shme_4M 3197 nop 3198 3199 /* 3200 * 512K hash 3201 */ 3202 3203 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3204 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_shme_l512K, tsb_shme_checktte, 3205 sfmmu_suspend_tl, tsb_shme_4M) 3206 /* NOT REACHED */ 3207 3208tsb_shme_4M: 3209 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3210 and %g4, HAT_4M_FLAG, %g5 3211 brz,pn %g5, tsb_shme_32M 3212 nop 32134: 3214 /* 3215 * 4M hash 3216 */ 3217 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3218 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_shme_l4M, tsb_shme_checktte, 3219 sfmmu_suspend_tl, tsb_shme_32M) 3220 /* NOT REACHED */ 3221 3222tsb_shme_32M: 3223 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3224 and %g4, HAT_32M_FLAG, %g5 3225 brz,pn %g5, tsb_shme_256M 3226 nop 3227 3228 /* 3229 * 32M hash 3230 */ 3231 3232 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3233 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_shme_l32M, tsb_shme_checktte, 3234 sfmmu_suspend_tl, tsb_shme_256M) 3235 /* NOT REACHED */ 3236 3237tsb_shme_256M: 3238 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3239 and %g4, HAT_256M_FLAG, %g5 3240 brz,pn %g5, tsb_pagefault 3241 nop 3242 3243 /* 3244 * 256M hash 3245 */ 3246 3247 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3248 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_shme_l256M, tsb_shme_checktte, 3249 sfmmu_suspend_tl, tsb_pagefault) 3250 /* NOT REACHED */ 3251 3252tsb_shme_checktte: 3253 3254 brgez,pn %g3, tsb_pagefault 3255 rdpr %tt, %g7 3256 /* 3257 * g1 = ctx1 flag 3258 * g3 = tte 3259 * g4 = tte pa 3260 * g6 = tsbmiss area 3261 * g7 = tt 3262 */ 3263 3264 brz,pt %g1, tsb_validtte 3265 nop 3266 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3267 or %g1, HAT_CHKCTX1_FLAG, %g1 3268 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3269 3270 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3271#endif /* sun4u && !UTSB_PHYS */ 3272 3273tsb_validtte: 3274 /* 3275 * g3 = tte 3276 * g4 = tte pa 3277 * g6 = tsbmiss area 3278 * g7 = tt 3279 */ 3280 3281 /* 3282 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3283 */ 3284 cmp %g7, FAST_PROT_TT 3285 bne,pt %icc, 4f 3286 nop 3287 3288 TTE_SET_REFMOD_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_refmod, 3289 tsb_protfault) 3290 3291 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3292#ifdef sun4v 3293 MMU_FAULT_STATUS_AREA(%g7) 3294 ldx [%g7 + MMFSA_D_ADDR], %g5 /* load fault addr for later */ 3295#else /* sun4v */ 3296 mov MMU_TAG_ACCESS, %g5 3297 ldxa [%g5]ASI_DMMU, %g5 3298#endif /* sun4v */ 3299 ba,pt %xcc, tsb_update_tl1 3300 nop 33014: 3302 /* 3303 * If ITLB miss check exec bit. 3304 * If not set treat as invalid TTE. 3305 */ 3306 cmp %g7, T_INSTR_MMU_MISS 3307 be,pn %icc, 5f 3308 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3309 cmp %g7, FAST_IMMU_MISS_TT 3310 bne,pt %icc, 3f 3311 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 33125: 3313 bz,pn %icc, tsb_protfault 3314 nop 3315 33163: 3317 /* 3318 * Set reference bit if not already set 3319 */ 3320 TTE_SET_REF_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_ref) 3321 3322 /* 3323 * Now, load into TSB/TLB. At this point: 3324 * g3 = tte 3325 * g4 = patte 3326 * g6 = tsbmiss area 3327 */ 3328 rdpr %tt, %g7 3329#ifdef sun4v 3330 MMU_FAULT_STATUS_AREA(%g2) 3331 cmp %g7, T_INSTR_MMU_MISS 3332 be,a,pt %icc, 9f 3333 nop 3334 cmp %g7, FAST_IMMU_MISS_TT 3335 be,a,pt %icc, 9f 3336 nop 3337 add %g2, MMFSA_D_, %g2 33389: 3339 ldx [%g2 + MMFSA_CTX_], %g7 3340 sllx %g7, TTARGET_CTX_SHIFT, %g7 3341 ldx [%g2 + MMFSA_ADDR_], %g2 3342 mov %g2, %g5 ! load the fault addr for later use 3343 srlx %g2, TTARGET_VA_SHIFT, %g2 3344 or %g2, %g7, %g2 3345#else /* sun4v */ 3346 mov MMU_TAG_ACCESS, %g5 3347 cmp %g7, FAST_IMMU_MISS_TT 3348 be,a,pt %icc, 9f 3349 ldxa [%g0]ASI_IMMU, %g2 3350 ldxa [%g0]ASI_DMMU, %g2 3351 ba,pt %icc, tsb_update_tl1 3352 ldxa [%g5]ASI_DMMU, %g5 33539: 3354 ldxa [%g5]ASI_IMMU, %g5 3355#endif /* sun4v */ 3356 3357tsb_update_tl1: 3358 srlx %g2, TTARGET_CTX_SHIFT, %g7 3359 brz,pn %g7, tsb_kernel 3360#ifdef sun4v 3361 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3362#else /* sun4v */ 3363 srlx %g3, TTE_SZ_SHFT, %g7 3364#endif /* sun4v */ 3365 3366tsb_user: 3367#ifdef sun4v 3368 cmp %g7, TTE4M 3369 bge,pn %icc, tsb_user4m 3370 nop 3371#else /* sun4v */ 3372 cmp %g7, TTESZ_VALID | TTE4M 3373 be,pn %icc, tsb_user4m 3374 srlx %g3, TTE_SZ2_SHFT, %g7 3375 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3376#ifdef ITLB_32M_256M_SUPPORT 3377 bnz,pn %icc, tsb_user4m 3378 nop 3379#else /* ITLB_32M_256M_SUPPORT */ 3380 bnz,a,pn %icc, tsb_user_pn_synth 3381 nop 3382#endif /* ITLB_32M_256M_SUPPORT */ 3383#endif /* sun4v */ 3384 3385tsb_user8k: 3386#if defined(sun4v) || defined(UTSB_PHYS) 3387 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3388 and %g7, HAT_CHKCTX1_FLAG, %g1 3389 brz,a,pn %g1, 1f 3390 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3391 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3392 brlz,a,pn %g1, ptl1_panic ! if no shared 3RD tsb 3393 mov PTL1_NO_SCDTSB8K, %g1 ! panic 3394 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 33951: 3396#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3397 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3398#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3399 3400#ifndef UTSB_PHYS 3401 mov ASI_N, %g7 ! user TSBs accessed by VA 3402 mov %g7, %asi 3403#endif /* !UTSB_PHYS */ 3404 3405 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l3) 3406 3407 rdpr %tt, %g5 3408#ifdef sun4v 3409 cmp %g5, T_INSTR_MMU_MISS 3410 be,a,pn %xcc, 9f 3411 mov %g3, %g5 3412#endif /* sun4v */ 3413 cmp %g5, FAST_IMMU_MISS_TT 3414 be,pn %xcc, 9f 3415 mov %g3, %g5 3416 3417 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3418 ! trapstat wants TTE in %g5 3419 retry 34209: 3421 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3422 ! trapstat wants TTE in %g5 3423 retry 3424 3425tsb_user4m: 3426#if defined(sun4v) || defined(UTSB_PHYS) 3427 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3428 and %g7, HAT_CHKCTX1_FLAG, %g1 3429 brz,a,pn %g1, 4f 3430 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3431 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr 3432 brlz,a,pn %g1, 5f ! if no shared 4TH TSB 3433 nop 3434 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 3435 3436#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3437 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3438#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 34394: 3440 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3441 nop 3442 3443#ifndef UTSB_PHYS 3444 mov ASI_N, %g7 ! user TSBs accessed by VA 3445 mov %g7, %asi 3446#endif /* UTSB_PHYS */ 3447 3448 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l4) 3449 34505: 3451 rdpr %tt, %g5 3452#ifdef sun4v 3453 cmp %g5, T_INSTR_MMU_MISS 3454 be,a,pn %xcc, 9f 3455 mov %g3, %g5 3456#endif /* sun4v */ 3457 cmp %g5, FAST_IMMU_MISS_TT 3458 be,pn %xcc, 9f 3459 mov %g3, %g5 3460 3461 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3462 ! trapstat wants TTE in %g5 3463 retry 34649: 3465 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3466 ! trapstat wants TTE in %g5 3467 retry 3468 3469#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3470 /* 3471 * Panther ITLB synthesis. 3472 * The Panther 32M and 256M ITLB code simulates these two large page 3473 * sizes with 4M pages, to provide support for programs, for example 3474 * Java, that may copy instructions into a 32M or 256M data page and 3475 * then execute them. The code below generates the 4M pfn bits and 3476 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3477 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3478 * are ignored by the hardware. 3479 * 3480 * Now, load into TSB/TLB. At this point: 3481 * g2 = tagtarget 3482 * g3 = tte 3483 * g4 = patte 3484 * g5 = tt 3485 * g6 = tsbmiss area 3486 */ 3487tsb_user_pn_synth: 3488 rdpr %tt, %g5 3489 cmp %g5, FAST_IMMU_MISS_TT 3490 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3491 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3492 bz,pn %icc, 4b /* if not, been here before */ 3493 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3494 brlz,a,pn %g1, 5f /* no 2nd tsb */ 3495 mov %g3, %g5 3496 3497 mov MMU_TAG_ACCESS, %g7 3498 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3499 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3500 3501 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3502 mov %g7, %asi 3503 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l5) /* update TSB */ 35045: 3505 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3506 retry 3507 3508tsb_user_itlb_synth: 3509 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = 2ND TSB */ 3510 3511 mov MMU_TAG_ACCESS, %g7 3512 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3513 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3514 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3515 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3516 3517 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3518 mov %g7, %asi 3519 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l6) /* update TSB */ 35207: 3521 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3522 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3523 retry 3524#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3525 3526tsb_kernel: 3527 rdpr %tt, %g5 3528#ifdef sun4v 3529 cmp %g7, TTE4M 3530 bge,pn %icc, 5f 3531#else 3532 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3533 be,pn %icc, 5f 3534#endif /* sun4v */ 3535 nop 3536 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8K TSB ptr 3537 ba,pt %xcc, 6f 3538 nop 35395: 3540 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4M TSB ptr 3541 brlz,pn %g1, 3f /* skip programming if 4M TSB ptr is -1 */ 3542 nop 35436: 3544#ifndef sun4v 3545tsb_kernel_patch_asi: 3546 or %g0, RUNTIME_PATCH, %g6 3547 mov %g6, %asi ! XXX avoid writing to %asi !! 3548#endif 3549 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l7) 35503: 3551#ifdef sun4v 3552 cmp %g5, T_INSTR_MMU_MISS 3553 be,a,pn %icc, 1f 3554 mov %g3, %g5 ! trapstat wants TTE in %g5 3555#endif /* sun4v */ 3556 cmp %g5, FAST_IMMU_MISS_TT 3557 be,pn %icc, 1f 3558 mov %g3, %g5 ! trapstat wants TTE in %g5 3559 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3560 ! trapstat wants TTE in %g5 3561 retry 35621: 3563 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3564 ! trapstat wants TTE in %g5 3565 retry 3566 3567tsb_ism: 3568 /* 3569 * This is an ISM [i|d]tlb miss. We optimize for largest 3570 * page size down to smallest. 3571 * 3572 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3573 * register 3574 * g3 = ismmap->ism_seg 3575 * g4 = physical address of ismmap->ism_sfmmu 3576 * g6 = tsbmiss area 3577 */ 3578 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3579 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3580 mov PTL1_BAD_ISM, %g1 3581 /* g5 = pa of imap_vb_shift */ 3582 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3583 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3584 srlx %g3, %g4, %g3 /* clr size field */ 3585 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3586 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3587 and %g2, %g1, %g4 /* g4 = ctx number */ 3588 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3589 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3590 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3591 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3592 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3593#if defined(sun4v) || defined(UTSB_PHYS) 3594 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3595 brz,pt %g5, tsb_chk4M_ism 3596 nop 3597 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3598 or %g5, HAT_CHKCTX1_FLAG, %g5 3599 stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3600 rdpr %tt, %g5 3601 SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl) 3602#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3603 3604 /* 3605 * ISM pages are always locked down. 3606 * If we can't find the tte then pagefault 3607 * and let the spt segment driver resolve it. 3608 * 3609 * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3610 * g4 = imap_hatflags 3611 * g6 = tsb miss area 3612 * g7 = ISM hatid 3613 */ 3614 3615tsb_chk4M_ism: 3616 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3617 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3618 nop 3619 3620tsb_ism_32M: 3621 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3622 brz,pn %g5, tsb_ism_256M 3623 nop 3624 3625 /* 3626 * 32M hash. 3627 */ 3628 3629 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT32M, 3630 TTE32M, %g5, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3631 tsb_ism_4M) 3632 /* NOT REACHED */ 3633 3634tsb_ism_32M_found: 3635 brlz,a,pt %g3, tsb_validtte 3636 rdpr %tt, %g7 3637 ba,pt %xcc, tsb_ism_4M 3638 nop 3639 3640tsb_ism_256M: 3641 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3642 brz,a,pn %g5, ptl1_panic 3643 mov PTL1_BAD_ISM, %g1 3644 3645 /* 3646 * 256M hash. 3647 */ 3648 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT256M, 3649 TTE256M, %g5, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3650 tsb_ism_4M) 3651 3652tsb_ism_256M_found: 3653 brlz,a,pt %g3, tsb_validtte 3654 rdpr %tt, %g7 3655 3656tsb_ism_4M: 3657 /* 3658 * 4M hash. 3659 */ 3660 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT4M, 3661 TTE4M, %g5, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3662 tsb_ism_8K) 3663 /* NOT REACHED */ 3664 3665tsb_ism_4M_found: 3666 brlz,a,pt %g3, tsb_validtte 3667 rdpr %tt, %g7 3668 3669tsb_ism_8K: 3670 /* 3671 * 8K and 64K hash. 3672 */ 3673 3674 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT64K, 3675 TTE64K, %g5, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3676 tsb_pagefault) 3677 /* NOT REACHED */ 3678 3679tsb_ism_8K_found: 3680 brlz,a,pt %g3, tsb_validtte 3681 rdpr %tt, %g7 3682 3683tsb_pagefault: 3684 rdpr %tt, %g7 3685 cmp %g7, FAST_PROT_TT 3686 be,a,pn %icc, tsb_protfault 3687 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3688 3689tsb_protfault: 3690 /* 3691 * we get here if we couldn't find a valid tte in the hash. 3692 * 3693 * If user and we are at tl>1 we go to window handling code. 3694 * 3695 * If kernel and the fault is on the same page as our stack 3696 * pointer, then we know the stack is bad and the trap handler 3697 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3698 * 3699 * If this is a kernel trap and tl>1, panic. 3700 * 3701 * Otherwise we call pagefault. 3702 */ 3703 cmp %g7, FAST_IMMU_MISS_TT 3704#ifdef sun4v 3705 MMU_FAULT_STATUS_AREA(%g4) 3706 ldx [%g4 + MMFSA_I_CTX], %g5 3707 ldx [%g4 + MMFSA_D_CTX], %g4 3708 move %icc, %g5, %g4 3709 cmp %g7, T_INSTR_MMU_MISS 3710 move %icc, %g5, %g4 3711#else 3712 mov MMU_TAG_ACCESS, %g4 3713 ldxa [%g4]ASI_DMMU, %g2 3714 ldxa [%g4]ASI_IMMU, %g5 3715 move %icc, %g5, %g2 3716 cmp %g7, T_INSTR_MMU_MISS 3717 move %icc, %g5, %g2 3718 sllx %g2, TAGACC_CTX_LSHIFT, %g4 3719#endif /* sun4v */ 3720 brnz,pn %g4, 3f /* skip if not kernel */ 3721 rdpr %tl, %g5 3722 3723 add %sp, STACK_BIAS, %g3 3724 srlx %g3, MMU_PAGESHIFT, %g3 3725 srlx %g2, MMU_PAGESHIFT, %g4 3726 cmp %g3, %g4 3727 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 3728 mov PTL1_BAD_STACK, %g1 3729 3730 cmp %g5, 1 3731 ble,pt %icc, 2f 3732 nop 3733 TSTAT_CHECK_TL1(2f, %g1, %g2) 3734 rdpr %tt, %g2 3735 cmp %g2, FAST_PROT_TT 3736 mov PTL1_BAD_KPROT_FAULT, %g1 3737 movne %icc, PTL1_BAD_KMISS, %g1 3738 ba,pt %icc, ptl1_panic 3739 nop 3740 37412: 3742 /* 3743 * We are taking a pagefault in the kernel on a kernel address. If 3744 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 3745 * want to call sfmmu_pagefault -- we will instead note that a fault 3746 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 3747 * (instead of a "retry"). This will step over the faulting 3748 * instruction. 3749 */ 3750 CPU_INDEX(%g1, %g2) 3751 set cpu_core, %g2 3752 sllx %g1, CPU_CORE_SHIFT, %g1 3753 add %g1, %g2, %g1 3754 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3755 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3756 bz sfmmu_pagefault 3757 or %g2, CPU_DTRACE_BADADDR, %g2 3758 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3759 GET_MMU_D_ADDR(%g3, %g4) 3760 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3761 done 3762 37633: 3764 cmp %g5, 1 3765 ble,pt %icc, 4f 3766 nop 3767 TSTAT_CHECK_TL1(4f, %g1, %g2) 3768 ba,pt %icc, sfmmu_window_trap 3769 nop 3770 37714: 3772 /* 3773 * We are taking a pagefault on a non-kernel address. If we are in 3774 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 3775 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 3776 */ 3777 CPU_INDEX(%g1, %g2) 3778 set cpu_core, %g2 3779 sllx %g1, CPU_CORE_SHIFT, %g1 3780 add %g1, %g2, %g1 3781 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3782 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3783 bz sfmmu_mmu_trap 3784 or %g2, CPU_DTRACE_BADADDR, %g2 3785 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3786 GET_MMU_D_ADDR(%g3, %g4) 3787 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3788 3789 /* 3790 * Be sure that we're actually taking this miss from the kernel -- 3791 * otherwise we have managed to return to user-level with 3792 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3793 */ 3794 rdpr %tstate, %g2 3795 btst TSTATE_PRIV, %g2 3796 bz,a ptl1_panic 3797 mov PTL1_BAD_DTRACE_FLAGS, %g1 3798 done 3799 3800 ALTENTRY(tsb_tl0_noctxt) 3801 /* 3802 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 3803 * if it is, indicated that we have faulted and issue a done. 3804 */ 3805 CPU_INDEX(%g5, %g6) 3806 set cpu_core, %g6 3807 sllx %g5, CPU_CORE_SHIFT, %g5 3808 add %g5, %g6, %g5 3809 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 3810 andcc %g6, CPU_DTRACE_NOFAULT, %g0 3811 bz 1f 3812 or %g6, CPU_DTRACE_BADADDR, %g6 3813 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 3814 GET_MMU_D_ADDR(%g3, %g4) 3815 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 3816 3817 /* 3818 * Be sure that we're actually taking this miss from the kernel -- 3819 * otherwise we have managed to return to user-level with 3820 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3821 */ 3822 rdpr %tstate, %g5 3823 btst TSTATE_PRIV, %g5 3824 bz,a ptl1_panic 3825 mov PTL1_BAD_DTRACE_FLAGS, %g1 3826 TSTAT_CHECK_TL1(2f, %g1, %g2); 38272: 3828 done 3829 38301: 3831 rdpr %tt, %g5 3832 cmp %g5, FAST_IMMU_MISS_TT 3833#ifdef sun4v 3834 MMU_FAULT_STATUS_AREA(%g2) 3835 be,a,pt %icc, 2f 3836 ldx [%g2 + MMFSA_I_CTX], %g3 3837 cmp %g5, T_INSTR_MMU_MISS 3838 be,a,pt %icc, 2f 3839 ldx [%g2 + MMFSA_I_CTX], %g3 3840 ldx [%g2 + MMFSA_D_CTX], %g3 38412: 3842#else 3843 mov MMU_TAG_ACCESS, %g2 3844 be,a,pt %icc, 2f 3845 ldxa [%g2]ASI_IMMU, %g3 3846 ldxa [%g2]ASI_DMMU, %g3 38472: sllx %g3, TAGACC_CTX_LSHIFT, %g3 3848#endif /* sun4v */ 3849 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 3850 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 3851 rdpr %tl, %g5 3852 cmp %g5, 1 3853 ble,pt %icc, sfmmu_mmu_trap 3854 nop 3855 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 3856 ba,pt %icc, sfmmu_window_trap 3857 nop 3858 SET_SIZE(sfmmu_tsb_miss) 3859#endif /* lint */ 3860 3861#if defined (lint) 3862/* 3863 * This routine will look for a user or kernel vaddr in the hash 3864 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 3865 * grab any locks. It should only be used by other sfmmu routines. 3866 */ 3867/* ARGSUSED */ 3868pfn_t 3869sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 3870{ 3871 return(0); 3872} 3873 3874/* ARGSUSED */ 3875pfn_t 3876sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno) 3877{ 3878 return(0); 3879} 3880 3881#else /* lint */ 3882 3883 ENTRY_NP(sfmmu_vatopfn) 3884 /* 3885 * disable interrupts 3886 */ 3887 rdpr %pstate, %o3 3888#ifdef DEBUG 3889 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 3890#endif 3891 /* 3892 * disable interrupts to protect the TSBMISS area 3893 */ 3894 andn %o3, PSTATE_IE, %o5 3895 wrpr %o5, 0, %pstate 3896 3897 /* 3898 * o0 = vaddr 3899 * o1 = sfmmup 3900 * o2 = ttep 3901 */ 3902 CPU_TSBMISS_AREA(%g1, %o5) 3903 ldn [%g1 + TSBMISS_KHATID], %o4 3904 cmp %o4, %o1 3905 bne,pn %ncc, vatopfn_nokernel 3906 mov TTE64K, %g5 /* g5 = rehash # */ 3907 mov %g1,%o5 /* o5 = tsbmiss_area */ 3908 /* 3909 * o0 = vaddr 3910 * o1 & o4 = hatid 3911 * o2 = ttep 3912 * o5 = tsbmiss area 3913 */ 3914 mov HBLK_RANGE_SHIFT, %g6 39151: 3916 3917 /* 3918 * o0 = vaddr 3919 * o1 = sfmmup 3920 * o2 = ttep 3921 * o3 = old %pstate 3922 * o4 = hatid 3923 * o5 = tsbmiss 3924 * g5 = rehash # 3925 * g6 = hmeshift 3926 * 3927 * The first arg to GET_TTE is actually tagaccess register 3928 * not just vaddr. Since this call is for kernel we need to clear 3929 * any lower vaddr bits that would be interpreted as ctx bits. 3930 */ 3931 set TAGACC_CTX_MASK, %g1 3932 andn %o0, %g1, %o0 3933 GET_TTE(%o0, %o4, %g1, %g2, %o5, %g4, %g6, %g5, %g3, 3934 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 3935 3936kvtop_hblk_found: 3937 /* 3938 * o0 = vaddr 3939 * o1 = sfmmup 3940 * o2 = ttep 3941 * g1 = tte 3942 * g2 = tte pa 3943 * g3 = scratch 3944 * o2 = tsbmiss area 3945 * o1 = hat id 3946 */ 3947 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 3948 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3949 stx %g1,[%o2] /* put tte into *ttep */ 3950 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 3951 /* 3952 * o0 = vaddr 3953 * o1 = sfmmup 3954 * o2 = ttep 3955 * g1 = pfn 3956 */ 3957 ba,pt %xcc, 6f 3958 mov %g1, %o0 3959 3960kvtop_nohblk: 3961 /* 3962 * we get here if we couldn't find valid hblk in hash. We rehash 3963 * if neccesary. 3964 */ 3965 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 3966#ifdef sun4v 3967 cmp %g5, MAX_HASHCNT 3968#else 3969 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 3970#endif /* sun4v */ 3971 be,a,pn %icc, 6f 3972 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3973 mov %o1, %o4 /* restore hatid */ 3974#ifdef sun4v 3975 add %g5, 2, %g5 3976 cmp %g5, 3 3977 move %icc, MMU_PAGESHIFT4M, %g6 3978 ba,pt %icc, 1b 3979 movne %icc, MMU_PAGESHIFT256M, %g6 3980#else 3981 inc %g5 3982 cmp %g5, 2 3983 move %icc, MMU_PAGESHIFT512K, %g6 3984 ba,pt %icc, 1b 3985 movne %icc, MMU_PAGESHIFT4M, %g6 3986#endif /* sun4v */ 39876: 3988 retl 3989 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3990 3991tsb_suspend: 3992 /* 3993 * o0 = vaddr 3994 * o1 = sfmmup 3995 * o2 = ttep 3996 * g1 = tte 3997 * g2 = tte pa 3998 * g3 = tte va 3999 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 4000 */ 4001 stx %g1,[%o2] /* put tte into *ttep */ 4002 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 4003 sub %g0, 1, %o0 /* output = PFN_INVALID */ 4004 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 40058: 4006 retl 4007 wrpr %g0, %o3, %pstate /* enable interrupts */ 4008 4009vatopfn_nokernel: 4010 /* 4011 * This routine does NOT support user addresses 4012 * There is a routine in C that supports this. 4013 * The only reason why we don't have the C routine 4014 * support kernel addresses as well is because 4015 * we do va_to_pa while holding the hashlock. 4016 */ 4017 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4018 save %sp, -SA(MINFRAME), %sp 4019 sethi %hi(sfmmu_panic3), %o0 4020 call panic 4021 or %o0, %lo(sfmmu_panic3), %o0 4022 4023 SET_SIZE(sfmmu_vatopfn) 4024 4025 /* 4026 * %o0 = vaddr 4027 * %o1 = hashno (aka szc) 4028 * 4029 * 4030 * This routine is similar to sfmmu_vatopfn() but will only look for 4031 * a kernel vaddr in the hash structure for the specified rehash value. 4032 * It's just an optimization for the case when pagesize for a given 4033 * va range is already known (e.g. large page heap) and we don't want 4034 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 4035 * 4036 * Returns valid pfn or PFN_INVALID if 4037 * tte for specified rehash # is not found, invalid or suspended. 4038 */ 4039 ENTRY_NP(sfmmu_kvaszc2pfn) 4040 /* 4041 * disable interrupts 4042 */ 4043 rdpr %pstate, %o3 4044#ifdef DEBUG 4045 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 4046#endif 4047 /* 4048 * disable interrupts to protect the TSBMISS area 4049 */ 4050 andn %o3, PSTATE_IE, %o5 4051 wrpr %o5, 0, %pstate 4052 4053 CPU_TSBMISS_AREA(%g1, %o5) 4054 ldn [%g1 + TSBMISS_KHATID], %o4 4055 sll %o1, 1, %g6 4056 add %g6, %o1, %g6 4057 add %g6, MMU_PAGESHIFT, %g6 4058 /* 4059 * %o0 = vaddr 4060 * %o1 = hashno 4061 * %o3 = old %pstate 4062 * %o4 = ksfmmup 4063 * %g1 = tsbmiss area 4064 * %g6 = hmeshift 4065 */ 4066 4067 /* 4068 * The first arg to GET_TTE is actually tagaccess register 4069 * not just vaddr. Since this call is for kernel we need to clear 4070 * any lower vaddr bits that would be interpreted as ctx bits. 4071 */ 4072 srlx %o0, MMU_PAGESHIFT, %o0 4073 sllx %o0, MMU_PAGESHIFT, %o0 4074 GET_TTE(%o0, %o4, %g3, %g4, %g1, %o5, %g6, %o1, %g5, 4075 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 4076 kvaszc2pfn_nohblk) 4077 4078kvaszc2pfn_hblk_found: 4079 /* 4080 * %g3 = tte 4081 * %o0 = vaddr 4082 */ 4083 brgez,a,pn %g3, 1f /* check if tte is invalid */ 4084 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4085 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 4086 /* 4087 * g3 = pfn 4088 */ 4089 ba,pt %xcc, 1f 4090 mov %g3, %o0 4091 4092kvaszc2pfn_nohblk: 4093 mov -1, %o0 4094 40951: 4096 retl 4097 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4098 4099 SET_SIZE(sfmmu_kvaszc2pfn) 4100 4101#endif /* lint */ 4102 4103 4104 4105#if !defined(lint) 4106 4107/* 4108 * kpm lock used between trap level tsbmiss handler and kpm C level. 4109 */ 4110#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 4111 mov 0xff, tmp1 ;\ 4112label1: ;\ 4113 casa [kpmlckp]asi, %g0, tmp1 ;\ 4114 brnz,pn tmp1, label1 ;\ 4115 mov 0xff, tmp1 ;\ 4116 membar #LoadLoad 4117 4118#define KPMLOCK_EXIT(kpmlckp, asi) \ 4119 membar #LoadStore|#StoreStore ;\ 4120 sta %g0, [kpmlckp]asi 4121 4122/* 4123 * Lookup a memseg for a given pfn and if found, return the physical 4124 * address of the corresponding struct memseg in mseg, otherwise 4125 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 4126 * tsbmp, %asi is assumed to be ASI_MEM. 4127 * This lookup is done by strictly traversing only the physical memseg 4128 * linkage. The more generic approach, to check the virtual linkage 4129 * before using the physical (used e.g. with hmehash buckets), cannot 4130 * be used here. Memory DR operations can run in parallel to this 4131 * lookup w/o any locks and updates of the physical and virtual linkage 4132 * cannot be done atomically wrt. to each other. Because physical 4133 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 4134 * as "physical NULL" pointer. 4135 */ 4136#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 4137 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 4138 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 4139 udivx pfn, mseg, mseg ;\ 4140 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 4141 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 4142 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 4143 add tmp1, mseg, tmp1 ;\ 4144 ldxa [tmp1]%asi, mseg ;\ 4145 cmp mseg, MSEG_NULLPTR_PA ;\ 4146 be,pn %xcc, label/**/1 /* if not found */ ;\ 4147 nop ;\ 4148 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4149 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4150 blu,pn %xcc, label/**/1 ;\ 4151 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4152 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4153 bgeu,pn %xcc, label/**/1 ;\ 4154 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 4155 mulx tmp1, PAGE_SIZE, tmp1 ;\ 4156 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 4157 add tmp2, tmp1, tmp1 /* pp */ ;\ 4158 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 4159 cmp tmp2, pfn ;\ 4160 be,pt %xcc, label/**/_ok /* found */ ;\ 4161label/**/1: ;\ 4162 /* brute force lookup */ ;\ 4163 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 4164 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 4165label/**/2: ;\ 4166 cmp mseg, MSEG_NULLPTR_PA ;\ 4167 be,pn %xcc, label/**/_ok /* if not found */ ;\ 4168 nop ;\ 4169 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4170 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4171 blu,a,pt %xcc, label/**/2 ;\ 4172 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4173 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4174 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4175 bgeu,a,pt %xcc, label/**/2 ;\ 4176 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4177label/**/_ok: 4178 4179 /* 4180 * kpm tsb miss handler large pages 4181 * g1 = 8K kpm TSB entry pointer 4182 * g2 = tag access register 4183 * g3 = 4M kpm TSB entry pointer 4184 */ 4185 ALTENTRY(sfmmu_kpm_dtsb_miss) 4186 TT_TRACE(trace_tsbmiss) 4187 4188 CPU_INDEX(%g7, %g6) 4189 sethi %hi(kpmtsbm_area), %g6 4190 sllx %g7, KPMTSBM_SHIFT, %g7 4191 or %g6, %lo(kpmtsbm_area), %g6 4192 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4193 4194 /* check enable flag */ 4195 ldub [%g6 + KPMTSBM_FLAGS], %g4 4196 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4197 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4198 nop 4199 4200 /* VA range check */ 4201 ldx [%g6 + KPMTSBM_VBASE], %g7 4202 cmp %g2, %g7 4203 blu,pn %xcc, sfmmu_tsb_miss 4204 ldx [%g6 + KPMTSBM_VEND], %g5 4205 cmp %g2, %g5 4206 bgeu,pn %xcc, sfmmu_tsb_miss 4207 stx %g3, [%g6 + KPMTSBM_TSBPTR] 4208 4209 /* 4210 * check TL tsbmiss handling flag 4211 * bump tsbmiss counter 4212 */ 4213 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4214#ifdef DEBUG 4215 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4216 inc %g5 4217 brz,pn %g3, sfmmu_kpm_exception 4218 st %g5, [%g6 + KPMTSBM_TSBMISS] 4219#else 4220 inc %g5 4221 st %g5, [%g6 + KPMTSBM_TSBMISS] 4222#endif 4223 /* 4224 * At this point: 4225 * g1 = 8K kpm TSB pointer (not used) 4226 * g2 = tag access register 4227 * g3 = clobbered 4228 * g6 = per-CPU kpm tsbmiss area 4229 * g7 = kpm_vbase 4230 */ 4231 4232 /* vaddr2pfn */ 4233 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4234 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4235 srax %g4, %g3, %g2 /* which alias range (r) */ 4236 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4237 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4238 4239 /* 4240 * Setup %asi 4241 * mseg_pa = page_numtomemseg_nolock(pfn) 4242 * if (mseg_pa == NULL) sfmmu_kpm_exception 4243 * g2=pfn 4244 */ 4245 mov ASI_MEM, %asi 4246 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4247 cmp %g3, MSEG_NULLPTR_PA 4248 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4249 nop 4250 4251 /* 4252 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4253 * g2=pfn g3=mseg_pa 4254 */ 4255 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4256 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4257 srlx %g2, %g5, %g4 4258 sllx %g4, %g5, %g4 4259 sub %g4, %g7, %g4 4260 srlx %g4, %g5, %g4 4261 4262 /* 4263 * Validate inx value 4264 * g2=pfn g3=mseg_pa g4=inx 4265 */ 4266#ifdef DEBUG 4267 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4268 cmp %g4, %g5 /* inx - nkpmpgs */ 4269 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4270 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4271#else 4272 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4273#endif 4274 /* 4275 * kp = &mseg_pa->kpm_pages[inx] 4276 */ 4277 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4278 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4279 add %g5, %g4, %g5 /* kp */ 4280 4281 /* 4282 * KPMP_HASH(kp) 4283 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4284 */ 4285 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4286 sub %g7, 1, %g7 /* mask */ 4287 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4288 add %g5, %g1, %g5 /* y = ksp + x */ 4289 and %g5, %g7, %g5 /* hashinx = y & mask */ 4290 4291 /* 4292 * Calculate physical kpm_page pointer 4293 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4294 */ 4295 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4296 add %g1, %g4, %g1 /* kp_pa */ 4297 4298 /* 4299 * Calculate physical hash lock address 4300 * g1=kp_refcntc_pa g2=pfn g5=hashinx 4301 */ 4302 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4303 sllx %g5, KPMHLK_SHIFT, %g5 4304 add %g4, %g5, %g3 4305 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4306 4307 /* 4308 * Assemble tte 4309 * g1=kp_pa g2=pfn g3=hlck_pa 4310 */ 4311#ifdef sun4v 4312 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4313 sllx %g5, 32, %g5 4314 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4315 or %g4, TTE4M, %g4 4316 or %g5, %g4, %g5 4317#else 4318 sethi %hi(TTE_VALID_INT), %g4 4319 mov TTE4M, %g5 4320 sllx %g5, TTE_SZ_SHFT_INT, %g5 4321 or %g5, %g4, %g5 /* upper part */ 4322 sllx %g5, 32, %g5 4323 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4324 or %g5, %g4, %g5 4325#endif 4326 sllx %g2, MMU_PAGESHIFT, %g4 4327 or %g5, %g4, %g5 /* tte */ 4328 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4329 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4330 4331 /* 4332 * tsb dropin 4333 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4334 */ 4335 4336 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4337 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4338 4339 /* use C-handler if there's no go for dropin */ 4340 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4341 cmp %g7, -1 4342 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4343 nop 4344 4345#ifdef DEBUG 4346 /* double check refcnt */ 4347 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4348 brz,pn %g7, 5f /* let C-handler deal with this */ 4349 nop 4350#endif 4351 4352#ifndef sun4v 4353 ldub [%g6 + KPMTSBM_FLAGS], %g7 4354 mov ASI_N, %g1 4355 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4356 movnz %icc, ASI_MEM, %g1 4357 mov %g1, %asi 4358#endif 4359 4360 /* 4361 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4362 * If we fail to lock the TSB entry then just load the tte into the 4363 * TLB. 4364 */ 4365 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l1) 4366 4367 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4368 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4369locked_tsb_l1: 4370 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4371 4372 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4373 KPMLOCK_EXIT(%g3, ASI_MEM) 4374 4375 /* 4376 * If trapstat is running, we need to shift the %tpc and %tnpc to 4377 * point to trapstat's TSB miss return code (note that trapstat 4378 * itself will patch the correct offset to add). 4379 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4380 */ 4381 rdpr %tl, %g7 4382 cmp %g7, 1 4383 ble %icc, 0f 4384 sethi %hi(KERNELBASE), %g6 4385 rdpr %tpc, %g7 4386 or %g6, %lo(KERNELBASE), %g6 4387 cmp %g7, %g6 4388 bgeu %xcc, 0f 4389 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4390 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4391 wrpr %g7, %tpc 4392 add %g7, 4, %g7 4393 wrpr %g7, %tnpc 43940: 4395 retry 43965: 4397 /* g3=hlck_pa */ 4398 KPMLOCK_EXIT(%g3, ASI_MEM) 4399 ba,pt %icc, sfmmu_kpm_exception 4400 nop 4401 SET_SIZE(sfmmu_kpm_dtsb_miss) 4402 4403 /* 4404 * kpm tsbmiss handler for smallpages 4405 * g1 = 8K kpm TSB pointer 4406 * g2 = tag access register 4407 * g3 = 4M kpm TSB pointer 4408 */ 4409 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4410 TT_TRACE(trace_tsbmiss) 4411 CPU_INDEX(%g7, %g6) 4412 sethi %hi(kpmtsbm_area), %g6 4413 sllx %g7, KPMTSBM_SHIFT, %g7 4414 or %g6, %lo(kpmtsbm_area), %g6 4415 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4416 4417 /* check enable flag */ 4418 ldub [%g6 + KPMTSBM_FLAGS], %g4 4419 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4420 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4421 nop 4422 4423 /* 4424 * VA range check 4425 * On fail: goto sfmmu_tsb_miss 4426 */ 4427 ldx [%g6 + KPMTSBM_VBASE], %g7 4428 cmp %g2, %g7 4429 blu,pn %xcc, sfmmu_tsb_miss 4430 ldx [%g6 + KPMTSBM_VEND], %g5 4431 cmp %g2, %g5 4432 bgeu,pn %xcc, sfmmu_tsb_miss 4433 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4434 4435 /* 4436 * check TL tsbmiss handling flag 4437 * bump tsbmiss counter 4438 */ 4439 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4440#ifdef DEBUG 4441 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4442 inc %g5 4443 brz,pn %g1, sfmmu_kpm_exception 4444 st %g5, [%g6 + KPMTSBM_TSBMISS] 4445#else 4446 inc %g5 4447 st %g5, [%g6 + KPMTSBM_TSBMISS] 4448#endif 4449 /* 4450 * At this point: 4451 * g1 = clobbered 4452 * g2 = tag access register 4453 * g3 = 4M kpm TSB pointer (not used) 4454 * g6 = per-CPU kpm tsbmiss area 4455 * g7 = kpm_vbase 4456 */ 4457 4458 /* 4459 * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr) 4460 * which is defined in mach_kpm.h. Any changes in that macro 4461 * should also be ported back to this assembly code. 4462 */ 4463 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 /* g3 = kpm_size_shift */ 4464 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4465 srax %g4, %g3, %g7 /* which alias range (r) */ 4466 brz,pt %g7, 2f 4467 sethi %hi(vac_colors_mask), %g5 4468 ld [%g5 + %lo(vac_colors_mask)], %g5 4469 4470 srlx %g2, MMU_PAGESHIFT, %g1 /* vaddr >> MMU_PAGESHIFT */ 4471 and %g1, %g5, %g1 /* g1 = v */ 4472 sllx %g7, %g3, %g5 /* g5 = r << kpm_size_shift */ 4473 cmp %g7, %g1 /* if (r > v) */ 4474 bleu,pn %xcc, 1f 4475 sub %g4, %g5, %g4 /* paddr -= r << kpm_size_shift */ 4476 sub %g7, %g1, %g5 /* g5 = r - v */ 4477 sllx %g5, MMU_PAGESHIFT, %g7 /* (r-v) << MMU_PAGESHIFT */ 4478 add %g4, %g7, %g4 /* paddr += (r-v)<<MMU_PAGESHIFT */ 4479 ba 2f 4480 nop 44811: 4482 sllx %g7, MMU_PAGESHIFT, %g5 /* else */ 4483 sub %g4, %g5, %g4 /* paddr -= r << MMU_PAGESHIFT */ 4484 4485 /* 4486 * paddr2pfn 4487 * g1 = vcolor (not used) 4488 * g2 = tag access register 4489 * g3 = clobbered 4490 * g4 = paddr 4491 * g5 = clobbered 4492 * g6 = per-CPU kpm tsbmiss area 4493 * g7 = clobbered 4494 */ 44952: 4496 srlx %g4, MMU_PAGESHIFT, %g2 /* g2 = pfn */ 4497 4498 /* 4499 * Setup %asi 4500 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4501 * if (mseg not found) sfmmu_kpm_exception 4502 * g2=pfn g6=per-CPU kpm tsbmiss area 4503 * g4 g5 g7 for scratch use. 4504 */ 4505 mov ASI_MEM, %asi 4506 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4507 cmp %g3, MSEG_NULLPTR_PA 4508 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4509 nop 4510 4511 /* 4512 * inx = pfn - mseg_pa->kpm_pbase 4513 * g2=pfn g3=mseg_pa g6=per-CPU kpm tsbmiss area 4514 */ 4515 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4516 sub %g2, %g7, %g4 4517 4518#ifdef DEBUG 4519 /* 4520 * Validate inx value 4521 * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area 4522 */ 4523 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4524 cmp %g4, %g5 /* inx - nkpmpgs */ 4525 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4526 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4527#else 4528 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4529#endif 4530 /* ksp = &mseg_pa->kpm_spages[inx] */ 4531 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4532 add %g5, %g4, %g5 /* ksp */ 4533 4534 /* 4535 * KPMP_SHASH(kp) 4536 * g2=pfn g3=mseg_pa g4=inx g5=ksp 4537 * g6=per-CPU kpm tsbmiss area g7=kpmp_stable_sz 4538 */ 4539 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4540 sub %g7, 1, %g7 /* mask */ 4541 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4542 add %g5, %g1, %g5 /* y = ksp + x */ 4543 and %g5, %g7, %g5 /* hashinx = y & mask */ 4544 4545 /* 4546 * Calculate physical kpm_spage pointer 4547 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4548 * g6=per-CPU kpm tsbmiss area 4549 */ 4550 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4551 add %g1, %g4, %g1 /* ksp_pa */ 4552 4553 /* 4554 * Calculate physical hash lock address. 4555 * Note: Changes in kpm_shlk_t must be reflected here. 4556 * g1=ksp_pa g2=pfn g5=hashinx 4557 * g6=per-CPU kpm tsbmiss area 4558 */ 4559 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4560 sllx %g5, KPMSHLK_SHIFT, %g5 4561 add %g4, %g5, %g3 /* hlck_pa */ 4562 4563 /* 4564 * Assemble non-cacheable tte initially 4565 * g1=ksp_pa g2=pfn g3=hlck_pa 4566 * g6=per-CPU kpm tsbmiss area 4567 */ 4568 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4569 sllx %g5, 32, %g5 4570 mov (TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4571 or %g5, %g4, %g5 4572 sllx %g2, MMU_PAGESHIFT, %g4 4573 or %g5, %g4, %g5 /* tte */ 4574 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4575 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4576 4577 /* 4578 * tsb dropin 4579 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable) 4580 * g6=per-CPU kpm tsbmiss area g7=scratch register 4581 */ 4582 4583 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4584 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4585 4586 /* use C-handler if there's no go for dropin */ 4587 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4588 andcc %g7, KPM_MAPPED_GO, %g0 /* go or no go ? */ 4589 bz,pt %icc, 5f /* no go */ 4590 nop 4591 and %g7, KPM_MAPPED_MASK, %g7 /* go */ 4592 cmp %g7, KPM_MAPPEDS /* cacheable ? */ 4593 be,a,pn %xcc, 3f 4594 or %g5, TTE_CV_INT, %g5 /* cacheable */ 45953: 4596#ifndef sun4v 4597 ldub [%g6 + KPMTSBM_FLAGS], %g7 4598 mov ASI_N, %g1 4599 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4600 movnz %icc, ASI_MEM, %g1 4601 mov %g1, %asi 4602#endif 4603 4604 /* 4605 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4606 * If we fail to lock the TSB entry then just load the tte into the 4607 * TLB. 4608 */ 4609 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l2) 4610 4611 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4612 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4613locked_tsb_l2: 4614 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4615 4616 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4617 KPMLOCK_EXIT(%g3, ASI_MEM) 4618 4619 /* 4620 * If trapstat is running, we need to shift the %tpc and %tnpc to 4621 * point to trapstat's TSB miss return code (note that trapstat 4622 * itself will patch the correct offset to add). 4623 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4624 */ 4625 rdpr %tl, %g7 4626 cmp %g7, 1 4627 ble %icc, 0f 4628 sethi %hi(KERNELBASE), %g6 4629 rdpr %tpc, %g7 4630 or %g6, %lo(KERNELBASE), %g6 4631 cmp %g7, %g6 4632 bgeu %xcc, 0f 4633 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4634 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4635 wrpr %g7, %tpc 4636 add %g7, 4, %g7 4637 wrpr %g7, %tnpc 46380: 4639 retry 46405: 4641 /* g3=hlck_pa */ 4642 KPMLOCK_EXIT(%g3, ASI_MEM) 4643 ba,pt %icc, sfmmu_kpm_exception 4644 nop 4645 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4646 4647#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4648#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4649#endif 4650 4651#endif /* lint */ 4652 4653#ifdef lint 4654/* 4655 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4656 * Called from C-level, sets/clears "go" indication for trap level handler. 4657 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 4658 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 4659 * Assumes khl_mutex is held when called from C-level. 4660 */ 4661/* ARGSUSED */ 4662void 4663sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 4664{ 4665} 4666 4667/* 4668 * kpm_smallpages: stores val to byte at address mapped within 4669 * low level lock brackets. The old value is returned. 4670 * Called from C-level. 4671 */ 4672/* ARGSUSED */ 4673int 4674sfmmu_kpm_stsbmtl(uchar_t *mapped, uint_t *kshl_lock, int val) 4675{ 4676 return (0); 4677} 4678 4679#else /* lint */ 4680 4681 .seg ".data" 4682sfmmu_kpm_tsbmtl_panic: 4683 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4684 .byte 0 4685sfmmu_kpm_stsbmtl_panic: 4686 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4687 .byte 0 4688 .align 4 4689 .seg ".text" 4690 4691 ENTRY_NP(sfmmu_kpm_tsbmtl) 4692 rdpr %pstate, %o3 4693 /* 4694 * %o0 = &kp_refcntc 4695 * %o1 = &khl_lock 4696 * %o2 = 0/1 (off/on) 4697 * %o3 = pstate save 4698 */ 4699#ifdef DEBUG 4700 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4701 bnz,pt %icc, 1f /* disabled, panic */ 4702 nop 4703 save %sp, -SA(MINFRAME), %sp 4704 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4705 call panic 4706 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4707 ret 4708 restore 47091: 4710#endif /* DEBUG */ 4711 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4712 4713 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4714 mov -1, %o5 4715 brz,a %o2, 2f 4716 mov 0, %o5 47172: 4718 sth %o5, [%o0] 4719 KPMLOCK_EXIT(%o1, ASI_N) 4720 4721 retl 4722 wrpr %g0, %o3, %pstate /* enable interrupts */ 4723 SET_SIZE(sfmmu_kpm_tsbmtl) 4724 4725 ENTRY_NP(sfmmu_kpm_stsbmtl) 4726 rdpr %pstate, %o3 4727 /* 4728 * %o0 = &mapped 4729 * %o1 = &kshl_lock 4730 * %o2 = val 4731 * %o3 = pstate save 4732 */ 4733#ifdef DEBUG 4734 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4735 bnz,pt %icc, 1f /* disabled, panic */ 4736 nop 4737 save %sp, -SA(MINFRAME), %sp 4738 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4739 call panic 4740 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4741 ret 4742 restore 47431: 4744#endif /* DEBUG */ 4745 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4746 4747 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4748 ldsb [%o0], %o5 4749 stb %o2, [%o0] 4750 KPMLOCK_EXIT(%o1, ASI_N) 4751 4752 and %o5, KPM_MAPPED_MASK, %o0 /* return old val */ 4753 retl 4754 wrpr %g0, %o3, %pstate /* enable interrupts */ 4755 SET_SIZE(sfmmu_kpm_stsbmtl) 4756 4757#endif /* lint */ 4758 4759#ifndef lint 4760#ifdef sun4v 4761 /* 4762 * User/kernel data miss w// multiple TSBs 4763 * The first probe covers 8K, 64K, and 512K page sizes, 4764 * because 64K and 512K mappings are replicated off 8K 4765 * pointer. Second probe covers 4M page size only. 4766 * 4767 * MMU fault area contains miss address and context. 4768 */ 4769 ALTENTRY(sfmmu_slow_dmmu_miss) 4770 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 4771 4772slow_miss_common: 4773 /* 4774 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 4775 * %g3 = ctx (cannot be INVALID_CONTEXT) 4776 */ 4777 brnz,pt %g3, 8f ! check for user context 4778 nop 4779 4780 /* 4781 * Kernel miss 4782 * Get 8K and 4M TSB pointers in %g1 and %g3 and 4783 * branch to sfmmu_tsb_miss_tt to handle it. 4784 */ 4785 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4786sfmmu_dslow_patch_ktsb_base: 4787 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 4788sfmmu_dslow_patch_ktsb_szcode: 4789 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 4790 4791 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 4792 ! %g1 = First TSB entry pointer, as TSB miss handler expects 4793 4794 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4795sfmmu_dslow_patch_ktsb4m_base: 4796 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 4797sfmmu_dslow_patch_ktsb4m_szcode: 4798 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 4799 4800 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 4801 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 4802 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4803 .empty 4804 48058: 4806 /* 4807 * User miss 4808 * Get first TSB pointer in %g1 4809 * Get second TSB pointer (or NULL if no second TSB) in %g3 4810 * Branch to sfmmu_tsb_miss_tt to handle it 4811 */ 4812 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 4813 /* %g1 = first TSB entry ptr now, %g2 preserved */ 4814 4815 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 4816 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 4817 nop 4818 4819 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 4820 /* %g3 = second TSB entry ptr now, %g2 preserved */ 48219: 4822 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4823 .empty 4824 SET_SIZE(sfmmu_slow_dmmu_miss) 4825 4826 4827 /* 4828 * User/kernel instruction miss w/ multiple TSBs 4829 * The first probe covers 8K, 64K, and 512K page sizes, 4830 * because 64K and 512K mappings are replicated off 8K 4831 * pointer. Second probe covers 4M page size only. 4832 * 4833 * MMU fault area contains miss address and context. 4834 */ 4835 ALTENTRY(sfmmu_slow_immu_miss) 4836 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 4837 ba,a,pt %xcc, slow_miss_common 4838 SET_SIZE(sfmmu_slow_immu_miss) 4839 4840#endif /* sun4v */ 4841#endif /* lint */ 4842 4843#ifndef lint 4844 4845/* 4846 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 4847 */ 4848 .seg ".data" 4849 .align 64 4850 .global tsbmiss_area 4851tsbmiss_area: 4852 .skip (TSBMISS_SIZE * NCPU) 4853 4854 .align 64 4855 .global kpmtsbm_area 4856kpmtsbm_area: 4857 .skip (KPMTSBM_SIZE * NCPU) 4858#endif /* lint */ 4859