1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26/* 27 * SFMMU primitives. These primitives should only be used by sfmmu 28 * routines. 29 */ 30 31#if defined(lint) 32#include <sys/types.h> 33#else /* lint */ 34#include "assym.h" 35#endif /* lint */ 36 37#include <sys/asm_linkage.h> 38#include <sys/machtrap.h> 39#include <sys/machasi.h> 40#include <sys/sun4asi.h> 41#include <sys/pte.h> 42#include <sys/mmu.h> 43#include <vm/hat_sfmmu.h> 44#include <vm/seg_spt.h> 45#include <sys/machparam.h> 46#include <sys/privregs.h> 47#include <sys/scb.h> 48#include <sys/intreg.h> 49#include <sys/machthread.h> 50#include <sys/intr.h> 51#include <sys/clock.h> 52#include <sys/trapstat.h> 53 54#ifdef TRAPTRACE 55#include <sys/traptrace.h> 56 57/* 58 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 59 */ 60#define TT_TRACE(label) \ 61 ba label ;\ 62 rd %pc, %g7 63#else 64 65#define TT_TRACE(label) 66 67#endif /* TRAPTRACE */ 68 69#ifndef lint 70 71#if (TTE_SUSPEND_SHIFT > 0) 72#define TTE_SUSPEND_INT_SHIFT(reg) \ 73 sllx reg, TTE_SUSPEND_SHIFT, reg 74#else 75#define TTE_SUSPEND_INT_SHIFT(reg) 76#endif 77 78#endif /* lint */ 79 80#ifndef lint 81 82/* 83 * Assumes TSBE_TAG is 0 84 * Assumes TSBE_INTHI is 0 85 * Assumes TSBREG.split is 0 86 */ 87 88#if TSBE_TAG != 0 89#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 90#endif 91 92#if TSBTAG_INTHI != 0 93#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 94#endif 95 96/* 97 * The following code assumes the tsb is not split. 98 * 99 * With TSBs no longer shared between processes, it's no longer 100 * necessary to hash the context bits into the tsb index to get 101 * tsb coloring; the new implementation treats the TSB as a 102 * direct-mapped, virtually-addressed cache. 103 * 104 * In: 105 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 106 * tsbbase = base address of TSB (clobbered) 107 * tagacc = tag access register (clobbered) 108 * szc = size code of TSB (ro) 109 * tmp = scratch reg 110 * Out: 111 * tsbbase = pointer to entry in TSB 112 */ 113#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 114 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 115 srlx tagacc, vpshift, tagacc ;\ 116 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 117 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 118 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 119 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 120 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 121 122/* 123 * When the kpm TSB is used it is assumed that it is direct mapped 124 * using (vaddr>>vpshift)%tsbsz as the index. 125 * 126 * Note that, for now, the kpm TSB and kernel TSB are the same for 127 * each mapping size. However that need not always be the case. If 128 * the trap handlers are updated to search a different TSB for kpm 129 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 130 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 131 * 132 * In: 133 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 134 * vaddr = virtual address (clobbered) 135 * tsbp, szc, tmp = scratch 136 * Out: 137 * tsbp = pointer to entry in TSB 138 */ 139#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 140 cmp vpshift, MMU_PAGESHIFT ;\ 141 bne,pn %icc, 1f /* branch if large case */ ;\ 142 sethi %hi(kpmsm_tsbsz), szc ;\ 143 sethi %hi(kpmsm_tsbbase), tsbp ;\ 144 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 145 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 146 ba,pt %icc, 2f ;\ 147 nop ;\ 1481: sethi %hi(kpm_tsbsz), szc ;\ 149 sethi %hi(kpm_tsbbase), tsbp ;\ 150 ld [szc + %lo(kpm_tsbsz)], szc ;\ 151 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1522: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 153 154/* 155 * Lock the TSBE at virtual address tsbep. 156 * 157 * tsbep = TSBE va (ro) 158 * tmp1, tmp2 = scratch registers (clobbered) 159 * label = label to use for branches (text) 160 * %asi = ASI to use for TSB access 161 * 162 * NOTE that we flush the TSB using fast VIS instructions that 163 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 164 * not be treated as a locked entry or we'll get stuck spinning on 165 * an entry that isn't locked but really invalid. 166 */ 167 168#if defined(UTSB_PHYS) 169 170#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 171 lda [tsbep]ASI_MEM, tmp1 ;\ 172label: ;\ 173 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 174 cmp tmp1, tmp2 ;\ 175 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 176 lda [tsbep]ASI_MEM, tmp1 ;\ 177 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 178 cmp tmp1, tmp2 ;\ 179 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 180 lda [tsbep]ASI_MEM, tmp1 ;\ 181 /* tsbe lock acquired */ ;\ 182 membar #StoreStore 183 184#else /* UTSB_PHYS */ 185 186#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 187 lda [tsbep]%asi, tmp1 ;\ 188label: ;\ 189 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 190 cmp tmp1, tmp2 ;\ 191 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 192 lda [tsbep]%asi, tmp1 ;\ 193 casa [tsbep]%asi, tmp1, tmp2 ;\ 194 cmp tmp1, tmp2 ;\ 195 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 196 lda [tsbep]%asi, tmp1 ;\ 197 /* tsbe lock acquired */ ;\ 198 membar #StoreStore 199 200#endif /* UTSB_PHYS */ 201 202/* 203 * Atomically write TSBE at virtual address tsbep. 204 * 205 * tsbep = TSBE va (ro) 206 * tte = TSBE TTE (ro) 207 * tagtarget = TSBE tag (ro) 208 * %asi = ASI to use for TSB access 209 */ 210 211#if defined(UTSB_PHYS) 212 213#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 214 add tsbep, TSBE_TTE, tmp1 ;\ 215 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 216 membar #StoreStore ;\ 217 add tsbep, TSBE_TAG, tmp1 ;\ 218 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 219 220#else /* UTSB_PHYS */ 221 222#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 223 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 224 membar #StoreStore ;\ 225 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 226 227#endif /* UTSB_PHYS */ 228 229/* 230 * Load an entry into the TSB at TL > 0. 231 * 232 * tsbep = pointer to the TSBE to load as va (ro) 233 * tte = value of the TTE retrieved and loaded (wo) 234 * tagtarget = tag target register. To get TSBE tag to load, 235 * we need to mask off the context and leave only the va (clobbered) 236 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 237 * tmp1, tmp2 = scratch registers 238 * label = label to use for branches (text) 239 * %asi = ASI to use for TSB access 240 */ 241 242#if defined(UTSB_PHYS) 243 244#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 245 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 246 /* ;\ 247 * I don't need to update the TSB then check for the valid tte. ;\ 248 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 249 * we always invalidate the hash table before we unload the TSB.;\ 250 */ ;\ 251 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 252 ldxa [ttepa]ASI_MEM, tte ;\ 253 TTE_CLR_SOFTEXEC_ML(tte) ;\ 254 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 255 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 256 add tsbep, TSBE_TAG, tmp1 ;\ 257 brgez,a,pn tte, label/**/f ;\ 258 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 259 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 260label: 261 262#else /* UTSB_PHYS */ 263 264#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 265 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 266 /* ;\ 267 * I don't need to update the TSB then check for the valid tte. ;\ 268 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 269 * we always invalidate the hash table before we unload the TSB.;\ 270 */ ;\ 271 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 272 ldxa [ttepa]ASI_MEM, tte ;\ 273 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 274 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 275 brgez,a,pn tte, label/**/f ;\ 276 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 277 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 278label: 279 280#endif /* UTSB_PHYS */ 281 282/* 283 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 284 * for ITLB synthesis. 285 * 286 * tsbep = pointer to the TSBE to load as va (ro) 287 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 288 * with exec_perm turned off and exec_synth turned on 289 * tagtarget = tag target register. To get TSBE tag to load, 290 * we need to mask off the context and leave only the va (clobbered) 291 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 292 * tmp1, tmp2 = scratch registers 293 * label = label to use for branch (text) 294 * %asi = ASI to use for TSB access 295 */ 296 297#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 298 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 299 /* ;\ 300 * I don't need to update the TSB then check for the valid tte. ;\ 301 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 302 * we always invalidate the hash table before we unload the TSB.;\ 303 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 304 * and exec_synth bit to 1. ;\ 305 */ ;\ 306 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 307 mov tte, tmp1 ;\ 308 ldxa [ttepa]ASI_MEM, tte ;\ 309 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 310 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 311 brgez,a,pn tte, label/**/f ;\ 312 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 313 or tte, tmp1, tte ;\ 314 andn tte, TTE_EXECPRM_INT, tte ;\ 315 or tte, TTE_E_SYNTH_INT, tte ;\ 316 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 317label: 318 319/* 320 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 321 * 322 * tte = value of the TTE, used to get tte_size bits (ro) 323 * tagaccess = tag access register, used to get 4M pfn bits (ro) 324 * pfn = 4M pfn bits shifted to offset for tte (out) 325 * tmp1 = scratch register 326 * label = label to use for branch (text) 327 */ 328 329#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 330 /* ;\ 331 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 332 * Return them, shifted, in pfn. ;\ 333 */ ;\ 334 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 335 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 336 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 337 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 338 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 339 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 340label: ;\ 341 sllx tagaccess, MMU_PAGESHIFT4M, pfn 342 343/* 344 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 345 * for ITLB synthesis. 346 * 347 * tte = value of the TTE, used to get tte_size bits (rw) 348 * tmp1 = scratch register 349 */ 350 351#define SET_TTE4M_PN(tte, tmp) \ 352 /* ;\ 353 * Set 4M pagesize tte bits. ;\ 354 */ ;\ 355 set TTE4M, tmp ;\ 356 sllx tmp, TTE_SZ_SHFT, tmp ;\ 357 or tte, tmp, tte 358 359/* 360 * Load an entry into the TSB at TL=0. 361 * 362 * tsbep = pointer to the TSBE to load as va (ro) 363 * tteva = pointer to the TTE to load as va (ro) 364 * tagtarget = TSBE tag to load (which contains no context), synthesized 365 * to match va of MMU tag target register only (ro) 366 * tmp1, tmp2 = scratch registers (clobbered) 367 * label = label to use for branches (text) 368 * %asi = ASI to use for TSB access 369 */ 370 371#if defined(UTSB_PHYS) 372 373#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 374 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 375 ldx [tteva], tteva /* load tte */ ;\ 376 TTE_CLR_SOFTEXEC_ML(tteva) ;\ 377 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 378 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 379 add tsbep, TSBE_TAG, tmp1 ;\ 380 brgez,a,pn tteva, label/**/f ;\ 381 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 382 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 383label: 384 385#else /* UTSB_PHYS */ 386 387#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 388 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 389 ldx [tteva], tteva /* load tte */ ;\ 390 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 391 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 392 brgez,a,pn tteva, label/**/f ;\ 393 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 394 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 395label: 396 397#endif /* UTSB_PHYS */ 398 399/* 400 * Invalidate a TSB entry in the TSB. 401 * 402 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 403 * about this earlier to ensure this is true. Thus when we are 404 * directly referencing tsbep below, we are referencing the tte_tag 405 * field of the TSBE. If this offset ever changes, the code below 406 * will need to be modified. 407 * 408 * tsbep = pointer to TSBE as va (ro) 409 * tag = invalidation is done if this matches the TSBE tag (ro) 410 * tmp1 - tmp3 = scratch registers (clobbered) 411 * label = label name to use for branches (text) 412 * %asi = ASI to use for TSB access 413 */ 414 415#if defined(UTSB_PHYS) 416 417#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 418 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 419 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 420label/**/1: ;\ 421 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 422 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 423 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 424 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 425 cmp tag, tmp3 /* compare tags */ ;\ 426 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 427 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 428 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 429 cmp tmp1, tmp3 /* if not successful */ ;\ 430 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 431 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 432label/**/2: 433 434#else /* UTSB_PHYS */ 435 436#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 437 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 438 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 439label/**/1: ;\ 440 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 441 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 442 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 443 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 444 cmp tag, tmp3 /* compare tags */ ;\ 445 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 446 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 447 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 448 cmp tmp1, tmp3 /* if not successful */ ;\ 449 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 450 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 451label/**/2: 452 453#endif /* UTSB_PHYS */ 454 455#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 456#error - TSB_SOFTSZ_MASK too small 457#endif 458 459 460/* 461 * An implementation of setx which will be hot patched at run time. 462 * since it is being hot patched, there is no value passed in. 463 * Thus, essentially we are implementing 464 * setx value, tmp, dest 465 * where value is RUNTIME_PATCH (aka 0) in this case. 466 */ 467#define RUNTIME_PATCH_SETX(dest, tmp) \ 468 sethi %hh(RUNTIME_PATCH), tmp ;\ 469 sethi %lm(RUNTIME_PATCH), dest ;\ 470 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 471 or dest, %lo(RUNTIME_PATCH), dest ;\ 472 sllx tmp, 32, tmp ;\ 473 nop /* for perf reasons */ ;\ 474 or tmp, dest, dest /* contents of patched value */ 475 476#endif /* lint */ 477 478 479#if defined (lint) 480 481/* 482 * sfmmu related subroutines 483 */ 484uint_t 485sfmmu_disable_intrs() 486{ return(0); } 487 488/* ARGSUSED */ 489void 490sfmmu_enable_intrs(uint_t pstate_save) 491{} 492 493/* ARGSUSED */ 494int 495sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag) 496{ return(0); } 497 498/* 499 * Use cas, if tte has changed underneath us then reread and try again. 500 * In the case of a retry, it will update sttep with the new original. 501 */ 502/* ARGSUSED */ 503int 504sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 505{ return(0); } 506 507/* 508 * Use cas, if tte has changed underneath us then return 1, else return 0 509 */ 510/* ARGSUSED */ 511int 512sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 513{ return(0); } 514 515/* ARGSUSED */ 516void 517sfmmu_copytte(tte_t *sttep, tte_t *dttep) 518{} 519 520/*ARGSUSED*/ 521struct tsbe * 522sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 523{ return(0); } 524 525/*ARGSUSED*/ 526uint64_t 527sfmmu_make_tsbtag(caddr_t va) 528{ return(0); } 529 530#else /* lint */ 531 532 .seg ".data" 533 .global sfmmu_panic1 534sfmmu_panic1: 535 .asciz "sfmmu_asm: interrupts already disabled" 536 537 .global sfmmu_panic3 538sfmmu_panic3: 539 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 540 541 .global sfmmu_panic4 542sfmmu_panic4: 543 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 544 545 .global sfmmu_panic5 546sfmmu_panic5: 547 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 548 549 .global sfmmu_panic6 550sfmmu_panic6: 551 .asciz "sfmmu_asm: interrupts not disabled" 552 553 .global sfmmu_panic7 554sfmmu_panic7: 555 .asciz "sfmmu_asm: kernel as" 556 557 .global sfmmu_panic8 558sfmmu_panic8: 559 .asciz "sfmmu_asm: gnum is zero" 560 561 .global sfmmu_panic9 562sfmmu_panic9: 563 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 564 565 .global sfmmu_panic10 566sfmmu_panic10: 567 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 568 569 ENTRY(sfmmu_disable_intrs) 570 rdpr %pstate, %o0 571#ifdef DEBUG 572 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 573#endif /* DEBUG */ 574 retl 575 wrpr %o0, PSTATE_IE, %pstate 576 SET_SIZE(sfmmu_disable_intrs) 577 578 ENTRY(sfmmu_enable_intrs) 579 retl 580 wrpr %g0, %o0, %pstate 581 SET_SIZE(sfmmu_enable_intrs) 582 583/* 584 * This routine is called both by resume() and sfmmu_get_ctx() to 585 * allocate a new context for the process on a MMU. 586 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 587 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 588 * is the case when sfmmu_alloc_ctx is called from resume(). 589 * 590 * The caller must disable interrupts before entering this routine. 591 * To reduce ctx switch overhead, the code contains both 'fast path' and 592 * 'slow path' code. The fast path code covers the common case where only 593 * a quick check is needed and the real ctx allocation is not required. 594 * It can be done without holding the per-process (PP) lock. 595 * The 'slow path' code must be protected by the PP Lock and performs ctx 596 * allocation. 597 * Hardware context register and HAT mmu cnum are updated accordingly. 598 * 599 * %o0 - sfmmup 600 * %o1 - allocflag 601 * %o2 - CPU 602 * %o3 - sfmmu private/shared flag 603 * 604 * ret - 0: no ctx is allocated 605 * 1: a ctx is allocated 606 */ 607 ENTRY_NP(sfmmu_alloc_ctx) 608 609#ifdef DEBUG 610 sethi %hi(ksfmmup), %g1 611 ldx [%g1 + %lo(ksfmmup)], %g1 612 cmp %g1, %o0 613 bne,pt %xcc, 0f 614 nop 615 616 sethi %hi(panicstr), %g1 ! if kernel as, panic 617 ldx [%g1 + %lo(panicstr)], %g1 618 tst %g1 619 bnz,pn %icc, 7f 620 nop 621 622 sethi %hi(sfmmu_panic7), %o0 623 call panic 624 or %o0, %lo(sfmmu_panic7), %o0 625 6267: 627 retl 628 mov %g0, %o0 ! %o0 = ret = 0 629 6300: 631 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 632#endif /* DEBUG */ 633 634 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 635 636 ! load global mmu_ctxp info 637 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 638 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 639 640 ! load global mmu_ctxp gnum 641 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 642 643#ifdef DEBUG 644 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 645 bne,pt %xcc, 3f 646 nop 647 648 sethi %hi(panicstr), %g1 ! test if panicstr is already set 649 ldx [%g1 + %lo(panicstr)], %g1 650 tst %g1 651 bnz,pn %icc, 1f 652 nop 653 654 sethi %hi(sfmmu_panic8), %o0 655 call panic 656 or %o0, %lo(sfmmu_panic8), %o0 6571: 658 retl 659 mov %g0, %o0 ! %o0 = ret = 0 6603: 661#endif 662 663 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 664 665 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 666 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 667 668 /* 669 * %g5 = sfmmu gnum returned 670 * %g6 = sfmmu cnum returned 671 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 672 * %g4 = scratch 673 * 674 * Fast path code, do a quick check. 675 */ 676 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 677 678 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 679 bne,pt %icc, 1f ! valid hat cnum, check gnum 680 nop 681 682 ! cnum == INVALID, check allocflag 683 mov %g0, %g4 ! %g4 = ret = 0 684 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 685 mov %g6, %o1 686 687 ! (invalid HAT cnum) && (allocflag == 1) 688 ba,pt %icc, 2f 689 nop 6901: 691 ! valid HAT cnum, check gnum 692 cmp %g5, %o4 693 mov 1, %g4 !%g4 = ret = 1 694 be,a,pt %icc, 8f ! gnum unchanged, go to done 695 mov %g6, %o1 696 6972: 698 /* 699 * Grab per process (PP) sfmmu_ctx_lock spinlock, 700 * followed by the 'slow path' code. 701 */ 702 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 7033: 704 brz %g3, 5f 705 nop 7064: 707 brnz,a,pt %g3, 4b ! spin if lock is 1 708 ldub [%o0 + SFMMU_CTX_LOCK], %g3 709 ba %xcc, 3b ! retry the lock 710 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 711 7125: 713 membar #LoadLoad 714 /* 715 * %g5 = sfmmu gnum returned 716 * %g6 = sfmmu cnum returned 717 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 718 * %g4 = scratch 719 */ 720 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 721 722 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 723 bne,pt %icc, 1f ! valid hat cnum, check gnum 724 nop 725 726 ! cnum == INVALID, check allocflag 727 mov %g0, %g4 ! %g4 = ret = 0 728 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 729 mov %g6, %o1 730 731 ! (invalid HAT cnum) && (allocflag == 1) 732 ba,pt %icc, 6f 733 nop 7341: 735 ! valid HAT cnum, check gnum 736 cmp %g5, %o4 737 mov 1, %g4 ! %g4 = ret = 1 738 be,a,pt %icc, 2f ! gnum unchanged, go to done 739 mov %g6, %o1 740 741 ba,pt %icc, 6f 742 nop 7432: 744 membar #LoadStore|#StoreStore 745 ba,pt %icc, 8f 746 clrb [%o0 + SFMMU_CTX_LOCK] 7476: 748 /* 749 * We get here if we do not have a valid context, or 750 * the HAT gnum does not match global gnum. We hold 751 * sfmmu_ctx_lock spinlock. Allocate that context. 752 * 753 * %o3 = mmu_ctxp 754 */ 755 add %o3, MMU_CTX_CNUM, %g3 756 ld [%o3 + MMU_CTX_NCTXS], %g4 757 758 /* 759 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 760 * %g3 = mmu cnum address 761 * %g4 = mmu nctxs 762 * 763 * %o0 = sfmmup 764 * %o1 = mmu current cnum value (used as new cnum) 765 * %o4 = mmu gnum 766 * 767 * %o5 = scratch 768 */ 769 ld [%g3], %o1 7700: 771 cmp %o1, %g4 772 bl,a,pt %icc, 1f 773 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 774 775 /* 776 * cnum reachs max, bail, so wrap around can be performed later. 777 */ 778 set INVALID_CONTEXT, %o1 779 mov %g0, %g4 ! %g4 = ret = 0 780 781 membar #LoadStore|#StoreStore 782 ba,pt %icc, 8f 783 clrb [%o0 + SFMMU_CTX_LOCK] 7841: 785 ! %g3 = addr of mmu_ctxp->cnum 786 ! %o5 = mmu_ctxp->cnum + 1 787 cas [%g3], %o1, %o5 788 cmp %o1, %o5 789 bne,a,pn %xcc, 0b ! cas failed 790 ld [%g3], %o1 791 792#ifdef DEBUG 793 set MAX_SFMMU_CTX_VAL, %o5 794 cmp %o1, %o5 795 ble,pt %icc, 2f 796 nop 797 798 sethi %hi(sfmmu_panic9), %o0 799 call panic 800 or %o0, %lo(sfmmu_panic9), %o0 8012: 802#endif 803 ! update hat gnum and cnum 804 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 805 or %o4, %o1, %o4 806 stx %o4, [%g2 + SFMMU_CTXS] 807 808 membar #LoadStore|#StoreStore 809 clrb [%o0 + SFMMU_CTX_LOCK] 810 811 mov 1, %g4 ! %g4 = ret = 1 8128: 813 /* 814 * program the secondary context register 815 * 816 * %o1 = cnum 817 * %g1 = sfmmu private/shared flag (0:private, 1:shared) 818 */ 819 820 /* 821 * When we come here and context is invalid, we want to set both 822 * private and shared ctx regs to INVALID. In order to 823 * do so, we set the sfmmu priv/shared flag to 'private' regardless 824 * so that private ctx reg will be set to invalid. 825 * Note that on sun4v values written to private context register are 826 * automatically written to corresponding shared context register as 827 * well. On sun4u SET_SECCTX() will invalidate shared context register 828 * when it sets a private secondary context register. 829 */ 830 831 cmp %o1, INVALID_CONTEXT 832 be,a,pn %icc, 9f 833 clr %g1 8349: 835 836#ifdef sun4u 837 ldub [%o0 + SFMMU_CEXT], %o2 838 sll %o2, CTXREG_EXT_SHIFT, %o2 839 or %o1, %o2, %o1 840#endif /* sun4u */ 841 842 SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1) 843 844 retl 845 mov %g4, %o0 ! %o0 = ret 846 847 SET_SIZE(sfmmu_alloc_ctx) 848 849 ENTRY_NP(sfmmu_modifytte) 850 ldx [%o2], %g3 /* current */ 851 ldx [%o0], %g1 /* original */ 8522: 853 ldx [%o1], %g2 /* modified */ 854 cmp %g2, %g3 /* is modified = current? */ 855 be,a,pt %xcc,1f /* yes, don't write */ 856 stx %g3, [%o0] /* update new original */ 857 casx [%o2], %g1, %g2 858 cmp %g1, %g2 859 be,pt %xcc, 1f /* cas succeeded - return */ 860 nop 861 ldx [%o2], %g3 /* new current */ 862 stx %g3, [%o0] /* save as new original */ 863 ba,pt %xcc, 2b 864 mov %g3, %g1 8651: retl 866 membar #StoreLoad 867 SET_SIZE(sfmmu_modifytte) 868 869 ENTRY_NP(sfmmu_modifytte_try) 870 ldx [%o1], %g2 /* modified */ 871 ldx [%o2], %g3 /* current */ 872 ldx [%o0], %g1 /* original */ 873 cmp %g3, %g2 /* is modified = current? */ 874 be,a,pn %xcc,1f /* yes, don't write */ 875 mov 0, %o1 /* as if cas failed. */ 876 877 casx [%o2], %g1, %g2 878 membar #StoreLoad 879 cmp %g1, %g2 880 movne %xcc, -1, %o1 /* cas failed. */ 881 move %xcc, 1, %o1 /* cas succeeded. */ 8821: 883 stx %g2, [%o0] /* report "current" value */ 884 retl 885 mov %o1, %o0 886 SET_SIZE(sfmmu_modifytte_try) 887 888 ENTRY_NP(sfmmu_copytte) 889 ldx [%o0], %g1 890 retl 891 stx %g1, [%o1] 892 SET_SIZE(sfmmu_copytte) 893 894 895 /* 896 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 897 * %o0 = TSB base address (in), pointer to TSB entry (out) 898 * %o1 = vaddr (in) 899 * %o2 = vpshift (in) 900 * %o3 = tsb size code (in) 901 * %o4 = scratch register 902 */ 903 ENTRY_NP(sfmmu_get_tsbe) 904 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 905 retl 906 nop 907 SET_SIZE(sfmmu_get_tsbe) 908 909 /* 910 * Return a TSB tag for the given va. 911 * %o0 = va (in/clobbered) 912 * %o0 = va shifted to be in tsb tag format (with no context) (out) 913 */ 914 ENTRY_NP(sfmmu_make_tsbtag) 915 retl 916 srln %o0, TTARGET_VA_SHIFT, %o0 917 SET_SIZE(sfmmu_make_tsbtag) 918 919#endif /* lint */ 920 921/* 922 * Other sfmmu primitives 923 */ 924 925 926#if defined (lint) 927void 928sfmmu_patch_ktsb(void) 929{ 930} 931 932void 933sfmmu_kpm_patch_tlbm(void) 934{ 935} 936 937void 938sfmmu_kpm_patch_tsbm(void) 939{ 940} 941 942void 943sfmmu_patch_shctx(void) 944{ 945} 946 947/* ARGSUSED */ 948void 949sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 950{ 951} 952 953/* ARGSUSED */ 954void 955sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 956{ 957} 958 959/* ARGSUSED */ 960void 961sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 962{ 963} 964 965/* ARGSUSED */ 966void 967sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 968{ 969} 970 971#else /* lint */ 972 973#define I_SIZE 4 974 975 ENTRY_NP(sfmmu_fix_ktlb_traptable) 976 /* 977 * %o0 = start of patch area 978 * %o1 = size code of TSB to patch 979 * %o3 = scratch 980 */ 981 /* fix sll */ 982 ld [%o0], %o3 /* get sll */ 983 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 984 st %o3, [%o0] /* write sll */ 985 flush %o0 986 /* fix srl */ 987 add %o0, I_SIZE, %o0 /* goto next instr. */ 988 ld [%o0], %o3 /* get srl */ 989 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 990 st %o3, [%o0] /* write srl */ 991 retl 992 flush %o0 993 SET_SIZE(sfmmu_fix_ktlb_traptable) 994 995 ENTRY_NP(sfmmu_fixup_ktsbbase) 996 /* 997 * %o0 = start of patch area 998 * %o5 = kernel virtual or physical tsb base address 999 * %o2, %o3 are used as scratch registers. 1000 */ 1001 /* fixup sethi instruction */ 1002 ld [%o0], %o3 1003 srl %o5, 10, %o2 ! offset is bits 32:10 1004 or %o3, %o2, %o3 ! set imm22 1005 st %o3, [%o0] 1006 /* fixup offset of lduw/ldx */ 1007 add %o0, I_SIZE, %o0 ! next instr 1008 ld [%o0], %o3 1009 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 1010 or %o3, %o2, %o3 1011 st %o3, [%o0] 1012 retl 1013 flush %o0 1014 SET_SIZE(sfmmu_fixup_ktsbbase) 1015 1016 ENTRY_NP(sfmmu_fixup_setx) 1017 /* 1018 * %o0 = start of patch area 1019 * %o4 = 64 bit value to patch 1020 * %o2, %o3 are used as scratch registers. 1021 * 1022 * Note: Assuming that all parts of the instructions which need to be 1023 * patched correspond to RUNTIME_PATCH (aka 0) 1024 * 1025 * Note the implementation of setx which is being patched is as follows: 1026 * 1027 * sethi %hh(RUNTIME_PATCH), tmp 1028 * sethi %lm(RUNTIME_PATCH), dest 1029 * or tmp, %hm(RUNTIME_PATCH), tmp 1030 * or dest, %lo(RUNTIME_PATCH), dest 1031 * sllx tmp, 32, tmp 1032 * nop 1033 * or tmp, dest, dest 1034 * 1035 * which differs from the implementation in the 1036 * "SPARC Architecture Manual" 1037 */ 1038 /* fixup sethi instruction */ 1039 ld [%o0], %o3 1040 srlx %o4, 42, %o2 ! bits [63:42] 1041 or %o3, %o2, %o3 ! set imm22 1042 st %o3, [%o0] 1043 /* fixup sethi instruction */ 1044 add %o0, I_SIZE, %o0 ! next instr 1045 ld [%o0], %o3 1046 sllx %o4, 32, %o2 ! clear upper bits 1047 srlx %o2, 42, %o2 ! bits [31:10] 1048 or %o3, %o2, %o3 ! set imm22 1049 st %o3, [%o0] 1050 /* fixup or instruction */ 1051 add %o0, I_SIZE, %o0 ! next instr 1052 ld [%o0], %o3 1053 srlx %o4, 32, %o2 ! bits [63:32] 1054 and %o2, 0x3ff, %o2 ! bits [41:32] 1055 or %o3, %o2, %o3 ! set imm 1056 st %o3, [%o0] 1057 /* fixup or instruction */ 1058 add %o0, I_SIZE, %o0 ! next instr 1059 ld [%o0], %o3 1060 and %o4, 0x3ff, %o2 ! bits [9:0] 1061 or %o3, %o2, %o3 ! set imm 1062 st %o3, [%o0] 1063 retl 1064 flush %o0 1065 SET_SIZE(sfmmu_fixup_setx) 1066 1067 ENTRY_NP(sfmmu_fixup_or) 1068 /* 1069 * %o0 = start of patch area 1070 * %o4 = 32 bit value to patch 1071 * %o2, %o3 are used as scratch registers. 1072 * Note: Assuming that all parts of the instructions which need to be 1073 * patched correspond to RUNTIME_PATCH (aka 0) 1074 */ 1075 ld [%o0], %o3 1076 and %o4, 0x3ff, %o2 ! bits [9:0] 1077 or %o3, %o2, %o3 ! set imm 1078 st %o3, [%o0] 1079 retl 1080 flush %o0 1081 SET_SIZE(sfmmu_fixup_or) 1082 1083 ENTRY_NP(sfmmu_fixup_shiftx) 1084 /* 1085 * %o0 = start of patch area 1086 * %o4 = signed int immediate value to add to sllx/srlx imm field 1087 * %o2, %o3 are used as scratch registers. 1088 * 1089 * sllx/srlx store the 6 bit immediate value in the lowest order bits 1090 * so we do a simple add. The caller must be careful to prevent 1091 * overflow, which could easily occur if the initial value is nonzero! 1092 */ 1093 ld [%o0], %o3 ! %o3 = instruction to patch 1094 and %o3, 0x3f, %o2 ! %o2 = existing imm value 1095 add %o2, %o4, %o2 ! %o2 = new imm value 1096 andn %o3, 0x3f, %o3 ! clear old imm value 1097 and %o2, 0x3f, %o2 ! truncate new imm value 1098 or %o3, %o2, %o3 ! set new imm value 1099 st %o3, [%o0] ! store updated instruction 1100 retl 1101 flush %o0 1102 SET_SIZE(sfmmu_fixup_shiftx) 1103 1104 ENTRY_NP(sfmmu_fixup_mmu_asi) 1105 /* 1106 * Patch imm_asi of all ldda instructions in the MMU 1107 * trap handlers. We search MMU_PATCH_INSTR instructions 1108 * starting from the itlb miss handler (trap 0x64). 1109 * %o0 = address of tt[0,1]_itlbmiss 1110 * %o1 = imm_asi to setup, shifted by appropriate offset. 1111 * %o3 = number of instructions to search 1112 * %o4 = reserved by caller: called from leaf routine 1113 */ 11141: ldsw [%o0], %o2 ! load instruction to %o2 1115 brgez,pt %o2, 2f 1116 srl %o2, 30, %o5 1117 btst 1, %o5 ! test bit 30; skip if not set 1118 bz,pt %icc, 2f 1119 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1120 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1121 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1122 brnz,pt %o5, 2f ! skip if not a match 1123 or %o2, %o1, %o2 ! or in imm_asi 1124 st %o2, [%o0] ! write patched instruction 11252: dec %o3 1126 brnz,a,pt %o3, 1b ! loop until we're done 1127 add %o0, I_SIZE, %o0 1128 retl 1129 flush %o0 1130 SET_SIZE(sfmmu_fixup_mmu_asi) 1131 1132 /* 1133 * Patch immediate ASI used to access the TSB in the 1134 * trap table. 1135 * inputs: %o0 = value of ktsb_phys 1136 */ 1137 ENTRY_NP(sfmmu_patch_mmu_asi) 1138 mov %o7, %o4 ! save return pc in %o4 1139 movrnz %o0, ASI_QUAD_LDD_PHYS, %o3 1140 movrz %o0, ASI_NQUAD_LD, %o3 1141 sll %o3, 5, %o1 ! imm_asi offset 1142 mov 6, %o3 ! number of instructions 1143 sethi %hi(dktsb), %o0 ! to search 1144 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1145 or %o0, %lo(dktsb), %o0 1146 mov 6, %o3 ! number of instructions 1147 sethi %hi(dktsb4m), %o0 ! to search 1148 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1149 or %o0, %lo(dktsb4m), %o0 1150 mov 6, %o3 ! number of instructions 1151 sethi %hi(iktsb), %o0 ! to search 1152 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1153 or %o0, %lo(iktsb), %o0 1154 mov 6, %o3 ! number of instructions 1155 sethi %hi(iktsb4m), %o0 ! to search 1156 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1157 or %o0, %lo(iktsb4m), %o0 1158 mov %o4, %o7 ! retore return pc -- leaf 1159 retl 1160 nop 1161 SET_SIZE(sfmmu_patch_mmu_asi) 1162 1163 ENTRY_NP(sfmmu_patch_ktsb) 1164 /* 1165 * We need to fix iktsb, dktsb, et. al. 1166 */ 1167 save %sp, -SA(MINFRAME), %sp 1168 set ktsb_phys, %o1 1169 ld [%o1], %o4 1170 set ktsb_base, %o5 1171 set ktsb4m_base, %l1 1172 brz,pt %o4, 1f 1173 nop 1174 set ktsb_pbase, %o5 1175 set ktsb4m_pbase, %l1 11761: 1177 sethi %hi(ktsb_szcode), %o1 1178 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1179 1180 sethi %hi(iktsb), %o0 1181 call sfmmu_fix_ktlb_traptable 1182 or %o0, %lo(iktsb), %o0 1183 1184 sethi %hi(dktsb), %o0 1185 call sfmmu_fix_ktlb_traptable 1186 or %o0, %lo(dktsb), %o0 1187 1188 sethi %hi(ktsb4m_szcode), %o1 1189 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1190 1191 sethi %hi(iktsb4m), %o0 1192 call sfmmu_fix_ktlb_traptable 1193 or %o0, %lo(iktsb4m), %o0 1194 1195 sethi %hi(dktsb4m), %o0 1196 call sfmmu_fix_ktlb_traptable 1197 or %o0, %lo(dktsb4m), %o0 1198 1199#ifndef sun4v 1200 mov ASI_N, %o2 1201 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1202 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1203 sethi %hi(tsb_kernel_patch_asi), %o0 1204 call sfmmu_fixup_or 1205 or %o0, %lo(tsb_kernel_patch_asi), %o0 1206#endif /* !sun4v */ 1207 1208 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1209 1210 sethi %hi(dktsbbase), %o0 1211 call sfmmu_fixup_setx ! patch value of ktsb base addr 1212 or %o0, %lo(dktsbbase), %o0 1213 1214 sethi %hi(iktsbbase), %o0 1215 call sfmmu_fixup_setx ! patch value of ktsb base addr 1216 or %o0, %lo(iktsbbase), %o0 1217 1218 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1219 call sfmmu_fixup_setx ! patch value of ktsb base addr 1220 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1221 1222#ifdef sun4v 1223 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1224 call sfmmu_fixup_setx ! patch value of ktsb base addr 1225 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1226#endif /* sun4v */ 1227 1228 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1229 1230 sethi %hi(dktsb4mbase), %o0 1231 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1232 or %o0, %lo(dktsb4mbase), %o0 1233 1234 sethi %hi(iktsb4mbase), %o0 1235 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1236 or %o0, %lo(iktsb4mbase), %o0 1237 1238 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1239 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1240 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1241 1242#ifdef sun4v 1243 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1244 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1245 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1246#endif /* sun4v */ 1247 1248 set ktsb_szcode, %o4 1249 ld [%o4], %o4 1250 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1251 call sfmmu_fixup_or ! patch value of ktsb_szcode 1252 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1253 1254#ifdef sun4v 1255 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1256 call sfmmu_fixup_or ! patch value of ktsb_szcode 1257 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1258#endif /* sun4v */ 1259 1260 set ktsb4m_szcode, %o4 1261 ld [%o4], %o4 1262 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1263 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1264 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1265 1266#ifdef sun4v 1267 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1268 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1269 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1270#endif /* sun4v */ 1271 1272 ret 1273 restore 1274 SET_SIZE(sfmmu_patch_ktsb) 1275 1276 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1277 /* 1278 * Fixup trap handlers in common segkpm case. This is reserved 1279 * for future use should kpm TSB be changed to be other than the 1280 * kernel TSB. 1281 */ 1282 retl 1283 nop 1284 SET_SIZE(sfmmu_kpm_patch_tlbm) 1285 1286 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1287 /* 1288 * nop the branch to sfmmu_kpm_dtsb_miss_small 1289 * in the case where we are using large pages for 1290 * seg_kpm (and hence must probe the second TSB for 1291 * seg_kpm VAs) 1292 */ 1293 set dktsb4m_kpmcheck_small, %o0 1294 MAKE_NOP_INSTR(%o1) 1295 st %o1, [%o0] 1296 flush %o0 1297 retl 1298 nop 1299 SET_SIZE(sfmmu_kpm_patch_tsbm) 1300 1301 ENTRY_NP(sfmmu_patch_utsb) 1302#ifdef UTSB_PHYS 1303 retl 1304 nop 1305#else /* UTSB_PHYS */ 1306 /* 1307 * We need to hot patch utsb_vabase and utsb4m_vabase 1308 */ 1309 save %sp, -SA(MINFRAME), %sp 1310 1311 /* patch value of utsb_vabase */ 1312 set utsb_vabase, %o1 1313 ldx [%o1], %o4 1314 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1315 call sfmmu_fixup_setx 1316 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1317 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1318 call sfmmu_fixup_setx 1319 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1320 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1321 call sfmmu_fixup_setx 1322 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1323 1324 /* patch value of utsb4m_vabase */ 1325 set utsb4m_vabase, %o1 1326 ldx [%o1], %o4 1327 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1328 call sfmmu_fixup_setx 1329 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1330 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1331 call sfmmu_fixup_setx 1332 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1333 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1334 call sfmmu_fixup_setx 1335 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1336 1337 /* 1338 * Patch TSB base register masks and shifts if needed. 1339 * By default the TSB base register contents are set up for 4M slab. 1340 * If we're using a smaller slab size and reserved VA range we need 1341 * to patch up those values here. 1342 */ 1343 set tsb_slab_shift, %o1 1344 set MMU_PAGESHIFT4M, %o4 1345 lduw [%o1], %o3 1346 subcc %o4, %o3, %o4 1347 bz,pt %icc, 1f 1348 /* delay slot safe */ 1349 1350 /* patch reserved VA range size if needed. */ 1351 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1352 call sfmmu_fixup_shiftx 1353 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1354 call sfmmu_fixup_shiftx 1355 add %o0, I_SIZE, %o0 1356 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1357 call sfmmu_fixup_shiftx 1358 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1359 call sfmmu_fixup_shiftx 1360 add %o0, I_SIZE, %o0 13611: 1362 /* patch TSBREG_VAMASK used to set up TSB base register */ 1363 set tsb_slab_mask, %o1 1364 ldx [%o1], %o4 1365 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1366 call sfmmu_fixup_or 1367 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1368 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1369 call sfmmu_fixup_or 1370 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1371 1372 ret 1373 restore 1374#endif /* UTSB_PHYS */ 1375 SET_SIZE(sfmmu_patch_utsb) 1376 1377 ENTRY_NP(sfmmu_patch_shctx) 1378#ifdef sun4u 1379 retl 1380 nop 1381#else /* sun4u */ 1382 set sfmmu_shctx_cpu_mondo_patch, %o0 1383 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1384 st %o1, [%o0] 1385 flush %o0 1386 MAKE_NOP_INSTR(%o1) 1387 add %o0, I_SIZE, %o0 ! next instr 1388 st %o1, [%o0] 1389 flush %o0 1390 1391 set sfmmu_shctx_user_rtt_patch, %o0 1392 st %o1, [%o0] ! nop 1st instruction 1393 flush %o0 1394 add %o0, I_SIZE, %o0 1395 st %o1, [%o0] ! nop 2nd instruction 1396 flush %o0 1397 add %o0, I_SIZE, %o0 1398 st %o1, [%o0] ! nop 3rd instruction 1399 flush %o0 1400 add %o0, I_SIZE, %o0 1401 st %o1, [%o0] ! nop 4th instruction 1402 retl 1403 flush %o0 1404#endif /* sun4u */ 1405 SET_SIZE(sfmmu_patch_shctx) 1406 1407 /* 1408 * Routine that loads an entry into a tsb using virtual addresses. 1409 * Locking is required since all cpus can use the same TSB. 1410 * Note that it is no longer required to have a valid context 1411 * when calling this function. 1412 */ 1413 ENTRY_NP(sfmmu_load_tsbe) 1414 /* 1415 * %o0 = pointer to tsbe to load 1416 * %o1 = tsb tag 1417 * %o2 = virtual pointer to TTE 1418 * %o3 = 1 if physical address in %o0 else 0 1419 */ 1420 rdpr %pstate, %o5 1421#ifdef DEBUG 1422 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1423#endif /* DEBUG */ 1424 1425 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1426 1427 SETUP_TSB_ASI(%o3, %g3) 1428 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1) 1429 1430 wrpr %g0, %o5, %pstate /* enable interrupts */ 1431 1432 retl 1433 membar #StoreStore|#StoreLoad 1434 SET_SIZE(sfmmu_load_tsbe) 1435 1436 /* 1437 * Flush TSB of a given entry if the tag matches. 1438 */ 1439 ENTRY(sfmmu_unload_tsbe) 1440 /* 1441 * %o0 = pointer to tsbe to be flushed 1442 * %o1 = tag to match 1443 * %o2 = 1 if physical address in %o0 else 0 1444 */ 1445 SETUP_TSB_ASI(%o2, %g1) 1446 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1447 retl 1448 membar #StoreStore|#StoreLoad 1449 SET_SIZE(sfmmu_unload_tsbe) 1450 1451 /* 1452 * Routine that loads a TTE into the kpm TSB from C code. 1453 * Locking is required since kpm TSB is shared among all CPUs. 1454 */ 1455 ENTRY_NP(sfmmu_kpm_load_tsb) 1456 /* 1457 * %o0 = vaddr 1458 * %o1 = ttep 1459 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1460 */ 1461 rdpr %pstate, %o5 ! %o5 = saved pstate 1462#ifdef DEBUG 1463 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1464#endif /* DEBUG */ 1465 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1466 1467#ifndef sun4v 1468 sethi %hi(ktsb_phys), %o4 1469 mov ASI_N, %o3 1470 ld [%o4 + %lo(ktsb_phys)], %o4 1471 movrnz %o4, ASI_MEM, %o3 1472 mov %o3, %asi 1473#endif /* !sun4v */ 1474 mov %o0, %g1 ! %g1 = vaddr 1475 1476 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1477 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1478 /* %g2 = tsbep, %g1 clobbered */ 1479 1480 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1481 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1482 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1) 1483 1484 wrpr %g0, %o5, %pstate ! enable interrupts 1485 retl 1486 membar #StoreStore|#StoreLoad 1487 SET_SIZE(sfmmu_kpm_load_tsb) 1488 1489 /* 1490 * Routine that shoots down a TTE in the kpm TSB or in the 1491 * kernel TSB depending on virtpg. Locking is required since 1492 * kpm/kernel TSB is shared among all CPUs. 1493 */ 1494 ENTRY_NP(sfmmu_kpm_unload_tsb) 1495 /* 1496 * %o0 = vaddr 1497 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1498 */ 1499#ifndef sun4v 1500 sethi %hi(ktsb_phys), %o4 1501 mov ASI_N, %o3 1502 ld [%o4 + %lo(ktsb_phys)], %o4 1503 movrnz %o4, ASI_MEM, %o3 1504 mov %o3, %asi 1505#endif /* !sun4v */ 1506 mov %o0, %g1 ! %g1 = vaddr 1507 1508 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1509 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1510 /* %g2 = tsbep, %g1 clobbered */ 1511 1512 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1513 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1514 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1515 1516 retl 1517 membar #StoreStore|#StoreLoad 1518 SET_SIZE(sfmmu_kpm_unload_tsb) 1519 1520#endif /* lint */ 1521 1522 1523#if defined (lint) 1524 1525/*ARGSUSED*/ 1526pfn_t 1527sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1528{ return(0); } 1529 1530#else /* lint */ 1531 1532 ENTRY_NP(sfmmu_ttetopfn) 1533 ldx [%o0], %g1 /* read tte */ 1534 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1535 /* 1536 * g1 = pfn 1537 */ 1538 retl 1539 mov %g1, %o0 1540 SET_SIZE(sfmmu_ttetopfn) 1541 1542#endif /* !lint */ 1543 1544 1545#if defined (lint) 1546/* 1547 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the 1548 * the hash list. 1549 */ 1550/* ARGSUSED */ 1551void 1552sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1553 uint64_t hblkpa) 1554{ 1555} 1556 1557/* 1558 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the 1559 * hash list. 1560 */ 1561/* ARGSUSED */ 1562void 1563sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1564 uint64_t hblkpa, struct hme_blk *prev_hblkp) 1565{ 1566} 1567#else /* lint */ 1568 1569/* 1570 * Functions to grab/release hme bucket list lock. I only use a byte 1571 * instead of the whole int because eventually we might want to 1572 * put some counters on the other bytes (of course, these routines would 1573 * have to change). The code that grab this lock should execute 1574 * with interrupts disabled and hold the lock for the least amount of time 1575 * possible. 1576 */ 1577 1578/* 1579 * Even though hmeh_listlock is updated using pa there's no need to flush 1580 * dcache since hmeh_listlock will be restored to the original value (0) 1581 * before interrupts are reenabled. 1582 */ 1583 1584/* 1585 * For sparcv9 hme hash buckets may not be in the nucleus. hme hash update 1586 * routines still use virtual addresses to update the bucket fields. But they 1587 * must not cause a TLB miss after grabbing the low level bucket lock. To 1588 * achieve this we must make sure the bucket structure is completely within an 1589 * 8K page. 1590 */ 1591 1592#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1)) 1593#error - the size of hmehash_bucket structure is not power of 2 1594#endif 1595 1596/* 1597 * Enable backoff to significantly reduce locking overhead and reduce a chance 1598 * of xcall timeout. This is only enabled for sun4v as a Makefile compile- 1599 * time option. 1600 * The rd %ccr is better for performance compared to a non pipeline releasing 1601 * tight spin on N2/VF. 1602 * Backoff based fix is a temporary solution and doesn't allow scaling above 1603 * lock saturation point. The final fix is to eliminate HMELOCK_ENTER() 1604 * to avoid xcall timeouts and improve GET_TTE() performance. 1605 */ 1606 1607#ifdef HMELOCK_BACKOFF_ENABLE 1608 1609#define HMELOCK_BACKOFF(reg, val) \ 1610 set val, reg ;\ 1611 rd %ccr, %g0 ;\ 1612 brnz reg, .-4 ;\ 1613 dec reg 1614 1615#define CAS_HME(tmp1, tmp2, exitlabel, asi) \ 1616 mov 0xff, tmp2 ;\ 1617 casa [tmp1]asi, %g0, tmp2 ;\ 1618 brz,a,pt tmp2, exitlabel ;\ 1619 membar #LoadLoad 1620 1621#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label, asi) \ 1622 mov 0xff, tmp2 ;\ 1623 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1624 casa [tmp1]asi, %g0, tmp2 ;\ 1625 brz,a,pt tmp2, label/**/2 ;\ 1626 membar #LoadLoad ;\ 1627 HMELOCK_BACKOFF(tmp2,0x8) ;\ 1628 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1629 HMELOCK_BACKOFF(tmp2,0x10) ;\ 1630 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1631 HMELOCK_BACKOFF(tmp2,0x20) ;\ 1632 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1633 HMELOCK_BACKOFF(tmp2,0x40) ;\ 1634 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1635 HMELOCK_BACKOFF(tmp2,0x80) ;\ 1636 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1637label/**/1: ;\ 1638 HMELOCK_BACKOFF(tmp2,0x100) ;\ 1639 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1640 HMELOCK_BACKOFF(tmp2,0x200) ;\ 1641 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1642 HMELOCK_BACKOFF(tmp2,0x400) ;\ 1643 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1644 HMELOCK_BACKOFF(tmp2,0x800) ;\ 1645 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1646 HMELOCK_BACKOFF(tmp2,0x1000) ;\ 1647 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1648 HMELOCK_BACKOFF(tmp2,0x2000) ;\ 1649 mov 0xff, tmp2 ;\ 1650 casa [tmp1]asi, %g0, tmp2 ;\ 1651 brnz,pn tmp2, label/**/1 /* reset backoff */ ;\ 1652 membar #LoadLoad ;\ 1653label/**/2: 1654 1655#else /* HMELOCK_BACKOFF_ENABLE */ 1656 1657#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi) \ 1658 mov 0xff, tmp2 ;\ 1659 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1660label1: ;\ 1661 casa [tmp1]asi, %g0, tmp2 ;\ 1662 brnz,pn tmp2, label1 ;\ 1663 mov 0xff, tmp2 ;\ 1664 membar #LoadLoad 1665 1666#endif /* HMELOCK_BACKOFF_ENABLE */ 1667 1668#define HMELOCK_EXIT(hmebp, tmp1, asi) \ 1669 membar #LoadStore|#StoreStore ;\ 1670 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1671 sta %g0, [tmp1]asi 1672 1673 .seg ".data" 1674hblk_add_panic1: 1675 .ascii "sfmmu_hblk_hash_add: interrupts disabled" 1676 .byte 0 1677hblk_add_panic2: 1678 .ascii "sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not" 1679 .byte 0 1680 .align 4 1681 .seg ".text" 1682 1683 ENTRY_NP(sfmmu_hblk_hash_add) 1684 /* 1685 * %o0 = hmebp 1686 * %o1 = hmeblkp 1687 * %o2 = hblkpa 1688 */ 1689 rdpr %pstate, %o5 1690#ifdef DEBUG 1691 andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ 1692 bnz,pt %icc, 3f /* disabled, panic */ 1693 nop 1694 save %sp, -SA(MINFRAME), %sp 1695 sethi %hi(hblk_add_panic1), %o0 1696 call panic 1697 or %o0, %lo(hblk_add_panic1), %o0 1698 ret 1699 restore 1700 17013: 1702#endif /* DEBUG */ 1703 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1704 mov %o2, %g1 1705 1706 /* 1707 * g1 = hblkpa 1708 */ 1709 ldn [%o0 + HMEBUCK_HBLK], %o4 /* next hmeblk */ 1710 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = next hblkpa */ 1711#ifdef DEBUG 1712 cmp %o4, %g0 1713 bne,pt %xcc, 1f 1714 nop 1715 brz,pt %g2, 1f 1716 nop 1717 wrpr %g0, %o5, %pstate /* enable interrupts */ 1718 save %sp, -SA(MINFRAME), %sp 1719 sethi %hi(hblk_add_panic2), %o0 1720 call panic 1721 or %o0, %lo(hblk_add_panic2), %o0 1722 ret 1723 restore 17241: 1725#endif /* DEBUG */ 1726 /* 1727 * We update hmeblks entries before grabbing lock because the stores 1728 * could take a tlb miss and require the hash lock. The buckets 1729 * are part of the nucleus so we are cool with those stores. 1730 * 1731 * if buckets are not part of the nucleus our game is to 1732 * not touch any other page via va until we drop the lock. 1733 * This guarantees we won't get a tlb miss before the lock release 1734 * since interrupts are disabled. 1735 */ 1736 stn %o4, [%o1 + HMEBLK_NEXT] /* update hmeblk's next */ 1737 stx %g2, [%o1 + HMEBLK_NEXTPA] /* update hmeblk's next pa */ 1738 HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N) 1739 stn %o1, [%o0 + HMEBUCK_HBLK] /* update bucket hblk next */ 1740 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* add hmeblk to list */ 1741 HMELOCK_EXIT(%o0, %g2, ASI_N) 1742 retl 1743 wrpr %g0, %o5, %pstate /* enable interrupts */ 1744 SET_SIZE(sfmmu_hblk_hash_add) 1745 1746 ENTRY_NP(sfmmu_hblk_hash_rm) 1747 /* 1748 * This function removes an hmeblk from the hash chain. 1749 * It is written to guarantee we don't take a tlb miss 1750 * by using physical addresses to update the list. 1751 * 1752 * %o0 = hmebp 1753 * %o1 = hmeblkp 1754 * %o2 = hmeblkp previous pa 1755 * %o3 = hmeblkp previous 1756 */ 1757 1758 mov %o3, %o4 /* o4 = hmeblkp previous */ 1759 1760 rdpr %pstate, %o5 1761#ifdef DEBUG 1762 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1) 1763#endif /* DEBUG */ 1764 /* 1765 * disable interrupts, clear Address Mask to access 64 bit physaddr 1766 */ 1767 andn %o5, PSTATE_IE, %g1 1768 wrpr %g1, 0, %pstate 1769 1770#ifndef sun4v 1771 sethi %hi(dcache_line_mask), %g4 1772 ld [%g4 + %lo(dcache_line_mask)], %g4 1773#endif /* sun4v */ 1774 1775 /* 1776 * if buckets are not part of the nucleus our game is to 1777 * not touch any other page via va until we drop the lock. 1778 * This guarantees we won't get a tlb miss before the lock release 1779 * since interrupts are disabled. 1780 */ 1781 HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N) 1782 ldn [%o0 + HMEBUCK_HBLK], %g2 /* first hmeblk in list */ 1783 cmp %g2, %o1 1784 bne,pt %ncc,1f 1785 mov ASI_MEM, %asi 1786 /* 1787 * hmeblk is first on list 1788 */ 1789 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = hmeblk pa */ 1790 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1791 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1792 stn %o3, [%o0 + HMEBUCK_HBLK] /* write va */ 1793 ba,pt %xcc, 2f 1794 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* write pa */ 17951: 1796 /* hmeblk is not first on list */ 1797 1798 mov %o2, %g3 1799#ifndef sun4v 1800 GET_CPU_IMPL(%g2) 1801 cmp %g2, CHEETAH_IMPL 1802 bge,a,pt %icc, hblk_hash_rm_1 1803 and %o4, %g4, %g2 1804 cmp %g2, SPITFIRE_IMPL 1805 blt %icc, hblk_hash_rm_2 /* no flushing needed for OPL */ 1806 and %o4, %g4, %g2 1807 stxa %g0, [%g2]ASI_DC_TAG /* flush prev pa from dcache */ 1808 add %o4, HMEBLK_NEXT, %o4 1809 and %o4, %g4, %g2 1810 ba hblk_hash_rm_2 1811 stxa %g0, [%g2]ASI_DC_TAG /* flush prev va from dcache */ 1812hblk_hash_rm_1: 1813 1814 stxa %g0, [%g3]ASI_DC_INVAL /* flush prev pa from dcache */ 1815 membar #Sync 1816 add %g3, HMEBLK_NEXT, %g2 1817 stxa %g0, [%g2]ASI_DC_INVAL /* flush prev va from dcache */ 1818hblk_hash_rm_2: 1819 membar #Sync 1820#endif /* sun4v */ 1821 ldxa [%g3 + HMEBLK_NEXTPA] %asi, %g2 /* g2 = hmeblk pa */ 1822 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1823 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1824 stna %o3, [%g3 + HMEBLK_NEXT] %asi /* write va */ 1825 stxa %g1, [%g3 + HMEBLK_NEXTPA] %asi /* write pa */ 18262: 1827 HMELOCK_EXIT(%o0, %g2, ASI_N) 1828 retl 1829 wrpr %g0, %o5, %pstate /* enable interrupts */ 1830 SET_SIZE(sfmmu_hblk_hash_rm) 1831 1832#endif /* lint */ 1833 1834/* 1835 * These macros are used to update global sfmmu hme hash statistics 1836 * in perf critical paths. It is only enabled in debug kernels or 1837 * if SFMMU_STAT_GATHER is defined 1838 */ 1839#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1840#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1841 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1842 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1843 cmp tmp1, hatid ;\ 1844 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1845 set sfmmu_global_stat, tmp1 ;\ 1846 add tmp1, tmp2, tmp1 ;\ 1847 ld [tmp1], tmp2 ;\ 1848 inc tmp2 ;\ 1849 st tmp2, [tmp1] 1850 1851#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1852 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1853 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1854 cmp tmp1, hatid ;\ 1855 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1856 set sfmmu_global_stat, tmp1 ;\ 1857 add tmp1, tmp2, tmp1 ;\ 1858 ld [tmp1], tmp2 ;\ 1859 inc tmp2 ;\ 1860 st tmp2, [tmp1] 1861 1862 1863#else /* DEBUG || SFMMU_STAT_GATHER */ 1864 1865#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1866 1867#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1868 1869#endif /* DEBUG || SFMMU_STAT_GATHER */ 1870 1871/* 1872 * This macro is used to update global sfmmu kstas in non 1873 * perf critical areas so they are enabled all the time 1874 */ 1875#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1876 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1877 add tmp1, statname, tmp1 ;\ 1878 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1879 inc tmp2 ;\ 1880 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1881 1882/* 1883 * These macros are used to update per cpu stats in non perf 1884 * critical areas so they are enabled all the time 1885 */ 1886#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1887 ld [tsbarea + stat], tmp1 ;\ 1888 inc tmp1 ;\ 1889 st tmp1, [tsbarea + stat] 1890 1891/* 1892 * These macros are used to update per cpu stats in non perf 1893 * critical areas so they are enabled all the time 1894 */ 1895#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1896 lduh [tsbarea + stat], tmp1 ;\ 1897 inc tmp1 ;\ 1898 stuh tmp1, [tsbarea + stat] 1899 1900#if defined(KPM_TLBMISS_STATS_GATHER) 1901 /* 1902 * Count kpm dtlb misses separately to allow a different 1903 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1904 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1905 */ 1906#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1907 brgez tagacc, label /* KPM VA? */ ;\ 1908 nop ;\ 1909 CPU_INDEX(tmp1, tsbma) ;\ 1910 sethi %hi(kpmtsbm_area), tsbma ;\ 1911 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1912 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1913 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1914 /* VA range check */ ;\ 1915 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1916 cmp tagacc, val ;\ 1917 blu,pn %xcc, label ;\ 1918 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1919 cmp tagacc, tmp1 ;\ 1920 bgeu,pn %xcc, label ;\ 1921 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1922 inc val ;\ 1923 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1924label: 1925#else 1926#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1927#endif /* KPM_TLBMISS_STATS_GATHER */ 1928 1929#if defined (lint) 1930/* 1931 * The following routines are jumped to from the mmu trap handlers to do 1932 * the setting up to call systrap. They are separate routines instead of 1933 * being part of the handlers because the handlers would exceed 32 1934 * instructions and since this is part of the slow path the jump 1935 * cost is irrelevant. 1936 */ 1937void 1938sfmmu_pagefault(void) 1939{ 1940} 1941 1942void 1943sfmmu_mmu_trap(void) 1944{ 1945} 1946 1947void 1948sfmmu_window_trap(void) 1949{ 1950} 1951 1952void 1953sfmmu_kpm_exception(void) 1954{ 1955} 1956 1957#else /* lint */ 1958 1959#ifdef PTL1_PANIC_DEBUG 1960 .seg ".data" 1961 .global test_ptl1_panic 1962test_ptl1_panic: 1963 .word 0 1964 .align 8 1965 1966 .seg ".text" 1967 .align 4 1968#endif /* PTL1_PANIC_DEBUG */ 1969 1970 1971 ENTRY_NP(sfmmu_pagefault) 1972 SET_GL_REG(1) 1973 USE_ALTERNATE_GLOBALS(%g5) 1974 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1975 rdpr %tt, %g6 1976 cmp %g6, FAST_IMMU_MISS_TT 1977 be,a,pn %icc, 1f 1978 mov T_INSTR_MMU_MISS, %g3 1979 cmp %g6, T_INSTR_MMU_MISS 1980 be,a,pn %icc, 1f 1981 mov T_INSTR_MMU_MISS, %g3 1982 mov %g5, %g2 1983 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1984 cmp %g6, FAST_DMMU_MISS_TT 1985 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1986 cmp %g6, T_DATA_MMU_MISS 1987 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1988 1989#ifdef PTL1_PANIC_DEBUG 1990 /* check if we want to test the tl1 panic */ 1991 sethi %hi(test_ptl1_panic), %g4 1992 ld [%g4 + %lo(test_ptl1_panic)], %g1 1993 st %g0, [%g4 + %lo(test_ptl1_panic)] 1994 cmp %g1, %g0 1995 bne,a,pn %icc, ptl1_panic 1996 or %g0, PTL1_BAD_DEBUG, %g1 1997#endif /* PTL1_PANIC_DEBUG */ 19981: 1999 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 2000 /* 2001 * g2 = tag access reg 2002 * g3.l = type 2003 * g3.h = 0 2004 */ 2005 sethi %hi(trap), %g1 2006 or %g1, %lo(trap), %g1 20072: 2008 ba,pt %xcc, sys_trap 2009 mov -1, %g4 2010 SET_SIZE(sfmmu_pagefault) 2011 2012 ENTRY_NP(sfmmu_mmu_trap) 2013 SET_GL_REG(1) 2014 USE_ALTERNATE_GLOBALS(%g5) 2015 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 2016 rdpr %tt, %g6 2017 cmp %g6, FAST_IMMU_MISS_TT 2018 be,a,pn %icc, 1f 2019 mov T_INSTR_MMU_MISS, %g3 2020 cmp %g6, T_INSTR_MMU_MISS 2021 be,a,pn %icc, 1f 2022 mov T_INSTR_MMU_MISS, %g3 2023 mov %g5, %g2 2024 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 2025 cmp %g6, FAST_DMMU_MISS_TT 2026 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 2027 cmp %g6, T_DATA_MMU_MISS 2028 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 20291: 2030 /* 2031 * g2 = tag access reg 2032 * g3 = type 2033 */ 2034 sethi %hi(sfmmu_tsbmiss_exception), %g1 2035 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 2036 ba,pt %xcc, sys_trap 2037 mov -1, %g4 2038 /*NOTREACHED*/ 2039 SET_SIZE(sfmmu_mmu_trap) 2040 2041 ENTRY_NP(sfmmu_suspend_tl) 2042 SET_GL_REG(1) 2043 USE_ALTERNATE_GLOBALS(%g5) 2044 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 2045 rdpr %tt, %g6 2046 cmp %g6, FAST_IMMU_MISS_TT 2047 be,a,pn %icc, 1f 2048 mov T_INSTR_MMU_MISS, %g3 2049 mov %g5, %g2 2050 cmp %g6, FAST_DMMU_MISS_TT 2051 move %icc, T_DATA_MMU_MISS, %g3 2052 movne %icc, T_DATA_PROT, %g3 20531: 2054 sethi %hi(sfmmu_tsbmiss_suspended), %g1 2055 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 2056 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 2057 ba,pt %xcc, sys_trap 2058 mov PIL_15, %g4 2059 /*NOTREACHED*/ 2060 SET_SIZE(sfmmu_suspend_tl) 2061 2062 /* 2063 * No %g registers in use at this point. 2064 */ 2065 ENTRY_NP(sfmmu_window_trap) 2066 rdpr %tpc, %g1 2067#ifdef sun4v 2068#ifdef DEBUG 2069 /* We assume previous %gl was 1 */ 2070 rdpr %tstate, %g4 2071 srlx %g4, TSTATE_GL_SHIFT, %g4 2072 and %g4, TSTATE_GL_MASK, %g4 2073 cmp %g4, 1 2074 bne,a,pn %icc, ptl1_panic 2075 mov PTL1_BAD_WTRAP, %g1 2076#endif /* DEBUG */ 2077 /* user miss at tl>1. better be the window handler or user_rtt */ 2078 /* in user_rtt? */ 2079 set rtt_fill_start, %g4 2080 cmp %g1, %g4 2081 blu,pn %xcc, 6f 2082 .empty 2083 set rtt_fill_end, %g4 2084 cmp %g1, %g4 2085 bgeu,pn %xcc, 6f 2086 nop 2087 set fault_rtt_fn1, %g1 2088 wrpr %g0, %g1, %tnpc 2089 ba,a 7f 20906: 2091 ! must save this trap level before descending trap stack 2092 ! no need to save %tnpc, either overwritten or discarded 2093 ! already got it: rdpr %tpc, %g1 2094 rdpr %tstate, %g6 2095 rdpr %tt, %g7 2096 ! trap level saved, go get underlying trap type 2097 rdpr %tl, %g5 2098 sub %g5, 1, %g3 2099 wrpr %g3, %tl 2100 rdpr %tt, %g2 2101 wrpr %g5, %tl 2102 ! restore saved trap level 2103 wrpr %g1, %tpc 2104 wrpr %g6, %tstate 2105 wrpr %g7, %tt 2106#else /* sun4v */ 2107 /* user miss at tl>1. better be the window handler */ 2108 rdpr %tl, %g5 2109 sub %g5, 1, %g3 2110 wrpr %g3, %tl 2111 rdpr %tt, %g2 2112 wrpr %g5, %tl 2113#endif /* sun4v */ 2114 and %g2, WTRAP_TTMASK, %g4 2115 cmp %g4, WTRAP_TYPE 2116 bne,pn %xcc, 1f 2117 nop 2118 /* tpc should be in the trap table */ 2119 set trap_table, %g4 2120 cmp %g1, %g4 2121 blt,pn %xcc, 1f 2122 .empty 2123 set etrap_table, %g4 2124 cmp %g1, %g4 2125 bge,pn %xcc, 1f 2126 .empty 2127 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 2128 add %g1, WTRAP_FAULTOFF, %g1 2129 wrpr %g0, %g1, %tnpc 21307: 2131 /* 2132 * some wbuf handlers will call systrap to resolve the fault 2133 * we pass the trap type so they figure out the correct parameters. 2134 * g5 = trap type, g6 = tag access reg 2135 */ 2136 2137 /* 2138 * only use g5, g6, g7 registers after we have switched to alternate 2139 * globals. 2140 */ 2141 SET_GL_REG(1) 2142 USE_ALTERNATE_GLOBALS(%g5) 2143 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 2144 rdpr %tt, %g7 2145 cmp %g7, FAST_IMMU_MISS_TT 2146 be,a,pn %icc, ptl1_panic 2147 mov PTL1_BAD_WTRAP, %g1 2148 cmp %g7, T_INSTR_MMU_MISS 2149 be,a,pn %icc, ptl1_panic 2150 mov PTL1_BAD_WTRAP, %g1 2151 mov T_DATA_PROT, %g5 2152 cmp %g7, FAST_DMMU_MISS_TT 2153 move %icc, T_DATA_MMU_MISS, %g5 2154 cmp %g7, T_DATA_MMU_MISS 2155 move %icc, T_DATA_MMU_MISS, %g5 2156 ! XXXQ AGS re-check out this one 2157 done 21581: 2159 CPU_PADDR(%g1, %g4) 2160 add %g1, CPU_TL1_HDLR, %g1 2161 lda [%g1]ASI_MEM, %g4 2162 brnz,a,pt %g4, sfmmu_mmu_trap 2163 sta %g0, [%g1]ASI_MEM 2164 ba,pt %icc, ptl1_panic 2165 mov PTL1_BAD_TRAP, %g1 2166 SET_SIZE(sfmmu_window_trap) 2167 2168 ENTRY_NP(sfmmu_kpm_exception) 2169 /* 2170 * We have accessed an unmapped segkpm address or a legal segkpm 2171 * address which is involved in a VAC alias conflict prevention. 2172 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 2173 * set. If it is, we will instead note that a fault has occurred 2174 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 2175 * a "retry"). This will step over the faulting instruction. 2176 * Note that this means that a legal segkpm address involved in 2177 * a VAC alias conflict prevention (a rare case to begin with) 2178 * cannot be used in DTrace. 2179 */ 2180 CPU_INDEX(%g1, %g2) 2181 set cpu_core, %g2 2182 sllx %g1, CPU_CORE_SHIFT, %g1 2183 add %g1, %g2, %g1 2184 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 2185 andcc %g2, CPU_DTRACE_NOFAULT, %g0 2186 bz 0f 2187 or %g2, CPU_DTRACE_BADADDR, %g2 2188 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 2189 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 2190 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 2191 done 21920: 2193 TSTAT_CHECK_TL1(1f, %g1, %g2) 21941: 2195 SET_GL_REG(1) 2196 USE_ALTERNATE_GLOBALS(%g5) 2197 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 2198 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 2199 /* 2200 * g2=tagacc g3.l=type g3.h=0 2201 */ 2202 sethi %hi(trap), %g1 2203 or %g1, %lo(trap), %g1 2204 ba,pt %xcc, sys_trap 2205 mov -1, %g4 2206 SET_SIZE(sfmmu_kpm_exception) 2207 2208#endif /* lint */ 2209 2210#if defined (lint) 2211 2212void 2213sfmmu_tsb_miss(void) 2214{ 2215} 2216 2217void 2218sfmmu_kpm_dtsb_miss(void) 2219{ 2220} 2221 2222void 2223sfmmu_kpm_dtsb_miss_small(void) 2224{ 2225} 2226 2227#else /* lint */ 2228 2229#if (IMAP_SEG != 0) 2230#error - ism_map->ism_seg offset is not zero 2231#endif 2232 2233/* 2234 * Copies ism mapping for this ctx in param "ism" if this is a ISM 2235 * tlb miss and branches to label "ismhit". If this is not an ISM 2236 * process or an ISM tlb miss it falls thru. 2237 * 2238 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 2239 * this process. 2240 * If so, it will branch to label "ismhit". If not, it will fall through. 2241 * 2242 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 2243 * so that any other threads of this process will not try and walk the ism 2244 * maps while they are being changed. 2245 * 2246 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 2247 * will make sure of that. This means we can terminate our search on 2248 * the first zero mapping we find. 2249 * 2250 * Parameters: 2251 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 2252 * tsbmiss = address of tsb miss area (in) 2253 * ismseg = contents of ism_seg for this ism map (out) 2254 * ismhat = physical address of imap_ismhat for this ism map (out) 2255 * tmp1 = scratch reg (CLOBBERED) 2256 * tmp2 = scratch reg (CLOBBERED) 2257 * tmp3 = scratch reg (CLOBBERED) 2258 * label: temporary labels 2259 * ismhit: label where to jump to if an ism dtlb miss 2260 * exitlabel:label where to jump if hat is busy due to hat_unshare. 2261 */ 2262#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 2263 label, ismhit) \ 2264 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 2265 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 2266 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 2267label/**/1: ;\ 2268 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 2269 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 2270label/**/2: ;\ 2271 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 2272 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 2273 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 2274 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 2275 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 2276 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 2277 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 2278 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 2279 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 2280 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 2281 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 2282 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 2283 ;\ 2284 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 2285 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 2286 cmp ismhat, tmp1 ;\ 2287 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 2288 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 2289 ;\ 2290 add tmp3, IBLK_NEXTPA, tmp1 ;\ 2291 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 2292 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 2293 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 2294label/**/3: 2295 2296/* 2297 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 2298 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 2299 * Parameters: 2300 * tagacc = reg containing virtual address 2301 * hatid = reg containing sfmmu pointer 2302 * hmeshift = constant/register to shift vaddr to obtain vapg 2303 * hmebp = register where bucket pointer will be stored 2304 * vapg = register where virtual page will be stored 2305 * tmp1, tmp2 = tmp registers 2306 */ 2307 2308 2309#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 2310 vapg, label, tmp1, tmp2) \ 2311 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 2312 brnz,a,pt tmp1, label/**/1 ;\ 2313 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 2314 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 2315 ba,pt %xcc, label/**/2 ;\ 2316 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 2317label/**/1: ;\ 2318 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 2319label/**/2: ;\ 2320 srlx tagacc, hmeshift, vapg ;\ 2321 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 2322 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 2323 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 2324 add hmebp, tmp1, hmebp 2325 2326/* 2327 * hashtag includes bspage + hashno (64 bits). 2328 */ 2329 2330#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 2331 sllx vapg, hmeshift, vapg ;\ 2332 mov hashno, hblktag ;\ 2333 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 2334 or vapg, hblktag, hblktag 2335 2336/* 2337 * Function to traverse hmeblk hash link list and find corresponding match. 2338 * The search is done using physical pointers. It returns the physical address 2339 * and virtual address pointers to the hmeblk that matches with the tag 2340 * provided. 2341 * Parameters: 2342 * hmebp = register that points to hme hash bucket, also used as 2343 * tmp reg (clobbered) 2344 * hmeblktag = register with hmeblk tag match 2345 * hatid = register with hatid 2346 * hmeblkpa = register where physical ptr will be stored 2347 * hmeblkva = register where virtual ptr will be stored 2348 * tmp1 = tmp reg 2349 * label: temporary label 2350 */ 2351 2352#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2353 tsbarea, tmp1, label) \ 2354 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 2355 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2356 add hmebp, HMEBUCK_HBLK, hmeblkva ;\ 2357 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2358 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2359label/**/1: ;\ 2360 brz,pn hmeblkva, label/**/2 ;\ 2361 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2362 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 2363 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2364 add hmebp, CLONGSIZE, hmebp ;\ 2365 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 2366 xor tmp1, hmeblktag, tmp1 ;\ 2367 xor hmebp, hatid, hmebp ;\ 2368 or hmebp, tmp1, hmebp ;\ 2369 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 2370 add hmeblkpa, HMEBLK_NEXT, hmebp ;\ 2371 ldna [hmebp]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2372 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 2373 ba,pt %xcc, label/**/1 ;\ 2374 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2375label/**/2: 2376 2377/* 2378 * Function to traverse hmeblk hash link list and find corresponding match. 2379 * The search is done using physical pointers. It returns the physical address 2380 * and virtual address pointers to the hmeblk that matches with the tag 2381 * provided. 2382 * Parameters: 2383 * hmeblktag = register with hmeblk tag match (rid field is 0) 2384 * hatid = register with hatid (pointer to SRD) 2385 * hmeblkpa = register where physical ptr will be stored 2386 * hmeblkva = register where virtual ptr will be stored 2387 * tmp1 = tmp reg 2388 * tmp2 = tmp reg 2389 * label: temporary label 2390 */ 2391 2392#define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2393 tsbarea, tmp1, tmp2, label) \ 2394label/**/1: ;\ 2395 brz,pn hmeblkva, label/**/4 ;\ 2396 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 2397 add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 2398 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2399 add tmp2, CLONGSIZE, tmp2 ;\ 2400 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 2401 xor tmp1, hmeblktag, tmp1 ;\ 2402 xor tmp2, hatid, tmp2 ;\ 2403 brz,pn tmp2, label/**/3 /* branch on hit */ ;\ 2404 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2405label/**/2: ;\ 2406 ldna [tmp2]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2407 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2408 ba,pt %xcc, label/**/1 ;\ 2409 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2410label/**/3: ;\ 2411 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 2412 bgeu,pt %xcc, label/**/2 ;\ 2413 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2414 and tmp1, BT_ULMASK, tmp2 ;\ 2415 srlx tmp1, BT_ULSHIFT, tmp1 ;\ 2416 sllx tmp1, CLONGSHIFT, tmp1 ;\ 2417 add tsbarea, tmp1, tmp1 ;\ 2418 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 2419 srlx tmp1, tmp2, tmp1 ;\ 2420 btst 0x1, tmp1 ;\ 2421 bz,pn %xcc, label/**/2 ;\ 2422 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2423label/**/4: 2424 2425#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2426#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2427#endif 2428 2429/* 2430 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2431 * he offset for the corresponding hment. 2432 * Parameters: 2433 * In: 2434 * vaddr = register with virtual address 2435 * hmeblkpa = physical pointer to hme_blk 2436 * Out: 2437 * hmentoff = register where hment offset will be stored 2438 * hmemisc = hblk_misc 2439 * Scratch: 2440 * tmp1 2441 */ 2442#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2443 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2444 lda [hmentoff]ASI_MEM, hmemisc ;\ 2445 andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2446 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2447 or %g0, HMEBLK_HME1, hmentoff ;\ 2448 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2449 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2450 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2451 add tmp1, HMEBLK_HME1, hmentoff ;\ 2452label1: 2453 2454/* 2455 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2456 * 2457 * tagacc = (pseudo-)tag access register (in) 2458 * hatid = sfmmu pointer for TSB miss (in) 2459 * tte = tte for TLB miss if found, otherwise clobbered (out) 2460 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2461 * hmeblkva = VA of hment if found, otherwise clobbered (out) 2462 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2463 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2464 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2465 * for this page size. 2466 * hashno = constant/register hash number 2467 * label = temporary label for branching within macro. 2468 * foundlabel = label to jump to when tte is found. 2469 * suspendlabel= label to jump to when tte is suspended. 2470 * exitlabel = label to jump to when tte is not found. 2471 * 2472 */ 2473#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmemisc, \ 2474 hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \ 2475 ;\ 2476 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2477 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2478 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2479 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2480 ;\ 2481 /* ;\ 2482 * tagacc = tagacc ;\ 2483 * hatid = hatid ;\ 2484 * tsbarea = tsbarea ;\ 2485 * tte = hmebp (hme bucket pointer) ;\ 2486 * hmeblkpa = vapg (virtual page) ;\ 2487 * hmemisc, hmeblkva = scratch ;\ 2488 */ ;\ 2489 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2490 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2491 ;\ 2492 /* ;\ 2493 * tagacc = tagacc ;\ 2494 * hatid = hatid ;\ 2495 * tte = hmebp ;\ 2496 * hmeblkpa = CLOBBERED ;\ 2497 * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2498 * hmeblkva = scratch ;\ 2499 */ ;\ 2500 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2501 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2502 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, hmeblkva, \ 2503 tsbarea, tagacc, label/**/1) ;\ 2504 /* ;\ 2505 * tagacc = CLOBBERED ;\ 2506 * tte = CLOBBERED ;\ 2507 * hmeblkpa = hmeblkpa ;\ 2508 * hmeblkva = hmeblkva ;\ 2509 */ ;\ 2510 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2511 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2512 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2513 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2514 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2515 nop ;\ 2516label/**/4: ;\ 2517 /* ;\ 2518 * We have found the hmeblk containing the hment. ;\ 2519 * Now we calculate the corresponding tte. ;\ 2520 * ;\ 2521 * tagacc = tagacc ;\ 2522 * hatid = hatid ;\ 2523 * tte = clobbered ;\ 2524 * hmeblkpa = hmeblkpa ;\ 2525 * hmemisc = hblktag ;\ 2526 * hmeblkva = hmeblkva ;\ 2527 */ ;\ 2528 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2529 label/**/2) ;\ 2530 ;\ 2531 /* ;\ 2532 * tagacc = tagacc ;\ 2533 * hatid = hmentoff ;\ 2534 * tte = clobbered ;\ 2535 * hmeblkpa = hmeblkpa ;\ 2536 * hmemisc = hblk_misc ;\ 2537 * hmeblkva = hmeblkva ;\ 2538 */ ;\ 2539 ;\ 2540 add hatid, SFHME_TTE, hatid ;\ 2541 add hmeblkpa, hatid, hmeblkpa ;\ 2542 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2543 add hmeblkva, hatid, hmeblkva ;\ 2544 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2545 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2546 set TTE_SUSPEND, hatid ;\ 2547 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2548 btst tte, hatid ;\ 2549 bz,pt %xcc, foundlabel ;\ 2550 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2551 ;\ 2552 /* ;\ 2553 * Mapping is suspended, so goto suspend label. ;\ 2554 */ ;\ 2555 ba,pt %xcc, suspendlabel ;\ 2556 nop 2557 2558/* 2559 * GET_SHME_TTE is similar to GET_TTE() except it searches 2560 * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2561 * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2562 * either 0 (not part of scd) or 1 (part of scd). 2563 */ 2564#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, \ 2565 hmemisc, hmeshift, hashno, label, foundlabel, \ 2566 suspendlabel, exitlabel) \ 2567 ;\ 2568 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2569 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2570 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2571 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2572 ;\ 2573 /* ;\ 2574 * tagacc = tagacc ;\ 2575 * hatid = hatid ;\ 2576 * tsbarea = tsbarea ;\ 2577 * tte = hmebp (hme bucket pointer) ;\ 2578 * hmeblkpa = vapg (virtual page) ;\ 2579 * hmemisc, hmeblkva = scratch ;\ 2580 */ ;\ 2581 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2582 ;\ 2583 /* ;\ 2584 * tagacc = tagacc ;\ 2585 * hatid = hatid ;\ 2586 * tsbarea = tsbarea ;\ 2587 * tte = hmebp ;\ 2588 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2589 * hmeblkpa = CLOBBERED ;\ 2590 * hmeblkva = scratch ;\ 2591 */ ;\ 2592 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2593 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2594 ;\ 2595 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2596 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2597 add tte, HMEBUCK_HBLK, hmeblkva ;\ 2598 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2599 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2600 ;\ 2601label/**/8: ;\ 2602 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, hmeblkva, \ 2603 tsbarea, tagacc, tte, label/**/1) ;\ 2604 /* ;\ 2605 * tagacc = CLOBBERED ;\ 2606 * tte = CLOBBERED ;\ 2607 * hmeblkpa = hmeblkpa ;\ 2608 * hmeblkva = hmeblkva ;\ 2609 */ ;\ 2610 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2611 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2612 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2613 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2614 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2615 nop ;\ 2616label/**/4: ;\ 2617 /* ;\ 2618 * We have found the hmeblk containing the hment. ;\ 2619 * Now we calculate the corresponding tte. ;\ 2620 * ;\ 2621 * tagacc = tagacc ;\ 2622 * hatid = hatid ;\ 2623 * tte = clobbered ;\ 2624 * hmeblkpa = hmeblkpa ;\ 2625 * hmemisc = hblktag ;\ 2626 * hmeblkva = hmeblkva ;\ 2627 * tsbarea = tsbmiss area ;\ 2628 */ ;\ 2629 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2630 label/**/2) ;\ 2631 ;\ 2632 /* ;\ 2633 * tagacc = tagacc ;\ 2634 * hatid = hmentoff ;\ 2635 * tte = clobbered ;\ 2636 * hmeblkpa = hmeblkpa ;\ 2637 * hmemisc = hblk_misc ;\ 2638 * hmeblkva = hmeblkva ;\ 2639 * tsbarea = tsbmiss area ;\ 2640 */ ;\ 2641 ;\ 2642 add hatid, SFHME_TTE, hatid ;\ 2643 add hmeblkpa, hatid, hmeblkpa ;\ 2644 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2645 brlz,pt tte, label/**/6 ;\ 2646 add hmeblkva, hatid, hmeblkva ;\ 2647 btst HBLK_SZMASK, hmemisc ;\ 2648 bnz,a,pt %icc, label/**/7 ;\ 2649 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2650 ;\ 2651 /* ;\ 2652 * We found an invalid 8K tte in shme. ;\ 2653 * it may not belong to shme's region since ;\ 2654 * region size/alignment granularity is 8K but different ;\ 2655 * regions don't share hmeblks. Continue the search. ;\ 2656 */ ;\ 2657 sub hmeblkpa, hatid, hmeblkpa ;\ 2658 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2659 srlx tagacc, hmeshift, tte ;\ 2660 add hmeblkpa, HMEBLK_NEXT, hmeblkva ;\ 2661 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2662 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2663 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2664 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2665 ba,a,pt %xcc, label/**/8 ;\ 2666label/**/6: ;\ 2667 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2668 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2669label/**/7: ;\ 2670 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2671 set TTE_SUSPEND, hatid ;\ 2672 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2673 btst tte, hatid ;\ 2674 bz,pt %xcc, foundlabel ;\ 2675 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2676 ;\ 2677 /* ;\ 2678 * Mapping is suspended, so goto suspend label. ;\ 2679 */ ;\ 2680 ba,pt %xcc, suspendlabel ;\ 2681 nop 2682 2683 /* 2684 * KERNEL PROTECTION HANDLER 2685 * 2686 * g1 = tsb8k pointer register (clobbered) 2687 * g2 = tag access register (ro) 2688 * g3 - g7 = scratch registers 2689 * 2690 * Note: This function is patched at runtime for performance reasons. 2691 * Any changes here require sfmmu_patch_ktsb fixed. 2692 */ 2693 ENTRY_NP(sfmmu_kprot_trap) 2694 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2695sfmmu_kprot_patch_ktsb_base: 2696 RUNTIME_PATCH_SETX(%g1, %g6) 2697 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2698sfmmu_kprot_patch_ktsb_szcode: 2699 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2700 2701 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2702 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2703 2704 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2705sfmmu_kprot_patch_ktsb4m_base: 2706 RUNTIME_PATCH_SETX(%g3, %g6) 2707 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2708sfmmu_kprot_patch_ktsb4m_szcode: 2709 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2710 2711 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2712 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2713 2714 CPU_TSBMISS_AREA(%g6, %g7) 2715 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2716 ba,pt %xcc, sfmmu_tsb_miss_tt 2717 nop 2718 2719 /* 2720 * USER PROTECTION HANDLER 2721 * 2722 * g1 = tsb8k pointer register (ro) 2723 * g2 = tag access register (ro) 2724 * g3 = faulting context (clobbered, currently not used) 2725 * g4 - g7 = scratch registers 2726 */ 2727 ALTENTRY(sfmmu_uprot_trap) 2728#ifdef sun4v 2729 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2730 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2731 2732 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2733 brlz,pt %g3, 9f /* check for 2nd TSB */ 2734 nop 2735 2736 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2737 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2738 2739#else /* sun4v */ 2740#ifdef UTSB_PHYS 2741 /* g1 = first TSB entry ptr */ 2742 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2743 brlz,pt %g3, 9f /* check for 2nd TSB */ 2744 nop 2745 2746 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2747 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2748#else /* UTSB_PHYS */ 2749 brgez,pt %g1, 9f /* check for 2nd TSB */ 2750 mov -1, %g3 /* set second tsbe ptr to -1 */ 2751 2752 mov %g2, %g7 2753 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2754 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2755 mov %g1, %g7 2756 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2757#endif /* UTSB_PHYS */ 2758#endif /* sun4v */ 27599: 2760 CPU_TSBMISS_AREA(%g6, %g7) 2761 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2762 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2763 nop 2764 2765 /* 2766 * Kernel 8K page iTLB miss. We also get here if we took a 2767 * fast instruction access mmu miss trap while running in 2768 * invalid context. 2769 * 2770 * %g1 = 8K TSB pointer register (not used, clobbered) 2771 * %g2 = tag access register (used) 2772 * %g3 = faulting context id (used) 2773 * %g7 = TSB tag to match (used) 2774 */ 2775 .align 64 2776 ALTENTRY(sfmmu_kitlb_miss) 2777 brnz,pn %g3, tsb_tl0_noctxt 2778 nop 2779 2780 /* kernel miss */ 2781 /* get kernel tsb pointer */ 2782 /* we patch the next set of instructions at run time */ 2783 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2784iktsbbase: 2785 RUNTIME_PATCH_SETX(%g4, %g5) 2786 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2787 2788iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2789 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2790 or %g4, %g1, %g1 ! form tsb ptr 2791 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2792 cmp %g4, %g7 2793 bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2794 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2795 2796 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2797 bz,pn %icc, exec_fault 2798 nop 2799 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2800 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2801 retry 2802 2803iktsb4mbase: 2804 RUNTIME_PATCH_SETX(%g4, %g6) 2805 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2806iktsb4m: 2807 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2808 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2809 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2810 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2811 cmp %g4, %g7 2812 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2813 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2814 bz,pn %icc, exec_fault 2815 nop 2816 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2817 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2818 retry 2819 2820 /* 2821 * Kernel dTLB miss. We also get here if we took a fast data 2822 * access mmu miss trap while running in invalid context. 2823 * 2824 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2825 * We select the TSB miss handler to branch to depending on 2826 * the virtual address of the access. In the future it may 2827 * be desirable to separate kpm TTEs into their own TSB, 2828 * in which case all that needs to be done is to set 2829 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2830 * early in the miss if we detect a kpm VA to a new handler. 2831 * 2832 * %g1 = 8K TSB pointer register (not used, clobbered) 2833 * %g2 = tag access register (used) 2834 * %g3 = faulting context id (used) 2835 */ 2836 .align 64 2837 ALTENTRY(sfmmu_kdtlb_miss) 2838 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2839 nop 2840 2841 /* Gather some stats for kpm misses in the TLB. */ 2842 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2843 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2844 2845 /* 2846 * Get first TSB offset and look for 8K/64K/512K mapping 2847 * using the 8K virtual page as the index. 2848 * 2849 * We patch the next set of instructions at run time; 2850 * any changes here require sfmmu_patch_ktsb changes too. 2851 */ 2852dktsbbase: 2853 RUNTIME_PATCH_SETX(%g7, %g6) 2854 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2855 2856dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2857 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2858 2859 /* 2860 * At this point %g1 is our index into the TSB. 2861 * We just masked off enough bits of the VA depending 2862 * on our TSB size code. 2863 */ 2864 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2865 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2866 cmp %g6, %g4 ! compare tag 2867 bne,pn %xcc, dktsb4m_kpmcheck_small 2868 add %g7, %g1, %g1 /* form tsb ptr */ 2869 TT_TRACE(trace_tsbhit) 2870 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2871 /* trapstat expects tte in %g5 */ 2872 retry 2873 2874 /* 2875 * If kpm is using large pages, the following instruction needs 2876 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2877 * so that we will probe the 4M TSB regardless of the VA. In 2878 * the case kpm is using small pages, we know no large kernel 2879 * mappings are located above 0x80000000.00000000 so we skip the 2880 * probe as an optimization. 2881 */ 2882dktsb4m_kpmcheck_small: 2883 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2884 /* delay slot safe, below */ 2885 2886 /* 2887 * Get second TSB offset and look for 4M mapping 2888 * using 4M virtual page as the TSB index. 2889 * 2890 * Here: 2891 * %g1 = 8K TSB pointer. Don't squash it. 2892 * %g2 = tag access register (we still need it) 2893 */ 2894 srlx %g2, MMU_PAGESHIFT4M, %g3 2895 2896 /* 2897 * We patch the next set of instructions at run time; 2898 * any changes here require sfmmu_patch_ktsb changes too. 2899 */ 2900dktsb4mbase: 2901 RUNTIME_PATCH_SETX(%g7, %g6) 2902 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2903dktsb4m: 2904 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2905 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2906 2907 /* 2908 * At this point %g3 is our index into the TSB. 2909 * We just masked off enough bits of the VA depending 2910 * on our TSB size code. 2911 */ 2912 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2913 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2914 cmp %g6, %g4 ! compare tag 2915 2916dktsb4m_tsbmiss: 2917 bne,pn %xcc, dktsb4m_kpmcheck 2918 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2919 TT_TRACE(trace_tsbhit) 2920 /* we don't check TTE size here since we assume 4M TSB is separate */ 2921 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2922 /* trapstat expects tte in %g5 */ 2923 retry 2924 2925 /* 2926 * So, we failed to find a valid TTE to match the faulting 2927 * address in either TSB. There are a few cases that could land 2928 * us here: 2929 * 2930 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2931 * to sfmmu_tsb_miss_tt to handle the miss. 2932 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2933 * 4M TSB. Let segkpm handle it. 2934 * 2935 * Note that we shouldn't land here in the case of a kpm VA when 2936 * kpm_smallpages is active -- we handled that case earlier at 2937 * dktsb4m_kpmcheck_small. 2938 * 2939 * At this point: 2940 * g1 = 8K-indexed primary TSB pointer 2941 * g2 = tag access register 2942 * g3 = 4M-indexed secondary TSB pointer 2943 */ 2944dktsb4m_kpmcheck: 2945 cmp %g2, %g0 2946 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2947 nop 2948 ba,a,pt %icc, sfmmu_tsb_miss_tt 2949 nop 2950 2951#ifdef sun4v 2952 /* 2953 * User instruction miss w/ single TSB. 2954 * The first probe covers 8K, 64K, and 512K page sizes, 2955 * because 64K and 512K mappings are replicated off 8K 2956 * pointer. 2957 * 2958 * g1 = tsb8k pointer register 2959 * g2 = tag access register 2960 * g3 - g6 = scratch registers 2961 * g7 = TSB tag to match 2962 */ 2963 .align 64 2964 ALTENTRY(sfmmu_uitlb_fastpath) 2965 2966 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2967 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2968 ba,pn %xcc, sfmmu_tsb_miss_tt 2969 mov -1, %g3 2970 2971 /* 2972 * User data miss w/ single TSB. 2973 * The first probe covers 8K, 64K, and 512K page sizes, 2974 * because 64K and 512K mappings are replicated off 8K 2975 * pointer. 2976 * 2977 * g1 = tsb8k pointer register 2978 * g2 = tag access register 2979 * g3 - g6 = scratch registers 2980 * g7 = TSB tag to match 2981 */ 2982 .align 64 2983 ALTENTRY(sfmmu_udtlb_fastpath) 2984 2985 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2986 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2987 ba,pn %xcc, sfmmu_tsb_miss_tt 2988 mov -1, %g3 2989 2990 /* 2991 * User instruction miss w/ multiple TSBs (sun4v). 2992 * The first probe covers 8K, 64K, and 512K page sizes, 2993 * because 64K and 512K mappings are replicated off 8K 2994 * pointer. Second probe covers 4M page size only. 2995 * 2996 * Just like sfmmu_udtlb_slowpath, except: 2997 * o Uses ASI_ITLB_IN 2998 * o checks for execute permission 2999 * o No ISM prediction. 3000 * 3001 * g1 = tsb8k pointer register 3002 * g2 = tag access register 3003 * g3 - g6 = scratch registers 3004 * g7 = TSB tag to match 3005 */ 3006 .align 64 3007 ALTENTRY(sfmmu_uitlb_slowpath) 3008 3009 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 3010 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 3011 /* g4 - g5 = clobbered here */ 3012 3013 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3014 /* g1 = first TSB pointer, g3 = second TSB pointer */ 3015 srlx %g2, TAG_VALO_SHIFT, %g7 3016 PROBE_2ND_ITSB(%g3, %g7) 3017 /* NOT REACHED */ 3018 3019#else /* sun4v */ 3020 3021 /* 3022 * User instruction miss w/ multiple TSBs (sun4u). 3023 * The first probe covers 8K, 64K, and 512K page sizes, 3024 * because 64K and 512K mappings are replicated off 8K 3025 * pointer. Probe of 1st TSB has already been done prior to entry 3026 * into this routine. For the UTSB_PHYS case we probe up to 3 3027 * valid other TSBs in the following order: 3028 * 1) shared TSB for 4M-256M pages 3029 * 2) private TSB for 4M-256M pages 3030 * 3) shared TSB for 8K-512K pages 3031 * 3032 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs 3033 * 4M-256M pages. 3034 * 3035 * Just like sfmmu_udtlb_slowpath, except: 3036 * o Uses ASI_ITLB_IN 3037 * o checks for execute permission 3038 * o No ISM prediction. 3039 * 3040 * g1 = tsb8k pointer register 3041 * g2 = tag access register 3042 * g4 - g6 = scratch registers 3043 * g7 = TSB tag to match 3044 */ 3045 .align 64 3046 ALTENTRY(sfmmu_uitlb_slowpath) 3047 3048#ifdef UTSB_PHYS 3049 3050 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3051 brlz,pt %g6, 1f 3052 nop 3053 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3054 PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail) 30551: 3056 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3057 brlz,pt %g3, 2f 3058 nop 3059 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3060 PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail) 30612: 3062 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3063 brlz,pt %g6, sfmmu_tsb_miss_tt 3064 nop 3065 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3066 PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail) 3067 ba,pn %xcc, sfmmu_tsb_miss_tt 3068 nop 3069 3070#else /* UTSB_PHYS */ 3071 mov %g1, %g3 /* save tsb8k reg in %g3 */ 3072 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 3073 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 3074 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 3075 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 3076 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 3077 /* g1 = first TSB pointer, g3 = second TSB pointer */ 3078 srlx %g2, TAG_VALO_SHIFT, %g7 3079 PROBE_2ND_ITSB(%g3, %g7, isynth) 3080 ba,pn %xcc, sfmmu_tsb_miss_tt 3081 nop 3082 3083#endif /* UTSB_PHYS */ 3084#endif /* sun4v */ 3085 3086#if defined(sun4u) && defined(UTSB_PHYS) 3087 3088 /* 3089 * We come here for ism predict DTLB_MISS case or if 3090 * if probe in first TSB failed. 3091 */ 3092 3093 .align 64 3094 ALTENTRY(sfmmu_udtlb_slowpath_noismpred) 3095 3096 /* 3097 * g1 = tsb8k pointer register 3098 * g2 = tag access register 3099 * g4 - %g6 = scratch registers 3100 * g7 = TSB tag to match 3101 */ 3102 3103 /* 3104 * ISM non-predict probe order 3105 * probe 1ST_TSB (8K index) 3106 * probe 2ND_TSB (4M index) 3107 * probe 4TH_TSB (4M index) 3108 * probe 3RD_TSB (8K index) 3109 * 3110 * We already probed first TSB in DTLB_MISS handler. 3111 */ 3112 3113 /* 3114 * Private 2ND TSB 4M-256 pages 3115 */ 3116 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3117 brlz,pt %g3, 1f 3118 nop 3119 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3120 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3121 3122 /* 3123 * Shared Context 4TH TSB 4M-256 pages 3124 */ 31251: 3126 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3127 brlz,pt %g6, 2f 3128 nop 3129 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3130 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail) 3131 3132 /* 3133 * Shared Context 3RD TSB 8K-512K pages 3134 */ 31352: 3136 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3137 brlz,pt %g6, sfmmu_tsb_miss_tt 3138 nop 3139 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3140 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail) 3141 ba,pn %xcc, sfmmu_tsb_miss_tt 3142 nop 3143 3144 .align 64 3145 ALTENTRY(sfmmu_udtlb_slowpath_ismpred) 3146 3147 /* 3148 * g1 = tsb8k pointer register 3149 * g2 = tag access register 3150 * g4 - g6 = scratch registers 3151 * g7 = TSB tag to match 3152 */ 3153 3154 /* 3155 * ISM predict probe order 3156 * probe 4TH_TSB (4M index) 3157 * probe 2ND_TSB (4M index) 3158 * probe 1ST_TSB (8K index) 3159 * probe 3RD_TSB (8K index) 3160 3161 /* 3162 * Shared Context 4TH TSB 4M-256 pages 3163 */ 3164 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3165 brlz,pt %g6, 4f 3166 nop 3167 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3168 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2) 3169 3170 /* 3171 * Private 2ND TSB 4M-256 pages 3172 */ 31734: 3174 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3175 brlz,pt %g3, 5f 3176 nop 3177 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3178 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2) 3179 31805: 3181 PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2) 3182 3183 /* 3184 * Shared Context 3RD TSB 8K-512K pages 3185 */ 3186 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3187 brlz,pt %g6, 6f 3188 nop 3189 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3190 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2) 31916: 3192 ba,pn %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */ 3193 nop 3194 3195#else /* sun4u && UTSB_PHYS */ 3196 3197 .align 64 3198 ALTENTRY(sfmmu_udtlb_slowpath) 3199 3200 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 3201 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 3202 mov %g1, %g3 3203 3204udtlb_miss_probefirst: 3205 /* 3206 * g1 = 8K TSB pointer register 3207 * g2 = tag access register 3208 * g3 = (potentially) second TSB entry ptr 3209 * g6 = ism pred. 3210 * g7 = vpg_4m 3211 */ 3212#ifdef sun4v 3213 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 3214 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3215 3216 /* 3217 * Here: 3218 * g1 = first TSB pointer 3219 * g2 = tag access reg 3220 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3221 */ 3222 brgz,pn %g6, sfmmu_tsb_miss_tt 3223 nop 3224#else /* sun4v */ 3225 mov %g1, %g4 3226 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 3227 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3228 3229 /* 3230 * Here: 3231 * g1 = first TSB pointer 3232 * g2 = tag access reg 3233 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3234 */ 3235 brgz,pn %g6, sfmmu_tsb_miss_tt 3236 nop 3237 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 3238 /* fall through in 8K->4M probe order */ 3239#endif /* sun4v */ 3240 3241udtlb_miss_probesecond: 3242 /* 3243 * Look in the second TSB for the TTE 3244 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 3245 * g2 = tag access reg 3246 * g3 = 8K TSB pointer register 3247 * g6 = ism pred. 3248 * g7 = vpg_4m 3249 */ 3250#ifdef sun4v 3251 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 3252 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3253 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 3254#else /* sun4v */ 3255 mov %g3, %g7 3256 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 3257 /* %g2 clobbered, %g3 =second tsbe ptr */ 3258 mov MMU_TAG_ACCESS, %g2 3259 ldxa [%g2]ASI_DMMU, %g2 3260#endif /* sun4v */ 3261 3262 srlx %g2, TAG_VALO_SHIFT, %g7 3263 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3264 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 3265 brgz,pn %g6, udtlb_miss_probefirst 3266 nop 3267 3268 /* fall through to sfmmu_tsb_miss_tt */ 3269#endif /* sun4u && UTSB_PHYS */ 3270 3271 3272 ALTENTRY(sfmmu_tsb_miss_tt) 3273 TT_TRACE(trace_tsbmiss) 3274 /* 3275 * We get here if there is a TSB miss OR a write protect trap. 3276 * 3277 * g1 = First TSB entry pointer 3278 * g2 = tag access register 3279 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 3280 * g4 - g7 = scratch registers 3281 */ 3282 3283 ALTENTRY(sfmmu_tsb_miss) 3284 3285 /* 3286 * If trapstat is running, we need to shift the %tpc and %tnpc to 3287 * point to trapstat's TSB miss return code (note that trapstat 3288 * itself will patch the correct offset to add). 3289 */ 3290 rdpr %tl, %g7 3291 cmp %g7, 1 3292 ble,pt %xcc, 0f 3293 sethi %hi(KERNELBASE), %g6 3294 rdpr %tpc, %g7 3295 or %g6, %lo(KERNELBASE), %g6 3296 cmp %g7, %g6 3297 bgeu,pt %xcc, 0f 3298 /* delay slot safe */ 3299 3300 ALTENTRY(tsbmiss_trapstat_patch_point) 3301 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 3302 wrpr %g7, %tpc 3303 add %g7, 4, %g7 3304 wrpr %g7, %tnpc 33050: 3306 CPU_TSBMISS_AREA(%g6, %g7) 3307 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save 1ST tsb pointer */ 3308 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save 2ND tsb pointer */ 3309 3310 sllx %g2, TAGACC_CTX_LSHIFT, %g3 3311 brz,a,pn %g3, 1f /* skip ahead if kernel */ 3312 ldn [%g6 + TSBMISS_KHATID], %g7 3313 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 3314 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 3315 3316 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 3317 3318 cmp %g3, INVALID_CONTEXT 3319 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 3320 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 3321 3322#if defined(sun4v) || defined(UTSB_PHYS) 3323 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 3324 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 3325 stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 3326#endif /* sun4v || UTSB_PHYS */ 3327 3328 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 3329 /* 3330 * The miss wasn't in an ISM segment. 3331 * 3332 * %g1 %g3, %g4, %g5, %g7 all clobbered 3333 * %g2 = (pseudo) tag access 3334 */ 3335 3336 ba,pt %icc, 2f 3337 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 3338 33391: 3340 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 3341 /* 3342 * 8K and 64K hash. 3343 */ 33442: 3345 3346 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3347 MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte, 3348 sfmmu_suspend_tl, tsb_512K) 3349 /* NOT REACHED */ 3350 3351tsb_512K: 3352 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3353 brz,pn %g5, 3f 3354 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3355 and %g4, HAT_512K_FLAG, %g5 3356 3357 /* 3358 * Note that there is a small window here where we may have 3359 * a 512k page in the hash list but have not set the HAT_512K_FLAG 3360 * flag yet, so we will skip searching the 512k hash list. 3361 * In this case we will end up in pagefault which will find 3362 * the mapping and return. So, in this instance we will end up 3363 * spending a bit more time resolving this TSB miss, but it can 3364 * only happen once per process and even then, the chances of that 3365 * are very small, so it's not worth the extra overhead it would 3366 * take to close this window. 3367 */ 3368 brz,pn %g5, tsb_4M 3369 nop 33703: 3371 /* 3372 * 512K hash 3373 */ 3374 3375 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3376 MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte, 3377 sfmmu_suspend_tl, tsb_4M) 3378 /* NOT REACHED */ 3379 3380tsb_4M: 3381 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3382 brz,pn %g5, 4f 3383 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3384 and %g4, HAT_4M_FLAG, %g5 3385 brz,pn %g5, tsb_32M 3386 nop 33874: 3388 /* 3389 * 4M hash 3390 */ 3391 3392 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3393 MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte, 3394 sfmmu_suspend_tl, tsb_32M) 3395 /* NOT REACHED */ 3396 3397tsb_32M: 3398 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3399#ifdef sun4v 3400 brz,pn %g5, 6f 3401#else 3402 brz,pn %g5, tsb_pagefault 3403#endif 3404 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3405 and %g4, HAT_32M_FLAG, %g5 3406 brz,pn %g5, tsb_256M 3407 nop 34085: 3409 /* 3410 * 32M hash 3411 */ 3412 3413 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3414 MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte, 3415 sfmmu_suspend_tl, tsb_256M) 3416 /* NOT REACHED */ 3417 3418#if defined(sun4u) && !defined(UTSB_PHYS) 3419#define tsb_shme tsb_pagefault 3420#endif 3421tsb_256M: 3422 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3423 and %g4, HAT_256M_FLAG, %g5 3424 brz,pn %g5, tsb_shme 3425 nop 34266: 3427 /* 3428 * 256M hash 3429 */ 3430 3431 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3432 MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte, 3433 sfmmu_suspend_tl, tsb_shme) 3434 /* NOT REACHED */ 3435 3436tsb_checktte: 3437 /* 3438 * g1 = hblk_misc 3439 * g2 = tagacc 3440 * g3 = tte 3441 * g4 = tte pa 3442 * g5 = tte va 3443 * g6 = tsbmiss area 3444 * g7 = hatid 3445 */ 3446 brlz,a,pt %g3, tsb_validtte 3447 rdpr %tt, %g7 3448 3449#if defined(sun4u) && !defined(UTSB_PHYS) 3450#undef tsb_shme 3451 ba tsb_pagefault 3452 nop 3453#else /* sun4u && !UTSB_PHYS */ 3454 3455tsb_shme: 3456 /* 3457 * g2 = tagacc 3458 * g6 = tsbmiss area 3459 */ 3460 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3461 brz,pn %g5, tsb_pagefault 3462 nop 3463 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3464 brz,pn %g7, tsb_pagefault 3465 nop 3466 3467 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3468 MMU_PAGESHIFT64K, TTE64K, tsb_shme_l8K, tsb_shme_checktte, 3469 sfmmu_suspend_tl, tsb_shme_512K) 3470 /* NOT REACHED */ 3471 3472tsb_shme_512K: 3473 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3474 and %g4, HAT_512K_FLAG, %g5 3475 brz,pn %g5, tsb_shme_4M 3476 nop 3477 3478 /* 3479 * 512K hash 3480 */ 3481 3482 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3483 MMU_PAGESHIFT512K, TTE512K, tsb_shme_l512K, tsb_shme_checktte, 3484 sfmmu_suspend_tl, tsb_shme_4M) 3485 /* NOT REACHED */ 3486 3487tsb_shme_4M: 3488 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3489 and %g4, HAT_4M_FLAG, %g5 3490 brz,pn %g5, tsb_shme_32M 3491 nop 34924: 3493 /* 3494 * 4M hash 3495 */ 3496 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3497 MMU_PAGESHIFT4M, TTE4M, tsb_shme_l4M, tsb_shme_checktte, 3498 sfmmu_suspend_tl, tsb_shme_32M) 3499 /* NOT REACHED */ 3500 3501tsb_shme_32M: 3502 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3503 and %g4, HAT_32M_FLAG, %g5 3504 brz,pn %g5, tsb_shme_256M 3505 nop 3506 3507 /* 3508 * 32M hash 3509 */ 3510 3511 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3512 MMU_PAGESHIFT32M, TTE32M, tsb_shme_l32M, tsb_shme_checktte, 3513 sfmmu_suspend_tl, tsb_shme_256M) 3514 /* NOT REACHED */ 3515 3516tsb_shme_256M: 3517 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3518 and %g4, HAT_256M_FLAG, %g5 3519 brz,pn %g5, tsb_pagefault 3520 nop 3521 3522 /* 3523 * 256M hash 3524 */ 3525 3526 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3527 MMU_PAGESHIFT256M, TTE256M, tsb_shme_l256M, tsb_shme_checktte, 3528 sfmmu_suspend_tl, tsb_pagefault) 3529 /* NOT REACHED */ 3530 3531tsb_shme_checktte: 3532 3533 brgez,pn %g3, tsb_pagefault 3534 rdpr %tt, %g7 3535 /* 3536 * g1 = ctx1 flag 3537 * g3 = tte 3538 * g4 = tte pa 3539 * g5 = tte va 3540 * g6 = tsbmiss area 3541 * g7 = tt 3542 */ 3543 3544 brz,pt %g1, tsb_validtte 3545 nop 3546 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3547 or %g1, HAT_CHKCTX1_FLAG, %g1 3548 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3549 3550 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3551#endif /* sun4u && !UTSB_PHYS */ 3552 3553tsb_validtte: 3554 /* 3555 * g3 = tte 3556 * g4 = tte pa 3557 * g5 = tte va 3558 * g6 = tsbmiss area 3559 * g7 = tt 3560 */ 3561 3562 /* 3563 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3564 */ 3565 cmp %g7, FAST_PROT_TT 3566 bne,pt %icc, 4f 3567 nop 3568 3569 TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod, 3570 tsb_protfault) 3571 3572 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3573#ifdef sun4v 3574 MMU_FAULT_STATUS_AREA(%g7) 3575 ldx [%g7 + MMFSA_D_ADDR], %g5 /* load fault addr for later */ 3576#else /* sun4v */ 3577 mov MMU_TAG_ACCESS, %g5 3578 ldxa [%g5]ASI_DMMU, %g5 3579#endif /* sun4v */ 3580 ba,pt %xcc, tsb_update_tl1 3581 nop 35824: 3583 /* 3584 * ITLB translation was found but execute permission is 3585 * disabled. If we have software execute permission (soft exec 3586 * bit is set), then enable hardware execute permission. 3587 * Otherwise continue with a protection violation. 3588 */ 3589 cmp %g7, T_INSTR_MMU_MISS 3590 be,pn %icc, 5f 3591 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3592 cmp %g7, FAST_IMMU_MISS_TT 3593 bne,pt %icc, 3f 3594 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 35955: 3596 bnz,pn %icc, 3f 3597 TTE_CHK_SOFTEXEC_ML(%g3) /* check soft execute */ 3598 bz,pn %icc, tsb_protfault 3599 nop 3600 TTE_SET_EXEC_ML(%g3, %g4, %g7, tsb_lset_exec) 36013: 3602 /* 3603 * Set reference bit if not already set 3604 */ 3605 TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref) 3606 3607 /* 3608 * Now, load into TSB/TLB. At this point: 3609 * g3 = tte 3610 * g4 = patte 3611 * g6 = tsbmiss area 3612 */ 3613 rdpr %tt, %g7 3614#ifdef sun4v 3615 MMU_FAULT_STATUS_AREA(%g2) 3616 cmp %g7, T_INSTR_MMU_MISS 3617 be,a,pt %icc, 9f 3618 nop 3619 cmp %g7, FAST_IMMU_MISS_TT 3620 be,a,pt %icc, 9f 3621 nop 3622 add %g2, MMFSA_D_, %g2 36239: 3624 ldx [%g2 + MMFSA_CTX_], %g7 3625 sllx %g7, TTARGET_CTX_SHIFT, %g7 3626 ldx [%g2 + MMFSA_ADDR_], %g2 3627 mov %g2, %g5 ! load the fault addr for later use 3628 srlx %g2, TTARGET_VA_SHIFT, %g2 3629 or %g2, %g7, %g2 3630#else /* sun4v */ 3631 mov MMU_TAG_ACCESS, %g5 3632 cmp %g7, FAST_IMMU_MISS_TT 3633 be,a,pt %icc, 9f 3634 ldxa [%g0]ASI_IMMU, %g2 3635 ldxa [%g0]ASI_DMMU, %g2 3636 ba,pt %icc, tsb_update_tl1 3637 ldxa [%g5]ASI_DMMU, %g5 36389: 3639 ldxa [%g5]ASI_IMMU, %g5 3640#endif /* sun4v */ 3641 3642tsb_update_tl1: 3643 TTE_CLR_SOFTEXEC_ML(%g3) 3644 srlx %g2, TTARGET_CTX_SHIFT, %g7 3645 brz,pn %g7, tsb_kernel 3646#ifdef sun4v 3647 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3648#else /* sun4v */ 3649 srlx %g3, TTE_SZ_SHFT, %g7 3650#endif /* sun4v */ 3651 3652tsb_user: 3653#ifdef sun4v 3654 cmp %g7, TTE4M 3655 bge,pn %icc, tsb_user4m 3656 nop 3657#else /* sun4v */ 3658 cmp %g7, TTESZ_VALID | TTE4M 3659 be,pn %icc, tsb_user4m 3660 srlx %g3, TTE_SZ2_SHFT, %g7 3661 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3662#ifdef ITLB_32M_256M_SUPPORT 3663 bnz,pn %icc, tsb_user4m 3664 nop 3665#else /* ITLB_32M_256M_SUPPORT */ 3666 bnz,a,pn %icc, tsb_user_pn_synth 3667 nop 3668#endif /* ITLB_32M_256M_SUPPORT */ 3669#endif /* sun4v */ 3670 3671tsb_user8k: 3672#if defined(sun4v) || defined(UTSB_PHYS) 3673 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3674 and %g7, HAT_CHKCTX1_FLAG, %g1 3675 brz,a,pn %g1, 1f 3676 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3677 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3678 brlz,a,pn %g1, ptl1_panic ! if no shared 3RD tsb 3679 mov PTL1_NO_SCDTSB8K, %g1 ! panic 3680 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 36811: 3682#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3683 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3684#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3685 3686#ifndef UTSB_PHYS 3687 mov ASI_N, %g7 ! user TSBs accessed by VA 3688 mov %g7, %asi 3689#endif /* !UTSB_PHYS */ 3690 3691 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5) 3692 3693 rdpr %tt, %g5 3694#ifdef sun4v 3695 cmp %g5, T_INSTR_MMU_MISS 3696 be,a,pn %xcc, 9f 3697 mov %g3, %g5 3698#endif /* sun4v */ 3699 cmp %g5, FAST_IMMU_MISS_TT 3700 be,pn %xcc, 9f 3701 mov %g3, %g5 3702 3703 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3704 ! trapstat wants TTE in %g5 3705 retry 37069: 3707 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3708 ! trapstat wants TTE in %g5 3709 retry 3710 3711tsb_user4m: 3712#if defined(sun4v) || defined(UTSB_PHYS) 3713 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3714 and %g7, HAT_CHKCTX1_FLAG, %g1 3715 brz,a,pn %g1, 4f 3716 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3717 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr 3718 brlz,a,pn %g1, 5f ! if no shared 4TH TSB 3719 nop 3720 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 3721 3722#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3723 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3724#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 37254: 3726 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3727 nop 3728 3729#ifndef UTSB_PHYS 3730 mov ASI_N, %g7 ! user TSBs accessed by VA 3731 mov %g7, %asi 3732#endif /* UTSB_PHYS */ 3733 3734 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6) 3735 37365: 3737 rdpr %tt, %g5 3738#ifdef sun4v 3739 cmp %g5, T_INSTR_MMU_MISS 3740 be,a,pn %xcc, 9f 3741 mov %g3, %g5 3742#endif /* sun4v */ 3743 cmp %g5, FAST_IMMU_MISS_TT 3744 be,pn %xcc, 9f 3745 mov %g3, %g5 3746 3747 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3748 ! trapstat wants TTE in %g5 3749 retry 37509: 3751 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3752 ! trapstat wants TTE in %g5 3753 retry 3754 3755#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3756 /* 3757 * Panther ITLB synthesis. 3758 * The Panther 32M and 256M ITLB code simulates these two large page 3759 * sizes with 4M pages, to provide support for programs, for example 3760 * Java, that may copy instructions into a 32M or 256M data page and 3761 * then execute them. The code below generates the 4M pfn bits and 3762 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3763 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3764 * are ignored by the hardware. 3765 * 3766 * Now, load into TSB/TLB. At this point: 3767 * g2 = tagtarget 3768 * g3 = tte 3769 * g4 = patte 3770 * g5 = tt 3771 * g6 = tsbmiss area 3772 */ 3773tsb_user_pn_synth: 3774 rdpr %tt, %g5 3775 cmp %g5, FAST_IMMU_MISS_TT 3776 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3777 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3778 bz,pn %icc, 4b /* if not, been here before */ 3779 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3780 brlz,a,pn %g1, 5f /* no 2nd tsb */ 3781 mov %g3, %g5 3782 3783 mov MMU_TAG_ACCESS, %g7 3784 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3785 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3786 3787 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3788 mov %g7, %asi 3789 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */ 37905: 3791 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3792 retry 3793 3794tsb_user_itlb_synth: 3795 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = 2ND TSB */ 3796 3797 mov MMU_TAG_ACCESS, %g7 3798 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3799 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3800 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3801 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3802 3803 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3804 mov %g7, %asi 3805 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */ 38067: 3807 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3808 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3809 retry 3810#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3811 3812tsb_kernel: 3813 rdpr %tt, %g5 3814#ifdef sun4v 3815 cmp %g7, TTE4M 3816 bge,pn %icc, 5f 3817#else 3818 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3819 be,pn %icc, 5f 3820#endif /* sun4v */ 3821 nop 3822 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8K TSB ptr 3823 ba,pt %xcc, 6f 3824 nop 38255: 3826 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4M TSB ptr 3827 brlz,pn %g1, 3f /* skip programming if 4M TSB ptr is -1 */ 3828 nop 38296: 3830#ifndef sun4v 3831tsb_kernel_patch_asi: 3832 or %g0, RUNTIME_PATCH, %g6 3833 mov %g6, %asi ! XXX avoid writing to %asi !! 3834#endif 3835 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7) 38363: 3837#ifdef sun4v 3838 cmp %g5, T_INSTR_MMU_MISS 3839 be,a,pn %icc, 1f 3840 mov %g3, %g5 ! trapstat wants TTE in %g5 3841#endif /* sun4v */ 3842 cmp %g5, FAST_IMMU_MISS_TT 3843 be,pn %icc, 1f 3844 mov %g3, %g5 ! trapstat wants TTE in %g5 3845 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3846 ! trapstat wants TTE in %g5 3847 retry 38481: 3849 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3850 ! trapstat wants TTE in %g5 3851 retry 3852 3853tsb_ism: 3854 /* 3855 * This is an ISM [i|d]tlb miss. We optimize for largest 3856 * page size down to smallest. 3857 * 3858 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3859 * register 3860 * g3 = ismmap->ism_seg 3861 * g4 = physical address of ismmap->ism_sfmmu 3862 * g6 = tsbmiss area 3863 */ 3864 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3865 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3866 mov PTL1_BAD_ISM, %g1 3867 /* g5 = pa of imap_vb_shift */ 3868 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3869 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3870 srlx %g3, %g4, %g3 /* clr size field */ 3871 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3872 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3873 and %g2, %g1, %g4 /* g4 = ctx number */ 3874 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3875 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3876 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3877 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3878 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3879#if defined(sun4v) || defined(UTSB_PHYS) 3880 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3881 brz,pt %g5, tsb_chk4M_ism 3882 nop 3883 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3884 or %g5, HAT_CHKCTX1_FLAG, %g5 3885 stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3886 rdpr %tt, %g5 3887 SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl) 3888#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3889 3890 /* 3891 * ISM pages are always locked down. 3892 * If we can't find the tte then pagefault 3893 * and let the spt segment driver resolve it. 3894 * 3895 * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3896 * g4 = imap_hatflags 3897 * g6 = tsb miss area 3898 * g7 = ISM hatid 3899 */ 3900 3901tsb_chk4M_ism: 3902 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3903 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3904 nop 3905 3906tsb_ism_32M: 3907 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3908 brz,pn %g5, tsb_ism_256M 3909 nop 3910 3911 /* 3912 * 32M hash. 3913 */ 3914 3915 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M, 3916 TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3917 tsb_ism_4M) 3918 /* NOT REACHED */ 3919 3920tsb_ism_32M_found: 3921 brlz,a,pt %g3, tsb_validtte 3922 rdpr %tt, %g7 3923 ba,pt %xcc, tsb_ism_4M 3924 nop 3925 3926tsb_ism_256M: 3927 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3928 brz,a,pn %g5, ptl1_panic 3929 mov PTL1_BAD_ISM, %g1 3930 3931 /* 3932 * 256M hash. 3933 */ 3934 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M, 3935 TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3936 tsb_ism_4M) 3937 3938tsb_ism_256M_found: 3939 brlz,a,pt %g3, tsb_validtte 3940 rdpr %tt, %g7 3941 3942tsb_ism_4M: 3943 /* 3944 * 4M hash. 3945 */ 3946 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M, 3947 TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3948 tsb_ism_8K) 3949 /* NOT REACHED */ 3950 3951tsb_ism_4M_found: 3952 brlz,a,pt %g3, tsb_validtte 3953 rdpr %tt, %g7 3954 3955tsb_ism_8K: 3956 /* 3957 * 8K and 64K hash. 3958 */ 3959 3960 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K, 3961 TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3962 tsb_pagefault) 3963 /* NOT REACHED */ 3964 3965tsb_ism_8K_found: 3966 brlz,a,pt %g3, tsb_validtte 3967 rdpr %tt, %g7 3968 3969tsb_pagefault: 3970 rdpr %tt, %g7 3971 cmp %g7, FAST_PROT_TT 3972 be,a,pn %icc, tsb_protfault 3973 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3974 3975tsb_protfault: 3976 /* 3977 * we get here if we couldn't find a valid tte in the hash. 3978 * 3979 * If user and we are at tl>1 we go to window handling code. 3980 * 3981 * If kernel and the fault is on the same page as our stack 3982 * pointer, then we know the stack is bad and the trap handler 3983 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3984 * 3985 * If this is a kernel trap and tl>1, panic. 3986 * 3987 * Otherwise we call pagefault. 3988 */ 3989 cmp %g7, FAST_IMMU_MISS_TT 3990#ifdef sun4v 3991 MMU_FAULT_STATUS_AREA(%g4) 3992 ldx [%g4 + MMFSA_I_CTX], %g5 3993 ldx [%g4 + MMFSA_D_CTX], %g4 3994 move %icc, %g5, %g4 3995 cmp %g7, T_INSTR_MMU_MISS 3996 move %icc, %g5, %g4 3997#else 3998 mov MMU_TAG_ACCESS, %g4 3999 ldxa [%g4]ASI_DMMU, %g2 4000 ldxa [%g4]ASI_IMMU, %g5 4001 move %icc, %g5, %g2 4002 cmp %g7, T_INSTR_MMU_MISS 4003 move %icc, %g5, %g2 4004 sllx %g2, TAGACC_CTX_LSHIFT, %g4 4005#endif /* sun4v */ 4006 brnz,pn %g4, 3f /* skip if not kernel */ 4007 rdpr %tl, %g5 4008 4009 add %sp, STACK_BIAS, %g3 4010 srlx %g3, MMU_PAGESHIFT, %g3 4011 srlx %g2, MMU_PAGESHIFT, %g4 4012 cmp %g3, %g4 4013 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 4014 mov PTL1_BAD_STACK, %g1 4015 4016 cmp %g5, 1 4017 ble,pt %icc, 2f 4018 nop 4019 TSTAT_CHECK_TL1(2f, %g1, %g2) 4020 rdpr %tt, %g2 4021 cmp %g2, FAST_PROT_TT 4022 mov PTL1_BAD_KPROT_FAULT, %g1 4023 movne %icc, PTL1_BAD_KMISS, %g1 4024 ba,pt %icc, ptl1_panic 4025 nop 4026 40272: 4028 /* 4029 * We are taking a pagefault in the kernel on a kernel address. If 4030 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 4031 * want to call sfmmu_pagefault -- we will instead note that a fault 4032 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 4033 * (instead of a "retry"). This will step over the faulting 4034 * instruction. 4035 */ 4036 CPU_INDEX(%g1, %g2) 4037 set cpu_core, %g2 4038 sllx %g1, CPU_CORE_SHIFT, %g1 4039 add %g1, %g2, %g1 4040 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 4041 andcc %g2, CPU_DTRACE_NOFAULT, %g0 4042 bz sfmmu_pagefault 4043 or %g2, CPU_DTRACE_BADADDR, %g2 4044 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 4045 GET_MMU_D_ADDR(%g3, %g4) 4046 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 4047 done 4048 40493: 4050 cmp %g5, 1 4051 ble,pt %icc, 4f 4052 nop 4053 TSTAT_CHECK_TL1(4f, %g1, %g2) 4054 ba,pt %icc, sfmmu_window_trap 4055 nop 4056 40574: 4058 /* 4059 * We are taking a pagefault on a non-kernel address. If we are in 4060 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 4061 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 4062 */ 4063 CPU_INDEX(%g1, %g2) 4064 set cpu_core, %g2 4065 sllx %g1, CPU_CORE_SHIFT, %g1 4066 add %g1, %g2, %g1 4067 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 4068 andcc %g2, CPU_DTRACE_NOFAULT, %g0 4069 bz sfmmu_mmu_trap 4070 or %g2, CPU_DTRACE_BADADDR, %g2 4071 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 4072 GET_MMU_D_ADDR(%g3, %g4) 4073 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 4074 4075 /* 4076 * Be sure that we're actually taking this miss from the kernel -- 4077 * otherwise we have managed to return to user-level with 4078 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 4079 */ 4080 rdpr %tstate, %g2 4081 btst TSTATE_PRIV, %g2 4082 bz,a ptl1_panic 4083 mov PTL1_BAD_DTRACE_FLAGS, %g1 4084 done 4085 4086 ALTENTRY(tsb_tl0_noctxt) 4087 /* 4088 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 4089 * if it is, indicated that we have faulted and issue a done. 4090 */ 4091 CPU_INDEX(%g5, %g6) 4092 set cpu_core, %g6 4093 sllx %g5, CPU_CORE_SHIFT, %g5 4094 add %g5, %g6, %g5 4095 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 4096 andcc %g6, CPU_DTRACE_NOFAULT, %g0 4097 bz 1f 4098 or %g6, CPU_DTRACE_BADADDR, %g6 4099 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 4100 GET_MMU_D_ADDR(%g3, %g4) 4101 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 4102 4103 /* 4104 * Be sure that we're actually taking this miss from the kernel -- 4105 * otherwise we have managed to return to user-level with 4106 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 4107 */ 4108 rdpr %tstate, %g5 4109 btst TSTATE_PRIV, %g5 4110 bz,a ptl1_panic 4111 mov PTL1_BAD_DTRACE_FLAGS, %g1 4112 TSTAT_CHECK_TL1(2f, %g1, %g2); 41132: 4114 done 4115 41161: 4117 rdpr %tt, %g5 4118 cmp %g5, FAST_IMMU_MISS_TT 4119#ifdef sun4v 4120 MMU_FAULT_STATUS_AREA(%g2) 4121 be,a,pt %icc, 2f 4122 ldx [%g2 + MMFSA_I_CTX], %g3 4123 cmp %g5, T_INSTR_MMU_MISS 4124 be,a,pt %icc, 2f 4125 ldx [%g2 + MMFSA_I_CTX], %g3 4126 ldx [%g2 + MMFSA_D_CTX], %g3 41272: 4128#else 4129 mov MMU_TAG_ACCESS, %g2 4130 be,a,pt %icc, 2f 4131 ldxa [%g2]ASI_IMMU, %g3 4132 ldxa [%g2]ASI_DMMU, %g3 41332: sllx %g3, TAGACC_CTX_LSHIFT, %g3 4134#endif /* sun4v */ 4135 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 4136 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 4137 rdpr %tl, %g5 4138 cmp %g5, 1 4139 ble,pt %icc, sfmmu_mmu_trap 4140 nop 4141 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 4142 ba,pt %icc, sfmmu_window_trap 4143 nop 4144 SET_SIZE(sfmmu_tsb_miss) 4145#endif /* lint */ 4146 4147#if defined (lint) 4148/* 4149 * This routine will look for a user or kernel vaddr in the hash 4150 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 4151 * grab any locks. It should only be used by other sfmmu routines. 4152 */ 4153/* ARGSUSED */ 4154pfn_t 4155sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 4156{ 4157 return(0); 4158} 4159 4160/* ARGSUSED */ 4161pfn_t 4162sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno) 4163{ 4164 return(0); 4165} 4166 4167#else /* lint */ 4168 4169 ENTRY_NP(sfmmu_vatopfn) 4170 /* 4171 * disable interrupts 4172 */ 4173 rdpr %pstate, %o3 4174#ifdef DEBUG 4175 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 4176#endif 4177 /* 4178 * disable interrupts to protect the TSBMISS area 4179 */ 4180 andn %o3, PSTATE_IE, %o5 4181 wrpr %o5, 0, %pstate 4182 4183 /* 4184 * o0 = vaddr 4185 * o1 = sfmmup 4186 * o2 = ttep 4187 */ 4188 CPU_TSBMISS_AREA(%g1, %o5) 4189 ldn [%g1 + TSBMISS_KHATID], %o4 4190 cmp %o4, %o1 4191 bne,pn %ncc, vatopfn_nokernel 4192 mov TTE64K, %g5 /* g5 = rehash # */ 4193 mov %g1,%o5 /* o5 = tsbmiss_area */ 4194 /* 4195 * o0 = vaddr 4196 * o1 & o4 = hatid 4197 * o2 = ttep 4198 * o5 = tsbmiss area 4199 */ 4200 mov HBLK_RANGE_SHIFT, %g6 42011: 4202 4203 /* 4204 * o0 = vaddr 4205 * o1 = sfmmup 4206 * o2 = ttep 4207 * o3 = old %pstate 4208 * o4 = hatid 4209 * o5 = tsbmiss 4210 * g5 = rehash # 4211 * g6 = hmeshift 4212 * 4213 * The first arg to GET_TTE is actually tagaccess register 4214 * not just vaddr. Since this call is for kernel we need to clear 4215 * any lower vaddr bits that would be interpreted as ctx bits. 4216 */ 4217 set TAGACC_CTX_MASK, %g1 4218 andn %o0, %g1, %o0 4219 GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5, 4220 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 4221 4222kvtop_hblk_found: 4223 /* 4224 * o0 = vaddr 4225 * o1 = sfmmup 4226 * o2 = ttep 4227 * g1 = tte 4228 * g2 = tte pa 4229 * g3 = tte va 4230 * o2 = tsbmiss area 4231 * o1 = hat id 4232 */ 4233 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 4234 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4235 stx %g1,[%o2] /* put tte into *ttep */ 4236 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 4237 /* 4238 * o0 = vaddr 4239 * o1 = sfmmup 4240 * o2 = ttep 4241 * g1 = pfn 4242 */ 4243 ba,pt %xcc, 6f 4244 mov %g1, %o0 4245 4246kvtop_nohblk: 4247 /* 4248 * we get here if we couldn't find valid hblk in hash. We rehash 4249 * if neccesary. 4250 */ 4251 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 4252#ifdef sun4v 4253 cmp %g5, MAX_HASHCNT 4254#else 4255 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 4256#endif /* sun4v */ 4257 be,a,pn %icc, 6f 4258 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4259 mov %o1, %o4 /* restore hatid */ 4260#ifdef sun4v 4261 add %g5, 2, %g5 4262 cmp %g5, 3 4263 move %icc, MMU_PAGESHIFT4M, %g6 4264 ba,pt %icc, 1b 4265 movne %icc, MMU_PAGESHIFT256M, %g6 4266#else 4267 inc %g5 4268 cmp %g5, 2 4269 move %icc, MMU_PAGESHIFT512K, %g6 4270 ba,pt %icc, 1b 4271 movne %icc, MMU_PAGESHIFT4M, %g6 4272#endif /* sun4v */ 42736: 4274 retl 4275 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4276 4277tsb_suspend: 4278 /* 4279 * o0 = vaddr 4280 * o1 = sfmmup 4281 * o2 = ttep 4282 * g1 = tte 4283 * g2 = tte pa 4284 * g3 = tte va 4285 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 4286 */ 4287 stx %g1,[%o2] /* put tte into *ttep */ 4288 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 4289 sub %g0, 1, %o0 /* output = PFN_INVALID */ 4290 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 42918: 4292 retl 4293 wrpr %g0, %o3, %pstate /* enable interrupts */ 4294 4295vatopfn_nokernel: 4296 /* 4297 * This routine does NOT support user addresses 4298 * There is a routine in C that supports this. 4299 * The only reason why we don't have the C routine 4300 * support kernel addresses as well is because 4301 * we do va_to_pa while holding the hashlock. 4302 */ 4303 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4304 save %sp, -SA(MINFRAME), %sp 4305 sethi %hi(sfmmu_panic3), %o0 4306 call panic 4307 or %o0, %lo(sfmmu_panic3), %o0 4308 4309 SET_SIZE(sfmmu_vatopfn) 4310 4311 /* 4312 * %o0 = vaddr 4313 * %o1 = hashno (aka szc) 4314 * 4315 * 4316 * This routine is similar to sfmmu_vatopfn() but will only look for 4317 * a kernel vaddr in the hash structure for the specified rehash value. 4318 * It's just an optimization for the case when pagesize for a given 4319 * va range is already known (e.g. large page heap) and we don't want 4320 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 4321 * 4322 * Returns valid pfn or PFN_INVALID if 4323 * tte for specified rehash # is not found, invalid or suspended. 4324 */ 4325 ENTRY_NP(sfmmu_kvaszc2pfn) 4326 /* 4327 * disable interrupts 4328 */ 4329 rdpr %pstate, %o3 4330#ifdef DEBUG 4331 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 4332#endif 4333 /* 4334 * disable interrupts to protect the TSBMISS area 4335 */ 4336 andn %o3, PSTATE_IE, %o5 4337 wrpr %o5, 0, %pstate 4338 4339 CPU_TSBMISS_AREA(%g1, %o5) 4340 ldn [%g1 + TSBMISS_KHATID], %o4 4341 sll %o1, 1, %g6 4342 add %g6, %o1, %g6 4343 add %g6, MMU_PAGESHIFT, %g6 4344 /* 4345 * %o0 = vaddr 4346 * %o1 = hashno 4347 * %o3 = old %pstate 4348 * %o4 = ksfmmup 4349 * %g1 = tsbmiss area 4350 * %g6 = hmeshift 4351 */ 4352 4353 /* 4354 * The first arg to GET_TTE is actually tagaccess register 4355 * not just vaddr. Since this call is for kernel we need to clear 4356 * any lower vaddr bits that would be interpreted as ctx bits. 4357 */ 4358 srlx %o0, MMU_PAGESHIFT, %o0 4359 sllx %o0, MMU_PAGESHIFT, %o0 4360 GET_TTE(%o0, %o4, %g3, %g4, %g5, %g1, %o5, %g6, %o1, 4361 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 4362 kvaszc2pfn_nohblk) 4363 4364kvaszc2pfn_hblk_found: 4365 /* 4366 * %g3 = tte 4367 * %o0 = vaddr 4368 */ 4369 brgez,a,pn %g3, 1f /* check if tte is invalid */ 4370 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4371 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 4372 /* 4373 * g3 = pfn 4374 */ 4375 ba,pt %xcc, 1f 4376 mov %g3, %o0 4377 4378kvaszc2pfn_nohblk: 4379 mov -1, %o0 4380 43811: 4382 retl 4383 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4384 4385 SET_SIZE(sfmmu_kvaszc2pfn) 4386 4387#endif /* lint */ 4388 4389 4390 4391#if !defined(lint) 4392 4393/* 4394 * kpm lock used between trap level tsbmiss handler and kpm C level. 4395 */ 4396#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 4397 mov 0xff, tmp1 ;\ 4398label1: ;\ 4399 casa [kpmlckp]asi, %g0, tmp1 ;\ 4400 brnz,pn tmp1, label1 ;\ 4401 mov 0xff, tmp1 ;\ 4402 membar #LoadLoad 4403 4404#define KPMLOCK_EXIT(kpmlckp, asi) \ 4405 membar #LoadStore|#StoreStore ;\ 4406 sta %g0, [kpmlckp]asi 4407 4408/* 4409 * Lookup a memseg for a given pfn and if found, return the physical 4410 * address of the corresponding struct memseg in mseg, otherwise 4411 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 4412 * tsbmp, %asi is assumed to be ASI_MEM. 4413 * This lookup is done by strictly traversing only the physical memseg 4414 * linkage. The more generic approach, to check the virtual linkage 4415 * before using the physical (used e.g. with hmehash buckets), cannot 4416 * be used here. Memory DR operations can run in parallel to this 4417 * lookup w/o any locks and updates of the physical and virtual linkage 4418 * cannot be done atomically wrt. to each other. Because physical 4419 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 4420 * as "physical NULL" pointer. 4421 */ 4422#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 4423 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 4424 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 4425 udivx pfn, mseg, mseg ;\ 4426 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 4427 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 4428 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 4429 add tmp1, mseg, tmp1 ;\ 4430 ldxa [tmp1]%asi, mseg ;\ 4431 cmp mseg, MSEG_NULLPTR_PA ;\ 4432 be,pn %xcc, label/**/1 /* if not found */ ;\ 4433 nop ;\ 4434 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4435 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4436 blu,pn %xcc, label/**/1 ;\ 4437 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4438 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4439 bgeu,pn %xcc, label/**/1 ;\ 4440 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 4441 mulx tmp1, PAGE_SIZE, tmp1 ;\ 4442 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 4443 add tmp2, tmp1, tmp1 /* pp */ ;\ 4444 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 4445 cmp tmp2, pfn ;\ 4446 be,pt %xcc, label/**/_ok /* found */ ;\ 4447label/**/1: ;\ 4448 /* brute force lookup */ ;\ 4449 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 4450 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 4451label/**/2: ;\ 4452 cmp mseg, MSEG_NULLPTR_PA ;\ 4453 be,pn %xcc, label/**/_ok /* if not found */ ;\ 4454 nop ;\ 4455 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4456 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4457 blu,a,pt %xcc, label/**/2 ;\ 4458 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4459 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4460 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4461 bgeu,a,pt %xcc, label/**/2 ;\ 4462 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4463label/**/_ok: 4464 4465 /* 4466 * kpm tsb miss handler large pages 4467 * g1 = 8K kpm TSB entry pointer 4468 * g2 = tag access register 4469 * g3 = 4M kpm TSB entry pointer 4470 */ 4471 ALTENTRY(sfmmu_kpm_dtsb_miss) 4472 TT_TRACE(trace_tsbmiss) 4473 4474 CPU_INDEX(%g7, %g6) 4475 sethi %hi(kpmtsbm_area), %g6 4476 sllx %g7, KPMTSBM_SHIFT, %g7 4477 or %g6, %lo(kpmtsbm_area), %g6 4478 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4479 4480 /* check enable flag */ 4481 ldub [%g6 + KPMTSBM_FLAGS], %g4 4482 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4483 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4484 nop 4485 4486 /* VA range check */ 4487 ldx [%g6 + KPMTSBM_VBASE], %g7 4488 cmp %g2, %g7 4489 blu,pn %xcc, sfmmu_tsb_miss 4490 ldx [%g6 + KPMTSBM_VEND], %g5 4491 cmp %g2, %g5 4492 bgeu,pn %xcc, sfmmu_tsb_miss 4493 stx %g3, [%g6 + KPMTSBM_TSBPTR] 4494 4495 /* 4496 * check TL tsbmiss handling flag 4497 * bump tsbmiss counter 4498 */ 4499 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4500#ifdef DEBUG 4501 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4502 inc %g5 4503 brz,pn %g3, sfmmu_kpm_exception 4504 st %g5, [%g6 + KPMTSBM_TSBMISS] 4505#else 4506 inc %g5 4507 st %g5, [%g6 + KPMTSBM_TSBMISS] 4508#endif 4509 /* 4510 * At this point: 4511 * g1 = 8K kpm TSB pointer (not used) 4512 * g2 = tag access register 4513 * g3 = clobbered 4514 * g6 = per-CPU kpm tsbmiss area 4515 * g7 = kpm_vbase 4516 */ 4517 4518 /* vaddr2pfn */ 4519 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4520 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4521 srax %g4, %g3, %g2 /* which alias range (r) */ 4522 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4523 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4524 4525 /* 4526 * Setup %asi 4527 * mseg_pa = page_numtomemseg_nolock(pfn) 4528 * if (mseg_pa == NULL) sfmmu_kpm_exception 4529 * g2=pfn 4530 */ 4531 mov ASI_MEM, %asi 4532 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4533 cmp %g3, MSEG_NULLPTR_PA 4534 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4535 nop 4536 4537 /* 4538 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4539 * g2=pfn g3=mseg_pa 4540 */ 4541 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4542 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4543 srlx %g2, %g5, %g4 4544 sllx %g4, %g5, %g4 4545 sub %g4, %g7, %g4 4546 srlx %g4, %g5, %g4 4547 4548 /* 4549 * Validate inx value 4550 * g2=pfn g3=mseg_pa g4=inx 4551 */ 4552#ifdef DEBUG 4553 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4554 cmp %g4, %g5 /* inx - nkpmpgs */ 4555 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4556 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4557#else 4558 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4559#endif 4560 /* 4561 * kp = &mseg_pa->kpm_pages[inx] 4562 */ 4563 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4564 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4565 add %g5, %g4, %g5 /* kp */ 4566 4567 /* 4568 * KPMP_HASH(kp) 4569 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4570 */ 4571 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4572 sub %g7, 1, %g7 /* mask */ 4573 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4574 add %g5, %g1, %g5 /* y = ksp + x */ 4575 and %g5, %g7, %g5 /* hashinx = y & mask */ 4576 4577 /* 4578 * Calculate physical kpm_page pointer 4579 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4580 */ 4581 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4582 add %g1, %g4, %g1 /* kp_pa */ 4583 4584 /* 4585 * Calculate physical hash lock address 4586 * g1=kp_refcntc_pa g2=pfn g5=hashinx 4587 */ 4588 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4589 sllx %g5, KPMHLK_SHIFT, %g5 4590 add %g4, %g5, %g3 4591 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4592 4593 /* 4594 * Assemble tte 4595 * g1=kp_pa g2=pfn g3=hlck_pa 4596 */ 4597#ifdef sun4v 4598 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4599 sllx %g5, 32, %g5 4600 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4601 or %g4, TTE4M, %g4 4602 or %g5, %g4, %g5 4603#else 4604 sethi %hi(TTE_VALID_INT), %g4 4605 mov TTE4M, %g5 4606 sllx %g5, TTE_SZ_SHFT_INT, %g5 4607 or %g5, %g4, %g5 /* upper part */ 4608 sllx %g5, 32, %g5 4609 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4610 or %g5, %g4, %g5 4611#endif 4612 sllx %g2, MMU_PAGESHIFT, %g4 4613 or %g5, %g4, %g5 /* tte */ 4614 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4615 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4616 4617 /* 4618 * tsb dropin 4619 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4620 */ 4621 4622 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4623 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4624 4625 /* use C-handler if there's no go for dropin */ 4626 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4627 cmp %g7, -1 4628 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4629 nop 4630 4631#ifdef DEBUG 4632 /* double check refcnt */ 4633 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4634 brz,pn %g7, 5f /* let C-handler deal with this */ 4635 nop 4636#endif 4637 4638#ifndef sun4v 4639 ldub [%g6 + KPMTSBM_FLAGS], %g7 4640 mov ASI_N, %g1 4641 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4642 movnz %icc, ASI_MEM, %g1 4643 mov %g1, %asi 4644#endif 4645 4646 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4647 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4648 4649 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4650 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4651 4652 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4653 4654 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4655 KPMLOCK_EXIT(%g3, ASI_MEM) 4656 4657 /* 4658 * If trapstat is running, we need to shift the %tpc and %tnpc to 4659 * point to trapstat's TSB miss return code (note that trapstat 4660 * itself will patch the correct offset to add). 4661 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4662 */ 4663 rdpr %tl, %g7 4664 cmp %g7, 1 4665 ble %icc, 0f 4666 sethi %hi(KERNELBASE), %g6 4667 rdpr %tpc, %g7 4668 or %g6, %lo(KERNELBASE), %g6 4669 cmp %g7, %g6 4670 bgeu %xcc, 0f 4671 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4672 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4673 wrpr %g7, %tpc 4674 add %g7, 4, %g7 4675 wrpr %g7, %tnpc 46760: 4677 retry 46785: 4679 /* g3=hlck_pa */ 4680 KPMLOCK_EXIT(%g3, ASI_MEM) 4681 ba,pt %icc, sfmmu_kpm_exception 4682 nop 4683 SET_SIZE(sfmmu_kpm_dtsb_miss) 4684 4685 /* 4686 * kpm tsbmiss handler for smallpages 4687 * g1 = 8K kpm TSB pointer 4688 * g2 = tag access register 4689 * g3 = 4M kpm TSB pointer 4690 */ 4691 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4692 TT_TRACE(trace_tsbmiss) 4693 CPU_INDEX(%g7, %g6) 4694 sethi %hi(kpmtsbm_area), %g6 4695 sllx %g7, KPMTSBM_SHIFT, %g7 4696 or %g6, %lo(kpmtsbm_area), %g6 4697 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4698 4699 /* check enable flag */ 4700 ldub [%g6 + KPMTSBM_FLAGS], %g4 4701 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4702 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4703 nop 4704 4705 /* 4706 * VA range check 4707 * On fail: goto sfmmu_tsb_miss 4708 */ 4709 ldx [%g6 + KPMTSBM_VBASE], %g7 4710 cmp %g2, %g7 4711 blu,pn %xcc, sfmmu_tsb_miss 4712 ldx [%g6 + KPMTSBM_VEND], %g5 4713 cmp %g2, %g5 4714 bgeu,pn %xcc, sfmmu_tsb_miss 4715 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4716 4717 /* 4718 * check TL tsbmiss handling flag 4719 * bump tsbmiss counter 4720 */ 4721 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4722#ifdef DEBUG 4723 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4724 inc %g5 4725 brz,pn %g1, sfmmu_kpm_exception 4726 st %g5, [%g6 + KPMTSBM_TSBMISS] 4727#else 4728 inc %g5 4729 st %g5, [%g6 + KPMTSBM_TSBMISS] 4730#endif 4731 /* 4732 * At this point: 4733 * g1 = clobbered 4734 * g2 = tag access register 4735 * g3 = 4M kpm TSB pointer (not used) 4736 * g6 = per-CPU kpm tsbmiss area 4737 * g7 = kpm_vbase 4738 */ 4739 4740 /* 4741 * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr) 4742 * which is defined in mach_kpm.h. Any changes in that macro 4743 * should also be ported back to this assembly code. 4744 */ 4745 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 /* g3 = kpm_size_shift */ 4746 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4747 srax %g4, %g3, %g7 /* which alias range (r) */ 4748 brz,pt %g7, 2f 4749 sethi %hi(vac_colors_mask), %g5 4750 ld [%g5 + %lo(vac_colors_mask)], %g5 4751 4752 srlx %g2, MMU_PAGESHIFT, %g1 /* vaddr >> MMU_PAGESHIFT */ 4753 and %g1, %g5, %g1 /* g1 = v */ 4754 sllx %g7, %g3, %g5 /* g5 = r << kpm_size_shift */ 4755 cmp %g7, %g1 /* if (r > v) */ 4756 bleu,pn %xcc, 1f 4757 sub %g4, %g5, %g4 /* paddr -= r << kpm_size_shift */ 4758 sub %g7, %g1, %g5 /* g5 = r - v */ 4759 sllx %g5, MMU_PAGESHIFT, %g7 /* (r-v) << MMU_PAGESHIFT */ 4760 add %g4, %g7, %g4 /* paddr += (r-v)<<MMU_PAGESHIFT */ 4761 ba 2f 4762 nop 47631: 4764 sllx %g7, MMU_PAGESHIFT, %g5 /* else */ 4765 sub %g4, %g5, %g4 /* paddr -= r << MMU_PAGESHIFT */ 4766 4767 /* 4768 * paddr2pfn 4769 * g1 = vcolor (not used) 4770 * g2 = tag access register 4771 * g3 = clobbered 4772 * g4 = paddr 4773 * g5 = clobbered 4774 * g6 = per-CPU kpm tsbmiss area 4775 * g7 = clobbered 4776 */ 47772: 4778 srlx %g4, MMU_PAGESHIFT, %g2 /* g2 = pfn */ 4779 4780 /* 4781 * Setup %asi 4782 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4783 * if (mseg not found) sfmmu_kpm_exception 4784 * g2=pfn g6=per-CPU kpm tsbmiss area 4785 * g4 g5 g7 for scratch use. 4786 */ 4787 mov ASI_MEM, %asi 4788 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4789 cmp %g3, MSEG_NULLPTR_PA 4790 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4791 nop 4792 4793 /* 4794 * inx = pfn - mseg_pa->kpm_pbase 4795 * g2=pfn g3=mseg_pa g6=per-CPU kpm tsbmiss area 4796 */ 4797 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4798 sub %g2, %g7, %g4 4799 4800#ifdef DEBUG 4801 /* 4802 * Validate inx value 4803 * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area 4804 */ 4805 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4806 cmp %g4, %g5 /* inx - nkpmpgs */ 4807 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4808 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4809#else 4810 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4811#endif 4812 /* ksp = &mseg_pa->kpm_spages[inx] */ 4813 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4814 add %g5, %g4, %g5 /* ksp */ 4815 4816 /* 4817 * KPMP_SHASH(kp) 4818 * g2=pfn g3=mseg_pa g4=inx g5=ksp 4819 * g6=per-CPU kpm tsbmiss area g7=kpmp_stable_sz 4820 */ 4821 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4822 sub %g7, 1, %g7 /* mask */ 4823 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4824 add %g5, %g1, %g5 /* y = ksp + x */ 4825 and %g5, %g7, %g5 /* hashinx = y & mask */ 4826 4827 /* 4828 * Calculate physical kpm_spage pointer 4829 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4830 * g6=per-CPU kpm tsbmiss area 4831 */ 4832 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4833 add %g1, %g4, %g1 /* ksp_pa */ 4834 4835 /* 4836 * Calculate physical hash lock address. 4837 * Note: Changes in kpm_shlk_t must be reflected here. 4838 * g1=ksp_pa g2=pfn g5=hashinx 4839 * g6=per-CPU kpm tsbmiss area 4840 */ 4841 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4842 sllx %g5, KPMSHLK_SHIFT, %g5 4843 add %g4, %g5, %g3 /* hlck_pa */ 4844 4845 /* 4846 * Assemble non-cacheable tte initially 4847 * g1=ksp_pa g2=pfn g3=hlck_pa 4848 * g6=per-CPU kpm tsbmiss area 4849 */ 4850 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4851 sllx %g5, 32, %g5 4852 mov (TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4853 or %g5, %g4, %g5 4854 sllx %g2, MMU_PAGESHIFT, %g4 4855 or %g5, %g4, %g5 /* tte */ 4856 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4857 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4858 4859 /* 4860 * tsb dropin 4861 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable) 4862 * g6=per-CPU kpm tsbmiss area g7=scratch register 4863 */ 4864 4865 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4866 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4867 4868 /* use C-handler if there's no go for dropin */ 4869 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4870 andcc %g7, KPM_MAPPED_GO, %g0 /* go or no go ? */ 4871 bz,pt %icc, 5f /* no go */ 4872 nop 4873 and %g7, KPM_MAPPED_MASK, %g7 /* go */ 4874 cmp %g7, KPM_MAPPEDS /* cacheable ? */ 4875 be,a,pn %xcc, 3f 4876 or %g5, TTE_CV_INT, %g5 /* cacheable */ 48773: 4878#ifndef sun4v 4879 ldub [%g6 + KPMTSBM_FLAGS], %g7 4880 mov ASI_N, %g1 4881 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4882 movnz %icc, ASI_MEM, %g1 4883 mov %g1, %asi 4884#endif 4885 4886 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4887 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4888 4889 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4890 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4891 4892 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4893 4894 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4895 KPMLOCK_EXIT(%g3, ASI_MEM) 4896 4897 /* 4898 * If trapstat is running, we need to shift the %tpc and %tnpc to 4899 * point to trapstat's TSB miss return code (note that trapstat 4900 * itself will patch the correct offset to add). 4901 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4902 */ 4903 rdpr %tl, %g7 4904 cmp %g7, 1 4905 ble %icc, 0f 4906 sethi %hi(KERNELBASE), %g6 4907 rdpr %tpc, %g7 4908 or %g6, %lo(KERNELBASE), %g6 4909 cmp %g7, %g6 4910 bgeu %xcc, 0f 4911 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4912 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4913 wrpr %g7, %tpc 4914 add %g7, 4, %g7 4915 wrpr %g7, %tnpc 49160: 4917 retry 49185: 4919 /* g3=hlck_pa */ 4920 KPMLOCK_EXIT(%g3, ASI_MEM) 4921 ba,pt %icc, sfmmu_kpm_exception 4922 nop 4923 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4924 4925#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4926#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4927#endif 4928 4929#endif /* lint */ 4930 4931#ifdef lint 4932/* 4933 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4934 * Called from C-level, sets/clears "go" indication for trap level handler. 4935 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 4936 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 4937 * Assumes khl_mutex is held when called from C-level. 4938 */ 4939/* ARGSUSED */ 4940void 4941sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 4942{ 4943} 4944 4945/* 4946 * kpm_smallpages: stores val to byte at address mapped within 4947 * low level lock brackets. The old value is returned. 4948 * Called from C-level. 4949 */ 4950/* ARGSUSED */ 4951int 4952sfmmu_kpm_stsbmtl(uchar_t *mapped, uint_t *kshl_lock, int val) 4953{ 4954 return (0); 4955} 4956 4957#else /* lint */ 4958 4959 .seg ".data" 4960sfmmu_kpm_tsbmtl_panic: 4961 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4962 .byte 0 4963sfmmu_kpm_stsbmtl_panic: 4964 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4965 .byte 0 4966 .align 4 4967 .seg ".text" 4968 4969 ENTRY_NP(sfmmu_kpm_tsbmtl) 4970 rdpr %pstate, %o3 4971 /* 4972 * %o0 = &kp_refcntc 4973 * %o1 = &khl_lock 4974 * %o2 = 0/1 (off/on) 4975 * %o3 = pstate save 4976 */ 4977#ifdef DEBUG 4978 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4979 bnz,pt %icc, 1f /* disabled, panic */ 4980 nop 4981 save %sp, -SA(MINFRAME), %sp 4982 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4983 call panic 4984 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4985 ret 4986 restore 49871: 4988#endif /* DEBUG */ 4989 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4990 4991 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4992 mov -1, %o5 4993 brz,a %o2, 2f 4994 mov 0, %o5 49952: 4996 sth %o5, [%o0] 4997 KPMLOCK_EXIT(%o1, ASI_N) 4998 4999 retl 5000 wrpr %g0, %o3, %pstate /* enable interrupts */ 5001 SET_SIZE(sfmmu_kpm_tsbmtl) 5002 5003 ENTRY_NP(sfmmu_kpm_stsbmtl) 5004 rdpr %pstate, %o3 5005 /* 5006 * %o0 = &mapped 5007 * %o1 = &kshl_lock 5008 * %o2 = val 5009 * %o3 = pstate save 5010 */ 5011#ifdef DEBUG 5012 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 5013 bnz,pt %icc, 1f /* disabled, panic */ 5014 nop 5015 save %sp, -SA(MINFRAME), %sp 5016 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 5017 call panic 5018 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 5019 ret 5020 restore 50211: 5022#endif /* DEBUG */ 5023 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 5024 5025 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 5026 ldsb [%o0], %o5 5027 stb %o2, [%o0] 5028 KPMLOCK_EXIT(%o1, ASI_N) 5029 5030 and %o5, KPM_MAPPED_MASK, %o0 /* return old val */ 5031 retl 5032 wrpr %g0, %o3, %pstate /* enable interrupts */ 5033 SET_SIZE(sfmmu_kpm_stsbmtl) 5034 5035#endif /* lint */ 5036 5037#ifndef lint 5038#ifdef sun4v 5039 /* 5040 * User/kernel data miss w// multiple TSBs 5041 * The first probe covers 8K, 64K, and 512K page sizes, 5042 * because 64K and 512K mappings are replicated off 8K 5043 * pointer. Second probe covers 4M page size only. 5044 * 5045 * MMU fault area contains miss address and context. 5046 */ 5047 ALTENTRY(sfmmu_slow_dmmu_miss) 5048 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 5049 5050slow_miss_common: 5051 /* 5052 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 5053 * %g3 = ctx (cannot be INVALID_CONTEXT) 5054 */ 5055 brnz,pt %g3, 8f ! check for user context 5056 nop 5057 5058 /* 5059 * Kernel miss 5060 * Get 8K and 4M TSB pointers in %g1 and %g3 and 5061 * branch to sfmmu_tsb_miss_tt to handle it. 5062 */ 5063 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 5064sfmmu_dslow_patch_ktsb_base: 5065 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 5066sfmmu_dslow_patch_ktsb_szcode: 5067 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 5068 5069 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 5070 ! %g1 = First TSB entry pointer, as TSB miss handler expects 5071 5072 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 5073sfmmu_dslow_patch_ktsb4m_base: 5074 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 5075sfmmu_dslow_patch_ktsb4m_szcode: 5076 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 5077 5078 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 5079 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 5080 ba,a,pt %xcc, sfmmu_tsb_miss_tt 5081 .empty 5082 50838: 5084 /* 5085 * User miss 5086 * Get first TSB pointer in %g1 5087 * Get second TSB pointer (or NULL if no second TSB) in %g3 5088 * Branch to sfmmu_tsb_miss_tt to handle it 5089 */ 5090 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 5091 /* %g1 = first TSB entry ptr now, %g2 preserved */ 5092 5093 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 5094 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 5095 nop 5096 5097 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 5098 /* %g3 = second TSB entry ptr now, %g2 preserved */ 50999: 5100 ba,a,pt %xcc, sfmmu_tsb_miss_tt 5101 .empty 5102 SET_SIZE(sfmmu_slow_dmmu_miss) 5103 5104 5105 /* 5106 * User/kernel instruction miss w/ multiple TSBs 5107 * The first probe covers 8K, 64K, and 512K page sizes, 5108 * because 64K and 512K mappings are replicated off 8K 5109 * pointer. Second probe covers 4M page size only. 5110 * 5111 * MMU fault area contains miss address and context. 5112 */ 5113 ALTENTRY(sfmmu_slow_immu_miss) 5114 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 5115 ba,a,pt %xcc, slow_miss_common 5116 SET_SIZE(sfmmu_slow_immu_miss) 5117 5118#endif /* sun4v */ 5119#endif /* lint */ 5120 5121#ifndef lint 5122 5123/* 5124 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 5125 */ 5126 .seg ".data" 5127 .align 64 5128 .global tsbmiss_area 5129tsbmiss_area: 5130 .skip (TSBMISS_SIZE * NCPU) 5131 5132 .align 64 5133 .global kpmtsbm_area 5134kpmtsbm_area: 5135 .skip (KPMTSBM_SIZE * NCPU) 5136#endif /* lint */ 5137