1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#pragma ident "%Z%%M% %I% %E% SMI" 27 28/* 29 * SFMMU primitives. These primitives should only be used by sfmmu 30 * routines. 31 */ 32 33#if defined(lint) 34#include <sys/types.h> 35#else /* lint */ 36#include "assym.h" 37#endif /* lint */ 38 39#include <sys/asm_linkage.h> 40#include <sys/machtrap.h> 41#include <sys/machasi.h> 42#include <sys/sun4asi.h> 43#include <sys/pte.h> 44#include <sys/mmu.h> 45#include <vm/hat_sfmmu.h> 46#include <vm/seg_spt.h> 47#include <sys/machparam.h> 48#include <sys/privregs.h> 49#include <sys/scb.h> 50#include <sys/intreg.h> 51#include <sys/machthread.h> 52#include <sys/intr.h> 53#include <sys/clock.h> 54#include <sys/trapstat.h> 55 56#ifdef TRAPTRACE 57#include <sys/traptrace.h> 58 59/* 60 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 61 */ 62#define TT_TRACE(label) \ 63 ba label ;\ 64 rd %pc, %g7 65#else 66 67#define TT_TRACE(label) 68 69#endif /* TRAPTRACE */ 70 71#ifndef lint 72 73#if (TTE_SUSPEND_SHIFT > 0) 74#define TTE_SUSPEND_INT_SHIFT(reg) \ 75 sllx reg, TTE_SUSPEND_SHIFT, reg 76#else 77#define TTE_SUSPEND_INT_SHIFT(reg) 78#endif 79 80#endif /* lint */ 81 82#ifndef lint 83 84/* 85 * Assumes TSBE_TAG is 0 86 * Assumes TSBE_INTHI is 0 87 * Assumes TSBREG.split is 0 88 */ 89 90#if TSBE_TAG != 0 91#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 92#endif 93 94#if TSBTAG_INTHI != 0 95#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 96#endif 97 98/* 99 * The following code assumes the tsb is not split. 100 * 101 * With TSBs no longer shared between processes, it's no longer 102 * necessary to hash the context bits into the tsb index to get 103 * tsb coloring; the new implementation treats the TSB as a 104 * direct-mapped, virtually-addressed cache. 105 * 106 * In: 107 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 108 * tsbbase = base address of TSB (clobbered) 109 * tagacc = tag access register (clobbered) 110 * szc = size code of TSB (ro) 111 * tmp = scratch reg 112 * Out: 113 * tsbbase = pointer to entry in TSB 114 */ 115#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 116 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 117 srlx tagacc, vpshift, tagacc ;\ 118 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 119 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 120 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 121 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 122 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 123 124/* 125 * When the kpm TSB is used it is assumed that it is direct mapped 126 * using (vaddr>>vpshift)%tsbsz as the index. 127 * 128 * Note that, for now, the kpm TSB and kernel TSB are the same for 129 * each mapping size. However that need not always be the case. If 130 * the trap handlers are updated to search a different TSB for kpm 131 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 132 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 133 * 134 * In: 135 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 136 * vaddr = virtual address (clobbered) 137 * tsbp, szc, tmp = scratch 138 * Out: 139 * tsbp = pointer to entry in TSB 140 */ 141#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 142 cmp vpshift, MMU_PAGESHIFT ;\ 143 bne,pn %icc, 1f /* branch if large case */ ;\ 144 sethi %hi(kpmsm_tsbsz), szc ;\ 145 sethi %hi(kpmsm_tsbbase), tsbp ;\ 146 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 147 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 148 ba,pt %icc, 2f ;\ 149 nop ;\ 1501: sethi %hi(kpm_tsbsz), szc ;\ 151 sethi %hi(kpm_tsbbase), tsbp ;\ 152 ld [szc + %lo(kpm_tsbsz)], szc ;\ 153 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1542: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 155 156/* 157 * Lock the TSBE at virtual address tsbep. 158 * 159 * tsbep = TSBE va (ro) 160 * tmp1, tmp2 = scratch registers (clobbered) 161 * label = label to use for branches (text) 162 * %asi = ASI to use for TSB access 163 * 164 * NOTE that we flush the TSB using fast VIS instructions that 165 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 166 * not be treated as a locked entry or we'll get stuck spinning on 167 * an entry that isn't locked but really invalid. 168 */ 169 170#if defined(UTSB_PHYS) 171 172#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 173 lda [tsbep]ASI_MEM, tmp1 ;\ 174label: ;\ 175 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 176 cmp tmp1, tmp2 ;\ 177 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 178 lda [tsbep]ASI_MEM, tmp1 ;\ 179 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 180 cmp tmp1, tmp2 ;\ 181 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 182 lda [tsbep]ASI_MEM, tmp1 ;\ 183 /* tsbe lock acquired */ ;\ 184 membar #StoreStore 185 186#else /* UTSB_PHYS */ 187 188#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 189 lda [tsbep]%asi, tmp1 ;\ 190label: ;\ 191 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 192 cmp tmp1, tmp2 ;\ 193 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 194 lda [tsbep]%asi, tmp1 ;\ 195 casa [tsbep]%asi, tmp1, tmp2 ;\ 196 cmp tmp1, tmp2 ;\ 197 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 198 lda [tsbep]%asi, tmp1 ;\ 199 /* tsbe lock acquired */ ;\ 200 membar #StoreStore 201 202#endif /* UTSB_PHYS */ 203 204/* 205 * Atomically write TSBE at virtual address tsbep. 206 * 207 * tsbep = TSBE va (ro) 208 * tte = TSBE TTE (ro) 209 * tagtarget = TSBE tag (ro) 210 * %asi = ASI to use for TSB access 211 */ 212 213#if defined(UTSB_PHYS) 214 215#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 216 add tsbep, TSBE_TTE, tmp1 ;\ 217 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 218 membar #StoreStore ;\ 219 add tsbep, TSBE_TAG, tmp1 ;\ 220 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 221 222#else /* UTSB_PHYS */ 223 224#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 225 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 226 membar #StoreStore ;\ 227 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 228 229#endif /* UTSB_PHYS */ 230 231/* 232 * Load an entry into the TSB at TL > 0. 233 * 234 * tsbep = pointer to the TSBE to load as va (ro) 235 * tte = value of the TTE retrieved and loaded (wo) 236 * tagtarget = tag target register. To get TSBE tag to load, 237 * we need to mask off the context and leave only the va (clobbered) 238 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 239 * tmp1, tmp2 = scratch registers 240 * label = label to use for branches (text) 241 * %asi = ASI to use for TSB access 242 */ 243 244#if defined(UTSB_PHYS) 245 246#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 247 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 248 /* ;\ 249 * I don't need to update the TSB then check for the valid tte. ;\ 250 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 251 * we always invalidate the hash table before we unload the TSB.;\ 252 */ ;\ 253 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 254 ldxa [ttepa]ASI_MEM, tte ;\ 255 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 256 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 257 add tsbep, TSBE_TAG, tmp1 ;\ 258 brgez,a,pn tte, label/**/f ;\ 259 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 260 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 261label: 262 263#else /* UTSB_PHYS */ 264 265#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 266 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 267 /* ;\ 268 * I don't need to update the TSB then check for the valid tte. ;\ 269 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 270 * we always invalidate the hash table before we unload the TSB.;\ 271 */ ;\ 272 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 273 ldxa [ttepa]ASI_MEM, tte ;\ 274 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 275 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 276 brgez,a,pn tte, label/**/f ;\ 277 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 278 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 279label: 280 281#endif /* UTSB_PHYS */ 282 283/* 284 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 285 * for ITLB synthesis. 286 * 287 * tsbep = pointer to the TSBE to load as va (ro) 288 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 289 * with exec_perm turned off and exec_synth turned on 290 * tagtarget = tag target register. To get TSBE tag to load, 291 * we need to mask off the context and leave only the va (clobbered) 292 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 293 * tmp1, tmp2 = scratch registers 294 * label = label to use for branch (text) 295 * %asi = ASI to use for TSB access 296 */ 297 298#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 299 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 300 /* ;\ 301 * I don't need to update the TSB then check for the valid tte. ;\ 302 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 303 * we always invalidate the hash table before we unload the TSB.;\ 304 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 305 * and exec_synth bit to 1. ;\ 306 */ ;\ 307 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 308 mov tte, tmp1 ;\ 309 ldxa [ttepa]ASI_MEM, tte ;\ 310 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 311 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 312 brgez,a,pn tte, label/**/f ;\ 313 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 314 or tte, tmp1, tte ;\ 315 andn tte, TTE_EXECPRM_INT, tte ;\ 316 or tte, TTE_E_SYNTH_INT, tte ;\ 317 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 318label: 319 320/* 321 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 322 * 323 * tte = value of the TTE, used to get tte_size bits (ro) 324 * tagaccess = tag access register, used to get 4M pfn bits (ro) 325 * pfn = 4M pfn bits shifted to offset for tte (out) 326 * tmp1 = scratch register 327 * label = label to use for branch (text) 328 */ 329 330#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 331 /* ;\ 332 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 333 * Return them, shifted, in pfn. ;\ 334 */ ;\ 335 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 336 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 337 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 338 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 339 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 340 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 341label: ;\ 342 sllx tagaccess, MMU_PAGESHIFT4M, pfn 343 344/* 345 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 346 * for ITLB synthesis. 347 * 348 * tte = value of the TTE, used to get tte_size bits (rw) 349 * tmp1 = scratch register 350 */ 351 352#define SET_TTE4M_PN(tte, tmp) \ 353 /* ;\ 354 * Set 4M pagesize tte bits. ;\ 355 */ ;\ 356 set TTE4M, tmp ;\ 357 sllx tmp, TTE_SZ_SHFT, tmp ;\ 358 or tte, tmp, tte 359 360/* 361 * Load an entry into the TSB at TL=0. 362 * 363 * tsbep = pointer to the TSBE to load as va (ro) 364 * tteva = pointer to the TTE to load as va (ro) 365 * tagtarget = TSBE tag to load (which contains no context), synthesized 366 * to match va of MMU tag target register only (ro) 367 * tmp1, tmp2 = scratch registers (clobbered) 368 * label = label to use for branches (text) 369 * %asi = ASI to use for TSB access 370 */ 371 372#if defined(UTSB_PHYS) 373 374#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 375 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 376 ldx [tteva], tteva /* load tte */ ;\ 377 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 378 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 379 add tsbep, TSBE_TAG, tmp1 ;\ 380 brgez,a,pn tteva, label/**/f ;\ 381 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 382 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 383label: 384 385#else /* UTSB_PHYS */ 386 387#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 388 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 389 ldx [tteva], tteva /* load tte */ ;\ 390 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 391 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 392 brgez,a,pn tteva, label/**/f ;\ 393 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 394 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 395label: 396 397#endif /* UTSB_PHYS */ 398 399/* 400 * Invalidate a TSB entry in the TSB. 401 * 402 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 403 * about this earlier to ensure this is true. Thus when we are 404 * directly referencing tsbep below, we are referencing the tte_tag 405 * field of the TSBE. If this offset ever changes, the code below 406 * will need to be modified. 407 * 408 * tsbep = pointer to TSBE as va (ro) 409 * tag = invalidation is done if this matches the TSBE tag (ro) 410 * tmp1 - tmp3 = scratch registers (clobbered) 411 * label = label name to use for branches (text) 412 * %asi = ASI to use for TSB access 413 */ 414 415#if defined(UTSB_PHYS) 416 417#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 418 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 419 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 420label/**/1: ;\ 421 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 422 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 423 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 424 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 425 cmp tag, tmp3 /* compare tags */ ;\ 426 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 427 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 428 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 429 cmp tmp1, tmp3 /* if not successful */ ;\ 430 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 431 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 432label/**/2: 433 434#else /* UTSB_PHYS */ 435 436#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 437 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 438 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 439label/**/1: ;\ 440 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 441 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 442 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 443 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 444 cmp tag, tmp3 /* compare tags */ ;\ 445 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 446 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 447 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 448 cmp tmp1, tmp3 /* if not successful */ ;\ 449 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 450 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 451label/**/2: 452 453#endif /* UTSB_PHYS */ 454 455#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 456#error - TSB_SOFTSZ_MASK too small 457#endif 458 459 460/* 461 * An implementation of setx which will be hot patched at run time. 462 * since it is being hot patched, there is no value passed in. 463 * Thus, essentially we are implementing 464 * setx value, tmp, dest 465 * where value is RUNTIME_PATCH (aka 0) in this case. 466 */ 467#define RUNTIME_PATCH_SETX(dest, tmp) \ 468 sethi %hh(RUNTIME_PATCH), tmp ;\ 469 sethi %lm(RUNTIME_PATCH), dest ;\ 470 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 471 or dest, %lo(RUNTIME_PATCH), dest ;\ 472 sllx tmp, 32, tmp ;\ 473 nop /* for perf reasons */ ;\ 474 or tmp, dest, dest /* contents of patched value */ 475 476 477#endif (lint) 478 479 480#if defined (lint) 481 482/* 483 * sfmmu related subroutines 484 */ 485uint_t 486sfmmu_disable_intrs() 487{ return(0); } 488 489/* ARGSUSED */ 490void 491sfmmu_enable_intrs(uint_t pstate_save) 492{} 493 494/* ARGSUSED */ 495int 496sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag) 497{ return(0); } 498 499/* 500 * Use cas, if tte has changed underneath us then reread and try again. 501 * In the case of a retry, it will update sttep with the new original. 502 */ 503/* ARGSUSED */ 504int 505sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 506{ return(0); } 507 508/* 509 * Use cas, if tte has changed underneath us then return 1, else return 0 510 */ 511/* ARGSUSED */ 512int 513sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 514{ return(0); } 515 516/* ARGSUSED */ 517void 518sfmmu_copytte(tte_t *sttep, tte_t *dttep) 519{} 520 521/*ARGSUSED*/ 522struct tsbe * 523sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 524{ return(0); } 525 526/*ARGSUSED*/ 527uint64_t 528sfmmu_make_tsbtag(caddr_t va) 529{ return(0); } 530 531#else /* lint */ 532 533 .seg ".data" 534 .global sfmmu_panic1 535sfmmu_panic1: 536 .asciz "sfmmu_asm: interrupts already disabled" 537 538 .global sfmmu_panic3 539sfmmu_panic3: 540 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 541 542 .global sfmmu_panic4 543sfmmu_panic4: 544 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 545 546 .global sfmmu_panic5 547sfmmu_panic5: 548 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 549 550 .global sfmmu_panic6 551sfmmu_panic6: 552 .asciz "sfmmu_asm: interrupts not disabled" 553 554 .global sfmmu_panic7 555sfmmu_panic7: 556 .asciz "sfmmu_asm: kernel as" 557 558 .global sfmmu_panic8 559sfmmu_panic8: 560 .asciz "sfmmu_asm: gnum is zero" 561 562 .global sfmmu_panic9 563sfmmu_panic9: 564 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 565 566 .global sfmmu_panic10 567sfmmu_panic10: 568 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 569 570 ENTRY(sfmmu_disable_intrs) 571 rdpr %pstate, %o0 572#ifdef DEBUG 573 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 574#endif /* DEBUG */ 575 retl 576 wrpr %o0, PSTATE_IE, %pstate 577 SET_SIZE(sfmmu_disable_intrs) 578 579 ENTRY(sfmmu_enable_intrs) 580 retl 581 wrpr %g0, %o0, %pstate 582 SET_SIZE(sfmmu_enable_intrs) 583 584/* 585 * This routine is called both by resume() and sfmmu_get_ctx() to 586 * allocate a new context for the process on a MMU. 587 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 588 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 589 * is the case when sfmmu_alloc_ctx is called from resume(). 590 * 591 * The caller must disable interrupts before entering this routine. 592 * To reduce ctx switch overhead, the code contains both 'fast path' and 593 * 'slow path' code. The fast path code covers the common case where only 594 * a quick check is needed and the real ctx allocation is not required. 595 * It can be done without holding the per-process (PP) lock. 596 * The 'slow path' code must be protected by the PP Lock and performs ctx 597 * allocation. 598 * Hardware context register and HAT mmu cnum are updated accordingly. 599 * 600 * %o0 - sfmmup 601 * %o1 - allocflag 602 * %o2 - CPU 603 * %o3 - sfmmu private/shared flag 604 * 605 * ret - 0: no ctx is allocated 606 * 1: a ctx is allocated 607 */ 608 ENTRY_NP(sfmmu_alloc_ctx) 609 610#ifdef DEBUG 611 sethi %hi(ksfmmup), %g1 612 ldx [%g1 + %lo(ksfmmup)], %g1 613 cmp %g1, %o0 614 bne,pt %xcc, 0f 615 nop 616 617 sethi %hi(panicstr), %g1 ! if kernel as, panic 618 ldx [%g1 + %lo(panicstr)], %g1 619 tst %g1 620 bnz,pn %icc, 7f 621 nop 622 623 sethi %hi(sfmmu_panic7), %o0 624 call panic 625 or %o0, %lo(sfmmu_panic7), %o0 626 6277: 628 retl 629 mov %g0, %o0 ! %o0 = ret = 0 630 6310: 632 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 633#endif /* DEBUG */ 634 635 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 636 637 ! load global mmu_ctxp info 638 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 639 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 640 641 ! load global mmu_ctxp gnum 642 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 643 644#ifdef DEBUG 645 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 646 bne,pt %xcc, 3f 647 nop 648 649 sethi %hi(panicstr), %g1 ! test if panicstr is already set 650 ldx [%g1 + %lo(panicstr)], %g1 651 tst %g1 652 bnz,pn %icc, 1f 653 nop 654 655 sethi %hi(sfmmu_panic8), %o0 656 call panic 657 or %o0, %lo(sfmmu_panic8), %o0 6581: 659 retl 660 mov %g0, %o0 ! %o0 = ret = 0 6613: 662#endif 663 664 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 665 666 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 667 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 668 669 /* 670 * %g5 = sfmmu gnum returned 671 * %g6 = sfmmu cnum returned 672 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 673 * %g4 = scratch 674 * 675 * Fast path code, do a quick check. 676 */ 677 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 678 679 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 680 bne,pt %icc, 1f ! valid hat cnum, check gnum 681 nop 682 683 ! cnum == INVALID, check allocflag 684 mov %g0, %g4 ! %g4 = ret = 0 685 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 686 mov %g6, %o1 687 688 ! (invalid HAT cnum) && (allocflag == 1) 689 ba,pt %icc, 2f 690 nop 6911: 692 ! valid HAT cnum, check gnum 693 cmp %g5, %o4 694 mov 1, %g4 !%g4 = ret = 1 695 be,a,pt %icc, 8f ! gnum unchanged, go to done 696 mov %g6, %o1 697 6982: 699 /* 700 * Grab per process (PP) sfmmu_ctx_lock spinlock, 701 * followed by the 'slow path' code. 702 */ 703 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 7043: 705 brz %g3, 5f 706 nop 7074: 708 brnz,a,pt %g3, 4b ! spin if lock is 1 709 ldub [%o0 + SFMMU_CTX_LOCK], %g3 710 ba %xcc, 3b ! retry the lock 711 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 712 7135: 714 membar #LoadLoad 715 /* 716 * %g5 = sfmmu gnum returned 717 * %g6 = sfmmu cnum returned 718 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 719 * %g4 = scratch 720 */ 721 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 722 723 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 724 bne,pt %icc, 1f ! valid hat cnum, check gnum 725 nop 726 727 ! cnum == INVALID, check allocflag 728 mov %g0, %g4 ! %g4 = ret = 0 729 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 730 mov %g6, %o1 731 732 ! (invalid HAT cnum) && (allocflag == 1) 733 ba,pt %icc, 6f 734 nop 7351: 736 ! valid HAT cnum, check gnum 737 cmp %g5, %o4 738 mov 1, %g4 ! %g4 = ret = 1 739 be,a,pt %icc, 2f ! gnum unchanged, go to done 740 mov %g6, %o1 741 742 ba,pt %icc, 6f 743 nop 7442: 745 membar #LoadStore|#StoreStore 746 ba,pt %icc, 8f 747 clrb [%o0 + SFMMU_CTX_LOCK] 7486: 749 /* 750 * We get here if we do not have a valid context, or 751 * the HAT gnum does not match global gnum. We hold 752 * sfmmu_ctx_lock spinlock. Allocate that context. 753 * 754 * %o3 = mmu_ctxp 755 */ 756 add %o3, MMU_CTX_CNUM, %g3 757 ld [%o3 + MMU_CTX_NCTXS], %g4 758 759 /* 760 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 761 * %g3 = mmu cnum address 762 * %g4 = mmu nctxs 763 * 764 * %o0 = sfmmup 765 * %o1 = mmu current cnum value (used as new cnum) 766 * %o4 = mmu gnum 767 * 768 * %o5 = scratch 769 */ 770 ld [%g3], %o1 7710: 772 cmp %o1, %g4 773 bl,a,pt %icc, 1f 774 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 775 776 /* 777 * cnum reachs max, bail, so wrap around can be performed later. 778 */ 779 set INVALID_CONTEXT, %o1 780 /* 781 * When the routine is called by shared ctx, we want to set 782 * both private and shared ctx regs to INVALID. In order to 783 * do so, we set the sfmmu priv/shared flag to 'private' regardless. 784 * so that private ctx reg will be set to invalid. 785 * Note that values written to private context register are 786 * automatically written to shared context register as well. 787 */ 788 mov %g0, %g1 ! %g1 = sfmmu private/shared flag 789 mov %g0, %g4 ! %g4 = ret = 0 790 791 membar #LoadStore|#StoreStore 792 ba,pt %icc, 8f 793 clrb [%o0 + SFMMU_CTX_LOCK] 7941: 795 ! %g3 = addr of mmu_ctxp->cnum 796 ! %o5 = mmu_ctxp->cnum + 1 797 cas [%g3], %o1, %o5 798 cmp %o1, %o5 799 bne,a,pn %xcc, 0b ! cas failed 800 ld [%g3], %o1 801 802#ifdef DEBUG 803 set MAX_SFMMU_CTX_VAL, %o5 804 cmp %o1, %o5 805 ble,pt %icc, 2f 806 nop 807 808 sethi %hi(sfmmu_panic9), %o0 809 call panic 810 or %o0, %lo(sfmmu_panic9), %o0 8112: 812#endif 813 ! update hat gnum and cnum 814 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 815 or %o4, %o1, %o4 816 stx %o4, [%g2 + SFMMU_CTXS] 817 818 membar #LoadStore|#StoreStore 819 clrb [%o0 + SFMMU_CTX_LOCK] 820 821 mov 1, %g4 ! %g4 = ret = 1 8228: 823 /* 824 * program the secondary context register 825 * 826 * %o1 = cnum 827 * %g1 = sfmmu private/shared flag (0:private, 1:shared) 828 */ 829 830#ifdef sun4u 831 ldub [%o0 + SFMMU_CEXT], %o2 832 sll %o2, CTXREG_EXT_SHIFT, %o2 833 or %o1, %o2, %o1 834#endif /* sun4u */ 835 836 SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1) 837 838 retl 839 mov %g4, %o0 ! %o0 = ret 840 841 SET_SIZE(sfmmu_alloc_ctx) 842 843 ENTRY_NP(sfmmu_modifytte) 844 ldx [%o2], %g3 /* current */ 845 ldx [%o0], %g1 /* original */ 8462: 847 ldx [%o1], %g2 /* modified */ 848 cmp %g2, %g3 /* is modified = current? */ 849 be,a,pt %xcc,1f /* yes, don't write */ 850 stx %g3, [%o0] /* update new original */ 851 casx [%o2], %g1, %g2 852 cmp %g1, %g2 853 be,pt %xcc, 1f /* cas succeeded - return */ 854 nop 855 ldx [%o2], %g3 /* new current */ 856 stx %g3, [%o0] /* save as new original */ 857 ba,pt %xcc, 2b 858 mov %g3, %g1 8591: retl 860 membar #StoreLoad 861 SET_SIZE(sfmmu_modifytte) 862 863 ENTRY_NP(sfmmu_modifytte_try) 864 ldx [%o1], %g2 /* modified */ 865 ldx [%o2], %g3 /* current */ 866 ldx [%o0], %g1 /* original */ 867 cmp %g3, %g2 /* is modified = current? */ 868 be,a,pn %xcc,1f /* yes, don't write */ 869 mov 0, %o1 /* as if cas failed. */ 870 871 casx [%o2], %g1, %g2 872 membar #StoreLoad 873 cmp %g1, %g2 874 movne %xcc, -1, %o1 /* cas failed. */ 875 move %xcc, 1, %o1 /* cas succeeded. */ 8761: 877 stx %g2, [%o0] /* report "current" value */ 878 retl 879 mov %o1, %o0 880 SET_SIZE(sfmmu_modifytte_try) 881 882 ENTRY_NP(sfmmu_copytte) 883 ldx [%o0], %g1 884 retl 885 stx %g1, [%o1] 886 SET_SIZE(sfmmu_copytte) 887 888 889 /* 890 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 891 * %o0 = TSB base address (in), pointer to TSB entry (out) 892 * %o1 = vaddr (in) 893 * %o2 = vpshift (in) 894 * %o3 = tsb size code (in) 895 * %o4 = scratch register 896 */ 897 ENTRY_NP(sfmmu_get_tsbe) 898 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 899 retl 900 nop 901 SET_SIZE(sfmmu_get_tsbe) 902 903 /* 904 * Return a TSB tag for the given va. 905 * %o0 = va (in/clobbered) 906 * %o0 = va shifted to be in tsb tag format (with no context) (out) 907 */ 908 ENTRY_NP(sfmmu_make_tsbtag) 909 retl 910 srln %o0, TTARGET_VA_SHIFT, %o0 911 SET_SIZE(sfmmu_make_tsbtag) 912 913#endif /* lint */ 914 915/* 916 * Other sfmmu primitives 917 */ 918 919 920#if defined (lint) 921void 922sfmmu_patch_ktsb(void) 923{ 924} 925 926void 927sfmmu_kpm_patch_tlbm(void) 928{ 929} 930 931void 932sfmmu_kpm_patch_tsbm(void) 933{ 934} 935 936void 937sfmmu_patch_shctx(void) 938{ 939} 940 941/* ARGSUSED */ 942void 943sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 944{ 945} 946 947/* ARGSUSED */ 948void 949sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 950{ 951} 952 953/* ARGSUSED */ 954void 955sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 956{ 957} 958 959/* ARGSUSED */ 960void 961sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 962{ 963} 964 965#else /* lint */ 966 967#define I_SIZE 4 968 969 ENTRY_NP(sfmmu_fix_ktlb_traptable) 970 /* 971 * %o0 = start of patch area 972 * %o1 = size code of TSB to patch 973 * %o3 = scratch 974 */ 975 /* fix sll */ 976 ld [%o0], %o3 /* get sll */ 977 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 978 st %o3, [%o0] /* write sll */ 979 flush %o0 980 /* fix srl */ 981 add %o0, I_SIZE, %o0 /* goto next instr. */ 982 ld [%o0], %o3 /* get srl */ 983 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 984 st %o3, [%o0] /* write srl */ 985 retl 986 flush %o0 987 SET_SIZE(sfmmu_fix_ktlb_traptable) 988 989 ENTRY_NP(sfmmu_fixup_ktsbbase) 990 /* 991 * %o0 = start of patch area 992 * %o5 = kernel virtual or physical tsb base address 993 * %o2, %o3 are used as scratch registers. 994 */ 995 /* fixup sethi instruction */ 996 ld [%o0], %o3 997 srl %o5, 10, %o2 ! offset is bits 32:10 998 or %o3, %o2, %o3 ! set imm22 999 st %o3, [%o0] 1000 /* fixup offset of lduw/ldx */ 1001 add %o0, I_SIZE, %o0 ! next instr 1002 ld [%o0], %o3 1003 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 1004 or %o3, %o2, %o3 1005 st %o3, [%o0] 1006 retl 1007 flush %o0 1008 SET_SIZE(sfmmu_fixup_ktsbbase) 1009 1010 ENTRY_NP(sfmmu_fixup_setx) 1011 /* 1012 * %o0 = start of patch area 1013 * %o4 = 64 bit value to patch 1014 * %o2, %o3 are used as scratch registers. 1015 * 1016 * Note: Assuming that all parts of the instructions which need to be 1017 * patched correspond to RUNTIME_PATCH (aka 0) 1018 * 1019 * Note the implementation of setx which is being patched is as follows: 1020 * 1021 * sethi %hh(RUNTIME_PATCH), tmp 1022 * sethi %lm(RUNTIME_PATCH), dest 1023 * or tmp, %hm(RUNTIME_PATCH), tmp 1024 * or dest, %lo(RUNTIME_PATCH), dest 1025 * sllx tmp, 32, tmp 1026 * nop 1027 * or tmp, dest, dest 1028 * 1029 * which differs from the implementation in the 1030 * "SPARC Architecture Manual" 1031 */ 1032 /* fixup sethi instruction */ 1033 ld [%o0], %o3 1034 srlx %o4, 42, %o2 ! bits [63:42] 1035 or %o3, %o2, %o3 ! set imm22 1036 st %o3, [%o0] 1037 /* fixup sethi instruction */ 1038 add %o0, I_SIZE, %o0 ! next instr 1039 ld [%o0], %o3 1040 sllx %o4, 32, %o2 ! clear upper bits 1041 srlx %o2, 42, %o2 ! bits [31:10] 1042 or %o3, %o2, %o3 ! set imm22 1043 st %o3, [%o0] 1044 /* fixup or instruction */ 1045 add %o0, I_SIZE, %o0 ! next instr 1046 ld [%o0], %o3 1047 srlx %o4, 32, %o2 ! bits [63:32] 1048 and %o2, 0x3ff, %o2 ! bits [41:32] 1049 or %o3, %o2, %o3 ! set imm 1050 st %o3, [%o0] 1051 /* fixup or instruction */ 1052 add %o0, I_SIZE, %o0 ! next instr 1053 ld [%o0], %o3 1054 and %o4, 0x3ff, %o2 ! bits [9:0] 1055 or %o3, %o2, %o3 ! set imm 1056 st %o3, [%o0] 1057 retl 1058 flush %o0 1059 SET_SIZE(sfmmu_fixup_setx) 1060 1061 ENTRY_NP(sfmmu_fixup_or) 1062 /* 1063 * %o0 = start of patch area 1064 * %o4 = 32 bit value to patch 1065 * %o2, %o3 are used as scratch registers. 1066 * Note: Assuming that all parts of the instructions which need to be 1067 * patched correspond to RUNTIME_PATCH (aka 0) 1068 */ 1069 ld [%o0], %o3 1070 and %o4, 0x3ff, %o2 ! bits [9:0] 1071 or %o3, %o2, %o3 ! set imm 1072 st %o3, [%o0] 1073 retl 1074 flush %o0 1075 SET_SIZE(sfmmu_fixup_or) 1076 1077 ENTRY_NP(sfmmu_fixup_shiftx) 1078 /* 1079 * %o0 = start of patch area 1080 * %o4 = signed int immediate value to add to sllx/srlx imm field 1081 * %o2, %o3 are used as scratch registers. 1082 * 1083 * sllx/srlx store the 6 bit immediate value in the lowest order bits 1084 * so we do a simple add. The caller must be careful to prevent 1085 * overflow, which could easily occur if the initial value is nonzero! 1086 */ 1087 ld [%o0], %o3 ! %o3 = instruction to patch 1088 and %o3, 0x3f, %o2 ! %o2 = existing imm value 1089 add %o2, %o4, %o2 ! %o2 = new imm value 1090 andn %o3, 0x3f, %o3 ! clear old imm value 1091 and %o2, 0x3f, %o2 ! truncate new imm value 1092 or %o3, %o2, %o3 ! set new imm value 1093 st %o3, [%o0] ! store updated instruction 1094 retl 1095 flush %o0 1096 SET_SIZE(sfmmu_fixup_shiftx) 1097 1098 ENTRY_NP(sfmmu_fixup_mmu_asi) 1099 /* 1100 * Patch imm_asi of all ldda instructions in the MMU 1101 * trap handlers. We search MMU_PATCH_INSTR instructions 1102 * starting from the itlb miss handler (trap 0x64). 1103 * %o0 = address of tt[0,1]_itlbmiss 1104 * %o1 = imm_asi to setup, shifted by appropriate offset. 1105 * %o3 = number of instructions to search 1106 * %o4 = reserved by caller: called from leaf routine 1107 */ 11081: ldsw [%o0], %o2 ! load instruction to %o2 1109 brgez,pt %o2, 2f 1110 srl %o2, 30, %o5 1111 btst 1, %o5 ! test bit 30; skip if not set 1112 bz,pt %icc, 2f 1113 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1114 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1115 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1116 brnz,pt %o5, 2f ! skip if not a match 1117 or %o2, %o1, %o2 ! or in imm_asi 1118 st %o2, [%o0] ! write patched instruction 11192: dec %o3 1120 brnz,a,pt %o3, 1b ! loop until we're done 1121 add %o0, I_SIZE, %o0 1122 retl 1123 flush %o0 1124 SET_SIZE(sfmmu_fixup_mmu_asi) 1125 1126 /* 1127 * Patch immediate ASI used to access the TSB in the 1128 * trap table. 1129 * inputs: %o0 = value of ktsb_phys 1130 */ 1131 ENTRY_NP(sfmmu_patch_mmu_asi) 1132 mov %o7, %o4 ! save return pc in %o4 1133 movrnz %o0, ASI_QUAD_LDD_PHYS, %o3 1134 movrz %o0, ASI_NQUAD_LD, %o3 1135 sll %o3, 5, %o1 ! imm_asi offset 1136 mov 6, %o3 ! number of instructions 1137 sethi %hi(dktsb), %o0 ! to search 1138 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1139 or %o0, %lo(dktsb), %o0 1140 mov 6, %o3 ! number of instructions 1141 sethi %hi(dktsb4m), %o0 ! to search 1142 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1143 or %o0, %lo(dktsb4m), %o0 1144 mov 6, %o3 ! number of instructions 1145 sethi %hi(iktsb), %o0 ! to search 1146 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1147 or %o0, %lo(iktsb), %o0 1148 mov 6, %o3 ! number of instructions 1149 sethi %hi(iktsb4m), %o0 ! to search 1150 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1151 or %o0, %lo(iktsb4m), %o0 1152 mov %o4, %o7 ! retore return pc -- leaf 1153 retl 1154 nop 1155 SET_SIZE(sfmmu_patch_mmu_asi) 1156 1157 ENTRY_NP(sfmmu_patch_ktsb) 1158 /* 1159 * We need to fix iktsb, dktsb, et. al. 1160 */ 1161 save %sp, -SA(MINFRAME), %sp 1162 set ktsb_phys, %o1 1163 ld [%o1], %o4 1164 set ktsb_base, %o5 1165 set ktsb4m_base, %l1 1166 brz,pt %o4, 1f 1167 nop 1168 set ktsb_pbase, %o5 1169 set ktsb4m_pbase, %l1 11701: 1171 sethi %hi(ktsb_szcode), %o1 1172 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1173 1174 sethi %hi(iktsb), %o0 1175 call sfmmu_fix_ktlb_traptable 1176 or %o0, %lo(iktsb), %o0 1177 1178 sethi %hi(dktsb), %o0 1179 call sfmmu_fix_ktlb_traptable 1180 or %o0, %lo(dktsb), %o0 1181 1182 sethi %hi(ktsb4m_szcode), %o1 1183 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1184 1185 sethi %hi(iktsb4m), %o0 1186 call sfmmu_fix_ktlb_traptable 1187 or %o0, %lo(iktsb4m), %o0 1188 1189 sethi %hi(dktsb4m), %o0 1190 call sfmmu_fix_ktlb_traptable 1191 or %o0, %lo(dktsb4m), %o0 1192 1193#ifndef sun4v 1194 mov ASI_N, %o2 1195 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1196 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1197 sethi %hi(tsb_kernel_patch_asi), %o0 1198 call sfmmu_fixup_or 1199 or %o0, %lo(tsb_kernel_patch_asi), %o0 1200#endif /* !sun4v */ 1201 1202 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1203 1204 sethi %hi(dktsbbase), %o0 1205 call sfmmu_fixup_setx ! patch value of ktsb base addr 1206 or %o0, %lo(dktsbbase), %o0 1207 1208 sethi %hi(iktsbbase), %o0 1209 call sfmmu_fixup_setx ! patch value of ktsb base addr 1210 or %o0, %lo(iktsbbase), %o0 1211 1212 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1213 call sfmmu_fixup_setx ! patch value of ktsb base addr 1214 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1215 1216#ifdef sun4v 1217 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1218 call sfmmu_fixup_setx ! patch value of ktsb base addr 1219 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1220#endif /* sun4v */ 1221 1222 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1223 1224 sethi %hi(dktsb4mbase), %o0 1225 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1226 or %o0, %lo(dktsb4mbase), %o0 1227 1228 sethi %hi(iktsb4mbase), %o0 1229 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1230 or %o0, %lo(iktsb4mbase), %o0 1231 1232 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1233 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1234 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1235 1236#ifdef sun4v 1237 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1238 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1239 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1240#endif /* sun4v */ 1241 1242 set ktsb_szcode, %o4 1243 ld [%o4], %o4 1244 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1245 call sfmmu_fixup_or ! patch value of ktsb_szcode 1246 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1247 1248#ifdef sun4v 1249 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1250 call sfmmu_fixup_or ! patch value of ktsb_szcode 1251 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1252#endif /* sun4v */ 1253 1254 set ktsb4m_szcode, %o4 1255 ld [%o4], %o4 1256 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1257 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1258 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1259 1260#ifdef sun4v 1261 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1262 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1263 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1264#endif /* sun4v */ 1265 1266 ret 1267 restore 1268 SET_SIZE(sfmmu_patch_ktsb) 1269 1270 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1271 /* 1272 * Fixup trap handlers in common segkpm case. This is reserved 1273 * for future use should kpm TSB be changed to be other than the 1274 * kernel TSB. 1275 */ 1276 retl 1277 nop 1278 SET_SIZE(sfmmu_kpm_patch_tlbm) 1279 1280 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1281 /* 1282 * nop the branch to sfmmu_kpm_dtsb_miss_small 1283 * in the case where we are using large pages for 1284 * seg_kpm (and hence must probe the second TSB for 1285 * seg_kpm VAs) 1286 */ 1287 set dktsb4m_kpmcheck_small, %o0 1288 MAKE_NOP_INSTR(%o1) 1289 st %o1, [%o0] 1290 flush %o0 1291 retl 1292 nop 1293 SET_SIZE(sfmmu_kpm_patch_tsbm) 1294 1295 ENTRY_NP(sfmmu_patch_utsb) 1296#ifdef UTSB_PHYS 1297 retl 1298 nop 1299#else /* UTSB_PHYS */ 1300 /* 1301 * We need to hot patch utsb_vabase and utsb4m_vabase 1302 */ 1303 save %sp, -SA(MINFRAME), %sp 1304 1305 /* patch value of utsb_vabase */ 1306 set utsb_vabase, %o1 1307 ldx [%o1], %o4 1308 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1309 call sfmmu_fixup_setx 1310 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1311 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1312 call sfmmu_fixup_setx 1313 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1314 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1315 call sfmmu_fixup_setx 1316 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1317 1318 /* patch value of utsb4m_vabase */ 1319 set utsb4m_vabase, %o1 1320 ldx [%o1], %o4 1321 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1322 call sfmmu_fixup_setx 1323 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1324 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1325 call sfmmu_fixup_setx 1326 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1327 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1328 call sfmmu_fixup_setx 1329 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1330 1331 /* 1332 * Patch TSB base register masks and shifts if needed. 1333 * By default the TSB base register contents are set up for 4M slab. 1334 * If we're using a smaller slab size and reserved VA range we need 1335 * to patch up those values here. 1336 */ 1337 set tsb_slab_shift, %o1 1338 set MMU_PAGESHIFT4M, %o4 1339 lduw [%o1], %o3 1340 subcc %o4, %o3, %o4 1341 bz,pt %icc, 1f 1342 /* delay slot safe */ 1343 1344 /* patch reserved VA range size if needed. */ 1345 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1346 call sfmmu_fixup_shiftx 1347 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1348 call sfmmu_fixup_shiftx 1349 add %o0, I_SIZE, %o0 1350 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1351 call sfmmu_fixup_shiftx 1352 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1353 call sfmmu_fixup_shiftx 1354 add %o0, I_SIZE, %o0 13551: 1356 /* patch TSBREG_VAMASK used to set up TSB base register */ 1357 set tsb_slab_mask, %o1 1358 ldx [%o1], %o4 1359 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1360 call sfmmu_fixup_or 1361 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1362 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1363 call sfmmu_fixup_or 1364 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1365 1366 ret 1367 restore 1368#endif /* UTSB_PHYS */ 1369 SET_SIZE(sfmmu_patch_utsb) 1370 1371 ENTRY_NP(sfmmu_patch_shctx) 1372#ifdef sun4u 1373 retl 1374 nop 1375#else /* sun4u */ 1376 set sfmmu_shctx_cpu_mondo_patch, %o0 1377 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1378 st %o1, [%o0] 1379 flush %o0 1380 MAKE_NOP_INSTR(%o1) 1381 add %o0, I_SIZE, %o0 ! next instr 1382 st %o1, [%o0] 1383 flush %o0 1384 1385 set sfmmu_shctx_user_rtt_patch, %o0 1386 st %o1, [%o0] ! nop 1st instruction 1387 flush %o0 1388 add %o0, I_SIZE, %o0 1389 st %o1, [%o0] ! nop 2nd instruction 1390 flush %o0 1391 add %o0, I_SIZE, %o0 1392 st %o1, [%o0] ! nop 3rd instruction 1393 flush %o0 1394 add %o0, I_SIZE, %o0 1395 st %o1, [%o0] ! nop 4th instruction 1396 flush %o0 1397 add %o0, I_SIZE, %o0 1398 st %o1, [%o0] ! nop 5th instruction 1399 retl 1400 flush %o0 1401#endif /* sun4u */ 1402 SET_SIZE(sfmmu_patch_shctx) 1403 1404 /* 1405 * Routine that loads an entry into a tsb using virtual addresses. 1406 * Locking is required since all cpus can use the same TSB. 1407 * Note that it is no longer required to have a valid context 1408 * when calling this function. 1409 */ 1410 ENTRY_NP(sfmmu_load_tsbe) 1411 /* 1412 * %o0 = pointer to tsbe to load 1413 * %o1 = tsb tag 1414 * %o2 = virtual pointer to TTE 1415 * %o3 = 1 if physical address in %o0 else 0 1416 */ 1417 rdpr %pstate, %o5 1418#ifdef DEBUG 1419 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1420#endif /* DEBUG */ 1421 1422 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1423 1424 SETUP_TSB_ASI(%o3, %g3) 1425 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1) 1426 1427 wrpr %g0, %o5, %pstate /* enable interrupts */ 1428 1429 retl 1430 membar #StoreStore|#StoreLoad 1431 SET_SIZE(sfmmu_load_tsbe) 1432 1433 /* 1434 * Flush TSB of a given entry if the tag matches. 1435 */ 1436 ENTRY(sfmmu_unload_tsbe) 1437 /* 1438 * %o0 = pointer to tsbe to be flushed 1439 * %o1 = tag to match 1440 * %o2 = 1 if physical address in %o0 else 0 1441 */ 1442 SETUP_TSB_ASI(%o2, %g1) 1443 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1444 retl 1445 membar #StoreStore|#StoreLoad 1446 SET_SIZE(sfmmu_unload_tsbe) 1447 1448 /* 1449 * Routine that loads a TTE into the kpm TSB from C code. 1450 * Locking is required since kpm TSB is shared among all CPUs. 1451 */ 1452 ENTRY_NP(sfmmu_kpm_load_tsb) 1453 /* 1454 * %o0 = vaddr 1455 * %o1 = ttep 1456 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1457 */ 1458 rdpr %pstate, %o5 ! %o5 = saved pstate 1459#ifdef DEBUG 1460 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1461#endif /* DEBUG */ 1462 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1463 1464#ifndef sun4v 1465 sethi %hi(ktsb_phys), %o4 1466 mov ASI_N, %o3 1467 ld [%o4 + %lo(ktsb_phys)], %o4 1468 movrnz %o4, ASI_MEM, %o3 1469 mov %o3, %asi 1470#endif /* !sun4v */ 1471 mov %o0, %g1 ! %g1 = vaddr 1472 1473 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1474 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1475 /* %g2 = tsbep, %g1 clobbered */ 1476 1477 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1478 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1479 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1) 1480 1481 wrpr %g0, %o5, %pstate ! enable interrupts 1482 retl 1483 membar #StoreStore|#StoreLoad 1484 SET_SIZE(sfmmu_kpm_load_tsb) 1485 1486 /* 1487 * Routine that shoots down a TTE in the kpm TSB or in the 1488 * kernel TSB depending on virtpg. Locking is required since 1489 * kpm/kernel TSB is shared among all CPUs. 1490 */ 1491 ENTRY_NP(sfmmu_kpm_unload_tsb) 1492 /* 1493 * %o0 = vaddr 1494 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1495 */ 1496#ifndef sun4v 1497 sethi %hi(ktsb_phys), %o4 1498 mov ASI_N, %o3 1499 ld [%o4 + %lo(ktsb_phys)], %o4 1500 movrnz %o4, ASI_MEM, %o3 1501 mov %o3, %asi 1502#endif /* !sun4v */ 1503 mov %o0, %g1 ! %g1 = vaddr 1504 1505 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1506 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1507 /* %g2 = tsbep, %g1 clobbered */ 1508 1509 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1510 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1511 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1512 1513 retl 1514 membar #StoreStore|#StoreLoad 1515 SET_SIZE(sfmmu_kpm_unload_tsb) 1516 1517#endif /* lint */ 1518 1519 1520#if defined (lint) 1521 1522/*ARGSUSED*/ 1523pfn_t 1524sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1525{ return(0); } 1526 1527#else /* lint */ 1528 1529 ENTRY_NP(sfmmu_ttetopfn) 1530 ldx [%o0], %g1 /* read tte */ 1531 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1532 /* 1533 * g1 = pfn 1534 */ 1535 retl 1536 mov %g1, %o0 1537 SET_SIZE(sfmmu_ttetopfn) 1538 1539#endif /* !lint */ 1540 1541 1542#if defined (lint) 1543/* 1544 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the 1545 * the hash list. 1546 */ 1547/* ARGSUSED */ 1548void 1549sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1550 uint64_t hblkpa) 1551{ 1552} 1553 1554/* 1555 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the 1556 * hash list. 1557 */ 1558/* ARGSUSED */ 1559void 1560sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1561 uint64_t hblkpa, struct hme_blk *prev_hblkp) 1562{ 1563} 1564#else /* lint */ 1565 1566/* 1567 * Functions to grab/release hme bucket list lock. I only use a byte 1568 * instead of the whole int because eventually we might want to 1569 * put some counters on the other bytes (of course, these routines would 1570 * have to change). The code that grab this lock should execute 1571 * with interrupts disabled and hold the lock for the least amount of time 1572 * possible. 1573 */ 1574 1575/* 1576 * Even though hmeh_listlock is updated using pa there's no need to flush 1577 * dcache since hmeh_listlock will be restored to the original value (0) 1578 * before interrupts are reenabled. 1579 */ 1580 1581/* 1582 * For sparcv9 hme hash buckets may not be in the nucleus. hme hash update 1583 * routines still use virtual addresses to update the bucket fields. But they 1584 * must not cause a TLB miss after grabbing the low level bucket lock. To 1585 * achieve this we must make sure the bucket structure is completely within an 1586 * 8K page. 1587 */ 1588 1589#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1)) 1590#error - the size of hmehash_bucket structure is not power of 2 1591#endif 1592 1593#ifdef HMELOCK_BACKOFF_ENABLE 1594 1595#define HMELOCK_BACKOFF(reg, val) \ 1596 set val, reg ;\ 1597 brnz reg, . ;\ 1598 dec reg 1599 1600#define CAS_HME(tmp1, tmp2, exitlabel, asi) \ 1601 mov 0xff, tmp2 ;\ 1602 casa [tmp1]asi, %g0, tmp2 ;\ 1603 brz,a,pt tmp2, exitlabel ;\ 1604 membar #LoadLoad 1605 1606#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label, asi) \ 1607 mov 0xff, tmp2 ;\ 1608 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1609 casa [tmp1]asi, %g0, tmp2 ;\ 1610 brz,a,pt tmp2, label/**/2 ;\ 1611 membar #LoadLoad ;\ 1612 HMELOCK_BACKOFF(tmp2,0x80) ;\ 1613 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1614 HMELOCK_BACKOFF(tmp2,0x100) ;\ 1615 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1616 HMELOCK_BACKOFF(tmp2,0x200) ;\ 1617 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1618label/**/1: ;\ 1619 HMELOCK_BACKOFF(tmp2,0x400) ;\ 1620 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1621 HMELOCK_BACKOFF(tmp2,0x800) ;\ 1622 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1623 HMELOCK_BACKOFF(tmp2,0x1000) ;\ 1624 CAS_HME(tmp1, tmp2, label/**/2, asi) ;\ 1625 HMELOCK_BACKOFF(tmp2,0x2000) ;\ 1626 mov 0xff, tmp2 ;\ 1627 casa [tmp1]asi, %g0, tmp2 ;\ 1628 brnz,pn tmp2, label/**/1 /* reset backoff */ ;\ 1629 membar #LoadLoad ;\ 1630label/**/2: 1631 1632#else /* HMELOCK_BACKOFF_ENABLE */ 1633 1634#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi) \ 1635 mov 0xff, tmp2 ;\ 1636 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1637label1: ;\ 1638 casa [tmp1]asi, %g0, tmp2 ;\ 1639 brnz,pn tmp2, label1 ;\ 1640 mov 0xff, tmp2 ;\ 1641 membar #LoadLoad 1642 1643#endif /* HMELOCK_BACKOFF_ENABLE */ 1644 1645#define HMELOCK_EXIT(hmebp, tmp1, asi) \ 1646 membar #LoadStore|#StoreStore ;\ 1647 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1648 sta %g0, [tmp1]asi 1649 1650 .seg ".data" 1651hblk_add_panic1: 1652 .ascii "sfmmu_hblk_hash_add: interrupts disabled" 1653 .byte 0 1654hblk_add_panic2: 1655 .ascii "sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not" 1656 .byte 0 1657 .align 4 1658 .seg ".text" 1659 1660 ENTRY_NP(sfmmu_hblk_hash_add) 1661 /* 1662 * %o0 = hmebp 1663 * %o1 = hmeblkp 1664 * %o2 = hblkpa 1665 */ 1666 rdpr %pstate, %o5 1667#ifdef DEBUG 1668 andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ 1669 bnz,pt %icc, 3f /* disabled, panic */ 1670 nop 1671 save %sp, -SA(MINFRAME), %sp 1672 sethi %hi(hblk_add_panic1), %o0 1673 call panic 1674 or %o0, %lo(hblk_add_panic1), %o0 1675 ret 1676 restore 1677 16783: 1679#endif /* DEBUG */ 1680 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1681 mov %o2, %g1 1682 1683 /* 1684 * g1 = hblkpa 1685 */ 1686 ldn [%o0 + HMEBUCK_HBLK], %o4 /* next hmeblk */ 1687 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = next hblkpa */ 1688#ifdef DEBUG 1689 cmp %o4, %g0 1690 bne,pt %xcc, 1f 1691 nop 1692 brz,pt %g2, 1f 1693 nop 1694 wrpr %g0, %o5, %pstate /* enable interrupts */ 1695 save %sp, -SA(MINFRAME), %sp 1696 sethi %hi(hblk_add_panic2), %o0 1697 call panic 1698 or %o0, %lo(hblk_add_panic2), %o0 1699 ret 1700 restore 17011: 1702#endif /* DEBUG */ 1703 /* 1704 * We update hmeblks entries before grabbing lock because the stores 1705 * could take a tlb miss and require the hash lock. The buckets 1706 * are part of the nucleus so we are cool with those stores. 1707 * 1708 * if buckets are not part of the nucleus our game is to 1709 * not touch any other page via va until we drop the lock. 1710 * This guarantees we won't get a tlb miss before the lock release 1711 * since interrupts are disabled. 1712 */ 1713 stn %o4, [%o1 + HMEBLK_NEXT] /* update hmeblk's next */ 1714 stx %g2, [%o1 + HMEBLK_NEXTPA] /* update hmeblk's next pa */ 1715 HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N) 1716 stn %o1, [%o0 + HMEBUCK_HBLK] /* update bucket hblk next */ 1717 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* add hmeblk to list */ 1718 HMELOCK_EXIT(%o0, %g2, ASI_N) 1719 retl 1720 wrpr %g0, %o5, %pstate /* enable interrupts */ 1721 SET_SIZE(sfmmu_hblk_hash_add) 1722 1723 ENTRY_NP(sfmmu_hblk_hash_rm) 1724 /* 1725 * This function removes an hmeblk from the hash chain. 1726 * It is written to guarantee we don't take a tlb miss 1727 * by using physical addresses to update the list. 1728 * 1729 * %o0 = hmebp 1730 * %o1 = hmeblkp 1731 * %o2 = hmeblkp previous pa 1732 * %o3 = hmeblkp previous 1733 */ 1734 1735 mov %o3, %o4 /* o4 = hmeblkp previous */ 1736 1737 rdpr %pstate, %o5 1738#ifdef DEBUG 1739 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1) 1740#endif /* DEBUG */ 1741 /* 1742 * disable interrupts, clear Address Mask to access 64 bit physaddr 1743 */ 1744 andn %o5, PSTATE_IE, %g1 1745 wrpr %g1, 0, %pstate 1746 1747#ifndef sun4v 1748 sethi %hi(dcache_line_mask), %g4 1749 ld [%g4 + %lo(dcache_line_mask)], %g4 1750#endif /* sun4v */ 1751 1752 /* 1753 * if buckets are not part of the nucleus our game is to 1754 * not touch any other page via va until we drop the lock. 1755 * This guarantees we won't get a tlb miss before the lock release 1756 * since interrupts are disabled. 1757 */ 1758 HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N) 1759 ldn [%o0 + HMEBUCK_HBLK], %g2 /* first hmeblk in list */ 1760 cmp %g2, %o1 1761 bne,pt %ncc,1f 1762 mov ASI_MEM, %asi 1763 /* 1764 * hmeblk is first on list 1765 */ 1766 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = hmeblk pa */ 1767 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1768 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1769 stn %o3, [%o0 + HMEBUCK_HBLK] /* write va */ 1770 ba,pt %xcc, 2f 1771 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* write pa */ 17721: 1773 /* hmeblk is not first on list */ 1774 1775 mov %o2, %g3 1776#ifndef sun4v 1777 GET_CPU_IMPL(%g2) 1778 cmp %g2, CHEETAH_IMPL 1779 bge,a,pt %icc, hblk_hash_rm_1 1780 and %o4, %g4, %g2 1781 cmp %g2, SPITFIRE_IMPL 1782 blt %icc, hblk_hash_rm_2 /* no flushing needed for OPL */ 1783 and %o4, %g4, %g2 1784 stxa %g0, [%g2]ASI_DC_TAG /* flush prev pa from dcache */ 1785 add %o4, HMEBLK_NEXT, %o4 1786 and %o4, %g4, %g2 1787 ba hblk_hash_rm_2 1788 stxa %g0, [%g2]ASI_DC_TAG /* flush prev va from dcache */ 1789hblk_hash_rm_1: 1790 1791 stxa %g0, [%g3]ASI_DC_INVAL /* flush prev pa from dcache */ 1792 membar #Sync 1793 add %g3, HMEBLK_NEXT, %g2 1794 stxa %g0, [%g2]ASI_DC_INVAL /* flush prev va from dcache */ 1795hblk_hash_rm_2: 1796 membar #Sync 1797#endif /* sun4v */ 1798 ldxa [%g3 + HMEBLK_NEXTPA] %asi, %g2 /* g2 = hmeblk pa */ 1799 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1800 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1801 stna %o3, [%g3 + HMEBLK_NEXT] %asi /* write va */ 1802 stxa %g1, [%g3 + HMEBLK_NEXTPA] %asi /* write pa */ 18032: 1804 HMELOCK_EXIT(%o0, %g2, ASI_N) 1805 retl 1806 wrpr %g0, %o5, %pstate /* enable interrupts */ 1807 SET_SIZE(sfmmu_hblk_hash_rm) 1808 1809#endif /* lint */ 1810 1811/* 1812 * These macros are used to update global sfmmu hme hash statistics 1813 * in perf critical paths. It is only enabled in debug kernels or 1814 * if SFMMU_STAT_GATHER is defined 1815 */ 1816#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1817#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1818 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1819 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1820 cmp tmp1, hatid ;\ 1821 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1822 set sfmmu_global_stat, tmp1 ;\ 1823 add tmp1, tmp2, tmp1 ;\ 1824 ld [tmp1], tmp2 ;\ 1825 inc tmp2 ;\ 1826 st tmp2, [tmp1] 1827 1828#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1829 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1830 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1831 cmp tmp1, hatid ;\ 1832 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1833 set sfmmu_global_stat, tmp1 ;\ 1834 add tmp1, tmp2, tmp1 ;\ 1835 ld [tmp1], tmp2 ;\ 1836 inc tmp2 ;\ 1837 st tmp2, [tmp1] 1838 1839 1840#else /* DEBUG || SFMMU_STAT_GATHER */ 1841 1842#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1843 1844#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1845 1846#endif /* DEBUG || SFMMU_STAT_GATHER */ 1847 1848/* 1849 * This macro is used to update global sfmmu kstas in non 1850 * perf critical areas so they are enabled all the time 1851 */ 1852#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1853 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1854 add tmp1, statname, tmp1 ;\ 1855 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1856 inc tmp2 ;\ 1857 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1858 1859/* 1860 * These macros are used to update per cpu stats in non perf 1861 * critical areas so they are enabled all the time 1862 */ 1863#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1864 ld [tsbarea + stat], tmp1 ;\ 1865 inc tmp1 ;\ 1866 st tmp1, [tsbarea + stat] 1867 1868/* 1869 * These macros are used to update per cpu stats in non perf 1870 * critical areas so they are enabled all the time 1871 */ 1872#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1873 lduh [tsbarea + stat], tmp1 ;\ 1874 inc tmp1 ;\ 1875 stuh tmp1, [tsbarea + stat] 1876 1877#if defined(KPM_TLBMISS_STATS_GATHER) 1878 /* 1879 * Count kpm dtlb misses separately to allow a different 1880 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1881 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1882 */ 1883#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1884 brgez tagacc, label /* KPM VA? */ ;\ 1885 nop ;\ 1886 CPU_INDEX(tmp1, tsbma) ;\ 1887 sethi %hi(kpmtsbm_area), tsbma ;\ 1888 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1889 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1890 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1891 /* VA range check */ ;\ 1892 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1893 cmp tagacc, val ;\ 1894 blu,pn %xcc, label ;\ 1895 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1896 cmp tagacc, tmp1 ;\ 1897 bgeu,pn %xcc, label ;\ 1898 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1899 inc val ;\ 1900 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1901label: 1902#else 1903#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1904#endif /* KPM_TLBMISS_STATS_GATHER */ 1905 1906#if defined (lint) 1907/* 1908 * The following routines are jumped to from the mmu trap handlers to do 1909 * the setting up to call systrap. They are separate routines instead of 1910 * being part of the handlers because the handlers would exceed 32 1911 * instructions and since this is part of the slow path the jump 1912 * cost is irrelevant. 1913 */ 1914void 1915sfmmu_pagefault(void) 1916{ 1917} 1918 1919void 1920sfmmu_mmu_trap(void) 1921{ 1922} 1923 1924void 1925sfmmu_window_trap(void) 1926{ 1927} 1928 1929void 1930sfmmu_kpm_exception(void) 1931{ 1932} 1933 1934#else /* lint */ 1935 1936#ifdef PTL1_PANIC_DEBUG 1937 .seg ".data" 1938 .global test_ptl1_panic 1939test_ptl1_panic: 1940 .word 0 1941 .align 8 1942 1943 .seg ".text" 1944 .align 4 1945#endif /* PTL1_PANIC_DEBUG */ 1946 1947 1948 ENTRY_NP(sfmmu_pagefault) 1949 SET_GL_REG(1) 1950 USE_ALTERNATE_GLOBALS(%g5) 1951 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1952 rdpr %tt, %g6 1953 cmp %g6, FAST_IMMU_MISS_TT 1954 be,a,pn %icc, 1f 1955 mov T_INSTR_MMU_MISS, %g3 1956 cmp %g6, T_INSTR_MMU_MISS 1957 be,a,pn %icc, 1f 1958 mov T_INSTR_MMU_MISS, %g3 1959 mov %g5, %g2 1960 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1961 cmp %g6, FAST_DMMU_MISS_TT 1962 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1963 cmp %g6, T_DATA_MMU_MISS 1964 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1965 1966#ifdef PTL1_PANIC_DEBUG 1967 /* check if we want to test the tl1 panic */ 1968 sethi %hi(test_ptl1_panic), %g4 1969 ld [%g4 + %lo(test_ptl1_panic)], %g1 1970 st %g0, [%g4 + %lo(test_ptl1_panic)] 1971 cmp %g1, %g0 1972 bne,a,pn %icc, ptl1_panic 1973 or %g0, PTL1_BAD_DEBUG, %g1 1974#endif /* PTL1_PANIC_DEBUG */ 19751: 1976 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 1977 /* 1978 * g2 = tag access reg 1979 * g3.l = type 1980 * g3.h = 0 1981 */ 1982 sethi %hi(trap), %g1 1983 or %g1, %lo(trap), %g1 19842: 1985 ba,pt %xcc, sys_trap 1986 mov -1, %g4 1987 SET_SIZE(sfmmu_pagefault) 1988 1989 ENTRY_NP(sfmmu_mmu_trap) 1990 SET_GL_REG(1) 1991 USE_ALTERNATE_GLOBALS(%g5) 1992 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 1993 rdpr %tt, %g6 1994 cmp %g6, FAST_IMMU_MISS_TT 1995 be,a,pn %icc, 1f 1996 mov T_INSTR_MMU_MISS, %g3 1997 cmp %g6, T_INSTR_MMU_MISS 1998 be,a,pn %icc, 1f 1999 mov T_INSTR_MMU_MISS, %g3 2000 mov %g5, %g2 2001 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 2002 cmp %g6, FAST_DMMU_MISS_TT 2003 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 2004 cmp %g6, T_DATA_MMU_MISS 2005 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 20061: 2007 /* 2008 * g2 = tag access reg 2009 * g3 = type 2010 */ 2011 sethi %hi(sfmmu_tsbmiss_exception), %g1 2012 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 2013 ba,pt %xcc, sys_trap 2014 mov -1, %g4 2015 /*NOTREACHED*/ 2016 SET_SIZE(sfmmu_mmu_trap) 2017 2018 ENTRY_NP(sfmmu_suspend_tl) 2019 SET_GL_REG(1) 2020 USE_ALTERNATE_GLOBALS(%g5) 2021 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 2022 rdpr %tt, %g6 2023 cmp %g6, FAST_IMMU_MISS_TT 2024 be,a,pn %icc, 1f 2025 mov T_INSTR_MMU_MISS, %g3 2026 mov %g5, %g2 2027 cmp %g6, FAST_DMMU_MISS_TT 2028 move %icc, T_DATA_MMU_MISS, %g3 2029 movne %icc, T_DATA_PROT, %g3 20301: 2031 sethi %hi(sfmmu_tsbmiss_suspended), %g1 2032 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 2033 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 2034 ba,pt %xcc, sys_trap 2035 mov PIL_15, %g4 2036 /*NOTREACHED*/ 2037 SET_SIZE(sfmmu_suspend_tl) 2038 2039 /* 2040 * No %g registers in use at this point. 2041 */ 2042 ENTRY_NP(sfmmu_window_trap) 2043 rdpr %tpc, %g1 2044#ifdef sun4v 2045#ifdef DEBUG 2046 /* We assume previous %gl was 1 */ 2047 rdpr %tstate, %g4 2048 srlx %g4, TSTATE_GL_SHIFT, %g4 2049 and %g4, TSTATE_GL_MASK, %g4 2050 cmp %g4, 1 2051 bne,a,pn %icc, ptl1_panic 2052 mov PTL1_BAD_WTRAP, %g1 2053#endif /* DEBUG */ 2054 /* user miss at tl>1. better be the window handler or user_rtt */ 2055 /* in user_rtt? */ 2056 set rtt_fill_start, %g4 2057 cmp %g1, %g4 2058 blu,pn %xcc, 6f 2059 .empty 2060 set rtt_fill_end, %g4 2061 cmp %g1, %g4 2062 bgeu,pn %xcc, 6f 2063 nop 2064 set fault_rtt_fn1, %g1 2065 wrpr %g0, %g1, %tnpc 2066 ba,a 7f 20676: 2068 ! must save this trap level before descending trap stack 2069 ! no need to save %tnpc, either overwritten or discarded 2070 ! already got it: rdpr %tpc, %g1 2071 rdpr %tstate, %g6 2072 rdpr %tt, %g7 2073 ! trap level saved, go get underlying trap type 2074 rdpr %tl, %g5 2075 sub %g5, 1, %g3 2076 wrpr %g3, %tl 2077 rdpr %tt, %g2 2078 wrpr %g5, %tl 2079 ! restore saved trap level 2080 wrpr %g1, %tpc 2081 wrpr %g6, %tstate 2082 wrpr %g7, %tt 2083#else /* sun4v */ 2084 /* user miss at tl>1. better be the window handler */ 2085 rdpr %tl, %g5 2086 sub %g5, 1, %g3 2087 wrpr %g3, %tl 2088 rdpr %tt, %g2 2089 wrpr %g5, %tl 2090#endif /* sun4v */ 2091 and %g2, WTRAP_TTMASK, %g4 2092 cmp %g4, WTRAP_TYPE 2093 bne,pn %xcc, 1f 2094 nop 2095 /* tpc should be in the trap table */ 2096 set trap_table, %g4 2097 cmp %g1, %g4 2098 blt,pn %xcc, 1f 2099 .empty 2100 set etrap_table, %g4 2101 cmp %g1, %g4 2102 bge,pn %xcc, 1f 2103 .empty 2104 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 2105 add %g1, WTRAP_FAULTOFF, %g1 2106 wrpr %g0, %g1, %tnpc 21077: 2108 /* 2109 * some wbuf handlers will call systrap to resolve the fault 2110 * we pass the trap type so they figure out the correct parameters. 2111 * g5 = trap type, g6 = tag access reg 2112 */ 2113 2114 /* 2115 * only use g5, g6, g7 registers after we have switched to alternate 2116 * globals. 2117 */ 2118 SET_GL_REG(1) 2119 USE_ALTERNATE_GLOBALS(%g5) 2120 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 2121 rdpr %tt, %g7 2122 cmp %g7, FAST_IMMU_MISS_TT 2123 be,a,pn %icc, ptl1_panic 2124 mov PTL1_BAD_WTRAP, %g1 2125 cmp %g7, T_INSTR_MMU_MISS 2126 be,a,pn %icc, ptl1_panic 2127 mov PTL1_BAD_WTRAP, %g1 2128 mov T_DATA_PROT, %g5 2129 cmp %g7, FAST_DMMU_MISS_TT 2130 move %icc, T_DATA_MMU_MISS, %g5 2131 cmp %g7, T_DATA_MMU_MISS 2132 move %icc, T_DATA_MMU_MISS, %g5 2133 ! XXXQ AGS re-check out this one 2134 done 21351: 2136 CPU_PADDR(%g1, %g4) 2137 add %g1, CPU_TL1_HDLR, %g1 2138 lda [%g1]ASI_MEM, %g4 2139 brnz,a,pt %g4, sfmmu_mmu_trap 2140 sta %g0, [%g1]ASI_MEM 2141 ba,pt %icc, ptl1_panic 2142 mov PTL1_BAD_TRAP, %g1 2143 SET_SIZE(sfmmu_window_trap) 2144 2145 ENTRY_NP(sfmmu_kpm_exception) 2146 /* 2147 * We have accessed an unmapped segkpm address or a legal segkpm 2148 * address which is involved in a VAC alias conflict prevention. 2149 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 2150 * set. If it is, we will instead note that a fault has occurred 2151 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 2152 * a "retry"). This will step over the faulting instruction. 2153 * Note that this means that a legal segkpm address involved in 2154 * a VAC alias conflict prevention (a rare case to begin with) 2155 * cannot be used in DTrace. 2156 */ 2157 CPU_INDEX(%g1, %g2) 2158 set cpu_core, %g2 2159 sllx %g1, CPU_CORE_SHIFT, %g1 2160 add %g1, %g2, %g1 2161 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 2162 andcc %g2, CPU_DTRACE_NOFAULT, %g0 2163 bz 0f 2164 or %g2, CPU_DTRACE_BADADDR, %g2 2165 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 2166 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 2167 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 2168 done 21690: 2170 TSTAT_CHECK_TL1(1f, %g1, %g2) 21711: 2172 SET_GL_REG(1) 2173 USE_ALTERNATE_GLOBALS(%g5) 2174 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 2175 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 2176 /* 2177 * g2=tagacc g3.l=type g3.h=0 2178 */ 2179 sethi %hi(trap), %g1 2180 or %g1, %lo(trap), %g1 2181 ba,pt %xcc, sys_trap 2182 mov -1, %g4 2183 SET_SIZE(sfmmu_kpm_exception) 2184 2185#endif /* lint */ 2186 2187#if defined (lint) 2188 2189void 2190sfmmu_tsb_miss(void) 2191{ 2192} 2193 2194void 2195sfmmu_kpm_dtsb_miss(void) 2196{ 2197} 2198 2199void 2200sfmmu_kpm_dtsb_miss_small(void) 2201{ 2202} 2203 2204#else /* lint */ 2205 2206#if (IMAP_SEG != 0) 2207#error - ism_map->ism_seg offset is not zero 2208#endif 2209 2210/* 2211 * Copies ism mapping for this ctx in param "ism" if this is a ISM 2212 * tlb miss and branches to label "ismhit". If this is not an ISM 2213 * process or an ISM tlb miss it falls thru. 2214 * 2215 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 2216 * this process. 2217 * If so, it will branch to label "ismhit". If not, it will fall through. 2218 * 2219 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 2220 * so that any other threads of this process will not try and walk the ism 2221 * maps while they are being changed. 2222 * 2223 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 2224 * will make sure of that. This means we can terminate our search on 2225 * the first zero mapping we find. 2226 * 2227 * Parameters: 2228 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 2229 * tsbmiss = address of tsb miss area (in) 2230 * ismseg = contents of ism_seg for this ism map (out) 2231 * ismhat = physical address of imap_ismhat for this ism map (out) 2232 * tmp1 = scratch reg (CLOBBERED) 2233 * tmp2 = scratch reg (CLOBBERED) 2234 * tmp3 = scratch reg (CLOBBERED) 2235 * label: temporary labels 2236 * ismhit: label where to jump to if an ism dtlb miss 2237 * exitlabel:label where to jump if hat is busy due to hat_unshare. 2238 */ 2239#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 2240 label, ismhit) \ 2241 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 2242 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 2243 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 2244label/**/1: ;\ 2245 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 2246 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 2247label/**/2: ;\ 2248 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 2249 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 2250 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 2251 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 2252 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 2253 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 2254 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 2255 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 2256 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 2257 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 2258 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 2259 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 2260 ;\ 2261 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 2262 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 2263 cmp ismhat, tmp1 ;\ 2264 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 2265 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 2266 ;\ 2267 add tmp3, IBLK_NEXTPA, tmp1 ;\ 2268 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 2269 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 2270 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 2271label/**/3: 2272 2273/* 2274 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 2275 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 2276 * Parameters: 2277 * tagacc = reg containing virtual address 2278 * hatid = reg containing sfmmu pointer 2279 * hmeshift = constant/register to shift vaddr to obtain vapg 2280 * hmebp = register where bucket pointer will be stored 2281 * vapg = register where virtual page will be stored 2282 * tmp1, tmp2 = tmp registers 2283 */ 2284 2285 2286#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 2287 vapg, label, tmp1, tmp2) \ 2288 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 2289 brnz,a,pt tmp1, label/**/1 ;\ 2290 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 2291 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 2292 ba,pt %xcc, label/**/2 ;\ 2293 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 2294label/**/1: ;\ 2295 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 2296label/**/2: ;\ 2297 srlx tagacc, hmeshift, vapg ;\ 2298 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 2299 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 2300 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 2301 add hmebp, tmp1, hmebp 2302 2303/* 2304 * hashtag includes bspage + hashno (64 bits). 2305 */ 2306 2307#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 2308 sllx vapg, hmeshift, vapg ;\ 2309 mov hashno, hblktag ;\ 2310 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 2311 or vapg, hblktag, hblktag 2312 2313/* 2314 * Function to traverse hmeblk hash link list and find corresponding match. 2315 * The search is done using physical pointers. It returns the physical address 2316 * and virtual address pointers to the hmeblk that matches with the tag 2317 * provided. 2318 * Parameters: 2319 * hmebp = register that points to hme hash bucket, also used as 2320 * tmp reg (clobbered) 2321 * hmeblktag = register with hmeblk tag match 2322 * hatid = register with hatid 2323 * hmeblkpa = register where physical ptr will be stored 2324 * hmeblkva = register where virtual ptr will be stored 2325 * tmp1 = tmp reg 2326 * label: temporary label 2327 */ 2328 2329#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2330 tsbarea, tmp1, label) \ 2331 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 2332 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2333 add hmebp, HMEBUCK_HBLK, hmeblkva ;\ 2334 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2335 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2336label/**/1: ;\ 2337 brz,pn hmeblkva, label/**/2 ;\ 2338 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2339 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 2340 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2341 add hmebp, CLONGSIZE, hmebp ;\ 2342 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 2343 xor tmp1, hmeblktag, tmp1 ;\ 2344 xor hmebp, hatid, hmebp ;\ 2345 or hmebp, tmp1, hmebp ;\ 2346 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 2347 add hmeblkpa, HMEBLK_NEXT, hmebp ;\ 2348 ldna [hmebp]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2349 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 2350 ba,pt %xcc, label/**/1 ;\ 2351 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2352label/**/2: 2353 2354/* 2355 * Function to traverse hmeblk hash link list and find corresponding match. 2356 * The search is done using physical pointers. It returns the physical address 2357 * and virtual address pointers to the hmeblk that matches with the tag 2358 * provided. 2359 * Parameters: 2360 * hmeblktag = register with hmeblk tag match (rid field is 0) 2361 * hatid = register with hatid (pointer to SRD) 2362 * hmeblkpa = register where physical ptr will be stored 2363 * hmeblkva = register where virtual ptr will be stored 2364 * tmp1 = tmp reg 2365 * tmp2 = tmp reg 2366 * label: temporary label 2367 */ 2368 2369#define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, hmeblkva, \ 2370 tsbarea, tmp1, tmp2, label) \ 2371label/**/1: ;\ 2372 brz,pn hmeblkva, label/**/4 ;\ 2373 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 2374 add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 2375 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2376 add tmp2, CLONGSIZE, tmp2 ;\ 2377 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 2378 xor tmp1, hmeblktag, tmp1 ;\ 2379 xor tmp2, hatid, tmp2 ;\ 2380 brz,pn tmp2, label/**/3 /* branch on hit */ ;\ 2381 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2382label/**/2: ;\ 2383 ldna [tmp2]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 2384 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2385 ba,pt %xcc, label/**/1 ;\ 2386 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2387label/**/3: ;\ 2388 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 2389 bgeu,pt %xcc, label/**/2 ;\ 2390 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2391 and tmp1, BT_ULMASK, tmp2 ;\ 2392 srlx tmp1, BT_ULSHIFT, tmp1 ;\ 2393 sllx tmp1, CLONGSHIFT, tmp1 ;\ 2394 add tsbarea, tmp1, tmp1 ;\ 2395 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 2396 srlx tmp1, tmp2, tmp1 ;\ 2397 btst 0x1, tmp1 ;\ 2398 bz,pn %xcc, label/**/2 ;\ 2399 add hmeblkpa, HMEBLK_NEXT, tmp2 ;\ 2400label/**/4: 2401 2402#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2403#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2404#endif 2405 2406/* 2407 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2408 * he offset for the corresponding hment. 2409 * Parameters: 2410 * In: 2411 * vaddr = register with virtual address 2412 * hmeblkpa = physical pointer to hme_blk 2413 * Out: 2414 * hmentoff = register where hment offset will be stored 2415 * hmemisc = hblk_misc 2416 * Scratch: 2417 * tmp1 2418 */ 2419#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2420 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2421 lda [hmentoff]ASI_MEM, hmemisc ;\ 2422 andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2423 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2424 or %g0, HMEBLK_HME1, hmentoff ;\ 2425 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2426 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2427 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2428 add tmp1, HMEBLK_HME1, hmentoff ;\ 2429label1: 2430 2431/* 2432 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2433 * 2434 * tagacc = (pseudo-)tag access register (in) 2435 * hatid = sfmmu pointer for TSB miss (in) 2436 * tte = tte for TLB miss if found, otherwise clobbered (out) 2437 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2438 * hmeblkva = VA of hment if found, otherwise clobbered (out) 2439 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2440 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2441 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2442 * for this page size. 2443 * hashno = constant/register hash number 2444 * label = temporary label for branching within macro. 2445 * foundlabel = label to jump to when tte is found. 2446 * suspendlabel= label to jump to when tte is suspended. 2447 * exitlabel = label to jump to when tte is not found. 2448 * 2449 */ 2450#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmemisc, \ 2451 hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \ 2452 ;\ 2453 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2454 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2455 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2456 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2457 ;\ 2458 /* ;\ 2459 * tagacc = tagacc ;\ 2460 * hatid = hatid ;\ 2461 * tsbarea = tsbarea ;\ 2462 * tte = hmebp (hme bucket pointer) ;\ 2463 * hmeblkpa = vapg (virtual page) ;\ 2464 * hmemisc, hmeblkva = scratch ;\ 2465 */ ;\ 2466 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2467 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2468 ;\ 2469 /* ;\ 2470 * tagacc = tagacc ;\ 2471 * hatid = hatid ;\ 2472 * tte = hmebp ;\ 2473 * hmeblkpa = CLOBBERED ;\ 2474 * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2475 * hmeblkva = scratch ;\ 2476 */ ;\ 2477 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2478 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2479 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, hmeblkva, \ 2480 tsbarea, tagacc, label/**/1) ;\ 2481 /* ;\ 2482 * tagacc = CLOBBERED ;\ 2483 * tte = CLOBBERED ;\ 2484 * hmeblkpa = hmeblkpa ;\ 2485 * hmeblkva = hmeblkva ;\ 2486 */ ;\ 2487 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2488 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2489 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2490 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2491 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2492 nop ;\ 2493label/**/4: ;\ 2494 /* ;\ 2495 * We have found the hmeblk containing the hment. ;\ 2496 * Now we calculate the corresponding tte. ;\ 2497 * ;\ 2498 * tagacc = tagacc ;\ 2499 * hatid = hatid ;\ 2500 * tte = clobbered ;\ 2501 * hmeblkpa = hmeblkpa ;\ 2502 * hmemisc = hblktag ;\ 2503 * hmeblkva = hmeblkva ;\ 2504 */ ;\ 2505 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2506 label/**/2) ;\ 2507 ;\ 2508 /* ;\ 2509 * tagacc = tagacc ;\ 2510 * hatid = hmentoff ;\ 2511 * tte = clobbered ;\ 2512 * hmeblkpa = hmeblkpa ;\ 2513 * hmemisc = hblk_misc ;\ 2514 * hmeblkva = hmeblkva ;\ 2515 */ ;\ 2516 ;\ 2517 add hatid, SFHME_TTE, hatid ;\ 2518 add hmeblkpa, hatid, hmeblkpa ;\ 2519 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2520 add hmeblkva, hatid, hmeblkva ;\ 2521 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2522 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2523 set TTE_SUSPEND, hatid ;\ 2524 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2525 btst tte, hatid ;\ 2526 bz,pt %xcc, foundlabel ;\ 2527 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2528 ;\ 2529 /* ;\ 2530 * Mapping is suspended, so goto suspend label. ;\ 2531 */ ;\ 2532 ba,pt %xcc, suspendlabel ;\ 2533 nop 2534 2535/* 2536 * GET_SHME_TTE is similar to GET_TTE() except it searches 2537 * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2538 * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2539 * either 0 (not part of scd) or 1 (part of scd). 2540 */ 2541#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, \ 2542 hmemisc, hmeshift, hashno, label, foundlabel, \ 2543 suspendlabel, exitlabel) \ 2544 ;\ 2545 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2546 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2547 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2548 hmeblkpa, label/**/5, hmemisc, hmeblkva) ;\ 2549 ;\ 2550 /* ;\ 2551 * tagacc = tagacc ;\ 2552 * hatid = hatid ;\ 2553 * tsbarea = tsbarea ;\ 2554 * tte = hmebp (hme bucket pointer) ;\ 2555 * hmeblkpa = vapg (virtual page) ;\ 2556 * hmemisc, hmeblkva = scratch ;\ 2557 */ ;\ 2558 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2559 ;\ 2560 /* ;\ 2561 * tagacc = tagacc ;\ 2562 * hatid = hatid ;\ 2563 * tsbarea = tsbarea ;\ 2564 * tte = hmebp ;\ 2565 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2566 * hmeblkpa = CLOBBERED ;\ 2567 * hmeblkva = scratch ;\ 2568 */ ;\ 2569 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2570 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2571 ;\ 2572 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2573 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2574 add tte, HMEBUCK_HBLK, hmeblkva ;\ 2575 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2576 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2577 ;\ 2578label/**/8: ;\ 2579 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, hmeblkva, \ 2580 tsbarea, tagacc, tte, label/**/1) ;\ 2581 /* ;\ 2582 * tagacc = CLOBBERED ;\ 2583 * tte = CLOBBERED ;\ 2584 * hmeblkpa = hmeblkpa ;\ 2585 * hmeblkva = hmeblkva ;\ 2586 */ ;\ 2587 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2588 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2589 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2590 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2591 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2592 nop ;\ 2593label/**/4: ;\ 2594 /* ;\ 2595 * We have found the hmeblk containing the hment. ;\ 2596 * Now we calculate the corresponding tte. ;\ 2597 * ;\ 2598 * tagacc = tagacc ;\ 2599 * hatid = hatid ;\ 2600 * tte = clobbered ;\ 2601 * hmeblkpa = hmeblkpa ;\ 2602 * hmemisc = hblktag ;\ 2603 * hmeblkva = hmeblkva ;\ 2604 * tsbarea = tsbmiss area ;\ 2605 */ ;\ 2606 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2607 label/**/2) ;\ 2608 ;\ 2609 /* ;\ 2610 * tagacc = tagacc ;\ 2611 * hatid = hmentoff ;\ 2612 * tte = clobbered ;\ 2613 * hmeblkpa = hmeblkpa ;\ 2614 * hmemisc = hblk_misc ;\ 2615 * hmeblkva = hmeblkva ;\ 2616 * tsbarea = tsbmiss area ;\ 2617 */ ;\ 2618 ;\ 2619 add hatid, SFHME_TTE, hatid ;\ 2620 add hmeblkpa, hatid, hmeblkpa ;\ 2621 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2622 brlz,pt tte, label/**/6 ;\ 2623 add hmeblkva, hatid, hmeblkva ;\ 2624 btst HBLK_SZMASK, hmemisc ;\ 2625 bnz,a,pt %icc, label/**/7 ;\ 2626 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2627 ;\ 2628 /* ;\ 2629 * We found an invalid 8K tte in shme. ;\ 2630 * it may not belong to shme's region since ;\ 2631 * region size/alignment granularity is 8K but different ;\ 2632 * regions don't share hmeblks. Continue the search. ;\ 2633 */ ;\ 2634 sub hmeblkpa, hatid, hmeblkpa ;\ 2635 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2636 srlx tagacc, hmeshift, tte ;\ 2637 add hmeblkpa, HMEBLK_NEXT, hmeblkva ;\ 2638 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 2639 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2640 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2641 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2642 ba,a,pt %xcc, label/**/8 ;\ 2643label/**/6: ;\ 2644 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2645 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2646label/**/7: ;\ 2647 HMELOCK_EXIT(hatid, hatid, ASI_MEM) /* drop lock */ ;\ 2648 set TTE_SUSPEND, hatid ;\ 2649 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2650 btst tte, hatid ;\ 2651 bz,pt %xcc, foundlabel ;\ 2652 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2653 ;\ 2654 /* ;\ 2655 * Mapping is suspended, so goto suspend label. ;\ 2656 */ ;\ 2657 ba,pt %xcc, suspendlabel ;\ 2658 nop 2659 2660 /* 2661 * KERNEL PROTECTION HANDLER 2662 * 2663 * g1 = tsb8k pointer register (clobbered) 2664 * g2 = tag access register (ro) 2665 * g3 - g7 = scratch registers 2666 * 2667 * Note: This function is patched at runtime for performance reasons. 2668 * Any changes here require sfmmu_patch_ktsb fixed. 2669 */ 2670 ENTRY_NP(sfmmu_kprot_trap) 2671 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2672sfmmu_kprot_patch_ktsb_base: 2673 RUNTIME_PATCH_SETX(%g1, %g6) 2674 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2675sfmmu_kprot_patch_ktsb_szcode: 2676 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2677 2678 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2679 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2680 2681 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2682sfmmu_kprot_patch_ktsb4m_base: 2683 RUNTIME_PATCH_SETX(%g3, %g6) 2684 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2685sfmmu_kprot_patch_ktsb4m_szcode: 2686 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2687 2688 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2689 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2690 2691 CPU_TSBMISS_AREA(%g6, %g7) 2692 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2693 ba,pt %xcc, sfmmu_tsb_miss_tt 2694 nop 2695 2696 /* 2697 * USER PROTECTION HANDLER 2698 * 2699 * g1 = tsb8k pointer register (ro) 2700 * g2 = tag access register (ro) 2701 * g3 = faulting context (clobbered, currently not used) 2702 * g4 - g7 = scratch registers 2703 */ 2704 ALTENTRY(sfmmu_uprot_trap) 2705#ifdef sun4v 2706 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2707 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2708 2709 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2710 brlz,pt %g3, 9f /* check for 2nd TSB */ 2711 nop 2712 2713 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2714 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2715 2716#else /* sun4v */ 2717#ifdef UTSB_PHYS 2718 /* g1 = first TSB entry ptr */ 2719 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2720 brlz,pt %g3, 9f /* check for 2nd TSB */ 2721 nop 2722 2723 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2724 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2725#else /* UTSB_PHYS */ 2726 brgez,pt %g1, 9f /* check for 2nd TSB */ 2727 mov -1, %g3 /* set second tsbe ptr to -1 */ 2728 2729 mov %g2, %g7 2730 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2731 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2732 mov %g1, %g7 2733 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2734#endif /* UTSB_PHYS */ 2735#endif /* sun4v */ 27369: 2737 CPU_TSBMISS_AREA(%g6, %g7) 2738 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2739 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2740 nop 2741 2742 /* 2743 * Kernel 8K page iTLB miss. We also get here if we took a 2744 * fast instruction access mmu miss trap while running in 2745 * invalid context. 2746 * 2747 * %g1 = 8K TSB pointer register (not used, clobbered) 2748 * %g2 = tag access register (used) 2749 * %g3 = faulting context id (used) 2750 * %g7 = TSB tag to match (used) 2751 */ 2752 .align 64 2753 ALTENTRY(sfmmu_kitlb_miss) 2754 brnz,pn %g3, tsb_tl0_noctxt 2755 nop 2756 2757 /* kernel miss */ 2758 /* get kernel tsb pointer */ 2759 /* we patch the next set of instructions at run time */ 2760 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2761iktsbbase: 2762 RUNTIME_PATCH_SETX(%g4, %g5) 2763 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2764 2765iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2766 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2767 or %g4, %g1, %g1 ! form tsb ptr 2768 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2769 cmp %g4, %g7 2770 bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2771 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2772 2773 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2774 bz,pn %icc, exec_fault 2775 nop 2776 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2777 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2778 retry 2779 2780iktsb4mbase: 2781 RUNTIME_PATCH_SETX(%g4, %g6) 2782 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2783iktsb4m: 2784 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2785 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2786 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2787 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2788 cmp %g4, %g7 2789 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2790 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2791 bz,pn %icc, exec_fault 2792 nop 2793 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2794 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2795 retry 2796 2797 /* 2798 * Kernel dTLB miss. We also get here if we took a fast data 2799 * access mmu miss trap while running in invalid context. 2800 * 2801 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2802 * We select the TSB miss handler to branch to depending on 2803 * the virtual address of the access. In the future it may 2804 * be desirable to separate kpm TTEs into their own TSB, 2805 * in which case all that needs to be done is to set 2806 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2807 * early in the miss if we detect a kpm VA to a new handler. 2808 * 2809 * %g1 = 8K TSB pointer register (not used, clobbered) 2810 * %g2 = tag access register (used) 2811 * %g3 = faulting context id (used) 2812 */ 2813 .align 64 2814 ALTENTRY(sfmmu_kdtlb_miss) 2815 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2816 nop 2817 2818 /* Gather some stats for kpm misses in the TLB. */ 2819 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2820 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2821 2822 /* 2823 * Get first TSB offset and look for 8K/64K/512K mapping 2824 * using the 8K virtual page as the index. 2825 * 2826 * We patch the next set of instructions at run time; 2827 * any changes here require sfmmu_patch_ktsb changes too. 2828 */ 2829dktsbbase: 2830 RUNTIME_PATCH_SETX(%g7, %g6) 2831 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2832 2833dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2834 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2835 2836 /* 2837 * At this point %g1 is our index into the TSB. 2838 * We just masked off enough bits of the VA depending 2839 * on our TSB size code. 2840 */ 2841 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2842 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2843 cmp %g6, %g4 ! compare tag 2844 bne,pn %xcc, dktsb4m_kpmcheck_small 2845 add %g7, %g1, %g1 /* form tsb ptr */ 2846 TT_TRACE(trace_tsbhit) 2847 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2848 /* trapstat expects tte in %g5 */ 2849 retry 2850 2851 /* 2852 * If kpm is using large pages, the following instruction needs 2853 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2854 * so that we will probe the 4M TSB regardless of the VA. In 2855 * the case kpm is using small pages, we know no large kernel 2856 * mappings are located above 0x80000000.00000000 so we skip the 2857 * probe as an optimization. 2858 */ 2859dktsb4m_kpmcheck_small: 2860 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2861 /* delay slot safe, below */ 2862 2863 /* 2864 * Get second TSB offset and look for 4M mapping 2865 * using 4M virtual page as the TSB index. 2866 * 2867 * Here: 2868 * %g1 = 8K TSB pointer. Don't squash it. 2869 * %g2 = tag access register (we still need it) 2870 */ 2871 srlx %g2, MMU_PAGESHIFT4M, %g3 2872 2873 /* 2874 * We patch the next set of instructions at run time; 2875 * any changes here require sfmmu_patch_ktsb changes too. 2876 */ 2877dktsb4mbase: 2878 RUNTIME_PATCH_SETX(%g7, %g6) 2879 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2880dktsb4m: 2881 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2882 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2883 2884 /* 2885 * At this point %g3 is our index into the TSB. 2886 * We just masked off enough bits of the VA depending 2887 * on our TSB size code. 2888 */ 2889 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2890 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2891 cmp %g6, %g4 ! compare tag 2892 2893dktsb4m_tsbmiss: 2894 bne,pn %xcc, dktsb4m_kpmcheck 2895 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2896 TT_TRACE(trace_tsbhit) 2897 /* we don't check TTE size here since we assume 4M TSB is separate */ 2898 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2899 /* trapstat expects tte in %g5 */ 2900 retry 2901 2902 /* 2903 * So, we failed to find a valid TTE to match the faulting 2904 * address in either TSB. There are a few cases that could land 2905 * us here: 2906 * 2907 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2908 * to sfmmu_tsb_miss_tt to handle the miss. 2909 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2910 * 4M TSB. Let segkpm handle it. 2911 * 2912 * Note that we shouldn't land here in the case of a kpm VA when 2913 * kpm_smallpages is active -- we handled that case earlier at 2914 * dktsb4m_kpmcheck_small. 2915 * 2916 * At this point: 2917 * g1 = 8K-indexed primary TSB pointer 2918 * g2 = tag access register 2919 * g3 = 4M-indexed secondary TSB pointer 2920 */ 2921dktsb4m_kpmcheck: 2922 cmp %g2, %g0 2923 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2924 nop 2925 ba,a,pt %icc, sfmmu_tsb_miss_tt 2926 nop 2927 2928#ifdef sun4v 2929 /* 2930 * User instruction miss w/ single TSB. 2931 * The first probe covers 8K, 64K, and 512K page sizes, 2932 * because 64K and 512K mappings are replicated off 8K 2933 * pointer. 2934 * 2935 * g1 = tsb8k pointer register 2936 * g2 = tag access register 2937 * g3 - g6 = scratch registers 2938 * g7 = TSB tag to match 2939 */ 2940 .align 64 2941 ALTENTRY(sfmmu_uitlb_fastpath) 2942 2943 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2944 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2945 ba,pn %xcc, sfmmu_tsb_miss_tt 2946 mov -1, %g3 2947 2948 /* 2949 * User data miss w/ single TSB. 2950 * The first probe covers 8K, 64K, and 512K page sizes, 2951 * because 64K and 512K mappings are replicated off 8K 2952 * pointer. 2953 * 2954 * g1 = tsb8k pointer register 2955 * g2 = tag access register 2956 * g3 - g6 = scratch registers 2957 * g7 = TSB tag to match 2958 */ 2959 .align 64 2960 ALTENTRY(sfmmu_udtlb_fastpath) 2961 2962 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2963 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2964 ba,pn %xcc, sfmmu_tsb_miss_tt 2965 mov -1, %g3 2966 2967 /* 2968 * User instruction miss w/ multiple TSBs (sun4v). 2969 * The first probe covers 8K, 64K, and 512K page sizes, 2970 * because 64K and 512K mappings are replicated off 8K 2971 * pointer. Second probe covers 4M page size only. 2972 * 2973 * Just like sfmmu_udtlb_slowpath, except: 2974 * o Uses ASI_ITLB_IN 2975 * o checks for execute permission 2976 * o No ISM prediction. 2977 * 2978 * g1 = tsb8k pointer register 2979 * g2 = tag access register 2980 * g3 - g6 = scratch registers 2981 * g7 = TSB tag to match 2982 */ 2983 .align 64 2984 ALTENTRY(sfmmu_uitlb_slowpath) 2985 2986 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2987 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2988 /* g4 - g5 = clobbered here */ 2989 2990 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2991 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2992 srlx %g2, TAG_VALO_SHIFT, %g7 2993 PROBE_2ND_ITSB(%g3, %g7) 2994 /* NOT REACHED */ 2995 2996#else /* sun4v */ 2997 2998 /* 2999 * User instruction miss w/ multiple TSBs (sun4u). 3000 * The first probe covers 8K, 64K, and 512K page sizes, 3001 * because 64K and 512K mappings are replicated off 8K 3002 * pointer. Probe of 1st TSB has already been done prior to entry 3003 * into this routine. For the UTSB_PHYS case we probe up to 3 3004 * valid other TSBs in the following order: 3005 * 1) shared TSB for 4M-256M pages 3006 * 2) private TSB for 4M-256M pages 3007 * 3) shared TSB for 8K-512K pages 3008 * 3009 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs 3010 * 4M-256M pages. 3011 * 3012 * Just like sfmmu_udtlb_slowpath, except: 3013 * o Uses ASI_ITLB_IN 3014 * o checks for execute permission 3015 * o No ISM prediction. 3016 * 3017 * g1 = tsb8k pointer register 3018 * g2 = tag access register 3019 * g4 - g6 = scratch registers 3020 * g7 = TSB tag to match 3021 */ 3022 .align 64 3023 ALTENTRY(sfmmu_uitlb_slowpath) 3024 3025#ifdef UTSB_PHYS 3026 3027 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3028 brlz,pt %g6, 1f 3029 nop 3030 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3031 PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail) 30321: 3033 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3034 brlz,pt %g3, 2f 3035 nop 3036 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3037 PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail) 30382: 3039 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3040 brlz,pt %g6, sfmmu_tsb_miss_tt 3041 nop 3042 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3043 PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail) 3044 ba,pn %xcc, sfmmu_tsb_miss_tt 3045 nop 3046 3047#else /* UTSB_PHYS */ 3048 mov %g1, %g3 /* save tsb8k reg in %g3 */ 3049 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 3050 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 3051 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 3052 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 3053 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 3054 /* g1 = first TSB pointer, g3 = second TSB pointer */ 3055 srlx %g2, TAG_VALO_SHIFT, %g7 3056 PROBE_2ND_ITSB(%g3, %g7, isynth) 3057 ba,pn %xcc, sfmmu_tsb_miss_tt 3058 nop 3059 3060#endif /* UTSB_PHYS */ 3061#endif /* sun4v */ 3062 3063#if defined(sun4u) && defined(UTSB_PHYS) 3064 3065 /* 3066 * We come here for ism predict DTLB_MISS case or if 3067 * if probe in first TSB failed. 3068 */ 3069 3070 .align 64 3071 ALTENTRY(sfmmu_udtlb_slowpath_noismpred) 3072 3073 /* 3074 * g1 = tsb8k pointer register 3075 * g2 = tag access register 3076 * g4 - %g6 = scratch registers 3077 * g7 = TSB tag to match 3078 */ 3079 3080 /* 3081 * ISM non-predict probe order 3082 * probe 1ST_TSB (8K index) 3083 * probe 2ND_TSB (4M index) 3084 * probe 4TH_TSB (4M index) 3085 * probe 3RD_TSB (8K index) 3086 * 3087 * We already probed first TSB in DTLB_MISS handler. 3088 */ 3089 3090 /* 3091 * Private 2ND TSB 4M-256 pages 3092 */ 3093 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3094 brlz,pt %g3, 1f 3095 nop 3096 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3097 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3098 3099 /* 3100 * Shared Context 4TH TSB 4M-256 pages 3101 */ 31021: 3103 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3104 brlz,pt %g6, 2f 3105 nop 3106 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3107 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail) 3108 3109 /* 3110 * Shared Context 3RD TSB 8K-512K pages 3111 */ 31122: 3113 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3114 brlz,pt %g6, sfmmu_tsb_miss_tt 3115 nop 3116 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3117 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail) 3118 ba,pn %xcc, sfmmu_tsb_miss_tt 3119 nop 3120 3121 .align 64 3122 ALTENTRY(sfmmu_udtlb_slowpath_ismpred) 3123 3124 /* 3125 * g1 = tsb8k pointer register 3126 * g2 = tag access register 3127 * g4 - g6 = scratch registers 3128 * g7 = TSB tag to match 3129 */ 3130 3131 /* 3132 * ISM predict probe order 3133 * probe 4TH_TSB (4M index) 3134 * probe 2ND_TSB (4M index) 3135 * probe 1ST_TSB (8K index) 3136 * probe 3RD_TSB (8K index) 3137 3138 /* 3139 * Shared Context 4TH TSB 4M-256 pages 3140 */ 3141 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 3142 brlz,pt %g6, 4f 3143 nop 3144 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 3145 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2) 3146 3147 /* 3148 * Private 2ND TSB 4M-256 pages 3149 */ 31504: 3151 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 3152 brlz,pt %g3, 5f 3153 nop 3154 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3155 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2) 3156 31575: 3158 PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2) 3159 3160 /* 3161 * Shared Context 3RD TSB 8K-512K pages 3162 */ 3163 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 3164 brlz,pt %g6, 6f 3165 nop 3166 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 3167 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2) 31686: 3169 ba,pn %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */ 3170 nop 3171 3172#else /* sun4u && UTSB_PHYS */ 3173 3174 .align 64 3175 ALTENTRY(sfmmu_udtlb_slowpath) 3176 3177 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 3178 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 3179 mov %g1, %g3 3180 3181udtlb_miss_probefirst: 3182 /* 3183 * g1 = 8K TSB pointer register 3184 * g2 = tag access register 3185 * g3 = (potentially) second TSB entry ptr 3186 * g6 = ism pred. 3187 * g7 = vpg_4m 3188 */ 3189#ifdef sun4v 3190 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 3191 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3192 3193 /* 3194 * Here: 3195 * g1 = first TSB pointer 3196 * g2 = tag access reg 3197 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3198 */ 3199 brgz,pn %g6, sfmmu_tsb_miss_tt 3200 nop 3201#else /* sun4v */ 3202 mov %g1, %g4 3203 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 3204 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 3205 3206 /* 3207 * Here: 3208 * g1 = first TSB pointer 3209 * g2 = tag access reg 3210 * g3 = second TSB ptr IFF ISM pred. (else don't care) 3211 */ 3212 brgz,pn %g6, sfmmu_tsb_miss_tt 3213 nop 3214 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 3215 /* fall through in 8K->4M probe order */ 3216#endif /* sun4v */ 3217 3218udtlb_miss_probesecond: 3219 /* 3220 * Look in the second TSB for the TTE 3221 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 3222 * g2 = tag access reg 3223 * g3 = 8K TSB pointer register 3224 * g6 = ism pred. 3225 * g7 = vpg_4m 3226 */ 3227#ifdef sun4v 3228 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 3229 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 3230 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 3231#else /* sun4v */ 3232 mov %g3, %g7 3233 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 3234 /* %g2 clobbered, %g3 =second tsbe ptr */ 3235 mov MMU_TAG_ACCESS, %g2 3236 ldxa [%g2]ASI_DMMU, %g2 3237#endif /* sun4v */ 3238 3239 srlx %g2, TAG_VALO_SHIFT, %g7 3240 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3241 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 3242 brgz,pn %g6, udtlb_miss_probefirst 3243 nop 3244 3245 /* fall through to sfmmu_tsb_miss_tt */ 3246#endif /* sun4u && UTSB_PHYS */ 3247 3248 3249 ALTENTRY(sfmmu_tsb_miss_tt) 3250 TT_TRACE(trace_tsbmiss) 3251 /* 3252 * We get here if there is a TSB miss OR a write protect trap. 3253 * 3254 * g1 = First TSB entry pointer 3255 * g2 = tag access register 3256 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 3257 * g4 - g7 = scratch registers 3258 */ 3259 3260 ALTENTRY(sfmmu_tsb_miss) 3261 3262 /* 3263 * If trapstat is running, we need to shift the %tpc and %tnpc to 3264 * point to trapstat's TSB miss return code (note that trapstat 3265 * itself will patch the correct offset to add). 3266 */ 3267 rdpr %tl, %g7 3268 cmp %g7, 1 3269 ble,pt %xcc, 0f 3270 sethi %hi(KERNELBASE), %g6 3271 rdpr %tpc, %g7 3272 or %g6, %lo(KERNELBASE), %g6 3273 cmp %g7, %g6 3274 bgeu,pt %xcc, 0f 3275 /* delay slot safe */ 3276 3277 ALTENTRY(tsbmiss_trapstat_patch_point) 3278 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 3279 wrpr %g7, %tpc 3280 add %g7, 4, %g7 3281 wrpr %g7, %tnpc 32820: 3283 CPU_TSBMISS_AREA(%g6, %g7) 3284 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save 1ST tsb pointer */ 3285 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save 2ND tsb pointer */ 3286 3287 sllx %g2, TAGACC_CTX_LSHIFT, %g3 3288 brz,a,pn %g3, 1f /* skip ahead if kernel */ 3289 ldn [%g6 + TSBMISS_KHATID], %g7 3290 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 3291 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 3292 3293 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 3294 3295 cmp %g3, INVALID_CONTEXT 3296 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 3297 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 3298 3299#if defined(sun4v) || defined(UTSB_PHYS) 3300 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 3301 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 3302 stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 3303#endif /* sun4v || UTSB_PHYS */ 3304 3305 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 3306 /* 3307 * The miss wasn't in an ISM segment. 3308 * 3309 * %g1 %g3, %g4, %g5, %g7 all clobbered 3310 * %g2 = (pseudo) tag access 3311 */ 3312 3313 ba,pt %icc, 2f 3314 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 3315 33161: 3317 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 3318 /* 3319 * 8K and 64K hash. 3320 */ 33212: 3322 3323 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3324 MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte, 3325 sfmmu_suspend_tl, tsb_512K) 3326 /* NOT REACHED */ 3327 3328tsb_512K: 3329 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3330 brz,pn %g5, 3f 3331 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3332 and %g4, HAT_512K_FLAG, %g5 3333 3334 /* 3335 * Note that there is a small window here where we may have 3336 * a 512k page in the hash list but have not set the HAT_512K_FLAG 3337 * flag yet, so we will skip searching the 512k hash list. 3338 * In this case we will end up in pagefault which will find 3339 * the mapping and return. So, in this instance we will end up 3340 * spending a bit more time resolving this TSB miss, but it can 3341 * only happen once per process and even then, the chances of that 3342 * are very small, so it's not worth the extra overhead it would 3343 * take to close this window. 3344 */ 3345 brz,pn %g5, tsb_4M 3346 nop 33473: 3348 /* 3349 * 512K hash 3350 */ 3351 3352 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3353 MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte, 3354 sfmmu_suspend_tl, tsb_4M) 3355 /* NOT REACHED */ 3356 3357tsb_4M: 3358 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3359 brz,pn %g5, 4f 3360 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3361 and %g4, HAT_4M_FLAG, %g5 3362 brz,pn %g5, tsb_32M 3363 nop 33644: 3365 /* 3366 * 4M hash 3367 */ 3368 3369 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3370 MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte, 3371 sfmmu_suspend_tl, tsb_32M) 3372 /* NOT REACHED */ 3373 3374tsb_32M: 3375 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3376#ifdef sun4v 3377 brz,pn %g5, 6f 3378#else 3379 brz,pn %g5, tsb_pagefault 3380#endif 3381 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3382 and %g4, HAT_32M_FLAG, %g5 3383 brz,pn %g5, tsb_256M 3384 nop 33855: 3386 /* 3387 * 32M hash 3388 */ 3389 3390 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3391 MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte, 3392 sfmmu_suspend_tl, tsb_256M) 3393 /* NOT REACHED */ 3394 3395#if defined(sun4u) && !defined(UTSB_PHYS) 3396#define tsb_shme tsb_pagefault 3397#endif 3398tsb_256M: 3399 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3400 and %g4, HAT_256M_FLAG, %g5 3401 brz,pn %g5, tsb_shme 3402 nop 34036: 3404 /* 3405 * 256M hash 3406 */ 3407 3408 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3409 MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte, 3410 sfmmu_suspend_tl, tsb_shme) 3411 /* NOT REACHED */ 3412 3413tsb_checktte: 3414 /* 3415 * g1 = hblk_misc 3416 * g2 = tagacc 3417 * g3 = tte 3418 * g4 = tte pa 3419 * g5 = tte va 3420 * g6 = tsbmiss area 3421 * g7 = hatid 3422 */ 3423 brlz,a,pt %g3, tsb_validtte 3424 rdpr %tt, %g7 3425 3426#if defined(sun4u) && !defined(UTSB_PHYS) 3427#undef tsb_shme 3428 ba tsb_pagefault 3429 nop 3430#else /* sun4u && !UTSB_PHYS */ 3431 3432tsb_shme: 3433 /* 3434 * g2 = tagacc 3435 * g6 = tsbmiss area 3436 */ 3437 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3438 brz,pn %g5, tsb_pagefault 3439 nop 3440 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3441 brz,pn %g7, tsb_pagefault 3442 nop 3443 3444 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3445 MMU_PAGESHIFT64K, TTE64K, tsb_shme_l8K, tsb_shme_checktte, 3446 sfmmu_suspend_tl, tsb_shme_512K) 3447 /* NOT REACHED */ 3448 3449tsb_shme_512K: 3450 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3451 and %g4, HAT_512K_FLAG, %g5 3452 brz,pn %g5, tsb_shme_4M 3453 nop 3454 3455 /* 3456 * 512K hash 3457 */ 3458 3459 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3460 MMU_PAGESHIFT512K, TTE512K, tsb_shme_l512K, tsb_shme_checktte, 3461 sfmmu_suspend_tl, tsb_shme_4M) 3462 /* NOT REACHED */ 3463 3464tsb_shme_4M: 3465 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3466 and %g4, HAT_4M_FLAG, %g5 3467 brz,pn %g5, tsb_shme_32M 3468 nop 34694: 3470 /* 3471 * 4M hash 3472 */ 3473 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3474 MMU_PAGESHIFT4M, TTE4M, tsb_shme_l4M, tsb_shme_checktte, 3475 sfmmu_suspend_tl, tsb_shme_32M) 3476 /* NOT REACHED */ 3477 3478tsb_shme_32M: 3479 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3480 and %g4, HAT_32M_FLAG, %g5 3481 brz,pn %g5, tsb_shme_256M 3482 nop 3483 3484 /* 3485 * 32M hash 3486 */ 3487 3488 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3489 MMU_PAGESHIFT32M, TTE32M, tsb_shme_l32M, tsb_shme_checktte, 3490 sfmmu_suspend_tl, tsb_shme_256M) 3491 /* NOT REACHED */ 3492 3493tsb_shme_256M: 3494 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3495 and %g4, HAT_256M_FLAG, %g5 3496 brz,pn %g5, tsb_pagefault 3497 nop 3498 3499 /* 3500 * 256M hash 3501 */ 3502 3503 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 3504 MMU_PAGESHIFT256M, TTE256M, tsb_shme_l256M, tsb_shme_checktte, 3505 sfmmu_suspend_tl, tsb_pagefault) 3506 /* NOT REACHED */ 3507 3508tsb_shme_checktte: 3509 3510 brgez,pn %g3, tsb_pagefault 3511 rdpr %tt, %g7 3512 /* 3513 * g1 = ctx1 flag 3514 * g3 = tte 3515 * g4 = tte pa 3516 * g5 = tte va 3517 * g6 = tsbmiss area 3518 * g7 = tt 3519 */ 3520 3521 brz,pt %g1, tsb_validtte 3522 nop 3523 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3524 or %g1, HAT_CHKCTX1_FLAG, %g1 3525 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3526 3527 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3528#endif /* sun4u && !UTSB_PHYS */ 3529 3530tsb_validtte: 3531 /* 3532 * g3 = tte 3533 * g4 = tte pa 3534 * g5 = tte va 3535 * g6 = tsbmiss area 3536 * g7 = tt 3537 */ 3538 3539 /* 3540 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3541 */ 3542 cmp %g7, FAST_PROT_TT 3543 bne,pt %icc, 4f 3544 nop 3545 3546 TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod, 3547 tsb_protfault) 3548 3549 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3550#ifdef sun4v 3551 MMU_FAULT_STATUS_AREA(%g7) 3552 ldx [%g7 + MMFSA_D_ADDR], %g5 /* load fault addr for later */ 3553#else /* sun4v */ 3554 mov MMU_TAG_ACCESS, %g5 3555 ldxa [%g5]ASI_DMMU, %g5 3556#endif /* sun4v */ 3557 ba,pt %xcc, tsb_update_tl1 3558 nop 35594: 3560 /* 3561 * If ITLB miss check exec bit. 3562 * If not set treat as invalid TTE. 3563 */ 3564 cmp %g7, T_INSTR_MMU_MISS 3565 be,pn %icc, 5f 3566 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3567 cmp %g7, FAST_IMMU_MISS_TT 3568 bne,pt %icc, 3f 3569 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 35705: 3571 bz,pn %icc, tsb_protfault 3572 nop 3573 35743: 3575 /* 3576 * Set reference bit if not already set 3577 */ 3578 TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref) 3579 3580 /* 3581 * Now, load into TSB/TLB. At this point: 3582 * g3 = tte 3583 * g4 = patte 3584 * g6 = tsbmiss area 3585 */ 3586 rdpr %tt, %g7 3587#ifdef sun4v 3588 MMU_FAULT_STATUS_AREA(%g2) 3589 cmp %g7, T_INSTR_MMU_MISS 3590 be,a,pt %icc, 9f 3591 nop 3592 cmp %g7, FAST_IMMU_MISS_TT 3593 be,a,pt %icc, 9f 3594 nop 3595 add %g2, MMFSA_D_, %g2 35969: 3597 ldx [%g2 + MMFSA_CTX_], %g7 3598 sllx %g7, TTARGET_CTX_SHIFT, %g7 3599 ldx [%g2 + MMFSA_ADDR_], %g2 3600 mov %g2, %g5 ! load the fault addr for later use 3601 srlx %g2, TTARGET_VA_SHIFT, %g2 3602 or %g2, %g7, %g2 3603#else /* sun4v */ 3604 mov MMU_TAG_ACCESS, %g5 3605 cmp %g7, FAST_IMMU_MISS_TT 3606 be,a,pt %icc, 9f 3607 ldxa [%g0]ASI_IMMU, %g2 3608 ldxa [%g0]ASI_DMMU, %g2 3609 ba,pt %icc, tsb_update_tl1 3610 ldxa [%g5]ASI_DMMU, %g5 36119: 3612 ldxa [%g5]ASI_IMMU, %g5 3613#endif /* sun4v */ 3614 3615tsb_update_tl1: 3616 srlx %g2, TTARGET_CTX_SHIFT, %g7 3617 brz,pn %g7, tsb_kernel 3618#ifdef sun4v 3619 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3620#else /* sun4v */ 3621 srlx %g3, TTE_SZ_SHFT, %g7 3622#endif /* sun4v */ 3623 3624tsb_user: 3625#ifdef sun4v 3626 cmp %g7, TTE4M 3627 bge,pn %icc, tsb_user4m 3628 nop 3629#else /* sun4v */ 3630 cmp %g7, TTESZ_VALID | TTE4M 3631 be,pn %icc, tsb_user4m 3632 srlx %g3, TTE_SZ2_SHFT, %g7 3633 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3634#ifdef ITLB_32M_256M_SUPPORT 3635 bnz,pn %icc, tsb_user4m 3636 nop 3637#else /* ITLB_32M_256M_SUPPORT */ 3638 bnz,a,pn %icc, tsb_user_pn_synth 3639 nop 3640#endif /* ITLB_32M_256M_SUPPORT */ 3641#endif /* sun4v */ 3642 3643tsb_user8k: 3644#if defined(sun4v) || defined(UTSB_PHYS) 3645 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3646 and %g7, HAT_CHKCTX1_FLAG, %g1 3647 brz,a,pn %g1, 1f 3648 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3649 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3650 brlz,a,pn %g1, ptl1_panic ! if no shared 3RD tsb 3651 mov PTL1_NO_SCDTSB8K, %g1 ! panic 3652 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 36531: 3654#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3655 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3656#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3657 3658#ifndef UTSB_PHYS 3659 mov ASI_N, %g7 ! user TSBs accessed by VA 3660 mov %g7, %asi 3661#endif /* !UTSB_PHYS */ 3662 3663 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5) 3664 3665 rdpr %tt, %g5 3666#ifdef sun4v 3667 cmp %g5, T_INSTR_MMU_MISS 3668 be,a,pn %xcc, 9f 3669 mov %g3, %g5 3670#endif /* sun4v */ 3671 cmp %g5, FAST_IMMU_MISS_TT 3672 be,pn %xcc, 9f 3673 mov %g3, %g5 3674 3675 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3676 ! trapstat wants TTE in %g5 3677 retry 36789: 3679 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3680 ! trapstat wants TTE in %g5 3681 retry 3682 3683tsb_user4m: 3684#if defined(sun4v) || defined(UTSB_PHYS) 3685 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3686 and %g7, HAT_CHKCTX1_FLAG, %g1 3687 brz,a,pn %g1, 4f 3688 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3689 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr 3690 brlz,a,pn %g1, 5f ! if no shared 4TH TSB 3691 nop 3692 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 36934: 3694#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3695 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3696#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3697 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3698 nop 3699 3700#ifndef UTSB_PHYS 3701 mov ASI_N, %g7 ! user TSBs accessed by VA 3702 mov %g7, %asi 3703#endif /* UTSB_PHYS */ 3704 3705 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6) 3706 37075: 3708 rdpr %tt, %g5 3709#ifdef sun4v 3710 cmp %g5, T_INSTR_MMU_MISS 3711 be,a,pn %xcc, 9f 3712 mov %g3, %g5 3713#endif /* sun4v */ 3714 cmp %g5, FAST_IMMU_MISS_TT 3715 be,pn %xcc, 9f 3716 mov %g3, %g5 3717 3718 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3719 ! trapstat wants TTE in %g5 3720 retry 37219: 3722 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3723 ! trapstat wants TTE in %g5 3724 retry 3725 3726#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3727 /* 3728 * Panther ITLB synthesis. 3729 * The Panther 32M and 256M ITLB code simulates these two large page 3730 * sizes with 4M pages, to provide support for programs, for example 3731 * Java, that may copy instructions into a 32M or 256M data page and 3732 * then execute them. The code below generates the 4M pfn bits and 3733 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3734 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3735 * are ignored by the hardware. 3736 * 3737 * Now, load into TSB/TLB. At this point: 3738 * g2 = tagtarget 3739 * g3 = tte 3740 * g4 = patte 3741 * g5 = tt 3742 * g6 = tsbmiss area 3743 */ 3744tsb_user_pn_synth: 3745 rdpr %tt, %g5 3746 cmp %g5, FAST_IMMU_MISS_TT 3747 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3748 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3749 bz,pn %icc, 4b /* if not, been here before */ 3750 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3751 brlz,a,pn %g1, 5f /* no 2nd tsb */ 3752 mov %g3, %g5 3753 3754 mov MMU_TAG_ACCESS, %g7 3755 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3756 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3757 3758 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3759 mov %g7, %asi 3760 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */ 37615: 3762 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3763 retry 3764 3765tsb_user_itlb_synth: 3766 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = 2ND TSB */ 3767 3768 mov MMU_TAG_ACCESS, %g7 3769 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3770 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3771 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3772 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3773 3774 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3775 mov %g7, %asi 3776 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */ 37777: 3778 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3779 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3780 retry 3781#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3782 3783tsb_kernel: 3784 rdpr %tt, %g5 3785#ifdef sun4v 3786 cmp %g7, TTE4M 3787 bge,pn %icc, 5f 3788#else 3789 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3790 be,pn %icc, 5f 3791#endif /* sun4v */ 3792 nop 3793 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8K TSB ptr 3794 ba,pt %xcc, 6f 3795 nop 37965: 3797 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4M TSB ptr 3798 brlz,pn %g1, 3f /* skip programming if 4M TSB ptr is -1 */ 3799 nop 38006: 3801#ifndef sun4v 3802tsb_kernel_patch_asi: 3803 or %g0, RUNTIME_PATCH, %g6 3804 mov %g6, %asi ! XXX avoid writing to %asi !! 3805#endif 3806 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7) 38073: 3808#ifdef sun4v 3809 cmp %g5, T_INSTR_MMU_MISS 3810 be,a,pn %icc, 1f 3811 mov %g3, %g5 ! trapstat wants TTE in %g5 3812#endif /* sun4v */ 3813 cmp %g5, FAST_IMMU_MISS_TT 3814 be,pn %icc, 1f 3815 mov %g3, %g5 ! trapstat wants TTE in %g5 3816 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3817 ! trapstat wants TTE in %g5 3818 retry 38191: 3820 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3821 ! trapstat wants TTE in %g5 3822 retry 3823 3824tsb_ism: 3825 /* 3826 * This is an ISM [i|d]tlb miss. We optimize for largest 3827 * page size down to smallest. 3828 * 3829 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3830 * register 3831 * g3 = ismmap->ism_seg 3832 * g4 = physical address of ismmap->ism_sfmmu 3833 * g6 = tsbmiss area 3834 */ 3835 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3836 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3837 mov PTL1_BAD_ISM, %g1 3838 /* g5 = pa of imap_vb_shift */ 3839 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3840 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3841 srlx %g3, %g4, %g3 /* clr size field */ 3842 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3843 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3844 and %g2, %g1, %g4 /* g4 = ctx number */ 3845 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3846 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3847 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3848 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3849 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3850#if defined(sun4v) || defined(UTSB_PHYS) 3851 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3852 brz,pt %g5, tsb_chk4M_ism 3853 nop 3854 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3855 or %g5, HAT_CHKCTX1_FLAG, %g5 3856 stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3857 rdpr %tt, %g5 3858 SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl) 3859#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3860 3861 /* 3862 * ISM pages are always locked down. 3863 * If we can't find the tte then pagefault 3864 * and let the spt segment driver resolve it. 3865 * 3866 * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3867 * g4 = imap_hatflags 3868 * g6 = tsb miss area 3869 * g7 = ISM hatid 3870 */ 3871 3872tsb_chk4M_ism: 3873 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3874 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3875 nop 3876 3877tsb_ism_32M: 3878 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3879 brz,pn %g5, tsb_ism_256M 3880 nop 3881 3882 /* 3883 * 32M hash. 3884 */ 3885 3886 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M, 3887 TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3888 tsb_ism_4M) 3889 /* NOT REACHED */ 3890 3891tsb_ism_32M_found: 3892 brlz,a,pt %g3, tsb_validtte 3893 rdpr %tt, %g7 3894 ba,pt %xcc, tsb_ism_4M 3895 nop 3896 3897tsb_ism_256M: 3898 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3899 brz,a,pn %g5, ptl1_panic 3900 mov PTL1_BAD_ISM, %g1 3901 3902 /* 3903 * 256M hash. 3904 */ 3905 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M, 3906 TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3907 tsb_ism_4M) 3908 3909tsb_ism_256M_found: 3910 brlz,a,pt %g3, tsb_validtte 3911 rdpr %tt, %g7 3912 3913tsb_ism_4M: 3914 /* 3915 * 4M hash. 3916 */ 3917 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M, 3918 TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3919 tsb_ism_8K) 3920 /* NOT REACHED */ 3921 3922tsb_ism_4M_found: 3923 brlz,a,pt %g3, tsb_validtte 3924 rdpr %tt, %g7 3925 3926tsb_ism_8K: 3927 /* 3928 * 8K and 64K hash. 3929 */ 3930 3931 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K, 3932 TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3933 tsb_pagefault) 3934 /* NOT REACHED */ 3935 3936tsb_ism_8K_found: 3937 brlz,a,pt %g3, tsb_validtte 3938 rdpr %tt, %g7 3939 3940tsb_pagefault: 3941 rdpr %tt, %g7 3942 cmp %g7, FAST_PROT_TT 3943 be,a,pn %icc, tsb_protfault 3944 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3945 3946tsb_protfault: 3947 /* 3948 * we get here if we couldn't find a valid tte in the hash. 3949 * 3950 * If user and we are at tl>1 we go to window handling code. 3951 * 3952 * If kernel and the fault is on the same page as our stack 3953 * pointer, then we know the stack is bad and the trap handler 3954 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3955 * 3956 * If this is a kernel trap and tl>1, panic. 3957 * 3958 * Otherwise we call pagefault. 3959 */ 3960 cmp %g7, FAST_IMMU_MISS_TT 3961#ifdef sun4v 3962 MMU_FAULT_STATUS_AREA(%g4) 3963 ldx [%g4 + MMFSA_I_CTX], %g5 3964 ldx [%g4 + MMFSA_D_CTX], %g4 3965 move %icc, %g5, %g4 3966 cmp %g7, T_INSTR_MMU_MISS 3967 move %icc, %g5, %g4 3968#else 3969 mov MMU_TAG_ACCESS, %g4 3970 ldxa [%g4]ASI_DMMU, %g2 3971 ldxa [%g4]ASI_IMMU, %g5 3972 move %icc, %g5, %g2 3973 cmp %g7, T_INSTR_MMU_MISS 3974 move %icc, %g5, %g2 3975 sllx %g2, TAGACC_CTX_LSHIFT, %g4 3976#endif /* sun4v */ 3977 brnz,pn %g4, 3f /* skip if not kernel */ 3978 rdpr %tl, %g5 3979 3980 add %sp, STACK_BIAS, %g3 3981 srlx %g3, MMU_PAGESHIFT, %g3 3982 srlx %g2, MMU_PAGESHIFT, %g4 3983 cmp %g3, %g4 3984 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 3985 mov PTL1_BAD_STACK, %g1 3986 3987 cmp %g5, 1 3988 ble,pt %icc, 2f 3989 nop 3990 TSTAT_CHECK_TL1(2f, %g1, %g2) 3991 rdpr %tt, %g2 3992 cmp %g2, FAST_PROT_TT 3993 mov PTL1_BAD_KPROT_FAULT, %g1 3994 movne %icc, PTL1_BAD_KMISS, %g1 3995 ba,pt %icc, ptl1_panic 3996 nop 3997 39982: 3999 /* 4000 * We are taking a pagefault in the kernel on a kernel address. If 4001 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 4002 * want to call sfmmu_pagefault -- we will instead note that a fault 4003 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 4004 * (instead of a "retry"). This will step over the faulting 4005 * instruction. 4006 */ 4007 CPU_INDEX(%g1, %g2) 4008 set cpu_core, %g2 4009 sllx %g1, CPU_CORE_SHIFT, %g1 4010 add %g1, %g2, %g1 4011 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 4012 andcc %g2, CPU_DTRACE_NOFAULT, %g0 4013 bz sfmmu_pagefault 4014 or %g2, CPU_DTRACE_BADADDR, %g2 4015 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 4016 GET_MMU_D_ADDR(%g3, %g4) 4017 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 4018 done 4019 40203: 4021 cmp %g5, 1 4022 ble,pt %icc, 4f 4023 nop 4024 TSTAT_CHECK_TL1(4f, %g1, %g2) 4025 ba,pt %icc, sfmmu_window_trap 4026 nop 4027 40284: 4029 /* 4030 * We are taking a pagefault on a non-kernel address. If we are in 4031 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 4032 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 4033 */ 4034 CPU_INDEX(%g1, %g2) 4035 set cpu_core, %g2 4036 sllx %g1, CPU_CORE_SHIFT, %g1 4037 add %g1, %g2, %g1 4038 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 4039 andcc %g2, CPU_DTRACE_NOFAULT, %g0 4040 bz sfmmu_mmu_trap 4041 or %g2, CPU_DTRACE_BADADDR, %g2 4042 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 4043 GET_MMU_D_ADDR(%g3, %g4) 4044 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 4045 4046 /* 4047 * Be sure that we're actually taking this miss from the kernel -- 4048 * otherwise we have managed to return to user-level with 4049 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 4050 */ 4051 rdpr %tstate, %g2 4052 btst TSTATE_PRIV, %g2 4053 bz,a ptl1_panic 4054 mov PTL1_BAD_DTRACE_FLAGS, %g1 4055 done 4056 4057 ALTENTRY(tsb_tl0_noctxt) 4058 /* 4059 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 4060 * if it is, indicated that we have faulted and issue a done. 4061 */ 4062 CPU_INDEX(%g5, %g6) 4063 set cpu_core, %g6 4064 sllx %g5, CPU_CORE_SHIFT, %g5 4065 add %g5, %g6, %g5 4066 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 4067 andcc %g6, CPU_DTRACE_NOFAULT, %g0 4068 bz 1f 4069 or %g6, CPU_DTRACE_BADADDR, %g6 4070 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 4071 GET_MMU_D_ADDR(%g3, %g4) 4072 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 4073 4074 /* 4075 * Be sure that we're actually taking this miss from the kernel -- 4076 * otherwise we have managed to return to user-level with 4077 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 4078 */ 4079 rdpr %tstate, %g5 4080 btst TSTATE_PRIV, %g5 4081 bz,a ptl1_panic 4082 mov PTL1_BAD_DTRACE_FLAGS, %g1 4083 TSTAT_CHECK_TL1(2f, %g1, %g2); 40842: 4085 done 4086 40871: 4088 rdpr %tt, %g5 4089 cmp %g5, FAST_IMMU_MISS_TT 4090#ifdef sun4v 4091 MMU_FAULT_STATUS_AREA(%g2) 4092 be,a,pt %icc, 2f 4093 ldx [%g2 + MMFSA_I_CTX], %g3 4094 cmp %g5, T_INSTR_MMU_MISS 4095 be,a,pt %icc, 2f 4096 ldx [%g2 + MMFSA_I_CTX], %g3 4097 ldx [%g2 + MMFSA_D_CTX], %g3 40982: 4099#else 4100 mov MMU_TAG_ACCESS, %g2 4101 be,a,pt %icc, 2f 4102 ldxa [%g2]ASI_IMMU, %g3 4103 ldxa [%g2]ASI_DMMU, %g3 41042: sllx %g3, TAGACC_CTX_LSHIFT, %g3 4105#endif /* sun4v */ 4106 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 4107 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 4108 rdpr %tl, %g5 4109 cmp %g5, 1 4110 ble,pt %icc, sfmmu_mmu_trap 4111 nop 4112 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 4113 ba,pt %icc, sfmmu_window_trap 4114 nop 4115 SET_SIZE(sfmmu_tsb_miss) 4116#endif /* lint */ 4117 4118#if defined (lint) 4119/* 4120 * This routine will look for a user or kernel vaddr in the hash 4121 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 4122 * grab any locks. It should only be used by other sfmmu routines. 4123 */ 4124/* ARGSUSED */ 4125pfn_t 4126sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 4127{ 4128 return(0); 4129} 4130 4131/* ARGSUSED */ 4132pfn_t 4133sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno) 4134{ 4135 return(0); 4136} 4137 4138#else /* lint */ 4139 4140 ENTRY_NP(sfmmu_vatopfn) 4141 /* 4142 * disable interrupts 4143 */ 4144 rdpr %pstate, %o3 4145#ifdef DEBUG 4146 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 4147#endif 4148 /* 4149 * disable interrupts to protect the TSBMISS area 4150 */ 4151 andn %o3, PSTATE_IE, %o5 4152 wrpr %o5, 0, %pstate 4153 4154 /* 4155 * o0 = vaddr 4156 * o1 = sfmmup 4157 * o2 = ttep 4158 */ 4159 CPU_TSBMISS_AREA(%g1, %o5) 4160 ldn [%g1 + TSBMISS_KHATID], %o4 4161 cmp %o4, %o1 4162 bne,pn %ncc, vatopfn_nokernel 4163 mov TTE64K, %g5 /* g5 = rehash # */ 4164 mov %g1,%o5 /* o5 = tsbmiss_area */ 4165 /* 4166 * o0 = vaddr 4167 * o1 & o4 = hatid 4168 * o2 = ttep 4169 * o5 = tsbmiss area 4170 */ 4171 mov HBLK_RANGE_SHIFT, %g6 41721: 4173 4174 /* 4175 * o0 = vaddr 4176 * o1 = sfmmup 4177 * o2 = ttep 4178 * o3 = old %pstate 4179 * o4 = hatid 4180 * o5 = tsbmiss 4181 * g5 = rehash # 4182 * g6 = hmeshift 4183 * 4184 * The first arg to GET_TTE is actually tagaccess register 4185 * not just vaddr. Since this call is for kernel we need to clear 4186 * any lower vaddr bits that would be interpreted as ctx bits. 4187 */ 4188 set TAGACC_CTX_MASK, %g1 4189 andn %o0, %g1, %o0 4190 GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5, 4191 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 4192 4193kvtop_hblk_found: 4194 /* 4195 * o0 = vaddr 4196 * o1 = sfmmup 4197 * o2 = ttep 4198 * g1 = tte 4199 * g2 = tte pa 4200 * g3 = tte va 4201 * o2 = tsbmiss area 4202 * o1 = hat id 4203 */ 4204 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 4205 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4206 stx %g1,[%o2] /* put tte into *ttep */ 4207 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 4208 /* 4209 * o0 = vaddr 4210 * o1 = sfmmup 4211 * o2 = ttep 4212 * g1 = pfn 4213 */ 4214 ba,pt %xcc, 6f 4215 mov %g1, %o0 4216 4217kvtop_nohblk: 4218 /* 4219 * we get here if we couldn't find valid hblk in hash. We rehash 4220 * if neccesary. 4221 */ 4222 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 4223#ifdef sun4v 4224 cmp %g5, MAX_HASHCNT 4225#else 4226 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 4227#endif /* sun4v */ 4228 be,a,pn %icc, 6f 4229 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4230 mov %o1, %o4 /* restore hatid */ 4231#ifdef sun4v 4232 add %g5, 2, %g5 4233 cmp %g5, 3 4234 move %icc, MMU_PAGESHIFT4M, %g6 4235 ba,pt %icc, 1b 4236 movne %icc, MMU_PAGESHIFT256M, %g6 4237#else 4238 inc %g5 4239 cmp %g5, 2 4240 move %icc, MMU_PAGESHIFT512K, %g6 4241 ba,pt %icc, 1b 4242 movne %icc, MMU_PAGESHIFT4M, %g6 4243#endif /* sun4v */ 42446: 4245 retl 4246 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4247 4248tsb_suspend: 4249 /* 4250 * o0 = vaddr 4251 * o1 = sfmmup 4252 * o2 = ttep 4253 * g1 = tte 4254 * g2 = tte pa 4255 * g3 = tte va 4256 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 4257 */ 4258 stx %g1,[%o2] /* put tte into *ttep */ 4259 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 4260 sub %g0, 1, %o0 /* output = PFN_INVALID */ 4261 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 42628: 4263 retl 4264 wrpr %g0, %o3, %pstate /* enable interrupts */ 4265 4266vatopfn_nokernel: 4267 /* 4268 * This routine does NOT support user addresses 4269 * There is a routine in C that supports this. 4270 * The only reason why we don't have the C routine 4271 * support kernel addresses as well is because 4272 * we do va_to_pa while holding the hashlock. 4273 */ 4274 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4275 save %sp, -SA(MINFRAME), %sp 4276 sethi %hi(sfmmu_panic3), %o0 4277 call panic 4278 or %o0, %lo(sfmmu_panic3), %o0 4279 4280 SET_SIZE(sfmmu_vatopfn) 4281 4282 /* 4283 * %o0 = vaddr 4284 * %o1 = hashno (aka szc) 4285 * 4286 * 4287 * This routine is similar to sfmmu_vatopfn() but will only look for 4288 * a kernel vaddr in the hash structure for the specified rehash value. 4289 * It's just an optimization for the case when pagesize for a given 4290 * va range is already known (e.g. large page heap) and we don't want 4291 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 4292 * 4293 * Returns valid pfn or PFN_INVALID if 4294 * tte for specified rehash # is not found, invalid or suspended. 4295 */ 4296 ENTRY_NP(sfmmu_kvaszc2pfn) 4297 /* 4298 * disable interrupts 4299 */ 4300 rdpr %pstate, %o3 4301#ifdef DEBUG 4302 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 4303#endif 4304 /* 4305 * disable interrupts to protect the TSBMISS area 4306 */ 4307 andn %o3, PSTATE_IE, %o5 4308 wrpr %o5, 0, %pstate 4309 4310 CPU_TSBMISS_AREA(%g1, %o5) 4311 ldn [%g1 + TSBMISS_KHATID], %o4 4312 sll %o1, 1, %g6 4313 add %g6, %o1, %g6 4314 add %g6, MMU_PAGESHIFT, %g6 4315 /* 4316 * %o0 = vaddr 4317 * %o1 = hashno 4318 * %o3 = old %pstate 4319 * %o4 = ksfmmup 4320 * %g1 = tsbmiss area 4321 * %g6 = hmeshift 4322 */ 4323 4324 /* 4325 * The first arg to GET_TTE is actually tagaccess register 4326 * not just vaddr. Since this call is for kernel we need to clear 4327 * any lower vaddr bits that would be interpreted as ctx bits. 4328 */ 4329 srlx %o0, MMU_PAGESHIFT, %o0 4330 sllx %o0, MMU_PAGESHIFT, %o0 4331 GET_TTE(%o0, %o4, %g3, %g4, %g5, %g1, %o5, %g6, %o1, 4332 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 4333 kvaszc2pfn_nohblk) 4334 4335kvaszc2pfn_hblk_found: 4336 /* 4337 * %g3 = tte 4338 * %o0 = vaddr 4339 */ 4340 brgez,a,pn %g3, 1f /* check if tte is invalid */ 4341 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4342 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 4343 /* 4344 * g3 = pfn 4345 */ 4346 ba,pt %xcc, 1f 4347 mov %g3, %o0 4348 4349kvaszc2pfn_nohblk: 4350 mov -1, %o0 4351 43521: 4353 retl 4354 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4355 4356 SET_SIZE(sfmmu_kvaszc2pfn) 4357 4358#endif /* lint */ 4359 4360 4361 4362#if !defined(lint) 4363 4364/* 4365 * kpm lock used between trap level tsbmiss handler and kpm C level. 4366 */ 4367#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 4368 mov 0xff, tmp1 ;\ 4369label1: ;\ 4370 casa [kpmlckp]asi, %g0, tmp1 ;\ 4371 brnz,pn tmp1, label1 ;\ 4372 mov 0xff, tmp1 ;\ 4373 membar #LoadLoad 4374 4375#define KPMLOCK_EXIT(kpmlckp, asi) \ 4376 membar #LoadStore|#StoreStore ;\ 4377 sta %g0, [kpmlckp]asi 4378 4379/* 4380 * Lookup a memseg for a given pfn and if found, return the physical 4381 * address of the corresponding struct memseg in mseg, otherwise 4382 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 4383 * tsbmp, %asi is assumed to be ASI_MEM. 4384 * This lookup is done by strictly traversing only the physical memseg 4385 * linkage. The more generic approach, to check the virtual linkage 4386 * before using the physical (used e.g. with hmehash buckets), cannot 4387 * be used here. Memory DR operations can run in parallel to this 4388 * lookup w/o any locks and updates of the physical and virtual linkage 4389 * cannot be done atomically wrt. to each other. Because physical 4390 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 4391 * as "physical NULL" pointer. 4392 */ 4393#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 4394 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 4395 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 4396 udivx pfn, mseg, mseg ;\ 4397 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 4398 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 4399 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 4400 add tmp1, mseg, tmp1 ;\ 4401 ldxa [tmp1]%asi, mseg ;\ 4402 cmp mseg, MSEG_NULLPTR_PA ;\ 4403 be,pn %xcc, label/**/1 /* if not found */ ;\ 4404 nop ;\ 4405 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4406 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4407 blu,pn %xcc, label/**/1 ;\ 4408 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4409 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4410 bgeu,pn %xcc, label/**/1 ;\ 4411 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 4412 mulx tmp1, PAGE_SIZE, tmp1 ;\ 4413 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 4414 add tmp2, tmp1, tmp1 /* pp */ ;\ 4415 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 4416 cmp tmp2, pfn ;\ 4417 be,pt %xcc, label/**/_ok /* found */ ;\ 4418label/**/1: ;\ 4419 /* brute force lookup */ ;\ 4420 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 4421 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 4422label/**/2: ;\ 4423 cmp mseg, MSEG_NULLPTR_PA ;\ 4424 be,pn %xcc, label/**/_ok /* if not found */ ;\ 4425 nop ;\ 4426 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4427 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4428 blu,a,pt %xcc, label/**/2 ;\ 4429 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4430 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4431 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4432 bgeu,a,pt %xcc, label/**/2 ;\ 4433 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4434label/**/_ok: 4435 4436 /* 4437 * kpm tsb miss handler large pages 4438 * g1 = 8K kpm TSB entry pointer 4439 * g2 = tag access register 4440 * g3 = 4M kpm TSB entry pointer 4441 */ 4442 ALTENTRY(sfmmu_kpm_dtsb_miss) 4443 TT_TRACE(trace_tsbmiss) 4444 4445 CPU_INDEX(%g7, %g6) 4446 sethi %hi(kpmtsbm_area), %g6 4447 sllx %g7, KPMTSBM_SHIFT, %g7 4448 or %g6, %lo(kpmtsbm_area), %g6 4449 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4450 4451 /* check enable flag */ 4452 ldub [%g6 + KPMTSBM_FLAGS], %g4 4453 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4454 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4455 nop 4456 4457 /* VA range check */ 4458 ldx [%g6 + KPMTSBM_VBASE], %g7 4459 cmp %g2, %g7 4460 blu,pn %xcc, sfmmu_tsb_miss 4461 ldx [%g6 + KPMTSBM_VEND], %g5 4462 cmp %g2, %g5 4463 bgeu,pn %xcc, sfmmu_tsb_miss 4464 stx %g3, [%g6 + KPMTSBM_TSBPTR] 4465 4466 /* 4467 * check TL tsbmiss handling flag 4468 * bump tsbmiss counter 4469 */ 4470 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4471#ifdef DEBUG 4472 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4473 inc %g5 4474 brz,pn %g3, sfmmu_kpm_exception 4475 st %g5, [%g6 + KPMTSBM_TSBMISS] 4476#else 4477 inc %g5 4478 st %g5, [%g6 + KPMTSBM_TSBMISS] 4479#endif 4480 /* 4481 * At this point: 4482 * g1 = 8K kpm TSB pointer (not used) 4483 * g2 = tag access register 4484 * g3 = clobbered 4485 * g6 = per-CPU kpm tsbmiss area 4486 * g7 = kpm_vbase 4487 */ 4488 4489 /* vaddr2pfn */ 4490 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4491 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4492 srax %g4, %g3, %g2 /* which alias range (r) */ 4493 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4494 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4495 4496 /* 4497 * Setup %asi 4498 * mseg_pa = page_numtomemseg_nolock(pfn) 4499 * if (mseg_pa == NULL) sfmmu_kpm_exception 4500 * g2=pfn 4501 */ 4502 mov ASI_MEM, %asi 4503 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4504 cmp %g3, MSEG_NULLPTR_PA 4505 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4506 nop 4507 4508 /* 4509 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4510 * g2=pfn g3=mseg_pa 4511 */ 4512 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4513 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4514 srlx %g2, %g5, %g4 4515 sllx %g4, %g5, %g4 4516 sub %g4, %g7, %g4 4517 srlx %g4, %g5, %g4 4518 4519 /* 4520 * Validate inx value 4521 * g2=pfn g3=mseg_pa g4=inx 4522 */ 4523#ifdef DEBUG 4524 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4525 cmp %g4, %g5 /* inx - nkpmpgs */ 4526 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4527 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4528#else 4529 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4530#endif 4531 /* 4532 * kp = &mseg_pa->kpm_pages[inx] 4533 */ 4534 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4535 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4536 add %g5, %g4, %g5 /* kp */ 4537 4538 /* 4539 * KPMP_HASH(kp) 4540 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4541 */ 4542 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4543 sub %g7, 1, %g7 /* mask */ 4544 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4545 add %g5, %g1, %g5 /* y = ksp + x */ 4546 and %g5, %g7, %g5 /* hashinx = y & mask */ 4547 4548 /* 4549 * Calculate physical kpm_page pointer 4550 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4551 */ 4552 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4553 add %g1, %g4, %g1 /* kp_pa */ 4554 4555 /* 4556 * Calculate physical hash lock address 4557 * g1=kp_refcntc_pa g2=pfn g5=hashinx 4558 */ 4559 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4560 sllx %g5, KPMHLK_SHIFT, %g5 4561 add %g4, %g5, %g3 4562 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4563 4564 /* 4565 * Assemble tte 4566 * g1=kp_pa g2=pfn g3=hlck_pa 4567 */ 4568#ifdef sun4v 4569 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4570 sllx %g5, 32, %g5 4571 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4572 or %g4, TTE4M, %g4 4573 or %g5, %g4, %g5 4574#else 4575 sethi %hi(TTE_VALID_INT), %g4 4576 mov TTE4M, %g5 4577 sllx %g5, TTE_SZ_SHFT_INT, %g5 4578 or %g5, %g4, %g5 /* upper part */ 4579 sllx %g5, 32, %g5 4580 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4581 or %g5, %g4, %g5 4582#endif 4583 sllx %g2, MMU_PAGESHIFT, %g4 4584 or %g5, %g4, %g5 /* tte */ 4585 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4586 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4587 4588 /* 4589 * tsb dropin 4590 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4591 */ 4592 4593 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4594 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4595 4596 /* use C-handler if there's no go for dropin */ 4597 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4598 cmp %g7, -1 4599 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4600 nop 4601 4602#ifdef DEBUG 4603 /* double check refcnt */ 4604 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4605 brz,pn %g7, 5f /* let C-handler deal with this */ 4606 nop 4607#endif 4608 4609#ifndef sun4v 4610 ldub [%g6 + KPMTSBM_FLAGS], %g7 4611 mov ASI_N, %g1 4612 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4613 movnz %icc, ASI_MEM, %g1 4614 mov %g1, %asi 4615#endif 4616 4617 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4618 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4619 4620 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4621 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4622 4623 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4624 4625 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4626 KPMLOCK_EXIT(%g3, ASI_MEM) 4627 4628 /* 4629 * If trapstat is running, we need to shift the %tpc and %tnpc to 4630 * point to trapstat's TSB miss return code (note that trapstat 4631 * itself will patch the correct offset to add). 4632 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4633 */ 4634 rdpr %tl, %g7 4635 cmp %g7, 1 4636 ble %icc, 0f 4637 sethi %hi(KERNELBASE), %g6 4638 rdpr %tpc, %g7 4639 or %g6, %lo(KERNELBASE), %g6 4640 cmp %g7, %g6 4641 bgeu %xcc, 0f 4642 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4643 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4644 wrpr %g7, %tpc 4645 add %g7, 4, %g7 4646 wrpr %g7, %tnpc 46470: 4648 retry 46495: 4650 /* g3=hlck_pa */ 4651 KPMLOCK_EXIT(%g3, ASI_MEM) 4652 ba,pt %icc, sfmmu_kpm_exception 4653 nop 4654 SET_SIZE(sfmmu_kpm_dtsb_miss) 4655 4656 /* 4657 * kpm tsbmiss handler for smallpages 4658 * g1 = 8K kpm TSB pointer 4659 * g2 = tag access register 4660 * g3 = 4M kpm TSB pointer 4661 */ 4662 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4663 TT_TRACE(trace_tsbmiss) 4664 CPU_INDEX(%g7, %g6) 4665 sethi %hi(kpmtsbm_area), %g6 4666 sllx %g7, KPMTSBM_SHIFT, %g7 4667 or %g6, %lo(kpmtsbm_area), %g6 4668 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4669 4670 /* check enable flag */ 4671 ldub [%g6 + KPMTSBM_FLAGS], %g4 4672 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4673 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4674 nop 4675 4676 /* 4677 * VA range check 4678 * On fail: goto sfmmu_tsb_miss 4679 */ 4680 ldx [%g6 + KPMTSBM_VBASE], %g7 4681 cmp %g2, %g7 4682 blu,pn %xcc, sfmmu_tsb_miss 4683 ldx [%g6 + KPMTSBM_VEND], %g5 4684 cmp %g2, %g5 4685 bgeu,pn %xcc, sfmmu_tsb_miss 4686 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4687 4688 /* 4689 * check TL tsbmiss handling flag 4690 * bump tsbmiss counter 4691 */ 4692 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4693#ifdef DEBUG 4694 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4695 inc %g5 4696 brz,pn %g1, sfmmu_kpm_exception 4697 st %g5, [%g6 + KPMTSBM_TSBMISS] 4698#else 4699 inc %g5 4700 st %g5, [%g6 + KPMTSBM_TSBMISS] 4701#endif 4702 /* 4703 * At this point: 4704 * g1 = clobbered 4705 * g2 = tag access register 4706 * g3 = 4M kpm TSB pointer (not used) 4707 * g6 = per-CPU kpm tsbmiss area 4708 * g7 = kpm_vbase 4709 */ 4710 4711 /* vaddr2pfn */ 4712 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4713 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4714 srax %g4, %g3, %g2 /* which alias range (r) */ 4715 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4716 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4717 4718 /* 4719 * Setup %asi 4720 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4721 * if (mseg not found) sfmmu_kpm_exception 4722 * g2=pfn 4723 */ 4724 mov ASI_MEM, %asi 4725 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4726 cmp %g3, MSEG_NULLPTR_PA 4727 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4728 nop 4729 4730 /* 4731 * inx = pfn - mseg_pa->kpm_pbase 4732 * g2=pfn g3=mseg_pa 4733 */ 4734 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4735 sub %g2, %g7, %g4 4736 4737#ifdef DEBUG 4738 /* 4739 * Validate inx value 4740 * g2=pfn g3=mseg_pa g4=inx 4741 */ 4742 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4743 cmp %g4, %g5 /* inx - nkpmpgs */ 4744 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4745 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4746#else 4747 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4748#endif 4749 /* ksp = &mseg_pa->kpm_spages[inx] */ 4750 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4751 add %g5, %g4, %g5 /* ksp */ 4752 4753 /* 4754 * KPMP_SHASH(kp) 4755 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz 4756 */ 4757 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4758 sub %g7, 1, %g7 /* mask */ 4759 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4760 add %g5, %g1, %g5 /* y = ksp + x */ 4761 and %g5, %g7, %g5 /* hashinx = y & mask */ 4762 4763 /* 4764 * Calculate physical kpm_spage pointer 4765 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4766 */ 4767 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4768 add %g1, %g4, %g1 /* ksp_pa */ 4769 4770 /* 4771 * Calculate physical hash lock address. 4772 * Note: Changes in kpm_shlk_t must be reflected here. 4773 * g1=ksp_pa g2=pfn g5=hashinx 4774 */ 4775 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4776 sllx %g5, KPMSHLK_SHIFT, %g5 4777 add %g4, %g5, %g3 /* hlck_pa */ 4778 4779 /* 4780 * Assemble tte 4781 * g1=ksp_pa g2=pfn g3=hlck_pa 4782 */ 4783 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4784 sllx %g5, 32, %g5 4785 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4786 or %g5, %g4, %g5 4787 sllx %g2, MMU_PAGESHIFT, %g4 4788 or %g5, %g4, %g5 /* tte */ 4789 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4790 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4791 4792 /* 4793 * tsb dropin 4794 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte 4795 */ 4796 4797 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4798 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4799 4800 /* use C-handler if there's no go for dropin */ 4801 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4802 cmp %g7, -1 4803 bne,pn %xcc, 5f 4804 nop 4805 4806#ifndef sun4v 4807 ldub [%g6 + KPMTSBM_FLAGS], %g7 4808 mov ASI_N, %g1 4809 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4810 movnz %icc, ASI_MEM, %g1 4811 mov %g1, %asi 4812#endif 4813 4814 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 4815 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 4816 4817 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4818 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4819 4820 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4821 4822 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4823 KPMLOCK_EXIT(%g3, ASI_MEM) 4824 4825 /* 4826 * If trapstat is running, we need to shift the %tpc and %tnpc to 4827 * point to trapstat's TSB miss return code (note that trapstat 4828 * itself will patch the correct offset to add). 4829 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4830 */ 4831 rdpr %tl, %g7 4832 cmp %g7, 1 4833 ble %icc, 0f 4834 sethi %hi(KERNELBASE), %g6 4835 rdpr %tpc, %g7 4836 or %g6, %lo(KERNELBASE), %g6 4837 cmp %g7, %g6 4838 bgeu %xcc, 0f 4839 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4840 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4841 wrpr %g7, %tpc 4842 add %g7, 4, %g7 4843 wrpr %g7, %tnpc 48440: 4845 retry 48465: 4847 /* g3=hlck_pa */ 4848 KPMLOCK_EXIT(%g3, ASI_MEM) 4849 ba,pt %icc, sfmmu_kpm_exception 4850 nop 4851 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4852 4853#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4854#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4855#endif 4856 4857#endif /* lint */ 4858 4859#ifdef lint 4860/* 4861 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4862 * Called from C-level, sets/clears "go" indication for trap level handler. 4863 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 4864 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 4865 * Assumes khl_mutex is held when called from C-level. 4866 */ 4867/* ARGSUSED */ 4868void 4869sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 4870{ 4871} 4872 4873/* 4874 * kpm_smallpages: stores val to byte at address mapped within 4875 * low level lock brackets. The old value is returned. 4876 * Called from C-level. 4877 */ 4878/* ARGSUSED */ 4879int 4880sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val) 4881{ 4882 return (0); 4883} 4884 4885#else /* lint */ 4886 4887 .seg ".data" 4888sfmmu_kpm_tsbmtl_panic: 4889 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4890 .byte 0 4891sfmmu_kpm_stsbmtl_panic: 4892 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4893 .byte 0 4894 .align 4 4895 .seg ".text" 4896 4897 ENTRY_NP(sfmmu_kpm_tsbmtl) 4898 rdpr %pstate, %o3 4899 /* 4900 * %o0 = &kp_refcntc 4901 * %o1 = &khl_lock 4902 * %o2 = 0/1 (off/on) 4903 * %o3 = pstate save 4904 */ 4905#ifdef DEBUG 4906 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4907 bnz,pt %icc, 1f /* disabled, panic */ 4908 nop 4909 save %sp, -SA(MINFRAME), %sp 4910 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4911 call panic 4912 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4913 ret 4914 restore 49151: 4916#endif /* DEBUG */ 4917 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4918 4919 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4920 mov -1, %o5 4921 brz,a %o2, 2f 4922 mov 0, %o5 49232: 4924 sth %o5, [%o0] 4925 KPMLOCK_EXIT(%o1, ASI_N) 4926 4927 retl 4928 wrpr %g0, %o3, %pstate /* enable interrupts */ 4929 SET_SIZE(sfmmu_kpm_tsbmtl) 4930 4931 ENTRY_NP(sfmmu_kpm_stsbmtl) 4932 rdpr %pstate, %o3 4933 /* 4934 * %o0 = &mapped 4935 * %o1 = &kshl_lock 4936 * %o2 = val 4937 * %o3 = pstate save 4938 */ 4939#ifdef DEBUG 4940 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4941 bnz,pt %icc, 1f /* disabled, panic */ 4942 nop 4943 save %sp, -SA(MINFRAME), %sp 4944 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4945 call panic 4946 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4947 ret 4948 restore 49491: 4950#endif /* DEBUG */ 4951 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4952 4953 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4954 ldsb [%o0], %o5 4955 stb %o2, [%o0] 4956 KPMLOCK_EXIT(%o1, ASI_N) 4957 4958 mov %o5, %o0 /* return old val */ 4959 retl 4960 wrpr %g0, %o3, %pstate /* enable interrupts */ 4961 SET_SIZE(sfmmu_kpm_stsbmtl) 4962 4963#endif /* lint */ 4964 4965#ifndef lint 4966#ifdef sun4v 4967 /* 4968 * User/kernel data miss w// multiple TSBs 4969 * The first probe covers 8K, 64K, and 512K page sizes, 4970 * because 64K and 512K mappings are replicated off 8K 4971 * pointer. Second probe covers 4M page size only. 4972 * 4973 * MMU fault area contains miss address and context. 4974 */ 4975 ALTENTRY(sfmmu_slow_dmmu_miss) 4976 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 4977 4978slow_miss_common: 4979 /* 4980 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 4981 * %g3 = ctx (cannot be INVALID_CONTEXT) 4982 */ 4983 brnz,pt %g3, 8f ! check for user context 4984 nop 4985 4986 /* 4987 * Kernel miss 4988 * Get 8K and 4M TSB pointers in %g1 and %g3 and 4989 * branch to sfmmu_tsb_miss_tt to handle it. 4990 */ 4991 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4992sfmmu_dslow_patch_ktsb_base: 4993 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 4994sfmmu_dslow_patch_ktsb_szcode: 4995 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 4996 4997 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 4998 ! %g1 = First TSB entry pointer, as TSB miss handler expects 4999 5000 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 5001sfmmu_dslow_patch_ktsb4m_base: 5002 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 5003sfmmu_dslow_patch_ktsb4m_szcode: 5004 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 5005 5006 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 5007 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 5008 ba,a,pt %xcc, sfmmu_tsb_miss_tt 5009 .empty 5010 50118: 5012 /* 5013 * User miss 5014 * Get first TSB pointer in %g1 5015 * Get second TSB pointer (or NULL if no second TSB) in %g3 5016 * Branch to sfmmu_tsb_miss_tt to handle it 5017 */ 5018 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 5019 /* %g1 = first TSB entry ptr now, %g2 preserved */ 5020 5021 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 5022 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 5023 nop 5024 5025 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 5026 /* %g3 = second TSB entry ptr now, %g2 preserved */ 50279: 5028 ba,a,pt %xcc, sfmmu_tsb_miss_tt 5029 .empty 5030 SET_SIZE(sfmmu_slow_dmmu_miss) 5031 5032 5033 /* 5034 * User/kernel instruction miss w/ multiple TSBs 5035 * The first probe covers 8K, 64K, and 512K page sizes, 5036 * because 64K and 512K mappings are replicated off 8K 5037 * pointer. Second probe covers 4M page size only. 5038 * 5039 * MMU fault area contains miss address and context. 5040 */ 5041 ALTENTRY(sfmmu_slow_immu_miss) 5042 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 5043 ba,a,pt %xcc, slow_miss_common 5044 SET_SIZE(sfmmu_slow_immu_miss) 5045 5046#endif /* sun4v */ 5047#endif /* lint */ 5048 5049#ifndef lint 5050 5051/* 5052 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 5053 */ 5054 .seg ".data" 5055 .align 64 5056 .global tsbmiss_area 5057tsbmiss_area: 5058 .skip (TSBMISS_SIZE * NCPU) 5059 5060 .align 64 5061 .global kpmtsbm_area 5062kpmtsbm_area: 5063 .skip (KPMTSBM_SIZE * NCPU) 5064#endif /* lint */ 5065