1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26/* 27 * SFMMU primitives. These primitives should only be used by sfmmu 28 * routines. 29 */ 30 31#if defined(lint) 32#include <sys/types.h> 33#else /* lint */ 34#include "assym.h" 35#endif /* lint */ 36 37#include <sys/asm_linkage.h> 38#include <sys/machtrap.h> 39#include <sys/machasi.h> 40#include <sys/sun4asi.h> 41#include <sys/pte.h> 42#include <sys/mmu.h> 43#include <vm/hat_sfmmu.h> 44#include <vm/seg_spt.h> 45#include <sys/machparam.h> 46#include <sys/privregs.h> 47#include <sys/scb.h> 48#include <sys/intreg.h> 49#include <sys/machthread.h> 50#include <sys/intr.h> 51#include <sys/clock.h> 52#include <sys/trapstat.h> 53 54#ifdef TRAPTRACE 55#include <sys/traptrace.h> 56 57/* 58 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 59 */ 60#define TT_TRACE(label) \ 61 ba label ;\ 62 rd %pc, %g7 63#else 64 65#define TT_TRACE(label) 66 67#endif /* TRAPTRACE */ 68 69#ifndef lint 70 71#if (TTE_SUSPEND_SHIFT > 0) 72#define TTE_SUSPEND_INT_SHIFT(reg) \ 73 sllx reg, TTE_SUSPEND_SHIFT, reg 74#else 75#define TTE_SUSPEND_INT_SHIFT(reg) 76#endif 77 78#endif /* lint */ 79 80#ifndef lint 81 82/* 83 * Assumes TSBE_TAG is 0 84 * Assumes TSBE_INTHI is 0 85 * Assumes TSBREG.split is 0 86 */ 87 88#if TSBE_TAG != 0 89#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 90#endif 91 92#if TSBTAG_INTHI != 0 93#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 94#endif 95 96/* 97 * The following code assumes the tsb is not split. 98 * 99 * With TSBs no longer shared between processes, it's no longer 100 * necessary to hash the context bits into the tsb index to get 101 * tsb coloring; the new implementation treats the TSB as a 102 * direct-mapped, virtually-addressed cache. 103 * 104 * In: 105 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 106 * tsbbase = base address of TSB (clobbered) 107 * tagacc = tag access register (clobbered) 108 * szc = size code of TSB (ro) 109 * tmp = scratch reg 110 * Out: 111 * tsbbase = pointer to entry in TSB 112 */ 113#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 114 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 115 srlx tagacc, vpshift, tagacc ;\ 116 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 117 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 118 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 119 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 120 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 121 122/* 123 * When the kpm TSB is used it is assumed that it is direct mapped 124 * using (vaddr>>vpshift)%tsbsz as the index. 125 * 126 * Note that, for now, the kpm TSB and kernel TSB are the same for 127 * each mapping size. However that need not always be the case. If 128 * the trap handlers are updated to search a different TSB for kpm 129 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 130 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 131 * 132 * In: 133 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 134 * vaddr = virtual address (clobbered) 135 * tsbp, szc, tmp = scratch 136 * Out: 137 * tsbp = pointer to entry in TSB 138 */ 139#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 140 cmp vpshift, MMU_PAGESHIFT ;\ 141 bne,pn %icc, 1f /* branch if large case */ ;\ 142 sethi %hi(kpmsm_tsbsz), szc ;\ 143 sethi %hi(kpmsm_tsbbase), tsbp ;\ 144 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 145 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 146 ba,pt %icc, 2f ;\ 147 nop ;\ 1481: sethi %hi(kpm_tsbsz), szc ;\ 149 sethi %hi(kpm_tsbbase), tsbp ;\ 150 ld [szc + %lo(kpm_tsbsz)], szc ;\ 151 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1522: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 153 154/* 155 * Lock the TSBE at virtual address tsbep. 156 * 157 * tsbep = TSBE va (ro) 158 * tmp1, tmp2 = scratch registers (clobbered) 159 * label = label to jump to if we fail to lock the tsb entry 160 * %asi = ASI to use for TSB access 161 * 162 * NOTE that we flush the TSB using fast VIS instructions that 163 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 164 * not be treated as a locked entry or we'll get stuck spinning on 165 * an entry that isn't locked but really invalid. 166 */ 167 168#if defined(UTSB_PHYS) 169 170#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 171 lda [tsbep]ASI_MEM, tmp1 ;\ 172 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 173 cmp tmp1, tmp2 ;\ 174 be,a,pn %icc, label /* if locked ignore */ ;\ 175 nop ;\ 176 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 177 cmp tmp1, tmp2 ;\ 178 bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 179 nop ;\ 180 /* tsbe lock acquired */ ;\ 181 membar #StoreStore 182 183#else /* UTSB_PHYS */ 184 185#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 186 lda [tsbep]%asi, tmp1 ;\ 187 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 188 cmp tmp1, tmp2 ;\ 189 be,a,pn %icc, label /* if locked ignore */ ;\ 190 nop ;\ 191 casa [tsbep]%asi, tmp1, tmp2 ;\ 192 cmp tmp1, tmp2 ;\ 193 bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 194 nop ;\ 195 /* tsbe lock acquired */ ;\ 196 membar #StoreStore 197 198#endif /* UTSB_PHYS */ 199 200/* 201 * Atomically write TSBE at virtual address tsbep. 202 * 203 * tsbep = TSBE va (ro) 204 * tte = TSBE TTE (ro) 205 * tagtarget = TSBE tag (ro) 206 * %asi = ASI to use for TSB access 207 */ 208 209#if defined(UTSB_PHYS) 210 211#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 212 add tsbep, TSBE_TTE, tmp1 ;\ 213 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 214 membar #StoreStore ;\ 215 add tsbep, TSBE_TAG, tmp1 ;\ 216 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 217 218#else /* UTSB_PHYS */ 219 220#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 221 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 222 membar #StoreStore ;\ 223 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 224 225#endif /* UTSB_PHYS */ 226 227/* 228 * Load an entry into the TSB at TL > 0. 229 * 230 * tsbep = pointer to the TSBE to load as va (ro) 231 * tte = value of the TTE retrieved and loaded (wo) 232 * tagtarget = tag target register. To get TSBE tag to load, 233 * we need to mask off the context and leave only the va (clobbered) 234 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 235 * tmp1, tmp2 = scratch registers 236 * label = label to jump to if we fail to lock the tsb entry 237 * %asi = ASI to use for TSB access 238 */ 239 240#if defined(UTSB_PHYS) 241 242#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 243 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 244 /* ;\ 245 * I don't need to update the TSB then check for the valid tte. ;\ 246 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 247 * we always invalidate the hash table before we unload the TSB.;\ 248 */ ;\ 249 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 250 ldxa [ttepa]ASI_MEM, tte ;\ 251 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 252 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 253 add tsbep, TSBE_TAG, tmp1 ;\ 254 brgez,a,pn tte, label ;\ 255 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 256 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 257label: 258 259#else /* UTSB_PHYS */ 260 261#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 262 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 263 /* ;\ 264 * I don't need to update the TSB then check for the valid tte. ;\ 265 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 266 * we always invalidate the hash table before we unload the TSB.;\ 267 */ ;\ 268 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 269 ldxa [ttepa]ASI_MEM, tte ;\ 270 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 271 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 272 brgez,a,pn tte, label ;\ 273 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 274 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 275label: 276 277#endif /* UTSB_PHYS */ 278 279/* 280 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 281 * for ITLB synthesis. 282 * 283 * tsbep = pointer to the TSBE to load as va (ro) 284 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 285 * with exec_perm turned off and exec_synth turned on 286 * tagtarget = tag target register. To get TSBE tag to load, 287 * we need to mask off the context and leave only the va (clobbered) 288 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 289 * tmp1, tmp2 = scratch registers 290 * label = label to use for branch (text) 291 * %asi = ASI to use for TSB access 292 */ 293 294#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 295 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 296 /* ;\ 297 * I don't need to update the TSB then check for the valid tte. ;\ 298 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 299 * we always invalidate the hash table before we unload the TSB.;\ 300 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 301 * and exec_synth bit to 1. ;\ 302 */ ;\ 303 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 304 mov tte, tmp1 ;\ 305 ldxa [ttepa]ASI_MEM, tte ;\ 306 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 307 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 308 brgez,a,pn tte, label ;\ 309 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 310 or tte, tmp1, tte ;\ 311 andn tte, TTE_EXECPRM_INT, tte ;\ 312 or tte, TTE_E_SYNTH_INT, tte ;\ 313 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 314label: 315 316/* 317 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 318 * 319 * tte = value of the TTE, used to get tte_size bits (ro) 320 * tagaccess = tag access register, used to get 4M pfn bits (ro) 321 * pfn = 4M pfn bits shifted to offset for tte (out) 322 * tmp1 = scratch register 323 * label = label to use for branch (text) 324 */ 325 326#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 327 /* ;\ 328 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 329 * Return them, shifted, in pfn. ;\ 330 */ ;\ 331 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 332 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 333 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 334 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 335 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 336 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 337label: ;\ 338 sllx tagaccess, MMU_PAGESHIFT4M, pfn 339 340/* 341 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 342 * for ITLB synthesis. 343 * 344 * tte = value of the TTE, used to get tte_size bits (rw) 345 * tmp1 = scratch register 346 */ 347 348#define SET_TTE4M_PN(tte, tmp) \ 349 /* ;\ 350 * Set 4M pagesize tte bits. ;\ 351 */ ;\ 352 set TTE4M, tmp ;\ 353 sllx tmp, TTE_SZ_SHFT, tmp ;\ 354 or tte, tmp, tte 355 356/* 357 * Load an entry into the TSB at TL=0. 358 * 359 * tsbep = pointer to the TSBE to load as va (ro) 360 * tteva = pointer to the TTE to load as va (ro) 361 * tagtarget = TSBE tag to load (which contains no context), synthesized 362 * to match va of MMU tag target register only (ro) 363 * tmp1, tmp2 = scratch registers (clobbered) 364 * label = label to use for branches (text) 365 * %asi = ASI to use for TSB access 366 */ 367 368#if defined(UTSB_PHYS) 369 370#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 371 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 372 ldx [tteva], tteva /* load tte */ ;\ 373 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 374 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 375 add tsbep, TSBE_TAG, tmp1 ;\ 376 brgez,a,pn tteva, label ;\ 377 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 378 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 379label: 380 381#else /* UTSB_PHYS */ 382 383#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 384 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 385 ldx [tteva], tteva /* load tte */ ;\ 386 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 387 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 388 brgez,a,pn tteva, label ;\ 389 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 390 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 391label: 392 393#endif /* UTSB_PHYS */ 394 395/* 396 * Invalidate a TSB entry in the TSB. 397 * 398 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 399 * about this earlier to ensure this is true. Thus when we are 400 * directly referencing tsbep below, we are referencing the tte_tag 401 * field of the TSBE. If this offset ever changes, the code below 402 * will need to be modified. 403 * 404 * tsbep = pointer to TSBE as va (ro) 405 * tag = invalidation is done if this matches the TSBE tag (ro) 406 * tmp1 - tmp3 = scratch registers (clobbered) 407 * label = label name to use for branches (text) 408 * %asi = ASI to use for TSB access 409 */ 410 411#if defined(UTSB_PHYS) 412 413#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 414 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 415 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 416label/**/1: ;\ 417 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 418 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 419 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 420 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 421 cmp tag, tmp3 /* compare tags */ ;\ 422 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 423 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 424 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 425 cmp tmp1, tmp3 /* if not successful */ ;\ 426 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 427 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 428label/**/2: 429 430#else /* UTSB_PHYS */ 431 432#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 433 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 434 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 435label/**/1: ;\ 436 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 437 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 438 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 439 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 440 cmp tag, tmp3 /* compare tags */ ;\ 441 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 442 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 443 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 444 cmp tmp1, tmp3 /* if not successful */ ;\ 445 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 446 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 447label/**/2: 448 449#endif /* UTSB_PHYS */ 450 451#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 452#error - TSB_SOFTSZ_MASK too small 453#endif 454 455 456/* 457 * An implementation of setx which will be hot patched at run time. 458 * since it is being hot patched, there is no value passed in. 459 * Thus, essentially we are implementing 460 * setx value, tmp, dest 461 * where value is RUNTIME_PATCH (aka 0) in this case. 462 */ 463#define RUNTIME_PATCH_SETX(dest, tmp) \ 464 sethi %hh(RUNTIME_PATCH), tmp ;\ 465 sethi %lm(RUNTIME_PATCH), dest ;\ 466 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 467 or dest, %lo(RUNTIME_PATCH), dest ;\ 468 sllx tmp, 32, tmp ;\ 469 nop /* for perf reasons */ ;\ 470 or tmp, dest, dest /* contents of patched value */ 471 472#endif /* lint */ 473 474 475#if defined (lint) 476 477/* 478 * sfmmu related subroutines 479 */ 480uint_t 481sfmmu_disable_intrs() 482{ return(0); } 483 484/* ARGSUSED */ 485void 486sfmmu_enable_intrs(uint_t pstate_save) 487{} 488 489/* ARGSUSED */ 490int 491sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag) 492{ return(0); } 493 494/* 495 * Use cas, if tte has changed underneath us then reread and try again. 496 * In the case of a retry, it will update sttep with the new original. 497 */ 498/* ARGSUSED */ 499int 500sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 501{ return(0); } 502 503/* 504 * Use cas, if tte has changed underneath us then return 1, else return 0 505 */ 506/* ARGSUSED */ 507int 508sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 509{ return(0); } 510 511/* ARGSUSED */ 512void 513sfmmu_copytte(tte_t *sttep, tte_t *dttep) 514{} 515 516/*ARGSUSED*/ 517struct tsbe * 518sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 519{ return(0); } 520 521/*ARGSUSED*/ 522uint64_t 523sfmmu_make_tsbtag(caddr_t va) 524{ return(0); } 525 526#else /* lint */ 527 528 .seg ".data" 529 .global sfmmu_panic1 530sfmmu_panic1: 531 .asciz "sfmmu_asm: interrupts already disabled" 532 533 .global sfmmu_panic3 534sfmmu_panic3: 535 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 536 537 .global sfmmu_panic4 538sfmmu_panic4: 539 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 540 541 .global sfmmu_panic5 542sfmmu_panic5: 543 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 544 545 .global sfmmu_panic6 546sfmmu_panic6: 547 .asciz "sfmmu_asm: interrupts not disabled" 548 549 .global sfmmu_panic7 550sfmmu_panic7: 551 .asciz "sfmmu_asm: kernel as" 552 553 .global sfmmu_panic8 554sfmmu_panic8: 555 .asciz "sfmmu_asm: gnum is zero" 556 557 .global sfmmu_panic9 558sfmmu_panic9: 559 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 560 561 .global sfmmu_panic10 562sfmmu_panic10: 563 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 564 565 .global sfmmu_panic11 566sfmmu_panic11: 567 .asciz "sfmmu_asm: ktsb_phys must not be 0 on a sun4v platform" 568 569 ENTRY(sfmmu_disable_intrs) 570 rdpr %pstate, %o0 571#ifdef DEBUG 572 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 573#endif /* DEBUG */ 574 retl 575 wrpr %o0, PSTATE_IE, %pstate 576 SET_SIZE(sfmmu_disable_intrs) 577 578 ENTRY(sfmmu_enable_intrs) 579 retl 580 wrpr %g0, %o0, %pstate 581 SET_SIZE(sfmmu_enable_intrs) 582 583/* 584 * This routine is called both by resume() and sfmmu_get_ctx() to 585 * allocate a new context for the process on a MMU. 586 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 587 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 588 * is the case when sfmmu_alloc_ctx is called from resume(). 589 * 590 * The caller must disable interrupts before entering this routine. 591 * To reduce ctx switch overhead, the code contains both 'fast path' and 592 * 'slow path' code. The fast path code covers the common case where only 593 * a quick check is needed and the real ctx allocation is not required. 594 * It can be done without holding the per-process (PP) lock. 595 * The 'slow path' code must be protected by the PP Lock and performs ctx 596 * allocation. 597 * Hardware context register and HAT mmu cnum are updated accordingly. 598 * 599 * %o0 - sfmmup 600 * %o1 - allocflag 601 * %o2 - CPU 602 * %o3 - sfmmu private/shared flag 603 * 604 * ret - 0: no ctx is allocated 605 * 1: a ctx is allocated 606 */ 607 ENTRY_NP(sfmmu_alloc_ctx) 608 609#ifdef DEBUG 610 sethi %hi(ksfmmup), %g1 611 ldx [%g1 + %lo(ksfmmup)], %g1 612 cmp %g1, %o0 613 bne,pt %xcc, 0f 614 nop 615 616 sethi %hi(panicstr), %g1 ! if kernel as, panic 617 ldx [%g1 + %lo(panicstr)], %g1 618 tst %g1 619 bnz,pn %icc, 7f 620 nop 621 622 sethi %hi(sfmmu_panic7), %o0 623 call panic 624 or %o0, %lo(sfmmu_panic7), %o0 625 6267: 627 retl 628 mov %g0, %o0 ! %o0 = ret = 0 629 6300: 631 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 632#endif /* DEBUG */ 633 634 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 635 636 ! load global mmu_ctxp info 637 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 638 639#ifdef sun4v 640 /* During suspend on sun4v, context domains can be temporary removed */ 641 brz,a,pn %o3, 0f 642 nop 643#endif 644 645 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 646 647 ! load global mmu_ctxp gnum 648 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 649 650#ifdef DEBUG 651 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 652 bne,pt %xcc, 3f 653 nop 654 655 sethi %hi(panicstr), %g1 ! test if panicstr is already set 656 ldx [%g1 + %lo(panicstr)], %g1 657 tst %g1 658 bnz,pn %icc, 1f 659 nop 660 661 sethi %hi(sfmmu_panic8), %o0 662 call panic 663 or %o0, %lo(sfmmu_panic8), %o0 6641: 665 retl 666 mov %g0, %o0 ! %o0 = ret = 0 6673: 668#endif 669 670 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 671 672 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 673 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 674 675 /* 676 * %g5 = sfmmu gnum returned 677 * %g6 = sfmmu cnum returned 678 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 679 * %g4 = scratch 680 * 681 * Fast path code, do a quick check. 682 */ 683 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 684 685 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 686 bne,pt %icc, 1f ! valid hat cnum, check gnum 687 nop 688 689 ! cnum == INVALID, check allocflag 690 mov %g0, %g4 ! %g4 = ret = 0 691 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 692 mov %g6, %o1 693 694 ! (invalid HAT cnum) && (allocflag == 1) 695 ba,pt %icc, 2f 696 nop 697#ifdef sun4v 6980: 699 set INVALID_CONTEXT, %o1 700 membar #LoadStore|#StoreStore 701 ba,pt %icc, 8f 702 mov %g0, %g4 ! %g4 = ret = 0 703#endif 7041: 705 ! valid HAT cnum, check gnum 706 cmp %g5, %o4 707 mov 1, %g4 !%g4 = ret = 1 708 be,a,pt %icc, 8f ! gnum unchanged, go to done 709 mov %g6, %o1 710 7112: 712 /* 713 * Grab per process (PP) sfmmu_ctx_lock spinlock, 714 * followed by the 'slow path' code. 715 */ 716 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 7173: 718 brz %g3, 5f 719 nop 7204: 721 brnz,a,pt %g3, 4b ! spin if lock is 1 722 ldub [%o0 + SFMMU_CTX_LOCK], %g3 723 ba %xcc, 3b ! retry the lock 724 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 725 7265: 727 membar #LoadLoad 728 /* 729 * %g5 = sfmmu gnum returned 730 * %g6 = sfmmu cnum returned 731 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 732 * %g4 = scratch 733 */ 734 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 735 736 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 737 bne,pt %icc, 1f ! valid hat cnum, check gnum 738 nop 739 740 ! cnum == INVALID, check allocflag 741 mov %g0, %g4 ! %g4 = ret = 0 742 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 743 mov %g6, %o1 744 745 ! (invalid HAT cnum) && (allocflag == 1) 746 ba,pt %icc, 6f 747 nop 7481: 749 ! valid HAT cnum, check gnum 750 cmp %g5, %o4 751 mov 1, %g4 ! %g4 = ret = 1 752 be,a,pt %icc, 2f ! gnum unchanged, go to done 753 mov %g6, %o1 754 755 ba,pt %icc, 6f 756 nop 7572: 758 membar #LoadStore|#StoreStore 759 ba,pt %icc, 8f 760 clrb [%o0 + SFMMU_CTX_LOCK] 7616: 762 /* 763 * We get here if we do not have a valid context, or 764 * the HAT gnum does not match global gnum. We hold 765 * sfmmu_ctx_lock spinlock. Allocate that context. 766 * 767 * %o3 = mmu_ctxp 768 */ 769 add %o3, MMU_CTX_CNUM, %g3 770 ld [%o3 + MMU_CTX_NCTXS], %g4 771 772 /* 773 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 774 * %g3 = mmu cnum address 775 * %g4 = mmu nctxs 776 * 777 * %o0 = sfmmup 778 * %o1 = mmu current cnum value (used as new cnum) 779 * %o4 = mmu gnum 780 * 781 * %o5 = scratch 782 */ 783 ld [%g3], %o1 7840: 785 cmp %o1, %g4 786 bl,a,pt %icc, 1f 787 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 788 789 /* 790 * cnum reachs max, bail, so wrap around can be performed later. 791 */ 792 set INVALID_CONTEXT, %o1 793 mov %g0, %g4 ! %g4 = ret = 0 794 795 membar #LoadStore|#StoreStore 796 ba,pt %icc, 8f 797 clrb [%o0 + SFMMU_CTX_LOCK] 7981: 799 ! %g3 = addr of mmu_ctxp->cnum 800 ! %o5 = mmu_ctxp->cnum + 1 801 cas [%g3], %o1, %o5 802 cmp %o1, %o5 803 bne,a,pn %xcc, 0b ! cas failed 804 ld [%g3], %o1 805 806#ifdef DEBUG 807 set MAX_SFMMU_CTX_VAL, %o5 808 cmp %o1, %o5 809 ble,pt %icc, 2f 810 nop 811 812 sethi %hi(sfmmu_panic9), %o0 813 call panic 814 or %o0, %lo(sfmmu_panic9), %o0 8152: 816#endif 817 ! update hat gnum and cnum 818 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 819 or %o4, %o1, %o4 820 stx %o4, [%g2 + SFMMU_CTXS] 821 822 membar #LoadStore|#StoreStore 823 clrb [%o0 + SFMMU_CTX_LOCK] 824 825 mov 1, %g4 ! %g4 = ret = 1 8268: 827 /* 828 * program the secondary context register 829 * 830 * %o1 = cnum 831 * %g1 = sfmmu private/shared flag (0:private, 1:shared) 832 */ 833 834 /* 835 * When we come here and context is invalid, we want to set both 836 * private and shared ctx regs to INVALID. In order to 837 * do so, we set the sfmmu priv/shared flag to 'private' regardless 838 * so that private ctx reg will be set to invalid. 839 * Note that on sun4v values written to private context register are 840 * automatically written to corresponding shared context register as 841 * well. On sun4u SET_SECCTX() will invalidate shared context register 842 * when it sets a private secondary context register. 843 */ 844 845 cmp %o1, INVALID_CONTEXT 846 be,a,pn %icc, 9f 847 clr %g1 8489: 849 850#ifdef sun4u 851 ldub [%o0 + SFMMU_CEXT], %o2 852 sll %o2, CTXREG_EXT_SHIFT, %o2 853 or %o1, %o2, %o1 854#endif /* sun4u */ 855 856 SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1) 857 858 retl 859 mov %g4, %o0 ! %o0 = ret 860 861 SET_SIZE(sfmmu_alloc_ctx) 862 863 ENTRY_NP(sfmmu_modifytte) 864 ldx [%o2], %g3 /* current */ 865 ldx [%o0], %g1 /* original */ 8662: 867 ldx [%o1], %g2 /* modified */ 868 cmp %g2, %g3 /* is modified = current? */ 869 be,a,pt %xcc,1f /* yes, don't write */ 870 stx %g3, [%o0] /* update new original */ 871 casx [%o2], %g1, %g2 872 cmp %g1, %g2 873 be,pt %xcc, 1f /* cas succeeded - return */ 874 nop 875 ldx [%o2], %g3 /* new current */ 876 stx %g3, [%o0] /* save as new original */ 877 ba,pt %xcc, 2b 878 mov %g3, %g1 8791: retl 880 membar #StoreLoad 881 SET_SIZE(sfmmu_modifytte) 882 883 ENTRY_NP(sfmmu_modifytte_try) 884 ldx [%o1], %g2 /* modified */ 885 ldx [%o2], %g3 /* current */ 886 ldx [%o0], %g1 /* original */ 887 cmp %g3, %g2 /* is modified = current? */ 888 be,a,pn %xcc,1f /* yes, don't write */ 889 mov 0, %o1 /* as if cas failed. */ 890 891 casx [%o2], %g1, %g2 892 membar #StoreLoad 893 cmp %g1, %g2 894 movne %xcc, -1, %o1 /* cas failed. */ 895 move %xcc, 1, %o1 /* cas succeeded. */ 8961: 897 stx %g2, [%o0] /* report "current" value */ 898 retl 899 mov %o1, %o0 900 SET_SIZE(sfmmu_modifytte_try) 901 902 ENTRY_NP(sfmmu_copytte) 903 ldx [%o0], %g1 904 retl 905 stx %g1, [%o1] 906 SET_SIZE(sfmmu_copytte) 907 908 909 /* 910 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 911 * %o0 = TSB base address (in), pointer to TSB entry (out) 912 * %o1 = vaddr (in) 913 * %o2 = vpshift (in) 914 * %o3 = tsb size code (in) 915 * %o4 = scratch register 916 */ 917 ENTRY_NP(sfmmu_get_tsbe) 918 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 919 retl 920 nop 921 SET_SIZE(sfmmu_get_tsbe) 922 923 /* 924 * Return a TSB tag for the given va. 925 * %o0 = va (in/clobbered) 926 * %o0 = va shifted to be in tsb tag format (with no context) (out) 927 */ 928 ENTRY_NP(sfmmu_make_tsbtag) 929 retl 930 srln %o0, TTARGET_VA_SHIFT, %o0 931 SET_SIZE(sfmmu_make_tsbtag) 932 933#endif /* lint */ 934 935/* 936 * Other sfmmu primitives 937 */ 938 939 940#if defined (lint) 941void 942sfmmu_patch_ktsb(void) 943{ 944} 945 946void 947sfmmu_kpm_patch_tlbm(void) 948{ 949} 950 951void 952sfmmu_kpm_patch_tsbm(void) 953{ 954} 955 956void 957sfmmu_patch_shctx(void) 958{ 959} 960 961/* ARGSUSED */ 962void 963sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 964{ 965} 966 967/* ARGSUSED */ 968void 969sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 970{ 971} 972 973/* ARGSUSED */ 974void 975sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 976{ 977} 978 979/* ARGSUSED */ 980void 981sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 982{ 983} 984 985#else /* lint */ 986 987#define I_SIZE 4 988 989 ENTRY_NP(sfmmu_fix_ktlb_traptable) 990 /* 991 * %o0 = start of patch area 992 * %o1 = size code of TSB to patch 993 * %o3 = scratch 994 */ 995 /* fix sll */ 996 ld [%o0], %o3 /* get sll */ 997 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 998 st %o3, [%o0] /* write sll */ 999 flush %o0 1000 /* fix srl */ 1001 add %o0, I_SIZE, %o0 /* goto next instr. */ 1002 ld [%o0], %o3 /* get srl */ 1003 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 1004 st %o3, [%o0] /* write srl */ 1005 retl 1006 flush %o0 1007 SET_SIZE(sfmmu_fix_ktlb_traptable) 1008 1009 ENTRY_NP(sfmmu_fixup_ktsbbase) 1010 /* 1011 * %o0 = start of patch area 1012 * %o5 = kernel virtual or physical tsb base address 1013 * %o2, %o3 are used as scratch registers. 1014 */ 1015 /* fixup sethi instruction */ 1016 ld [%o0], %o3 1017 srl %o5, 10, %o2 ! offset is bits 32:10 1018 or %o3, %o2, %o3 ! set imm22 1019 st %o3, [%o0] 1020 /* fixup offset of lduw/ldx */ 1021 add %o0, I_SIZE, %o0 ! next instr 1022 ld [%o0], %o3 1023 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 1024 or %o3, %o2, %o3 1025 st %o3, [%o0] 1026 retl 1027 flush %o0 1028 SET_SIZE(sfmmu_fixup_ktsbbase) 1029 1030 ENTRY_NP(sfmmu_fixup_setx) 1031 /* 1032 * %o0 = start of patch area 1033 * %o4 = 64 bit value to patch 1034 * %o2, %o3 are used as scratch registers. 1035 * 1036 * Note: Assuming that all parts of the instructions which need to be 1037 * patched correspond to RUNTIME_PATCH (aka 0) 1038 * 1039 * Note the implementation of setx which is being patched is as follows: 1040 * 1041 * sethi %hh(RUNTIME_PATCH), tmp 1042 * sethi %lm(RUNTIME_PATCH), dest 1043 * or tmp, %hm(RUNTIME_PATCH), tmp 1044 * or dest, %lo(RUNTIME_PATCH), dest 1045 * sllx tmp, 32, tmp 1046 * nop 1047 * or tmp, dest, dest 1048 * 1049 * which differs from the implementation in the 1050 * "SPARC Architecture Manual" 1051 */ 1052 /* fixup sethi instruction */ 1053 ld [%o0], %o3 1054 srlx %o4, 42, %o2 ! bits [63:42] 1055 or %o3, %o2, %o3 ! set imm22 1056 st %o3, [%o0] 1057 /* fixup sethi instruction */ 1058 add %o0, I_SIZE, %o0 ! next instr 1059 ld [%o0], %o3 1060 sllx %o4, 32, %o2 ! clear upper bits 1061 srlx %o2, 42, %o2 ! bits [31:10] 1062 or %o3, %o2, %o3 ! set imm22 1063 st %o3, [%o0] 1064 /* fixup or instruction */ 1065 add %o0, I_SIZE, %o0 ! next instr 1066 ld [%o0], %o3 1067 srlx %o4, 32, %o2 ! bits [63:32] 1068 and %o2, 0x3ff, %o2 ! bits [41:32] 1069 or %o3, %o2, %o3 ! set imm 1070 st %o3, [%o0] 1071 /* fixup or instruction */ 1072 add %o0, I_SIZE, %o0 ! next instr 1073 ld [%o0], %o3 1074 and %o4, 0x3ff, %o2 ! bits [9:0] 1075 or %o3, %o2, %o3 ! set imm 1076 st %o3, [%o0] 1077 retl 1078 flush %o0 1079 SET_SIZE(sfmmu_fixup_setx) 1080 1081 ENTRY_NP(sfmmu_fixup_or) 1082 /* 1083 * %o0 = start of patch area 1084 * %o4 = 32 bit value to patch 1085 * %o2, %o3 are used as scratch registers. 1086 * Note: Assuming that all parts of the instructions which need to be 1087 * patched correspond to RUNTIME_PATCH (aka 0) 1088 */ 1089 ld [%o0], %o3 1090 and %o4, 0x3ff, %o2 ! bits [9:0] 1091 or %o3, %o2, %o3 ! set imm 1092 st %o3, [%o0] 1093 retl 1094 flush %o0 1095 SET_SIZE(sfmmu_fixup_or) 1096 1097 ENTRY_NP(sfmmu_fixup_shiftx) 1098 /* 1099 * %o0 = start of patch area 1100 * %o4 = signed int immediate value to add to sllx/srlx imm field 1101 * %o2, %o3 are used as scratch registers. 1102 * 1103 * sllx/srlx store the 6 bit immediate value in the lowest order bits 1104 * so we do a simple add. The caller must be careful to prevent 1105 * overflow, which could easily occur if the initial value is nonzero! 1106 */ 1107 ld [%o0], %o3 ! %o3 = instruction to patch 1108 and %o3, 0x3f, %o2 ! %o2 = existing imm value 1109 add %o2, %o4, %o2 ! %o2 = new imm value 1110 andn %o3, 0x3f, %o3 ! clear old imm value 1111 and %o2, 0x3f, %o2 ! truncate new imm value 1112 or %o3, %o2, %o3 ! set new imm value 1113 st %o3, [%o0] ! store updated instruction 1114 retl 1115 flush %o0 1116 SET_SIZE(sfmmu_fixup_shiftx) 1117 1118 ENTRY_NP(sfmmu_fixup_mmu_asi) 1119 /* 1120 * Patch imm_asi of all ldda instructions in the MMU 1121 * trap handlers. We search MMU_PATCH_INSTR instructions 1122 * starting from the itlb miss handler (trap 0x64). 1123 * %o0 = address of tt[0,1]_itlbmiss 1124 * %o1 = imm_asi to setup, shifted by appropriate offset. 1125 * %o3 = number of instructions to search 1126 * %o4 = reserved by caller: called from leaf routine 1127 */ 11281: ldsw [%o0], %o2 ! load instruction to %o2 1129 brgez,pt %o2, 2f 1130 srl %o2, 30, %o5 1131 btst 1, %o5 ! test bit 30; skip if not set 1132 bz,pt %icc, 2f 1133 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1134 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1135 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1136 brnz,pt %o5, 2f ! skip if not a match 1137 or %o2, %o1, %o2 ! or in imm_asi 1138 st %o2, [%o0] ! write patched instruction 11392: dec %o3 1140 brnz,a,pt %o3, 1b ! loop until we're done 1141 add %o0, I_SIZE, %o0 1142 retl 1143 flush %o0 1144 SET_SIZE(sfmmu_fixup_mmu_asi) 1145 1146 /* 1147 * Patch immediate ASI used to access the TSB in the 1148 * trap table. 1149 * inputs: %o0 = value of ktsb_phys 1150 */ 1151 ENTRY_NP(sfmmu_patch_mmu_asi) 1152 mov %o7, %o4 ! save return pc in %o4 1153 mov ASI_QUAD_LDD_PHYS, %o3 ! set QUAD_LDD_PHYS by default 1154 1155#ifdef sun4v 1156 1157 /* 1158 * Check ktsb_phys. It must be non-zero for sun4v, panic if not. 1159 */ 1160 1161 brnz,pt %o0, do_patch 1162 nop 1163 1164 sethi %hi(sfmmu_panic11), %o0 1165 call panic 1166 or %o0, %lo(sfmmu_panic11), %o0 1167do_patch: 1168 1169#else /* sun4v */ 1170 /* 1171 * Some non-sun4v platforms deploy virtual ktsb (ktsb_phys==0). 1172 * Note that ASI_NQUAD_LD is not defined/used for sun4v 1173 */ 1174 movrz %o0, ASI_NQUAD_LD, %o3 1175 1176#endif /* sun4v */ 1177 1178 sll %o3, 5, %o1 ! imm_asi offset 1179 mov 6, %o3 ! number of instructions 1180 sethi %hi(dktsb), %o0 ! to search 1181 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1182 or %o0, %lo(dktsb), %o0 1183 mov 6, %o3 ! number of instructions 1184 sethi %hi(dktsb4m), %o0 ! to search 1185 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1186 or %o0, %lo(dktsb4m), %o0 1187 mov 6, %o3 ! number of instructions 1188 sethi %hi(iktsb), %o0 ! to search 1189 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1190 or %o0, %lo(iktsb), %o0 1191 mov 6, %o3 ! number of instructions 1192 sethi %hi(iktsb4m), %o0 ! to search 1193 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1194 or %o0, %lo(iktsb4m), %o0 1195 mov %o4, %o7 ! retore return pc -- leaf 1196 retl 1197 nop 1198 SET_SIZE(sfmmu_patch_mmu_asi) 1199 1200 1201 ENTRY_NP(sfmmu_patch_ktsb) 1202 /* 1203 * We need to fix iktsb, dktsb, et. al. 1204 */ 1205 save %sp, -SA(MINFRAME), %sp 1206 set ktsb_phys, %o1 1207 ld [%o1], %o4 1208 set ktsb_base, %o5 1209 set ktsb4m_base, %l1 1210 brz,pt %o4, 1f 1211 nop 1212 set ktsb_pbase, %o5 1213 set ktsb4m_pbase, %l1 12141: 1215 sethi %hi(ktsb_szcode), %o1 1216 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1217 1218 sethi %hi(iktsb), %o0 1219 call sfmmu_fix_ktlb_traptable 1220 or %o0, %lo(iktsb), %o0 1221 1222 sethi %hi(dktsb), %o0 1223 call sfmmu_fix_ktlb_traptable 1224 or %o0, %lo(dktsb), %o0 1225 1226 sethi %hi(ktsb4m_szcode), %o1 1227 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1228 1229 sethi %hi(iktsb4m), %o0 1230 call sfmmu_fix_ktlb_traptable 1231 or %o0, %lo(iktsb4m), %o0 1232 1233 sethi %hi(dktsb4m), %o0 1234 call sfmmu_fix_ktlb_traptable 1235 or %o0, %lo(dktsb4m), %o0 1236 1237#ifndef sun4v 1238 mov ASI_N, %o2 1239 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1240 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1241 sethi %hi(tsb_kernel_patch_asi), %o0 1242 call sfmmu_fixup_or 1243 or %o0, %lo(tsb_kernel_patch_asi), %o0 1244#endif /* !sun4v */ 1245 1246 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1247 1248 sethi %hi(dktsbbase), %o0 1249 call sfmmu_fixup_setx ! patch value of ktsb base addr 1250 or %o0, %lo(dktsbbase), %o0 1251 1252 sethi %hi(iktsbbase), %o0 1253 call sfmmu_fixup_setx ! patch value of ktsb base addr 1254 or %o0, %lo(iktsbbase), %o0 1255 1256 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1257 call sfmmu_fixup_setx ! patch value of ktsb base addr 1258 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1259 1260#ifdef sun4v 1261 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1262 call sfmmu_fixup_setx ! patch value of ktsb base addr 1263 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1264#endif /* sun4v */ 1265 1266 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1267 1268 sethi %hi(dktsb4mbase), %o0 1269 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1270 or %o0, %lo(dktsb4mbase), %o0 1271 1272 sethi %hi(iktsb4mbase), %o0 1273 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1274 or %o0, %lo(iktsb4mbase), %o0 1275 1276 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1277 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1278 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1279 1280#ifdef sun4v 1281 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1282 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1283 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1284#endif /* sun4v */ 1285 1286 set ktsb_szcode, %o4 1287 ld [%o4], %o4 1288 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1289 call sfmmu_fixup_or ! patch value of ktsb_szcode 1290 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1291 1292#ifdef sun4v 1293 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1294 call sfmmu_fixup_or ! patch value of ktsb_szcode 1295 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1296#endif /* sun4v */ 1297 1298 set ktsb4m_szcode, %o4 1299 ld [%o4], %o4 1300 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1301 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1302 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1303 1304#ifdef sun4v 1305 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1306 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1307 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1308#endif /* sun4v */ 1309 1310 ret 1311 restore 1312 SET_SIZE(sfmmu_patch_ktsb) 1313 1314 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1315 /* 1316 * Fixup trap handlers in common segkpm case. This is reserved 1317 * for future use should kpm TSB be changed to be other than the 1318 * kernel TSB. 1319 */ 1320 retl 1321 nop 1322 SET_SIZE(sfmmu_kpm_patch_tlbm) 1323 1324 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1325 /* 1326 * nop the branch to sfmmu_kpm_dtsb_miss_small 1327 * in the case where we are using large pages for 1328 * seg_kpm (and hence must probe the second TSB for 1329 * seg_kpm VAs) 1330 */ 1331 set dktsb4m_kpmcheck_small, %o0 1332 MAKE_NOP_INSTR(%o1) 1333 st %o1, [%o0] 1334 flush %o0 1335 retl 1336 nop 1337 SET_SIZE(sfmmu_kpm_patch_tsbm) 1338 1339 ENTRY_NP(sfmmu_patch_utsb) 1340#ifdef UTSB_PHYS 1341 retl 1342 nop 1343#else /* UTSB_PHYS */ 1344 /* 1345 * We need to hot patch utsb_vabase and utsb4m_vabase 1346 */ 1347 save %sp, -SA(MINFRAME), %sp 1348 1349 /* patch value of utsb_vabase */ 1350 set utsb_vabase, %o1 1351 ldx [%o1], %o4 1352 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1353 call sfmmu_fixup_setx 1354 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1355 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1356 call sfmmu_fixup_setx 1357 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1358 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1359 call sfmmu_fixup_setx 1360 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1361 1362 /* patch value of utsb4m_vabase */ 1363 set utsb4m_vabase, %o1 1364 ldx [%o1], %o4 1365 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1366 call sfmmu_fixup_setx 1367 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1368 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1369 call sfmmu_fixup_setx 1370 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1371 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1372 call sfmmu_fixup_setx 1373 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1374 1375 /* 1376 * Patch TSB base register masks and shifts if needed. 1377 * By default the TSB base register contents are set up for 4M slab. 1378 * If we're using a smaller slab size and reserved VA range we need 1379 * to patch up those values here. 1380 */ 1381 set tsb_slab_shift, %o1 1382 set MMU_PAGESHIFT4M, %o4 1383 lduw [%o1], %o3 1384 subcc %o4, %o3, %o4 1385 bz,pt %icc, 1f 1386 /* delay slot safe */ 1387 1388 /* patch reserved VA range size if needed. */ 1389 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1390 call sfmmu_fixup_shiftx 1391 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1392 call sfmmu_fixup_shiftx 1393 add %o0, I_SIZE, %o0 1394 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1395 call sfmmu_fixup_shiftx 1396 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1397 call sfmmu_fixup_shiftx 1398 add %o0, I_SIZE, %o0 13991: 1400 /* patch TSBREG_VAMASK used to set up TSB base register */ 1401 set tsb_slab_mask, %o1 1402 ldx [%o1], %o4 1403 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1404 call sfmmu_fixup_or 1405 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1406 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1407 call sfmmu_fixup_or 1408 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1409 1410 ret 1411 restore 1412#endif /* UTSB_PHYS */ 1413 SET_SIZE(sfmmu_patch_utsb) 1414 1415 ENTRY_NP(sfmmu_patch_shctx) 1416#ifdef sun4u 1417 retl 1418 nop 1419#else /* sun4u */ 1420 set sfmmu_shctx_cpu_mondo_patch, %o0 1421 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1422 st %o1, [%o0] 1423 flush %o0 1424 MAKE_NOP_INSTR(%o1) 1425 add %o0, I_SIZE, %o0 ! next instr 1426 st %o1, [%o0] 1427 flush %o0 1428 1429 set sfmmu_shctx_user_rtt_patch, %o0 1430 st %o1, [%o0] ! nop 1st instruction 1431 flush %o0 1432 add %o0, I_SIZE, %o0 1433 st %o1, [%o0] ! nop 2nd instruction 1434 flush %o0 1435 add %o0, I_SIZE, %o0 1436 st %o1, [%o0] ! nop 3rd instruction 1437 flush %o0 1438 add %o0, I_SIZE, %o0 1439 st %o1, [%o0] ! nop 4th instruction 1440 flush %o0 1441 add %o0, I_SIZE, %o0 1442 st %o1, [%o0] ! nop 5th instruction 1443 flush %o0 1444 add %o0, I_SIZE, %o0 1445 st %o1, [%o0] ! nop 6th instruction 1446 retl 1447 flush %o0 1448#endif /* sun4u */ 1449 SET_SIZE(sfmmu_patch_shctx) 1450 1451 /* 1452 * Routine that loads an entry into a tsb using virtual addresses. 1453 * Locking is required since all cpus can use the same TSB. 1454 * Note that it is no longer required to have a valid context 1455 * when calling this function. 1456 */ 1457 ENTRY_NP(sfmmu_load_tsbe) 1458 /* 1459 * %o0 = pointer to tsbe to load 1460 * %o1 = tsb tag 1461 * %o2 = virtual pointer to TTE 1462 * %o3 = 1 if physical address in %o0 else 0 1463 */ 1464 rdpr %pstate, %o5 1465#ifdef DEBUG 1466 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1467#endif /* DEBUG */ 1468 1469 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1470 1471 SETUP_TSB_ASI(%o3, %g3) 1472 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, locked_tsb_l8) 1473 1474 wrpr %g0, %o5, %pstate /* enable interrupts */ 1475 1476 retl 1477 membar #StoreStore|#StoreLoad 1478 SET_SIZE(sfmmu_load_tsbe) 1479 1480 /* 1481 * Flush TSB of a given entry if the tag matches. 1482 */ 1483 ENTRY(sfmmu_unload_tsbe) 1484 /* 1485 * %o0 = pointer to tsbe to be flushed 1486 * %o1 = tag to match 1487 * %o2 = 1 if physical address in %o0 else 0 1488 */ 1489 SETUP_TSB_ASI(%o2, %g1) 1490 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1491 retl 1492 membar #StoreStore|#StoreLoad 1493 SET_SIZE(sfmmu_unload_tsbe) 1494 1495 /* 1496 * Routine that loads a TTE into the kpm TSB from C code. 1497 * Locking is required since kpm TSB is shared among all CPUs. 1498 */ 1499 ENTRY_NP(sfmmu_kpm_load_tsb) 1500 /* 1501 * %o0 = vaddr 1502 * %o1 = ttep 1503 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1504 */ 1505 rdpr %pstate, %o5 ! %o5 = saved pstate 1506#ifdef DEBUG 1507 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1508#endif /* DEBUG */ 1509 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1510 1511#ifndef sun4v 1512 sethi %hi(ktsb_phys), %o4 1513 mov ASI_N, %o3 1514 ld [%o4 + %lo(ktsb_phys)], %o4 1515 movrnz %o4, ASI_MEM, %o3 1516 mov %o3, %asi 1517#endif /* !sun4v */ 1518 mov %o0, %g1 ! %g1 = vaddr 1519 1520 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1521 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1522 /* %g2 = tsbep, %g1 clobbered */ 1523 1524 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1525 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1526 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, locked_tsb_l9) 1527 1528 wrpr %g0, %o5, %pstate ! enable interrupts 1529 retl 1530 membar #StoreStore|#StoreLoad 1531 SET_SIZE(sfmmu_kpm_load_tsb) 1532 1533 /* 1534 * Routine that shoots down a TTE in the kpm TSB or in the 1535 * kernel TSB depending on virtpg. Locking is required since 1536 * kpm/kernel TSB is shared among all CPUs. 1537 */ 1538 ENTRY_NP(sfmmu_kpm_unload_tsb) 1539 /* 1540 * %o0 = vaddr 1541 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1542 */ 1543#ifndef sun4v 1544 sethi %hi(ktsb_phys), %o4 1545 mov ASI_N, %o3 1546 ld [%o4 + %lo(ktsb_phys)], %o4 1547 movrnz %o4, ASI_MEM, %o3 1548 mov %o3, %asi 1549#endif /* !sun4v */ 1550 mov %o0, %g1 ! %g1 = vaddr 1551 1552 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1553 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1554 /* %g2 = tsbep, %g1 clobbered */ 1555 1556 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1557 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1558 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1559 1560 retl 1561 membar #StoreStore|#StoreLoad 1562 SET_SIZE(sfmmu_kpm_unload_tsb) 1563 1564#endif /* lint */ 1565 1566 1567#if defined (lint) 1568 1569/*ARGSUSED*/ 1570pfn_t 1571sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1572{ return(0); } 1573 1574#else /* lint */ 1575 1576 ENTRY_NP(sfmmu_ttetopfn) 1577 ldx [%o0], %g1 /* read tte */ 1578 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1579 /* 1580 * g1 = pfn 1581 */ 1582 retl 1583 mov %g1, %o0 1584 SET_SIZE(sfmmu_ttetopfn) 1585 1586#endif /* !lint */ 1587 1588/* 1589 * These macros are used to update global sfmmu hme hash statistics 1590 * in perf critical paths. It is only enabled in debug kernels or 1591 * if SFMMU_STAT_GATHER is defined 1592 */ 1593#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1594#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1595 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1596 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1597 cmp tmp1, hatid ;\ 1598 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1599 set sfmmu_global_stat, tmp1 ;\ 1600 add tmp1, tmp2, tmp1 ;\ 1601 ld [tmp1], tmp2 ;\ 1602 inc tmp2 ;\ 1603 st tmp2, [tmp1] 1604 1605#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1606 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1607 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1608 cmp tmp1, hatid ;\ 1609 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1610 set sfmmu_global_stat, tmp1 ;\ 1611 add tmp1, tmp2, tmp1 ;\ 1612 ld [tmp1], tmp2 ;\ 1613 inc tmp2 ;\ 1614 st tmp2, [tmp1] 1615 1616 1617#else /* DEBUG || SFMMU_STAT_GATHER */ 1618 1619#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1620 1621#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1622 1623#endif /* DEBUG || SFMMU_STAT_GATHER */ 1624 1625/* 1626 * This macro is used to update global sfmmu kstas in non 1627 * perf critical areas so they are enabled all the time 1628 */ 1629#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1630 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1631 add tmp1, statname, tmp1 ;\ 1632 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1633 inc tmp2 ;\ 1634 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1635 1636/* 1637 * These macros are used to update per cpu stats in non perf 1638 * critical areas so they are enabled all the time 1639 */ 1640#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1641 ld [tsbarea + stat], tmp1 ;\ 1642 inc tmp1 ;\ 1643 st tmp1, [tsbarea + stat] 1644 1645/* 1646 * These macros are used to update per cpu stats in non perf 1647 * critical areas so they are enabled all the time 1648 */ 1649#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1650 lduh [tsbarea + stat], tmp1 ;\ 1651 inc tmp1 ;\ 1652 stuh tmp1, [tsbarea + stat] 1653 1654#if defined(KPM_TLBMISS_STATS_GATHER) 1655 /* 1656 * Count kpm dtlb misses separately to allow a different 1657 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1658 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1659 */ 1660#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1661 brgez tagacc, label /* KPM VA? */ ;\ 1662 nop ;\ 1663 CPU_INDEX(tmp1, tsbma) ;\ 1664 sethi %hi(kpmtsbm_area), tsbma ;\ 1665 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1666 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1667 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1668 /* VA range check */ ;\ 1669 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1670 cmp tagacc, val ;\ 1671 blu,pn %xcc, label ;\ 1672 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1673 cmp tagacc, tmp1 ;\ 1674 bgeu,pn %xcc, label ;\ 1675 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1676 inc val ;\ 1677 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1678label: 1679#else 1680#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1681#endif /* KPM_TLBMISS_STATS_GATHER */ 1682 1683#if defined (lint) 1684/* 1685 * The following routines are jumped to from the mmu trap handlers to do 1686 * the setting up to call systrap. They are separate routines instead of 1687 * being part of the handlers because the handlers would exceed 32 1688 * instructions and since this is part of the slow path the jump 1689 * cost is irrelevant. 1690 */ 1691void 1692sfmmu_pagefault(void) 1693{ 1694} 1695 1696void 1697sfmmu_mmu_trap(void) 1698{ 1699} 1700 1701void 1702sfmmu_window_trap(void) 1703{ 1704} 1705 1706void 1707sfmmu_kpm_exception(void) 1708{ 1709} 1710 1711#else /* lint */ 1712 1713#ifdef PTL1_PANIC_DEBUG 1714 .seg ".data" 1715 .global test_ptl1_panic 1716test_ptl1_panic: 1717 .word 0 1718 .align 8 1719 1720 .seg ".text" 1721 .align 4 1722#endif /* PTL1_PANIC_DEBUG */ 1723 1724 1725 ENTRY_NP(sfmmu_pagefault) 1726 SET_GL_REG(1) 1727 USE_ALTERNATE_GLOBALS(%g5) 1728 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1729 rdpr %tt, %g6 1730 cmp %g6, FAST_IMMU_MISS_TT 1731 be,a,pn %icc, 1f 1732 mov T_INSTR_MMU_MISS, %g3 1733 cmp %g6, T_INSTR_MMU_MISS 1734 be,a,pn %icc, 1f 1735 mov T_INSTR_MMU_MISS, %g3 1736 mov %g5, %g2 1737 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1738 cmp %g6, FAST_DMMU_MISS_TT 1739 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1740 cmp %g6, T_DATA_MMU_MISS 1741 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1742 1743#ifdef PTL1_PANIC_DEBUG 1744 /* check if we want to test the tl1 panic */ 1745 sethi %hi(test_ptl1_panic), %g4 1746 ld [%g4 + %lo(test_ptl1_panic)], %g1 1747 st %g0, [%g4 + %lo(test_ptl1_panic)] 1748 cmp %g1, %g0 1749 bne,a,pn %icc, ptl1_panic 1750 or %g0, PTL1_BAD_DEBUG, %g1 1751#endif /* PTL1_PANIC_DEBUG */ 17521: 1753 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 1754 /* 1755 * g2 = tag access reg 1756 * g3.l = type 1757 * g3.h = 0 1758 */ 1759 sethi %hi(trap), %g1 1760 or %g1, %lo(trap), %g1 17612: 1762 ba,pt %xcc, sys_trap 1763 mov -1, %g4 1764 SET_SIZE(sfmmu_pagefault) 1765 1766 ENTRY_NP(sfmmu_mmu_trap) 1767 SET_GL_REG(1) 1768 USE_ALTERNATE_GLOBALS(%g5) 1769 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 1770 rdpr %tt, %g6 1771 cmp %g6, FAST_IMMU_MISS_TT 1772 be,a,pn %icc, 1f 1773 mov T_INSTR_MMU_MISS, %g3 1774 cmp %g6, T_INSTR_MMU_MISS 1775 be,a,pn %icc, 1f 1776 mov T_INSTR_MMU_MISS, %g3 1777 mov %g5, %g2 1778 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1779 cmp %g6, FAST_DMMU_MISS_TT 1780 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1781 cmp %g6, T_DATA_MMU_MISS 1782 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 17831: 1784 /* 1785 * g2 = tag access reg 1786 * g3 = type 1787 */ 1788 sethi %hi(sfmmu_tsbmiss_exception), %g1 1789 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 1790 ba,pt %xcc, sys_trap 1791 mov -1, %g4 1792 /*NOTREACHED*/ 1793 SET_SIZE(sfmmu_mmu_trap) 1794 1795 ENTRY_NP(sfmmu_suspend_tl) 1796 SET_GL_REG(1) 1797 USE_ALTERNATE_GLOBALS(%g5) 1798 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 1799 rdpr %tt, %g6 1800 cmp %g6, FAST_IMMU_MISS_TT 1801 be,a,pn %icc, 1f 1802 mov T_INSTR_MMU_MISS, %g3 1803 mov %g5, %g2 1804 cmp %g6, FAST_DMMU_MISS_TT 1805 move %icc, T_DATA_MMU_MISS, %g3 1806 movne %icc, T_DATA_PROT, %g3 18071: 1808 sethi %hi(sfmmu_tsbmiss_suspended), %g1 1809 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 1810 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 1811 ba,pt %xcc, sys_trap 1812 mov PIL_15, %g4 1813 /*NOTREACHED*/ 1814 SET_SIZE(sfmmu_suspend_tl) 1815 1816 /* 1817 * No %g registers in use at this point. 1818 */ 1819 ENTRY_NP(sfmmu_window_trap) 1820 rdpr %tpc, %g1 1821#ifdef sun4v 1822#ifdef DEBUG 1823 /* We assume previous %gl was 1 */ 1824 rdpr %tstate, %g4 1825 srlx %g4, TSTATE_GL_SHIFT, %g4 1826 and %g4, TSTATE_GL_MASK, %g4 1827 cmp %g4, 1 1828 bne,a,pn %icc, ptl1_panic 1829 mov PTL1_BAD_WTRAP, %g1 1830#endif /* DEBUG */ 1831 /* user miss at tl>1. better be the window handler or user_rtt */ 1832 /* in user_rtt? */ 1833 set rtt_fill_start, %g4 1834 cmp %g1, %g4 1835 blu,pn %xcc, 6f 1836 .empty 1837 set rtt_fill_end, %g4 1838 cmp %g1, %g4 1839 bgeu,pn %xcc, 6f 1840 nop 1841 set fault_rtt_fn1, %g1 1842 wrpr %g0, %g1, %tnpc 1843 ba,a 7f 18446: 1845 ! must save this trap level before descending trap stack 1846 ! no need to save %tnpc, either overwritten or discarded 1847 ! already got it: rdpr %tpc, %g1 1848 rdpr %tstate, %g6 1849 rdpr %tt, %g7 1850 ! trap level saved, go get underlying trap type 1851 rdpr %tl, %g5 1852 sub %g5, 1, %g3 1853 wrpr %g3, %tl 1854 rdpr %tt, %g2 1855 wrpr %g5, %tl 1856 ! restore saved trap level 1857 wrpr %g1, %tpc 1858 wrpr %g6, %tstate 1859 wrpr %g7, %tt 1860#else /* sun4v */ 1861 /* user miss at tl>1. better be the window handler */ 1862 rdpr %tl, %g5 1863 sub %g5, 1, %g3 1864 wrpr %g3, %tl 1865 rdpr %tt, %g2 1866 wrpr %g5, %tl 1867#endif /* sun4v */ 1868 and %g2, WTRAP_TTMASK, %g4 1869 cmp %g4, WTRAP_TYPE 1870 bne,pn %xcc, 1f 1871 nop 1872 /* tpc should be in the trap table */ 1873 set trap_table, %g4 1874 cmp %g1, %g4 1875 blt,pn %xcc, 1f 1876 .empty 1877 set etrap_table, %g4 1878 cmp %g1, %g4 1879 bge,pn %xcc, 1f 1880 .empty 1881 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 1882 add %g1, WTRAP_FAULTOFF, %g1 1883 wrpr %g0, %g1, %tnpc 18847: 1885 /* 1886 * some wbuf handlers will call systrap to resolve the fault 1887 * we pass the trap type so they figure out the correct parameters. 1888 * g5 = trap type, g6 = tag access reg 1889 */ 1890 1891 /* 1892 * only use g5, g6, g7 registers after we have switched to alternate 1893 * globals. 1894 */ 1895 SET_GL_REG(1) 1896 USE_ALTERNATE_GLOBALS(%g5) 1897 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 1898 rdpr %tt, %g7 1899 cmp %g7, FAST_IMMU_MISS_TT 1900 be,a,pn %icc, ptl1_panic 1901 mov PTL1_BAD_WTRAP, %g1 1902 cmp %g7, T_INSTR_MMU_MISS 1903 be,a,pn %icc, ptl1_panic 1904 mov PTL1_BAD_WTRAP, %g1 1905 mov T_DATA_PROT, %g5 1906 cmp %g7, FAST_DMMU_MISS_TT 1907 move %icc, T_DATA_MMU_MISS, %g5 1908 cmp %g7, T_DATA_MMU_MISS 1909 move %icc, T_DATA_MMU_MISS, %g5 1910 ! XXXQ AGS re-check out this one 1911 done 19121: 1913 CPU_PADDR(%g1, %g4) 1914 add %g1, CPU_TL1_HDLR, %g1 1915 lda [%g1]ASI_MEM, %g4 1916 brnz,a,pt %g4, sfmmu_mmu_trap 1917 sta %g0, [%g1]ASI_MEM 1918 ba,pt %icc, ptl1_panic 1919 mov PTL1_BAD_TRAP, %g1 1920 SET_SIZE(sfmmu_window_trap) 1921 1922 ENTRY_NP(sfmmu_kpm_exception) 1923 /* 1924 * We have accessed an unmapped segkpm address or a legal segkpm 1925 * address which is involved in a VAC alias conflict prevention. 1926 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 1927 * set. If it is, we will instead note that a fault has occurred 1928 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 1929 * a "retry"). This will step over the faulting instruction. 1930 * Note that this means that a legal segkpm address involved in 1931 * a VAC alias conflict prevention (a rare case to begin with) 1932 * cannot be used in DTrace. 1933 */ 1934 CPU_INDEX(%g1, %g2) 1935 set cpu_core, %g2 1936 sllx %g1, CPU_CORE_SHIFT, %g1 1937 add %g1, %g2, %g1 1938 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 1939 andcc %g2, CPU_DTRACE_NOFAULT, %g0 1940 bz 0f 1941 or %g2, CPU_DTRACE_BADADDR, %g2 1942 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 1943 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 1944 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 1945 done 19460: 1947 TSTAT_CHECK_TL1(1f, %g1, %g2) 19481: 1949 SET_GL_REG(1) 1950 USE_ALTERNATE_GLOBALS(%g5) 1951 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 1952 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1953 /* 1954 * g2=tagacc g3.l=type g3.h=0 1955 */ 1956 sethi %hi(trap), %g1 1957 or %g1, %lo(trap), %g1 1958 ba,pt %xcc, sys_trap 1959 mov -1, %g4 1960 SET_SIZE(sfmmu_kpm_exception) 1961 1962#endif /* lint */ 1963 1964#if defined (lint) 1965 1966void 1967sfmmu_tsb_miss(void) 1968{ 1969} 1970 1971void 1972sfmmu_kpm_dtsb_miss(void) 1973{ 1974} 1975 1976void 1977sfmmu_kpm_dtsb_miss_small(void) 1978{ 1979} 1980 1981#else /* lint */ 1982 1983#if (IMAP_SEG != 0) 1984#error - ism_map->ism_seg offset is not zero 1985#endif 1986 1987/* 1988 * Copies ism mapping for this ctx in param "ism" if this is a ISM 1989 * tlb miss and branches to label "ismhit". If this is not an ISM 1990 * process or an ISM tlb miss it falls thru. 1991 * 1992 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 1993 * this process. 1994 * If so, it will branch to label "ismhit". If not, it will fall through. 1995 * 1996 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 1997 * so that any other threads of this process will not try and walk the ism 1998 * maps while they are being changed. 1999 * 2000 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 2001 * will make sure of that. This means we can terminate our search on 2002 * the first zero mapping we find. 2003 * 2004 * Parameters: 2005 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 2006 * tsbmiss = address of tsb miss area (in) 2007 * ismseg = contents of ism_seg for this ism map (out) 2008 * ismhat = physical address of imap_ismhat for this ism map (out) 2009 * tmp1 = scratch reg (CLOBBERED) 2010 * tmp2 = scratch reg (CLOBBERED) 2011 * tmp3 = scratch reg (CLOBBERED) 2012 * label: temporary labels 2013 * ismhit: label where to jump to if an ism dtlb miss 2014 * exitlabel:label where to jump if hat is busy due to hat_unshare. 2015 */ 2016#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 2017 label, ismhit) \ 2018 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 2019 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 2020 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 2021label/**/1: ;\ 2022 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 2023 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 2024label/**/2: ;\ 2025 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 2026 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 2027 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 2028 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 2029 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 2030 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 2031 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 2032 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 2033 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 2034 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 2035 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 2036 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 2037 ;\ 2038 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 2039 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 2040 cmp ismhat, tmp1 ;\ 2041 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 2042 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 2043 ;\ 2044 add tmp3, IBLK_NEXTPA, tmp1 ;\ 2045 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 2046 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 2047 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 2048label/**/3: 2049 2050/* 2051 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 2052 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 2053 * Parameters: 2054 * tagacc = reg containing virtual address 2055 * hatid = reg containing sfmmu pointer 2056 * hmeshift = constant/register to shift vaddr to obtain vapg 2057 * hmebp = register where bucket pointer will be stored 2058 * vapg = register where virtual page will be stored 2059 * tmp1, tmp2 = tmp registers 2060 */ 2061 2062 2063#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 2064 vapg, label, tmp1, tmp2) \ 2065 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 2066 brnz,a,pt tmp1, label/**/1 ;\ 2067 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 2068 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 2069 ba,pt %xcc, label/**/2 ;\ 2070 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 2071label/**/1: ;\ 2072 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 2073label/**/2: ;\ 2074 srlx tagacc, hmeshift, vapg ;\ 2075 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 2076 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 2077 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 2078 add hmebp, tmp1, hmebp 2079 2080/* 2081 * hashtag includes bspage + hashno (64 bits). 2082 */ 2083 2084#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 2085 sllx vapg, hmeshift, vapg ;\ 2086 mov hashno, hblktag ;\ 2087 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 2088 or vapg, hblktag, hblktag 2089 2090/* 2091 * Function to traverse hmeblk hash link list and find corresponding match. 2092 * The search is done using physical pointers. It returns the physical address 2093 * pointer to the hmeblk that matches with the tag provided. 2094 * Parameters: 2095 * hmebp = register that points to hme hash bucket, also used as 2096 * tmp reg (clobbered) 2097 * hmeblktag = register with hmeblk tag match 2098 * hatid = register with hatid 2099 * hmeblkpa = register where physical ptr will be stored 2100 * tmp1 = tmp reg 2101 * label: temporary label 2102 */ 2103 2104#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, tsbarea, \ 2105 tmp1, label) \ 2106 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 2107 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2108 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2109label/**/1: ;\ 2110 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2111 be,pn %xcc, label/**/2 ;\ 2112 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 2113 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 2114 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2115 add hmebp, CLONGSIZE, hmebp ;\ 2116 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 2117 xor tmp1, hmeblktag, tmp1 ;\ 2118 xor hmebp, hatid, hmebp ;\ 2119 or hmebp, tmp1, hmebp ;\ 2120 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 2121 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 2122 ba,pt %xcc, label/**/1 ;\ 2123 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2124label/**/2: 2125 2126/* 2127 * Function to traverse hmeblk hash link list and find corresponding match. 2128 * The search is done using physical pointers. It returns the physical address 2129 * pointer to the hmeblk that matches with the tag 2130 * provided. 2131 * Parameters: 2132 * hmeblktag = register with hmeblk tag match (rid field is 0) 2133 * hatid = register with hatid (pointer to SRD) 2134 * hmeblkpa = register where physical ptr will be stored 2135 * tmp1 = tmp reg 2136 * tmp2 = tmp reg 2137 * label: temporary label 2138 */ 2139 2140#define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, tsbarea, \ 2141 tmp1, tmp2, label) \ 2142label/**/1: ;\ 2143 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2144 be,pn %xcc, label/**/4 ;\ 2145 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 2146 add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 2147 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 2148 add tmp2, CLONGSIZE, tmp2 ;\ 2149 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 2150 xor tmp1, hmeblktag, tmp1 ;\ 2151 xor tmp2, hatid, tmp2 ;\ 2152 brz,pn tmp2, label/**/3 /* branch on hit */ ;\ 2153 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2154label/**/2: ;\ 2155 ba,pt %xcc, label/**/1 ;\ 2156 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 2157label/**/3: ;\ 2158 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 2159 bgeu,pt %xcc, label/**/2 ;\ 2160 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2161 and tmp1, BT_ULMASK, tmp2 ;\ 2162 srlx tmp1, BT_ULSHIFT, tmp1 ;\ 2163 sllx tmp1, CLONGSHIFT, tmp1 ;\ 2164 add tsbarea, tmp1, tmp1 ;\ 2165 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 2166 srlx tmp1, tmp2, tmp1 ;\ 2167 btst 0x1, tmp1 ;\ 2168 bz,pn %xcc, label/**/2 ;\ 2169 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2170label/**/4: 2171 2172#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2173#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2174#endif 2175 2176/* 2177 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2178 * he offset for the corresponding hment. 2179 * Parameters: 2180 * In: 2181 * vaddr = register with virtual address 2182 * hmeblkpa = physical pointer to hme_blk 2183 * Out: 2184 * hmentoff = register where hment offset will be stored 2185 * hmemisc = hblk_misc 2186 * Scratch: 2187 * tmp1 2188 */ 2189#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2190 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2191 lda [hmentoff]ASI_MEM, hmemisc ;\ 2192 andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2193 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2194 or %g0, HMEBLK_HME1, hmentoff ;\ 2195 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2196 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2197 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2198 add tmp1, HMEBLK_HME1, hmentoff ;\ 2199label1: 2200 2201/* 2202 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2203 * 2204 * tagacc = (pseudo-)tag access register (in) 2205 * hatid = sfmmu pointer for TSB miss (in) 2206 * tte = tte for TLB miss if found, otherwise clobbered (out) 2207 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2208 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2209 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2210 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2211 * for this page size. 2212 * hashno = constant/register hash number 2213 * tmp = temp value - clobbered 2214 * label = temporary label for branching within macro. 2215 * foundlabel = label to jump to when tte is found. 2216 * suspendlabel= label to jump to when tte is suspended. 2217 * exitlabel = label to jump to when tte is not found. 2218 * 2219 */ 2220#define GET_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, hmeshift, \ 2221 hashno, tmp, label, foundlabel, suspendlabel, exitlabel) \ 2222 ;\ 2223 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2224 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2225 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2226 hmeblkpa, label/**/5, hmemisc, tmp) ;\ 2227 ;\ 2228 /* ;\ 2229 * tagacc = tagacc ;\ 2230 * hatid = hatid ;\ 2231 * tsbarea = tsbarea ;\ 2232 * tte = hmebp (hme bucket pointer) ;\ 2233 * hmeblkpa = vapg (virtual page) ;\ 2234 * hmemisc, tmp = scratch ;\ 2235 */ ;\ 2236 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2237 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2238 ;\ 2239 /* ;\ 2240 * tagacc = tagacc ;\ 2241 * hatid = hatid ;\ 2242 * tte = hmebp ;\ 2243 * hmeblkpa = CLOBBERED ;\ 2244 * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2245 * tmp = scratch ;\ 2246 */ ;\ 2247 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2248 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, \ 2249 tsbarea, tagacc, label/**/1) ;\ 2250 /* ;\ 2251 * tagacc = CLOBBERED ;\ 2252 * tte = CLOBBERED ;\ 2253 * hmeblkpa = hmeblkpa ;\ 2254 * tmp = scratch ;\ 2255 */ ;\ 2256 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2257 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\ 2258 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2259 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2260 nop ;\ 2261label/**/4: ;\ 2262 /* ;\ 2263 * We have found the hmeblk containing the hment. ;\ 2264 * Now we calculate the corresponding tte. ;\ 2265 * ;\ 2266 * tagacc = tagacc ;\ 2267 * hatid = hatid ;\ 2268 * tte = clobbered ;\ 2269 * hmeblkpa = hmeblkpa ;\ 2270 * hmemisc = hblktag ;\ 2271 * tmp = scratch ;\ 2272 */ ;\ 2273 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2274 label/**/2) ;\ 2275 ;\ 2276 /* ;\ 2277 * tagacc = tagacc ;\ 2278 * hatid = hmentoff ;\ 2279 * tte = clobbered ;\ 2280 * hmeblkpa = hmeblkpa ;\ 2281 * hmemisc = hblk_misc ;\ 2282 * tmp = scratch ;\ 2283 */ ;\ 2284 ;\ 2285 add hatid, SFHME_TTE, hatid ;\ 2286 add hmeblkpa, hatid, hmeblkpa ;\ 2287 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2288 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2289 set TTE_SUSPEND, hatid ;\ 2290 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2291 btst tte, hatid ;\ 2292 bz,pt %xcc, foundlabel ;\ 2293 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2294 ;\ 2295 /* ;\ 2296 * Mapping is suspended, so goto suspend label. ;\ 2297 */ ;\ 2298 ba,pt %xcc, suspendlabel ;\ 2299 nop 2300 2301/* 2302 * GET_SHME_TTE is similar to GET_TTE() except it searches 2303 * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2304 * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2305 * either 0 (not part of scd) or 1 (part of scd). 2306 */ 2307#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, \ 2308 hmeshift, hashno, tmp, label, foundlabel, \ 2309 suspendlabel, exitlabel) \ 2310 ;\ 2311 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2312 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2313 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2314 hmeblkpa, label/**/5, hmemisc, tmp) ;\ 2315 ;\ 2316 /* ;\ 2317 * tagacc = tagacc ;\ 2318 * hatid = hatid ;\ 2319 * tsbarea = tsbarea ;\ 2320 * tte = hmebp (hme bucket pointer) ;\ 2321 * hmeblkpa = vapg (virtual page) ;\ 2322 * hmemisc, tmp = scratch ;\ 2323 */ ;\ 2324 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2325 ;\ 2326 /* ;\ 2327 * tagacc = tagacc ;\ 2328 * hatid = hatid ;\ 2329 * tsbarea = tsbarea ;\ 2330 * tte = hmebp ;\ 2331 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2332 * hmeblkpa = CLOBBERED ;\ 2333 * tmp = scratch ;\ 2334 */ ;\ 2335 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2336 ;\ 2337 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2338 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2339 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2340 ;\ 2341label/**/8: ;\ 2342 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, \ 2343 tsbarea, tagacc, tte, label/**/1) ;\ 2344 /* ;\ 2345 * tagacc = CLOBBERED ;\ 2346 * tte = CLOBBERED ;\ 2347 * hmeblkpa = hmeblkpa ;\ 2348 * tmp = scratch ;\ 2349 */ ;\ 2350 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2351 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\ 2352 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2353 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2354 nop ;\ 2355label/**/4: ;\ 2356 /* ;\ 2357 * We have found the hmeblk containing the hment. ;\ 2358 * Now we calculate the corresponding tte. ;\ 2359 * ;\ 2360 * tagacc = tagacc ;\ 2361 * hatid = hatid ;\ 2362 * tte = clobbered ;\ 2363 * hmeblkpa = hmeblkpa ;\ 2364 * hmemisc = hblktag ;\ 2365 * tsbarea = tsbmiss area ;\ 2366 * tmp = scratch ;\ 2367 */ ;\ 2368 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2369 label/**/2) ;\ 2370 ;\ 2371 /* ;\ 2372 * tagacc = tagacc ;\ 2373 * hatid = hmentoff ;\ 2374 * tte = clobbered ;\ 2375 * hmeblkpa = hmeblkpa ;\ 2376 * hmemisc = hblk_misc ;\ 2377 * tsbarea = tsbmiss area ;\ 2378 * tmp = scratch ;\ 2379 */ ;\ 2380 ;\ 2381 add hatid, SFHME_TTE, hatid ;\ 2382 add hmeblkpa, hatid, hmeblkpa ;\ 2383 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2384 brlz,pt tte, label/**/6 ;\ 2385 nop ;\ 2386 btst HBLK_SZMASK, hmemisc ;\ 2387 bnz,a,pt %icc, label/**/7 ;\ 2388 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2389 ;\ 2390 /* ;\ 2391 * We found an invalid 8K tte in shme. ;\ 2392 * it may not belong to shme's region since ;\ 2393 * region size/alignment granularity is 8K but different ;\ 2394 * regions don't share hmeblks. Continue the search. ;\ 2395 */ ;\ 2396 sub hmeblkpa, hatid, hmeblkpa ;\ 2397 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2398 srlx tagacc, hmeshift, tte ;\ 2399 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2400 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2401 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2402 ba,a,pt %xcc, label/**/8 ;\ 2403label/**/6: ;\ 2404 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2405 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2406label/**/7: ;\ 2407 set TTE_SUSPEND, hatid ;\ 2408 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2409 btst tte, hatid ;\ 2410 bz,pt %xcc, foundlabel ;\ 2411 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2412 ;\ 2413 /* ;\ 2414 * Mapping is suspended, so goto suspend label. ;\ 2415 */ ;\ 2416 ba,pt %xcc, suspendlabel ;\ 2417 nop 2418 2419 /* 2420 * KERNEL PROTECTION HANDLER 2421 * 2422 * g1 = tsb8k pointer register (clobbered) 2423 * g2 = tag access register (ro) 2424 * g3 - g7 = scratch registers 2425 * 2426 * Note: This function is patched at runtime for performance reasons. 2427 * Any changes here require sfmmu_patch_ktsb fixed. 2428 */ 2429 ENTRY_NP(sfmmu_kprot_trap) 2430 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2431sfmmu_kprot_patch_ktsb_base: 2432 RUNTIME_PATCH_SETX(%g1, %g6) 2433 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2434sfmmu_kprot_patch_ktsb_szcode: 2435 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2436 2437 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2438 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2439 2440 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2441sfmmu_kprot_patch_ktsb4m_base: 2442 RUNTIME_PATCH_SETX(%g3, %g6) 2443 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2444sfmmu_kprot_patch_ktsb4m_szcode: 2445 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2446 2447 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2448 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2449 2450 CPU_TSBMISS_AREA(%g6, %g7) 2451 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2452 ba,pt %xcc, sfmmu_tsb_miss_tt 2453 nop 2454 2455 /* 2456 * USER PROTECTION HANDLER 2457 * 2458 * g1 = tsb8k pointer register (ro) 2459 * g2 = tag access register (ro) 2460 * g3 = faulting context (clobbered, currently not used) 2461 * g4 - g7 = scratch registers 2462 */ 2463 ALTENTRY(sfmmu_uprot_trap) 2464#ifdef sun4v 2465 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2466 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2467 2468 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2469 brlz,pt %g3, 9f /* check for 2nd TSB */ 2470 nop 2471 2472 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2473 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2474 2475#else /* sun4v */ 2476#ifdef UTSB_PHYS 2477 /* g1 = first TSB entry ptr */ 2478 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2479 brlz,pt %g3, 9f /* check for 2nd TSB */ 2480 nop 2481 2482 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2483 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2484#else /* UTSB_PHYS */ 2485 brgez,pt %g1, 9f /* check for 2nd TSB */ 2486 mov -1, %g3 /* set second tsbe ptr to -1 */ 2487 2488 mov %g2, %g7 2489 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2490 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2491 mov %g1, %g7 2492 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2493#endif /* UTSB_PHYS */ 2494#endif /* sun4v */ 24959: 2496 CPU_TSBMISS_AREA(%g6, %g7) 2497 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2498 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2499 nop 2500 2501 /* 2502 * Kernel 8K page iTLB miss. We also get here if we took a 2503 * fast instruction access mmu miss trap while running in 2504 * invalid context. 2505 * 2506 * %g1 = 8K TSB pointer register (not used, clobbered) 2507 * %g2 = tag access register (used) 2508 * %g3 = faulting context id (used) 2509 * %g7 = TSB tag to match (used) 2510 */ 2511 .align 64 2512 ALTENTRY(sfmmu_kitlb_miss) 2513 brnz,pn %g3, tsb_tl0_noctxt 2514 nop 2515 2516 /* kernel miss */ 2517 /* get kernel tsb pointer */ 2518 /* we patch the next set of instructions at run time */ 2519 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2520iktsbbase: 2521 RUNTIME_PATCH_SETX(%g4, %g5) 2522 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2523 2524iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2525 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2526 or %g4, %g1, %g1 ! form tsb ptr 2527 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2528 cmp %g4, %g7 2529 bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2530 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2531 2532 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2533 bz,pn %icc, exec_fault 2534 nop 2535 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2536 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2537 retry 2538 2539iktsb4mbase: 2540 RUNTIME_PATCH_SETX(%g4, %g6) 2541 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2542iktsb4m: 2543 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2544 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2545 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2546 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2547 cmp %g4, %g7 2548 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2549 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2550 bz,pn %icc, exec_fault 2551 nop 2552 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2553 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2554 retry 2555 2556 /* 2557 * Kernel dTLB miss. We also get here if we took a fast data 2558 * access mmu miss trap while running in invalid context. 2559 * 2560 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2561 * We select the TSB miss handler to branch to depending on 2562 * the virtual address of the access. In the future it may 2563 * be desirable to separate kpm TTEs into their own TSB, 2564 * in which case all that needs to be done is to set 2565 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2566 * early in the miss if we detect a kpm VA to a new handler. 2567 * 2568 * %g1 = 8K TSB pointer register (not used, clobbered) 2569 * %g2 = tag access register (used) 2570 * %g3 = faulting context id (used) 2571 */ 2572 .align 64 2573 ALTENTRY(sfmmu_kdtlb_miss) 2574 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2575 nop 2576 2577 /* Gather some stats for kpm misses in the TLB. */ 2578 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2579 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2580 2581 /* 2582 * Get first TSB offset and look for 8K/64K/512K mapping 2583 * using the 8K virtual page as the index. 2584 * 2585 * We patch the next set of instructions at run time; 2586 * any changes here require sfmmu_patch_ktsb changes too. 2587 */ 2588dktsbbase: 2589 RUNTIME_PATCH_SETX(%g7, %g6) 2590 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2591 2592dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2593 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2594 2595 /* 2596 * At this point %g1 is our index into the TSB. 2597 * We just masked off enough bits of the VA depending 2598 * on our TSB size code. 2599 */ 2600 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2601 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2602 cmp %g6, %g4 ! compare tag 2603 bne,pn %xcc, dktsb4m_kpmcheck_small 2604 add %g7, %g1, %g1 /* form tsb ptr */ 2605 TT_TRACE(trace_tsbhit) 2606 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2607 /* trapstat expects tte in %g5 */ 2608 retry 2609 2610 /* 2611 * If kpm is using large pages, the following instruction needs 2612 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2613 * so that we will probe the 4M TSB regardless of the VA. In 2614 * the case kpm is using small pages, we know no large kernel 2615 * mappings are located above 0x80000000.00000000 so we skip the 2616 * probe as an optimization. 2617 */ 2618dktsb4m_kpmcheck_small: 2619 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2620 /* delay slot safe, below */ 2621 2622 /* 2623 * Get second TSB offset and look for 4M mapping 2624 * using 4M virtual page as the TSB index. 2625 * 2626 * Here: 2627 * %g1 = 8K TSB pointer. Don't squash it. 2628 * %g2 = tag access register (we still need it) 2629 */ 2630 srlx %g2, MMU_PAGESHIFT4M, %g3 2631 2632 /* 2633 * We patch the next set of instructions at run time; 2634 * any changes here require sfmmu_patch_ktsb changes too. 2635 */ 2636dktsb4mbase: 2637 RUNTIME_PATCH_SETX(%g7, %g6) 2638 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2639dktsb4m: 2640 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2641 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2642 2643 /* 2644 * At this point %g3 is our index into the TSB. 2645 * We just masked off enough bits of the VA depending 2646 * on our TSB size code. 2647 */ 2648 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2649 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2650 cmp %g6, %g4 ! compare tag 2651 2652dktsb4m_tsbmiss: 2653 bne,pn %xcc, dktsb4m_kpmcheck 2654 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2655 TT_TRACE(trace_tsbhit) 2656 /* we don't check TTE size here since we assume 4M TSB is separate */ 2657 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2658 /* trapstat expects tte in %g5 */ 2659 retry 2660 2661 /* 2662 * So, we failed to find a valid TTE to match the faulting 2663 * address in either TSB. There are a few cases that could land 2664 * us here: 2665 * 2666 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2667 * to sfmmu_tsb_miss_tt to handle the miss. 2668 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2669 * 4M TSB. Let segkpm handle it. 2670 * 2671 * Note that we shouldn't land here in the case of a kpm VA when 2672 * kpm_smallpages is active -- we handled that case earlier at 2673 * dktsb4m_kpmcheck_small. 2674 * 2675 * At this point: 2676 * g1 = 8K-indexed primary TSB pointer 2677 * g2 = tag access register 2678 * g3 = 4M-indexed secondary TSB pointer 2679 */ 2680dktsb4m_kpmcheck: 2681 cmp %g2, %g0 2682 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2683 nop 2684 ba,a,pt %icc, sfmmu_tsb_miss_tt 2685 nop 2686 2687#ifdef sun4v 2688 /* 2689 * User instruction miss w/ single TSB. 2690 * The first probe covers 8K, 64K, and 512K page sizes, 2691 * because 64K and 512K mappings are replicated off 8K 2692 * pointer. 2693 * 2694 * g1 = tsb8k pointer register 2695 * g2 = tag access register 2696 * g3 - g6 = scratch registers 2697 * g7 = TSB tag to match 2698 */ 2699 .align 64 2700 ALTENTRY(sfmmu_uitlb_fastpath) 2701 2702 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2703 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2704 ba,pn %xcc, sfmmu_tsb_miss_tt 2705 mov -1, %g3 2706 2707 /* 2708 * User data miss w/ single TSB. 2709 * The first probe covers 8K, 64K, and 512K page sizes, 2710 * because 64K and 512K mappings are replicated off 8K 2711 * pointer. 2712 * 2713 * g1 = tsb8k pointer register 2714 * g2 = tag access register 2715 * g3 - g6 = scratch registers 2716 * g7 = TSB tag to match 2717 */ 2718 .align 64 2719 ALTENTRY(sfmmu_udtlb_fastpath) 2720 2721 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2722 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2723 ba,pn %xcc, sfmmu_tsb_miss_tt 2724 mov -1, %g3 2725 2726 /* 2727 * User instruction miss w/ multiple TSBs (sun4v). 2728 * The first probe covers 8K, 64K, and 512K page sizes, 2729 * because 64K and 512K mappings are replicated off 8K 2730 * pointer. Second probe covers 4M page size only. 2731 * 2732 * Just like sfmmu_udtlb_slowpath, except: 2733 * o Uses ASI_ITLB_IN 2734 * o checks for execute permission 2735 * o No ISM prediction. 2736 * 2737 * g1 = tsb8k pointer register 2738 * g2 = tag access register 2739 * g3 - g6 = scratch registers 2740 * g7 = TSB tag to match 2741 */ 2742 .align 64 2743 ALTENTRY(sfmmu_uitlb_slowpath) 2744 2745 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2746 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2747 /* g4 - g5 = clobbered here */ 2748 2749 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2750 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2751 srlx %g2, TAG_VALO_SHIFT, %g7 2752 PROBE_2ND_ITSB(%g3, %g7) 2753 /* NOT REACHED */ 2754 2755#else /* sun4v */ 2756 2757 /* 2758 * User instruction miss w/ multiple TSBs (sun4u). 2759 * The first probe covers 8K, 64K, and 512K page sizes, 2760 * because 64K and 512K mappings are replicated off 8K 2761 * pointer. Probe of 1st TSB has already been done prior to entry 2762 * into this routine. For the UTSB_PHYS case we probe up to 3 2763 * valid other TSBs in the following order: 2764 * 1) shared TSB for 4M-256M pages 2765 * 2) private TSB for 4M-256M pages 2766 * 3) shared TSB for 8K-512K pages 2767 * 2768 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs 2769 * 4M-256M pages. 2770 * 2771 * Just like sfmmu_udtlb_slowpath, except: 2772 * o Uses ASI_ITLB_IN 2773 * o checks for execute permission 2774 * o No ISM prediction. 2775 * 2776 * g1 = tsb8k pointer register 2777 * g2 = tag access register 2778 * g4 - g6 = scratch registers 2779 * g7 = TSB tag to match 2780 */ 2781 .align 64 2782 ALTENTRY(sfmmu_uitlb_slowpath) 2783 2784#ifdef UTSB_PHYS 2785 2786 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2787 brlz,pt %g6, 1f 2788 nop 2789 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2790 PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail) 27911: 2792 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2793 brlz,pt %g3, 2f 2794 nop 2795 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2796 PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail) 27972: 2798 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2799 brlz,pt %g6, sfmmu_tsb_miss_tt 2800 nop 2801 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2802 PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail) 2803 ba,pn %xcc, sfmmu_tsb_miss_tt 2804 nop 2805 2806#else /* UTSB_PHYS */ 2807 mov %g1, %g3 /* save tsb8k reg in %g3 */ 2808 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 2809 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2810 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 2811 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 2812 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 2813 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2814 srlx %g2, TAG_VALO_SHIFT, %g7 2815 PROBE_2ND_ITSB(%g3, %g7, isynth) 2816 ba,pn %xcc, sfmmu_tsb_miss_tt 2817 nop 2818 2819#endif /* UTSB_PHYS */ 2820#endif /* sun4v */ 2821 2822#if defined(sun4u) && defined(UTSB_PHYS) 2823 2824 /* 2825 * We come here for ism predict DTLB_MISS case or if 2826 * if probe in first TSB failed. 2827 */ 2828 2829 .align 64 2830 ALTENTRY(sfmmu_udtlb_slowpath_noismpred) 2831 2832 /* 2833 * g1 = tsb8k pointer register 2834 * g2 = tag access register 2835 * g4 - %g6 = scratch registers 2836 * g7 = TSB tag to match 2837 */ 2838 2839 /* 2840 * ISM non-predict probe order 2841 * probe 1ST_TSB (8K index) 2842 * probe 2ND_TSB (4M index) 2843 * probe 4TH_TSB (4M index) 2844 * probe 3RD_TSB (8K index) 2845 * 2846 * We already probed first TSB in DTLB_MISS handler. 2847 */ 2848 2849 /* 2850 * Private 2ND TSB 4M-256 pages 2851 */ 2852 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2853 brlz,pt %g3, 1f 2854 nop 2855 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2856 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2857 2858 /* 2859 * Shared Context 4TH TSB 4M-256 pages 2860 */ 28611: 2862 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2863 brlz,pt %g6, 2f 2864 nop 2865 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2866 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail) 2867 2868 /* 2869 * Shared Context 3RD TSB 8K-512K pages 2870 */ 28712: 2872 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2873 brlz,pt %g6, sfmmu_tsb_miss_tt 2874 nop 2875 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2876 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail) 2877 ba,pn %xcc, sfmmu_tsb_miss_tt 2878 nop 2879 2880 .align 64 2881 ALTENTRY(sfmmu_udtlb_slowpath_ismpred) 2882 2883 /* 2884 * g1 = tsb8k pointer register 2885 * g2 = tag access register 2886 * g4 - g6 = scratch registers 2887 * g7 = TSB tag to match 2888 */ 2889 2890 /* 2891 * ISM predict probe order 2892 * probe 4TH_TSB (4M index) 2893 * probe 2ND_TSB (4M index) 2894 * probe 1ST_TSB (8K index) 2895 * probe 3RD_TSB (8K index) 2896 2897 /* 2898 * Shared Context 4TH TSB 4M-256 pages 2899 */ 2900 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2901 brlz,pt %g6, 4f 2902 nop 2903 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2904 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2) 2905 2906 /* 2907 * Private 2ND TSB 4M-256 pages 2908 */ 29094: 2910 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2911 brlz,pt %g3, 5f 2912 nop 2913 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2914 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2) 2915 29165: 2917 PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2) 2918 2919 /* 2920 * Shared Context 3RD TSB 8K-512K pages 2921 */ 2922 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2923 brlz,pt %g6, 6f 2924 nop 2925 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2926 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2) 29276: 2928 ba,pn %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */ 2929 nop 2930 2931#else /* sun4u && UTSB_PHYS */ 2932 2933 .align 64 2934 ALTENTRY(sfmmu_udtlb_slowpath) 2935 2936 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 2937 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 2938 mov %g1, %g3 2939 2940udtlb_miss_probefirst: 2941 /* 2942 * g1 = 8K TSB pointer register 2943 * g2 = tag access register 2944 * g3 = (potentially) second TSB entry ptr 2945 * g6 = ism pred. 2946 * g7 = vpg_4m 2947 */ 2948#ifdef sun4v 2949 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2950 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2951 2952 /* 2953 * Here: 2954 * g1 = first TSB pointer 2955 * g2 = tag access reg 2956 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2957 */ 2958 brgz,pn %g6, sfmmu_tsb_miss_tt 2959 nop 2960#else /* sun4v */ 2961 mov %g1, %g4 2962 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 2963 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2964 2965 /* 2966 * Here: 2967 * g1 = first TSB pointer 2968 * g2 = tag access reg 2969 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2970 */ 2971 brgz,pn %g6, sfmmu_tsb_miss_tt 2972 nop 2973 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 2974 /* fall through in 8K->4M probe order */ 2975#endif /* sun4v */ 2976 2977udtlb_miss_probesecond: 2978 /* 2979 * Look in the second TSB for the TTE 2980 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 2981 * g2 = tag access reg 2982 * g3 = 8K TSB pointer register 2983 * g6 = ism pred. 2984 * g7 = vpg_4m 2985 */ 2986#ifdef sun4v 2987 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 2988 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2989 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 2990#else /* sun4v */ 2991 mov %g3, %g7 2992 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 2993 /* %g2 clobbered, %g3 =second tsbe ptr */ 2994 mov MMU_TAG_ACCESS, %g2 2995 ldxa [%g2]ASI_DMMU, %g2 2996#endif /* sun4v */ 2997 2998 srlx %g2, TAG_VALO_SHIFT, %g7 2999 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 3000 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 3001 brgz,pn %g6, udtlb_miss_probefirst 3002 nop 3003 3004 /* fall through to sfmmu_tsb_miss_tt */ 3005#endif /* sun4u && UTSB_PHYS */ 3006 3007 3008 ALTENTRY(sfmmu_tsb_miss_tt) 3009 TT_TRACE(trace_tsbmiss) 3010 /* 3011 * We get here if there is a TSB miss OR a write protect trap. 3012 * 3013 * g1 = First TSB entry pointer 3014 * g2 = tag access register 3015 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 3016 * g4 - g7 = scratch registers 3017 */ 3018 3019 ALTENTRY(sfmmu_tsb_miss) 3020 3021 /* 3022 * If trapstat is running, we need to shift the %tpc and %tnpc to 3023 * point to trapstat's TSB miss return code (note that trapstat 3024 * itself will patch the correct offset to add). 3025 */ 3026 rdpr %tl, %g7 3027 cmp %g7, 1 3028 ble,pt %xcc, 0f 3029 sethi %hi(KERNELBASE), %g6 3030 rdpr %tpc, %g7 3031 or %g6, %lo(KERNELBASE), %g6 3032 cmp %g7, %g6 3033 bgeu,pt %xcc, 0f 3034 /* delay slot safe */ 3035 3036 ALTENTRY(tsbmiss_trapstat_patch_point) 3037 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 3038 wrpr %g7, %tpc 3039 add %g7, 4, %g7 3040 wrpr %g7, %tnpc 30410: 3042 CPU_TSBMISS_AREA(%g6, %g7) 3043 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save 1ST tsb pointer */ 3044 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save 2ND tsb pointer */ 3045 3046 sllx %g2, TAGACC_CTX_LSHIFT, %g3 3047 brz,a,pn %g3, 1f /* skip ahead if kernel */ 3048 ldn [%g6 + TSBMISS_KHATID], %g7 3049 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 3050 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 3051 3052 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 3053 3054 cmp %g3, INVALID_CONTEXT 3055 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 3056 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 3057 3058#if defined(sun4v) || defined(UTSB_PHYS) 3059 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 3060 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 3061 stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 3062#endif /* sun4v || UTSB_PHYS */ 3063 3064 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 3065 /* 3066 * The miss wasn't in an ISM segment. 3067 * 3068 * %g1 %g3, %g4, %g5, %g7 all clobbered 3069 * %g2 = (pseudo) tag access 3070 */ 3071 3072 ba,pt %icc, 2f 3073 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 3074 30751: 3076 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 3077 /* 3078 * 8K and 64K hash. 3079 */ 30802: 3081 3082 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3083 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_l8K, tsb_checktte, 3084 sfmmu_suspend_tl, tsb_512K) 3085 /* NOT REACHED */ 3086 3087tsb_512K: 3088 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3089 brz,pn %g5, 3f 3090 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3091 and %g4, HAT_512K_FLAG, %g5 3092 3093 /* 3094 * Note that there is a small window here where we may have 3095 * a 512k page in the hash list but have not set the HAT_512K_FLAG 3096 * flag yet, so we will skip searching the 512k hash list. 3097 * In this case we will end up in pagefault which will find 3098 * the mapping and return. So, in this instance we will end up 3099 * spending a bit more time resolving this TSB miss, but it can 3100 * only happen once per process and even then, the chances of that 3101 * are very small, so it's not worth the extra overhead it would 3102 * take to close this window. 3103 */ 3104 brz,pn %g5, tsb_4M 3105 nop 31063: 3107 /* 3108 * 512K hash 3109 */ 3110 3111 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3112 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_l512K, tsb_checktte, 3113 sfmmu_suspend_tl, tsb_4M) 3114 /* NOT REACHED */ 3115 3116tsb_4M: 3117 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3118 brz,pn %g5, 4f 3119 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3120 and %g4, HAT_4M_FLAG, %g5 3121 brz,pn %g5, tsb_32M 3122 nop 31234: 3124 /* 3125 * 4M hash 3126 */ 3127 3128 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3129 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_l4M, tsb_checktte, 3130 sfmmu_suspend_tl, tsb_32M) 3131 /* NOT REACHED */ 3132 3133tsb_32M: 3134 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3135#ifdef sun4v 3136 brz,pn %g5, 6f 3137#else 3138 brz,pn %g5, tsb_pagefault 3139#endif 3140 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3141 and %g4, HAT_32M_FLAG, %g5 3142 brz,pn %g5, tsb_256M 3143 nop 31445: 3145 /* 3146 * 32M hash 3147 */ 3148 3149 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3150 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_l32M, tsb_checktte, 3151 sfmmu_suspend_tl, tsb_256M) 3152 /* NOT REACHED */ 3153 3154#if defined(sun4u) && !defined(UTSB_PHYS) 3155#define tsb_shme tsb_pagefault 3156#endif 3157tsb_256M: 3158 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 3159 and %g4, HAT_256M_FLAG, %g5 3160 brz,pn %g5, tsb_shme 3161 nop 31626: 3163 /* 3164 * 256M hash 3165 */ 3166 3167 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3168 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_l256M, tsb_checktte, 3169 sfmmu_suspend_tl, tsb_shme) 3170 /* NOT REACHED */ 3171 3172tsb_checktte: 3173 /* 3174 * g1 = hblk_misc 3175 * g2 = tagacc 3176 * g3 = tte 3177 * g4 = tte pa 3178 * g6 = tsbmiss area 3179 * g7 = hatid 3180 */ 3181 brlz,a,pt %g3, tsb_validtte 3182 rdpr %tt, %g7 3183 3184#if defined(sun4u) && !defined(UTSB_PHYS) 3185#undef tsb_shme 3186 ba tsb_pagefault 3187 nop 3188#else /* sun4u && !UTSB_PHYS */ 3189 3190tsb_shme: 3191 /* 3192 * g2 = tagacc 3193 * g6 = tsbmiss area 3194 */ 3195 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3196 brz,pn %g5, tsb_pagefault 3197 nop 3198 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3199 brz,pn %g7, tsb_pagefault 3200 nop 3201 3202 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3203 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_shme_l8K, tsb_shme_checktte, 3204 sfmmu_suspend_tl, tsb_shme_512K) 3205 /* NOT REACHED */ 3206 3207tsb_shme_512K: 3208 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3209 and %g4, HAT_512K_FLAG, %g5 3210 brz,pn %g5, tsb_shme_4M 3211 nop 3212 3213 /* 3214 * 512K hash 3215 */ 3216 3217 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3218 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_shme_l512K, tsb_shme_checktte, 3219 sfmmu_suspend_tl, tsb_shme_4M) 3220 /* NOT REACHED */ 3221 3222tsb_shme_4M: 3223 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3224 and %g4, HAT_4M_FLAG, %g5 3225 brz,pn %g5, tsb_shme_32M 3226 nop 32274: 3228 /* 3229 * 4M hash 3230 */ 3231 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3232 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_shme_l4M, tsb_shme_checktte, 3233 sfmmu_suspend_tl, tsb_shme_32M) 3234 /* NOT REACHED */ 3235 3236tsb_shme_32M: 3237 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3238 and %g4, HAT_32M_FLAG, %g5 3239 brz,pn %g5, tsb_shme_256M 3240 nop 3241 3242 /* 3243 * 32M hash 3244 */ 3245 3246 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3247 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_shme_l32M, tsb_shme_checktte, 3248 sfmmu_suspend_tl, tsb_shme_256M) 3249 /* NOT REACHED */ 3250 3251tsb_shme_256M: 3252 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3253 and %g4, HAT_256M_FLAG, %g5 3254 brz,pn %g5, tsb_pagefault 3255 nop 3256 3257 /* 3258 * 256M hash 3259 */ 3260 3261 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3262 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_shme_l256M, tsb_shme_checktte, 3263 sfmmu_suspend_tl, tsb_pagefault) 3264 /* NOT REACHED */ 3265 3266tsb_shme_checktte: 3267 3268 brgez,pn %g3, tsb_pagefault 3269 rdpr %tt, %g7 3270 /* 3271 * g1 = ctx1 flag 3272 * g3 = tte 3273 * g4 = tte pa 3274 * g6 = tsbmiss area 3275 * g7 = tt 3276 */ 3277 3278 brz,pt %g1, tsb_validtte 3279 nop 3280 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3281 or %g1, HAT_CHKCTX1_FLAG, %g1 3282 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3283 3284 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3285#endif /* sun4u && !UTSB_PHYS */ 3286 3287tsb_validtte: 3288 /* 3289 * g3 = tte 3290 * g4 = tte pa 3291 * g6 = tsbmiss area 3292 * g7 = tt 3293 */ 3294 3295 /* 3296 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3297 */ 3298 cmp %g7, FAST_PROT_TT 3299 bne,pt %icc, 4f 3300 nop 3301 3302 TTE_SET_REFMOD_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_refmod, 3303 tsb_protfault) 3304 3305 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3306#ifdef sun4v 3307 MMU_FAULT_STATUS_AREA(%g7) 3308 ldx [%g7 + MMFSA_D_ADDR], %g5 /* load fault addr for later */ 3309#else /* sun4v */ 3310 mov MMU_TAG_ACCESS, %g5 3311 ldxa [%g5]ASI_DMMU, %g5 3312#endif /* sun4v */ 3313 ba,pt %xcc, tsb_update_tl1 3314 nop 33154: 3316 /* 3317 * If ITLB miss check exec bit. 3318 * If not set treat as invalid TTE. 3319 */ 3320 cmp %g7, T_INSTR_MMU_MISS 3321 be,pn %icc, 5f 3322 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3323 cmp %g7, FAST_IMMU_MISS_TT 3324 bne,pt %icc, 3f 3325 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 33265: 3327 bz,pn %icc, tsb_protfault 3328 nop 3329 33303: 3331 /* 3332 * Set reference bit if not already set 3333 */ 3334 TTE_SET_REF_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_ref) 3335 3336 /* 3337 * Now, load into TSB/TLB. At this point: 3338 * g3 = tte 3339 * g4 = patte 3340 * g6 = tsbmiss area 3341 */ 3342 rdpr %tt, %g7 3343#ifdef sun4v 3344 MMU_FAULT_STATUS_AREA(%g2) 3345 cmp %g7, T_INSTR_MMU_MISS 3346 be,a,pt %icc, 9f 3347 nop 3348 cmp %g7, FAST_IMMU_MISS_TT 3349 be,a,pt %icc, 9f 3350 nop 3351 add %g2, MMFSA_D_, %g2 33529: 3353 ldx [%g2 + MMFSA_CTX_], %g7 3354 sllx %g7, TTARGET_CTX_SHIFT, %g7 3355 ldx [%g2 + MMFSA_ADDR_], %g2 3356 mov %g2, %g5 ! load the fault addr for later use 3357 srlx %g2, TTARGET_VA_SHIFT, %g2 3358 or %g2, %g7, %g2 3359#else /* sun4v */ 3360 mov MMU_TAG_ACCESS, %g5 3361 cmp %g7, FAST_IMMU_MISS_TT 3362 be,a,pt %icc, 9f 3363 ldxa [%g0]ASI_IMMU, %g2 3364 ldxa [%g0]ASI_DMMU, %g2 3365 ba,pt %icc, tsb_update_tl1 3366 ldxa [%g5]ASI_DMMU, %g5 33679: 3368 ldxa [%g5]ASI_IMMU, %g5 3369#endif /* sun4v */ 3370 3371tsb_update_tl1: 3372 srlx %g2, TTARGET_CTX_SHIFT, %g7 3373 brz,pn %g7, tsb_kernel 3374#ifdef sun4v 3375 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3376#else /* sun4v */ 3377 srlx %g3, TTE_SZ_SHFT, %g7 3378#endif /* sun4v */ 3379 3380tsb_user: 3381#ifdef sun4v 3382 cmp %g7, TTE4M 3383 bge,pn %icc, tsb_user4m 3384 nop 3385#else /* sun4v */ 3386 cmp %g7, TTESZ_VALID | TTE4M 3387 be,pn %icc, tsb_user4m 3388 srlx %g3, TTE_SZ2_SHFT, %g7 3389 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3390#ifdef ITLB_32M_256M_SUPPORT 3391 bnz,pn %icc, tsb_user4m 3392 nop 3393#else /* ITLB_32M_256M_SUPPORT */ 3394 bnz,a,pn %icc, tsb_user_pn_synth 3395 nop 3396#endif /* ITLB_32M_256M_SUPPORT */ 3397#endif /* sun4v */ 3398 3399tsb_user8k: 3400#if defined(sun4v) || defined(UTSB_PHYS) 3401 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3402 and %g7, HAT_CHKCTX1_FLAG, %g1 3403 brz,a,pn %g1, 1f 3404 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3405 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3406 brlz,a,pn %g1, ptl1_panic ! if no shared 3RD tsb 3407 mov PTL1_NO_SCDTSB8K, %g1 ! panic 3408 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 34091: 3410#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3411 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3412#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3413 3414#ifndef UTSB_PHYS 3415 mov ASI_N, %g7 ! user TSBs accessed by VA 3416 mov %g7, %asi 3417#endif /* !UTSB_PHYS */ 3418 3419 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l3) 3420 3421 rdpr %tt, %g5 3422#ifdef sun4v 3423 cmp %g5, T_INSTR_MMU_MISS 3424 be,a,pn %xcc, 9f 3425 mov %g3, %g5 3426#endif /* sun4v */ 3427 cmp %g5, FAST_IMMU_MISS_TT 3428 be,pn %xcc, 9f 3429 mov %g3, %g5 3430 3431 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3432 ! trapstat wants TTE in %g5 3433 retry 34349: 3435 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3436 ! trapstat wants TTE in %g5 3437 retry 3438 3439tsb_user4m: 3440#if defined(sun4v) || defined(UTSB_PHYS) 3441 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3442 and %g7, HAT_CHKCTX1_FLAG, %g1 3443 brz,a,pn %g1, 4f 3444 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3445 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr 3446 brlz,a,pn %g1, 5f ! if no shared 4TH TSB 3447 nop 3448 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 3449 3450#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3451 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3452#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 34534: 3454 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3455 nop 3456 3457#ifndef UTSB_PHYS 3458 mov ASI_N, %g7 ! user TSBs accessed by VA 3459 mov %g7, %asi 3460#endif /* UTSB_PHYS */ 3461 3462 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l4) 3463 34645: 3465 rdpr %tt, %g5 3466#ifdef sun4v 3467 cmp %g5, T_INSTR_MMU_MISS 3468 be,a,pn %xcc, 9f 3469 mov %g3, %g5 3470#endif /* sun4v */ 3471 cmp %g5, FAST_IMMU_MISS_TT 3472 be,pn %xcc, 9f 3473 mov %g3, %g5 3474 3475 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3476 ! trapstat wants TTE in %g5 3477 retry 34789: 3479 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3480 ! trapstat wants TTE in %g5 3481 retry 3482 3483#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3484 /* 3485 * Panther ITLB synthesis. 3486 * The Panther 32M and 256M ITLB code simulates these two large page 3487 * sizes with 4M pages, to provide support for programs, for example 3488 * Java, that may copy instructions into a 32M or 256M data page and 3489 * then execute them. The code below generates the 4M pfn bits and 3490 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3491 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3492 * are ignored by the hardware. 3493 * 3494 * Now, load into TSB/TLB. At this point: 3495 * g2 = tagtarget 3496 * g3 = tte 3497 * g4 = patte 3498 * g5 = tt 3499 * g6 = tsbmiss area 3500 */ 3501tsb_user_pn_synth: 3502 rdpr %tt, %g5 3503 cmp %g5, FAST_IMMU_MISS_TT 3504 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3505 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3506 bz,pn %icc, 4b /* if not, been here before */ 3507 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3508 brlz,a,pn %g1, 5f /* no 2nd tsb */ 3509 mov %g3, %g5 3510 3511 mov MMU_TAG_ACCESS, %g7 3512 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3513 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3514 3515 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3516 mov %g7, %asi 3517 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l5) /* update TSB */ 35185: 3519 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3520 retry 3521 3522tsb_user_itlb_synth: 3523 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = 2ND TSB */ 3524 3525 mov MMU_TAG_ACCESS, %g7 3526 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3527 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3528 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3529 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3530 3531 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3532 mov %g7, %asi 3533 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l6) /* update TSB */ 35347: 3535 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3536 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3537 retry 3538#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3539 3540tsb_kernel: 3541 rdpr %tt, %g5 3542#ifdef sun4v 3543 cmp %g7, TTE4M 3544 bge,pn %icc, 5f 3545#else 3546 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3547 be,pn %icc, 5f 3548#endif /* sun4v */ 3549 nop 3550 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8K TSB ptr 3551 ba,pt %xcc, 6f 3552 nop 35535: 3554 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4M TSB ptr 3555 brlz,pn %g1, 3f /* skip programming if 4M TSB ptr is -1 */ 3556 nop 35576: 3558#ifndef sun4v 3559tsb_kernel_patch_asi: 3560 or %g0, RUNTIME_PATCH, %g6 3561 mov %g6, %asi ! XXX avoid writing to %asi !! 3562#endif 3563 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l7) 35643: 3565#ifdef sun4v 3566 cmp %g5, T_INSTR_MMU_MISS 3567 be,a,pn %icc, 1f 3568 mov %g3, %g5 ! trapstat wants TTE in %g5 3569#endif /* sun4v */ 3570 cmp %g5, FAST_IMMU_MISS_TT 3571 be,pn %icc, 1f 3572 mov %g3, %g5 ! trapstat wants TTE in %g5 3573 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3574 ! trapstat wants TTE in %g5 3575 retry 35761: 3577 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3578 ! trapstat wants TTE in %g5 3579 retry 3580 3581tsb_ism: 3582 /* 3583 * This is an ISM [i|d]tlb miss. We optimize for largest 3584 * page size down to smallest. 3585 * 3586 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3587 * register 3588 * g3 = ismmap->ism_seg 3589 * g4 = physical address of ismmap->ism_sfmmu 3590 * g6 = tsbmiss area 3591 */ 3592 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3593 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3594 mov PTL1_BAD_ISM, %g1 3595 /* g5 = pa of imap_vb_shift */ 3596 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3597 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3598 srlx %g3, %g4, %g3 /* clr size field */ 3599 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3600 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3601 and %g2, %g1, %g4 /* g4 = ctx number */ 3602 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3603 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3604 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3605 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3606 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3607#if defined(sun4v) || defined(UTSB_PHYS) 3608 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3609 brz,pt %g5, tsb_chk4M_ism 3610 nop 3611 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3612 or %g5, HAT_CHKCTX1_FLAG, %g5 3613 stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3614 rdpr %tt, %g5 3615 SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl) 3616#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3617 3618 /* 3619 * ISM pages are always locked down. 3620 * If we can't find the tte then pagefault 3621 * and let the spt segment driver resolve it. 3622 * 3623 * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3624 * g4 = imap_hatflags 3625 * g6 = tsb miss area 3626 * g7 = ISM hatid 3627 */ 3628 3629tsb_chk4M_ism: 3630 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3631 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3632 nop 3633 3634tsb_ism_32M: 3635 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3636 brz,pn %g5, tsb_ism_256M 3637 nop 3638 3639 /* 3640 * 32M hash. 3641 */ 3642 3643 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT32M, 3644 TTE32M, %g5, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3645 tsb_ism_4M) 3646 /* NOT REACHED */ 3647 3648tsb_ism_32M_found: 3649 brlz,a,pt %g3, tsb_validtte 3650 rdpr %tt, %g7 3651 ba,pt %xcc, tsb_ism_4M 3652 nop 3653 3654tsb_ism_256M: 3655 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3656 brz,a,pn %g5, ptl1_panic 3657 mov PTL1_BAD_ISM, %g1 3658 3659 /* 3660 * 256M hash. 3661 */ 3662 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT256M, 3663 TTE256M, %g5, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3664 tsb_ism_4M) 3665 3666tsb_ism_256M_found: 3667 brlz,a,pt %g3, tsb_validtte 3668 rdpr %tt, %g7 3669 3670tsb_ism_4M: 3671 /* 3672 * 4M hash. 3673 */ 3674 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT4M, 3675 TTE4M, %g5, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3676 tsb_ism_8K) 3677 /* NOT REACHED */ 3678 3679tsb_ism_4M_found: 3680 brlz,a,pt %g3, tsb_validtte 3681 rdpr %tt, %g7 3682 3683tsb_ism_8K: 3684 /* 3685 * 8K and 64K hash. 3686 */ 3687 3688 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT64K, 3689 TTE64K, %g5, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3690 tsb_pagefault) 3691 /* NOT REACHED */ 3692 3693tsb_ism_8K_found: 3694 brlz,a,pt %g3, tsb_validtte 3695 rdpr %tt, %g7 3696 3697tsb_pagefault: 3698 rdpr %tt, %g7 3699 cmp %g7, FAST_PROT_TT 3700 be,a,pn %icc, tsb_protfault 3701 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3702 3703tsb_protfault: 3704 /* 3705 * we get here if we couldn't find a valid tte in the hash. 3706 * 3707 * If user and we are at tl>1 we go to window handling code. 3708 * 3709 * If kernel and the fault is on the same page as our stack 3710 * pointer, then we know the stack is bad and the trap handler 3711 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3712 * 3713 * If this is a kernel trap and tl>1, panic. 3714 * 3715 * Otherwise we call pagefault. 3716 */ 3717 cmp %g7, FAST_IMMU_MISS_TT 3718#ifdef sun4v 3719 MMU_FAULT_STATUS_AREA(%g4) 3720 ldx [%g4 + MMFSA_I_CTX], %g5 3721 ldx [%g4 + MMFSA_D_CTX], %g4 3722 move %icc, %g5, %g4 3723 cmp %g7, T_INSTR_MMU_MISS 3724 move %icc, %g5, %g4 3725#else 3726 mov MMU_TAG_ACCESS, %g4 3727 ldxa [%g4]ASI_DMMU, %g2 3728 ldxa [%g4]ASI_IMMU, %g5 3729 move %icc, %g5, %g2 3730 cmp %g7, T_INSTR_MMU_MISS 3731 move %icc, %g5, %g2 3732 sllx %g2, TAGACC_CTX_LSHIFT, %g4 3733#endif /* sun4v */ 3734 brnz,pn %g4, 3f /* skip if not kernel */ 3735 rdpr %tl, %g5 3736 3737 add %sp, STACK_BIAS, %g3 3738 srlx %g3, MMU_PAGESHIFT, %g3 3739 srlx %g2, MMU_PAGESHIFT, %g4 3740 cmp %g3, %g4 3741 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 3742 mov PTL1_BAD_STACK, %g1 3743 3744 cmp %g5, 1 3745 ble,pt %icc, 2f 3746 nop 3747 TSTAT_CHECK_TL1(2f, %g1, %g2) 3748 rdpr %tt, %g2 3749 cmp %g2, FAST_PROT_TT 3750 mov PTL1_BAD_KPROT_FAULT, %g1 3751 movne %icc, PTL1_BAD_KMISS, %g1 3752 ba,pt %icc, ptl1_panic 3753 nop 3754 37552: 3756 /* 3757 * We are taking a pagefault in the kernel on a kernel address. If 3758 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 3759 * want to call sfmmu_pagefault -- we will instead note that a fault 3760 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 3761 * (instead of a "retry"). This will step over the faulting 3762 * instruction. 3763 */ 3764 CPU_INDEX(%g1, %g2) 3765 set cpu_core, %g2 3766 sllx %g1, CPU_CORE_SHIFT, %g1 3767 add %g1, %g2, %g1 3768 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3769 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3770 bz sfmmu_pagefault 3771 or %g2, CPU_DTRACE_BADADDR, %g2 3772 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3773 GET_MMU_D_ADDR(%g3, %g4) 3774 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3775 done 3776 37773: 3778 cmp %g5, 1 3779 ble,pt %icc, 4f 3780 nop 3781 TSTAT_CHECK_TL1(4f, %g1, %g2) 3782 ba,pt %icc, sfmmu_window_trap 3783 nop 3784 37854: 3786 /* 3787 * We are taking a pagefault on a non-kernel address. If we are in 3788 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 3789 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 3790 */ 3791 CPU_INDEX(%g1, %g2) 3792 set cpu_core, %g2 3793 sllx %g1, CPU_CORE_SHIFT, %g1 3794 add %g1, %g2, %g1 3795 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3796 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3797 bz sfmmu_mmu_trap 3798 or %g2, CPU_DTRACE_BADADDR, %g2 3799 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3800 GET_MMU_D_ADDR(%g3, %g4) 3801 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3802 3803 /* 3804 * Be sure that we're actually taking this miss from the kernel -- 3805 * otherwise we have managed to return to user-level with 3806 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3807 */ 3808 rdpr %tstate, %g2 3809 btst TSTATE_PRIV, %g2 3810 bz,a ptl1_panic 3811 mov PTL1_BAD_DTRACE_FLAGS, %g1 3812 done 3813 3814 ALTENTRY(tsb_tl0_noctxt) 3815 /* 3816 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 3817 * if it is, indicated that we have faulted and issue a done. 3818 */ 3819 CPU_INDEX(%g5, %g6) 3820 set cpu_core, %g6 3821 sllx %g5, CPU_CORE_SHIFT, %g5 3822 add %g5, %g6, %g5 3823 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 3824 andcc %g6, CPU_DTRACE_NOFAULT, %g0 3825 bz 1f 3826 or %g6, CPU_DTRACE_BADADDR, %g6 3827 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 3828 GET_MMU_D_ADDR(%g3, %g4) 3829 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 3830 3831 /* 3832 * Be sure that we're actually taking this miss from the kernel -- 3833 * otherwise we have managed to return to user-level with 3834 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3835 */ 3836 rdpr %tstate, %g5 3837 btst TSTATE_PRIV, %g5 3838 bz,a ptl1_panic 3839 mov PTL1_BAD_DTRACE_FLAGS, %g1 3840 TSTAT_CHECK_TL1(2f, %g1, %g2); 38412: 3842 done 3843 38441: 3845 rdpr %tt, %g5 3846 cmp %g5, FAST_IMMU_MISS_TT 3847#ifdef sun4v 3848 MMU_FAULT_STATUS_AREA(%g2) 3849 be,a,pt %icc, 2f 3850 ldx [%g2 + MMFSA_I_CTX], %g3 3851 cmp %g5, T_INSTR_MMU_MISS 3852 be,a,pt %icc, 2f 3853 ldx [%g2 + MMFSA_I_CTX], %g3 3854 ldx [%g2 + MMFSA_D_CTX], %g3 38552: 3856#else 3857 mov MMU_TAG_ACCESS, %g2 3858 be,a,pt %icc, 2f 3859 ldxa [%g2]ASI_IMMU, %g3 3860 ldxa [%g2]ASI_DMMU, %g3 38612: sllx %g3, TAGACC_CTX_LSHIFT, %g3 3862#endif /* sun4v */ 3863 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 3864 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 3865 rdpr %tl, %g5 3866 cmp %g5, 1 3867 ble,pt %icc, sfmmu_mmu_trap 3868 nop 3869 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 3870 ba,pt %icc, sfmmu_window_trap 3871 nop 3872 SET_SIZE(sfmmu_tsb_miss) 3873#endif /* lint */ 3874 3875#if defined (lint) 3876/* 3877 * This routine will look for a user or kernel vaddr in the hash 3878 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 3879 * grab any locks. It should only be used by other sfmmu routines. 3880 */ 3881/* ARGSUSED */ 3882pfn_t 3883sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 3884{ 3885 return(0); 3886} 3887 3888/* ARGSUSED */ 3889pfn_t 3890sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno) 3891{ 3892 return(0); 3893} 3894 3895#else /* lint */ 3896 3897 ENTRY_NP(sfmmu_vatopfn) 3898 /* 3899 * disable interrupts 3900 */ 3901 rdpr %pstate, %o3 3902#ifdef DEBUG 3903 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 3904#endif 3905 /* 3906 * disable interrupts to protect the TSBMISS area 3907 */ 3908 andn %o3, PSTATE_IE, %o5 3909 wrpr %o5, 0, %pstate 3910 3911 /* 3912 * o0 = vaddr 3913 * o1 = sfmmup 3914 * o2 = ttep 3915 */ 3916 CPU_TSBMISS_AREA(%g1, %o5) 3917 ldn [%g1 + TSBMISS_KHATID], %o4 3918 cmp %o4, %o1 3919 bne,pn %ncc, vatopfn_nokernel 3920 mov TTE64K, %g5 /* g5 = rehash # */ 3921 mov %g1,%o5 /* o5 = tsbmiss_area */ 3922 /* 3923 * o0 = vaddr 3924 * o1 & o4 = hatid 3925 * o2 = ttep 3926 * o5 = tsbmiss area 3927 */ 3928 mov HBLK_RANGE_SHIFT, %g6 39291: 3930 3931 /* 3932 * o0 = vaddr 3933 * o1 = sfmmup 3934 * o2 = ttep 3935 * o3 = old %pstate 3936 * o4 = hatid 3937 * o5 = tsbmiss 3938 * g5 = rehash # 3939 * g6 = hmeshift 3940 * 3941 * The first arg to GET_TTE is actually tagaccess register 3942 * not just vaddr. Since this call is for kernel we need to clear 3943 * any lower vaddr bits that would be interpreted as ctx bits. 3944 */ 3945 set TAGACC_CTX_MASK, %g1 3946 andn %o0, %g1, %o0 3947 GET_TTE(%o0, %o4, %g1, %g2, %o5, %g4, %g6, %g5, %g3, 3948 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 3949 3950kvtop_hblk_found: 3951 /* 3952 * o0 = vaddr 3953 * o1 = sfmmup 3954 * o2 = ttep 3955 * g1 = tte 3956 * g2 = tte pa 3957 * g3 = scratch 3958 * o2 = tsbmiss area 3959 * o1 = hat id 3960 */ 3961 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 3962 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3963 stx %g1,[%o2] /* put tte into *ttep */ 3964 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 3965 /* 3966 * o0 = vaddr 3967 * o1 = sfmmup 3968 * o2 = ttep 3969 * g1 = pfn 3970 */ 3971 ba,pt %xcc, 6f 3972 mov %g1, %o0 3973 3974kvtop_nohblk: 3975 /* 3976 * we get here if we couldn't find valid hblk in hash. We rehash 3977 * if neccesary. 3978 */ 3979 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 3980#ifdef sun4v 3981 cmp %g5, MAX_HASHCNT 3982#else 3983 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 3984#endif /* sun4v */ 3985 be,a,pn %icc, 6f 3986 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3987 mov %o1, %o4 /* restore hatid */ 3988#ifdef sun4v 3989 add %g5, 2, %g5 3990 cmp %g5, 3 3991 move %icc, MMU_PAGESHIFT4M, %g6 3992 ba,pt %icc, 1b 3993 movne %icc, MMU_PAGESHIFT256M, %g6 3994#else 3995 inc %g5 3996 cmp %g5, 2 3997 move %icc, MMU_PAGESHIFT512K, %g6 3998 ba,pt %icc, 1b 3999 movne %icc, MMU_PAGESHIFT4M, %g6 4000#endif /* sun4v */ 40016: 4002 retl 4003 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4004 4005tsb_suspend: 4006 /* 4007 * o0 = vaddr 4008 * o1 = sfmmup 4009 * o2 = ttep 4010 * g1 = tte 4011 * g2 = tte pa 4012 * g3 = tte va 4013 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 4014 */ 4015 stx %g1,[%o2] /* put tte into *ttep */ 4016 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 4017 sub %g0, 1, %o0 /* output = PFN_INVALID */ 4018 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 40198: 4020 retl 4021 wrpr %g0, %o3, %pstate /* enable interrupts */ 4022 4023vatopfn_nokernel: 4024 /* 4025 * This routine does NOT support user addresses 4026 * There is a routine in C that supports this. 4027 * The only reason why we don't have the C routine 4028 * support kernel addresses as well is because 4029 * we do va_to_pa while holding the hashlock. 4030 */ 4031 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4032 save %sp, -SA(MINFRAME), %sp 4033 sethi %hi(sfmmu_panic3), %o0 4034 call panic 4035 or %o0, %lo(sfmmu_panic3), %o0 4036 4037 SET_SIZE(sfmmu_vatopfn) 4038 4039 /* 4040 * %o0 = vaddr 4041 * %o1 = hashno (aka szc) 4042 * 4043 * 4044 * This routine is similar to sfmmu_vatopfn() but will only look for 4045 * a kernel vaddr in the hash structure for the specified rehash value. 4046 * It's just an optimization for the case when pagesize for a given 4047 * va range is already known (e.g. large page heap) and we don't want 4048 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 4049 * 4050 * Returns valid pfn or PFN_INVALID if 4051 * tte for specified rehash # is not found, invalid or suspended. 4052 */ 4053 ENTRY_NP(sfmmu_kvaszc2pfn) 4054 /* 4055 * disable interrupts 4056 */ 4057 rdpr %pstate, %o3 4058#ifdef DEBUG 4059 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 4060#endif 4061 /* 4062 * disable interrupts to protect the TSBMISS area 4063 */ 4064 andn %o3, PSTATE_IE, %o5 4065 wrpr %o5, 0, %pstate 4066 4067 CPU_TSBMISS_AREA(%g1, %o5) 4068 ldn [%g1 + TSBMISS_KHATID], %o4 4069 sll %o1, 1, %g6 4070 add %g6, %o1, %g6 4071 add %g6, MMU_PAGESHIFT, %g6 4072 /* 4073 * %o0 = vaddr 4074 * %o1 = hashno 4075 * %o3 = old %pstate 4076 * %o4 = ksfmmup 4077 * %g1 = tsbmiss area 4078 * %g6 = hmeshift 4079 */ 4080 4081 /* 4082 * The first arg to GET_TTE is actually tagaccess register 4083 * not just vaddr. Since this call is for kernel we need to clear 4084 * any lower vaddr bits that would be interpreted as ctx bits. 4085 */ 4086 srlx %o0, MMU_PAGESHIFT, %o0 4087 sllx %o0, MMU_PAGESHIFT, %o0 4088 GET_TTE(%o0, %o4, %g3, %g4, %g1, %o5, %g6, %o1, %g5, 4089 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 4090 kvaszc2pfn_nohblk) 4091 4092kvaszc2pfn_hblk_found: 4093 /* 4094 * %g3 = tte 4095 * %o0 = vaddr 4096 */ 4097 brgez,a,pn %g3, 1f /* check if tte is invalid */ 4098 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 4099 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 4100 /* 4101 * g3 = pfn 4102 */ 4103 ba,pt %xcc, 1f 4104 mov %g3, %o0 4105 4106kvaszc2pfn_nohblk: 4107 mov -1, %o0 4108 41091: 4110 retl 4111 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 4112 4113 SET_SIZE(sfmmu_kvaszc2pfn) 4114 4115#endif /* lint */ 4116 4117 4118 4119#if !defined(lint) 4120 4121/* 4122 * kpm lock used between trap level tsbmiss handler and kpm C level. 4123 */ 4124#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 4125 mov 0xff, tmp1 ;\ 4126label1: ;\ 4127 casa [kpmlckp]asi, %g0, tmp1 ;\ 4128 brnz,pn tmp1, label1 ;\ 4129 mov 0xff, tmp1 ;\ 4130 membar #LoadLoad 4131 4132#define KPMLOCK_EXIT(kpmlckp, asi) \ 4133 membar #LoadStore|#StoreStore ;\ 4134 sta %g0, [kpmlckp]asi 4135 4136/* 4137 * Lookup a memseg for a given pfn and if found, return the physical 4138 * address of the corresponding struct memseg in mseg, otherwise 4139 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 4140 * tsbmp, %asi is assumed to be ASI_MEM. 4141 * This lookup is done by strictly traversing only the physical memseg 4142 * linkage. The more generic approach, to check the virtual linkage 4143 * before using the physical (used e.g. with hmehash buckets), cannot 4144 * be used here. Memory DR operations can run in parallel to this 4145 * lookup w/o any locks and updates of the physical and virtual linkage 4146 * cannot be done atomically wrt. to each other. Because physical 4147 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 4148 * as "physical NULL" pointer. 4149 */ 4150#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 4151 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 4152 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 4153 udivx pfn, mseg, mseg ;\ 4154 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 4155 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 4156 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 4157 add tmp1, mseg, tmp1 ;\ 4158 ldxa [tmp1]%asi, mseg ;\ 4159 cmp mseg, MSEG_NULLPTR_PA ;\ 4160 be,pn %xcc, label/**/1 /* if not found */ ;\ 4161 nop ;\ 4162 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4163 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4164 blu,pn %xcc, label/**/1 ;\ 4165 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4166 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4167 bgeu,pn %xcc, label/**/1 ;\ 4168 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 4169 mulx tmp1, PAGE_SIZE, tmp1 ;\ 4170 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 4171 add tmp2, tmp1, tmp1 /* pp */ ;\ 4172 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 4173 cmp tmp2, pfn ;\ 4174 be,pt %xcc, label/**/_ok /* found */ ;\ 4175label/**/1: ;\ 4176 /* brute force lookup */ ;\ 4177 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 4178 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 4179label/**/2: ;\ 4180 cmp mseg, MSEG_NULLPTR_PA ;\ 4181 be,pn %xcc, label/**/_ok /* if not found */ ;\ 4182 nop ;\ 4183 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 4184 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 4185 blu,a,pt %xcc, label/**/2 ;\ 4186 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4187 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 4188 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 4189 bgeu,a,pt %xcc, label/**/2 ;\ 4190 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 4191label/**/_ok: 4192 4193 /* 4194 * kpm tsb miss handler large pages 4195 * g1 = 8K kpm TSB entry pointer 4196 * g2 = tag access register 4197 * g3 = 4M kpm TSB entry pointer 4198 */ 4199 ALTENTRY(sfmmu_kpm_dtsb_miss) 4200 TT_TRACE(trace_tsbmiss) 4201 4202 CPU_INDEX(%g7, %g6) 4203 sethi %hi(kpmtsbm_area), %g6 4204 sllx %g7, KPMTSBM_SHIFT, %g7 4205 or %g6, %lo(kpmtsbm_area), %g6 4206 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4207 4208 /* check enable flag */ 4209 ldub [%g6 + KPMTSBM_FLAGS], %g4 4210 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4211 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4212 nop 4213 4214 /* VA range check */ 4215 ldx [%g6 + KPMTSBM_VBASE], %g7 4216 cmp %g2, %g7 4217 blu,pn %xcc, sfmmu_tsb_miss 4218 ldx [%g6 + KPMTSBM_VEND], %g5 4219 cmp %g2, %g5 4220 bgeu,pn %xcc, sfmmu_tsb_miss 4221 stx %g3, [%g6 + KPMTSBM_TSBPTR] 4222 4223 /* 4224 * check TL tsbmiss handling flag 4225 * bump tsbmiss counter 4226 */ 4227 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4228#ifdef DEBUG 4229 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4230 inc %g5 4231 brz,pn %g3, sfmmu_kpm_exception 4232 st %g5, [%g6 + KPMTSBM_TSBMISS] 4233#else 4234 inc %g5 4235 st %g5, [%g6 + KPMTSBM_TSBMISS] 4236#endif 4237 /* 4238 * At this point: 4239 * g1 = 8K kpm TSB pointer (not used) 4240 * g2 = tag access register 4241 * g3 = clobbered 4242 * g6 = per-CPU kpm tsbmiss area 4243 * g7 = kpm_vbase 4244 */ 4245 4246 /* vaddr2pfn */ 4247 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4248 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4249 srax %g4, %g3, %g2 /* which alias range (r) */ 4250 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4251 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4252 4253 /* 4254 * Setup %asi 4255 * mseg_pa = page_numtomemseg_nolock(pfn) 4256 * if (mseg_pa == NULL) sfmmu_kpm_exception 4257 * g2=pfn 4258 */ 4259 mov ASI_MEM, %asi 4260 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4261 cmp %g3, MSEG_NULLPTR_PA 4262 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4263 nop 4264 4265 /* 4266 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4267 * g2=pfn g3=mseg_pa 4268 */ 4269 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4270 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4271 srlx %g2, %g5, %g4 4272 sllx %g4, %g5, %g4 4273 sub %g4, %g7, %g4 4274 srlx %g4, %g5, %g4 4275 4276 /* 4277 * Validate inx value 4278 * g2=pfn g3=mseg_pa g4=inx 4279 */ 4280#ifdef DEBUG 4281 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4282 cmp %g4, %g5 /* inx - nkpmpgs */ 4283 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4284 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4285#else 4286 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4287#endif 4288 /* 4289 * kp = &mseg_pa->kpm_pages[inx] 4290 */ 4291 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4292 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4293 add %g5, %g4, %g5 /* kp */ 4294 4295 /* 4296 * KPMP_HASH(kp) 4297 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4298 */ 4299 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4300 sub %g7, 1, %g7 /* mask */ 4301 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4302 add %g5, %g1, %g5 /* y = ksp + x */ 4303 and %g5, %g7, %g5 /* hashinx = y & mask */ 4304 4305 /* 4306 * Calculate physical kpm_page pointer 4307 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4308 */ 4309 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4310 add %g1, %g4, %g1 /* kp_pa */ 4311 4312 /* 4313 * Calculate physical hash lock address 4314 * g1=kp_refcntc_pa g2=pfn g5=hashinx 4315 */ 4316 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4317 sllx %g5, KPMHLK_SHIFT, %g5 4318 add %g4, %g5, %g3 4319 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4320 4321 /* 4322 * Assemble tte 4323 * g1=kp_pa g2=pfn g3=hlck_pa 4324 */ 4325#ifdef sun4v 4326 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4327 sllx %g5, 32, %g5 4328 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4329 or %g4, TTE4M, %g4 4330 or %g5, %g4, %g5 4331#else 4332 sethi %hi(TTE_VALID_INT), %g4 4333 mov TTE4M, %g5 4334 sllx %g5, TTE_SZ_SHFT_INT, %g5 4335 or %g5, %g4, %g5 /* upper part */ 4336 sllx %g5, 32, %g5 4337 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4338 or %g5, %g4, %g5 4339#endif 4340 sllx %g2, MMU_PAGESHIFT, %g4 4341 or %g5, %g4, %g5 /* tte */ 4342 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4343 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4344 4345 /* 4346 * tsb dropin 4347 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4348 */ 4349 4350 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4351 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4352 4353 /* use C-handler if there's no go for dropin */ 4354 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4355 cmp %g7, -1 4356 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4357 nop 4358 4359#ifdef DEBUG 4360 /* double check refcnt */ 4361 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4362 brz,pn %g7, 5f /* let C-handler deal with this */ 4363 nop 4364#endif 4365 4366#ifndef sun4v 4367 ldub [%g6 + KPMTSBM_FLAGS], %g7 4368 mov ASI_N, %g1 4369 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4370 movnz %icc, ASI_MEM, %g1 4371 mov %g1, %asi 4372#endif 4373 4374 /* 4375 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4376 * If we fail to lock the TSB entry then just load the tte into the 4377 * TLB. 4378 */ 4379 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l1) 4380 4381 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4382 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4383locked_tsb_l1: 4384 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4385 4386 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4387 KPMLOCK_EXIT(%g3, ASI_MEM) 4388 4389 /* 4390 * If trapstat is running, we need to shift the %tpc and %tnpc to 4391 * point to trapstat's TSB miss return code (note that trapstat 4392 * itself will patch the correct offset to add). 4393 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4394 */ 4395 rdpr %tl, %g7 4396 cmp %g7, 1 4397 ble %icc, 0f 4398 sethi %hi(KERNELBASE), %g6 4399 rdpr %tpc, %g7 4400 or %g6, %lo(KERNELBASE), %g6 4401 cmp %g7, %g6 4402 bgeu %xcc, 0f 4403 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4404 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4405 wrpr %g7, %tpc 4406 add %g7, 4, %g7 4407 wrpr %g7, %tnpc 44080: 4409 retry 44105: 4411 /* g3=hlck_pa */ 4412 KPMLOCK_EXIT(%g3, ASI_MEM) 4413 ba,pt %icc, sfmmu_kpm_exception 4414 nop 4415 SET_SIZE(sfmmu_kpm_dtsb_miss) 4416 4417 /* 4418 * kpm tsbmiss handler for smallpages 4419 * g1 = 8K kpm TSB pointer 4420 * g2 = tag access register 4421 * g3 = 4M kpm TSB pointer 4422 */ 4423 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4424 TT_TRACE(trace_tsbmiss) 4425 CPU_INDEX(%g7, %g6) 4426 sethi %hi(kpmtsbm_area), %g6 4427 sllx %g7, KPMTSBM_SHIFT, %g7 4428 or %g6, %lo(kpmtsbm_area), %g6 4429 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4430 4431 /* check enable flag */ 4432 ldub [%g6 + KPMTSBM_FLAGS], %g4 4433 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4434 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4435 nop 4436 4437 /* 4438 * VA range check 4439 * On fail: goto sfmmu_tsb_miss 4440 */ 4441 ldx [%g6 + KPMTSBM_VBASE], %g7 4442 cmp %g2, %g7 4443 blu,pn %xcc, sfmmu_tsb_miss 4444 ldx [%g6 + KPMTSBM_VEND], %g5 4445 cmp %g2, %g5 4446 bgeu,pn %xcc, sfmmu_tsb_miss 4447 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4448 4449 /* 4450 * check TL tsbmiss handling flag 4451 * bump tsbmiss counter 4452 */ 4453 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4454#ifdef DEBUG 4455 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4456 inc %g5 4457 brz,pn %g1, sfmmu_kpm_exception 4458 st %g5, [%g6 + KPMTSBM_TSBMISS] 4459#else 4460 inc %g5 4461 st %g5, [%g6 + KPMTSBM_TSBMISS] 4462#endif 4463 /* 4464 * At this point: 4465 * g1 = clobbered 4466 * g2 = tag access register 4467 * g3 = 4M kpm TSB pointer (not used) 4468 * g6 = per-CPU kpm tsbmiss area 4469 * g7 = kpm_vbase 4470 */ 4471 4472 /* 4473 * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr) 4474 * which is defined in mach_kpm.h. Any changes in that macro 4475 * should also be ported back to this assembly code. 4476 */ 4477 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 /* g3 = kpm_size_shift */ 4478 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4479 srax %g4, %g3, %g7 /* which alias range (r) */ 4480 brz,pt %g7, 2f 4481 sethi %hi(vac_colors_mask), %g5 4482 ld [%g5 + %lo(vac_colors_mask)], %g5 4483 4484 srlx %g2, MMU_PAGESHIFT, %g1 /* vaddr >> MMU_PAGESHIFT */ 4485 and %g1, %g5, %g1 /* g1 = v */ 4486 sllx %g7, %g3, %g5 /* g5 = r << kpm_size_shift */ 4487 cmp %g7, %g1 /* if (r > v) */ 4488 bleu,pn %xcc, 1f 4489 sub %g4, %g5, %g4 /* paddr -= r << kpm_size_shift */ 4490 sub %g7, %g1, %g5 /* g5 = r - v */ 4491 sllx %g5, MMU_PAGESHIFT, %g7 /* (r-v) << MMU_PAGESHIFT */ 4492 add %g4, %g7, %g4 /* paddr += (r-v)<<MMU_PAGESHIFT */ 4493 ba 2f 4494 nop 44951: 4496 sllx %g7, MMU_PAGESHIFT, %g5 /* else */ 4497 sub %g4, %g5, %g4 /* paddr -= r << MMU_PAGESHIFT */ 4498 4499 /* 4500 * paddr2pfn 4501 * g1 = vcolor (not used) 4502 * g2 = tag access register 4503 * g3 = clobbered 4504 * g4 = paddr 4505 * g5 = clobbered 4506 * g6 = per-CPU kpm tsbmiss area 4507 * g7 = clobbered 4508 */ 45092: 4510 srlx %g4, MMU_PAGESHIFT, %g2 /* g2 = pfn */ 4511 4512 /* 4513 * Setup %asi 4514 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4515 * if (mseg not found) sfmmu_kpm_exception 4516 * g2=pfn g6=per-CPU kpm tsbmiss area 4517 * g4 g5 g7 for scratch use. 4518 */ 4519 mov ASI_MEM, %asi 4520 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4521 cmp %g3, MSEG_NULLPTR_PA 4522 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4523 nop 4524 4525 /* 4526 * inx = pfn - mseg_pa->kpm_pbase 4527 * g2=pfn g3=mseg_pa g6=per-CPU kpm tsbmiss area 4528 */ 4529 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4530 sub %g2, %g7, %g4 4531 4532#ifdef DEBUG 4533 /* 4534 * Validate inx value 4535 * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area 4536 */ 4537 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4538 cmp %g4, %g5 /* inx - nkpmpgs */ 4539 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4540 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4541#else 4542 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4543#endif 4544 /* ksp = &mseg_pa->kpm_spages[inx] */ 4545 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4546 add %g5, %g4, %g5 /* ksp */ 4547 4548 /* 4549 * KPMP_SHASH(kp) 4550 * g2=pfn g3=mseg_pa g4=inx g5=ksp 4551 * g6=per-CPU kpm tsbmiss area g7=kpmp_stable_sz 4552 */ 4553 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4554 sub %g7, 1, %g7 /* mask */ 4555 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4556 add %g5, %g1, %g5 /* y = ksp + x */ 4557 and %g5, %g7, %g5 /* hashinx = y & mask */ 4558 4559 /* 4560 * Calculate physical kpm_spage pointer 4561 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4562 * g6=per-CPU kpm tsbmiss area 4563 */ 4564 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4565 add %g1, %g4, %g1 /* ksp_pa */ 4566 4567 /* 4568 * Calculate physical hash lock address. 4569 * Note: Changes in kpm_shlk_t must be reflected here. 4570 * g1=ksp_pa g2=pfn g5=hashinx 4571 * g6=per-CPU kpm tsbmiss area 4572 */ 4573 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4574 sllx %g5, KPMSHLK_SHIFT, %g5 4575 add %g4, %g5, %g3 /* hlck_pa */ 4576 4577 /* 4578 * Assemble non-cacheable tte initially 4579 * g1=ksp_pa g2=pfn g3=hlck_pa 4580 * g6=per-CPU kpm tsbmiss area 4581 */ 4582 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4583 sllx %g5, 32, %g5 4584 mov (TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4585 or %g5, %g4, %g5 4586 sllx %g2, MMU_PAGESHIFT, %g4 4587 or %g5, %g4, %g5 /* tte */ 4588 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4589 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4590 4591 /* 4592 * tsb dropin 4593 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable) 4594 * g6=per-CPU kpm tsbmiss area g7=scratch register 4595 */ 4596 4597 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4598 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4599 4600 /* use C-handler if there's no go for dropin */ 4601 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4602 andcc %g7, KPM_MAPPED_GO, %g0 /* go or no go ? */ 4603 bz,pt %icc, 5f /* no go */ 4604 nop 4605 and %g7, KPM_MAPPED_MASK, %g7 /* go */ 4606 cmp %g7, KPM_MAPPEDS /* cacheable ? */ 4607 be,a,pn %xcc, 3f 4608 or %g5, TTE_CV_INT, %g5 /* cacheable */ 46093: 4610#ifndef sun4v 4611 ldub [%g6 + KPMTSBM_FLAGS], %g7 4612 mov ASI_N, %g1 4613 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4614 movnz %icc, ASI_MEM, %g1 4615 mov %g1, %asi 4616#endif 4617 4618 /* 4619 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4620 * If we fail to lock the TSB entry then just load the tte into the 4621 * TLB. 4622 */ 4623 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l2) 4624 4625 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4626 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4627locked_tsb_l2: 4628 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4629 4630 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4631 KPMLOCK_EXIT(%g3, ASI_MEM) 4632 4633 /* 4634 * If trapstat is running, we need to shift the %tpc and %tnpc to 4635 * point to trapstat's TSB miss return code (note that trapstat 4636 * itself will patch the correct offset to add). 4637 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4638 */ 4639 rdpr %tl, %g7 4640 cmp %g7, 1 4641 ble %icc, 0f 4642 sethi %hi(KERNELBASE), %g6 4643 rdpr %tpc, %g7 4644 or %g6, %lo(KERNELBASE), %g6 4645 cmp %g7, %g6 4646 bgeu %xcc, 0f 4647 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4648 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4649 wrpr %g7, %tpc 4650 add %g7, 4, %g7 4651 wrpr %g7, %tnpc 46520: 4653 retry 46545: 4655 /* g3=hlck_pa */ 4656 KPMLOCK_EXIT(%g3, ASI_MEM) 4657 ba,pt %icc, sfmmu_kpm_exception 4658 nop 4659 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4660 4661#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4662#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4663#endif 4664 4665#endif /* lint */ 4666 4667#ifdef lint 4668/* 4669 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4670 * Called from C-level, sets/clears "go" indication for trap level handler. 4671 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 4672 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 4673 * Assumes khl_mutex is held when called from C-level. 4674 */ 4675/* ARGSUSED */ 4676void 4677sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 4678{ 4679} 4680 4681/* 4682 * kpm_smallpages: stores val to byte at address mapped within 4683 * low level lock brackets. The old value is returned. 4684 * Called from C-level. 4685 */ 4686/* ARGSUSED */ 4687int 4688sfmmu_kpm_stsbmtl(uchar_t *mapped, uint_t *kshl_lock, int val) 4689{ 4690 return (0); 4691} 4692 4693#else /* lint */ 4694 4695 .seg ".data" 4696sfmmu_kpm_tsbmtl_panic: 4697 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4698 .byte 0 4699sfmmu_kpm_stsbmtl_panic: 4700 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4701 .byte 0 4702 .align 4 4703 .seg ".text" 4704 4705 ENTRY_NP(sfmmu_kpm_tsbmtl) 4706 rdpr %pstate, %o3 4707 /* 4708 * %o0 = &kp_refcntc 4709 * %o1 = &khl_lock 4710 * %o2 = 0/1 (off/on) 4711 * %o3 = pstate save 4712 */ 4713#ifdef DEBUG 4714 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4715 bnz,pt %icc, 1f /* disabled, panic */ 4716 nop 4717 save %sp, -SA(MINFRAME), %sp 4718 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4719 call panic 4720 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4721 ret 4722 restore 47231: 4724#endif /* DEBUG */ 4725 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4726 4727 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4728 mov -1, %o5 4729 brz,a %o2, 2f 4730 mov 0, %o5 47312: 4732 sth %o5, [%o0] 4733 KPMLOCK_EXIT(%o1, ASI_N) 4734 4735 retl 4736 wrpr %g0, %o3, %pstate /* enable interrupts */ 4737 SET_SIZE(sfmmu_kpm_tsbmtl) 4738 4739 ENTRY_NP(sfmmu_kpm_stsbmtl) 4740 rdpr %pstate, %o3 4741 /* 4742 * %o0 = &mapped 4743 * %o1 = &kshl_lock 4744 * %o2 = val 4745 * %o3 = pstate save 4746 */ 4747#ifdef DEBUG 4748 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4749 bnz,pt %icc, 1f /* disabled, panic */ 4750 nop 4751 save %sp, -SA(MINFRAME), %sp 4752 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4753 call panic 4754 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4755 ret 4756 restore 47571: 4758#endif /* DEBUG */ 4759 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4760 4761 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4762 ldsb [%o0], %o5 4763 stb %o2, [%o0] 4764 KPMLOCK_EXIT(%o1, ASI_N) 4765 4766 and %o5, KPM_MAPPED_MASK, %o0 /* return old val */ 4767 retl 4768 wrpr %g0, %o3, %pstate /* enable interrupts */ 4769 SET_SIZE(sfmmu_kpm_stsbmtl) 4770 4771#endif /* lint */ 4772 4773#ifndef lint 4774#ifdef sun4v 4775 /* 4776 * User/kernel data miss w// multiple TSBs 4777 * The first probe covers 8K, 64K, and 512K page sizes, 4778 * because 64K and 512K mappings are replicated off 8K 4779 * pointer. Second probe covers 4M page size only. 4780 * 4781 * MMU fault area contains miss address and context. 4782 */ 4783 ALTENTRY(sfmmu_slow_dmmu_miss) 4784 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 4785 4786slow_miss_common: 4787 /* 4788 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 4789 * %g3 = ctx (cannot be INVALID_CONTEXT) 4790 */ 4791 brnz,pt %g3, 8f ! check for user context 4792 nop 4793 4794 /* 4795 * Kernel miss 4796 * Get 8K and 4M TSB pointers in %g1 and %g3 and 4797 * branch to sfmmu_tsb_miss_tt to handle it. 4798 */ 4799 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4800sfmmu_dslow_patch_ktsb_base: 4801 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 4802sfmmu_dslow_patch_ktsb_szcode: 4803 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 4804 4805 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 4806 ! %g1 = First TSB entry pointer, as TSB miss handler expects 4807 4808 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4809sfmmu_dslow_patch_ktsb4m_base: 4810 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 4811sfmmu_dslow_patch_ktsb4m_szcode: 4812 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 4813 4814 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 4815 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 4816 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4817 .empty 4818 48198: 4820 /* 4821 * User miss 4822 * Get first TSB pointer in %g1 4823 * Get second TSB pointer (or NULL if no second TSB) in %g3 4824 * Branch to sfmmu_tsb_miss_tt to handle it 4825 */ 4826 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 4827 /* %g1 = first TSB entry ptr now, %g2 preserved */ 4828 4829 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 4830 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 4831 nop 4832 4833 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 4834 /* %g3 = second TSB entry ptr now, %g2 preserved */ 48359: 4836 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4837 .empty 4838 SET_SIZE(sfmmu_slow_dmmu_miss) 4839 4840 4841 /* 4842 * User/kernel instruction miss w/ multiple TSBs 4843 * The first probe covers 8K, 64K, and 512K page sizes, 4844 * because 64K and 512K mappings are replicated off 8K 4845 * pointer. Second probe covers 4M page size only. 4846 * 4847 * MMU fault area contains miss address and context. 4848 */ 4849 ALTENTRY(sfmmu_slow_immu_miss) 4850 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 4851 ba,a,pt %xcc, slow_miss_common 4852 SET_SIZE(sfmmu_slow_immu_miss) 4853 4854#endif /* sun4v */ 4855#endif /* lint */ 4856 4857#ifndef lint 4858 4859/* 4860 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 4861 */ 4862 .seg ".data" 4863 .align 64 4864 .global tsbmiss_area 4865tsbmiss_area: 4866 .skip (TSBMISS_SIZE * NCPU) 4867 4868 .align 64 4869 .global kpmtsbm_area 4870kpmtsbm_area: 4871 .skip (KPMTSBM_SIZE * NCPU) 4872#endif /* lint */ 4873