1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22/* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27#pragma ident "%Z%%M% %I% %E% SMI" 28 29/* 30 * SFMMU primitives. These primitives should only be used by sfmmu 31 * routines. 32 */ 33 34#if defined(lint) 35#include <sys/types.h> 36#else /* lint */ 37#include "assym.h" 38#endif /* lint */ 39 40#include <sys/asm_linkage.h> 41#include <sys/machtrap.h> 42#include <sys/machasi.h> 43#include <sys/sun4asi.h> 44#include <sys/pte.h> 45#include <sys/mmu.h> 46#include <vm/hat_sfmmu.h> 47#include <vm/seg_spt.h> 48#include <sys/machparam.h> 49#include <sys/privregs.h> 50#include <sys/scb.h> 51#include <sys/intreg.h> 52#include <sys/machthread.h> 53#include <sys/intr.h> 54#include <sys/clock.h> 55#include <sys/trapstat.h> 56 57#ifdef TRAPTRACE 58#include <sys/traptrace.h> 59 60/* 61 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 62 */ 63#define TT_TRACE(label) \ 64 ba label ;\ 65 rd %pc, %g7 66#else 67 68#define TT_TRACE(label) 69 70#endif /* TRAPTRACE */ 71 72#ifndef lint 73 74#if (TTE_SUSPEND_SHIFT > 0) 75#define TTE_SUSPEND_INT_SHIFT(reg) \ 76 sllx reg, TTE_SUSPEND_SHIFT, reg 77#else 78#define TTE_SUSPEND_INT_SHIFT(reg) 79#endif 80 81#endif /* lint */ 82 83#ifndef lint 84 85/* 86 * Assumes TSBE_TAG is 0 87 * Assumes TSBE_INTHI is 0 88 * Assumes TSBREG.split is 0 89 */ 90 91#if TSBE_TAG != 0 92#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 93#endif 94 95#if TSBTAG_INTHI != 0 96#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 97#endif 98 99/* 100 * The following code assumes the tsb is not split. 101 * 102 * With TSBs no longer shared between processes, it's no longer 103 * necessary to hash the context bits into the tsb index to get 104 * tsb coloring; the new implementation treats the TSB as a 105 * direct-mapped, virtually-addressed cache. 106 * 107 * In: 108 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 109 * tsbbase = base address of TSB (clobbered) 110 * tagacc = tag access register (clobbered) 111 * szc = size code of TSB (ro) 112 * tmp = scratch reg 113 * Out: 114 * tsbbase = pointer to entry in TSB 115 */ 116#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 117 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 118 srlx tagacc, vpshift, tagacc ;\ 119 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 120 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 121 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 122 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 123 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 124 125/* 126 * When the kpm TSB is used it is assumed that it is direct mapped 127 * using (vaddr>>vpshift)%tsbsz as the index. 128 * 129 * Note that, for now, the kpm TSB and kernel TSB are the same for 130 * each mapping size. However that need not always be the case. If 131 * the trap handlers are updated to search a different TSB for kpm 132 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 133 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 134 * 135 * In: 136 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 137 * vaddr = virtual address (clobbered) 138 * tsbp, szc, tmp = scratch 139 * Out: 140 * tsbp = pointer to entry in TSB 141 */ 142#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 143 cmp vpshift, MMU_PAGESHIFT ;\ 144 bne,pn %icc, 1f /* branch if large case */ ;\ 145 sethi %hi(kpmsm_tsbsz), szc ;\ 146 sethi %hi(kpmsm_tsbbase), tsbp ;\ 147 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 148 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 149 ba,pt %icc, 2f ;\ 150 nop ;\ 1511: sethi %hi(kpm_tsbsz), szc ;\ 152 sethi %hi(kpm_tsbbase), tsbp ;\ 153 ld [szc + %lo(kpm_tsbsz)], szc ;\ 154 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 1552: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 156 157/* 158 * Lock the TSBE at virtual address tsbep. 159 * 160 * tsbep = TSBE va (ro) 161 * tmp1, tmp2 = scratch registers (clobbered) 162 * label = label to use for branches (text) 163 * %asi = ASI to use for TSB access 164 * 165 * NOTE that we flush the TSB using fast VIS instructions that 166 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 167 * not be treated as a locked entry or we'll get stuck spinning on 168 * an entry that isn't locked but really invalid. 169 */ 170 171#if defined(UTSB_PHYS) 172 173#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 174 lda [tsbep]ASI_MEM, tmp1 ;\ 175label: ;\ 176 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 177 cmp tmp1, tmp2 ;\ 178 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 179 lda [tsbep]ASI_MEM, tmp1 ;\ 180 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 181 cmp tmp1, tmp2 ;\ 182 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 183 lda [tsbep]ASI_MEM, tmp1 ;\ 184 /* tsbe lock acquired */ ;\ 185 membar #StoreStore 186 187#else /* UTSB_PHYS */ 188 189#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 190 lda [tsbep]%asi, tmp1 ;\ 191label: ;\ 192 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 193 cmp tmp1, tmp2 ;\ 194 be,a,pn %icc, label/**/b /* if locked spin */ ;\ 195 lda [tsbep]%asi, tmp1 ;\ 196 casa [tsbep]%asi, tmp1, tmp2 ;\ 197 cmp tmp1, tmp2 ;\ 198 bne,a,pn %icc, label/**/b /* didn't lock so try again */ ;\ 199 lda [tsbep]%asi, tmp1 ;\ 200 /* tsbe lock acquired */ ;\ 201 membar #StoreStore 202 203#endif /* UTSB_PHYS */ 204 205/* 206 * Atomically write TSBE at virtual address tsbep. 207 * 208 * tsbep = TSBE va (ro) 209 * tte = TSBE TTE (ro) 210 * tagtarget = TSBE tag (ro) 211 * %asi = ASI to use for TSB access 212 */ 213 214#if defined(UTSB_PHYS) 215 216#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 217 add tsbep, TSBE_TTE, tmp1 ;\ 218 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 219 membar #StoreStore ;\ 220 add tsbep, TSBE_TAG, tmp1 ;\ 221 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 222 223#else /* UTSB_PHYS */ 224 225#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 226 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 227 membar #StoreStore ;\ 228 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 229 230#endif /* UTSB_PHYS */ 231 232/* 233 * Load an entry into the TSB at TL > 0. 234 * 235 * tsbep = pointer to the TSBE to load as va (ro) 236 * tte = value of the TTE retrieved and loaded (wo) 237 * tagtarget = tag target register. To get TSBE tag to load, 238 * we need to mask off the context and leave only the va (clobbered) 239 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 240 * tmp1, tmp2 = scratch registers 241 * label = label to use for branches (text) 242 * %asi = ASI to use for TSB access 243 */ 244 245#if defined(UTSB_PHYS) 246 247#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 248 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 249 /* ;\ 250 * I don't need to update the TSB then check for the valid tte. ;\ 251 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 252 * we always invalidate the hash table before we unload the TSB.;\ 253 */ ;\ 254 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 255 ldxa [ttepa]ASI_MEM, tte ;\ 256 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 257 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 258 add tsbep, TSBE_TAG, tmp1 ;\ 259 brgez,a,pn tte, label/**/f ;\ 260 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 261 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 262label: 263 264#else /* UTSB_PHYS */ 265 266#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 267 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 268 /* ;\ 269 * I don't need to update the TSB then check for the valid tte. ;\ 270 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 271 * we always invalidate the hash table before we unload the TSB.;\ 272 */ ;\ 273 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 274 ldxa [ttepa]ASI_MEM, tte ;\ 275 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 276 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 277 brgez,a,pn tte, label/**/f ;\ 278 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 279 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 280label: 281 282/* 283 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 284 * for ITLB synthesis. 285 * 286 * tsbep = pointer to the TSBE to load as va (ro) 287 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 288 * with exec_perm turned off and exec_synth turned on 289 * tagtarget = tag target register. To get TSBE tag to load, 290 * we need to mask off the context and leave only the va (clobbered) 291 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 292 * tmp1, tmp2 = scratch registers 293 * label = label to use for branch (text) 294 * %asi = ASI to use for TSB access 295 */ 296 297#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 298 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 299 /* ;\ 300 * I don't need to update the TSB then check for the valid tte. ;\ 301 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 302 * we always invalidate the hash table before we unload the TSB.;\ 303 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 304 * and exec_synth bit to 1. ;\ 305 */ ;\ 306 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 307 mov tte, tmp1 ;\ 308 ldxa [ttepa]ASI_MEM, tte ;\ 309 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 310 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 311 brgez,a,pn tte, label/**/f ;\ 312 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 313 or tte, tmp1, tte ;\ 314 andn tte, TTE_EXECPRM_INT, tte ;\ 315 or tte, TTE_E_SYNTH_INT, tte ;\ 316 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 317label: 318 319/* 320 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 321 * 322 * tte = value of the TTE, used to get tte_size bits (ro) 323 * tagaccess = tag access register, used to get 4M pfn bits (ro) 324 * pfn = 4M pfn bits shifted to offset for tte (out) 325 * tmp1 = scratch register 326 * label = label to use for branch (text) 327 */ 328 329#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 330 /* ;\ 331 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 332 * Return them, shifted, in pfn. ;\ 333 */ ;\ 334 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 335 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 336 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 337 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 338 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 339 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 340label: ;\ 341 sllx tagaccess, MMU_PAGESHIFT4M, pfn 342 343/* 344 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 345 * for ITLB synthesis. 346 * 347 * tte = value of the TTE, used to get tte_size bits (rw) 348 * tmp1 = scratch register 349 */ 350 351#define SET_TTE4M_PN(tte, tmp) \ 352 /* ;\ 353 * Set 4M pagesize tte bits. ;\ 354 */ ;\ 355 set TTE4M, tmp ;\ 356 sllx tmp, TTE_SZ_SHFT, tmp ;\ 357 or tte, tmp, tte 358 359#endif /* UTSB_PHYS */ 360 361/* 362 * Load an entry into the TSB at TL=0. 363 * 364 * tsbep = pointer to the TSBE to load as va (ro) 365 * tteva = pointer to the TTE to load as va (ro) 366 * tagtarget = TSBE tag to load (which contains no context), synthesized 367 * to match va of MMU tag target register only (ro) 368 * tmp1, tmp2 = scratch registers (clobbered) 369 * label = label to use for branches (text) 370 * %asi = ASI to use for TSB access 371 */ 372 373#if defined(UTSB_PHYS) 374 375#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 376 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 377 ldx [tteva], tteva /* load tte */ ;\ 378 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 379 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 380 add tsbep, TSBE_TAG, tmp1 ;\ 381 brgez,a,pn tteva, label/**/f ;\ 382 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 383 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 384label: 385 386#else /* UTSB_PHYS */ 387 388#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 389 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 390 ldx [tteva], tteva /* load tte */ ;\ 391 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 392 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 393 brgez,a,pn tteva, label/**/f ;\ 394 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 395 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 396label: 397 398#endif /* UTSB_PHYS */ 399 400/* 401 * Invalidate a TSB entry in the TSB. 402 * 403 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 404 * about this earlier to ensure this is true. Thus when we are 405 * directly referencing tsbep below, we are referencing the tte_tag 406 * field of the TSBE. If this offset ever changes, the code below 407 * will need to be modified. 408 * 409 * tsbep = pointer to TSBE as va (ro) 410 * tag = invalidation is done if this matches the TSBE tag (ro) 411 * tmp1 - tmp3 = scratch registers (clobbered) 412 * label = label name to use for branches (text) 413 * %asi = ASI to use for TSB access 414 */ 415 416#if defined(UTSB_PHYS) 417 418#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 419 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 420 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 421label/**/1: ;\ 422 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 423 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 424 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 425 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 426 cmp tag, tmp3 /* compare tags */ ;\ 427 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 428 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 429 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 430 cmp tmp1, tmp3 /* if not successful */ ;\ 431 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 432 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 433label/**/2: 434 435#else /* UTSB_PHYS */ 436 437#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 438 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 439 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 440label/**/1: ;\ 441 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 442 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 443 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 444 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 445 cmp tag, tmp3 /* compare tags */ ;\ 446 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 447 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 448 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 449 cmp tmp1, tmp3 /* if not successful */ ;\ 450 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 451 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 452label/**/2: 453 454#endif /* UTSB_PHYS */ 455 456#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 457#error - TSB_SOFTSZ_MASK too small 458#endif 459 460 461/* 462 * An implementation of setx which will be hot patched at run time. 463 * since it is being hot patched, there is no value passed in. 464 * Thus, essentially we are implementing 465 * setx value, tmp, dest 466 * where value is RUNTIME_PATCH (aka 0) in this case. 467 */ 468#define RUNTIME_PATCH_SETX(dest, tmp) \ 469 sethi %hh(RUNTIME_PATCH), tmp ;\ 470 sethi %lm(RUNTIME_PATCH), dest ;\ 471 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 472 or dest, %lo(RUNTIME_PATCH), dest ;\ 473 sllx tmp, 32, tmp ;\ 474 nop /* for perf reasons */ ;\ 475 or tmp, dest, dest /* contents of patched value */ 476 477 478#endif (lint) 479 480 481#if defined (lint) 482 483/* 484 * sfmmu related subroutines 485 */ 486 487/* 488 * Use cas, if tte has changed underneath us then reread and try again. 489 * In the case of a retry, it will update sttep with the new original. 490 */ 491/* ARGSUSED */ 492int 493sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 494{ return(0); } 495 496/* 497 * Use cas, if tte has changed underneath us then return 1, else return 0 498 */ 499/* ARGSUSED */ 500int 501sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep) 502{ return(0); } 503 504/* ARGSUSED */ 505void 506sfmmu_copytte(tte_t *sttep, tte_t *dttep) 507{} 508 509/*ARGSUSED*/ 510struct tsbe * 511sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc) 512{ return(0); } 513 514/*ARGSUSED*/ 515uint64_t 516sfmmu_make_tsbtag(caddr_t va) 517{ return(0); } 518 519#else /* lint */ 520 521 .seg ".data" 522 .global sfmmu_panic1 523sfmmu_panic1: 524 .asciz "sfmmu_asm: interrupts already disabled" 525 526 .global sfmmu_panic3 527sfmmu_panic3: 528 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 529 530 .global sfmmu_panic4 531sfmmu_panic4: 532 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 533 534 .global sfmmu_panic5 535sfmmu_panic5: 536 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 537 538 539 ENTRY_NP(sfmmu_modifytte) 540 ldx [%o2], %g3 /* current */ 541 ldx [%o0], %g1 /* original */ 5422: 543 ldx [%o1], %g2 /* modified */ 544 cmp %g2, %g3 /* is modified = current? */ 545 be,a,pt %xcc,1f /* yes, don't write */ 546 stx %g3, [%o0] /* update new original */ 547 casx [%o2], %g1, %g2 548 cmp %g1, %g2 549 be,pt %xcc, 1f /* cas succeeded - return */ 550 nop 551 ldx [%o2], %g3 /* new current */ 552 stx %g3, [%o0] /* save as new original */ 553 ba,pt %xcc, 2b 554 mov %g3, %g1 5551: retl 556 membar #StoreLoad 557 SET_SIZE(sfmmu_modifytte) 558 559 ENTRY_NP(sfmmu_modifytte_try) 560 ldx [%o1], %g2 /* modified */ 561 ldx [%o2], %g3 /* current */ 562 ldx [%o0], %g1 /* original */ 563 cmp %g3, %g2 /* is modified = current? */ 564 be,a,pn %xcc,1f /* yes, don't write */ 565 mov 0, %o1 /* as if cas failed. */ 566 567 casx [%o2], %g1, %g2 568 membar #StoreLoad 569 cmp %g1, %g2 570 movne %xcc, -1, %o1 /* cas failed. */ 571 move %xcc, 1, %o1 /* cas succeeded. */ 5721: 573 stx %g2, [%o0] /* report "current" value */ 574 retl 575 mov %o1, %o0 576 SET_SIZE(sfmmu_modifytte_try) 577 578 ENTRY_NP(sfmmu_copytte) 579 ldx [%o0], %g1 580 retl 581 stx %g1, [%o1] 582 SET_SIZE(sfmmu_copytte) 583 584 585 /* 586 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 587 * %o0 = TSB base address (in), pointer to TSB entry (out) 588 * %o1 = vaddr (in) 589 * %o2 = vpshift (in) 590 * %o3 = tsb size code (in) 591 * %o4 = scratch register 592 */ 593 ENTRY_NP(sfmmu_get_tsbe) 594 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 595 retl 596 nop 597 SET_SIZE(sfmmu_get_tsbe) 598 599 /* 600 * Return a TSB tag for the given va. 601 * %o0 = va (in/clobbered) 602 * %o0 = va shifted to be in tsb tag format (with no context) (out) 603 */ 604 ENTRY_NP(sfmmu_make_tsbtag) 605 retl 606 srln %o0, TTARGET_VA_SHIFT, %o0 607 SET_SIZE(sfmmu_make_tsbtag) 608 609#endif /* lint */ 610 611/* 612 * Other sfmmu primitives 613 */ 614 615 616#if defined (lint) 617void 618sfmmu_patch_ktsb(void) 619{ 620} 621 622void 623sfmmu_kpm_patch_tlbm(void) 624{ 625} 626 627void 628sfmmu_kpm_patch_tsbm(void) 629{ 630} 631 632/* ARGSUSED */ 633void 634sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys) 635{ 636} 637 638/* ARGSUSED */ 639void 640sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys) 641{ 642} 643 644/* ARGSUSED */ 645void 646sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift) 647{ 648} 649 650/* ARGSUSED */ 651void 652sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) 653{ 654} 655 656#else /* lint */ 657 658#define I_SIZE 4 659 660 ENTRY_NP(sfmmu_fix_ktlb_traptable) 661 /* 662 * %o0 = start of patch area 663 * %o1 = size code of TSB to patch 664 * %o3 = scratch 665 */ 666 /* fix sll */ 667 ld [%o0], %o3 /* get sll */ 668 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 669 st %o3, [%o0] /* write sll */ 670 flush %o0 671 /* fix srl */ 672 add %o0, I_SIZE, %o0 /* goto next instr. */ 673 ld [%o0], %o3 /* get srl */ 674 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 675 st %o3, [%o0] /* write srl */ 676 retl 677 flush %o0 678 SET_SIZE(sfmmu_fix_ktlb_traptable) 679 680 ENTRY_NP(sfmmu_fixup_ktsbbase) 681 /* 682 * %o0 = start of patch area 683 * %o5 = kernel virtual or physical tsb base address 684 * %o2, %o3 are used as scratch registers. 685 */ 686 /* fixup sethi instruction */ 687 ld [%o0], %o3 688 srl %o5, 10, %o2 ! offset is bits 32:10 689 or %o3, %o2, %o3 ! set imm22 690 st %o3, [%o0] 691 /* fixup offset of lduw/ldx */ 692 add %o0, I_SIZE, %o0 ! next instr 693 ld [%o0], %o3 694 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 695 or %o3, %o2, %o3 696 st %o3, [%o0] 697 retl 698 flush %o0 699 SET_SIZE(sfmmu_fixup_ktsbbase) 700 701 ENTRY_NP(sfmmu_fixup_setx) 702 /* 703 * %o0 = start of patch area 704 * %o4 = 64 bit value to patch 705 * %o2, %o3 are used as scratch registers. 706 * 707 * Note: Assuming that all parts of the instructions which need to be 708 * patched correspond to RUNTIME_PATCH (aka 0) 709 * 710 * Note the implementation of setx which is being patched is as follows: 711 * 712 * sethi %hh(RUNTIME_PATCH), tmp 713 * sethi %lm(RUNTIME_PATCH), dest 714 * or tmp, %hm(RUNTIME_PATCH), tmp 715 * or dest, %lo(RUNTIME_PATCH), dest 716 * sllx tmp, 32, tmp 717 * nop 718 * or tmp, dest, dest 719 * 720 * which differs from the implementation in the 721 * "SPARC Architecture Manual" 722 */ 723 /* fixup sethi instruction */ 724 ld [%o0], %o3 725 srlx %o4, 42, %o2 ! bits [63:42] 726 or %o3, %o2, %o3 ! set imm22 727 st %o3, [%o0] 728 /* fixup sethi instruction */ 729 add %o0, I_SIZE, %o0 ! next instr 730 ld [%o0], %o3 731 sllx %o4, 32, %o2 ! clear upper bits 732 srlx %o2, 42, %o2 ! bits [31:10] 733 or %o3, %o2, %o3 ! set imm22 734 st %o3, [%o0] 735 /* fixup or instruction */ 736 add %o0, I_SIZE, %o0 ! next instr 737 ld [%o0], %o3 738 srlx %o4, 32, %o2 ! bits [63:32] 739 and %o2, 0x3ff, %o2 ! bits [41:32] 740 or %o3, %o2, %o3 ! set imm 741 st %o3, [%o0] 742 /* fixup or instruction */ 743 add %o0, I_SIZE, %o0 ! next instr 744 ld [%o0], %o3 745 and %o4, 0x3ff, %o2 ! bits [9:0] 746 or %o3, %o2, %o3 ! set imm 747 st %o3, [%o0] 748 retl 749 flush %o0 750 SET_SIZE(sfmmu_fixup_setx) 751 752 ENTRY_NP(sfmmu_fixup_or) 753 /* 754 * %o0 = start of patch area 755 * %o4 = 32 bit value to patch 756 * %o2, %o3 are used as scratch registers. 757 * Note: Assuming that all parts of the instructions which need to be 758 * patched correspond to RUNTIME_PATCH (aka 0) 759 */ 760 ld [%o0], %o3 761 and %o4, 0x3ff, %o2 ! bits [9:0] 762 or %o3, %o2, %o3 ! set imm 763 st %o3, [%o0] 764 retl 765 flush %o0 766 SET_SIZE(sfmmu_fixup_or) 767 768 ENTRY_NP(sfmmu_fixup_shiftx) 769 /* 770 * %o0 = start of patch area 771 * %o4 = signed int immediate value to add to sllx/srlx imm field 772 * %o2, %o3 are used as scratch registers. 773 * 774 * sllx/srlx store the 6 bit immediate value in the lowest order bits 775 * so we do a simple add. The caller must be careful to prevent 776 * overflow, which could easily occur if the initial value is nonzero! 777 */ 778 ld [%o0], %o3 ! %o3 = instruction to patch 779 and %o3, 0x3f, %o2 ! %o2 = existing imm value 780 add %o2, %o4, %o2 ! %o2 = new imm value 781 andn %o3, 0x3f, %o3 ! clear old imm value 782 and %o2, 0x3f, %o2 ! truncate new imm value 783 or %o3, %o2, %o3 ! set new imm value 784 st %o3, [%o0] ! store updated instruction 785 retl 786 flush %o0 787 SET_SIZE(sfmmu_fixup_shiftx) 788 789 ENTRY_NP(sfmmu_fixup_mmu_asi) 790 /* 791 * Patch imm_asi of all ldda instructions in the MMU 792 * trap handlers. We search MMU_PATCH_INSTR instructions 793 * starting from the itlb miss handler (trap 0x64). 794 * %o0 = address of tt[0,1]_itlbmiss 795 * %o1 = imm_asi to setup, shifted by appropriate offset. 796 * %o3 = number of instructions to search 797 * %o4 = reserved by caller: called from leaf routine 798 */ 7991: ldsw [%o0], %o2 ! load instruction to %o2 800 brgez,pt %o2, 2f 801 srl %o2, 30, %o5 802 btst 1, %o5 ! test bit 30; skip if not set 803 bz,pt %icc, 2f 804 sllx %o2, 39, %o5 ! bit 24 -> bit 63 805 srlx %o5, 58, %o5 ! isolate op3 part of opcode 806 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 807 brnz,pt %o5, 2f ! skip if not a match 808 or %o2, %o1, %o2 ! or in imm_asi 809 st %o2, [%o0] ! write patched instruction 8102: dec %o3 811 brnz,a,pt %o3, 1b ! loop until we're done 812 add %o0, I_SIZE, %o0 813 retl 814 flush %o0 815 SET_SIZE(sfmmu_fixup_mmu_asi) 816 817 /* 818 * Patch immediate ASI used to access the TSB in the 819 * trap table. 820 * inputs: %o0 = value of ktsb_phys 821 */ 822 ENTRY_NP(sfmmu_patch_mmu_asi) 823 mov %o7, %o4 ! save return pc in %o4 824 movrnz %o0, ASI_QUAD_LDD_PHYS, %o3 825 movrz %o0, ASI_NQUAD_LD, %o3 826 sll %o3, 5, %o1 ! imm_asi offset 827 mov 6, %o3 ! number of instructions 828 sethi %hi(dktsb), %o0 ! to search 829 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 830 or %o0, %lo(dktsb), %o0 831 mov 6, %o3 ! number of instructions 832 sethi %hi(dktsb4m), %o0 ! to search 833 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 834 or %o0, %lo(dktsb4m), %o0 835 mov 6, %o3 ! number of instructions 836 sethi %hi(iktsb), %o0 ! to search 837 call sfmmu_fixup_mmu_asi ! patch kitlb miss 838 or %o0, %lo(iktsb), %o0 839 mov %o4, %o7 ! retore return pc -- leaf 840 retl 841 nop 842 SET_SIZE(sfmmu_patch_mmu_asi) 843 844 ENTRY_NP(sfmmu_patch_ktsb) 845 /* 846 * We need to fix iktsb, dktsb, et. al. 847 */ 848 save %sp, -SA(MINFRAME), %sp 849 set ktsb_phys, %o1 850 ld [%o1], %o4 851 set ktsb_base, %o5 852 set ktsb4m_base, %l1 853 brz,pt %o4, 1f 854 nop 855 set ktsb_pbase, %o5 856 set ktsb4m_pbase, %l1 8571: 858 sethi %hi(ktsb_szcode), %o1 859 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 860 861 sethi %hi(iktsb), %o0 862 call sfmmu_fix_ktlb_traptable 863 or %o0, %lo(iktsb), %o0 864 865 sethi %hi(dktsb), %o0 866 call sfmmu_fix_ktlb_traptable 867 or %o0, %lo(dktsb), %o0 868 869 sethi %hi(ktsb4m_szcode), %o1 870 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 871 872 sethi %hi(dktsb4m), %o0 873 call sfmmu_fix_ktlb_traptable 874 or %o0, %lo(dktsb4m), %o0 875 876#ifndef sun4v 877 mov ASI_N, %o2 878 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 879 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 880 sethi %hi(tsb_kernel_patch_asi), %o0 881 call sfmmu_fixup_or 882 or %o0, %lo(tsb_kernel_patch_asi), %o0 883#endif 884 885 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 886 887 sethi %hi(dktsbbase), %o0 888 call sfmmu_fixup_setx ! patch value of ktsb base addr 889 or %o0, %lo(dktsbbase), %o0 890 891 sethi %hi(iktsbbase), %o0 892 call sfmmu_fixup_setx ! patch value of ktsb base addr 893 or %o0, %lo(iktsbbase), %o0 894 895 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 896 call sfmmu_fixup_setx ! patch value of ktsb base addr 897 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 898 899#ifdef sun4v 900 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 901 call sfmmu_fixup_setx ! patch value of ktsb base addr 902 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 903#endif /* sun4v */ 904 905 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 906 907 sethi %hi(dktsb4mbase), %o0 908 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 909 or %o0, %lo(dktsb4mbase), %o0 910 911 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 912 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 913 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 914 915#ifdef sun4v 916 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 917 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 918 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 919#endif /* sun4v */ 920 921 set ktsb_szcode, %o4 922 ld [%o4], %o4 923 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 924 call sfmmu_fixup_or ! patch value of ktsb_szcode 925 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 926 927#ifdef sun4v 928 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 929 call sfmmu_fixup_or ! patch value of ktsb_szcode 930 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 931#endif /* sun4v */ 932 933 set ktsb4m_szcode, %o4 934 ld [%o4], %o4 935 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 936 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 937 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 938 939#ifdef sun4v 940 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 941 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 942 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 943#endif /* sun4v */ 944 945 ret 946 restore 947 SET_SIZE(sfmmu_patch_ktsb) 948 949 ENTRY_NP(sfmmu_kpm_patch_tlbm) 950 /* 951 * Fixup trap handlers in common segkpm case. This is reserved 952 * for future use should kpm TSB be changed to be other than the 953 * kernel TSB. 954 */ 955 retl 956 nop 957 SET_SIZE(sfmmu_kpm_patch_tlbm) 958 959 ENTRY_NP(sfmmu_kpm_patch_tsbm) 960 /* 961 * nop the branch to sfmmu_kpm_dtsb_miss_small 962 * in the case where we are using large pages for 963 * seg_kpm (and hence must probe the second TSB for 964 * seg_kpm VAs) 965 */ 966 set dktsb4m_kpmcheck_small, %o0 967 MAKE_NOP_INSTR(%o1) 968 st %o1, [%o0] 969 flush %o0 970 retl 971 nop 972 SET_SIZE(sfmmu_kpm_patch_tsbm) 973 974 ENTRY_NP(sfmmu_patch_utsb) 975#ifdef sun4v 976 retl 977 nop 978#else /* sun4v */ 979 /* 980 * We need to hot patch utsb_vabase and utsb4m_vabase 981 */ 982 save %sp, -SA(MINFRAME), %sp 983 984 /* patch value of utsb_vabase */ 985 set utsb_vabase, %o1 986 ldx [%o1], %o4 987 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 988 call sfmmu_fixup_setx 989 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 990 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 991 call sfmmu_fixup_setx 992 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 993 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 994 call sfmmu_fixup_setx 995 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 996 997 /* patch value of utsb4m_vabase */ 998 set utsb4m_vabase, %o1 999 ldx [%o1], %o4 1000 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1001 call sfmmu_fixup_setx 1002 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1003 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1004 call sfmmu_fixup_setx 1005 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1006 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1007 call sfmmu_fixup_setx 1008 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1009 1010 /* 1011 * Patch TSB base register masks and shifts if needed. 1012 * By default the TSB base register contents are set up for 4M slab. 1013 * If we're using a smaller slab size and reserved VA range we need 1014 * to patch up those values here. 1015 */ 1016 set tsb_slab_shift, %o1 1017 set MMU_PAGESHIFT4M, %o4 1018 ldsw [%o1], %o3 1019 subcc %o4, %o3, %o4 1020 bz,pt %icc, 1f 1021 /* delay slot safe */ 1022 1023 /* patch reserved VA range size if needed. */ 1024 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1025 call sfmmu_fixup_shiftx 1026 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1027 call sfmmu_fixup_shiftx 1028 add %o0, I_SIZE, %o0 1029 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1030 call sfmmu_fixup_shiftx 1031 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1032 call sfmmu_fixup_shiftx 1033 add %o0, I_SIZE, %o0 10341: 1035 /* patch TSBREG_VAMASK used to set up TSB base register */ 1036 set tsb_slab_mask, %o1 1037 lduw [%o1], %o4 1038 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1039 call sfmmu_fixup_or 1040 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1041 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1042 call sfmmu_fixup_or 1043 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1044 1045 ret 1046 restore 1047#endif /* sun4v */ 1048 SET_SIZE(sfmmu_patch_utsb) 1049 1050 1051 /* 1052 * Routine that loads an entry into a tsb using virtual addresses. 1053 * Locking is required since all cpus can use the same TSB. 1054 * Note that it is no longer required to have a valid context 1055 * when calling this function. 1056 */ 1057 ENTRY_NP(sfmmu_load_tsbe) 1058 /* 1059 * %o0 = pointer to tsbe to load 1060 * %o1 = tsb tag 1061 * %o2 = virtual pointer to TTE 1062 * %o3 = 1 if physical address in %o0 else 0 1063 */ 1064 rdpr %pstate, %o5 1065#ifdef DEBUG 1066 andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ 1067 bnz,pt %icc, 1f /* disabled, panic */ 1068 nop 1069 1070 sethi %hi(panicstr), %g1 1071 ldx [%g1 + %lo(panicstr)], %g1 1072 tst %g1 1073 bnz,pt %icc, 1f 1074 nop 1075 1076 save %sp, -SA(MINFRAME), %sp 1077 sethi %hi(sfmmu_panic1), %o0 1078 call panic 1079 or %o0, %lo(sfmmu_panic1), %o0 10801: 1081#endif /* DEBUG */ 1082 1083 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1084 1085 SETUP_TSB_ASI(%o3, %g3) 1086 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1) 1087 1088 wrpr %g0, %o5, %pstate /* enable interrupts */ 1089 1090 retl 1091 membar #StoreStore|#StoreLoad 1092 SET_SIZE(sfmmu_load_tsbe) 1093 1094 /* 1095 * Flush TSB of a given entry if the tag matches. 1096 */ 1097 ENTRY(sfmmu_unload_tsbe) 1098 /* 1099 * %o0 = pointer to tsbe to be flushed 1100 * %o1 = tag to match 1101 * %o2 = 1 if physical address in %o0 else 0 1102 */ 1103 SETUP_TSB_ASI(%o2, %g1) 1104 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1105 retl 1106 membar #StoreStore|#StoreLoad 1107 SET_SIZE(sfmmu_unload_tsbe) 1108 1109 /* 1110 * Routine that loads a TTE into the kpm TSB from C code. 1111 * Locking is required since kpm TSB is shared among all CPUs. 1112 */ 1113 ENTRY_NP(sfmmu_kpm_load_tsb) 1114 /* 1115 * %o0 = vaddr 1116 * %o1 = ttep 1117 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1118 */ 1119 rdpr %pstate, %o5 ! %o5 = saved pstate 1120#ifdef DEBUG 1121 andcc %o5, PSTATE_IE, %g0 ! if interrupts already 1122 bnz,pt %icc, 1f ! disabled, panic 1123 nop 1124 1125 sethi %hi(panicstr), %g1 1126 ldx [%g1 + %lo(panicstr)], %g1 1127 tst %g1 1128 bnz,pt %icc, 1f 1129 nop 1130 1131 save %sp, -SA(MINFRAME), %sp 1132 sethi %hi(sfmmu_panic1), %o0 1133 call panic 1134 or %o0, %lo(sfmmu_panic1), %o0 11351: 1136#endif /* DEBUG */ 1137 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1138 1139#ifndef sun4v 1140 sethi %hi(ktsb_phys), %o4 1141 mov ASI_N, %o3 1142 ld [%o4 + %lo(ktsb_phys)], %o4 1143 movrnz %o4, ASI_MEM, %o3 1144 mov %o3, %asi 1145#endif 1146 mov %o0, %g1 ! %g1 = vaddr 1147 1148 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1149 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1150 /* %g2 = tsbep, %g1 clobbered */ 1151 1152 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1153 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1154 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1) 1155 1156 wrpr %g0, %o5, %pstate ! enable interrupts 1157 retl 1158 membar #StoreStore|#StoreLoad 1159 SET_SIZE(sfmmu_kpm_load_tsb) 1160 1161 /* 1162 * Routine that shoots down a TTE in the kpm TSB or in the 1163 * kernel TSB depending on virtpg. Locking is required since 1164 * kpm/kernel TSB is shared among all CPUs. 1165 */ 1166 ENTRY_NP(sfmmu_kpm_unload_tsb) 1167 /* 1168 * %o0 = vaddr 1169 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1170 */ 1171#ifndef sun4v 1172 sethi %hi(ktsb_phys), %o4 1173 mov ASI_N, %o3 1174 ld [%o4 + %lo(ktsb_phys)], %o4 1175 movrnz %o4, ASI_MEM, %o3 1176 mov %o3, %asi 1177#endif 1178 mov %o0, %g1 ! %g1 = vaddr 1179 1180 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1181 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1182 /* %g2 = tsbep, %g1 clobbered */ 1183 1184 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1185 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1186 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1187 1188 retl 1189 membar #StoreStore|#StoreLoad 1190 SET_SIZE(sfmmu_kpm_unload_tsb) 1191 1192#endif /* lint */ 1193 1194 1195#if defined (lint) 1196 1197/*ARGSUSED*/ 1198pfn_t 1199sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr) 1200{ return(0); } 1201 1202#else /* lint */ 1203 1204 ENTRY_NP(sfmmu_ttetopfn) 1205 ldx [%o0], %g1 /* read tte */ 1206 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1207 /* 1208 * g1 = pfn 1209 */ 1210 retl 1211 mov %g1, %o0 1212 SET_SIZE(sfmmu_ttetopfn) 1213 1214#endif /* !lint */ 1215 1216 1217#if defined (lint) 1218/* 1219 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the 1220 * the hash list. 1221 */ 1222/* ARGSUSED */ 1223void 1224sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1225 uint64_t hblkpa) 1226{ 1227} 1228 1229/* 1230 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the 1231 * hash list. 1232 */ 1233/* ARGSUSED */ 1234void 1235sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 1236 uint64_t hblkpa, struct hme_blk *prev_hblkp) 1237{ 1238} 1239#else /* lint */ 1240 1241/* 1242 * Functions to grab/release hme bucket list lock. I only use a byte 1243 * instead of the whole int because eventually we might want to 1244 * put some counters on the other bytes (of course, these routines would 1245 * have to change). The code that grab this lock should execute 1246 * with interrupts disabled and hold the lock for the least amount of time 1247 * possible. 1248 */ 1249 1250/* 1251 * Even though hmeh_listlock is updated using pa there's no need to flush 1252 * dcache since hmeh_listlock will be restored to the original value (0) 1253 * before interrupts are reenabled. 1254 */ 1255 1256/* 1257 * For sparcv9 hme hash buckets may not be in the nucleus. hme hash update 1258 * routines still use virtual addresses to update the bucket fields. But they 1259 * must not cause a TLB miss after grabbing the low level bucket lock. To 1260 * achieve this we must make sure the bucket structure is completely within an 1261 * 8K page. 1262 */ 1263 1264#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1)) 1265#error - the size of hmehash_bucket structure is not power of 2 1266#endif 1267 1268#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi) \ 1269 mov 0xff, tmp2 ;\ 1270 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1271label1: ;\ 1272 casa [tmp1]asi, %g0, tmp2 ;\ 1273 brnz,pn tmp2, label1 ;\ 1274 mov 0xff, tmp2 ;\ 1275 membar #LoadLoad 1276 1277#define HMELOCK_EXIT(hmebp, tmp1, asi) \ 1278 membar #LoadStore|#StoreStore ;\ 1279 add hmebp, HMEBUCK_LOCK, tmp1 ;\ 1280 sta %g0, [tmp1]asi 1281 1282 .seg ".data" 1283hblk_add_panic1: 1284 .ascii "sfmmu_hblk_hash_add: interrupts disabled" 1285 .byte 0 1286hblk_add_panic2: 1287 .ascii "sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not" 1288 .byte 0 1289 .align 4 1290 .seg ".text" 1291 1292 ENTRY_NP(sfmmu_hblk_hash_add) 1293 /* 1294 * %o0 = hmebp 1295 * %o1 = hmeblkp 1296 * %o2 = hblkpa 1297 */ 1298 rdpr %pstate, %o5 1299#ifdef DEBUG 1300 andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ 1301 bnz,pt %icc, 3f /* disabled, panic */ 1302 nop 1303 save %sp, -SA(MINFRAME), %sp 1304 sethi %hi(hblk_add_panic1), %o0 1305 call panic 1306 or %o0, %lo(hblk_add_panic1), %o0 1307 ret 1308 restore 1309 13103: 1311#endif /* DEBUG */ 1312 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1313 mov %o2, %g1 1314 1315 /* 1316 * g1 = hblkpa 1317 */ 1318 ldn [%o0 + HMEBUCK_HBLK], %o4 /* next hmeblk */ 1319 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = next hblkpa */ 1320#ifdef DEBUG 1321 cmp %o4, %g0 1322 bne,pt %xcc, 1f 1323 nop 1324 brz,pt %g2, 1f 1325 nop 1326 wrpr %g0, %o5, %pstate /* enable interrupts */ 1327 save %sp, -SA(MINFRAME), %sp 1328 sethi %hi(hblk_add_panic2), %o0 1329 call panic 1330 or %o0, %lo(hblk_add_panic2), %o0 1331 ret 1332 restore 13331: 1334#endif /* DEBUG */ 1335 /* 1336 * We update hmeblks entries before grabbing lock because the stores 1337 * could take a tlb miss and require the hash lock. The buckets 1338 * are part of the nucleus so we are cool with those stores. 1339 * 1340 * if buckets are not part of the nucleus our game is to 1341 * not touch any other page via va until we drop the lock. 1342 * This guarantees we won't get a tlb miss before the lock release 1343 * since interrupts are disabled. 1344 */ 1345 stn %o4, [%o1 + HMEBLK_NEXT] /* update hmeblk's next */ 1346 stx %g2, [%o1 + HMEBLK_NEXTPA] /* update hmeblk's next pa */ 1347 HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N) 1348 stn %o1, [%o0 + HMEBUCK_HBLK] /* update bucket hblk next */ 1349 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* add hmeblk to list */ 1350 HMELOCK_EXIT(%o0, %g2, ASI_N) 1351 retl 1352 wrpr %g0, %o5, %pstate /* enable interrupts */ 1353 SET_SIZE(sfmmu_hblk_hash_add) 1354 1355 ENTRY_NP(sfmmu_hblk_hash_rm) 1356 /* 1357 * This function removes an hmeblk from the hash chain. 1358 * It is written to guarantee we don't take a tlb miss 1359 * by using physical addresses to update the list. 1360 * 1361 * %o0 = hmebp 1362 * %o1 = hmeblkp 1363 * %o2 = hmeblkp previous pa 1364 * %o3 = hmeblkp previous 1365 */ 1366 1367 mov %o3, %o4 /* o4 = hmeblkp previous */ 1368 1369 rdpr %pstate, %o5 1370#ifdef DEBUG 1371 andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ 1372 bnz,pt %icc, 3f /* disabled, panic */ 1373 nop 1374 1375 sethi %hi(panicstr), %g1 1376 ldx [%g1 + %lo(panicstr)], %g1 1377 tst %g1 1378 bnz,pt %icc, 3f 1379 nop 1380 1381 sethi %hi(sfmmu_panic1), %o0 1382 call panic 1383 or %o0, %lo(sfmmu_panic1), %o0 13843: 1385#endif /* DEBUG */ 1386 /* 1387 * disable interrupts, clear Address Mask to access 64 bit physaddr 1388 */ 1389 andn %o5, PSTATE_IE, %g1 1390 wrpr %g1, 0, %pstate 1391 1392#ifndef sun4v 1393 sethi %hi(dcache_line_mask), %g4 1394 ld [%g4 + %lo(dcache_line_mask)], %g4 1395#endif /* sun4v */ 1396 1397 /* 1398 * if buckets are not part of the nucleus our game is to 1399 * not touch any other page via va until we drop the lock. 1400 * This guarantees we won't get a tlb miss before the lock release 1401 * since interrupts are disabled. 1402 */ 1403 HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N) 1404 ldn [%o0 + HMEBUCK_HBLK], %g2 /* first hmeblk in list */ 1405 cmp %g2, %o1 1406 bne,pt %ncc,1f 1407 mov ASI_MEM, %asi 1408 /* 1409 * hmeblk is first on list 1410 */ 1411 ldx [%o0 + HMEBUCK_NEXTPA], %g2 /* g2 = hmeblk pa */ 1412 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1413 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1414 stn %o3, [%o0 + HMEBUCK_HBLK] /* write va */ 1415 ba,pt %xcc, 2f 1416 stx %g1, [%o0 + HMEBUCK_NEXTPA] /* write pa */ 14171: 1418 /* hmeblk is not first on list */ 1419 1420 mov %o2, %g3 1421#ifndef sun4v 1422 GET_CPU_IMPL(%g2) 1423 cmp %g2, CHEETAH_IMPL 1424 bge %icc, hblk_hash_rm_1 1425 and %o4, %g4, %g2 1426 stxa %g0, [%g2]ASI_DC_TAG /* flush prev pa from dcache */ 1427 add %o4, HMEBLK_NEXT, %o4 1428 and %o4, %g4, %g2 1429 ba hblk_hash_rm_2 1430 stxa %g0, [%g2]ASI_DC_TAG /* flush prev va from dcache */ 1431hblk_hash_rm_1: 1432 1433 stxa %g0, [%g3]ASI_DC_INVAL /* flush prev pa from dcache */ 1434 membar #Sync 1435 add %g3, HMEBLK_NEXT, %g2 1436 stxa %g0, [%g2]ASI_DC_INVAL /* flush prev va from dcache */ 1437hblk_hash_rm_2: 1438 membar #Sync 1439#endif /* sun4v */ 1440 ldxa [%g3 + HMEBLK_NEXTPA] %asi, %g2 /* g2 = hmeblk pa */ 1441 ldna [%g2 + HMEBLK_NEXT] %asi, %o3 /* read next hmeblk va */ 1442 ldxa [%g2 + HMEBLK_NEXTPA] %asi, %g1 /* read next hmeblk pa */ 1443 stna %o3, [%g3 + HMEBLK_NEXT] %asi /* write va */ 1444 stxa %g1, [%g3 + HMEBLK_NEXTPA] %asi /* write pa */ 14452: 1446 HMELOCK_EXIT(%o0, %g2, ASI_N) 1447 retl 1448 wrpr %g0, %o5, %pstate /* enable interrupts */ 1449 SET_SIZE(sfmmu_hblk_hash_rm) 1450 1451#endif /* lint */ 1452 1453/* 1454 * These macros are used to update global sfmmu hme hash statistics 1455 * in perf critical paths. It is only enabled in debug kernels or 1456 * if SFMMU_STAT_GATHER is defined 1457 */ 1458#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1459#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1460 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1461 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1462 cmp tmp1, hatid ;\ 1463 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1464 set sfmmu_global_stat, tmp1 ;\ 1465 add tmp1, tmp2, tmp1 ;\ 1466 ld [tmp1], tmp2 ;\ 1467 inc tmp2 ;\ 1468 st tmp2, [tmp1] 1469 1470#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1471 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1472 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1473 cmp tmp1, hatid ;\ 1474 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1475 set sfmmu_global_stat, tmp1 ;\ 1476 add tmp1, tmp2, tmp1 ;\ 1477 ld [tmp1], tmp2 ;\ 1478 inc tmp2 ;\ 1479 st tmp2, [tmp1] 1480 1481 1482#else /* DEBUG || SFMMU_STAT_GATHER */ 1483 1484#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1485 1486#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1487 1488#endif /* DEBUG || SFMMU_STAT_GATHER */ 1489 1490/* 1491 * This macro is used to update global sfmmu kstas in non 1492 * perf critical areas so they are enabled all the time 1493 */ 1494#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1495 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1496 add tmp1, statname, tmp1 ;\ 1497 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1498 inc tmp2 ;\ 1499 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1500 1501/* 1502 * These macros are used to update per cpu stats in non perf 1503 * critical areas so they are enabled all the time 1504 */ 1505#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1506 ld [tsbarea + stat], tmp1 ;\ 1507 inc tmp1 ;\ 1508 st tmp1, [tsbarea + stat] 1509 1510/* 1511 * These macros are used to update per cpu stats in non perf 1512 * critical areas so they are enabled all the time 1513 */ 1514#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1515 lduh [tsbarea + stat], tmp1 ;\ 1516 inc tmp1 ;\ 1517 stuh tmp1, [tsbarea + stat] 1518 1519#if defined(KPM_TLBMISS_STATS_GATHER) 1520 /* 1521 * Count kpm dtlb misses separately to allow a different 1522 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1523 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1524 */ 1525#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1526 brgez tagacc, label /* KPM VA? */ ;\ 1527 nop ;\ 1528 CPU_INDEX(tmp1, tsbma) ;\ 1529 sethi %hi(kpmtsbm_area), tsbma ;\ 1530 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1531 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1532 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1533 /* VA range check */ ;\ 1534 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1535 cmp tagacc, val ;\ 1536 blu,pn %xcc, label ;\ 1537 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1538 cmp tagacc, tmp1 ;\ 1539 bgeu,pn %xcc, label ;\ 1540 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1541 inc val ;\ 1542 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1543label: 1544#else 1545#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1546#endif /* KPM_TLBMISS_STATS_GATHER */ 1547 1548#if defined (lint) 1549/* 1550 * The following routines are jumped to from the mmu trap handlers to do 1551 * the setting up to call systrap. They are separate routines instead of 1552 * being part of the handlers because the handlers would exceed 32 1553 * instructions and since this is part of the slow path the jump 1554 * cost is irrelevant. 1555 */ 1556void 1557sfmmu_pagefault(void) 1558{ 1559} 1560 1561void 1562sfmmu_mmu_trap(void) 1563{ 1564} 1565 1566void 1567sfmmu_window_trap(void) 1568{ 1569} 1570 1571void 1572sfmmu_kpm_exception(void) 1573{ 1574} 1575 1576#else /* lint */ 1577 1578#ifdef PTL1_PANIC_DEBUG 1579 .seg ".data" 1580 .global test_ptl1_panic 1581test_ptl1_panic: 1582 .word 0 1583 .align 8 1584 1585 .seg ".text" 1586 .align 4 1587#endif /* PTL1_PANIC_DEBUG */ 1588 1589 1590 ENTRY_NP(sfmmu_pagefault) 1591 USE_ALTERNATE_GLOBALS(%g5) 1592 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1593 rdpr %tt, %g6 1594 cmp %g6, FAST_IMMU_MISS_TT 1595 be,a,pn %icc, 1f 1596 mov T_INSTR_MMU_MISS, %g3 1597 cmp %g6, T_INSTR_MMU_MISS 1598 be,a,pn %icc, 1f 1599 mov T_INSTR_MMU_MISS, %g3 1600 mov %g5, %g2 1601 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1602 cmp %g6, FAST_DMMU_MISS_TT 1603 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1604 cmp %g6, T_DATA_MMU_MISS 1605 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1606 1607#ifdef PTL1_PANIC_DEBUG 1608 /* check if we want to test the tl1 panic */ 1609 sethi %hi(test_ptl1_panic), %g4 1610 ld [%g4 + %lo(test_ptl1_panic)], %g1 1611 st %g0, [%g4 + %lo(test_ptl1_panic)] 1612 cmp %g1, %g0 1613 bne,a,pn %icc, ptl1_panic 1614 or %g0, PTL1_BAD_DEBUG, %g1 1615#endif /* PTL1_PANIC_DEBUG */ 16161: 1617 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 1618 /* 1619 * g2 = tag access reg 1620 * g3.l = type 1621 * g3.h = 0 1622 */ 1623 sethi %hi(trap), %g1 1624 or %g1, %lo(trap), %g1 16252: 1626 ba,pt %xcc, sys_trap 1627 mov -1, %g4 1628 SET_SIZE(sfmmu_pagefault) 1629 1630 ENTRY_NP(sfmmu_mmu_trap) 1631 USE_ALTERNATE_GLOBALS(%g5) 1632 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 1633 rdpr %tt, %g6 1634 cmp %g6, FAST_IMMU_MISS_TT 1635 be,a,pn %icc, 1f 1636 mov T_INSTR_MMU_MISS, %g3 1637 cmp %g6, T_INSTR_MMU_MISS 1638 be,a,pn %icc, 1f 1639 mov T_INSTR_MMU_MISS, %g3 1640 mov %g5, %g2 1641 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1642 cmp %g6, FAST_DMMU_MISS_TT 1643 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1644 cmp %g6, T_DATA_MMU_MISS 1645 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 16461: 1647 /* 1648 * g2 = tag access reg 1649 * g3 = type 1650 */ 1651 sethi %hi(sfmmu_tsbmiss_exception), %g1 1652 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 1653 ba,pt %xcc, sys_trap 1654 mov -1, %g4 1655 /*NOTREACHED*/ 1656 SET_SIZE(sfmmu_mmu_trap) 1657 1658 ENTRY_NP(sfmmu_suspend_tl) 1659 USE_ALTERNATE_GLOBALS(%g5) 1660 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 1661 rdpr %tt, %g6 1662 cmp %g6, FAST_IMMU_MISS_TT 1663 be,a,pn %icc, 1f 1664 mov T_INSTR_MMU_MISS, %g3 1665 mov %g5, %g2 1666 cmp %g6, FAST_DMMU_MISS_TT 1667 move %icc, T_DATA_MMU_MISS, %g3 1668 movne %icc, T_DATA_PROT, %g3 16691: 1670 sethi %hi(sfmmu_tsbmiss_suspended), %g1 1671 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 1672 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 1673 ba,pt %xcc, sys_trap 1674 mov PIL_15, %g4 1675 /*NOTREACHED*/ 1676 SET_SIZE(sfmmu_suspend_tl) 1677 1678 /* 1679 * No %g registers in use at this point. 1680 */ 1681 ENTRY_NP(sfmmu_window_trap) 1682 rdpr %tpc, %g1 1683#ifdef sun4v 1684#ifdef DEBUG 1685 /* We assume previous %gl was 1 */ 1686 rdpr %tstate, %g4 1687 srlx %g4, TSTATE_GL_SHIFT, %g4 1688 and %g4, TSTATE_GL_MASK, %g4 1689 cmp %g4, 1 1690 bne,a,pn %icc, ptl1_panic 1691 mov PTL1_BAD_WTRAP, %g1 1692#endif /* DEBUG */ 1693 /* user miss at tl>1. better be the window handler or user_rtt */ 1694 /* in user_rtt? */ 1695 set rtt_fill_start, %g4 1696 cmp %g1, %g4 1697 blu,pn %xcc, 6f 1698 .empty 1699 set rtt_fill_end, %g4 1700 cmp %g1, %g4 1701 bgeu,pn %xcc, 6f 1702 nop 1703 set fault_rtt_fn1, %g1 1704 wrpr %g0, %g1, %tnpc 1705 ba,a 7f 17066: 1707 ! must save this trap level before descending trap stack 1708 ! no need to save %tnpc, either overwritten or discarded 1709 ! already got it: rdpr %tpc, %g1 1710 rdpr %tstate, %g6 1711 rdpr %tt, %g7 1712 ! trap level saved, go get underlying trap type 1713 rdpr %tl, %g5 1714 sub %g5, 1, %g3 1715 wrpr %g3, %tl 1716 rdpr %tt, %g2 1717 wrpr %g5, %tl 1718 ! restore saved trap level 1719 wrpr %g1, %tpc 1720 wrpr %g6, %tstate 1721 wrpr %g7, %tt 1722#else /* sun4v */ 1723 /* user miss at tl>1. better be the window handler */ 1724 rdpr %tl, %g5 1725 sub %g5, 1, %g3 1726 wrpr %g3, %tl 1727 rdpr %tt, %g2 1728 wrpr %g5, %tl 1729#endif /* sun4v */ 1730 and %g2, WTRAP_TTMASK, %g4 1731 cmp %g4, WTRAP_TYPE 1732 bne,pn %xcc, 1f 1733 nop 1734 /* tpc should be in the trap table */ 1735 set trap_table, %g4 1736 cmp %g1, %g4 1737 blt,pn %xcc, 1f 1738 .empty 1739 set etrap_table, %g4 1740 cmp %g1, %g4 1741 bge,pn %xcc, 1f 1742 .empty 1743 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 1744 add %g1, WTRAP_FAULTOFF, %g1 1745 wrpr %g0, %g1, %tnpc 17467: 1747 /* 1748 * some wbuf handlers will call systrap to resolve the fault 1749 * we pass the trap type so they figure out the correct parameters. 1750 * g5 = trap type, g6 = tag access reg 1751 */ 1752 1753 /* 1754 * only use g5, g6, g7 registers after we have switched to alternate 1755 * globals. 1756 */ 1757 SET_GL_REG(1) 1758 USE_ALTERNATE_GLOBALS(%g5) 1759 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 1760 rdpr %tt, %g7 1761 cmp %g7, FAST_IMMU_MISS_TT 1762 be,a,pn %icc, ptl1_panic 1763 mov PTL1_BAD_WTRAP, %g1 1764 cmp %g7, T_INSTR_MMU_MISS 1765 be,a,pn %icc, ptl1_panic 1766 mov PTL1_BAD_WTRAP, %g1 1767 mov T_DATA_PROT, %g5 1768 cmp %g7, FAST_DMMU_MISS_TT 1769 move %icc, T_DATA_MMU_MISS, %g5 1770 cmp %g7, T_DATA_MMU_MISS 1771 move %icc, T_DATA_MMU_MISS, %g5 1772 ! XXXQ AGS re-check out this one 1773 done 17741: 1775 CPU_ADDR(%g1, %g4) 1776 ld [%g1 + CPU_TL1_HDLR], %g4 1777 brnz,a,pt %g4, sfmmu_mmu_trap 1778 st %g0, [%g1 + CPU_TL1_HDLR] 1779 ba,pt %icc, ptl1_panic 1780 mov PTL1_BAD_TRAP, %g1 1781 SET_SIZE(sfmmu_window_trap) 1782 1783 ENTRY_NP(sfmmu_kpm_exception) 1784 /* 1785 * We have accessed an unmapped segkpm address or a legal segkpm 1786 * address which is involved in a VAC alias conflict prevention. 1787 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 1788 * set. If it is, we will instead note that a fault has occurred 1789 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 1790 * a "retry"). This will step over the faulting instruction. 1791 * Note that this means that a legal segkpm address involved in 1792 * a VAC alias conflict prevention (a rare case to begin with) 1793 * cannot be used in DTrace. 1794 */ 1795 CPU_INDEX(%g1, %g2) 1796 set cpu_core, %g2 1797 sllx %g1, CPU_CORE_SHIFT, %g1 1798 add %g1, %g2, %g1 1799 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 1800 andcc %g2, CPU_DTRACE_NOFAULT, %g0 1801 bz 0f 1802 or %g2, CPU_DTRACE_BADADDR, %g2 1803 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 1804 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 1805 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 1806 done 18070: 1808 TSTAT_CHECK_TL1(1f, %g1, %g2) 18091: 1810 USE_ALTERNATE_GLOBALS(%g5) 1811 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 1812 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1813 /* 1814 * g2=tagacc g3.l=type g3.h=0 1815 */ 1816 sethi %hi(trap), %g1 1817 or %g1, %lo(trap), %g1 1818 ba,pt %xcc, sys_trap 1819 mov -1, %g4 1820 SET_SIZE(sfmmu_kpm_exception) 1821 1822#endif /* lint */ 1823 1824#if defined (lint) 1825 1826void 1827sfmmu_tsb_miss(void) 1828{ 1829} 1830 1831void 1832sfmmu_kpm_dtsb_miss(void) 1833{ 1834} 1835 1836void 1837sfmmu_kpm_dtsb_miss_small(void) 1838{ 1839} 1840 1841#else /* lint */ 1842 1843 1844#if (CTX_SIZE != (1 << CTX_SZ_SHIFT)) 1845#error - size of context struct does not match with CTX_SZ_SHIFT 1846#endif 1847 1848#if (IMAP_SEG != 0) 1849#error - ism_map->ism_seg offset is not zero 1850#endif 1851 1852/* 1853 * Copies ism mapping for this ctx in param "ism" if this is a ISM 1854 * tlb miss and branches to label "ismhit". If this is not an ISM 1855 * process or an ISM tlb miss it falls thru. 1856 * 1857 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 1858 * this process. 1859 * If so, it will branch to label "ismhit". If not, it will fall through. 1860 * 1861 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 1862 * so that any other threads of this process will not try and walk the ism 1863 * maps while they are being changed. 1864 * 1865 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 1866 * will make sure of that. This means we can terminate our search on 1867 * the first zero mapping we find. 1868 * 1869 * Parameters: 1870 * tagacc = tag access register (vaddr + ctx) (in) 1871 * tsbmiss = address of tsb miss area (in) 1872 * ismseg = contents of ism_seg for this ism map (out) 1873 * ismhat = physical address of imap_ismhat for this ism map (out) 1874 * tmp1 = scratch reg (CLOBBERED) 1875 * tmp2 = scratch reg (CLOBBERED) 1876 * tmp3 = scratch reg (CLOBBERED) 1877 * label: temporary labels 1878 * ismhit: label where to jump to if an ism dtlb miss 1879 * exitlabel:label where to jump if hat is busy due to hat_unshare. 1880 */ 1881#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 1882 label, ismhit) \ 1883 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 1884 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 1885 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 1886label/**/1: ;\ 1887 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 1888 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 1889label/**/2: ;\ 1890 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 1891 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 1892 lduha [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 1893 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 1894 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 1895 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 1896 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 1897 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 1898 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 1899 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 1900 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 1901 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 1902 ;\ 1903 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 1904 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 1905 cmp ismhat, tmp1 ;\ 1906 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 1907 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 1908 ;\ 1909 add tmp3, IBLK_NEXTPA, tmp1 ;\ 1910 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 1911 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 1912 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 1913label/**/3: 1914 1915/* 1916 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 1917 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 1918 * Parameters: 1919 * vaddr = reg containing virtual address 1920 * hatid = reg containing sfmmu pointer 1921 * hmeshift = constant/register to shift vaddr to obtain vapg 1922 * hmebp = register where bucket pointer will be stored 1923 * vapg = register where virtual page will be stored 1924 * tmp1, tmp2 = tmp registers 1925 */ 1926 1927 1928#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 1929 vapg, label, tmp1, tmp2) \ 1930 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 1931 brnz,a,pt tmp1, label/**/1 ;\ 1932 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 1933 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 1934 ba,pt %xcc, label/**/2 ;\ 1935 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 1936label/**/1: ;\ 1937 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 1938label/**/2: ;\ 1939 srlx tagacc, hmeshift, vapg ;\ 1940 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 1941 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 1942 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 1943 add hmebp, tmp1, hmebp 1944 1945/* 1946 * hashtag includes bspage + hashno (64 bits). 1947 */ 1948 1949#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 1950 sllx vapg, hmeshift, vapg ;\ 1951 or vapg, hashno, hblktag 1952 1953/* 1954 * Function to traverse hmeblk hash link list and find corresponding match. 1955 * The search is done using physical pointers. It returns the physical address 1956 * and virtual address pointers to the hmeblk that matches with the tag 1957 * provided. 1958 * Parameters: 1959 * hmebp = register that points to hme hash bucket, also used as 1960 * tmp reg (clobbered) 1961 * hmeblktag = register with hmeblk tag match 1962 * hatid = register with hatid 1963 * hmeblkpa = register where physical ptr will be stored 1964 * hmeblkva = register where virtual ptr will be stored 1965 * tmp1 = tmp reg 1966 * label: temporary label 1967 */ 1968 1969#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva, \ 1970 tsbarea, tmp1, label) \ 1971 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 1972 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 1973 add hmebp, HMEBUCK_HBLK, hmeblkva ;\ 1974 ldxa [hmeblkva]ASI_MEM, hmeblkva ;\ 1975 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 1976label/**/1: ;\ 1977 brz,pn hmeblkva, label/**/2 ;\ 1978 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 1979 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 1980 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 1981 add hmebp, CLONGSIZE, hmebp ;\ 1982 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 1983 xor tmp1, hmeblktag, tmp1 ;\ 1984 xor hmebp, hatid, hmebp ;\ 1985 or hmebp, tmp1, hmebp ;\ 1986 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 1987 add hmeblkpa, HMEBLK_NEXT, hmebp ;\ 1988 ldna [hmebp]ASI_MEM, hmeblkva /* hmeblk ptr va */ ;\ 1989 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 1990 ba,pt %xcc, label/**/1 ;\ 1991 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 1992label/**/2: 1993 1994 1995#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 1996#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 1997#endif 1998 1999/* 2000 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2001 * he offset for the corresponding hment. 2002 * Parameters: 2003 * vaddr = register with virtual address 2004 * hmeblkpa = physical pointer to hme_blk 2005 * hment = register where address of hment will be stored 2006 * hmentoff = register where hment offset will be stored 2007 * label1 = temporary label 2008 */ 2009#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, tmp1, label1) \ 2010 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2011 lda [hmentoff]ASI_MEM, tmp1 ;\ 2012 andcc tmp1, HBLK_SZMASK, %g0 /* tmp1 = get_hblk_sz(%g5) */ ;\ 2013 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2014 or %g0, HMEBLK_HME1, hmentoff ;\ 2015 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2016 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2017 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2018 add tmp1, HMEBLK_HME1, hmentoff ;\ 2019label1: 2020 2021/* 2022 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2023 * 2024 * tagacc = tag access register (vaddr + ctx) (in) 2025 * hatid = sfmmu pointer for TSB miss (in) 2026 * tte = tte for TLB miss if found, otherwise clobbered (out) 2027 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2028 * hmeblkva = VA of hment if found, otherwise clobbered (out) 2029 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2030 * hmentoff = temporarily stores hment offset (clobbered) 2031 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2032 * for this page size. 2033 * hashno = constant/register hash number 2034 * label = temporary label for branching within macro. 2035 * foundlabel = label to jump to when tte is found. 2036 * suspendlabel= label to jump to when tte is suspended. 2037 * exitlabel = label to jump to when tte is not found. The hmebp lock 2038 * is still held at this time. 2039 * 2040 * The caller should set up the tsbmiss->scratch[2] field correctly before 2041 * calling this funciton (aka TSBMISS_SCRATCH + TSBMISS_HATID) 2042 */ 2043#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmentoff, \ 2044 hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \ 2045 ;\ 2046 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2047 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2048 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2049 hmeblkpa, label/**/5, hmentoff, hmeblkva) ;\ 2050 ;\ 2051 /* ;\ 2052 * tagacc = tagacc ;\ 2053 * hatid = hatid ;\ 2054 * tsbarea = tsbarea ;\ 2055 * tte = hmebp (hme bucket pointer) ;\ 2056 * hmeblkpa = vapg (virtual page) ;\ 2057 * hmentoff, hmeblkva = scratch ;\ 2058 */ ;\ 2059 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmentoff) ;\ 2060 ;\ 2061 /* ;\ 2062 * tagacc = tagacc ;\ 2063 * hatid = hatid ;\ 2064 * tte = hmebp ;\ 2065 * hmeblkpa = CLOBBERED ;\ 2066 * hmentoff = htag_bspage & hashno ;\ 2067 * hmeblkva = scratch ;\ 2068 */ ;\ 2069 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2070 HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM) ;\ 2071 HMEHASH_SEARCH(tte, hmentoff, hatid, hmeblkpa, hmeblkva, \ 2072 tsbarea, tagacc, label/**/1) ;\ 2073 /* ;\ 2074 * tagacc = CLOBBERED ;\ 2075 * tte = CLOBBERED ;\ 2076 * hmeblkpa = hmeblkpa ;\ 2077 * hmeblkva = hmeblkva ;\ 2078 */ ;\ 2079 brnz,pt hmeblkva, label/**/4 /* branch if hmeblk found */ ;\ 2080 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2081 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva ;\ 2082 HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM) /* drop lock */ ;\ 2083 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2084 nop ;\ 2085label/**/4: ;\ 2086 /* ;\ 2087 * We have found the hmeblk containing the hment. ;\ 2088 * Now we calculate the corresponding tte. ;\ 2089 * ;\ 2090 * tagacc = tagacc ;\ 2091 * hatid = clobbered ;\ 2092 * tte = hmebp ;\ 2093 * hmeblkpa = hmeblkpa ;\ 2094 * hmentoff = hblktag ;\ 2095 * hmeblkva = hmeblkva ;\ 2096 */ ;\ 2097 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hmentoff, hatid, label/**/2) ;\ 2098 ;\ 2099 add hmentoff, SFHME_TTE, hmentoff ;\ 2100 add hmeblkpa, hmentoff, hmeblkpa ;\ 2101 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2102 add hmeblkva, hmentoff, hmeblkva ;\ 2103 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2104 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmentoff ;\ 2105 HMELOCK_EXIT(hmentoff, hmentoff, ASI_MEM) /* drop lock */ ;\ 2106 set TTE_SUSPEND, hmentoff ;\ 2107 TTE_SUSPEND_INT_SHIFT(hmentoff) ;\ 2108 btst tte, hmentoff ;\ 2109 bz,pt %xcc, foundlabel ;\ 2110 nop ;\ 2111 ;\ 2112 /* ;\ 2113 * Mapping is suspended, so goto suspend label. ;\ 2114 */ ;\ 2115 ba,pt %xcc, suspendlabel ;\ 2116 nop 2117 2118 /* 2119 * KERNEL PROTECTION HANDLER 2120 * 2121 * g1 = tsb8k pointer register (clobbered) 2122 * g2 = tag access register (ro) 2123 * g3 - g7 = scratch registers 2124 * 2125 * Note: This function is patched at runtime for performance reasons. 2126 * Any changes here require sfmmu_patch_ktsb fixed. 2127 */ 2128 ENTRY_NP(sfmmu_kprot_trap) 2129 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2130sfmmu_kprot_patch_ktsb_base: 2131 RUNTIME_PATCH_SETX(%g1, %g6) 2132 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2133sfmmu_kprot_patch_ktsb_szcode: 2134 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2135 2136 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2137 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2138 2139 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2140sfmmu_kprot_patch_ktsb4m_base: 2141 RUNTIME_PATCH_SETX(%g3, %g6) 2142 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2143sfmmu_kprot_patch_ktsb4m_szcode: 2144 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2145 2146 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2147 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2148 2149 CPU_TSBMISS_AREA(%g6, %g7) 2150 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2151 ba,pt %xcc, sfmmu_tsb_miss_tt 2152 nop 2153 2154 /* 2155 * USER PROTECTION HANDLER 2156 * 2157 * g1 = tsb8k pointer register (ro) 2158 * g2 = tag access register (ro) 2159 * g3 = faulting context (clobbered, currently not used) 2160 * g4 - g7 = scratch registers 2161 */ 2162 ALTENTRY(sfmmu_uprot_trap) 2163#ifdef sun4v 2164 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2165 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2166 2167 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2168 brlz,pt %g3, 9f /* check for 2nd TSB */ 2169 mov %g0, %g3 /* clear second tsbe ptr */ 2170 2171 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2172 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2173 2174#else /* sun4v */ 2175 2176 brgez,pt %g1, 9f /* check for 2nd TSB */ 2177 mov %g0, %g3 /* clear second tsbe ptr */ 2178 2179 mov %g2, %g7 2180 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2181 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2182 mov %g1, %g7 2183 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2184 2185#endif /* sun4v */ 21869: 2187 CPU_TSBMISS_AREA(%g6, %g7) 2188 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2189 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2190 nop 2191 2192 /* 2193 * Kernel 8K page iTLB miss. We also get here if we took a 2194 * fast instruction access mmu miss trap while running in 2195 * invalid context. 2196 * 2197 * %g1 = 8K TSB pointer register (not used, clobbered) 2198 * %g2 = tag access register (used) 2199 * %g3 = faulting context id (used) 2200 * %g7 = 4M virtual page number for tag matching (used) 2201 */ 2202 .align 64 2203 ALTENTRY(sfmmu_kitlb_miss) 2204 brnz,pn %g3, tsb_tl0_noctxt 2205 nop 2206 2207 /* kernel miss */ 2208 /* get kernel tsb pointer */ 2209 /* we patch the next set of instructions at run time */ 2210 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2211iktsbbase: 2212 RUNTIME_PATCH_SETX(%g4, %g5) 2213 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2214 2215iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2216 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2217 or %g4, %g1, %g1 ! form tsb ptr 2218 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2219 cmp %g4, %g7 2220 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2221 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2222 bz,pn %icc, exec_fault 2223 nop 2224 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2225 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2226 retry 2227 2228 /* 2229 * Kernel dTLB miss. We also get here if we took a fast data 2230 * access mmu miss trap while running in invalid context. 2231 * 2232 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2233 * We select the TSB miss handler to branch to depending on 2234 * the virtual address of the access. In the future it may 2235 * be desirable to separate kpm TTEs into their own TSB, 2236 * in which case all that needs to be done is to set 2237 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2238 * early in the miss if we detect a kpm VA to a new handler. 2239 * 2240 * %g1 = 8K TSB pointer register (not used, clobbered) 2241 * %g2 = tag access register (used) 2242 * %g3 = faulting context id (used) 2243 */ 2244 .align 64 2245 ALTENTRY(sfmmu_kdtlb_miss) 2246 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2247 nop 2248 2249 /* Gather some stats for kpm misses in the TLB. */ 2250 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2251 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2252 2253 /* 2254 * Get first TSB offset and look for 8K/64K/512K mapping 2255 * using the 8K virtual page as the index. 2256 * 2257 * We patch the next set of instructions at run time; 2258 * any changes here require sfmmu_patch_ktsb changes too. 2259 */ 2260dktsbbase: 2261 RUNTIME_PATCH_SETX(%g7, %g6) 2262 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2263 2264dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2265 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2266 2267 /* 2268 * At this point %g1 is our index into the TSB. 2269 * We just masked off enough bits of the VA depending 2270 * on our TSB size code. 2271 */ 2272 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2273 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2274 cmp %g6, %g4 ! compare tag 2275 bne,pn %xcc, dktsb4m_kpmcheck_small 2276 add %g7, %g1, %g1 /* form tsb ptr */ 2277 TT_TRACE(trace_tsbhit) 2278 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2279 /* trapstat expects tte in %g5 */ 2280 retry 2281 2282 /* 2283 * If kpm is using large pages, the following instruction needs 2284 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2285 * so that we will probe the 4M TSB regardless of the VA. In 2286 * the case kpm is using small pages, we know no large kernel 2287 * mappings are located above 0x80000000.00000000 so we skip the 2288 * probe as an optimization. 2289 */ 2290dktsb4m_kpmcheck_small: 2291 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2292 /* delay slot safe, below */ 2293 2294 /* 2295 * Get second TSB offset and look for 4M mapping 2296 * using 4M virtual page as the TSB index. 2297 * 2298 * Here: 2299 * %g1 = 8K TSB pointer. Don't squash it. 2300 * %g2 = tag access register (we still need it) 2301 */ 2302 srlx %g2, MMU_PAGESHIFT4M, %g3 2303 2304 /* 2305 * We patch the next set of instructions at run time; 2306 * any changes here require sfmmu_patch_ktsb changes too. 2307 */ 2308dktsb4mbase: 2309 RUNTIME_PATCH_SETX(%g7, %g6) 2310 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2311dktsb4m: 2312 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2313 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2314 2315 /* 2316 * At this point %g3 is our index into the TSB. 2317 * We just masked off enough bits of the VA depending 2318 * on our TSB size code. 2319 */ 2320 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2321 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2322 cmp %g6, %g4 ! compare tag 2323 2324dktsb4m_tsbmiss: 2325 bne,pn %xcc, dktsb4m_kpmcheck 2326 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2327 TT_TRACE(trace_tsbhit) 2328 /* we don't check TTE size here since we assume 4M TSB is separate */ 2329 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2330 /* trapstat expects tte in %g5 */ 2331 retry 2332 2333 /* 2334 * So, we failed to find a valid TTE to match the faulting 2335 * address in either TSB. There are a few cases that could land 2336 * us here: 2337 * 2338 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2339 * to sfmmu_tsb_miss_tt to handle the miss. 2340 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2341 * 4M TSB. Let segkpm handle it. 2342 * 2343 * Note that we shouldn't land here in the case of a kpm VA when 2344 * kpm_smallpages is active -- we handled that case earlier at 2345 * dktsb4m_kpmcheck_small. 2346 * 2347 * At this point: 2348 * g1 = 8K-indexed primary TSB pointer 2349 * g2 = tag access register 2350 * g3 = 4M-indexed secondary TSB pointer 2351 */ 2352dktsb4m_kpmcheck: 2353 cmp %g2, %g0 2354 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2355 nop 2356 ba,a,pt %icc, sfmmu_tsb_miss_tt 2357 nop 2358 2359#ifdef sun4v 2360 /* 2361 * User instruction miss w/ single TSB. 2362 * The first probe covers 8K, 64K, and 512K page sizes, 2363 * because 64K and 512K mappings are replicated off 8K 2364 * pointer. 2365 * 2366 * g1 = tsb8k pointer register 2367 * g2 = tag access register 2368 * g3 - g6 = scratch registers 2369 * g7 = TSB tag to match 2370 */ 2371 .align 64 2372 ALTENTRY(sfmmu_uitlb_fastpath) 2373 2374 SETUP_UTSB_ATOMIC_ASI(%g4, %g5) 2375 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2376 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2377 ba,pn %xcc, sfmmu_tsb_miss_tt 2378 mov %g0, %g3 2379 2380 /* 2381 * User data miss w/ single TSB. 2382 * The first probe covers 8K, 64K, and 512K page sizes, 2383 * because 64K and 512K mappings are replicated off 8K 2384 * pointer. 2385 * 2386 * g1 = tsb8k pointer register 2387 * g2 = tag access register 2388 * g3 - g6 = scratch registers 2389 * g7 = TSB tag to match 2390 */ 2391 .align 64 2392 ALTENTRY(sfmmu_udtlb_fastpath) 2393 2394 SETUP_UTSB_ATOMIC_ASI(%g4, %g6) 2395 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2396 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2397 ba,pn %xcc, sfmmu_tsb_miss_tt 2398 mov %g0, %g3 2399 2400#endif /* sun4v */ 2401 2402 /* 2403 * User instruction miss w/ multiple TSBs. 2404 * The first probe covers 8K, 64K, and 512K page sizes, 2405 * because 64K and 512K mappings are replicated off 8K 2406 * pointer. Second probe covers 4M page size only. 2407 * 2408 * Just like sfmmu_udtlb_slowpath, except: 2409 * o Uses ASI_ITLB_IN 2410 * o checks for execute permission 2411 * o No ISM prediction. 2412 * 2413 * g1 = tsb8k pointer register 2414 * g2 = tag access register 2415 * g3 - g6 = scratch registers 2416 * g7 = TSB tag to match 2417 */ 2418 .align 64 2419 ALTENTRY(sfmmu_uitlb_slowpath) 2420 2421#ifdef sun4v 2422 SETUP_UTSB_ATOMIC_ASI(%g4, %g5) 2423 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2424 2425 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2426 /* g4 - g5 = clobbered here */ 2427 2428 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2429 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2430 srlx %g2, TAG_VALO_SHIFT, %g7 2431 PROBE_2ND_ITSB(%g3, %g7) 2432 /* NOT REACHED */ 2433#else /* sun4v */ 2434 mov %g1, %g3 /* save tsb8k reg in %g3 */ 2435 SETUP_UTSB_ATOMIC_ASI(%g4, %g5) 2436 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 2437 2438 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2439 /* g4 - g5 = clobbered here */ 2440 2441 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 2442 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 2443 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 2444 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2445 srlx %g2, TAG_VALO_SHIFT, %g7 2446 PROBE_2ND_ITSB(%g3, %g7, isynth) 2447 /* NOT REACHED */ 2448#endif /* sun4v */ 2449 2450 /* 2451 * User data miss w/ multiple TSBs. 2452 * The first probe covers 8K, 64K, and 512K page sizes, 2453 * because 64K and 512K mappings are replicated off 8K 2454 * pointer. Second probe covers 4M page size only. 2455 * 2456 * We consider probing for 4M pages first if the VA falls 2457 * in a range that's likely to be ISM. 2458 * 2459 * g1 = tsb8k pointer register 2460 * g2 = tag access register 2461 * g3 - g6 = scratch registers 2462 * g7 = TSB tag to match 2463 */ 2464 .align 64 2465 ALTENTRY(sfmmu_udtlb_slowpath) 2466 2467 SETUP_UTSB_ATOMIC_ASI(%g4, %g6) 2468 2469 /* 2470 * Check for ISM. If it exists, look for 4M mappings in the second TSB 2471 * first, then probe for other mappings in the first TSB if that fails. 2472 */ 2473 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 2474 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 2475 mov %g1, %g3 2476 2477udtlb_miss_probefirst: 2478 /* 2479 * g1 = 8K TSB pointer register 2480 * g2 = tag access register 2481 * g3 = (potentially) second TSB entry ptr 2482 * g6 = ism pred. 2483 * g7 = vpg_4m 2484 */ 2485#ifdef sun4v 2486 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2487 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2488 2489 /* 2490 * Here: 2491 * g1 = first TSB pointer 2492 * g2 = tag access reg 2493 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2494 */ 2495 brgz,pn %g6, sfmmu_tsb_miss_tt 2496 nop 2497#else /* sun4v */ 2498 mov %g1, %g4 2499 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 2500 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2501 2502 /* 2503 * Here: 2504 * g1 = first TSB pointer 2505 * g2 = tag access reg 2506 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2507 */ 2508 brgz,pn %g6, sfmmu_tsb_miss_tt 2509 nop 2510 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 2511 /* fall through in 8K->4M probe order */ 2512#endif /* sun4v */ 2513 2514udtlb_miss_probesecond: 2515 /* 2516 * Look in the second TSB for the TTE 2517 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 2518 * g2 = tag access reg 2519 * g3 = 8K TSB pointer register 2520 * g6 = ism pred. 2521 * g7 = vpg_4m 2522 */ 2523#ifdef sun4v 2524 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 2525 /* tagacc (%g2) not destroyed */ 2526 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2527 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 2528#else 2529 mov %g3, %g7 2530 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 2531 /* %g2 clobbered, %g3 =second tsbe ptr */ 2532 mov MMU_TAG_ACCESS, %g2 2533 ldxa [%g2]ASI_DMMU, %g2 2534#endif 2535 2536 srlx %g2, TAG_VALO_SHIFT, %g7 2537 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2538 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 2539 brgz,pn %g6, udtlb_miss_probefirst 2540 nop 2541 2542 /* fall through to sfmmu_tsb_miss_tt */ 2543 2544 ALTENTRY(sfmmu_tsb_miss_tt) 2545 TT_TRACE(trace_tsbmiss) 2546 /* 2547 * We get here if there is a TSB miss OR a write protect trap. 2548 * 2549 * g1 = First TSB entry pointer 2550 * g2 = tag access register 2551 * g3 = 4M TSB entry pointer; NULL if no 2nd TSB 2552 * g4 - g7 = scratch registers 2553 */ 2554 2555 ALTENTRY(sfmmu_tsb_miss) 2556 2557 /* 2558 * If trapstat is running, we need to shift the %tpc and %tnpc to 2559 * point to trapstat's TSB miss return code (note that trapstat 2560 * itself will patch the correct offset to add). 2561 */ 2562 rdpr %tl, %g7 2563 cmp %g7, 1 2564 ble,pt %xcc, 0f 2565 sethi %hi(KERNELBASE), %g6 2566 rdpr %tpc, %g7 2567 or %g6, %lo(KERNELBASE), %g6 2568 cmp %g7, %g6 2569 bgeu,pt %xcc, 0f 2570 /* delay slot safe */ 2571 2572 ALTENTRY(tsbmiss_trapstat_patch_point) 2573 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 2574 wrpr %g7, %tpc 2575 add %g7, 4, %g7 2576 wrpr %g7, %tnpc 25770: 2578 CPU_TSBMISS_AREA(%g6, %g7) 2579 2580 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save first tsb pointer */ 2581 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save second tsb pointer */ 2582 2583 sllx %g2, TAGACC_CTX_LSHIFT, %g3 2584 brz,a,pn %g3, 1f /* skip ahead if kernel */ 2585 ldn [%g6 + TSBMISS_KHATID], %g7 2586 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 2587 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 2588 2589 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 2590 2591 cmp %g3, INVALID_CONTEXT 2592 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 2593 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 2594 2595 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 2596 /* 2597 * The miss wasn't in an ISM segment. 2598 * 2599 * %g1 %g3, %g4, %g5, %g7 all clobbered 2600 * %g2 = tag access (vaddr + ctx) 2601 */ 2602 2603 ba,pt %icc, 2f 2604 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 2605 26061: 2607 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 2608 /* 2609 * 8K and 64K hash. 2610 */ 26112: 2612 2613 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 2614 MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte, 2615 sfmmu_suspend_tl, tsb_512K) 2616 /* NOT REACHED */ 2617 2618tsb_512K: 2619 ldn [%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3 2620 sllx %g3, TAGACC_CTX_LSHIFT, %g5 2621 brz,pn %g5, 3f 2622 lduh [%g6 + TSBMISS_HATFLAGS], %g4 2623 and %g4, HAT_512K_FLAG, %g5 2624 2625 /* 2626 * Note that there is a small window here where we may have 2627 * a 512k page in the hash list but have not set the HAT_512K_FLAG 2628 * flag yet, so we will skip searching the 512k hash list. 2629 * In this case we will end up in pagefault which will find 2630 * the mapping and return. So, in this instance we will end up 2631 * spending a bit more time resolving this TSB miss, but it can 2632 * only happen once per process and even then, the chances of that 2633 * are very small, so it's not worth the extra overhead it would 2634 * take to close this window. 2635 */ 2636 brz,pn %g5, tsb_4M 2637 nop 26383: 2639 /* 2640 * 512K hash 2641 */ 2642 2643 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 2644 MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte, 2645 sfmmu_suspend_tl, tsb_4M) 2646 /* NOT REACHED */ 2647 2648tsb_4M: 2649 ldn [%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3 2650 sllx %g3, TAGACC_CTX_LSHIFT, %g5 2651 brz,pn %g5, 4f 2652 lduh [%g6 + TSBMISS_HATFLAGS], %g4 2653 and %g4, HAT_4M_FLAG, %g5 2654 brz,pn %g5, tsb_32M 2655 nop 26564: 2657 /* 2658 * 4M hash 2659 */ 2660 2661 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 2662 MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte, 2663 sfmmu_suspend_tl, tsb_32M) 2664 /* NOT REACHED */ 2665 2666tsb_32M: 2667#ifndef sun4v 2668 GET_CPU_IMPL(%g5) 2669 cmp %g5, PANTHER_IMPL 2670 bne,pt %xcc, tsb_pagefault 2671 nop 2672#endif 2673 2674 ldn [%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3 2675 sllx %g3, TAGACC_CTX_LSHIFT, %g5 2676#ifdef sun4v 2677 brz,pn %g5, 6f 2678#else 2679 brz,pn %g5, tsb_pagefault 2680#endif 2681 lduh [%g6 + TSBMISS_HATFLAGS], %g4 2682 and %g4, HAT_32M_FLAG, %g5 2683 brz,pn %g5, tsb_256M 2684 nop 26855: 2686 /* 2687 * 32M hash 2688 */ 2689 2690 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 2691 MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte, 2692 sfmmu_suspend_tl, tsb_256M) 2693 /* NOT REACHED */ 2694 2695tsb_256M: 2696 lduh [%g6 + TSBMISS_HATFLAGS], %g4 2697 and %g4, HAT_256M_FLAG, %g5 2698 brz,pn %g5, tsb_pagefault 2699 nop 27006: 2701 /* 2702 * 256M hash 2703 */ 2704 2705 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, 2706 MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte, 2707 sfmmu_suspend_tl, tsb_pagefault) 2708 /* NOT REACHED */ 2709 2710tsb_checktte: 2711 /* 2712 * g3 = tte 2713 * g4 = tte pa 2714 * g5 = tte va 2715 * g6 = tsbmiss area 2716 */ 2717 brgez,pn %g3, tsb_pagefault /* if tte invalid branch */ 2718 nop 2719 2720tsb_validtte: 2721 /* 2722 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 2723 */ 2724 rdpr %tt, %g7 2725 cmp %g7, FAST_PROT_TT 2726 bne,pt %icc, 4f 2727 nop 2728 2729 TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod, 2730 tsb_protfault) 2731 2732 rdpr %tt, %g5 2733 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 2734 ba,pt %xcc, tsb_update_tl1 2735 nop 2736 27374: 2738 /* 2739 * If ITLB miss check exec bit. 2740 * If not set treat as invalid TTE. 2741 */ 2742 cmp %g7, T_INSTR_MMU_MISS 2743 be,pn %icc, 5f 2744 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 2745 cmp %g7, FAST_IMMU_MISS_TT 2746 bne,pt %icc, 3f 2747 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 27485: 2749 bz,pn %icc, tsb_protfault 2750 nop 2751 27523: 2753 /* 2754 * Set reference bit if not already set 2755 */ 2756 TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref) 2757 2758 /* 2759 * Now, load into TSB/TLB. At this point: 2760 * g3 = tte 2761 * g4 = patte 2762 * g6 = tsbmiss area 2763 */ 2764 rdpr %tt, %g5 2765#ifdef sun4v 2766 MMU_FAULT_STATUS_AREA(%g2) 2767 cmp %g5, T_INSTR_MMU_MISS 2768 be,a,pt %icc, 9f 2769 nop 2770 cmp %g5, FAST_IMMU_MISS_TT 2771 be,a,pt %icc, 9f 2772 nop 2773 add %g2, MMFSA_D_, %g2 27749: 2775 ldx [%g2 + MMFSA_CTX_], %g7 2776 sllx %g7, TTARGET_CTX_SHIFT, %g7 2777 ldx [%g2 + MMFSA_ADDR_], %g2 2778 srlx %g2, TTARGET_VA_SHIFT, %g2 2779 or %g2, %g7, %g2 2780#else 2781 cmp %g5, FAST_IMMU_MISS_TT 2782 be,a,pt %icc, tsb_update_tl1 2783 ldxa [%g0]ASI_IMMU, %g2 2784 ldxa [%g0]ASI_DMMU, %g2 2785#endif 2786tsb_update_tl1: 2787 srlx %g2, TTARGET_CTX_SHIFT, %g7 2788 brz,pn %g7, tsb_kernel 2789#ifdef sun4v 2790 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 2791#else 2792 srlx %g3, TTE_SZ_SHFT, %g7 2793#endif 2794 2795tsb_user: 2796#ifdef sun4v 2797 cmp %g7, TTE4M 2798 bge,pn %icc, tsb_user4m 2799 nop 2800#else 2801 cmp %g7, TTESZ_VALID | TTE4M 2802 be,pn %icc, tsb_user4m 2803 srlx %g3, TTE_SZ2_SHFT, %g7 2804 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 2805 bnz,a,pn %icc, tsb_user_pn_synth 2806 cmp %g5, FAST_IMMU_MISS_TT 2807#endif 2808 2809tsb_user8k: 2810 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = first TSB ptr 2811 2812#ifndef sun4v 2813 mov ASI_N, %g7 ! user TSBs always accessed by VA 2814 mov %g7, %asi 2815#endif /* sun4v */ 2816 2817 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5) 2818 2819#ifdef sun4v 2820 cmp %g5, T_INSTR_MMU_MISS 2821 be,a,pn %xcc, 9f 2822 mov %g3, %g5 2823#endif /* sun4v */ 2824 cmp %g5, FAST_IMMU_MISS_TT 2825 be,pn %xcc, 9f 2826 mov %g3, %g5 2827 2828 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2829 ! trapstat wants TTE in %g5 2830 retry 28319: 2832 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2833 ! trapstat wants TTE in %g5 2834 retry 2835 2836tsb_user4m: 2837 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 28384: 2839 brz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 2840 nop 2841 2842#ifndef sun4v 2843 mov ASI_N, %g7 ! user TSBs always accessed by VA 2844 mov %g7, %asi 2845#endif 2846 2847 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6) 2848 28495: 2850#ifdef sun4v 2851 cmp %g5, T_INSTR_MMU_MISS 2852 be,a,pn %xcc, 9f 2853 mov %g3, %g5 2854#endif /* sun4v */ 2855 cmp %g5, FAST_IMMU_MISS_TT 2856 be,pn %xcc, 9f 2857 mov %g3, %g5 2858 2859 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2860 ! trapstat wants TTE in %g5 2861 retry 28629: 2863 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2864 ! trapstat wants TTE in %g5 2865 retry 2866 2867#ifndef sun4v 2868 /* 2869 * Panther ITLB synthesis. 2870 * The Panther 32M and 256M ITLB code simulates these two large page 2871 * sizes with 4M pages, to provide support for programs, for example 2872 * Java, that may copy instructions into a 32M or 256M data page and 2873 * then execute them. The code below generates the 4M pfn bits and 2874 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 2875 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 2876 * are ignored by the hardware. 2877 * 2878 * Now, load into TSB/TLB. At this point: 2879 * g2 = tagtarget 2880 * g3 = tte 2881 * g4 = patte 2882 * g5 = tt 2883 * g6 = tsbmiss area 2884 */ 2885tsb_user_pn_synth: 2886 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 2887 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 2888 bz,pn %icc, 4b /* if not, been here before */ 2889 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 2890 brz,a,pn %g1, 5f /* no 2nd tsb */ 2891 mov %g3, %g5 2892 2893 mov MMU_TAG_ACCESS, %g7 2894 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 2895 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 2896 2897 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 2898 mov %g7, %asi 2899 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */ 29005: 2901 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2902 retry 2903 2904tsb_user_itlb_synth: 2905 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 2906 2907 mov MMU_TAG_ACCESS, %g7 2908 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 2909 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 2910 brz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 2911 or %g5, %g3, %g5 /* add 4M bits to TTE */ 2912 2913 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 2914 mov %g7, %asi 2915 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */ 29167: 2917 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 2918 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2919 retry 2920#endif 2921 2922tsb_kernel: ! no 32M or 256M support 2923#ifdef sun4v 2924 cmp %g7, TTE4M 2925#else 2926 cmp %g7, TTESZ_VALID | TTE4M 2927#endif 2928 be,pn %icc, 5f 2929 nop 2930 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8k tsbptr 2931 ba,pt %xcc, 6f 2932 nop 29335: 2934 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4m tsbptr 2935 brz,pn %g1, 3f /* skip programming if 4m TSB ptr is NULL */ 2936 nop 29376: 2938#ifndef sun4v 2939tsb_kernel_patch_asi: 2940 or %g0, RUNTIME_PATCH, %g6 2941 mov %g6, %asi ! XXX avoid writing to %asi !! 2942#endif 2943 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7) 29443: 2945#ifdef sun4v 2946 cmp %g5, T_INSTR_MMU_MISS 2947 be,a,pn %icc, 1f 2948 mov %g3, %g5 ! trapstat wants TTE in %g5 2949#endif /* sun4v */ 2950 cmp %g5, FAST_IMMU_MISS_TT 2951 be,pn %icc, 1f 2952 mov %g3, %g5 ! trapstat wants TTE in %g5 2953 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2954 ! trapstat wants TTE in %g5 2955 retry 29561: 2957 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2958 ! trapstat wants TTE in %g5 2959 retry 2960 2961tsb_ism: 2962 /* 2963 * This is an ISM [i|d]tlb miss. We optimize for largest 2964 * page size down to smallest. 2965 * 2966 * g2 = vaddr + ctx aka tag access register 2967 * g3 = ismmap->ism_seg 2968 * g4 = physical address of ismmap->ism_sfmmu 2969 * g6 = tsbmiss area 2970 */ 2971 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 2972 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 2973 mov PTL1_BAD_ISM, %g1 2974 /* g5 = pa of imap_vb_shift */ 2975 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 2976 lduha [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 2977 srlx %g3, %g4, %g3 /* clr size field */ 2978 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 2979 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 2980 and %g2, %g1, %g4 /* g4 = ctx number */ 2981 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 2982 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 2983 or %g2, %g4, %g2 /* g2 = tagacc (vaddr + ctx) */ 2984 2985 /* 2986 * ISM pages are always locked down. 2987 * If we can't find the tte then pagefault 2988 * and let the spt segment driver resovle it. 2989 * 2990 * g2 = ISM vaddr (offset in ISM seg) 2991 * g6 = tsb miss area 2992 * g7 = ISM hatid 2993 */ 2994 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 2995 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 2996 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 2997 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 2998 nop 2999 3000tsb_ism_32M: 3001 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3002 brz,pn %g5, tsb_ism_256M 3003 nop 3004 3005 /* 3006 * 32M hash. 3007 */ 3008 3009 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M, 3010 TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3011 tsb_ism_4M) 3012 /* NOT REACHED */ 3013 3014tsb_ism_32M_found: 3015 brlz,pt %g3, tsb_validtte 3016 nop 3017 ba,pt %xcc, tsb_ism_4M 3018 nop 3019 3020tsb_ism_256M: 3021 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3022 brz,a,pn %g5, ptl1_panic 3023 mov PTL1_BAD_ISM, %g1 3024 3025 /* 3026 * 256M hash. 3027 */ 3028 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M, 3029 TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3030 tsb_ism_4M) 3031 3032tsb_ism_256M_found: 3033 brlz,pt %g3, tsb_validtte 3034 nop 3035 3036tsb_ism_4M: 3037 /* 3038 * 4M hash. 3039 */ 3040 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M, 3041 TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3042 tsb_ism_8K) 3043 /* NOT REACHED */ 3044 3045tsb_ism_4M_found: 3046 brlz,pt %g3, tsb_validtte 3047 nop 3048 3049tsb_ism_8K: 3050 /* 3051 * 8K and 64K hash. 3052 */ 3053 3054 GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K, 3055 TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3056 tsb_pagefault) 3057 /* NOT REACHED */ 3058 3059tsb_ism_8K_found: 3060 brlz,pt %g3, tsb_validtte 3061 nop 3062 3063tsb_pagefault: 3064 rdpr %tt, %g7 3065 cmp %g7, FAST_PROT_TT 3066 be,a,pn %icc, tsb_protfault 3067 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3068 3069tsb_protfault: 3070 /* 3071 * we get here if we couldn't find a valid tte in the hash. 3072 * 3073 * If user and we are at tl>1 we go to window handling code. 3074 * 3075 * If kernel and the fault is on the same page as our stack 3076 * pointer, then we know the stack is bad and the trap handler 3077 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3078 * 3079 * If this is a kernel trap and tl>1, panic. 3080 * 3081 * Otherwise we call pagefault. 3082 */ 3083 cmp %g7, FAST_IMMU_MISS_TT 3084#ifdef sun4v 3085 MMU_FAULT_STATUS_AREA(%g4) 3086 ldx [%g4 + MMFSA_I_CTX], %g5 3087 ldx [%g4 + MMFSA_D_CTX], %g4 3088 move %icc, %g5, %g4 3089 cmp %g7, T_INSTR_MMU_MISS 3090 move %icc, %g5, %g4 3091#else 3092 mov MMU_TAG_ACCESS, %g4 3093 ldxa [%g4]ASI_DMMU, %g2 3094 ldxa [%g4]ASI_IMMU, %g5 3095 move %icc, %g5, %g2 3096 cmp %g7, T_INSTR_MMU_MISS 3097 move %icc, %g5, %g2 3098 sllx %g2, TAGACC_CTX_LSHIFT, %g4 3099#endif 3100 brnz,pn %g4, 3f /* skip if not kernel */ 3101 rdpr %tl, %g5 3102 3103 add %sp, STACK_BIAS, %g3 3104 srlx %g3, MMU_PAGESHIFT, %g3 3105 srlx %g2, MMU_PAGESHIFT, %g4 3106 cmp %g3, %g4 3107 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 3108 mov PTL1_BAD_STACK, %g1 3109 3110 cmp %g5, 1 3111 ble,pt %icc, 2f 3112 nop 3113 TSTAT_CHECK_TL1(2f, %g1, %g2) 3114 rdpr %tt, %g2 3115 cmp %g2, FAST_PROT_TT 3116 mov PTL1_BAD_KPROT_FAULT, %g1 3117 movne %icc, PTL1_BAD_KMISS, %g1 3118 ba,pt %icc, ptl1_panic 3119 nop 3120 31212: 3122 /* 3123 * We are taking a pagefault in the kernel on a kernel address. If 3124 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 3125 * want to call sfmmu_pagefault -- we will instead note that a fault 3126 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 3127 * (instead of a "retry"). This will step over the faulting 3128 * instruction. 3129 */ 3130 CPU_INDEX(%g1, %g2) 3131 set cpu_core, %g2 3132 sllx %g1, CPU_CORE_SHIFT, %g1 3133 add %g1, %g2, %g1 3134 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3135 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3136 bz sfmmu_pagefault 3137 or %g2, CPU_DTRACE_BADADDR, %g2 3138 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3139 GET_MMU_D_ADDR(%g3, %g4) 3140 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3141 done 3142 31433: 3144 cmp %g5, 1 3145 ble,pt %icc, 4f 3146 nop 3147 TSTAT_CHECK_TL1(4f, %g1, %g2) 3148 ba,pt %icc, sfmmu_window_trap 3149 nop 3150 31514: 3152 /* 3153 * We are taking a pagefault on a non-kernel address. If we are in 3154 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 3155 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 3156 */ 3157 CPU_INDEX(%g1, %g2) 3158 set cpu_core, %g2 3159 sllx %g1, CPU_CORE_SHIFT, %g1 3160 add %g1, %g2, %g1 3161 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3162 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3163 bz sfmmu_pagefault 3164 or %g2, CPU_DTRACE_BADADDR, %g2 3165 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3166 GET_MMU_D_ADDR(%g3, %g4) 3167 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3168 3169 /* 3170 * Be sure that we're actually taking this miss from the kernel -- 3171 * otherwise we have managed to return to user-level with 3172 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3173 */ 3174 rdpr %tstate, %g2 3175 btst TSTATE_PRIV, %g2 3176 bz,a ptl1_panic 3177 mov PTL1_BAD_DTRACE_FLAGS, %g1 3178 done 3179 3180 ALTENTRY(tsb_tl0_noctxt) 3181 /* 3182 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 3183 * if it is, indicated that we have faulted and issue a done. 3184 */ 3185 CPU_INDEX(%g5, %g6) 3186 set cpu_core, %g6 3187 sllx %g5, CPU_CORE_SHIFT, %g5 3188 add %g5, %g6, %g5 3189 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 3190 andcc %g6, CPU_DTRACE_NOFAULT, %g0 3191 bz 1f 3192 or %g6, CPU_DTRACE_BADADDR, %g6 3193 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 3194 GET_MMU_D_ADDR(%g3, %g4) 3195 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 3196 3197 /* 3198 * Be sure that we're actually taking this miss from the kernel -- 3199 * otherwise we have managed to return to user-level with 3200 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3201 */ 3202 rdpr %tstate, %g5 3203 btst TSTATE_PRIV, %g5 3204 bz,a ptl1_panic 3205 mov PTL1_BAD_DTRACE_FLAGS, %g1 3206 done 3207 32081: 3209 rdpr %tt, %g5 3210 cmp %g5, FAST_IMMU_MISS_TT 3211#ifdef sun4v 3212 MMU_FAULT_STATUS_AREA(%g2) 3213 be,a,pt %icc, 2f 3214 ldx [%g2 + MMFSA_I_CTX], %g3 3215 cmp %g5, T_INSTR_MMU_MISS 3216 be,a,pt %icc, 2f 3217 ldx [%g2 + MMFSA_I_CTX], %g3 3218 ldx [%g2 + MMFSA_D_CTX], %g3 32192: 3220#else 3221 mov MMU_TAG_ACCESS, %g2 3222 be,a,pt %icc, 2f 3223 ldxa [%g2]ASI_IMMU, %g3 3224 ldxa [%g2]ASI_DMMU, %g3 32252: sllx %g3, TAGACC_CTX_LSHIFT, %g3 3226#endif 3227 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 3228 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 3229 rdpr %tl, %g5 3230 cmp %g5, 1 3231 ble,pt %icc, sfmmu_mmu_trap 3232 nop 3233 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 3234 ba,pt %icc, sfmmu_window_trap 3235 nop 3236 SET_SIZE(sfmmu_tsb_miss) 3237 3238#if (1<< TSBMISS_SHIFT) != TSBMISS_SIZE 3239#error - TSBMISS_SHIFT does not correspond to size of tsbmiss struct 3240#endif 3241 3242#endif /* lint */ 3243 3244#if defined (lint) 3245/* 3246 * This routine will look for a user or kernel vaddr in the hash 3247 * structure. It returns a valid pfn or PFN_INVALID. It doesn't 3248 * grab any locks. It should only be used by other sfmmu routines. 3249 */ 3250/* ARGSUSED */ 3251pfn_t 3252sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) 3253{ 3254 return(0); 3255} 3256 3257#else /* lint */ 3258 3259 ENTRY_NP(sfmmu_vatopfn) 3260 /* 3261 * disable interrupts 3262 */ 3263 rdpr %pstate, %o3 3264#ifdef DEBUG 3265 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 3266 bnz,pt %icc, 1f /* disabled, panic */ 3267 nop 3268 3269 sethi %hi(panicstr), %g1 3270 ldx [%g1 + %lo(panicstr)], %g1 3271 tst %g1 3272 bnz,pt %icc, 1f 3273 nop 3274 3275 save %sp, -SA(MINFRAME), %sp 3276 sethi %hi(sfmmu_panic1), %o0 3277 call panic 3278 or %o0, %lo(sfmmu_panic1), %o0 32791: 3280#endif 3281 /* 3282 * disable interrupts to protect the TSBMISS area 3283 */ 3284 andn %o3, PSTATE_IE, %o5 3285 wrpr %o5, 0, %pstate 3286 3287 /* 3288 * o0 = vaddr 3289 * o1 = sfmmup 3290 * o2 = ttep 3291 */ 3292 CPU_TSBMISS_AREA(%g1, %o5) 3293 ldn [%g1 + TSBMISS_KHATID], %o4 3294 cmp %o4, %o1 3295 bne,pn %ncc, vatopfn_nokernel 3296 mov TTE64K, %g5 /* g5 = rehash # */ 3297 mov %g1,%o5 /* o5 = tsbmiss_area */ 3298 /* 3299 * o0 = vaddr 3300 * o1 & o4 = hatid 3301 * o2 = ttep 3302 * o5 = tsbmiss area 3303 */ 3304 mov HBLK_RANGE_SHIFT, %g6 33051: 3306 3307 /* 3308 * o0 = vaddr 3309 * o1 = sfmmup 3310 * o2 = ttep 3311 * o3 = old %pstate 3312 * o4 = hatid 3313 * o5 = tsbmiss 3314 * g5 = rehash # 3315 * g6 = hmeshift 3316 * 3317 * The first arg to GET_TTE is actually tagaccess register 3318 * not just vaddr. Since this call is for kernel we need to clear 3319 * any lower vaddr bits that would be interpreted as ctx bits. 3320 */ 3321 set TAGACC_CTX_MASK, %g1 3322 andn %o0, %g1, %o0 3323 GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5, 3324 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 3325 3326kvtop_hblk_found: 3327 /* 3328 * o0 = vaddr 3329 * o1 = sfmmup 3330 * o2 = ttep 3331 * g1 = tte 3332 * g2 = tte pa 3333 * g3 = tte va 3334 * o2 = tsbmiss area 3335 * o1 = hat id 3336 */ 3337 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 3338 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3339 stx %g1,[%o2] /* put tte into *ttep */ 3340 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 3341 /* 3342 * o0 = vaddr 3343 * o1 = sfmmup 3344 * o2 = ttep 3345 * g1 = pfn 3346 */ 3347 ba,pt %xcc, 6f 3348 mov %g1, %o0 3349 3350kvtop_nohblk: 3351 /* 3352 * we get here if we couldn't find valid hblk in hash. We rehash 3353 * if neccesary. 3354 */ 3355 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 3356#ifdef sun4v 3357 cmp %g5, MAX_HASHCNT 3358#else 3359 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 3360#endif 3361 be,a,pn %icc, 6f 3362 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3363 mov %o1, %o4 /* restore hatid */ 3364#ifdef sun4v 3365 add %g5, 2, %g5 3366 cmp %g5, 3 3367 move %icc, MMU_PAGESHIFT4M, %g6 3368 ba,pt %icc, 1b 3369 movne %icc, MMU_PAGESHIFT256M, %g6 3370#else 3371 inc %g5 3372 cmp %g5, 2 3373 move %icc, MMU_PAGESHIFT512K, %g6 3374 ba,pt %icc, 1b 3375 movne %icc, MMU_PAGESHIFT4M, %g6 3376#endif 33776: 3378 retl 3379 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3380 3381tsb_suspend: 3382 /* 3383 * o0 = vaddr 3384 * o1 = sfmmup 3385 * o2 = ttep 3386 * g1 = tte 3387 * g2 = tte pa 3388 * g3 = tte va 3389 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 3390 */ 3391 stx %g1,[%o2] /* put tte into *ttep */ 3392 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 3393 sub %g0, 1, %o0 /* output = -1 (PFN_INVALID) */ 3394 TTETOPFN(%g1, %o0, vatopfn_l3, %g2, %g3, %g4) 3395 /* 3396 * o0 = PFN return value PFN_INVALID, PFN_SUSPENDED, or pfn# 3397 * o1 = sfmmup 3398 * o2 = ttep 3399 * g1 = pfn 3400 */ 3401 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 34028: 3403 retl 3404 wrpr %g0, %o3, %pstate /* enable interrupts */ 3405 3406vatopfn_nokernel: 3407 /* 3408 * This routine does NOT support user addresses 3409 * There is a routine in C that supports this. 3410 * The only reason why we don't have the C routine 3411 * support kernel addresses as well is because 3412 * we do va_to_pa while holding the hashlock. 3413 */ 3414 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3415 save %sp, -SA(MINFRAME), %sp 3416 sethi %hi(sfmmu_panic3), %o0 3417 call panic 3418 or %o0, %lo(sfmmu_panic3), %o0 3419 3420 SET_SIZE(sfmmu_vatopfn) 3421#endif /* lint */ 3422 3423 3424 3425#if !defined(lint) 3426 3427/* 3428 * kpm lock used between trap level tsbmiss handler and kpm C level. 3429 */ 3430#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 3431 mov 0xff, tmp1 ;\ 3432label1: ;\ 3433 casa [kpmlckp]asi, %g0, tmp1 ;\ 3434 brnz,pn tmp1, label1 ;\ 3435 mov 0xff, tmp1 ;\ 3436 membar #LoadLoad 3437 3438#define KPMLOCK_EXIT(kpmlckp, asi) \ 3439 membar #LoadStore|#StoreStore ;\ 3440 sta %g0, [kpmlckp]asi 3441 3442/* 3443 * Lookup a memseg for a given pfn and if found, return the physical 3444 * address of the corresponding struct memseg in mseg, otherwise 3445 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 3446 * tsbmp, %asi is assumed to be ASI_MEM. 3447 * This lookup is done by strictly traversing only the physical memseg 3448 * linkage. The more generic approach, to check the virtual linkage 3449 * before using the physical (used e.g. with hmehash buckets), cannot 3450 * be used here. Memory DR operations can run in parallel to this 3451 * lookup w/o any locks and updates of the physical and virtual linkage 3452 * cannot be done atomically wrt. to each other. Because physical 3453 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 3454 * as "physical NULL" pointer. 3455 */ 3456#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 3457 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 3458 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 3459 udivx pfn, mseg, mseg ;\ 3460 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 3461 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 3462 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 3463 add tmp1, mseg, tmp1 ;\ 3464 ldxa [tmp1]%asi, mseg ;\ 3465 cmp mseg, MSEG_NULLPTR_PA ;\ 3466 be,pn %xcc, label/**/1 /* if not found */ ;\ 3467 nop ;\ 3468 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 3469 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 3470 blu,pn %xcc, label/**/1 ;\ 3471 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 3472 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 3473 bgeu,pn %xcc, label/**/1 ;\ 3474 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 3475 mulx tmp1, PAGE_SIZE, tmp1 ;\ 3476 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 3477 add tmp2, tmp1, tmp1 /* pp */ ;\ 3478 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 3479 cmp tmp2, pfn ;\ 3480 be,pt %xcc, label/**/_ok /* found */ ;\ 3481label/**/1: ;\ 3482 /* brute force lookup */ ;\ 3483 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 3484 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 3485label/**/2: ;\ 3486 cmp mseg, MSEG_NULLPTR_PA ;\ 3487 be,pn %xcc, label/**/_ok /* if not found */ ;\ 3488 nop ;\ 3489 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 3490 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 3491 blu,a,pt %xcc, label/**/2 ;\ 3492 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 3493 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 3494 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 3495 bgeu,a,pt %xcc, label/**/2 ;\ 3496 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 3497label/**/_ok: 3498 3499 /* 3500 * kpm tsb miss handler large pages 3501 * g1 = 8K kpm TSB entry pointer 3502 * g2 = tag access register 3503 * g3 = 4M kpm TSB entry pointer 3504 */ 3505 ALTENTRY(sfmmu_kpm_dtsb_miss) 3506 TT_TRACE(trace_tsbmiss) 3507 3508 CPU_INDEX(%g7, %g6) 3509 sethi %hi(kpmtsbm_area), %g6 3510 sllx %g7, KPMTSBM_SHIFT, %g7 3511 or %g6, %lo(kpmtsbm_area), %g6 3512 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 3513 3514 /* check enable flag */ 3515 ldub [%g6 + KPMTSBM_FLAGS], %g4 3516 and %g4, KPMTSBM_ENABLE_FLAG, %g5 3517 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 3518 nop 3519 3520 /* VA range check */ 3521 ldx [%g6 + KPMTSBM_VBASE], %g7 3522 cmp %g2, %g7 3523 blu,pn %xcc, sfmmu_tsb_miss 3524 ldx [%g6 + KPMTSBM_VEND], %g5 3525 cmp %g2, %g5 3526 bgeu,pn %xcc, sfmmu_tsb_miss 3527 stx %g3, [%g6 + KPMTSBM_TSBPTR] 3528 3529 /* 3530 * check TL tsbmiss handling flag 3531 * bump tsbmiss counter 3532 */ 3533 lduw [%g6 + KPMTSBM_TSBMISS], %g5 3534#ifdef DEBUG 3535 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 3536 inc %g5 3537 brz,pn %g3, sfmmu_kpm_exception 3538 st %g5, [%g6 + KPMTSBM_TSBMISS] 3539#else 3540 inc %g5 3541 st %g5, [%g6 + KPMTSBM_TSBMISS] 3542#endif 3543 /* 3544 * At this point: 3545 * g1 = 8K kpm TSB pointer (not used) 3546 * g2 = tag access register 3547 * g3 = clobbered 3548 * g6 = per-CPU kpm tsbmiss area 3549 * g7 = kpm_vbase 3550 */ 3551 3552 /* vaddr2pfn */ 3553 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 3554 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 3555 srax %g4, %g3, %g2 /* which alias range (r) */ 3556 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 3557 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 3558 3559 /* 3560 * Setup %asi 3561 * mseg_pa = page_numtomemseg_nolock(pfn) 3562 * if (mseg_pa == NULL) sfmmu_kpm_exception 3563 * g2=pfn 3564 */ 3565 mov ASI_MEM, %asi 3566 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 3567 cmp %g3, MSEG_NULLPTR_PA 3568 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 3569 nop 3570 3571 /* 3572 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 3573 * g2=pfn g3=mseg_pa 3574 */ 3575 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 3576 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 3577 srlx %g2, %g5, %g4 3578 sllx %g4, %g5, %g4 3579 sub %g4, %g7, %g4 3580 srlx %g4, %g5, %g4 3581 3582 /* 3583 * Validate inx value 3584 * g2=pfn g3=mseg_pa g4=inx 3585 */ 3586#ifdef DEBUG 3587 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 3588 cmp %g4, %g5 /* inx - nkpmpgs */ 3589 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 3590 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 3591#else 3592 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 3593#endif 3594 /* 3595 * kp = &mseg_pa->kpm_pages[inx] 3596 */ 3597 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 3598 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 3599 add %g5, %g4, %g5 /* kp */ 3600 3601 /* 3602 * KPMP_HASH(kp) 3603 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 3604 */ 3605 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 3606 sub %g7, 1, %g7 /* mask */ 3607 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 3608 add %g5, %g1, %g5 /* y = ksp + x */ 3609 and %g5, %g7, %g5 /* hashinx = y & mask */ 3610 3611 /* 3612 * Calculate physical kpm_page pointer 3613 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 3614 */ 3615 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 3616 add %g1, %g4, %g1 /* kp_pa */ 3617 3618 /* 3619 * Calculate physical hash lock address 3620 * g1=kp_refcntc_pa g2=pfn g5=hashinx 3621 */ 3622 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 3623 sllx %g5, KPMHLK_SHIFT, %g5 3624 add %g4, %g5, %g3 3625 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 3626 3627 /* 3628 * Assemble tte 3629 * g1=kp_pa g2=pfn g3=hlck_pa 3630 */ 3631#ifdef sun4v 3632 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 3633 sllx %g5, 32, %g5 3634 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 3635 or %g4, TTE4M, %g4 3636 or %g5, %g4, %g5 3637#else 3638 sethi %hi(TTE_VALID_INT), %g4 3639 mov TTE4M, %g5 3640 sllx %g5, TTE_SZ_SHFT_INT, %g5 3641 or %g5, %g4, %g5 /* upper part */ 3642 sllx %g5, 32, %g5 3643 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 3644 or %g5, %g4, %g5 3645#endif 3646 sllx %g2, MMU_PAGESHIFT, %g4 3647 or %g5, %g4, %g5 /* tte */ 3648 ldx [%g6 + KPMTSBM_TSBPTR], %g4 3649 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3650 3651 /* 3652 * tsb dropin 3653 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 3654 */ 3655 3656 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 3657 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 3658 3659 /* use C-handler if there's no go for dropin */ 3660 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 3661 cmp %g7, -1 3662 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 3663 nop 3664 3665#ifdef DEBUG 3666 /* double check refcnt */ 3667 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 3668 brz,pn %g7, 5f /* let C-handler deal with this */ 3669 nop 3670#endif 3671 3672#ifndef sun4v 3673 ldub [%g6 + KPMTSBM_FLAGS], %g7 3674 mov ASI_N, %g1 3675 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 3676 movnz %icc, ASI_MEM, %g1 3677 mov %g1, %asi 3678#endif 3679 3680 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 3681 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 3682 3683 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 3684 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 3685 3686 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 3687 3688 /* KPMLOCK_EXIT(kpmlckp, asi) */ 3689 KPMLOCK_EXIT(%g3, ASI_MEM) 3690 3691 /* 3692 * If trapstat is running, we need to shift the %tpc and %tnpc to 3693 * point to trapstat's TSB miss return code (note that trapstat 3694 * itself will patch the correct offset to add). 3695 * Note: TTE is expected in %g5 (allows per pagesize reporting). 3696 */ 3697 rdpr %tl, %g7 3698 cmp %g7, 1 3699 ble %icc, 0f 3700 sethi %hi(KERNELBASE), %g6 3701 rdpr %tpc, %g7 3702 or %g6, %lo(KERNELBASE), %g6 3703 cmp %g7, %g6 3704 bgeu %xcc, 0f 3705 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 3706 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 3707 wrpr %g7, %tpc 3708 add %g7, 4, %g7 3709 wrpr %g7, %tnpc 37100: 3711 retry 37125: 3713 /* g3=hlck_pa */ 3714 KPMLOCK_EXIT(%g3, ASI_MEM) 3715 ba,pt %icc, sfmmu_kpm_exception 3716 nop 3717 SET_SIZE(sfmmu_kpm_dtsb_miss) 3718 3719 /* 3720 * kpm tsbmiss handler for smallpages 3721 * g1 = 8K kpm TSB pointer 3722 * g2 = tag access register 3723 * g3 = 4M kpm TSB pointer 3724 */ 3725 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 3726 TT_TRACE(trace_tsbmiss) 3727 CPU_INDEX(%g7, %g6) 3728 sethi %hi(kpmtsbm_area), %g6 3729 sllx %g7, KPMTSBM_SHIFT, %g7 3730 or %g6, %lo(kpmtsbm_area), %g6 3731 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 3732 3733 /* check enable flag */ 3734 ldub [%g6 + KPMTSBM_FLAGS], %g4 3735 and %g4, KPMTSBM_ENABLE_FLAG, %g5 3736 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 3737 nop 3738 3739 /* 3740 * VA range check 3741 * On fail: goto sfmmu_tsb_miss 3742 */ 3743 ldx [%g6 + KPMTSBM_VBASE], %g7 3744 cmp %g2, %g7 3745 blu,pn %xcc, sfmmu_tsb_miss 3746 ldx [%g6 + KPMTSBM_VEND], %g5 3747 cmp %g2, %g5 3748 bgeu,pn %xcc, sfmmu_tsb_miss 3749 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 3750 3751 /* 3752 * check TL tsbmiss handling flag 3753 * bump tsbmiss counter 3754 */ 3755 lduw [%g6 + KPMTSBM_TSBMISS], %g5 3756#ifdef DEBUG 3757 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 3758 inc %g5 3759 brz,pn %g1, sfmmu_kpm_exception 3760 st %g5, [%g6 + KPMTSBM_TSBMISS] 3761#else 3762 inc %g5 3763 st %g5, [%g6 + KPMTSBM_TSBMISS] 3764#endif 3765 /* 3766 * At this point: 3767 * g1 = clobbered 3768 * g2 = tag access register 3769 * g3 = 4M kpm TSB pointer (not used) 3770 * g6 = per-CPU kpm tsbmiss area 3771 * g7 = kpm_vbase 3772 */ 3773 3774 /* vaddr2pfn */ 3775 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 3776 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 3777 srax %g4, %g3, %g2 /* which alias range (r) */ 3778 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 3779 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 3780 3781 /* 3782 * Setup %asi 3783 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 3784 * if (mseg not found) sfmmu_kpm_exception 3785 * g2=pfn 3786 */ 3787 mov ASI_MEM, %asi 3788 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 3789 cmp %g3, MSEG_NULLPTR_PA 3790 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 3791 nop 3792 3793 /* 3794 * inx = pfn - mseg_pa->kpm_pbase 3795 * g2=pfn g3=mseg_pa 3796 */ 3797 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 3798 sub %g2, %g7, %g4 3799 3800#ifdef DEBUG 3801 /* 3802 * Validate inx value 3803 * g2=pfn g3=mseg_pa g4=inx 3804 */ 3805 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 3806 cmp %g4, %g5 /* inx - nkpmpgs */ 3807 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 3808 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 3809#else 3810 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 3811#endif 3812 /* ksp = &mseg_pa->kpm_spages[inx] */ 3813 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 3814 add %g5, %g4, %g5 /* ksp */ 3815 3816 /* 3817 * KPMP_SHASH(kp) 3818 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz 3819 */ 3820 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 3821 sub %g7, 1, %g7 /* mask */ 3822 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 3823 add %g5, %g1, %g5 /* y = ksp + x */ 3824 and %g5, %g7, %g5 /* hashinx = y & mask */ 3825 3826 /* 3827 * Calculate physical kpm_spage pointer 3828 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 3829 */ 3830 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 3831 add %g1, %g4, %g1 /* ksp_pa */ 3832 3833 /* 3834 * Calculate physical hash lock address. 3835 * Note: Changes in kpm_shlk_t must be reflected here. 3836 * g1=ksp_pa g2=pfn g5=hashinx 3837 */ 3838 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 3839 sllx %g5, KPMSHLK_SHIFT, %g5 3840 add %g4, %g5, %g3 /* hlck_pa */ 3841 3842 /* 3843 * Assemble tte 3844 * g1=ksp_pa g2=pfn g3=hlck_pa 3845 */ 3846 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 3847 sllx %g5, 32, %g5 3848 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 3849 or %g5, %g4, %g5 3850 sllx %g2, MMU_PAGESHIFT, %g4 3851 or %g5, %g4, %g5 /* tte */ 3852 ldx [%g6 + KPMTSBM_TSBPTR], %g4 3853 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3854 3855 /* 3856 * tsb dropin 3857 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte 3858 */ 3859 3860 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 3861 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 3862 3863 /* use C-handler if there's no go for dropin */ 3864 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 3865 cmp %g7, -1 3866 bne,pn %xcc, 5f 3867 nop 3868 3869#ifndef sun4v 3870 ldub [%g6 + KPMTSBM_FLAGS], %g7 3871 mov ASI_N, %g1 3872 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 3873 movnz %icc, ASI_MEM, %g1 3874 mov %g1, %asi 3875#endif 3876 3877 /* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */ 3878 TSB_LOCK_ENTRY(%g4, %g1, %g7, 6) 3879 3880 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 3881 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 3882 3883 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 3884 3885 /* KPMLOCK_EXIT(kpmlckp, asi) */ 3886 KPMLOCK_EXIT(%g3, ASI_MEM) 3887 3888 /* 3889 * If trapstat is running, we need to shift the %tpc and %tnpc to 3890 * point to trapstat's TSB miss return code (note that trapstat 3891 * itself will patch the correct offset to add). 3892 * Note: TTE is expected in %g5 (allows per pagesize reporting). 3893 */ 3894 rdpr %tl, %g7 3895 cmp %g7, 1 3896 ble %icc, 0f 3897 sethi %hi(KERNELBASE), %g6 3898 rdpr %tpc, %g7 3899 or %g6, %lo(KERNELBASE), %g6 3900 cmp %g7, %g6 3901 bgeu %xcc, 0f 3902 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 3903 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 3904 wrpr %g7, %tpc 3905 add %g7, 4, %g7 3906 wrpr %g7, %tnpc 39070: 3908 retry 39095: 3910 /* g3=hlck_pa */ 3911 KPMLOCK_EXIT(%g3, ASI_MEM) 3912 ba,pt %icc, sfmmu_kpm_exception 3913 nop 3914 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 3915 3916#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 3917#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 3918#endif 3919 3920#endif /* lint */ 3921 3922#ifdef lint 3923/* 3924 * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 3925 * Called from C-level, sets/clears "go" indication for trap level handler. 3926 * khl_lock is a low level spin lock to protect the kp_tsbmtl field. 3927 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level. 3928 * Assumes khl_mutex is held when called from C-level. 3929 */ 3930/* ARGSUSED */ 3931void 3932sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd) 3933{ 3934} 3935 3936/* 3937 * kpm_smallpages: stores val to byte at address mapped within 3938 * low level lock brackets. The old value is returned. 3939 * Called from C-level. 3940 */ 3941/* ARGSUSED */ 3942int 3943sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val) 3944{ 3945 return (0); 3946} 3947 3948#else /* lint */ 3949 3950 .seg ".data" 3951sfmmu_kpm_tsbmtl_panic: 3952 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 3953 .byte 0 3954sfmmu_kpm_stsbmtl_panic: 3955 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 3956 .byte 0 3957 .align 4 3958 .seg ".text" 3959 3960 ENTRY_NP(sfmmu_kpm_tsbmtl) 3961 rdpr %pstate, %o3 3962 /* 3963 * %o0 = &kp_refcntc 3964 * %o1 = &khl_lock 3965 * %o2 = 0/1 (off/on) 3966 * %o3 = pstate save 3967 */ 3968#ifdef DEBUG 3969 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 3970 bnz,pt %icc, 1f /* disabled, panic */ 3971 nop 3972 save %sp, -SA(MINFRAME), %sp 3973 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 3974 call panic 3975 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 3976 ret 3977 restore 39781: 3979#endif /* DEBUG */ 3980 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 3981 3982 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 3983 mov -1, %o5 3984 brz,a %o2, 2f 3985 mov 0, %o5 39862: 3987 sth %o5, [%o0] 3988 KPMLOCK_EXIT(%o1, ASI_N) 3989 3990 retl 3991 wrpr %g0, %o3, %pstate /* enable interrupts */ 3992 SET_SIZE(sfmmu_kpm_tsbmtl) 3993 3994 ENTRY_NP(sfmmu_kpm_stsbmtl) 3995 rdpr %pstate, %o3 3996 /* 3997 * %o0 = &mapped 3998 * %o1 = &kshl_lock 3999 * %o2 = val 4000 * %o3 = pstate save 4001 */ 4002#ifdef DEBUG 4003 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4004 bnz,pt %icc, 1f /* disabled, panic */ 4005 nop 4006 save %sp, -SA(MINFRAME), %sp 4007 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4008 call panic 4009 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4010 ret 4011 restore 40121: 4013#endif /* DEBUG */ 4014 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4015 4016 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4017 ldsb [%o0], %o5 4018 stb %o2, [%o0] 4019 KPMLOCK_EXIT(%o1, ASI_N) 4020 4021 mov %o5, %o0 /* return old val */ 4022 retl 4023 wrpr %g0, %o3, %pstate /* enable interrupts */ 4024 SET_SIZE(sfmmu_kpm_stsbmtl) 4025 4026#endif /* lint */ 4027 4028#ifndef lint 4029#ifdef sun4v 4030 /* 4031 * User/kernel data miss w// multiple TSBs 4032 * The first probe covers 8K, 64K, and 512K page sizes, 4033 * because 64K and 512K mappings are replicated off 8K 4034 * pointer. Second probe covers 4M page size only. 4035 * 4036 * MMU fault area contains miss address and context. 4037 */ 4038 ALTENTRY(sfmmu_slow_dmmu_miss) 4039 GET_MMU_D_TAGACC_CTX(%g2, %g3) ! %g2 = tagacc, %g3 = ctx 4040 4041slow_miss_common: 4042 /* 4043 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 4044 * %g3 = ctx (cannot be INVALID_CONTEXT) 4045 */ 4046 brnz,pt %g3, 8f ! check for user context 4047 nop 4048 4049 /* 4050 * Kernel miss 4051 * Get 8K and 4M TSB pointers in %g1 and %g3 and 4052 * branch to sfmmu_tsb_miss_tt to handle it. 4053 */ 4054 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4055sfmmu_dslow_patch_ktsb_base: 4056 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 4057sfmmu_dslow_patch_ktsb_szcode: 4058 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 4059 4060 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 4061 ! %g1 = First TSB entry pointer, as TSB miss handler expects 4062 4063 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4064sfmmu_dslow_patch_ktsb4m_base: 4065 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 4066sfmmu_dslow_patch_ktsb4m_szcode: 4067 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 4068 4069 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 4070 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 4071 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4072 .empty 4073 40748: 4075 /* 4076 * User miss 4077 * Get first TSB pointer in %g1 4078 * Get second TSB pointer (or NULL if no second TSB) in %g3 4079 * Branch to sfmmu_tsb_miss_tt to handle it 4080 */ 4081 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 4082 /* %g1 = first TSB entry ptr now, %g2 preserved */ 4083 4084 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 4085 brlz,a,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 4086 mov %g0, %g3 4087 4088 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 4089 /* %g3 = second TSB entry ptr now, %g2 preserved */ 40909: 4091 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4092 .empty 4093 SET_SIZE(sfmmu_slow_dmmu_miss) 4094 4095 4096 /* 4097 * User/kernel instruction miss w/ multiple TSBs 4098 * The first probe covers 8K, 64K, and 512K page sizes, 4099 * because 64K and 512K mappings are replicated off 8K 4100 * pointer. Second probe covers 4M page size only. 4101 * 4102 * MMU fault area contains miss address and context. 4103 */ 4104 ALTENTRY(sfmmu_slow_immu_miss) 4105 MMU_FAULT_STATUS_AREA(%g2) 4106 ldx [%g2 + MMFSA_I_CTX], %g3 4107 ldx [%g2 + MMFSA_I_ADDR], %g2 4108 srlx %g2, MMU_PAGESHIFT, %g2 ! align address to page boundry 4109 sllx %g2, MMU_PAGESHIFT, %g2 4110 ba,pt %xcc, slow_miss_common 4111 or %g2, %g3, %g2 4112 SET_SIZE(sfmmu_slow_immu_miss) 4113 4114#endif /* sun4v */ 4115#endif /* lint */ 4116 4117#ifndef lint 4118 4119/* 4120 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 4121 */ 4122 .seg ".data" 4123 .align 64 4124 .global tsbmiss_area 4125tsbmiss_area: 4126 .skip (TSBMISS_SIZE * NCPU) 4127 4128 .align 64 4129 .global kpmtsbm_area 4130kpmtsbm_area: 4131 .skip (KPMTSBM_SIZE * NCPU) 4132#endif /* lint */ 4133