1*5d9d9091SRichard Lowe/* 2*5d9d9091SRichard Lowe * CDDL HEADER START 3*5d9d9091SRichard Lowe * 4*5d9d9091SRichard Lowe * The contents of this file are subject to the terms of the 5*5d9d9091SRichard Lowe * Common Development and Distribution License (the "License"). 6*5d9d9091SRichard Lowe * You may not use this file except in compliance with the License. 7*5d9d9091SRichard Lowe * 8*5d9d9091SRichard Lowe * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9*5d9d9091SRichard Lowe * or http://www.opensolaris.org/os/licensing. 10*5d9d9091SRichard Lowe * See the License for the specific language governing permissions 11*5d9d9091SRichard Lowe * and limitations under the License. 12*5d9d9091SRichard Lowe * 13*5d9d9091SRichard Lowe * When distributing Covered Code, include this CDDL HEADER in each 14*5d9d9091SRichard Lowe * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15*5d9d9091SRichard Lowe * If applicable, add the following below this CDDL HEADER, with the 16*5d9d9091SRichard Lowe * fields enclosed by brackets "[]" replaced with your own identifying 17*5d9d9091SRichard Lowe * information: Portions Copyright [yyyy] [name of copyright owner] 18*5d9d9091SRichard Lowe * 19*5d9d9091SRichard Lowe * CDDL HEADER END 20*5d9d9091SRichard Lowe */ 21*5d9d9091SRichard Lowe/* 22*5d9d9091SRichard Lowe * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23*5d9d9091SRichard Lowe * Use is subject to license terms. 24*5d9d9091SRichard Lowe * Copyright (c) 2016 by Delphix. All rights reserved. 25*5d9d9091SRichard Lowe */ 26*5d9d9091SRichard Lowe 27*5d9d9091SRichard Lowe/* 28*5d9d9091SRichard Lowe * SFMMU primitives. These primitives should only be used by sfmmu 29*5d9d9091SRichard Lowe * routines. 30*5d9d9091SRichard Lowe */ 31*5d9d9091SRichard Lowe 32*5d9d9091SRichard Lowe#include "assym.h" 33*5d9d9091SRichard Lowe 34*5d9d9091SRichard Lowe#include <sys/asm_linkage.h> 35*5d9d9091SRichard Lowe#include <sys/machtrap.h> 36*5d9d9091SRichard Lowe#include <sys/machasi.h> 37*5d9d9091SRichard Lowe#include <sys/sun4asi.h> 38*5d9d9091SRichard Lowe#include <sys/pte.h> 39*5d9d9091SRichard Lowe#include <sys/mmu.h> 40*5d9d9091SRichard Lowe#include <vm/hat_sfmmu.h> 41*5d9d9091SRichard Lowe#include <vm/seg_spt.h> 42*5d9d9091SRichard Lowe#include <sys/machparam.h> 43*5d9d9091SRichard Lowe#include <sys/privregs.h> 44*5d9d9091SRichard Lowe#include <sys/scb.h> 45*5d9d9091SRichard Lowe#include <sys/intreg.h> 46*5d9d9091SRichard Lowe#include <sys/machthread.h> 47*5d9d9091SRichard Lowe#include <sys/intr.h> 48*5d9d9091SRichard Lowe#include <sys/clock.h> 49*5d9d9091SRichard Lowe#include <sys/trapstat.h> 50*5d9d9091SRichard Lowe 51*5d9d9091SRichard Lowe#ifdef TRAPTRACE 52*5d9d9091SRichard Lowe#include <sys/traptrace.h> 53*5d9d9091SRichard Lowe 54*5d9d9091SRichard Lowe/* 55*5d9d9091SRichard Lowe * Tracing macro. Adds two instructions if TRAPTRACE is defined. 56*5d9d9091SRichard Lowe */ 57*5d9d9091SRichard Lowe#define TT_TRACE(label) \ 58*5d9d9091SRichard Lowe ba label ;\ 59*5d9d9091SRichard Lowe rd %pc, %g7 60*5d9d9091SRichard Lowe#else 61*5d9d9091SRichard Lowe 62*5d9d9091SRichard Lowe#define TT_TRACE(label) 63*5d9d9091SRichard Lowe 64*5d9d9091SRichard Lowe#endif /* TRAPTRACE */ 65*5d9d9091SRichard Lowe 66*5d9d9091SRichard Lowe#if (TTE_SUSPEND_SHIFT > 0) 67*5d9d9091SRichard Lowe#define TTE_SUSPEND_INT_SHIFT(reg) \ 68*5d9d9091SRichard Lowe sllx reg, TTE_SUSPEND_SHIFT, reg 69*5d9d9091SRichard Lowe#else 70*5d9d9091SRichard Lowe#define TTE_SUSPEND_INT_SHIFT(reg) 71*5d9d9091SRichard Lowe#endif 72*5d9d9091SRichard Lowe 73*5d9d9091SRichard Lowe/* 74*5d9d9091SRichard Lowe * Assumes TSBE_TAG is 0 75*5d9d9091SRichard Lowe * Assumes TSBE_INTHI is 0 76*5d9d9091SRichard Lowe * Assumes TSBREG.split is 0 77*5d9d9091SRichard Lowe */ 78*5d9d9091SRichard Lowe 79*5d9d9091SRichard Lowe#if TSBE_TAG != 0 80*5d9d9091SRichard Lowe#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 81*5d9d9091SRichard Lowe#endif 82*5d9d9091SRichard Lowe 83*5d9d9091SRichard Lowe#if TSBTAG_INTHI != 0 84*5d9d9091SRichard Lowe#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 85*5d9d9091SRichard Lowe#endif 86*5d9d9091SRichard Lowe 87*5d9d9091SRichard Lowe/* 88*5d9d9091SRichard Lowe * The following code assumes the tsb is not split. 89*5d9d9091SRichard Lowe * 90*5d9d9091SRichard Lowe * With TSBs no longer shared between processes, it's no longer 91*5d9d9091SRichard Lowe * necessary to hash the context bits into the tsb index to get 92*5d9d9091SRichard Lowe * tsb coloring; the new implementation treats the TSB as a 93*5d9d9091SRichard Lowe * direct-mapped, virtually-addressed cache. 94*5d9d9091SRichard Lowe * 95*5d9d9091SRichard Lowe * In: 96*5d9d9091SRichard Lowe * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 97*5d9d9091SRichard Lowe * tsbbase = base address of TSB (clobbered) 98*5d9d9091SRichard Lowe * tagacc = tag access register (clobbered) 99*5d9d9091SRichard Lowe * szc = size code of TSB (ro) 100*5d9d9091SRichard Lowe * tmp = scratch reg 101*5d9d9091SRichard Lowe * Out: 102*5d9d9091SRichard Lowe * tsbbase = pointer to entry in TSB 103*5d9d9091SRichard Lowe */ 104*5d9d9091SRichard Lowe#define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 105*5d9d9091SRichard Lowe mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 106*5d9d9091SRichard Lowe srlx tagacc, vpshift, tagacc ;\ 107*5d9d9091SRichard Lowe sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 108*5d9d9091SRichard Lowe sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 109*5d9d9091SRichard Lowe and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 110*5d9d9091SRichard Lowe sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 111*5d9d9091SRichard Lowe add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 112*5d9d9091SRichard Lowe 113*5d9d9091SRichard Lowe/* 114*5d9d9091SRichard Lowe * When the kpm TSB is used it is assumed that it is direct mapped 115*5d9d9091SRichard Lowe * using (vaddr>>vpshift)%tsbsz as the index. 116*5d9d9091SRichard Lowe * 117*5d9d9091SRichard Lowe * Note that, for now, the kpm TSB and kernel TSB are the same for 118*5d9d9091SRichard Lowe * each mapping size. However that need not always be the case. If 119*5d9d9091SRichard Lowe * the trap handlers are updated to search a different TSB for kpm 120*5d9d9091SRichard Lowe * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 121*5d9d9091SRichard Lowe * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 122*5d9d9091SRichard Lowe * 123*5d9d9091SRichard Lowe * In: 124*5d9d9091SRichard Lowe * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 125*5d9d9091SRichard Lowe * vaddr = virtual address (clobbered) 126*5d9d9091SRichard Lowe * tsbp, szc, tmp = scratch 127*5d9d9091SRichard Lowe * Out: 128*5d9d9091SRichard Lowe * tsbp = pointer to entry in TSB 129*5d9d9091SRichard Lowe */ 130*5d9d9091SRichard Lowe#define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 131*5d9d9091SRichard Lowe cmp vpshift, MMU_PAGESHIFT ;\ 132*5d9d9091SRichard Lowe bne,pn %icc, 1f /* branch if large case */ ;\ 133*5d9d9091SRichard Lowe sethi %hi(kpmsm_tsbsz), szc ;\ 134*5d9d9091SRichard Lowe sethi %hi(kpmsm_tsbbase), tsbp ;\ 135*5d9d9091SRichard Lowe ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 136*5d9d9091SRichard Lowe ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 137*5d9d9091SRichard Lowe ba,pt %icc, 2f ;\ 138*5d9d9091SRichard Lowe nop ;\ 139*5d9d9091SRichard Lowe1: sethi %hi(kpm_tsbsz), szc ;\ 140*5d9d9091SRichard Lowe sethi %hi(kpm_tsbbase), tsbp ;\ 141*5d9d9091SRichard Lowe ld [szc + %lo(kpm_tsbsz)], szc ;\ 142*5d9d9091SRichard Lowe ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 143*5d9d9091SRichard Lowe2: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 144*5d9d9091SRichard Lowe 145*5d9d9091SRichard Lowe/* 146*5d9d9091SRichard Lowe * Lock the TSBE at virtual address tsbep. 147*5d9d9091SRichard Lowe * 148*5d9d9091SRichard Lowe * tsbep = TSBE va (ro) 149*5d9d9091SRichard Lowe * tmp1, tmp2 = scratch registers (clobbered) 150*5d9d9091SRichard Lowe * label = label to jump to if we fail to lock the tsb entry 151*5d9d9091SRichard Lowe * %asi = ASI to use for TSB access 152*5d9d9091SRichard Lowe * 153*5d9d9091SRichard Lowe * NOTE that we flush the TSB using fast VIS instructions that 154*5d9d9091SRichard Lowe * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 155*5d9d9091SRichard Lowe * not be treated as a locked entry or we'll get stuck spinning on 156*5d9d9091SRichard Lowe * an entry that isn't locked but really invalid. 157*5d9d9091SRichard Lowe */ 158*5d9d9091SRichard Lowe 159*5d9d9091SRichard Lowe#if defined(UTSB_PHYS) 160*5d9d9091SRichard Lowe 161*5d9d9091SRichard Lowe#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 162*5d9d9091SRichard Lowe lda [tsbep]ASI_MEM, tmp1 ;\ 163*5d9d9091SRichard Lowe sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 164*5d9d9091SRichard Lowe cmp tmp1, tmp2 ;\ 165*5d9d9091SRichard Lowe be,a,pn %icc, label /* if locked ignore */ ;\ 166*5d9d9091SRichard Lowe nop ;\ 167*5d9d9091SRichard Lowe casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 168*5d9d9091SRichard Lowe cmp tmp1, tmp2 ;\ 169*5d9d9091SRichard Lowe bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 170*5d9d9091SRichard Lowe nop ;\ 171*5d9d9091SRichard Lowe /* tsbe lock acquired */ ;\ 172*5d9d9091SRichard Lowe membar #StoreStore 173*5d9d9091SRichard Lowe 174*5d9d9091SRichard Lowe#else /* UTSB_PHYS */ 175*5d9d9091SRichard Lowe 176*5d9d9091SRichard Lowe#define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 177*5d9d9091SRichard Lowe lda [tsbep]%asi, tmp1 ;\ 178*5d9d9091SRichard Lowe sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 179*5d9d9091SRichard Lowe cmp tmp1, tmp2 ;\ 180*5d9d9091SRichard Lowe be,a,pn %icc, label /* if locked ignore */ ;\ 181*5d9d9091SRichard Lowe nop ;\ 182*5d9d9091SRichard Lowe casa [tsbep]%asi, tmp1, tmp2 ;\ 183*5d9d9091SRichard Lowe cmp tmp1, tmp2 ;\ 184*5d9d9091SRichard Lowe bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 185*5d9d9091SRichard Lowe nop ;\ 186*5d9d9091SRichard Lowe /* tsbe lock acquired */ ;\ 187*5d9d9091SRichard Lowe membar #StoreStore 188*5d9d9091SRichard Lowe 189*5d9d9091SRichard Lowe#endif /* UTSB_PHYS */ 190*5d9d9091SRichard Lowe 191*5d9d9091SRichard Lowe/* 192*5d9d9091SRichard Lowe * Atomically write TSBE at virtual address tsbep. 193*5d9d9091SRichard Lowe * 194*5d9d9091SRichard Lowe * tsbep = TSBE va (ro) 195*5d9d9091SRichard Lowe * tte = TSBE TTE (ro) 196*5d9d9091SRichard Lowe * tagtarget = TSBE tag (ro) 197*5d9d9091SRichard Lowe * %asi = ASI to use for TSB access 198*5d9d9091SRichard Lowe */ 199*5d9d9091SRichard Lowe 200*5d9d9091SRichard Lowe#if defined(UTSB_PHYS) 201*5d9d9091SRichard Lowe 202*5d9d9091SRichard Lowe#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 203*5d9d9091SRichard Lowe add tsbep, TSBE_TTE, tmp1 ;\ 204*5d9d9091SRichard Lowe stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 205*5d9d9091SRichard Lowe membar #StoreStore ;\ 206*5d9d9091SRichard Lowe add tsbep, TSBE_TAG, tmp1 ;\ 207*5d9d9091SRichard Lowe stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 208*5d9d9091SRichard Lowe 209*5d9d9091SRichard Lowe#else /* UTSB_PHYS */ 210*5d9d9091SRichard Lowe 211*5d9d9091SRichard Lowe#define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 212*5d9d9091SRichard Lowe stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 213*5d9d9091SRichard Lowe membar #StoreStore ;\ 214*5d9d9091SRichard Lowe stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 215*5d9d9091SRichard Lowe 216*5d9d9091SRichard Lowe#endif /* UTSB_PHYS */ 217*5d9d9091SRichard Lowe 218*5d9d9091SRichard Lowe/* 219*5d9d9091SRichard Lowe * Load an entry into the TSB at TL > 0. 220*5d9d9091SRichard Lowe * 221*5d9d9091SRichard Lowe * tsbep = pointer to the TSBE to load as va (ro) 222*5d9d9091SRichard Lowe * tte = value of the TTE retrieved and loaded (wo) 223*5d9d9091SRichard Lowe * tagtarget = tag target register. To get TSBE tag to load, 224*5d9d9091SRichard Lowe * we need to mask off the context and leave only the va (clobbered) 225*5d9d9091SRichard Lowe * ttepa = pointer to the TTE to retrieve/load as pa (ro) 226*5d9d9091SRichard Lowe * tmp1, tmp2 = scratch registers 227*5d9d9091SRichard Lowe * label = label to jump to if we fail to lock the tsb entry 228*5d9d9091SRichard Lowe * %asi = ASI to use for TSB access 229*5d9d9091SRichard Lowe */ 230*5d9d9091SRichard Lowe 231*5d9d9091SRichard Lowe#if defined(UTSB_PHYS) 232*5d9d9091SRichard Lowe 233*5d9d9091SRichard Lowe#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 234*5d9d9091SRichard Lowe TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 235*5d9d9091SRichard Lowe /* ;\ 236*5d9d9091SRichard Lowe * I don't need to update the TSB then check for the valid tte. ;\ 237*5d9d9091SRichard Lowe * TSB invalidate will spin till the entry is unlocked. Note, ;\ 238*5d9d9091SRichard Lowe * we always invalidate the hash table before we unload the TSB.;\ 239*5d9d9091SRichard Lowe */ ;\ 240*5d9d9091SRichard Lowe sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 241*5d9d9091SRichard Lowe ldxa [ttepa]ASI_MEM, tte ;\ 242*5d9d9091SRichard Lowe srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 243*5d9d9091SRichard Lowe sethi %hi(TSBTAG_INVALID), tmp2 ;\ 244*5d9d9091SRichard Lowe add tsbep, TSBE_TAG, tmp1 ;\ 245*5d9d9091SRichard Lowe brgez,a,pn tte, label ;\ 246*5d9d9091SRichard Lowe sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 247*5d9d9091SRichard Lowe TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 248*5d9d9091SRichard Lowelabel: 249*5d9d9091SRichard Lowe 250*5d9d9091SRichard Lowe#else /* UTSB_PHYS */ 251*5d9d9091SRichard Lowe 252*5d9d9091SRichard Lowe#define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 253*5d9d9091SRichard Lowe TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 254*5d9d9091SRichard Lowe /* ;\ 255*5d9d9091SRichard Lowe * I don't need to update the TSB then check for the valid tte. ;\ 256*5d9d9091SRichard Lowe * TSB invalidate will spin till the entry is unlocked. Note, ;\ 257*5d9d9091SRichard Lowe * we always invalidate the hash table before we unload the TSB.;\ 258*5d9d9091SRichard Lowe */ ;\ 259*5d9d9091SRichard Lowe sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 260*5d9d9091SRichard Lowe ldxa [ttepa]ASI_MEM, tte ;\ 261*5d9d9091SRichard Lowe srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 262*5d9d9091SRichard Lowe sethi %hi(TSBTAG_INVALID), tmp2 ;\ 263*5d9d9091SRichard Lowe brgez,a,pn tte, label ;\ 264*5d9d9091SRichard Lowe sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 265*5d9d9091SRichard Lowe TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 266*5d9d9091SRichard Lowelabel: 267*5d9d9091SRichard Lowe 268*5d9d9091SRichard Lowe#endif /* UTSB_PHYS */ 269*5d9d9091SRichard Lowe 270*5d9d9091SRichard Lowe/* 271*5d9d9091SRichard Lowe * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 272*5d9d9091SRichard Lowe * for ITLB synthesis. 273*5d9d9091SRichard Lowe * 274*5d9d9091SRichard Lowe * tsbep = pointer to the TSBE to load as va (ro) 275*5d9d9091SRichard Lowe * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 276*5d9d9091SRichard Lowe * with exec_perm turned off and exec_synth turned on 277*5d9d9091SRichard Lowe * tagtarget = tag target register. To get TSBE tag to load, 278*5d9d9091SRichard Lowe * we need to mask off the context and leave only the va (clobbered) 279*5d9d9091SRichard Lowe * ttepa = pointer to the TTE to retrieve/load as pa (ro) 280*5d9d9091SRichard Lowe * tmp1, tmp2 = scratch registers 281*5d9d9091SRichard Lowe * label = label to use for branch (text) 282*5d9d9091SRichard Lowe * %asi = ASI to use for TSB access 283*5d9d9091SRichard Lowe */ 284*5d9d9091SRichard Lowe 285*5d9d9091SRichard Lowe#define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 286*5d9d9091SRichard Lowe TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 287*5d9d9091SRichard Lowe /* ;\ 288*5d9d9091SRichard Lowe * I don't need to update the TSB then check for the valid tte. ;\ 289*5d9d9091SRichard Lowe * TSB invalidate will spin till the entry is unlocked. Note, ;\ 290*5d9d9091SRichard Lowe * we always invalidate the hash table before we unload the TSB.;\ 291*5d9d9091SRichard Lowe * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 292*5d9d9091SRichard Lowe * and exec_synth bit to 1. ;\ 293*5d9d9091SRichard Lowe */ ;\ 294*5d9d9091SRichard Lowe sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 295*5d9d9091SRichard Lowe mov tte, tmp1 ;\ 296*5d9d9091SRichard Lowe ldxa [ttepa]ASI_MEM, tte ;\ 297*5d9d9091SRichard Lowe srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 298*5d9d9091SRichard Lowe sethi %hi(TSBTAG_INVALID), tmp2 ;\ 299*5d9d9091SRichard Lowe brgez,a,pn tte, label ;\ 300*5d9d9091SRichard Lowe sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 301*5d9d9091SRichard Lowe or tte, tmp1, tte ;\ 302*5d9d9091SRichard Lowe andn tte, TTE_EXECPRM_INT, tte ;\ 303*5d9d9091SRichard Lowe or tte, TTE_E_SYNTH_INT, tte ;\ 304*5d9d9091SRichard Lowe TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 305*5d9d9091SRichard Lowelabel: 306*5d9d9091SRichard Lowe 307*5d9d9091SRichard Lowe/* 308*5d9d9091SRichard Lowe * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 309*5d9d9091SRichard Lowe * 310*5d9d9091SRichard Lowe * tte = value of the TTE, used to get tte_size bits (ro) 311*5d9d9091SRichard Lowe * tagaccess = tag access register, used to get 4M pfn bits (ro) 312*5d9d9091SRichard Lowe * pfn = 4M pfn bits shifted to offset for tte (out) 313*5d9d9091SRichard Lowe * tmp1 = scratch register 314*5d9d9091SRichard Lowe * label = label to use for branch (text) 315*5d9d9091SRichard Lowe */ 316*5d9d9091SRichard Lowe 317*5d9d9091SRichard Lowe#define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 318*5d9d9091SRichard Lowe /* ;\ 319*5d9d9091SRichard Lowe * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 320*5d9d9091SRichard Lowe * Return them, shifted, in pfn. ;\ 321*5d9d9091SRichard Lowe */ ;\ 322*5d9d9091SRichard Lowe srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 323*5d9d9091SRichard Lowe srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 324*5d9d9091SRichard Lowe andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 325*5d9d9091SRichard Lowe bz,a,pt %icc, label##f /* if 0, is */ ;\ 326*5d9d9091SRichard Lowe and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 327*5d9d9091SRichard Lowe and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 328*5d9d9091SRichard Lowelabel: ;\ 329*5d9d9091SRichard Lowe sllx tagaccess, MMU_PAGESHIFT4M, pfn 330*5d9d9091SRichard Lowe 331*5d9d9091SRichard Lowe/* 332*5d9d9091SRichard Lowe * Add 4M TTE size code to a tte for a Panther 32M/256M page, 333*5d9d9091SRichard Lowe * for ITLB synthesis. 334*5d9d9091SRichard Lowe * 335*5d9d9091SRichard Lowe * tte = value of the TTE, used to get tte_size bits (rw) 336*5d9d9091SRichard Lowe * tmp1 = scratch register 337*5d9d9091SRichard Lowe */ 338*5d9d9091SRichard Lowe 339*5d9d9091SRichard Lowe#define SET_TTE4M_PN(tte, tmp) \ 340*5d9d9091SRichard Lowe /* ;\ 341*5d9d9091SRichard Lowe * Set 4M pagesize tte bits. ;\ 342*5d9d9091SRichard Lowe */ ;\ 343*5d9d9091SRichard Lowe set TTE4M, tmp ;\ 344*5d9d9091SRichard Lowe sllx tmp, TTE_SZ_SHFT, tmp ;\ 345*5d9d9091SRichard Lowe or tte, tmp, tte 346*5d9d9091SRichard Lowe 347*5d9d9091SRichard Lowe/* 348*5d9d9091SRichard Lowe * Load an entry into the TSB at TL=0. 349*5d9d9091SRichard Lowe * 350*5d9d9091SRichard Lowe * tsbep = pointer to the TSBE to load as va (ro) 351*5d9d9091SRichard Lowe * tteva = pointer to the TTE to load as va (ro) 352*5d9d9091SRichard Lowe * tagtarget = TSBE tag to load (which contains no context), synthesized 353*5d9d9091SRichard Lowe * to match va of MMU tag target register only (ro) 354*5d9d9091SRichard Lowe * tmp1, tmp2 = scratch registers (clobbered) 355*5d9d9091SRichard Lowe * label = label to use for branches (text) 356*5d9d9091SRichard Lowe * %asi = ASI to use for TSB access 357*5d9d9091SRichard Lowe */ 358*5d9d9091SRichard Lowe 359*5d9d9091SRichard Lowe#if defined(UTSB_PHYS) 360*5d9d9091SRichard Lowe 361*5d9d9091SRichard Lowe#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 362*5d9d9091SRichard Lowe /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 363*5d9d9091SRichard Lowe ldx [tteva], tteva /* load tte */ ;\ 364*5d9d9091SRichard Lowe TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 365*5d9d9091SRichard Lowe sethi %hi(TSBTAG_INVALID), tmp2 ;\ 366*5d9d9091SRichard Lowe add tsbep, TSBE_TAG, tmp1 ;\ 367*5d9d9091SRichard Lowe brgez,a,pn tteva, label ;\ 368*5d9d9091SRichard Lowe sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 369*5d9d9091SRichard Lowe TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 370*5d9d9091SRichard Lowelabel: 371*5d9d9091SRichard Lowe 372*5d9d9091SRichard Lowe#else /* UTSB_PHYS */ 373*5d9d9091SRichard Lowe 374*5d9d9091SRichard Lowe#define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 375*5d9d9091SRichard Lowe /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 376*5d9d9091SRichard Lowe ldx [tteva], tteva /* load tte */ ;\ 377*5d9d9091SRichard Lowe TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 378*5d9d9091SRichard Lowe sethi %hi(TSBTAG_INVALID), tmp2 ;\ 379*5d9d9091SRichard Lowe brgez,a,pn tteva, label ;\ 380*5d9d9091SRichard Lowe sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 381*5d9d9091SRichard Lowe TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 382*5d9d9091SRichard Lowelabel: 383*5d9d9091SRichard Lowe 384*5d9d9091SRichard Lowe#endif /* UTSB_PHYS */ 385*5d9d9091SRichard Lowe 386*5d9d9091SRichard Lowe/* 387*5d9d9091SRichard Lowe * Invalidate a TSB entry in the TSB. 388*5d9d9091SRichard Lowe * 389*5d9d9091SRichard Lowe * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 390*5d9d9091SRichard Lowe * about this earlier to ensure this is true. Thus when we are 391*5d9d9091SRichard Lowe * directly referencing tsbep below, we are referencing the tte_tag 392*5d9d9091SRichard Lowe * field of the TSBE. If this offset ever changes, the code below 393*5d9d9091SRichard Lowe * will need to be modified. 394*5d9d9091SRichard Lowe * 395*5d9d9091SRichard Lowe * tsbep = pointer to TSBE as va (ro) 396*5d9d9091SRichard Lowe * tag = invalidation is done if this matches the TSBE tag (ro) 397*5d9d9091SRichard Lowe * tmp1 - tmp3 = scratch registers (clobbered) 398*5d9d9091SRichard Lowe * label = label name to use for branches (text) 399*5d9d9091SRichard Lowe * %asi = ASI to use for TSB access 400*5d9d9091SRichard Lowe */ 401*5d9d9091SRichard Lowe 402*5d9d9091SRichard Lowe#if defined(UTSB_PHYS) 403*5d9d9091SRichard Lowe 404*5d9d9091SRichard Lowe#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 405*5d9d9091SRichard Lowe lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 406*5d9d9091SRichard Lowe sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 407*5d9d9091SRichard Lowelabel##1: ;\ 408*5d9d9091SRichard Lowe cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 409*5d9d9091SRichard Lowe be,a,pn %icc, label##1 /* so, loop until unlocked */ ;\ 410*5d9d9091SRichard Lowe lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 411*5d9d9091SRichard Lowe ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 412*5d9d9091SRichard Lowe cmp tag, tmp3 /* compare tags */ ;\ 413*5d9d9091SRichard Lowe bne,pt %xcc, label##2 /* if different, do nothing */ ;\ 414*5d9d9091SRichard Lowe sethi %hi(TSBTAG_INVALID), tmp3 ;\ 415*5d9d9091SRichard Lowe casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 416*5d9d9091SRichard Lowe cmp tmp1, tmp3 /* if not successful */ ;\ 417*5d9d9091SRichard Lowe bne,a,pn %icc, label##1 /* start over from the top */ ;\ 418*5d9d9091SRichard Lowe lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 419*5d9d9091SRichard Lowelabel##2: 420*5d9d9091SRichard Lowe 421*5d9d9091SRichard Lowe#else /* UTSB_PHYS */ 422*5d9d9091SRichard Lowe 423*5d9d9091SRichard Lowe#define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 424*5d9d9091SRichard Lowe lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 425*5d9d9091SRichard Lowe sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 426*5d9d9091SRichard Lowelabel##1: ;\ 427*5d9d9091SRichard Lowe cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 428*5d9d9091SRichard Lowe be,a,pn %icc, label##1 /* so, loop until unlocked */ ;\ 429*5d9d9091SRichard Lowe lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 430*5d9d9091SRichard Lowe ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 431*5d9d9091SRichard Lowe cmp tag, tmp3 /* compare tags */ ;\ 432*5d9d9091SRichard Lowe bne,pt %xcc, label##2 /* if different, do nothing */ ;\ 433*5d9d9091SRichard Lowe sethi %hi(TSBTAG_INVALID), tmp3 ;\ 434*5d9d9091SRichard Lowe casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 435*5d9d9091SRichard Lowe cmp tmp1, tmp3 /* if not successful */ ;\ 436*5d9d9091SRichard Lowe bne,a,pn %icc, label##1 /* start over from the top */ ;\ 437*5d9d9091SRichard Lowe lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 438*5d9d9091SRichard Lowelabel##2: 439*5d9d9091SRichard Lowe 440*5d9d9091SRichard Lowe#endif /* UTSB_PHYS */ 441*5d9d9091SRichard Lowe 442*5d9d9091SRichard Lowe#if TSB_SOFTSZ_MASK < TSB_SZ_MASK 443*5d9d9091SRichard Lowe#error - TSB_SOFTSZ_MASK too small 444*5d9d9091SRichard Lowe#endif 445*5d9d9091SRichard Lowe 446*5d9d9091SRichard Lowe 447*5d9d9091SRichard Lowe/* 448*5d9d9091SRichard Lowe * An implementation of setx which will be hot patched at run time. 449*5d9d9091SRichard Lowe * since it is being hot patched, there is no value passed in. 450*5d9d9091SRichard Lowe * Thus, essentially we are implementing 451*5d9d9091SRichard Lowe * setx value, tmp, dest 452*5d9d9091SRichard Lowe * where value is RUNTIME_PATCH (aka 0) in this case. 453*5d9d9091SRichard Lowe */ 454*5d9d9091SRichard Lowe#define RUNTIME_PATCH_SETX(dest, tmp) \ 455*5d9d9091SRichard Lowe sethi %hh(RUNTIME_PATCH), tmp ;\ 456*5d9d9091SRichard Lowe sethi %lm(RUNTIME_PATCH), dest ;\ 457*5d9d9091SRichard Lowe or tmp, %hm(RUNTIME_PATCH), tmp ;\ 458*5d9d9091SRichard Lowe or dest, %lo(RUNTIME_PATCH), dest ;\ 459*5d9d9091SRichard Lowe sllx tmp, 32, tmp ;\ 460*5d9d9091SRichard Lowe nop /* for perf reasons */ ;\ 461*5d9d9091SRichard Lowe or tmp, dest, dest /* contents of patched value */ 462*5d9d9091SRichard Lowe 463*5d9d9091SRichard Lowe 464*5d9d9091SRichard Lowe .seg ".data" 465*5d9d9091SRichard Lowe .global sfmmu_panic1 466*5d9d9091SRichard Lowesfmmu_panic1: 467*5d9d9091SRichard Lowe .asciz "sfmmu_asm: interrupts already disabled" 468*5d9d9091SRichard Lowe 469*5d9d9091SRichard Lowe .global sfmmu_panic3 470*5d9d9091SRichard Lowesfmmu_panic3: 471*5d9d9091SRichard Lowe .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 472*5d9d9091SRichard Lowe 473*5d9d9091SRichard Lowe .global sfmmu_panic4 474*5d9d9091SRichard Lowesfmmu_panic4: 475*5d9d9091SRichard Lowe .asciz "sfmmu_asm: 4M tsb pointer mis-match" 476*5d9d9091SRichard Lowe 477*5d9d9091SRichard Lowe .global sfmmu_panic5 478*5d9d9091SRichard Lowesfmmu_panic5: 479*5d9d9091SRichard Lowe .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 480*5d9d9091SRichard Lowe 481*5d9d9091SRichard Lowe .global sfmmu_panic6 482*5d9d9091SRichard Lowesfmmu_panic6: 483*5d9d9091SRichard Lowe .asciz "sfmmu_asm: interrupts not disabled" 484*5d9d9091SRichard Lowe 485*5d9d9091SRichard Lowe .global sfmmu_panic7 486*5d9d9091SRichard Lowesfmmu_panic7: 487*5d9d9091SRichard Lowe .asciz "sfmmu_asm: kernel as" 488*5d9d9091SRichard Lowe 489*5d9d9091SRichard Lowe .global sfmmu_panic8 490*5d9d9091SRichard Lowesfmmu_panic8: 491*5d9d9091SRichard Lowe .asciz "sfmmu_asm: gnum is zero" 492*5d9d9091SRichard Lowe 493*5d9d9091SRichard Lowe .global sfmmu_panic9 494*5d9d9091SRichard Lowesfmmu_panic9: 495*5d9d9091SRichard Lowe .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 496*5d9d9091SRichard Lowe 497*5d9d9091SRichard Lowe .global sfmmu_panic10 498*5d9d9091SRichard Lowesfmmu_panic10: 499*5d9d9091SRichard Lowe .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 500*5d9d9091SRichard Lowe 501*5d9d9091SRichard Lowe .global sfmmu_panic11 502*5d9d9091SRichard Lowesfmmu_panic11: 503*5d9d9091SRichard Lowe .asciz "sfmmu_asm: ktsb_phys must not be 0 on a sun4v platform" 504*5d9d9091SRichard Lowe 505*5d9d9091SRichard Lowe ENTRY(sfmmu_disable_intrs) 506*5d9d9091SRichard Lowe rdpr %pstate, %o0 507*5d9d9091SRichard Lowe#ifdef DEBUG 508*5d9d9091SRichard Lowe PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 509*5d9d9091SRichard Lowe#endif /* DEBUG */ 510*5d9d9091SRichard Lowe retl 511*5d9d9091SRichard Lowe wrpr %o0, PSTATE_IE, %pstate 512*5d9d9091SRichard Lowe SET_SIZE(sfmmu_disable_intrs) 513*5d9d9091SRichard Lowe 514*5d9d9091SRichard Lowe ENTRY(sfmmu_enable_intrs) 515*5d9d9091SRichard Lowe retl 516*5d9d9091SRichard Lowe wrpr %g0, %o0, %pstate 517*5d9d9091SRichard Lowe SET_SIZE(sfmmu_enable_intrs) 518*5d9d9091SRichard Lowe 519*5d9d9091SRichard Lowe/* 520*5d9d9091SRichard Lowe * This routine is called both by resume() and sfmmu_get_ctx() to 521*5d9d9091SRichard Lowe * allocate a new context for the process on a MMU. 522*5d9d9091SRichard Lowe * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 523*5d9d9091SRichard Lowe * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 524*5d9d9091SRichard Lowe * is the case when sfmmu_alloc_ctx is called from resume(). 525*5d9d9091SRichard Lowe * 526*5d9d9091SRichard Lowe * The caller must disable interrupts before entering this routine. 527*5d9d9091SRichard Lowe * To reduce ctx switch overhead, the code contains both 'fast path' and 528*5d9d9091SRichard Lowe * 'slow path' code. The fast path code covers the common case where only 529*5d9d9091SRichard Lowe * a quick check is needed and the real ctx allocation is not required. 530*5d9d9091SRichard Lowe * It can be done without holding the per-process (PP) lock. 531*5d9d9091SRichard Lowe * The 'slow path' code must be protected by the PP Lock and performs ctx 532*5d9d9091SRichard Lowe * allocation. 533*5d9d9091SRichard Lowe * Hardware context register and HAT mmu cnum are updated accordingly. 534*5d9d9091SRichard Lowe * 535*5d9d9091SRichard Lowe * %o0 - sfmmup 536*5d9d9091SRichard Lowe * %o1 - allocflag 537*5d9d9091SRichard Lowe * %o2 - CPU 538*5d9d9091SRichard Lowe * %o3 - sfmmu private/shared flag 539*5d9d9091SRichard Lowe * 540*5d9d9091SRichard Lowe * ret - 0: no ctx is allocated 541*5d9d9091SRichard Lowe * 1: a ctx is allocated 542*5d9d9091SRichard Lowe */ 543*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_alloc_ctx) 544*5d9d9091SRichard Lowe 545*5d9d9091SRichard Lowe#ifdef DEBUG 546*5d9d9091SRichard Lowe sethi %hi(ksfmmup), %g1 547*5d9d9091SRichard Lowe ldx [%g1 + %lo(ksfmmup)], %g1 548*5d9d9091SRichard Lowe cmp %g1, %o0 549*5d9d9091SRichard Lowe bne,pt %xcc, 0f 550*5d9d9091SRichard Lowe nop 551*5d9d9091SRichard Lowe 552*5d9d9091SRichard Lowe sethi %hi(panicstr), %g1 ! if kernel as, panic 553*5d9d9091SRichard Lowe ldx [%g1 + %lo(panicstr)], %g1 554*5d9d9091SRichard Lowe tst %g1 555*5d9d9091SRichard Lowe bnz,pn %icc, 7f 556*5d9d9091SRichard Lowe nop 557*5d9d9091SRichard Lowe 558*5d9d9091SRichard Lowe sethi %hi(sfmmu_panic7), %o0 559*5d9d9091SRichard Lowe call panic 560*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_panic7), %o0 561*5d9d9091SRichard Lowe 562*5d9d9091SRichard Lowe7: 563*5d9d9091SRichard Lowe retl 564*5d9d9091SRichard Lowe mov %g0, %o0 ! %o0 = ret = 0 565*5d9d9091SRichard Lowe 566*5d9d9091SRichard Lowe0: 567*5d9d9091SRichard Lowe PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 568*5d9d9091SRichard Lowe#endif /* DEBUG */ 569*5d9d9091SRichard Lowe 570*5d9d9091SRichard Lowe mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 571*5d9d9091SRichard Lowe 572*5d9d9091SRichard Lowe ! load global mmu_ctxp info 573*5d9d9091SRichard Lowe ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 574*5d9d9091SRichard Lowe 575*5d9d9091SRichard Lowe#ifdef sun4v 576*5d9d9091SRichard Lowe /* During suspend on sun4v, context domains can be temporary removed */ 577*5d9d9091SRichard Lowe brz,a,pn %o3, 0f 578*5d9d9091SRichard Lowe nop 579*5d9d9091SRichard Lowe#endif 580*5d9d9091SRichard Lowe 581*5d9d9091SRichard Lowe lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 582*5d9d9091SRichard Lowe 583*5d9d9091SRichard Lowe ! load global mmu_ctxp gnum 584*5d9d9091SRichard Lowe ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 585*5d9d9091SRichard Lowe 586*5d9d9091SRichard Lowe#ifdef DEBUG 587*5d9d9091SRichard Lowe cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 588*5d9d9091SRichard Lowe bne,pt %xcc, 3f 589*5d9d9091SRichard Lowe nop 590*5d9d9091SRichard Lowe 591*5d9d9091SRichard Lowe sethi %hi(panicstr), %g1 ! test if panicstr is already set 592*5d9d9091SRichard Lowe ldx [%g1 + %lo(panicstr)], %g1 593*5d9d9091SRichard Lowe tst %g1 594*5d9d9091SRichard Lowe bnz,pn %icc, 1f 595*5d9d9091SRichard Lowe nop 596*5d9d9091SRichard Lowe 597*5d9d9091SRichard Lowe sethi %hi(sfmmu_panic8), %o0 598*5d9d9091SRichard Lowe call panic 599*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_panic8), %o0 600*5d9d9091SRichard Lowe1: 601*5d9d9091SRichard Lowe retl 602*5d9d9091SRichard Lowe mov %g0, %o0 ! %o0 = ret = 0 603*5d9d9091SRichard Lowe3: 604*5d9d9091SRichard Lowe#endif 605*5d9d9091SRichard Lowe 606*5d9d9091SRichard Lowe ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 607*5d9d9091SRichard Lowe 608*5d9d9091SRichard Lowe sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 609*5d9d9091SRichard Lowe add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 610*5d9d9091SRichard Lowe 611*5d9d9091SRichard Lowe /* 612*5d9d9091SRichard Lowe * %g5 = sfmmu gnum returned 613*5d9d9091SRichard Lowe * %g6 = sfmmu cnum returned 614*5d9d9091SRichard Lowe * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 615*5d9d9091SRichard Lowe * %g4 = scratch 616*5d9d9091SRichard Lowe * 617*5d9d9091SRichard Lowe * Fast path code, do a quick check. 618*5d9d9091SRichard Lowe */ 619*5d9d9091SRichard Lowe SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 620*5d9d9091SRichard Lowe 621*5d9d9091SRichard Lowe cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 622*5d9d9091SRichard Lowe bne,pt %icc, 1f ! valid hat cnum, check gnum 623*5d9d9091SRichard Lowe nop 624*5d9d9091SRichard Lowe 625*5d9d9091SRichard Lowe ! cnum == INVALID, check allocflag 626*5d9d9091SRichard Lowe mov %g0, %g4 ! %g4 = ret = 0 627*5d9d9091SRichard Lowe brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 628*5d9d9091SRichard Lowe mov %g6, %o1 629*5d9d9091SRichard Lowe 630*5d9d9091SRichard Lowe ! (invalid HAT cnum) && (allocflag == 1) 631*5d9d9091SRichard Lowe ba,pt %icc, 2f 632*5d9d9091SRichard Lowe nop 633*5d9d9091SRichard Lowe#ifdef sun4v 634*5d9d9091SRichard Lowe0: 635*5d9d9091SRichard Lowe set INVALID_CONTEXT, %o1 636*5d9d9091SRichard Lowe membar #LoadStore|#StoreStore 637*5d9d9091SRichard Lowe ba,pt %icc, 8f 638*5d9d9091SRichard Lowe mov %g0, %g4 ! %g4 = ret = 0 639*5d9d9091SRichard Lowe#endif 640*5d9d9091SRichard Lowe1: 641*5d9d9091SRichard Lowe ! valid HAT cnum, check gnum 642*5d9d9091SRichard Lowe cmp %g5, %o4 643*5d9d9091SRichard Lowe mov 1, %g4 !%g4 = ret = 1 644*5d9d9091SRichard Lowe be,a,pt %icc, 8f ! gnum unchanged, go to done 645*5d9d9091SRichard Lowe mov %g6, %o1 646*5d9d9091SRichard Lowe 647*5d9d9091SRichard Lowe2: 648*5d9d9091SRichard Lowe /* 649*5d9d9091SRichard Lowe * Grab per process (PP) sfmmu_ctx_lock spinlock, 650*5d9d9091SRichard Lowe * followed by the 'slow path' code. 651*5d9d9091SRichard Lowe */ 652*5d9d9091SRichard Lowe ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 653*5d9d9091SRichard Lowe3: 654*5d9d9091SRichard Lowe brz %g3, 5f 655*5d9d9091SRichard Lowe nop 656*5d9d9091SRichard Lowe4: 657*5d9d9091SRichard Lowe brnz,a,pt %g3, 4b ! spin if lock is 1 658*5d9d9091SRichard Lowe ldub [%o0 + SFMMU_CTX_LOCK], %g3 659*5d9d9091SRichard Lowe ba %xcc, 3b ! retry the lock 660*5d9d9091SRichard Lowe ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 661*5d9d9091SRichard Lowe 662*5d9d9091SRichard Lowe5: 663*5d9d9091SRichard Lowe membar #LoadLoad 664*5d9d9091SRichard Lowe /* 665*5d9d9091SRichard Lowe * %g5 = sfmmu gnum returned 666*5d9d9091SRichard Lowe * %g6 = sfmmu cnum returned 667*5d9d9091SRichard Lowe * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 668*5d9d9091SRichard Lowe * %g4 = scratch 669*5d9d9091SRichard Lowe */ 670*5d9d9091SRichard Lowe SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 671*5d9d9091SRichard Lowe 672*5d9d9091SRichard Lowe cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 673*5d9d9091SRichard Lowe bne,pt %icc, 1f ! valid hat cnum, check gnum 674*5d9d9091SRichard Lowe nop 675*5d9d9091SRichard Lowe 676*5d9d9091SRichard Lowe ! cnum == INVALID, check allocflag 677*5d9d9091SRichard Lowe mov %g0, %g4 ! %g4 = ret = 0 678*5d9d9091SRichard Lowe brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 679*5d9d9091SRichard Lowe mov %g6, %o1 680*5d9d9091SRichard Lowe 681*5d9d9091SRichard Lowe ! (invalid HAT cnum) && (allocflag == 1) 682*5d9d9091SRichard Lowe ba,pt %icc, 6f 683*5d9d9091SRichard Lowe nop 684*5d9d9091SRichard Lowe1: 685*5d9d9091SRichard Lowe ! valid HAT cnum, check gnum 686*5d9d9091SRichard Lowe cmp %g5, %o4 687*5d9d9091SRichard Lowe mov 1, %g4 ! %g4 = ret = 1 688*5d9d9091SRichard Lowe be,a,pt %icc, 2f ! gnum unchanged, go to done 689*5d9d9091SRichard Lowe mov %g6, %o1 690*5d9d9091SRichard Lowe 691*5d9d9091SRichard Lowe ba,pt %icc, 6f 692*5d9d9091SRichard Lowe nop 693*5d9d9091SRichard Lowe2: 694*5d9d9091SRichard Lowe membar #LoadStore|#StoreStore 695*5d9d9091SRichard Lowe ba,pt %icc, 8f 696*5d9d9091SRichard Lowe clrb [%o0 + SFMMU_CTX_LOCK] 697*5d9d9091SRichard Lowe6: 698*5d9d9091SRichard Lowe /* 699*5d9d9091SRichard Lowe * We get here if we do not have a valid context, or 700*5d9d9091SRichard Lowe * the HAT gnum does not match global gnum. We hold 701*5d9d9091SRichard Lowe * sfmmu_ctx_lock spinlock. Allocate that context. 702*5d9d9091SRichard Lowe * 703*5d9d9091SRichard Lowe * %o3 = mmu_ctxp 704*5d9d9091SRichard Lowe */ 705*5d9d9091SRichard Lowe add %o3, MMU_CTX_CNUM, %g3 706*5d9d9091SRichard Lowe ld [%o3 + MMU_CTX_NCTXS], %g4 707*5d9d9091SRichard Lowe 708*5d9d9091SRichard Lowe /* 709*5d9d9091SRichard Lowe * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 710*5d9d9091SRichard Lowe * %g3 = mmu cnum address 711*5d9d9091SRichard Lowe * %g4 = mmu nctxs 712*5d9d9091SRichard Lowe * 713*5d9d9091SRichard Lowe * %o0 = sfmmup 714*5d9d9091SRichard Lowe * %o1 = mmu current cnum value (used as new cnum) 715*5d9d9091SRichard Lowe * %o4 = mmu gnum 716*5d9d9091SRichard Lowe * 717*5d9d9091SRichard Lowe * %o5 = scratch 718*5d9d9091SRichard Lowe */ 719*5d9d9091SRichard Lowe ld [%g3], %o1 720*5d9d9091SRichard Lowe0: 721*5d9d9091SRichard Lowe cmp %o1, %g4 722*5d9d9091SRichard Lowe bl,a,pt %icc, 1f 723*5d9d9091SRichard Lowe add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 724*5d9d9091SRichard Lowe 725*5d9d9091SRichard Lowe /* 726*5d9d9091SRichard Lowe * cnum reachs max, bail, so wrap around can be performed later. 727*5d9d9091SRichard Lowe */ 728*5d9d9091SRichard Lowe set INVALID_CONTEXT, %o1 729*5d9d9091SRichard Lowe mov %g0, %g4 ! %g4 = ret = 0 730*5d9d9091SRichard Lowe 731*5d9d9091SRichard Lowe membar #LoadStore|#StoreStore 732*5d9d9091SRichard Lowe ba,pt %icc, 8f 733*5d9d9091SRichard Lowe clrb [%o0 + SFMMU_CTX_LOCK] 734*5d9d9091SRichard Lowe1: 735*5d9d9091SRichard Lowe ! %g3 = addr of mmu_ctxp->cnum 736*5d9d9091SRichard Lowe ! %o5 = mmu_ctxp->cnum + 1 737*5d9d9091SRichard Lowe cas [%g3], %o1, %o5 738*5d9d9091SRichard Lowe cmp %o1, %o5 739*5d9d9091SRichard Lowe bne,a,pn %xcc, 0b ! cas failed 740*5d9d9091SRichard Lowe ld [%g3], %o1 741*5d9d9091SRichard Lowe 742*5d9d9091SRichard Lowe#ifdef DEBUG 743*5d9d9091SRichard Lowe set MAX_SFMMU_CTX_VAL, %o5 744*5d9d9091SRichard Lowe cmp %o1, %o5 745*5d9d9091SRichard Lowe ble,pt %icc, 2f 746*5d9d9091SRichard Lowe nop 747*5d9d9091SRichard Lowe 748*5d9d9091SRichard Lowe sethi %hi(sfmmu_panic9), %o0 749*5d9d9091SRichard Lowe call panic 750*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_panic9), %o0 751*5d9d9091SRichard Lowe2: 752*5d9d9091SRichard Lowe#endif 753*5d9d9091SRichard Lowe ! update hat gnum and cnum 754*5d9d9091SRichard Lowe sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 755*5d9d9091SRichard Lowe or %o4, %o1, %o4 756*5d9d9091SRichard Lowe stx %o4, [%g2 + SFMMU_CTXS] 757*5d9d9091SRichard Lowe 758*5d9d9091SRichard Lowe membar #LoadStore|#StoreStore 759*5d9d9091SRichard Lowe clrb [%o0 + SFMMU_CTX_LOCK] 760*5d9d9091SRichard Lowe 761*5d9d9091SRichard Lowe mov 1, %g4 ! %g4 = ret = 1 762*5d9d9091SRichard Lowe8: 763*5d9d9091SRichard Lowe /* 764*5d9d9091SRichard Lowe * program the secondary context register 765*5d9d9091SRichard Lowe * 766*5d9d9091SRichard Lowe * %o1 = cnum 767*5d9d9091SRichard Lowe * %g1 = sfmmu private/shared flag (0:private, 1:shared) 768*5d9d9091SRichard Lowe */ 769*5d9d9091SRichard Lowe 770*5d9d9091SRichard Lowe /* 771*5d9d9091SRichard Lowe * When we come here and context is invalid, we want to set both 772*5d9d9091SRichard Lowe * private and shared ctx regs to INVALID. In order to 773*5d9d9091SRichard Lowe * do so, we set the sfmmu priv/shared flag to 'private' regardless 774*5d9d9091SRichard Lowe * so that private ctx reg will be set to invalid. 775*5d9d9091SRichard Lowe * Note that on sun4v values written to private context register are 776*5d9d9091SRichard Lowe * automatically written to corresponding shared context register as 777*5d9d9091SRichard Lowe * well. On sun4u SET_SECCTX() will invalidate shared context register 778*5d9d9091SRichard Lowe * when it sets a private secondary context register. 779*5d9d9091SRichard Lowe */ 780*5d9d9091SRichard Lowe 781*5d9d9091SRichard Lowe cmp %o1, INVALID_CONTEXT 782*5d9d9091SRichard Lowe be,a,pn %icc, 9f 783*5d9d9091SRichard Lowe clr %g1 784*5d9d9091SRichard Lowe9: 785*5d9d9091SRichard Lowe 786*5d9d9091SRichard Lowe#ifdef sun4u 787*5d9d9091SRichard Lowe ldub [%o0 + SFMMU_CEXT], %o2 788*5d9d9091SRichard Lowe sll %o2, CTXREG_EXT_SHIFT, %o2 789*5d9d9091SRichard Lowe or %o1, %o2, %o1 790*5d9d9091SRichard Lowe#endif /* sun4u */ 791*5d9d9091SRichard Lowe 792*5d9d9091SRichard Lowe SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1) 793*5d9d9091SRichard Lowe 794*5d9d9091SRichard Lowe retl 795*5d9d9091SRichard Lowe mov %g4, %o0 ! %o0 = ret 796*5d9d9091SRichard Lowe 797*5d9d9091SRichard Lowe SET_SIZE(sfmmu_alloc_ctx) 798*5d9d9091SRichard Lowe 799*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_modifytte) 800*5d9d9091SRichard Lowe ldx [%o2], %g3 /* current */ 801*5d9d9091SRichard Lowe ldx [%o0], %g1 /* original */ 802*5d9d9091SRichard Lowe2: 803*5d9d9091SRichard Lowe ldx [%o1], %g2 /* modified */ 804*5d9d9091SRichard Lowe cmp %g2, %g3 /* is modified = current? */ 805*5d9d9091SRichard Lowe be,a,pt %xcc,1f /* yes, don't write */ 806*5d9d9091SRichard Lowe stx %g3, [%o0] /* update new original */ 807*5d9d9091SRichard Lowe casx [%o2], %g1, %g2 808*5d9d9091SRichard Lowe cmp %g1, %g2 809*5d9d9091SRichard Lowe be,pt %xcc, 1f /* cas succeeded - return */ 810*5d9d9091SRichard Lowe nop 811*5d9d9091SRichard Lowe ldx [%o2], %g3 /* new current */ 812*5d9d9091SRichard Lowe stx %g3, [%o0] /* save as new original */ 813*5d9d9091SRichard Lowe ba,pt %xcc, 2b 814*5d9d9091SRichard Lowe mov %g3, %g1 815*5d9d9091SRichard Lowe1: retl 816*5d9d9091SRichard Lowe membar #StoreLoad 817*5d9d9091SRichard Lowe SET_SIZE(sfmmu_modifytte) 818*5d9d9091SRichard Lowe 819*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_modifytte_try) 820*5d9d9091SRichard Lowe ldx [%o1], %g2 /* modified */ 821*5d9d9091SRichard Lowe ldx [%o2], %g3 /* current */ 822*5d9d9091SRichard Lowe ldx [%o0], %g1 /* original */ 823*5d9d9091SRichard Lowe cmp %g3, %g2 /* is modified = current? */ 824*5d9d9091SRichard Lowe be,a,pn %xcc,1f /* yes, don't write */ 825*5d9d9091SRichard Lowe mov 0, %o1 /* as if cas failed. */ 826*5d9d9091SRichard Lowe 827*5d9d9091SRichard Lowe casx [%o2], %g1, %g2 828*5d9d9091SRichard Lowe membar #StoreLoad 829*5d9d9091SRichard Lowe cmp %g1, %g2 830*5d9d9091SRichard Lowe movne %xcc, -1, %o1 /* cas failed. */ 831*5d9d9091SRichard Lowe move %xcc, 1, %o1 /* cas succeeded. */ 832*5d9d9091SRichard Lowe1: 833*5d9d9091SRichard Lowe stx %g2, [%o0] /* report "current" value */ 834*5d9d9091SRichard Lowe retl 835*5d9d9091SRichard Lowe mov %o1, %o0 836*5d9d9091SRichard Lowe SET_SIZE(sfmmu_modifytte_try) 837*5d9d9091SRichard Lowe 838*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_copytte) 839*5d9d9091SRichard Lowe ldx [%o0], %g1 840*5d9d9091SRichard Lowe retl 841*5d9d9091SRichard Lowe stx %g1, [%o1] 842*5d9d9091SRichard Lowe SET_SIZE(sfmmu_copytte) 843*5d9d9091SRichard Lowe 844*5d9d9091SRichard Lowe 845*5d9d9091SRichard Lowe /* 846*5d9d9091SRichard Lowe * Calculate a TSB entry pointer for the given TSB, va, pagesize. 847*5d9d9091SRichard Lowe * %o0 = TSB base address (in), pointer to TSB entry (out) 848*5d9d9091SRichard Lowe * %o1 = vaddr (in) 849*5d9d9091SRichard Lowe * %o2 = vpshift (in) 850*5d9d9091SRichard Lowe * %o3 = tsb size code (in) 851*5d9d9091SRichard Lowe * %o4 = scratch register 852*5d9d9091SRichard Lowe */ 853*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_get_tsbe) 854*5d9d9091SRichard Lowe GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 855*5d9d9091SRichard Lowe retl 856*5d9d9091SRichard Lowe nop 857*5d9d9091SRichard Lowe SET_SIZE(sfmmu_get_tsbe) 858*5d9d9091SRichard Lowe 859*5d9d9091SRichard Lowe /* 860*5d9d9091SRichard Lowe * Return a TSB tag for the given va. 861*5d9d9091SRichard Lowe * %o0 = va (in/clobbered) 862*5d9d9091SRichard Lowe * %o0 = va shifted to be in tsb tag format (with no context) (out) 863*5d9d9091SRichard Lowe */ 864*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_make_tsbtag) 865*5d9d9091SRichard Lowe retl 866*5d9d9091SRichard Lowe srln %o0, TTARGET_VA_SHIFT, %o0 867*5d9d9091SRichard Lowe SET_SIZE(sfmmu_make_tsbtag) 868*5d9d9091SRichard Lowe 869*5d9d9091SRichard Lowe/* 870*5d9d9091SRichard Lowe * Other sfmmu primitives 871*5d9d9091SRichard Lowe */ 872*5d9d9091SRichard Lowe 873*5d9d9091SRichard Lowe 874*5d9d9091SRichard Lowe#define I_SIZE 4 875*5d9d9091SRichard Lowe 876*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_fix_ktlb_traptable) 877*5d9d9091SRichard Lowe /* 878*5d9d9091SRichard Lowe * %o0 = start of patch area 879*5d9d9091SRichard Lowe * %o1 = size code of TSB to patch 880*5d9d9091SRichard Lowe * %o3 = scratch 881*5d9d9091SRichard Lowe */ 882*5d9d9091SRichard Lowe /* fix sll */ 883*5d9d9091SRichard Lowe ld [%o0], %o3 /* get sll */ 884*5d9d9091SRichard Lowe sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 885*5d9d9091SRichard Lowe st %o3, [%o0] /* write sll */ 886*5d9d9091SRichard Lowe flush %o0 887*5d9d9091SRichard Lowe /* fix srl */ 888*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 /* goto next instr. */ 889*5d9d9091SRichard Lowe ld [%o0], %o3 /* get srl */ 890*5d9d9091SRichard Lowe sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 891*5d9d9091SRichard Lowe st %o3, [%o0] /* write srl */ 892*5d9d9091SRichard Lowe retl 893*5d9d9091SRichard Lowe flush %o0 894*5d9d9091SRichard Lowe SET_SIZE(sfmmu_fix_ktlb_traptable) 895*5d9d9091SRichard Lowe 896*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_fixup_ktsbbase) 897*5d9d9091SRichard Lowe /* 898*5d9d9091SRichard Lowe * %o0 = start of patch area 899*5d9d9091SRichard Lowe * %o5 = kernel virtual or physical tsb base address 900*5d9d9091SRichard Lowe * %o2, %o3 are used as scratch registers. 901*5d9d9091SRichard Lowe */ 902*5d9d9091SRichard Lowe /* fixup sethi instruction */ 903*5d9d9091SRichard Lowe ld [%o0], %o3 904*5d9d9091SRichard Lowe srl %o5, 10, %o2 ! offset is bits 32:10 905*5d9d9091SRichard Lowe or %o3, %o2, %o3 ! set imm22 906*5d9d9091SRichard Lowe st %o3, [%o0] 907*5d9d9091SRichard Lowe /* fixup offset of lduw/ldx */ 908*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 ! next instr 909*5d9d9091SRichard Lowe ld [%o0], %o3 910*5d9d9091SRichard Lowe and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 911*5d9d9091SRichard Lowe or %o3, %o2, %o3 912*5d9d9091SRichard Lowe st %o3, [%o0] 913*5d9d9091SRichard Lowe retl 914*5d9d9091SRichard Lowe flush %o0 915*5d9d9091SRichard Lowe SET_SIZE(sfmmu_fixup_ktsbbase) 916*5d9d9091SRichard Lowe 917*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_fixup_setx) 918*5d9d9091SRichard Lowe /* 919*5d9d9091SRichard Lowe * %o0 = start of patch area 920*5d9d9091SRichard Lowe * %o4 = 64 bit value to patch 921*5d9d9091SRichard Lowe * %o2, %o3 are used as scratch registers. 922*5d9d9091SRichard Lowe * 923*5d9d9091SRichard Lowe * Note: Assuming that all parts of the instructions which need to be 924*5d9d9091SRichard Lowe * patched correspond to RUNTIME_PATCH (aka 0) 925*5d9d9091SRichard Lowe * 926*5d9d9091SRichard Lowe * Note the implementation of setx which is being patched is as follows: 927*5d9d9091SRichard Lowe * 928*5d9d9091SRichard Lowe * sethi %hh(RUNTIME_PATCH), tmp 929*5d9d9091SRichard Lowe * sethi %lm(RUNTIME_PATCH), dest 930*5d9d9091SRichard Lowe * or tmp, %hm(RUNTIME_PATCH), tmp 931*5d9d9091SRichard Lowe * or dest, %lo(RUNTIME_PATCH), dest 932*5d9d9091SRichard Lowe * sllx tmp, 32, tmp 933*5d9d9091SRichard Lowe * nop 934*5d9d9091SRichard Lowe * or tmp, dest, dest 935*5d9d9091SRichard Lowe * 936*5d9d9091SRichard Lowe * which differs from the implementation in the 937*5d9d9091SRichard Lowe * "SPARC Architecture Manual" 938*5d9d9091SRichard Lowe */ 939*5d9d9091SRichard Lowe /* fixup sethi instruction */ 940*5d9d9091SRichard Lowe ld [%o0], %o3 941*5d9d9091SRichard Lowe srlx %o4, 42, %o2 ! bits [63:42] 942*5d9d9091SRichard Lowe or %o3, %o2, %o3 ! set imm22 943*5d9d9091SRichard Lowe st %o3, [%o0] 944*5d9d9091SRichard Lowe /* fixup sethi instruction */ 945*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 ! next instr 946*5d9d9091SRichard Lowe ld [%o0], %o3 947*5d9d9091SRichard Lowe sllx %o4, 32, %o2 ! clear upper bits 948*5d9d9091SRichard Lowe srlx %o2, 42, %o2 ! bits [31:10] 949*5d9d9091SRichard Lowe or %o3, %o2, %o3 ! set imm22 950*5d9d9091SRichard Lowe st %o3, [%o0] 951*5d9d9091SRichard Lowe /* fixup or instruction */ 952*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 ! next instr 953*5d9d9091SRichard Lowe ld [%o0], %o3 954*5d9d9091SRichard Lowe srlx %o4, 32, %o2 ! bits [63:32] 955*5d9d9091SRichard Lowe and %o2, 0x3ff, %o2 ! bits [41:32] 956*5d9d9091SRichard Lowe or %o3, %o2, %o3 ! set imm 957*5d9d9091SRichard Lowe st %o3, [%o0] 958*5d9d9091SRichard Lowe /* fixup or instruction */ 959*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 ! next instr 960*5d9d9091SRichard Lowe ld [%o0], %o3 961*5d9d9091SRichard Lowe and %o4, 0x3ff, %o2 ! bits [9:0] 962*5d9d9091SRichard Lowe or %o3, %o2, %o3 ! set imm 963*5d9d9091SRichard Lowe st %o3, [%o0] 964*5d9d9091SRichard Lowe retl 965*5d9d9091SRichard Lowe flush %o0 966*5d9d9091SRichard Lowe SET_SIZE(sfmmu_fixup_setx) 967*5d9d9091SRichard Lowe 968*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_fixup_or) 969*5d9d9091SRichard Lowe /* 970*5d9d9091SRichard Lowe * %o0 = start of patch area 971*5d9d9091SRichard Lowe * %o4 = 32 bit value to patch 972*5d9d9091SRichard Lowe * %o2, %o3 are used as scratch registers. 973*5d9d9091SRichard Lowe * Note: Assuming that all parts of the instructions which need to be 974*5d9d9091SRichard Lowe * patched correspond to RUNTIME_PATCH (aka 0) 975*5d9d9091SRichard Lowe */ 976*5d9d9091SRichard Lowe ld [%o0], %o3 977*5d9d9091SRichard Lowe and %o4, 0x3ff, %o2 ! bits [9:0] 978*5d9d9091SRichard Lowe or %o3, %o2, %o3 ! set imm 979*5d9d9091SRichard Lowe st %o3, [%o0] 980*5d9d9091SRichard Lowe retl 981*5d9d9091SRichard Lowe flush %o0 982*5d9d9091SRichard Lowe SET_SIZE(sfmmu_fixup_or) 983*5d9d9091SRichard Lowe 984*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_fixup_shiftx) 985*5d9d9091SRichard Lowe /* 986*5d9d9091SRichard Lowe * %o0 = start of patch area 987*5d9d9091SRichard Lowe * %o4 = signed int immediate value to add to sllx/srlx imm field 988*5d9d9091SRichard Lowe * %o2, %o3 are used as scratch registers. 989*5d9d9091SRichard Lowe * 990*5d9d9091SRichard Lowe * sllx/srlx store the 6 bit immediate value in the lowest order bits 991*5d9d9091SRichard Lowe * so we do a simple add. The caller must be careful to prevent 992*5d9d9091SRichard Lowe * overflow, which could easily occur if the initial value is nonzero! 993*5d9d9091SRichard Lowe */ 994*5d9d9091SRichard Lowe ld [%o0], %o3 ! %o3 = instruction to patch 995*5d9d9091SRichard Lowe and %o3, 0x3f, %o2 ! %o2 = existing imm value 996*5d9d9091SRichard Lowe add %o2, %o4, %o2 ! %o2 = new imm value 997*5d9d9091SRichard Lowe andn %o3, 0x3f, %o3 ! clear old imm value 998*5d9d9091SRichard Lowe and %o2, 0x3f, %o2 ! truncate new imm value 999*5d9d9091SRichard Lowe or %o3, %o2, %o3 ! set new imm value 1000*5d9d9091SRichard Lowe st %o3, [%o0] ! store updated instruction 1001*5d9d9091SRichard Lowe retl 1002*5d9d9091SRichard Lowe flush %o0 1003*5d9d9091SRichard Lowe SET_SIZE(sfmmu_fixup_shiftx) 1004*5d9d9091SRichard Lowe 1005*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_fixup_mmu_asi) 1006*5d9d9091SRichard Lowe /* 1007*5d9d9091SRichard Lowe * Patch imm_asi of all ldda instructions in the MMU 1008*5d9d9091SRichard Lowe * trap handlers. We search MMU_PATCH_INSTR instructions 1009*5d9d9091SRichard Lowe * starting from the itlb miss handler (trap 0x64). 1010*5d9d9091SRichard Lowe * %o0 = address of tt[0,1]_itlbmiss 1011*5d9d9091SRichard Lowe * %o1 = imm_asi to setup, shifted by appropriate offset. 1012*5d9d9091SRichard Lowe * %o3 = number of instructions to search 1013*5d9d9091SRichard Lowe * %o4 = reserved by caller: called from leaf routine 1014*5d9d9091SRichard Lowe */ 1015*5d9d9091SRichard Lowe1: ldsw [%o0], %o2 ! load instruction to %o2 1016*5d9d9091SRichard Lowe brgez,pt %o2, 2f 1017*5d9d9091SRichard Lowe srl %o2, 30, %o5 1018*5d9d9091SRichard Lowe btst 1, %o5 ! test bit 30; skip if not set 1019*5d9d9091SRichard Lowe bz,pt %icc, 2f 1020*5d9d9091SRichard Lowe sllx %o2, 39, %o5 ! bit 24 -> bit 63 1021*5d9d9091SRichard Lowe srlx %o5, 58, %o5 ! isolate op3 part of opcode 1022*5d9d9091SRichard Lowe xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1023*5d9d9091SRichard Lowe brnz,pt %o5, 2f ! skip if not a match 1024*5d9d9091SRichard Lowe or %o2, %o1, %o2 ! or in imm_asi 1025*5d9d9091SRichard Lowe st %o2, [%o0] ! write patched instruction 1026*5d9d9091SRichard Lowe2: dec %o3 1027*5d9d9091SRichard Lowe brnz,a,pt %o3, 1b ! loop until we're done 1028*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 1029*5d9d9091SRichard Lowe retl 1030*5d9d9091SRichard Lowe flush %o0 1031*5d9d9091SRichard Lowe SET_SIZE(sfmmu_fixup_mmu_asi) 1032*5d9d9091SRichard Lowe 1033*5d9d9091SRichard Lowe /* 1034*5d9d9091SRichard Lowe * Patch immediate ASI used to access the TSB in the 1035*5d9d9091SRichard Lowe * trap table. 1036*5d9d9091SRichard Lowe * inputs: %o0 = value of ktsb_phys 1037*5d9d9091SRichard Lowe */ 1038*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_patch_mmu_asi) 1039*5d9d9091SRichard Lowe mov %o7, %o4 ! save return pc in %o4 1040*5d9d9091SRichard Lowe mov ASI_QUAD_LDD_PHYS, %o3 ! set QUAD_LDD_PHYS by default 1041*5d9d9091SRichard Lowe 1042*5d9d9091SRichard Lowe#ifdef sun4v 1043*5d9d9091SRichard Lowe 1044*5d9d9091SRichard Lowe /* 1045*5d9d9091SRichard Lowe * Check ktsb_phys. It must be non-zero for sun4v, panic if not. 1046*5d9d9091SRichard Lowe */ 1047*5d9d9091SRichard Lowe 1048*5d9d9091SRichard Lowe brnz,pt %o0, do_patch 1049*5d9d9091SRichard Lowe nop 1050*5d9d9091SRichard Lowe 1051*5d9d9091SRichard Lowe sethi %hi(sfmmu_panic11), %o0 1052*5d9d9091SRichard Lowe call panic 1053*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_panic11), %o0 1054*5d9d9091SRichard Lowedo_patch: 1055*5d9d9091SRichard Lowe 1056*5d9d9091SRichard Lowe#else /* sun4v */ 1057*5d9d9091SRichard Lowe /* 1058*5d9d9091SRichard Lowe * Some non-sun4v platforms deploy virtual ktsb (ktsb_phys==0). 1059*5d9d9091SRichard Lowe * Note that ASI_NQUAD_LD is not defined/used for sun4v 1060*5d9d9091SRichard Lowe */ 1061*5d9d9091SRichard Lowe movrz %o0, ASI_NQUAD_LD, %o3 1062*5d9d9091SRichard Lowe 1063*5d9d9091SRichard Lowe#endif /* sun4v */ 1064*5d9d9091SRichard Lowe 1065*5d9d9091SRichard Lowe sll %o3, 5, %o1 ! imm_asi offset 1066*5d9d9091SRichard Lowe mov 6, %o3 ! number of instructions 1067*5d9d9091SRichard Lowe sethi %hi(dktsb), %o0 ! to search 1068*5d9d9091SRichard Lowe call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1069*5d9d9091SRichard Lowe or %o0, %lo(dktsb), %o0 1070*5d9d9091SRichard Lowe mov 6, %o3 ! number of instructions 1071*5d9d9091SRichard Lowe sethi %hi(dktsb4m), %o0 ! to search 1072*5d9d9091SRichard Lowe call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1073*5d9d9091SRichard Lowe or %o0, %lo(dktsb4m), %o0 1074*5d9d9091SRichard Lowe mov 6, %o3 ! number of instructions 1075*5d9d9091SRichard Lowe sethi %hi(iktsb), %o0 ! to search 1076*5d9d9091SRichard Lowe call sfmmu_fixup_mmu_asi ! patch kitlb miss 1077*5d9d9091SRichard Lowe or %o0, %lo(iktsb), %o0 1078*5d9d9091SRichard Lowe mov 6, %o3 ! number of instructions 1079*5d9d9091SRichard Lowe sethi %hi(iktsb4m), %o0 ! to search 1080*5d9d9091SRichard Lowe call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1081*5d9d9091SRichard Lowe or %o0, %lo(iktsb4m), %o0 1082*5d9d9091SRichard Lowe mov %o4, %o7 ! retore return pc -- leaf 1083*5d9d9091SRichard Lowe retl 1084*5d9d9091SRichard Lowe nop 1085*5d9d9091SRichard Lowe SET_SIZE(sfmmu_patch_mmu_asi) 1086*5d9d9091SRichard Lowe 1087*5d9d9091SRichard Lowe 1088*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_patch_ktsb) 1089*5d9d9091SRichard Lowe /* 1090*5d9d9091SRichard Lowe * We need to fix iktsb, dktsb, et. al. 1091*5d9d9091SRichard Lowe */ 1092*5d9d9091SRichard Lowe save %sp, -SA(MINFRAME), %sp 1093*5d9d9091SRichard Lowe set ktsb_phys, %o1 1094*5d9d9091SRichard Lowe ld [%o1], %o4 1095*5d9d9091SRichard Lowe set ktsb_base, %o5 1096*5d9d9091SRichard Lowe set ktsb4m_base, %l1 1097*5d9d9091SRichard Lowe brz,pt %o4, 1f 1098*5d9d9091SRichard Lowe nop 1099*5d9d9091SRichard Lowe set ktsb_pbase, %o5 1100*5d9d9091SRichard Lowe set ktsb4m_pbase, %l1 1101*5d9d9091SRichard Lowe1: 1102*5d9d9091SRichard Lowe sethi %hi(ktsb_szcode), %o1 1103*5d9d9091SRichard Lowe ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1104*5d9d9091SRichard Lowe 1105*5d9d9091SRichard Lowe sethi %hi(iktsb), %o0 1106*5d9d9091SRichard Lowe call sfmmu_fix_ktlb_traptable 1107*5d9d9091SRichard Lowe or %o0, %lo(iktsb), %o0 1108*5d9d9091SRichard Lowe 1109*5d9d9091SRichard Lowe sethi %hi(dktsb), %o0 1110*5d9d9091SRichard Lowe call sfmmu_fix_ktlb_traptable 1111*5d9d9091SRichard Lowe or %o0, %lo(dktsb), %o0 1112*5d9d9091SRichard Lowe 1113*5d9d9091SRichard Lowe sethi %hi(ktsb4m_szcode), %o1 1114*5d9d9091SRichard Lowe ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1115*5d9d9091SRichard Lowe 1116*5d9d9091SRichard Lowe sethi %hi(iktsb4m), %o0 1117*5d9d9091SRichard Lowe call sfmmu_fix_ktlb_traptable 1118*5d9d9091SRichard Lowe or %o0, %lo(iktsb4m), %o0 1119*5d9d9091SRichard Lowe 1120*5d9d9091SRichard Lowe sethi %hi(dktsb4m), %o0 1121*5d9d9091SRichard Lowe call sfmmu_fix_ktlb_traptable 1122*5d9d9091SRichard Lowe or %o0, %lo(dktsb4m), %o0 1123*5d9d9091SRichard Lowe 1124*5d9d9091SRichard Lowe#ifndef sun4v 1125*5d9d9091SRichard Lowe mov ASI_N, %o2 1126*5d9d9091SRichard Lowe movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1127*5d9d9091SRichard Lowe mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1128*5d9d9091SRichard Lowe sethi %hi(tsb_kernel_patch_asi), %o0 1129*5d9d9091SRichard Lowe call sfmmu_fixup_or 1130*5d9d9091SRichard Lowe or %o0, %lo(tsb_kernel_patch_asi), %o0 1131*5d9d9091SRichard Lowe#endif /* !sun4v */ 1132*5d9d9091SRichard Lowe 1133*5d9d9091SRichard Lowe ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1134*5d9d9091SRichard Lowe 1135*5d9d9091SRichard Lowe sethi %hi(dktsbbase), %o0 1136*5d9d9091SRichard Lowe call sfmmu_fixup_setx ! patch value of ktsb base addr 1137*5d9d9091SRichard Lowe or %o0, %lo(dktsbbase), %o0 1138*5d9d9091SRichard Lowe 1139*5d9d9091SRichard Lowe sethi %hi(iktsbbase), %o0 1140*5d9d9091SRichard Lowe call sfmmu_fixup_setx ! patch value of ktsb base addr 1141*5d9d9091SRichard Lowe or %o0, %lo(iktsbbase), %o0 1142*5d9d9091SRichard Lowe 1143*5d9d9091SRichard Lowe sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1144*5d9d9091SRichard Lowe call sfmmu_fixup_setx ! patch value of ktsb base addr 1145*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1146*5d9d9091SRichard Lowe 1147*5d9d9091SRichard Lowe#ifdef sun4v 1148*5d9d9091SRichard Lowe sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1149*5d9d9091SRichard Lowe call sfmmu_fixup_setx ! patch value of ktsb base addr 1150*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1151*5d9d9091SRichard Lowe#endif /* sun4v */ 1152*5d9d9091SRichard Lowe 1153*5d9d9091SRichard Lowe ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1154*5d9d9091SRichard Lowe 1155*5d9d9091SRichard Lowe sethi %hi(dktsb4mbase), %o0 1156*5d9d9091SRichard Lowe call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1157*5d9d9091SRichard Lowe or %o0, %lo(dktsb4mbase), %o0 1158*5d9d9091SRichard Lowe 1159*5d9d9091SRichard Lowe sethi %hi(iktsb4mbase), %o0 1160*5d9d9091SRichard Lowe call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1161*5d9d9091SRichard Lowe or %o0, %lo(iktsb4mbase), %o0 1162*5d9d9091SRichard Lowe 1163*5d9d9091SRichard Lowe sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1164*5d9d9091SRichard Lowe call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1165*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1166*5d9d9091SRichard Lowe 1167*5d9d9091SRichard Lowe#ifdef sun4v 1168*5d9d9091SRichard Lowe sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1169*5d9d9091SRichard Lowe call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1170*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1171*5d9d9091SRichard Lowe#endif /* sun4v */ 1172*5d9d9091SRichard Lowe 1173*5d9d9091SRichard Lowe set ktsb_szcode, %o4 1174*5d9d9091SRichard Lowe ld [%o4], %o4 1175*5d9d9091SRichard Lowe sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1176*5d9d9091SRichard Lowe call sfmmu_fixup_or ! patch value of ktsb_szcode 1177*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1178*5d9d9091SRichard Lowe 1179*5d9d9091SRichard Lowe#ifdef sun4v 1180*5d9d9091SRichard Lowe sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1181*5d9d9091SRichard Lowe call sfmmu_fixup_or ! patch value of ktsb_szcode 1182*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1183*5d9d9091SRichard Lowe#endif /* sun4v */ 1184*5d9d9091SRichard Lowe 1185*5d9d9091SRichard Lowe set ktsb4m_szcode, %o4 1186*5d9d9091SRichard Lowe ld [%o4], %o4 1187*5d9d9091SRichard Lowe sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1188*5d9d9091SRichard Lowe call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1189*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1190*5d9d9091SRichard Lowe 1191*5d9d9091SRichard Lowe#ifdef sun4v 1192*5d9d9091SRichard Lowe sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1193*5d9d9091SRichard Lowe call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1194*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1195*5d9d9091SRichard Lowe#endif /* sun4v */ 1196*5d9d9091SRichard Lowe 1197*5d9d9091SRichard Lowe ret 1198*5d9d9091SRichard Lowe restore 1199*5d9d9091SRichard Lowe SET_SIZE(sfmmu_patch_ktsb) 1200*5d9d9091SRichard Lowe 1201*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_kpm_patch_tlbm) 1202*5d9d9091SRichard Lowe /* 1203*5d9d9091SRichard Lowe * Fixup trap handlers in common segkpm case. This is reserved 1204*5d9d9091SRichard Lowe * for future use should kpm TSB be changed to be other than the 1205*5d9d9091SRichard Lowe * kernel TSB. 1206*5d9d9091SRichard Lowe */ 1207*5d9d9091SRichard Lowe retl 1208*5d9d9091SRichard Lowe nop 1209*5d9d9091SRichard Lowe SET_SIZE(sfmmu_kpm_patch_tlbm) 1210*5d9d9091SRichard Lowe 1211*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_kpm_patch_tsbm) 1212*5d9d9091SRichard Lowe /* 1213*5d9d9091SRichard Lowe * nop the branch to sfmmu_kpm_dtsb_miss_small 1214*5d9d9091SRichard Lowe * in the case where we are using large pages for 1215*5d9d9091SRichard Lowe * seg_kpm (and hence must probe the second TSB for 1216*5d9d9091SRichard Lowe * seg_kpm VAs) 1217*5d9d9091SRichard Lowe */ 1218*5d9d9091SRichard Lowe set dktsb4m_kpmcheck_small, %o0 1219*5d9d9091SRichard Lowe MAKE_NOP_INSTR(%o1) 1220*5d9d9091SRichard Lowe st %o1, [%o0] 1221*5d9d9091SRichard Lowe flush %o0 1222*5d9d9091SRichard Lowe retl 1223*5d9d9091SRichard Lowe nop 1224*5d9d9091SRichard Lowe SET_SIZE(sfmmu_kpm_patch_tsbm) 1225*5d9d9091SRichard Lowe 1226*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_patch_utsb) 1227*5d9d9091SRichard Lowe#ifdef UTSB_PHYS 1228*5d9d9091SRichard Lowe retl 1229*5d9d9091SRichard Lowe nop 1230*5d9d9091SRichard Lowe#else /* UTSB_PHYS */ 1231*5d9d9091SRichard Lowe /* 1232*5d9d9091SRichard Lowe * We need to hot patch utsb_vabase and utsb4m_vabase 1233*5d9d9091SRichard Lowe */ 1234*5d9d9091SRichard Lowe save %sp, -SA(MINFRAME), %sp 1235*5d9d9091SRichard Lowe 1236*5d9d9091SRichard Lowe /* patch value of utsb_vabase */ 1237*5d9d9091SRichard Lowe set utsb_vabase, %o1 1238*5d9d9091SRichard Lowe ldx [%o1], %o4 1239*5d9d9091SRichard Lowe sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1240*5d9d9091SRichard Lowe call sfmmu_fixup_setx 1241*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1242*5d9d9091SRichard Lowe sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1243*5d9d9091SRichard Lowe call sfmmu_fixup_setx 1244*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1245*5d9d9091SRichard Lowe sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1246*5d9d9091SRichard Lowe call sfmmu_fixup_setx 1247*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1248*5d9d9091SRichard Lowe 1249*5d9d9091SRichard Lowe /* patch value of utsb4m_vabase */ 1250*5d9d9091SRichard Lowe set utsb4m_vabase, %o1 1251*5d9d9091SRichard Lowe ldx [%o1], %o4 1252*5d9d9091SRichard Lowe sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1253*5d9d9091SRichard Lowe call sfmmu_fixup_setx 1254*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1255*5d9d9091SRichard Lowe sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1256*5d9d9091SRichard Lowe call sfmmu_fixup_setx 1257*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1258*5d9d9091SRichard Lowe sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1259*5d9d9091SRichard Lowe call sfmmu_fixup_setx 1260*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1261*5d9d9091SRichard Lowe 1262*5d9d9091SRichard Lowe /* 1263*5d9d9091SRichard Lowe * Patch TSB base register masks and shifts if needed. 1264*5d9d9091SRichard Lowe * By default the TSB base register contents are set up for 4M slab. 1265*5d9d9091SRichard Lowe * If we're using a smaller slab size and reserved VA range we need 1266*5d9d9091SRichard Lowe * to patch up those values here. 1267*5d9d9091SRichard Lowe */ 1268*5d9d9091SRichard Lowe set tsb_slab_shift, %o1 1269*5d9d9091SRichard Lowe set MMU_PAGESHIFT4M, %o4 1270*5d9d9091SRichard Lowe lduw [%o1], %o3 1271*5d9d9091SRichard Lowe subcc %o4, %o3, %o4 1272*5d9d9091SRichard Lowe bz,pt %icc, 1f 1273*5d9d9091SRichard Lowe /* delay slot safe */ 1274*5d9d9091SRichard Lowe 1275*5d9d9091SRichard Lowe /* patch reserved VA range size if needed. */ 1276*5d9d9091SRichard Lowe sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1277*5d9d9091SRichard Lowe call sfmmu_fixup_shiftx 1278*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1279*5d9d9091SRichard Lowe call sfmmu_fixup_shiftx 1280*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 1281*5d9d9091SRichard Lowe sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1282*5d9d9091SRichard Lowe call sfmmu_fixup_shiftx 1283*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1284*5d9d9091SRichard Lowe call sfmmu_fixup_shiftx 1285*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 1286*5d9d9091SRichard Lowe1: 1287*5d9d9091SRichard Lowe /* patch TSBREG_VAMASK used to set up TSB base register */ 1288*5d9d9091SRichard Lowe set tsb_slab_mask, %o1 1289*5d9d9091SRichard Lowe ldx [%o1], %o4 1290*5d9d9091SRichard Lowe sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1291*5d9d9091SRichard Lowe call sfmmu_fixup_or 1292*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1293*5d9d9091SRichard Lowe sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1294*5d9d9091SRichard Lowe call sfmmu_fixup_or 1295*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1296*5d9d9091SRichard Lowe 1297*5d9d9091SRichard Lowe ret 1298*5d9d9091SRichard Lowe restore 1299*5d9d9091SRichard Lowe#endif /* UTSB_PHYS */ 1300*5d9d9091SRichard Lowe SET_SIZE(sfmmu_patch_utsb) 1301*5d9d9091SRichard Lowe 1302*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_patch_shctx) 1303*5d9d9091SRichard Lowe#ifdef sun4u 1304*5d9d9091SRichard Lowe retl 1305*5d9d9091SRichard Lowe nop 1306*5d9d9091SRichard Lowe#else /* sun4u */ 1307*5d9d9091SRichard Lowe set sfmmu_shctx_cpu_mondo_patch, %o0 1308*5d9d9091SRichard Lowe MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1309*5d9d9091SRichard Lowe st %o1, [%o0] 1310*5d9d9091SRichard Lowe flush %o0 1311*5d9d9091SRichard Lowe MAKE_NOP_INSTR(%o1) 1312*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 ! next instr 1313*5d9d9091SRichard Lowe st %o1, [%o0] 1314*5d9d9091SRichard Lowe flush %o0 1315*5d9d9091SRichard Lowe 1316*5d9d9091SRichard Lowe set sfmmu_shctx_user_rtt_patch, %o0 1317*5d9d9091SRichard Lowe st %o1, [%o0] ! nop 1st instruction 1318*5d9d9091SRichard Lowe flush %o0 1319*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 1320*5d9d9091SRichard Lowe st %o1, [%o0] ! nop 2nd instruction 1321*5d9d9091SRichard Lowe flush %o0 1322*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 1323*5d9d9091SRichard Lowe st %o1, [%o0] ! nop 3rd instruction 1324*5d9d9091SRichard Lowe flush %o0 1325*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 1326*5d9d9091SRichard Lowe st %o1, [%o0] ! nop 4th instruction 1327*5d9d9091SRichard Lowe flush %o0 1328*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 1329*5d9d9091SRichard Lowe st %o1, [%o0] ! nop 5th instruction 1330*5d9d9091SRichard Lowe flush %o0 1331*5d9d9091SRichard Lowe add %o0, I_SIZE, %o0 1332*5d9d9091SRichard Lowe st %o1, [%o0] ! nop 6th instruction 1333*5d9d9091SRichard Lowe retl 1334*5d9d9091SRichard Lowe flush %o0 1335*5d9d9091SRichard Lowe#endif /* sun4u */ 1336*5d9d9091SRichard Lowe SET_SIZE(sfmmu_patch_shctx) 1337*5d9d9091SRichard Lowe 1338*5d9d9091SRichard Lowe /* 1339*5d9d9091SRichard Lowe * Routine that loads an entry into a tsb using virtual addresses. 1340*5d9d9091SRichard Lowe * Locking is required since all cpus can use the same TSB. 1341*5d9d9091SRichard Lowe * Note that it is no longer required to have a valid context 1342*5d9d9091SRichard Lowe * when calling this function. 1343*5d9d9091SRichard Lowe */ 1344*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_load_tsbe) 1345*5d9d9091SRichard Lowe /* 1346*5d9d9091SRichard Lowe * %o0 = pointer to tsbe to load 1347*5d9d9091SRichard Lowe * %o1 = tsb tag 1348*5d9d9091SRichard Lowe * %o2 = virtual pointer to TTE 1349*5d9d9091SRichard Lowe * %o3 = 1 if physical address in %o0 else 0 1350*5d9d9091SRichard Lowe */ 1351*5d9d9091SRichard Lowe rdpr %pstate, %o5 1352*5d9d9091SRichard Lowe#ifdef DEBUG 1353*5d9d9091SRichard Lowe PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1354*5d9d9091SRichard Lowe#endif /* DEBUG */ 1355*5d9d9091SRichard Lowe 1356*5d9d9091SRichard Lowe wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1357*5d9d9091SRichard Lowe 1358*5d9d9091SRichard Lowe SETUP_TSB_ASI(%o3, %g3) 1359*5d9d9091SRichard Lowe TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, locked_tsb_l8) 1360*5d9d9091SRichard Lowe 1361*5d9d9091SRichard Lowe wrpr %g0, %o5, %pstate /* enable interrupts */ 1362*5d9d9091SRichard Lowe 1363*5d9d9091SRichard Lowe retl 1364*5d9d9091SRichard Lowe membar #StoreStore|#StoreLoad 1365*5d9d9091SRichard Lowe SET_SIZE(sfmmu_load_tsbe) 1366*5d9d9091SRichard Lowe 1367*5d9d9091SRichard Lowe /* 1368*5d9d9091SRichard Lowe * Flush TSB of a given entry if the tag matches. 1369*5d9d9091SRichard Lowe */ 1370*5d9d9091SRichard Lowe ENTRY(sfmmu_unload_tsbe) 1371*5d9d9091SRichard Lowe /* 1372*5d9d9091SRichard Lowe * %o0 = pointer to tsbe to be flushed 1373*5d9d9091SRichard Lowe * %o1 = tag to match 1374*5d9d9091SRichard Lowe * %o2 = 1 if physical address in %o0 else 0 1375*5d9d9091SRichard Lowe */ 1376*5d9d9091SRichard Lowe SETUP_TSB_ASI(%o2, %g1) 1377*5d9d9091SRichard Lowe TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1378*5d9d9091SRichard Lowe retl 1379*5d9d9091SRichard Lowe membar #StoreStore|#StoreLoad 1380*5d9d9091SRichard Lowe SET_SIZE(sfmmu_unload_tsbe) 1381*5d9d9091SRichard Lowe 1382*5d9d9091SRichard Lowe /* 1383*5d9d9091SRichard Lowe * Routine that loads a TTE into the kpm TSB from C code. 1384*5d9d9091SRichard Lowe * Locking is required since kpm TSB is shared among all CPUs. 1385*5d9d9091SRichard Lowe */ 1386*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_kpm_load_tsb) 1387*5d9d9091SRichard Lowe /* 1388*5d9d9091SRichard Lowe * %o0 = vaddr 1389*5d9d9091SRichard Lowe * %o1 = ttep 1390*5d9d9091SRichard Lowe * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1391*5d9d9091SRichard Lowe */ 1392*5d9d9091SRichard Lowe rdpr %pstate, %o5 ! %o5 = saved pstate 1393*5d9d9091SRichard Lowe#ifdef DEBUG 1394*5d9d9091SRichard Lowe PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1395*5d9d9091SRichard Lowe#endif /* DEBUG */ 1396*5d9d9091SRichard Lowe wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1397*5d9d9091SRichard Lowe 1398*5d9d9091SRichard Lowe#ifndef sun4v 1399*5d9d9091SRichard Lowe sethi %hi(ktsb_phys), %o4 1400*5d9d9091SRichard Lowe mov ASI_N, %o3 1401*5d9d9091SRichard Lowe ld [%o4 + %lo(ktsb_phys)], %o4 1402*5d9d9091SRichard Lowe movrnz %o4, ASI_MEM, %o3 1403*5d9d9091SRichard Lowe mov %o3, %asi 1404*5d9d9091SRichard Lowe#endif /* !sun4v */ 1405*5d9d9091SRichard Lowe mov %o0, %g1 ! %g1 = vaddr 1406*5d9d9091SRichard Lowe 1407*5d9d9091SRichard Lowe /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1408*5d9d9091SRichard Lowe GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1409*5d9d9091SRichard Lowe /* %g2 = tsbep, %g1 clobbered */ 1410*5d9d9091SRichard Lowe 1411*5d9d9091SRichard Lowe srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1412*5d9d9091SRichard Lowe /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1413*5d9d9091SRichard Lowe TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, locked_tsb_l9) 1414*5d9d9091SRichard Lowe 1415*5d9d9091SRichard Lowe wrpr %g0, %o5, %pstate ! enable interrupts 1416*5d9d9091SRichard Lowe retl 1417*5d9d9091SRichard Lowe membar #StoreStore|#StoreLoad 1418*5d9d9091SRichard Lowe SET_SIZE(sfmmu_kpm_load_tsb) 1419*5d9d9091SRichard Lowe 1420*5d9d9091SRichard Lowe /* 1421*5d9d9091SRichard Lowe * Routine that shoots down a TTE in the kpm TSB or in the 1422*5d9d9091SRichard Lowe * kernel TSB depending on virtpg. Locking is required since 1423*5d9d9091SRichard Lowe * kpm/kernel TSB is shared among all CPUs. 1424*5d9d9091SRichard Lowe */ 1425*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_kpm_unload_tsb) 1426*5d9d9091SRichard Lowe /* 1427*5d9d9091SRichard Lowe * %o0 = vaddr 1428*5d9d9091SRichard Lowe * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1429*5d9d9091SRichard Lowe */ 1430*5d9d9091SRichard Lowe#ifndef sun4v 1431*5d9d9091SRichard Lowe sethi %hi(ktsb_phys), %o4 1432*5d9d9091SRichard Lowe mov ASI_N, %o3 1433*5d9d9091SRichard Lowe ld [%o4 + %lo(ktsb_phys)], %o4 1434*5d9d9091SRichard Lowe movrnz %o4, ASI_MEM, %o3 1435*5d9d9091SRichard Lowe mov %o3, %asi 1436*5d9d9091SRichard Lowe#endif /* !sun4v */ 1437*5d9d9091SRichard Lowe mov %o0, %g1 ! %g1 = vaddr 1438*5d9d9091SRichard Lowe 1439*5d9d9091SRichard Lowe /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1440*5d9d9091SRichard Lowe GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1441*5d9d9091SRichard Lowe /* %g2 = tsbep, %g1 clobbered */ 1442*5d9d9091SRichard Lowe 1443*5d9d9091SRichard Lowe srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1444*5d9d9091SRichard Lowe /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1445*5d9d9091SRichard Lowe TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1446*5d9d9091SRichard Lowe 1447*5d9d9091SRichard Lowe retl 1448*5d9d9091SRichard Lowe membar #StoreStore|#StoreLoad 1449*5d9d9091SRichard Lowe SET_SIZE(sfmmu_kpm_unload_tsb) 1450*5d9d9091SRichard Lowe 1451*5d9d9091SRichard Lowe 1452*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_ttetopfn) 1453*5d9d9091SRichard Lowe ldx [%o0], %g1 /* read tte */ 1454*5d9d9091SRichard Lowe TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1455*5d9d9091SRichard Lowe /* 1456*5d9d9091SRichard Lowe * g1 = pfn 1457*5d9d9091SRichard Lowe */ 1458*5d9d9091SRichard Lowe retl 1459*5d9d9091SRichard Lowe mov %g1, %o0 1460*5d9d9091SRichard Lowe SET_SIZE(sfmmu_ttetopfn) 1461*5d9d9091SRichard Lowe 1462*5d9d9091SRichard Lowe/* 1463*5d9d9091SRichard Lowe * These macros are used to update global sfmmu hme hash statistics 1464*5d9d9091SRichard Lowe * in perf critical paths. It is only enabled in debug kernels or 1465*5d9d9091SRichard Lowe * if SFMMU_STAT_GATHER is defined 1466*5d9d9091SRichard Lowe */ 1467*5d9d9091SRichard Lowe#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1468*5d9d9091SRichard Lowe#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1469*5d9d9091SRichard Lowe ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1470*5d9d9091SRichard Lowe mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1471*5d9d9091SRichard Lowe cmp tmp1, hatid ;\ 1472*5d9d9091SRichard Lowe movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1473*5d9d9091SRichard Lowe set sfmmu_global_stat, tmp1 ;\ 1474*5d9d9091SRichard Lowe add tmp1, tmp2, tmp1 ;\ 1475*5d9d9091SRichard Lowe ld [tmp1], tmp2 ;\ 1476*5d9d9091SRichard Lowe inc tmp2 ;\ 1477*5d9d9091SRichard Lowe st tmp2, [tmp1] 1478*5d9d9091SRichard Lowe 1479*5d9d9091SRichard Lowe#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1480*5d9d9091SRichard Lowe ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1481*5d9d9091SRichard Lowe mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1482*5d9d9091SRichard Lowe cmp tmp1, hatid ;\ 1483*5d9d9091SRichard Lowe movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1484*5d9d9091SRichard Lowe set sfmmu_global_stat, tmp1 ;\ 1485*5d9d9091SRichard Lowe add tmp1, tmp2, tmp1 ;\ 1486*5d9d9091SRichard Lowe ld [tmp1], tmp2 ;\ 1487*5d9d9091SRichard Lowe inc tmp2 ;\ 1488*5d9d9091SRichard Lowe st tmp2, [tmp1] 1489*5d9d9091SRichard Lowe 1490*5d9d9091SRichard Lowe 1491*5d9d9091SRichard Lowe#else /* DEBUG || SFMMU_STAT_GATHER */ 1492*5d9d9091SRichard Lowe 1493*5d9d9091SRichard Lowe#define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1494*5d9d9091SRichard Lowe 1495*5d9d9091SRichard Lowe#define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1496*5d9d9091SRichard Lowe 1497*5d9d9091SRichard Lowe#endif /* DEBUG || SFMMU_STAT_GATHER */ 1498*5d9d9091SRichard Lowe 1499*5d9d9091SRichard Lowe/* 1500*5d9d9091SRichard Lowe * This macro is used to update global sfmmu kstas in non 1501*5d9d9091SRichard Lowe * perf critical areas so they are enabled all the time 1502*5d9d9091SRichard Lowe */ 1503*5d9d9091SRichard Lowe#define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1504*5d9d9091SRichard Lowe sethi %hi(sfmmu_global_stat), tmp1 ;\ 1505*5d9d9091SRichard Lowe add tmp1, statname, tmp1 ;\ 1506*5d9d9091SRichard Lowe ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1507*5d9d9091SRichard Lowe inc tmp2 ;\ 1508*5d9d9091SRichard Lowe st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1509*5d9d9091SRichard Lowe 1510*5d9d9091SRichard Lowe/* 1511*5d9d9091SRichard Lowe * These macros are used to update per cpu stats in non perf 1512*5d9d9091SRichard Lowe * critical areas so they are enabled all the time 1513*5d9d9091SRichard Lowe */ 1514*5d9d9091SRichard Lowe#define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1515*5d9d9091SRichard Lowe ld [tsbarea + stat], tmp1 ;\ 1516*5d9d9091SRichard Lowe inc tmp1 ;\ 1517*5d9d9091SRichard Lowe st tmp1, [tsbarea + stat] 1518*5d9d9091SRichard Lowe 1519*5d9d9091SRichard Lowe/* 1520*5d9d9091SRichard Lowe * These macros are used to update per cpu stats in non perf 1521*5d9d9091SRichard Lowe * critical areas so they are enabled all the time 1522*5d9d9091SRichard Lowe */ 1523*5d9d9091SRichard Lowe#define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1524*5d9d9091SRichard Lowe lduh [tsbarea + stat], tmp1 ;\ 1525*5d9d9091SRichard Lowe inc tmp1 ;\ 1526*5d9d9091SRichard Lowe stuh tmp1, [tsbarea + stat] 1527*5d9d9091SRichard Lowe 1528*5d9d9091SRichard Lowe#if defined(KPM_TLBMISS_STATS_GATHER) 1529*5d9d9091SRichard Lowe /* 1530*5d9d9091SRichard Lowe * Count kpm dtlb misses separately to allow a different 1531*5d9d9091SRichard Lowe * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1532*5d9d9091SRichard Lowe * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1533*5d9d9091SRichard Lowe */ 1534*5d9d9091SRichard Lowe#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1535*5d9d9091SRichard Lowe brgez tagacc, label /* KPM VA? */ ;\ 1536*5d9d9091SRichard Lowe nop ;\ 1537*5d9d9091SRichard Lowe CPU_INDEX(tmp1, tsbma) ;\ 1538*5d9d9091SRichard Lowe sethi %hi(kpmtsbm_area), tsbma ;\ 1539*5d9d9091SRichard Lowe sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1540*5d9d9091SRichard Lowe or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1541*5d9d9091SRichard Lowe add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1542*5d9d9091SRichard Lowe /* VA range check */ ;\ 1543*5d9d9091SRichard Lowe ldx [tsbma + KPMTSBM_VBASE], val ;\ 1544*5d9d9091SRichard Lowe cmp tagacc, val ;\ 1545*5d9d9091SRichard Lowe blu,pn %xcc, label ;\ 1546*5d9d9091SRichard Lowe ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1547*5d9d9091SRichard Lowe cmp tagacc, tmp1 ;\ 1548*5d9d9091SRichard Lowe bgeu,pn %xcc, label ;\ 1549*5d9d9091SRichard Lowe lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1550*5d9d9091SRichard Lowe inc val ;\ 1551*5d9d9091SRichard Lowe st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1552*5d9d9091SRichard Lowelabel: 1553*5d9d9091SRichard Lowe#else 1554*5d9d9091SRichard Lowe#define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1555*5d9d9091SRichard Lowe#endif /* KPM_TLBMISS_STATS_GATHER */ 1556*5d9d9091SRichard Lowe 1557*5d9d9091SRichard Lowe#ifdef PTL1_PANIC_DEBUG 1558*5d9d9091SRichard Lowe .seg ".data" 1559*5d9d9091SRichard Lowe .global test_ptl1_panic 1560*5d9d9091SRichard Lowetest_ptl1_panic: 1561*5d9d9091SRichard Lowe .word 0 1562*5d9d9091SRichard Lowe .align 8 1563*5d9d9091SRichard Lowe 1564*5d9d9091SRichard Lowe .seg ".text" 1565*5d9d9091SRichard Lowe .align 4 1566*5d9d9091SRichard Lowe#endif /* PTL1_PANIC_DEBUG */ 1567*5d9d9091SRichard Lowe 1568*5d9d9091SRichard Lowe /* 1569*5d9d9091SRichard Lowe * The following routines are jumped to from the mmu trap handlers to do 1570*5d9d9091SRichard Lowe * the setting up to call systrap. They are separate routines instead 1571*5d9d9091SRichard Lowe * of being part of the handlers because the handlers would exceed 32 1572*5d9d9091SRichard Lowe * instructions and since this is part of the slow path the jump cost is 1573*5d9d9091SRichard Lowe * irrelevant. 1574*5d9d9091SRichard Lowe */ 1575*5d9d9091SRichard Lowe 1576*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_pagefault) 1577*5d9d9091SRichard Lowe SET_GL_REG(1) 1578*5d9d9091SRichard Lowe USE_ALTERNATE_GLOBALS(%g5) 1579*5d9d9091SRichard Lowe GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1580*5d9d9091SRichard Lowe rdpr %tt, %g6 1581*5d9d9091SRichard Lowe cmp %g6, FAST_IMMU_MISS_TT 1582*5d9d9091SRichard Lowe be,a,pn %icc, 1f 1583*5d9d9091SRichard Lowe mov T_INSTR_MMU_MISS, %g3 1584*5d9d9091SRichard Lowe cmp %g6, T_INSTR_MMU_MISS 1585*5d9d9091SRichard Lowe be,a,pn %icc, 1f 1586*5d9d9091SRichard Lowe mov T_INSTR_MMU_MISS, %g3 1587*5d9d9091SRichard Lowe mov %g5, %g2 1588*5d9d9091SRichard Lowe mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1589*5d9d9091SRichard Lowe cmp %g6, FAST_DMMU_MISS_TT 1590*5d9d9091SRichard Lowe move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1591*5d9d9091SRichard Lowe cmp %g6, T_DATA_MMU_MISS 1592*5d9d9091SRichard Lowe move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1593*5d9d9091SRichard Lowe 1594*5d9d9091SRichard Lowe#ifdef PTL1_PANIC_DEBUG 1595*5d9d9091SRichard Lowe /* check if we want to test the tl1 panic */ 1596*5d9d9091SRichard Lowe sethi %hi(test_ptl1_panic), %g4 1597*5d9d9091SRichard Lowe ld [%g4 + %lo(test_ptl1_panic)], %g1 1598*5d9d9091SRichard Lowe st %g0, [%g4 + %lo(test_ptl1_panic)] 1599*5d9d9091SRichard Lowe cmp %g1, %g0 1600*5d9d9091SRichard Lowe bne,a,pn %icc, ptl1_panic 1601*5d9d9091SRichard Lowe or %g0, PTL1_BAD_DEBUG, %g1 1602*5d9d9091SRichard Lowe#endif /* PTL1_PANIC_DEBUG */ 1603*5d9d9091SRichard Lowe1: 1604*5d9d9091SRichard Lowe HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 1605*5d9d9091SRichard Lowe /* 1606*5d9d9091SRichard Lowe * g2 = tag access reg 1607*5d9d9091SRichard Lowe * g3.l = type 1608*5d9d9091SRichard Lowe * g3.h = 0 1609*5d9d9091SRichard Lowe */ 1610*5d9d9091SRichard Lowe sethi %hi(trap), %g1 1611*5d9d9091SRichard Lowe or %g1, %lo(trap), %g1 1612*5d9d9091SRichard Lowe2: 1613*5d9d9091SRichard Lowe ba,pt %xcc, sys_trap 1614*5d9d9091SRichard Lowe mov -1, %g4 1615*5d9d9091SRichard Lowe SET_SIZE(sfmmu_pagefault) 1616*5d9d9091SRichard Lowe 1617*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_mmu_trap) 1618*5d9d9091SRichard Lowe SET_GL_REG(1) 1619*5d9d9091SRichard Lowe USE_ALTERNATE_GLOBALS(%g5) 1620*5d9d9091SRichard Lowe GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 1621*5d9d9091SRichard Lowe rdpr %tt, %g6 1622*5d9d9091SRichard Lowe cmp %g6, FAST_IMMU_MISS_TT 1623*5d9d9091SRichard Lowe be,a,pn %icc, 1f 1624*5d9d9091SRichard Lowe mov T_INSTR_MMU_MISS, %g3 1625*5d9d9091SRichard Lowe cmp %g6, T_INSTR_MMU_MISS 1626*5d9d9091SRichard Lowe be,a,pn %icc, 1f 1627*5d9d9091SRichard Lowe mov T_INSTR_MMU_MISS, %g3 1628*5d9d9091SRichard Lowe mov %g5, %g2 1629*5d9d9091SRichard Lowe mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1630*5d9d9091SRichard Lowe cmp %g6, FAST_DMMU_MISS_TT 1631*5d9d9091SRichard Lowe move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1632*5d9d9091SRichard Lowe cmp %g6, T_DATA_MMU_MISS 1633*5d9d9091SRichard Lowe move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1634*5d9d9091SRichard Lowe1: 1635*5d9d9091SRichard Lowe /* 1636*5d9d9091SRichard Lowe * g2 = tag access reg 1637*5d9d9091SRichard Lowe * g3 = type 1638*5d9d9091SRichard Lowe */ 1639*5d9d9091SRichard Lowe sethi %hi(sfmmu_tsbmiss_exception), %g1 1640*5d9d9091SRichard Lowe or %g1, %lo(sfmmu_tsbmiss_exception), %g1 1641*5d9d9091SRichard Lowe ba,pt %xcc, sys_trap 1642*5d9d9091SRichard Lowe mov -1, %g4 1643*5d9d9091SRichard Lowe /*NOTREACHED*/ 1644*5d9d9091SRichard Lowe SET_SIZE(sfmmu_mmu_trap) 1645*5d9d9091SRichard Lowe 1646*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_suspend_tl) 1647*5d9d9091SRichard Lowe SET_GL_REG(1) 1648*5d9d9091SRichard Lowe USE_ALTERNATE_GLOBALS(%g5) 1649*5d9d9091SRichard Lowe GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 1650*5d9d9091SRichard Lowe rdpr %tt, %g6 1651*5d9d9091SRichard Lowe cmp %g6, FAST_IMMU_MISS_TT 1652*5d9d9091SRichard Lowe be,a,pn %icc, 1f 1653*5d9d9091SRichard Lowe mov T_INSTR_MMU_MISS, %g3 1654*5d9d9091SRichard Lowe mov %g5, %g2 1655*5d9d9091SRichard Lowe cmp %g6, FAST_DMMU_MISS_TT 1656*5d9d9091SRichard Lowe move %icc, T_DATA_MMU_MISS, %g3 1657*5d9d9091SRichard Lowe movne %icc, T_DATA_PROT, %g3 1658*5d9d9091SRichard Lowe1: 1659*5d9d9091SRichard Lowe sethi %hi(sfmmu_tsbmiss_suspended), %g1 1660*5d9d9091SRichard Lowe or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 1661*5d9d9091SRichard Lowe /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 1662*5d9d9091SRichard Lowe ba,pt %xcc, sys_trap 1663*5d9d9091SRichard Lowe mov PIL_15, %g4 1664*5d9d9091SRichard Lowe /*NOTREACHED*/ 1665*5d9d9091SRichard Lowe SET_SIZE(sfmmu_suspend_tl) 1666*5d9d9091SRichard Lowe 1667*5d9d9091SRichard Lowe /* 1668*5d9d9091SRichard Lowe * No %g registers in use at this point. 1669*5d9d9091SRichard Lowe */ 1670*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_window_trap) 1671*5d9d9091SRichard Lowe rdpr %tpc, %g1 1672*5d9d9091SRichard Lowe#ifdef sun4v 1673*5d9d9091SRichard Lowe#ifdef DEBUG 1674*5d9d9091SRichard Lowe /* We assume previous %gl was 1 */ 1675*5d9d9091SRichard Lowe rdpr %tstate, %g4 1676*5d9d9091SRichard Lowe srlx %g4, TSTATE_GL_SHIFT, %g4 1677*5d9d9091SRichard Lowe and %g4, TSTATE_GL_MASK, %g4 1678*5d9d9091SRichard Lowe cmp %g4, 1 1679*5d9d9091SRichard Lowe bne,a,pn %icc, ptl1_panic 1680*5d9d9091SRichard Lowe mov PTL1_BAD_WTRAP, %g1 1681*5d9d9091SRichard Lowe#endif /* DEBUG */ 1682*5d9d9091SRichard Lowe /* user miss at tl>1. better be the window handler or user_rtt */ 1683*5d9d9091SRichard Lowe /* in user_rtt? */ 1684*5d9d9091SRichard Lowe set rtt_fill_start, %g4 1685*5d9d9091SRichard Lowe cmp %g1, %g4 1686*5d9d9091SRichard Lowe blu,pn %xcc, 6f 1687*5d9d9091SRichard Lowe .empty 1688*5d9d9091SRichard Lowe set rtt_fill_end, %g4 1689*5d9d9091SRichard Lowe cmp %g1, %g4 1690*5d9d9091SRichard Lowe bgeu,pn %xcc, 6f 1691*5d9d9091SRichard Lowe nop 1692*5d9d9091SRichard Lowe set fault_rtt_fn1, %g1 1693*5d9d9091SRichard Lowe wrpr %g0, %g1, %tnpc 1694*5d9d9091SRichard Lowe ba,a 7f 1695*5d9d9091SRichard Lowe6: 1696*5d9d9091SRichard Lowe ! must save this trap level before descending trap stack 1697*5d9d9091SRichard Lowe ! no need to save %tnpc, either overwritten or discarded 1698*5d9d9091SRichard Lowe ! already got it: rdpr %tpc, %g1 1699*5d9d9091SRichard Lowe rdpr %tstate, %g6 1700*5d9d9091SRichard Lowe rdpr %tt, %g7 1701*5d9d9091SRichard Lowe ! trap level saved, go get underlying trap type 1702*5d9d9091SRichard Lowe rdpr %tl, %g5 1703*5d9d9091SRichard Lowe sub %g5, 1, %g3 1704*5d9d9091SRichard Lowe wrpr %g3, %tl 1705*5d9d9091SRichard Lowe rdpr %tt, %g2 1706*5d9d9091SRichard Lowe wrpr %g5, %tl 1707*5d9d9091SRichard Lowe ! restore saved trap level 1708*5d9d9091SRichard Lowe wrpr %g1, %tpc 1709*5d9d9091SRichard Lowe wrpr %g6, %tstate 1710*5d9d9091SRichard Lowe wrpr %g7, %tt 1711*5d9d9091SRichard Lowe#else /* sun4v */ 1712*5d9d9091SRichard Lowe /* user miss at tl>1. better be the window handler */ 1713*5d9d9091SRichard Lowe rdpr %tl, %g5 1714*5d9d9091SRichard Lowe sub %g5, 1, %g3 1715*5d9d9091SRichard Lowe wrpr %g3, %tl 1716*5d9d9091SRichard Lowe rdpr %tt, %g2 1717*5d9d9091SRichard Lowe wrpr %g5, %tl 1718*5d9d9091SRichard Lowe#endif /* sun4v */ 1719*5d9d9091SRichard Lowe and %g2, WTRAP_TTMASK, %g4 1720*5d9d9091SRichard Lowe cmp %g4, WTRAP_TYPE 1721*5d9d9091SRichard Lowe bne,pn %xcc, 1f 1722*5d9d9091SRichard Lowe nop 1723*5d9d9091SRichard Lowe /* tpc should be in the trap table */ 1724*5d9d9091SRichard Lowe set trap_table, %g4 1725*5d9d9091SRichard Lowe cmp %g1, %g4 1726*5d9d9091SRichard Lowe blt,pn %xcc, 1f 1727*5d9d9091SRichard Lowe .empty 1728*5d9d9091SRichard Lowe set etrap_table, %g4 1729*5d9d9091SRichard Lowe cmp %g1, %g4 1730*5d9d9091SRichard Lowe bge,pn %xcc, 1f 1731*5d9d9091SRichard Lowe .empty 1732*5d9d9091SRichard Lowe andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 1733*5d9d9091SRichard Lowe add %g1, WTRAP_FAULTOFF, %g1 1734*5d9d9091SRichard Lowe wrpr %g0, %g1, %tnpc 1735*5d9d9091SRichard Lowe7: 1736*5d9d9091SRichard Lowe /* 1737*5d9d9091SRichard Lowe * some wbuf handlers will call systrap to resolve the fault 1738*5d9d9091SRichard Lowe * we pass the trap type so they figure out the correct parameters. 1739*5d9d9091SRichard Lowe * g5 = trap type, g6 = tag access reg 1740*5d9d9091SRichard Lowe */ 1741*5d9d9091SRichard Lowe 1742*5d9d9091SRichard Lowe /* 1743*5d9d9091SRichard Lowe * only use g5, g6, g7 registers after we have switched to alternate 1744*5d9d9091SRichard Lowe * globals. 1745*5d9d9091SRichard Lowe */ 1746*5d9d9091SRichard Lowe SET_GL_REG(1) 1747*5d9d9091SRichard Lowe USE_ALTERNATE_GLOBALS(%g5) 1748*5d9d9091SRichard Lowe GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 1749*5d9d9091SRichard Lowe rdpr %tt, %g7 1750*5d9d9091SRichard Lowe cmp %g7, FAST_IMMU_MISS_TT 1751*5d9d9091SRichard Lowe be,a,pn %icc, ptl1_panic 1752*5d9d9091SRichard Lowe mov PTL1_BAD_WTRAP, %g1 1753*5d9d9091SRichard Lowe cmp %g7, T_INSTR_MMU_MISS 1754*5d9d9091SRichard Lowe be,a,pn %icc, ptl1_panic 1755*5d9d9091SRichard Lowe mov PTL1_BAD_WTRAP, %g1 1756*5d9d9091SRichard Lowe mov T_DATA_PROT, %g5 1757*5d9d9091SRichard Lowe cmp %g7, FAST_DMMU_MISS_TT 1758*5d9d9091SRichard Lowe move %icc, T_DATA_MMU_MISS, %g5 1759*5d9d9091SRichard Lowe cmp %g7, T_DATA_MMU_MISS 1760*5d9d9091SRichard Lowe move %icc, T_DATA_MMU_MISS, %g5 1761*5d9d9091SRichard Lowe ! XXXQ AGS re-check out this one 1762*5d9d9091SRichard Lowe done 1763*5d9d9091SRichard Lowe1: 1764*5d9d9091SRichard Lowe CPU_PADDR(%g1, %g4) 1765*5d9d9091SRichard Lowe add %g1, CPU_TL1_HDLR, %g1 1766*5d9d9091SRichard Lowe lda [%g1]ASI_MEM, %g4 1767*5d9d9091SRichard Lowe brnz,a,pt %g4, sfmmu_mmu_trap 1768*5d9d9091SRichard Lowe sta %g0, [%g1]ASI_MEM 1769*5d9d9091SRichard Lowe ba,pt %icc, ptl1_panic 1770*5d9d9091SRichard Lowe mov PTL1_BAD_TRAP, %g1 1771*5d9d9091SRichard Lowe SET_SIZE(sfmmu_window_trap) 1772*5d9d9091SRichard Lowe 1773*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_kpm_exception) 1774*5d9d9091SRichard Lowe /* 1775*5d9d9091SRichard Lowe * We have accessed an unmapped segkpm address or a legal segkpm 1776*5d9d9091SRichard Lowe * address which is involved in a VAC alias conflict prevention. 1777*5d9d9091SRichard Lowe * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 1778*5d9d9091SRichard Lowe * set. If it is, we will instead note that a fault has occurred 1779*5d9d9091SRichard Lowe * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 1780*5d9d9091SRichard Lowe * a "retry"). This will step over the faulting instruction. 1781*5d9d9091SRichard Lowe * Note that this means that a legal segkpm address involved in 1782*5d9d9091SRichard Lowe * a VAC alias conflict prevention (a rare case to begin with) 1783*5d9d9091SRichard Lowe * cannot be used in DTrace. 1784*5d9d9091SRichard Lowe */ 1785*5d9d9091SRichard Lowe CPU_INDEX(%g1, %g2) 1786*5d9d9091SRichard Lowe set cpu_core, %g2 1787*5d9d9091SRichard Lowe sllx %g1, CPU_CORE_SHIFT, %g1 1788*5d9d9091SRichard Lowe add %g1, %g2, %g1 1789*5d9d9091SRichard Lowe lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 1790*5d9d9091SRichard Lowe andcc %g2, CPU_DTRACE_NOFAULT, %g0 1791*5d9d9091SRichard Lowe bz 0f 1792*5d9d9091SRichard Lowe or %g2, CPU_DTRACE_BADADDR, %g2 1793*5d9d9091SRichard Lowe stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 1794*5d9d9091SRichard Lowe GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 1795*5d9d9091SRichard Lowe stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 1796*5d9d9091SRichard Lowe done 1797*5d9d9091SRichard Lowe0: 1798*5d9d9091SRichard Lowe TSTAT_CHECK_TL1(1f, %g1, %g2) 1799*5d9d9091SRichard Lowe1: 1800*5d9d9091SRichard Lowe SET_GL_REG(1) 1801*5d9d9091SRichard Lowe USE_ALTERNATE_GLOBALS(%g5) 1802*5d9d9091SRichard Lowe GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 1803*5d9d9091SRichard Lowe mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1804*5d9d9091SRichard Lowe /* 1805*5d9d9091SRichard Lowe * g2=tagacc g3.l=type g3.h=0 1806*5d9d9091SRichard Lowe */ 1807*5d9d9091SRichard Lowe sethi %hi(trap), %g1 1808*5d9d9091SRichard Lowe or %g1, %lo(trap), %g1 1809*5d9d9091SRichard Lowe ba,pt %xcc, sys_trap 1810*5d9d9091SRichard Lowe mov -1, %g4 1811*5d9d9091SRichard Lowe SET_SIZE(sfmmu_kpm_exception) 1812*5d9d9091SRichard Lowe 1813*5d9d9091SRichard Lowe#if (IMAP_SEG != 0) 1814*5d9d9091SRichard Lowe#error - ism_map->ism_seg offset is not zero 1815*5d9d9091SRichard Lowe#endif 1816*5d9d9091SRichard Lowe 1817*5d9d9091SRichard Lowe/* 1818*5d9d9091SRichard Lowe * Copies ism mapping for this ctx in param "ism" if this is a ISM 1819*5d9d9091SRichard Lowe * tlb miss and branches to label "ismhit". If this is not an ISM 1820*5d9d9091SRichard Lowe * process or an ISM tlb miss it falls thru. 1821*5d9d9091SRichard Lowe * 1822*5d9d9091SRichard Lowe * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 1823*5d9d9091SRichard Lowe * this process. 1824*5d9d9091SRichard Lowe * If so, it will branch to label "ismhit". If not, it will fall through. 1825*5d9d9091SRichard Lowe * 1826*5d9d9091SRichard Lowe * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 1827*5d9d9091SRichard Lowe * so that any other threads of this process will not try and walk the ism 1828*5d9d9091SRichard Lowe * maps while they are being changed. 1829*5d9d9091SRichard Lowe * 1830*5d9d9091SRichard Lowe * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 1831*5d9d9091SRichard Lowe * will make sure of that. This means we can terminate our search on 1832*5d9d9091SRichard Lowe * the first zero mapping we find. 1833*5d9d9091SRichard Lowe * 1834*5d9d9091SRichard Lowe * Parameters: 1835*5d9d9091SRichard Lowe * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 1836*5d9d9091SRichard Lowe * tsbmiss = address of tsb miss area (in) 1837*5d9d9091SRichard Lowe * ismseg = contents of ism_seg for this ism map (out) 1838*5d9d9091SRichard Lowe * ismhat = physical address of imap_ismhat for this ism map (out) 1839*5d9d9091SRichard Lowe * tmp1 = scratch reg (CLOBBERED) 1840*5d9d9091SRichard Lowe * tmp2 = scratch reg (CLOBBERED) 1841*5d9d9091SRichard Lowe * tmp3 = scratch reg (CLOBBERED) 1842*5d9d9091SRichard Lowe * label: temporary labels 1843*5d9d9091SRichard Lowe * ismhit: label where to jump to if an ism dtlb miss 1844*5d9d9091SRichard Lowe * exitlabel:label where to jump if hat is busy due to hat_unshare. 1845*5d9d9091SRichard Lowe */ 1846*5d9d9091SRichard Lowe#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 1847*5d9d9091SRichard Lowe label, ismhit) \ 1848*5d9d9091SRichard Lowe ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 1849*5d9d9091SRichard Lowe brlz,pt tmp1, label##3 /* exit if -1 */ ;\ 1850*5d9d9091SRichard Lowe add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 1851*5d9d9091SRichard Lowelabel##1: ;\ 1852*5d9d9091SRichard Lowe ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 1853*5d9d9091SRichard Lowe mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 1854*5d9d9091SRichard Lowelabel##2: ;\ 1855*5d9d9091SRichard Lowe brz,pt ismseg, label##3 /* no mapping */ ;\ 1856*5d9d9091SRichard Lowe add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 1857*5d9d9091SRichard Lowe lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 1858*5d9d9091SRichard Lowe srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 1859*5d9d9091SRichard Lowe srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 1860*5d9d9091SRichard Lowe sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 1861*5d9d9091SRichard Lowe add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 1862*5d9d9091SRichard Lowe lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 1863*5d9d9091SRichard Lowe and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 1864*5d9d9091SRichard Lowe cmp tmp2, tmp1 /* check va <= offset*/ ;\ 1865*5d9d9091SRichard Lowe blu,a,pt %xcc, ismhit /* ism hit */ ;\ 1866*5d9d9091SRichard Lowe add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 1867*5d9d9091SRichard Lowe ;\ 1868*5d9d9091SRichard Lowe add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 1869*5d9d9091SRichard Lowe add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 1870*5d9d9091SRichard Lowe cmp ismhat, tmp1 ;\ 1871*5d9d9091SRichard Lowe bl,pt %xcc, label##2 /* keep looking */ ;\ 1872*5d9d9091SRichard Lowe ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 1873*5d9d9091SRichard Lowe ;\ 1874*5d9d9091SRichard Lowe add tmp3, IBLK_NEXTPA, tmp1 ;\ 1875*5d9d9091SRichard Lowe ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 1876*5d9d9091SRichard Lowe brgez,pt tmp1, label##1 /* continue if not -1*/ ;\ 1877*5d9d9091SRichard Lowe add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 1878*5d9d9091SRichard Lowelabel##3: 1879*5d9d9091SRichard Lowe 1880*5d9d9091SRichard Lowe/* 1881*5d9d9091SRichard Lowe * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 1882*5d9d9091SRichard Lowe * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 1883*5d9d9091SRichard Lowe * Parameters: 1884*5d9d9091SRichard Lowe * tagacc = reg containing virtual address 1885*5d9d9091SRichard Lowe * hatid = reg containing sfmmu pointer 1886*5d9d9091SRichard Lowe * hmeshift = constant/register to shift vaddr to obtain vapg 1887*5d9d9091SRichard Lowe * hmebp = register where bucket pointer will be stored 1888*5d9d9091SRichard Lowe * vapg = register where virtual page will be stored 1889*5d9d9091SRichard Lowe * tmp1, tmp2 = tmp registers 1890*5d9d9091SRichard Lowe */ 1891*5d9d9091SRichard Lowe 1892*5d9d9091SRichard Lowe 1893*5d9d9091SRichard Lowe#define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 1894*5d9d9091SRichard Lowe vapg, label, tmp1, tmp2) \ 1895*5d9d9091SRichard Lowe sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 1896*5d9d9091SRichard Lowe brnz,a,pt tmp1, label##1 ;\ 1897*5d9d9091SRichard Lowe ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 1898*5d9d9091SRichard Lowe ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 1899*5d9d9091SRichard Lowe ba,pt %xcc, label##2 ;\ 1900*5d9d9091SRichard Lowe ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 1901*5d9d9091SRichard Lowelabel##1: ;\ 1902*5d9d9091SRichard Lowe ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 1903*5d9d9091SRichard Lowelabel##2: ;\ 1904*5d9d9091SRichard Lowe srlx tagacc, hmeshift, vapg ;\ 1905*5d9d9091SRichard Lowe xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 1906*5d9d9091SRichard Lowe and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 1907*5d9d9091SRichard Lowe mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 1908*5d9d9091SRichard Lowe add hmebp, tmp1, hmebp 1909*5d9d9091SRichard Lowe 1910*5d9d9091SRichard Lowe/* 1911*5d9d9091SRichard Lowe * hashtag includes bspage + hashno (64 bits). 1912*5d9d9091SRichard Lowe */ 1913*5d9d9091SRichard Lowe 1914*5d9d9091SRichard Lowe#define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 1915*5d9d9091SRichard Lowe sllx vapg, hmeshift, vapg ;\ 1916*5d9d9091SRichard Lowe mov hashno, hblktag ;\ 1917*5d9d9091SRichard Lowe sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 1918*5d9d9091SRichard Lowe or vapg, hblktag, hblktag 1919*5d9d9091SRichard Lowe 1920*5d9d9091SRichard Lowe/* 1921*5d9d9091SRichard Lowe * Function to traverse hmeblk hash link list and find corresponding match. 1922*5d9d9091SRichard Lowe * The search is done using physical pointers. It returns the physical address 1923*5d9d9091SRichard Lowe * pointer to the hmeblk that matches with the tag provided. 1924*5d9d9091SRichard Lowe * Parameters: 1925*5d9d9091SRichard Lowe * hmebp = register that points to hme hash bucket, also used as 1926*5d9d9091SRichard Lowe * tmp reg (clobbered) 1927*5d9d9091SRichard Lowe * hmeblktag = register with hmeblk tag match 1928*5d9d9091SRichard Lowe * hatid = register with hatid 1929*5d9d9091SRichard Lowe * hmeblkpa = register where physical ptr will be stored 1930*5d9d9091SRichard Lowe * tmp1 = tmp reg 1931*5d9d9091SRichard Lowe * label: temporary label 1932*5d9d9091SRichard Lowe */ 1933*5d9d9091SRichard Lowe 1934*5d9d9091SRichard Lowe#define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, tsbarea, \ 1935*5d9d9091SRichard Lowe tmp1, label) \ 1936*5d9d9091SRichard Lowe add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 1937*5d9d9091SRichard Lowe ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 1938*5d9d9091SRichard Lowe HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 1939*5d9d9091SRichard Lowelabel##1: ;\ 1940*5d9d9091SRichard Lowe cmp hmeblkpa, HMEBLK_ENDPA ;\ 1941*5d9d9091SRichard Lowe be,pn %xcc, label##2 ;\ 1942*5d9d9091SRichard Lowe HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 1943*5d9d9091SRichard Lowe add hmeblkpa, HMEBLK_TAG, hmebp ;\ 1944*5d9d9091SRichard Lowe ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 1945*5d9d9091SRichard Lowe add hmebp, CLONGSIZE, hmebp ;\ 1946*5d9d9091SRichard Lowe ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 1947*5d9d9091SRichard Lowe xor tmp1, hmeblktag, tmp1 ;\ 1948*5d9d9091SRichard Lowe xor hmebp, hatid, hmebp ;\ 1949*5d9d9091SRichard Lowe or hmebp, tmp1, hmebp ;\ 1950*5d9d9091SRichard Lowe brz,pn hmebp, label##2 /* branch on hit */ ;\ 1951*5d9d9091SRichard Lowe add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 1952*5d9d9091SRichard Lowe ba,pt %xcc, label##1 ;\ 1953*5d9d9091SRichard Lowe ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 1954*5d9d9091SRichard Lowelabel##2: 1955*5d9d9091SRichard Lowe 1956*5d9d9091SRichard Lowe/* 1957*5d9d9091SRichard Lowe * Function to traverse hmeblk hash link list and find corresponding match. 1958*5d9d9091SRichard Lowe * The search is done using physical pointers. It returns the physical address 1959*5d9d9091SRichard Lowe * pointer to the hmeblk that matches with the tag 1960*5d9d9091SRichard Lowe * provided. 1961*5d9d9091SRichard Lowe * Parameters: 1962*5d9d9091SRichard Lowe * hmeblktag = register with hmeblk tag match (rid field is 0) 1963*5d9d9091SRichard Lowe * hatid = register with hatid (pointer to SRD) 1964*5d9d9091SRichard Lowe * hmeblkpa = register where physical ptr will be stored 1965*5d9d9091SRichard Lowe * tmp1 = tmp reg 1966*5d9d9091SRichard Lowe * tmp2 = tmp reg 1967*5d9d9091SRichard Lowe * label: temporary label 1968*5d9d9091SRichard Lowe */ 1969*5d9d9091SRichard Lowe 1970*5d9d9091SRichard Lowe#define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, tsbarea, \ 1971*5d9d9091SRichard Lowe tmp1, tmp2, label) \ 1972*5d9d9091SRichard Lowelabel##1: ;\ 1973*5d9d9091SRichard Lowe cmp hmeblkpa, HMEBLK_ENDPA ;\ 1974*5d9d9091SRichard Lowe be,pn %xcc, label##4 ;\ 1975*5d9d9091SRichard Lowe HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 1976*5d9d9091SRichard Lowe add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 1977*5d9d9091SRichard Lowe ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 1978*5d9d9091SRichard Lowe add tmp2, CLONGSIZE, tmp2 ;\ 1979*5d9d9091SRichard Lowe ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 1980*5d9d9091SRichard Lowe xor tmp1, hmeblktag, tmp1 ;\ 1981*5d9d9091SRichard Lowe xor tmp2, hatid, tmp2 ;\ 1982*5d9d9091SRichard Lowe brz,pn tmp2, label##3 /* branch on hit */ ;\ 1983*5d9d9091SRichard Lowe add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 1984*5d9d9091SRichard Lowelabel##2: ;\ 1985*5d9d9091SRichard Lowe ba,pt %xcc, label##1 ;\ 1986*5d9d9091SRichard Lowe ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 1987*5d9d9091SRichard Lowelabel##3: ;\ 1988*5d9d9091SRichard Lowe cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 1989*5d9d9091SRichard Lowe bgeu,pt %xcc, label##2 ;\ 1990*5d9d9091SRichard Lowe add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 1991*5d9d9091SRichard Lowe and tmp1, BT_ULMASK, tmp2 ;\ 1992*5d9d9091SRichard Lowe srlx tmp1, BT_ULSHIFT, tmp1 ;\ 1993*5d9d9091SRichard Lowe sllx tmp1, CLONGSHIFT, tmp1 ;\ 1994*5d9d9091SRichard Lowe add tsbarea, tmp1, tmp1 ;\ 1995*5d9d9091SRichard Lowe ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 1996*5d9d9091SRichard Lowe srlx tmp1, tmp2, tmp1 ;\ 1997*5d9d9091SRichard Lowe btst 0x1, tmp1 ;\ 1998*5d9d9091SRichard Lowe bz,pn %xcc, label##2 ;\ 1999*5d9d9091SRichard Lowe add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 2000*5d9d9091SRichard Lowelabel##4: 2001*5d9d9091SRichard Lowe 2002*5d9d9091SRichard Lowe#if ((1 << SFHME_SHIFT) != SFHME_SIZE) 2003*5d9d9091SRichard Lowe#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 2004*5d9d9091SRichard Lowe#endif 2005*5d9d9091SRichard Lowe 2006*5d9d9091SRichard Lowe/* 2007*5d9d9091SRichard Lowe * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2008*5d9d9091SRichard Lowe * the offset for the corresponding hment. 2009*5d9d9091SRichard Lowe * Parameters: 2010*5d9d9091SRichard Lowe * In: 2011*5d9d9091SRichard Lowe * vaddr = register with virtual address 2012*5d9d9091SRichard Lowe * hmeblkpa = physical pointer to hme_blk 2013*5d9d9091SRichard Lowe * Out: 2014*5d9d9091SRichard Lowe * hmentoff = register where hment offset will be stored 2015*5d9d9091SRichard Lowe * hmemisc = hblk_misc 2016*5d9d9091SRichard Lowe * Scratch: 2017*5d9d9091SRichard Lowe * tmp1 2018*5d9d9091SRichard Lowe */ 2019*5d9d9091SRichard Lowe#define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2020*5d9d9091SRichard Lowe add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2021*5d9d9091SRichard Lowe lda [hmentoff]ASI_MEM, hmemisc ;\ 2022*5d9d9091SRichard Lowe andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2023*5d9d9091SRichard Lowe bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2024*5d9d9091SRichard Lowe or %g0, HMEBLK_HME1, hmentoff ;\ 2025*5d9d9091SRichard Lowe srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2026*5d9d9091SRichard Lowe and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2027*5d9d9091SRichard Lowe sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2028*5d9d9091SRichard Lowe add tmp1, HMEBLK_HME1, hmentoff ;\ 2029*5d9d9091SRichard Lowelabel1: 2030*5d9d9091SRichard Lowe 2031*5d9d9091SRichard Lowe/* 2032*5d9d9091SRichard Lowe * GET_TTE is a macro that returns a TTE given a tag and hatid. 2033*5d9d9091SRichard Lowe * 2034*5d9d9091SRichard Lowe * tagacc = (pseudo-)tag access register (in) 2035*5d9d9091SRichard Lowe * hatid = sfmmu pointer for TSB miss (in) 2036*5d9d9091SRichard Lowe * tte = tte for TLB miss if found, otherwise clobbered (out) 2037*5d9d9091SRichard Lowe * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2038*5d9d9091SRichard Lowe * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2039*5d9d9091SRichard Lowe * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2040*5d9d9091SRichard Lowe * hmeshift = constant/register to shift VA to obtain the virtual pfn 2041*5d9d9091SRichard Lowe * for this page size. 2042*5d9d9091SRichard Lowe * hashno = constant/register hash number 2043*5d9d9091SRichard Lowe * tmp = temp value - clobbered 2044*5d9d9091SRichard Lowe * label = temporary label for branching within macro. 2045*5d9d9091SRichard Lowe * foundlabel = label to jump to when tte is found. 2046*5d9d9091SRichard Lowe * suspendlabel= label to jump to when tte is suspended. 2047*5d9d9091SRichard Lowe * exitlabel = label to jump to when tte is not found. 2048*5d9d9091SRichard Lowe * 2049*5d9d9091SRichard Lowe */ 2050*5d9d9091SRichard Lowe#define GET_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, hmeshift, \ 2051*5d9d9091SRichard Lowe hashno, tmp, label, foundlabel, suspendlabel, exitlabel) \ 2052*5d9d9091SRichard Lowe ;\ 2053*5d9d9091SRichard Lowe stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2054*5d9d9091SRichard Lowe stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2055*5d9d9091SRichard Lowe HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2056*5d9d9091SRichard Lowe hmeblkpa, label##5, hmemisc, tmp) ;\ 2057*5d9d9091SRichard Lowe ;\ 2058*5d9d9091SRichard Lowe /* ;\ 2059*5d9d9091SRichard Lowe * tagacc = tagacc ;\ 2060*5d9d9091SRichard Lowe * hatid = hatid ;\ 2061*5d9d9091SRichard Lowe * tsbarea = tsbarea ;\ 2062*5d9d9091SRichard Lowe * tte = hmebp (hme bucket pointer) ;\ 2063*5d9d9091SRichard Lowe * hmeblkpa = vapg (virtual page) ;\ 2064*5d9d9091SRichard Lowe * hmemisc, tmp = scratch ;\ 2065*5d9d9091SRichard Lowe */ ;\ 2066*5d9d9091SRichard Lowe MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2067*5d9d9091SRichard Lowe or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2068*5d9d9091SRichard Lowe ;\ 2069*5d9d9091SRichard Lowe /* ;\ 2070*5d9d9091SRichard Lowe * tagacc = tagacc ;\ 2071*5d9d9091SRichard Lowe * hatid = hatid ;\ 2072*5d9d9091SRichard Lowe * tte = hmebp ;\ 2073*5d9d9091SRichard Lowe * hmeblkpa = CLOBBERED ;\ 2074*5d9d9091SRichard Lowe * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2075*5d9d9091SRichard Lowe * tmp = scratch ;\ 2076*5d9d9091SRichard Lowe */ ;\ 2077*5d9d9091SRichard Lowe stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2078*5d9d9091SRichard Lowe HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, \ 2079*5d9d9091SRichard Lowe tsbarea, tagacc, label##1) ;\ 2080*5d9d9091SRichard Lowe /* ;\ 2081*5d9d9091SRichard Lowe * tagacc = CLOBBERED ;\ 2082*5d9d9091SRichard Lowe * tte = CLOBBERED ;\ 2083*5d9d9091SRichard Lowe * hmeblkpa = hmeblkpa ;\ 2084*5d9d9091SRichard Lowe * tmp = scratch ;\ 2085*5d9d9091SRichard Lowe */ ;\ 2086*5d9d9091SRichard Lowe cmp hmeblkpa, HMEBLK_ENDPA ;\ 2087*5d9d9091SRichard Lowe bne,pn %xcc, label##4 /* branch if hmeblk found */ ;\ 2088*5d9d9091SRichard Lowe ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2089*5d9d9091SRichard Lowe ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2090*5d9d9091SRichard Lowe nop ;\ 2091*5d9d9091SRichard Lowelabel##4: ;\ 2092*5d9d9091SRichard Lowe /* ;\ 2093*5d9d9091SRichard Lowe * We have found the hmeblk containing the hment. ;\ 2094*5d9d9091SRichard Lowe * Now we calculate the corresponding tte. ;\ 2095*5d9d9091SRichard Lowe * ;\ 2096*5d9d9091SRichard Lowe * tagacc = tagacc ;\ 2097*5d9d9091SRichard Lowe * hatid = hatid ;\ 2098*5d9d9091SRichard Lowe * tte = clobbered ;\ 2099*5d9d9091SRichard Lowe * hmeblkpa = hmeblkpa ;\ 2100*5d9d9091SRichard Lowe * hmemisc = hblktag ;\ 2101*5d9d9091SRichard Lowe * tmp = scratch ;\ 2102*5d9d9091SRichard Lowe */ ;\ 2103*5d9d9091SRichard Lowe HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2104*5d9d9091SRichard Lowe label##2) ;\ 2105*5d9d9091SRichard Lowe ;\ 2106*5d9d9091SRichard Lowe /* ;\ 2107*5d9d9091SRichard Lowe * tagacc = tagacc ;\ 2108*5d9d9091SRichard Lowe * hatid = hmentoff ;\ 2109*5d9d9091SRichard Lowe * tte = clobbered ;\ 2110*5d9d9091SRichard Lowe * hmeblkpa = hmeblkpa ;\ 2111*5d9d9091SRichard Lowe * hmemisc = hblk_misc ;\ 2112*5d9d9091SRichard Lowe * tmp = scratch ;\ 2113*5d9d9091SRichard Lowe */ ;\ 2114*5d9d9091SRichard Lowe ;\ 2115*5d9d9091SRichard Lowe add hatid, SFHME_TTE, hatid ;\ 2116*5d9d9091SRichard Lowe add hmeblkpa, hatid, hmeblkpa ;\ 2117*5d9d9091SRichard Lowe ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2118*5d9d9091SRichard Lowe ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2119*5d9d9091SRichard Lowe set TTE_SUSPEND, hatid ;\ 2120*5d9d9091SRichard Lowe TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2121*5d9d9091SRichard Lowe btst tte, hatid ;\ 2122*5d9d9091SRichard Lowe bz,pt %xcc, foundlabel ;\ 2123*5d9d9091SRichard Lowe ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2124*5d9d9091SRichard Lowe ;\ 2125*5d9d9091SRichard Lowe /* ;\ 2126*5d9d9091SRichard Lowe * Mapping is suspended, so goto suspend label. ;\ 2127*5d9d9091SRichard Lowe */ ;\ 2128*5d9d9091SRichard Lowe ba,pt %xcc, suspendlabel ;\ 2129*5d9d9091SRichard Lowe nop 2130*5d9d9091SRichard Lowe 2131*5d9d9091SRichard Lowe/* 2132*5d9d9091SRichard Lowe * GET_SHME_TTE is similar to GET_TTE() except it searches 2133*5d9d9091SRichard Lowe * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2134*5d9d9091SRichard Lowe * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2135*5d9d9091SRichard Lowe * either 0 (not part of scd) or 1 (part of scd). 2136*5d9d9091SRichard Lowe */ 2137*5d9d9091SRichard Lowe#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, \ 2138*5d9d9091SRichard Lowe hmeshift, hashno, tmp, label, foundlabel, \ 2139*5d9d9091SRichard Lowe suspendlabel, exitlabel) \ 2140*5d9d9091SRichard Lowe ;\ 2141*5d9d9091SRichard Lowe stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2142*5d9d9091SRichard Lowe stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2143*5d9d9091SRichard Lowe HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2144*5d9d9091SRichard Lowe hmeblkpa, label##5, hmemisc, tmp) ;\ 2145*5d9d9091SRichard Lowe ;\ 2146*5d9d9091SRichard Lowe /* ;\ 2147*5d9d9091SRichard Lowe * tagacc = tagacc ;\ 2148*5d9d9091SRichard Lowe * hatid = hatid ;\ 2149*5d9d9091SRichard Lowe * tsbarea = tsbarea ;\ 2150*5d9d9091SRichard Lowe * tte = hmebp (hme bucket pointer) ;\ 2151*5d9d9091SRichard Lowe * hmeblkpa = vapg (virtual page) ;\ 2152*5d9d9091SRichard Lowe * hmemisc, tmp = scratch ;\ 2153*5d9d9091SRichard Lowe */ ;\ 2154*5d9d9091SRichard Lowe MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2155*5d9d9091SRichard Lowe ;\ 2156*5d9d9091SRichard Lowe /* ;\ 2157*5d9d9091SRichard Lowe * tagacc = tagacc ;\ 2158*5d9d9091SRichard Lowe * hatid = hatid ;\ 2159*5d9d9091SRichard Lowe * tsbarea = tsbarea ;\ 2160*5d9d9091SRichard Lowe * tte = hmebp ;\ 2161*5d9d9091SRichard Lowe * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2162*5d9d9091SRichard Lowe * hmeblkpa = CLOBBERED ;\ 2163*5d9d9091SRichard Lowe * tmp = scratch ;\ 2164*5d9d9091SRichard Lowe */ ;\ 2165*5d9d9091SRichard Lowe stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2166*5d9d9091SRichard Lowe ;\ 2167*5d9d9091SRichard Lowe add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2168*5d9d9091SRichard Lowe ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2169*5d9d9091SRichard Lowe HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2170*5d9d9091SRichard Lowe ;\ 2171*5d9d9091SRichard Lowelabel##8: ;\ 2172*5d9d9091SRichard Lowe HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, \ 2173*5d9d9091SRichard Lowe tsbarea, tagacc, tte, label##1) ;\ 2174*5d9d9091SRichard Lowe /* ;\ 2175*5d9d9091SRichard Lowe * tagacc = CLOBBERED ;\ 2176*5d9d9091SRichard Lowe * tte = CLOBBERED ;\ 2177*5d9d9091SRichard Lowe * hmeblkpa = hmeblkpa ;\ 2178*5d9d9091SRichard Lowe * tmp = scratch ;\ 2179*5d9d9091SRichard Lowe */ ;\ 2180*5d9d9091SRichard Lowe cmp hmeblkpa, HMEBLK_ENDPA ;\ 2181*5d9d9091SRichard Lowe bne,pn %xcc, label##4 /* branch if hmeblk found */ ;\ 2182*5d9d9091SRichard Lowe ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2183*5d9d9091SRichard Lowe ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2184*5d9d9091SRichard Lowe nop ;\ 2185*5d9d9091SRichard Lowelabel##4: ;\ 2186*5d9d9091SRichard Lowe /* ;\ 2187*5d9d9091SRichard Lowe * We have found the hmeblk containing the hment. ;\ 2188*5d9d9091SRichard Lowe * Now we calculate the corresponding tte. ;\ 2189*5d9d9091SRichard Lowe * ;\ 2190*5d9d9091SRichard Lowe * tagacc = tagacc ;\ 2191*5d9d9091SRichard Lowe * hatid = hatid ;\ 2192*5d9d9091SRichard Lowe * tte = clobbered ;\ 2193*5d9d9091SRichard Lowe * hmeblkpa = hmeblkpa ;\ 2194*5d9d9091SRichard Lowe * hmemisc = hblktag ;\ 2195*5d9d9091SRichard Lowe * tsbarea = tsbmiss area ;\ 2196*5d9d9091SRichard Lowe * tmp = scratch ;\ 2197*5d9d9091SRichard Lowe */ ;\ 2198*5d9d9091SRichard Lowe HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2199*5d9d9091SRichard Lowe label##2) ;\ 2200*5d9d9091SRichard Lowe ;\ 2201*5d9d9091SRichard Lowe /* ;\ 2202*5d9d9091SRichard Lowe * tagacc = tagacc ;\ 2203*5d9d9091SRichard Lowe * hatid = hmentoff ;\ 2204*5d9d9091SRichard Lowe * tte = clobbered ;\ 2205*5d9d9091SRichard Lowe * hmeblkpa = hmeblkpa ;\ 2206*5d9d9091SRichard Lowe * hmemisc = hblk_misc ;\ 2207*5d9d9091SRichard Lowe * tsbarea = tsbmiss area ;\ 2208*5d9d9091SRichard Lowe * tmp = scratch ;\ 2209*5d9d9091SRichard Lowe */ ;\ 2210*5d9d9091SRichard Lowe ;\ 2211*5d9d9091SRichard Lowe add hatid, SFHME_TTE, hatid ;\ 2212*5d9d9091SRichard Lowe add hmeblkpa, hatid, hmeblkpa ;\ 2213*5d9d9091SRichard Lowe ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2214*5d9d9091SRichard Lowe brlz,pt tte, label##6 ;\ 2215*5d9d9091SRichard Lowe nop ;\ 2216*5d9d9091SRichard Lowe btst HBLK_SZMASK, hmemisc ;\ 2217*5d9d9091SRichard Lowe bnz,a,pt %icc, label##7 ;\ 2218*5d9d9091SRichard Lowe ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2219*5d9d9091SRichard Lowe ;\ 2220*5d9d9091SRichard Lowe /* ;\ 2221*5d9d9091SRichard Lowe * We found an invalid 8K tte in shme. ;\ 2222*5d9d9091SRichard Lowe * it may not belong to shme's region since ;\ 2223*5d9d9091SRichard Lowe * region size/alignment granularity is 8K but different ;\ 2224*5d9d9091SRichard Lowe * regions don't share hmeblks. Continue the search. ;\ 2225*5d9d9091SRichard Lowe */ ;\ 2226*5d9d9091SRichard Lowe sub hmeblkpa, hatid, hmeblkpa ;\ 2227*5d9d9091SRichard Lowe ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2228*5d9d9091SRichard Lowe srlx tagacc, hmeshift, tte ;\ 2229*5d9d9091SRichard Lowe add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2230*5d9d9091SRichard Lowe ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2231*5d9d9091SRichard Lowe MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2232*5d9d9091SRichard Lowe ba,a,pt %xcc, label##8 ;\ 2233*5d9d9091SRichard Lowelabel##6: ;\ 2234*5d9d9091SRichard Lowe GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2235*5d9d9091SRichard Lowe ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2236*5d9d9091SRichard Lowelabel##7: ;\ 2237*5d9d9091SRichard Lowe set TTE_SUSPEND, hatid ;\ 2238*5d9d9091SRichard Lowe TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2239*5d9d9091SRichard Lowe btst tte, hatid ;\ 2240*5d9d9091SRichard Lowe bz,pt %xcc, foundlabel ;\ 2241*5d9d9091SRichard Lowe ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2242*5d9d9091SRichard Lowe ;\ 2243*5d9d9091SRichard Lowe /* ;\ 2244*5d9d9091SRichard Lowe * Mapping is suspended, so goto suspend label. ;\ 2245*5d9d9091SRichard Lowe */ ;\ 2246*5d9d9091SRichard Lowe ba,pt %xcc, suspendlabel ;\ 2247*5d9d9091SRichard Lowe nop 2248*5d9d9091SRichard Lowe 2249*5d9d9091SRichard Lowe /* 2250*5d9d9091SRichard Lowe * KERNEL PROTECTION HANDLER 2251*5d9d9091SRichard Lowe * 2252*5d9d9091SRichard Lowe * g1 = tsb8k pointer register (clobbered) 2253*5d9d9091SRichard Lowe * g2 = tag access register (ro) 2254*5d9d9091SRichard Lowe * g3 - g7 = scratch registers 2255*5d9d9091SRichard Lowe * 2256*5d9d9091SRichard Lowe * Note: This function is patched at runtime for performance reasons. 2257*5d9d9091SRichard Lowe * Any changes here require sfmmu_patch_ktsb fixed. 2258*5d9d9091SRichard Lowe */ 2259*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_kprot_trap) 2260*5d9d9091SRichard Lowe mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2261*5d9d9091SRichard Lowesfmmu_kprot_patch_ktsb_base: 2262*5d9d9091SRichard Lowe RUNTIME_PATCH_SETX(%g1, %g6) 2263*5d9d9091SRichard Lowe /* %g1 = contents of ktsb_base or ktsb_pbase */ 2264*5d9d9091SRichard Lowesfmmu_kprot_patch_ktsb_szcode: 2265*5d9d9091SRichard Lowe or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2266*5d9d9091SRichard Lowe 2267*5d9d9091SRichard Lowe GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2268*5d9d9091SRichard Lowe ! %g1 = First TSB entry pointer, as TSB miss handler expects 2269*5d9d9091SRichard Lowe 2270*5d9d9091SRichard Lowe mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2271*5d9d9091SRichard Lowesfmmu_kprot_patch_ktsb4m_base: 2272*5d9d9091SRichard Lowe RUNTIME_PATCH_SETX(%g3, %g6) 2273*5d9d9091SRichard Lowe /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2274*5d9d9091SRichard Lowesfmmu_kprot_patch_ktsb4m_szcode: 2275*5d9d9091SRichard Lowe or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2276*5d9d9091SRichard Lowe 2277*5d9d9091SRichard Lowe GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2278*5d9d9091SRichard Lowe ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2279*5d9d9091SRichard Lowe 2280*5d9d9091SRichard Lowe CPU_TSBMISS_AREA(%g6, %g7) 2281*5d9d9091SRichard Lowe HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2282*5d9d9091SRichard Lowe ba,pt %xcc, sfmmu_tsb_miss_tt 2283*5d9d9091SRichard Lowe nop 2284*5d9d9091SRichard Lowe 2285*5d9d9091SRichard Lowe /* 2286*5d9d9091SRichard Lowe * USER PROTECTION HANDLER 2287*5d9d9091SRichard Lowe * 2288*5d9d9091SRichard Lowe * g1 = tsb8k pointer register (ro) 2289*5d9d9091SRichard Lowe * g2 = tag access register (ro) 2290*5d9d9091SRichard Lowe * g3 = faulting context (clobbered, currently not used) 2291*5d9d9091SRichard Lowe * g4 - g7 = scratch registers 2292*5d9d9091SRichard Lowe */ 2293*5d9d9091SRichard Lowe ALTENTRY(sfmmu_uprot_trap) 2294*5d9d9091SRichard Lowe#ifdef sun4v 2295*5d9d9091SRichard Lowe GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2296*5d9d9091SRichard Lowe /* %g1 = first TSB entry ptr now, %g2 preserved */ 2297*5d9d9091SRichard Lowe 2298*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2299*5d9d9091SRichard Lowe brlz,pt %g3, 9f /* check for 2nd TSB */ 2300*5d9d9091SRichard Lowe nop 2301*5d9d9091SRichard Lowe 2302*5d9d9091SRichard Lowe GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2303*5d9d9091SRichard Lowe /* %g3 = second TSB entry ptr now, %g2 preserved */ 2304*5d9d9091SRichard Lowe 2305*5d9d9091SRichard Lowe#else /* sun4v */ 2306*5d9d9091SRichard Lowe#ifdef UTSB_PHYS 2307*5d9d9091SRichard Lowe /* g1 = first TSB entry ptr */ 2308*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2309*5d9d9091SRichard Lowe brlz,pt %g3, 9f /* check for 2nd TSB */ 2310*5d9d9091SRichard Lowe nop 2311*5d9d9091SRichard Lowe 2312*5d9d9091SRichard Lowe GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2313*5d9d9091SRichard Lowe /* %g3 = second TSB entry ptr now, %g2 preserved */ 2314*5d9d9091SRichard Lowe#else /* UTSB_PHYS */ 2315*5d9d9091SRichard Lowe brgez,pt %g1, 9f /* check for 2nd TSB */ 2316*5d9d9091SRichard Lowe mov -1, %g3 /* set second tsbe ptr to -1 */ 2317*5d9d9091SRichard Lowe 2318*5d9d9091SRichard Lowe mov %g2, %g7 2319*5d9d9091SRichard Lowe GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2320*5d9d9091SRichard Lowe /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2321*5d9d9091SRichard Lowe mov %g1, %g7 2322*5d9d9091SRichard Lowe GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2323*5d9d9091SRichard Lowe#endif /* UTSB_PHYS */ 2324*5d9d9091SRichard Lowe#endif /* sun4v */ 2325*5d9d9091SRichard Lowe9: 2326*5d9d9091SRichard Lowe CPU_TSBMISS_AREA(%g6, %g7) 2327*5d9d9091SRichard Lowe HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2328*5d9d9091SRichard Lowe ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2329*5d9d9091SRichard Lowe nop 2330*5d9d9091SRichard Lowe 2331*5d9d9091SRichard Lowe /* 2332*5d9d9091SRichard Lowe * Kernel 8K page iTLB miss. We also get here if we took a 2333*5d9d9091SRichard Lowe * fast instruction access mmu miss trap while running in 2334*5d9d9091SRichard Lowe * invalid context. 2335*5d9d9091SRichard Lowe * 2336*5d9d9091SRichard Lowe * %g1 = 8K TSB pointer register (not used, clobbered) 2337*5d9d9091SRichard Lowe * %g2 = tag access register (used) 2338*5d9d9091SRichard Lowe * %g3 = faulting context id (used) 2339*5d9d9091SRichard Lowe * %g7 = TSB tag to match (used) 2340*5d9d9091SRichard Lowe */ 2341*5d9d9091SRichard Lowe .align 64 2342*5d9d9091SRichard Lowe ALTENTRY(sfmmu_kitlb_miss) 2343*5d9d9091SRichard Lowe brnz,pn %g3, tsb_tl0_noctxt 2344*5d9d9091SRichard Lowe nop 2345*5d9d9091SRichard Lowe 2346*5d9d9091SRichard Lowe /* kernel miss */ 2347*5d9d9091SRichard Lowe /* get kernel tsb pointer */ 2348*5d9d9091SRichard Lowe /* we patch the next set of instructions at run time */ 2349*5d9d9091SRichard Lowe /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2350*5d9d9091SRichard Loweiktsbbase: 2351*5d9d9091SRichard Lowe RUNTIME_PATCH_SETX(%g4, %g5) 2352*5d9d9091SRichard Lowe /* %g4 = contents of ktsb_base or ktsb_pbase */ 2353*5d9d9091SRichard Lowe 2354*5d9d9091SRichard Loweiktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2355*5d9d9091SRichard Lowe srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2356*5d9d9091SRichard Lowe or %g4, %g1, %g1 ! form tsb ptr 2357*5d9d9091SRichard Lowe ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2358*5d9d9091SRichard Lowe cmp %g4, %g7 2359*5d9d9091SRichard Lowe bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2360*5d9d9091SRichard Lowe srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2361*5d9d9091SRichard Lowe 2362*5d9d9091SRichard Lowe andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2363*5d9d9091SRichard Lowe bz,pn %icc, exec_fault 2364*5d9d9091SRichard Lowe nop 2365*5d9d9091SRichard Lowe TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2366*5d9d9091SRichard Lowe ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2367*5d9d9091SRichard Lowe retry 2368*5d9d9091SRichard Lowe 2369*5d9d9091SRichard Loweiktsb4mbase: 2370*5d9d9091SRichard Lowe RUNTIME_PATCH_SETX(%g4, %g6) 2371*5d9d9091SRichard Lowe /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2372*5d9d9091SRichard Loweiktsb4m: 2373*5d9d9091SRichard Lowe sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2374*5d9d9091SRichard Lowe srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2375*5d9d9091SRichard Lowe add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2376*5d9d9091SRichard Lowe ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2377*5d9d9091SRichard Lowe cmp %g4, %g7 2378*5d9d9091SRichard Lowe bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2379*5d9d9091SRichard Lowe andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2380*5d9d9091SRichard Lowe bz,pn %icc, exec_fault 2381*5d9d9091SRichard Lowe nop 2382*5d9d9091SRichard Lowe TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2383*5d9d9091SRichard Lowe ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2384*5d9d9091SRichard Lowe retry 2385*5d9d9091SRichard Lowe 2386*5d9d9091SRichard Lowe /* 2387*5d9d9091SRichard Lowe * Kernel dTLB miss. We also get here if we took a fast data 2388*5d9d9091SRichard Lowe * access mmu miss trap while running in invalid context. 2389*5d9d9091SRichard Lowe * 2390*5d9d9091SRichard Lowe * Note: for now we store kpm TTEs in the kernel TSB as usual. 2391*5d9d9091SRichard Lowe * We select the TSB miss handler to branch to depending on 2392*5d9d9091SRichard Lowe * the virtual address of the access. In the future it may 2393*5d9d9091SRichard Lowe * be desirable to separate kpm TTEs into their own TSB, 2394*5d9d9091SRichard Lowe * in which case all that needs to be done is to set 2395*5d9d9091SRichard Lowe * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2396*5d9d9091SRichard Lowe * early in the miss if we detect a kpm VA to a new handler. 2397*5d9d9091SRichard Lowe * 2398*5d9d9091SRichard Lowe * %g1 = 8K TSB pointer register (not used, clobbered) 2399*5d9d9091SRichard Lowe * %g2 = tag access register (used) 2400*5d9d9091SRichard Lowe * %g3 = faulting context id (used) 2401*5d9d9091SRichard Lowe */ 2402*5d9d9091SRichard Lowe .align 64 2403*5d9d9091SRichard Lowe ALTENTRY(sfmmu_kdtlb_miss) 2404*5d9d9091SRichard Lowe brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2405*5d9d9091SRichard Lowe nop 2406*5d9d9091SRichard Lowe 2407*5d9d9091SRichard Lowe /* Gather some stats for kpm misses in the TLB. */ 2408*5d9d9091SRichard Lowe /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2409*5d9d9091SRichard Lowe KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2410*5d9d9091SRichard Lowe 2411*5d9d9091SRichard Lowe /* 2412*5d9d9091SRichard Lowe * Get first TSB offset and look for 8K/64K/512K mapping 2413*5d9d9091SRichard Lowe * using the 8K virtual page as the index. 2414*5d9d9091SRichard Lowe * 2415*5d9d9091SRichard Lowe * We patch the next set of instructions at run time; 2416*5d9d9091SRichard Lowe * any changes here require sfmmu_patch_ktsb changes too. 2417*5d9d9091SRichard Lowe */ 2418*5d9d9091SRichard Lowedktsbbase: 2419*5d9d9091SRichard Lowe RUNTIME_PATCH_SETX(%g7, %g6) 2420*5d9d9091SRichard Lowe /* %g7 = contents of ktsb_base or ktsb_pbase */ 2421*5d9d9091SRichard Lowe 2422*5d9d9091SRichard Lowedktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2423*5d9d9091SRichard Lowe srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2424*5d9d9091SRichard Lowe 2425*5d9d9091SRichard Lowe /* 2426*5d9d9091SRichard Lowe * At this point %g1 is our index into the TSB. 2427*5d9d9091SRichard Lowe * We just masked off enough bits of the VA depending 2428*5d9d9091SRichard Lowe * on our TSB size code. 2429*5d9d9091SRichard Lowe */ 2430*5d9d9091SRichard Lowe ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2431*5d9d9091SRichard Lowe srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2432*5d9d9091SRichard Lowe cmp %g6, %g4 ! compare tag 2433*5d9d9091SRichard Lowe bne,pn %xcc, dktsb4m_kpmcheck_small 2434*5d9d9091SRichard Lowe add %g7, %g1, %g1 /* form tsb ptr */ 2435*5d9d9091SRichard Lowe TT_TRACE(trace_tsbhit) 2436*5d9d9091SRichard Lowe DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2437*5d9d9091SRichard Lowe /* trapstat expects tte in %g5 */ 2438*5d9d9091SRichard Lowe retry 2439*5d9d9091SRichard Lowe 2440*5d9d9091SRichard Lowe /* 2441*5d9d9091SRichard Lowe * If kpm is using large pages, the following instruction needs 2442*5d9d9091SRichard Lowe * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2443*5d9d9091SRichard Lowe * so that we will probe the 4M TSB regardless of the VA. In 2444*5d9d9091SRichard Lowe * the case kpm is using small pages, we know no large kernel 2445*5d9d9091SRichard Lowe * mappings are located above 0x80000000.00000000 so we skip the 2446*5d9d9091SRichard Lowe * probe as an optimization. 2447*5d9d9091SRichard Lowe */ 2448*5d9d9091SRichard Lowedktsb4m_kpmcheck_small: 2449*5d9d9091SRichard Lowe brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2450*5d9d9091SRichard Lowe /* delay slot safe, below */ 2451*5d9d9091SRichard Lowe 2452*5d9d9091SRichard Lowe /* 2453*5d9d9091SRichard Lowe * Get second TSB offset and look for 4M mapping 2454*5d9d9091SRichard Lowe * using 4M virtual page as the TSB index. 2455*5d9d9091SRichard Lowe * 2456*5d9d9091SRichard Lowe * Here: 2457*5d9d9091SRichard Lowe * %g1 = 8K TSB pointer. Don't squash it. 2458*5d9d9091SRichard Lowe * %g2 = tag access register (we still need it) 2459*5d9d9091SRichard Lowe */ 2460*5d9d9091SRichard Lowe srlx %g2, MMU_PAGESHIFT4M, %g3 2461*5d9d9091SRichard Lowe 2462*5d9d9091SRichard Lowe /* 2463*5d9d9091SRichard Lowe * We patch the next set of instructions at run time; 2464*5d9d9091SRichard Lowe * any changes here require sfmmu_patch_ktsb changes too. 2465*5d9d9091SRichard Lowe */ 2466*5d9d9091SRichard Lowedktsb4mbase: 2467*5d9d9091SRichard Lowe RUNTIME_PATCH_SETX(%g7, %g6) 2468*5d9d9091SRichard Lowe /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2469*5d9d9091SRichard Lowedktsb4m: 2470*5d9d9091SRichard Lowe sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2471*5d9d9091SRichard Lowe srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2472*5d9d9091SRichard Lowe 2473*5d9d9091SRichard Lowe /* 2474*5d9d9091SRichard Lowe * At this point %g3 is our index into the TSB. 2475*5d9d9091SRichard Lowe * We just masked off enough bits of the VA depending 2476*5d9d9091SRichard Lowe * on our TSB size code. 2477*5d9d9091SRichard Lowe */ 2478*5d9d9091SRichard Lowe ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2479*5d9d9091SRichard Lowe srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2480*5d9d9091SRichard Lowe cmp %g6, %g4 ! compare tag 2481*5d9d9091SRichard Lowe 2482*5d9d9091SRichard Lowedktsb4m_tsbmiss: 2483*5d9d9091SRichard Lowe bne,pn %xcc, dktsb4m_kpmcheck 2484*5d9d9091SRichard Lowe add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2485*5d9d9091SRichard Lowe TT_TRACE(trace_tsbhit) 2486*5d9d9091SRichard Lowe /* we don't check TTE size here since we assume 4M TSB is separate */ 2487*5d9d9091SRichard Lowe DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2488*5d9d9091SRichard Lowe /* trapstat expects tte in %g5 */ 2489*5d9d9091SRichard Lowe retry 2490*5d9d9091SRichard Lowe 2491*5d9d9091SRichard Lowe /* 2492*5d9d9091SRichard Lowe * So, we failed to find a valid TTE to match the faulting 2493*5d9d9091SRichard Lowe * address in either TSB. There are a few cases that could land 2494*5d9d9091SRichard Lowe * us here: 2495*5d9d9091SRichard Lowe * 2496*5d9d9091SRichard Lowe * 1) This is a kernel VA below 0x80000000.00000000. We branch 2497*5d9d9091SRichard Lowe * to sfmmu_tsb_miss_tt to handle the miss. 2498*5d9d9091SRichard Lowe * 2) We missed on a kpm VA, and we didn't find the mapping in the 2499*5d9d9091SRichard Lowe * 4M TSB. Let segkpm handle it. 2500*5d9d9091SRichard Lowe * 2501*5d9d9091SRichard Lowe * Note that we shouldn't land here in the case of a kpm VA when 2502*5d9d9091SRichard Lowe * kpm_smallpages is active -- we handled that case earlier at 2503*5d9d9091SRichard Lowe * dktsb4m_kpmcheck_small. 2504*5d9d9091SRichard Lowe * 2505*5d9d9091SRichard Lowe * At this point: 2506*5d9d9091SRichard Lowe * g1 = 8K-indexed primary TSB pointer 2507*5d9d9091SRichard Lowe * g2 = tag access register 2508*5d9d9091SRichard Lowe * g3 = 4M-indexed secondary TSB pointer 2509*5d9d9091SRichard Lowe */ 2510*5d9d9091SRichard Lowedktsb4m_kpmcheck: 2511*5d9d9091SRichard Lowe cmp %g2, %g0 2512*5d9d9091SRichard Lowe bl,pn %xcc, sfmmu_kpm_dtsb_miss 2513*5d9d9091SRichard Lowe nop 2514*5d9d9091SRichard Lowe ba,a,pt %icc, sfmmu_tsb_miss_tt 2515*5d9d9091SRichard Lowe nop 2516*5d9d9091SRichard Lowe 2517*5d9d9091SRichard Lowe#ifdef sun4v 2518*5d9d9091SRichard Lowe /* 2519*5d9d9091SRichard Lowe * User instruction miss w/ single TSB. 2520*5d9d9091SRichard Lowe * The first probe covers 8K, 64K, and 512K page sizes, 2521*5d9d9091SRichard Lowe * because 64K and 512K mappings are replicated off 8K 2522*5d9d9091SRichard Lowe * pointer. 2523*5d9d9091SRichard Lowe * 2524*5d9d9091SRichard Lowe * g1 = tsb8k pointer register 2525*5d9d9091SRichard Lowe * g2 = tag access register 2526*5d9d9091SRichard Lowe * g3 - g6 = scratch registers 2527*5d9d9091SRichard Lowe * g7 = TSB tag to match 2528*5d9d9091SRichard Lowe */ 2529*5d9d9091SRichard Lowe .align 64 2530*5d9d9091SRichard Lowe ALTENTRY(sfmmu_uitlb_fastpath) 2531*5d9d9091SRichard Lowe 2532*5d9d9091SRichard Lowe PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2533*5d9d9091SRichard Lowe /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2534*5d9d9091SRichard Lowe ba,pn %xcc, sfmmu_tsb_miss_tt 2535*5d9d9091SRichard Lowe mov -1, %g3 2536*5d9d9091SRichard Lowe 2537*5d9d9091SRichard Lowe /* 2538*5d9d9091SRichard Lowe * User data miss w/ single TSB. 2539*5d9d9091SRichard Lowe * The first probe covers 8K, 64K, and 512K page sizes, 2540*5d9d9091SRichard Lowe * because 64K and 512K mappings are replicated off 8K 2541*5d9d9091SRichard Lowe * pointer. 2542*5d9d9091SRichard Lowe * 2543*5d9d9091SRichard Lowe * g1 = tsb8k pointer register 2544*5d9d9091SRichard Lowe * g2 = tag access register 2545*5d9d9091SRichard Lowe * g3 - g6 = scratch registers 2546*5d9d9091SRichard Lowe * g7 = TSB tag to match 2547*5d9d9091SRichard Lowe */ 2548*5d9d9091SRichard Lowe .align 64 2549*5d9d9091SRichard Lowe ALTENTRY(sfmmu_udtlb_fastpath) 2550*5d9d9091SRichard Lowe 2551*5d9d9091SRichard Lowe PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2552*5d9d9091SRichard Lowe /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2553*5d9d9091SRichard Lowe ba,pn %xcc, sfmmu_tsb_miss_tt 2554*5d9d9091SRichard Lowe mov -1, %g3 2555*5d9d9091SRichard Lowe 2556*5d9d9091SRichard Lowe /* 2557*5d9d9091SRichard Lowe * User instruction miss w/ multiple TSBs (sun4v). 2558*5d9d9091SRichard Lowe * The first probe covers 8K, 64K, and 512K page sizes, 2559*5d9d9091SRichard Lowe * because 64K and 512K mappings are replicated off 8K 2560*5d9d9091SRichard Lowe * pointer. Second probe covers 4M page size only. 2561*5d9d9091SRichard Lowe * 2562*5d9d9091SRichard Lowe * Just like sfmmu_udtlb_slowpath, except: 2563*5d9d9091SRichard Lowe * o Uses ASI_ITLB_IN 2564*5d9d9091SRichard Lowe * o checks for execute permission 2565*5d9d9091SRichard Lowe * o No ISM prediction. 2566*5d9d9091SRichard Lowe * 2567*5d9d9091SRichard Lowe * g1 = tsb8k pointer register 2568*5d9d9091SRichard Lowe * g2 = tag access register 2569*5d9d9091SRichard Lowe * g3 - g6 = scratch registers 2570*5d9d9091SRichard Lowe * g7 = TSB tag to match 2571*5d9d9091SRichard Lowe */ 2572*5d9d9091SRichard Lowe .align 64 2573*5d9d9091SRichard Lowe ALTENTRY(sfmmu_uitlb_slowpath) 2574*5d9d9091SRichard Lowe 2575*5d9d9091SRichard Lowe GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2576*5d9d9091SRichard Lowe PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2577*5d9d9091SRichard Lowe /* g4 - g5 = clobbered here */ 2578*5d9d9091SRichard Lowe 2579*5d9d9091SRichard Lowe GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2580*5d9d9091SRichard Lowe /* g1 = first TSB pointer, g3 = second TSB pointer */ 2581*5d9d9091SRichard Lowe srlx %g2, TAG_VALO_SHIFT, %g7 2582*5d9d9091SRichard Lowe PROBE_2ND_ITSB(%g3, %g7) 2583*5d9d9091SRichard Lowe /* NOT REACHED */ 2584*5d9d9091SRichard Lowe 2585*5d9d9091SRichard Lowe#else /* sun4v */ 2586*5d9d9091SRichard Lowe 2587*5d9d9091SRichard Lowe /* 2588*5d9d9091SRichard Lowe * User instruction miss w/ multiple TSBs (sun4u). 2589*5d9d9091SRichard Lowe * The first probe covers 8K, 64K, and 512K page sizes, 2590*5d9d9091SRichard Lowe * because 64K and 512K mappings are replicated off 8K 2591*5d9d9091SRichard Lowe * pointer. Probe of 1st TSB has already been done prior to entry 2592*5d9d9091SRichard Lowe * into this routine. For the UTSB_PHYS case we probe up to 3 2593*5d9d9091SRichard Lowe * valid other TSBs in the following order: 2594*5d9d9091SRichard Lowe * 1) shared TSB for 4M-256M pages 2595*5d9d9091SRichard Lowe * 2) private TSB for 4M-256M pages 2596*5d9d9091SRichard Lowe * 3) shared TSB for 8K-512K pages 2597*5d9d9091SRichard Lowe * 2598*5d9d9091SRichard Lowe * For the non UTSB_PHYS case we probe the 2nd TSB here that backs 2599*5d9d9091SRichard Lowe * 4M-256M pages. 2600*5d9d9091SRichard Lowe * 2601*5d9d9091SRichard Lowe * Just like sfmmu_udtlb_slowpath, except: 2602*5d9d9091SRichard Lowe * o Uses ASI_ITLB_IN 2603*5d9d9091SRichard Lowe * o checks for execute permission 2604*5d9d9091SRichard Lowe * o No ISM prediction. 2605*5d9d9091SRichard Lowe * 2606*5d9d9091SRichard Lowe * g1 = tsb8k pointer register 2607*5d9d9091SRichard Lowe * g2 = tag access register 2608*5d9d9091SRichard Lowe * g4 - g6 = scratch registers 2609*5d9d9091SRichard Lowe * g7 = TSB tag to match 2610*5d9d9091SRichard Lowe */ 2611*5d9d9091SRichard Lowe .align 64 2612*5d9d9091SRichard Lowe ALTENTRY(sfmmu_uitlb_slowpath) 2613*5d9d9091SRichard Lowe 2614*5d9d9091SRichard Lowe#ifdef UTSB_PHYS 2615*5d9d9091SRichard Lowe 2616*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2617*5d9d9091SRichard Lowe brlz,pt %g6, 1f 2618*5d9d9091SRichard Lowe nop 2619*5d9d9091SRichard Lowe GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2620*5d9d9091SRichard Lowe PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail) 2621*5d9d9091SRichard Lowe1: 2622*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2623*5d9d9091SRichard Lowe brlz,pt %g3, 2f 2624*5d9d9091SRichard Lowe nop 2625*5d9d9091SRichard Lowe GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2626*5d9d9091SRichard Lowe PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail) 2627*5d9d9091SRichard Lowe2: 2628*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2629*5d9d9091SRichard Lowe brlz,pt %g6, sfmmu_tsb_miss_tt 2630*5d9d9091SRichard Lowe nop 2631*5d9d9091SRichard Lowe GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2632*5d9d9091SRichard Lowe PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail) 2633*5d9d9091SRichard Lowe ba,pn %xcc, sfmmu_tsb_miss_tt 2634*5d9d9091SRichard Lowe nop 2635*5d9d9091SRichard Lowe 2636*5d9d9091SRichard Lowe#else /* UTSB_PHYS */ 2637*5d9d9091SRichard Lowe mov %g1, %g3 /* save tsb8k reg in %g3 */ 2638*5d9d9091SRichard Lowe GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 2639*5d9d9091SRichard Lowe PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2640*5d9d9091SRichard Lowe mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 2641*5d9d9091SRichard Lowe mov %g3, %g7 /* copy tsb8k reg in %g7 */ 2642*5d9d9091SRichard Lowe GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 2643*5d9d9091SRichard Lowe /* g1 = first TSB pointer, g3 = second TSB pointer */ 2644*5d9d9091SRichard Lowe srlx %g2, TAG_VALO_SHIFT, %g7 2645*5d9d9091SRichard Lowe PROBE_2ND_ITSB(%g3, %g7, isynth) 2646*5d9d9091SRichard Lowe ba,pn %xcc, sfmmu_tsb_miss_tt 2647*5d9d9091SRichard Lowe nop 2648*5d9d9091SRichard Lowe 2649*5d9d9091SRichard Lowe#endif /* UTSB_PHYS */ 2650*5d9d9091SRichard Lowe#endif /* sun4v */ 2651*5d9d9091SRichard Lowe 2652*5d9d9091SRichard Lowe#if defined(sun4u) && defined(UTSB_PHYS) 2653*5d9d9091SRichard Lowe 2654*5d9d9091SRichard Lowe /* 2655*5d9d9091SRichard Lowe * We come here for ism predict DTLB_MISS case or if 2656*5d9d9091SRichard Lowe * if probe in first TSB failed. 2657*5d9d9091SRichard Lowe */ 2658*5d9d9091SRichard Lowe 2659*5d9d9091SRichard Lowe .align 64 2660*5d9d9091SRichard Lowe ALTENTRY(sfmmu_udtlb_slowpath_noismpred) 2661*5d9d9091SRichard Lowe 2662*5d9d9091SRichard Lowe /* 2663*5d9d9091SRichard Lowe * g1 = tsb8k pointer register 2664*5d9d9091SRichard Lowe * g2 = tag access register 2665*5d9d9091SRichard Lowe * g4 - %g6 = scratch registers 2666*5d9d9091SRichard Lowe * g7 = TSB tag to match 2667*5d9d9091SRichard Lowe */ 2668*5d9d9091SRichard Lowe 2669*5d9d9091SRichard Lowe /* 2670*5d9d9091SRichard Lowe * ISM non-predict probe order 2671*5d9d9091SRichard Lowe * probe 1ST_TSB (8K index) 2672*5d9d9091SRichard Lowe * probe 2ND_TSB (4M index) 2673*5d9d9091SRichard Lowe * probe 4TH_TSB (4M index) 2674*5d9d9091SRichard Lowe * probe 3RD_TSB (8K index) 2675*5d9d9091SRichard Lowe * 2676*5d9d9091SRichard Lowe * We already probed first TSB in DTLB_MISS handler. 2677*5d9d9091SRichard Lowe */ 2678*5d9d9091SRichard Lowe 2679*5d9d9091SRichard Lowe /* 2680*5d9d9091SRichard Lowe * Private 2ND TSB 4M-256 pages 2681*5d9d9091SRichard Lowe */ 2682*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2683*5d9d9091SRichard Lowe brlz,pt %g3, 1f 2684*5d9d9091SRichard Lowe nop 2685*5d9d9091SRichard Lowe GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2686*5d9d9091SRichard Lowe PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2687*5d9d9091SRichard Lowe 2688*5d9d9091SRichard Lowe /* 2689*5d9d9091SRichard Lowe * Shared Context 4TH TSB 4M-256 pages 2690*5d9d9091SRichard Lowe */ 2691*5d9d9091SRichard Lowe1: 2692*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2693*5d9d9091SRichard Lowe brlz,pt %g6, 2f 2694*5d9d9091SRichard Lowe nop 2695*5d9d9091SRichard Lowe GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2696*5d9d9091SRichard Lowe PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail) 2697*5d9d9091SRichard Lowe 2698*5d9d9091SRichard Lowe /* 2699*5d9d9091SRichard Lowe * Shared Context 3RD TSB 8K-512K pages 2700*5d9d9091SRichard Lowe */ 2701*5d9d9091SRichard Lowe2: 2702*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2703*5d9d9091SRichard Lowe brlz,pt %g6, sfmmu_tsb_miss_tt 2704*5d9d9091SRichard Lowe nop 2705*5d9d9091SRichard Lowe GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2706*5d9d9091SRichard Lowe PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail) 2707*5d9d9091SRichard Lowe ba,pn %xcc, sfmmu_tsb_miss_tt 2708*5d9d9091SRichard Lowe nop 2709*5d9d9091SRichard Lowe 2710*5d9d9091SRichard Lowe .align 64 2711*5d9d9091SRichard Lowe ALTENTRY(sfmmu_udtlb_slowpath_ismpred) 2712*5d9d9091SRichard Lowe 2713*5d9d9091SRichard Lowe /* 2714*5d9d9091SRichard Lowe * g1 = tsb8k pointer register 2715*5d9d9091SRichard Lowe * g2 = tag access register 2716*5d9d9091SRichard Lowe * g4 - g6 = scratch registers 2717*5d9d9091SRichard Lowe * g7 = TSB tag to match 2718*5d9d9091SRichard Lowe */ 2719*5d9d9091SRichard Lowe 2720*5d9d9091SRichard Lowe /* 2721*5d9d9091SRichard Lowe * ISM predict probe order 2722*5d9d9091SRichard Lowe * probe 4TH_TSB (4M index) 2723*5d9d9091SRichard Lowe * probe 2ND_TSB (4M index) 2724*5d9d9091SRichard Lowe * probe 1ST_TSB (8K index) 2725*5d9d9091SRichard Lowe * probe 3RD_TSB (8K index) 2726*5d9d9091SRichard Lowe 2727*5d9d9091SRichard Lowe /* 2728*5d9d9091SRichard Lowe * Shared Context 4TH TSB 4M-256 pages 2729*5d9d9091SRichard Lowe */ 2730*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2731*5d9d9091SRichard Lowe brlz,pt %g6, 4f 2732*5d9d9091SRichard Lowe nop 2733*5d9d9091SRichard Lowe GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2734*5d9d9091SRichard Lowe PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2) 2735*5d9d9091SRichard Lowe 2736*5d9d9091SRichard Lowe /* 2737*5d9d9091SRichard Lowe * Private 2ND TSB 4M-256 pages 2738*5d9d9091SRichard Lowe */ 2739*5d9d9091SRichard Lowe4: 2740*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2741*5d9d9091SRichard Lowe brlz,pt %g3, 5f 2742*5d9d9091SRichard Lowe nop 2743*5d9d9091SRichard Lowe GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2744*5d9d9091SRichard Lowe PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2) 2745*5d9d9091SRichard Lowe 2746*5d9d9091SRichard Lowe5: 2747*5d9d9091SRichard Lowe PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2) 2748*5d9d9091SRichard Lowe 2749*5d9d9091SRichard Lowe /* 2750*5d9d9091SRichard Lowe * Shared Context 3RD TSB 8K-512K pages 2751*5d9d9091SRichard Lowe */ 2752*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2753*5d9d9091SRichard Lowe brlz,pt %g6, 6f 2754*5d9d9091SRichard Lowe nop 2755*5d9d9091SRichard Lowe GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2756*5d9d9091SRichard Lowe PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2) 2757*5d9d9091SRichard Lowe6: 2758*5d9d9091SRichard Lowe ba,pn %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */ 2759*5d9d9091SRichard Lowe nop 2760*5d9d9091SRichard Lowe 2761*5d9d9091SRichard Lowe#else /* sun4u && UTSB_PHYS */ 2762*5d9d9091SRichard Lowe 2763*5d9d9091SRichard Lowe .align 64 2764*5d9d9091SRichard Lowe ALTENTRY(sfmmu_udtlb_slowpath) 2765*5d9d9091SRichard Lowe 2766*5d9d9091SRichard Lowe srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 2767*5d9d9091SRichard Lowe brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 2768*5d9d9091SRichard Lowe mov %g1, %g3 2769*5d9d9091SRichard Lowe 2770*5d9d9091SRichard Loweudtlb_miss_probefirst: 2771*5d9d9091SRichard Lowe /* 2772*5d9d9091SRichard Lowe * g1 = 8K TSB pointer register 2773*5d9d9091SRichard Lowe * g2 = tag access register 2774*5d9d9091SRichard Lowe * g3 = (potentially) second TSB entry ptr 2775*5d9d9091SRichard Lowe * g6 = ism pred. 2776*5d9d9091SRichard Lowe * g7 = vpg_4m 2777*5d9d9091SRichard Lowe */ 2778*5d9d9091SRichard Lowe#ifdef sun4v 2779*5d9d9091SRichard Lowe GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2780*5d9d9091SRichard Lowe PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2781*5d9d9091SRichard Lowe 2782*5d9d9091SRichard Lowe /* 2783*5d9d9091SRichard Lowe * Here: 2784*5d9d9091SRichard Lowe * g1 = first TSB pointer 2785*5d9d9091SRichard Lowe * g2 = tag access reg 2786*5d9d9091SRichard Lowe * g3 = second TSB ptr IFF ISM pred. (else don't care) 2787*5d9d9091SRichard Lowe */ 2788*5d9d9091SRichard Lowe brgz,pn %g6, sfmmu_tsb_miss_tt 2789*5d9d9091SRichard Lowe nop 2790*5d9d9091SRichard Lowe#else /* sun4v */ 2791*5d9d9091SRichard Lowe mov %g1, %g4 2792*5d9d9091SRichard Lowe GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 2793*5d9d9091SRichard Lowe PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2794*5d9d9091SRichard Lowe 2795*5d9d9091SRichard Lowe /* 2796*5d9d9091SRichard Lowe * Here: 2797*5d9d9091SRichard Lowe * g1 = first TSB pointer 2798*5d9d9091SRichard Lowe * g2 = tag access reg 2799*5d9d9091SRichard Lowe * g3 = second TSB ptr IFF ISM pred. (else don't care) 2800*5d9d9091SRichard Lowe */ 2801*5d9d9091SRichard Lowe brgz,pn %g6, sfmmu_tsb_miss_tt 2802*5d9d9091SRichard Lowe nop 2803*5d9d9091SRichard Lowe ldxa [%g0]ASI_DMMU_TSB_8K, %g3 2804*5d9d9091SRichard Lowe /* fall through in 8K->4M probe order */ 2805*5d9d9091SRichard Lowe#endif /* sun4v */ 2806*5d9d9091SRichard Lowe 2807*5d9d9091SRichard Loweudtlb_miss_probesecond: 2808*5d9d9091SRichard Lowe /* 2809*5d9d9091SRichard Lowe * Look in the second TSB for the TTE 2810*5d9d9091SRichard Lowe * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 2811*5d9d9091SRichard Lowe * g2 = tag access reg 2812*5d9d9091SRichard Lowe * g3 = 8K TSB pointer register 2813*5d9d9091SRichard Lowe * g6 = ism pred. 2814*5d9d9091SRichard Lowe * g7 = vpg_4m 2815*5d9d9091SRichard Lowe */ 2816*5d9d9091SRichard Lowe#ifdef sun4v 2817*5d9d9091SRichard Lowe /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 2818*5d9d9091SRichard Lowe GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2819*5d9d9091SRichard Lowe /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 2820*5d9d9091SRichard Lowe#else /* sun4v */ 2821*5d9d9091SRichard Lowe mov %g3, %g7 2822*5d9d9091SRichard Lowe GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 2823*5d9d9091SRichard Lowe /* %g2 clobbered, %g3 =second tsbe ptr */ 2824*5d9d9091SRichard Lowe mov MMU_TAG_ACCESS, %g2 2825*5d9d9091SRichard Lowe ldxa [%g2]ASI_DMMU, %g2 2826*5d9d9091SRichard Lowe#endif /* sun4v */ 2827*5d9d9091SRichard Lowe 2828*5d9d9091SRichard Lowe srlx %g2, TAG_VALO_SHIFT, %g7 2829*5d9d9091SRichard Lowe PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2830*5d9d9091SRichard Lowe /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 2831*5d9d9091SRichard Lowe brgz,pn %g6, udtlb_miss_probefirst 2832*5d9d9091SRichard Lowe nop 2833*5d9d9091SRichard Lowe 2834*5d9d9091SRichard Lowe /* fall through to sfmmu_tsb_miss_tt */ 2835*5d9d9091SRichard Lowe#endif /* sun4u && UTSB_PHYS */ 2836*5d9d9091SRichard Lowe 2837*5d9d9091SRichard Lowe 2838*5d9d9091SRichard Lowe ALTENTRY(sfmmu_tsb_miss_tt) 2839*5d9d9091SRichard Lowe TT_TRACE(trace_tsbmiss) 2840*5d9d9091SRichard Lowe /* 2841*5d9d9091SRichard Lowe * We get here if there is a TSB miss OR a write protect trap. 2842*5d9d9091SRichard Lowe * 2843*5d9d9091SRichard Lowe * g1 = First TSB entry pointer 2844*5d9d9091SRichard Lowe * g2 = tag access register 2845*5d9d9091SRichard Lowe * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 2846*5d9d9091SRichard Lowe * g4 - g7 = scratch registers 2847*5d9d9091SRichard Lowe */ 2848*5d9d9091SRichard Lowe 2849*5d9d9091SRichard Lowe ALTENTRY(sfmmu_tsb_miss) 2850*5d9d9091SRichard Lowe 2851*5d9d9091SRichard Lowe /* 2852*5d9d9091SRichard Lowe * If trapstat is running, we need to shift the %tpc and %tnpc to 2853*5d9d9091SRichard Lowe * point to trapstat's TSB miss return code (note that trapstat 2854*5d9d9091SRichard Lowe * itself will patch the correct offset to add). 2855*5d9d9091SRichard Lowe */ 2856*5d9d9091SRichard Lowe rdpr %tl, %g7 2857*5d9d9091SRichard Lowe cmp %g7, 1 2858*5d9d9091SRichard Lowe ble,pt %xcc, 0f 2859*5d9d9091SRichard Lowe sethi %hi(KERNELBASE), %g6 2860*5d9d9091SRichard Lowe rdpr %tpc, %g7 2861*5d9d9091SRichard Lowe or %g6, %lo(KERNELBASE), %g6 2862*5d9d9091SRichard Lowe cmp %g7, %g6 2863*5d9d9091SRichard Lowe bgeu,pt %xcc, 0f 2864*5d9d9091SRichard Lowe /* delay slot safe */ 2865*5d9d9091SRichard Lowe 2866*5d9d9091SRichard Lowe ALTENTRY(tsbmiss_trapstat_patch_point) 2867*5d9d9091SRichard Lowe add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 2868*5d9d9091SRichard Lowe wrpr %g7, %tpc 2869*5d9d9091SRichard Lowe add %g7, 4, %g7 2870*5d9d9091SRichard Lowe wrpr %g7, %tnpc 2871*5d9d9091SRichard Lowe0: 2872*5d9d9091SRichard Lowe CPU_TSBMISS_AREA(%g6, %g7) 2873*5d9d9091SRichard Lowe stn %g1, [%g6 + TSBMISS_TSBPTR] /* save 1ST tsb pointer */ 2874*5d9d9091SRichard Lowe stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save 2ND tsb pointer */ 2875*5d9d9091SRichard Lowe 2876*5d9d9091SRichard Lowe sllx %g2, TAGACC_CTX_LSHIFT, %g3 2877*5d9d9091SRichard Lowe brz,a,pn %g3, 1f /* skip ahead if kernel */ 2878*5d9d9091SRichard Lowe ldn [%g6 + TSBMISS_KHATID], %g7 2879*5d9d9091SRichard Lowe srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 2880*5d9d9091SRichard Lowe ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 2881*5d9d9091SRichard Lowe 2882*5d9d9091SRichard Lowe HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 2883*5d9d9091SRichard Lowe 2884*5d9d9091SRichard Lowe cmp %g3, INVALID_CONTEXT 2885*5d9d9091SRichard Lowe be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 2886*5d9d9091SRichard Lowe stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 2887*5d9d9091SRichard Lowe 2888*5d9d9091SRichard Lowe#if defined(sun4v) || defined(UTSB_PHYS) 2889*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 2890*5d9d9091SRichard Lowe andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 2891*5d9d9091SRichard Lowe stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 2892*5d9d9091SRichard Lowe#endif /* sun4v || UTSB_PHYS */ 2893*5d9d9091SRichard Lowe 2894*5d9d9091SRichard Lowe ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 2895*5d9d9091SRichard Lowe /* 2896*5d9d9091SRichard Lowe * The miss wasn't in an ISM segment. 2897*5d9d9091SRichard Lowe * 2898*5d9d9091SRichard Lowe * %g1 %g3, %g4, %g5, %g7 all clobbered 2899*5d9d9091SRichard Lowe * %g2 = (pseudo) tag access 2900*5d9d9091SRichard Lowe */ 2901*5d9d9091SRichard Lowe 2902*5d9d9091SRichard Lowe ba,pt %icc, 2f 2903*5d9d9091SRichard Lowe ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 2904*5d9d9091SRichard Lowe 2905*5d9d9091SRichard Lowe1: 2906*5d9d9091SRichard Lowe HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 2907*5d9d9091SRichard Lowe /* 2908*5d9d9091SRichard Lowe * 8K and 64K hash. 2909*5d9d9091SRichard Lowe */ 2910*5d9d9091SRichard Lowe2: 2911*5d9d9091SRichard Lowe 2912*5d9d9091SRichard Lowe GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 2913*5d9d9091SRichard Lowe MMU_PAGESHIFT64K, TTE64K, %g5, tsb_l8K, tsb_checktte, 2914*5d9d9091SRichard Lowe sfmmu_suspend_tl, tsb_512K) 2915*5d9d9091SRichard Lowe /* NOT REACHED */ 2916*5d9d9091SRichard Lowe 2917*5d9d9091SRichard Lowetsb_512K: 2918*5d9d9091SRichard Lowe sllx %g2, TAGACC_CTX_LSHIFT, %g5 2919*5d9d9091SRichard Lowe brz,pn %g5, 3f 2920*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 2921*5d9d9091SRichard Lowe and %g4, HAT_512K_FLAG, %g5 2922*5d9d9091SRichard Lowe 2923*5d9d9091SRichard Lowe /* 2924*5d9d9091SRichard Lowe * Note that there is a small window here where we may have 2925*5d9d9091SRichard Lowe * a 512k page in the hash list but have not set the HAT_512K_FLAG 2926*5d9d9091SRichard Lowe * flag yet, so we will skip searching the 512k hash list. 2927*5d9d9091SRichard Lowe * In this case we will end up in pagefault which will find 2928*5d9d9091SRichard Lowe * the mapping and return. So, in this instance we will end up 2929*5d9d9091SRichard Lowe * spending a bit more time resolving this TSB miss, but it can 2930*5d9d9091SRichard Lowe * only happen once per process and even then, the chances of that 2931*5d9d9091SRichard Lowe * are very small, so it's not worth the extra overhead it would 2932*5d9d9091SRichard Lowe * take to close this window. 2933*5d9d9091SRichard Lowe */ 2934*5d9d9091SRichard Lowe brz,pn %g5, tsb_4M 2935*5d9d9091SRichard Lowe nop 2936*5d9d9091SRichard Lowe3: 2937*5d9d9091SRichard Lowe /* 2938*5d9d9091SRichard Lowe * 512K hash 2939*5d9d9091SRichard Lowe */ 2940*5d9d9091SRichard Lowe 2941*5d9d9091SRichard Lowe GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 2942*5d9d9091SRichard Lowe MMU_PAGESHIFT512K, TTE512K, %g5, tsb_l512K, tsb_checktte, 2943*5d9d9091SRichard Lowe sfmmu_suspend_tl, tsb_4M) 2944*5d9d9091SRichard Lowe /* NOT REACHED */ 2945*5d9d9091SRichard Lowe 2946*5d9d9091SRichard Lowetsb_4M: 2947*5d9d9091SRichard Lowe sllx %g2, TAGACC_CTX_LSHIFT, %g5 2948*5d9d9091SRichard Lowe brz,pn %g5, 4f 2949*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 2950*5d9d9091SRichard Lowe and %g4, HAT_4M_FLAG, %g5 2951*5d9d9091SRichard Lowe brz,pn %g5, tsb_32M 2952*5d9d9091SRichard Lowe nop 2953*5d9d9091SRichard Lowe4: 2954*5d9d9091SRichard Lowe /* 2955*5d9d9091SRichard Lowe * 4M hash 2956*5d9d9091SRichard Lowe */ 2957*5d9d9091SRichard Lowe 2958*5d9d9091SRichard Lowe GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 2959*5d9d9091SRichard Lowe MMU_PAGESHIFT4M, TTE4M, %g5, tsb_l4M, tsb_checktte, 2960*5d9d9091SRichard Lowe sfmmu_suspend_tl, tsb_32M) 2961*5d9d9091SRichard Lowe /* NOT REACHED */ 2962*5d9d9091SRichard Lowe 2963*5d9d9091SRichard Lowetsb_32M: 2964*5d9d9091SRichard Lowe sllx %g2, TAGACC_CTX_LSHIFT, %g5 2965*5d9d9091SRichard Lowe#ifdef sun4v 2966*5d9d9091SRichard Lowe brz,pn %g5, 6f 2967*5d9d9091SRichard Lowe#else 2968*5d9d9091SRichard Lowe brz,pn %g5, tsb_pagefault 2969*5d9d9091SRichard Lowe#endif 2970*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 2971*5d9d9091SRichard Lowe and %g4, HAT_32M_FLAG, %g5 2972*5d9d9091SRichard Lowe brz,pn %g5, tsb_256M 2973*5d9d9091SRichard Lowe nop 2974*5d9d9091SRichard Lowe5: 2975*5d9d9091SRichard Lowe /* 2976*5d9d9091SRichard Lowe * 32M hash 2977*5d9d9091SRichard Lowe */ 2978*5d9d9091SRichard Lowe 2979*5d9d9091SRichard Lowe GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 2980*5d9d9091SRichard Lowe MMU_PAGESHIFT32M, TTE32M, %g5, tsb_l32M, tsb_checktte, 2981*5d9d9091SRichard Lowe sfmmu_suspend_tl, tsb_256M) 2982*5d9d9091SRichard Lowe /* NOT REACHED */ 2983*5d9d9091SRichard Lowe 2984*5d9d9091SRichard Lowe#if defined(sun4u) && !defined(UTSB_PHYS) 2985*5d9d9091SRichard Lowe#define tsb_shme tsb_pagefault 2986*5d9d9091SRichard Lowe#endif 2987*5d9d9091SRichard Lowetsb_256M: 2988*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 2989*5d9d9091SRichard Lowe and %g4, HAT_256M_FLAG, %g5 2990*5d9d9091SRichard Lowe brz,pn %g5, tsb_shme 2991*5d9d9091SRichard Lowe nop 2992*5d9d9091SRichard Lowe6: 2993*5d9d9091SRichard Lowe /* 2994*5d9d9091SRichard Lowe * 256M hash 2995*5d9d9091SRichard Lowe */ 2996*5d9d9091SRichard Lowe 2997*5d9d9091SRichard Lowe GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 2998*5d9d9091SRichard Lowe MMU_PAGESHIFT256M, TTE256M, %g5, tsb_l256M, tsb_checktte, 2999*5d9d9091SRichard Lowe sfmmu_suspend_tl, tsb_shme) 3000*5d9d9091SRichard Lowe /* NOT REACHED */ 3001*5d9d9091SRichard Lowe 3002*5d9d9091SRichard Lowetsb_checktte: 3003*5d9d9091SRichard Lowe /* 3004*5d9d9091SRichard Lowe * g1 = hblk_misc 3005*5d9d9091SRichard Lowe * g2 = tagacc 3006*5d9d9091SRichard Lowe * g3 = tte 3007*5d9d9091SRichard Lowe * g4 = tte pa 3008*5d9d9091SRichard Lowe * g6 = tsbmiss area 3009*5d9d9091SRichard Lowe * g7 = hatid 3010*5d9d9091SRichard Lowe */ 3011*5d9d9091SRichard Lowe brlz,a,pt %g3, tsb_validtte 3012*5d9d9091SRichard Lowe rdpr %tt, %g7 3013*5d9d9091SRichard Lowe 3014*5d9d9091SRichard Lowe#if defined(sun4u) && !defined(UTSB_PHYS) 3015*5d9d9091SRichard Lowe#undef tsb_shme 3016*5d9d9091SRichard Lowe ba tsb_pagefault 3017*5d9d9091SRichard Lowe nop 3018*5d9d9091SRichard Lowe#else /* sun4u && !UTSB_PHYS */ 3019*5d9d9091SRichard Lowe 3020*5d9d9091SRichard Lowetsb_shme: 3021*5d9d9091SRichard Lowe /* 3022*5d9d9091SRichard Lowe * g2 = tagacc 3023*5d9d9091SRichard Lowe * g6 = tsbmiss area 3024*5d9d9091SRichard Lowe */ 3025*5d9d9091SRichard Lowe sllx %g2, TAGACC_CTX_LSHIFT, %g5 3026*5d9d9091SRichard Lowe brz,pn %g5, tsb_pagefault 3027*5d9d9091SRichard Lowe nop 3028*5d9d9091SRichard Lowe ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3029*5d9d9091SRichard Lowe brz,pn %g7, tsb_pagefault 3030*5d9d9091SRichard Lowe nop 3031*5d9d9091SRichard Lowe 3032*5d9d9091SRichard Lowe GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3033*5d9d9091SRichard Lowe MMU_PAGESHIFT64K, TTE64K, %g5, tsb_shme_l8K, tsb_shme_checktte, 3034*5d9d9091SRichard Lowe sfmmu_suspend_tl, tsb_shme_512K) 3035*5d9d9091SRichard Lowe /* NOT REACHED */ 3036*5d9d9091SRichard Lowe 3037*5d9d9091SRichard Lowetsb_shme_512K: 3038*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3039*5d9d9091SRichard Lowe and %g4, HAT_512K_FLAG, %g5 3040*5d9d9091SRichard Lowe brz,pn %g5, tsb_shme_4M 3041*5d9d9091SRichard Lowe nop 3042*5d9d9091SRichard Lowe 3043*5d9d9091SRichard Lowe /* 3044*5d9d9091SRichard Lowe * 512K hash 3045*5d9d9091SRichard Lowe */ 3046*5d9d9091SRichard Lowe 3047*5d9d9091SRichard Lowe GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3048*5d9d9091SRichard Lowe MMU_PAGESHIFT512K, TTE512K, %g5, tsb_shme_l512K, tsb_shme_checktte, 3049*5d9d9091SRichard Lowe sfmmu_suspend_tl, tsb_shme_4M) 3050*5d9d9091SRichard Lowe /* NOT REACHED */ 3051*5d9d9091SRichard Lowe 3052*5d9d9091SRichard Lowetsb_shme_4M: 3053*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3054*5d9d9091SRichard Lowe and %g4, HAT_4M_FLAG, %g5 3055*5d9d9091SRichard Lowe brz,pn %g5, tsb_shme_32M 3056*5d9d9091SRichard Lowe nop 3057*5d9d9091SRichard Lowe4: 3058*5d9d9091SRichard Lowe /* 3059*5d9d9091SRichard Lowe * 4M hash 3060*5d9d9091SRichard Lowe */ 3061*5d9d9091SRichard Lowe GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3062*5d9d9091SRichard Lowe MMU_PAGESHIFT4M, TTE4M, %g5, tsb_shme_l4M, tsb_shme_checktte, 3063*5d9d9091SRichard Lowe sfmmu_suspend_tl, tsb_shme_32M) 3064*5d9d9091SRichard Lowe /* NOT REACHED */ 3065*5d9d9091SRichard Lowe 3066*5d9d9091SRichard Lowetsb_shme_32M: 3067*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3068*5d9d9091SRichard Lowe and %g4, HAT_32M_FLAG, %g5 3069*5d9d9091SRichard Lowe brz,pn %g5, tsb_shme_256M 3070*5d9d9091SRichard Lowe nop 3071*5d9d9091SRichard Lowe 3072*5d9d9091SRichard Lowe /* 3073*5d9d9091SRichard Lowe * 32M hash 3074*5d9d9091SRichard Lowe */ 3075*5d9d9091SRichard Lowe 3076*5d9d9091SRichard Lowe GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3077*5d9d9091SRichard Lowe MMU_PAGESHIFT32M, TTE32M, %g5, tsb_shme_l32M, tsb_shme_checktte, 3078*5d9d9091SRichard Lowe sfmmu_suspend_tl, tsb_shme_256M) 3079*5d9d9091SRichard Lowe /* NOT REACHED */ 3080*5d9d9091SRichard Lowe 3081*5d9d9091SRichard Lowetsb_shme_256M: 3082*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3083*5d9d9091SRichard Lowe and %g4, HAT_256M_FLAG, %g5 3084*5d9d9091SRichard Lowe brz,pn %g5, tsb_pagefault 3085*5d9d9091SRichard Lowe nop 3086*5d9d9091SRichard Lowe 3087*5d9d9091SRichard Lowe /* 3088*5d9d9091SRichard Lowe * 256M hash 3089*5d9d9091SRichard Lowe */ 3090*5d9d9091SRichard Lowe 3091*5d9d9091SRichard Lowe GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3092*5d9d9091SRichard Lowe MMU_PAGESHIFT256M, TTE256M, %g5, tsb_shme_l256M, tsb_shme_checktte, 3093*5d9d9091SRichard Lowe sfmmu_suspend_tl, tsb_pagefault) 3094*5d9d9091SRichard Lowe /* NOT REACHED */ 3095*5d9d9091SRichard Lowe 3096*5d9d9091SRichard Lowetsb_shme_checktte: 3097*5d9d9091SRichard Lowe 3098*5d9d9091SRichard Lowe brgez,pn %g3, tsb_pagefault 3099*5d9d9091SRichard Lowe rdpr %tt, %g7 3100*5d9d9091SRichard Lowe /* 3101*5d9d9091SRichard Lowe * g1 = ctx1 flag 3102*5d9d9091SRichard Lowe * g3 = tte 3103*5d9d9091SRichard Lowe * g4 = tte pa 3104*5d9d9091SRichard Lowe * g6 = tsbmiss area 3105*5d9d9091SRichard Lowe * g7 = tt 3106*5d9d9091SRichard Lowe */ 3107*5d9d9091SRichard Lowe 3108*5d9d9091SRichard Lowe brz,pt %g1, tsb_validtte 3109*5d9d9091SRichard Lowe nop 3110*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3111*5d9d9091SRichard Lowe or %g1, HAT_CHKCTX1_FLAG, %g1 3112*5d9d9091SRichard Lowe stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3113*5d9d9091SRichard Lowe 3114*5d9d9091SRichard Lowe SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3115*5d9d9091SRichard Lowe#endif /* sun4u && !UTSB_PHYS */ 3116*5d9d9091SRichard Lowe 3117*5d9d9091SRichard Lowetsb_validtte: 3118*5d9d9091SRichard Lowe /* 3119*5d9d9091SRichard Lowe * g3 = tte 3120*5d9d9091SRichard Lowe * g4 = tte pa 3121*5d9d9091SRichard Lowe * g6 = tsbmiss area 3122*5d9d9091SRichard Lowe * g7 = tt 3123*5d9d9091SRichard Lowe */ 3124*5d9d9091SRichard Lowe 3125*5d9d9091SRichard Lowe /* 3126*5d9d9091SRichard Lowe * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3127*5d9d9091SRichard Lowe */ 3128*5d9d9091SRichard Lowe cmp %g7, FAST_PROT_TT 3129*5d9d9091SRichard Lowe bne,pt %icc, 4f 3130*5d9d9091SRichard Lowe nop 3131*5d9d9091SRichard Lowe 3132*5d9d9091SRichard Lowe TTE_SET_REFMOD_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_refmod, 3133*5d9d9091SRichard Lowe tsb_protfault) 3134*5d9d9091SRichard Lowe 3135*5d9d9091SRichard Lowe GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3136*5d9d9091SRichard Lowe#ifdef sun4v 3137*5d9d9091SRichard Lowe MMU_FAULT_STATUS_AREA(%g7) 3138*5d9d9091SRichard Lowe ldx [%g7 + MMFSA_D_ADDR], %g5 /* load fault addr for later */ 3139*5d9d9091SRichard Lowe#else /* sun4v */ 3140*5d9d9091SRichard Lowe mov MMU_TAG_ACCESS, %g5 3141*5d9d9091SRichard Lowe ldxa [%g5]ASI_DMMU, %g5 3142*5d9d9091SRichard Lowe#endif /* sun4v */ 3143*5d9d9091SRichard Lowe ba,pt %xcc, tsb_update_tl1 3144*5d9d9091SRichard Lowe nop 3145*5d9d9091SRichard Lowe4: 3146*5d9d9091SRichard Lowe /* 3147*5d9d9091SRichard Lowe * If ITLB miss check exec bit. 3148*5d9d9091SRichard Lowe * If not set treat as invalid TTE. 3149*5d9d9091SRichard Lowe */ 3150*5d9d9091SRichard Lowe cmp %g7, T_INSTR_MMU_MISS 3151*5d9d9091SRichard Lowe be,pn %icc, 5f 3152*5d9d9091SRichard Lowe andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3153*5d9d9091SRichard Lowe cmp %g7, FAST_IMMU_MISS_TT 3154*5d9d9091SRichard Lowe bne,pt %icc, 3f 3155*5d9d9091SRichard Lowe andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3156*5d9d9091SRichard Lowe5: 3157*5d9d9091SRichard Lowe bz,pn %icc, tsb_protfault 3158*5d9d9091SRichard Lowe nop 3159*5d9d9091SRichard Lowe 3160*5d9d9091SRichard Lowe3: 3161*5d9d9091SRichard Lowe /* 3162*5d9d9091SRichard Lowe * Set reference bit if not already set 3163*5d9d9091SRichard Lowe */ 3164*5d9d9091SRichard Lowe TTE_SET_REF_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_ref) 3165*5d9d9091SRichard Lowe 3166*5d9d9091SRichard Lowe /* 3167*5d9d9091SRichard Lowe * Now, load into TSB/TLB. At this point: 3168*5d9d9091SRichard Lowe * g3 = tte 3169*5d9d9091SRichard Lowe * g4 = patte 3170*5d9d9091SRichard Lowe * g6 = tsbmiss area 3171*5d9d9091SRichard Lowe */ 3172*5d9d9091SRichard Lowe rdpr %tt, %g7 3173*5d9d9091SRichard Lowe#ifdef sun4v 3174*5d9d9091SRichard Lowe MMU_FAULT_STATUS_AREA(%g2) 3175*5d9d9091SRichard Lowe cmp %g7, T_INSTR_MMU_MISS 3176*5d9d9091SRichard Lowe be,a,pt %icc, 9f 3177*5d9d9091SRichard Lowe nop 3178*5d9d9091SRichard Lowe cmp %g7, FAST_IMMU_MISS_TT 3179*5d9d9091SRichard Lowe be,a,pt %icc, 9f 3180*5d9d9091SRichard Lowe nop 3181*5d9d9091SRichard Lowe add %g2, MMFSA_D_, %g2 3182*5d9d9091SRichard Lowe9: 3183*5d9d9091SRichard Lowe ldx [%g2 + MMFSA_CTX_], %g7 3184*5d9d9091SRichard Lowe sllx %g7, TTARGET_CTX_SHIFT, %g7 3185*5d9d9091SRichard Lowe ldx [%g2 + MMFSA_ADDR_], %g2 3186*5d9d9091SRichard Lowe mov %g2, %g5 ! load the fault addr for later use 3187*5d9d9091SRichard Lowe srlx %g2, TTARGET_VA_SHIFT, %g2 3188*5d9d9091SRichard Lowe or %g2, %g7, %g2 3189*5d9d9091SRichard Lowe#else /* sun4v */ 3190*5d9d9091SRichard Lowe mov MMU_TAG_ACCESS, %g5 3191*5d9d9091SRichard Lowe cmp %g7, FAST_IMMU_MISS_TT 3192*5d9d9091SRichard Lowe be,a,pt %icc, 9f 3193*5d9d9091SRichard Lowe ldxa [%g0]ASI_IMMU, %g2 3194*5d9d9091SRichard Lowe ldxa [%g0]ASI_DMMU, %g2 3195*5d9d9091SRichard Lowe ba,pt %icc, tsb_update_tl1 3196*5d9d9091SRichard Lowe ldxa [%g5]ASI_DMMU, %g5 3197*5d9d9091SRichard Lowe9: 3198*5d9d9091SRichard Lowe ldxa [%g5]ASI_IMMU, %g5 3199*5d9d9091SRichard Lowe#endif /* sun4v */ 3200*5d9d9091SRichard Lowe 3201*5d9d9091SRichard Lowetsb_update_tl1: 3202*5d9d9091SRichard Lowe srlx %g2, TTARGET_CTX_SHIFT, %g7 3203*5d9d9091SRichard Lowe brz,pn %g7, tsb_kernel 3204*5d9d9091SRichard Lowe#ifdef sun4v 3205*5d9d9091SRichard Lowe and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3206*5d9d9091SRichard Lowe#else /* sun4v */ 3207*5d9d9091SRichard Lowe srlx %g3, TTE_SZ_SHFT, %g7 3208*5d9d9091SRichard Lowe#endif /* sun4v */ 3209*5d9d9091SRichard Lowe 3210*5d9d9091SRichard Lowetsb_user: 3211*5d9d9091SRichard Lowe#ifdef sun4v 3212*5d9d9091SRichard Lowe cmp %g7, TTE4M 3213*5d9d9091SRichard Lowe bge,pn %icc, tsb_user4m 3214*5d9d9091SRichard Lowe nop 3215*5d9d9091SRichard Lowe#else /* sun4v */ 3216*5d9d9091SRichard Lowe cmp %g7, TTESZ_VALID | TTE4M 3217*5d9d9091SRichard Lowe be,pn %icc, tsb_user4m 3218*5d9d9091SRichard Lowe srlx %g3, TTE_SZ2_SHFT, %g7 3219*5d9d9091SRichard Lowe andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3220*5d9d9091SRichard Lowe#ifdef ITLB_32M_256M_SUPPORT 3221*5d9d9091SRichard Lowe bnz,pn %icc, tsb_user4m 3222*5d9d9091SRichard Lowe nop 3223*5d9d9091SRichard Lowe#else /* ITLB_32M_256M_SUPPORT */ 3224*5d9d9091SRichard Lowe bnz,a,pn %icc, tsb_user_pn_synth 3225*5d9d9091SRichard Lowe nop 3226*5d9d9091SRichard Lowe#endif /* ITLB_32M_256M_SUPPORT */ 3227*5d9d9091SRichard Lowe#endif /* sun4v */ 3228*5d9d9091SRichard Lowe 3229*5d9d9091SRichard Lowetsb_user8k: 3230*5d9d9091SRichard Lowe#if defined(sun4v) || defined(UTSB_PHYS) 3231*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3232*5d9d9091SRichard Lowe and %g7, HAT_CHKCTX1_FLAG, %g1 3233*5d9d9091SRichard Lowe brz,a,pn %g1, 1f 3234*5d9d9091SRichard Lowe ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3235*5d9d9091SRichard Lowe GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3236*5d9d9091SRichard Lowe brlz,a,pn %g1, ptl1_panic ! if no shared 3RD tsb 3237*5d9d9091SRichard Lowe mov PTL1_NO_SCDTSB8K, %g1 ! panic 3238*5d9d9091SRichard Lowe GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 3239*5d9d9091SRichard Lowe1: 3240*5d9d9091SRichard Lowe#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3241*5d9d9091SRichard Lowe ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3242*5d9d9091SRichard Lowe#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3243*5d9d9091SRichard Lowe 3244*5d9d9091SRichard Lowe#ifndef UTSB_PHYS 3245*5d9d9091SRichard Lowe mov ASI_N, %g7 ! user TSBs accessed by VA 3246*5d9d9091SRichard Lowe mov %g7, %asi 3247*5d9d9091SRichard Lowe#endif /* !UTSB_PHYS */ 3248*5d9d9091SRichard Lowe 3249*5d9d9091SRichard Lowe TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l3) 3250*5d9d9091SRichard Lowe 3251*5d9d9091SRichard Lowe rdpr %tt, %g5 3252*5d9d9091SRichard Lowe#ifdef sun4v 3253*5d9d9091SRichard Lowe cmp %g5, T_INSTR_MMU_MISS 3254*5d9d9091SRichard Lowe be,a,pn %xcc, 9f 3255*5d9d9091SRichard Lowe mov %g3, %g5 3256*5d9d9091SRichard Lowe#endif /* sun4v */ 3257*5d9d9091SRichard Lowe cmp %g5, FAST_IMMU_MISS_TT 3258*5d9d9091SRichard Lowe be,pn %xcc, 9f 3259*5d9d9091SRichard Lowe mov %g3, %g5 3260*5d9d9091SRichard Lowe 3261*5d9d9091SRichard Lowe DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3262*5d9d9091SRichard Lowe ! trapstat wants TTE in %g5 3263*5d9d9091SRichard Lowe retry 3264*5d9d9091SRichard Lowe9: 3265*5d9d9091SRichard Lowe ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3266*5d9d9091SRichard Lowe ! trapstat wants TTE in %g5 3267*5d9d9091SRichard Lowe retry 3268*5d9d9091SRichard Lowe 3269*5d9d9091SRichard Lowetsb_user4m: 3270*5d9d9091SRichard Lowe#if defined(sun4v) || defined(UTSB_PHYS) 3271*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3272*5d9d9091SRichard Lowe and %g7, HAT_CHKCTX1_FLAG, %g1 3273*5d9d9091SRichard Lowe brz,a,pn %g1, 4f 3274*5d9d9091SRichard Lowe ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3275*5d9d9091SRichard Lowe GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr 3276*5d9d9091SRichard Lowe brlz,a,pn %g1, 5f ! if no shared 4TH TSB 3277*5d9d9091SRichard Lowe nop 3278*5d9d9091SRichard Lowe GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 3279*5d9d9091SRichard Lowe 3280*5d9d9091SRichard Lowe#else /* defined(sun4v) || defined(UTSB_PHYS) */ 3281*5d9d9091SRichard Lowe ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3282*5d9d9091SRichard Lowe#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3283*5d9d9091SRichard Lowe4: 3284*5d9d9091SRichard Lowe brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3285*5d9d9091SRichard Lowe nop 3286*5d9d9091SRichard Lowe 3287*5d9d9091SRichard Lowe#ifndef UTSB_PHYS 3288*5d9d9091SRichard Lowe mov ASI_N, %g7 ! user TSBs accessed by VA 3289*5d9d9091SRichard Lowe mov %g7, %asi 3290*5d9d9091SRichard Lowe#endif /* UTSB_PHYS */ 3291*5d9d9091SRichard Lowe 3292*5d9d9091SRichard Lowe TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l4) 3293*5d9d9091SRichard Lowe 3294*5d9d9091SRichard Lowe5: 3295*5d9d9091SRichard Lowe rdpr %tt, %g5 3296*5d9d9091SRichard Lowe#ifdef sun4v 3297*5d9d9091SRichard Lowe cmp %g5, T_INSTR_MMU_MISS 3298*5d9d9091SRichard Lowe be,a,pn %xcc, 9f 3299*5d9d9091SRichard Lowe mov %g3, %g5 3300*5d9d9091SRichard Lowe#endif /* sun4v */ 3301*5d9d9091SRichard Lowe cmp %g5, FAST_IMMU_MISS_TT 3302*5d9d9091SRichard Lowe be,pn %xcc, 9f 3303*5d9d9091SRichard Lowe mov %g3, %g5 3304*5d9d9091SRichard Lowe 3305*5d9d9091SRichard Lowe DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3306*5d9d9091SRichard Lowe ! trapstat wants TTE in %g5 3307*5d9d9091SRichard Lowe retry 3308*5d9d9091SRichard Lowe9: 3309*5d9d9091SRichard Lowe ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3310*5d9d9091SRichard Lowe ! trapstat wants TTE in %g5 3311*5d9d9091SRichard Lowe retry 3312*5d9d9091SRichard Lowe 3313*5d9d9091SRichard Lowe#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3314*5d9d9091SRichard Lowe /* 3315*5d9d9091SRichard Lowe * Panther ITLB synthesis. 3316*5d9d9091SRichard Lowe * The Panther 32M and 256M ITLB code simulates these two large page 3317*5d9d9091SRichard Lowe * sizes with 4M pages, to provide support for programs, for example 3318*5d9d9091SRichard Lowe * Java, that may copy instructions into a 32M or 256M data page and 3319*5d9d9091SRichard Lowe * then execute them. The code below generates the 4M pfn bits and 3320*5d9d9091SRichard Lowe * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3321*5d9d9091SRichard Lowe * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3322*5d9d9091SRichard Lowe * are ignored by the hardware. 3323*5d9d9091SRichard Lowe * 3324*5d9d9091SRichard Lowe * Now, load into TSB/TLB. At this point: 3325*5d9d9091SRichard Lowe * g2 = tagtarget 3326*5d9d9091SRichard Lowe * g3 = tte 3327*5d9d9091SRichard Lowe * g4 = patte 3328*5d9d9091SRichard Lowe * g5 = tt 3329*5d9d9091SRichard Lowe * g6 = tsbmiss area 3330*5d9d9091SRichard Lowe */ 3331*5d9d9091SRichard Lowetsb_user_pn_synth: 3332*5d9d9091SRichard Lowe rdpr %tt, %g5 3333*5d9d9091SRichard Lowe cmp %g5, FAST_IMMU_MISS_TT 3334*5d9d9091SRichard Lowe be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3335*5d9d9091SRichard Lowe andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3336*5d9d9091SRichard Lowe bz,pn %icc, 4b /* if not, been here before */ 3337*5d9d9091SRichard Lowe ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3338*5d9d9091SRichard Lowe brlz,a,pn %g1, 5f /* no 2nd tsb */ 3339*5d9d9091SRichard Lowe mov %g3, %g5 3340*5d9d9091SRichard Lowe 3341*5d9d9091SRichard Lowe mov MMU_TAG_ACCESS, %g7 3342*5d9d9091SRichard Lowe ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3343*5d9d9091SRichard Lowe GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3344*5d9d9091SRichard Lowe 3345*5d9d9091SRichard Lowe mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3346*5d9d9091SRichard Lowe mov %g7, %asi 3347*5d9d9091SRichard Lowe TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l5) /* update TSB */ 3348*5d9d9091SRichard Lowe5: 3349*5d9d9091SRichard Lowe DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3350*5d9d9091SRichard Lowe retry 3351*5d9d9091SRichard Lowe 3352*5d9d9091SRichard Lowetsb_user_itlb_synth: 3353*5d9d9091SRichard Lowe ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = 2ND TSB */ 3354*5d9d9091SRichard Lowe 3355*5d9d9091SRichard Lowe mov MMU_TAG_ACCESS, %g7 3356*5d9d9091SRichard Lowe ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3357*5d9d9091SRichard Lowe GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3358*5d9d9091SRichard Lowe brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3359*5d9d9091SRichard Lowe or %g5, %g3, %g5 /* add 4M bits to TTE */ 3360*5d9d9091SRichard Lowe 3361*5d9d9091SRichard Lowe mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3362*5d9d9091SRichard Lowe mov %g7, %asi 3363*5d9d9091SRichard Lowe TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l6) /* update TSB */ 3364*5d9d9091SRichard Lowe7: 3365*5d9d9091SRichard Lowe SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3366*5d9d9091SRichard Lowe ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3367*5d9d9091SRichard Lowe retry 3368*5d9d9091SRichard Lowe#endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3369*5d9d9091SRichard Lowe 3370*5d9d9091SRichard Lowetsb_kernel: 3371*5d9d9091SRichard Lowe rdpr %tt, %g5 3372*5d9d9091SRichard Lowe#ifdef sun4v 3373*5d9d9091SRichard Lowe cmp %g7, TTE4M 3374*5d9d9091SRichard Lowe bge,pn %icc, 5f 3375*5d9d9091SRichard Lowe#else 3376*5d9d9091SRichard Lowe cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3377*5d9d9091SRichard Lowe be,pn %icc, 5f 3378*5d9d9091SRichard Lowe#endif /* sun4v */ 3379*5d9d9091SRichard Lowe nop 3380*5d9d9091SRichard Lowe ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8K TSB ptr 3381*5d9d9091SRichard Lowe ba,pt %xcc, 6f 3382*5d9d9091SRichard Lowe nop 3383*5d9d9091SRichard Lowe5: 3384*5d9d9091SRichard Lowe ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4M TSB ptr 3385*5d9d9091SRichard Lowe brlz,pn %g1, 3f /* skip programming if 4M TSB ptr is -1 */ 3386*5d9d9091SRichard Lowe nop 3387*5d9d9091SRichard Lowe6: 3388*5d9d9091SRichard Lowe#ifndef sun4v 3389*5d9d9091SRichard Lowetsb_kernel_patch_asi: 3390*5d9d9091SRichard Lowe or %g0, RUNTIME_PATCH, %g6 3391*5d9d9091SRichard Lowe mov %g6, %asi ! XXX avoid writing to %asi !! 3392*5d9d9091SRichard Lowe#endif 3393*5d9d9091SRichard Lowe TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l7) 3394*5d9d9091SRichard Lowe3: 3395*5d9d9091SRichard Lowe#ifdef sun4v 3396*5d9d9091SRichard Lowe cmp %g5, T_INSTR_MMU_MISS 3397*5d9d9091SRichard Lowe be,a,pn %icc, 1f 3398*5d9d9091SRichard Lowe mov %g3, %g5 ! trapstat wants TTE in %g5 3399*5d9d9091SRichard Lowe#endif /* sun4v */ 3400*5d9d9091SRichard Lowe cmp %g5, FAST_IMMU_MISS_TT 3401*5d9d9091SRichard Lowe be,pn %icc, 1f 3402*5d9d9091SRichard Lowe mov %g3, %g5 ! trapstat wants TTE in %g5 3403*5d9d9091SRichard Lowe DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3404*5d9d9091SRichard Lowe ! trapstat wants TTE in %g5 3405*5d9d9091SRichard Lowe retry 3406*5d9d9091SRichard Lowe1: 3407*5d9d9091SRichard Lowe ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3408*5d9d9091SRichard Lowe ! trapstat wants TTE in %g5 3409*5d9d9091SRichard Lowe retry 3410*5d9d9091SRichard Lowe 3411*5d9d9091SRichard Lowetsb_ism: 3412*5d9d9091SRichard Lowe /* 3413*5d9d9091SRichard Lowe * This is an ISM [i|d]tlb miss. We optimize for largest 3414*5d9d9091SRichard Lowe * page size down to smallest. 3415*5d9d9091SRichard Lowe * 3416*5d9d9091SRichard Lowe * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3417*5d9d9091SRichard Lowe * register 3418*5d9d9091SRichard Lowe * g3 = ismmap->ism_seg 3419*5d9d9091SRichard Lowe * g4 = physical address of ismmap->ism_sfmmu 3420*5d9d9091SRichard Lowe * g6 = tsbmiss area 3421*5d9d9091SRichard Lowe */ 3422*5d9d9091SRichard Lowe ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3423*5d9d9091SRichard Lowe brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3424*5d9d9091SRichard Lowe mov PTL1_BAD_ISM, %g1 3425*5d9d9091SRichard Lowe /* g5 = pa of imap_vb_shift */ 3426*5d9d9091SRichard Lowe sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3427*5d9d9091SRichard Lowe lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3428*5d9d9091SRichard Lowe srlx %g3, %g4, %g3 /* clr size field */ 3429*5d9d9091SRichard Lowe set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3430*5d9d9091SRichard Lowe sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3431*5d9d9091SRichard Lowe and %g2, %g1, %g4 /* g4 = ctx number */ 3432*5d9d9091SRichard Lowe andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3433*5d9d9091SRichard Lowe sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3434*5d9d9091SRichard Lowe or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3435*5d9d9091SRichard Lowe sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3436*5d9d9091SRichard Lowe lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3437*5d9d9091SRichard Lowe#if defined(sun4v) || defined(UTSB_PHYS) 3438*5d9d9091SRichard Lowe and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3439*5d9d9091SRichard Lowe brz,pt %g5, tsb_chk4M_ism 3440*5d9d9091SRichard Lowe nop 3441*5d9d9091SRichard Lowe ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3442*5d9d9091SRichard Lowe or %g5, HAT_CHKCTX1_FLAG, %g5 3443*5d9d9091SRichard Lowe stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3444*5d9d9091SRichard Lowe rdpr %tt, %g5 3445*5d9d9091SRichard Lowe SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl) 3446*5d9d9091SRichard Lowe#endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3447*5d9d9091SRichard Lowe 3448*5d9d9091SRichard Lowe /* 3449*5d9d9091SRichard Lowe * ISM pages are always locked down. 3450*5d9d9091SRichard Lowe * If we can't find the tte then pagefault 3451*5d9d9091SRichard Lowe * and let the spt segment driver resolve it. 3452*5d9d9091SRichard Lowe * 3453*5d9d9091SRichard Lowe * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3454*5d9d9091SRichard Lowe * g4 = imap_hatflags 3455*5d9d9091SRichard Lowe * g6 = tsb miss area 3456*5d9d9091SRichard Lowe * g7 = ISM hatid 3457*5d9d9091SRichard Lowe */ 3458*5d9d9091SRichard Lowe 3459*5d9d9091SRichard Lowetsb_chk4M_ism: 3460*5d9d9091SRichard Lowe and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3461*5d9d9091SRichard Lowe brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3462*5d9d9091SRichard Lowe nop 3463*5d9d9091SRichard Lowe 3464*5d9d9091SRichard Lowetsb_ism_32M: 3465*5d9d9091SRichard Lowe and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3466*5d9d9091SRichard Lowe brz,pn %g5, tsb_ism_256M 3467*5d9d9091SRichard Lowe nop 3468*5d9d9091SRichard Lowe 3469*5d9d9091SRichard Lowe /* 3470*5d9d9091SRichard Lowe * 32M hash. 3471*5d9d9091SRichard Lowe */ 3472*5d9d9091SRichard Lowe 3473*5d9d9091SRichard Lowe GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT32M, 3474*5d9d9091SRichard Lowe TTE32M, %g5, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3475*5d9d9091SRichard Lowe tsb_ism_4M) 3476*5d9d9091SRichard Lowe /* NOT REACHED */ 3477*5d9d9091SRichard Lowe 3478*5d9d9091SRichard Lowetsb_ism_32M_found: 3479*5d9d9091SRichard Lowe brlz,a,pt %g3, tsb_validtte 3480*5d9d9091SRichard Lowe rdpr %tt, %g7 3481*5d9d9091SRichard Lowe ba,pt %xcc, tsb_ism_4M 3482*5d9d9091SRichard Lowe nop 3483*5d9d9091SRichard Lowe 3484*5d9d9091SRichard Lowetsb_ism_256M: 3485*5d9d9091SRichard Lowe and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3486*5d9d9091SRichard Lowe brz,a,pn %g5, ptl1_panic 3487*5d9d9091SRichard Lowe mov PTL1_BAD_ISM, %g1 3488*5d9d9091SRichard Lowe 3489*5d9d9091SRichard Lowe /* 3490*5d9d9091SRichard Lowe * 256M hash. 3491*5d9d9091SRichard Lowe */ 3492*5d9d9091SRichard Lowe GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT256M, 3493*5d9d9091SRichard Lowe TTE256M, %g5, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3494*5d9d9091SRichard Lowe tsb_ism_4M) 3495*5d9d9091SRichard Lowe 3496*5d9d9091SRichard Lowetsb_ism_256M_found: 3497*5d9d9091SRichard Lowe brlz,a,pt %g3, tsb_validtte 3498*5d9d9091SRichard Lowe rdpr %tt, %g7 3499*5d9d9091SRichard Lowe 3500*5d9d9091SRichard Lowetsb_ism_4M: 3501*5d9d9091SRichard Lowe /* 3502*5d9d9091SRichard Lowe * 4M hash. 3503*5d9d9091SRichard Lowe */ 3504*5d9d9091SRichard Lowe GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT4M, 3505*5d9d9091SRichard Lowe TTE4M, %g5, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3506*5d9d9091SRichard Lowe tsb_ism_8K) 3507*5d9d9091SRichard Lowe /* NOT REACHED */ 3508*5d9d9091SRichard Lowe 3509*5d9d9091SRichard Lowetsb_ism_4M_found: 3510*5d9d9091SRichard Lowe brlz,a,pt %g3, tsb_validtte 3511*5d9d9091SRichard Lowe rdpr %tt, %g7 3512*5d9d9091SRichard Lowe 3513*5d9d9091SRichard Lowetsb_ism_8K: 3514*5d9d9091SRichard Lowe /* 3515*5d9d9091SRichard Lowe * 8K and 64K hash. 3516*5d9d9091SRichard Lowe */ 3517*5d9d9091SRichard Lowe 3518*5d9d9091SRichard Lowe GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT64K, 3519*5d9d9091SRichard Lowe TTE64K, %g5, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3520*5d9d9091SRichard Lowe tsb_pagefault) 3521*5d9d9091SRichard Lowe /* NOT REACHED */ 3522*5d9d9091SRichard Lowe 3523*5d9d9091SRichard Lowetsb_ism_8K_found: 3524*5d9d9091SRichard Lowe brlz,a,pt %g3, tsb_validtte 3525*5d9d9091SRichard Lowe rdpr %tt, %g7 3526*5d9d9091SRichard Lowe 3527*5d9d9091SRichard Lowetsb_pagefault: 3528*5d9d9091SRichard Lowe rdpr %tt, %g7 3529*5d9d9091SRichard Lowe cmp %g7, FAST_PROT_TT 3530*5d9d9091SRichard Lowe be,a,pn %icc, tsb_protfault 3531*5d9d9091SRichard Lowe wrpr %g0, FAST_DMMU_MISS_TT, %tt 3532*5d9d9091SRichard Lowe 3533*5d9d9091SRichard Lowetsb_protfault: 3534*5d9d9091SRichard Lowe /* 3535*5d9d9091SRichard Lowe * we get here if we couldn't find a valid tte in the hash. 3536*5d9d9091SRichard Lowe * 3537*5d9d9091SRichard Lowe * If user and we are at tl>1 we go to window handling code. 3538*5d9d9091SRichard Lowe * 3539*5d9d9091SRichard Lowe * If kernel and the fault is on the same page as our stack 3540*5d9d9091SRichard Lowe * pointer, then we know the stack is bad and the trap handler 3541*5d9d9091SRichard Lowe * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3542*5d9d9091SRichard Lowe * 3543*5d9d9091SRichard Lowe * If this is a kernel trap and tl>1, panic. 3544*5d9d9091SRichard Lowe * 3545*5d9d9091SRichard Lowe * Otherwise we call pagefault. 3546*5d9d9091SRichard Lowe */ 3547*5d9d9091SRichard Lowe cmp %g7, FAST_IMMU_MISS_TT 3548*5d9d9091SRichard Lowe#ifdef sun4v 3549*5d9d9091SRichard Lowe MMU_FAULT_STATUS_AREA(%g4) 3550*5d9d9091SRichard Lowe ldx [%g4 + MMFSA_I_CTX], %g5 3551*5d9d9091SRichard Lowe ldx [%g4 + MMFSA_D_CTX], %g4 3552*5d9d9091SRichard Lowe move %icc, %g5, %g4 3553*5d9d9091SRichard Lowe cmp %g7, T_INSTR_MMU_MISS 3554*5d9d9091SRichard Lowe move %icc, %g5, %g4 3555*5d9d9091SRichard Lowe#else 3556*5d9d9091SRichard Lowe mov MMU_TAG_ACCESS, %g4 3557*5d9d9091SRichard Lowe ldxa [%g4]ASI_DMMU, %g2 3558*5d9d9091SRichard Lowe ldxa [%g4]ASI_IMMU, %g5 3559*5d9d9091SRichard Lowe move %icc, %g5, %g2 3560*5d9d9091SRichard Lowe cmp %g7, T_INSTR_MMU_MISS 3561*5d9d9091SRichard Lowe move %icc, %g5, %g2 3562*5d9d9091SRichard Lowe sllx %g2, TAGACC_CTX_LSHIFT, %g4 3563*5d9d9091SRichard Lowe#endif /* sun4v */ 3564*5d9d9091SRichard Lowe brnz,pn %g4, 3f /* skip if not kernel */ 3565*5d9d9091SRichard Lowe rdpr %tl, %g5 3566*5d9d9091SRichard Lowe 3567*5d9d9091SRichard Lowe add %sp, STACK_BIAS, %g3 3568*5d9d9091SRichard Lowe srlx %g3, MMU_PAGESHIFT, %g3 3569*5d9d9091SRichard Lowe srlx %g2, MMU_PAGESHIFT, %g4 3570*5d9d9091SRichard Lowe cmp %g3, %g4 3571*5d9d9091SRichard Lowe be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 3572*5d9d9091SRichard Lowe mov PTL1_BAD_STACK, %g1 3573*5d9d9091SRichard Lowe 3574*5d9d9091SRichard Lowe cmp %g5, 1 3575*5d9d9091SRichard Lowe ble,pt %icc, 2f 3576*5d9d9091SRichard Lowe nop 3577*5d9d9091SRichard Lowe TSTAT_CHECK_TL1(2f, %g1, %g2) 3578*5d9d9091SRichard Lowe rdpr %tt, %g2 3579*5d9d9091SRichard Lowe cmp %g2, FAST_PROT_TT 3580*5d9d9091SRichard Lowe mov PTL1_BAD_KPROT_FAULT, %g1 3581*5d9d9091SRichard Lowe movne %icc, PTL1_BAD_KMISS, %g1 3582*5d9d9091SRichard Lowe ba,pt %icc, ptl1_panic 3583*5d9d9091SRichard Lowe nop 3584*5d9d9091SRichard Lowe 3585*5d9d9091SRichard Lowe2: 3586*5d9d9091SRichard Lowe /* 3587*5d9d9091SRichard Lowe * We are taking a pagefault in the kernel on a kernel address. If 3588*5d9d9091SRichard Lowe * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 3589*5d9d9091SRichard Lowe * want to call sfmmu_pagefault -- we will instead note that a fault 3590*5d9d9091SRichard Lowe * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 3591*5d9d9091SRichard Lowe * (instead of a "retry"). This will step over the faulting 3592*5d9d9091SRichard Lowe * instruction. 3593*5d9d9091SRichard Lowe */ 3594*5d9d9091SRichard Lowe CPU_INDEX(%g1, %g2) 3595*5d9d9091SRichard Lowe set cpu_core, %g2 3596*5d9d9091SRichard Lowe sllx %g1, CPU_CORE_SHIFT, %g1 3597*5d9d9091SRichard Lowe add %g1, %g2, %g1 3598*5d9d9091SRichard Lowe lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3599*5d9d9091SRichard Lowe andcc %g2, CPU_DTRACE_NOFAULT, %g0 3600*5d9d9091SRichard Lowe bz sfmmu_pagefault 3601*5d9d9091SRichard Lowe or %g2, CPU_DTRACE_BADADDR, %g2 3602*5d9d9091SRichard Lowe stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3603*5d9d9091SRichard Lowe GET_MMU_D_ADDR(%g3, %g4) 3604*5d9d9091SRichard Lowe stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3605*5d9d9091SRichard Lowe done 3606*5d9d9091SRichard Lowe 3607*5d9d9091SRichard Lowe3: 3608*5d9d9091SRichard Lowe cmp %g5, 1 3609*5d9d9091SRichard Lowe ble,pt %icc, 4f 3610*5d9d9091SRichard Lowe nop 3611*5d9d9091SRichard Lowe TSTAT_CHECK_TL1(4f, %g1, %g2) 3612*5d9d9091SRichard Lowe ba,pt %icc, sfmmu_window_trap 3613*5d9d9091SRichard Lowe nop 3614*5d9d9091SRichard Lowe 3615*5d9d9091SRichard Lowe4: 3616*5d9d9091SRichard Lowe /* 3617*5d9d9091SRichard Lowe * We are taking a pagefault on a non-kernel address. If we are in 3618*5d9d9091SRichard Lowe * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 3619*5d9d9091SRichard Lowe * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 3620*5d9d9091SRichard Lowe */ 3621*5d9d9091SRichard Lowe CPU_INDEX(%g1, %g2) 3622*5d9d9091SRichard Lowe set cpu_core, %g2 3623*5d9d9091SRichard Lowe sllx %g1, CPU_CORE_SHIFT, %g1 3624*5d9d9091SRichard Lowe add %g1, %g2, %g1 3625*5d9d9091SRichard Lowe lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3626*5d9d9091SRichard Lowe andcc %g2, CPU_DTRACE_NOFAULT, %g0 3627*5d9d9091SRichard Lowe bz sfmmu_mmu_trap 3628*5d9d9091SRichard Lowe or %g2, CPU_DTRACE_BADADDR, %g2 3629*5d9d9091SRichard Lowe stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3630*5d9d9091SRichard Lowe GET_MMU_D_ADDR(%g3, %g4) 3631*5d9d9091SRichard Lowe stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3632*5d9d9091SRichard Lowe 3633*5d9d9091SRichard Lowe /* 3634*5d9d9091SRichard Lowe * Be sure that we're actually taking this miss from the kernel -- 3635*5d9d9091SRichard Lowe * otherwise we have managed to return to user-level with 3636*5d9d9091SRichard Lowe * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3637*5d9d9091SRichard Lowe */ 3638*5d9d9091SRichard Lowe rdpr %tstate, %g2 3639*5d9d9091SRichard Lowe btst TSTATE_PRIV, %g2 3640*5d9d9091SRichard Lowe bz,a ptl1_panic 3641*5d9d9091SRichard Lowe mov PTL1_BAD_DTRACE_FLAGS, %g1 3642*5d9d9091SRichard Lowe done 3643*5d9d9091SRichard Lowe 3644*5d9d9091SRichard Lowe ALTENTRY(tsb_tl0_noctxt) 3645*5d9d9091SRichard Lowe /* 3646*5d9d9091SRichard Lowe * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 3647*5d9d9091SRichard Lowe * if it is, indicated that we have faulted and issue a done. 3648*5d9d9091SRichard Lowe */ 3649*5d9d9091SRichard Lowe CPU_INDEX(%g5, %g6) 3650*5d9d9091SRichard Lowe set cpu_core, %g6 3651*5d9d9091SRichard Lowe sllx %g5, CPU_CORE_SHIFT, %g5 3652*5d9d9091SRichard Lowe add %g5, %g6, %g5 3653*5d9d9091SRichard Lowe lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 3654*5d9d9091SRichard Lowe andcc %g6, CPU_DTRACE_NOFAULT, %g0 3655*5d9d9091SRichard Lowe bz 1f 3656*5d9d9091SRichard Lowe or %g6, CPU_DTRACE_BADADDR, %g6 3657*5d9d9091SRichard Lowe stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 3658*5d9d9091SRichard Lowe GET_MMU_D_ADDR(%g3, %g4) 3659*5d9d9091SRichard Lowe stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 3660*5d9d9091SRichard Lowe 3661*5d9d9091SRichard Lowe /* 3662*5d9d9091SRichard Lowe * Be sure that we're actually taking this miss from the kernel -- 3663*5d9d9091SRichard Lowe * otherwise we have managed to return to user-level with 3664*5d9d9091SRichard Lowe * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3665*5d9d9091SRichard Lowe */ 3666*5d9d9091SRichard Lowe rdpr %tstate, %g5 3667*5d9d9091SRichard Lowe btst TSTATE_PRIV, %g5 3668*5d9d9091SRichard Lowe bz,a ptl1_panic 3669*5d9d9091SRichard Lowe mov PTL1_BAD_DTRACE_FLAGS, %g1 3670*5d9d9091SRichard Lowe TSTAT_CHECK_TL1(2f, %g1, %g2); 3671*5d9d9091SRichard Lowe2: 3672*5d9d9091SRichard Lowe done 3673*5d9d9091SRichard Lowe 3674*5d9d9091SRichard Lowe1: 3675*5d9d9091SRichard Lowe rdpr %tt, %g5 3676*5d9d9091SRichard Lowe cmp %g5, FAST_IMMU_MISS_TT 3677*5d9d9091SRichard Lowe#ifdef sun4v 3678*5d9d9091SRichard Lowe MMU_FAULT_STATUS_AREA(%g2) 3679*5d9d9091SRichard Lowe be,a,pt %icc, 2f 3680*5d9d9091SRichard Lowe ldx [%g2 + MMFSA_I_CTX], %g3 3681*5d9d9091SRichard Lowe cmp %g5, T_INSTR_MMU_MISS 3682*5d9d9091SRichard Lowe be,a,pt %icc, 2f 3683*5d9d9091SRichard Lowe ldx [%g2 + MMFSA_I_CTX], %g3 3684*5d9d9091SRichard Lowe ldx [%g2 + MMFSA_D_CTX], %g3 3685*5d9d9091SRichard Lowe2: 3686*5d9d9091SRichard Lowe#else 3687*5d9d9091SRichard Lowe mov MMU_TAG_ACCESS, %g2 3688*5d9d9091SRichard Lowe be,a,pt %icc, 2f 3689*5d9d9091SRichard Lowe ldxa [%g2]ASI_IMMU, %g3 3690*5d9d9091SRichard Lowe ldxa [%g2]ASI_DMMU, %g3 3691*5d9d9091SRichard Lowe2: sllx %g3, TAGACC_CTX_LSHIFT, %g3 3692*5d9d9091SRichard Lowe#endif /* sun4v */ 3693*5d9d9091SRichard Lowe brz,a,pn %g3, ptl1_panic ! panic if called for kernel 3694*5d9d9091SRichard Lowe mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 3695*5d9d9091SRichard Lowe rdpr %tl, %g5 3696*5d9d9091SRichard Lowe cmp %g5, 1 3697*5d9d9091SRichard Lowe ble,pt %icc, sfmmu_mmu_trap 3698*5d9d9091SRichard Lowe nop 3699*5d9d9091SRichard Lowe TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 3700*5d9d9091SRichard Lowe ba,pt %icc, sfmmu_window_trap 3701*5d9d9091SRichard Lowe nop 3702*5d9d9091SRichard Lowe SET_SIZE(sfmmu_tsb_miss) 3703*5d9d9091SRichard Lowe 3704*5d9d9091SRichard Lowe /* 3705*5d9d9091SRichard Lowe * This routine will look for a user or kernel vaddr in the hash 3706*5d9d9091SRichard Lowe * structure. It returns a valid pfn or PFN_INVALID. It doesn't grab 3707*5d9d9091SRichard Lowe * any locks. It should only be used by other sfmmu routines. 3708*5d9d9091SRichard Lowe */ 3709*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_vatopfn) 3710*5d9d9091SRichard Lowe /* 3711*5d9d9091SRichard Lowe * disable interrupts 3712*5d9d9091SRichard Lowe */ 3713*5d9d9091SRichard Lowe rdpr %pstate, %o3 3714*5d9d9091SRichard Lowe#ifdef DEBUG 3715*5d9d9091SRichard Lowe PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 3716*5d9d9091SRichard Lowe#endif 3717*5d9d9091SRichard Lowe /* 3718*5d9d9091SRichard Lowe * disable interrupts to protect the TSBMISS area 3719*5d9d9091SRichard Lowe */ 3720*5d9d9091SRichard Lowe andn %o3, PSTATE_IE, %o5 3721*5d9d9091SRichard Lowe wrpr %o5, 0, %pstate 3722*5d9d9091SRichard Lowe 3723*5d9d9091SRichard Lowe /* 3724*5d9d9091SRichard Lowe * o0 = vaddr 3725*5d9d9091SRichard Lowe * o1 = sfmmup 3726*5d9d9091SRichard Lowe * o2 = ttep 3727*5d9d9091SRichard Lowe */ 3728*5d9d9091SRichard Lowe CPU_TSBMISS_AREA(%g1, %o5) 3729*5d9d9091SRichard Lowe ldn [%g1 + TSBMISS_KHATID], %o4 3730*5d9d9091SRichard Lowe cmp %o4, %o1 3731*5d9d9091SRichard Lowe bne,pn %ncc, vatopfn_nokernel 3732*5d9d9091SRichard Lowe mov TTE64K, %g5 /* g5 = rehash # */ 3733*5d9d9091SRichard Lowe mov %g1,%o5 /* o5 = tsbmiss_area */ 3734*5d9d9091SRichard Lowe /* 3735*5d9d9091SRichard Lowe * o0 = vaddr 3736*5d9d9091SRichard Lowe * o1 & o4 = hatid 3737*5d9d9091SRichard Lowe * o2 = ttep 3738*5d9d9091SRichard Lowe * o5 = tsbmiss area 3739*5d9d9091SRichard Lowe */ 3740*5d9d9091SRichard Lowe mov HBLK_RANGE_SHIFT, %g6 3741*5d9d9091SRichard Lowe1: 3742*5d9d9091SRichard Lowe 3743*5d9d9091SRichard Lowe /* 3744*5d9d9091SRichard Lowe * o0 = vaddr 3745*5d9d9091SRichard Lowe * o1 = sfmmup 3746*5d9d9091SRichard Lowe * o2 = ttep 3747*5d9d9091SRichard Lowe * o3 = old %pstate 3748*5d9d9091SRichard Lowe * o4 = hatid 3749*5d9d9091SRichard Lowe * o5 = tsbmiss 3750*5d9d9091SRichard Lowe * g5 = rehash # 3751*5d9d9091SRichard Lowe * g6 = hmeshift 3752*5d9d9091SRichard Lowe * 3753*5d9d9091SRichard Lowe * The first arg to GET_TTE is actually tagaccess register 3754*5d9d9091SRichard Lowe * not just vaddr. Since this call is for kernel we need to clear 3755*5d9d9091SRichard Lowe * any lower vaddr bits that would be interpreted as ctx bits. 3756*5d9d9091SRichard Lowe */ 3757*5d9d9091SRichard Lowe set TAGACC_CTX_MASK, %g1 3758*5d9d9091SRichard Lowe andn %o0, %g1, %o0 3759*5d9d9091SRichard Lowe GET_TTE(%o0, %o4, %g1, %g2, %o5, %g4, %g6, %g5, %g3, 3760*5d9d9091SRichard Lowe vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 3761*5d9d9091SRichard Lowe 3762*5d9d9091SRichard Lowekvtop_hblk_found: 3763*5d9d9091SRichard Lowe /* 3764*5d9d9091SRichard Lowe * o0 = vaddr 3765*5d9d9091SRichard Lowe * o1 = sfmmup 3766*5d9d9091SRichard Lowe * o2 = ttep 3767*5d9d9091SRichard Lowe * g1 = tte 3768*5d9d9091SRichard Lowe * g2 = tte pa 3769*5d9d9091SRichard Lowe * g3 = scratch 3770*5d9d9091SRichard Lowe * o2 = tsbmiss area 3771*5d9d9091SRichard Lowe * o1 = hat id 3772*5d9d9091SRichard Lowe */ 3773*5d9d9091SRichard Lowe brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 3774*5d9d9091SRichard Lowe mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3775*5d9d9091SRichard Lowe stx %g1,[%o2] /* put tte into *ttep */ 3776*5d9d9091SRichard Lowe TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 3777*5d9d9091SRichard Lowe /* 3778*5d9d9091SRichard Lowe * o0 = vaddr 3779*5d9d9091SRichard Lowe * o1 = sfmmup 3780*5d9d9091SRichard Lowe * o2 = ttep 3781*5d9d9091SRichard Lowe * g1 = pfn 3782*5d9d9091SRichard Lowe */ 3783*5d9d9091SRichard Lowe ba,pt %xcc, 6f 3784*5d9d9091SRichard Lowe mov %g1, %o0 3785*5d9d9091SRichard Lowe 3786*5d9d9091SRichard Lowekvtop_nohblk: 3787*5d9d9091SRichard Lowe /* 3788*5d9d9091SRichard Lowe * we get here if we couldn't find valid hblk in hash. We rehash 3789*5d9d9091SRichard Lowe * if neccesary. 3790*5d9d9091SRichard Lowe */ 3791*5d9d9091SRichard Lowe ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 3792*5d9d9091SRichard Lowe#ifdef sun4v 3793*5d9d9091SRichard Lowe cmp %g5, MAX_HASHCNT 3794*5d9d9091SRichard Lowe#else 3795*5d9d9091SRichard Lowe cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 3796*5d9d9091SRichard Lowe#endif /* sun4v */ 3797*5d9d9091SRichard Lowe be,a,pn %icc, 6f 3798*5d9d9091SRichard Lowe mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3799*5d9d9091SRichard Lowe mov %o1, %o4 /* restore hatid */ 3800*5d9d9091SRichard Lowe#ifdef sun4v 3801*5d9d9091SRichard Lowe add %g5, 2, %g5 3802*5d9d9091SRichard Lowe cmp %g5, 3 3803*5d9d9091SRichard Lowe move %icc, MMU_PAGESHIFT4M, %g6 3804*5d9d9091SRichard Lowe ba,pt %icc, 1b 3805*5d9d9091SRichard Lowe movne %icc, MMU_PAGESHIFT256M, %g6 3806*5d9d9091SRichard Lowe#else 3807*5d9d9091SRichard Lowe inc %g5 3808*5d9d9091SRichard Lowe cmp %g5, 2 3809*5d9d9091SRichard Lowe move %icc, MMU_PAGESHIFT512K, %g6 3810*5d9d9091SRichard Lowe ba,pt %icc, 1b 3811*5d9d9091SRichard Lowe movne %icc, MMU_PAGESHIFT4M, %g6 3812*5d9d9091SRichard Lowe#endif /* sun4v */ 3813*5d9d9091SRichard Lowe6: 3814*5d9d9091SRichard Lowe retl 3815*5d9d9091SRichard Lowe wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3816*5d9d9091SRichard Lowe 3817*5d9d9091SRichard Lowetsb_suspend: 3818*5d9d9091SRichard Lowe /* 3819*5d9d9091SRichard Lowe * o0 = vaddr 3820*5d9d9091SRichard Lowe * o1 = sfmmup 3821*5d9d9091SRichard Lowe * o2 = ttep 3822*5d9d9091SRichard Lowe * g1 = tte 3823*5d9d9091SRichard Lowe * g2 = tte pa 3824*5d9d9091SRichard Lowe * g3 = tte va 3825*5d9d9091SRichard Lowe * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 3826*5d9d9091SRichard Lowe */ 3827*5d9d9091SRichard Lowe stx %g1,[%o2] /* put tte into *ttep */ 3828*5d9d9091SRichard Lowe brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 3829*5d9d9091SRichard Lowe sub %g0, 1, %o0 /* output = PFN_INVALID */ 3830*5d9d9091SRichard Lowe sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 3831*5d9d9091SRichard Lowe8: 3832*5d9d9091SRichard Lowe retl 3833*5d9d9091SRichard Lowe wrpr %g0, %o3, %pstate /* enable interrupts */ 3834*5d9d9091SRichard Lowe 3835*5d9d9091SRichard Lowevatopfn_nokernel: 3836*5d9d9091SRichard Lowe /* 3837*5d9d9091SRichard Lowe * This routine does NOT support user addresses 3838*5d9d9091SRichard Lowe * There is a routine in C that supports this. 3839*5d9d9091SRichard Lowe * The only reason why we don't have the C routine 3840*5d9d9091SRichard Lowe * support kernel addresses as well is because 3841*5d9d9091SRichard Lowe * we do va_to_pa while holding the hashlock. 3842*5d9d9091SRichard Lowe */ 3843*5d9d9091SRichard Lowe wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3844*5d9d9091SRichard Lowe save %sp, -SA(MINFRAME), %sp 3845*5d9d9091SRichard Lowe sethi %hi(sfmmu_panic3), %o0 3846*5d9d9091SRichard Lowe call panic 3847*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_panic3), %o0 3848*5d9d9091SRichard Lowe 3849*5d9d9091SRichard Lowe SET_SIZE(sfmmu_vatopfn) 3850*5d9d9091SRichard Lowe 3851*5d9d9091SRichard Lowe /* 3852*5d9d9091SRichard Lowe * %o0 = vaddr 3853*5d9d9091SRichard Lowe * %o1 = hashno (aka szc) 3854*5d9d9091SRichard Lowe * 3855*5d9d9091SRichard Lowe * 3856*5d9d9091SRichard Lowe * This routine is similar to sfmmu_vatopfn() but will only look for 3857*5d9d9091SRichard Lowe * a kernel vaddr in the hash structure for the specified rehash value. 3858*5d9d9091SRichard Lowe * It's just an optimization for the case when pagesize for a given 3859*5d9d9091SRichard Lowe * va range is already known (e.g. large page heap) and we don't want 3860*5d9d9091SRichard Lowe * to start the search with rehash value 1 as sfmmu_vatopfn() does. 3861*5d9d9091SRichard Lowe * 3862*5d9d9091SRichard Lowe * Returns valid pfn or PFN_INVALID if 3863*5d9d9091SRichard Lowe * tte for specified rehash # is not found, invalid or suspended. 3864*5d9d9091SRichard Lowe */ 3865*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_kvaszc2pfn) 3866*5d9d9091SRichard Lowe /* 3867*5d9d9091SRichard Lowe * disable interrupts 3868*5d9d9091SRichard Lowe */ 3869*5d9d9091SRichard Lowe rdpr %pstate, %o3 3870*5d9d9091SRichard Lowe#ifdef DEBUG 3871*5d9d9091SRichard Lowe PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 3872*5d9d9091SRichard Lowe#endif 3873*5d9d9091SRichard Lowe /* 3874*5d9d9091SRichard Lowe * disable interrupts to protect the TSBMISS area 3875*5d9d9091SRichard Lowe */ 3876*5d9d9091SRichard Lowe andn %o3, PSTATE_IE, %o5 3877*5d9d9091SRichard Lowe wrpr %o5, 0, %pstate 3878*5d9d9091SRichard Lowe 3879*5d9d9091SRichard Lowe CPU_TSBMISS_AREA(%g1, %o5) 3880*5d9d9091SRichard Lowe ldn [%g1 + TSBMISS_KHATID], %o4 3881*5d9d9091SRichard Lowe sll %o1, 1, %g6 3882*5d9d9091SRichard Lowe add %g6, %o1, %g6 3883*5d9d9091SRichard Lowe add %g6, MMU_PAGESHIFT, %g6 3884*5d9d9091SRichard Lowe /* 3885*5d9d9091SRichard Lowe * %o0 = vaddr 3886*5d9d9091SRichard Lowe * %o1 = hashno 3887*5d9d9091SRichard Lowe * %o3 = old %pstate 3888*5d9d9091SRichard Lowe * %o4 = ksfmmup 3889*5d9d9091SRichard Lowe * %g1 = tsbmiss area 3890*5d9d9091SRichard Lowe * %g6 = hmeshift 3891*5d9d9091SRichard Lowe */ 3892*5d9d9091SRichard Lowe 3893*5d9d9091SRichard Lowe /* 3894*5d9d9091SRichard Lowe * The first arg to GET_TTE is actually tagaccess register 3895*5d9d9091SRichard Lowe * not just vaddr. Since this call is for kernel we need to clear 3896*5d9d9091SRichard Lowe * any lower vaddr bits that would be interpreted as ctx bits. 3897*5d9d9091SRichard Lowe */ 3898*5d9d9091SRichard Lowe srlx %o0, MMU_PAGESHIFT, %o0 3899*5d9d9091SRichard Lowe sllx %o0, MMU_PAGESHIFT, %o0 3900*5d9d9091SRichard Lowe GET_TTE(%o0, %o4, %g3, %g4, %g1, %o5, %g6, %o1, %g5, 3901*5d9d9091SRichard Lowe kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 3902*5d9d9091SRichard Lowe kvaszc2pfn_nohblk) 3903*5d9d9091SRichard Lowe 3904*5d9d9091SRichard Lowekvaszc2pfn_hblk_found: 3905*5d9d9091SRichard Lowe /* 3906*5d9d9091SRichard Lowe * %g3 = tte 3907*5d9d9091SRichard Lowe * %o0 = vaddr 3908*5d9d9091SRichard Lowe */ 3909*5d9d9091SRichard Lowe brgez,a,pn %g3, 1f /* check if tte is invalid */ 3910*5d9d9091SRichard Lowe mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3911*5d9d9091SRichard Lowe TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 3912*5d9d9091SRichard Lowe /* 3913*5d9d9091SRichard Lowe * g3 = pfn 3914*5d9d9091SRichard Lowe */ 3915*5d9d9091SRichard Lowe ba,pt %xcc, 1f 3916*5d9d9091SRichard Lowe mov %g3, %o0 3917*5d9d9091SRichard Lowe 3918*5d9d9091SRichard Lowekvaszc2pfn_nohblk: 3919*5d9d9091SRichard Lowe mov -1, %o0 3920*5d9d9091SRichard Lowe 3921*5d9d9091SRichard Lowe1: 3922*5d9d9091SRichard Lowe retl 3923*5d9d9091SRichard Lowe wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3924*5d9d9091SRichard Lowe 3925*5d9d9091SRichard Lowe SET_SIZE(sfmmu_kvaszc2pfn) 3926*5d9d9091SRichard Lowe 3927*5d9d9091SRichard Lowe 3928*5d9d9091SRichard Lowe 3929*5d9d9091SRichard Lowe/* 3930*5d9d9091SRichard Lowe * kpm lock used between trap level tsbmiss handler and kpm C level. 3931*5d9d9091SRichard Lowe */ 3932*5d9d9091SRichard Lowe#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 3933*5d9d9091SRichard Lowe mov 0xff, tmp1 ;\ 3934*5d9d9091SRichard Lowelabel1: ;\ 3935*5d9d9091SRichard Lowe casa [kpmlckp]asi, %g0, tmp1 ;\ 3936*5d9d9091SRichard Lowe brnz,pn tmp1, label1 ;\ 3937*5d9d9091SRichard Lowe mov 0xff, tmp1 ;\ 3938*5d9d9091SRichard Lowe membar #LoadLoad 3939*5d9d9091SRichard Lowe 3940*5d9d9091SRichard Lowe#define KPMLOCK_EXIT(kpmlckp, asi) \ 3941*5d9d9091SRichard Lowe membar #LoadStore|#StoreStore ;\ 3942*5d9d9091SRichard Lowe sta %g0, [kpmlckp]asi 3943*5d9d9091SRichard Lowe 3944*5d9d9091SRichard Lowe/* 3945*5d9d9091SRichard Lowe * Lookup a memseg for a given pfn and if found, return the physical 3946*5d9d9091SRichard Lowe * address of the corresponding struct memseg in mseg, otherwise 3947*5d9d9091SRichard Lowe * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 3948*5d9d9091SRichard Lowe * tsbmp, %asi is assumed to be ASI_MEM. 3949*5d9d9091SRichard Lowe * This lookup is done by strictly traversing only the physical memseg 3950*5d9d9091SRichard Lowe * linkage. The more generic approach, to check the virtual linkage 3951*5d9d9091SRichard Lowe * before using the physical (used e.g. with hmehash buckets), cannot 3952*5d9d9091SRichard Lowe * be used here. Memory DR operations can run in parallel to this 3953*5d9d9091SRichard Lowe * lookup w/o any locks and updates of the physical and virtual linkage 3954*5d9d9091SRichard Lowe * cannot be done atomically wrt. to each other. Because physical 3955*5d9d9091SRichard Lowe * address zero can be valid physical address, MSEG_NULLPTR_PA acts 3956*5d9d9091SRichard Lowe * as "physical NULL" pointer. 3957*5d9d9091SRichard Lowe */ 3958*5d9d9091SRichard Lowe#define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 3959*5d9d9091SRichard Lowe sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 3960*5d9d9091SRichard Lowe ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 3961*5d9d9091SRichard Lowe udivx pfn, mseg, mseg ;\ 3962*5d9d9091SRichard Lowe ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 3963*5d9d9091SRichard Lowe and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 3964*5d9d9091SRichard Lowe sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 3965*5d9d9091SRichard Lowe add tmp1, mseg, tmp1 ;\ 3966*5d9d9091SRichard Lowe ldxa [tmp1]%asi, mseg ;\ 3967*5d9d9091SRichard Lowe cmp mseg, MSEG_NULLPTR_PA ;\ 3968*5d9d9091SRichard Lowe be,pn %xcc, label##1 /* if not found */ ;\ 3969*5d9d9091SRichard Lowe nop ;\ 3970*5d9d9091SRichard Lowe ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 3971*5d9d9091SRichard Lowe cmp pfn, tmp1 /* pfn - pages_base */ ;\ 3972*5d9d9091SRichard Lowe blu,pn %xcc, label##1 ;\ 3973*5d9d9091SRichard Lowe ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 3974*5d9d9091SRichard Lowe cmp pfn, tmp2 /* pfn - pages_end */ ;\ 3975*5d9d9091SRichard Lowe bgeu,pn %xcc, label##1 ;\ 3976*5d9d9091SRichard Lowe sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 3977*5d9d9091SRichard Lowe mulx tmp1, PAGE_SIZE, tmp1 ;\ 3978*5d9d9091SRichard Lowe ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 3979*5d9d9091SRichard Lowe add tmp2, tmp1, tmp1 /* pp */ ;\ 3980*5d9d9091SRichard Lowe lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 3981*5d9d9091SRichard Lowe cmp tmp2, pfn ;\ 3982*5d9d9091SRichard Lowe be,pt %xcc, label##_ok /* found */ ;\ 3983*5d9d9091SRichard Lowelabel##1: ;\ 3984*5d9d9091SRichard Lowe /* brute force lookup */ ;\ 3985*5d9d9091SRichard Lowe sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 3986*5d9d9091SRichard Lowe ldx [tmp3 + %lo(memsegspa)], mseg ;\ 3987*5d9d9091SRichard Lowelabel##2: ;\ 3988*5d9d9091SRichard Lowe cmp mseg, MSEG_NULLPTR_PA ;\ 3989*5d9d9091SRichard Lowe be,pn %xcc, label##_ok /* if not found */ ;\ 3990*5d9d9091SRichard Lowe nop ;\ 3991*5d9d9091SRichard Lowe ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 3992*5d9d9091SRichard Lowe cmp pfn, tmp1 /* pfn - pages_base */ ;\ 3993*5d9d9091SRichard Lowe blu,a,pt %xcc, label##2 ;\ 3994*5d9d9091SRichard Lowe ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 3995*5d9d9091SRichard Lowe ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 3996*5d9d9091SRichard Lowe cmp pfn, tmp2 /* pfn - pages_end */ ;\ 3997*5d9d9091SRichard Lowe bgeu,a,pt %xcc, label##2 ;\ 3998*5d9d9091SRichard Lowe ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 3999*5d9d9091SRichard Lowelabel##_ok: 4000*5d9d9091SRichard Lowe 4001*5d9d9091SRichard Lowe /* 4002*5d9d9091SRichard Lowe * kpm tsb miss handler large pages 4003*5d9d9091SRichard Lowe * g1 = 8K kpm TSB entry pointer 4004*5d9d9091SRichard Lowe * g2 = tag access register 4005*5d9d9091SRichard Lowe * g3 = 4M kpm TSB entry pointer 4006*5d9d9091SRichard Lowe */ 4007*5d9d9091SRichard Lowe ALTENTRY(sfmmu_kpm_dtsb_miss) 4008*5d9d9091SRichard Lowe TT_TRACE(trace_tsbmiss) 4009*5d9d9091SRichard Lowe 4010*5d9d9091SRichard Lowe CPU_INDEX(%g7, %g6) 4011*5d9d9091SRichard Lowe sethi %hi(kpmtsbm_area), %g6 4012*5d9d9091SRichard Lowe sllx %g7, KPMTSBM_SHIFT, %g7 4013*5d9d9091SRichard Lowe or %g6, %lo(kpmtsbm_area), %g6 4014*5d9d9091SRichard Lowe add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4015*5d9d9091SRichard Lowe 4016*5d9d9091SRichard Lowe /* check enable flag */ 4017*5d9d9091SRichard Lowe ldub [%g6 + KPMTSBM_FLAGS], %g4 4018*5d9d9091SRichard Lowe and %g4, KPMTSBM_ENABLE_FLAG, %g5 4019*5d9d9091SRichard Lowe brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4020*5d9d9091SRichard Lowe nop 4021*5d9d9091SRichard Lowe 4022*5d9d9091SRichard Lowe /* VA range check */ 4023*5d9d9091SRichard Lowe ldx [%g6 + KPMTSBM_VBASE], %g7 4024*5d9d9091SRichard Lowe cmp %g2, %g7 4025*5d9d9091SRichard Lowe blu,pn %xcc, sfmmu_tsb_miss 4026*5d9d9091SRichard Lowe ldx [%g6 + KPMTSBM_VEND], %g5 4027*5d9d9091SRichard Lowe cmp %g2, %g5 4028*5d9d9091SRichard Lowe bgeu,pn %xcc, sfmmu_tsb_miss 4029*5d9d9091SRichard Lowe stx %g3, [%g6 + KPMTSBM_TSBPTR] 4030*5d9d9091SRichard Lowe 4031*5d9d9091SRichard Lowe /* 4032*5d9d9091SRichard Lowe * check TL tsbmiss handling flag 4033*5d9d9091SRichard Lowe * bump tsbmiss counter 4034*5d9d9091SRichard Lowe */ 4035*5d9d9091SRichard Lowe lduw [%g6 + KPMTSBM_TSBMISS], %g5 4036*5d9d9091SRichard Lowe#ifdef DEBUG 4037*5d9d9091SRichard Lowe and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4038*5d9d9091SRichard Lowe inc %g5 4039*5d9d9091SRichard Lowe brz,pn %g3, sfmmu_kpm_exception 4040*5d9d9091SRichard Lowe st %g5, [%g6 + KPMTSBM_TSBMISS] 4041*5d9d9091SRichard Lowe#else 4042*5d9d9091SRichard Lowe inc %g5 4043*5d9d9091SRichard Lowe st %g5, [%g6 + KPMTSBM_TSBMISS] 4044*5d9d9091SRichard Lowe#endif 4045*5d9d9091SRichard Lowe /* 4046*5d9d9091SRichard Lowe * At this point: 4047*5d9d9091SRichard Lowe * g1 = 8K kpm TSB pointer (not used) 4048*5d9d9091SRichard Lowe * g2 = tag access register 4049*5d9d9091SRichard Lowe * g3 = clobbered 4050*5d9d9091SRichard Lowe * g6 = per-CPU kpm tsbmiss area 4051*5d9d9091SRichard Lowe * g7 = kpm_vbase 4052*5d9d9091SRichard Lowe */ 4053*5d9d9091SRichard Lowe 4054*5d9d9091SRichard Lowe /* vaddr2pfn */ 4055*5d9d9091SRichard Lowe ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4056*5d9d9091SRichard Lowe sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4057*5d9d9091SRichard Lowe srax %g4, %g3, %g2 /* which alias range (r) */ 4058*5d9d9091SRichard Lowe brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4059*5d9d9091SRichard Lowe srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4060*5d9d9091SRichard Lowe 4061*5d9d9091SRichard Lowe /* 4062*5d9d9091SRichard Lowe * Setup %asi 4063*5d9d9091SRichard Lowe * mseg_pa = page_numtomemseg_nolock(pfn) 4064*5d9d9091SRichard Lowe * if (mseg_pa == NULL) sfmmu_kpm_exception 4065*5d9d9091SRichard Lowe * g2=pfn 4066*5d9d9091SRichard Lowe */ 4067*5d9d9091SRichard Lowe mov ASI_MEM, %asi 4068*5d9d9091SRichard Lowe PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4069*5d9d9091SRichard Lowe cmp %g3, MSEG_NULLPTR_PA 4070*5d9d9091SRichard Lowe be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4071*5d9d9091SRichard Lowe nop 4072*5d9d9091SRichard Lowe 4073*5d9d9091SRichard Lowe /* 4074*5d9d9091SRichard Lowe * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4075*5d9d9091SRichard Lowe * g2=pfn g3=mseg_pa 4076*5d9d9091SRichard Lowe */ 4077*5d9d9091SRichard Lowe ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4078*5d9d9091SRichard Lowe ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4079*5d9d9091SRichard Lowe srlx %g2, %g5, %g4 4080*5d9d9091SRichard Lowe sllx %g4, %g5, %g4 4081*5d9d9091SRichard Lowe sub %g4, %g7, %g4 4082*5d9d9091SRichard Lowe srlx %g4, %g5, %g4 4083*5d9d9091SRichard Lowe 4084*5d9d9091SRichard Lowe /* 4085*5d9d9091SRichard Lowe * Validate inx value 4086*5d9d9091SRichard Lowe * g2=pfn g3=mseg_pa g4=inx 4087*5d9d9091SRichard Lowe */ 4088*5d9d9091SRichard Lowe#ifdef DEBUG 4089*5d9d9091SRichard Lowe ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4090*5d9d9091SRichard Lowe cmp %g4, %g5 /* inx - nkpmpgs */ 4091*5d9d9091SRichard Lowe bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4092*5d9d9091SRichard Lowe ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4093*5d9d9091SRichard Lowe#else 4094*5d9d9091SRichard Lowe ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4095*5d9d9091SRichard Lowe#endif 4096*5d9d9091SRichard Lowe /* 4097*5d9d9091SRichard Lowe * kp = &mseg_pa->kpm_pages[inx] 4098*5d9d9091SRichard Lowe */ 4099*5d9d9091SRichard Lowe sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4100*5d9d9091SRichard Lowe ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4101*5d9d9091SRichard Lowe add %g5, %g4, %g5 /* kp */ 4102*5d9d9091SRichard Lowe 4103*5d9d9091SRichard Lowe /* 4104*5d9d9091SRichard Lowe * KPMP_HASH(kp) 4105*5d9d9091SRichard Lowe * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4106*5d9d9091SRichard Lowe */ 4107*5d9d9091SRichard Lowe ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4108*5d9d9091SRichard Lowe sub %g7, 1, %g7 /* mask */ 4109*5d9d9091SRichard Lowe srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4110*5d9d9091SRichard Lowe add %g5, %g1, %g5 /* y = ksp + x */ 4111*5d9d9091SRichard Lowe and %g5, %g7, %g5 /* hashinx = y & mask */ 4112*5d9d9091SRichard Lowe 4113*5d9d9091SRichard Lowe /* 4114*5d9d9091SRichard Lowe * Calculate physical kpm_page pointer 4115*5d9d9091SRichard Lowe * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4116*5d9d9091SRichard Lowe */ 4117*5d9d9091SRichard Lowe ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4118*5d9d9091SRichard Lowe add %g1, %g4, %g1 /* kp_pa */ 4119*5d9d9091SRichard Lowe 4120*5d9d9091SRichard Lowe /* 4121*5d9d9091SRichard Lowe * Calculate physical hash lock address 4122*5d9d9091SRichard Lowe * g1=kp_refcntc_pa g2=pfn g5=hashinx 4123*5d9d9091SRichard Lowe */ 4124*5d9d9091SRichard Lowe ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4125*5d9d9091SRichard Lowe sllx %g5, KPMHLK_SHIFT, %g5 4126*5d9d9091SRichard Lowe add %g4, %g5, %g3 4127*5d9d9091SRichard Lowe add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4128*5d9d9091SRichard Lowe 4129*5d9d9091SRichard Lowe /* 4130*5d9d9091SRichard Lowe * Assemble tte 4131*5d9d9091SRichard Lowe * g1=kp_pa g2=pfn g3=hlck_pa 4132*5d9d9091SRichard Lowe */ 4133*5d9d9091SRichard Lowe#ifdef sun4v 4134*5d9d9091SRichard Lowe sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4135*5d9d9091SRichard Lowe sllx %g5, 32, %g5 4136*5d9d9091SRichard Lowe mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4137*5d9d9091SRichard Lowe or %g4, TTE4M, %g4 4138*5d9d9091SRichard Lowe or %g5, %g4, %g5 4139*5d9d9091SRichard Lowe#else 4140*5d9d9091SRichard Lowe sethi %hi(TTE_VALID_INT), %g4 4141*5d9d9091SRichard Lowe mov TTE4M, %g5 4142*5d9d9091SRichard Lowe sllx %g5, TTE_SZ_SHFT_INT, %g5 4143*5d9d9091SRichard Lowe or %g5, %g4, %g5 /* upper part */ 4144*5d9d9091SRichard Lowe sllx %g5, 32, %g5 4145*5d9d9091SRichard Lowe mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4146*5d9d9091SRichard Lowe or %g5, %g4, %g5 4147*5d9d9091SRichard Lowe#endif 4148*5d9d9091SRichard Lowe sllx %g2, MMU_PAGESHIFT, %g4 4149*5d9d9091SRichard Lowe or %g5, %g4, %g5 /* tte */ 4150*5d9d9091SRichard Lowe ldx [%g6 + KPMTSBM_TSBPTR], %g4 4151*5d9d9091SRichard Lowe GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4152*5d9d9091SRichard Lowe 4153*5d9d9091SRichard Lowe /* 4154*5d9d9091SRichard Lowe * tsb dropin 4155*5d9d9091SRichard Lowe * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4156*5d9d9091SRichard Lowe */ 4157*5d9d9091SRichard Lowe 4158*5d9d9091SRichard Lowe /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4159*5d9d9091SRichard Lowe KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4160*5d9d9091SRichard Lowe 4161*5d9d9091SRichard Lowe /* use C-handler if there's no go for dropin */ 4162*5d9d9091SRichard Lowe ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4163*5d9d9091SRichard Lowe cmp %g7, -1 4164*5d9d9091SRichard Lowe bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4165*5d9d9091SRichard Lowe nop 4166*5d9d9091SRichard Lowe 4167*5d9d9091SRichard Lowe#ifdef DEBUG 4168*5d9d9091SRichard Lowe /* double check refcnt */ 4169*5d9d9091SRichard Lowe ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4170*5d9d9091SRichard Lowe brz,pn %g7, 5f /* let C-handler deal with this */ 4171*5d9d9091SRichard Lowe nop 4172*5d9d9091SRichard Lowe#endif 4173*5d9d9091SRichard Lowe 4174*5d9d9091SRichard Lowe#ifndef sun4v 4175*5d9d9091SRichard Lowe ldub [%g6 + KPMTSBM_FLAGS], %g7 4176*5d9d9091SRichard Lowe mov ASI_N, %g1 4177*5d9d9091SRichard Lowe andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4178*5d9d9091SRichard Lowe movnz %icc, ASI_MEM, %g1 4179*5d9d9091SRichard Lowe mov %g1, %asi 4180*5d9d9091SRichard Lowe#endif 4181*5d9d9091SRichard Lowe 4182*5d9d9091SRichard Lowe /* 4183*5d9d9091SRichard Lowe * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4184*5d9d9091SRichard Lowe * If we fail to lock the TSB entry then just load the tte into the 4185*5d9d9091SRichard Lowe * TLB. 4186*5d9d9091SRichard Lowe */ 4187*5d9d9091SRichard Lowe TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l1) 4188*5d9d9091SRichard Lowe 4189*5d9d9091SRichard Lowe /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4190*5d9d9091SRichard Lowe TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4191*5d9d9091SRichard Lowelocked_tsb_l1: 4192*5d9d9091SRichard Lowe DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4193*5d9d9091SRichard Lowe 4194*5d9d9091SRichard Lowe /* KPMLOCK_EXIT(kpmlckp, asi) */ 4195*5d9d9091SRichard Lowe KPMLOCK_EXIT(%g3, ASI_MEM) 4196*5d9d9091SRichard Lowe 4197*5d9d9091SRichard Lowe /* 4198*5d9d9091SRichard Lowe * If trapstat is running, we need to shift the %tpc and %tnpc to 4199*5d9d9091SRichard Lowe * point to trapstat's TSB miss return code (note that trapstat 4200*5d9d9091SRichard Lowe * itself will patch the correct offset to add). 4201*5d9d9091SRichard Lowe * Note: TTE is expected in %g5 (allows per pagesize reporting). 4202*5d9d9091SRichard Lowe */ 4203*5d9d9091SRichard Lowe rdpr %tl, %g7 4204*5d9d9091SRichard Lowe cmp %g7, 1 4205*5d9d9091SRichard Lowe ble %icc, 0f 4206*5d9d9091SRichard Lowe sethi %hi(KERNELBASE), %g6 4207*5d9d9091SRichard Lowe rdpr %tpc, %g7 4208*5d9d9091SRichard Lowe or %g6, %lo(KERNELBASE), %g6 4209*5d9d9091SRichard Lowe cmp %g7, %g6 4210*5d9d9091SRichard Lowe bgeu %xcc, 0f 4211*5d9d9091SRichard Lowe ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4212*5d9d9091SRichard Lowe add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4213*5d9d9091SRichard Lowe wrpr %g7, %tpc 4214*5d9d9091SRichard Lowe add %g7, 4, %g7 4215*5d9d9091SRichard Lowe wrpr %g7, %tnpc 4216*5d9d9091SRichard Lowe0: 4217*5d9d9091SRichard Lowe retry 4218*5d9d9091SRichard Lowe5: 4219*5d9d9091SRichard Lowe /* g3=hlck_pa */ 4220*5d9d9091SRichard Lowe KPMLOCK_EXIT(%g3, ASI_MEM) 4221*5d9d9091SRichard Lowe ba,pt %icc, sfmmu_kpm_exception 4222*5d9d9091SRichard Lowe nop 4223*5d9d9091SRichard Lowe SET_SIZE(sfmmu_kpm_dtsb_miss) 4224*5d9d9091SRichard Lowe 4225*5d9d9091SRichard Lowe /* 4226*5d9d9091SRichard Lowe * kpm tsbmiss handler for smallpages 4227*5d9d9091SRichard Lowe * g1 = 8K kpm TSB pointer 4228*5d9d9091SRichard Lowe * g2 = tag access register 4229*5d9d9091SRichard Lowe * g3 = 4M kpm TSB pointer 4230*5d9d9091SRichard Lowe */ 4231*5d9d9091SRichard Lowe ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4232*5d9d9091SRichard Lowe TT_TRACE(trace_tsbmiss) 4233*5d9d9091SRichard Lowe CPU_INDEX(%g7, %g6) 4234*5d9d9091SRichard Lowe sethi %hi(kpmtsbm_area), %g6 4235*5d9d9091SRichard Lowe sllx %g7, KPMTSBM_SHIFT, %g7 4236*5d9d9091SRichard Lowe or %g6, %lo(kpmtsbm_area), %g6 4237*5d9d9091SRichard Lowe add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4238*5d9d9091SRichard Lowe 4239*5d9d9091SRichard Lowe /* check enable flag */ 4240*5d9d9091SRichard Lowe ldub [%g6 + KPMTSBM_FLAGS], %g4 4241*5d9d9091SRichard Lowe and %g4, KPMTSBM_ENABLE_FLAG, %g5 4242*5d9d9091SRichard Lowe brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4243*5d9d9091SRichard Lowe nop 4244*5d9d9091SRichard Lowe 4245*5d9d9091SRichard Lowe /* 4246*5d9d9091SRichard Lowe * VA range check 4247*5d9d9091SRichard Lowe * On fail: goto sfmmu_tsb_miss 4248*5d9d9091SRichard Lowe */ 4249*5d9d9091SRichard Lowe ldx [%g6 + KPMTSBM_VBASE], %g7 4250*5d9d9091SRichard Lowe cmp %g2, %g7 4251*5d9d9091SRichard Lowe blu,pn %xcc, sfmmu_tsb_miss 4252*5d9d9091SRichard Lowe ldx [%g6 + KPMTSBM_VEND], %g5 4253*5d9d9091SRichard Lowe cmp %g2, %g5 4254*5d9d9091SRichard Lowe bgeu,pn %xcc, sfmmu_tsb_miss 4255*5d9d9091SRichard Lowe stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4256*5d9d9091SRichard Lowe 4257*5d9d9091SRichard Lowe /* 4258*5d9d9091SRichard Lowe * check TL tsbmiss handling flag 4259*5d9d9091SRichard Lowe * bump tsbmiss counter 4260*5d9d9091SRichard Lowe */ 4261*5d9d9091SRichard Lowe lduw [%g6 + KPMTSBM_TSBMISS], %g5 4262*5d9d9091SRichard Lowe#ifdef DEBUG 4263*5d9d9091SRichard Lowe and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4264*5d9d9091SRichard Lowe inc %g5 4265*5d9d9091SRichard Lowe brz,pn %g1, sfmmu_kpm_exception 4266*5d9d9091SRichard Lowe st %g5, [%g6 + KPMTSBM_TSBMISS] 4267*5d9d9091SRichard Lowe#else 4268*5d9d9091SRichard Lowe inc %g5 4269*5d9d9091SRichard Lowe st %g5, [%g6 + KPMTSBM_TSBMISS] 4270*5d9d9091SRichard Lowe#endif 4271*5d9d9091SRichard Lowe /* 4272*5d9d9091SRichard Lowe * At this point: 4273*5d9d9091SRichard Lowe * g1 = clobbered 4274*5d9d9091SRichard Lowe * g2 = tag access register 4275*5d9d9091SRichard Lowe * g3 = 4M kpm TSB pointer (not used) 4276*5d9d9091SRichard Lowe * g6 = per-CPU kpm tsbmiss area 4277*5d9d9091SRichard Lowe * g7 = kpm_vbase 4278*5d9d9091SRichard Lowe */ 4279*5d9d9091SRichard Lowe 4280*5d9d9091SRichard Lowe /* 4281*5d9d9091SRichard Lowe * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr) 4282*5d9d9091SRichard Lowe * which is defined in mach_kpm.h. Any changes in that macro 4283*5d9d9091SRichard Lowe * should also be ported back to this assembly code. 4284*5d9d9091SRichard Lowe */ 4285*5d9d9091SRichard Lowe ldub [%g6 + KPMTSBM_SZSHIFT], %g3 /* g3 = kpm_size_shift */ 4286*5d9d9091SRichard Lowe sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4287*5d9d9091SRichard Lowe srax %g4, %g3, %g7 /* which alias range (r) */ 4288*5d9d9091SRichard Lowe brz,pt %g7, 2f 4289*5d9d9091SRichard Lowe sethi %hi(vac_colors_mask), %g5 4290*5d9d9091SRichard Lowe ld [%g5 + %lo(vac_colors_mask)], %g5 4291*5d9d9091SRichard Lowe 4292*5d9d9091SRichard Lowe srlx %g2, MMU_PAGESHIFT, %g1 /* vaddr >> MMU_PAGESHIFT */ 4293*5d9d9091SRichard Lowe and %g1, %g5, %g1 /* g1 = v */ 4294*5d9d9091SRichard Lowe sllx %g7, %g3, %g5 /* g5 = r << kpm_size_shift */ 4295*5d9d9091SRichard Lowe cmp %g7, %g1 /* if (r > v) */ 4296*5d9d9091SRichard Lowe bleu,pn %xcc, 1f 4297*5d9d9091SRichard Lowe sub %g4, %g5, %g4 /* paddr -= r << kpm_size_shift */ 4298*5d9d9091SRichard Lowe sub %g7, %g1, %g5 /* g5 = r - v */ 4299*5d9d9091SRichard Lowe sllx %g5, MMU_PAGESHIFT, %g7 /* (r-v) << MMU_PAGESHIFT */ 4300*5d9d9091SRichard Lowe add %g4, %g7, %g4 /* paddr += (r-v)<<MMU_PAGESHIFT */ 4301*5d9d9091SRichard Lowe ba 2f 4302*5d9d9091SRichard Lowe nop 4303*5d9d9091SRichard Lowe1: 4304*5d9d9091SRichard Lowe sllx %g7, MMU_PAGESHIFT, %g5 /* else */ 4305*5d9d9091SRichard Lowe sub %g4, %g5, %g4 /* paddr -= r << MMU_PAGESHIFT */ 4306*5d9d9091SRichard Lowe 4307*5d9d9091SRichard Lowe /* 4308*5d9d9091SRichard Lowe * paddr2pfn 4309*5d9d9091SRichard Lowe * g1 = vcolor (not used) 4310*5d9d9091SRichard Lowe * g2 = tag access register 4311*5d9d9091SRichard Lowe * g3 = clobbered 4312*5d9d9091SRichard Lowe * g4 = paddr 4313*5d9d9091SRichard Lowe * g5 = clobbered 4314*5d9d9091SRichard Lowe * g6 = per-CPU kpm tsbmiss area 4315*5d9d9091SRichard Lowe * g7 = clobbered 4316*5d9d9091SRichard Lowe */ 4317*5d9d9091SRichard Lowe2: 4318*5d9d9091SRichard Lowe srlx %g4, MMU_PAGESHIFT, %g2 /* g2 = pfn */ 4319*5d9d9091SRichard Lowe 4320*5d9d9091SRichard Lowe /* 4321*5d9d9091SRichard Lowe * Setup %asi 4322*5d9d9091SRichard Lowe * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4323*5d9d9091SRichard Lowe * if (mseg not found) sfmmu_kpm_exception 4324*5d9d9091SRichard Lowe * g2=pfn g6=per-CPU kpm tsbmiss area 4325*5d9d9091SRichard Lowe * g4 g5 g7 for scratch use. 4326*5d9d9091SRichard Lowe */ 4327*5d9d9091SRichard Lowe mov ASI_MEM, %asi 4328*5d9d9091SRichard Lowe PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4329*5d9d9091SRichard Lowe cmp %g3, MSEG_NULLPTR_PA 4330*5d9d9091SRichard Lowe be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4331*5d9d9091SRichard Lowe nop 4332*5d9d9091SRichard Lowe 4333*5d9d9091SRichard Lowe /* 4334*5d9d9091SRichard Lowe * inx = pfn - mseg_pa->kpm_pbase 4335*5d9d9091SRichard Lowe * g2=pfn g3=mseg_pa g6=per-CPU kpm tsbmiss area 4336*5d9d9091SRichard Lowe */ 4337*5d9d9091SRichard Lowe ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4338*5d9d9091SRichard Lowe sub %g2, %g7, %g4 4339*5d9d9091SRichard Lowe 4340*5d9d9091SRichard Lowe#ifdef DEBUG 4341*5d9d9091SRichard Lowe /* 4342*5d9d9091SRichard Lowe * Validate inx value 4343*5d9d9091SRichard Lowe * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area 4344*5d9d9091SRichard Lowe */ 4345*5d9d9091SRichard Lowe ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4346*5d9d9091SRichard Lowe cmp %g4, %g5 /* inx - nkpmpgs */ 4347*5d9d9091SRichard Lowe bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4348*5d9d9091SRichard Lowe ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4349*5d9d9091SRichard Lowe#else 4350*5d9d9091SRichard Lowe ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4351*5d9d9091SRichard Lowe#endif 4352*5d9d9091SRichard Lowe /* ksp = &mseg_pa->kpm_spages[inx] */ 4353*5d9d9091SRichard Lowe ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4354*5d9d9091SRichard Lowe add %g5, %g4, %g5 /* ksp */ 4355*5d9d9091SRichard Lowe 4356*5d9d9091SRichard Lowe /* 4357*5d9d9091SRichard Lowe * KPMP_SHASH(kp) 4358*5d9d9091SRichard Lowe * g2=pfn g3=mseg_pa g4=inx g5=ksp 4359*5d9d9091SRichard Lowe * g6=per-CPU kpm tsbmiss area g7=kpmp_stable_sz 4360*5d9d9091SRichard Lowe */ 4361*5d9d9091SRichard Lowe ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4362*5d9d9091SRichard Lowe sub %g7, 1, %g7 /* mask */ 4363*5d9d9091SRichard Lowe sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4364*5d9d9091SRichard Lowe add %g5, %g1, %g5 /* y = ksp + x */ 4365*5d9d9091SRichard Lowe and %g5, %g7, %g5 /* hashinx = y & mask */ 4366*5d9d9091SRichard Lowe 4367*5d9d9091SRichard Lowe /* 4368*5d9d9091SRichard Lowe * Calculate physical kpm_spage pointer 4369*5d9d9091SRichard Lowe * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4370*5d9d9091SRichard Lowe * g6=per-CPU kpm tsbmiss area 4371*5d9d9091SRichard Lowe */ 4372*5d9d9091SRichard Lowe ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4373*5d9d9091SRichard Lowe add %g1, %g4, %g1 /* ksp_pa */ 4374*5d9d9091SRichard Lowe 4375*5d9d9091SRichard Lowe /* 4376*5d9d9091SRichard Lowe * Calculate physical hash lock address. 4377*5d9d9091SRichard Lowe * Note: Changes in kpm_shlk_t must be reflected here. 4378*5d9d9091SRichard Lowe * g1=ksp_pa g2=pfn g5=hashinx 4379*5d9d9091SRichard Lowe * g6=per-CPU kpm tsbmiss area 4380*5d9d9091SRichard Lowe */ 4381*5d9d9091SRichard Lowe ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4382*5d9d9091SRichard Lowe sllx %g5, KPMSHLK_SHIFT, %g5 4383*5d9d9091SRichard Lowe add %g4, %g5, %g3 /* hlck_pa */ 4384*5d9d9091SRichard Lowe 4385*5d9d9091SRichard Lowe /* 4386*5d9d9091SRichard Lowe * Assemble non-cacheable tte initially 4387*5d9d9091SRichard Lowe * g1=ksp_pa g2=pfn g3=hlck_pa 4388*5d9d9091SRichard Lowe * g6=per-CPU kpm tsbmiss area 4389*5d9d9091SRichard Lowe */ 4390*5d9d9091SRichard Lowe sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4391*5d9d9091SRichard Lowe sllx %g5, 32, %g5 4392*5d9d9091SRichard Lowe mov (TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4393*5d9d9091SRichard Lowe or %g5, %g4, %g5 4394*5d9d9091SRichard Lowe sllx %g2, MMU_PAGESHIFT, %g4 4395*5d9d9091SRichard Lowe or %g5, %g4, %g5 /* tte */ 4396*5d9d9091SRichard Lowe ldx [%g6 + KPMTSBM_TSBPTR], %g4 4397*5d9d9091SRichard Lowe GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4398*5d9d9091SRichard Lowe 4399*5d9d9091SRichard Lowe /* 4400*5d9d9091SRichard Lowe * tsb dropin 4401*5d9d9091SRichard Lowe * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable) 4402*5d9d9091SRichard Lowe * g6=per-CPU kpm tsbmiss area g7=scratch register 4403*5d9d9091SRichard Lowe */ 4404*5d9d9091SRichard Lowe 4405*5d9d9091SRichard Lowe /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4406*5d9d9091SRichard Lowe KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4407*5d9d9091SRichard Lowe 4408*5d9d9091SRichard Lowe /* use C-handler if there's no go for dropin */ 4409*5d9d9091SRichard Lowe ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4410*5d9d9091SRichard Lowe andcc %g7, KPM_MAPPED_GO, %g0 /* go or no go ? */ 4411*5d9d9091SRichard Lowe bz,pt %icc, 5f /* no go */ 4412*5d9d9091SRichard Lowe nop 4413*5d9d9091SRichard Lowe and %g7, KPM_MAPPED_MASK, %g7 /* go */ 4414*5d9d9091SRichard Lowe cmp %g7, KPM_MAPPEDS /* cacheable ? */ 4415*5d9d9091SRichard Lowe be,a,pn %xcc, 3f 4416*5d9d9091SRichard Lowe or %g5, TTE_CV_INT, %g5 /* cacheable */ 4417*5d9d9091SRichard Lowe3: 4418*5d9d9091SRichard Lowe#ifndef sun4v 4419*5d9d9091SRichard Lowe ldub [%g6 + KPMTSBM_FLAGS], %g7 4420*5d9d9091SRichard Lowe mov ASI_N, %g1 4421*5d9d9091SRichard Lowe andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4422*5d9d9091SRichard Lowe movnz %icc, ASI_MEM, %g1 4423*5d9d9091SRichard Lowe mov %g1, %asi 4424*5d9d9091SRichard Lowe#endif 4425*5d9d9091SRichard Lowe 4426*5d9d9091SRichard Lowe /* 4427*5d9d9091SRichard Lowe * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4428*5d9d9091SRichard Lowe * If we fail to lock the TSB entry then just load the tte into the 4429*5d9d9091SRichard Lowe * TLB. 4430*5d9d9091SRichard Lowe */ 4431*5d9d9091SRichard Lowe TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l2) 4432*5d9d9091SRichard Lowe 4433*5d9d9091SRichard Lowe /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4434*5d9d9091SRichard Lowe TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4435*5d9d9091SRichard Lowelocked_tsb_l2: 4436*5d9d9091SRichard Lowe DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4437*5d9d9091SRichard Lowe 4438*5d9d9091SRichard Lowe /* KPMLOCK_EXIT(kpmlckp, asi) */ 4439*5d9d9091SRichard Lowe KPMLOCK_EXIT(%g3, ASI_MEM) 4440*5d9d9091SRichard Lowe 4441*5d9d9091SRichard Lowe /* 4442*5d9d9091SRichard Lowe * If trapstat is running, we need to shift the %tpc and %tnpc to 4443*5d9d9091SRichard Lowe * point to trapstat's TSB miss return code (note that trapstat 4444*5d9d9091SRichard Lowe * itself will patch the correct offset to add). 4445*5d9d9091SRichard Lowe * Note: TTE is expected in %g5 (allows per pagesize reporting). 4446*5d9d9091SRichard Lowe */ 4447*5d9d9091SRichard Lowe rdpr %tl, %g7 4448*5d9d9091SRichard Lowe cmp %g7, 1 4449*5d9d9091SRichard Lowe ble %icc, 0f 4450*5d9d9091SRichard Lowe sethi %hi(KERNELBASE), %g6 4451*5d9d9091SRichard Lowe rdpr %tpc, %g7 4452*5d9d9091SRichard Lowe or %g6, %lo(KERNELBASE), %g6 4453*5d9d9091SRichard Lowe cmp %g7, %g6 4454*5d9d9091SRichard Lowe bgeu %xcc, 0f 4455*5d9d9091SRichard Lowe ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4456*5d9d9091SRichard Lowe add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4457*5d9d9091SRichard Lowe wrpr %g7, %tpc 4458*5d9d9091SRichard Lowe add %g7, 4, %g7 4459*5d9d9091SRichard Lowe wrpr %g7, %tnpc 4460*5d9d9091SRichard Lowe0: 4461*5d9d9091SRichard Lowe retry 4462*5d9d9091SRichard Lowe5: 4463*5d9d9091SRichard Lowe /* g3=hlck_pa */ 4464*5d9d9091SRichard Lowe KPMLOCK_EXIT(%g3, ASI_MEM) 4465*5d9d9091SRichard Lowe ba,pt %icc, sfmmu_kpm_exception 4466*5d9d9091SRichard Lowe nop 4467*5d9d9091SRichard Lowe SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4468*5d9d9091SRichard Lowe 4469*5d9d9091SRichard Lowe#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4470*5d9d9091SRichard Lowe#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4471*5d9d9091SRichard Lowe#endif 4472*5d9d9091SRichard Lowe 4473*5d9d9091SRichard Lowe .seg ".data" 4474*5d9d9091SRichard Lowesfmmu_kpm_tsbmtl_panic: 4475*5d9d9091SRichard Lowe .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4476*5d9d9091SRichard Lowe .byte 0 4477*5d9d9091SRichard Lowesfmmu_kpm_stsbmtl_panic: 4478*5d9d9091SRichard Lowe .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4479*5d9d9091SRichard Lowe .byte 0 4480*5d9d9091SRichard Lowe .align 4 4481*5d9d9091SRichard Lowe .seg ".text" 4482*5d9d9091SRichard Lowe 4483*5d9d9091SRichard Lowe /* 4484*5d9d9091SRichard Lowe * Enable/disable tsbmiss handling at trap level for a kpm (large) page. 4485*5d9d9091SRichard Lowe * Called from C-level, sets/clears "go" indication for trap level 4486*5d9d9091SRichard Lowe * handler. khl_lock is a low level spin lock to protect the kp_tsbmtl 4487*5d9d9091SRichard Lowe * field. Assumed that &kp->kp_refcntc is checked for zero or -1 at 4488*5d9d9091SRichard Lowe * C-level. Assumes khl_mutex is held when called from C-level. 4489*5d9d9091SRichard Lowe */ 4490*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_kpm_tsbmtl) 4491*5d9d9091SRichard Lowe rdpr %pstate, %o3 4492*5d9d9091SRichard Lowe /* 4493*5d9d9091SRichard Lowe * %o0 = &kp_refcntc 4494*5d9d9091SRichard Lowe * %o1 = &khl_lock 4495*5d9d9091SRichard Lowe * %o2 = 0/1 (off/on) 4496*5d9d9091SRichard Lowe * %o3 = pstate save 4497*5d9d9091SRichard Lowe */ 4498*5d9d9091SRichard Lowe#ifdef DEBUG 4499*5d9d9091SRichard Lowe andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4500*5d9d9091SRichard Lowe bnz,pt %icc, 1f /* disabled, panic */ 4501*5d9d9091SRichard Lowe nop 4502*5d9d9091SRichard Lowe save %sp, -SA(MINFRAME), %sp 4503*5d9d9091SRichard Lowe sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4504*5d9d9091SRichard Lowe call panic 4505*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4506*5d9d9091SRichard Lowe ret 4507*5d9d9091SRichard Lowe restore 4508*5d9d9091SRichard Lowe1: 4509*5d9d9091SRichard Lowe#endif /* DEBUG */ 4510*5d9d9091SRichard Lowe wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4511*5d9d9091SRichard Lowe 4512*5d9d9091SRichard Lowe KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4513*5d9d9091SRichard Lowe mov -1, %o5 4514*5d9d9091SRichard Lowe brz,a %o2, 2f 4515*5d9d9091SRichard Lowe mov 0, %o5 4516*5d9d9091SRichard Lowe2: 4517*5d9d9091SRichard Lowe sth %o5, [%o0] 4518*5d9d9091SRichard Lowe KPMLOCK_EXIT(%o1, ASI_N) 4519*5d9d9091SRichard Lowe 4520*5d9d9091SRichard Lowe retl 4521*5d9d9091SRichard Lowe wrpr %g0, %o3, %pstate /* enable interrupts */ 4522*5d9d9091SRichard Lowe SET_SIZE(sfmmu_kpm_tsbmtl) 4523*5d9d9091SRichard Lowe 4524*5d9d9091SRichard Lowe /* 4525*5d9d9091SRichard Lowe * kpm_smallpages: stores val to byte at address mapped within low level 4526*5d9d9091SRichard Lowe * lock brackets. The old value is returned. Called from C-level. 4527*5d9d9091SRichard Lowe */ 4528*5d9d9091SRichard Lowe ENTRY_NP(sfmmu_kpm_stsbmtl) 4529*5d9d9091SRichard Lowe rdpr %pstate, %o3 4530*5d9d9091SRichard Lowe /* 4531*5d9d9091SRichard Lowe * %o0 = &mapped 4532*5d9d9091SRichard Lowe * %o1 = &kshl_lock 4533*5d9d9091SRichard Lowe * %o2 = val 4534*5d9d9091SRichard Lowe * %o3 = pstate save 4535*5d9d9091SRichard Lowe */ 4536*5d9d9091SRichard Lowe#ifdef DEBUG 4537*5d9d9091SRichard Lowe andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4538*5d9d9091SRichard Lowe bnz,pt %icc, 1f /* disabled, panic */ 4539*5d9d9091SRichard Lowe nop 4540*5d9d9091SRichard Lowe save %sp, -SA(MINFRAME), %sp 4541*5d9d9091SRichard Lowe sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4542*5d9d9091SRichard Lowe call panic 4543*5d9d9091SRichard Lowe or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4544*5d9d9091SRichard Lowe ret 4545*5d9d9091SRichard Lowe restore 4546*5d9d9091SRichard Lowe1: 4547*5d9d9091SRichard Lowe#endif /* DEBUG */ 4548*5d9d9091SRichard Lowe wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4549*5d9d9091SRichard Lowe 4550*5d9d9091SRichard Lowe KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4551*5d9d9091SRichard Lowe ldsb [%o0], %o5 4552*5d9d9091SRichard Lowe stb %o2, [%o0] 4553*5d9d9091SRichard Lowe KPMLOCK_EXIT(%o1, ASI_N) 4554*5d9d9091SRichard Lowe 4555*5d9d9091SRichard Lowe and %o5, KPM_MAPPED_MASK, %o0 /* return old val */ 4556*5d9d9091SRichard Lowe retl 4557*5d9d9091SRichard Lowe wrpr %g0, %o3, %pstate /* enable interrupts */ 4558*5d9d9091SRichard Lowe SET_SIZE(sfmmu_kpm_stsbmtl) 4559*5d9d9091SRichard Lowe 4560*5d9d9091SRichard Lowe#ifdef sun4v 4561*5d9d9091SRichard Lowe /* 4562*5d9d9091SRichard Lowe * User/kernel data miss w// multiple TSBs 4563*5d9d9091SRichard Lowe * The first probe covers 8K, 64K, and 512K page sizes, 4564*5d9d9091SRichard Lowe * because 64K and 512K mappings are replicated off 8K 4565*5d9d9091SRichard Lowe * pointer. Second probe covers 4M page size only. 4566*5d9d9091SRichard Lowe * 4567*5d9d9091SRichard Lowe * MMU fault area contains miss address and context. 4568*5d9d9091SRichard Lowe */ 4569*5d9d9091SRichard Lowe ALTENTRY(sfmmu_slow_dmmu_miss) 4570*5d9d9091SRichard Lowe GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 4571*5d9d9091SRichard Lowe 4572*5d9d9091SRichard Loweslow_miss_common: 4573*5d9d9091SRichard Lowe /* 4574*5d9d9091SRichard Lowe * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 4575*5d9d9091SRichard Lowe * %g3 = ctx (cannot be INVALID_CONTEXT) 4576*5d9d9091SRichard Lowe */ 4577*5d9d9091SRichard Lowe brnz,pt %g3, 8f ! check for user context 4578*5d9d9091SRichard Lowe nop 4579*5d9d9091SRichard Lowe 4580*5d9d9091SRichard Lowe /* 4581*5d9d9091SRichard Lowe * Kernel miss 4582*5d9d9091SRichard Lowe * Get 8K and 4M TSB pointers in %g1 and %g3 and 4583*5d9d9091SRichard Lowe * branch to sfmmu_tsb_miss_tt to handle it. 4584*5d9d9091SRichard Lowe */ 4585*5d9d9091SRichard Lowe mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4586*5d9d9091SRichard Lowesfmmu_dslow_patch_ktsb_base: 4587*5d9d9091SRichard Lowe RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 4588*5d9d9091SRichard Lowesfmmu_dslow_patch_ktsb_szcode: 4589*5d9d9091SRichard Lowe or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 4590*5d9d9091SRichard Lowe 4591*5d9d9091SRichard Lowe GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 4592*5d9d9091SRichard Lowe ! %g1 = First TSB entry pointer, as TSB miss handler expects 4593*5d9d9091SRichard Lowe 4594*5d9d9091SRichard Lowe mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4595*5d9d9091SRichard Lowesfmmu_dslow_patch_ktsb4m_base: 4596*5d9d9091SRichard Lowe RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 4597*5d9d9091SRichard Lowesfmmu_dslow_patch_ktsb4m_szcode: 4598*5d9d9091SRichard Lowe or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 4599*5d9d9091SRichard Lowe 4600*5d9d9091SRichard Lowe GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 4601*5d9d9091SRichard Lowe ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 4602*5d9d9091SRichard Lowe ba,a,pt %xcc, sfmmu_tsb_miss_tt 4603*5d9d9091SRichard Lowe .empty 4604*5d9d9091SRichard Lowe 4605*5d9d9091SRichard Lowe8: 4606*5d9d9091SRichard Lowe /* 4607*5d9d9091SRichard Lowe * User miss 4608*5d9d9091SRichard Lowe * Get first TSB pointer in %g1 4609*5d9d9091SRichard Lowe * Get second TSB pointer (or NULL if no second TSB) in %g3 4610*5d9d9091SRichard Lowe * Branch to sfmmu_tsb_miss_tt to handle it 4611*5d9d9091SRichard Lowe */ 4612*5d9d9091SRichard Lowe GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 4613*5d9d9091SRichard Lowe /* %g1 = first TSB entry ptr now, %g2 preserved */ 4614*5d9d9091SRichard Lowe 4615*5d9d9091SRichard Lowe GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 4616*5d9d9091SRichard Lowe brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 4617*5d9d9091SRichard Lowe nop 4618*5d9d9091SRichard Lowe 4619*5d9d9091SRichard Lowe GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 4620*5d9d9091SRichard Lowe /* %g3 = second TSB entry ptr now, %g2 preserved */ 4621*5d9d9091SRichard Lowe9: 4622*5d9d9091SRichard Lowe ba,a,pt %xcc, sfmmu_tsb_miss_tt 4623*5d9d9091SRichard Lowe .empty 4624*5d9d9091SRichard Lowe SET_SIZE(sfmmu_slow_dmmu_miss) 4625*5d9d9091SRichard Lowe 4626*5d9d9091SRichard Lowe 4627*5d9d9091SRichard Lowe /* 4628*5d9d9091SRichard Lowe * User/kernel instruction miss w/ multiple TSBs 4629*5d9d9091SRichard Lowe * The first probe covers 8K, 64K, and 512K page sizes, 4630*5d9d9091SRichard Lowe * because 64K and 512K mappings are replicated off 8K 4631*5d9d9091SRichard Lowe * pointer. Second probe covers 4M page size only. 4632*5d9d9091SRichard Lowe * 4633*5d9d9091SRichard Lowe * MMU fault area contains miss address and context. 4634*5d9d9091SRichard Lowe */ 4635*5d9d9091SRichard Lowe ALTENTRY(sfmmu_slow_immu_miss) 4636*5d9d9091SRichard Lowe GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 4637*5d9d9091SRichard Lowe ba,a,pt %xcc, slow_miss_common 4638*5d9d9091SRichard Lowe SET_SIZE(sfmmu_slow_immu_miss) 4639*5d9d9091SRichard Lowe 4640*5d9d9091SRichard Lowe#endif /* sun4v */ 4641*5d9d9091SRichard Lowe 4642*5d9d9091SRichard Lowe/* 4643*5d9d9091SRichard Lowe * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 4644*5d9d9091SRichard Lowe */ 4645*5d9d9091SRichard Lowe .seg ".data" 4646*5d9d9091SRichard Lowe .align 64 4647*5d9d9091SRichard Lowe .global tsbmiss_area 4648*5d9d9091SRichard Lowetsbmiss_area: 4649*5d9d9091SRichard Lowe .skip (TSBMISS_SIZE * NCPU) 4650*5d9d9091SRichard Lowe 4651*5d9d9091SRichard Lowe .align 64 4652*5d9d9091SRichard Lowe .global kpmtsbm_area 4653*5d9d9091SRichard Lowekpmtsbm_area: 4654*5d9d9091SRichard Lowe .skip (KPMTSBM_SIZE * NCPU) 4655