xref: /titanic_51/usr/src/uts/sfmmu/ml/sfmmu_asm.s (revision 99ba4d70906e5f67853c25ade6ebf4da3e5be4d6)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26/*
27 * SFMMU primitives.  These primitives should only be used by sfmmu
28 * routines.
29 */
30
31#if defined(lint)
32#include <sys/types.h>
33#else	/* lint */
34#include "assym.h"
35#endif	/* lint */
36
37#include <sys/asm_linkage.h>
38#include <sys/machtrap.h>
39#include <sys/machasi.h>
40#include <sys/sun4asi.h>
41#include <sys/pte.h>
42#include <sys/mmu.h>
43#include <vm/hat_sfmmu.h>
44#include <vm/seg_spt.h>
45#include <sys/machparam.h>
46#include <sys/privregs.h>
47#include <sys/scb.h>
48#include <sys/intreg.h>
49#include <sys/machthread.h>
50#include <sys/intr.h>
51#include <sys/clock.h>
52#include <sys/trapstat.h>
53
54#ifdef TRAPTRACE
55#include <sys/traptrace.h>
56
57/*
58 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
59 */
60#define	TT_TRACE(label)		\
61	ba	label		;\
62	rd	%pc, %g7
63#else
64
65#define	TT_TRACE(label)
66
67#endif /* TRAPTRACE */
68
69#ifndef	lint
70
71#if (TTE_SUSPEND_SHIFT > 0)
72#define	TTE_SUSPEND_INT_SHIFT(reg)				\
73	sllx	reg, TTE_SUSPEND_SHIFT, reg
74#else
75#define	TTE_SUSPEND_INT_SHIFT(reg)
76#endif
77
78#endif /* lint */
79
80#ifndef	lint
81
82/*
83 * Assumes TSBE_TAG is 0
84 * Assumes TSBE_INTHI is 0
85 * Assumes TSBREG.split is 0
86 */
87
88#if TSBE_TAG != 0
89#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
90#endif
91
92#if TSBTAG_INTHI != 0
93#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
94#endif
95
96/*
97 * The following code assumes the tsb is not split.
98 *
99 * With TSBs no longer shared between processes, it's no longer
100 * necessary to hash the context bits into the tsb index to get
101 * tsb coloring; the new implementation treats the TSB as a
102 * direct-mapped, virtually-addressed cache.
103 *
104 * In:
105 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
106 *    tsbbase = base address of TSB (clobbered)
107 *    tagacc = tag access register (clobbered)
108 *    szc = size code of TSB (ro)
109 *    tmp = scratch reg
110 * Out:
111 *    tsbbase = pointer to entry in TSB
112 */
113#define	GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp)		\
114	mov	TSB_ENTRIES(0), tmp	/* nentries in TSB size 0 */	;\
115	srlx	tagacc, vpshift, tagacc 				;\
116	sllx	tmp, szc, tmp		/* tmp = nentries in TSB */	;\
117	sub	tmp, 1, tmp		/* mask = nentries - 1 */	;\
118	and	tagacc, tmp, tmp	/* tsbent = virtpage & mask */	;\
119	sllx	tmp, TSB_ENTRY_SHIFT, tmp	/* entry num --> ptr */	;\
120	add	tsbbase, tmp, tsbbase	/* add entry offset to TSB base */
121
122/*
123 * When the kpm TSB is used it is assumed that it is direct mapped
124 * using (vaddr>>vpshift)%tsbsz as the index.
125 *
126 * Note that, for now, the kpm TSB and kernel TSB are the same for
127 * each mapping size.  However that need not always be the case.  If
128 * the trap handlers are updated to search a different TSB for kpm
129 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
130 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
131 *
132 * In:
133 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
134 *    vaddr = virtual address (clobbered)
135 *    tsbp, szc, tmp = scratch
136 * Out:
137 *    tsbp = pointer to entry in TSB
138 */
139#define	GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)		\
140	cmp	vpshift, MMU_PAGESHIFT					;\
141	bne,pn	%icc, 1f		/* branch if large case */	;\
142	  sethi	%hi(kpmsm_tsbsz), szc					;\
143	sethi	%hi(kpmsm_tsbbase), tsbp				;\
144	ld	[szc + %lo(kpmsm_tsbsz)], szc				;\
145	ldx	[tsbp + %lo(kpmsm_tsbbase)], tsbp			;\
146	ba,pt	%icc, 2f						;\
147	  nop								;\
1481:	sethi	%hi(kpm_tsbsz), szc					;\
149	sethi	%hi(kpm_tsbbase), tsbp					;\
150	ld	[szc + %lo(kpm_tsbsz)], szc				;\
151	ldx	[tsbp + %lo(kpm_tsbbase)], tsbp				;\
1522:	GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)
153
154/*
155 * Lock the TSBE at virtual address tsbep.
156 *
157 * tsbep = TSBE va (ro)
158 * tmp1, tmp2 = scratch registers (clobbered)
159 * label = label to use for branches (text)
160 * %asi = ASI to use for TSB access
161 *
162 * NOTE that we flush the TSB using fast VIS instructions that
163 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
164 * not be treated as a locked entry or we'll get stuck spinning on
165 * an entry that isn't locked but really invalid.
166 */
167
168#if defined(UTSB_PHYS)
169
170#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
171	lda	[tsbep]ASI_MEM, tmp1					;\
172label:									;\
173	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
174	cmp	tmp1, tmp2 						;\
175	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
176	  lda	[tsbep]ASI_MEM, tmp1					;\
177	casa	[tsbep]ASI_MEM, tmp1, tmp2				;\
178	cmp	tmp1, tmp2 						;\
179	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
180	  lda	[tsbep]ASI_MEM, tmp1					;\
181	/* tsbe lock acquired */					;\
182	membar #StoreStore
183
184#else /* UTSB_PHYS */
185
186#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
187	lda	[tsbep]%asi, tmp1					;\
188label:									;\
189	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
190	cmp	tmp1, tmp2 						;\
191	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
192	  lda	[tsbep]%asi, tmp1					;\
193	casa	[tsbep]%asi, tmp1, tmp2					;\
194	cmp	tmp1, tmp2 						;\
195	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
196	  lda	[tsbep]%asi, tmp1					;\
197	/* tsbe lock acquired */					;\
198	membar #StoreStore
199
200#endif /* UTSB_PHYS */
201
202/*
203 * Atomically write TSBE at virtual address tsbep.
204 *
205 * tsbep = TSBE va (ro)
206 * tte = TSBE TTE (ro)
207 * tagtarget = TSBE tag (ro)
208 * %asi = ASI to use for TSB access
209 */
210
211#if defined(UTSB_PHYS)
212
213#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		\
214	add	tsbep, TSBE_TTE, tmp1					;\
215	stxa	tte, [tmp1]ASI_MEM		/* write tte data */	;\
216	membar #StoreStore						;\
217	add	tsbep, TSBE_TAG, tmp1					;\
218	stxa	tagtarget, [tmp1]ASI_MEM	/* write tte tag & unlock */
219
220#else /* UTSB_PHYS */
221
222#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1)		\
223	stxa	tte, [tsbep + TSBE_TTE]%asi	/* write tte data */	;\
224	membar #StoreStore						;\
225	stxa	tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */
226
227#endif /* UTSB_PHYS */
228
229/*
230 * Load an entry into the TSB at TL > 0.
231 *
232 * tsbep = pointer to the TSBE to load as va (ro)
233 * tte = value of the TTE retrieved and loaded (wo)
234 * tagtarget = tag target register.  To get TSBE tag to load,
235 *   we need to mask off the context and leave only the va (clobbered)
236 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
237 * tmp1, tmp2 = scratch registers
238 * label = label to use for branches (text)
239 * %asi = ASI to use for TSB access
240 */
241
242#if defined(UTSB_PHYS)
243
244#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
245	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
246	/*								;\
247	 * I don't need to update the TSB then check for the valid tte.	;\
248	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
249	 * we always invalidate the hash table before we unload the TSB.;\
250	 */								;\
251	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
252	ldxa	[ttepa]ASI_MEM, tte					;\
253	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
254	sethi	%hi(TSBTAG_INVALID), tmp2				;\
255	add	tsbep, TSBE_TAG, tmp1					;\
256	brgez,a,pn tte, label/**/f					;\
257	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
258	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
259label:
260
261#else /* UTSB_PHYS */
262
263#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
264	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
265	/*								;\
266	 * I don't need to update the TSB then check for the valid tte.	;\
267	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
268	 * we always invalidate the hash table before we unload the TSB.;\
269	 */								;\
270	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
271	ldxa	[ttepa]ASI_MEM, tte					;\
272	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
273	sethi	%hi(TSBTAG_INVALID), tmp2				;\
274	brgez,a,pn tte, label/**/f					;\
275	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
276	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
277label:
278
279#endif /* UTSB_PHYS */
280
281/*
282 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0,
283 *   for ITLB synthesis.
284 *
285 * tsbep = pointer to the TSBE to load as va (ro)
286 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out)
287 *   with exec_perm turned off and exec_synth turned on
288 * tagtarget = tag target register.  To get TSBE tag to load,
289 *   we need to mask off the context and leave only the va (clobbered)
290 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
291 * tmp1, tmp2 = scratch registers
292 * label = label to use for branch (text)
293 * %asi = ASI to use for TSB access
294 */
295
296#define	TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
297	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
298	/*								;\
299	 * I don't need to update the TSB then check for the valid tte.	;\
300	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
301	 * we always invalidate the hash table before we unload the TSB.;\
302	 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0	;\
303	 * and exec_synth bit to 1.					;\
304	 */								;\
305	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
306	mov	tte, tmp1						;\
307	ldxa	[ttepa]ASI_MEM, tte					;\
308	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
309	sethi	%hi(TSBTAG_INVALID), tmp2				;\
310	brgez,a,pn tte, label/**/f					;\
311	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
312	or	tte, tmp1, tte						;\
313	andn	tte, TTE_EXECPRM_INT, tte				;\
314	or	tte, TTE_E_SYNTH_INT, tte				;\
315	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
316label:
317
318/*
319 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis.
320 *
321 * tte = value of the TTE, used to get tte_size bits (ro)
322 * tagaccess = tag access register, used to get 4M pfn bits (ro)
323 * pfn = 4M pfn bits shifted to offset for tte (out)
324 * tmp1 = scratch register
325 * label = label to use for branch (text)
326 */
327
328#define	GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label)			\
329	/*								;\
330	 * Get 4M bits from tagaccess for 32M, 256M pagesizes.		;\
331	 * Return them, shifted, in pfn.				;\
332	 */								;\
333	srlx	tagaccess, MMU_PAGESHIFT4M, tagaccess			;\
334	srlx	tte, TTE_SZ_SHFT, tmp		/* isolate the */	;\
335	andcc	tmp, TTE_SZ_BITS, %g0		/* tte_size bits */	;\
336	bz,a,pt %icc, label/**/f		/* if 0, is */		;\
337	  and	tagaccess, 0x7, tagaccess	/* 32M page size */	;\
338	and	tagaccess, 0x3f, tagaccess /* else 256M page size */	;\
339label:									;\
340	sllx	tagaccess, MMU_PAGESHIFT4M, pfn
341
342/*
343 * Add 4M TTE size code to a tte for a Panther 32M/256M page,
344 * for ITLB synthesis.
345 *
346 * tte = value of the TTE, used to get tte_size bits (rw)
347 * tmp1 = scratch register
348 */
349
350#define	SET_TTE4M_PN(tte, tmp)						\
351	/*								;\
352	 * Set 4M pagesize tte bits. 					;\
353	 */								;\
354	set	TTE4M, tmp						;\
355	sllx	tmp, TTE_SZ_SHFT, tmp					;\
356	or	tte, tmp, tte
357
358/*
359 * Load an entry into the TSB at TL=0.
360 *
361 * tsbep = pointer to the TSBE to load as va (ro)
362 * tteva = pointer to the TTE to load as va (ro)
363 * tagtarget = TSBE tag to load (which contains no context), synthesized
364 * to match va of MMU tag target register only (ro)
365 * tmp1, tmp2 = scratch registers (clobbered)
366 * label = label to use for branches (text)
367 * %asi = ASI to use for TSB access
368 */
369
370#if defined(UTSB_PHYS)
371
372#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
373	/* can't rd tteva after locking tsb because it can tlb miss */	;\
374	ldx	[tteva], tteva			/* load tte */		;\
375	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
376	sethi	%hi(TSBTAG_INVALID), tmp2				;\
377	add	tsbep, TSBE_TAG, tmp1					;\
378	brgez,a,pn tteva, label/**/f					;\
379	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
380	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
381label:
382
383#else /* UTSB_PHYS */
384
385#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
386	/* can't rd tteva after locking tsb because it can tlb miss */	;\
387	ldx	[tteva], tteva			/* load tte */		;\
388	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
389	sethi	%hi(TSBTAG_INVALID), tmp2				;\
390	brgez,a,pn tteva, label/**/f					;\
391	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
392	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
393label:
394
395#endif /* UTSB_PHYS */
396
397/*
398 * Invalidate a TSB entry in the TSB.
399 *
400 * NOTE: TSBE_TAG is assumed to be zero.  There is a compile time check
401 *	 about this earlier to ensure this is true.  Thus when we are
402 *	 directly referencing tsbep below, we are referencing the tte_tag
403 *	 field of the TSBE.  If this  offset ever changes, the code below
404 *	 will need to be modified.
405 *
406 * tsbep = pointer to TSBE as va (ro)
407 * tag = invalidation is done if this matches the TSBE tag (ro)
408 * tmp1 - tmp3 = scratch registers (clobbered)
409 * label = label name to use for branches (text)
410 * %asi = ASI to use for TSB access
411 */
412
413#if defined(UTSB_PHYS)
414
415#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
416	lda	[tsbep]ASI_MEM, tmp1	/* tmp1 = tsbe tag */		;\
417	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
418label/**/1:								;\
419	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
420	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
421	  lda	[tsbep]ASI_MEM, tmp1	/* reloading value each time */	;\
422	ldxa	[tsbep]ASI_MEM, tmp3	/* tmp3 = tsbe tag */		;\
423	cmp	tag, tmp3		/* compare tags */		;\
424	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
425	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
426	casa	[tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */	;\
427	cmp	tmp1, tmp3		/* if not successful */		;\
428	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
429	  lda	[tsbep]ASI_MEM, tmp1	/* reloading tsbe tag */	;\
430label/**/2:
431
432#else /* UTSB_PHYS */
433
434#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
435	lda	[tsbep]%asi, tmp1	/* tmp1 = tsbe tag */		;\
436	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
437label/**/1:								;\
438	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
439	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
440	  lda	[tsbep]%asi, tmp1	/* reloading value each time */	;\
441	ldxa	[tsbep]%asi, tmp3	/* tmp3 = tsbe tag */		;\
442	cmp	tag, tmp3		/* compare tags */		;\
443	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
444	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
445	casa	[tsbep]%asi, tmp1, tmp3	/* try to set tag invalid */	;\
446	cmp	tmp1, tmp3		/* if not successful */		;\
447	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
448	  lda	[tsbep]%asi, tmp1	/* reloading tsbe tag */	;\
449label/**/2:
450
451#endif /* UTSB_PHYS */
452
453#if TSB_SOFTSZ_MASK < TSB_SZ_MASK
454#error	- TSB_SOFTSZ_MASK too small
455#endif
456
457
458/*
459 * An implementation of setx which will be hot patched at run time.
460 * since it is being hot patched, there is no value passed in.
461 * Thus, essentially we are implementing
462 *	setx value, tmp, dest
463 * where value is RUNTIME_PATCH (aka 0) in this case.
464 */
465#define	RUNTIME_PATCH_SETX(dest, tmp)					\
466	sethi	%hh(RUNTIME_PATCH), tmp					;\
467	sethi	%lm(RUNTIME_PATCH), dest				;\
468	or	tmp, %hm(RUNTIME_PATCH), tmp				;\
469	or	dest, %lo(RUNTIME_PATCH), dest				;\
470	sllx	tmp, 32, tmp						;\
471	nop				/* for perf reasons */		;\
472	or	tmp, dest, dest		/* contents of patched value */
473
474
475#endif (lint)
476
477
478#if defined (lint)
479
480/*
481 * sfmmu related subroutines
482 */
483uint_t
484sfmmu_disable_intrs()
485{ return(0); }
486
487/* ARGSUSED */
488void
489sfmmu_enable_intrs(uint_t pstate_save)
490{}
491
492/* ARGSUSED */
493int
494sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag)
495{ return(0); }
496
497/*
498 * Use cas, if tte has changed underneath us then reread and try again.
499 * In the case of a retry, it will update sttep with the new original.
500 */
501/* ARGSUSED */
502int
503sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
504{ return(0); }
505
506/*
507 * Use cas, if tte has changed underneath us then return 1, else return 0
508 */
509/* ARGSUSED */
510int
511sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
512{ return(0); }
513
514/* ARGSUSED */
515void
516sfmmu_copytte(tte_t *sttep, tte_t *dttep)
517{}
518
519/*ARGSUSED*/
520struct tsbe *
521sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc)
522{ return(0); }
523
524/*ARGSUSED*/
525uint64_t
526sfmmu_make_tsbtag(caddr_t va)
527{ return(0); }
528
529#else	/* lint */
530
531	.seg	".data"
532	.global	sfmmu_panic1
533sfmmu_panic1:
534	.asciz	"sfmmu_asm: interrupts already disabled"
535
536	.global	sfmmu_panic3
537sfmmu_panic3:
538	.asciz	"sfmmu_asm: sfmmu_vatopfn called for user"
539
540	.global	sfmmu_panic4
541sfmmu_panic4:
542	.asciz	"sfmmu_asm: 4M tsb pointer mis-match"
543
544	.global	sfmmu_panic5
545sfmmu_panic5:
546	.asciz	"sfmmu_asm: no unlocked TTEs in TLB 0"
547
548	.global	sfmmu_panic6
549sfmmu_panic6:
550	.asciz	"sfmmu_asm: interrupts not disabled"
551
552	.global	sfmmu_panic7
553sfmmu_panic7:
554	.asciz	"sfmmu_asm: kernel as"
555
556	.global	sfmmu_panic8
557sfmmu_panic8:
558	.asciz	"sfmmu_asm: gnum is zero"
559
560	.global	sfmmu_panic9
561sfmmu_panic9:
562	.asciz	"sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL"
563
564	.global	sfmmu_panic10
565sfmmu_panic10:
566	.asciz	"sfmmu_asm: valid SCD with no 3rd scd TSB"
567
568        ENTRY(sfmmu_disable_intrs)
569        rdpr    %pstate, %o0
570#ifdef DEBUG
571	PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1)
572#endif /* DEBUG */
573        retl
574          wrpr   %o0, PSTATE_IE, %pstate
575        SET_SIZE(sfmmu_disable_intrs)
576
577	ENTRY(sfmmu_enable_intrs)
578        retl
579          wrpr    %g0, %o0, %pstate
580        SET_SIZE(sfmmu_enable_intrs)
581
582/*
583 * This routine is called both by resume() and sfmmu_get_ctx() to
584 * allocate a new context for the process on a MMU.
585 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID .
586 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which
587 * is the case when sfmmu_alloc_ctx is called from resume().
588 *
589 * The caller must disable interrupts before entering this routine.
590 * To reduce ctx switch overhead, the code contains both 'fast path' and
591 * 'slow path' code. The fast path code covers the common case where only
592 * a quick check is needed and the real ctx allocation is not required.
593 * It can be done without holding the per-process (PP) lock.
594 * The 'slow path' code must be protected by the PP Lock and performs ctx
595 * allocation.
596 * Hardware context register and HAT mmu cnum are updated accordingly.
597 *
598 * %o0 - sfmmup
599 * %o1 - allocflag
600 * %o2 - CPU
601 * %o3 - sfmmu private/shared flag
602 *
603 * ret - 0: no ctx is allocated
604 *       1: a ctx is allocated
605 */
606        ENTRY_NP(sfmmu_alloc_ctx)
607
608#ifdef DEBUG
609	sethi   %hi(ksfmmup), %g1
610	ldx     [%g1 + %lo(ksfmmup)], %g1
611	cmp     %g1, %o0
612	bne,pt   %xcc, 0f
613	  nop
614
615	sethi   %hi(panicstr), %g1		! if kernel as, panic
616        ldx     [%g1 + %lo(panicstr)], %g1
617        tst     %g1
618        bnz,pn  %icc, 7f
619          nop
620
621	sethi	%hi(sfmmu_panic7), %o0
622	call	panic
623	  or	%o0, %lo(sfmmu_panic7), %o0
624
6257:
626	retl
627	  mov	%g0, %o0			! %o0 = ret = 0
628
6290:
630	PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1)
631#endif /* DEBUG */
632
633	mov	%o3, %g1			! save sfmmu pri/sh flag in %g1
634
635	! load global mmu_ctxp info
636	ldx	[%o2 + CPU_MMU_CTXP], %o3		! %o3 = mmu_ctx_t ptr
637        lduw	[%o2 + CPU_MMU_IDX], %g2		! %g2 = mmu index
638
639	! load global mmu_ctxp gnum
640	ldx	[%o3 + MMU_CTX_GNUM], %o4		! %o4 = mmu_ctxp->gnum
641
642#ifdef DEBUG
643	cmp	%o4, %g0		! mmu_ctxp->gnum should never be 0
644	bne,pt	%xcc, 3f
645	  nop
646
647	sethi   %hi(panicstr), %g1	! test if panicstr is already set
648        ldx     [%g1 + %lo(panicstr)], %g1
649        tst     %g1
650        bnz,pn  %icc, 1f
651          nop
652
653	sethi	%hi(sfmmu_panic8), %o0
654	call	panic
655	  or	%o0, %lo(sfmmu_panic8), %o0
6561:
657	retl
658	  mov	%g0, %o0			! %o0 = ret = 0
6593:
660#endif
661
662	! load HAT sfmmu_ctxs[mmuid] gnum, cnum
663
664	sllx	%g2, SFMMU_MMU_CTX_SHIFT, %g2
665	add	%o0, %g2, %g2		! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
666
667	/*
668	 * %g5 = sfmmu gnum returned
669	 * %g6 = sfmmu cnum returned
670	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
671	 * %g4 = scratch
672	 *
673	 * Fast path code, do a quick check.
674	 */
675	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
676
677	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
678	bne,pt	%icc, 1f			! valid hat cnum, check gnum
679	  nop
680
681	! cnum == INVALID, check allocflag
682	mov	%g0, %g4	! %g4 = ret = 0
683	brz,pt  %o1, 8f		! allocflag == 0, skip ctx allocation, bail
684	  mov	%g6, %o1
685
686	! (invalid HAT cnum) && (allocflag == 1)
687	ba,pt	%icc, 2f
688	  nop
6891:
690	! valid HAT cnum, check gnum
691	cmp	%g5, %o4
692	mov	1, %g4				!%g4 = ret = 1
693	be,a,pt	%icc, 8f			! gnum unchanged, go to done
694	  mov	%g6, %o1
695
6962:
697	/*
698	 * Grab per process (PP) sfmmu_ctx_lock spinlock,
699	 * followed by the 'slow path' code.
700	 */
701	ldstub	[%o0 + SFMMU_CTX_LOCK], %g3	! %g3 = per process (PP) lock
7023:
703	brz	%g3, 5f
704	  nop
7054:
706	brnz,a,pt       %g3, 4b				! spin if lock is 1
707	  ldub	[%o0 + SFMMU_CTX_LOCK], %g3
708	ba	%xcc, 3b				! retry the lock
709	  ldstub	[%o0 + SFMMU_CTX_LOCK], %g3    ! %g3 = PP lock
710
7115:
712	membar  #LoadLoad
713	/*
714	 * %g5 = sfmmu gnum returned
715	 * %g6 = sfmmu cnum returned
716	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
717	 * %g4 = scratch
718	 */
719	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
720
721	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
722	bne,pt	%icc, 1f			! valid hat cnum, check gnum
723	  nop
724
725	! cnum == INVALID, check allocflag
726	mov	%g0, %g4	! %g4 = ret = 0
727	brz,pt	%o1, 2f		! allocflag == 0, called from resume, set hw
728	  mov	%g6, %o1
729
730	! (invalid HAT cnum) && (allocflag == 1)
731	ba,pt	%icc, 6f
732	  nop
7331:
734	! valid HAT cnum, check gnum
735	cmp	%g5, %o4
736	mov	1, %g4				! %g4 = ret  = 1
737	be,a,pt	%icc, 2f			! gnum unchanged, go to done
738	  mov	%g6, %o1
739
740	ba,pt	%icc, 6f
741	  nop
7422:
743	membar  #LoadStore|#StoreStore
744	ba,pt %icc, 8f
745	  clrb  [%o0 + SFMMU_CTX_LOCK]
7466:
747	/*
748	 * We get here if we do not have a valid context, or
749	 * the HAT gnum does not match global gnum. We hold
750	 * sfmmu_ctx_lock spinlock. Allocate that context.
751	 *
752	 * %o3 = mmu_ctxp
753	 */
754	add	%o3, MMU_CTX_CNUM, %g3
755	ld	[%o3 + MMU_CTX_NCTXS], %g4
756
757	/*
758         * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS;
759         * %g3 = mmu cnum address
760	 * %g4 = mmu nctxs
761	 *
762	 * %o0 = sfmmup
763	 * %o1 = mmu current cnum value (used as new cnum)
764	 * %o4 = mmu gnum
765	 *
766	 * %o5 = scratch
767	 */
768	ld	[%g3], %o1
7690:
770	cmp	%o1, %g4
771	bl,a,pt %icc, 1f
772	  add	%o1, 1, %o5		! %o5 = mmu_ctxp->cnum + 1
773
774	/*
775	 * cnum reachs max, bail, so wrap around can be performed later.
776	 */
777	set	INVALID_CONTEXT, %o1
778	mov	%g0, %g4		! %g4 = ret = 0
779
780	membar  #LoadStore|#StoreStore
781	ba,pt	%icc, 8f
782	  clrb	[%o0 + SFMMU_CTX_LOCK]
7831:
784	! %g3 = addr of mmu_ctxp->cnum
785	! %o5 = mmu_ctxp->cnum + 1
786	cas	[%g3], %o1, %o5
787	cmp	%o1, %o5
788	bne,a,pn %xcc, 0b	! cas failed
789	  ld	[%g3], %o1
790
791#ifdef DEBUG
792        set	MAX_SFMMU_CTX_VAL, %o5
793	cmp	%o1, %o5
794	ble,pt %icc, 2f
795	  nop
796
797	sethi	%hi(sfmmu_panic9), %o0
798	call	panic
799	  or	%o0, %lo(sfmmu_panic9), %o0
8002:
801#endif
802	! update hat gnum and cnum
803	sllx	%o4, SFMMU_MMU_GNUM_RSHIFT, %o4
804	or	%o4, %o1, %o4
805	stx	%o4, [%g2 + SFMMU_CTXS]
806
807	membar  #LoadStore|#StoreStore
808	clrb	[%o0 + SFMMU_CTX_LOCK]
809
810	mov	1, %g4			! %g4 = ret = 1
8118:
812	/*
813	 * program the secondary context register
814	 *
815	 * %o1 = cnum
816	 * %g1 = sfmmu private/shared flag (0:private,  1:shared)
817	 */
818
819	/*
820	 * When we come here and context is invalid, we want to set both
821	 * private and shared ctx regs to INVALID. In order to
822	 * do so, we set the sfmmu priv/shared flag to 'private' regardless
823	 * so that private ctx reg will be set to invalid.
824	 * Note that on sun4v values written to private context register are
825	 * automatically written to corresponding shared context register as
826	 * well. On sun4u SET_SECCTX() will invalidate shared context register
827	 * when it sets a private secondary context register.
828	 */
829
830	cmp	%o1, INVALID_CONTEXT
831	be,a,pn	%icc, 9f
832	  clr	%g1
8339:
834
835#ifdef	sun4u
836	ldub	[%o0 + SFMMU_CEXT], %o2
837	sll	%o2, CTXREG_EXT_SHIFT, %o2
838	or	%o1, %o2, %o1
839#endif /* sun4u */
840
841	SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1)
842
843        retl
844          mov   %g4, %o0                        ! %o0 = ret
845
846	SET_SIZE(sfmmu_alloc_ctx)
847
848	ENTRY_NP(sfmmu_modifytte)
849	ldx	[%o2], %g3			/* current */
850	ldx	[%o0], %g1			/* original */
8512:
852	ldx	[%o1], %g2			/* modified */
853	cmp	%g2, %g3			/* is modified = current? */
854	be,a,pt	%xcc,1f				/* yes, don't write */
855	stx	%g3, [%o0]			/* update new original */
856	casx	[%o2], %g1, %g2
857	cmp	%g1, %g2
858	be,pt	%xcc, 1f			/* cas succeeded - return */
859	  nop
860	ldx	[%o2], %g3			/* new current */
861	stx	%g3, [%o0]			/* save as new original */
862	ba,pt	%xcc, 2b
863	  mov	%g3, %g1
8641:	retl
865	membar	#StoreLoad
866	SET_SIZE(sfmmu_modifytte)
867
868	ENTRY_NP(sfmmu_modifytte_try)
869	ldx	[%o1], %g2			/* modified */
870	ldx	[%o2], %g3			/* current */
871	ldx	[%o0], %g1			/* original */
872	cmp	%g3, %g2			/* is modified = current? */
873	be,a,pn %xcc,1f				/* yes, don't write */
874	mov	0, %o1				/* as if cas failed. */
875
876	casx	[%o2], %g1, %g2
877	membar	#StoreLoad
878	cmp	%g1, %g2
879	movne	%xcc, -1, %o1			/* cas failed. */
880	move	%xcc, 1, %o1			/* cas succeeded. */
8811:
882	stx	%g2, [%o0]			/* report "current" value */
883	retl
884	mov	%o1, %o0
885	SET_SIZE(sfmmu_modifytte_try)
886
887	ENTRY_NP(sfmmu_copytte)
888	ldx	[%o0], %g1
889	retl
890	stx	%g1, [%o1]
891	SET_SIZE(sfmmu_copytte)
892
893
894	/*
895	 * Calculate a TSB entry pointer for the given TSB, va, pagesize.
896	 * %o0 = TSB base address (in), pointer to TSB entry (out)
897	 * %o1 = vaddr (in)
898	 * %o2 = vpshift (in)
899	 * %o3 = tsb size code (in)
900	 * %o4 = scratch register
901	 */
902	ENTRY_NP(sfmmu_get_tsbe)
903	GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
904	retl
905	nop
906	SET_SIZE(sfmmu_get_tsbe)
907
908	/*
909	 * Return a TSB tag for the given va.
910	 * %o0 = va (in/clobbered)
911	 * %o0 = va shifted to be in tsb tag format (with no context) (out)
912	 */
913	ENTRY_NP(sfmmu_make_tsbtag)
914	retl
915	srln	%o0, TTARGET_VA_SHIFT, %o0
916	SET_SIZE(sfmmu_make_tsbtag)
917
918#endif /* lint */
919
920/*
921 * Other sfmmu primitives
922 */
923
924
925#if defined (lint)
926void
927sfmmu_patch_ktsb(void)
928{
929}
930
931void
932sfmmu_kpm_patch_tlbm(void)
933{
934}
935
936void
937sfmmu_kpm_patch_tsbm(void)
938{
939}
940
941void
942sfmmu_patch_shctx(void)
943{
944}
945
946/* ARGSUSED */
947void
948sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys)
949{
950}
951
952/* ARGSUSED */
953void
954sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys)
955{
956}
957
958/* ARGSUSED */
959void
960sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift)
961{
962}
963
964/* ARGSUSED */
965void
966sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
967{
968}
969
970#else /* lint */
971
972#define	I_SIZE		4
973
974	ENTRY_NP(sfmmu_fix_ktlb_traptable)
975	/*
976	 * %o0 = start of patch area
977	 * %o1 = size code of TSB to patch
978	 * %o3 = scratch
979	 */
980	/* fix sll */
981	ld	[%o0], %o3			/* get sll */
982	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
983	st	%o3, [%o0]			/* write sll */
984	flush	%o0
985	/* fix srl */
986	add	%o0, I_SIZE, %o0		/* goto next instr. */
987	ld	[%o0], %o3			/* get srl */
988	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
989	st	%o3, [%o0]			/* write srl */
990	retl
991	flush	%o0
992	SET_SIZE(sfmmu_fix_ktlb_traptable)
993
994	ENTRY_NP(sfmmu_fixup_ktsbbase)
995	/*
996	 * %o0 = start of patch area
997	 * %o5 = kernel virtual or physical tsb base address
998	 * %o2, %o3 are used as scratch registers.
999	 */
1000	/* fixup sethi instruction */
1001	ld	[%o0], %o3
1002	srl	%o5, 10, %o2			! offset is bits 32:10
1003	or	%o3, %o2, %o3			! set imm22
1004	st	%o3, [%o0]
1005	/* fixup offset of lduw/ldx */
1006	add	%o0, I_SIZE, %o0		! next instr
1007	ld	[%o0], %o3
1008	and	%o5, 0x3ff, %o2			! set imm13 to bits 9:0
1009	or	%o3, %o2, %o3
1010	st	%o3, [%o0]
1011	retl
1012	flush	%o0
1013	SET_SIZE(sfmmu_fixup_ktsbbase)
1014
1015	ENTRY_NP(sfmmu_fixup_setx)
1016	/*
1017	 * %o0 = start of patch area
1018	 * %o4 = 64 bit value to patch
1019	 * %o2, %o3 are used as scratch registers.
1020	 *
1021	 * Note: Assuming that all parts of the instructions which need to be
1022	 *	 patched correspond to RUNTIME_PATCH (aka 0)
1023	 *
1024	 * Note the implementation of setx which is being patched is as follows:
1025	 *
1026	 * sethi   %hh(RUNTIME_PATCH), tmp
1027	 * sethi   %lm(RUNTIME_PATCH), dest
1028	 * or      tmp, %hm(RUNTIME_PATCH), tmp
1029	 * or      dest, %lo(RUNTIME_PATCH), dest
1030	 * sllx    tmp, 32, tmp
1031	 * nop
1032	 * or      tmp, dest, dest
1033	 *
1034	 * which differs from the implementation in the
1035	 * "SPARC Architecture Manual"
1036	 */
1037	/* fixup sethi instruction */
1038	ld	[%o0], %o3
1039	srlx	%o4, 42, %o2			! bits [63:42]
1040	or	%o3, %o2, %o3			! set imm22
1041	st	%o3, [%o0]
1042	/* fixup sethi instruction */
1043	add	%o0, I_SIZE, %o0		! next instr
1044	ld	[%o0], %o3
1045	sllx	%o4, 32, %o2			! clear upper bits
1046	srlx	%o2, 42, %o2			! bits [31:10]
1047	or	%o3, %o2, %o3			! set imm22
1048	st	%o3, [%o0]
1049	/* fixup or instruction */
1050	add	%o0, I_SIZE, %o0		! next instr
1051	ld	[%o0], %o3
1052	srlx	%o4, 32, %o2			! bits [63:32]
1053	and	%o2, 0x3ff, %o2			! bits [41:32]
1054	or	%o3, %o2, %o3			! set imm
1055	st	%o3, [%o0]
1056	/* fixup or instruction */
1057	add	%o0, I_SIZE, %o0		! next instr
1058	ld	[%o0], %o3
1059	and	%o4, 0x3ff, %o2			! bits [9:0]
1060	or	%o3, %o2, %o3			! set imm
1061	st	%o3, [%o0]
1062	retl
1063	flush	%o0
1064	SET_SIZE(sfmmu_fixup_setx)
1065
1066	ENTRY_NP(sfmmu_fixup_or)
1067	/*
1068	 * %o0 = start of patch area
1069	 * %o4 = 32 bit value to patch
1070	 * %o2, %o3 are used as scratch registers.
1071	 * Note: Assuming that all parts of the instructions which need to be
1072	 *	 patched correspond to RUNTIME_PATCH (aka 0)
1073	 */
1074	ld	[%o0], %o3
1075	and	%o4, 0x3ff, %o2			! bits [9:0]
1076	or	%o3, %o2, %o3			! set imm
1077	st	%o3, [%o0]
1078	retl
1079	flush	%o0
1080	SET_SIZE(sfmmu_fixup_or)
1081
1082	ENTRY_NP(sfmmu_fixup_shiftx)
1083	/*
1084	 * %o0 = start of patch area
1085	 * %o4 = signed int immediate value to add to sllx/srlx imm field
1086	 * %o2, %o3 are used as scratch registers.
1087	 *
1088	 * sllx/srlx store the 6 bit immediate value in the lowest order bits
1089	 * so we do a simple add.  The caller must be careful to prevent
1090	 * overflow, which could easily occur if the initial value is nonzero!
1091	 */
1092	ld	[%o0], %o3			! %o3 = instruction to patch
1093	and	%o3, 0x3f, %o2			! %o2 = existing imm value
1094	add	%o2, %o4, %o2			! %o2 = new imm value
1095	andn	%o3, 0x3f, %o3			! clear old imm value
1096	and	%o2, 0x3f, %o2			! truncate new imm value
1097	or	%o3, %o2, %o3			! set new imm value
1098	st	%o3, [%o0]			! store updated instruction
1099	retl
1100	flush	%o0
1101	SET_SIZE(sfmmu_fixup_shiftx)
1102
1103	ENTRY_NP(sfmmu_fixup_mmu_asi)
1104	/*
1105	 * Patch imm_asi of all ldda instructions in the MMU
1106	 * trap handlers.  We search MMU_PATCH_INSTR instructions
1107	 * starting from the itlb miss handler (trap 0x64).
1108	 * %o0 = address of tt[0,1]_itlbmiss
1109	 * %o1 = imm_asi to setup, shifted by appropriate offset.
1110	 * %o3 = number of instructions to search
1111	 * %o4 = reserved by caller: called from leaf routine
1112	 */
11131:	ldsw	[%o0], %o2			! load instruction to %o2
1114	brgez,pt %o2, 2f
1115	  srl	%o2, 30, %o5
1116	btst	1, %o5				! test bit 30; skip if not set
1117	bz,pt	%icc, 2f
1118	  sllx	%o2, 39, %o5			! bit 24 -> bit 63
1119	srlx	%o5, 58, %o5			! isolate op3 part of opcode
1120	xor	%o5, 0x13, %o5			! 01 0011 binary == ldda
1121	brnz,pt	%o5, 2f				! skip if not a match
1122	  or	%o2, %o1, %o2			! or in imm_asi
1123	st	%o2, [%o0]			! write patched instruction
11242:	dec	%o3
1125	brnz,a,pt %o3, 1b			! loop until we're done
1126	  add	%o0, I_SIZE, %o0
1127	retl
1128	flush	%o0
1129	SET_SIZE(sfmmu_fixup_mmu_asi)
1130
1131	/*
1132	 * Patch immediate ASI used to access the TSB in the
1133	 * trap table.
1134	 * inputs: %o0 = value of ktsb_phys
1135	 */
1136	ENTRY_NP(sfmmu_patch_mmu_asi)
1137	mov	%o7, %o4			! save return pc in %o4
1138	movrnz	%o0, ASI_QUAD_LDD_PHYS, %o3
1139	movrz	%o0, ASI_NQUAD_LD, %o3
1140	sll	%o3, 5, %o1			! imm_asi offset
1141	mov	6, %o3				! number of instructions
1142	sethi	%hi(dktsb), %o0			! to search
1143	call	sfmmu_fixup_mmu_asi		! patch kdtlb miss
1144	  or	%o0, %lo(dktsb), %o0
1145	mov	6, %o3				! number of instructions
1146	sethi	%hi(dktsb4m), %o0		! to search
1147	call	sfmmu_fixup_mmu_asi		! patch kdtlb4m miss
1148	  or	%o0, %lo(dktsb4m), %o0
1149	mov	6, %o3				! number of instructions
1150	sethi	%hi(iktsb), %o0			! to search
1151	call	sfmmu_fixup_mmu_asi		! patch kitlb miss
1152	  or	%o0, %lo(iktsb), %o0
1153	mov	6, %o3				! number of instructions
1154	sethi	%hi(iktsb4m), %o0		! to search
1155	call	sfmmu_fixup_mmu_asi		! patch kitlb4m miss
1156	  or	%o0, %lo(iktsb4m), %o0
1157	mov	%o4, %o7			! retore return pc -- leaf
1158	retl
1159	nop
1160	SET_SIZE(sfmmu_patch_mmu_asi)
1161
1162	ENTRY_NP(sfmmu_patch_ktsb)
1163	/*
1164	 * We need to fix iktsb, dktsb, et. al.
1165	 */
1166	save	%sp, -SA(MINFRAME), %sp
1167	set	ktsb_phys, %o1
1168	ld	[%o1], %o4
1169	set	ktsb_base, %o5
1170	set	ktsb4m_base, %l1
1171	brz,pt	%o4, 1f
1172	  nop
1173	set	ktsb_pbase, %o5
1174	set	ktsb4m_pbase, %l1
11751:
1176	sethi	%hi(ktsb_szcode), %o1
1177	ld	[%o1 + %lo(ktsb_szcode)], %o1	/* %o1 = ktsb size code */
1178
1179	sethi	%hi(iktsb), %o0
1180	call	sfmmu_fix_ktlb_traptable
1181	  or	%o0, %lo(iktsb), %o0
1182
1183	sethi	%hi(dktsb), %o0
1184	call	sfmmu_fix_ktlb_traptable
1185	  or	%o0, %lo(dktsb), %o0
1186
1187	sethi	%hi(ktsb4m_szcode), %o1
1188	ld	[%o1 + %lo(ktsb4m_szcode)], %o1	/* %o1 = ktsb4m size code */
1189
1190	sethi	%hi(iktsb4m), %o0
1191	call	sfmmu_fix_ktlb_traptable
1192	  or	%o0, %lo(iktsb4m), %o0
1193
1194	sethi	%hi(dktsb4m), %o0
1195	call	sfmmu_fix_ktlb_traptable
1196	  or	%o0, %lo(dktsb4m), %o0
1197
1198#ifndef sun4v
1199	mov	ASI_N, %o2
1200	movrnz	%o4, ASI_MEM, %o2	! setup kernel 32bit ASI to patch
1201	mov	%o2, %o4		! sfmmu_fixup_or needs this in %o4
1202	sethi	%hi(tsb_kernel_patch_asi), %o0
1203	call	sfmmu_fixup_or
1204	  or	%o0, %lo(tsb_kernel_patch_asi), %o0
1205#endif /* !sun4v */
1206
1207	ldx 	[%o5], %o4		! load ktsb base addr (VA or PA)
1208
1209	sethi	%hi(dktsbbase), %o0
1210	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1211	  or	%o0, %lo(dktsbbase), %o0
1212
1213	sethi	%hi(iktsbbase), %o0
1214	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1215	  or	%o0, %lo(iktsbbase), %o0
1216
1217	sethi	%hi(sfmmu_kprot_patch_ktsb_base), %o0
1218	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1219	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
1220
1221#ifdef sun4v
1222	sethi	%hi(sfmmu_dslow_patch_ktsb_base), %o0
1223	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1224	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
1225#endif /* sun4v */
1226
1227	ldx 	[%l1], %o4		! load ktsb4m base addr (VA or PA)
1228
1229	sethi	%hi(dktsb4mbase), %o0
1230	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1231	  or	%o0, %lo(dktsb4mbase), %o0
1232
1233	sethi	%hi(iktsb4mbase), %o0
1234	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1235	  or	%o0, %lo(iktsb4mbase), %o0
1236
1237	sethi	%hi(sfmmu_kprot_patch_ktsb4m_base), %o0
1238	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1239	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
1240
1241#ifdef sun4v
1242	sethi	%hi(sfmmu_dslow_patch_ktsb4m_base), %o0
1243	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1244	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
1245#endif /* sun4v */
1246
1247	set	ktsb_szcode, %o4
1248	ld	[%o4], %o4
1249	sethi	%hi(sfmmu_kprot_patch_ktsb_szcode), %o0
1250	call	sfmmu_fixup_or		! patch value of ktsb_szcode
1251	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
1252
1253#ifdef sun4v
1254	sethi	%hi(sfmmu_dslow_patch_ktsb_szcode), %o0
1255	call	sfmmu_fixup_or		! patch value of ktsb_szcode
1256	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
1257#endif /* sun4v */
1258
1259	set	ktsb4m_szcode, %o4
1260	ld	[%o4], %o4
1261	sethi	%hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1262	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
1263	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1264
1265#ifdef sun4v
1266	sethi	%hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1267	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
1268	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1269#endif /* sun4v */
1270
1271	ret
1272	restore
1273	SET_SIZE(sfmmu_patch_ktsb)
1274
1275	ENTRY_NP(sfmmu_kpm_patch_tlbm)
1276	/*
1277	 * Fixup trap handlers in common segkpm case.  This is reserved
1278	 * for future use should kpm TSB be changed to be other than the
1279	 * kernel TSB.
1280	 */
1281	retl
1282	nop
1283	SET_SIZE(sfmmu_kpm_patch_tlbm)
1284
1285	ENTRY_NP(sfmmu_kpm_patch_tsbm)
1286	/*
1287	 * nop the branch to sfmmu_kpm_dtsb_miss_small
1288	 * in the case where we are using large pages for
1289	 * seg_kpm (and hence must probe the second TSB for
1290	 * seg_kpm VAs)
1291	 */
1292	set	dktsb4m_kpmcheck_small, %o0
1293	MAKE_NOP_INSTR(%o1)
1294	st	%o1, [%o0]
1295	flush	%o0
1296	retl
1297	nop
1298	SET_SIZE(sfmmu_kpm_patch_tsbm)
1299
1300	ENTRY_NP(sfmmu_patch_utsb)
1301#ifdef UTSB_PHYS
1302	retl
1303	nop
1304#else /* UTSB_PHYS */
1305	/*
1306	 * We need to hot patch utsb_vabase and utsb4m_vabase
1307	 */
1308	save	%sp, -SA(MINFRAME), %sp
1309
1310	/* patch value of utsb_vabase */
1311	set	utsb_vabase, %o1
1312	ldx	[%o1], %o4
1313	sethi	%hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1314	call	sfmmu_fixup_setx
1315	  or	%o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1316	sethi	%hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1317	call	sfmmu_fixup_setx
1318	  or	%o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1319	sethi	%hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1320	call	sfmmu_fixup_setx
1321	  or	%o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1322
1323	/* patch value of utsb4m_vabase */
1324	set	utsb4m_vabase, %o1
1325	ldx	[%o1], %o4
1326	sethi	%hi(sfmmu_uprot_get_2nd_tsb_base), %o0
1327	call	sfmmu_fixup_setx
1328	  or	%o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
1329	sethi	%hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
1330	call	sfmmu_fixup_setx
1331	  or	%o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
1332	sethi	%hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
1333	call	sfmmu_fixup_setx
1334	  or	%o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
1335
1336	/*
1337	 * Patch TSB base register masks and shifts if needed.
1338	 * By default the TSB base register contents are set up for 4M slab.
1339	 * If we're using a smaller slab size and reserved VA range we need
1340	 * to patch up those values here.
1341	 */
1342	set	tsb_slab_shift, %o1
1343	set	MMU_PAGESHIFT4M, %o4
1344	lduw	[%o1], %o3
1345	subcc	%o4, %o3, %o4
1346	bz,pt	%icc, 1f
1347	  /* delay slot safe */
1348
1349	/* patch reserved VA range size if needed. */
1350	sethi	%hi(sfmmu_tsb_1st_resv_offset), %o0
1351	call	sfmmu_fixup_shiftx
1352	  or	%o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
1353	call	sfmmu_fixup_shiftx
1354	  add	%o0, I_SIZE, %o0
1355	sethi	%hi(sfmmu_tsb_2nd_resv_offset), %o0
1356	call	sfmmu_fixup_shiftx
1357	  or	%o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
1358	call	sfmmu_fixup_shiftx
1359	  add	%o0, I_SIZE, %o0
13601:
1361	/* patch TSBREG_VAMASK used to set up TSB base register */
1362	set	tsb_slab_mask, %o1
1363	ldx	[%o1], %o4
1364	sethi	%hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
1365	call	sfmmu_fixup_or
1366	  or	%o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
1367	sethi	%hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1368	call	sfmmu_fixup_or
1369	  or	%o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1370
1371	ret
1372	restore
1373#endif /* UTSB_PHYS */
1374	SET_SIZE(sfmmu_patch_utsb)
1375
1376	ENTRY_NP(sfmmu_patch_shctx)
1377#ifdef sun4u
1378	retl
1379	  nop
1380#else /* sun4u */
1381	set	sfmmu_shctx_cpu_mondo_patch, %o0
1382	MAKE_JMP_INSTR(5, %o1, %o2)	! jmp       %g5
1383	st	%o1, [%o0]
1384	flush	%o0
1385	MAKE_NOP_INSTR(%o1)
1386	add	%o0, I_SIZE, %o0	! next instr
1387	st	%o1, [%o0]
1388	flush	%o0
1389
1390	set	sfmmu_shctx_user_rtt_patch, %o0
1391	st      %o1, [%o0]		! nop 1st instruction
1392	flush	%o0
1393	add     %o0, I_SIZE, %o0
1394	st      %o1, [%o0]		! nop 2nd instruction
1395	flush	%o0
1396	add     %o0, I_SIZE, %o0
1397	st      %o1, [%o0]		! nop 3rd instruction
1398	flush	%o0
1399	add     %o0, I_SIZE, %o0
1400	st      %o1, [%o0]		! nop 4th instruction
1401	flush	%o0
1402	add     %o0, I_SIZE, %o0
1403	st      %o1, [%o0]		! nop 5th instruction
1404	retl
1405	  flush	%o0
1406#endif /* sun4u */
1407	SET_SIZE(sfmmu_patch_shctx)
1408
1409	/*
1410	 * Routine that loads an entry into a tsb using virtual addresses.
1411	 * Locking is required since all cpus can use the same TSB.
1412	 * Note that it is no longer required to have a valid context
1413	 * when calling this function.
1414	 */
1415	ENTRY_NP(sfmmu_load_tsbe)
1416	/*
1417	 * %o0 = pointer to tsbe to load
1418	 * %o1 = tsb tag
1419	 * %o2 = virtual pointer to TTE
1420	 * %o3 = 1 if physical address in %o0 else 0
1421	 */
1422	rdpr	%pstate, %o5
1423#ifdef DEBUG
1424	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1)
1425#endif /* DEBUG */
1426
1427	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1428
1429	SETUP_TSB_ASI(%o3, %g3)
1430	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1)
1431
1432	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1433
1434	retl
1435	membar	#StoreStore|#StoreLoad
1436	SET_SIZE(sfmmu_load_tsbe)
1437
1438	/*
1439	 * Flush TSB of a given entry if the tag matches.
1440	 */
1441	ENTRY(sfmmu_unload_tsbe)
1442	/*
1443	 * %o0 = pointer to tsbe to be flushed
1444	 * %o1 = tag to match
1445	 * %o2 = 1 if physical address in %o0 else 0
1446	 */
1447	SETUP_TSB_ASI(%o2, %g1)
1448	TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
1449	retl
1450	membar	#StoreStore|#StoreLoad
1451	SET_SIZE(sfmmu_unload_tsbe)
1452
1453	/*
1454	 * Routine that loads a TTE into the kpm TSB from C code.
1455	 * Locking is required since kpm TSB is shared among all CPUs.
1456	 */
1457	ENTRY_NP(sfmmu_kpm_load_tsb)
1458	/*
1459	 * %o0 = vaddr
1460	 * %o1 = ttep
1461	 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
1462	 */
1463	rdpr	%pstate, %o5			! %o5 = saved pstate
1464#ifdef DEBUG
1465	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1)
1466#endif /* DEBUG */
1467	wrpr	%o5, PSTATE_IE, %pstate		! disable interrupts
1468
1469#ifndef sun4v
1470	sethi	%hi(ktsb_phys), %o4
1471	mov	ASI_N, %o3
1472	ld	[%o4 + %lo(ktsb_phys)], %o4
1473	movrnz	%o4, ASI_MEM, %o3
1474	mov	%o3, %asi
1475#endif /* !sun4v */
1476	mov	%o0, %g1			! %g1 = vaddr
1477
1478	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1479	GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
1480	/* %g2 = tsbep, %g1 clobbered */
1481
1482	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1483	/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
1484	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1)
1485
1486	wrpr	%g0, %o5, %pstate		! enable interrupts
1487	retl
1488	  membar #StoreStore|#StoreLoad
1489	SET_SIZE(sfmmu_kpm_load_tsb)
1490
1491	/*
1492	 * Routine that shoots down a TTE in the kpm TSB or in the
1493	 * kernel TSB depending on virtpg. Locking is required since
1494	 * kpm/kernel TSB is shared among all CPUs.
1495	 */
1496	ENTRY_NP(sfmmu_kpm_unload_tsb)
1497	/*
1498	 * %o0 = vaddr
1499	 * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
1500	 */
1501#ifndef sun4v
1502	sethi	%hi(ktsb_phys), %o4
1503	mov	ASI_N, %o3
1504	ld	[%o4 + %lo(ktsb_phys)], %o4
1505	movrnz	%o4, ASI_MEM, %o3
1506	mov	%o3, %asi
1507#endif /* !sun4v */
1508	mov	%o0, %g1			! %g1 = vaddr
1509
1510	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1511	GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
1512	/* %g2 = tsbep, %g1 clobbered */
1513
1514	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1515	/* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
1516	TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)
1517
1518	retl
1519	  membar	#StoreStore|#StoreLoad
1520	SET_SIZE(sfmmu_kpm_unload_tsb)
1521
1522#endif /* lint */
1523
1524
1525#if defined (lint)
1526
1527/*ARGSUSED*/
1528pfn_t
1529sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr)
1530{ return(0); }
1531
1532#else /* lint */
1533
1534	ENTRY_NP(sfmmu_ttetopfn)
1535	ldx	[%o0], %g1			/* read tte */
1536	TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
1537	/*
1538	 * g1 = pfn
1539	 */
1540	retl
1541	mov	%g1, %o0
1542	SET_SIZE(sfmmu_ttetopfn)
1543
1544#endif /* !lint */
1545
1546
1547#if defined (lint)
1548/*
1549 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the
1550 * the hash list.
1551 */
1552/* ARGSUSED */
1553void
1554sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1555	uint64_t hblkpa)
1556{
1557}
1558
1559/*
1560 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the
1561 * hash list.
1562 */
1563/* ARGSUSED */
1564void
1565sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1566	uint64_t hblkpa, struct hme_blk *prev_hblkp)
1567{
1568}
1569#else /* lint */
1570
1571/*
1572 * Functions to grab/release hme bucket list lock.  I only use a byte
1573 * instead of the whole int because eventually we might want to
1574 * put some counters on the other bytes (of course, these routines would
1575 * have to change).  The code that grab this lock should execute
1576 * with interrupts disabled and hold the lock for the least amount of time
1577 * possible.
1578 */
1579
1580/*
1581 * Even though hmeh_listlock is updated using pa there's no need to flush
1582 * dcache since hmeh_listlock will be restored to the original value (0)
1583 * before interrupts are reenabled.
1584 */
1585
1586/*
1587 * For sparcv9 hme hash buckets may not be in the nucleus.  hme hash update
1588 * routines still use virtual addresses to update the bucket fields. But they
1589 * must not cause a TLB miss after grabbing the low level bucket lock. To
1590 * achieve this we must make sure the bucket structure is completely within an
1591 * 8K page.
1592 */
1593
1594#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1))
1595#error - the size of hmehash_bucket structure is not power of 2
1596#endif
1597
1598/*
1599 * Enable backoff to significantly reduce locking overhead and reduce a chance
1600 * of xcall timeout. This is only enabled for sun4v as a Makefile compile-
1601 * time option.
1602 * The rd %ccr is better for performance compared to a non pipeline releasing
1603 * tight spin on N2/VF.
1604 * Backoff based fix is a temporary solution and doesn't allow scaling above
1605 * lock saturation point. The final fix is to eliminate HMELOCK_ENTER()
1606 * to avoid xcall timeouts and improve GET_TTE() performance.
1607 */
1608
1609#ifdef HMELOCK_BACKOFF_ENABLE
1610
1611#define HMELOCK_BACKOFF(reg, val)                               \
1612	set     val, reg                                        ;\
1613	rd	%ccr, %g0                                       ;\
1614	brnz	reg, .-4                                        ;\
1615	dec	reg
1616
1617#define CAS_HME(tmp1, tmp2, exitlabel, asi)                     \
1618	mov     0xff, tmp2                                      ;\
1619	casa    [tmp1]asi, %g0, tmp2                            ;\
1620	brz,a,pt tmp2, exitlabel                                ;\
1621	membar  #LoadLoad
1622
1623#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label, asi)            \
1624	mov     0xff, tmp2                                      ;\
1625	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1626	casa    [tmp1]asi, %g0, tmp2                            ;\
1627	brz,a,pt tmp2, label/**/2                               ;\
1628	membar  #LoadLoad                                       ;\
1629	HMELOCK_BACKOFF(tmp2,0x8)                               ;\
1630	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1631	HMELOCK_BACKOFF(tmp2,0x10)                              ;\
1632	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1633	HMELOCK_BACKOFF(tmp2,0x20)                              ;\
1634	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1635	HMELOCK_BACKOFF(tmp2,0x40)                              ;\
1636	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1637	HMELOCK_BACKOFF(tmp2,0x80)                              ;\
1638	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1639label/**/1:                                                     ;\
1640	HMELOCK_BACKOFF(tmp2,0x100)                             ;\
1641	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1642	HMELOCK_BACKOFF(tmp2,0x200)                             ;\
1643	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1644	HMELOCK_BACKOFF(tmp2,0x400)                             ;\
1645	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1646	HMELOCK_BACKOFF(tmp2,0x800)                             ;\
1647	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1648	HMELOCK_BACKOFF(tmp2,0x1000)                            ;\
1649	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1650	HMELOCK_BACKOFF(tmp2,0x2000)                            ;\
1651	mov     0xff, tmp2                                      ;\
1652	casa    [tmp1]asi, %g0, tmp2                            ;\
1653	brnz,pn tmp2, label/**/1     /* reset backoff */        ;\
1654	membar  #LoadLoad                                       ;\
1655label/**/2:
1656
1657#else /* HMELOCK_BACKOFF_ENABLE */
1658
1659#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi)           \
1660	mov     0xff, tmp2                                      ;\
1661	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1662label1:                                                         ;\
1663	casa    [tmp1]asi, %g0, tmp2                            ;\
1664	brnz,pn tmp2, label1                                    ;\
1665	mov     0xff, tmp2                                      ;\
1666	membar  #LoadLoad
1667
1668#endif /* HMELOCK_BACKOFF_ENABLE */
1669
1670#define HMELOCK_EXIT(hmebp, tmp1, asi)                          \
1671	membar  #LoadStore|#StoreStore                          ;\
1672	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1673	sta     %g0, [tmp1]asi
1674
1675	.seg	".data"
1676hblk_add_panic1:
1677	.ascii	"sfmmu_hblk_hash_add: interrupts disabled"
1678	.byte	0
1679hblk_add_panic2:
1680	.ascii	"sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not"
1681	.byte	0
1682	.align	4
1683	.seg	".text"
1684
1685	ENTRY_NP(sfmmu_hblk_hash_add)
1686	/*
1687	 * %o0 = hmebp
1688	 * %o1 = hmeblkp
1689	 * %o2 = hblkpa
1690	 */
1691	rdpr	%pstate, %o5
1692#ifdef DEBUG
1693	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1694	bnz,pt %icc, 3f				/* disabled, panic	 */
1695	  nop
1696	save	%sp, -SA(MINFRAME), %sp
1697	sethi	%hi(hblk_add_panic1), %o0
1698	call	panic
1699	 or	%o0, %lo(hblk_add_panic1), %o0
1700	ret
1701	restore
1702
17033:
1704#endif /* DEBUG */
1705	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1706	mov	%o2, %g1
1707
1708	/*
1709	 * g1 = hblkpa
1710	 */
1711	ldn	[%o0 + HMEBUCK_HBLK], %o4	/* next hmeblk */
1712	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = next hblkpa */
1713#ifdef	DEBUG
1714	cmp	%o4, %g0
1715	bne,pt %xcc, 1f
1716	 nop
1717	brz,pt %g2, 1f
1718	 nop
1719	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1720	save	%sp, -SA(MINFRAME), %sp
1721	sethi	%hi(hblk_add_panic2), %o0
1722	call	panic
1723	  or	%o0, %lo(hblk_add_panic2), %o0
1724	ret
1725	restore
17261:
1727#endif /* DEBUG */
1728	/*
1729	 * We update hmeblks entries before grabbing lock because the stores
1730	 * could take a tlb miss and require the hash lock.  The buckets
1731	 * are part of the nucleus so we are cool with those stores.
1732	 *
1733	 * if buckets are not part of the nucleus our game is to
1734	 * not touch any other page via va until we drop the lock.
1735	 * This guarantees we won't get a tlb miss before the lock release
1736	 * since interrupts are disabled.
1737	 */
1738	stn	%o4, [%o1 + HMEBLK_NEXT]	/* update hmeblk's next */
1739	stx	%g2, [%o1 + HMEBLK_NEXTPA]	/* update hmeblk's next pa */
1740	HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N)
1741	stn	%o1, [%o0 + HMEBUCK_HBLK]	/* update bucket hblk next */
1742	stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* add hmeblk to list */
1743	HMELOCK_EXIT(%o0, %g2, ASI_N)
1744	retl
1745	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1746	SET_SIZE(sfmmu_hblk_hash_add)
1747
1748	ENTRY_NP(sfmmu_hblk_hash_rm)
1749	/*
1750	 * This function removes an hmeblk from the hash chain.
1751	 * It is written to guarantee we don't take a tlb miss
1752	 * by using physical addresses to update the list.
1753	 *
1754	 * %o0 = hmebp
1755	 * %o1 = hmeblkp
1756	 * %o2 = hmeblkp previous pa
1757	 * %o3 = hmeblkp previous
1758	 */
1759
1760	mov	%o3, %o4			/* o4 = hmeblkp previous */
1761
1762	rdpr	%pstate, %o5
1763#ifdef DEBUG
1764	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1)
1765#endif /* DEBUG */
1766	/*
1767	 * disable interrupts, clear Address Mask to access 64 bit physaddr
1768	 */
1769	andn    %o5, PSTATE_IE, %g1
1770	wrpr    %g1, 0, %pstate
1771
1772#ifndef sun4v
1773	sethi   %hi(dcache_line_mask), %g4
1774	ld      [%g4 + %lo(dcache_line_mask)], %g4
1775#endif /* sun4v */
1776
1777	/*
1778	 * if buckets are not part of the nucleus our game is to
1779	 * not touch any other page via va until we drop the lock.
1780	 * This guarantees we won't get a tlb miss before the lock release
1781	 * since interrupts are disabled.
1782	 */
1783	HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N)
1784	ldn	[%o0 + HMEBUCK_HBLK], %g2	/* first hmeblk in list */
1785	cmp	%g2, %o1
1786	bne,pt	%ncc,1f
1787	 mov	ASI_MEM, %asi
1788	/*
1789	 * hmeblk is first on list
1790	 */
1791	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = hmeblk pa */
1792	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1793	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1794	stn	%o3, [%o0 + HMEBUCK_HBLK]	/* write va */
1795	ba,pt	%xcc, 2f
1796	  stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* write pa */
17971:
1798	/* hmeblk is not first on list */
1799
1800	mov	%o2, %g3
1801#ifndef sun4v
1802	GET_CPU_IMPL(%g2)
1803	cmp 	%g2, CHEETAH_IMPL
1804	bge,a,pt %icc, hblk_hash_rm_1
1805	  and	%o4, %g4, %g2
1806	cmp	%g2, SPITFIRE_IMPL
1807	blt	%icc, hblk_hash_rm_2		/* no flushing needed for OPL */
1808	  and	%o4, %g4, %g2
1809	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev pa from dcache */
1810	add	%o4, HMEBLK_NEXT, %o4
1811	and	%o4, %g4, %g2
1812	ba	hblk_hash_rm_2
1813	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev va from dcache */
1814hblk_hash_rm_1:
1815
1816	stxa	%g0, [%g3]ASI_DC_INVAL		/* flush prev pa from dcache */
1817	membar	#Sync
1818	add     %g3, HMEBLK_NEXT, %g2
1819	stxa	%g0, [%g2]ASI_DC_INVAL		/* flush prev va from dcache */
1820hblk_hash_rm_2:
1821	membar	#Sync
1822#endif /* sun4v */
1823	ldxa	[%g3 + HMEBLK_NEXTPA] %asi, %g2	/* g2 = hmeblk pa */
1824	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1825	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1826	stna	%o3, [%g3 + HMEBLK_NEXT] %asi	/* write va */
1827	stxa	%g1, [%g3 + HMEBLK_NEXTPA] %asi	/* write pa */
18282:
1829	HMELOCK_EXIT(%o0, %g2, ASI_N)
1830	retl
1831	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1832	SET_SIZE(sfmmu_hblk_hash_rm)
1833
1834#endif /* lint */
1835
1836/*
1837 * These macros are used to update global sfmmu hme hash statistics
1838 * in perf critical paths. It is only enabled in debug kernels or
1839 * if SFMMU_STAT_GATHER is defined
1840 */
1841#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
1842#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1843	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1844	mov	HATSTAT_KHASH_SEARCH, tmp2				;\
1845	cmp	tmp1, hatid						;\
1846	movne	%ncc, HATSTAT_UHASH_SEARCH, tmp2			;\
1847	set	sfmmu_global_stat, tmp1					;\
1848	add	tmp1, tmp2, tmp1					;\
1849	ld	[tmp1], tmp2						;\
1850	inc	tmp2							;\
1851	st	tmp2, [tmp1]
1852
1853#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1854	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1855	mov	HATSTAT_KHASH_LINKS, tmp2				;\
1856	cmp	tmp1, hatid						;\
1857	movne	%ncc, HATSTAT_UHASH_LINKS, tmp2				;\
1858	set	sfmmu_global_stat, tmp1					;\
1859	add	tmp1, tmp2, tmp1					;\
1860	ld	[tmp1], tmp2						;\
1861	inc	tmp2							;\
1862	st	tmp2, [tmp1]
1863
1864
1865#else /* DEBUG || SFMMU_STAT_GATHER */
1866
1867#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1868
1869#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1870
1871#endif  /* DEBUG || SFMMU_STAT_GATHER */
1872
1873/*
1874 * This macro is used to update global sfmmu kstas in non
1875 * perf critical areas so they are enabled all the time
1876 */
1877#define	HAT_GLOBAL_STAT(statname, tmp1, tmp2)				\
1878	sethi	%hi(sfmmu_global_stat), tmp1				;\
1879	add	tmp1, statname, tmp1					;\
1880	ld	[tmp1 + %lo(sfmmu_global_stat)], tmp2			;\
1881	inc	tmp2							;\
1882	st	tmp2, [tmp1 + %lo(sfmmu_global_stat)]
1883
1884/*
1885 * These macros are used to update per cpu stats in non perf
1886 * critical areas so they are enabled all the time
1887 */
1888#define	HAT_PERCPU_STAT32(tsbarea, stat, tmp1)				\
1889	ld	[tsbarea + stat], tmp1					;\
1890	inc	tmp1							;\
1891	st	tmp1, [tsbarea + stat]
1892
1893/*
1894 * These macros are used to update per cpu stats in non perf
1895 * critical areas so they are enabled all the time
1896 */
1897#define	HAT_PERCPU_STAT16(tsbarea, stat, tmp1)				\
1898	lduh	[tsbarea + stat], tmp1					;\
1899	inc	tmp1							;\
1900	stuh	tmp1, [tsbarea + stat]
1901
1902#if defined(KPM_TLBMISS_STATS_GATHER)
1903	/*
1904	 * Count kpm dtlb misses separately to allow a different
1905	 * evaluation of hme and kpm tlbmisses. kpm tsb hits can
1906	 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
1907	 */
1908#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)		\
1909	brgez	tagacc, label	/* KPM VA? */				;\
1910	nop								;\
1911	CPU_INDEX(tmp1, tsbma)						;\
1912	sethi	%hi(kpmtsbm_area), tsbma				;\
1913	sllx	tmp1, KPMTSBM_SHIFT, tmp1				;\
1914	or	tsbma, %lo(kpmtsbm_area), tsbma				;\
1915	add	tsbma, tmp1, tsbma		/* kpmtsbm area */	;\
1916	/* VA range check */						;\
1917	ldx	[tsbma + KPMTSBM_VBASE], val				;\
1918	cmp	tagacc, val						;\
1919	blu,pn	%xcc, label						;\
1920	  ldx	[tsbma + KPMTSBM_VEND], tmp1				;\
1921	cmp	tagacc, tmp1						;\
1922	bgeu,pn	%xcc, label						;\
1923	  lduw	[tsbma + KPMTSBM_DTLBMISS], val				;\
1924	inc	val							;\
1925	st	val, [tsbma + KPMTSBM_DTLBMISS]				;\
1926label:
1927#else
1928#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
1929#endif	/* KPM_TLBMISS_STATS_GATHER */
1930
1931#if defined (lint)
1932/*
1933 * The following routines are jumped to from the mmu trap handlers to do
1934 * the setting up to call systrap.  They are separate routines instead of
1935 * being part of the handlers because the handlers would exceed 32
1936 * instructions and since this is part of the slow path the jump
1937 * cost is irrelevant.
1938 */
1939void
1940sfmmu_pagefault(void)
1941{
1942}
1943
1944void
1945sfmmu_mmu_trap(void)
1946{
1947}
1948
1949void
1950sfmmu_window_trap(void)
1951{
1952}
1953
1954void
1955sfmmu_kpm_exception(void)
1956{
1957}
1958
1959#else /* lint */
1960
1961#ifdef	PTL1_PANIC_DEBUG
1962	.seg	".data"
1963	.global	test_ptl1_panic
1964test_ptl1_panic:
1965	.word	0
1966	.align	8
1967
1968	.seg	".text"
1969	.align	4
1970#endif	/* PTL1_PANIC_DEBUG */
1971
1972
1973	ENTRY_NP(sfmmu_pagefault)
1974	SET_GL_REG(1)
1975	USE_ALTERNATE_GLOBALS(%g5)
1976	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
1977	rdpr	%tt, %g6
1978	cmp	%g6, FAST_IMMU_MISS_TT
1979	be,a,pn	%icc, 1f
1980	  mov	T_INSTR_MMU_MISS, %g3
1981	cmp	%g6, T_INSTR_MMU_MISS
1982	be,a,pn	%icc, 1f
1983	  mov	T_INSTR_MMU_MISS, %g3
1984	mov	%g5, %g2
1985	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1986	cmp	%g6, FAST_DMMU_MISS_TT
1987	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1988	cmp	%g6, T_DATA_MMU_MISS
1989	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1990
1991#ifdef  PTL1_PANIC_DEBUG
1992	/* check if we want to test the tl1 panic */
1993	sethi	%hi(test_ptl1_panic), %g4
1994	ld	[%g4 + %lo(test_ptl1_panic)], %g1
1995	st	%g0, [%g4 + %lo(test_ptl1_panic)]
1996	cmp	%g1, %g0
1997	bne,a,pn %icc, ptl1_panic
1998	  or	%g0, PTL1_BAD_DEBUG, %g1
1999#endif	/* PTL1_PANIC_DEBUG */
20001:
2001	HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
2002	/*
2003	 * g2 = tag access reg
2004	 * g3.l = type
2005	 * g3.h = 0
2006	 */
2007	sethi	%hi(trap), %g1
2008	or	%g1, %lo(trap), %g1
20092:
2010	ba,pt	%xcc, sys_trap
2011	  mov	-1, %g4
2012	SET_SIZE(sfmmu_pagefault)
2013
2014	ENTRY_NP(sfmmu_mmu_trap)
2015	SET_GL_REG(1)
2016	USE_ALTERNATE_GLOBALS(%g5)
2017	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
2018	rdpr	%tt, %g6
2019	cmp	%g6, FAST_IMMU_MISS_TT
2020	be,a,pn	%icc, 1f
2021	  mov	T_INSTR_MMU_MISS, %g3
2022	cmp	%g6, T_INSTR_MMU_MISS
2023	be,a,pn	%icc, 1f
2024	  mov	T_INSTR_MMU_MISS, %g3
2025	mov	%g5, %g2
2026	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
2027	cmp	%g6, FAST_DMMU_MISS_TT
2028	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
2029	cmp	%g6, T_DATA_MMU_MISS
2030	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
20311:
2032	/*
2033	 * g2 = tag access reg
2034	 * g3 = type
2035	 */
2036	sethi	%hi(sfmmu_tsbmiss_exception), %g1
2037	or	%g1, %lo(sfmmu_tsbmiss_exception), %g1
2038	ba,pt	%xcc, sys_trap
2039	  mov	-1, %g4
2040	/*NOTREACHED*/
2041	SET_SIZE(sfmmu_mmu_trap)
2042
2043	ENTRY_NP(sfmmu_suspend_tl)
2044	SET_GL_REG(1)
2045	USE_ALTERNATE_GLOBALS(%g5)
2046	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
2047	rdpr	%tt, %g6
2048	cmp	%g6, FAST_IMMU_MISS_TT
2049	be,a,pn	%icc, 1f
2050	  mov	T_INSTR_MMU_MISS, %g3
2051	mov	%g5, %g2
2052	cmp	%g6, FAST_DMMU_MISS_TT
2053	move	%icc, T_DATA_MMU_MISS, %g3
2054	movne	%icc, T_DATA_PROT, %g3
20551:
2056	sethi	%hi(sfmmu_tsbmiss_suspended), %g1
2057	or	%g1, %lo(sfmmu_tsbmiss_suspended), %g1
2058	/* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
2059	ba,pt	%xcc, sys_trap
2060	  mov	PIL_15, %g4
2061	/*NOTREACHED*/
2062	SET_SIZE(sfmmu_suspend_tl)
2063
2064	/*
2065	 * No %g registers in use at this point.
2066	 */
2067	ENTRY_NP(sfmmu_window_trap)
2068	rdpr	%tpc, %g1
2069#ifdef sun4v
2070#ifdef DEBUG
2071	/* We assume previous %gl was 1 */
2072	rdpr	%tstate, %g4
2073	srlx	%g4, TSTATE_GL_SHIFT, %g4
2074	and	%g4, TSTATE_GL_MASK, %g4
2075	cmp	%g4, 1
2076	bne,a,pn %icc, ptl1_panic
2077	  mov	PTL1_BAD_WTRAP, %g1
2078#endif /* DEBUG */
2079	/* user miss at tl>1. better be the window handler or user_rtt */
2080	/* in user_rtt? */
2081	set	rtt_fill_start, %g4
2082	cmp	%g1, %g4
2083	blu,pn %xcc, 6f
2084	 .empty
2085	set	rtt_fill_end, %g4
2086	cmp	%g1, %g4
2087	bgeu,pn %xcc, 6f
2088	 nop
2089	set	fault_rtt_fn1, %g1
2090	wrpr	%g0, %g1, %tnpc
2091	ba,a	7f
20926:
2093	! must save this trap level before descending trap stack
2094	! no need to save %tnpc, either overwritten or discarded
2095	! already got it: rdpr	%tpc, %g1
2096	rdpr	%tstate, %g6
2097	rdpr	%tt, %g7
2098	! trap level saved, go get underlying trap type
2099	rdpr	%tl, %g5
2100	sub	%g5, 1, %g3
2101	wrpr	%g3, %tl
2102	rdpr	%tt, %g2
2103	wrpr	%g5, %tl
2104	! restore saved trap level
2105	wrpr	%g1, %tpc
2106	wrpr	%g6, %tstate
2107	wrpr	%g7, %tt
2108#else /* sun4v */
2109	/* user miss at tl>1. better be the window handler */
2110	rdpr	%tl, %g5
2111	sub	%g5, 1, %g3
2112	wrpr	%g3, %tl
2113	rdpr	%tt, %g2
2114	wrpr	%g5, %tl
2115#endif /* sun4v */
2116	and	%g2, WTRAP_TTMASK, %g4
2117	cmp	%g4, WTRAP_TYPE
2118	bne,pn	%xcc, 1f
2119	 nop
2120	/* tpc should be in the trap table */
2121	set	trap_table, %g4
2122	cmp	%g1, %g4
2123	blt,pn %xcc, 1f
2124	 .empty
2125	set	etrap_table, %g4
2126	cmp	%g1, %g4
2127	bge,pn %xcc, 1f
2128	 .empty
2129	andn	%g1, WTRAP_ALIGN, %g1	/* 128 byte aligned */
2130	add	%g1, WTRAP_FAULTOFF, %g1
2131	wrpr	%g0, %g1, %tnpc
21327:
2133	/*
2134	 * some wbuf handlers will call systrap to resolve the fault
2135	 * we pass the trap type so they figure out the correct parameters.
2136	 * g5 = trap type, g6 = tag access reg
2137	 */
2138
2139	/*
2140	 * only use g5, g6, g7 registers after we have switched to alternate
2141	 * globals.
2142	 */
2143	SET_GL_REG(1)
2144	USE_ALTERNATE_GLOBALS(%g5)
2145	GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
2146	rdpr	%tt, %g7
2147	cmp	%g7, FAST_IMMU_MISS_TT
2148	be,a,pn	%icc, ptl1_panic
2149	  mov	PTL1_BAD_WTRAP, %g1
2150	cmp	%g7, T_INSTR_MMU_MISS
2151	be,a,pn	%icc, ptl1_panic
2152	  mov	PTL1_BAD_WTRAP, %g1
2153	mov	T_DATA_PROT, %g5
2154	cmp	%g7, FAST_DMMU_MISS_TT
2155	move	%icc, T_DATA_MMU_MISS, %g5
2156	cmp	%g7, T_DATA_MMU_MISS
2157	move	%icc, T_DATA_MMU_MISS, %g5
2158	! XXXQ AGS re-check out this one
2159	done
21601:
2161	CPU_PADDR(%g1, %g4)
2162	add	%g1, CPU_TL1_HDLR, %g1
2163	lda	[%g1]ASI_MEM, %g4
2164	brnz,a,pt %g4, sfmmu_mmu_trap
2165	  sta	%g0, [%g1]ASI_MEM
2166	ba,pt	%icc, ptl1_panic
2167	  mov	PTL1_BAD_TRAP, %g1
2168	SET_SIZE(sfmmu_window_trap)
2169
2170	ENTRY_NP(sfmmu_kpm_exception)
2171	/*
2172	 * We have accessed an unmapped segkpm address or a legal segkpm
2173	 * address which is involved in a VAC alias conflict prevention.
2174	 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
2175	 * set. If it is, we will instead note that a fault has occurred
2176	 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
2177	 * a "retry"). This will step over the faulting instruction.
2178	 * Note that this means that a legal segkpm address involved in
2179	 * a VAC alias conflict prevention (a rare case to begin with)
2180	 * cannot be used in DTrace.
2181	 */
2182	CPU_INDEX(%g1, %g2)
2183	set	cpu_core, %g2
2184	sllx	%g1, CPU_CORE_SHIFT, %g1
2185	add	%g1, %g2, %g1
2186	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
2187	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
2188	bz	0f
2189	or	%g2, CPU_DTRACE_BADADDR, %g2
2190	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
2191	GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
2192	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
2193	done
21940:
2195	TSTAT_CHECK_TL1(1f, %g1, %g2)
21961:
2197	SET_GL_REG(1)
2198	USE_ALTERNATE_GLOBALS(%g5)
2199	GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
2200	mov	T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
2201	/*
2202	 * g2=tagacc g3.l=type g3.h=0
2203	 */
2204	sethi	%hi(trap), %g1
2205	or	%g1, %lo(trap), %g1
2206	ba,pt	%xcc, sys_trap
2207	mov	-1, %g4
2208	SET_SIZE(sfmmu_kpm_exception)
2209
2210#endif /* lint */
2211
2212#if defined (lint)
2213
2214void
2215sfmmu_tsb_miss(void)
2216{
2217}
2218
2219void
2220sfmmu_kpm_dtsb_miss(void)
2221{
2222}
2223
2224void
2225sfmmu_kpm_dtsb_miss_small(void)
2226{
2227}
2228
2229#else /* lint */
2230
2231#if (IMAP_SEG != 0)
2232#error - ism_map->ism_seg offset is not zero
2233#endif
2234
2235/*
2236 * Copies ism mapping for this ctx in param "ism" if this is a ISM
2237 * tlb miss and branches to label "ismhit". If this is not an ISM
2238 * process or an ISM tlb miss it falls thru.
2239 *
2240 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
2241 * this process.
2242 * If so, it will branch to label "ismhit".  If not, it will fall through.
2243 *
2244 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
2245 * so that any other threads of this process will not try and walk the ism
2246 * maps while they are being changed.
2247 *
2248 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
2249 *       will make sure of that. This means we can terminate our search on
2250 *       the first zero mapping we find.
2251 *
2252 * Parameters:
2253 * tagacc	= (pseudo-)tag access register (vaddr + ctx) (in)
2254 * tsbmiss	= address of tsb miss area (in)
2255 * ismseg	= contents of ism_seg for this ism map (out)
2256 * ismhat	= physical address of imap_ismhat for this ism map (out)
2257 * tmp1		= scratch reg (CLOBBERED)
2258 * tmp2		= scratch reg (CLOBBERED)
2259 * tmp3		= scratch reg (CLOBBERED)
2260 * label:    temporary labels
2261 * ismhit:   label where to jump to if an ism dtlb miss
2262 * exitlabel:label where to jump if hat is busy due to hat_unshare.
2263 */
2264#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
2265	label, ismhit)							\
2266	ldx	[tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */	;\
2267	brlz,pt  tmp1, label/**/3		/* exit if -1 */	;\
2268	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0] */	;\
2269label/**/1:								;\
2270	ldxa	[ismhat]ASI_MEM, ismseg	/* ismblk.map[0].ism_seg */	;\
2271	mov	tmp1, tmp3	/* update current ismblkpa head */	;\
2272label/**/2:								;\
2273	brz,pt  ismseg, label/**/3		/* no mapping */	;\
2274	  add	ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */	;\
2275	lduba	[tmp1]ASI_MEM, tmp1 		/* tmp1 = vb shift*/	;\
2276	srlx	ismseg, tmp1, tmp2		/* tmp2 = vbase */	;\
2277	srlx	tagacc, tmp1, tmp1		/* tmp1 =  va seg*/	;\
2278	sub	tmp1, tmp2, tmp2		/* tmp2 = va - vbase */	;\
2279	add	ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */	;\
2280	lda	[tmp1]ASI_MEM, tmp1		/* tmp1 = sz_mask */	;\
2281	and	ismseg, tmp1, tmp1		/* tmp1 = size */	;\
2282	cmp	tmp2, tmp1		 	/* check va <= offset*/	;\
2283	blu,a,pt  %xcc, ismhit			/* ism hit */		;\
2284	  add	ismhat, IMAP_ISMHAT, ismhat 	/* ismhat = &ism_sfmmu*/ ;\
2285									;\
2286	add	ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ 	;\
2287	add	tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1	;\
2288	cmp	ismhat, tmp1						;\
2289	bl,pt	%xcc, label/**/2		/* keep looking  */	;\
2290	  ldxa	[ismhat]ASI_MEM, ismseg	/* ismseg = map[ismhat] */	;\
2291									;\
2292	add	tmp3, IBLK_NEXTPA, tmp1					;\
2293	ldxa	[tmp1]ASI_MEM, tmp1		/* check blk->nextpa */	;\
2294	brgez,pt tmp1, label/**/1		/* continue if not -1*/	;\
2295	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0]*/	;\
2296label/**/3:
2297
2298/*
2299 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
2300 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
2301 * Parameters:
2302 * tagacc = reg containing virtual address
2303 * hatid = reg containing sfmmu pointer
2304 * hmeshift = constant/register to shift vaddr to obtain vapg
2305 * hmebp = register where bucket pointer will be stored
2306 * vapg = register where virtual page will be stored
2307 * tmp1, tmp2 = tmp registers
2308 */
2309
2310
2311#define	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp,	\
2312	vapg, label, tmp1, tmp2)					\
2313	sllx	tagacc, TAGACC_CTX_LSHIFT, tmp1				;\
2314	brnz,a,pt tmp1, label/**/1					;\
2315	  ld    [tsbarea + TSBMISS_UHASHSZ], hmebp			;\
2316	ld	[tsbarea + TSBMISS_KHASHSZ], hmebp			;\
2317	ba,pt	%xcc, label/**/2					;\
2318	  ldx	[tsbarea + TSBMISS_KHASHSTART], tmp1			;\
2319label/**/1:								;\
2320	ldx	[tsbarea + TSBMISS_UHASHSTART], tmp1			;\
2321label/**/2:								;\
2322	srlx	tagacc, hmeshift, vapg					;\
2323	xor	vapg, hatid, tmp2	/* hatid ^ (vaddr >> shift) */	;\
2324	and	tmp2, hmebp, hmebp	/* index into hme_hash */	;\
2325	mulx	hmebp, HMEBUCK_SIZE, hmebp				;\
2326	add	hmebp, tmp1, hmebp
2327
2328/*
2329 * hashtag includes bspage + hashno (64 bits).
2330 */
2331
2332#define	MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag)		\
2333	sllx	vapg, hmeshift, vapg					;\
2334	mov	hashno, hblktag						;\
2335	sllx	hblktag, HTAG_REHASH_SHIFT, hblktag			;\
2336	or	vapg, hblktag, hblktag
2337
2338/*
2339 * Function to traverse hmeblk hash link list and find corresponding match.
2340 * The search is done using physical pointers. It returns the physical address
2341 * and virtual address pointers to the hmeblk that matches with the tag
2342 * provided.
2343 * Parameters:
2344 * hmebp	= register that points to hme hash bucket, also used as
2345 *		  tmp reg (clobbered)
2346 * hmeblktag	= register with hmeblk tag match
2347 * hatid	= register with hatid
2348 * hmeblkpa	= register where physical ptr will be stored
2349 * hmeblkva	= register where virtual ptr will be stored
2350 * tmp1		= tmp reg
2351 * label: temporary label
2352 */
2353
2354#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva,	\
2355	tsbarea, tmp1, label)					 	\
2356	add     hmebp, HMEBUCK_NEXTPA, hmeblkpa				;\
2357	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
2358	add     hmebp, HMEBUCK_HBLK, hmeblkva				;\
2359	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
2360	HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
2361label/**/1:								;\
2362	brz,pn	hmeblkva, label/**/2					;\
2363	HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
2364	add	hmeblkpa, HMEBLK_TAG, hmebp				;\
2365	ldxa	[hmebp]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
2366	add	hmebp, CLONGSIZE, hmebp					;\
2367	ldxa	[hmebp]ASI_MEM, hmebp 	/* read 2nd part of tag */	;\
2368	xor	tmp1, hmeblktag, tmp1					;\
2369	xor	hmebp, hatid, hmebp					;\
2370	or	hmebp, tmp1, hmebp					;\
2371	brz,pn	hmebp, label/**/2	/* branch on hit */		;\
2372	  add	hmeblkpa, HMEBLK_NEXT, hmebp				;\
2373	ldna	[hmebp]ASI_MEM, hmeblkva	/* hmeblk ptr va */	;\
2374	add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
2375	ba,pt	%xcc, label/**/1					;\
2376	  ldxa	[hmebp]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */	;\
2377label/**/2:
2378
2379/*
2380 * Function to traverse hmeblk hash link list and find corresponding match.
2381 * The search is done using physical pointers. It returns the physical address
2382 * and virtual address pointers to the hmeblk that matches with the tag
2383 * provided.
2384 * Parameters:
2385 * hmeblktag	= register with hmeblk tag match (rid field is 0)
2386 * hatid	= register with hatid (pointer to SRD)
2387 * hmeblkpa	= register where physical ptr will be stored
2388 * hmeblkva	= register where virtual ptr will be stored
2389 * tmp1		= tmp reg
2390 * tmp2		= tmp reg
2391 * label: temporary label
2392 */
2393
2394#define	HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, hmeblkva,	\
2395	tsbarea, tmp1, tmp2, label)			 		\
2396label/**/1:								;\
2397	brz,pn	hmeblkva, label/**/4					;\
2398	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			;\
2399	add	hmeblkpa, HMEBLK_TAG, tmp2				;\
2400	ldxa	[tmp2]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
2401	add	tmp2, CLONGSIZE, tmp2					;\
2402	ldxa	[tmp2]ASI_MEM, tmp2 	/* read 2nd part of tag */	;\
2403	xor	tmp1, hmeblktag, tmp1					;\
2404	xor	tmp2, hatid, tmp2					;\
2405	brz,pn	tmp2, label/**/3	/* branch on hit */		;\
2406	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
2407label/**/2:								;\
2408	ldna	[tmp2]ASI_MEM, hmeblkva	/* hmeblk ptr va */		;\
2409	add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
2410	ba,pt	%xcc, label/**/1					;\
2411	  ldxa	[tmp2]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */		;\
2412label/**/3:								;\
2413	cmp	tmp1, SFMMU_MAX_HME_REGIONS				;\
2414	bgeu,pt	%xcc, label/**/2					;\
2415	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
2416	and	tmp1, BT_ULMASK, tmp2					;\
2417	srlx	tmp1, BT_ULSHIFT, tmp1					;\
2418	sllx	tmp1, CLONGSHIFT, tmp1					;\
2419	add	tsbarea, tmp1, tmp1					;\
2420	ldx	[tmp1 + TSBMISS_SHMERMAP], tmp1				;\
2421	srlx	tmp1, tmp2, tmp1					;\
2422	btst	0x1, tmp1						;\
2423	bz,pn	%xcc, label/**/2					;\
2424	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
2425label/**/4:
2426
2427#if ((1 << SFHME_SHIFT) != SFHME_SIZE)
2428#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
2429#endif
2430
2431/*
2432 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
2433 * he offset for the corresponding hment.
2434 * Parameters:
2435 * In:
2436 *	vaddr = register with virtual address
2437 *	hmeblkpa = physical pointer to hme_blk
2438 * Out:
2439 *	hmentoff = register where hment offset will be stored
2440 *	hmemisc = hblk_misc
2441 * Scratch:
2442 *	tmp1
2443 */
2444#define	HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\
2445	add	hmeblkpa, HMEBLK_MISC, hmentoff				;\
2446	lda	[hmentoff]ASI_MEM, hmemisc 				;\
2447	andcc	hmemisc, HBLK_SZMASK, %g0				;\
2448	bnz,a,pn  %icc, label1		/* if sz != TTE8K branch */	;\
2449	  or	%g0, HMEBLK_HME1, hmentoff				;\
2450	srl	vaddr, MMU_PAGESHIFT, tmp1				;\
2451	and	tmp1, NHMENTS - 1, tmp1		/* tmp1 = index */	;\
2452	sllx	tmp1, SFHME_SHIFT, tmp1					;\
2453	add	tmp1, HMEBLK_HME1, hmentoff				;\
2454label1:
2455
2456/*
2457 * GET_TTE is a macro that returns a TTE given a tag and hatid.
2458 *
2459 * tagacc	= (pseudo-)tag access register (in)
2460 * hatid	= sfmmu pointer for TSB miss (in)
2461 * tte		= tte for TLB miss if found, otherwise clobbered (out)
2462 * hmeblkpa	= PA of hment if found, otherwise clobbered (out)
2463 * hmeblkva	= VA of hment if found, otherwise clobbered (out)
2464 * tsbarea	= pointer to the tsbmiss area for this cpu. (in)
2465 * hmemisc	= hblk_misc if TTE is found (out), otherwise clobbered
2466 * hmeshift	= constant/register to shift VA to obtain the virtual pfn
2467 *		  for this page size.
2468 * hashno	= constant/register hash number
2469 * label	= temporary label for branching within macro.
2470 * foundlabel	= label to jump to when tte is found.
2471 * suspendlabel= label to jump to when tte is suspended.
2472 * exitlabel	= label to jump to when tte is not found.
2473 *
2474 */
2475#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmemisc, \
2476		hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \
2477									;\
2478	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2479	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2480	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2481		hmeblkpa, label/**/5, hmemisc, hmeblkva)		;\
2482									;\
2483	/*								;\
2484	 * tagacc = tagacc						;\
2485	 * hatid = hatid						;\
2486	 * tsbarea = tsbarea						;\
2487	 * tte   = hmebp (hme bucket pointer)				;\
2488	 * hmeblkpa  = vapg  (virtual page)				;\
2489	 * hmemisc, hmeblkva = scratch					;\
2490	 */								;\
2491	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
2492	or	hmemisc, SFMMU_INVALID_SHMERID, hmemisc			;\
2493									;\
2494	/*								;\
2495	 * tagacc = tagacc						;\
2496	 * hatid = hatid						;\
2497	 * tte   = hmebp						;\
2498	 * hmeblkpa  = CLOBBERED					;\
2499	 * hmemisc  = htag_bspage+hashno+invalid_rid			;\
2500	 * hmeblkva  = scratch						;\
2501	 */								;\
2502	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2503	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
2504	HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, hmeblkva, 	\
2505		tsbarea, tagacc, label/**/1)				;\
2506	/*								;\
2507	 * tagacc = CLOBBERED						;\
2508	 * tte = CLOBBERED						;\
2509	 * hmeblkpa = hmeblkpa						;\
2510	 * hmeblkva = hmeblkva						;\
2511	 */								;\
2512	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
2513	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2514	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
2515	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
2516	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2517	  nop								;\
2518label/**/4:								;\
2519	/*								;\
2520	 * We have found the hmeblk containing the hment.		;\
2521	 * Now we calculate the corresponding tte.			;\
2522	 *								;\
2523	 * tagacc = tagacc						;\
2524	 * hatid = hatid						;\
2525	 * tte   = clobbered						;\
2526	 * hmeblkpa  = hmeblkpa						;\
2527	 * hmemisc  = hblktag						;\
2528	 * hmeblkva  = hmeblkva 					;\
2529	 */								;\
2530	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
2531		label/**/2)						;\
2532									;\
2533	/*								;\
2534	 * tagacc = tagacc						;\
2535	 * hatid = hmentoff						;\
2536	 * tte   = clobbered						;\
2537	 * hmeblkpa  = hmeblkpa						;\
2538	 * hmemisc  = hblk_misc						;\
2539	 * hmeblkva  = hmeblkva 					;\
2540	 */								;\
2541									;\
2542	add	hatid, SFHME_TTE, hatid					;\
2543	add	hmeblkpa, hatid, hmeblkpa				;\
2544	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2545	add	hmeblkva, hatid, hmeblkva				;\
2546	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2547	HMELOCK_EXIT(hatid, hatid, ASI_MEM)	/* drop lock */		;\
2548	set	TTE_SUSPEND, hatid					;\
2549	TTE_SUSPEND_INT_SHIFT(hatid)					;\
2550	btst	tte, hatid						;\
2551	bz,pt	%xcc, foundlabel					;\
2552	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2553									;\
2554	/*								;\
2555	 * Mapping is suspended, so goto suspend label.			;\
2556	 */								;\
2557	ba,pt	%xcc, suspendlabel					;\
2558	  nop
2559
2560/*
2561 * GET_SHME_TTE is similar to GET_TTE() except it searches
2562 * shared hmeblks via HMEHASH_SEARCH_SHME() macro.
2563 * If valid tte is found, hmemisc = shctx flag, i.e., shme is
2564 * either 0 (not part of scd) or 1 (part of scd).
2565 */
2566#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea,	\
2567		hmemisc, hmeshift, hashno, label, foundlabel,		\
2568		suspendlabel, exitlabel)				\
2569									;\
2570	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2571	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2572	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2573		hmeblkpa, label/**/5, hmemisc, hmeblkva)		;\
2574									;\
2575	/*								;\
2576	 * tagacc = tagacc						;\
2577	 * hatid = hatid						;\
2578	 * tsbarea = tsbarea						;\
2579	 * tte   = hmebp (hme bucket pointer)				;\
2580	 * hmeblkpa  = vapg  (virtual page)				;\
2581	 * hmemisc, hmeblkva = scratch					;\
2582	 */								;\
2583	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
2584									;\
2585	/*								;\
2586	 * tagacc = tagacc						;\
2587	 * hatid = hatid						;\
2588	 * tsbarea = tsbarea						;\
2589	 * tte   = hmebp						;\
2590	 * hmemisc  = htag_bspage + hashno + 0 (for rid)		;\
2591	 * hmeblkpa  = CLOBBERED					;\
2592	 * hmeblkva  = scratch						;\
2593	 */								;\
2594	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2595	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
2596									;\
2597	add     tte, HMEBUCK_NEXTPA, hmeblkpa				;\
2598	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
2599	add     tte, HMEBUCK_HBLK, hmeblkva				;\
2600	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
2601	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte)			;\
2602									;\
2603label/**/8:								;\
2604	HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, hmeblkva, 	\
2605		tsbarea, tagacc, tte, label/**/1)			;\
2606	/*								;\
2607	 * tagacc = CLOBBERED						;\
2608	 * tte = CLOBBERED						;\
2609	 * hmeblkpa = hmeblkpa						;\
2610	 * hmeblkva = hmeblkva						;\
2611	 */								;\
2612	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
2613	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2614	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
2615	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
2616	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2617	  nop								;\
2618label/**/4:								;\
2619	/*								;\
2620	 * We have found the hmeblk containing the hment.		;\
2621	 * Now we calculate the corresponding tte.			;\
2622	 *								;\
2623	 * tagacc = tagacc						;\
2624	 * hatid = hatid						;\
2625	 * tte   = clobbered						;\
2626	 * hmeblkpa  = hmeblkpa						;\
2627	 * hmemisc  = hblktag						;\
2628	 * hmeblkva  = hmeblkva 					;\
2629	 * tsbarea = tsbmiss area					;\
2630	 */								;\
2631	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
2632		label/**/2)						;\
2633									;\
2634	/*								;\
2635	 * tagacc = tagacc						;\
2636	 * hatid = hmentoff						;\
2637	 * tte = clobbered						;\
2638	 * hmeblkpa  = hmeblkpa						;\
2639	 * hmemisc  = hblk_misc						;\
2640	 * hmeblkva  = hmeblkva						;\
2641	 * tsbarea = tsbmiss area					;\
2642	 */								;\
2643									;\
2644	add	hatid, SFHME_TTE, hatid					;\
2645	add	hmeblkpa, hatid, hmeblkpa				;\
2646	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2647	brlz,pt tte, label/**/6						;\
2648	  add	hmeblkva, hatid, hmeblkva				;\
2649	btst	HBLK_SZMASK, hmemisc					;\
2650	bnz,a,pt %icc, label/**/7					;\
2651	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2652									;\
2653	/*								;\
2654 	 * We found an invalid 8K tte in shme.				;\
2655	 * it may not belong to shme's region since			;\
2656	 * region size/alignment granularity is 8K but different	;\
2657	 * regions don't share hmeblks. Continue the search.		;\
2658	 */								;\
2659	sub	hmeblkpa, hatid, hmeblkpa				;\
2660	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2661	srlx	tagacc, hmeshift, tte					;\
2662	add	hmeblkpa, HMEBLK_NEXT, hmeblkva				;\
2663	ldxa	[hmeblkva]ASI_MEM, hmeblkva				;\
2664	add	hmeblkpa, HMEBLK_NEXTPA, hmeblkpa			;\
2665	ldxa	[hmeblkpa]ASI_MEM, hmeblkpa				;\
2666	MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc)		;\
2667	ba,a,pt	%xcc, label/**/8					;\
2668label/**/6:								;\
2669	GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc)		;\
2670	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2671label/**/7:								;\
2672	HMELOCK_EXIT(hatid, hatid, ASI_MEM)	/* drop lock */		;\
2673	set	TTE_SUSPEND, hatid					;\
2674	TTE_SUSPEND_INT_SHIFT(hatid)					;\
2675	btst	tte, hatid						;\
2676	bz,pt	%xcc, foundlabel					;\
2677	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2678									;\
2679	/*								;\
2680	 * Mapping is suspended, so goto suspend label.			;\
2681	 */								;\
2682	ba,pt	%xcc, suspendlabel					;\
2683	  nop
2684
2685	/*
2686	 * KERNEL PROTECTION HANDLER
2687	 *
2688	 * g1 = tsb8k pointer register (clobbered)
2689	 * g2 = tag access register (ro)
2690	 * g3 - g7 = scratch registers
2691	 *
2692	 * Note: This function is patched at runtime for performance reasons.
2693	 * 	 Any changes here require sfmmu_patch_ktsb fixed.
2694	 */
2695	ENTRY_NP(sfmmu_kprot_trap)
2696	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2697sfmmu_kprot_patch_ktsb_base:
2698	RUNTIME_PATCH_SETX(%g1, %g6)
2699	/* %g1 = contents of ktsb_base or ktsb_pbase */
2700sfmmu_kprot_patch_ktsb_szcode:
2701	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
2702
2703	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
2704	! %g1 = First TSB entry pointer, as TSB miss handler expects
2705
2706	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2707sfmmu_kprot_patch_ktsb4m_base:
2708	RUNTIME_PATCH_SETX(%g3, %g6)
2709	/* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
2710sfmmu_kprot_patch_ktsb4m_szcode:
2711	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
2712
2713	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
2714	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
2715
2716        CPU_TSBMISS_AREA(%g6, %g7)
2717        HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
2718	ba,pt	%xcc, sfmmu_tsb_miss_tt
2719	  nop
2720
2721	/*
2722	 * USER PROTECTION HANDLER
2723	 *
2724	 * g1 = tsb8k pointer register (ro)
2725	 * g2 = tag access register (ro)
2726	 * g3 = faulting context (clobbered, currently not used)
2727	 * g4 - g7 = scratch registers
2728	 */
2729	ALTENTRY(sfmmu_uprot_trap)
2730#ifdef sun4v
2731	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2732	/* %g1 = first TSB entry ptr now, %g2 preserved */
2733
2734	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
2735	brlz,pt %g3, 9f				/* check for 2nd TSB */
2736	  nop
2737
2738	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2739	/* %g3 = second TSB entry ptr now, %g2 preserved */
2740
2741#else /* sun4v */
2742#ifdef UTSB_PHYS
2743	/* g1 = first TSB entry ptr */
2744	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2745	brlz,pt %g3, 9f			/* check for 2nd TSB */
2746	  nop
2747
2748	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2749	/* %g3 = second TSB entry ptr now, %g2 preserved */
2750#else /* UTSB_PHYS */
2751	brgez,pt %g1, 9f		/* check for 2nd TSB */
2752	  mov	-1, %g3			/* set second tsbe ptr to -1 */
2753
2754	mov	%g2, %g7
2755	GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
2756	/* %g3 = second TSB entry ptr now, %g7 clobbered */
2757	mov	%g1, %g7
2758	GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
2759#endif /* UTSB_PHYS */
2760#endif /* sun4v */
27619:
2762	CPU_TSBMISS_AREA(%g6, %g7)
2763	HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
2764	ba,pt	%xcc, sfmmu_tsb_miss_tt		/* branch TSB miss handler */
2765	  nop
2766
2767	/*
2768	 * Kernel 8K page iTLB miss.  We also get here if we took a
2769	 * fast instruction access mmu miss trap while running in
2770	 * invalid context.
2771	 *
2772	 * %g1 = 8K TSB pointer register (not used, clobbered)
2773	 * %g2 = tag access register (used)
2774	 * %g3 = faulting context id (used)
2775	 * %g7 = TSB tag to match (used)
2776	 */
2777	.align	64
2778	ALTENTRY(sfmmu_kitlb_miss)
2779	brnz,pn %g3, tsb_tl0_noctxt
2780	  nop
2781
2782	/* kernel miss */
2783	/* get kernel tsb pointer */
2784	/* we patch the next set of instructions at run time */
2785	/* NOTE: any changes here require sfmmu_patch_ktsb fixed */
2786iktsbbase:
2787	RUNTIME_PATCH_SETX(%g4, %g5)
2788	/* %g4 = contents of ktsb_base or ktsb_pbase */
2789
2790iktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2791	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2792	or	%g4, %g1, %g1			! form tsb ptr
2793	ldda	[%g1]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2794	cmp	%g4, %g7
2795	bne,pn	%xcc, iktsb4mbase		! check 4m ktsb
2796	  srlx    %g2, MMU_PAGESHIFT4M, %g3	! use 4m virt-page as TSB index
2797
2798	andcc %g5, TTE_EXECPRM_INT, %g0		! check exec bit
2799	bz,pn	%icc, exec_fault
2800	  nop
2801	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2802	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2803	retry
2804
2805iktsb4mbase:
2806        RUNTIME_PATCH_SETX(%g4, %g6)
2807        /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */
2808iktsb4m:
2809	sllx    %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2810        srlx    %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2811	add	%g4, %g3, %g3			! %g3 = 4m tsbe ptr
2812	ldda	[%g3]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2813	cmp	%g4, %g7
2814	bne,pn	%xcc, sfmmu_tsb_miss_tt		! branch on miss
2815	  andcc %g5, TTE_EXECPRM_INT, %g0		! check exec bit
2816	bz,pn	%icc, exec_fault
2817	  nop
2818	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2819	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2820	retry
2821
2822	/*
2823	 * Kernel dTLB miss.  We also get here if we took a fast data
2824	 * access mmu miss trap while running in invalid context.
2825	 *
2826	 * Note: for now we store kpm TTEs in the kernel TSB as usual.
2827	 *	We select the TSB miss handler to branch to depending on
2828	 *	the virtual address of the access.  In the future it may
2829	 *	be desirable to separate kpm TTEs into their own TSB,
2830	 *	in which case all that needs to be done is to set
2831	 *	kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
2832	 *	early in the miss if we detect a kpm VA to a new handler.
2833	 *
2834	 * %g1 = 8K TSB pointer register (not used, clobbered)
2835	 * %g2 = tag access register (used)
2836	 * %g3 = faulting context id (used)
2837	 */
2838	.align	64
2839	ALTENTRY(sfmmu_kdtlb_miss)
2840	brnz,pn	%g3, tsb_tl0_noctxt		/* invalid context? */
2841	  nop
2842
2843	/* Gather some stats for kpm misses in the TLB. */
2844	/* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
2845	KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)
2846
2847	/*
2848	 * Get first TSB offset and look for 8K/64K/512K mapping
2849	 * using the 8K virtual page as the index.
2850	 *
2851	 * We patch the next set of instructions at run time;
2852	 * any changes here require sfmmu_patch_ktsb changes too.
2853	 */
2854dktsbbase:
2855	RUNTIME_PATCH_SETX(%g7, %g6)
2856	/* %g7 = contents of ktsb_base or ktsb_pbase */
2857
2858dktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2859	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2860
2861	/*
2862	 * At this point %g1 is our index into the TSB.
2863	 * We just masked off enough bits of the VA depending
2864	 * on our TSB size code.
2865	 */
2866	ldda	[%g7 + %g1]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2867	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2868	cmp	%g6, %g4			! compare tag
2869	bne,pn	%xcc, dktsb4m_kpmcheck_small
2870	  add	%g7, %g1, %g1			/* form tsb ptr */
2871	TT_TRACE(trace_tsbhit)
2872	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2873	/* trapstat expects tte in %g5 */
2874	retry
2875
2876	/*
2877	 * If kpm is using large pages, the following instruction needs
2878	 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
2879	 * so that we will probe the 4M TSB regardless of the VA.  In
2880	 * the case kpm is using small pages, we know no large kernel
2881	 * mappings are located above 0x80000000.00000000 so we skip the
2882	 * probe as an optimization.
2883	 */
2884dktsb4m_kpmcheck_small:
2885	brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
2886	  /* delay slot safe, below */
2887
2888	/*
2889	 * Get second TSB offset and look for 4M mapping
2890	 * using 4M virtual page as the TSB index.
2891	 *
2892	 * Here:
2893	 * %g1 = 8K TSB pointer.  Don't squash it.
2894	 * %g2 = tag access register (we still need it)
2895	 */
2896	srlx	%g2, MMU_PAGESHIFT4M, %g3
2897
2898	/*
2899	 * We patch the next set of instructions at run time;
2900	 * any changes here require sfmmu_patch_ktsb changes too.
2901	 */
2902dktsb4mbase:
2903	RUNTIME_PATCH_SETX(%g7, %g6)
2904	/* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
2905dktsb4m:
2906	sllx	%g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2907	srlx	%g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2908
2909	/*
2910	 * At this point %g3 is our index into the TSB.
2911	 * We just masked off enough bits of the VA depending
2912	 * on our TSB size code.
2913	 */
2914	ldda	[%g7 + %g3]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2915	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2916	cmp	%g6, %g4			! compare tag
2917
2918dktsb4m_tsbmiss:
2919	bne,pn	%xcc, dktsb4m_kpmcheck
2920	  add	%g7, %g3, %g3			! %g3 = kernel second TSB ptr
2921	TT_TRACE(trace_tsbhit)
2922	/* we don't check TTE size here since we assume 4M TSB is separate */
2923	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2924	/* trapstat expects tte in %g5 */
2925	retry
2926
2927	/*
2928	 * So, we failed to find a valid TTE to match the faulting
2929	 * address in either TSB.  There are a few cases that could land
2930	 * us here:
2931	 *
2932	 * 1) This is a kernel VA below 0x80000000.00000000.  We branch
2933	 *    to sfmmu_tsb_miss_tt to handle the miss.
2934	 * 2) We missed on a kpm VA, and we didn't find the mapping in the
2935	 *    4M TSB.  Let segkpm handle it.
2936	 *
2937	 * Note that we shouldn't land here in the case of a kpm VA when
2938	 * kpm_smallpages is active -- we handled that case earlier at
2939	 * dktsb4m_kpmcheck_small.
2940	 *
2941	 * At this point:
2942	 *  g1 = 8K-indexed primary TSB pointer
2943	 *  g2 = tag access register
2944	 *  g3 = 4M-indexed secondary TSB pointer
2945	 */
2946dktsb4m_kpmcheck:
2947	cmp	%g2, %g0
2948	bl,pn	%xcc, sfmmu_kpm_dtsb_miss
2949	  nop
2950	ba,a,pt	%icc, sfmmu_tsb_miss_tt
2951	  nop
2952
2953#ifdef sun4v
2954	/*
2955	 * User instruction miss w/ single TSB.
2956	 * The first probe covers 8K, 64K, and 512K page sizes,
2957	 * because 64K and 512K mappings are replicated off 8K
2958	 * pointer.
2959	 *
2960	 * g1 = tsb8k pointer register
2961	 * g2 = tag access register
2962	 * g3 - g6 = scratch registers
2963	 * g7 = TSB tag to match
2964	 */
2965	.align	64
2966	ALTENTRY(sfmmu_uitlb_fastpath)
2967
2968	PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
2969	/* g4 - g5 = clobbered by PROBE_1ST_ITSB */
2970	ba,pn	%xcc, sfmmu_tsb_miss_tt
2971	  mov	-1, %g3
2972
2973	/*
2974	 * User data miss w/ single TSB.
2975	 * The first probe covers 8K, 64K, and 512K page sizes,
2976	 * because 64K and 512K mappings are replicated off 8K
2977	 * pointer.
2978	 *
2979	 * g1 = tsb8k pointer register
2980	 * g2 = tag access register
2981	 * g3 - g6 = scratch registers
2982	 * g7 = TSB tag to match
2983	 */
2984	.align 64
2985	ALTENTRY(sfmmu_udtlb_fastpath)
2986
2987	PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
2988	/* g4 - g5 = clobbered by PROBE_1ST_DTSB */
2989	ba,pn	%xcc, sfmmu_tsb_miss_tt
2990	  mov	-1, %g3
2991
2992	/*
2993	 * User instruction miss w/ multiple TSBs (sun4v).
2994	 * The first probe covers 8K, 64K, and 512K page sizes,
2995	 * because 64K and 512K mappings are replicated off 8K
2996	 * pointer.  Second probe covers 4M page size only.
2997	 *
2998	 * Just like sfmmu_udtlb_slowpath, except:
2999	 *   o Uses ASI_ITLB_IN
3000	 *   o checks for execute permission
3001	 *   o No ISM prediction.
3002	 *
3003	 * g1 = tsb8k pointer register
3004	 * g2 = tag access register
3005	 * g3 - g6 = scratch registers
3006	 * g7 = TSB tag to match
3007	 */
3008	.align	64
3009	ALTENTRY(sfmmu_uitlb_slowpath)
3010
3011	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
3012	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
3013	/* g4 - g5 = clobbered here */
3014
3015	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3016	/* g1 = first TSB pointer, g3 = second TSB pointer */
3017	srlx	%g2, TAG_VALO_SHIFT, %g7
3018	PROBE_2ND_ITSB(%g3, %g7)
3019	/* NOT REACHED */
3020
3021#else /* sun4v */
3022
3023	/*
3024	 * User instruction miss w/ multiple TSBs (sun4u).
3025	 * The first probe covers 8K, 64K, and 512K page sizes,
3026	 * because 64K and 512K mappings are replicated off 8K
3027	 * pointer.  Probe of 1st TSB has already been done prior to entry
3028	 * into this routine. For the UTSB_PHYS case we probe up to 3
3029	 * valid other TSBs in the following order:
3030	 * 1) shared TSB for 4M-256M pages
3031	 * 2) private TSB for 4M-256M pages
3032	 * 3) shared TSB for 8K-512K pages
3033	 *
3034	 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs
3035	 * 4M-256M pages.
3036	 *
3037	 * Just like sfmmu_udtlb_slowpath, except:
3038	 *   o Uses ASI_ITLB_IN
3039	 *   o checks for execute permission
3040	 *   o No ISM prediction.
3041	 *
3042	 * g1 = tsb8k pointer register
3043	 * g2 = tag access register
3044	 * g4 - g6 = scratch registers
3045	 * g7 = TSB tag to match
3046	 */
3047	.align	64
3048	ALTENTRY(sfmmu_uitlb_slowpath)
3049
3050#ifdef UTSB_PHYS
3051
3052       GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
3053        brlz,pt %g6, 1f
3054          nop
3055        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
3056        PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail)
30571:
3058        GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
3059        brlz,pt %g3, 2f
3060          nop
3061        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3062        PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail)
30632:
3064        GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
3065        brlz,pt %g6, sfmmu_tsb_miss_tt
3066          nop
3067        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
3068        PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail)
3069        ba,pn   %xcc, sfmmu_tsb_miss_tt
3070          nop
3071
3072#else /* UTSB_PHYS */
3073	mov	%g1, %g3	/* save tsb8k reg in %g3 */
3074	GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
3075	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
3076	mov	%g2, %g6	/* GET_2ND_TSBE_PTR clobbers tagacc */
3077	mov	%g3, %g7	/* copy tsb8k reg in %g7 */
3078	GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
3079       /* g1 = first TSB pointer, g3 = second TSB pointer */
3080        srlx    %g2, TAG_VALO_SHIFT, %g7
3081        PROBE_2ND_ITSB(%g3, %g7, isynth)
3082	ba,pn	%xcc, sfmmu_tsb_miss_tt
3083	  nop
3084
3085#endif /* UTSB_PHYS */
3086#endif /* sun4v */
3087
3088#if defined(sun4u) && defined(UTSB_PHYS)
3089
3090        /*
3091	 * We come here for ism predict DTLB_MISS case or if
3092	 * if probe in first TSB failed.
3093         */
3094
3095        .align 64
3096        ALTENTRY(sfmmu_udtlb_slowpath_noismpred)
3097
3098	/*
3099         * g1 = tsb8k pointer register
3100         * g2 = tag access register
3101         * g4 - %g6 = scratch registers
3102         * g7 = TSB tag to match
3103	 */
3104
3105	/*
3106	 * ISM non-predict probe order
3107         * probe 1ST_TSB (8K index)
3108         * probe 2ND_TSB (4M index)
3109         * probe 4TH_TSB (4M index)
3110         * probe 3RD_TSB (8K index)
3111	 *
3112	 * We already probed first TSB in DTLB_MISS handler.
3113	 */
3114
3115        /*
3116         * Private 2ND TSB 4M-256 pages
3117         */
3118	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
3119	brlz,pt %g3, 1f
3120	  nop
3121        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3122        PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
3123
3124	/*
3125	 * Shared Context 4TH TSB 4M-256 pages
3126	 */
31271:
3128	GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
3129	brlz,pt %g6, 2f
3130	  nop
3131        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
3132        PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail)
3133
3134        /*
3135         * Shared Context 3RD TSB 8K-512K pages
3136         */
31372:
3138	GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
3139	brlz,pt %g6, sfmmu_tsb_miss_tt
3140	  nop
3141        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
3142        PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail)
3143	ba,pn	%xcc, sfmmu_tsb_miss_tt
3144	  nop
3145
3146	.align 64
3147        ALTENTRY(sfmmu_udtlb_slowpath_ismpred)
3148
3149	/*
3150         * g1 = tsb8k pointer register
3151         * g2 = tag access register
3152         * g4 - g6 = scratch registers
3153         * g7 = TSB tag to match
3154	 */
3155
3156	/*
3157	 * ISM predict probe order
3158	 * probe 4TH_TSB (4M index)
3159	 * probe 2ND_TSB (4M index)
3160	 * probe 1ST_TSB (8K index)
3161	 * probe 3RD_TSB (8K index)
3162
3163	/*
3164	 * Shared Context 4TH TSB 4M-256 pages
3165	 */
3166	GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
3167	brlz,pt %g6, 4f
3168	  nop
3169        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
3170        PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2)
3171
3172        /*
3173         * Private 2ND TSB 4M-256 pages
3174         */
31754:
3176	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
3177	brlz,pt %g3, 5f
3178	  nop
3179        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3180        PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2)
3181
31825:
3183        PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2)
3184
3185        /*
3186         * Shared Context 3RD TSB 8K-512K pages
3187         */
3188	GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
3189	brlz,pt %g6, 6f
3190	  nop
3191        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
3192        PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2)
31936:
3194	ba,pn	%xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */
3195	  nop
3196
3197#else /* sun4u && UTSB_PHYS */
3198
3199       .align 64
3200        ALTENTRY(sfmmu_udtlb_slowpath)
3201
3202	srax	%g2, PREDISM_BASESHIFT, %g6	/* g6 > 0 : ISM predicted */
3203	brgz,pn %g6, udtlb_miss_probesecond	/* check for ISM */
3204	  mov	%g1, %g3
3205
3206udtlb_miss_probefirst:
3207	/*
3208	 * g1 = 8K TSB pointer register
3209	 * g2 = tag access register
3210	 * g3 = (potentially) second TSB entry ptr
3211	 * g6 = ism pred.
3212	 * g7 = vpg_4m
3213	 */
3214#ifdef sun4v
3215	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
3216	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
3217
3218	/*
3219	 * Here:
3220	 *   g1 = first TSB pointer
3221	 *   g2 = tag access reg
3222	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
3223	 */
3224	brgz,pn	%g6, sfmmu_tsb_miss_tt
3225	  nop
3226#else /* sun4v */
3227	mov	%g1, %g4
3228	GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
3229	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
3230
3231	/*
3232	 * Here:
3233	 *   g1 = first TSB pointer
3234	 *   g2 = tag access reg
3235	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
3236	 */
3237	brgz,pn	%g6, sfmmu_tsb_miss_tt
3238	  nop
3239	ldxa	[%g0]ASI_DMMU_TSB_8K, %g3
3240	/* fall through in 8K->4M probe order */
3241#endif /* sun4v */
3242
3243udtlb_miss_probesecond:
3244	/*
3245	 * Look in the second TSB for the TTE
3246	 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
3247	 * g2 = tag access reg
3248	 * g3 = 8K TSB pointer register
3249	 * g6 = ism pred.
3250	 * g7 = vpg_4m
3251	 */
3252#ifdef sun4v
3253	/* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
3254	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3255	/* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
3256#else /* sun4v */
3257	mov	%g3, %g7
3258	GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
3259	/* %g2 clobbered, %g3 =second tsbe ptr */
3260	mov	MMU_TAG_ACCESS, %g2
3261	ldxa	[%g2]ASI_DMMU, %g2
3262#endif /* sun4v */
3263
3264	srlx	%g2, TAG_VALO_SHIFT, %g7
3265	PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
3266	/* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
3267	brgz,pn	%g6, udtlb_miss_probefirst
3268	  nop
3269
3270	/* fall through to sfmmu_tsb_miss_tt */
3271#endif /* sun4u && UTSB_PHYS */
3272
3273
3274	ALTENTRY(sfmmu_tsb_miss_tt)
3275	TT_TRACE(trace_tsbmiss)
3276	/*
3277	 * We get here if there is a TSB miss OR a write protect trap.
3278	 *
3279	 * g1 = First TSB entry pointer
3280	 * g2 = tag access register
3281	 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB
3282	 * g4 - g7 = scratch registers
3283	 */
3284
3285	ALTENTRY(sfmmu_tsb_miss)
3286
3287	/*
3288	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3289	 * point to trapstat's TSB miss return code (note that trapstat
3290	 * itself will patch the correct offset to add).
3291	 */
3292	rdpr	%tl, %g7
3293	cmp	%g7, 1
3294	ble,pt	%xcc, 0f
3295	  sethi	%hi(KERNELBASE), %g6
3296	rdpr	%tpc, %g7
3297	or	%g6, %lo(KERNELBASE), %g6
3298	cmp	%g7, %g6
3299	bgeu,pt	%xcc, 0f
3300	/* delay slot safe */
3301
3302	ALTENTRY(tsbmiss_trapstat_patch_point)
3303	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3304	wrpr	%g7, %tpc
3305	add	%g7, 4, %g7
3306	wrpr	%g7, %tnpc
33070:
3308	CPU_TSBMISS_AREA(%g6, %g7)
3309	stn	%g1, [%g6 + TSBMISS_TSBPTR]	/* save 1ST tsb pointer */
3310	stn	%g3, [%g6 + TSBMISS_TSBPTR4M]	/* save 2ND tsb pointer */
3311
3312	sllx	%g2, TAGACC_CTX_LSHIFT, %g3
3313	brz,a,pn %g3, 1f			/* skip ahead if kernel */
3314	  ldn	[%g6 + TSBMISS_KHATID], %g7
3315	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctxnum */
3316	ldn	[%g6 + TSBMISS_UHATID], %g7     /* g7 = hatid */
3317
3318	HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)
3319
3320	cmp	%g3, INVALID_CONTEXT
3321	be,pn	%icc, tsb_tl0_noctxt		/* no ctx miss exception */
3322	  stn	%g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]
3323
3324#if defined(sun4v) || defined(UTSB_PHYS)
3325        ldub    [%g6 + TSBMISS_URTTEFLAGS], %g7	/* clear ctx1 flag set from */
3326        andn    %g7, HAT_CHKCTX1_FLAG, %g7	/* the previous tsb miss    */
3327        stub    %g7, [%g6 + TSBMISS_URTTEFLAGS]
3328#endif /* sun4v || UTSB_PHYS */
3329
3330	ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
3331	/*
3332	 * The miss wasn't in an ISM segment.
3333	 *
3334	 * %g1 %g3, %g4, %g5, %g7 all clobbered
3335	 * %g2 = (pseudo) tag access
3336	 */
3337
3338	ba,pt	%icc, 2f
3339	  ldn	[%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7
3340
33411:
3342	HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
3343	/*
3344	 * 8K and 64K hash.
3345	 */
33462:
3347
3348	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3349		MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte,
3350		sfmmu_suspend_tl, tsb_512K)
3351	/* NOT REACHED */
3352
3353tsb_512K:
3354	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3355	brz,pn	%g5, 3f
3356	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3357	and	%g4, HAT_512K_FLAG, %g5
3358
3359	/*
3360	 * Note that there is a small window here where we may have
3361	 * a 512k page in the hash list but have not set the HAT_512K_FLAG
3362	 * flag yet, so we will skip searching the 512k hash list.
3363	 * In this case we will end up in pagefault which will find
3364	 * the mapping and return.  So, in this instance we will end up
3365	 * spending a bit more time resolving this TSB miss, but it can
3366	 * only happen once per process and even then, the chances of that
3367	 * are very small, so it's not worth the extra overhead it would
3368	 * take to close this window.
3369	 */
3370	brz,pn	%g5, tsb_4M
3371	  nop
33723:
3373	/*
3374	 * 512K hash
3375	 */
3376
3377	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3378		MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte,
3379		sfmmu_suspend_tl, tsb_4M)
3380	/* NOT REACHED */
3381
3382tsb_4M:
3383	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3384	brz,pn	%g5, 4f
3385	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3386	and	%g4, HAT_4M_FLAG, %g5
3387	brz,pn	%g5, tsb_32M
3388	  nop
33894:
3390	/*
3391	 * 4M hash
3392	 */
3393
3394	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3395		MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte,
3396		sfmmu_suspend_tl, tsb_32M)
3397	/* NOT REACHED */
3398
3399tsb_32M:
3400	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3401#ifdef	sun4v
3402        brz,pn	%g5, 6f
3403#else
3404	brz,pn  %g5, tsb_pagefault
3405#endif
3406	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3407	and	%g4, HAT_32M_FLAG, %g5
3408	brz,pn	%g5, tsb_256M
3409	  nop
34105:
3411	/*
3412	 * 32M hash
3413	 */
3414
3415	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3416		MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte,
3417		sfmmu_suspend_tl, tsb_256M)
3418	/* NOT REACHED */
3419
3420#if defined(sun4u) && !defined(UTSB_PHYS)
3421#define tsb_shme        tsb_pagefault
3422#endif
3423tsb_256M:
3424	ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3425	and	%g4, HAT_256M_FLAG, %g5
3426	brz,pn	%g5, tsb_shme
3427	  nop
34286:
3429	/*
3430	 * 256M hash
3431	 */
3432
3433	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3434	    MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte,
3435	    sfmmu_suspend_tl, tsb_shme)
3436	/* NOT REACHED */
3437
3438tsb_checktte:
3439	/*
3440	 * g1 = hblk_misc
3441	 * g2 = tagacc
3442	 * g3 = tte
3443	 * g4 = tte pa
3444	 * g5 = tte va
3445	 * g6 = tsbmiss area
3446	 * g7 = hatid
3447	 */
3448	brlz,a,pt %g3, tsb_validtte
3449	  rdpr	%tt, %g7
3450
3451#if defined(sun4u) && !defined(UTSB_PHYS)
3452#undef tsb_shme
3453	ba      tsb_pagefault
3454	  nop
3455#else /* sun4u && !UTSB_PHYS */
3456
3457tsb_shme:
3458	/*
3459	 * g2 = tagacc
3460	 * g6 = tsbmiss area
3461	 */
3462	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3463	brz,pn	%g5, tsb_pagefault
3464	  nop
3465	ldx	[%g6 + TSBMISS_SHARED_UHATID], %g7	/* g7 = srdp */
3466	brz,pn	%g7, tsb_pagefault
3467	  nop
3468
3469	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3470		MMU_PAGESHIFT64K, TTE64K, tsb_shme_l8K, tsb_shme_checktte,
3471		sfmmu_suspend_tl, tsb_shme_512K)
3472	/* NOT REACHED */
3473
3474tsb_shme_512K:
3475	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3476	and	%g4, HAT_512K_FLAG, %g5
3477	brz,pn	%g5, tsb_shme_4M
3478	  nop
3479
3480	/*
3481	 * 512K hash
3482	 */
3483
3484	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3485		MMU_PAGESHIFT512K, TTE512K, tsb_shme_l512K, tsb_shme_checktte,
3486		sfmmu_suspend_tl, tsb_shme_4M)
3487	/* NOT REACHED */
3488
3489tsb_shme_4M:
3490	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3491	and	%g4, HAT_4M_FLAG, %g5
3492	brz,pn	%g5, tsb_shme_32M
3493	  nop
34944:
3495	/*
3496	 * 4M hash
3497	 */
3498	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3499		MMU_PAGESHIFT4M, TTE4M, tsb_shme_l4M, tsb_shme_checktte,
3500		sfmmu_suspend_tl, tsb_shme_32M)
3501	/* NOT REACHED */
3502
3503tsb_shme_32M:
3504	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3505	and	%g4, HAT_32M_FLAG, %g5
3506	brz,pn	%g5, tsb_shme_256M
3507	  nop
3508
3509	/*
3510	 * 32M hash
3511	 */
3512
3513	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3514		MMU_PAGESHIFT32M, TTE32M, tsb_shme_l32M, tsb_shme_checktte,
3515		sfmmu_suspend_tl, tsb_shme_256M)
3516	/* NOT REACHED */
3517
3518tsb_shme_256M:
3519	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3520	and	%g4, HAT_256M_FLAG, %g5
3521	brz,pn	%g5, tsb_pagefault
3522	  nop
3523
3524	/*
3525	 * 256M hash
3526	 */
3527
3528	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3529	    MMU_PAGESHIFT256M, TTE256M, tsb_shme_l256M, tsb_shme_checktte,
3530	    sfmmu_suspend_tl, tsb_pagefault)
3531	/* NOT REACHED */
3532
3533tsb_shme_checktte:
3534
3535	brgez,pn %g3, tsb_pagefault
3536	  rdpr	%tt, %g7
3537	/*
3538	 * g1 = ctx1 flag
3539	 * g3 = tte
3540	 * g4 = tte pa
3541	 * g5 = tte va
3542	 * g6 = tsbmiss area
3543	 * g7 = tt
3544	 */
3545
3546	brz,pt  %g1, tsb_validtte
3547	  nop
3548	ldub    [%g6 + TSBMISS_URTTEFLAGS], %g1
3549	  or	%g1, HAT_CHKCTX1_FLAG, %g1
3550	stub    %g1, [%g6 + TSBMISS_URTTEFLAGS]
3551
3552	SAVE_CTX1(%g7, %g2, %g1, tsb_shmel)
3553#endif /* sun4u && !UTSB_PHYS */
3554
3555tsb_validtte:
3556	/*
3557	 * g3 = tte
3558	 * g4 = tte pa
3559	 * g5 = tte va
3560	 * g6 = tsbmiss area
3561	 * g7 = tt
3562	 */
3563
3564	/*
3565	 * Set ref/mod bits if this is a prot trap.  Usually, it isn't.
3566	 */
3567	cmp	%g7, FAST_PROT_TT
3568	bne,pt	%icc, 4f
3569	  nop
3570
3571	TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod,
3572	    tsb_protfault)
3573
3574	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3575#ifdef sun4v
3576	MMU_FAULT_STATUS_AREA(%g7)
3577	ldx	[%g7 + MMFSA_D_ADDR], %g5	/* load fault addr for later */
3578#else /* sun4v */
3579	mov     MMU_TAG_ACCESS, %g5
3580	ldxa    [%g5]ASI_DMMU, %g5
3581#endif /* sun4v */
3582	ba,pt	%xcc, tsb_update_tl1
3583	  nop
35844:
3585	/*
3586	 * If ITLB miss check exec bit.
3587	 * If not set treat as invalid TTE.
3588	 */
3589	cmp     %g7, T_INSTR_MMU_MISS
3590	be,pn	%icc, 5f
3591	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
3592	cmp     %g7, FAST_IMMU_MISS_TT
3593	bne,pt %icc, 3f
3594	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
35955:
3596	bz,pn %icc, tsb_protfault
3597	  nop
3598
35993:
3600	/*
3601	 * Set reference bit if not already set
3602	 */
3603	TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref)
3604
3605	/*
3606	 * Now, load into TSB/TLB.  At this point:
3607	 * g3 = tte
3608	 * g4 = patte
3609	 * g6 = tsbmiss area
3610	 */
3611	rdpr	%tt, %g7
3612#ifdef sun4v
3613	MMU_FAULT_STATUS_AREA(%g2)
3614	cmp	%g7, T_INSTR_MMU_MISS
3615	be,a,pt	%icc, 9f
3616	  nop
3617	cmp	%g7, FAST_IMMU_MISS_TT
3618	be,a,pt	%icc, 9f
3619	  nop
3620	add	%g2, MMFSA_D_, %g2
36219:
3622	ldx	[%g2 + MMFSA_CTX_], %g7
3623	sllx	%g7, TTARGET_CTX_SHIFT, %g7
3624	ldx	[%g2 + MMFSA_ADDR_], %g2
3625	mov	%g2, %g5		! load the fault addr for later use
3626	srlx	%g2, TTARGET_VA_SHIFT, %g2
3627	or	%g2, %g7, %g2
3628#else /* sun4v */
3629	mov     MMU_TAG_ACCESS, %g5
3630	cmp     %g7, FAST_IMMU_MISS_TT
3631	be,a,pt %icc, 9f
3632	   ldxa  [%g0]ASI_IMMU, %g2
3633	ldxa    [%g0]ASI_DMMU, %g2
3634	ba,pt   %icc, tsb_update_tl1
3635	   ldxa  [%g5]ASI_DMMU, %g5
36369:
3637	ldxa    [%g5]ASI_IMMU, %g5
3638#endif /* sun4v */
3639
3640tsb_update_tl1:
3641	srlx	%g2, TTARGET_CTX_SHIFT, %g7
3642	brz,pn	%g7, tsb_kernel
3643#ifdef sun4v
3644	  and	%g3, TTE_SZ_BITS, %g7	! assumes TTE_SZ_SHFT is 0
3645#else  /* sun4v */
3646	  srlx	%g3, TTE_SZ_SHFT, %g7
3647#endif /* sun4v */
3648
3649tsb_user:
3650#ifdef sun4v
3651	cmp	%g7, TTE4M
3652	bge,pn	%icc, tsb_user4m
3653	  nop
3654#else /* sun4v */
3655	cmp	%g7, TTESZ_VALID | TTE4M
3656	be,pn	%icc, tsb_user4m
3657	  srlx	%g3, TTE_SZ2_SHFT, %g7
3658	andcc	%g7, TTE_SZ2_BITS, %g7		! check 32/256MB
3659#ifdef ITLB_32M_256M_SUPPORT
3660	bnz,pn	%icc, tsb_user4m
3661	  nop
3662#else /* ITLB_32M_256M_SUPPORT */
3663	bnz,a,pn %icc, tsb_user_pn_synth
3664	 nop
3665#endif /* ITLB_32M_256M_SUPPORT */
3666#endif /* sun4v */
3667
3668tsb_user8k:
3669#if defined(sun4v) || defined(UTSB_PHYS)
3670	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g7
3671	and	%g7, HAT_CHKCTX1_FLAG, %g1
3672	brz,a,pn %g1, 1f
3673	  ldn	[%g6 + TSBMISS_TSBPTR], %g1		! g1 = 1ST TSB ptr
3674	GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1)
3675	brlz,a,pn %g1, ptl1_panic			! if no shared 3RD tsb
3676	  mov PTL1_NO_SCDTSB8K, %g1			! panic
3677        GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7)
36781:
3679#else /* defined(sun4v) || defined(UTSB_PHYS) */
3680	ldn   [%g6 + TSBMISS_TSBPTR], %g1             ! g1 = 1ST TSB ptr
3681#endif /* defined(sun4v) || defined(UTSB_PHYS) */
3682
3683#ifndef UTSB_PHYS
3684	mov	ASI_N, %g7	! user TSBs accessed by VA
3685	mov	%g7, %asi
3686#endif /* !UTSB_PHYS */
3687
3688	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5)
3689
3690	rdpr    %tt, %g5
3691#ifdef sun4v
3692	cmp	%g5, T_INSTR_MMU_MISS
3693	be,a,pn	%xcc, 9f
3694	  mov	%g3, %g5
3695#endif /* sun4v */
3696	cmp	%g5, FAST_IMMU_MISS_TT
3697	be,pn	%xcc, 9f
3698	  mov	%g3, %g5
3699
3700	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3701	! trapstat wants TTE in %g5
3702	retry
37039:
3704	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3705	! trapstat wants TTE in %g5
3706	retry
3707
3708tsb_user4m:
3709#if defined(sun4v) || defined(UTSB_PHYS)
3710	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g7
3711	and	%g7, HAT_CHKCTX1_FLAG, %g1
3712	brz,a,pn %g1, 4f
3713	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		! g1 = 2ND TSB ptr
3714	GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr
3715	brlz,a,pn %g1, 5f				! if no shared 4TH TSB
3716	  nop
3717        GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7)
3718
3719#else /* defined(sun4v) || defined(UTSB_PHYS) */
3720	ldn   [%g6 + TSBMISS_TSBPTR4M], %g1             ! g1 = 2ND TSB ptr
3721#endif /* defined(sun4v) || defined(UTSB_PHYS) */
37224:
3723	brlz,pn %g1, 5f	/* Check to see if we have 2nd TSB programmed */
3724	  nop
3725
3726#ifndef UTSB_PHYS
3727	mov	ASI_N, %g7	! user TSBs accessed by VA
3728	mov	%g7, %asi
3729#endif /* UTSB_PHYS */
3730
3731        TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6)
3732
37335:
3734	rdpr    %tt, %g5
3735#ifdef sun4v
3736        cmp     %g5, T_INSTR_MMU_MISS
3737        be,a,pn %xcc, 9f
3738          mov   %g3, %g5
3739#endif /* sun4v */
3740        cmp     %g5, FAST_IMMU_MISS_TT
3741        be,pn   %xcc, 9f
3742        mov     %g3, %g5
3743
3744        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3745        ! trapstat wants TTE in %g5
3746        retry
37479:
3748        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3749        ! trapstat wants TTE in %g5
3750        retry
3751
3752#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT)
3753	/*
3754	 * Panther ITLB synthesis.
3755	 * The Panther 32M and 256M ITLB code simulates these two large page
3756	 * sizes with 4M pages, to provide support for programs, for example
3757	 * Java, that may copy instructions into a 32M or 256M data page and
3758	 * then execute them. The code below generates the 4M pfn bits and
3759	 * saves them in the modified 32M/256M ttes in the TSB. If the tte is
3760	 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits
3761	 * are ignored by the hardware.
3762	 *
3763	 * Now, load into TSB/TLB.  At this point:
3764	 * g2 = tagtarget
3765	 * g3 = tte
3766	 * g4 = patte
3767	 * g5 = tt
3768	 * g6 = tsbmiss area
3769	 */
3770tsb_user_pn_synth:
3771	rdpr %tt, %g5
3772	cmp    %g5, FAST_IMMU_MISS_TT
3773	be,pt	%xcc, tsb_user_itlb_synth	/* ITLB miss */
3774	  andcc %g3, TTE_EXECPRM_INT, %g0	/* is execprm bit set */
3775	bz,pn %icc, 4b				/* if not, been here before */
3776	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	/* g1 = tsbp */
3777	brlz,a,pn %g1, 5f			/* no 2nd tsb */
3778	  mov	%g3, %g5
3779
3780	mov	MMU_TAG_ACCESS, %g7
3781	ldxa	[%g7]ASI_DMMU, %g6		/* get tag access va */
3782	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1)	/* make 4M pfn offset */
3783
3784	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
3785	mov	%g7, %asi
3786	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */
37875:
3788        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3789        retry
3790
3791tsb_user_itlb_synth:
3792	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 =  2ND TSB */
3793
3794	mov	MMU_TAG_ACCESS, %g7
3795	ldxa	[%g7]ASI_IMMU, %g6		/* get tag access va */
3796	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2)	/* make 4M pfn offset */
3797	brlz,a,pn %g1, 7f	/* Check to see if we have 2nd TSB programmed */
3798	  or	%g5, %g3, %g5			/* add 4M bits to TTE */
3799
3800	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
3801	mov	%g7, %asi
3802	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */
38037:
3804	SET_TTE4M_PN(%g5, %g7)			/* add TTE4M pagesize to TTE */
3805        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3806        retry
3807#endif /* sun4v && ITLB_32M_256M_SUPPORT */
3808
3809tsb_kernel:
3810	rdpr	%tt, %g5
3811#ifdef sun4v
3812	cmp	%g7, TTE4M
3813	bge,pn	%icc, 5f
3814#else
3815	cmp	%g7, TTESZ_VALID | TTE4M	! no 32M or 256M support
3816	be,pn	%icc, 5f
3817#endif /* sun4v */
3818	  nop
3819	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = 8K TSB ptr
3820	ba,pt	%xcc, 6f
3821	  nop
38225:
3823	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	! g1 = 4M TSB ptr
3824	brlz,pn	%g1, 3f		/* skip programming if 4M TSB ptr is -1 */
3825	  nop
38266:
3827#ifndef sun4v
3828tsb_kernel_patch_asi:
3829	or	%g0, RUNTIME_PATCH, %g6
3830	mov	%g6, %asi	! XXX avoid writing to %asi !!
3831#endif
3832	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7)
38333:
3834#ifdef sun4v
3835	cmp	%g5, T_INSTR_MMU_MISS
3836	be,a,pn	%icc, 1f
3837	  mov	%g3, %g5			! trapstat wants TTE in %g5
3838#endif /* sun4v */
3839	cmp	%g5, FAST_IMMU_MISS_TT
3840	be,pn	%icc, 1f
3841	  mov	%g3, %g5			! trapstat wants TTE in %g5
3842	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3843	! trapstat wants TTE in %g5
3844	retry
38451:
3846	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3847	! trapstat wants TTE in %g5
3848	retry
3849
3850tsb_ism:
3851	/*
3852	 * This is an ISM [i|d]tlb miss.  We optimize for largest
3853	 * page size down to smallest.
3854	 *
3855	 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access
3856	 *	register
3857	 * g3 = ismmap->ism_seg
3858	 * g4 = physical address of ismmap->ism_sfmmu
3859	 * g6 = tsbmiss area
3860	 */
3861	ldna	[%g4]ASI_MEM, %g7		/* g7 = ism hatid */
3862	brz,a,pn %g7, ptl1_panic		/* if zero jmp ahead */
3863	  mov	PTL1_BAD_ISM, %g1
3864						/* g5 = pa of imap_vb_shift */
3865	sub	%g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
3866	lduba	[%g5]ASI_MEM, %g4		/* g4 = imap_vb_shift */
3867	srlx	%g3, %g4, %g3			/* clr size field */
3868	set	TAGACC_CTX_MASK, %g1		/* mask off ctx number */
3869	sllx    %g3, %g4, %g3                   /* g3 = ism vbase */
3870	and     %g2, %g1, %g4                   /* g4 = ctx number */
3871	andn    %g2, %g1, %g1                   /* g1 = tlb miss vaddr */
3872	sub     %g1, %g3, %g2                   /* g2 = offset in ISM seg */
3873	or      %g2, %g4, %g2                   /* g2 = (pseudo-)tagacc */
3874	sub     %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
3875	lduha   [%g5]ASI_MEM, %g4               /* g5 = pa of imap_hatflags */
3876#if defined(sun4v) || defined(UTSB_PHYS)
3877	and     %g4, HAT_CTX1_FLAG, %g5         /* g5 = imap_hatflags */
3878	brz,pt %g5, tsb_chk4M_ism
3879	  nop
3880	ldub    [%g6 + TSBMISS_URTTEFLAGS], %g5
3881	or      %g5, HAT_CHKCTX1_FLAG, %g5
3882	stub    %g5, [%g6 + TSBMISS_URTTEFLAGS]
3883	rdpr    %tt, %g5
3884	SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl)
3885#endif /* defined(sun4v) || defined(UTSB_PHYS) */
3886
3887	/*
3888	 * ISM pages are always locked down.
3889	 * If we can't find the tte then pagefault
3890	 * and let the spt segment driver resolve it.
3891	 *
3892	 * g2 = tagacc w/ISM vaddr (offset in ISM seg)
3893	 * g4 = imap_hatflags
3894	 * g6 = tsb miss area
3895	 * g7 = ISM hatid
3896	 */
3897
3898tsb_chk4M_ism:
3899	and	%g4, HAT_4M_FLAG, %g5		/* g4 = imap_hatflags */
3900	brnz,pt	%g5, tsb_ism_4M			/* branch if 4M pages */
3901	  nop
3902
3903tsb_ism_32M:
3904	and	%g4, HAT_32M_FLAG, %g5		/* check default 32M next */
3905	brz,pn	%g5, tsb_ism_256M
3906	  nop
3907
3908	/*
3909	 * 32M hash.
3910	 */
3911
3912	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M,
3913	    TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
3914	    tsb_ism_4M)
3915	/* NOT REACHED */
3916
3917tsb_ism_32M_found:
3918	brlz,a,pt %g3, tsb_validtte
3919	  rdpr	%tt, %g7
3920	ba,pt	%xcc, tsb_ism_4M
3921	  nop
3922
3923tsb_ism_256M:
3924	and	%g4, HAT_256M_FLAG, %g5		/* 256M is last resort */
3925	brz,a,pn %g5, ptl1_panic
3926	  mov	PTL1_BAD_ISM, %g1
3927
3928	/*
3929	 * 256M hash.
3930	 */
3931	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M,
3932	    TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
3933	    tsb_ism_4M)
3934
3935tsb_ism_256M_found:
3936	brlz,a,pt %g3, tsb_validtte
3937	  rdpr	%tt, %g7
3938
3939tsb_ism_4M:
3940	/*
3941	 * 4M hash.
3942	 */
3943	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M,
3944	    TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
3945	    tsb_ism_8K)
3946	/* NOT REACHED */
3947
3948tsb_ism_4M_found:
3949	brlz,a,pt %g3, tsb_validtte
3950	  rdpr	%tt, %g7
3951
3952tsb_ism_8K:
3953	/*
3954	 * 8K and 64K hash.
3955	 */
3956
3957	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K,
3958	    TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
3959	    tsb_pagefault)
3960	/* NOT REACHED */
3961
3962tsb_ism_8K_found:
3963	brlz,a,pt %g3, tsb_validtte
3964	  rdpr	%tt, %g7
3965
3966tsb_pagefault:
3967	rdpr	%tt, %g7
3968	cmp	%g7, FAST_PROT_TT
3969	be,a,pn	%icc, tsb_protfault
3970	  wrpr	%g0, FAST_DMMU_MISS_TT, %tt
3971
3972tsb_protfault:
3973	/*
3974	 * we get here if we couldn't find a valid tte in the hash.
3975	 *
3976	 * If user and we are at tl>1 we go to window handling code.
3977	 *
3978	 * If kernel and the fault is on the same page as our stack
3979	 * pointer, then we know the stack is bad and the trap handler
3980	 * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
3981	 *
3982	 * If this is a kernel trap and tl>1, panic.
3983	 *
3984	 * Otherwise we call pagefault.
3985	 */
3986	cmp	%g7, FAST_IMMU_MISS_TT
3987#ifdef sun4v
3988	MMU_FAULT_STATUS_AREA(%g4)
3989	ldx	[%g4 + MMFSA_I_CTX], %g5
3990	ldx	[%g4 + MMFSA_D_CTX], %g4
3991	move	%icc, %g5, %g4
3992	cmp	%g7, T_INSTR_MMU_MISS
3993	move	%icc, %g5, %g4
3994#else
3995	mov	MMU_TAG_ACCESS, %g4
3996	ldxa	[%g4]ASI_DMMU, %g2
3997	ldxa	[%g4]ASI_IMMU, %g5
3998	move	%icc, %g5, %g2
3999	cmp	%g7, T_INSTR_MMU_MISS
4000	move	%icc, %g5, %g2
4001	sllx	%g2, TAGACC_CTX_LSHIFT, %g4
4002#endif /* sun4v */
4003	brnz,pn	%g4, 3f				/* skip if not kernel */
4004	  rdpr	%tl, %g5
4005
4006	add	%sp, STACK_BIAS, %g3
4007	srlx	%g3, MMU_PAGESHIFT, %g3
4008	srlx	%g2, MMU_PAGESHIFT, %g4
4009	cmp	%g3, %g4
4010	be,a,pn	%icc, ptl1_panic		/* panic if bad %sp */
4011	  mov	PTL1_BAD_STACK, %g1
4012
4013	cmp	%g5, 1
4014	ble,pt	%icc, 2f
4015	  nop
4016	TSTAT_CHECK_TL1(2f, %g1, %g2)
4017	rdpr	%tt, %g2
4018	cmp	%g2, FAST_PROT_TT
4019	mov	PTL1_BAD_KPROT_FAULT, %g1
4020	movne	%icc, PTL1_BAD_KMISS, %g1
4021	ba,pt	%icc, ptl1_panic
4022	  nop
4023
40242:
4025	/*
4026	 * We are taking a pagefault in the kernel on a kernel address.  If
4027	 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
4028	 * want to call sfmmu_pagefault -- we will instead note that a fault
4029	 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
4030	 * (instead of a "retry").  This will step over the faulting
4031	 * instruction.
4032	 */
4033	CPU_INDEX(%g1, %g2)
4034	set	cpu_core, %g2
4035	sllx	%g1, CPU_CORE_SHIFT, %g1
4036	add	%g1, %g2, %g1
4037	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
4038	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
4039	bz	sfmmu_pagefault
4040	or	%g2, CPU_DTRACE_BADADDR, %g2
4041	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
4042	GET_MMU_D_ADDR(%g3, %g4)
4043	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
4044	done
4045
40463:
4047	cmp	%g5, 1
4048	ble,pt	%icc, 4f
4049	  nop
4050	TSTAT_CHECK_TL1(4f, %g1, %g2)
4051	ba,pt	%icc, sfmmu_window_trap
4052	  nop
4053
40544:
4055	/*
4056	 * We are taking a pagefault on a non-kernel address.  If we are in
4057	 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
4058	 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
4059	 */
4060	CPU_INDEX(%g1, %g2)
4061	set	cpu_core, %g2
4062	sllx	%g1, CPU_CORE_SHIFT, %g1
4063	add	%g1, %g2, %g1
4064	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
4065	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
4066	bz	sfmmu_mmu_trap
4067	or	%g2, CPU_DTRACE_BADADDR, %g2
4068	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
4069	GET_MMU_D_ADDR(%g3, %g4)
4070	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
4071
4072	/*
4073	 * Be sure that we're actually taking this miss from the kernel --
4074	 * otherwise we have managed to return to user-level with
4075	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
4076	 */
4077	rdpr	%tstate, %g2
4078	btst	TSTATE_PRIV, %g2
4079	bz,a	ptl1_panic
4080	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
4081	done
4082
4083	ALTENTRY(tsb_tl0_noctxt)
4084	/*
4085	 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
4086	 * if it is, indicated that we have faulted and issue a done.
4087	 */
4088	CPU_INDEX(%g5, %g6)
4089	set	cpu_core, %g6
4090	sllx	%g5, CPU_CORE_SHIFT, %g5
4091	add	%g5, %g6, %g5
4092	lduh	[%g5 + CPUC_DTRACE_FLAGS], %g6
4093	andcc	%g6, CPU_DTRACE_NOFAULT, %g0
4094	bz	1f
4095	or	%g6, CPU_DTRACE_BADADDR, %g6
4096	stuh	%g6, [%g5 + CPUC_DTRACE_FLAGS]
4097	GET_MMU_D_ADDR(%g3, %g4)
4098	stx	%g3, [%g5 + CPUC_DTRACE_ILLVAL]
4099
4100	/*
4101	 * Be sure that we're actually taking this miss from the kernel --
4102	 * otherwise we have managed to return to user-level with
4103	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
4104	 */
4105	rdpr	%tstate, %g5
4106	btst	TSTATE_PRIV, %g5
4107	bz,a	ptl1_panic
4108	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
4109	TSTAT_CHECK_TL1(2f, %g1, %g2);
41102:
4111	done
4112
41131:
4114	rdpr	%tt, %g5
4115	cmp	%g5, FAST_IMMU_MISS_TT
4116#ifdef sun4v
4117	MMU_FAULT_STATUS_AREA(%g2)
4118	be,a,pt	%icc, 2f
4119	  ldx	[%g2 + MMFSA_I_CTX], %g3
4120	cmp	%g5, T_INSTR_MMU_MISS
4121	be,a,pt	%icc, 2f
4122	  ldx	[%g2 + MMFSA_I_CTX], %g3
4123	ldx	[%g2 + MMFSA_D_CTX], %g3
41242:
4125#else
4126	mov	MMU_TAG_ACCESS, %g2
4127	be,a,pt	%icc, 2f
4128	  ldxa	[%g2]ASI_IMMU, %g3
4129	ldxa	[%g2]ASI_DMMU, %g3
41302:	sllx	%g3, TAGACC_CTX_LSHIFT, %g3
4131#endif /* sun4v */
4132	brz,a,pn %g3, ptl1_panic		! panic if called for kernel
4133	  mov	PTL1_BAD_CTX_STEAL, %g1		! since kernel ctx was stolen
4134	rdpr	%tl, %g5
4135	cmp	%g5, 1
4136	ble,pt	%icc, sfmmu_mmu_trap
4137	  nop
4138	TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
4139	ba,pt	%icc, sfmmu_window_trap
4140	  nop
4141	SET_SIZE(sfmmu_tsb_miss)
4142#endif  /* lint */
4143
4144#if defined (lint)
4145/*
4146 * This routine will look for a user or kernel vaddr in the hash
4147 * structure.  It returns a valid pfn or PFN_INVALID.  It doesn't
4148 * grab any locks.  It should only be used by other sfmmu routines.
4149 */
4150/* ARGSUSED */
4151pfn_t
4152sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep)
4153{
4154	return(0);
4155}
4156
4157/* ARGSUSED */
4158pfn_t
4159sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno)
4160{
4161	return(0);
4162}
4163
4164#else /* lint */
4165
4166	ENTRY_NP(sfmmu_vatopfn)
4167 	/*
4168 	 * disable interrupts
4169 	 */
4170 	rdpr	%pstate, %o3
4171#ifdef DEBUG
4172	PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1)
4173#endif
4174	/*
4175	 * disable interrupts to protect the TSBMISS area
4176	 */
4177	andn    %o3, PSTATE_IE, %o5
4178	wrpr    %o5, 0, %pstate
4179
4180	/*
4181	 * o0 = vaddr
4182	 * o1 = sfmmup
4183	 * o2 = ttep
4184	 */
4185	CPU_TSBMISS_AREA(%g1, %o5)
4186	ldn	[%g1 + TSBMISS_KHATID], %o4
4187	cmp	%o4, %o1
4188	bne,pn	%ncc, vatopfn_nokernel
4189	  mov	TTE64K, %g5			/* g5 = rehash # */
4190	mov %g1,%o5				/* o5 = tsbmiss_area */
4191	/*
4192	 * o0 = vaddr
4193	 * o1 & o4 = hatid
4194	 * o2 = ttep
4195	 * o5 = tsbmiss area
4196	 */
4197	mov	HBLK_RANGE_SHIFT, %g6
41981:
4199
4200	/*
4201	 * o0 = vaddr
4202	 * o1 = sfmmup
4203	 * o2 = ttep
4204	 * o3 = old %pstate
4205	 * o4 = hatid
4206	 * o5 = tsbmiss
4207	 * g5 = rehash #
4208	 * g6 = hmeshift
4209	 *
4210	 * The first arg to GET_TTE is actually tagaccess register
4211	 * not just vaddr. Since this call is for kernel we need to clear
4212	 * any lower vaddr bits that would be interpreted as ctx bits.
4213	 */
4214	set     TAGACC_CTX_MASK, %g1
4215	andn    %o0, %g1, %o0
4216	GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5,
4217		vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
4218
4219kvtop_hblk_found:
4220	/*
4221	 * o0 = vaddr
4222	 * o1 = sfmmup
4223	 * o2 = ttep
4224	 * g1 = tte
4225	 * g2 = tte pa
4226	 * g3 = tte va
4227	 * o2 = tsbmiss area
4228	 * o1 = hat id
4229	 */
4230	brgez,a,pn %g1, 6f			/* if tte invalid goto tl0 */
4231	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
4232	stx %g1,[%o2]				/* put tte into *ttep */
4233	TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
4234	/*
4235	 * o0 = vaddr
4236	 * o1 = sfmmup
4237	 * o2 = ttep
4238	 * g1 = pfn
4239	 */
4240	ba,pt	%xcc, 6f
4241	  mov	%g1, %o0
4242
4243kvtop_nohblk:
4244	/*
4245	 * we get here if we couldn't find valid hblk in hash.  We rehash
4246	 * if neccesary.
4247	 */
4248	ldn	[%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
4249#ifdef sun4v
4250	cmp	%g5, MAX_HASHCNT
4251#else
4252	cmp	%g5, DEFAULT_MAX_HASHCNT	/* no 32/256M kernel pages */
4253#endif /* sun4v */
4254	be,a,pn	%icc, 6f
4255	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
4256	mov	%o1, %o4			/* restore hatid */
4257#ifdef sun4v
4258        add	%g5, 2, %g5
4259	cmp	%g5, 3
4260	move	%icc, MMU_PAGESHIFT4M, %g6
4261	ba,pt	%icc, 1b
4262	movne	%icc, MMU_PAGESHIFT256M, %g6
4263#else
4264        inc	%g5
4265	cmp	%g5, 2
4266	move	%icc, MMU_PAGESHIFT512K, %g6
4267	ba,pt	%icc, 1b
4268	movne	%icc, MMU_PAGESHIFT4M, %g6
4269#endif /* sun4v */
42706:
4271	retl
4272 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4273
4274tsb_suspend:
4275	/*
4276	 * o0 = vaddr
4277	 * o1 = sfmmup
4278	 * o2 = ttep
4279	 * g1 = tte
4280	 * g2 = tte pa
4281	 * g3 = tte va
4282	 * o2 = tsbmiss area  use o5 instead of o2 for tsbmiss
4283	 */
4284	stx %g1,[%o2]				/* put tte into *ttep */
4285	brgez,a,pn %g1, 8f			/* if tte invalid goto 8: */
4286	  sub	%g0, 1, %o0			/* output = PFN_INVALID */
4287	sub	%g0, 2, %o0			/* output = PFN_SUSPENDED */
42888:
4289	retl
4290	 wrpr	%g0, %o3, %pstate		/* enable interrupts */
4291
4292vatopfn_nokernel:
4293	/*
4294	 * This routine does NOT support user addresses
4295	 * There is a routine in C that supports this.
4296	 * The only reason why we don't have the C routine
4297	 * support kernel addresses as well is because
4298	 * we do va_to_pa while holding the hashlock.
4299	 */
4300 	wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4301	save	%sp, -SA(MINFRAME), %sp
4302	sethi	%hi(sfmmu_panic3), %o0
4303	call	panic
4304	 or	%o0, %lo(sfmmu_panic3), %o0
4305
4306	SET_SIZE(sfmmu_vatopfn)
4307
4308	/*
4309	 * %o0 = vaddr
4310	 * %o1 = hashno (aka szc)
4311	 *
4312	 *
4313	 * This routine is similar to sfmmu_vatopfn() but will only look for
4314	 * a kernel vaddr in the hash structure for the specified rehash value.
4315	 * It's just an optimization for the case when pagesize for a given
4316	 * va range is already known (e.g. large page heap) and we don't want
4317	 * to start the search with rehash value 1 as sfmmu_vatopfn() does.
4318	 *
4319	 * Returns valid pfn or PFN_INVALID if
4320	 * tte for specified rehash # is not found, invalid or suspended.
4321	 */
4322	ENTRY_NP(sfmmu_kvaszc2pfn)
4323 	/*
4324 	 * disable interrupts
4325 	 */
4326 	rdpr	%pstate, %o3
4327#ifdef DEBUG
4328	PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1)
4329#endif
4330	/*
4331	 * disable interrupts to protect the TSBMISS area
4332	 */
4333	andn    %o3, PSTATE_IE, %o5
4334	wrpr    %o5, 0, %pstate
4335
4336	CPU_TSBMISS_AREA(%g1, %o5)
4337	ldn	[%g1 + TSBMISS_KHATID], %o4
4338	sll	%o1, 1, %g6
4339	add	%g6, %o1, %g6
4340	add	%g6, MMU_PAGESHIFT, %g6
4341	/*
4342	 * %o0 = vaddr
4343	 * %o1 = hashno
4344	 * %o3 = old %pstate
4345	 * %o4 = ksfmmup
4346	 * %g1 = tsbmiss area
4347	 * %g6 = hmeshift
4348	 */
4349
4350	/*
4351	 * The first arg to GET_TTE is actually tagaccess register
4352	 * not just vaddr. Since this call is for kernel we need to clear
4353	 * any lower vaddr bits that would be interpreted as ctx bits.
4354	 */
4355	srlx	%o0, MMU_PAGESHIFT, %o0
4356	sllx	%o0, MMU_PAGESHIFT, %o0
4357	GET_TTE(%o0, %o4, %g3, %g4, %g5, %g1, %o5, %g6, %o1,
4358		kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk,
4359		kvaszc2pfn_nohblk)
4360
4361kvaszc2pfn_hblk_found:
4362	/*
4363	 * %g3 = tte
4364	 * %o0 = vaddr
4365	 */
4366	brgez,a,pn %g3, 1f			/* check if tte is invalid */
4367	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
4368	TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5)
4369	/*
4370	 * g3 = pfn
4371	 */
4372	ba,pt	%xcc, 1f
4373	  mov	%g3, %o0
4374
4375kvaszc2pfn_nohblk:
4376	mov	-1, %o0
4377
43781:
4379	retl
4380 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4381
4382	SET_SIZE(sfmmu_kvaszc2pfn)
4383
4384#endif /* lint */
4385
4386
4387
4388#if !defined(lint)
4389
4390/*
4391 * kpm lock used between trap level tsbmiss handler and kpm C level.
4392 */
4393#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi)			\
4394	mov     0xff, tmp1						;\
4395label1:									;\
4396	casa    [kpmlckp]asi, %g0, tmp1					;\
4397	brnz,pn tmp1, label1						;\
4398	mov     0xff, tmp1						;\
4399	membar  #LoadLoad
4400
4401#define KPMLOCK_EXIT(kpmlckp, asi)					\
4402	membar  #LoadStore|#StoreStore					;\
4403	sta     %g0, [kpmlckp]asi
4404
4405/*
4406 * Lookup a memseg for a given pfn and if found, return the physical
4407 * address of the corresponding struct memseg in mseg, otherwise
4408 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
4409 * tsbmp, %asi is assumed to be ASI_MEM.
4410 * This lookup is done by strictly traversing only the physical memseg
4411 * linkage. The more generic approach, to check the virtual linkage
4412 * before using the physical (used e.g. with hmehash buckets), cannot
4413 * be used here. Memory DR operations can run in parallel to this
4414 * lookup w/o any locks and updates of the physical and virtual linkage
4415 * cannot be done atomically wrt. to each other. Because physical
4416 * address zero can be valid physical address, MSEG_NULLPTR_PA acts
4417 * as "physical NULL" pointer.
4418 */
4419#define	PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
4420	sethi	%hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */	;\
4421	ldx	[tmp3 + %lo(mhash_per_slot)], mseg			;\
4422	udivx	pfn, mseg, mseg						;\
4423	ldx	[tsbmp + KPMTSBM_MSEGPHASHPA], tmp1			;\
4424	and	mseg, SFMMU_N_MEM_SLOTS - 1, mseg			;\
4425	sllx	mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg			;\
4426	add	tmp1, mseg, tmp1					;\
4427	ldxa	[tmp1]%asi, mseg					;\
4428	cmp	mseg, MSEG_NULLPTR_PA					;\
4429	be,pn	%xcc, label/**/1		/* if not found */	;\
4430	  nop								;\
4431	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
4432	cmp	pfn, tmp1			/* pfn - pages_base */	;\
4433	blu,pn	%xcc, label/**/1					;\
4434	  ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
4435	cmp	pfn, tmp2			/* pfn - pages_end */	;\
4436	bgeu,pn	%xcc, label/**/1					;\
4437	  sub	pfn, tmp1, tmp1			/* pfn - pages_base */	;\
4438	mulx	tmp1, PAGE_SIZE, tmp1					;\
4439	ldxa	[mseg + MEMSEG_PAGESPA]%asi, tmp2	/* pages */	;\
4440	add	tmp2, tmp1, tmp1			/* pp */	;\
4441	lduwa	[tmp1 + PAGE_PAGENUM]%asi, tmp2				;\
4442	cmp	tmp2, pfn						;\
4443	be,pt	%xcc, label/**/_ok			/* found */	;\
4444label/**/1:								;\
4445	/* brute force lookup */					;\
4446	sethi	%hi(memsegspa), tmp3 /* no tsbmp use due to DR */	;\
4447	ldx	[tmp3 + %lo(memsegspa)], mseg				;\
4448label/**/2:								;\
4449	cmp	mseg, MSEG_NULLPTR_PA					;\
4450	be,pn	%xcc, label/**/_ok		/* if not found */	;\
4451	  nop								;\
4452	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
4453	cmp	pfn, tmp1			/* pfn - pages_base */	;\
4454	blu,a,pt %xcc, label/**/2					;\
4455	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
4456	ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
4457	cmp	pfn, tmp2			/* pfn - pages_end */	;\
4458	bgeu,a,pt %xcc, label/**/2					;\
4459	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
4460label/**/_ok:
4461
4462	/*
4463	 * kpm tsb miss handler large pages
4464	 * g1 = 8K kpm TSB entry pointer
4465	 * g2 = tag access register
4466	 * g3 = 4M kpm TSB entry pointer
4467	 */
4468	ALTENTRY(sfmmu_kpm_dtsb_miss)
4469	TT_TRACE(trace_tsbmiss)
4470
4471	CPU_INDEX(%g7, %g6)
4472	sethi	%hi(kpmtsbm_area), %g6
4473	sllx	%g7, KPMTSBM_SHIFT, %g7
4474	or	%g6, %lo(kpmtsbm_area), %g6
4475	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
4476
4477	/* check enable flag */
4478	ldub	[%g6 + KPMTSBM_FLAGS], %g4
4479	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
4480	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
4481	  nop
4482
4483	/* VA range check */
4484	ldx	[%g6 + KPMTSBM_VBASE], %g7
4485	cmp	%g2, %g7
4486	blu,pn	%xcc, sfmmu_tsb_miss
4487	  ldx	[%g6 + KPMTSBM_VEND], %g5
4488	cmp	%g2, %g5
4489	bgeu,pn	%xcc, sfmmu_tsb_miss
4490	  stx	%g3, [%g6 + KPMTSBM_TSBPTR]
4491
4492	/*
4493	 * check TL tsbmiss handling flag
4494	 * bump tsbmiss counter
4495	 */
4496	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
4497#ifdef	DEBUG
4498	and	%g4, KPMTSBM_TLTSBM_FLAG, %g3
4499	inc	%g5
4500	brz,pn	%g3, sfmmu_kpm_exception
4501	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
4502#else
4503	inc	%g5
4504	st	%g5, [%g6 + KPMTSBM_TSBMISS]
4505#endif
4506	/*
4507	 * At this point:
4508	 *  g1 = 8K kpm TSB pointer (not used)
4509	 *  g2 = tag access register
4510	 *  g3 = clobbered
4511	 *  g6 = per-CPU kpm tsbmiss area
4512	 *  g7 = kpm_vbase
4513	 */
4514
4515	/* vaddr2pfn */
4516	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
4517	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
4518	srax    %g4, %g3, %g2			/* which alias range (r) */
4519	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
4520	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
4521
4522	/*
4523	 * Setup %asi
4524	 * mseg_pa = page_numtomemseg_nolock(pfn)
4525	 * if (mseg_pa == NULL) sfmmu_kpm_exception
4526	 * g2=pfn
4527	 */
4528	mov	ASI_MEM, %asi
4529	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
4530	cmp	%g3, MSEG_NULLPTR_PA
4531	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
4532	  nop
4533
4534	/*
4535	 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
4536	 * g2=pfn g3=mseg_pa
4537	 */
4538	ldub	[%g6 + KPMTSBM_KPMP2PSHFT], %g5
4539	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4540	srlx	%g2, %g5, %g4
4541	sllx	%g4, %g5, %g4
4542	sub	%g4, %g7, %g4
4543	srlx	%g4, %g5, %g4
4544
4545	/*
4546	 * Validate inx value
4547	 * g2=pfn g3=mseg_pa g4=inx
4548	 */
4549#ifdef	DEBUG
4550	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4551	cmp	%g4, %g5			/* inx - nkpmpgs */
4552	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
4553	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4554#else
4555	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4556#endif
4557	/*
4558	 * kp = &mseg_pa->kpm_pages[inx]
4559	 */
4560	sllx	%g4, KPMPAGE_SHIFT, %g4		/* kpm_pages offset */
4561	ldxa	[%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
4562	add	%g5, %g4, %g5			/* kp */
4563
4564	/*
4565	 * KPMP_HASH(kp)
4566	 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
4567	 */
4568	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
4569	sub	%g7, 1, %g7			/* mask */
4570	srlx	%g5, %g1, %g1			/* x = ksp >> kpmp_shift */
4571	add	%g5, %g1, %g5			/* y = ksp + x */
4572	and 	%g5, %g7, %g5			/* hashinx = y & mask */
4573
4574	/*
4575	 * Calculate physical kpm_page pointer
4576	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4577	 */
4578	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
4579	add	%g1, %g4, %g1			/* kp_pa */
4580
4581	/*
4582	 * Calculate physical hash lock address
4583	 * g1=kp_refcntc_pa g2=pfn g5=hashinx
4584	 */
4585	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
4586	sllx	%g5, KPMHLK_SHIFT, %g5
4587	add	%g4, %g5, %g3
4588	add	%g3, KPMHLK_LOCK, %g3		/* hlck_pa */
4589
4590	/*
4591	 * Assemble tte
4592	 * g1=kp_pa g2=pfn g3=hlck_pa
4593	 */
4594#ifdef sun4v
4595	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
4596	sllx	%g5, 32, %g5
4597	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4598	or	%g4, TTE4M, %g4
4599	or	%g5, %g4, %g5
4600#else
4601	sethi	%hi(TTE_VALID_INT), %g4
4602	mov	TTE4M, %g5
4603	sllx	%g5, TTE_SZ_SHFT_INT, %g5
4604	or	%g5, %g4, %g5			/* upper part */
4605	sllx	%g5, 32, %g5
4606	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4607	or	%g5, %g4, %g5
4608#endif
4609	sllx	%g2, MMU_PAGESHIFT, %g4
4610	or	%g5, %g4, %g5			/* tte */
4611	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
4612	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
4613
4614	/*
4615	 * tsb dropin
4616	 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
4617	 */
4618
4619	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4620	KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)
4621
4622	/* use C-handler if there's no go for dropin */
4623	ldsha	[%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
4624	cmp	%g7, -1
4625	bne,pn	%xcc, 5f	/* use C-handler if there's no go for dropin */
4626	  nop
4627
4628#ifdef	DEBUG
4629	/* double check refcnt */
4630	ldsha	[%g1 + KPMPAGE_REFCNT]%asi, %g7
4631	brz,pn	%g7, 5f			/* let C-handler deal with this */
4632	  nop
4633#endif
4634
4635#ifndef sun4v
4636	ldub	[%g6 + KPMTSBM_FLAGS], %g7
4637	mov	ASI_N, %g1
4638	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
4639	movnz	%icc, ASI_MEM, %g1
4640	mov	%g1, %asi
4641#endif
4642
4643	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
4644	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
4645
4646	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4647	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4648
4649	DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
4650
4651	/* KPMLOCK_EXIT(kpmlckp, asi) */
4652	KPMLOCK_EXIT(%g3, ASI_MEM)
4653
4654	/*
4655	 * If trapstat is running, we need to shift the %tpc and %tnpc to
4656	 * point to trapstat's TSB miss return code (note that trapstat
4657	 * itself will patch the correct offset to add).
4658	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4659	 */
4660	rdpr	%tl, %g7
4661	cmp	%g7, 1
4662	ble	%icc, 0f
4663	sethi	%hi(KERNELBASE), %g6
4664	rdpr	%tpc, %g7
4665	or	%g6, %lo(KERNELBASE), %g6
4666	cmp	%g7, %g6
4667	bgeu	%xcc, 0f
4668	ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
4669	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
4670	wrpr	%g7, %tpc
4671	add	%g7, 4, %g7
4672	wrpr	%g7, %tnpc
46730:
4674	retry
46755:
4676	/* g3=hlck_pa */
4677	KPMLOCK_EXIT(%g3, ASI_MEM)
4678	ba,pt	%icc, sfmmu_kpm_exception
4679	  nop
4680	SET_SIZE(sfmmu_kpm_dtsb_miss)
4681
4682	/*
4683	 * kpm tsbmiss handler for smallpages
4684	 * g1 = 8K kpm TSB pointer
4685	 * g2 = tag access register
4686	 * g3 = 4M kpm TSB pointer
4687	 */
4688	ALTENTRY(sfmmu_kpm_dtsb_miss_small)
4689	TT_TRACE(trace_tsbmiss)
4690	CPU_INDEX(%g7, %g6)
4691	sethi	%hi(kpmtsbm_area), %g6
4692	sllx	%g7, KPMTSBM_SHIFT, %g7
4693	or	%g6, %lo(kpmtsbm_area), %g6
4694	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
4695
4696	/* check enable flag */
4697	ldub	[%g6 + KPMTSBM_FLAGS], %g4
4698	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
4699	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
4700	  nop
4701
4702	/*
4703	 * VA range check
4704	 * On fail: goto sfmmu_tsb_miss
4705	 */
4706	ldx	[%g6 + KPMTSBM_VBASE], %g7
4707	cmp	%g2, %g7
4708	blu,pn	%xcc, sfmmu_tsb_miss
4709	  ldx	[%g6 + KPMTSBM_VEND], %g5
4710	cmp	%g2, %g5
4711	bgeu,pn	%xcc, sfmmu_tsb_miss
4712	  stx	%g1, [%g6 + KPMTSBM_TSBPTR]	/* save 8K kpm TSB pointer */
4713
4714	/*
4715	 * check TL tsbmiss handling flag
4716	 * bump tsbmiss counter
4717	 */
4718	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
4719#ifdef	DEBUG
4720	and	%g4, KPMTSBM_TLTSBM_FLAG, %g1
4721	inc	%g5
4722	brz,pn	%g1, sfmmu_kpm_exception
4723	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
4724#else
4725	inc	%g5
4726	st	%g5, [%g6 + KPMTSBM_TSBMISS]
4727#endif
4728	/*
4729	 * At this point:
4730	 *  g1 = clobbered
4731	 *  g2 = tag access register
4732	 *  g3 = 4M kpm TSB pointer (not used)
4733	 *  g6 = per-CPU kpm tsbmiss area
4734	 *  g7 = kpm_vbase
4735	 */
4736
4737	/*
4738	 * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr)
4739	 * which is defined in mach_kpm.h. Any changes in that macro
4740	 * should also be ported back to this assembly code.
4741	 */
4742	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3	/* g3 = kpm_size_shift */
4743	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
4744	srax    %g4, %g3, %g7			/* which alias range (r) */
4745	brz,pt	%g7, 2f
4746	  sethi   %hi(vac_colors_mask), %g5
4747	ld	[%g5 + %lo(vac_colors_mask)], %g5
4748
4749	srlx	%g2, MMU_PAGESHIFT, %g1		/* vaddr >> MMU_PAGESHIFT */
4750	and	%g1, %g5, %g1			/* g1 = v */
4751	sllx	%g7, %g3, %g5			/* g5 = r << kpm_size_shift */
4752	cmp	%g7, %g1			/* if (r > v) */
4753	bleu,pn %xcc, 1f
4754	  sub   %g4, %g5, %g4			/* paddr -= r << kpm_size_shift */
4755	sub	%g7, %g1, %g5			/* g5 = r - v */
4756	sllx	%g5, MMU_PAGESHIFT, %g7		/* (r-v) << MMU_PAGESHIFT */
4757	add	%g4, %g7, %g4			/* paddr += (r-v)<<MMU_PAGESHIFT */
4758	ba	2f
4759	  nop
47601:
4761	sllx	%g7, MMU_PAGESHIFT, %g5		/* else */
4762	sub	%g4, %g5, %g4			/* paddr -= r << MMU_PAGESHIFT */
4763
4764	/*
4765	 * paddr2pfn
4766	 *  g1 = vcolor (not used)
4767	 *  g2 = tag access register
4768	 *  g3 = clobbered
4769	 *  g4 = paddr
4770	 *  g5 = clobbered
4771	 *  g6 = per-CPU kpm tsbmiss area
4772	 *  g7 = clobbered
4773	 */
47742:
4775	srlx	%g4, MMU_PAGESHIFT, %g2		/* g2 = pfn */
4776
4777	/*
4778	 * Setup %asi
4779	 * mseg_pa = page_numtomemseg_nolock_pa(pfn)
4780	 * if (mseg not found) sfmmu_kpm_exception
4781	 * g2=pfn g6=per-CPU kpm tsbmiss area
4782	 * g4 g5 g7 for scratch use.
4783	 */
4784	mov	ASI_MEM, %asi
4785	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
4786	cmp	%g3, MSEG_NULLPTR_PA
4787	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
4788	  nop
4789
4790	/*
4791	 * inx = pfn - mseg_pa->kpm_pbase
4792	 * g2=pfn  g3=mseg_pa  g6=per-CPU kpm tsbmiss area
4793	 */
4794	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4795	sub	%g2, %g7, %g4
4796
4797#ifdef	DEBUG
4798	/*
4799	 * Validate inx value
4800	 * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area
4801	 */
4802	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4803	cmp	%g4, %g5			/* inx - nkpmpgs */
4804	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
4805	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4806#else
4807	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4808#endif
4809	/* ksp = &mseg_pa->kpm_spages[inx] */
4810	ldxa	[%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
4811	add	%g5, %g4, %g5			/* ksp */
4812
4813	/*
4814	 * KPMP_SHASH(kp)
4815	 * g2=pfn g3=mseg_pa g4=inx g5=ksp
4816	 * g6=per-CPU kpm tsbmiss area  g7=kpmp_stable_sz
4817	 */
4818	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
4819	sub	%g7, 1, %g7			/* mask */
4820	sllx	%g5, %g1, %g1			/* x = ksp << kpmp_shift */
4821	add	%g5, %g1, %g5			/* y = ksp + x */
4822	and 	%g5, %g7, %g5			/* hashinx = y & mask */
4823
4824	/*
4825	 * Calculate physical kpm_spage pointer
4826	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4827	 * g6=per-CPU kpm tsbmiss area
4828	 */
4829	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
4830	add	%g1, %g4, %g1			/* ksp_pa */
4831
4832	/*
4833	 * Calculate physical hash lock address.
4834	 * Note: Changes in kpm_shlk_t must be reflected here.
4835	 * g1=ksp_pa g2=pfn g5=hashinx
4836	 * g6=per-CPU kpm tsbmiss area
4837	 */
4838	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
4839	sllx	%g5, KPMSHLK_SHIFT, %g5
4840	add	%g4, %g5, %g3			/* hlck_pa */
4841
4842	/*
4843	 * Assemble non-cacheable tte initially
4844	 * g1=ksp_pa g2=pfn g3=hlck_pa
4845	 * g6=per-CPU kpm tsbmiss area
4846	 */
4847	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
4848	sllx	%g5, 32, %g5
4849	mov	(TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4850	or	%g5, %g4, %g5
4851	sllx	%g2, MMU_PAGESHIFT, %g4
4852	or	%g5, %g4, %g5			/* tte */
4853	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
4854	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
4855
4856	/*
4857	 * tsb dropin
4858	 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable)
4859	 * g6=per-CPU kpm tsbmiss area  g7=scratch register
4860	 */
4861
4862	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4863	KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)
4864
4865	/* use C-handler if there's no go for dropin */
4866	ldsba	[%g1 + KPMSPAGE_MAPPED]%asi, %g7	/* kp_mapped */
4867	andcc	%g7, KPM_MAPPED_GO, %g0			/* go or no go ? */
4868	bz,pt	%icc, 5f				/* no go */
4869	  nop
4870	and	%g7, KPM_MAPPED_MASK, %g7		/* go */
4871	cmp	%g7, KPM_MAPPEDS			/* cacheable ? */
4872	be,a,pn	%xcc, 3f
4873	  or	%g5, TTE_CV_INT, %g5			/* cacheable */
48743:
4875#ifndef sun4v
4876	ldub	[%g6 + KPMTSBM_FLAGS], %g7
4877	mov	ASI_N, %g1
4878	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
4879	movnz	%icc, ASI_MEM, %g1
4880	mov	%g1, %asi
4881#endif
4882
4883	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
4884	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
4885
4886	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4887	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4888
4889	DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
4890
4891	/* KPMLOCK_EXIT(kpmlckp, asi) */
4892	KPMLOCK_EXIT(%g3, ASI_MEM)
4893
4894	/*
4895	 * If trapstat is running, we need to shift the %tpc and %tnpc to
4896	 * point to trapstat's TSB miss return code (note that trapstat
4897	 * itself will patch the correct offset to add).
4898	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4899	 */
4900	rdpr	%tl, %g7
4901	cmp	%g7, 1
4902	ble	%icc, 0f
4903	sethi	%hi(KERNELBASE), %g6
4904	rdpr	%tpc, %g7
4905	or	%g6, %lo(KERNELBASE), %g6
4906	cmp	%g7, %g6
4907	bgeu	%xcc, 0f
4908	ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
4909	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
4910	wrpr	%g7, %tpc
4911	add	%g7, 4, %g7
4912	wrpr	%g7, %tnpc
49130:
4914	retry
49155:
4916	/* g3=hlck_pa */
4917	KPMLOCK_EXIT(%g3, ASI_MEM)
4918	ba,pt	%icc, sfmmu_kpm_exception
4919	  nop
4920	SET_SIZE(sfmmu_kpm_dtsb_miss_small)
4921
4922#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
4923#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
4924#endif
4925
4926#endif /* lint */
4927
4928#ifdef	lint
4929/*
4930 * Enable/disable tsbmiss handling at trap level for a kpm (large) page.
4931 * Called from C-level, sets/clears "go" indication for trap level handler.
4932 * khl_lock is a low level spin lock to protect the kp_tsbmtl field.
4933 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level.
4934 * Assumes khl_mutex is held when called from C-level.
4935 */
4936/* ARGSUSED */
4937void
4938sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd)
4939{
4940}
4941
4942/*
4943 * kpm_smallpages: stores val to byte at address mapped within
4944 * low level lock brackets. The old value is returned.
4945 * Called from C-level.
4946 */
4947/* ARGSUSED */
4948int
4949sfmmu_kpm_stsbmtl(uchar_t *mapped, uint_t *kshl_lock, int val)
4950{
4951	return (0);
4952}
4953
4954#else /* lint */
4955
4956	.seg	".data"
4957sfmmu_kpm_tsbmtl_panic:
4958	.ascii	"sfmmu_kpm_tsbmtl: interrupts disabled"
4959	.byte	0
4960sfmmu_kpm_stsbmtl_panic:
4961	.ascii	"sfmmu_kpm_stsbmtl: interrupts disabled"
4962	.byte	0
4963	.align	4
4964	.seg	".text"
4965
4966	ENTRY_NP(sfmmu_kpm_tsbmtl)
4967	rdpr	%pstate, %o3
4968	/*
4969	 * %o0 = &kp_refcntc
4970	 * %o1 = &khl_lock
4971	 * %o2 = 0/1 (off/on)
4972	 * %o3 = pstate save
4973	 */
4974#ifdef DEBUG
4975	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4976	bnz,pt %icc, 1f				/* disabled, panic	 */
4977	  nop
4978	save	%sp, -SA(MINFRAME), %sp
4979	sethi	%hi(sfmmu_kpm_tsbmtl_panic), %o0
4980	call	panic
4981	 or	%o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
4982	ret
4983	restore
49841:
4985#endif /* DEBUG */
4986	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4987
4988	KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
4989	mov	-1, %o5
4990	brz,a	%o2, 2f
4991	  mov	0, %o5
49922:
4993	sth	%o5, [%o0]
4994	KPMLOCK_EXIT(%o1, ASI_N)
4995
4996	retl
4997	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4998	SET_SIZE(sfmmu_kpm_tsbmtl)
4999
5000	ENTRY_NP(sfmmu_kpm_stsbmtl)
5001	rdpr	%pstate, %o3
5002	/*
5003	 * %o0 = &mapped
5004	 * %o1 = &kshl_lock
5005	 * %o2 = val
5006	 * %o3 = pstate save
5007	 */
5008#ifdef DEBUG
5009	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
5010	bnz,pt %icc, 1f				/* disabled, panic	 */
5011	  nop
5012	save	%sp, -SA(MINFRAME), %sp
5013	sethi	%hi(sfmmu_kpm_stsbmtl_panic), %o0
5014	call	panic
5015	  or	%o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
5016	ret
5017	restore
50181:
5019#endif /* DEBUG */
5020	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
5021
5022	KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
5023	ldsb	[%o0], %o5
5024	stb	%o2, [%o0]
5025	KPMLOCK_EXIT(%o1, ASI_N)
5026
5027	and	%o5, KPM_MAPPED_MASK, %o0	/* return old val */
5028	retl
5029	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
5030	SET_SIZE(sfmmu_kpm_stsbmtl)
5031
5032#endif /* lint */
5033
5034#ifndef lint
5035#ifdef sun4v
5036	/*
5037	 * User/kernel data miss w// multiple TSBs
5038	 * The first probe covers 8K, 64K, and 512K page sizes,
5039	 * because 64K and 512K mappings are replicated off 8K
5040	 * pointer.  Second probe covers 4M page size only.
5041	 *
5042	 * MMU fault area contains miss address and context.
5043	 */
5044	ALTENTRY(sfmmu_slow_dmmu_miss)
5045	GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)	! %g2 = ptagacc, %g3 = ctx type
5046
5047slow_miss_common:
5048	/*
5049	 *  %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
5050	 *  %g3 = ctx (cannot be INVALID_CONTEXT)
5051	 */
5052	brnz,pt	%g3, 8f			! check for user context
5053	  nop
5054
5055	/*
5056	 * Kernel miss
5057	 * Get 8K and 4M TSB pointers in %g1 and %g3 and
5058	 * branch to sfmmu_tsb_miss_tt to handle it.
5059	 */
5060	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
5061sfmmu_dslow_patch_ktsb_base:
5062	RUNTIME_PATCH_SETX(%g1, %g6)	! %g1 = contents of ktsb_pbase
5063sfmmu_dslow_patch_ktsb_szcode:
5064	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
5065
5066	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
5067	! %g1 = First TSB entry pointer, as TSB miss handler expects
5068
5069	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
5070sfmmu_dslow_patch_ktsb4m_base:
5071	RUNTIME_PATCH_SETX(%g3, %g6)	! %g3 = contents of ktsb4m_pbase
5072sfmmu_dslow_patch_ktsb4m_szcode:
5073	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
5074
5075	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
5076	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
5077	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
5078	.empty
5079
50808:
5081	/*
5082	 * User miss
5083	 * Get first TSB pointer in %g1
5084	 * Get second TSB pointer (or NULL if no second TSB) in %g3
5085	 * Branch to sfmmu_tsb_miss_tt to handle it
5086	 */
5087	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
5088	/* %g1 = first TSB entry ptr now, %g2 preserved */
5089
5090	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
5091	brlz,pt %g3, sfmmu_tsb_miss_tt		/* done if no 2nd TSB */
5092	  nop
5093
5094	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
5095	/* %g3 = second TSB entry ptr now, %g2 preserved */
50969:
5097	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
5098	.empty
5099	SET_SIZE(sfmmu_slow_dmmu_miss)
5100
5101
5102	/*
5103	 * User/kernel instruction miss w/ multiple TSBs
5104	 * The first probe covers 8K, 64K, and 512K page sizes,
5105	 * because 64K and 512K mappings are replicated off 8K
5106	 * pointer.  Second probe covers 4M page size only.
5107	 *
5108	 * MMU fault area contains miss address and context.
5109	 */
5110	ALTENTRY(sfmmu_slow_immu_miss)
5111	GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3)
5112	ba,a,pt	%xcc, slow_miss_common
5113	SET_SIZE(sfmmu_slow_immu_miss)
5114
5115#endif /* sun4v */
5116#endif	/* lint */
5117
5118#ifndef lint
5119
5120/*
5121 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
5122 */
5123	.seg	".data"
5124	.align	64
5125	.global tsbmiss_area
5126tsbmiss_area:
5127	.skip	(TSBMISS_SIZE * NCPU)
5128
5129	.align	64
5130	.global kpmtsbm_area
5131kpmtsbm_area:
5132	.skip	(KPMTSBM_SIZE * NCPU)
5133#endif	/* lint */
5134