xref: /titanic_51/usr/src/uts/sfmmu/vm/hat_sfmmu.h (revision 9e59f930aa12797575f40ccaad097ac2dd7fc4d0)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * VM - Hardware Address Translation management.
28  *
29  * This file describes the contents of the sun-reference-mmu(sfmmu)-
30  * specific hat data structures and the sfmmu-specific hat procedures.
31  * The machine-independent interface is described in <vm/hat.h>.
32  */
33 
34 #ifndef	_VM_HAT_SFMMU_H
35 #define	_VM_HAT_SFMMU_H
36 
37 #pragma ident	"%Z%%M%	%I%	%E% SMI"
38 
39 #ifdef	__cplusplus
40 extern "C" {
41 #endif
42 
43 #ifndef _ASM
44 
45 #include <sys/types.h>
46 
47 #endif /* _ASM */
48 
49 #ifdef	_KERNEL
50 
51 #include <sys/pte.h>
52 #include <vm/mach_sfmmu.h>
53 #include <sys/mmu.h>
54 
55 /*
56  * Don't alter these without considering changes to ism_map_t.
57  */
58 #define	DEFAULT_ISM_PAGESIZE		MMU_PAGESIZE4M
59 #define	ISM_PG_SIZE(ism_vbshift)	(1 << ism_vbshift)
60 #define	ISM_SZ_MASK(ism_vbshift)	(ISM_PG_SIZE(ism_vbshift) - 1)
61 #define	ISM_MAP_SLOTS	8	/* Change this carefully. */
62 
63 #ifndef _ASM
64 
65 #include <sys/t_lock.h>
66 #include <vm/hat.h>
67 #include <vm/seg.h>
68 #include <sys/machparam.h>
69 #include <sys/systm.h>
70 #include <sys/x_call.h>
71 #include <vm/page.h>
72 #include <sys/ksynch.h>
73 
74 typedef struct hat sfmmu_t;
75 
76 /*
77  * SFMMU attributes for hat_memload/hat_devload
78  */
79 #define	SFMMU_UNCACHEPTTE	0x01000000	/* unencache in physical $ */
80 #define	SFMMU_UNCACHEVTTE	0x02000000	/* unencache in virtual $ */
81 #define	SFMMU_SIDEFFECT		0x04000000	/* set side effect bit */
82 #define	SFMMU_LOAD_ALLATTR	(HAT_PROT_MASK | HAT_ORDER_MASK |	\
83 		HAT_ENDIAN_MASK | HAT_NOFAULT | HAT_NOSYNC |		\
84 		SFMMU_UNCACHEPTTE | SFMMU_UNCACHEVTTE | SFMMU_SIDEFFECT)
85 
86 
87 /*
88  * sfmmu flags for hat_memload/hat_devload
89  */
90 #define	SFMMU_NO_TSBLOAD	0x08000000	/* do not preload tsb */
91 #define	SFMMU_LOAD_ALLFLAG	(HAT_LOAD | HAT_LOAD_LOCK |		\
92 		HAT_LOAD_ADV | HAT_LOAD_CONTIG | HAT_LOAD_NOCONSIST |	\
93 		HAT_LOAD_SHARE | HAT_LOAD_REMAP | SFMMU_NO_TSBLOAD |	\
94 		HAT_RELOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_TEXT)
95 
96 /*
97  * sfmmu internal flag to hat_pageunload that spares locked mappings
98  */
99 #define	SFMMU_KERNEL_RELOC	0x8000
100 
101 /*
102  * mode for sfmmu_chgattr
103  */
104 #define	SFMMU_SETATTR	0x0
105 #define	SFMMU_CLRATTR	0x1
106 #define	SFMMU_CHGATTR	0x2
107 
108 /*
109  * sfmmu specific flags for page_t
110  */
111 #define	P_PNC	0x8		/* non-caching is permanent bit */
112 #define	P_TNC	0x10		/* non-caching is temporary bit */
113 #define	P_KPMS	0x20		/* kpm mapped small (vac alias prevention) */
114 #define	P_KPMC	0x40		/* kpm conflict page (vac alias prevention) */
115 
116 #define	PP_GENERIC_ATTR(pp)	((pp)->p_nrm & (P_MOD | P_REF | P_RO))
117 #define	PP_ISMOD(pp)		((pp)->p_nrm & P_MOD)
118 #define	PP_ISREF(pp)		((pp)->p_nrm & P_REF)
119 #define	PP_ISRO(pp)		((pp)->p_nrm & P_RO)
120 #define	PP_ISNC(pp)		((pp)->p_nrm & (P_PNC|P_TNC))
121 #define	PP_ISPNC(pp)		((pp)->p_nrm & P_PNC)
122 #ifdef VAC
123 #define	PP_ISTNC(pp)		((pp)->p_nrm & P_TNC)
124 #endif
125 #define	PP_ISKPMS(pp)		((pp)->p_nrm & P_KPMS)
126 #define	PP_ISKPMC(pp)		((pp)->p_nrm & P_KPMC)
127 
128 #define	PP_SETMOD(pp)		((pp)->p_nrm |= P_MOD)
129 #define	PP_SETREF(pp)		((pp)->p_nrm |= P_REF)
130 #define	PP_SETREFMOD(pp)	((pp)->p_nrm |= (P_REF|P_MOD))
131 #define	PP_SETRO(pp)		((pp)->p_nrm |= P_RO)
132 #define	PP_SETREFRO(pp)		((pp)->p_nrm |= (P_REF|P_RO))
133 #define	PP_SETPNC(pp)		((pp)->p_nrm |= P_PNC)
134 #ifdef VAC
135 #define	PP_SETTNC(pp)		((pp)->p_nrm |= P_TNC)
136 #endif
137 #define	PP_SETKPMS(pp)		((pp)->p_nrm |= P_KPMS)
138 #define	PP_SETKPMC(pp)		((pp)->p_nrm |= P_KPMC)
139 
140 #define	PP_CLRMOD(pp)		((pp)->p_nrm &= ~P_MOD)
141 #define	PP_CLRREF(pp)		((pp)->p_nrm &= ~P_REF)
142 #define	PP_CLRREFMOD(pp)	((pp)->p_nrm &= ~(P_REF|P_MOD))
143 #define	PP_CLRRO(pp)		((pp)->p_nrm &= ~P_RO)
144 #define	PP_CLRPNC(pp)		((pp)->p_nrm &= ~P_PNC)
145 #ifdef VAC
146 #define	PP_CLRTNC(pp)		((pp)->p_nrm &= ~P_TNC)
147 #endif
148 #define	PP_CLRKPMS(pp)		((pp)->p_nrm &= ~P_KPMS)
149 #define	PP_CLRKPMC(pp)		((pp)->p_nrm &= ~P_KPMC)
150 
151 /*
152  * All shared memory segments attached with the SHM_SHARE_MMU flag (ISM)
153  * will be constrained to a 4M, 32M or 256M alignment. Also since every newly-
154  * created ISM segment is created out of a new address space at base va
155  * of 0 we don't need to store it.
156  */
157 #define	ISM_ALIGN(shift)	(1 << shift)	/* base va aligned to <n>M  */
158 #define	ISM_ALIGNED(shift, va)	(((uintptr_t)va & (ISM_ALIGN(shift) - 1)) == 0)
159 #define	ISM_SHIFT(shift, x)	((uintptr_t)x >> (shift))
160 
161 /*
162  * Pad locks out to cache sub-block boundaries to prevent
163  * false sharing, so several processes don't contend for
164  * the same line if they aren't using the same lock.  Since
165  * this is a typedef we also have a bit of freedom in
166  * changing lock implementations later if we decide it
167  * is necessary.
168  */
169 typedef struct hat_lock {
170 	kmutex_t hl_mutex;
171 	uchar_t hl_pad[64 - sizeof (kmutex_t)];
172 } hatlock_t;
173 
174 #define	HATLOCK_MUTEXP(hatlockp)	(&((hatlockp)->hl_mutex))
175 
176 /*
177  * All segments mapped with ISM are guaranteed to be 4M, 32M or 256M aligned.
178  * Also size is guaranteed to be in 4M, 32M or 256M chunks.
179  * ism_seg consists of the following members:
180  * [XX..22] base address of ism segment. XX is 63 or 31 depending whether
181  *	caddr_t is 64 bits or 32 bits.
182  * [21..0] size of segment.
183  *
184  * NOTE: Don't alter this structure without changing defines above and
185  * the tsb_miss and protection handlers.
186  */
187 typedef struct ism_map {
188 	uintptr_t	imap_seg;  	/* base va + sz of ISM segment */
189 	ushort_t	imap_vb_shift;	/* mmu_pageshift for ism page size */
190 	ushort_t	imap_hatflags;	/* primary ism page size */
191 	uint_t		imap_sz_mask;	/* mmu_pagemask for ism page size */
192 	sfmmu_t		*imap_ismhat; 	/* hat id of dummy ISM as */
193 	struct ism_ment	*imap_ment;	/* pointer to mapping list entry */
194 } ism_map_t;
195 
196 #define	ism_start(map)	((caddr_t)((map).imap_seg & \
197 				~ISM_SZ_MASK((map).imap_vb_shift)))
198 #define	ism_size(map)	((map).imap_seg & ISM_SZ_MASK((map).imap_vb_shift))
199 #define	ism_end(map)	((caddr_t)(ism_start(map) + (ism_size(map) * \
200 				ISM_PG_SIZE((map).imap_vb_shift))))
201 /*
202  * ISM mapping entry. Used to link all hat's sharing a ism_hat.
203  * Same function as the p_mapping list for a page.
204  */
205 typedef struct ism_ment {
206 	sfmmu_t		*iment_hat;	/* back pointer to hat_share() hat */
207 	caddr_t		iment_base_va;	/* hat's va base for this ism seg */
208 	struct ism_ment	*iment_next;	/* next ism map entry */
209 	struct ism_ment	*iment_prev;	/* prev ism map entry */
210 } ism_ment_t;
211 
212 /*
213  * ISM segment block. One will be hung off the sfmmu structure if a
214  * a process uses ISM.  More will be linked using ismblk_next if more
215  * than ISM_MAP_SLOTS segments are attached to this proc.
216  *
217  * All modifications to fields in this structure will be protected
218  * by the hat mutex.  In order to avoid grabbing this lock in low level
219  * routines (tsb miss/protection handlers and vatopfn) while not
220  * introducing any race conditions with hat_unshare, we will set
221  * CTX_ISM_BUSY bit in the ctx struct. Any mmu traps that occur
222  * for this ctx while this bit is set will be handled in sfmmu_tsb_excption
223  * where it will synchronize behind the hat mutex.
224  */
225 typedef struct ism_blk {
226 	ism_map_t		iblk_maps[ISM_MAP_SLOTS];
227 	struct ism_blk		*iblk_next;
228 	uint64_t		iblk_nextpa;
229 } ism_blk_t;
230 
231 /*
232  * TSB access information.  All fields are protected by the process's
233  * hat lock.
234  */
235 
236 struct tsb_info {
237 	caddr_t		tsb_va;		/* tsb base virtual address */
238 	uint64_t	tsb_pa;		/* tsb base physical address */
239 	struct tsb_info	*tsb_next;	/* next tsb used by this process */
240 	uint16_t	tsb_szc;	/* tsb size code */
241 	uint16_t	tsb_flags;	/* flags for this tsb; see below */
242 	uint_t		tsb_ttesz_mask;	/* page size masks; see below */
243 
244 	tte_t		tsb_tte;	/* tte to lock into DTLB */
245 	sfmmu_t		*tsb_sfmmu;	/* sfmmu */
246 	kmem_cache_t	*tsb_cache;	/* cache from which mem allocated */
247 	vmem_t		*tsb_vmp;	/* vmem arena from which mem alloc'd */
248 };
249 
250 /*
251  * Values for "tsb_ttesz_mask" bitmask.
252  */
253 #define	TSB8K	(1 << TTE8K)
254 #define	TSB64K  (1 << TTE64K)
255 #define	TSB512K (1 << TTE512K)
256 #define	TSB4M   (1 << TTE4M)
257 #define	TSB32M  (1 << TTE32M)
258 #define	TSB256M (1 << TTE256M)
259 
260 /*
261  * Values for "tsb_flags" field.
262  */
263 #define	TSB_RELOC_FLAG		0x1
264 #define	TSB_FLUSH_NEEDED	0x2
265 #define	TSB_SWAPPED	0x4
266 
267 /*
268  * Per-MMU context domain kstats.
269  *
270  * TSB Miss Exceptions
271  *	Number of times a TSB miss exception is handled in an MMU. See
272  *	sfmmu_tsbmiss_exception() for more details.
273  * TSB Raise Exception
274  *	Number of times the CPUs within an MMU are cross-called
275  *	to invalidate either a specific process context (when the process
276  *	switches MMU contexts) or the context of any process that is
277  *	running on those CPUs (as part of the MMU context wrap-around).
278  * Wrap Around
279  *	The number of times a wrap-around of MMU context happens.
280  */
281 typedef enum mmu_ctx_stat_types {
282 	MMU_CTX_TSB_EXCEPTIONS,		/* TSB miss exceptions handled */
283 	MMU_CTX_TSB_RAISE_EXCEPTION,	/* ctx invalidation cross calls */
284 	MMU_CTX_WRAP_AROUND,		/* wraparounds */
285 	MMU_CTX_NUM_STATS
286 } mmu_ctx_stat_t;
287 
288 /*
289  * Per-MMU context domain structure. This is instantiated the first time a CPU
290  * belonging to the MMU context domain is configured into the system, at boot
291  * time or at DR time.
292  *
293  * mmu_gnum
294  *	The current generation number for the context IDs on this MMU context
295  *	domain. It is protected by mmu_lock.
296  * mmu_cnum
297  *	The current cnum to be allocated on this MMU context domain. It
298  *	is protected via CAS.
299  * mmu_nctxs
300  *	The max number of context IDs supported on every CPU in this
301  *	MMU context domain. It is 8K except for Rock where it is 64K.
302  *      This is needed here in case the system supports mixed type of
303  *      processors/MMUs. It also helps to make ctx switch code access
304  *      fewer cache lines i.e. no need to retrieve it from some global nctxs.
305  * mmu_lock
306  *	The mutex spin lock used to serialize context ID wrap around
307  * mmu_idx
308  *	The index for this MMU context domain structure in the global array
309  *	mmu_ctxdoms.
310  * mmu_ncpus
311  *	The actual number of CPUs that have been configured in this
312  *	MMU context domain. This also acts as a reference count for the
313  *	structure. When the last CPU in an MMU context domain is unconfigured,
314  *	the structure is freed. It is protected by mmu_lock.
315  * mmu_cpuset
316  *	The CPU set of configured CPUs for this MMU context domain. Used
317  *	to cross-call all the CPUs in the MMU context domain to invalidate
318  *	context IDs during a wraparound operation. It is protected by mmu_lock.
319  */
320 
321 typedef struct mmu_ctx {
322 	uint64_t	mmu_gnum;
323 	uint_t		mmu_cnum;
324 	uint_t		mmu_nctxs;
325 	kmutex_t	mmu_lock;
326 	uint_t		mmu_idx;
327 	uint_t		mmu_ncpus;
328 	cpuset_t	mmu_cpuset;
329 	kstat_t		*mmu_kstat;
330 	kstat_named_t	mmu_kstat_data[MMU_CTX_NUM_STATS];
331 } mmu_ctx_t;
332 
333 #define	mmu_tsb_exceptions	\
334 		mmu_kstat_data[MMU_CTX_TSB_EXCEPTIONS].value.ui64
335 #define	mmu_tsb_raise_exception	\
336 		mmu_kstat_data[MMU_CTX_TSB_RAISE_EXCEPTION].value.ui64
337 #define	mmu_wrap_around		\
338 		mmu_kstat_data[MMU_CTX_WRAP_AROUND].value.ui64
339 
340 extern uint_t		max_mmu_ctxdoms;
341 extern mmu_ctx_t	**mmu_ctxs_tbl;
342 
343 extern void	sfmmu_cpu_init(cpu_t *);
344 extern void	sfmmu_cpu_cleanup(cpu_t *);
345 
346 /*
347  * The following structure is used to get MMU context domain information for
348  * a CPU from the platform.
349  *
350  * mmu_idx
351  *	The MMU context domain index within the global array mmu_ctxs
352  * mmu_nctxs
353  *	The number of context IDs supported in the MMU context domain
354  *	(64K for Rock)
355  */
356 typedef struct mmu_ctx_info {
357 	uint_t		mmu_idx;
358 	uint_t		mmu_nctxs;
359 } mmu_ctx_info_t;
360 
361 #pragma weak plat_cpuid_to_mmu_ctx_info
362 
363 extern void	plat_cpuid_to_mmu_ctx_info(processorid_t, mmu_ctx_info_t *);
364 
365 /*
366  * Each address space has an array of sfmmu_ctx_t structures, one structure
367  * per MMU context domain.
368  *
369  * cnum
370  *	The context ID allocated for an address space on an MMU context domain
371  * gnum
372  *	The generation number for the context ID in the MMU context domain.
373  *
374  * This structure needs to be a power-of-two in size.
375  */
376 typedef struct sfmmu_ctx {
377 	uint64_t	gnum:48;
378 	uint64_t	cnum:16;
379 } sfmmu_ctx_t;
380 
381 
382 /*
383  * The platform dependent hat structure.
384  * tte counts should be protected by cas.
385  * cpuset is protected by cas.
386  *
387  * Note that sfmmu_xhat_provider MUST be the first element.
388  */
389 struct hat {
390 	void		*sfmmu_xhat_provider;	/* NULL for CPU hat */
391 	cpuset_t	sfmmu_cpusran;	/* cpu bit mask for efficient xcalls */
392 	struct	as	*sfmmu_as;	/* as this hat provides mapping for */
393 	ulong_t		sfmmu_ttecnt[MMU_PAGE_SIZES]; /* per sz tte counts */
394 	ulong_t		sfmmu_ismttecnt[MMU_PAGE_SIZES]; /* est. ism ttes */
395 	union _h_un {
396 		ism_blk_t	*sfmmu_iblkp;  /* maps to ismhat(s) */
397 		ism_ment_t	*sfmmu_imentp; /* ism hat's mapping list */
398 	} h_un;
399 	uint_t		sfmmu_free:1;	/* hat to be freed - set on as_free */
400 	uint_t		sfmmu_ismhat:1;	/* hat is dummy ism hatid */
401 	uint_t		sfmmu_ctxflushed:1;	/* ctx has been flushed */
402 	uchar_t		sfmmu_rmstat;	/* refmod stats refcnt */
403 	uchar_t		sfmmu_clrstart;	/* start color bin for page coloring */
404 	ushort_t	sfmmu_clrbin;	/* per as phys page coloring bin */
405 	ushort_t	sfmmu_flags;	/* flags */
406 	struct tsb_info	*sfmmu_tsb;	/* list of per as tsbs */
407 	uint64_t	sfmmu_ismblkpa; /* pa of sfmmu_iblkp, or -1 */
408 	lock_t		sfmmu_ctx_lock;	/* sync ctx alloc and invalidation */
409 	kcondvar_t	sfmmu_tsb_cv;	/* signals TSB swapin or relocation */
410 	uchar_t		sfmmu_cext;	/* context page size encoding */
411 	uint8_t		sfmmu_pgsz[MMU_PAGE_SIZES];  /* ranking for MMU */
412 #ifdef sun4v
413 	struct hv_tsb_block sfmmu_hvblock;
414 #endif
415 	/*
416 	 * sfmmu_ctxs is a variable length array of max_mmu_ctxdoms # of
417 	 * elements. max_mmu_ctxdoms is determined at run-time.
418 	 * sfmmu_ctxs[1] is just the fist element of an array, it always
419 	 * has to be the last field to ensure that the memory allocated
420 	 * for sfmmu_ctxs is consecutive with the memory of the rest of
421 	 * the hat data structure.
422 	 */
423 	sfmmu_ctx_t	sfmmu_ctxs[1];
424 
425 };
426 
427 #define	sfmmu_iblk	h_un.sfmmu_iblkp
428 #define	sfmmu_iment	h_un.sfmmu_imentp
429 
430 /*
431  * bit mask for managing vac conflicts on large pages.
432  * bit 1 is for uncache flag.
433  * bits 2 through min(num of cache colors + 1,31) are
434  * for cache colors that have already been flushed.
435  */
436 #ifdef VAC
437 #define	CACHE_NUM_COLOR		(shm_alignment >> MMU_PAGESHIFT)
438 #else
439 #define	CACHE_NUM_COLOR		1
440 #endif
441 
442 #define	CACHE_VCOLOR_MASK(vcolor)	(2 << (vcolor & (CACHE_NUM_COLOR - 1)))
443 
444 #define	CacheColor_IsFlushed(flag, vcolor) \
445 					((flag) & CACHE_VCOLOR_MASK(vcolor))
446 
447 #define	CacheColor_SetFlushed(flag, vcolor) \
448 					((flag) |= CACHE_VCOLOR_MASK(vcolor))
449 /*
450  * Flags passed to sfmmu_page_cache to flush page from vac or not.
451  */
452 #define	CACHE_FLUSH	0
453 #define	CACHE_NO_FLUSH	1
454 
455 /*
456  * Flags passed to sfmmu_tlbcache_demap
457  */
458 #define	FLUSH_NECESSARY_CPUS	0
459 #define	FLUSH_ALL_CPUS		1
460 
461 #ifdef	DEBUG
462 /*
463  * For debugging purpose only. Maybe removed later.
464  */
465 struct ctx_trace {
466 	sfmmu_t		*sc_sfmmu_stolen;
467 	sfmmu_t		*sc_sfmmu_stealing;
468 	clock_t		sc_time;
469 	ushort_t	sc_type;
470 	ushort_t	sc_cnum;
471 };
472 #define	CTX_TRC_STEAL	0x1
473 #define	CTX_TRC_FREE	0x0
474 #define	TRSIZE	0x400
475 #define	NEXT_CTXTR(ptr)	(((ptr) >= ctx_trace_last) ? \
476 		ctx_trace_first : ((ptr) + 1))
477 #define	TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type) \
478 	mutex_enter(mutex);						\
479 	(ptr)->sc_sfmmu_stolen = (stolen_sfmmu);			\
480 	(ptr)->sc_sfmmu_stealing = (stealing_sfmmu);			\
481 	(ptr)->sc_cnum = (cnum);					\
482 	(ptr)->sc_type = (type);					\
483 	(ptr)->sc_time = lbolt;						\
484 	(ptr) = NEXT_CTXTR(ptr);					\
485 	num_ctx_stolen += (type);					\
486 	mutex_exit(mutex);
487 #else
488 
489 #define	TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type)
490 
491 #endif	/* DEBUG */
492 
493 #endif	/* !_ASM */
494 
495 /*
496  * Macros for sfmmup->sfmmu_flags access.  The macros that change the flags
497  * ASSERT() that we're holding the HAT lock before changing the flags;
498  * however callers that read the flags may do so without acquiring the lock
499  * in a fast path, and then recheck the flag after acquiring the lock in
500  * a slow path.
501  */
502 #define	SFMMU_FLAGS_ISSET(sfmmup, flags) \
503 	(((sfmmup)->sfmmu_flags & (flags)) == (flags))
504 
505 #define	SFMMU_FLAGS_CLEAR(sfmmup, flags) \
506 	(ASSERT(sfmmu_hat_lock_held((sfmmup))), \
507 	(sfmmup)->sfmmu_flags &= ~(flags))
508 
509 #define	SFMMU_FLAGS_SET(sfmmup, flags) \
510 	(ASSERT(sfmmu_hat_lock_held((sfmmup))), \
511 	(sfmmup)->sfmmu_flags |= (flags))
512 
513 /*
514  * sfmmu HAT flags
515  */
516 #define	HAT_64K_FLAG	0x01
517 #define	HAT_512K_FLAG	0x02
518 #define	HAT_4M_FLAG	0x04
519 #define	HAT_32M_FLAG	0x08
520 #define	HAT_256M_FLAG	0x10
521 #define	HAT_4MTEXT_FLAG	0x80
522 #define	HAT_SWAPPED	0x100	/* swapped out */
523 #define	HAT_SWAPIN	0x200	/* swapping in */
524 #define	HAT_BUSY	0x400	/* replacing TSB(s) */
525 #define	HAT_ISMBUSY	0x800	/* adding/removing/traversing ISM maps */
526 
527 #define	HAT_LGPG_FLAGS						\
528 	(HAT_64K_FLAG | HAT_512K_FLAG | HAT_4M_FLAG |		\
529 	    HAT_32M_FLAG | HAT_256M_FLAG)
530 
531 #define	HAT_FLAGS_MASK						\
532 	(HAT_LGPG_FLAGS | HAT_4MTEXT_FLAG | HAT_SWAPPED |	\
533 	    HAT_SWAPIN | HAT_BUSY | HAT_ISMBUSY)
534 
535 /*
536  * Context flags
537  */
538 #define	CTX_FREE_FLAG		0x1
539 #define	CTX_FLAGS_MASK		0x1
540 
541 #define	CTX_SET_FLAGS(ctx, flag)					\
542 {									\
543 	uint32_t old, new;						\
544 									\
545 	do {								\
546 		new = old = (ctx)->ctx_flags;				\
547 		new &= CTX_FLAGS_MASK;					\
548 		new |= flag;						\
549 		new = cas32(&(ctx)->ctx_flags, old, new);		\
550 	} while (new != old);						\
551 }
552 
553 #define	CTX_CLEAR_FLAGS(ctx, flag)					\
554 {									\
555 	uint32_t old, new;						\
556 									\
557 	do {								\
558 		new = old = (ctx)->ctx_flags;				\
559 		new &= CTX_FLAGS_MASK & ~(flag);			\
560 		new = cas32(&(ctx)->ctx_flags, old, new);		\
561 	} while (new != old);						\
562 }
563 
564 #define	ctxtoctxnum(ctx)	((ushort_t)((ctx) - ctxs))
565 
566 /*
567  * Defines needed for ctx stealing.
568  */
569 #define	GET_CTX_RETRY_CNT	100
570 
571 /*
572  * Starting with context 0, the first NUM_LOCKED_CTXS contexts
573  * are locked so that sfmmu_getctx can't steal any of these
574  * contexts.  At the time this software was being developed, the
575  * only context that needs to be locked is context 0 (the kernel
576  * context), and context 1 (reserved for stolen context). So this constant
577  * was originally defined to be 2.
578  */
579 #define	NUM_LOCKED_CTXS 2
580 #define	INVALID_CONTEXT	1
581 
582 #ifndef	_ASM
583 
584 /*
585  * Kernel page relocation stuff.
586  */
587 struct sfmmu_callback {
588 	int key;
589 	int (*prehandler)(caddr_t, uint_t, uint_t, void *);
590 	int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t);
591 	int (*errhandler)(caddr_t, uint_t, uint_t, void *);
592 	int capture_cpus;
593 };
594 
595 extern int sfmmu_max_cb_id;
596 extern struct sfmmu_callback *sfmmu_cb_table;
597 
598 extern int hat_kpr_enabled;
599 
600 struct pa_hment;
601 
602 /*
603  * RFE: With multihat gone we gain back an int.  We could use this to
604  * keep ref bits on a per cpu basis to eliminate xcalls.
605  */
606 struct sf_hment {
607 	tte_t hme_tte;			/* tte for this hment */
608 
609 	union {
610 		struct page *page;	/* what page this maps */
611 		struct pa_hment *data;	/* pa_hment */
612 	} sf_hment_un;
613 
614 	struct	sf_hment *hme_next;	/* next hment */
615 	struct	sf_hment *hme_prev;	/* prev hment */
616 };
617 
618 struct pa_hment {
619 	caddr_t		addr;		/* va */
620 	uint_t		len;		/* bytes */
621 	ushort_t	flags;		/* internal flags */
622 	ushort_t	refcnt;		/* reference count */
623 	id_t		cb_id;		/* callback id, table index */
624 	void		*pvt;		/* handler's private data */
625 	struct sf_hment	sfment;		/* corresponding dummy sf_hment */
626 };
627 
628 #define	hme_page		sf_hment_un.page
629 #define	hme_data		sf_hment_un.data
630 #define	hme_size(sfhmep)	((int)(TTE_CSZ(&(sfhmep)->hme_tte)))
631 #define	PAHME_SZ		(sizeof (struct pa_hment))
632 #define	SFHME_SZ		(sizeof (struct sf_hment))
633 
634 #define	IS_PAHME(hme)	((hme)->hme_tte.ll == 0)
635 
636 /*
637  * hmeblk_tag structure
638  * structure used to obtain a match on a hme_blk.  Currently consists of
639  * the address of the sfmmu struct (or hatid), the base page address of the
640  * hme_blk, and the rehash count.  The rehash count is actually only 2 bits
641  * and has the following meaning:
642  * 1 = 8k or 64k hash sequence.
643  * 2 = 512k hash sequence.
644  * 3 = 4M hash sequence.
645  * We require this count because we don't want to get a false hit on a 512K or
646  * 4M rehash with a base address corresponding to a 8k or 64k hmeblk.
647  * Note:  The ordering and size of the hmeblk_tag members are implictly known
648  * by the tsb miss handlers written in assembly.  Do not change this structure
649  * without checking those routines.  See HTAG_SFMMUPSZ define.
650  */
651 
652 typedef union {
653 	struct {
654 		uint64_t	hblk_basepg: 51, /* hme_blk base pg # */
655 				hblk_rehash: 13; /* rehash number */
656 		sfmmu_t		*sfmmup;
657 	} hblk_tag_un;
658 	uint64_t		htag_tag[2];
659 } hmeblk_tag;
660 
661 #define	htag_id		hblk_tag_un.sfmmup
662 #define	htag_bspage	hblk_tag_un.hblk_basepg
663 #define	htag_rehash	hblk_tag_un.hblk_rehash
664 
665 #define	HTAGS_EQ(tag1, tag2)	(((tag1.htag_tag[0] ^ tag2.htag_tag[0]) | \
666 				(tag1.htag_tag[1] ^ tag2.htag_tag[1])) == 0)
667 #define	HME_REHASH(sfmmup)						\
668 	((sfmmup)->sfmmu_ttecnt[TTE512K] != 0 ||			\
669 	(sfmmup)->sfmmu_ttecnt[TTE4M] != 0 ||				\
670 	(sfmmup)->sfmmu_ttecnt[TTE32M] != 0 ||				\
671 	(sfmmup)->sfmmu_ttecnt[TTE256M] != 0)
672 
673 #endif /* !_ASM */
674 
675 #define	NHMENTS		8		/* # of hments in an 8k hme_blk */
676 					/* needs to be multiple of 2 */
677 #ifndef	_ASM
678 
679 #ifdef	HBLK_TRACE
680 
681 #define	HBLK_LOCK		1
682 #define	HBLK_UNLOCK		0
683 #define	HBLK_STACK_DEPTH	6
684 #define	HBLK_AUDIT_CACHE_SIZE	16
685 #define	HBLK_LOCK_PATTERN	0xaaaaaaaa
686 #define	HBLK_UNLOCK_PATTERN	0xbbbbbbbb
687 
688 struct hblk_lockcnt_audit {
689 	int		flag;		/* lock or unlock */
690 	kthread_id_t	thread;
691 	int		depth;
692 	pc_t		stack[HBLK_STACK_DEPTH];
693 };
694 
695 #endif	/* HBLK_TRACE */
696 
697 
698 /*
699  * Hment block structure.
700  * The hme_blk is the node data structure which the hash structure
701  * mantains. An hme_blk can have 2 different sizes depending on the
702  * number of hments it implicitly contains.  When dealing with 64K, 512K,
703  * or 4M hments there is one hment per hme_blk.  When dealing with
704  * 8k hments we allocate an hme_blk plus an additional 7 hments to
705  * give us a total of 8 (NHMENTS) hments that can be referenced through a
706  * hme_blk.
707  *
708  * The hmeblk structure contains 2 tte reference counters used to determine if
709  * it is ok to free up the hmeblk.  Both counters have to be zero in order
710  * to be able to free up hmeblk.  They are protected by cas.
711  * hblk_hmecnt is the number of hments present on pp mapping lists.
712  * hblk_vcnt reflects number of valid ttes in hmeblk.
713  *
714  * The hmeblk now also has per tte lock cnts.  This is required because
715  * the counts can be high and there are not enough bits in the tte. When
716  * physio is fixed to not lock the translations we should be able to move
717  * the lock cnt back to the tte.  See bug id 1198554.
718  *
719  * Note that xhat_hme_blk's layout follows this structure: hme_blk_misc
720  * and sf_hment are at the same offsets in both structures. Whenever
721  * hme_blk is changed, xhat_hme_blk may need to be updated as well.
722  */
723 
724 struct hme_blk_misc {
725 	ushort_t locked_cnt;	/* HAT_LOAD_LOCK ref cnt */
726 	uint_t	notused:10;
727 	uint_t	xhat_bit:1;	/* set for an xhat hme_blk */
728 	uint_t	shadow_bit:1;	/* set for a shadow hme_blk */
729 	uint_t	nucleus_bit:1;	/* set for a nucleus hme_blk */
730 	uint_t	ttesize:3;	/* contains ttesz of hmeblk */
731 };
732 
733 struct hme_blk {
734 	uint64_t	hblk_nextpa;	/* physical address for hash list */
735 
736 	hmeblk_tag	hblk_tag;	/* tag used to obtain an hmeblk match */
737 
738 	struct hme_blk	*hblk_next;	/* on free list or on hash list */
739 					/* protected by hash lock */
740 
741 	struct hme_blk	*hblk_shadow;	/* pts to shadow hblk */
742 					/* protected by hash lock */
743 	uint_t		hblk_span;	/* span of memory hmeblk maps */
744 
745 	struct hme_blk_misc	hblk_misc;
746 
747 	union {
748 		struct {
749 			ushort_t hblk_hmecount;	/* hment on mlists counter */
750 			ushort_t hblk_validcnt;	/* valid tte reference count */
751 		} hblk_counts;
752 		uint_t		hblk_shadow_mask;
753 	} hblk_un;
754 
755 #ifdef	HBLK_TRACE
756 	kmutex_t	hblk_audit_lock;	/* lock to protect index */
757 	uint_t		hblk_audit_index;	/* index into audit_cache */
758 	struct	hblk_lockcnt_audit hblk_audit_cache[HBLK_AUDIT_CACHE_SIZE];
759 #endif	/* HBLK_AUDIT */
760 
761 	struct sf_hment hblk_hme[1];	/* hment array */
762 };
763 
764 #define	hblk_lckcnt	hblk_misc.locked_cnt
765 #define	hblk_xhat_bit   hblk_misc.xhat_bit
766 #define	hblk_shw_bit	hblk_misc.shadow_bit
767 #define	hblk_nuc_bit	hblk_misc.nucleus_bit
768 #define	hblk_ttesz	hblk_misc.ttesize
769 #define	hblk_hmecnt	hblk_un.hblk_counts.hblk_hmecount
770 #define	hblk_vcnt	hblk_un.hblk_counts.hblk_validcnt
771 #define	hblk_shw_mask	hblk_un.hblk_shadow_mask
772 
773 #define	MAX_HBLK_LCKCNT	0xFFFF
774 #define	HMEBLK_ALIGN	0x8		/* hmeblk has to be double aligned */
775 
776 #ifdef	HBLK_TRACE
777 
778 #define	HBLK_STACK_TRACE(hmeblkp, lock)					\
779 {									\
780 	int flag = lock;	/* to pacify lint */			\
781 	int audit_index;						\
782 									\
783 	mutex_enter(&hmeblkp->hblk_audit_lock);				\
784 	audit_index = hmeblkp->hblk_audit_index;			\
785 	hmeblkp->hblk_audit_index = ((hmeblkp->hblk_audit_index + 1) &	\
786 	    (HBLK_AUDIT_CACHE_SIZE - 1));				\
787 	mutex_exit(&hmeblkp->hblk_audit_lock);				\
788 									\
789 	if (flag)							\
790 		hmeblkp->hblk_audit_cache[audit_index].flag =		\
791 		    HBLK_LOCK_PATTERN;					\
792 	else								\
793 		hmeblkp->hblk_audit_cache[audit_index].flag =		\
794 		    HBLK_UNLOCK_PATTERN;				\
795 									\
796 	hmeblkp->hblk_audit_cache[audit_index].thread = curthread;	\
797 	hmeblkp->hblk_audit_cache[audit_index].depth =			\
798 	    getpcstack(hmeblkp->hblk_audit_cache[audit_index].stack,	\
799 	    HBLK_STACK_DEPTH);						\
800 }
801 
802 #else
803 
804 #define	HBLK_STACK_TRACE(hmeblkp, lock)
805 
806 #endif	/* HBLK_TRACE */
807 
808 #define	HMEHASH_FACTOR	16	/* used to calc # of buckets in hme hash */
809 
810 /*
811  * A maximum number of user hmeblks is defined in order to place an upper
812  * limit on how much nucleus memory is required and to avoid overflowing the
813  * tsbmiss uhashsz and khashsz data areas. The number below corresponds to
814  * the number of buckets required, for an average hash chain length of 4 on
815  * a 16TB machine.
816  */
817 
818 #define	MAX_UHME_BUCKETS	(0x1 << 30)
819 #define	MAX_KHME_BUCKETS	(0x1 << 30)
820 
821 /*
822  * The minimum number of kernel hash buckets.
823  */
824 #define	MIN_KHME_BUCKETS	0x800
825 
826 /*
827  * The number of hash buckets must be a power of 2. If the initial calculated
828  * value is less than USER_BUCKETS_THRESHOLD we round up to the next greater
829  * power of 2, otherwise we round down to avoid huge over allocations.
830  */
831 #define	USER_BUCKETS_THRESHOLD	(1<<22)
832 
833 #define	MAX_NUCUHME_BUCKETS	0x4000
834 #define	MAX_NUCKHME_BUCKETS	0x2000
835 
836 /*
837  * There are 2 locks in the hmehash bucket.  The hmehash_mutex is
838  * a regular mutex used to make sure operations on a hash link are only
839  * done by one thread.  Any operation which comes into the hat with
840  * a <vaddr, as> will grab the hmehash_mutex.  Normally one would expect
841  * the tsb miss handlers to grab the hash lock to make sure the hash list
842  * is consistent while we traverse it.  Unfortunately this can lead to
843  * deadlocks or recursive mutex enters since it is possible for
844  * someone holding the lock to take a tlb/tsb miss.
845  * To solve this problem we have added the hmehash_listlock.  This lock
846  * is only grabbed by the tsb miss handlers, vatopfn, and while
847  * adding/removing a hmeblk from the hash list. The code is written to
848  * guarantee we won't take a tlb miss while holding this lock.
849  */
850 struct hmehash_bucket {
851 	kmutex_t	hmehash_mutex;
852 	uint64_t	hmeh_nextpa;	/* physical address for hash list */
853 	struct hme_blk *hmeblkp;
854 	uint_t		hmeh_listlock;
855 };
856 
857 #endif /* !_ASM */
858 
859 /* Proc Count Project */
860 #define	SFMMU_PGCNT_MASK	0x3f
861 #define	SFMMU_PGCNT_SHIFT	6
862 #define	INVALID_MMU_ID		-1
863 #define	SFMMU_MMU_GNUM_RSHIFT	16
864 #define	SFMMU_MMU_CNUM_LSHIFT	(64 - SFMMU_MMU_GNUM_RSHIFT)
865 #define	MAX_SFMMU_CTX_VAL	((1 << 16) - 1) /* for sanity check */
866 #define	MAX_SFMMU_GNUM_VAL	((0x1UL << 48) - 1)
867 
868 /*
869  * The tsb miss handlers written in assembly know that sfmmup
870  * is a 64 bit ptr.
871  *
872  * The bspage and re-hash part is 64 bits, with the sfmmup being another 64
873  * bits.
874  */
875 #define	HTAG_SFMMUPSZ		0	/* Not really used for LP64 */
876 #define	HTAG_REHASHSZ		13
877 
878 /*
879  * Assembly routines need to be able to get to ttesz
880  */
881 #define	HBLK_SZMASK		0x7
882 
883 #ifndef _ASM
884 
885 /*
886  * Returns the number of bytes that an hmeblk spans given its tte size
887  */
888 #define	get_hblk_span(hmeblkp) ((hmeblkp)->hblk_span)
889 #define	get_hblk_ttesz(hmeblkp)	((hmeblkp)->hblk_ttesz)
890 #define	get_hblk_cache(hmeblkp)	(((hmeblkp)->hblk_ttesz == TTE8K) ? \
891 	sfmmu8_cache : sfmmu1_cache)
892 #define	HMEBLK_SPAN(ttesz)						\
893 	((ttesz == TTE8K)? (TTEBYTES(ttesz) * NHMENTS) : TTEBYTES(ttesz))
894 
895 #define	set_hblk_sz(hmeblkp, ttesz)				\
896 	(hmeblkp)->hblk_ttesz = (ttesz);			\
897 	(hmeblkp)->hblk_span = HMEBLK_SPAN(ttesz)
898 
899 #define	get_hblk_base(hmeblkp)					\
900 	((uintptr_t)(hmeblkp)->hblk_tag.htag_bspage << MMU_PAGESHIFT)
901 
902 #define	get_hblk_endaddr(hmeblkp)				\
903 	((caddr_t)(get_hblk_base(hmeblkp) + get_hblk_span(hmeblkp)))
904 
905 #define	in_hblk_range(hmeblkp, vaddr)					\
906 	(((uintptr_t)(vaddr) >= get_hblk_base(hmeblkp)) &&		\
907 	((uintptr_t)(vaddr) < (get_hblk_base(hmeblkp) +			\
908 	get_hblk_span(hmeblkp))))
909 
910 #define	tte_to_vaddr(hmeblkp, tte)	((caddr_t)(get_hblk_base(hmeblkp) \
911 	+ (TTEBYTES(TTE_CSZ(&tte)) * (tte).tte_hmenum)))
912 
913 #define	vaddr_to_vshift(hblktag, vaddr, shwsz)				\
914 	((((uintptr_t)(vaddr) >> MMU_PAGESHIFT) - (hblktag.htag_bspage)) >>\
915 	TTE_BSZS_SHIFT((shwsz) - 1))
916 
917 #define	HME8BLK_SZ	(sizeof (struct hme_blk) + \
918 			(NHMENTS - 1) * sizeof (struct sf_hment))
919 #define	HME1BLK_SZ	(sizeof (struct hme_blk))
920 #define	H8TOH1		(MMU_PAGESIZE4M / MMU_PAGESIZE)
921 #define	H1MIN		(2 + MAX_BIGKTSB_TTES)	/* nucleus text+data, ktsb */
922 
923 /*
924  * Hme_blk hash structure
925  * Active mappings are kept in a hash structure of hme_blks.  The hash
926  * function is based on (ctx, vaddr) The size of the hash table size is a
927  * power of 2 such that the average hash chain lenth is HMENT_HASHAVELEN.
928  * The hash actually consists of 2 separate hashes.  One hash is for the user
929  * address space and the other hash is for the kernel address space.
930  * The number of buckets are calculated at boot time and stored in the global
931  * variables "uhmehash_num" and "khmehash_num".  By making the hash table size
932  * a power of 2 we can use a simply & function to derive an index instead of
933  * a divide.
934  *
935  * HME_HASH_FUNCTION(hatid, vaddr, shift) returns a pointer to a hme_hash
936  * bucket.
937  * An hme hash bucket contains a pointer to an hme_blk and the mutex that
938  * protects the link list.
939  * Spitfire supports 4 page sizes.  8k and 64K pages only need one hash.
940  * 512K pages need 2 hashes and 4M pages need 3 hashes.
941  * The 'shift' parameter controls how many bits the vaddr will be shifted in
942  * the hash function. It is calculated in the HME_HASH_SHIFT(ttesz) function
943  * and it varies depending on the page size as follows:
944  *	8k pages:  	HBLK_RANGE_SHIFT
945  *	64k pages:	MMU_PAGESHIFT64K
946  *	512K pages:	MMU_PAGESHIFT512K
947  *	4M pages:	MMU_PAGESHIFT4M
948  * An assembly version of the hash function exists in sfmmu_ktsb_miss(). All
949  * changes should be reflected in both versions.  This function and the TSB
950  * miss handlers are the only places which know about the two hashes.
951  *
952  * HBLK_RANGE_SHIFT controls range of virtual addresses that will fall
953  * into the same bucket for a particular process.  It is currently set to
954  * be equivalent to 64K range or one hme_blk.
955  *
956  * The hme_blks in the hash are protected by a per hash bucket mutex
957  * known as SFMMU_HASH_LOCK.
958  * You need to acquire this lock before traversing the hash bucket link
959  * list, while adding/removing a hme_blk to the list, and while
960  * modifying an hme_blk.  A possible optimization is to replace these
961  * mutexes by readers/writer lock but right now it is not clear whether
962  * this is a win or not.
963  *
964  * The HME_HASH_TABLE_SEARCH will search the hash table for the
965  * hme_blk that contains the hment that corresponds to the passed
966  * ctx and vaddr.  It assumed the SFMMU_HASH_LOCK is held.
967  */
968 
969 #endif /* ! _ASM */
970 
971 #define	KHATID			ksfmmup
972 #define	UHMEHASH_SZ		uhmehash_num
973 #define	KHMEHASH_SZ		khmehash_num
974 #define	HMENT_HASHAVELEN	4
975 #define	HBLK_RANGE_SHIFT	MMU_PAGESHIFT64K /* shift for HBLK_BS_MASK */
976 #define	MAX_HASHCNT		5
977 #define	DEFAULT_MAX_HASHCNT	3
978 
979 #ifndef _ASM
980 
981 #define	HASHADDR_MASK(hashno)	TTE_PAGEMASK(hashno)
982 
983 #define	HME_HASH_SHIFT(ttesz)						\
984 	((ttesz == TTE8K)? HBLK_RANGE_SHIFT : TTE_PAGE_SHIFT(ttesz))	\
985 
986 #define	HME_HASH_ADDR(vaddr, hmeshift)					\
987 	((caddr_t)(((uintptr_t)(vaddr) >> (hmeshift)) << (hmeshift)))
988 
989 #define	HME_HASH_BSPAGE(vaddr, hmeshift)				\
990 	(((uintptr_t)(vaddr) >> (hmeshift)) << ((hmeshift) - MMU_PAGESHIFT))
991 
992 #define	HME_HASH_REHASH(ttesz)						\
993 	(((ttesz) < TTE512K)? 1 : (ttesz))
994 
995 #define	HME_HASH_FUNCTION(hatid, vaddr, shift)				\
996 	((hatid != KHATID)?						\
997 	(&uhme_hash[ (((uintptr_t)(hatid) ^	\
998 	    ((uintptr_t)vaddr >> (shift))) & UHMEHASH_SZ) ]):		\
999 	(&khme_hash[ (((uintptr_t)(hatid) ^	\
1000 	    ((uintptr_t)vaddr >> (shift))) & KHMEHASH_SZ) ]))
1001 
1002 /*
1003  * This macro will traverse a hmeblk hash link list looking for an hme_blk
1004  * that owns the specified vaddr and hatid.  If if doesn't find one , hmeblkp
1005  * will be set to NULL, otherwise it will point to the correct hme_blk.
1006  * This macro also cleans empty hblks.
1007  */
1008 #define	HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa,		\
1009 	pr_hblk, prevpa, listp)						\
1010 {									\
1011 	struct hme_blk *nx_hblk;					\
1012 	uint64_t 	nx_pa;						\
1013 									\
1014 	ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));				\
1015 	hblkp = hmebp->hmeblkp;						\
1016 	hblkpa = hmebp->hmeh_nextpa;					\
1017 	prevpa = 0;							\
1018 	pr_hblk = NULL;							\
1019 	while (hblkp) {							\
1020 		if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) {		\
1021 			/* found hme_blk */				\
1022 			break;						\
1023 		}							\
1024 		nx_hblk = hblkp->hblk_next;				\
1025 		nx_pa = hblkp->hblk_nextpa;				\
1026 		if (!hblkp->hblk_vcnt && !hblkp->hblk_hmecnt) {		\
1027 			sfmmu_hblk_hash_rm(hmebp, hblkp, prevpa, pr_hblk); \
1028 			sfmmu_hblk_free(hmebp, hblkp, hblkpa, listp);   \
1029 		} else {						\
1030 			pr_hblk = hblkp;				\
1031 			prevpa = hblkpa;				\
1032 		}							\
1033 		hblkp = nx_hblk;					\
1034 		hblkpa = nx_pa;						\
1035 	}								\
1036 }
1037 
1038 #define	HME_HASH_SEARCH(hmebp, hblktag, hblkp, listp)			\
1039 {									\
1040 	struct hme_blk *pr_hblk;					\
1041 	uint64_t hblkpa, prevpa;					\
1042 									\
1043 	HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa, pr_hblk,	\
1044 		prevpa, listp);						\
1045 }
1046 
1047 /*
1048  * This macro will traverse a hmeblk hash link list looking for an hme_blk
1049  * that owns the specified vaddr and hatid.  If if doesn't find one , hmeblkp
1050  * will be set to NULL, otherwise it will point to the correct hme_blk.
1051  * It doesn't remove empty hblks.
1052  */
1053 #define	HME_HASH_FAST_SEARCH(hmebp, hblktag, hblkp)			\
1054 	ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));				\
1055 	for (hblkp = hmebp->hmeblkp; hblkp;				\
1056 	    hblkp = hblkp->hblk_next) {					\
1057 		if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) {		\
1058 			/* found hme_blk */				\
1059 			break;						\
1060 		}							\
1061 	}								\
1062 
1063 
1064 #define	SFMMU_HASH_LOCK(hmebp)						\
1065 		(mutex_enter(&hmebp->hmehash_mutex))
1066 
1067 #define	SFMMU_HASH_UNLOCK(hmebp)					\
1068 		(mutex_exit(&hmebp->hmehash_mutex))
1069 
1070 #define	SFMMU_HASH_LOCK_TRYENTER(hmebp)					\
1071 		(mutex_tryenter(&hmebp->hmehash_mutex))
1072 
1073 #define	SFMMU_HASH_LOCK_ISHELD(hmebp)					\
1074 		(mutex_owned(&hmebp->hmehash_mutex))
1075 
1076 #define	SFMMU_XCALL_STATS(sfmmup)					\
1077 {									\
1078 	if (sfmmup == ksfmmup) {					\
1079 		SFMMU_STAT(sf_kernel_xcalls);				\
1080 	} else {							\
1081 		SFMMU_STAT(sf_user_xcalls);				\
1082 	}								\
1083 }
1084 
1085 #define	astosfmmu(as)		((as)->a_hat)
1086 #define	hblktosfmmu(hmeblkp)	((sfmmu_t *)(hmeblkp)->hblk_tag.htag_id)
1087 #define	sfmmutoas(sfmmup)	((sfmmup)->sfmmu_as)
1088 /*
1089  * We use the sfmmu data structure to keep the per as page coloring info.
1090  */
1091 #define	as_color_bin(as)	(astosfmmu(as)->sfmmu_clrbin)
1092 #define	as_color_start(as)	(astosfmmu(as)->sfmmu_clrstart)
1093 
1094 typedef struct {
1095 	char	h8[HME8BLK_SZ];
1096 } hblk8_t;
1097 
1098 typedef struct {
1099 	char	h1[HME1BLK_SZ];
1100 } hblk1_t;
1101 
1102 typedef struct {
1103 	ulong_t  	index;
1104 	ulong_t  	len;
1105 	hblk8_t		*list;
1106 } nucleus_hblk8_info_t;
1107 
1108 typedef struct {
1109 	ulong_t		index;
1110 	ulong_t		len;
1111 	hblk1_t		*list;
1112 } nucleus_hblk1_info_t;
1113 
1114 /*
1115  * This struct is used for accumlating information about a range
1116  * of pages that are unloading so that a single xcall can flush
1117  * the entire range from remote tlbs. A function that must demap
1118  * a range of virtual addresses declares one of these structures
1119  * and initializes using DEMP_RANGE_INIT(). It then passes a pointer to this
1120  * struct to the appropriate sfmmu_hblk_* level function which does
1121  * all the bookkeeping using the other macros. When the function has
1122  * finished the virtual address range, it needs to call DEMAP_RANGE_FLUSH()
1123  * macro to take care of any remaining unflushed mappings.
1124  *
1125  * The maximum range this struct can represent is the number of bits
1126  * in the dmr_bitvec field times the pagesize in dmr_pgsz. Currently, only
1127  * MMU_PAGESIZE pages are supported.
1128  *
1129  * Since there are now cases where it's no longer necessary to do
1130  * flushes (e.g. when the process isn't runnable because it's swapping
1131  * out or exiting) we allow these macros to take a NULL dmr input and do
1132  * nothing in that case.
1133  */
1134 typedef struct {
1135 	sfmmu_t		*dmr_sfmmup;	/* relevent hat */
1136 	caddr_t		dmr_addr;	/* beginning address */
1137 	caddr_t		dmr_endaddr;	/* ending  address */
1138 	ulong_t		dmr_bitvec;	/* valid pages found */
1139 	ulong_t		dmr_bit;	/* next page to examine */
1140 	ulong_t		dmr_maxbit;	/* highest page in range */
1141 	ulong_t		dmr_pgsz;	/* page size in range */
1142 } demap_range_t;
1143 
1144 #define	DMR_MAXBIT ((ulong_t)1<<63) /* dmr_bit high bit */
1145 
1146 #define	DEMAP_RANGE_INIT(sfmmup, dmrp) \
1147 	if ((dmrp) != NULL) { \
1148 	(dmrp)->dmr_sfmmup = (sfmmup); \
1149 	(dmrp)->dmr_bitvec = 0; \
1150 	(dmrp)->dmr_maxbit = sfmmu_dmr_maxbit; \
1151 	(dmrp)->dmr_pgsz = MMU_PAGESIZE; \
1152 	}
1153 
1154 #define	DEMAP_RANGE_PGSZ(dmrp) ((dmrp)? (dmrp)->dmr_pgsz : MMU_PAGESIZE)
1155 
1156 #define	DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr) \
1157 	if ((dmrp) != NULL) { \
1158 	if ((dmrp)->dmr_bitvec != 0 && (dmrp)->dmr_endaddr != (addr)) \
1159 		sfmmu_tlb_range_demap(dmrp); \
1160 	(dmrp)->dmr_endaddr = (endaddr); \
1161 	}
1162 
1163 #define	DEMAP_RANGE_FLUSH(dmrp) \
1164 	if ((dmrp) != NULL) { \
1165 		if ((dmrp)->dmr_bitvec != 0) \
1166 			sfmmu_tlb_range_demap(dmrp); \
1167 	}
1168 
1169 #define	DEMAP_RANGE_MARKPG(dmrp, addr) \
1170 	if ((dmrp) != NULL) { \
1171 		if ((dmrp)->dmr_bitvec == 0) { \
1172 			(dmrp)->dmr_addr = (addr); \
1173 			(dmrp)->dmr_bit = 1; \
1174 		} \
1175 		(dmrp)->dmr_bitvec |= (dmrp)->dmr_bit; \
1176 	}
1177 
1178 #define	DEMAP_RANGE_NEXTPG(dmrp) \
1179 	if ((dmrp) != NULL && (dmrp)->dmr_bitvec != 0) { \
1180 		if ((dmrp)->dmr_bit & (dmrp)->dmr_maxbit) { \
1181 			sfmmu_tlb_range_demap(dmrp); \
1182 		} else { \
1183 			(dmrp)->dmr_bit <<= 1; \
1184 		} \
1185 	}
1186 
1187 /*
1188  * TSB related structures
1189  *
1190  * The TSB is made up of tte entries.  Both the tag and data are present
1191  * in the TSB.  The TSB locking is managed as follows:
1192  * A software bit in the tsb tag is used to indicate that entry is locked.
1193  * If a cpu servicing a tsb miss reads a locked entry the tag compare will
1194  * fail forcing the cpu to go to the hat hash for the translation.
1195  * The cpu who holds the lock can then modify the data side, and the tag side.
1196  * The last write should be to the word containing the lock bit which will
1197  * clear the lock and allow the tsb entry to be read.  It is assumed that all
1198  * cpus reading the tsb will do so with atomic 128-bit loads.  An atomic 128
1199  * bit load is required to prevent the following from happening:
1200  *
1201  * cpu 0			cpu 1			comments
1202  *
1203  * ldx tag						tag unlocked
1204  *				ldstub lock		set lock
1205  *				stx data
1206  *				stx tag			unlock
1207  * ldx tag						incorrect tte!!!
1208  *
1209  * The software also maintains a bit in the tag to indicate an invalid
1210  * tsb entry.  The purpose of this bit is to allow the tsb invalidate code
1211  * to invalidate a tsb entry with a single cas.  See code for details.
1212  */
1213 
1214 union tsb_tag {
1215 	struct {
1216 		uint32_t	tag_res0:16;	/* reserved - context area */
1217 		uint32_t	tag_inv:1;	/* sw - invalid tsb entry */
1218 		uint32_t	tag_lock:1;	/* sw - locked tsb entry */
1219 		uint32_t	tag_res1:4;	/* reserved */
1220 		uint32_t	tag_va_hi:10;	/* va[63:54] */
1221 		uint32_t	tag_va_lo;	/* va[53:22] */
1222 	} tagbits;
1223 	struct tsb_tagints {
1224 		uint32_t	inthi;
1225 		uint32_t	intlo;
1226 	} tagints;
1227 };
1228 #define	tag_invalid		tagbits.tag_inv
1229 #define	tag_locked		tagbits.tag_lock
1230 #define	tag_vahi		tagbits.tag_va_hi
1231 #define	tag_valo		tagbits.tag_va_lo
1232 #define	tag_inthi		tagints.inthi
1233 #define	tag_intlo		tagints.intlo
1234 
1235 struct tsbe {
1236 	union tsb_tag	tte_tag;
1237 	tte_t		tte_data;
1238 };
1239 
1240 /*
1241  * A per cpu struct is kept that duplicates some info
1242  * used by the tl>0 tsb miss handlers plus it provides
1243  * a scratch area.  Its purpose is to minimize cache misses
1244  * in the tsb miss handler and is 128 bytes (2 e$ lines).
1245  *
1246  * There should be one allocated per cpu in nucleus memory
1247  * and should be aligned on an ecache line boundary.
1248  */
1249 struct tsbmiss {
1250 	sfmmu_t			*ksfmmup;	/* kernel hat id */
1251 	sfmmu_t			*usfmmup;	/* user hat id */
1252 	struct tsbe		*tsbptr;	/* hardware computed ptr */
1253 	struct tsbe		*tsbptr4m;	/* hardware computed ptr */
1254 	uint64_t		ismblkpa;
1255 	struct hmehash_bucket	*khashstart;
1256 	struct hmehash_bucket	*uhashstart;
1257 	uint_t			khashsz;
1258 	uint_t			uhashsz;
1259 	uint16_t 		dcache_line_mask; /* used to flush dcache */
1260 	uint16_t		hat_flags;
1261 	uint32_t		itlb_misses;
1262 	uint32_t		dtlb_misses;
1263 	uint32_t		utsb_misses;
1264 	uint32_t		ktsb_misses;
1265 	uint16_t		uprot_traps;
1266 	uint16_t		kprot_traps;
1267 
1268 	/*
1269 	 * scratch[0] -> TSB_TAGACC
1270 	 * scratch[1] -> TSBMISS_HMEBP
1271 	 * scratch[2] -> TSBMISS_HATID
1272 	 */
1273 	uintptr_t		scratch[3];
1274 	uint8_t			pad[0x10];
1275 };
1276 
1277 /*
1278  * A per cpu struct is kept for the use within the tl>0 kpm tsb
1279  * miss handler. Some members are duplicates of common data or
1280  * the physical addresses of common data. A few members are also
1281  * written by the tl>0 kpm tsb miss handler. Its purpose is to
1282  * minimize cache misses in the kpm tsb miss handler and occupies
1283  * one ecache line. There should be one allocated per cpu in
1284  * nucleus memory and it should be aligned on an ecache line
1285  * boundary. It is not merged w/ struct tsbmiss since there is
1286  * not much to share and the tsbmiss pathes are different, so
1287  * a kpm tlbmiss/tsbmiss only touches one cacheline, except for
1288  * (DEBUG || SFMMU_STAT_GATHER) where the dtlb_misses counter
1289  * of struct tsbmiss is used on every dtlb miss.
1290  */
1291 struct kpmtsbm {
1292 	caddr_t		vbase;		/* start of address kpm range */
1293 	caddr_t		vend;		/* end of address kpm range */
1294 	uchar_t		flags;		/* flags needed in TL tsbmiss handler */
1295 	uchar_t		sz_shift;	/* for single kpm window */
1296 	uchar_t		kpmp_shift;	/* hash lock shift */
1297 	uchar_t		kpmp2pshft;	/* kpm page to page shift */
1298 	uint_t		kpmp_table_sz;	/* size of kpmp_table or kpmp_stable */
1299 	uint64_t	kpmp_tablepa;	/* paddr of kpmp_table or kpmp_stable */
1300 	uint64_t	msegphashpa;	/* paddr of memseg_phash */
1301 	struct tsbe	*tsbptr;	/* saved ktsb pointer */
1302 	uint_t		kpm_dtlb_misses; /* kpm tlbmiss counter */
1303 	uint_t		kpm_tsb_misses;	/* kpm tsbmiss counter */
1304 	uintptr_t	pad[1];
1305 };
1306 
1307 extern uint_t  tsb_slab_size;
1308 extern uint_t  tsb_slab_shift;
1309 extern uint_t  tsb_slab_ttesz;
1310 extern uint_t  tsb_slab_pamask;
1311 
1312 #endif /* !_ASM */
1313 
1314 /*
1315  * Flags for TL kpm tsbmiss handler
1316  */
1317 #define	KPMTSBM_ENABLE_FLAG	0x01	/* bit copy of kpm_enable */
1318 #define	KPMTSBM_TLTSBM_FLAG	0x02	/* use TL tsbmiss handler */
1319 #define	KPMTSBM_TSBPHYS_FLAG	0x04	/* use ASI_MEM for TSB update */
1320 
1321 /*
1322  * The TSB
1323  * All TSB sizes supported by the hardware are now supported (8K - 1M).
1324  * For kernel TSBs we may go beyond the hardware supported sizes and support
1325  * larger TSBs via software.
1326  * All TTE sizes are supported in the TSB; the manner in which this is
1327  * done is cpu dependent.
1328  */
1329 #define	TSB_MIN_SZCODE		TSB_8K_SZCODE	/* min. supported TSB size */
1330 #define	TSB_MIN_OFFSET_MASK	(TSB_OFFSET_MASK(TSB_MIN_SZCODE))
1331 
1332 #define	UTSB_MAX_SZCODE		TSB_1M_SZCODE /* max. supported TSB size */
1333 #define	UTSB_MAX_OFFSET_MASK	(TSB_OFFSET_MASK(UTSB_MAX_SZCODE))
1334 
1335 #define	TSB_FREEMEM_MIN		0x1000		/* 32 mb */
1336 #define	TSB_FREEMEM_LARGE	0x10000		/* 512 mb */
1337 #define	TSB_8K_SZCODE		0		/* 512 entries */
1338 #define	TSB_16K_SZCODE		1		/* 1k entries */
1339 #define	TSB_32K_SZCODE		2		/* 2k entries */
1340 #define	TSB_64K_SZCODE		3		/* 4k entries */
1341 #define	TSB_128K_SZCODE		4		/* 8k entries */
1342 #define	TSB_256K_SZCODE		5		/* 16k entries */
1343 #define	TSB_512K_SZCODE		6		/* 32k entries */
1344 #define	TSB_1M_SZCODE		7		/* 64k entries */
1345 #define	TSB_2M_SZCODE		8		/* 128k entries */
1346 #define	TSB_4M_SZCODE		9		/* 256k entries */
1347 #define	TSB_ENTRY_SHIFT		4	/* each entry = 128 bits = 16 bytes */
1348 #define	TSB_ENTRY_SIZE		(1 << 4)
1349 #define	TSB_START_SIZE		9
1350 #define	TSB_ENTRIES(tsbsz)	(1 << (TSB_START_SIZE + tsbsz))
1351 #define	TSB_BYTES(tsbsz)	(TSB_ENTRIES(tsbsz) << TSB_ENTRY_SHIFT)
1352 #define	TSB_OFFSET_MASK(tsbsz)	(TSB_ENTRIES(tsbsz) - 1)
1353 #define	TSB_BASEADDR_MASK	((1 << 12) - 1)
1354 
1355 /*
1356  * sun4u platforms
1357  * ---------------
1358  * We now support two user TSBs with one TSB base register.
1359  * Hence the TSB base register is split up as follows:
1360  *
1361  * When only one TSB present:
1362  *   [63  62..42  41..13  12..4  3..0]
1363  *     ^   ^       ^       ^     ^
1364  *     |   |       |       |     |
1365  *     |   |       |       |     |_ TSB size code
1366  *     |   |       |       |
1367  *     |   |       |       |_ Reserved 0
1368  *     |   |       |
1369  *     |   |       |_ TSB VA[41..13]
1370  *     |   |
1371  *     |   |_ VA hole (Spitfire), zeros (Cheetah and beyond)
1372  *     |
1373  *     |_ 0
1374  *
1375  * When second TSB present:
1376  *   [63  62..42  41..33  32..29  28..22  21..13  12..4  3..0]
1377  *     ^   ^       ^       ^       ^       ^       ^     ^
1378  *     |   |       |       |       |       |       |     |
1379  *     |   |       |       |       |       |       |     |_ First TSB size code
1380  *     |   |       |       |       |       |       |
1381  *     |   |       |       |       |       |       |_ Reserved 0
1382  *     |   |       |       |       |       |
1383  *     |   |       |       |       |       |_ First TSB's VA[21..13]
1384  *     |   |       |       |       |
1385  *     |   |       |       |       |_ Reserved for future use
1386  *     |   |       |       |
1387  *     |   |       |       |_ Second TSB's size code
1388  *     |   |       |
1389  *     |   |       |_ Second TSB's VA[21..13]
1390  *     |   |
1391  *     |   |_ VA hole (Spitfire) / ones (Cheetah and beyond)
1392  *     |
1393  *     |_ 1
1394  *
1395  * Note that since we store 21..13 of each TSB's VA, TSBs and their slabs
1396  * may be up to 4M in size.  For now, only hardware supported TSB sizes
1397  * are supported, though the slabs are usually 4M in size.
1398  *
1399  * sun4u platforms that define UTSB_PHYS use physical addressing to access
1400  * the user TSBs at TL>0.  The first user TSB base is in the MMU I/D TSB Base
1401  * registers.  The second TSB base uses a dedicated scratchpad register which
1402  * requires a definition of SCRATCHPAD_UTSBREG in mach_sfmmu.h.  The layout for
1403  * both registers is equivalent to sun4v below, except the TSB PA range is
1404  * [46..13] for sun4u.
1405  *
1406  * sun4v platforms
1407  * ---------------
1408  * On sun4v platforms, we use two dedicated scratchpad registers as pseudo
1409  * hardware TSB base registers to hold up to two different user TSBs.
1410  *
1411  * Each register contains TSB's physical base and size code information
1412  * as follows:
1413  *
1414  *   [63..56  55..13  12..4  3..0]
1415  *      ^       ^       ^     ^
1416  *      |       |       |     |
1417  *      |       |       |     |_ TSB size code
1418  *      |       |       |
1419  *      |       |       |_ Reserved 0
1420  *      |       |
1421  *      |       |_ TSB PA[55..13]
1422  *      |
1423  *      |
1424  *      |
1425  *      |_ 0 for valid TSB
1426  *
1427  * Absence of a user TSB (primarily the second user TSB) is indicated by
1428  * storing a negative value in the TSB base register. This allows us to
1429  * check for presence of a user TSB by simply checking bit# 63.
1430  */
1431 #define	TSBREG_MSB_SHIFT	32		/* set upper bits */
1432 #define	TSBREG_MSB_CONST	0xfffff800	/* set bits 63..43 */
1433 #define	TSBREG_FIRTSB_SHIFT	42		/* to clear bits 63:22 */
1434 #define	TSBREG_SECTSB_MKSHIFT	20		/* 21:13 --> 41:33 */
1435 #define	TSBREG_SECTSB_LSHIFT	22		/* to clear bits 63:42 */
1436 #define	TSBREG_SECTSB_RSHIFT	(TSBREG_SECTSB_MKSHIFT + TSBREG_SECTSB_LSHIFT)
1437 						/* sectsb va -> bits 21:13 */
1438 						/* after clearing upper bits */
1439 #define	TSBREG_SECSZ_SHIFT	29		/* to get sectsb szc to 3:0 */
1440 #define	TSBREG_VAMASK_SHIFT	13		/* set up VA mask */
1441 
1442 #define	BIGKTSB_SZ_MASK		0xf
1443 #define	TSB_SOFTSZ_MASK		BIGKTSB_SZ_MASK
1444 #define	MIN_BIGKTSB_SZCODE	9	/* 256k entries */
1445 #define	MAX_BIGKTSB_SZCODE	11	/* 1024k entries */
1446 #define	MAX_BIGKTSB_TTES	(TSB_BYTES(MAX_BIGKTSB_SZCODE) / MMU_PAGESIZE4M)
1447 
1448 #define	TAG_VALO_SHIFT		22		/* tag's va are bits 63-22 */
1449 /*
1450  * sw bits used on tsb_tag - bit masks used only in assembly
1451  * use only a sethi for these fields.
1452  */
1453 #define	TSBTAG_INVALID	0x00008000		/* tsb_tag.tag_invalid */
1454 #define	TSBTAG_LOCKED	0x00004000		/* tsb_tag.tag_locked */
1455 
1456 #ifdef	_ASM
1457 
1458 /*
1459  * Marker to indicate that this instruction will be hot patched at runtime
1460  * to some other value.
1461  * This value must be zero since it fills in the imm bits of the target
1462  * instructions to be patched
1463  */
1464 #define	RUNTIME_PATCH	(0)
1465 
1466 /*
1467  * V9 defines nop instruction as the following, which we use
1468  * at runtime to nullify some instructions we don't want to
1469  * execute in the trap handlers on certain platforms.
1470  */
1471 #define	MAKE_NOP_INSTR(reg)	\
1472 	sethi	%hi(0x1000000), reg
1473 
1474 /*
1475  * Macro to get hat per-MMU cnum on this CPU.
1476  * sfmmu - In, pass in "sfmmup" from the caller.
1477  * cnum	- Out, return 'cnum' to the caller
1478  * scr	- scratch
1479  */
1480 #define	SFMMU_CPU_CNUM(sfmmu, cnum, scr)				      \
1481 	CPU_ADDR(scr, cnum);	/* scr = load CPU struct addr */	      \
1482 	ld	[scr + CPU_MMU_IDX], cnum;	/* cnum = mmuid */	      \
1483 	add	sfmmu, SFMMU_CTXS, scr;	/* scr = sfmmup->sfmmu_ctxs[] */      \
1484 	sllx    cnum, SFMMU_MMU_CTX_SHIFT, cnum;			      \
1485 	add	scr, cnum, scr;		/* scr = sfmmup->sfmmu_ctxs[id] */    \
1486 	ldx	[scr + SFMMU_MMU_GC_NUM], scr;	/* sfmmu_ctxs[id].gcnum */    \
1487 	sllx    scr, SFMMU_MMU_CNUM_LSHIFT, scr;			      \
1488 	srlx    scr, SFMMU_MMU_CNUM_LSHIFT, cnum;	/* cnum = sfmmu cnum */
1489 
1490 /*
1491  * Macro to get hat gnum & cnum assocaited with sfmmu_ctx[mmuid] entry
1492  * entry - In,  pass in (&sfmmu_ctxs[mmuid] - SFMMU_CTXS) from the caller.
1493  * gnum - Out, return sfmmu gnum
1494  * cnum - Out, return sfmmu cnum
1495  * reg	- scratch
1496  */
1497 #define	SFMMU_MMUID_GNUM_CNUM(entry, gnum, cnum, reg)			     \
1498 	ldx	[entry + SFMMU_CTXS], reg;  /* reg = sfmmu (gnum | cnum) */  \
1499 	srlx	reg, SFMMU_MMU_GNUM_RSHIFT, gnum;    /* gnum = sfmmu gnum */ \
1500 	sllx	reg, SFMMU_MMU_CNUM_LSHIFT, cnum;			     \
1501 	srlx	cnum, SFMMU_MMU_CNUM_LSHIFT, cnum;   /* cnum = sfmmu cnum */
1502 
1503 /*
1504  * Macro to get this CPU's tsbmiss area.
1505  */
1506 #define	CPU_TSBMISS_AREA(tsbmiss, tmp1)					\
1507 	CPU_INDEX(tmp1, tsbmiss);		/* tmp1 = cpu idx */	\
1508 	sethi	%hi(tsbmiss_area), tsbmiss;	/* tsbmiss base ptr */	\
1509 	sllx    tmp1, TSBMISS_SHIFT, tmp1;	/* byte offset */	\
1510 	or	tsbmiss, %lo(tsbmiss_area), tsbmiss;			\
1511 	add	tsbmiss, tmp1, tsbmiss		/* tsbmiss area of CPU */
1512 
1513 
1514 /*
1515  * Macro to set kernel context + page size codes in DMMU primary context
1516  * register. It is only necessary for sun4u because sun4v does not need
1517  * page size codes
1518  */
1519 #ifdef sun4v
1520 
1521 #define	SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3)
1522 
1523 #else
1524 
1525 #define	SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) \
1526 	sethi	%hi(kcontextreg), reg0;					\
1527 	ldx	[reg0 + %lo(kcontextreg)], reg0;			\
1528 	mov	MMU_PCONTEXT, reg1;					\
1529 	ldxa	[reg1]ASI_MMU_CTX, reg2;				\
1530 	xor	reg0, reg2, reg2;					\
1531 	brz	reg2, label3;						\
1532 	srlx	reg2, CTXREG_NEXT_SHIFT, reg2;				\
1533 	rdpr	%pstate, reg3;		/* disable interrupts */	\
1534 	btst	PSTATE_IE, reg3;					\
1535 /*CSTYLED*/								\
1536 	bnz,a,pt %icc, label1;						\
1537 	wrpr	reg3, PSTATE_IE, %pstate;				\
1538 /*CSTYLED*/								\
1539 label1:;								\
1540 	brz	reg2, label2;	   /* need demap if N_pgsz0/1 change */	\
1541 	sethi	%hi(FLUSH_ADDR), reg4;					\
1542 	mov	DEMAP_ALL_TYPE, reg2;					\
1543 	stxa	%g0, [reg2]ASI_DTLB_DEMAP;				\
1544 	stxa	%g0, [reg2]ASI_ITLB_DEMAP;				\
1545 /*CSTYLED*/								\
1546 label2:;								\
1547 	stxa	reg0, [reg1]ASI_MMU_CTX;				\
1548 	flush	reg4;							\
1549 	btst	PSTATE_IE, reg3;					\
1550 /*CSTYLED*/								\
1551 	bnz,a,pt %icc, label3;						\
1552 	wrpr	%g0, reg3, %pstate;	/* restore interrupt state */	\
1553 label3:;
1554 
1555 #endif
1556 
1557 /*
1558  * Macro to setup arguments with kernel sfmmup context + page size before
1559  * calling sfmmu_setctx_sec()
1560  */
1561 #ifdef sun4v
1562 #define	SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1)			\
1563 	set	KCONTEXT, arg0;					\
1564 	set	0, arg1;
1565 #else
1566 #define	SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1)			\
1567 	ldub	[sfmmup + SFMMU_CEXT], arg1;			\
1568 	set	KCONTEXT, arg0;					\
1569 	sll	arg1, CTXREG_EXT_SHIFT, arg1;
1570 #endif
1571 
1572 #define	PANIC_IF_INTR_DISABLED_PSTR(pstatereg, label, scr)	       	\
1573 	andcc	pstatereg, PSTATE_IE, %g0;	/* panic if intrs */	\
1574 /*CSTYLED*/								\
1575 	bnz,pt	%icc, label;			/* already disabled */	\
1576 	nop;								\
1577 									\
1578 	sethi	%hi(panicstr), scr;					\
1579 	ldx	[scr + %lo(panicstr)], scr;				\
1580 	tst	scr;							\
1581 /*CSTYLED*/								\
1582 	bnz,pt	%xcc, label;						\
1583 	nop;								\
1584 									\
1585 	save	%sp, -SA(MINFRAME), %sp;				\
1586 	sethi	%hi(sfmmu_panic1), %o0;					\
1587 	call	panic;							\
1588 	or	%o0, %lo(sfmmu_panic1), %o0;				\
1589 /*CSTYLED*/								\
1590 label:
1591 
1592 #define	PANIC_IF_INTR_ENABLED_PSTR(label, scr)				\
1593 	/*								\
1594 	 * The caller must have disabled interrupts.			\
1595 	 * If interrupts are not disabled, panic			\
1596 	 */								\
1597 	rdpr	%pstate, scr;						\
1598 	andcc	scr, PSTATE_IE, %g0;					\
1599 /*CSTYLED*/								\
1600 	bz,pt	%icc, label;						\
1601 	nop;								\
1602 									\
1603 	sethi	%hi(panicstr), scr;					\
1604 	ldx	[scr + %lo(panicstr)], scr;				\
1605 	tst	scr;							\
1606 /*CSTYLED*/								\
1607 	bnz,pt	%xcc, label;						\
1608 	nop;								\
1609 									\
1610 	sethi	%hi(sfmmu_panic6), %o0;					\
1611 	call	panic;							\
1612 	or	%o0, %lo(sfmmu_panic6), %o0;				\
1613 /*CSTYLED*/								\
1614 label:
1615 
1616 #endif	/* _ASM */
1617 
1618 #ifndef _ASM
1619 
1620 #ifdef VAC
1621 /*
1622  * Page coloring
1623  * The p_vcolor field of the page struct (1 byte) is used to store the
1624  * virtual page color.  This provides for 255 colors.  The value zero is
1625  * used to mean the page has no color - never been mapped or somehow
1626  * purified.
1627  */
1628 
1629 #define	PP_GET_VCOLOR(pp)	(((pp)->p_vcolor) - 1)
1630 #define	PP_NEWPAGE(pp)		(!(pp)->p_vcolor)
1631 #define	PP_SET_VCOLOR(pp, color)                                          \
1632 	((pp)->p_vcolor = ((color) + 1))
1633 
1634 /*
1635  * As mentioned p_vcolor == 0 means there is no color for this page.
1636  * But PP_SET_VCOLOR(pp, color) expects 'color' to be real color minus
1637  * one so we define this constant.
1638  */
1639 #define	NO_VCOLOR	(-1)
1640 
1641 #define	addr_to_vcolor(addr) \
1642 	(((uint_t)(uintptr_t)(addr) >> MMU_PAGESHIFT) & vac_colors_mask)
1643 #else	/* VAC */
1644 #define	addr_to_vcolor(addr)	(0)
1645 #endif	/* VAC */
1646 
1647 /*
1648  * The field p_index in the psm page structure is for large pages support.
1649  * P_index is a bit-vector of the different mapping sizes that a given page
1650  * is part of. An hme structure for a large mapping is only added in the
1651  * group leader page (first page). All pages covered by a given large mapping
1652  * have the corrosponding mapping bit set in their p_index field. This allows
1653  * us to only store an explicit hme structure in the leading page which
1654  * simplifies the mapping link list management. Furthermore, it provides us
1655  * a fast mechanism for determining the largest mapping a page is part of. For
1656  * exmaple, a page with a 64K and a 4M mappings has a p_index value of 0x0A.
1657  *
1658  * Implementation note: even though the first bit in p_index is reserved
1659  * for 8K mappings, it is NOT USED by the code and SHOULD NOT be set.
1660  * In addition, the upper four bits of the p_index field are used by the
1661  * code as temporaries
1662  */
1663 
1664 /*
1665  * Defines for psm page struct fields and large page support
1666  */
1667 #define	SFMMU_INDEX_SHIFT		6
1668 #define	SFMMU_INDEX_MASK		((1 << SFMMU_INDEX_SHIFT) - 1)
1669 
1670 /* Return the mapping index */
1671 #define	PP_MAPINDEX(pp)	((pp)->p_index & SFMMU_INDEX_MASK)
1672 
1673 /*
1674  * These macros rely on the following property:
1675  * All pages constituting a large page are covered by a virtually
1676  * contiguous set of page_t's.
1677  */
1678 
1679 /* Return the leader for this mapping size */
1680 #define	PP_GROUPLEADER(pp, sz) \
1681 	(&(pp)[-(int)(pp->p_pagenum & (TTEPAGES(sz)-1))])
1682 
1683 /* Return the root page for this page based on p_szc */
1684 #define	PP_PAGEROOT(pp)	((pp)->p_szc == 0 ? (pp) : \
1685 	PP_GROUPLEADER((pp), (pp)->p_szc))
1686 
1687 #define	PP_PAGENEXT_N(pp, n)	((pp) + (n))
1688 #define	PP_PAGENEXT(pp)		PP_PAGENEXT_N((pp), 1)
1689 
1690 #define	PP_PAGEPREV_N(pp, n)	((pp) - (n))
1691 #define	PP_PAGEPREV(pp)		PP_PAGEPREV_N((pp), 1)
1692 
1693 #define	PP_ISMAPPED_LARGE(pp)	(PP_MAPINDEX(pp) != 0)
1694 
1695 /* Need function to test the page mappping which takes p_index into account */
1696 #define	PP_ISMAPPED(pp)	((pp)->p_mapping || PP_ISMAPPED_LARGE(pp))
1697 
1698 /*
1699  * Don't call this macro with sz equal to zero. 8K mappings SHOULD NOT
1700  * set p_index field.
1701  */
1702 #define	PAGESZ_TO_INDEX(sz)	(1 << (sz))
1703 
1704 
1705 /*
1706  * prototypes for hat assembly routines.  Some of these are
1707  * known to machine dependent VM code.
1708  */
1709 extern uint64_t sfmmu_make_tsbtag(caddr_t);
1710 extern struct tsbe *
1711 		sfmmu_get_tsbe(uint64_t, caddr_t, int, int);
1712 extern void	sfmmu_load_tsbe(struct tsbe *, uint64_t, tte_t *, int);
1713 extern void	sfmmu_unload_tsbe(struct tsbe *, uint64_t, int);
1714 extern void	sfmmu_load_mmustate(sfmmu_t *);
1715 extern void	sfmmu_raise_tsb_exception(uint64_t, uint64_t);
1716 #ifndef sun4v
1717 extern void	sfmmu_itlb_ld_kva(caddr_t, tte_t *);
1718 extern void	sfmmu_dtlb_ld_kva(caddr_t, tte_t *);
1719 #endif /* sun4v */
1720 extern void	sfmmu_copytte(tte_t *, tte_t *);
1721 extern int	sfmmu_modifytte(tte_t *, tte_t *, tte_t *);
1722 extern int	sfmmu_modifytte_try(tte_t *, tte_t *, tte_t *);
1723 extern pfn_t	sfmmu_ttetopfn(tte_t *, caddr_t);
1724 extern void	sfmmu_hblk_hash_rm(struct hmehash_bucket *,
1725 			struct hme_blk *, uint64_t, struct hme_blk *);
1726 extern void	sfmmu_hblk_hash_add(struct hmehash_bucket *, struct hme_blk *,
1727 			uint64_t);
1728 extern uint_t	sfmmu_disable_intrs(void);
1729 extern void	sfmmu_enable_intrs(uint_t);
1730 /*
1731  * functions exported to machine dependent VM code
1732  */
1733 extern void	sfmmu_patch_ktsb(void);
1734 #ifndef UTSB_PHYS
1735 extern void	sfmmu_patch_utsb(void);
1736 #endif /* UTSB_PHYS */
1737 extern pfn_t	sfmmu_vatopfn(caddr_t, sfmmu_t *, tte_t *);
1738 extern void	sfmmu_vatopfn_suspended(caddr_t, sfmmu_t *, tte_t *);
1739 #ifdef	DEBUG
1740 extern void	sfmmu_check_kpfn(pfn_t);
1741 #else
1742 #define		sfmmu_check_kpfn(pfn)	/* disabled */
1743 #endif	/* DEBUG */
1744 extern void	sfmmu_memtte(tte_t *, pfn_t, uint_t, int);
1745 extern void	sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *,	uint_t);
1746 extern void	sfmmu_tsbmiss_exception(struct regs *, uintptr_t, uint_t);
1747 extern void	sfmmu_init_tsbs(void);
1748 extern caddr_t  sfmmu_ktsb_alloc(caddr_t);
1749 extern int	sfmmu_getctx_pri(void);
1750 extern int	sfmmu_getctx_sec(void);
1751 extern void	sfmmu_setctx_sec(int);
1752 extern void	sfmmu_inv_tsb(caddr_t, uint_t);
1753 extern void	sfmmu_init_ktsbinfo(void);
1754 extern int	sfmmu_setup_4lp(void);
1755 extern void	sfmmu_patch_mmu_asi(int);
1756 extern void	sfmmu_init_nucleus_hblks(caddr_t, size_t, int, int);
1757 extern void	sfmmu_cache_flushall(void);
1758 extern pgcnt_t  sfmmu_tte_cnt(sfmmu_t *, uint_t);
1759 extern void	*sfmmu_tsb_segkmem_alloc(vmem_t *, size_t, int);
1760 extern void	sfmmu_tsb_segkmem_free(vmem_t *, void *, size_t);
1761 extern void	sfmmu_reprog_pgsz_arr(sfmmu_t *, uint8_t *);
1762 
1763 extern void	hat_kern_setup(void);
1764 extern int	hat_page_relocate(page_t **, page_t **, spgcnt_t *);
1765 extern int	sfmmu_get_ppvcolor(struct page *);
1766 extern int	sfmmu_get_addrvcolor(caddr_t);
1767 extern int	sfmmu_hat_lock_held(sfmmu_t *);
1768 extern void	sfmmu_alloc_ctx(sfmmu_t *, int, struct cpu *);
1769 
1770 /*
1771  * Functions exported to xhat_sfmmu.c
1772  */
1773 extern kmutex_t *sfmmu_mlist_enter(page_t *);
1774 extern void	sfmmu_mlist_exit(kmutex_t *);
1775 extern int	sfmmu_mlist_held(struct page *);
1776 extern struct hme_blk *sfmmu_hmetohblk(struct sf_hment *);
1777 
1778 /*
1779  * MMU-specific functions optionally imported from the CPU module
1780  */
1781 #pragma weak mmu_large_pages_disabled
1782 #pragma weak mmu_set_ctx_page_sizes
1783 #pragma weak mmu_check_page_sizes
1784 
1785 extern int mmu_large_pages_disabled(uint_t);
1786 extern void mmu_set_ctx_page_sizes(sfmmu_t *);
1787 extern void mmu_check_page_sizes(sfmmu_t *, uint64_t *);
1788 
1789 extern sfmmu_t 		*ksfmmup;
1790 extern caddr_t		ktsb_base;
1791 extern uint64_t		ktsb_pbase;
1792 extern int		ktsb_sz;
1793 extern int		ktsb_szcode;
1794 extern caddr_t		ktsb4m_base;
1795 extern uint64_t		ktsb4m_pbase;
1796 extern int		ktsb4m_sz;
1797 extern int		ktsb4m_szcode;
1798 extern uint64_t		kpm_tsbbase;
1799 extern int		kpm_tsbsz;
1800 extern int		ktsb_phys;
1801 extern int		enable_bigktsb;
1802 #ifndef sun4v
1803 extern int		utsb_dtlb_ttenum;
1804 extern int		utsb4m_dtlb_ttenum;
1805 #endif /* sun4v */
1806 extern int		uhmehash_num;
1807 extern int		khmehash_num;
1808 extern struct hmehash_bucket *uhme_hash;
1809 extern struct hmehash_bucket *khme_hash;
1810 extern kmutex_t		*mml_table;
1811 extern uint_t		mml_table_sz;
1812 extern uint_t		mml_shift;
1813 extern uint_t		hblk_alloc_dynamic;
1814 extern struct tsbmiss	tsbmiss_area[NCPU];
1815 extern struct kpmtsbm	kpmtsbm_area[NCPU];
1816 extern int		tsb_max_growsize;
1817 #ifndef sun4v
1818 extern int		dtlb_resv_ttenum;
1819 extern caddr_t		utsb_vabase;
1820 extern caddr_t		utsb4m_vabase;
1821 #endif /* sun4v */
1822 extern vmem_t		*kmem_tsb_default_arena[];
1823 extern int		tsb_lgrp_affinity;
1824 
1825 /* kpm externals */
1826 extern pfn_t		sfmmu_kpm_vatopfn(caddr_t);
1827 extern void		sfmmu_kpm_patch_tlbm(void);
1828 extern void		sfmmu_kpm_patch_tsbm(void);
1829 extern void		sfmmu_kpm_load_tsb(caddr_t, tte_t *, int);
1830 extern void		sfmmu_kpm_unload_tsb(caddr_t, int);
1831 extern void		sfmmu_kpm_tsbmtl(short *, uint_t *, int);
1832 extern int		sfmmu_kpm_stsbmtl(char *, uint_t *, int);
1833 extern caddr_t		kpm_vbase;
1834 extern size_t		kpm_size;
1835 extern struct memseg	*memseg_hash[];
1836 extern uint64_t		memseg_phash[];
1837 extern kpm_hlk_t	*kpmp_table;
1838 extern kpm_shlk_t	*kpmp_stable;
1839 extern uint_t		kpmp_table_sz;
1840 extern uint_t		kpmp_stable_sz;
1841 extern uchar_t		kpmp_shift;
1842 
1843 #define	PP_ISMAPPED_KPM(pp)	((pp)->p_kpmref > 0)
1844 
1845 #define	IS_KPM_ALIAS_RANGE(vaddr)					\
1846 	(((vaddr) - kpm_vbase) >> (uintptr_t)kpm_size_shift > 0)
1847 
1848 #endif /* !_ASM */
1849 
1850 /* sfmmu_kpm_tsbmtl flags */
1851 #define	KPMTSBM_STOP		0
1852 #define	KPMTSBM_START		1
1853 
1854 /* kpm_smallpages kp_mapped values */
1855 #define	KPM_MAPPEDS		-1	/* small mapping valid, no conflict */
1856 #define	KPM_MAPPEDSC		1	/* small mapping valid, conflict */
1857 
1858 /* Physical memseg address NULL marker */
1859 #define	MSEG_NULLPTR_PA		-1
1860 
1861 /*
1862  * Memseg hash defines for kpm trap level tsbmiss handler.
1863  * Must be in sync w/ page.h .
1864  */
1865 #define	SFMMU_MEM_HASH_SHIFT		0x9
1866 #define	SFMMU_N_MEM_SLOTS		0x200
1867 #define	SFMMU_MEM_HASH_ENTRY_SHIFT	3
1868 
1869 #ifndef	_ASM
1870 #if (SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT)
1871 #error SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT
1872 #endif
1873 #if (SFMMU_N_MEM_SLOTS != N_MEM_SLOTS)
1874 #error SFMMU_N_MEM_SLOTS != N_MEM_SLOTS
1875 #endif
1876 
1877 /* Physical memseg address NULL marker */
1878 #define	SFMMU_MEMSEG_NULLPTR_PA		-1
1879 
1880 /*
1881  * Check KCONTEXT to be zero, asm parts depend on that assumption.
1882  */
1883 #if (KCONTEXT != 0)
1884 #error KCONTEXT != 0
1885 #endif
1886 #endif	/* !_ASM */
1887 
1888 
1889 #endif /* _KERNEL */
1890 
1891 #ifndef _ASM
1892 /*
1893  * ctx, hmeblk, mlistlock and other stats for sfmmu
1894  */
1895 struct sfmmu_global_stat {
1896 	int		sf_tsb_exceptions;	/* # of tsb exceptions */
1897 	int		sf_tsb_raise_exception;	/* # tsb exc. w/o TLB flush */
1898 
1899 	int		sf_pagefaults;		/* # of pagefaults */
1900 
1901 	int		sf_uhash_searches;	/* # of user hash searches */
1902 	int		sf_uhash_links;		/* # of user hash links */
1903 	int		sf_khash_searches;	/* # of kernel hash searches */
1904 	int		sf_khash_links;		/* # of kernel hash links */
1905 
1906 	int		sf_swapout;		/* # times hat swapped out */
1907 
1908 	int		sf_tsb_alloc;		/* # TSB allocations */
1909 	int		sf_tsb_allocfail;	/* # times TSB alloc fail */
1910 	int		sf_tsb_sectsb_create;	/* # times second TSB added */
1911 
1912 	int		sf_tteload8k;		/* calls to sfmmu_tteload */
1913 	int		sf_tteload64k;		/* calls to sfmmu_tteload */
1914 	int		sf_tteload512k;		/* calls to sfmmu_tteload */
1915 	int		sf_tteload4m;		/* calls to sfmmu_tteload */
1916 	int		sf_tteload32m;		/* calls to sfmmu_tteload */
1917 	int		sf_tteload256m;		/* calls to sfmmu_tteload */
1918 
1919 	int		sf_tsb_load8k;		/* # times loaded 8K tsbent */
1920 	int		sf_tsb_load4m;		/* # times loaded 4M tsbent */
1921 
1922 	int		sf_hblk_hit;		/* found hblk during tteload */
1923 	int		sf_hblk8_ncreate;	/* static hblk8's created */
1924 	int		sf_hblk8_nalloc;	/* static hblk8's allocated */
1925 	int		sf_hblk1_ncreate;	/* static hblk1's created */
1926 	int		sf_hblk1_nalloc;	/* static hblk1's allocated */
1927 	int		sf_hblk_slab_cnt;	/* sfmmu8_cache slab creates */
1928 	int		sf_hblk_reserve_cnt;	/* hblk_reserve usage */
1929 	int		sf_hblk_recurse_cnt;	/* hblk_reserve	owner reqs */
1930 	int		sf_hblk_reserve_hit;	/* hblk_reserve hash hits */
1931 	int		sf_get_free_success;	/* reserve list allocs */
1932 	int		sf_get_free_throttle;	/* fails due to throttling */
1933 	int		sf_get_free_fail;	/* fails due to empty list */
1934 	int		sf_put_free_success;	/* reserve list frees */
1935 	int		sf_put_free_fail;	/* fails due to full list */
1936 
1937 	int		sf_pgcolor_conflict;	/* VAC conflict resolution */
1938 	int		sf_uncache_conflict;	/* VAC conflict resolution */
1939 	int		sf_unload_conflict;	/* VAC unload resolution */
1940 	int		sf_ism_uncache;		/* VAC conflict resolution */
1941 	int		sf_ism_recache;		/* VAC conflict resolution */
1942 	int		sf_recache;		/* VAC conflict resolution */
1943 
1944 	int		sf_steal_count;		/* # of hblks stolen */
1945 
1946 	int		sf_pagesync;		/* # of pagesyncs */
1947 	int		sf_clrwrt;		/* # of clear write perms */
1948 	int		sf_pagesync_invalid;	/* pagesync with inv tte */
1949 
1950 	int		sf_kernel_xcalls;	/* # of kernel cross calls */
1951 	int		sf_user_xcalls;		/* # of user cross calls */
1952 
1953 	int		sf_tsb_grow;		/* # of user tsb grows */
1954 	int		sf_tsb_shrink;		/* # of user tsb shrinks */
1955 	int		sf_tsb_resize_failures;	/* # of user tsb resize */
1956 	int		sf_tsb_reloc;		/* # of user tsb relocations */
1957 
1958 	int		sf_user_vtop;		/* # of user vatopfn calls */
1959 
1960 	int		sf_ctx_inv;		/* #times invalidate MMU ctx */
1961 
1962 	int		sf_tlb_reprog_pgsz;	/* # times switch TLB pgsz */
1963 };
1964 
1965 struct sfmmu_tsbsize_stat {
1966 	int		sf_tsbsz_8k;
1967 	int		sf_tsbsz_16k;
1968 	int		sf_tsbsz_32k;
1969 	int		sf_tsbsz_64k;
1970 	int		sf_tsbsz_128k;
1971 	int		sf_tsbsz_256k;
1972 	int		sf_tsbsz_512k;
1973 	int		sf_tsbsz_1m;
1974 	int		sf_tsbsz_2m;
1975 	int		sf_tsbsz_4m;
1976 };
1977 
1978 struct sfmmu_percpu_stat {
1979 	int	sf_itlb_misses;		/* # of itlb misses */
1980 	int	sf_dtlb_misses;		/* # of dtlb misses */
1981 	int	sf_utsb_misses;		/* # of user tsb misses */
1982 	int	sf_ktsb_misses;		/* # of kernel tsb misses */
1983 	int	sf_tsb_hits;		/* # of tsb hits */
1984 	int	sf_umod_faults;		/* # of mod (prot viol) flts */
1985 	int	sf_kmod_faults;		/* # of mod (prot viol) flts */
1986 };
1987 
1988 #define	SFMMU_STAT(stat)		sfmmu_global_stat.stat++
1989 #define	SFMMU_STAT_ADD(stat, amount)	sfmmu_global_stat.stat += (amount)
1990 #define	SFMMU_STAT_SET(stat, count)	sfmmu_global_stat.stat = (count)
1991 
1992 #define	SFMMU_MMU_STAT(stat)		CPU->cpu_m.cpu_mmu_ctxp->stat++
1993 
1994 #endif /* !_ASM */
1995 
1996 #ifdef	__cplusplus
1997 }
1998 #endif
1999 
2000 #endif	/* _VM_HAT_SFMMU_H */
2001