xref: /titanic_41/usr/src/uts/sun4/vm/vm_dep.h (revision 15d9d0b528387242011cdcc6190c9e598cfe3a07)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * UNIX machine dependent virtual memory support.
28  */
29 
30 #ifndef	_VM_DEP_H
31 #define	_VM_DEP_H
32 
33 #pragma ident	"%Z%%M%	%I%	%E% SMI"
34 
35 #ifdef	__cplusplus
36 extern "C" {
37 #endif
38 
39 #include <vm/hat_sfmmu.h>
40 #include <sys/archsystm.h>
41 #include <sys/memnode.h>
42 
43 #define	GETTICK()	gettick()
44 
45 /*
46  * Per page size free lists. Allocated dynamically.
47  */
48 #define	MAX_MEM_TYPES	2	/* 0 = reloc, 1 = noreloc */
49 #define	MTYPE_RELOC	0
50 #define	MTYPE_NORELOC	1
51 
52 #define	PP_2_MTYPE(pp)	(PP_ISNORELOC(pp) ? MTYPE_NORELOC : MTYPE_RELOC)
53 
54 #define	MTYPE_INIT(mtype, vp, vaddr, flags, pgsz)			\
55 	mtype = (flags & PG_NORELOC) ? MTYPE_NORELOC : MTYPE_RELOC;
56 
57 /* mtype init for page_get_replacement_page */
58 #define	MTYPE_PGR_INIT(mtype, flags, pp, mnode, pgcnt)			\
59 	mtype = (flags & PG_NORELOC) ? MTYPE_NORELOC : MTYPE_RELOC;
60 
61 #define	MNODETYPE_2_PFN(mnode, mtype, pfnlo, pfnhi)			\
62 	ASSERT(mtype != MTYPE_NORELOC);					\
63 	pfnlo = mem_node_config[mnode].physbase;			\
64 	pfnhi = mem_node_config[mnode].physmax;
65 
66 /*
67  * candidate counters in vm_pagelist.c are indexed by color and range
68  */
69 #define	MAX_MNODE_MRANGES		MAX_MEM_TYPES
70 #define	MNODE_RANGE_CNT(mnode)		MAX_MNODE_MRANGES
71 #define	MNODE_MAX_MRANGE(mnode)		(MAX_MEM_TYPES - 1)
72 #define	MTYPE_2_MRANGE(mnode, mtype)	(mtype)
73 
74 /*
75  * Internal PG_ flags.
76  */
77 #define	PGI_RELOCONLY	0x10000	/* acts in the opposite sense to PG_NORELOC */
78 #define	PGI_NOCAGE	0x20000	/* indicates Cage is disabled */
79 #define	PGI_PGCPHIPRI	0x40000	/* page_get_contig_page priority allocation */
80 #define	PGI_PGCPSZC0	0x80000	/* relocate base pagesize page */
81 
82 /*
83  * PGI mtype flags - should not overlap PGI flags
84  */
85 #define	PGI_MT_RANGE	0x1000000	/* mtype range */
86 #define	PGI_MT_NEXT	0x2000000	/* get next mtype */
87 
88 extern page_t ***page_freelists[MMU_PAGE_SIZES][MAX_MEM_TYPES];
89 extern page_t ***page_cachelists[MAX_MEM_TYPES];
90 
91 #define	PAGE_FREELISTS(mnode, szc, color, mtype) \
92 	(*(page_freelists[szc][mtype][mnode] + (color)))
93 
94 #define	PAGE_CACHELISTS(mnode, color, mtype) \
95 	(*(page_cachelists[mtype][mnode] + (color)))
96 
97 /*
98  * There are 'page_colors' colors/bins.  Spread them out under a
99  * couple of locks.  There are mutexes for both the page freelist
100  * and the page cachelist.  We want enough locks to make contention
101  * reasonable, but not too many -- otherwise page_freelist_lock() gets
102  * so expensive that it becomes the bottleneck!
103  */
104 #define	NPC_MUTEX	16
105 
106 extern kmutex_t	*fpc_mutex[NPC_MUTEX];
107 extern kmutex_t	*cpc_mutex[NPC_MUTEX];
108 
109 /*
110  * Iterator provides the info needed to convert RA to PA.
111  * MEM_NODE_ITERATOR_INIT() should be called before
112  * PAGE_NEXT_PFN_FOR_COLOR() if pfn was not obtained via a previous
113  * PAGE_NEXT_PFN_FOR_COLOR() call. Iterator caches color 2 hash
114  * translations requiring initializer call if color or ceq_mask changes,
115  * even if pfn doesn't. MEM_NODE_ITERATOR_INIT() must also be called before
116  * PFN_2_COLOR() that uses a valid iterator argument.
117  */
118 #ifdef	sun4v
119 
120 typedef struct mem_node_iterator {
121 	uint_t mi_mnode;		/* mnode in which to iterate */
122 	int mi_init;			/* set to 1 when first init */
123 	int mi_last_mblock;		/* last mblock visited */
124 	uint_t mi_hash_ceq_mask;	/* cached copy of ceq_mask */
125 	uint_t mi_hash_color;		/* cached copy of color */
126 	uint_t mi_mnode_mask;		/* number of mask bits */
127 	uint_t mi_mnode_pfn_shift;	/* mnode position in pfn */
128 	pfn_t mi_mblock_base;		/* first valid pfn in current mblock */
129 	pfn_t mi_mblock_end;		/* last valid pfn in current mblock */
130 	pfn_t mi_ra_to_pa;		/* ra adjustment for current mblock */
131 	pfn_t mi_mnode_pfn_mask;	/* mask to obtain mnode id bits */
132 } mem_node_iterator_t;
133 
134 #define	MEM_NODE_ITERATOR_DECL(it) \
135 	mem_node_iterator_t it
136 #define	MEM_NODE_ITERATOR_INIT(pfn, mnode, it) \
137 	(pfn) = plat_mem_node_iterator_init((pfn), (mnode), (it), 1)
138 
139 extern pfn_t plat_mem_node_iterator_init(pfn_t, int,
140     mem_node_iterator_t *, int);
141 extern pfn_t plat_rapfn_to_papfn(pfn_t);
142 extern int interleaved_mnodes;
143 
144 #else	/* sun4v */
145 
146 #define	MEM_NODE_ITERATOR_DECL(it) \
147 	void *it = NULL
148 #define	MEM_NODE_ITERATOR_INIT(pfn, mnode, it)
149 
150 #endif	/* sun4v */
151 
152 /*
153  * Return the mnode limits so that hpc_counters length and base
154  * index can be determined. When interleaved_mnodes is set, we
155  * create an array only for the first mnode that exists. All other
156  * mnodes will share the array in this case.
157  * If interleaved_mnodes is not set, simply return the limits for
158  * the given mnode.
159  */
160 #define	HPM_COUNTERS_LIMITS(mnode, physbase, physmax, first)		\
161 	if (!interleaved_mnodes) {					\
162 		(physbase) = mem_node_config[(mnode)].physbase;		\
163 		(physmax) = mem_node_config[(mnode)].physmax;		\
164 		(first) = (mnode);					\
165 	} else if ((first) < 0) {					\
166 		mem_node_max_range(&(physbase), &(physmax));		\
167 		(first) = (mnode);					\
168 	}
169 
170 #define	PAGE_CTRS_WRITE_LOCK(mnode)					\
171 	if (!interleaved_mnodes) {					\
172 		rw_enter(&page_ctrs_rwlock[(mnode)], RW_WRITER);	\
173 		page_freelist_lock(mnode);				\
174 	} else {							\
175 		/* changing shared hpm_counters */			\
176 		int _i;							\
177 		for (_i = 0; _i < max_mem_nodes; _i++) {		\
178 			rw_enter(&page_ctrs_rwlock[_i], RW_WRITER);	\
179 			page_freelist_lock(_i);				\
180 		}							\
181 	}
182 
183 #define	PAGE_CTRS_WRITE_UNLOCK(mnode)					\
184 	if (!interleaved_mnodes) {					\
185 		page_freelist_unlock(mnode);				\
186 		rw_exit(&page_ctrs_rwlock[(mnode)]);			\
187 	} else {							\
188 		int _i;							\
189 		for (_i = 0; _i < max_mem_nodes; _i++) {		\
190 			page_freelist_unlock(_i);			\
191 			rw_exit(&page_ctrs_rwlock[_i]);			\
192 		}							\
193 	}
194 
195 /*
196  * cpu specific color conversion functions
197  */
198 extern uint_t page_get_nsz_color_mask_cpu(uchar_t, uint_t);
199 #pragma weak page_get_nsz_color_mask_cpu
200 
201 extern uint_t page_get_nsz_color_cpu(uchar_t, uint_t);
202 #pragma weak page_get_nsz_color_cpu
203 
204 extern uint_t page_get_color_shift_cpu(uchar_t, uchar_t);
205 #pragma weak page_get_color_shift_cpu
206 
207 extern uint_t page_convert_color_cpu(uint_t, uchar_t, uchar_t);
208 #pragma weak page_convert_color_cpu
209 
210 extern pfn_t page_next_pfn_for_color_cpu(pfn_t,
211     uchar_t, uint_t, uint_t, uint_t, void *);
212 #pragma weak page_next_pfn_for_color_cpu
213 
214 extern uint_t  page_pfn_2_color_cpu(pfn_t, uchar_t, void *);
215 #pragma weak page_pfn_2_color_cpu
216 
217 #define	PAGE_GET_COLOR_SHIFT(szc, nszc)				\
218 	((&page_get_color_shift_cpu != NULL) ?			\
219 	    page_get_color_shift_cpu(szc, nszc) :		\
220 	    (hw_page_array[(nszc)].hp_shift -			\
221 		hw_page_array[(szc)].hp_shift))
222 
223 #define	PAGE_CONVERT_COLOR(ncolor, szc, nszc)			\
224 	((&page_convert_color_cpu != NULL) ?			\
225 	    page_convert_color_cpu(ncolor, szc, nszc) :		\
226 	    ((ncolor) << PAGE_GET_COLOR_SHIFT((szc), (nszc))))
227 
228 #define	PFN_2_COLOR(pfn, szc, it)				\
229 	((&page_pfn_2_color_cpu != NULL) ?			\
230 	    page_pfn_2_color_cpu(pfn, szc, it) :		\
231 	    ((pfn & (hw_page_array[0].hp_colors - 1)) >>	\
232 		(hw_page_array[szc].hp_shift -			\
233 		    hw_page_array[0].hp_shift)))
234 
235 #define	PNUM_SIZE(szc)							\
236 	(hw_page_array[(szc)].hp_pgcnt)
237 #define	PNUM_SHIFT(szc)							\
238 	(hw_page_array[(szc)].hp_shift - hw_page_array[0].hp_shift)
239 #define	PAGE_GET_SHIFT(szc)						\
240 	(hw_page_array[(szc)].hp_shift)
241 #define	PAGE_GET_PAGECOLORS(szc)					\
242 	(hw_page_array[(szc)].hp_colors)
243 
244 /*
245  * This macro calculates the next sequential pfn with the specified
246  * color using color equivalency mask
247  */
248 #define	PAGE_NEXT_PFN_FOR_COLOR(pfn, szc, color, ceq_mask, color_mask, it)   \
249 	{                                                                    \
250 		ASSERT(((color) & ~(ceq_mask)) == 0);                        \
251 		if (&page_next_pfn_for_color_cpu == NULL) {                  \
252 			uint_t	pfn_shift = PAGE_BSZS_SHIFT(szc);            \
253 			pfn_t	spfn = pfn >> pfn_shift;                     \
254 			pfn_t	stride = (ceq_mask) + 1;                     \
255 			ASSERT((((ceq_mask) + 1) & (ceq_mask)) == 0);        \
256 			if (((spfn ^ (color)) & (ceq_mask)) == 0) {          \
257 				pfn += stride << pfn_shift;                  \
258 			} else {                                             \
259 				pfn = (spfn & ~(pfn_t)(ceq_mask)) | (color); \
260 				pfn = (pfn > spfn ? pfn : pfn + stride) <<   \
261 				    pfn_shift;                               \
262 			}                                                    \
263 		} else {                                                     \
264 		    pfn = page_next_pfn_for_color_cpu(pfn, szc, color,	     \
265 			ceq_mask, color_mask, it);			     \
266 		}                                                            \
267 	}
268 
269 /* get the color equivalency mask for the next szc */
270 #define	PAGE_GET_NSZ_MASK(szc, mask)                                         \
271 	((&page_get_nsz_color_mask_cpu == NULL) ?                            \
272 	    ((mask) >> (PAGE_GET_SHIFT((szc) + 1) - PAGE_GET_SHIFT(szc))) :  \
273 	    page_get_nsz_color_mask_cpu(szc, mask))
274 
275 /* get the color of the next szc */
276 #define	PAGE_GET_NSZ_COLOR(szc, color)                                       \
277 	((&page_get_nsz_color_cpu == NULL) ?                                 \
278 	    ((color) >> (PAGE_GET_SHIFT((szc) + 1) - PAGE_GET_SHIFT(szc))) : \
279 	    page_get_nsz_color_cpu(szc, color))
280 
281 /* Find the bin for the given page if it was of size szc */
282 #define	PP_2_BIN_SZC(pp, szc)	(PFN_2_COLOR(pp->p_pagenum, szc, (void *)(-1)))
283 
284 #define	PP_2_BIN(pp)		(PP_2_BIN_SZC(pp, pp->p_szc))
285 
286 #define	PP_2_MEM_NODE(pp)	(PFN_2_MEM_NODE(pp->p_pagenum))
287 
288 #define	PC_BIN_MUTEX(mnode, bin, flags) ((flags & PG_FREE_LIST) ?	\
289 	&fpc_mutex[(bin) & (NPC_MUTEX - 1)][mnode] :			\
290 	&cpc_mutex[(bin) & (NPC_MUTEX - 1)][mnode])
291 
292 #define	FPC_MUTEX(mnode, i)	(&fpc_mutex[i][mnode])
293 #define	CPC_MUTEX(mnode, i)	(&cpc_mutex[i][mnode])
294 
295 #define	PFN_BASE(pfnum, szc)	(pfnum & ~((1 << PAGE_BSZS_SHIFT(szc)) - 1))
296 
297 /*
298  * this structure is used for walking free page lists
299  * controls when to split large pages into smaller pages,
300  * and when to coalesce smaller pages into larger pages
301  */
302 typedef struct page_list_walker {
303 	uint_t	plw_colors;		/* num of colors for szc */
304 	uint_t  plw_color_mask;		/* colors-1 */
305 	uint_t	plw_bin_step;		/* next bin: 1 or 2 */
306 	uint_t  plw_count;		/* loop count */
307 	uint_t	plw_bin0;		/* starting bin */
308 	uint_t  plw_bin_marker;		/* bin after initial jump */
309 	uint_t  plw_bin_split_prev;	/* last bin we tried to split */
310 	uint_t  plw_do_split;		/* set if OK to split */
311 	uint_t  plw_split_next;		/* next bin to split */
312 	uint_t	plw_ceq_dif;		/* number of different color groups */
313 					/* to check */
314 	uint_t	plw_ceq_mask[MMU_PAGE_SIZES + 1]; /* color equiv mask */
315 	uint_t	plw_bins[MMU_PAGE_SIZES + 1];	/* num of bins */
316 } page_list_walker_t;
317 
318 void	page_list_walk_init(uchar_t szc, uint_t flags, uint_t bin,
319     int can_split, int use_ceq, page_list_walker_t *plw);
320 
321 typedef	char	hpmctr_t;
322 
323 #ifdef DEBUG
324 #define	CHK_LPG(pp, szc)	chk_lpg(pp, szc)
325 extern void	chk_lpg(page_t *, uchar_t);
326 #else
327 #define	CHK_LPG(pp, szc)
328 #endif
329 
330 /*
331  * page list count per mnode and type.
332  */
333 typedef	struct {
334 	pgcnt_t	plc_mt_pgmax;		/* max page cnt */
335 	pgcnt_t plc_mt_clpgcnt;		/* cache list cnt */
336 	pgcnt_t plc_mt_flpgcnt;		/* free list cnt - small pages */
337 	pgcnt_t plc_mt_lgpgcnt;		/* free list cnt - large pages */
338 #ifdef DEBUG
339 	struct {
340 		pgcnt_t plc_mts_pgcnt;	/* per page size count */
341 		int	plc_mts_colors;
342 		pgcnt_t	*plc_mtsc_pgcnt; /* per color bin count */
343 	} plc_mts[MMU_PAGE_SIZES];
344 #endif
345 } plcnt_t[MAX_MEM_NODES][MAX_MEM_TYPES];
346 
347 #ifdef DEBUG
348 
349 #define	PLCNT_SZ(ctrs_sz) {						\
350 	int	szc;							\
351 	for (szc = 0; szc < mmu_page_sizes; szc++) {			\
352 		int	colors = page_get_pagecolors(szc);		\
353 		ctrs_sz += (max_mem_nodes * MAX_MEM_TYPES *		\
354 		    colors * sizeof (pgcnt_t));				\
355 	}								\
356 }
357 
358 #define	PLCNT_INIT(base) {						\
359 	int	mn, mt, szc, colors;					\
360 	for (szc = 0; szc < mmu_page_sizes; szc++) {			\
361 		colors = page_get_pagecolors(szc);			\
362 		for (mn = 0; mn < max_mem_nodes; mn++) {		\
363 			for (mt = 0; mt < MAX_MEM_TYPES; mt++) {	\
364 				plcnt[mn][mt].plc_mts[szc].		\
365 				    plc_mts_colors = colors;		\
366 				plcnt[mn][mt].plc_mts[szc].		\
367 				    plc_mtsc_pgcnt = (pgcnt_t *)base;	\
368 				base += (colors * sizeof (pgcnt_t));	\
369 			}						\
370 		}							\
371 	}								\
372 }
373 
374 #define	PLCNT_DO(pp, mn, mtype, szc, cnt, flags) {			\
375 	int	bin = PP_2_BIN(pp);					\
376 	if (flags & PG_CACHE_LIST)					\
377 		atomic_add_long(&plcnt[mn][mtype].plc_mt_clpgcnt, cnt);	\
378 	else if (szc)							\
379 		atomic_add_long(&plcnt[mn][mtype].plc_mt_lgpgcnt, cnt);	\
380 	else								\
381 		atomic_add_long(&plcnt[mn][mtype].plc_mt_flpgcnt, cnt);	\
382 	atomic_add_long(&plcnt[mn][mtype].plc_mts[szc].plc_mts_pgcnt,	\
383 	    cnt);							\
384 	atomic_add_long(&plcnt[mn][mtype].plc_mts[szc].			\
385 	    plc_mtsc_pgcnt[bin], cnt);					\
386 }
387 
388 #else
389 
390 #define	PLCNT_SZ(ctrs_sz)
391 
392 #define	PLCNT_INIT(base)
393 
394 /* PG_FREE_LIST may not be explicitly set in flags for large pages */
395 
396 #define	PLCNT_DO(pp, mn, mtype, szc, cnt, flags) {			\
397 	if (flags & PG_CACHE_LIST)					\
398 		atomic_add_long(&plcnt[mn][mtype].plc_mt_clpgcnt, cnt);	\
399 	else if (szc)							\
400 		atomic_add_long(&plcnt[mn][mtype].plc_mt_lgpgcnt, cnt);	\
401 	else								\
402 		atomic_add_long(&plcnt[mn][mtype].plc_mt_flpgcnt, cnt);	\
403 }
404 
405 #endif
406 
407 #define	PLCNT_INCR(pp, mn, mtype, szc, flags) {				\
408 	long	cnt = (1 << PAGE_BSZS_SHIFT(szc));			\
409 	PLCNT_DO(pp, mn, mtype, szc, cnt, flags);			\
410 }
411 
412 #define	PLCNT_DECR(pp, mn, mtype, szc, flags) {				\
413 	long	cnt = ((-1) << PAGE_BSZS_SHIFT(szc));			\
414 	PLCNT_DO(pp, mn, mtype, szc, cnt, flags);			\
415 }
416 
417 /*
418  * macros to update page list max counts - done when pages transferred
419  * from RELOC to NORELOC mtype (kcage_init or kcage_assimilate_page).
420  */
421 
422 #define	PLCNT_XFER_NORELOC(pp) {					\
423 	long	cnt = (1 << PAGE_BSZS_SHIFT((pp)->p_szc));		\
424 	int	mn = PP_2_MEM_NODE(pp);					\
425 	atomic_add_long(&plcnt[mn][MTYPE_NORELOC].plc_mt_pgmax, cnt);	\
426 	atomic_add_long(&plcnt[mn][MTYPE_RELOC].plc_mt_pgmax, -cnt);	\
427 }
428 
429 /*
430  * macro to modify the page list max counts when memory is added to
431  * the page lists during startup (add_physmem) or during a DR operation
432  * when memory is added (kphysm_add_memory_dynamic) or deleted
433  * (kphysm_del_cleanup).
434  */
435 #define	PLCNT_MODIFY_MAX(pfn, cnt) {					       \
436 	spgcnt_t _cnt = (spgcnt_t)(cnt);				       \
437 	pgcnt_t _acnt = ABS(_cnt);					       \
438 	int _mn;							       \
439 	pgcnt_t _np;							       \
440 	if (&plat_mem_node_intersect_range != NULL) {			       \
441 		for (_mn = 0; _mn < max_mem_nodes; _mn++) {		       \
442 			plat_mem_node_intersect_range((pfn), _acnt, _mn, &_np);\
443 			if (_np == 0)					       \
444 				continue;				       \
445 			atomic_add_long(&plcnt[_mn][MTYPE_RELOC].plc_mt_pgmax, \
446 			    (_cnt < 0) ? -_np : _np);			       \
447 		}							       \
448 	} else {							       \
449 		pfn_t _pfn = (pfn);					       \
450 		pfn_t _endpfn = _pfn + _acnt;				       \
451 		while (_pfn < _endpfn) {				       \
452 			_mn = PFN_2_MEM_NODE(_pfn);			       \
453 			_np = MIN(_endpfn, mem_node_config[_mn].physmax + 1) - \
454 			    _pfn;					       \
455 			_pfn += _np;					       \
456 			atomic_add_long(&plcnt[_mn][MTYPE_RELOC].plc_mt_pgmax, \
457 			    (_cnt < 0) ? -_np : _np);			       \
458 		}							       \
459 	}								       \
460 }
461 
462 extern plcnt_t	plcnt;
463 
464 #define	MNODE_PGCNT(mn)							\
465 	(plcnt[mn][MTYPE_RELOC].plc_mt_clpgcnt +			\
466 	    plcnt[mn][MTYPE_NORELOC].plc_mt_clpgcnt +			\
467 	    plcnt[mn][MTYPE_RELOC].plc_mt_flpgcnt +			\
468 	    plcnt[mn][MTYPE_NORELOC].plc_mt_flpgcnt +			\
469 	    plcnt[mn][MTYPE_RELOC].plc_mt_lgpgcnt +			\
470 	    plcnt[mn][MTYPE_NORELOC].plc_mt_lgpgcnt)
471 
472 #define	MNODETYPE_PGCNT(mn, mtype)					\
473 	(plcnt[mn][mtype].plc_mt_clpgcnt +				\
474 	    plcnt[mn][mtype].plc_mt_flpgcnt +				\
475 	    plcnt[mn][mtype].plc_mt_lgpgcnt)
476 
477 /*
478  * macros to loop through the mtype range - MTYPE_START returns -1 in
479  * mtype if no pages in mnode/mtype and possibly NEXT mtype.
480  */
481 #define	MTYPE_START(mnode, mtype, flags) {				\
482 	if (plcnt[mnode][mtype].plc_mt_pgmax == 0) {			\
483 		ASSERT(mtype == MTYPE_RELOC ||				\
484 		    MNODETYPE_PGCNT(mnode, mtype) == 0 ||		\
485 		    plcnt[mnode][mtype].plc_mt_pgmax != 0);		\
486 		MTYPE_NEXT(mnode, mtype, flags);			\
487 	}								\
488 }
489 
490 /*
491  * if allocation from the RELOC pool failed and there is sufficient cage
492  * memory, attempt to allocate from the NORELOC pool.
493  */
494 #define	MTYPE_NEXT(mnode, mtype, flags) { 				\
495 	if (!(flags & (PG_NORELOC | PGI_NOCAGE | PGI_RELOCONLY)) &&	\
496 	    (kcage_freemem >= kcage_lotsfree)) {			\
497 		if (plcnt[mnode][MTYPE_NORELOC].plc_mt_pgmax == 0) {	\
498 			ASSERT(MNODETYPE_PGCNT(mnode, MTYPE_NORELOC) == 0 || \
499 			    plcnt[mnode][MTYPE_NORELOC].plc_mt_pgmax != 0);  \
500 			mtype = -1;					\
501 		} else {						\
502 			mtype = MTYPE_NORELOC;				\
503 			flags |= PG_NORELOC;				\
504 		}							\
505 	} else {							\
506 		mtype = -1;						\
507 	}								\
508 }
509 
510 /*
511  * get the ecache setsize for the current cpu.
512  */
513 #define	CPUSETSIZE()	(cpunodes[CPU->cpu_id].ecache_setsize)
514 
515 extern struct cpu	cpu0;
516 #define	CPU0		&cpu0
517 
518 #define	PAGE_BSZS_SHIFT(szc)	TTE_BSZS_SHIFT(szc)
519 /*
520  * For sfmmu each larger page is 8 times the size of the previous
521  * size page.
522  */
523 #define	FULL_REGION_CNT(rg_szc)	(8)
524 
525 /*
526  * The counter base must be per page_counter element to prevent
527  * races when re-indexing, and the base page size element should
528  * be aligned on a boundary of the given region size.
529  *
530  * We also round up the number of pages spanned by the counters
531  * for a given region to PC_BASE_ALIGN in certain situations to simplify
532  * the coding for some non-performance critical routines.
533  */
534 #define	PC_BASE_ALIGN		((pfn_t)1 << PAGE_BSZS_SHIFT(mmu_page_sizes-1))
535 #define	PC_BASE_ALIGN_MASK	(PC_BASE_ALIGN - 1)
536 
537 extern int ecache_alignsize;
538 #define	L2CACHE_ALIGN		ecache_alignsize
539 #define	L2CACHE_ALIGN_MAX	512
540 
541 extern int update_proc_pgcolorbase_after_fork;
542 extern int consistent_coloring;
543 extern uint_t vac_colors_mask;
544 extern int vac_size;
545 extern int vac_shift;
546 
547 /*
548  * Kernel mem segment in 64-bit space
549  */
550 extern caddr_t kmem64_base, kmem64_end, kmem64_aligned_end;
551 extern int kmem64_alignsize, kmem64_szc;
552 extern uint64_t kmem64_pabase;
553 extern int max_bootlp_tteszc;
554 
555 /*
556  * Maximum and default values for user heap, stack, private and shared
557  * anonymous memory, and user text and initialized data.
558  *
559  * Initial values are defined in architecture specific mach_vm_dep.c file.
560  * Used by map_pgsz*() routines.
561  */
562 extern size_t max_uheap_lpsize;
563 extern size_t default_uheap_lpsize;
564 extern size_t max_ustack_lpsize;
565 extern size_t default_ustack_lpsize;
566 extern size_t max_privmap_lpsize;
567 extern size_t max_uidata_lpsize;
568 extern size_t max_utext_lpsize;
569 extern size_t max_shm_lpsize;
570 
571 /*
572  * For adjusting the default lpsize, for DTLB-limited page sizes.
573  */
574 extern void adjust_data_maxlpsize(size_t ismpagesize);
575 
576 /*
577  * Sanity control. Don't use large pages regardless of user
578  * settings if there's less than priv or shm_lpg_min_physmem memory installed.
579  * The units for this variable are 8K pages.
580  */
581 extern pgcnt_t privm_lpg_min_physmem;
582 extern pgcnt_t shm_lpg_min_physmem;
583 
584 /*
585  * AS_2_BIN macro controls the page coloring policy.
586  * 0 (default) uses various vaddr bits
587  * 1 virtual=paddr
588  * 2 bin hopping
589  */
590 #define	AS_2_BIN(as, seg, vp, addr, bin, szc)				\
591 switch (consistent_coloring) {						\
592 	default:                                                        \
593 		cmn_err(CE_WARN,					\
594 			"AS_2_BIN: bad consistent coloring value");	\
595 		/* assume default algorithm -> continue */		\
596 	case 0: {                                                       \
597 		uint32_t ndx, new;					\
598 		int slew = 0;						\
599 		pfn_t pfn;                                              \
600                                                                         \
601 		if (vp != NULL && IS_SWAPVP(vp) &&			\
602 		    seg->s_ops == &segvn_ops)				\
603 			slew = as_color_bin(as);			\
604                                                                         \
605 		pfn = ((uintptr_t)addr >> MMU_PAGESHIFT) +		\
606 			(((uintptr_t)addr >> page_coloring_shift) <<	\
607 			(vac_shift - MMU_PAGESHIFT));			\
608 		if ((szc) == 0 || &page_pfn_2_color_cpu == NULL) {	\
609 			pfn += slew;					\
610 			bin = PFN_2_COLOR(pfn, szc, NULL);		\
611 		} else {						\
612 			bin = PFN_2_COLOR(pfn, szc, NULL);		\
613 			bin += slew >> (vac_shift - MMU_PAGESHIFT);	\
614 			bin &= hw_page_array[(szc)].hp_colors - 1;	\
615 		}							\
616 		break;                                                  \
617 	}                                                               \
618 	case 1:                                                         \
619 		bin = PFN_2_COLOR(((uintptr_t)addr >> MMU_PAGESHIFT),	\
620 		    szc, NULL);						\
621 		break;                                                  \
622 	case 2: {                                                       \
623 		int cnt = as_color_bin(as);				\
624 		uint_t color_mask = page_get_pagecolors(0) - 1;		\
625                                                                         \
626 		/* make sure physical color aligns with vac color */	\
627 		while ((cnt & vac_colors_mask) !=			\
628 		    addr_to_vcolor(addr)) {				\
629 			cnt++;						\
630 		}                                                       \
631 		bin = cnt = cnt & color_mask;			        \
632 		bin >>= PAGE_GET_COLOR_SHIFT(0, szc);                   \
633 		/* update per as page coloring fields */		\
634 		cnt = (cnt + 1) & color_mask;			        \
635 		if (cnt == (as_color_start(as) & color_mask)) {	        \
636 			cnt = as_color_start(as) = as_color_start(as) + \
637 				PGCLR_LOOPFACTOR;			\
638 		}                                                       \
639 		as_color_bin(as) = cnt & color_mask;		        \
640 		break;                                                  \
641 	}								\
642 }									\
643 	ASSERT(bin < page_get_pagecolors(szc));
644 
645 /*
646  * cpu private vm data - accessed thru CPU->cpu_vm_data
647  *	vc_pnum_memseg: tracks last memseg visited in page_numtopp_nolock()
648  *	vc_pnext_memseg: tracks last memseg visited in page_nextn()
649  *	vc_kmptr: unaligned kmem pointer for this vm_cpu_data_t
650  *	vc_kmsize: orignal kmem size for this vm_cpu_data_t
651  */
652 
653 typedef struct {
654 	struct memseg	*vc_pnum_memseg;
655 	struct memseg	*vc_pnext_memseg;
656 	void		*vc_kmptr;
657 	size_t		vc_kmsize;
658 } vm_cpu_data_t;
659 
660 /* allocation size to ensure vm_cpu_data_t resides in its own cache line */
661 #define	VM_CPU_DATA_PADSIZE						\
662 	(P2ROUNDUP(sizeof (vm_cpu_data_t), L2CACHE_ALIGN_MAX))
663 
664 /* for boot cpu before kmem is initialized */
665 extern char	vm_cpu_data0[];
666 
667 /*
668  * Function to get an ecache color bin: F(as, cnt, vcolor).
669  * the goal of this function is to:
670  * - to spread a processes' physical pages across the entire ecache to
671  *	maximize its use.
672  * - to minimize vac flushes caused when we reuse a physical page on a
673  *	different vac color than it was previously used.
674  * - to prevent all processes to use the same exact colors and trash each
675  *	other.
676  *
677  * cnt is a bin ptr kept on a per as basis.  As we page_create we increment
678  * the ptr so we spread out the physical pages to cover the entire ecache.
679  * The virtual color is made a subset of the physical color in order to
680  * in minimize virtual cache flushing.
681  * We add in the as to spread out different as.	 This happens when we
682  * initialize the start count value.
683  * sizeof(struct as) is 60 so we shift by 3 to get into the bit range
684  * that will tend to change.  For example, on spitfire based machines
685  * (vcshft == 1) contigous as are spread bu ~6 bins.
686  * vcshft provides for proper virtual color alignment.
687  * In theory cnt should be updated using cas only but if we are off by one
688  * or 2 it is no big deal.
689  * We also keep a start value which is used to randomize on what bin we
690  * start counting when it is time to start another loop. This avoids
691  * contigous allocations of ecache size to point to the same bin.
692  * Why 3? Seems work ok. Better than 7 or anything larger.
693  */
694 #define	PGCLR_LOOPFACTOR 3
695 
696 /*
697  * When a bin is empty, and we can't satisfy a color request correctly,
698  * we scan.  If we assume that the programs have reasonable spatial
699  * behavior, then it will not be a good idea to use the adjacent color.
700  * Using the adjacent color would result in virtually adjacent addresses
701  * mapping into the same spot in the cache.  So, if we stumble across
702  * an empty bin, skip a bunch before looking.  After the first skip,
703  * then just look one bin at a time so we don't miss our cache on
704  * every look. Be sure to check every bin.  Page_create() will panic
705  * if we miss a page.
706  *
707  * This also explains the `<=' in the for loops in both page_get_freelist()
708  * and page_get_cachelist().  Since we checked the target bin, skipped
709  * a bunch, then continued one a time, we wind up checking the target bin
710  * twice to make sure we get all of them bins.
711  */
712 #define	BIN_STEP	20
713 
714 #ifdef VM_STATS
715 struct vmm_vmstats_str {
716 	ulong_t pgf_alloc[MMU_PAGE_SIZES];	/* page_get_freelist */
717 	ulong_t pgf_allocok[MMU_PAGE_SIZES];
718 	ulong_t pgf_allocokrem[MMU_PAGE_SIZES];
719 	ulong_t pgf_allocfailed[MMU_PAGE_SIZES];
720 	ulong_t pgf_allocdeferred;
721 	ulong_t	pgf_allocretry[MMU_PAGE_SIZES];
722 	ulong_t pgc_alloc;			/* page_get_cachelist */
723 	ulong_t pgc_allocok;
724 	ulong_t pgc_allocokrem;
725 	ulong_t	pgc_allocokdeferred;
726 	ulong_t pgc_allocfailed;
727 	ulong_t	pgcp_alloc[MMU_PAGE_SIZES];	/* page_get_contig_pages */
728 	ulong_t	pgcp_allocfailed[MMU_PAGE_SIZES];
729 	ulong_t	pgcp_allocempty[MMU_PAGE_SIZES];
730 	ulong_t	pgcp_allocok[MMU_PAGE_SIZES];
731 	ulong_t	ptcp[MMU_PAGE_SIZES];		/* page_trylock_contig_pages */
732 	ulong_t	ptcpfreethresh[MMU_PAGE_SIZES];
733 	ulong_t	ptcpfailexcl[MMU_PAGE_SIZES];
734 	ulong_t	ptcpfailszc[MMU_PAGE_SIZES];
735 	ulong_t	ptcpfailcage[MMU_PAGE_SIZES];
736 	ulong_t	ptcpok[MMU_PAGE_SIZES];
737 	ulong_t	pgmf_alloc[MMU_PAGE_SIZES];	/* page_get_mnode_freelist */
738 	ulong_t	pgmf_allocfailed[MMU_PAGE_SIZES];
739 	ulong_t	pgmf_allocempty[MMU_PAGE_SIZES];
740 	ulong_t	pgmf_allocok[MMU_PAGE_SIZES];
741 	ulong_t	pgmc_alloc;			/* page_get_mnode_cachelist */
742 	ulong_t	pgmc_allocfailed;
743 	ulong_t	pgmc_allocempty;
744 	ulong_t	pgmc_allocok;
745 	ulong_t	pladd_free[MMU_PAGE_SIZES];	/* page_list_add/sub */
746 	ulong_t	plsub_free[MMU_PAGE_SIZES];
747 	ulong_t	pladd_cache;
748 	ulong_t	plsub_cache;
749 	ulong_t	plsubpages_szcbig;
750 	ulong_t	plsubpages_szc0;
751 	ulong_t	pfs_req[MMU_PAGE_SIZES];	/* page_freelist_split */
752 	ulong_t	pfs_demote[MMU_PAGE_SIZES];
753 	ulong_t	pfc_coalok[MMU_PAGE_SIZES][MAX_MNODE_MRANGES];
754 	ulong_t ppr_reloc[MMU_PAGE_SIZES];	/* page_relocate */
755 	ulong_t ppr_relocok[MMU_PAGE_SIZES];
756 	ulong_t ppr_relocnoroot[MMU_PAGE_SIZES];
757 	ulong_t ppr_reloc_replnoroot[MMU_PAGE_SIZES];
758 	ulong_t ppr_relocnolock[MMU_PAGE_SIZES];
759 	ulong_t ppr_relocnomem[MMU_PAGE_SIZES];
760 	ulong_t ppr_krelocfail[MMU_PAGE_SIZES];
761 	ulong_t ppr_copyfail;
762 	/* page coalesce counter */
763 	ulong_t	page_ctrs_coalesce[MMU_PAGE_SIZES][MAX_MNODE_MRANGES];
764 	/* candidates useful */
765 	ulong_t	page_ctrs_cands_skip[MMU_PAGE_SIZES][MAX_MNODE_MRANGES];
766 	/* ctrs changed after locking */
767 	ulong_t	page_ctrs_changed[MMU_PAGE_SIZES][MAX_MNODE_MRANGES];
768 	/* page_freelist_coalesce failed */
769 	ulong_t	page_ctrs_failed[MMU_PAGE_SIZES][MAX_MNODE_MRANGES];
770 	ulong_t	page_ctrs_coalesce_all;	/* page coalesce all counter */
771 	ulong_t	page_ctrs_cands_skip_all; /* candidates useful for all func */
772 };
773 extern struct vmm_vmstats_str vmm_vmstats;
774 #endif	/* VM_STATS */
775 
776 /*
777  * Used to hold off page relocations into the cage until OBP has completed
778  * its boot-time handoff of its resources to the kernel.
779  */
780 extern int page_relocate_ready;
781 
782 /*
783  * cpu/mmu-dependent vm variables may be reset at bootup.
784  */
785 extern uint_t mmu_page_sizes;
786 extern uint_t max_mmu_page_sizes;
787 extern uint_t mmu_hashcnt;
788 extern uint_t max_mmu_hashcnt;
789 extern size_t mmu_ism_pagesize;
790 extern int mmu_exported_pagesize_mask;
791 extern uint_t mmu_exported_page_sizes;
792 extern uint_t szc_2_userszc[];
793 extern uint_t userszc_2_szc[];
794 
795 #define	mmu_legacy_page_sizes	mmu_exported_page_sizes
796 #define	USERSZC_2_SZC(userszc)	(userszc_2_szc[userszc])
797 #define	SZC_2_USERSZC(szc)	(szc_2_userszc[szc])
798 
799 /*
800  * Platform specific page routines
801  */
802 extern void mach_page_add(page_t **, page_t *);
803 extern void mach_page_sub(page_t **, page_t *);
804 extern uint_t page_get_pagecolors(uint_t);
805 extern void ppcopy_kernel__relocatable(page_t *, page_t *);
806 #define	ppcopy_kernel(p1, p2)	ppcopy_kernel__relocatable(p1, p2)
807 
808 /*
809  * platform specific large pages for kernel heap support
810  */
811 extern size_t get_segkmem_lpsize(size_t lpsize);
812 extern size_t mmu_get_kernel_lpsize(size_t lpsize);
813 extern void mmu_init_kernel_pgsz(struct hat *hat);
814 extern void mmu_init_kcontext();
815 extern uint64_t kcontextreg;
816 
817 /*
818  * Nucleus data page allocator routines
819  */
820 extern void ndata_alloc_init(struct memlist *, uintptr_t, uintptr_t);
821 extern void *ndata_alloc(struct memlist *, size_t, size_t);
822 extern void *ndata_extra_base(struct memlist *, size_t, caddr_t);
823 extern size_t ndata_maxsize(struct memlist *);
824 extern size_t ndata_spare(struct memlist *, size_t, size_t);
825 
826 #ifdef	__cplusplus
827 }
828 #endif
829 
830 #endif	/* _VM_DEP_H */
831