xref: /illumos-gate/usr/src/uts/sun4v/cpu/niagara2.c (revision 59ac0c1669407488b67ae9e273667a340dccc611)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/types.h>
30 #include <sys/systm.h>
31 #include <sys/archsystm.h>
32 #include <sys/machparam.h>
33 #include <sys/machsystm.h>
34 #include <sys/cpu.h>
35 #include <sys/elf_SPARC.h>
36 #include <vm/hat_sfmmu.h>
37 #include <vm/page.h>
38 #include <vm/vm_dep.h>
39 #include <sys/cpuvar.h>
40 #include <sys/async.h>
41 #include <sys/cmn_err.h>
42 #include <sys/debug.h>
43 #include <sys/dditypes.h>
44 #include <sys/sunddi.h>
45 #include <sys/cpu_module.h>
46 #include <sys/prom_debug.h>
47 #include <sys/vmsystm.h>
48 #include <sys/prom_plat.h>
49 #include <sys/sysmacros.h>
50 #include <sys/intreg.h>
51 #include <sys/machtrap.h>
52 #include <sys/ontrap.h>
53 #include <sys/ivintr.h>
54 #include <sys/atomic.h>
55 #include <sys/panic.h>
56 #include <sys/dtrace.h>
57 #include <sys/simulate.h>
58 #include <sys/fault.h>
59 #include <sys/niagara2regs.h>
60 #include <sys/hsvc.h>
61 #include <sys/trapstat.h>
62 
63 uint_t root_phys_addr_lo_mask = 0xffffffffU;
64 #if defined(NIAGARA2_IMPL)
65 char cpu_module_name[] = "SUNW,UltraSPARC-T2";
66 #elif defined(VFALLS_IMPL)
67 char cpu_module_name[] = "SUNW,UltraSPARC-T2+";
68 #endif
69 
70 /*
71  * Hypervisor services information for the NIAGARA2 and Victoria Falls
72  * CPU module
73  */
74 static boolean_t cpu_hsvc_available = B_TRUE;
75 static uint64_t cpu_sup_minor;		/* Supported minor number */
76 #if defined(NIAGARA2_IMPL)
77 static hsvc_info_t cpu_hsvc = {
78 	HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR,
79 	NIAGARA2_HSVC_MINOR, cpu_module_name
80 };
81 #elif defined(VFALLS_IMPL)
82 static hsvc_info_t cpu_hsvc = {
83 	HSVC_REV_1, NULL, HSVC_GROUP_VFALLS_CPU, VFALLS_HSVC_MAJOR,
84 	VFALLS_HSVC_MINOR, cpu_module_name
85 };
86 #endif
87 
88 void
89 cpu_setup(void)
90 {
91 	extern int mmu_exported_pagesize_mask;
92 	extern int cpc_has_overflow_intr;
93 	extern size_t contig_mem_prealloc_base;
94 	int status;
95 
96 	/*
97 	 * Negotiate the API version for Niagara2 specific hypervisor
98 	 * services.
99 	 */
100 	status = hsvc_register(&cpu_hsvc, &cpu_sup_minor);
101 	if (status != 0) {
102 		cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services "
103 		    "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d",
104 		    cpu_hsvc.hsvc_modname, cpu_hsvc.hsvc_group,
105 		    cpu_hsvc.hsvc_major, cpu_hsvc.hsvc_minor, status);
106 		cpu_hsvc_available = B_FALSE;
107 	}
108 
109 	/*
110 	 * The setup common to all CPU modules is done in cpu_setup_common
111 	 * routine.
112 	 */
113 	cpu_setup_common(NULL);
114 
115 	cache |= (CACHE_PTAG | CACHE_IOCOHERENT);
116 
117 	if ((mmu_exported_pagesize_mask &
118 	    DEFAULT_SUN4V_MMU_PAGESIZE_MASK) !=
119 	    DEFAULT_SUN4V_MMU_PAGESIZE_MASK)
120 		cmn_err(CE_PANIC, "machine description"
121 		    " does not have required sun4v page sizes"
122 		    " 8K, 64K and 4M: MD mask is 0x%x",
123 		    mmu_exported_pagesize_mask);
124 
125 	cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT;
126 
127 	/*
128 	 * Niagara2 supports a 48-bit subset of the full 64-bit virtual
129 	 * address space. Virtual addresses between 0x0000800000000000
130 	 * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole"
131 	 * and must never be mapped. In addition, software must not use
132 	 * pages within 4GB of the VA hole as instruction pages to
133 	 * avoid problems with prefetching into the VA hole.
134 	 */
135 	hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32));
136 	hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32));
137 
138 	/*
139 	 * Niagara2 has a performance counter overflow interrupt
140 	 */
141 	cpc_has_overflow_intr = 1;
142 
143 	/*
144 	 * Enable 4M pages for OOB.
145 	 */
146 	max_uheap_lpsize = MMU_PAGESIZE4M;
147 	max_ustack_lpsize = MMU_PAGESIZE4M;
148 	max_privmap_lpsize = MMU_PAGESIZE4M;
149 
150 	contig_mem_prealloc_base = NIAGARA2_PREALLOC_BASE;
151 }
152 
153 /*
154  * Set the magic constants of the implementation.
155  */
156 void
157 cpu_fiximp(struct cpu_node *cpunode)
158 {
159 	/*
160 	 * The Cache node is optional in MD. Therefore in case "Cache"
161 	 * node does not exists in MD, set the default L2 cache associativity,
162 	 * size, linesize.
163 	 */
164 	if (cpunode->ecache_size == 0)
165 		cpunode->ecache_size = L2CACHE_SIZE;
166 	if (cpunode->ecache_linesize == 0)
167 		cpunode->ecache_linesize = L2CACHE_LINESIZE;
168 	if (cpunode->ecache_associativity == 0)
169 		cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY;
170 }
171 
172 void
173 cpu_map_exec_units(struct cpu *cp)
174 {
175 	ASSERT(MUTEX_HELD(&cpu_lock));
176 
177 	/*
178 	 * The cpu_ipipe and cpu_fpu fields are initialized based on
179 	 * the execution unit sharing information from the MD. They
180 	 * default to the CPU id in the absence of such information.
181 	 */
182 	cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping;
183 	if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND)
184 		cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id);
185 
186 	cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping;
187 	if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND)
188 		cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id);
189 
190 	/*
191 	 * Niagara 2 defines the core to be at the FPU level
192 	 */
193 	cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu;
194 
195 	/*
196 	 * The cpu_chip field is initialized based on the information
197 	 * in the MD and assume that all cpus within a chip
198 	 * share the same L2 cache. If no such info is available, we
199 	 * set the cpu to belong to the defacto chip 0.
200 	 */
201 	cp->cpu_m.cpu_chip = cpunodes[cp->cpu_id].l2_cache_mapping;
202 	if (cp->cpu_m.cpu_chip == NO_CHIP_MAPPING_FOUND)
203 		cp->cpu_m.cpu_chip = 0;
204 }
205 
206 static int cpucnt;
207 
208 void
209 cpu_init_private(struct cpu *cp)
210 {
211 	extern void niagara_kstat_init(void);
212 
213 	ASSERT(MUTEX_HELD(&cpu_lock));
214 
215 	cpu_map_exec_units(cp);
216 
217 	if ((cpucnt++ == 0) && (cpu_hsvc_available == B_TRUE))
218 		(void) niagara_kstat_init();
219 }
220 
221 /*ARGSUSED*/
222 void
223 cpu_uninit_private(struct cpu *cp)
224 {
225 	extern void niagara_kstat_fini(void);
226 
227 	ASSERT(MUTEX_HELD(&cpu_lock));
228 	if ((--cpucnt == 0) && (cpu_hsvc_available == B_TRUE))
229 		(void) niagara_kstat_fini();
230 }
231 
232 /*
233  * On Niagara2, any flush will cause all preceding stores to be
234  * synchronized wrt the i$, regardless of address or ASI.  In fact,
235  * the address is ignored, so we always flush address 0.
236  */
237 /*ARGSUSED*/
238 void
239 dtrace_flush_sec(uintptr_t addr)
240 {
241 	doflush(0);
242 }
243 
244 /*
245  * Trapstat support for Niagara2 processor
246  * The Niagara2 provides HWTW support for TSB lookup and with HWTW
247  * enabled no TSB hit information will be available. Therefore setting
248  * the time spent in TLB miss handler for TSB hits to 0.
249  */
250 int
251 cpu_trapstat_conf(int cmd)
252 {
253 	int status = 0;
254 
255 	switch (cmd) {
256 	case CPU_TSTATCONF_INIT:
257 	case CPU_TSTATCONF_FINI:
258 	case CPU_TSTATCONF_ENABLE:
259 	case CPU_TSTATCONF_DISABLE:
260 		break;
261 	default:
262 		status = EINVAL;
263 		break;
264 	}
265 	return (status);
266 }
267 
268 void
269 cpu_trapstat_data(void *buf, uint_t tstat_pgszs)
270 {
271 	tstat_pgszdata_t	*tstatp = (tstat_pgszdata_t *)buf;
272 	int	i;
273 
274 	for (i = 0; i < tstat_pgszs; i++, tstatp++) {
275 		tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0;
276 		tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0;
277 		tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0;
278 		tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0;
279 		tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0;
280 		tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0;
281 		tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0;
282 		tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0;
283 	}
284 }
285 
286 /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */
287 uint_t
288 page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc)
289 {
290 	uint_t color;
291 
292 	ASSERT(szc <= TTE256M);
293 
294 	pfn = PFN_BASE(pfn, szc);
295 	color = ((pfn >> 15) ^ pfn) & 0x1f;
296 	if (szc >= TTE4M)
297 		return (color);
298 
299 	color = (color << 2) | ((pfn >> 5) & 0x3);
300 
301 	return (szc <= TTE64K ? color : (color >> 1));
302 }
303 
304 #if TTE256M != 5
305 #error TTE256M is not 5
306 #endif
307 
308 uint_t
309 page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask)
310 {
311 	static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f};
312 	ASSERT(szc < TTE256M);
313 
314 	mask &= ni2_color_masks[szc];
315 	return ((szc == TTE64K || szc == TTE512K) ? (mask >> 1) : mask);
316 }
317 
318 uint_t
319 page_get_nsz_color_cpu(uchar_t szc, uint_t color)
320 {
321 	ASSERT(szc < TTE256M);
322 	return ((szc == TTE64K || szc == TTE512K) ? (color >> 1) : color);
323 }
324 
325 uint_t
326 page_get_color_shift_cpu(uchar_t szc, uchar_t nszc)
327 {
328 	ASSERT(nszc >= szc);
329 	ASSERT(nszc <= TTE256M);
330 
331 	if (szc == nszc)
332 		return (0);
333 	if (szc <= TTE64K)
334 		return ((nszc >= TTE4M) ? 2 : ((nszc >= TTE512K) ? 1 : 0));
335 	if (szc == TTE512K)
336 		return (1);
337 
338 	return (0);
339 }
340 
341 /*ARGSUSED*/
342 pfn_t
343 page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color,
344     uint_t ceq_mask, uint_t color_mask)
345 {
346 	pfn_t pstep = PNUM_SIZE(szc);
347 	pfn_t npfn, pfn_ceq_mask, pfn_color;
348 	pfn_t tmpmask, mask = (pfn_t)-1;
349 
350 	ASSERT((color & ~ceq_mask) == 0);
351 
352 	if (((page_pfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0) {
353 
354 		/* we start from the page with correct color */
355 		if (szc >= TTE512K) {
356 			if (szc >= TTE4M) {
357 				/* page color is PA[32:28] */
358 				pfn_ceq_mask = ceq_mask << 15;
359 			} else {
360 				/* page color is PA[32:28].PA[19:19] */
361 				pfn_ceq_mask = ((ceq_mask & 1) << 6) |
362 				    ((ceq_mask >> 1) << 15);
363 			}
364 			pfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask);
365 			return (pfn);
366 		} else {
367 			/*
368 			 * We deal 64K or 8K page. Check if we could the
369 			 * satisfy the request without changing PA[32:28]
370 			 */
371 			pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2);
372 			npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask);
373 
374 			if ((((npfn ^ pfn) >> 15) & 0x1f) == 0)
375 				return (npfn);
376 
377 			/*
378 			 * for next pfn we have to change bits PA[32:28]
379 			 * set PA[63:28] and PA[19:18] of the next pfn
380 			 */
381 			npfn = (pfn >> 15) << 15;
382 			npfn |= (ceq_mask & color & 3) << 5;
383 			pfn_ceq_mask = (szc == TTE8K) ? 0 :
384 			    (ceq_mask & 0x1c) << 13;
385 			npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask);
386 
387 			/*
388 			 * set bits PA[17:13] to match the color
389 			 */
390 			ceq_mask >>= 2;
391 			color = (color >> 2) & ceq_mask;
392 			npfn |= ((npfn >> 15) ^ color) & ceq_mask;
393 			return (npfn);
394 		}
395 	}
396 
397 	/*
398 	 * we start from the page with incorrect color - rare case
399 	 */
400 	if (szc >= TTE512K) {
401 		if (szc >= TTE4M) {
402 			/* page color is in bits PA[32:28] */
403 			npfn = ((pfn >> 20) << 20) | (color << 15);
404 			pfn_ceq_mask = (ceq_mask << 15) | 0x7fff;
405 		} else {
406 			/* try get the right color by changing bit PA[19:19] */
407 			npfn = pfn + pstep;
408 			if (((page_pfn_2_color_cpu(npfn, szc) ^ color) &
409 			    ceq_mask) == 0)
410 				return (npfn);
411 
412 			/* page color is PA[32:28].PA[19:19] */
413 			pfn_ceq_mask = ((ceq_mask & 1) << 6) |
414 			    ((ceq_mask >> 1) << 15) | (0xff << 7);
415 			pfn_color = ((color & 1) << 6) | ((color >> 1) << 15);
416 			npfn = ((pfn >> 20) << 20) | pfn_color;
417 		}
418 
419 		while (npfn <= pfn) {
420 			npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask);
421 		}
422 		return (npfn);
423 	}
424 
425 	/*
426 	 * We deal 64K or 8K page of incorrect color.
427 	 * Try correcting color without changing PA[32:28]
428 	 */
429 
430 	pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2);
431 	pfn_color = ((color & 3) << 5) | (color >> 2);
432 	npfn = (pfn & ~(pfn_t)0x7f);
433 	npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask;
434 	npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn;
435 
436 	if (((page_pfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0) {
437 
438 		/* the color is fixed - find the next page */
439 		while (npfn <= pfn) {
440 			npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask);
441 		}
442 		if ((((npfn ^ pfn) >> 15) & 0x1f) == 0)
443 			return (npfn);
444 	}
445 
446 	/* to fix the color need to touch PA[32:28] */
447 	npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) :
448 	    (((pfn >> 18) << 18) | ((color & 0x1c) << 13));
449 	tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13;
450 
451 	while (npfn <= pfn) {
452 		npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask);
453 	}
454 
455 	/* set bits PA[19:13] to match the color */
456 	npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask;
457 	npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn;
458 
459 	ASSERT(((page_pfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0);
460 
461 	return (npfn);
462 }
463 
464 /*
465  * init page coloring
466  */
467 void
468 page_coloring_init_cpu()
469 {
470 	int i;
471 
472 	hw_page_array[0].hp_colors = 1 << 7;
473 	hw_page_array[1].hp_colors = 1 << 7;
474 	hw_page_array[2].hp_colors = 1 << 6;
475 
476 	for (i = 3; i < mmu_page_sizes; i++) {
477 		hw_page_array[i].hp_colors = 1 << 5;
478 	}
479 }
480 
481 /*
482  * group colorequiv colors on N2 by low order bits of the color first
483  */
484 void
485 page_set_colorequiv_arr_cpu(void)
486 {
487 	static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0};
488 
489 	if (colorequiv > 1) {
490 		int i;
491 		uint_t sv_a = lowbit(colorequiv) - 1;
492 
493 		if (sv_a > 15)
494 			sv_a = 15;
495 
496 		for (i = 0; i < MMU_PAGE_SIZES; i++) {
497 			uint_t colors;
498 			uint_t a = sv_a;
499 
500 			if ((colors = hw_page_array[i].hp_colors) <= 1)
501 				continue;
502 			while ((colors >> a) == 0)
503 				a--;
504 			if (a > (colorequivszc[i] & 0xf) +
505 			    (colorequivszc[i] >> 4)) {
506 				if (a <= nequiv_shades_log2[i]) {
507 					colorequivszc[i] = (uchar_t)a;
508 				} else {
509 					colorequivszc[i] =
510 					    ((a - nequiv_shades_log2[i]) << 4) |
511 					    nequiv_shades_log2[i];
512 				}
513 			}
514 		}
515 	}
516 }
517