xref: /titanic_52/usr/src/uts/sun4u/cpu/us3_common_asm.s (revision 24fe0b3bf671e123467ce1df0b67cadd3614c8e4)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 *
25 * Assembly code support for Cheetah/Cheetah+ modules
26 */
27
28#if !defined(lint)
29#include "assym.h"
30#endif	/* !lint */
31
32#include <sys/asm_linkage.h>
33#include <sys/mmu.h>
34#include <vm/hat_sfmmu.h>
35#include <sys/machparam.h>
36#include <sys/machcpuvar.h>
37#include <sys/machthread.h>
38#include <sys/machtrap.h>
39#include <sys/privregs.h>
40#include <sys/trap.h>
41#include <sys/cheetahregs.h>
42#include <sys/us3_module.h>
43#include <sys/xc_impl.h>
44#include <sys/intreg.h>
45#include <sys/async.h>
46#include <sys/clock.h>
47#include <sys/cheetahasm.h>
48#include <sys/cmpregs.h>
49
50#ifdef TRAPTRACE
51#include <sys/traptrace.h>
52#endif /* TRAPTRACE */
53
54#if !defined(lint)
55
56/* BEGIN CSTYLED */
57
58#define	DCACHE_FLUSHPAGE(arg1, arg2, tmp1, tmp2, tmp3)			\
59	ldxa	[%g0]ASI_DCU, tmp1					;\
60	btst	DCU_DC, tmp1		/* is dcache enabled? */	;\
61	bz,pn	%icc, 1f						;\
62	ASM_LD(tmp1, dcache_linesize)					;\
63	ASM_LD(tmp2, dflush_type)					;\
64	cmp	tmp2, FLUSHPAGE_TYPE					;\
65	be,pt	%icc, 2f						;\
66	nop								;\
67	sllx	arg1, CHEETAH_DC_VBIT_SHIFT, arg1/* tag to compare */	;\
68	ASM_LD(tmp3, dcache_size)					;\
69	cmp	tmp2, FLUSHMATCH_TYPE					;\
70	be,pt	%icc, 3f						;\
71	nop								;\
72	/*								\
73	 * flushtype = FLUSHALL_TYPE, flush the whole thing		\
74	 * tmp3 = cache size						\
75	 * tmp1 = cache line size					\
76	 */								\
77	sub	tmp3, tmp1, tmp2					;\
784:									\
79	stxa	%g0, [tmp2]ASI_DC_TAG					;\
80	membar	#Sync							;\
81	cmp	%g0, tmp2						;\
82	bne,pt	%icc, 4b						;\
83	sub	tmp2, tmp1, tmp2					;\
84	ba,pt	%icc, 1f						;\
85	nop								;\
86	/*								\
87	 * flushtype = FLUSHPAGE_TYPE					\
88	 * arg1 = pfn							\
89	 * arg2 = virtual color						\
90	 * tmp1 = cache line size					\
91	 * tmp2 = tag from cache					\
92	 * tmp3 = counter						\
93	 */								\
942:									\
95	set	MMU_PAGESIZE, tmp3					;\
96        sllx    arg1, MMU_PAGESHIFT, arg1  /* pfn to 43 bit PA	   */   ;\
97	sub	tmp3, tmp1, tmp3					;\
984:									\
99	stxa	%g0, [arg1 + tmp3]ASI_DC_INVAL				;\
100	membar	#Sync							;\
1015:									\
102	cmp	%g0, tmp3						;\
103	bnz,pt	%icc, 4b		/* branch if not done */	;\
104	sub	tmp3, tmp1, tmp3					;\
105	ba,pt	%icc, 1f						;\
106	nop								;\
107	/*								\
108	 * flushtype = FLUSHMATCH_TYPE					\
109	 * arg1 = tag to compare against				\
110	 * tmp1 = cache line size					\
111	 * tmp3 = cache size						\
112	 * arg2 = counter						\
113	 * tmp2 = cache tag						\
114	 */								\
1153:									\
116	sub	tmp3, tmp1, arg2					;\
1174:									\
118	ldxa	[arg2]ASI_DC_TAG, tmp2		/* read tag */		;\
119	btst	CHEETAH_DC_VBIT_MASK, tmp2				;\
120	bz,pn	%icc, 5f		/* br if no valid sub-blocks */	;\
121	andn	tmp2, CHEETAH_DC_VBIT_MASK, tmp2 /* clear out v bits */	;\
122	cmp	tmp2, arg1						;\
123	bne,pn	%icc, 5f		/* branch if tag miss */	;\
124	nop								;\
125	stxa	%g0, [arg2]ASI_DC_TAG					;\
126	membar	#Sync							;\
1275:									\
128	cmp	%g0, arg2						;\
129	bne,pt	%icc, 4b		/* branch if not done */	;\
130	sub	arg2, tmp1, arg2					;\
1311:
132
133/*
134 * macro that flushes the entire dcache color
135 * dcache size = 64K, one way 16K
136 */
137#define DCACHE_FLUSHCOLOR(arg, way, tmp1, tmp2, tmp3)			\
138	ldxa	[%g0]ASI_DCU, tmp1;					\
139	btst	DCU_DC, tmp1;		/* is dcache enabled? */	\
140	bz,pn	%icc, 1f;						\
141	ASM_LD(tmp1, dcache_linesize)					\
142	set	MMU_PAGESIZE, tmp2;					\
143	/*								\
144	 * arg = virtual color						\
145	 * tmp2 = page size						\
146	 * tmp1 = cache line size					\
147	 */								\
148	sllx	arg, MMU_PAGESHIFT, arg; /* color to dcache page */	\
149	mov	way, tmp3;						\
150	sllx	tmp3, 14, tmp3;		 /* One way 16K */		\
151	or	arg, tmp3, arg;						\
152	sub	tmp2, tmp1, tmp2;					\
1532:									\
154	stxa	%g0, [arg + tmp2]ASI_DC_TAG;				\
155	membar	#Sync;							\
156	cmp	%g0, tmp2;						\
157	bne,pt	%icc, 2b;						\
158	  sub	tmp2, tmp1, tmp2;					\
1591:
160
161/* END CSTYLED */
162
163#endif	/* !lint */
164
165/*
166 * Cheetah MMU and Cache operations.
167 */
168
169#if defined(lint)
170
171/* ARGSUSED */
172void
173vtag_flushpage(caddr_t vaddr, uint64_t sfmmup)
174{}
175
176#else	/* lint */
177
178	ENTRY_NP(vtag_flushpage)
179	/*
180	 * flush page from the tlb
181	 *
182	 * %o0 = vaddr
183	 * %o1 = sfmmup
184	 */
185	rdpr	%pstate, %o5
186#ifdef DEBUG
187	PANIC_IF_INTR_DISABLED_PSTR(%o5, u3_di_label0, %g1)
188#endif /* DEBUG */
189	/*
190	 * disable ints
191	 */
192	andn	%o5, PSTATE_IE, %o4
193	wrpr	%o4, 0, %pstate
194
195	/*
196	 * Then, blow out the tlb
197	 * Interrupts are disabled to prevent the primary ctx register
198	 * from changing underneath us.
199	 */
200	sethi   %hi(ksfmmup), %o3
201        ldx     [%o3 + %lo(ksfmmup)], %o3
202        cmp     %o3, %o1
203        bne,pt   %xcc, 1f			! if not kernel as, go to 1
204	  sethi	%hi(FLUSH_ADDR), %o3
205	/*
206	 * For Kernel demaps use primary. type = page implicitly
207	 */
208	stxa	%g0, [%o0]ASI_DTLB_DEMAP	/* dmmu flush for KCONTEXT */
209	stxa	%g0, [%o0]ASI_ITLB_DEMAP	/* immu flush for KCONTEXT */
210	flush	%o3
211	retl
212	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
2131:
214	/*
215	 * User demap.  We need to set the primary context properly.
216	 * Secondary context cannot be used for Cheetah IMMU.
217	 * %o0 = vaddr
218	 * %o1 = sfmmup
219	 * %o3 = FLUSH_ADDR
220	 */
221	SFMMU_CPU_CNUM(%o1, %g1, %g2)		! %g1 = sfmmu cnum on this CPU
222
223	ldub	[%o1 + SFMMU_CEXT], %o4		! %o4 = sfmmup->sfmmu_cext
224	sll	%o4, CTXREG_EXT_SHIFT, %o4
225	or	%g1, %o4, %g1			! %g1 = primary pgsz | cnum
226
227	wrpr	%g0, 1, %tl
228	set	MMU_PCONTEXT, %o4
229	or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %o0, %o0
230	ldxa	[%o4]ASI_DMMU, %o2		! %o2 = save old ctxnum
231	srlx	%o2, CTXREG_NEXT_SHIFT, %o1	! need to preserve nucleus pgsz
232	sllx	%o1, CTXREG_NEXT_SHIFT, %o1	! %o1 = nucleus pgsz
233	or	%g1, %o1, %g1			! %g1 = nucleus pgsz | primary pgsz | cnum
234	stxa	%g1, [%o4]ASI_DMMU		! wr new ctxum
235
236	stxa	%g0, [%o0]ASI_DTLB_DEMAP
237	stxa	%g0, [%o0]ASI_ITLB_DEMAP
238	stxa	%o2, [%o4]ASI_DMMU		/* restore old ctxnum */
239	flush	%o3
240	wrpr	%g0, 0, %tl
241
242	retl
243	wrpr	%g0, %o5, %pstate		/* enable interrupts */
244	SET_SIZE(vtag_flushpage)
245
246#endif	/* lint */
247
248#if defined(lint)
249
250void
251vtag_flushall(void)
252{}
253
254#else	/* lint */
255
256	ENTRY_NP2(vtag_flushall, demap_all)
257	/*
258	 * flush the tlb
259	 */
260	sethi	%hi(FLUSH_ADDR), %o3
261	set	DEMAP_ALL_TYPE, %g1
262	stxa	%g0, [%g1]ASI_DTLB_DEMAP
263	stxa	%g0, [%g1]ASI_ITLB_DEMAP
264	flush	%o3
265	retl
266	nop
267	SET_SIZE(demap_all)
268	SET_SIZE(vtag_flushall)
269
270#endif	/* lint */
271
272
273#if defined(lint)
274
275/* ARGSUSED */
276void
277vtag_flushpage_tl1(uint64_t vaddr, uint64_t sfmmup)
278{}
279
280#else	/* lint */
281
282	ENTRY_NP(vtag_flushpage_tl1)
283	/*
284	 * x-trap to flush page from tlb and tsb
285	 *
286	 * %g1 = vaddr, zero-extended on 32-bit kernel
287	 * %g2 = sfmmup
288	 *
289	 * assumes TSBE_TAG = 0
290	 */
291	srln	%g1, MMU_PAGESHIFT, %g1
292
293	sethi   %hi(ksfmmup), %g3
294        ldx     [%g3 + %lo(ksfmmup)], %g3
295        cmp     %g3, %g2
296        bne,pt	%xcc, 1f                        ! if not kernel as, go to 1
297	  slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */
298
299	/* We need to demap in the kernel context */
300	or	DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1
301	stxa	%g0, [%g1]ASI_DTLB_DEMAP
302	stxa	%g0, [%g1]ASI_ITLB_DEMAP
303	retry
3041:
305	/* We need to demap in a user context */
306	or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1
307
308	SFMMU_CPU_CNUM(%g2, %g6, %g3)	! %g6 = sfmmu cnum on this CPU
309
310	ldub	[%g2 + SFMMU_CEXT], %g4		! %g4 = sfmmup->cext
311	sll	%g4, CTXREG_EXT_SHIFT, %g4
312	or	%g6, %g4, %g6			! %g6 = pgsz | cnum
313
314	set	MMU_PCONTEXT, %g4
315	ldxa	[%g4]ASI_DMMU, %g5		/* rd old ctxnum */
316	srlx	%g5, CTXREG_NEXT_SHIFT, %g2	/* %g2 = nucleus pgsz */
317	sllx	%g2, CTXREG_NEXT_SHIFT, %g2	/* preserve nucleus pgsz */
318	or	%g6, %g2, %g6			/* %g6 = nucleus pgsz | primary pgsz | cnum */
319	stxa	%g6, [%g4]ASI_DMMU		/* wr new ctxum */
320	stxa	%g0, [%g1]ASI_DTLB_DEMAP
321	stxa	%g0, [%g1]ASI_ITLB_DEMAP
322	stxa	%g5, [%g4]ASI_DMMU		/* restore old ctxnum */
323	retry
324	SET_SIZE(vtag_flushpage_tl1)
325
326#endif	/* lint */
327
328
329#if defined(lint)
330
331/* ARGSUSED */
332void
333vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t sfmmup_pgcnt)
334{}
335
336#else	/* lint */
337
338	ENTRY_NP(vtag_flush_pgcnt_tl1)
339	/*
340	 * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb
341	 *
342	 * %g1 = vaddr, zero-extended on 32-bit kernel
343	 * %g2 = <sfmmup58|pgcnt6>, (pgcnt - 1) is pass'ed in via pgcnt6 bits.
344	 *
345	 * NOTE: this handler relies on the fact that no
346	 *	interrupts or traps can occur during the loop
347	 *	issuing the TLB_DEMAP operations. It is assumed
348	 *	that interrupts are disabled and this code is
349	 *	fetching from the kernel locked text address.
350	 *
351	 * assumes TSBE_TAG = 0
352	 */
353	set	SFMMU_PGCNT_MASK, %g4
354	and	%g4, %g2, %g3			/* g3 = pgcnt - 1 */
355	add	%g3, 1, %g3			/* g3 = pgcnt */
356
357	andn	%g2, SFMMU_PGCNT_MASK, %g2	/* g2 = sfmmup */
358	srln	%g1, MMU_PAGESHIFT, %g1
359
360	sethi   %hi(ksfmmup), %g4
361        ldx     [%g4 + %lo(ksfmmup)], %g4
362        cmp     %g4, %g2
363        bne,pn   %xcc, 1f			/* if not kernel as, go to 1 */
364	  slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */
365
366	/* We need to demap in the kernel context */
367	or	DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1
368	set	MMU_PAGESIZE, %g2		/* g2 = pgsize */
369	sethi   %hi(FLUSH_ADDR), %g5
3704:
371	stxa	%g0, [%g1]ASI_DTLB_DEMAP
372	stxa	%g0, [%g1]ASI_ITLB_DEMAP
373	flush	%g5				! flush required by immu
374
375	deccc	%g3				/* decr pgcnt */
376	bnz,pt	%icc,4b
377	  add	%g1, %g2, %g1			/* next page */
378	retry
3791:
380	/*
381	 * We need to demap in a user context
382	 *
383	 * g2 = sfmmup
384	 * g3 = pgcnt
385	 */
386	SFMMU_CPU_CNUM(%g2, %g5, %g6)		! %g5 = sfmmu cnum on this CPU
387
388	or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1
389
390	ldub	[%g2 + SFMMU_CEXT], %g4		! %g4 = sfmmup->cext
391	sll	%g4, CTXREG_EXT_SHIFT, %g4
392	or	%g5, %g4, %g5
393
394	set	MMU_PCONTEXT, %g4
395	ldxa	[%g4]ASI_DMMU, %g6		/* rd old ctxnum */
396	srlx	%g6, CTXREG_NEXT_SHIFT, %g2	/* %g2 = nucleus pgsz */
397	sllx	%g2, CTXREG_NEXT_SHIFT, %g2	/* preserve nucleus pgsz */
398	or	%g5, %g2, %g5			/* %g5 = nucleus pgsz | primary pgsz | cnum */
399	stxa	%g5, [%g4]ASI_DMMU		/* wr new ctxum */
400
401	set	MMU_PAGESIZE, %g2		/* g2 = pgsize */
402	sethi   %hi(FLUSH_ADDR), %g5
4033:
404	stxa	%g0, [%g1]ASI_DTLB_DEMAP
405	stxa	%g0, [%g1]ASI_ITLB_DEMAP
406	flush	%g5				! flush required by immu
407
408	deccc	%g3				/* decr pgcnt */
409	bnz,pt	%icc,3b
410	  add	%g1, %g2, %g1			/* next page */
411
412	stxa	%g6, [%g4]ASI_DMMU		/* restore old ctxnum */
413	retry
414	SET_SIZE(vtag_flush_pgcnt_tl1)
415
416#endif	/* lint */
417
418#if defined(lint)
419
420/*ARGSUSED*/
421void
422vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2)
423{}
424
425#else	/* lint */
426
427	ENTRY_NP(vtag_flushall_tl1)
428	/*
429	 * x-trap to flush tlb
430	 */
431	set	DEMAP_ALL_TYPE, %g4
432	stxa	%g0, [%g4]ASI_DTLB_DEMAP
433	stxa	%g0, [%g4]ASI_ITLB_DEMAP
434	retry
435	SET_SIZE(vtag_flushall_tl1)
436
437#endif	/* lint */
438
439
440#if defined(lint)
441
442/* ARGSUSED */
443void
444vac_flushpage(pfn_t pfnum, int vcolor)
445{}
446
447#else	/* lint */
448
449/*
450 * vac_flushpage(pfnum, color)
451 *	Flush 1 8k page of the D-$ with physical page = pfnum
452 *	Algorithm:
453 *		The cheetah dcache is a 64k psuedo 4 way accaociative cache.
454 *		It is virtual indexed, physically tagged cache.
455 */
456	.seg	".data"
457	.align	8
458	.global	dflush_type
459dflush_type:
460	.word	FLUSHPAGE_TYPE
461
462	ENTRY(vac_flushpage)
463	/*
464	 * flush page from the d$
465	 *
466	 * %o0 = pfnum, %o1 = color
467	 */
468	DCACHE_FLUSHPAGE(%o0, %o1, %o2, %o3, %o4)
469	retl
470	  nop
471	SET_SIZE(vac_flushpage)
472
473#endif	/* lint */
474
475
476#if defined(lint)
477
478/* ARGSUSED */
479void
480vac_flushpage_tl1(uint64_t pfnum, uint64_t vcolor)
481{}
482
483#else	/* lint */
484
485	ENTRY_NP(vac_flushpage_tl1)
486	/*
487	 * x-trap to flush page from the d$
488	 *
489	 * %g1 = pfnum, %g2 = color
490	 */
491	DCACHE_FLUSHPAGE(%g1, %g2, %g3, %g4, %g5)
492	retry
493	SET_SIZE(vac_flushpage_tl1)
494
495#endif	/* lint */
496
497
498#if defined(lint)
499
500/* ARGSUSED */
501void
502vac_flushcolor(int vcolor, pfn_t pfnum)
503{}
504
505#else	/* lint */
506
507	ENTRY(vac_flushcolor)
508	/*
509	 * %o0 = vcolor
510	 */
511	DCACHE_FLUSHCOLOR(%o0, 0, %o1, %o2, %o3)
512	DCACHE_FLUSHCOLOR(%o0, 1, %o1, %o2, %o3)
513	DCACHE_FLUSHCOLOR(%o0, 2, %o1, %o2, %o3)
514	DCACHE_FLUSHCOLOR(%o0, 3, %o1, %o2, %o3)
515	retl
516	  nop
517	SET_SIZE(vac_flushcolor)
518
519#endif	/* lint */
520
521
522#if defined(lint)
523
524/* ARGSUSED */
525void
526vac_flushcolor_tl1(uint64_t vcolor, uint64_t pfnum)
527{}
528
529#else	/* lint */
530
531	ENTRY(vac_flushcolor_tl1)
532	/*
533	 * %g1 = vcolor
534	 */
535	DCACHE_FLUSHCOLOR(%g1, 0, %g2, %g3, %g4)
536	DCACHE_FLUSHCOLOR(%g1, 1, %g2, %g3, %g4)
537	DCACHE_FLUSHCOLOR(%g1, 2, %g2, %g3, %g4)
538	DCACHE_FLUSHCOLOR(%g1, 3, %g2, %g3, %g4)
539	retry
540	SET_SIZE(vac_flushcolor_tl1)
541
542#endif	/* lint */
543
544#if defined(lint)
545
546int
547idsr_busy(void)
548{
549	return (0);
550}
551
552#else	/* lint */
553
554/*
555 * Determine whether or not the IDSR is busy.
556 * Entry: no arguments
557 * Returns: 1 if busy, 0 otherwise
558 */
559	ENTRY(idsr_busy)
560	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
561	clr	%o0
562	btst	IDSR_BUSY, %g1
563	bz,a,pt	%xcc, 1f
564	mov	1, %o0
5651:
566	retl
567	nop
568	SET_SIZE(idsr_busy)
569
570#endif	/* lint */
571
572#if defined(lint)
573
574/* ARGSUSED */
575void
576init_mondo(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
577{}
578
579/* ARGSUSED */
580void
581init_mondo_nocheck(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
582{}
583
584#else	/* lint */
585
586	.global _dispatch_status_busy
587_dispatch_status_busy:
588	.asciz	"ASI_INTR_DISPATCH_STATUS error: busy"
589	.align	4
590
591/*
592 * Setup interrupt dispatch data registers
593 * Entry:
594 *	%o0 - function or inumber to call
595 *	%o1, %o2 - arguments (2 uint64_t's)
596 */
597	.seg "text"
598
599	ENTRY(init_mondo)
600#ifdef DEBUG
601	!
602	! IDSR should not be busy at the moment
603	!
604	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
605	btst	IDSR_BUSY, %g1
606	bz,pt	%xcc, 1f
607	nop
608	sethi	%hi(_dispatch_status_busy), %o0
609	call	panic
610	or	%o0, %lo(_dispatch_status_busy), %o0
611#endif /* DEBUG */
612
613	ALTENTRY(init_mondo_nocheck)
614	!
615	! interrupt vector dispatch data reg 0
616	!
6171:
618	mov	IDDR_0, %g1
619	mov	IDDR_1, %g2
620	mov	IDDR_2, %g3
621	stxa	%o0, [%g1]ASI_INTR_DISPATCH
622
623	!
624	! interrupt vector dispatch data reg 1
625	!
626	stxa	%o1, [%g2]ASI_INTR_DISPATCH
627
628	!
629	! interrupt vector dispatch data reg 2
630	!
631	stxa	%o2, [%g3]ASI_INTR_DISPATCH
632
633	membar	#Sync
634	retl
635	nop
636	SET_SIZE(init_mondo_nocheck)
637	SET_SIZE(init_mondo)
638
639#endif	/* lint */
640
641
642#if !(defined(JALAPENO) || defined(SERRANO))
643
644#if defined(lint)
645
646/* ARGSUSED */
647void
648shipit(int upaid, int bn)
649{ return; }
650
651#else	/* lint */
652
653/*
654 * Ship mondo to aid using busy/nack pair bn
655 */
656	ENTRY_NP(shipit)
657	sll	%o0, IDCR_PID_SHIFT, %g1	! IDCR<18:14> = agent id
658	sll	%o1, IDCR_BN_SHIFT, %g2		! IDCR<28:24> = b/n pair
659	or	%g1, IDCR_OFFSET, %g1		! IDCR<13:0> = 0x70
660	or	%g1, %g2, %g1
661	stxa	%g0, [%g1]ASI_INTR_DISPATCH	! interrupt vector dispatch
662	membar	#Sync
663	retl
664	nop
665	SET_SIZE(shipit)
666
667#endif	/* lint */
668
669#endif	/* !(JALAPENO || SERRANO) */
670
671
672#if defined(lint)
673
674/* ARGSUSED */
675void
676flush_instr_mem(caddr_t vaddr, size_t len)
677{}
678
679#else	/* lint */
680
681/*
682 * flush_instr_mem:
683 *	Flush 1 page of the I-$ starting at vaddr
684 * 	%o0 vaddr
685 *	%o1 bytes to be flushed
686 * UltraSPARC-III maintains consistency of the on-chip Instruction Cache with
687 * the stores from all processors so that a FLUSH instruction is only needed
688 * to ensure pipeline is consistent. This means a single flush is sufficient at
689 * the end of a sequence of stores that updates the instruction stream to
690 * ensure correct operation.
691 */
692
693	ENTRY(flush_instr_mem)
694	flush	%o0			! address irrelevant
695	retl
696	nop
697	SET_SIZE(flush_instr_mem)
698
699#endif	/* lint */
700
701
702#if defined(CPU_IMP_ECACHE_ASSOC)
703
704#if defined(lint)
705
706/* ARGSUSED */
707uint64_t
708get_ecache_ctrl(void)
709{ return (0); }
710
711#else	/* lint */
712
713	ENTRY(get_ecache_ctrl)
714	GET_CPU_IMPL(%o0)
715	cmp	%o0, JAGUAR_IMPL
716	!
717	! Putting an ASI access in the delay slot may
718	! cause it to be accessed, even when annulled.
719	!
720	bne	1f
721	  nop
722	ldxa	[%g0]ASI_EC_CFG_TIMING, %o0	! read Jaguar shared E$ ctrl reg
723	b	2f
724	  nop
7251:
726	ldxa	[%g0]ASI_EC_CTRL, %o0		! read Ch/Ch+ E$ control reg
7272:
728	retl
729	  nop
730	SET_SIZE(get_ecache_ctrl)
731
732#endif	/* lint */
733
734#endif	/* CPU_IMP_ECACHE_ASSOC */
735
736
737#if !(defined(JALAPENO) || defined(SERRANO))
738
739/*
740 * flush_ecache:
741 *	%o0 - 64 bit physical address
742 *	%o1 - ecache size
743 *	%o2 - ecache linesize
744 */
745#if defined(lint)
746
747/*ARGSUSED*/
748void
749flush_ecache(uint64_t physaddr, size_t ecache_size, size_t ecache_linesize)
750{}
751
752#else /* !lint */
753
754	ENTRY(flush_ecache)
755
756	/*
757	 * For certain CPU implementations, we have to flush the L2 cache
758	 * before flushing the ecache.
759	 */
760	PN_L2_FLUSHALL(%g3, %g4, %g5)
761
762	/*
763	 * Flush the entire Ecache using displacement flush.
764	 */
765	ECACHE_FLUSHALL(%o1, %o2, %o0, %o4)
766
767	retl
768	nop
769	SET_SIZE(flush_ecache)
770
771#endif /* lint */
772
773#endif	/* !(JALAPENO || SERRANO) */
774
775
776#if defined(lint)
777
778void
779flush_dcache(void)
780{}
781
782#else	/* lint */
783
784	ENTRY(flush_dcache)
785	ASM_LD(%o0, dcache_size)
786	ASM_LD(%o1, dcache_linesize)
787	CH_DCACHE_FLUSHALL(%o0, %o1, %o2)
788	retl
789	nop
790	SET_SIZE(flush_dcache)
791
792#endif	/* lint */
793
794
795#if defined(lint)
796
797void
798flush_icache(void)
799{}
800
801#else	/* lint */
802
803	ENTRY(flush_icache)
804	GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, flush_icache_1);
805	ld	[%o0 + CHPR_ICACHE_LINESIZE], %o1
806	ba,pt	%icc, 2f
807	  ld	[%o0 + CHPR_ICACHE_SIZE], %o0
808flush_icache_1:
809	ASM_LD(%o0, icache_size)
810	ASM_LD(%o1, icache_linesize)
8112:
812	CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4)
813	retl
814	nop
815	SET_SIZE(flush_icache)
816
817#endif	/* lint */
818
819#if defined(lint)
820
821/*ARGSUSED*/
822void
823kdi_flush_idcache(int dcache_size, int dcache_lsize, int icache_size,
824    int icache_lsize)
825{
826}
827
828#else	/* lint */
829
830	ENTRY(kdi_flush_idcache)
831	CH_DCACHE_FLUSHALL(%o0, %o1, %g1)
832	CH_ICACHE_FLUSHALL(%o2, %o3, %g1, %g2)
833	membar	#Sync
834	retl
835	nop
836	SET_SIZE(kdi_flush_idcache)
837
838#endif	/* lint */
839
840#if defined(lint)
841
842void
843flush_pcache(void)
844{}
845
846#else	/* lint */
847
848	ENTRY(flush_pcache)
849	PCACHE_FLUSHALL(%o0, %o1, %o2)
850	retl
851	nop
852	SET_SIZE(flush_pcache)
853
854#endif	/* lint */
855
856
857#if defined(CPU_IMP_L1_CACHE_PARITY)
858
859#if defined(lint)
860
861/* ARGSUSED */
862void
863get_dcache_dtag(uint32_t dcache_idx, uint64_t *data)
864{}
865
866#else	/* lint */
867
868/*
869 * Get dcache data and tag.  The Dcache data is a pointer to a ch_dc_data_t
870 * structure (see cheetahregs.h):
871 * The Dcache *should* be turned off when this code is executed.
872 */
873	.align	128
874	ENTRY(get_dcache_dtag)
875	rdpr	%pstate, %o5
876	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
877	wrpr	%g0, %o3, %pstate
878	b	1f
879	  stx	%o0, [%o1 + CH_DC_IDX]
880
881	.align	128
8821:
883	ldxa	[%o0]ASI_DC_TAG, %o2
884	stx	%o2, [%o1 + CH_DC_TAG]
885	membar	#Sync
886	ldxa	[%o0]ASI_DC_UTAG, %o2
887	membar	#Sync
888	stx	%o2, [%o1 + CH_DC_UTAG]
889	ldxa	[%o0]ASI_DC_SNP_TAG, %o2
890	stx	%o2, [%o1 + CH_DC_SNTAG]
891	add	%o1, CH_DC_DATA, %o1
892	clr	%o3
8932:
894	membar	#Sync				! required before ASI_DC_DATA
895	ldxa	[%o0 + %o3]ASI_DC_DATA, %o2
896	membar	#Sync				! required after ASI_DC_DATA
897	stx	%o2, [%o1 + %o3]
898	cmp	%o3, CH_DC_DATA_REG_SIZE - 8
899	blt	2b
900	  add	%o3, 8, %o3
901
902	/*
903	 * Unlike other CPUs in the family, D$ data parity bits for Panther
904	 * do not reside in the microtag. Instead, we have to read them
905	 * using the DC_data_parity bit of ASI_DCACHE_DATA. Also, instead
906	 * of just having 8 parity bits to protect all 32 bytes of data
907	 * per line, we now have 32 bits of parity.
908	 */
909	GET_CPU_IMPL(%o3)
910	cmp	%o3, PANTHER_IMPL
911	bne	4f
912	  clr	%o3
913
914	/*
915	 * move our pointer to the next field where we store parity bits
916	 * and add the offset of the last parity byte since we will be
917	 * storing all 4 parity bytes within one 64 bit field like this:
918	 *
919	 * +------+------------+------------+------------+------------+
920	 * |  -   | DC_parity  | DC_parity  | DC_parity  | DC_parity  |
921	 * |  -   | for word 3 | for word 2 | for word 1 | for word 0 |
922	 * +------+------------+------------+------------+------------+
923	 *  63:32     31:24        23:16         15:8          7:0
924	 */
925	add	%o1, CH_DC_PN_DATA_PARITY - CH_DC_DATA + 7, %o1
926
927	/* add the DC_data_parity bit into our working index */
928	mov	1, %o2
929	sll	%o2, PN_DC_DATA_PARITY_BIT_SHIFT, %o2
930	or	%o0, %o2, %o0
9313:
932	membar	#Sync				! required before ASI_DC_DATA
933	ldxa	[%o0 + %o3]ASI_DC_DATA, %o2
934	membar	#Sync				! required after ASI_DC_DATA
935	stb	%o2, [%o1]
936	dec	%o1
937	cmp	%o3, CH_DC_DATA_REG_SIZE - 8
938	blt	3b
939	  add	%o3, 8, %o3
9404:
941	retl
942	  wrpr	%g0, %o5, %pstate
943	SET_SIZE(get_dcache_dtag)
944
945#endif	/* lint */
946
947
948#if defined(lint)
949
950/* ARGSUSED */
951void
952get_icache_dtag(uint32_t ecache_idx, uint64_t *data)
953{}
954
955#else	/* lint */
956
957/*
958 * Get icache data and tag.  The data argument is a pointer to a ch_ic_data_t
959 * structure (see cheetahregs.h):
960 * The Icache *Must* be turned off when this function is called.
961 * This is because diagnostic accesses to the Icache interfere with cache
962 * consistency.
963 */
964	.align	128
965	ENTRY(get_icache_dtag)
966	rdpr	%pstate, %o5
967	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
968	wrpr	%g0, %o3, %pstate
969
970	stx	%o0, [%o1 + CH_IC_IDX]
971	ldxa	[%o0]ASI_IC_TAG, %o2
972	stx	%o2, [%o1 + CH_IC_PATAG]
973	add	%o0, CH_ICTAG_UTAG, %o0
974	ldxa	[%o0]ASI_IC_TAG, %o2
975	add	%o0, (CH_ICTAG_UPPER - CH_ICTAG_UTAG), %o0
976	stx	%o2, [%o1 + CH_IC_UTAG]
977	ldxa	[%o0]ASI_IC_TAG, %o2
978	add	%o0, (CH_ICTAG_LOWER - CH_ICTAG_UPPER), %o0
979	stx	%o2, [%o1 + CH_IC_UPPER]
980	ldxa	[%o0]ASI_IC_TAG, %o2
981	andn	%o0, CH_ICTAG_TMASK, %o0
982	stx	%o2, [%o1 + CH_IC_LOWER]
983	ldxa	[%o0]ASI_IC_SNP_TAG, %o2
984	stx	%o2, [%o1 + CH_IC_SNTAG]
985	add	%o1, CH_IC_DATA, %o1
986	clr	%o3
9872:
988	ldxa	[%o0 + %o3]ASI_IC_DATA, %o2
989	stx	%o2, [%o1 + %o3]
990	cmp	%o3, PN_IC_DATA_REG_SIZE - 8
991	blt	2b
992	  add	%o3, 8, %o3
993
994	retl
995	  wrpr	%g0, %o5, %pstate
996	SET_SIZE(get_icache_dtag)
997
998#endif	/* lint */
999
1000#if defined(lint)
1001
1002/* ARGSUSED */
1003void
1004get_pcache_dtag(uint32_t pcache_idx, uint64_t *data)
1005{}
1006
1007#else	/* lint */
1008
1009/*
1010 * Get pcache data and tags.
1011 * inputs:
1012 *   pcache_idx	- fully constructed VA for for accessing P$ diagnostic
1013 *		  registers. Contains PC_way and PC_addr shifted into
1014 *		  the correct bit positions. See the PRM for more details.
1015 *   data	- pointer to a ch_pc_data_t
1016 * structure (see cheetahregs.h):
1017 */
1018	.align	128
1019	ENTRY(get_pcache_dtag)
1020	rdpr	%pstate, %o5
1021	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
1022	wrpr	%g0, %o3, %pstate
1023
1024	stx	%o0, [%o1 + CH_PC_IDX]
1025	ldxa	[%o0]ASI_PC_STATUS_DATA, %o2
1026	stx	%o2, [%o1 + CH_PC_STATUS]
1027	ldxa	[%o0]ASI_PC_TAG, %o2
1028	stx	%o2, [%o1 + CH_PC_TAG]
1029	ldxa	[%o0]ASI_PC_SNP_TAG, %o2
1030	stx	%o2, [%o1 + CH_PC_SNTAG]
1031	add	%o1, CH_PC_DATA, %o1
1032	clr	%o3
10332:
1034	ldxa	[%o0 + %o3]ASI_PC_DATA, %o2
1035	stx	%o2, [%o1 + %o3]
1036	cmp	%o3, CH_PC_DATA_REG_SIZE - 8
1037	blt	2b
1038	  add	%o3, 8, %o3
1039
1040	retl
1041	  wrpr	%g0, %o5, %pstate
1042	SET_SIZE(get_pcache_dtag)
1043
1044#endif	/* lint */
1045
1046#endif	/* CPU_IMP_L1_CACHE_PARITY */
1047
1048#if defined(lint)
1049
1050/* ARGSUSED */
1051void
1052set_dcu(uint64_t dcu)
1053{}
1054
1055#else	/* lint */
1056
1057/*
1058 * re-enable the i$, d$, w$, and p$ according to bootup cache state.
1059 * Turn on WE, HPE, SPE, PE, IC, and DC bits defined as DCU_CACHE.
1060 *   %o0 - 64 bit constant
1061 */
1062	ENTRY(set_dcu)
1063	stxa	%o0, [%g0]ASI_DCU	! Store to DCU
1064	flush	%g0	/* flush required after changing the IC bit */
1065	retl
1066	nop
1067	SET_SIZE(set_dcu)
1068
1069#endif	/* lint */
1070
1071
1072#if defined(lint)
1073
1074uint64_t
1075get_dcu(void)
1076{
1077	return ((uint64_t)0);
1078}
1079
1080#else	/* lint */
1081
1082/*
1083 * Return DCU register.
1084 */
1085	ENTRY(get_dcu)
1086	ldxa	[%g0]ASI_DCU, %o0		/* DCU control register */
1087	retl
1088	nop
1089	SET_SIZE(get_dcu)
1090
1091#endif	/* lint */
1092
1093/*
1094 * Cheetah/Cheetah+ level 15 interrupt handler trap table entry.
1095 *
1096 * This handler is used to check for softints generated by error trap
1097 * handlers to report errors.  On Cheetah, this mechanism is used by the
1098 * Fast ECC at TL>0 error trap handler and, on Cheetah+, by both the Fast
1099 * ECC at TL>0 error and the I$/D$ parity error at TL>0 trap handlers.
1100 * NB: Must be 8 instructions or less to fit in trap table and code must
1101 *     be relocatable.
1102 */
1103#if defined(lint)
1104
1105void
1106ch_pil15_interrupt_instr(void)
1107{}
1108
1109#else	/* lint */
1110
1111	ENTRY_NP(ch_pil15_interrupt_instr)
1112	ASM_JMP(%g1, ch_pil15_interrupt)
1113	SET_SIZE(ch_pil15_interrupt_instr)
1114
1115#endif
1116
1117
1118#if defined(lint)
1119
1120void
1121ch_pil15_interrupt(void)
1122{}
1123
1124#else	/* lint */
1125
1126	ENTRY_NP(ch_pil15_interrupt)
1127
1128	/*
1129	 * Since pil_interrupt is hacked to assume that every level 15
1130	 * interrupt is generated by the CPU to indicate a performance
1131	 * counter overflow this gets ugly.  Before calling pil_interrupt
1132	 * the Error at TL>0 pending status is inspected.  If it is
1133	 * non-zero, then an error has occurred and it is handled.
1134	 * Otherwise control is transfered to pil_interrupt.  Note that if
1135	 * an error is detected pil_interrupt will not be called and
1136	 * overflow interrupts may be lost causing erroneous performance
1137	 * measurements.  However, error-recovery will have a detrimental
1138	 * effect on performance anyway.
1139	 */
1140	CPU_INDEX(%g1, %g4)
1141	set	ch_err_tl1_pending, %g4
1142	ldub	[%g1 + %g4], %g2
1143	brz	%g2, 1f
1144	  nop
1145
1146	/*
1147	 * We have a pending TL>0 error, clear the TL>0 pending status.
1148	 */
1149	stb	%g0, [%g1 + %g4]
1150
1151	/*
1152	 * Clear the softint.
1153	 */
1154	mov	1, %g5
1155	sll	%g5, PIL_15, %g5
1156	wr	%g5, CLEAR_SOFTINT
1157
1158	/*
1159	 * For Cheetah*, call cpu_tl1_error via systrap at PIL 15
1160	 * to process the Fast ECC/Cache Parity at TL>0 error.  Clear
1161	 * panic flag (%g2).
1162	 */
1163	set	cpu_tl1_error, %g1
1164	clr	%g2
1165	ba	sys_trap
1166	  mov	PIL_15, %g4
1167
11681:
1169	/*
1170	 * The logout is invalid.
1171	 *
1172	 * Call the default interrupt handler.
1173	 */
1174	sethi	%hi(pil_interrupt), %g1
1175	jmp	%g1 + %lo(pil_interrupt)
1176	  mov	PIL_15, %g4
1177
1178	SET_SIZE(ch_pil15_interrupt)
1179#endif
1180
1181
1182/*
1183 * Error Handling
1184 *
1185 * Cheetah provides error checking for all memory access paths between
1186 * the CPU, External Cache, Cheetah Data Switch and system bus. Error
1187 * information is logged in the AFSR, (also AFSR_EXT for Panther) and
1188 * AFAR and one of the following traps is generated (provided that it
1189 * is enabled in External Cache Error Enable Register) to handle that
1190 * error:
1191 * 1. trap 0x70: Precise trap
1192 *    tt0_fecc for errors at trap level(TL)>=0
1193 * 2. trap 0x0A and 0x32: Deferred trap
1194 *    async_err for errors at TL>=0
1195 * 3. trap 0x63: Disrupting trap
1196 *    ce_err for errors at TL=0
1197 *    (Note that trap 0x63 cannot happen at trap level > 0)
1198 *
1199 * Trap level one handlers panic the system except for the fast ecc
1200 * error handler which tries to recover from certain errors.
1201 */
1202
1203/*
1204 * FAST ECC TRAP STRATEGY:
1205 *
1206 * Software must handle single and multi bit errors which occur due to data
1207 * or instruction cache reads from the external cache. A single or multi bit
1208 * error occuring in one of these situations results in a precise trap.
1209 *
1210 * The basic flow of this trap handler is as follows:
1211 *
1212 * 1) Record the state and then turn off the Dcache and Icache.  The Dcache
1213 *    is disabled because bad data could have been installed.  The Icache is
1214 *    turned off because we want to capture the Icache line related to the
1215 *    AFAR.
1216 * 2) Disable trapping on CEEN/NCCEN errors during TL=0 processing.
1217 * 3) Park sibling core if caches are shared (to avoid race condition while
1218 *    accessing shared resources such as L3 data staging register during
1219 *    CPU logout.
1220 * 4) Read the AFAR and AFSR.
1221 * 5) If CPU logout structure is not being used, then:
1222 *    6) Clear all errors from the AFSR.
1223 *    7) Capture Ecache, Dcache and Icache lines in "CPU log out" structure.
1224 *    8) Flush Ecache then Flush Dcache and Icache and restore to previous
1225 *       state.
1226 *    9) Unpark sibling core if we parked it earlier.
1227 *    10) call cpu_fast_ecc_error via systrap at PIL 14 unless we're already
1228 *        running at PIL 15.
1229 * 6) Otherwise, if CPU logout structure is being used:
1230 *    7) Incriment the "logout busy count".
1231 *    8) Flush Ecache then Flush Dcache and Icache and restore to previous
1232 *       state.
1233 *    9) Unpark sibling core if we parked it earlier.
1234 *    10) Issue a retry since the other CPU error logging code will end up
1235 *       finding this error bit and logging information about it later.
1236 * 7) Alternatively (to 5 and 6 above), if the cpu_private struct is not
1237 *    yet initialized such that we can't even check the logout struct, then
1238 *    we place the clo_flags data into %g2 (sys_trap->have_win arg #1) and
1239 *    call cpu_fast_ecc_error via systrap. The clo_flags parameter is used
1240 *    to determine information such as TL, TT, CEEN and NCEEN settings, etc
1241 *    in the high level trap handler since we don't have access to detailed
1242 *    logout information in cases where the cpu_private struct is not yet
1243 *    initialized.
1244 *
1245 * We flush the E$ and D$ here on TL=1 code to prevent getting nested
1246 * Fast ECC traps in the TL=0 code.  If we get a Fast ECC event here in
1247 * the TL=1 code, we will go to the Fast ECC at TL>0 handler which,
1248 * since it is uses different code/data from this handler, has a better
1249 * chance of fixing things up than simply recursing through this code
1250 * again (this would probably cause an eventual kernel stack overflow).
1251 * If the Fast ECC at TL>0 handler encounters a Fast ECC error before it
1252 * can flush the E$ (or the error is a stuck-at bit), we will recurse in
1253 * the Fast ECC at TL>0 handler and eventually Red Mode.
1254 *
1255 * Note that for Cheetah (and only Cheetah), we use alias addresses for
1256 * flushing rather than ASI accesses (which don't exist on Cheetah).
1257 * Should we encounter a Fast ECC error within this handler on Cheetah,
1258 * there's a good chance it's within the ecache_flushaddr buffer (since
1259 * it's the largest piece of memory we touch in the handler and it is
1260 * usually kernel text/data).  For that reason the Fast ECC at TL>0
1261 * handler for Cheetah uses an alternate buffer: ecache_tl1_flushaddr.
1262 */
1263
1264/*
1265 * Cheetah ecc-protected E$ trap (Trap 70) at TL=0
1266 * tt0_fecc is replaced by fecc_err_instr in cpu_init_trap of the various
1267 * architecture-specific files.
1268 * NB: Must be 8 instructions or less to fit in trap table and code must
1269 *     be relocatable.
1270 */
1271
1272#if defined(lint)
1273
1274void
1275fecc_err_instr(void)
1276{}
1277
1278#else	/* lint */
1279
1280	ENTRY_NP(fecc_err_instr)
1281	membar	#Sync			! Cheetah requires membar #Sync
1282
1283	/*
1284	 * Save current DCU state.  Turn off the Dcache and Icache.
1285	 */
1286	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
1287	andn	%g1, DCU_DC + DCU_IC, %g4
1288	stxa	%g4, [%g0]ASI_DCU
1289	flush	%g0	/* flush required after changing the IC bit */
1290
1291	ASM_JMP(%g4, fast_ecc_err)
1292	SET_SIZE(fecc_err_instr)
1293
1294#endif	/* lint */
1295
1296
1297#if !(defined(JALAPENO) || defined(SERRANO))
1298
1299#if defined(lint)
1300
1301void
1302fast_ecc_err(void)
1303{}
1304
1305#else	/* lint */
1306
1307	.section ".text"
1308	.align	64
1309	ENTRY_NP(fast_ecc_err)
1310
1311	/*
1312	 * Turn off CEEN and NCEEN.
1313	 */
1314	ldxa	[%g0]ASI_ESTATE_ERR, %g3
1315	andn	%g3, EN_REG_NCEEN + EN_REG_CEEN, %g4
1316	stxa	%g4, [%g0]ASI_ESTATE_ERR
1317	membar	#Sync			! membar sync required
1318
1319	/*
1320	 * Check to see whether we need to park our sibling core
1321	 * before recording diagnostic information from caches
1322	 * which may be shared by both cores.
1323	 * We use %g1 to store information about whether or not
1324	 * we had to park the core (%g1 holds our DCUCR value and
1325	 * we only use bits from that register which are "reserved"
1326	 * to keep track of core parking) so that we know whether
1327	 * or not to unpark later. %g5 and %g4 are scratch registers.
1328	 */
1329	PARK_SIBLING_CORE(%g1, %g5, %g4)
1330
1331	/*
1332	 * Do the CPU log out capture.
1333	 *   %g3 = "failed?" return value.
1334	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
1335	 *         into this macro via %g4. Output only valid if cpu_private
1336	 *         struct has not been initialized.
1337	 *   CHPR_FECCTL0_LOGOUT = cpu logout structure offset input
1338	 *   %g4 = Trap information stored in the cpu logout flags field
1339	 *   %g5 = scr1
1340	 *   %g6 = scr2
1341	 *   %g3 = scr3
1342	 *   %g4 = scr4
1343	 */
1344	 /* store the CEEN and NCEEN values, TL=0 */
1345	and	%g3, EN_REG_CEEN + EN_REG_NCEEN, %g4
1346	set	CHPR_FECCTL0_LOGOUT, %g6
1347	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)
1348
1349	/*
1350	 * Flush the Ecache (and L2 cache for Panther) to get the error out
1351	 * of the Ecache.  If the UCC or UCU is on a dirty line, then the
1352	 * following flush will turn that into a WDC or WDU, respectively.
1353	 */
1354	PN_L2_FLUSHALL(%g4, %g5, %g6)
1355
1356	CPU_INDEX(%g4, %g5)
1357	mulx	%g4, CPU_NODE_SIZE, %g4
1358	set	cpunodes, %g5
1359	add	%g4, %g5, %g4
1360	ld	[%g4 + ECACHE_LINESIZE], %g5
1361	ld	[%g4 + ECACHE_SIZE], %g4
1362
1363	ASM_LDX(%g6, ecache_flushaddr)
1364	ECACHE_FLUSHALL(%g4, %g5, %g6, %g7)
1365
1366	/*
1367	 * Flush the Dcache.  Since bad data could have been installed in
1368	 * the Dcache we must flush it before re-enabling it.
1369	 */
1370	ASM_LD(%g5, dcache_size)
1371	ASM_LD(%g6, dcache_linesize)
1372	CH_DCACHE_FLUSHALL(%g5, %g6, %g7)
1373
1374	/*
1375	 * Flush the Icache.  Since we turned off the Icache to capture the
1376	 * Icache line it is now stale or corrupted and we must flush it
1377	 * before re-enabling it.
1378	 */
1379	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, fast_ecc_err_5);
1380	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
1381	ba,pt	%icc, 6f
1382	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
1383fast_ecc_err_5:
1384	ASM_LD(%g5, icache_size)
1385	ASM_LD(%g6, icache_linesize)
13866:
1387	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)
1388
1389	/*
1390	 * check to see whether we parked our sibling core at the start
1391	 * of this handler. If so, we need to unpark it here.
1392	 * We use DCUCR reserved bits (stored in %g1) to keep track of
1393	 * whether or not we need to unpark. %g5 and %g4 are scratch registers.
1394	 */
1395	UNPARK_SIBLING_CORE(%g1, %g5, %g4)
1396
1397	/*
1398	 * Restore the Dcache and Icache to the previous state.
1399	 */
1400	stxa	%g1, [%g0]ASI_DCU
1401	flush	%g0	/* flush required after changing the IC bit */
1402
1403	/*
1404	 * Make sure our CPU logout operation was successful.
1405	 */
1406	cmp	%g3, %g0
1407	be	8f
1408	  nop
1409
1410	/*
1411	 * If the logout structure had been busy, how many times have
1412	 * we tried to use it and failed (nesting count)? If we have
1413	 * already recursed a substantial number of times, then we can
1414	 * assume things are not going to get better by themselves and
1415	 * so it would be best to panic.
1416	 */
1417	cmp	%g3, CLO_NESTING_MAX
1418	blt	7f
1419	  nop
1420
1421        call ptl1_panic
1422          mov   PTL1_BAD_ECC, %g1
1423
14247:
1425	/*
1426	 * Otherwise, if the logout structure was busy but we have not
1427	 * nested more times than our maximum value, then we simply
1428	 * issue a retry. Our TL=0 trap handler code will check and
1429	 * clear the AFSR after it is done logging what is currently
1430	 * in the logout struct and handle this event at that time.
1431	 */
1432	retry
14338:
1434	/*
1435	 * Call cpu_fast_ecc_error via systrap at PIL 14 unless we're
1436	 * already at PIL 15.
1437	 */
1438	set	cpu_fast_ecc_error, %g1
1439	rdpr	%pil, %g4
1440	cmp	%g4, PIL_14
1441	ba	sys_trap
1442	  movl	%icc, PIL_14, %g4
1443
1444	SET_SIZE(fast_ecc_err)
1445
1446#endif	/* lint */
1447
1448#endif	/* !(JALAPENO || SERRANO) */
1449
1450
1451/*
1452 * Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy:
1453 *
1454 * The basic flow of this trap handler is as follows:
1455 *
1456 * 1) In the "trap 70" trap table code (fecc_err_tl1_instr), generate a
1457 *    software trap 0 ("ta 0") to buy an extra set of %tpc, etc. which we
1458 *    will use to save %g1 and %g2.
1459 * 2) At the software trap 0 at TL>0 trap table code (fecc_err_tl1_cont_instr),
1460 *    we save %g1+%g2 using %tpc, %tnpc + %tstate and jump to the fast ecc
1461 *    handler (using the just saved %g1).
1462 * 3) Turn off the Dcache if it was on and save the state of the Dcache
1463 *    (whether on or off) in Bit2 (CH_ERR_TSTATE_DC_ON) of %tstate.
1464 *    NB: we don't turn off the Icache because bad data is not installed nor
1465 *        will we be doing any diagnostic accesses.
1466 * 4) compute physical address of the per-cpu/per-tl save area using %g1+%g2
1467 * 5) Save %g1-%g7 into the per-cpu/per-tl save area (%g1 + %g2 from the
1468 *    %tpc, %tnpc, %tstate values previously saved).
1469 * 6) set %tl to %tl - 1.
1470 * 7) Save the appropriate flags and TPC in the ch_err_tl1_data structure.
1471 * 8) Save the value of CH_ERR_TSTATE_DC_ON in the ch_err_tl1_tmp field.
1472 * 9) For Cheetah and Jalapeno, read the AFAR and AFSR and clear.  For
1473 *    Cheetah+ (and later), read the shadow AFAR and AFSR but don't clear.
1474 *    Save the values in ch_err_tl1_data.  For Panther, read the shadow
1475 *    AFSR_EXT and save the value in ch_err_tl1_data.
1476 * 10) Disable CEEN/NCEEN to prevent any disrupting/deferred errors from
1477 *    being queued.  We'll report them via the AFSR/AFAR capture in step 13.
1478 * 11) Flush the Ecache.
1479 *    NB: the Ecache is flushed assuming the largest possible size with
1480 *        the smallest possible line size since access to the cpu_nodes may
1481 *        cause an unrecoverable DTLB miss.
1482 * 12) Reenable CEEN/NCEEN with the value saved from step 10.
1483 * 13) For Cheetah and Jalapeno, read the AFAR and AFSR and clear again.
1484 *    For Cheetah+ (and later), read the primary AFAR and AFSR and now clear.
1485 *    Save the read AFSR/AFAR values in ch_err_tl1_data.  For Panther,
1486 *    read and clear the primary AFSR_EXT and save it in ch_err_tl1_data.
1487 * 14) Flush and re-enable the Dcache if it was on at step 3.
1488 * 15) Do TRAPTRACE if enabled.
1489 * 16) Check if a UCU->WDU (or L3_UCU->WDU for Panther) happened, panic if so.
1490 * 17) Set the event pending flag in ch_err_tl1_pending[CPU]
1491 * 18) Cause a softint 15.  The pil15_interrupt handler will inspect the
1492 *    event pending flag and call cpu_tl1_error via systrap if set.
1493 * 19) Restore the registers from step 5 and issue retry.
1494 */
1495
1496/*
1497 * Cheetah ecc-protected E$ trap (Trap 70) at TL>0
1498 * tt1_fecc is replaced by fecc_err_tl1_instr in cpu_init_trap of the various
1499 * architecture-specific files.  This generates a "Software Trap 0" at TL>0,
1500 * which goes to fecc_err_tl1_cont_instr, and we continue the handling there.
1501 * NB: Must be 8 instructions or less to fit in trap table and code must
1502 *     be relocatable.
1503 */
1504
1505#if defined(lint)
1506
1507void
1508fecc_err_tl1_instr(void)
1509{}
1510
1511#else	/* lint */
1512
1513	ENTRY_NP(fecc_err_tl1_instr)
1514	CH_ERR_TL1_TRAPENTRY(SWTRAP_0);
1515	SET_SIZE(fecc_err_tl1_instr)
1516
1517#endif	/* lint */
1518
1519/*
1520 * Software trap 0 at TL>0.
1521 * tt1_swtrap0 is replaced by fecc_err_tl1_cont_instr in cpu_init_trap of
1522 * the various architecture-specific files.  This is used as a continuation
1523 * of the fast ecc handling where we've bought an extra TL level, so we can
1524 * use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
1525 * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
1526 * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
1527 * order two bits from %g1 and %g2 respectively).
1528 * NB: Must be 8 instructions or less to fit in trap table and code must
1529 *     be relocatable.
1530 */
1531#if defined(lint)
1532
1533void
1534fecc_err_tl1_cont_instr(void)
1535{}
1536
1537#else	/* lint */
1538
1539	ENTRY_NP(fecc_err_tl1_cont_instr)
1540	CH_ERR_TL1_SWTRAPENTRY(fast_ecc_tl1_err)
1541	SET_SIZE(fecc_err_tl1_cont_instr)
1542
1543#endif	/* lint */
1544
1545
1546#if defined(lint)
1547
1548void
1549ce_err(void)
1550{}
1551
1552#else	/* lint */
1553
1554/*
1555 * The ce_err function handles disrupting trap type 0x63 at TL=0.
1556 *
1557 * AFSR errors bits which cause this trap are:
1558 *	CE, EMC, EDU:ST, EDC, WDU, WDC, CPU, CPC, IVU, IVC
1559 *
1560 * NCEEN Bit of Cheetah External Cache Error Enable Register enables
1561 * the following AFSR disrupting traps: EDU:ST, WDU, CPU, IVU
1562 *
1563 * CEEN Bit of Cheetah External Cache Error Enable Register enables
1564 * the following AFSR disrupting traps: CE, EMC, EDC, WDC, CPC, IVC
1565 *
1566 * Cheetah+ also handles (No additional processing required):
1567 *    DUE, DTO, DBERR	(NCEEN controlled)
1568 *    THCE		(CEEN and ET_ECC_en controlled)
1569 *    TUE		(ET_ECC_en controlled)
1570 *
1571 * Panther further adds:
1572 *    IMU, L3_EDU, L3_WDU, L3_CPU		(NCEEN controlled)
1573 *    IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE	(CEEN controlled)
1574 *    TUE_SH, TUE		(NCEEN and L2_tag_ECC_en controlled)
1575 *    L3_TUE, L3_TUE_SH		(NCEEN and ET_ECC_en controlled)
1576 *    THCE			(CEEN and L2_tag_ECC_en controlled)
1577 *    L3_THCE			(CEEN and ET_ECC_en controlled)
1578 *
1579 * Steps:
1580 *	1. Disable hardware corrected disrupting errors only (CEEN)
1581 *	2. Park sibling core if caches are shared (to avoid race
1582 *	   condition while accessing shared resources such as L3
1583 *	   data staging register during CPU logout.
1584 *	3. If the CPU logout structure is not currently being used:
1585 *		4. Clear AFSR error bits
1586 *		5. Capture Ecache, Dcache and Icache lines associated
1587 *		   with AFAR.
1588 *		6. Unpark sibling core if we parked it earlier.
1589 *		7. call cpu_disrupting_error via sys_trap at PIL 14
1590 *		   unless we're already running at PIL 15.
1591 *	4. Otherwise, if the CPU logout structure is busy:
1592 *		5. Incriment "logout busy count" and place into %g3
1593 *		6. Unpark sibling core if we parked it earlier.
1594 *		7. Issue a retry since the other CPU error logging
1595 *		   code will end up finding this error bit and logging
1596 *		   information about it later.
1597 *	5. Alternatively (to 3 and 4 above), if the cpu_private struct is
1598 *         not yet initialized such that we can't even check the logout
1599 *         struct, then we place the clo_flags data into %g2
1600 *         (sys_trap->have_win arg #1) and call cpu_disrupting_error via
1601 *         systrap. The clo_flags parameter is used to determine information
1602 *         such as TL, TT, CEEN settings, etc in the high level trap
1603 *         handler since we don't have access to detailed logout information
1604 *         in cases where the cpu_private struct is not yet initialized.
1605 *
1606 * %g3: [ logout busy count ] - arg #2
1607 * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1
1608 */
1609
1610	.align	128
1611	ENTRY_NP(ce_err)
1612	membar	#Sync			! Cheetah requires membar #Sync
1613
1614	/*
1615	 * Disable trap on hardware corrected errors (CEEN) while at TL=0
1616	 * to prevent recursion.
1617	 */
1618	ldxa	[%g0]ASI_ESTATE_ERR, %g1
1619	bclr	EN_REG_CEEN, %g1
1620	stxa	%g1, [%g0]ASI_ESTATE_ERR
1621	membar	#Sync			! membar sync required
1622
1623	/*
1624	 * Save current DCU state.  Turn off Icache to allow capture of
1625	 * Icache data by DO_CPU_LOGOUT.
1626	 */
1627	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
1628	andn	%g1, DCU_IC, %g4
1629	stxa	%g4, [%g0]ASI_DCU
1630	flush	%g0	/* flush required after changing the IC bit */
1631
1632	/*
1633	 * Check to see whether we need to park our sibling core
1634	 * before recording diagnostic information from caches
1635	 * which may be shared by both cores.
1636	 * We use %g1 to store information about whether or not
1637	 * we had to park the core (%g1 holds our DCUCR value and
1638	 * we only use bits from that register which are "reserved"
1639	 * to keep track of core parking) so that we know whether
1640	 * or not to unpark later. %g5 and %g4 are scratch registers.
1641	 */
1642	PARK_SIBLING_CORE(%g1, %g5, %g4)
1643
1644	/*
1645	 * Do the CPU log out capture.
1646	 *   %g3 = "failed?" return value.
1647	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
1648	 *         into this macro via %g4. Output only valid if cpu_private
1649	 *         struct has not been initialized.
1650	 *   CHPR_CECC_LOGOUT = cpu logout structure offset input
1651	 *   %g4 = Trap information stored in the cpu logout flags field
1652	 *   %g5 = scr1
1653	 *   %g6 = scr2
1654	 *   %g3 = scr3
1655	 *   %g4 = scr4
1656	 */
1657	clr	%g4			! TL=0 bit in afsr
1658	set	CHPR_CECC_LOGOUT, %g6
1659	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)
1660
1661	/*
1662	 * Flush the Icache.  Since we turned off the Icache to capture the
1663	 * Icache line it is now stale or corrupted and we must flush it
1664	 * before re-enabling it.
1665	 */
1666	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, ce_err_1);
1667	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
1668	ba,pt	%icc, 2f
1669	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
1670ce_err_1:
1671	ASM_LD(%g5, icache_size)
1672	ASM_LD(%g6, icache_linesize)
16732:
1674	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)
1675
1676	/*
1677	 * check to see whether we parked our sibling core at the start
1678	 * of this handler. If so, we need to unpark it here.
1679	 * We use DCUCR reserved bits (stored in %g1) to keep track of
1680	 * whether or not we need to unpark. %g5 and %g4 are scratch registers.
1681	 */
1682	UNPARK_SIBLING_CORE(%g1, %g5, %g4)
1683
1684	/*
1685	 * Restore Icache to previous state.
1686	 */
1687	stxa	%g1, [%g0]ASI_DCU
1688	flush	%g0	/* flush required after changing the IC bit */
1689
1690	/*
1691	 * Make sure our CPU logout operation was successful.
1692	 */
1693	cmp	%g3, %g0
1694	be	4f
1695	  nop
1696
1697	/*
1698	 * If the logout structure had been busy, how many times have
1699	 * we tried to use it and failed (nesting count)? If we have
1700	 * already recursed a substantial number of times, then we can
1701	 * assume things are not going to get better by themselves and
1702	 * so it would be best to panic.
1703	 */
1704	cmp	%g3, CLO_NESTING_MAX
1705	blt	3f
1706	  nop
1707
1708        call ptl1_panic
1709          mov   PTL1_BAD_ECC, %g1
1710
17113:
1712	/*
1713	 * Otherwise, if the logout structure was busy but we have not
1714	 * nested more times than our maximum value, then we simply
1715	 * issue a retry. Our TL=0 trap handler code will check and
1716	 * clear the AFSR after it is done logging what is currently
1717	 * in the logout struct and handle this event at that time.
1718	 */
1719	retry
17204:
1721	/*
1722	 * Call cpu_disrupting_error via systrap at PIL 14 unless we're
1723	 * already at PIL 15.
1724	 */
1725	set	cpu_disrupting_error, %g1
1726	rdpr	%pil, %g4
1727	cmp	%g4, PIL_14
1728	ba	sys_trap
1729	  movl	%icc, PIL_14, %g4
1730	SET_SIZE(ce_err)
1731
1732#endif	/* lint */
1733
1734
1735#if defined(lint)
1736
1737/*
1738 * This trap cannot happen at TL>0 which means this routine will never
1739 * actually be called and so we treat this like a BAD TRAP panic.
1740 */
1741void
1742ce_err_tl1(void)
1743{}
1744
1745#else	/* lint */
1746
1747	.align	64
1748	ENTRY_NP(ce_err_tl1)
1749
1750        call ptl1_panic
1751          mov   PTL1_BAD_TRAP, %g1
1752
1753	SET_SIZE(ce_err_tl1)
1754
1755#endif	/* lint */
1756
1757
1758#if defined(lint)
1759
1760void
1761async_err(void)
1762{}
1763
1764#else	/* lint */
1765
1766/*
1767 * The async_err function handles deferred trap types 0xA
1768 * (instruction_access_error) and 0x32 (data_access_error) at TL>=0.
1769 *
1770 * AFSR errors bits which cause this trap are:
1771 *	UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR
1772 * On some platforms, EMU may causes cheetah to pull the error pin
1773 * never giving Solaris a chance to take a trap.
1774 *
1775 * NCEEN Bit of Cheetah External Cache Error Enable Register enables
1776 * the following AFSR deferred traps: UE, EMU, EDU:BLD, TO, BERR
1777 *
1778 * Steps:
1779 *	1. Disable CEEN and NCEEN errors to prevent recursive errors.
1780 *	2. Turn D$ off per Cheetah PRM P.5 Note 6, turn I$ off to capture
1781 *         I$ line in DO_CPU_LOGOUT.
1782 *	3. Park sibling core if caches are shared (to avoid race
1783 *	   condition while accessing shared resources such as L3
1784 *	   data staging register during CPU logout.
1785 *	4. If the CPU logout structure is not currently being used:
1786 *		5. Clear AFSR error bits
1787 *		6. Capture Ecache, Dcache and Icache lines associated
1788 *		   with AFAR.
1789 *		7. Unpark sibling core if we parked it earlier.
1790 *		8. call cpu_deferred_error via sys_trap.
1791 *	5. Otherwise, if the CPU logout structure is busy:
1792 *		6. Incriment "logout busy count"
1793 *		7. Unpark sibling core if we parked it earlier.
1794 *		8) Issue a retry since the other CPU error logging
1795 *		   code will end up finding this error bit and logging
1796 *		   information about it later.
1797 *      6. Alternatively (to 4 and 5 above), if the cpu_private struct is
1798 *         not yet initialized such that we can't even check the logout
1799 *         struct, then we place the clo_flags data into %g2
1800 *         (sys_trap->have_win arg #1) and call cpu_deferred_error via
1801 *         systrap. The clo_flags parameter is used to determine information
1802 *         such as TL, TT, CEEN settings, etc in the high level trap handler
1803 *         since we don't have access to detailed logout information in cases
1804 *         where the cpu_private struct is not yet initialized.
1805 *
1806 * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1
1807 * %g3: [ logout busy count ] - arg #2
1808 */
1809
1810	ENTRY_NP(async_err)
1811	membar	#Sync			! Cheetah requires membar #Sync
1812
1813	/*
1814	 * Disable CEEN and NCEEN.
1815	 */
1816	ldxa	[%g0]ASI_ESTATE_ERR, %g3
1817	andn	%g3, EN_REG_NCEEN + EN_REG_CEEN, %g4
1818	stxa	%g4, [%g0]ASI_ESTATE_ERR
1819	membar	#Sync			! membar sync required
1820
1821	/*
1822	 * Save current DCU state.
1823	 * Disable Icache to allow capture of Icache data by DO_CPU_LOGOUT.
1824	 * Do this regardless of whether this is a Data Access Error or
1825	 * Instruction Access Error Trap.
1826	 * Disable Dcache for both Data Access Error and Instruction Access
1827	 * Error per Cheetah PRM P.5 Note 6.
1828	 */
1829	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
1830	andn	%g1, DCU_IC + DCU_DC, %g4
1831	stxa	%g4, [%g0]ASI_DCU
1832	flush	%g0	/* flush required after changing the IC bit */
1833
1834	/*
1835	 * Check to see whether we need to park our sibling core
1836	 * before recording diagnostic information from caches
1837	 * which may be shared by both cores.
1838	 * We use %g1 to store information about whether or not
1839	 * we had to park the core (%g1 holds our DCUCR value and
1840	 * we only use bits from that register which are "reserved"
1841	 * to keep track of core parking) so that we know whether
1842	 * or not to unpark later. %g6 and %g4 are scratch registers.
1843	 */
1844	PARK_SIBLING_CORE(%g1, %g6, %g4)
1845
1846	/*
1847	 * Do the CPU logout capture.
1848	 *
1849	 *   %g3 = "failed?" return value.
1850	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
1851	 *         into this macro via %g4. Output only valid if cpu_private
1852	 *         struct has not been initialized.
1853	 *   CHPR_ASYNC_LOGOUT = cpu logout structure offset input
1854	 *   %g4 = Trap information stored in the cpu logout flags field
1855	 *   %g5 = scr1
1856	 *   %g6 = scr2
1857	 *   %g3 = scr3
1858	 *   %g4 = scr4
1859	 */
1860	andcc	%g5, T_TL1, %g0
1861	clr	%g6
1862	movnz	%xcc, 1, %g6			! set %g6 if T_TL1 set
1863	sllx	%g6, CLO_FLAGS_TL_SHIFT, %g6
1864	sllx	%g5, CLO_FLAGS_TT_SHIFT, %g4
1865	set	CLO_FLAGS_TT_MASK, %g2
1866	and	%g4, %g2, %g4			! ttype
1867	or	%g6, %g4, %g4			! TT and TL
1868	and	%g3, EN_REG_CEEN, %g3		! CEEN value
1869	or	%g3, %g4, %g4			! TT and TL and CEEN
1870	set	CHPR_ASYNC_LOGOUT, %g6
1871	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)
1872
1873	/*
1874	 * If the logout struct was busy, we may need to pass the
1875	 * TT, TL, and CEEN information to the TL=0 handler via
1876	 * systrap parameter so save it off here.
1877	 */
1878	cmp	%g3, %g0
1879	be	1f
1880	  nop
1881	sllx	%g4, 32, %g4
1882	or	%g4, %g3, %g3
18831:
1884	/*
1885	 * Flush the Icache.  Since we turned off the Icache to capture the
1886	 * Icache line it is now stale or corrupted and we must flush it
1887	 * before re-enabling it.
1888	 */
1889	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, async_err_1);
1890	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
1891	ba,pt	%icc, 2f
1892	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
1893async_err_1:
1894	ASM_LD(%g5, icache_size)
1895	ASM_LD(%g6, icache_linesize)
18962:
1897	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)
1898
1899	/*
1900	 * XXX - Don't we need to flush the Dcache before turning it back
1901	 *       on to avoid stale or corrupt data? Was this broken?
1902	 */
1903	/*
1904	 * Flush the Dcache before turning it back on since it may now
1905	 * contain stale or corrupt data.
1906	 */
1907	ASM_LD(%g5, dcache_size)
1908	ASM_LD(%g6, dcache_linesize)
1909	CH_DCACHE_FLUSHALL(%g5, %g6, %g7)
1910
1911	/*
1912	 * check to see whether we parked our sibling core at the start
1913	 * of this handler. If so, we need to unpark it here.
1914	 * We use DCUCR reserved bits (stored in %g1) to keep track of
1915	 * whether or not we need to unpark. %g5 and %g7 are scratch registers.
1916	 */
1917	UNPARK_SIBLING_CORE(%g1, %g5, %g7)
1918
1919	/*
1920	 * Restore Icache and Dcache to previous state.
1921	 */
1922	stxa	%g1, [%g0]ASI_DCU
1923	flush	%g0	/* flush required after changing the IC bit */
1924
1925	/*
1926	 * Make sure our CPU logout operation was successful.
1927	 */
1928	cmp	%g3, %g0
1929	be	4f
1930	  nop
1931
1932	/*
1933	 * If the logout structure had been busy, how many times have
1934	 * we tried to use it and failed (nesting count)? If we have
1935	 * already recursed a substantial number of times, then we can
1936	 * assume things are not going to get better by themselves and
1937	 * so it would be best to panic.
1938	 */
1939	cmp	%g3, CLO_NESTING_MAX
1940	blt	3f
1941	  nop
1942
1943        call ptl1_panic
1944          mov   PTL1_BAD_ECC, %g1
1945
19463:
1947	/*
1948	 * Otherwise, if the logout structure was busy but we have not
1949	 * nested more times than our maximum value, then we simply
1950	 * issue a retry. Our TL=0 trap handler code will check and
1951	 * clear the AFSR after it is done logging what is currently
1952	 * in the logout struct and handle this event at that time.
1953	 */
1954	retry
19554:
1956	RESET_USER_RTT_REGS(%g4, %g5, async_err_resetskip)
1957async_err_resetskip:
1958	set	cpu_deferred_error, %g1
1959	ba	sys_trap
1960	  mov	PIL_15, %g4		! run at pil 15
1961	SET_SIZE(async_err)
1962
1963#endif	/* lint */
1964
1965#if defined(CPU_IMP_L1_CACHE_PARITY)
1966
1967/*
1968 * D$ parity error trap (trap 71) at TL=0.
1969 * tt0_dperr is replaced by dcache_parity_instr in cpu_init_trap of
1970 * the various architecture-specific files.  This merely sets up the
1971 * arguments for cpu_parity_error and calls it via sys_trap.
1972 * NB: Must be 8 instructions or less to fit in trap table and code must
1973 *     be relocatable.
1974 */
1975#if defined(lint)
1976
1977void
1978dcache_parity_instr(void)
1979{}
1980
1981#else	/* lint */
1982	ENTRY_NP(dcache_parity_instr)
1983	membar	#Sync			! Cheetah+ requires membar #Sync
1984	set	cpu_parity_error, %g1
1985	or	%g0, CH_ERR_DPE, %g2
1986	rdpr	%tpc, %g3
1987	sethi	%hi(sys_trap), %g7
1988	jmp	%g7 + %lo(sys_trap)
1989	  mov	PIL_15, %g4		! run at pil 15
1990	SET_SIZE(dcache_parity_instr)
1991
1992#endif	/* lint */
1993
1994
1995/*
1996 * D$ parity error trap (trap 71) at TL>0.
1997 * tt1_dperr is replaced by dcache_parity_tl1_instr in cpu_init_trap of
1998 * the various architecture-specific files.  This generates a "Software
1999 * Trap 1" at TL>0, which goes to dcache_parity_tl1_cont_instr, and we
2000 * continue the handling there.
2001 * NB: Must be 8 instructions or less to fit in trap table and code must
2002 *     be relocatable.
2003 */
2004#if defined(lint)
2005
2006void
2007dcache_parity_tl1_instr(void)
2008{}
2009
2010#else	/* lint */
2011	ENTRY_NP(dcache_parity_tl1_instr)
2012	CH_ERR_TL1_TRAPENTRY(SWTRAP_1);
2013	SET_SIZE(dcache_parity_tl1_instr)
2014
2015#endif	/* lint */
2016
2017
2018/*
2019 * Software trap 1 at TL>0.
2020 * tt1_swtrap1 is replaced by dcache_parity_tl1_cont_instr in cpu_init_trap
2021 * of the various architecture-specific files.  This is used as a continuation
2022 * of the dcache parity handling where we've bought an extra TL level, so we
2023 * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
2024 * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
2025 * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
2026 * order two bits from %g1 and %g2 respectively).
2027 * NB: Must be 8 instructions or less to fit in trap table and code must
2028 *     be relocatable.
2029 */
2030#if defined(lint)
2031
2032void
2033dcache_parity_tl1_cont_instr(void)
2034{}
2035
2036#else	/* lint */
2037	ENTRY_NP(dcache_parity_tl1_cont_instr)
2038	CH_ERR_TL1_SWTRAPENTRY(dcache_parity_tl1_err);
2039	SET_SIZE(dcache_parity_tl1_cont_instr)
2040
2041#endif	/* lint */
2042
2043/*
2044 * D$ parity error at TL>0 handler
2045 * We get here via trap 71 at TL>0->Software trap 1 at TL>0.  We enter
2046 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
2047 */
2048#if defined(lint)
2049
2050void
2051dcache_parity_tl1_err(void)
2052{}
2053
2054#else	/* lint */
2055
2056	ENTRY_NP(dcache_parity_tl1_err)
2057
2058	/*
2059	 * This macro saves all the %g registers in the ch_err_tl1_data
2060	 * structure, updates the ch_err_tl1_flags and saves the %tpc in
2061	 * ch_err_tl1_tpc.  At the end of this macro, %g1 will point to
2062	 * the ch_err_tl1_data structure and %g2 will have the original
2063	 * flags in the ch_err_tl1_data structure.  All %g registers
2064	 * except for %g1 and %g2 will be available.
2065	 */
2066	CH_ERR_TL1_ENTER(CH_ERR_DPE);
2067
2068#ifdef TRAPTRACE
2069	/*
2070	 * Get current trap trace entry physical pointer.
2071	 */
2072	CPU_INDEX(%g6, %g5)
2073	sll	%g6, TRAPTR_SIZE_SHIFT, %g6
2074	set	trap_trace_ctl, %g5
2075	add	%g6, %g5, %g6
2076	ld	[%g6 + TRAPTR_LIMIT], %g5
2077	tst	%g5
2078	be	%icc, dpe_tl1_skip_tt
2079	  nop
2080	ldx	[%g6 + TRAPTR_PBASE], %g5
2081	ld	[%g6 + TRAPTR_OFFSET], %g4
2082	add	%g5, %g4, %g5
2083
2084	/*
2085	 * Create trap trace entry.
2086	 */
2087	rd	%asi, %g7
2088	wr	%g0, TRAPTR_ASI, %asi
2089	rd	STICK, %g4
2090	stxa	%g4, [%g5 + TRAP_ENT_TICK]%asi
2091	rdpr	%tl, %g4
2092	stha	%g4, [%g5 + TRAP_ENT_TL]%asi
2093	rdpr	%tt, %g4
2094	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
2095	rdpr	%tpc, %g4
2096	stna	%g4, [%g5 + TRAP_ENT_TPC]%asi
2097	rdpr	%tstate, %g4
2098	stxa	%g4, [%g5 + TRAP_ENT_TSTATE]%asi
2099	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
2100	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
2101	stna	%g0, [%g5 + TRAP_ENT_F1]%asi
2102	stna	%g0, [%g5 + TRAP_ENT_F2]%asi
2103	stna	%g0, [%g5 + TRAP_ENT_F3]%asi
2104	stna	%g0, [%g5 + TRAP_ENT_F4]%asi
2105	wr	%g0, %g7, %asi
2106
2107	/*
2108	 * Advance trap trace pointer.
2109	 */
2110	ld	[%g6 + TRAPTR_OFFSET], %g5
2111	ld	[%g6 + TRAPTR_LIMIT], %g4
2112	st	%g5, [%g6 + TRAPTR_LAST_OFFSET]
2113	add	%g5, TRAP_ENT_SIZE, %g5
2114	sub	%g4, TRAP_ENT_SIZE, %g4
2115	cmp	%g5, %g4
2116	movge	%icc, 0, %g5
2117	st	%g5, [%g6 + TRAPTR_OFFSET]
2118dpe_tl1_skip_tt:
2119#endif	/* TRAPTRACE */
2120
2121	/*
2122	 * I$ and D$ are automatically turned off by HW when the CPU hits
2123	 * a dcache or icache parity error so we will just leave those two
2124	 * off for now to avoid repeating this trap.
2125	 * For Panther, however, since we trap on P$ data parity errors
2126	 * and HW does not automatically disable P$, we need to disable it
2127	 * here so that we don't encounter any recursive traps when we
2128	 * issue the retry.
2129	 */
2130	ldxa	[%g0]ASI_DCU, %g3
2131	mov	1, %g4
2132	sllx	%g4, DCU_PE_SHIFT, %g4
2133	andn	%g3, %g4, %g3
2134	stxa	%g3, [%g0]ASI_DCU
2135	membar	#Sync
2136
2137	/*
2138	 * We fall into this macro if we've successfully logged the error in
2139	 * the ch_err_tl1_data structure and want the PIL15 softint to pick
2140	 * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
2141	 * Restores the %g registers and issues retry.
2142	 */
2143	CH_ERR_TL1_EXIT;
2144	SET_SIZE(dcache_parity_tl1_err)
2145
2146#endif	/* lint */
2147
2148/*
2149 * I$ parity error trap (trap 72) at TL=0.
2150 * tt0_iperr is replaced by icache_parity_instr in cpu_init_trap of
2151 * the various architecture-specific files.  This merely sets up the
2152 * arguments for cpu_parity_error and calls it via sys_trap.
2153 * NB: Must be 8 instructions or less to fit in trap table and code must
2154 *     be relocatable.
2155 */
2156#if defined(lint)
2157
2158void
2159icache_parity_instr(void)
2160{}
2161
2162#else	/* lint */
2163
2164	ENTRY_NP(icache_parity_instr)
2165	membar	#Sync			! Cheetah+ requires membar #Sync
2166	set	cpu_parity_error, %g1
2167	or	%g0, CH_ERR_IPE, %g2
2168	rdpr	%tpc, %g3
2169	sethi	%hi(sys_trap), %g7
2170	jmp	%g7 + %lo(sys_trap)
2171	  mov	PIL_15, %g4		! run at pil 15
2172	SET_SIZE(icache_parity_instr)
2173
2174#endif	/* lint */
2175
2176/*
2177 * I$ parity error trap (trap 72) at TL>0.
2178 * tt1_iperr is replaced by icache_parity_tl1_instr in cpu_init_trap of
2179 * the various architecture-specific files.  This generates a "Software
2180 * Trap 2" at TL>0, which goes to icache_parity_tl1_cont_instr, and we
2181 * continue the handling there.
2182 * NB: Must be 8 instructions or less to fit in trap table and code must
2183 *     be relocatable.
2184 */
2185#if defined(lint)
2186
2187void
2188icache_parity_tl1_instr(void)
2189{}
2190
2191#else	/* lint */
2192	ENTRY_NP(icache_parity_tl1_instr)
2193	CH_ERR_TL1_TRAPENTRY(SWTRAP_2);
2194	SET_SIZE(icache_parity_tl1_instr)
2195
2196#endif	/* lint */
2197
2198/*
2199 * Software trap 2 at TL>0.
2200 * tt1_swtrap2 is replaced by icache_parity_tl1_cont_instr in cpu_init_trap
2201 * of the various architecture-specific files.  This is used as a continuation
2202 * of the icache parity handling where we've bought an extra TL level, so we
2203 * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
2204 * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
2205 * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
2206 * order two bits from %g1 and %g2 respectively).
2207 * NB: Must be 8 instructions or less to fit in trap table and code must
2208 *     be relocatable.
2209 */
2210#if defined(lint)
2211
2212void
2213icache_parity_tl1_cont_instr(void)
2214{}
2215
2216#else	/* lint */
2217	ENTRY_NP(icache_parity_tl1_cont_instr)
2218	CH_ERR_TL1_SWTRAPENTRY(icache_parity_tl1_err);
2219	SET_SIZE(icache_parity_tl1_cont_instr)
2220
2221#endif	/* lint */
2222
2223
2224/*
2225 * I$ parity error at TL>0 handler
2226 * We get here via trap 72 at TL>0->Software trap 2 at TL>0.  We enter
2227 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
2228 */
2229#if defined(lint)
2230
2231void
2232icache_parity_tl1_err(void)
2233{}
2234
2235#else	/* lint */
2236
2237	ENTRY_NP(icache_parity_tl1_err)
2238
2239	/*
2240	 * This macro saves all the %g registers in the ch_err_tl1_data
2241	 * structure, updates the ch_err_tl1_flags and saves the %tpc in
2242	 * ch_err_tl1_tpc.  At the end of this macro, %g1 will point to
2243	 * the ch_err_tl1_data structure and %g2 will have the original
2244	 * flags in the ch_err_tl1_data structure.  All %g registers
2245	 * except for %g1 and %g2 will be available.
2246	 */
2247	CH_ERR_TL1_ENTER(CH_ERR_IPE);
2248
2249#ifdef TRAPTRACE
2250	/*
2251	 * Get current trap trace entry physical pointer.
2252	 */
2253	CPU_INDEX(%g6, %g5)
2254	sll	%g6, TRAPTR_SIZE_SHIFT, %g6
2255	set	trap_trace_ctl, %g5
2256	add	%g6, %g5, %g6
2257	ld	[%g6 + TRAPTR_LIMIT], %g5
2258	tst	%g5
2259	be	%icc, ipe_tl1_skip_tt
2260	  nop
2261	ldx	[%g6 + TRAPTR_PBASE], %g5
2262	ld	[%g6 + TRAPTR_OFFSET], %g4
2263	add	%g5, %g4, %g5
2264
2265	/*
2266	 * Create trap trace entry.
2267	 */
2268	rd	%asi, %g7
2269	wr	%g0, TRAPTR_ASI, %asi
2270	rd	STICK, %g4
2271	stxa	%g4, [%g5 + TRAP_ENT_TICK]%asi
2272	rdpr	%tl, %g4
2273	stha	%g4, [%g5 + TRAP_ENT_TL]%asi
2274	rdpr	%tt, %g4
2275	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
2276	rdpr	%tpc, %g4
2277	stna	%g4, [%g5 + TRAP_ENT_TPC]%asi
2278	rdpr	%tstate, %g4
2279	stxa	%g4, [%g5 + TRAP_ENT_TSTATE]%asi
2280	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
2281	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
2282	stna	%g0, [%g5 + TRAP_ENT_F1]%asi
2283	stna	%g0, [%g5 + TRAP_ENT_F2]%asi
2284	stna	%g0, [%g5 + TRAP_ENT_F3]%asi
2285	stna	%g0, [%g5 + TRAP_ENT_F4]%asi
2286	wr	%g0, %g7, %asi
2287
2288	/*
2289	 * Advance trap trace pointer.
2290	 */
2291	ld	[%g6 + TRAPTR_OFFSET], %g5
2292	ld	[%g6 + TRAPTR_LIMIT], %g4
2293	st	%g5, [%g6 + TRAPTR_LAST_OFFSET]
2294	add	%g5, TRAP_ENT_SIZE, %g5
2295	sub	%g4, TRAP_ENT_SIZE, %g4
2296	cmp	%g5, %g4
2297	movge	%icc, 0, %g5
2298	st	%g5, [%g6 + TRAPTR_OFFSET]
2299ipe_tl1_skip_tt:
2300#endif	/* TRAPTRACE */
2301
2302	/*
2303	 * We fall into this macro if we've successfully logged the error in
2304	 * the ch_err_tl1_data structure and want the PIL15 softint to pick
2305	 * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
2306	 * Restores the %g registers and issues retry.
2307	 */
2308	CH_ERR_TL1_EXIT;
2309
2310	SET_SIZE(icache_parity_tl1_err)
2311
2312#endif	/* lint */
2313
2314#endif	/* CPU_IMP_L1_CACHE_PARITY */
2315
2316
2317/*
2318 * The itlb_rd_entry and dtlb_rd_entry functions return the tag portion of the
2319 * tte, the virtual address, and the ctxnum of the specified tlb entry.  They
2320 * should only be used in places where you have no choice but to look at the
2321 * tlb itself.
2322 *
2323 * Note: These two routines are required by the Estar "cpr" loadable module.
2324 */
2325
2326#if defined(lint)
2327
2328/* ARGSUSED */
2329void
2330itlb_rd_entry(uint_t entry, tte_t *tte, uint64_t *va_tag)
2331{}
2332
2333#else	/* lint */
2334
2335	ENTRY_NP(itlb_rd_entry)
2336	sllx	%o0, 3, %o0
2337	ldxa	[%o0]ASI_ITLB_ACCESS, %g1
2338	stx	%g1, [%o1]
2339	ldxa	[%o0]ASI_ITLB_TAGREAD, %g2
2340	set	TAGREAD_CTX_MASK, %o4
2341	andn	%g2, %o4, %o5
2342	retl
2343	  stx	%o5, [%o2]
2344	SET_SIZE(itlb_rd_entry)
2345
2346#endif	/* lint */
2347
2348
2349#if defined(lint)
2350
2351/* ARGSUSED */
2352void
2353dtlb_rd_entry(uint_t entry, tte_t *tte, uint64_t *va_tag)
2354{}
2355
2356#else	/* lint */
2357
2358	ENTRY_NP(dtlb_rd_entry)
2359	sllx	%o0, 3, %o0
2360	ldxa	[%o0]ASI_DTLB_ACCESS, %g1
2361	stx	%g1, [%o1]
2362	ldxa	[%o0]ASI_DTLB_TAGREAD, %g2
2363	set	TAGREAD_CTX_MASK, %o4
2364	andn	%g2, %o4, %o5
2365	retl
2366	  stx	%o5, [%o2]
2367	SET_SIZE(dtlb_rd_entry)
2368#endif	/* lint */
2369
2370
2371#if !(defined(JALAPENO) || defined(SERRANO))
2372
2373#if defined(lint)
2374
2375uint64_t
2376get_safari_config(void)
2377{ return (0); }
2378
2379#else	/* lint */
2380
2381	ENTRY(get_safari_config)
2382	ldxa	[%g0]ASI_SAFARI_CONFIG, %o0
2383	retl
2384	nop
2385	SET_SIZE(get_safari_config)
2386
2387#endif	/* lint */
2388
2389
2390#if defined(lint)
2391
2392/* ARGSUSED */
2393void
2394set_safari_config(uint64_t safari_config)
2395{}
2396
2397#else	/* lint */
2398
2399	ENTRY(set_safari_config)
2400	stxa	%o0, [%g0]ASI_SAFARI_CONFIG
2401	membar	#Sync
2402	retl
2403	nop
2404	SET_SIZE(set_safari_config)
2405
2406#endif	/* lint */
2407
2408#endif	/* !(JALAPENO || SERRANO) */
2409
2410
2411#if defined(lint)
2412
2413void
2414cpu_cleartickpnt(void)
2415{}
2416
2417#else	/* lint */
2418	/*
2419	 * Clear the NPT (non-privileged trap) bit in the %tick/%stick
2420	 * registers. In an effort to make the change in the
2421	 * tick/stick counter as consistent as possible, we disable
2422	 * all interrupts while we're changing the registers. We also
2423	 * ensure that the read and write instructions are in the same
2424	 * line in the instruction cache.
2425	 */
2426	ENTRY_NP(cpu_clearticknpt)
2427	rdpr	%pstate, %g1		/* save processor state */
2428	andn	%g1, PSTATE_IE, %g3	/* turn off */
2429	wrpr	%g0, %g3, %pstate	/*   interrupts */
2430	rdpr	%tick, %g2		/* get tick register */
2431	brgez,pn %g2, 1f		/* if NPT bit off, we're done */
2432	mov	1, %g3			/* create mask */
2433	sllx	%g3, 63, %g3		/*   for NPT bit */
2434	ba,a,pt	%xcc, 2f
2435	.align	8			/* Ensure rd/wr in same i$ line */
24362:
2437	rdpr	%tick, %g2		/* get tick register */
2438	wrpr	%g3, %g2, %tick		/* write tick register, */
2439					/*   clearing NPT bit   */
24401:
2441	rd	STICK, %g2		/* get stick register */
2442	brgez,pn %g2, 3f		/* if NPT bit off, we're done */
2443	mov	1, %g3			/* create mask */
2444	sllx	%g3, 63, %g3		/*   for NPT bit */
2445	ba,a,pt	%xcc, 4f
2446	.align	8			/* Ensure rd/wr in same i$ line */
24474:
2448	rd	STICK, %g2		/* get stick register */
2449	wr	%g3, %g2, STICK		/* write stick register, */
2450					/*   clearing NPT bit   */
24513:
2452	jmp	%g4 + 4
2453	wrpr	%g0, %g1, %pstate	/* restore processor state */
2454
2455	SET_SIZE(cpu_clearticknpt)
2456
2457#endif	/* lint */
2458
2459
2460#if defined(CPU_IMP_L1_CACHE_PARITY)
2461
2462#if defined(lint)
2463/*
2464 * correct_dcache_parity(size_t size, size_t linesize)
2465 *
2466 * Correct D$ data parity by zeroing the data and initializing microtag
2467 * for all indexes and all ways of the D$.
2468 *
2469 */
2470/* ARGSUSED */
2471void
2472correct_dcache_parity(size_t size, size_t linesize)
2473{}
2474
2475#else	/* lint */
2476
2477	ENTRY(correct_dcache_parity)
2478	/*
2479	 * Register Usage:
2480	 *
2481	 * %o0 = input D$ size
2482	 * %o1 = input D$ line size
2483	 * %o2 = scratch
2484	 * %o3 = scratch
2485	 * %o4 = scratch
2486	 */
2487
2488	sub	%o0, %o1, %o0			! init cache line address
2489
2490	/*
2491	 * For Panther CPUs, we also need to clear the data parity bits
2492	 * using DC_data_parity bit of the ASI_DCACHE_DATA register.
2493	 */
2494	GET_CPU_IMPL(%o3)
2495	cmp	%o3, PANTHER_IMPL
2496	bne	1f
2497	  clr	%o3				! zero for non-Panther
2498	mov	1, %o3
2499	sll	%o3, PN_DC_DATA_PARITY_BIT_SHIFT, %o3
2500
25011:
2502	/*
2503	 * Set utag = way since it must be unique within an index.
2504	 */
2505	srl	%o0, 14, %o2			! get cache way (DC_way)
2506	membar	#Sync				! required before ASI_DC_UTAG
2507	stxa	%o2, [%o0]ASI_DC_UTAG		! set D$ utag = cache way
2508	membar	#Sync				! required after ASI_DC_UTAG
2509
2510	/*
2511	 * Zero line of D$ data (and data parity bits for Panther)
2512	 */
2513	sub	%o1, 8, %o2
2514	or	%o0, %o3, %o4			! same address + DC_data_parity
25152:
2516	membar	#Sync				! required before ASI_DC_DATA
2517	stxa	%g0, [%o0 + %o2]ASI_DC_DATA	! zero 8 bytes of D$ data
2518	membar	#Sync				! required after ASI_DC_DATA
2519	/*
2520	 * We also clear the parity bits if this is a panther. For non-Panther
2521	 * CPUs, we simply end up clearing the $data register twice.
2522	 */
2523	stxa	%g0, [%o4 + %o2]ASI_DC_DATA
2524	membar	#Sync
2525
2526	subcc	%o2, 8, %o2
2527	bge	2b
2528	nop
2529
2530	subcc	%o0, %o1, %o0
2531	bge	1b
2532	nop
2533
2534	retl
2535	  nop
2536	SET_SIZE(correct_dcache_parity)
2537
2538#endif	/* lint */
2539
2540#endif	/* CPU_IMP_L1_CACHE_PARITY */
2541
2542
2543#if defined(lint)
2544/*
2545 *  Get timestamp (stick).
2546 */
2547/* ARGSUSED */
2548void
2549stick_timestamp(int64_t *ts)
2550{
2551}
2552
2553#else	/* lint */
2554
2555	ENTRY_NP(stick_timestamp)
2556	rd	STICK, %g1	! read stick reg
2557	sllx	%g1, 1, %g1
2558	srlx	%g1, 1, %g1	! clear npt bit
2559
2560	retl
2561	stx     %g1, [%o0]	! store the timestamp
2562	SET_SIZE(stick_timestamp)
2563
2564#endif	/* lint */
2565
2566
2567#if defined(lint)
2568/*
2569 * Set STICK adjusted by skew.
2570 */
2571/* ARGSUSED */
2572void
2573stick_adj(int64_t skew)
2574{
2575}
2576
2577#else	/* lint */
2578
2579	ENTRY_NP(stick_adj)
2580	rdpr	%pstate, %g1		! save processor state
2581	andn	%g1, PSTATE_IE, %g3
2582	ba	1f			! cache align stick adj
2583	wrpr	%g0, %g3, %pstate	! turn off interrupts
2584
2585	.align	16
25861:	nop
2587
2588	rd	STICK, %g4		! read stick reg
2589	add	%g4, %o0, %o1		! adjust stick with skew
2590	wr	%o1, %g0, STICK		! write stick reg
2591
2592	retl
2593	wrpr	%g1, %pstate		! restore processor state
2594	SET_SIZE(stick_adj)
2595
2596#endif	/* lint */
2597
2598#if defined(lint)
2599/*
2600 * Debugger-specific stick retrieval
2601 */
2602/*ARGSUSED*/
2603int
2604kdi_get_stick(uint64_t *stickp)
2605{
2606	return (0);
2607}
2608
2609#else	/* lint */
2610
2611	ENTRY_NP(kdi_get_stick)
2612	rd	STICK, %g1
2613	stx	%g1, [%o0]
2614	retl
2615	mov	%g0, %o0
2616	SET_SIZE(kdi_get_stick)
2617
2618#endif	/* lint */
2619
2620#if defined(lint)
2621/*
2622 * Invalidate the specified line from the D$.
2623 *
2624 * Register usage:
2625 *	%o0 - index for the invalidation, specifies DC_way and DC_addr
2626 *
2627 * ASI_DC_TAG, 0x47, is used in the following manner. A 64-bit value is
2628 * stored to a particular DC_way and DC_addr in ASI_DC_TAG.
2629 *
2630 * The format of the stored 64-bit value is:
2631 *
2632 *	+----------+--------+----------+
2633 *	| Reserved | DC_tag | DC_valid |
2634 *	+----------+--------+----------+
2635 *       63      31 30     1	      0
2636 *
2637 * DC_tag is the 30-bit physical tag of the associated line.
2638 * DC_valid is the 1-bit valid field for both the physical and snoop tags.
2639 *
2640 * The format of the 64-bit DC_way and DC_addr into ASI_DC_TAG is:
2641 *
2642 *	+----------+--------+----------+----------+
2643 *	| Reserved | DC_way | DC_addr  | Reserved |
2644 *	+----------+--------+----------+----------+
2645 *       63      16 15    14 13       5 4        0
2646 *
2647 * DC_way is a 2-bit index that selects one of the 4 ways.
2648 * DC_addr is a 9-bit index that selects one of 512 tag/valid fields.
2649 *
2650 * Setting the DC_valid bit to zero for the specified DC_way and
2651 * DC_addr index into the D$ results in an invalidation of a D$ line.
2652 */
2653/*ARGSUSED*/
2654void
2655dcache_inval_line(int index)
2656{
2657}
2658#else	/* lint */
2659	ENTRY(dcache_inval_line)
2660	sll	%o0, 5, %o0		! shift index into DC_way and DC_addr
2661	stxa	%g0, [%o0]ASI_DC_TAG	! zero the DC_valid and DC_tag bits
2662	membar	#Sync
2663	retl
2664	nop
2665	SET_SIZE(dcache_inval_line)
2666#endif	/* lint */
2667
2668#if defined(lint)
2669/*
2670 * Invalidate the entire I$
2671 *
2672 * Register usage:
2673 *	%o0 - specifies IC_way, IC_addr, IC_tag
2674 *	%o1 - scratch
2675 *	%o2 - used to save and restore DCU value
2676 *	%o3 - scratch
2677 *	%o5 - used to save and restore PSTATE
2678 *
2679 * Due to the behavior of the I$ control logic when accessing ASI_IC_TAG,
2680 * the I$ should be turned off. Accesses to ASI_IC_TAG may collide and
2681 * block out snoops and invalidates to the I$, causing I$ consistency
2682 * to be broken. Before turning on the I$, all I$ lines must be invalidated.
2683 *
2684 * ASI_IC_TAG, 0x67, is used in the following manner. A 64-bit value is
2685 * stored to a particular IC_way, IC_addr, IC_tag in ASI_IC_TAG. The
2686 * info below describes store (write) use of ASI_IC_TAG. Note that read
2687 * use of ASI_IC_TAG behaves differently.
2688 *
2689 * The format of the stored 64-bit value is:
2690 *
2691 *	+----------+--------+---------------+-----------+
2692 *	| Reserved | Valid  | IC_vpred<7:0> | Undefined |
2693 *	+----------+--------+---------------+-----------+
2694 *       63      55    54    53           46 45        0
2695 *
2696 * Valid is the 1-bit valid field for both the physical and snoop tags.
2697 * IC_vpred is the 8-bit LPB bits for 8 instructions starting at
2698 *	the 32-byte boundary aligned address specified by IC_addr.
2699 *
2700 * The format of the 64-bit IC_way, IC_addr, IC_tag into ASI_IC_TAG is:
2701 *
2702 *	+----------+--------+---------+--------+---------+
2703 *	| Reserved | IC_way | IC_addr | IC_tag |Reserved |
2704 *	+----------+--------+---------+--------+---------+
2705 *       63      16 15    14 13      5 4      3 2       0
2706 *
2707 * IC_way is a 2-bit index that selects one of the 4 ways.
2708 * IC_addr[13:6] is an 8-bit index that selects one of 256 valid fields.
2709 * IC_addr[5] is a "don't care" for a store.
2710 * IC_tag set to 2 specifies that the stored value is to be interpreted
2711 *	as containing Valid and IC_vpred as described above.
2712 *
2713 * Setting the Valid bit to zero for the specified IC_way and
2714 * IC_addr index into the I$ results in an invalidation of an I$ line.
2715 */
2716/*ARGSUSED*/
2717void
2718icache_inval_all(void)
2719{
2720}
2721#else	/* lint */
2722	ENTRY(icache_inval_all)
2723	rdpr	%pstate, %o5
2724	andn	%o5, PSTATE_IE, %o3
2725	wrpr	%g0, %o3, %pstate	! clear IE bit
2726
2727	GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, icache_inval_all_1);
2728	ld	[%o0 + CHPR_ICACHE_LINESIZE], %o1
2729	ba,pt	%icc, 2f
2730	  ld	[%o0 + CHPR_ICACHE_SIZE], %o0
2731icache_inval_all_1:
2732	ASM_LD(%o0, icache_size)
2733	ASM_LD(%o1, icache_linesize)
27342:
2735	CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4)
2736
2737	retl
2738	wrpr	%g0, %o5, %pstate	! restore earlier pstate
2739	SET_SIZE(icache_inval_all)
2740#endif	/* lint */
2741
2742
2743#if defined(lint)
2744/* ARGSUSED */
2745void
2746cache_scrubreq_tl1(uint64_t inum, uint64_t index)
2747{
2748}
2749
2750#else	/* lint */
2751/*
2752 * cache_scrubreq_tl1 is the crosstrap handler called on offlined cpus via a
2753 * crosstrap.  It atomically increments the outstanding request counter and,
2754 * if there was not already an outstanding request, branches to setsoftint_tl1
2755 * to enqueue an intr_vec for the given inum.
2756 */
2757
2758	! Register usage:
2759	!
2760	! Arguments:
2761	! %g1 - inum
2762	! %g2 - index into chsm_outstanding array
2763	!
2764	! Internal:
2765	! %g2, %g3, %g5 - scratch
2766	! %g4 - ptr. to scrub_misc chsm_outstanding[index].
2767	! %g6 - setsoftint_tl1 address
2768
2769	ENTRY_NP(cache_scrubreq_tl1)
2770	mulx	%g2, CHSM_OUTSTANDING_INCR, %g2
2771	set	CHPR_SCRUB_MISC + CHSM_OUTSTANDING, %g3
2772	add	%g2, %g3, %g2
2773	GET_CPU_PRIVATE_PTR(%g2, %g4, %g5, 1f);
2774	ld	[%g4], %g2		! cpu's chsm_outstanding[index]
2775	!
2776	! no need to use atomic instructions for the following
2777	! increment - we're at tl1
2778	!
2779	add	%g2, 0x1, %g3
2780	brnz,pn	%g2, 1f			! no need to enqueue more intr_vec
2781	  st	%g3, [%g4]		! delay - store incremented counter
2782	ASM_JMP(%g6, setsoftint_tl1)
2783	! not reached
27841:
2785	retry
2786	SET_SIZE(cache_scrubreq_tl1)
2787
2788#endif	/* lint */
2789
2790
2791#if defined(lint)
2792
2793/* ARGSUSED */
2794void
2795get_cpu_error_state(ch_cpu_errors_t *cpu_error_regs)
2796{}
2797
2798#else	/* lint */
2799
2800/*
2801 * Get the error state for the processor.
2802 * Note that this must not be used at TL>0
2803 */
2804	ENTRY(get_cpu_error_state)
2805#if defined(CHEETAH_PLUS)
2806	set	ASI_SHADOW_REG_VA, %o2
2807	ldxa	[%o2]ASI_AFSR, %o1		! shadow afsr reg
2808	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR]
2809	ldxa	[%o2]ASI_AFAR, %o1		! shadow afar reg
2810	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR]
2811	GET_CPU_IMPL(%o3)	! Only panther has AFSR_EXT registers
2812	cmp	%o3, PANTHER_IMPL
2813	bne,a	1f
2814	  stx	%g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT]	! zero for non-PN
2815	set	ASI_AFSR_EXT_VA, %o2
2816	ldxa	[%o2]ASI_AFSR, %o1		! afsr_ext reg
2817	stx	%o1, [%o0 + CH_CPU_ERRORS_AFSR_EXT]
2818	set	ASI_SHADOW_AFSR_EXT_VA, %o2
2819	ldxa	[%o2]ASI_AFSR, %o1		! shadow afsr_ext reg
2820	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT]
2821	b	2f
2822	  nop
28231:
2824	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT] ! zero for non-PN
28252:
2826#else	/* CHEETAH_PLUS */
2827	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR]
2828	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR]
2829	stx	%g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT]
2830	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT]
2831#endif	/* CHEETAH_PLUS */
2832#if defined(SERRANO)
2833	/*
2834	 * Serrano has an afar2 which captures the address on FRC/FRU errors.
2835	 * We save this in the afar2 of the register save area.
2836	 */
2837	set	ASI_MCU_AFAR2_VA, %o2
2838	ldxa	[%o2]ASI_MCU_CTRL, %o1
2839	stx	%o1, [%o0 + CH_CPU_ERRORS_AFAR2]
2840#endif	/* SERRANO */
2841	ldxa	[%g0]ASI_AFSR, %o1		! primary afsr reg
2842	stx	%o1, [%o0 + CH_CPU_ERRORS_AFSR]
2843	ldxa	[%g0]ASI_AFAR, %o1		! primary afar reg
2844	retl
2845	stx	%o1, [%o0 + CH_CPU_ERRORS_AFAR]
2846	SET_SIZE(get_cpu_error_state)
2847#endif	/* lint */
2848
2849#if defined(lint)
2850
2851/*
2852 * Check a page of memory for errors.
2853 *
2854 * Load each 64 byte block from physical memory.
2855 * Check AFSR after each load to see if an error
2856 * was caused. If so, log/scrub that error.
2857 *
2858 * Used to determine if a page contains
2859 * CEs when CEEN is disabled.
2860 */
2861/*ARGSUSED*/
2862void
2863cpu_check_block(caddr_t va, uint_t psz)
2864{}
2865
2866#else	/* lint */
2867
2868	ENTRY(cpu_check_block)
2869	!
2870	! get a new window with room for the error regs
2871	!
2872	save	%sp, -SA(MINFRAME + CH_CPU_ERROR_SIZE), %sp
2873	srl	%i1, 6, %l4		! clear top bits of psz
2874					! and divide by 64
2875	rd	%fprs, %l2		! store FP
2876	wr	%g0, FPRS_FEF, %fprs	! enable FP
28771:
2878	ldda	[%i0]ASI_BLK_P, %d0	! load a block
2879	membar	#Sync
2880	ldxa    [%g0]ASI_AFSR, %l3	! read afsr reg
2881	brz,a,pt %l3, 2f		! check for error
2882	nop
2883
2884	!
2885	! if error, read the error regs and log it
2886	!
2887	call	get_cpu_error_state
2888	add	%fp, STACK_BIAS - CH_CPU_ERROR_SIZE, %o0
2889
2890	!
2891	! cpu_ce_detected(ch_cpu_errors_t *, flag)
2892	!
2893	call	cpu_ce_detected		! log the error
2894	mov	CE_CEEN_TIMEOUT, %o1
28952:
2896	dec	%l4			! next 64-byte block
2897	brnz,a,pt  %l4, 1b
2898	add	%i0, 64, %i0		! increment block addr
2899
2900	wr	%l2, %g0, %fprs		! restore FP
2901	ret
2902	restore
2903
2904	SET_SIZE(cpu_check_block)
2905
2906#endif	/* lint */
2907
2908#if defined(lint)
2909
2910/*
2911 * Perform a cpu logout called from C.  This is used where we did not trap
2912 * for the error but still want to gather "what we can".  Caller must make
2913 * sure cpu private area exists and that the indicated logout area is free
2914 * for use, and that we are unable to migrate cpus.
2915 */
2916/*ARGSUSED*/
2917void
2918cpu_delayed_logout(uint64_t afar, ch_cpu_logout_t *clop)
2919{ }
2920
2921#else
2922	ENTRY(cpu_delayed_logout)
2923	rdpr	%pstate, %o2
2924	andn	%o2, PSTATE_IE, %o2
2925	wrpr	%g0, %o2, %pstate		! disable interrupts
2926	PARK_SIBLING_CORE(%o2, %o3, %o4)	! %o2 has DCU value
2927	add	%o1, CH_CLO_DATA + CH_CHD_EC_DATA, %o1
2928	rd	%asi, %g1
2929	wr	%g0, ASI_P, %asi
2930	GET_ECACHE_DTAGS(%o0, %o1, %o3, %o4, %o5)
2931	wr	%g1, %asi
2932	UNPARK_SIBLING_CORE(%o2, %o3, %o4)	! can use %o2 again
2933	rdpr	%pstate, %o2
2934	or	%o2, PSTATE_IE, %o2
2935	wrpr	%g0, %o2, %pstate
2936	retl
2937	  nop
2938	SET_SIZE(cpu_delayed_logout)
2939
2940#endif	/* lint */
2941
2942#if defined(lint)
2943
2944/*ARGSUSED*/
2945int
2946dtrace_blksuword32(uintptr_t addr, uint32_t *data, int tryagain)
2947{ return (0); }
2948
2949#else
2950
2951	ENTRY(dtrace_blksuword32)
2952	save	%sp, -SA(MINFRAME + 4), %sp
2953
2954	rdpr	%pstate, %l1
2955	andn	%l1, PSTATE_IE, %l2		! disable interrupts to
2956	wrpr	%g0, %l2, %pstate		! protect our FPU diddling
2957
2958	rd	%fprs, %l0
2959	andcc	%l0, FPRS_FEF, %g0
2960	bz,a,pt	%xcc, 1f			! if the fpu is disabled
2961	wr	%g0, FPRS_FEF, %fprs		! ... enable the fpu
2962
2963	st	%f0, [%fp + STACK_BIAS - 4]	! save %f0 to the stack
29641:
2965	set	0f, %l5
2966        /*
2967         * We're about to write a block full or either total garbage
2968         * (not kernel data, don't worry) or user floating-point data
2969         * (so it only _looks_ like garbage).
2970         */
2971	ld	[%i1], %f0			! modify the block
2972	membar	#Sync
2973	stn	%l5, [THREAD_REG + T_LOFAULT]	! set up the lofault handler
2974	stda	%d0, [%i0]ASI_BLK_COMMIT_S	! store the modified block
2975	membar	#Sync
2976	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler
2977
2978	bz,a,pt	%xcc, 1f
2979	wr	%g0, %l0, %fprs			! restore %fprs
2980
2981	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
29821:
2983
2984	wrpr	%g0, %l1, %pstate		! restore interrupts
2985
2986	ret
2987	restore	%g0, %g0, %o0
2988
29890:
2990	membar	#Sync
2991	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler
2992
2993	bz,a,pt	%xcc, 1f
2994	wr	%g0, %l0, %fprs			! restore %fprs
2995
2996	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
29971:
2998
2999	wrpr	%g0, %l1, %pstate		! restore interrupts
3000
3001	/*
3002	 * If tryagain is set (%i2) we tail-call dtrace_blksuword32_err()
3003	 * which deals with watchpoints. Otherwise, just return -1.
3004	 */
3005	brnz,pt	%i2, 1f
3006	nop
3007	ret
3008	restore	%g0, -1, %o0
30091:
3010	call	dtrace_blksuword32_err
3011	restore
3012
3013	SET_SIZE(dtrace_blksuword32)
3014
3015#endif /* lint */
3016
3017#ifdef	CHEETAHPLUS_ERRATUM_25
3018
3019#if	defined(lint)
3020/*
3021 * Claim a chunk of physical address space.
3022 */
3023/*ARGSUSED*/
3024void
3025claimlines(uint64_t pa, size_t sz, int stride)
3026{}
3027#else	/* lint */
3028	ENTRY(claimlines)
30291:
3030	subcc	%o1, %o2, %o1
3031	add	%o0, %o1, %o3
3032	bgeu,a,pt	%xcc, 1b
3033	casxa	[%o3]ASI_MEM, %g0, %g0
3034	membar  #Sync
3035	retl
3036	nop
3037	SET_SIZE(claimlines)
3038#endif	/* lint */
3039
3040#if	defined(lint)
3041/*
3042 * CPU feature initialization,
3043 * turn BPE off,
3044 * get device id.
3045 */
3046/*ARGSUSED*/
3047void
3048cpu_feature_init(void)
3049{}
3050#else	/* lint */
3051	ENTRY(cpu_feature_init)
3052	save	%sp, -SA(MINFRAME), %sp
3053	sethi	%hi(cheetah_bpe_off), %o0
3054	ld	[%o0 + %lo(cheetah_bpe_off)], %o0
3055	brz	%o0, 1f
3056	nop
3057	rd	ASR_DISPATCH_CONTROL, %o0
3058	andn	%o0, ASR_DISPATCH_CONTROL_BPE, %o0
3059	wr	%o0, 0, ASR_DISPATCH_CONTROL
30601:
3061	!
3062	! get the device_id and store the device_id
3063	! in the appropriate cpunodes structure
3064	! given the cpus index
3065	!
3066	CPU_INDEX(%o0, %o1)
3067	mulx %o0, CPU_NODE_SIZE, %o0
3068	set  cpunodes + DEVICE_ID, %o1
3069	ldxa [%g0] ASI_DEVICE_SERIAL_ID, %o2
3070	stx  %o2, [%o0 + %o1]
3071#ifdef	CHEETAHPLUS_ERRATUM_34
3072	!
3073	! apply Cheetah+ erratum 34 workaround
3074	!
3075	call itlb_erratum34_fixup
3076	  nop
3077	call dtlb_erratum34_fixup
3078	  nop
3079#endif	/* CHEETAHPLUS_ERRATUM_34 */
3080	ret
3081	  restore
3082	SET_SIZE(cpu_feature_init)
3083#endif	/* lint */
3084
3085#if	defined(lint)
3086/*
3087 * Copy a tsb entry atomically, from src to dest.
3088 * src must be 128 bit aligned.
3089 */
3090/*ARGSUSED*/
3091void
3092copy_tsb_entry(uintptr_t src, uintptr_t dest)
3093{}
3094#else	/* lint */
3095	ENTRY(copy_tsb_entry)
3096	ldda	[%o0]ASI_NQUAD_LD, %o2		! %o2 = tag, %o3 = data
3097	stx	%o2, [%o1]
3098	stx	%o3, [%o1 + 8 ]
3099	retl
3100	nop
3101	SET_SIZE(copy_tsb_entry)
3102#endif	/* lint */
3103
3104#endif	/* CHEETAHPLUS_ERRATUM_25 */
3105
3106#ifdef	CHEETAHPLUS_ERRATUM_34
3107
3108#if	defined(lint)
3109
3110/*ARGSUSED*/
3111void
3112itlb_erratum34_fixup(void)
3113{}
3114
3115#else	/* lint */
3116
3117	!
3118	! In Cheetah+ erratum 34, under certain conditions an ITLB locked
3119	! index 0 TTE will erroneously be displaced when a new TTE is
3120	! loaded via ASI_ITLB_IN.  In order to avoid cheetah+ erratum 34,
3121	! locked index 0 TTEs must be relocated.
3122	!
3123	! NOTE: Care must be taken to avoid an ITLB miss in this routine.
3124	!
3125	ENTRY_NP(itlb_erratum34_fixup)
3126	rdpr	%pstate, %o3
3127#ifdef DEBUG
3128	PANIC_IF_INTR_DISABLED_PSTR(%o3, u3_di_label1, %g1)
3129#endif /* DEBUG */
3130	wrpr	%o3, PSTATE_IE, %pstate		! Disable interrupts
3131	ldxa	[%g0]ASI_ITLB_ACCESS, %o1	! %o1 = entry 0 data
3132	ldxa	[%g0]ASI_ITLB_TAGREAD, %o2	! %o2 = entry 0 tag
3133
3134	cmp	%o1, %g0			! Is this entry valid?
3135	bge	%xcc, 1f
3136	  andcc	%o1, TTE_LCK_INT, %g0		! Is this entry locked?
3137	bnz	%icc, 2f
3138	  nop
31391:
3140	retl					! Nope, outta here...
3141	  wrpr	%g0, %o3, %pstate		! Enable interrupts
31422:
3143	sethi	%hi(FLUSH_ADDR), %o4
3144	stxa	%g0, [%o2]ASI_ITLB_DEMAP	! Flush this mapping
3145	flush	%o4				! Flush required for I-MMU
3146	!
3147	! Start search from index 1 up.  This is because the kernel force
3148	! loads its text page at index 15 in sfmmu_kernel_remap() and we
3149	! don't want our relocated entry evicted later.
3150	!
3151	! NOTE: We assume that we'll be successful in finding an unlocked
3152	! or invalid entry.  If that isn't the case there are bound to
3153	! bigger problems.
3154	!
3155	set	(1 << 3), %g3
31563:
3157	ldxa	[%g3]ASI_ITLB_ACCESS, %o4	! Load TTE from t16
3158	!
3159	! If this entry isn't valid, we'll choose to displace it (regardless
3160	! of the lock bit).
3161	!
3162	cmp	%o4, %g0			! TTE is > 0 iff not valid
3163	bge	%xcc, 4f			! If invalid, go displace
3164	  andcc	%o4, TTE_LCK_INT, %g0		! Check for lock bit
3165	bnz,a	%icc, 3b			! If locked, look at next
3166	  add	%g3, (1 << 3), %g3		!  entry
31674:
3168	!
3169	! We found an unlocked or invalid entry; we'll explicitly load
3170	! the former index 0 entry here.
3171	!
3172	sethi	%hi(FLUSH_ADDR), %o4
3173	set	MMU_TAG_ACCESS, %g4
3174	stxa	%o2, [%g4]ASI_IMMU
3175	stxa	%o1, [%g3]ASI_ITLB_ACCESS
3176	flush	%o4				! Flush required for I-MMU
3177	retl
3178	  wrpr	%g0, %o3, %pstate		! Enable interrupts
3179	SET_SIZE(itlb_erratum34_fixup)
3180
3181#endif	/* lint */
3182
3183#if	defined(lint)
3184
3185/*ARGSUSED*/
3186void
3187dtlb_erratum34_fixup(void)
3188{}
3189
3190#else	/* lint */
3191
3192	!
3193	! In Cheetah+ erratum 34, under certain conditions a DTLB locked
3194	! index 0 TTE will erroneously be displaced when a new TTE is
3195	! loaded.  In order to avoid cheetah+ erratum 34, locked index 0
3196	! TTEs must be relocated.
3197	!
3198	ENTRY_NP(dtlb_erratum34_fixup)
3199	rdpr	%pstate, %o3
3200#ifdef DEBUG
3201	PANIC_IF_INTR_DISABLED_PSTR(%o3, u3_di_label2, %g1)
3202#endif /* DEBUG */
3203	wrpr	%o3, PSTATE_IE, %pstate		! Disable interrupts
3204	ldxa	[%g0]ASI_DTLB_ACCESS, %o1	! %o1 = entry 0 data
3205	ldxa	[%g0]ASI_DTLB_TAGREAD, %o2	! %o2 = entry 0 tag
3206
3207	cmp	%o1, %g0			! Is this entry valid?
3208	bge	%xcc, 1f
3209	  andcc	%o1, TTE_LCK_INT, %g0		! Is this entry locked?
3210	bnz	%icc, 2f
3211	  nop
32121:
3213	retl					! Nope, outta here...
3214	  wrpr	%g0, %o3, %pstate		! Enable interrupts
32152:
3216	stxa	%g0, [%o2]ASI_DTLB_DEMAP	! Flush this mapping
3217	membar	#Sync
3218	!
3219	! Start search from index 1 up.
3220	!
3221	! NOTE: We assume that we'll be successful in finding an unlocked
3222	! or invalid entry.  If that isn't the case there are bound to
3223	! bigger problems.
3224	!
3225	set	(1 << 3), %g3
32263:
3227	ldxa	[%g3]ASI_DTLB_ACCESS, %o4	! Load TTE from t16
3228	!
3229	! If this entry isn't valid, we'll choose to displace it (regardless
3230	! of the lock bit).
3231	!
3232	cmp	%o4, %g0			! TTE is > 0 iff not valid
3233	bge	%xcc, 4f			! If invalid, go displace
3234	  andcc	%o4, TTE_LCK_INT, %g0		! Check for lock bit
3235	bnz,a	%icc, 3b			! If locked, look at next
3236	  add	%g3, (1 << 3), %g3		!  entry
32374:
3238	!
3239	! We found an unlocked or invalid entry; we'll explicitly load
3240	! the former index 0 entry here.
3241	!
3242	set	MMU_TAG_ACCESS, %g4
3243	stxa	%o2, [%g4]ASI_DMMU
3244	stxa	%o1, [%g3]ASI_DTLB_ACCESS
3245	membar	#Sync
3246	retl
3247	  wrpr	%g0, %o3, %pstate		! Enable interrupts
3248	SET_SIZE(dtlb_erratum34_fixup)
3249
3250#endif	/* lint */
3251
3252#endif	/* CHEETAHPLUS_ERRATUM_34 */
3253
3254