xref: /titanic_41/usr/src/uts/sun4u/cpu/us3_common_asm.s (revision 3ee0e49223f178da635734759b9167f924321ff0)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 *
26 * Assembly code support for Cheetah/Cheetah+ modules
27 */
28
29#pragma ident	"%Z%%M%	%I%	%E% SMI"
30
31#if !defined(lint)
32#include "assym.h"
33#endif	/* !lint */
34
35#include <sys/asm_linkage.h>
36#include <sys/mmu.h>
37#include <vm/hat_sfmmu.h>
38#include <sys/machparam.h>
39#include <sys/machcpuvar.h>
40#include <sys/machthread.h>
41#include <sys/machtrap.h>
42#include <sys/privregs.h>
43#include <sys/trap.h>
44#include <sys/cheetahregs.h>
45#include <sys/us3_module.h>
46#include <sys/xc_impl.h>
47#include <sys/intreg.h>
48#include <sys/async.h>
49#include <sys/clock.h>
50#include <sys/cheetahasm.h>
51#include <sys/cmpregs.h>
52
53#ifdef TRAPTRACE
54#include <sys/traptrace.h>
55#endif /* TRAPTRACE */
56
57#if !defined(lint)
58
59/* BEGIN CSTYLED */
60
61#define	DCACHE_FLUSHPAGE(arg1, arg2, tmp1, tmp2, tmp3)			\
62	ldxa	[%g0]ASI_DCU, tmp1					;\
63	btst	DCU_DC, tmp1		/* is dcache enabled? */	;\
64	bz,pn	%icc, 1f						;\
65	ASM_LD(tmp1, dcache_linesize)					;\
66	ASM_LD(tmp2, dflush_type)					;\
67	cmp	tmp2, FLUSHPAGE_TYPE					;\
68	be,pt	%icc, 2f						;\
69	nop								;\
70	sllx	arg1, CHEETAH_DC_VBIT_SHIFT, arg1/* tag to compare */	;\
71	ASM_LD(tmp3, dcache_size)					;\
72	cmp	tmp2, FLUSHMATCH_TYPE					;\
73	be,pt	%icc, 3f						;\
74	nop								;\
75	/*								\
76	 * flushtype = FLUSHALL_TYPE, flush the whole thing		\
77	 * tmp3 = cache size						\
78	 * tmp1 = cache line size					\
79	 */								\
80	sub	tmp3, tmp1, tmp2					;\
814:									\
82	stxa	%g0, [tmp2]ASI_DC_TAG					;\
83	membar	#Sync							;\
84	cmp	%g0, tmp2						;\
85	bne,pt	%icc, 4b						;\
86	sub	tmp2, tmp1, tmp2					;\
87	ba,pt	%icc, 1f						;\
88	nop								;\
89	/*								\
90	 * flushtype = FLUSHPAGE_TYPE					\
91	 * arg1 = pfn							\
92	 * arg2 = virtual color						\
93	 * tmp1 = cache line size					\
94	 * tmp2 = tag from cache					\
95	 * tmp3 = counter						\
96	 */								\
972:									\
98	set	MMU_PAGESIZE, tmp3					;\
99        sllx    arg1, MMU_PAGESHIFT, arg1  /* pfn to 43 bit PA	   */   ;\
100	sub	tmp3, tmp1, tmp3					;\
1014:									\
102	stxa	%g0, [arg1 + tmp3]ASI_DC_INVAL				;\
103	membar	#Sync							;\
1045:									\
105	cmp	%g0, tmp3						;\
106	bnz,pt	%icc, 4b		/* branch if not done */	;\
107	sub	tmp3, tmp1, tmp3					;\
108	ba,pt	%icc, 1f						;\
109	nop								;\
110	/*								\
111	 * flushtype = FLUSHMATCH_TYPE					\
112	 * arg1 = tag to compare against				\
113	 * tmp1 = cache line size					\
114	 * tmp3 = cache size						\
115	 * arg2 = counter						\
116	 * tmp2 = cache tag						\
117	 */								\
1183:									\
119	sub	tmp3, tmp1, arg2					;\
1204:									\
121	ldxa	[arg2]ASI_DC_TAG, tmp2		/* read tag */		;\
122	btst	CHEETAH_DC_VBIT_MASK, tmp2				;\
123	bz,pn	%icc, 5f		/* br if no valid sub-blocks */	;\
124	andn	tmp2, CHEETAH_DC_VBIT_MASK, tmp2 /* clear out v bits */	;\
125	cmp	tmp2, arg1						;\
126	bne,pn	%icc, 5f		/* branch if tag miss */	;\
127	nop								;\
128	stxa	%g0, [arg2]ASI_DC_TAG					;\
129	membar	#Sync							;\
1305:									\
131	cmp	%g0, arg2						;\
132	bne,pt	%icc, 4b		/* branch if not done */	;\
133	sub	arg2, tmp1, arg2					;\
1341:
135
136
137/* END CSTYLED */
138
139#endif	/* !lint */
140
141/*
142 * Cheetah MMU and Cache operations.
143 */
144
145#if defined(lint)
146
147/* ARGSUSED */
148void
149vtag_flushpage(caddr_t vaddr, u_int ctxnum)
150{}
151
152#else	/* lint */
153
154	ENTRY_NP(vtag_flushpage)
155	/*
156	 * flush page from the tlb
157	 *
158	 * %o0 = vaddr
159	 * %o1 = ctxnum
160	 */
161	rdpr	%pstate, %o5
162#ifdef DEBUG
163	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
164	bnz,a,pt %icc, 3f			/* disabled, panic	 */
165	  nop
166	save	%sp, -SA(MINFRAME), %sp
167	sethi	%hi(sfmmu_panic1), %o0
168	call	panic
169	  or	%o0, %lo(sfmmu_panic1), %o0
170	ret
171	restore
1723:
173#endif /* DEBUG */
174	/*
175	 * disable ints
176	 */
177	andn	%o5, PSTATE_IE, %o4
178	wrpr	%o4, 0, %pstate
179
180	/*
181	 * Then, blow out the tlb
182	 * Interrupts are disabled to prevent the primary ctx register
183	 * from changing underneath us.
184	 */
185	brnz,pt	%o1, 1f			/* KCONTEXT */
186	sethi	%hi(FLUSH_ADDR), %o3
187	/*
188	 * For KCONTEXT demaps use primary. type = page implicitly
189	 */
190	stxa	%g0, [%o0]ASI_DTLB_DEMAP	/* dmmu flush for KCONTEXT */
191	stxa	%g0, [%o0]ASI_ITLB_DEMAP	/* immu flush for KCONTEXT */
192	flush	%o3
193	b	5f
194	nop
1951:
196	/*
197	 * User demap.  We need to set the primary context properly.
198	 * Secondary context cannot be used for Cheetah IMMU.
199	 * %o0 = vaddr
200	 * %o1 = ctxnum
201	 * %o3 = FLUSH_ADDR
202	 */
203	sethi	%hi(ctx_pgsz_array), %o4
204	ldn     [%o4 + %lo(ctx_pgsz_array)], %o4
205	brz	%o4, 2f
206	nop
207	ldub	[%o4 + %o1], %o4
208	sll	%o4, CTXREG_EXT_SHIFT, %o4
209	or	%o1, %o4, %o1
2102:
211	wrpr	%g0, 1, %tl
212	set	MMU_PCONTEXT, %o4
213	or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %o0, %o0
214	ldxa	[%o4]ASI_DMMU, %o2		/* rd old ctxnum */
215	stxa	%o1, [%o4]ASI_DMMU		/* wr new ctxum */
2164:
217	stxa	%g0, [%o0]ASI_DTLB_DEMAP
218	stxa	%g0, [%o0]ASI_ITLB_DEMAP
219	stxa	%o2, [%o4]ASI_DMMU		/* restore old ctxnum */
220	flush	%o3
221	wrpr	%g0, 0, %tl
2225:
223	retl
224	wrpr	%g0, %o5, %pstate		/* enable interrupts */
225	SET_SIZE(vtag_flushpage)
226
227#endif	/* lint */
228
229
230#if defined(lint)
231
232/* ARGSUSED */
233void
234vtag_flushctx(u_int ctxnum)
235{}
236
237#else	/* lint */
238
239	ENTRY_NP(vtag_flushctx)
240	/*
241	 * flush context from the tlb
242	 *
243	 * %o0 = ctxnum
244	 * We disable interrupts to prevent the primary ctx register changing
245	 * underneath us.
246	 */
247	sethi	%hi(FLUSH_ADDR), %o3
248	rdpr	%pstate, %o2
249
250#ifdef DEBUG
251	andcc	%o2, PSTATE_IE, %g0		/* if interrupts already */
252	bnz,a,pt %icc, 1f			/* disabled, panic	 */
253	  nop
254	sethi	%hi(sfmmu_panic1), %o0
255	call	panic
256	  or	%o0, %lo(sfmmu_panic1), %o0
2571:
258#endif /* DEBUG */
259
260	sethi	%hi(ctx_pgsz_array), %o4
261	ldn     [%o4 + %lo(ctx_pgsz_array)], %o4
262	brz	%o4, 2f
263	nop
264	ldub	[%o4 + %o0], %o4
265	sll	%o4, CTXREG_EXT_SHIFT, %o4
266	or	%o0, %o4, %o0
2672:
268	wrpr	%o2, PSTATE_IE, %pstate		/* disable interrupts */
269	set	MMU_PCONTEXT, %o4
270	set	DEMAP_CTX_TYPE | DEMAP_PRIMARY, %g1
271	wrpr	%g0, 1, %tl
272	ldxa	[%o4]ASI_DMMU, %o5		/* rd old ctxnum */
273	stxa	%o0, [%o4]ASI_DMMU		/* wr new ctxum */
2744:
275	stxa	%g0, [%g1]ASI_DTLB_DEMAP
276	stxa	%g0, [%g1]ASI_ITLB_DEMAP
277	stxa	%o5, [%o4]ASI_DMMU		/* restore old ctxnum */
278	flush	%o3
279	wrpr	%g0, 0, %tl
2805:
281	retl
282	wrpr	%g0, %o2, %pstate		/* enable interrupts */
283	SET_SIZE(vtag_flushctx)
284
285#endif	/* lint */
286
287
288#if defined(lint)
289
290void
291vtag_flushall(void)
292{}
293
294#else	/* lint */
295
296	ENTRY_NP2(vtag_flushall, demap_all)
297	/*
298	 * flush the tlb
299	 */
300	sethi	%hi(FLUSH_ADDR), %o3
301	set	DEMAP_ALL_TYPE, %g1
302	stxa	%g0, [%g1]ASI_DTLB_DEMAP
303	stxa	%g0, [%g1]ASI_ITLB_DEMAP
304	flush	%o3
305	retl
306	nop
307	SET_SIZE(demap_all)
308	SET_SIZE(vtag_flushall)
309
310#endif	/* lint */
311
312
313#if defined(lint)
314
315/* ARGSUSED */
316void
317vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum)
318{}
319
320#else	/* lint */
321
322	ENTRY_NP(vtag_flushpage_tl1)
323	/*
324	 * x-trap to flush page from tlb and tsb
325	 *
326	 * %g1 = vaddr, zero-extended on 32-bit kernel
327	 * %g2 = ctxnum
328	 *
329	 * assumes TSBE_TAG = 0
330	 */
331	srln	%g1, MMU_PAGESHIFT, %g1
332	brnz,pt	%g2, 1f				/* KCONTEXT */
333	  slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */
334
335	/* We need to demap in the kernel context */
336	or	DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1
337	stxa	%g0, [%g1]ASI_DTLB_DEMAP
338	stxa	%g0, [%g1]ASI_ITLB_DEMAP
339	retry
3401:
341	/* We need to demap in a user context */
342	or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1
343	sethi	%hi(ctx_pgsz_array), %g4
344	ldn     [%g4 + %lo(ctx_pgsz_array)], %g4
345	brz	%g4, 2f
346	nop
347	ldub	[%g4 + %g2], %g4
348	sll	%g4, CTXREG_EXT_SHIFT, %g4
349	or	%g2, %g4, %g2
3502:
351	set	MMU_PCONTEXT, %g4
352	ldxa	[%g4]ASI_DMMU, %g5		/* rd old ctxnum */
353	stxa	%g2, [%g4]ASI_DMMU		/* wr new ctxum */
354	stxa	%g0, [%g1]ASI_DTLB_DEMAP
355	stxa	%g0, [%g1]ASI_ITLB_DEMAP
356	stxa	%g5, [%g4]ASI_DMMU		/* restore old ctxnum */
357	retry
358	SET_SIZE(vtag_flushpage_tl1)
359
360#endif	/* lint */
361
362
363#if defined(lint)
364
365/* ARGSUSED */
366void
367vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt)
368{}
369
370#else	/* lint */
371
372	ENTRY_NP(vtag_flush_pgcnt_tl1)
373	/*
374	 * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb
375	 *
376	 * %g1 = vaddr, zero-extended on 32-bit kernel
377	 * %g2 = <zero32|ctx16|pgcnt16>
378	 *
379	 * NOTE: this handler relies on the fact that no
380	 *	interrupts or traps can occur during the loop
381	 *	issuing the TLB_DEMAP operations. It is assumed
382	 *	that interrupts are disabled and this code is
383	 *	fetching from the kernel locked text address.
384	 *
385	 * assumes TSBE_TAG = 0
386	 */
387	set	0xffff, %g4
388	and	%g4, %g2, %g3			/* g3 = pgcnt */
389	srln	%g2, 16, %g2			/* g2 = ctxnum */
390	srln	%g1, MMU_PAGESHIFT, %g1
391	brnz,pt	%g2, 1f				/* KCONTEXT? */
392	  slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */
393
394	/* We need to demap in the kernel context */
395	or	DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1
396	set	MMU_PAGESIZE, %g2		/* g2 = pgsize */
3974:
398	stxa	%g0, [%g1]ASI_DTLB_DEMAP
399	stxa	%g0, [%g1]ASI_ITLB_DEMAP
400	deccc	%g3				/* decr pgcnt */
401	bnz,pt	%icc,4b
402	  add	%g1, %g2, %g1			/* next page */
403	retry
4041:
405	/* We need to demap in a user context */
406	sethi	%hi(ctx_pgsz_array), %g4
407	ldn     [%g4 + %lo(ctx_pgsz_array)], %g4
408	brz	%g4, 2f
409	  or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1
410	ldub	[%g4 + %g2], %g4
411	sll	%g4, CTXREG_EXT_SHIFT, %g4
412	or	%g2, %g4, %g2
4132:
414	set	MMU_PCONTEXT, %g4
415	ldxa	[%g4]ASI_DMMU, %g5		/* rd old ctxnum */
416	stxa	%g2, [%g4]ASI_DMMU		/* wr new ctxum */
417
418	set	MMU_PAGESIZE, %g2		/* g2 = pgsize */
4193:
420	stxa	%g0, [%g1]ASI_DTLB_DEMAP
421	stxa	%g0, [%g1]ASI_ITLB_DEMAP
422	deccc	%g3				/* decr pgcnt */
423	bnz,pt	%icc,3b
424	  add	%g1, %g2, %g1			/* next page */
425
426	stxa	%g5, [%g4]ASI_DMMU		/* restore old ctxnum */
427	retry
428	SET_SIZE(vtag_flush_pgcnt_tl1)
429
430#endif	/* lint */
431
432
433#if defined(lint)
434
435/* ARGSUSED */
436void
437vtag_flushctx_tl1(uint64_t ctxnum, uint64_t dummy)
438{}
439
440#else	/* lint */
441
442	ENTRY_NP(vtag_flushctx_tl1)
443	/*
444	 * x-trap to flush context from tlb
445	 *
446	 * %g1 = ctxnum
447	 */
448	sethi	%hi(ctx_pgsz_array), %g4
449	ldn     [%g4 + %lo(ctx_pgsz_array)], %g4
450	brz	%g4, 2f
451	nop
452	ldub	[%g4 + %g1], %g4
453	sll	%g4, CTXREG_EXT_SHIFT, %g4
454	or	%g1, %g4, %g1
4552:
456	set	DEMAP_CTX_TYPE | DEMAP_PRIMARY, %g4
457	set	MMU_PCONTEXT, %g3
458	ldxa	[%g3]ASI_DMMU, %g5		/* rd old ctxnum */
459	stxa	%g1, [%g3]ASI_DMMU		/* wr new ctxum */
460	stxa	%g0, [%g4]ASI_DTLB_DEMAP
461	stxa	%g0, [%g4]ASI_ITLB_DEMAP
462	stxa	%g5, [%g3]ASI_DMMU		/* restore old ctxnum */
463	retry
464	SET_SIZE(vtag_flushctx_tl1)
465
466#endif	/* lint */
467
468
469#if defined(lint)
470
471/*ARGSUSED*/
472void
473vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2)
474{}
475
476#else	/* lint */
477
478	ENTRY_NP(vtag_flushall_tl1)
479	/*
480	 * x-trap to flush tlb
481	 */
482	set	DEMAP_ALL_TYPE, %g4
483	stxa	%g0, [%g4]ASI_DTLB_DEMAP
484	stxa	%g0, [%g4]ASI_ITLB_DEMAP
485	retry
486	SET_SIZE(vtag_flushall_tl1)
487
488#endif	/* lint */
489
490
491#if defined(lint)
492
493/* ARGSUSED */
494void
495vac_flushpage(pfn_t pfnum, int vcolor)
496{}
497
498#else	/* lint */
499
500/*
501 * vac_flushpage(pfnum, color)
502 *	Flush 1 8k page of the D-$ with physical page = pfnum
503 *	Algorithm:
504 *		The cheetah dcache is a 64k psuedo 4 way accaociative cache.
505 *		It is virtual indexed, physically tagged cache.
506 */
507	.seg	".data"
508	.align	8
509	.global	dflush_type
510dflush_type:
511	.word	FLUSHPAGE_TYPE
512
513	ENTRY(vac_flushpage)
514	/*
515	 * flush page from the d$
516	 *
517	 * %o0 = pfnum, %o1 = color
518	 */
519	DCACHE_FLUSHPAGE(%o0, %o1, %o2, %o3, %o4)
520	retl
521	  nop
522	SET_SIZE(vac_flushpage)
523
524#endif	/* lint */
525
526
527#if defined(lint)
528
529/* ARGSUSED */
530void
531vac_flushpage_tl1(uint64_t pfnum, uint64_t vcolor)
532{}
533
534#else	/* lint */
535
536	ENTRY_NP(vac_flushpage_tl1)
537	/*
538	 * x-trap to flush page from the d$
539	 *
540	 * %g1 = pfnum, %g2 = color
541	 */
542	DCACHE_FLUSHPAGE(%g1, %g2, %g3, %g4, %g5)
543	retry
544	SET_SIZE(vac_flushpage_tl1)
545
546#endif	/* lint */
547
548
549#if defined(lint)
550
551/* ARGSUSED */
552void
553vac_flushcolor(int vcolor, pfn_t pfnum)
554{}
555
556#else	/* lint */
557	/*
558	 * In UltraSPARC III flushcolor is same as as flushpage.
559	 * This is because we have an ASI to flush dcache using physical
560	 * address.
561	 * Flushing dcache using physical address is faster because we
562	 * don't have to deal with associativity of dcache.
563	 * The arguments to vac_flushpage() and vac_flushcolor() are same but
564	 * the order is reversed. this is because we maintain compatibility
565	 * with spitfire, in which vac_flushcolor has only one argument, namely
566	 * vcolor.
567	 */
568
569	ENTRY(vac_flushcolor)
570	/*
571	 * %o0 = vcolor, %o1 = pfnum
572	 */
573	DCACHE_FLUSHPAGE(%o1, %o0, %o2, %o3, %o4)
574	retl
575	  nop
576	SET_SIZE(vac_flushcolor)
577
578#endif	/* lint */
579
580
581#if defined(lint)
582
583/* ARGSUSED */
584void
585vac_flushcolor_tl1(uint64_t vcolor, uint64_t pfnum)
586{}
587
588#else	/* lint */
589
590	ENTRY(vac_flushcolor_tl1)
591	/*
592	 * %g1 = vcolor
593	 * %g2 = pfnum
594	 */
595	DCACHE_FLUSHPAGE(%g2, %g1, %g3, %g4, %g5)
596	retry
597	SET_SIZE(vac_flushcolor_tl1)
598
599#endif	/* lint */
600
601#if defined(lint)
602
603int
604idsr_busy(void)
605{
606	return (0);
607}
608
609#else	/* lint */
610
611/*
612 * Determine whether or not the IDSR is busy.
613 * Entry: no arguments
614 * Returns: 1 if busy, 0 otherwise
615 */
616	ENTRY(idsr_busy)
617	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
618	clr	%o0
619	btst	IDSR_BUSY, %g1
620	bz,a,pt	%xcc, 1f
621	mov	1, %o0
6221:
623	retl
624	nop
625	SET_SIZE(idsr_busy)
626
627#endif	/* lint */
628
629#if defined(lint)
630
631/* ARGSUSED */
632void
633init_mondo(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
634{}
635
636/* ARGSUSED */
637void
638init_mondo_nocheck(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
639{}
640
641#else	/* lint */
642
643	.global _dispatch_status_busy
644_dispatch_status_busy:
645	.asciz	"ASI_INTR_DISPATCH_STATUS error: busy"
646	.align	4
647
648/*
649 * Setup interrupt dispatch data registers
650 * Entry:
651 *	%o0 - function or inumber to call
652 *	%o1, %o2 - arguments (2 uint64_t's)
653 */
654	.seg "text"
655
656	ENTRY(init_mondo)
657#ifdef DEBUG
658	!
659	! IDSR should not be busy at the moment
660	!
661	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
662	btst	IDSR_BUSY, %g1
663	bz,pt	%xcc, 1f
664	nop
665	sethi	%hi(_dispatch_status_busy), %o0
666	call	panic
667	or	%o0, %lo(_dispatch_status_busy), %o0
668#endif /* DEBUG */
669
670	ALTENTRY(init_mondo_nocheck)
671	!
672	! interrupt vector dispatch data reg 0
673	!
6741:
675	mov	IDDR_0, %g1
676	mov	IDDR_1, %g2
677	mov	IDDR_2, %g3
678	stxa	%o0, [%g1]ASI_INTR_DISPATCH
679
680	!
681	! interrupt vector dispatch data reg 1
682	!
683	stxa	%o1, [%g2]ASI_INTR_DISPATCH
684
685	!
686	! interrupt vector dispatch data reg 2
687	!
688	stxa	%o2, [%g3]ASI_INTR_DISPATCH
689
690	membar	#Sync
691	retl
692	nop
693	SET_SIZE(init_mondo_nocheck)
694	SET_SIZE(init_mondo)
695
696#endif	/* lint */
697
698
699#if !(defined(JALAPENO) || defined(SERRANO))
700
701#if defined(lint)
702
703/* ARGSUSED */
704void
705shipit(int upaid, int bn)
706{ return; }
707
708#else	/* lint */
709
710/*
711 * Ship mondo to aid using busy/nack pair bn
712 */
713	ENTRY_NP(shipit)
714	sll	%o0, IDCR_PID_SHIFT, %g1	! IDCR<18:14> = agent id
715	sll	%o1, IDCR_BN_SHIFT, %g2		! IDCR<28:24> = b/n pair
716	or	%g1, IDCR_OFFSET, %g1		! IDCR<13:0> = 0x70
717	or	%g1, %g2, %g1
718	stxa	%g0, [%g1]ASI_INTR_DISPATCH	! interrupt vector dispatch
719	membar	#Sync
720	retl
721	nop
722	SET_SIZE(shipit)
723
724#endif	/* lint */
725
726#endif	/* !(JALAPENO || SERRANO) */
727
728
729#if defined(lint)
730
731/* ARGSUSED */
732void
733flush_instr_mem(caddr_t vaddr, size_t len)
734{}
735
736#else	/* lint */
737
738/*
739 * flush_instr_mem:
740 *	Flush 1 page of the I-$ starting at vaddr
741 * 	%o0 vaddr
742 *	%o1 bytes to be flushed
743 * UltraSPARC-III maintains consistency of the on-chip Instruction Cache with
744 * the stores from all processors so that a FLUSH instruction is only needed
745 * to ensure pipeline is consistent. This means a single flush is sufficient at
746 * the end of a sequence of stores that updates the instruction stream to
747 * ensure correct operation.
748 */
749
750	ENTRY(flush_instr_mem)
751	flush	%o0			! address irrelevent
752	retl
753	nop
754	SET_SIZE(flush_instr_mem)
755
756#endif	/* lint */
757
758
759#if defined(CPU_IMP_ECACHE_ASSOC)
760
761#if defined(lint)
762
763/* ARGSUSED */
764uint64_t
765get_ecache_ctrl(void)
766{ return (0); }
767
768#else	/* lint */
769
770	ENTRY(get_ecache_ctrl)
771	GET_CPU_IMPL(%o0)
772	cmp	%o0, JAGUAR_IMPL
773	!
774	! Putting an ASI access in the delay slot may
775	! cause it to be accessed, even when annulled.
776	!
777	bne	1f
778	  nop
779	ldxa	[%g0]ASI_EC_CFG_TIMING, %o0	! read Jaguar shared E$ ctrl reg
780	b	2f
781	  nop
7821:
783	ldxa	[%g0]ASI_EC_CTRL, %o0		! read Ch/Ch+ E$ control reg
7842:
785	retl
786	  nop
787	SET_SIZE(get_ecache_ctrl)
788
789#endif	/* lint */
790
791#endif	/* CPU_IMP_ECACHE_ASSOC */
792
793
794#if !(defined(JALAPENO) || defined(SERRANO))
795
796/*
797 * flush_ecache:
798 *	%o0 - 64 bit physical address
799 *	%o1 - ecache size
800 *	%o2 - ecache linesize
801 */
802#if defined(lint)
803
804/*ARGSUSED*/
805void
806flush_ecache(uint64_t physaddr, size_t ecache_size, size_t ecache_linesize)
807{}
808
809#else /* !lint */
810
811	ENTRY(flush_ecache)
812
813	/*
814	 * For certain CPU implementations, we have to flush the L2 cache
815	 * before flushing the ecache.
816	 */
817	PN_L2_FLUSHALL(%g3, %g4, %g5)
818
819	/*
820	 * Flush the entire Ecache using displacement flush.
821	 */
822	ECACHE_FLUSHALL(%o1, %o2, %o0, %o4)
823
824	retl
825	nop
826	SET_SIZE(flush_ecache)
827
828#endif /* lint */
829
830#endif	/* !(JALAPENO || SERRANO) */
831
832
833#if defined(lint)
834
835void
836flush_dcache(void)
837{}
838
839#else	/* lint */
840
841	ENTRY(flush_dcache)
842	ASM_LD(%o0, dcache_size)
843	ASM_LD(%o1, dcache_linesize)
844	CH_DCACHE_FLUSHALL(%o0, %o1, %o2)
845	retl
846	nop
847	SET_SIZE(flush_dcache)
848
849#endif	/* lint */
850
851
852#if defined(lint)
853
854void
855flush_icache(void)
856{}
857
858#else	/* lint */
859
860	ENTRY(flush_icache)
861	GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, flush_icache_1);
862	ld	[%o0 + CHPR_ICACHE_LINESIZE], %o1
863	ba,pt	%icc, 2f
864	  ld	[%o0 + CHPR_ICACHE_SIZE], %o0
865flush_icache_1:
866	ASM_LD(%o0, icache_size)
867	ASM_LD(%o1, icache_linesize)
8682:
869	CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4)
870	retl
871	nop
872	SET_SIZE(flush_icache)
873
874#endif	/* lint */
875
876#if defined(lint)
877
878/*ARGSUSED*/
879void
880kdi_flush_idcache(int dcache_size, int dcache_lsize, int icache_size,
881    int icache_lsize)
882{
883}
884
885#else	/* lint */
886
887	ENTRY(kdi_flush_idcache)
888	CH_DCACHE_FLUSHALL(%o0, %o1, %g1)
889	CH_ICACHE_FLUSHALL(%o2, %o3, %g1, %g2)
890	membar	#Sync
891	retl
892	nop
893	SET_SIZE(kdi_flush_idcache)
894
895#endif	/* lint */
896
897#if defined(lint)
898
899void
900flush_pcache(void)
901{}
902
903#else	/* lint */
904
905	ENTRY(flush_pcache)
906	PCACHE_FLUSHALL(%o0, %o1, %o2)
907	retl
908	nop
909	SET_SIZE(flush_pcache)
910
911#endif	/* lint */
912
913
914#if defined(CPU_IMP_L1_CACHE_PARITY)
915
916#if defined(lint)
917
918/* ARGSUSED */
919void
920get_dcache_dtag(uint32_t dcache_idx, uint64_t *data)
921{}
922
923#else	/* lint */
924
925/*
926 * Get dcache data and tag.  The Dcache data is a pointer to a ch_dc_data_t
927 * structure (see cheetahregs.h):
928 * The Dcache *should* be turned off when this code is executed.
929 */
930	.align	128
931	ENTRY(get_dcache_dtag)
932	rdpr	%pstate, %o5
933	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
934	wrpr	%g0, %o3, %pstate
935	b	1f
936	  stx	%o0, [%o1 + CH_DC_IDX]
937
938	.align	128
9391:
940	ldxa	[%o0]ASI_DC_TAG, %o2
941	stx	%o2, [%o1 + CH_DC_TAG]
942	membar	#Sync
943	ldxa	[%o0]ASI_DC_UTAG, %o2
944	membar	#Sync
945	stx	%o2, [%o1 + CH_DC_UTAG]
946	ldxa	[%o0]ASI_DC_SNP_TAG, %o2
947	stx	%o2, [%o1 + CH_DC_SNTAG]
948	add	%o1, CH_DC_DATA, %o1
949	clr	%o3
9502:
951	membar	#Sync				! required before ASI_DC_DATA
952	ldxa	[%o0 + %o3]ASI_DC_DATA, %o2
953	membar	#Sync				! required after ASI_DC_DATA
954	stx	%o2, [%o1 + %o3]
955	cmp	%o3, CH_DC_DATA_REG_SIZE - 8
956	blt	2b
957	  add	%o3, 8, %o3
958
959	/*
960	 * Unlike other CPUs in the family, D$ data parity bits for Panther
961	 * do not reside in the microtag. Instead, we have to read them
962	 * using the DC_data_parity bit of ASI_DCACHE_DATA. Also, instead
963	 * of just having 8 parity bits to protect all 32 bytes of data
964	 * per line, we now have 32 bits of parity.
965	 */
966	GET_CPU_IMPL(%o3)
967	cmp	%o3, PANTHER_IMPL
968	bne	4f
969	  clr	%o3
970
971	/*
972	 * move our pointer to the next field where we store parity bits
973	 * and add the offset of the last parity byte since we will be
974	 * storing all 4 parity bytes within one 64 bit field like this:
975	 *
976	 * +------+------------+------------+------------+------------+
977	 * |  -   | DC_parity  | DC_parity  | DC_parity  | DC_parity  |
978	 * |  -   | for word 3 | for word 2 | for word 1 | for word 0 |
979	 * +------+------------+------------+------------+------------+
980	 *  63:32     31:24        23:16         15:8          7:0
981	 */
982	add	%o1, CH_DC_PN_DATA_PARITY - CH_DC_DATA + 7, %o1
983
984	/* add the DC_data_parity bit into our working index */
985	mov	1, %o2
986	sll	%o2, PN_DC_DATA_PARITY_BIT_SHIFT, %o2
987	or	%o0, %o2, %o0
9883:
989	membar	#Sync				! required before ASI_DC_DATA
990	ldxa	[%o0 + %o3]ASI_DC_DATA, %o2
991	membar	#Sync				! required after ASI_DC_DATA
992	stb	%o2, [%o1]
993	dec	%o1
994	cmp	%o3, CH_DC_DATA_REG_SIZE - 8
995	blt	3b
996	  add	%o3, 8, %o3
9974:
998	retl
999	  wrpr	%g0, %o5, %pstate
1000	SET_SIZE(get_dcache_dtag)
1001
1002#endif	/* lint */
1003
1004
1005#if defined(lint)
1006
1007/* ARGSUSED */
1008void
1009get_icache_dtag(uint32_t ecache_idx, uint64_t *data)
1010{}
1011
1012#else	/* lint */
1013
1014/*
1015 * Get icache data and tag.  The data argument is a pointer to a ch_ic_data_t
1016 * structure (see cheetahregs.h):
1017 * The Icache *Must* be turned off when this function is called.
1018 * This is because diagnostic accesses to the Icache interfere with cache
1019 * consistency.
1020 */
1021	.align	128
1022	ENTRY(get_icache_dtag)
1023	rdpr	%pstate, %o5
1024	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
1025	wrpr	%g0, %o3, %pstate
1026
1027	stx	%o0, [%o1 + CH_IC_IDX]
1028	ldxa	[%o0]ASI_IC_TAG, %o2
1029	stx	%o2, [%o1 + CH_IC_PATAG]
1030	add	%o0, CH_ICTAG_UTAG, %o0
1031	ldxa	[%o0]ASI_IC_TAG, %o2
1032	add	%o0, (CH_ICTAG_UPPER - CH_ICTAG_UTAG), %o0
1033	stx	%o2, [%o1 + CH_IC_UTAG]
1034	ldxa	[%o0]ASI_IC_TAG, %o2
1035	add	%o0, (CH_ICTAG_LOWER - CH_ICTAG_UPPER), %o0
1036	stx	%o2, [%o1 + CH_IC_UPPER]
1037	ldxa	[%o0]ASI_IC_TAG, %o2
1038	andn	%o0, CH_ICTAG_TMASK, %o0
1039	stx	%o2, [%o1 + CH_IC_LOWER]
1040	ldxa	[%o0]ASI_IC_SNP_TAG, %o2
1041	stx	%o2, [%o1 + CH_IC_SNTAG]
1042	add	%o1, CH_IC_DATA, %o1
1043	clr	%o3
10442:
1045	ldxa	[%o0 + %o3]ASI_IC_DATA, %o2
1046	stx	%o2, [%o1 + %o3]
1047	cmp	%o3, PN_IC_DATA_REG_SIZE - 8
1048	blt	2b
1049	  add	%o3, 8, %o3
1050
1051	retl
1052	  wrpr	%g0, %o5, %pstate
1053	SET_SIZE(get_icache_dtag)
1054
1055#endif	/* lint */
1056
1057#if defined(lint)
1058
1059/* ARGSUSED */
1060void
1061get_pcache_dtag(uint32_t pcache_idx, uint64_t *data)
1062{}
1063
1064#else	/* lint */
1065
1066/*
1067 * Get pcache data and tags.
1068 * inputs:
1069 *   pcache_idx	- fully constructed VA for for accessing P$ diagnostic
1070 *		  registers. Contains PC_way and PC_addr shifted into
1071 *		  the correct bit positions. See the PRM for more details.
1072 *   data	- pointer to a ch_pc_data_t
1073 * structure (see cheetahregs.h):
1074 */
1075	.align	128
1076	ENTRY(get_pcache_dtag)
1077	rdpr	%pstate, %o5
1078	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
1079	wrpr	%g0, %o3, %pstate
1080
1081	stx	%o0, [%o1 + CH_PC_IDX]
1082	ldxa	[%o0]ASI_PC_STATUS_DATA, %o2
1083	stx	%o2, [%o1 + CH_PC_STATUS]
1084	ldxa	[%o0]ASI_PC_TAG, %o2
1085	stx	%o2, [%o1 + CH_PC_TAG]
1086	ldxa	[%o0]ASI_PC_SNP_TAG, %o2
1087	stx	%o2, [%o1 + CH_PC_SNTAG]
1088	add	%o1, CH_PC_DATA, %o1
1089	clr	%o3
10902:
1091	ldxa	[%o0 + %o3]ASI_PC_DATA, %o2
1092	stx	%o2, [%o1 + %o3]
1093	cmp	%o3, CH_PC_DATA_REG_SIZE - 8
1094	blt	2b
1095	  add	%o3, 8, %o3
1096
1097	retl
1098	  wrpr	%g0, %o5, %pstate
1099	SET_SIZE(get_pcache_dtag)
1100
1101#endif	/* lint */
1102
1103#endif	/* CPU_IMP_L1_CACHE_PARITY */
1104
1105#if defined(lint)
1106
1107/* ARGSUSED */
1108void
1109set_dcu(uint64_t dcu)
1110{}
1111
1112#else	/* lint */
1113
1114/*
1115 * re-enable the i$, d$, w$, and p$ according to bootup cache state.
1116 * Turn on WE, HPE, SPE, PE, IC, and DC bits defined as DCU_CACHE.
1117 *   %o0 - 64 bit constant
1118 */
1119	ENTRY(set_dcu)
1120	stxa	%o0, [%g0]ASI_DCU	! Store to DCU
1121	flush	%g0	/* flush required after changing the IC bit */
1122	retl
1123	nop
1124	SET_SIZE(set_dcu)
1125
1126#endif	/* lint */
1127
1128
1129#if defined(lint)
1130
1131uint64_t
1132get_dcu(void)
1133{
1134	return ((uint64_t)0);
1135}
1136
1137#else	/* lint */
1138
1139/*
1140 * Return DCU register.
1141 */
1142	ENTRY(get_dcu)
1143	ldxa	[%g0]ASI_DCU, %o0		/* DCU control register */
1144	retl
1145	nop
1146	SET_SIZE(get_dcu)
1147
1148#endif	/* lint */
1149
1150/*
1151 * Cheetah/Cheetah+ level 15 interrupt handler trap table entry.
1152 *
1153 * This handler is used to check for softints generated by error trap
1154 * handlers to report errors.  On Cheetah, this mechanism is used by the
1155 * Fast ECC at TL>0 error trap handler and, on Cheetah+, by both the Fast
1156 * ECC at TL>0 error and the I$/D$ parity error at TL>0 trap handlers.
1157 * NB: Must be 8 instructions or less to fit in trap table and code must
1158 *     be relocatable.
1159 */
1160#if defined(lint)
1161
1162void
1163ch_pil15_interrupt_instr(void)
1164{}
1165
1166#else	/* lint */
1167
1168	ENTRY_NP(ch_pil15_interrupt_instr)
1169	ASM_JMP(%g1, ch_pil15_interrupt)
1170	SET_SIZE(ch_pil15_interrupt_instr)
1171
1172#endif
1173
1174
1175#if defined(lint)
1176
1177void
1178ch_pil15_interrupt(void)
1179{}
1180
1181#else	/* lint */
1182
1183	ENTRY_NP(ch_pil15_interrupt)
1184
1185	/*
1186	 * Since pil_interrupt is hacked to assume that every level 15
1187	 * interrupt is generated by the CPU to indicate a performance
1188	 * counter overflow this gets ugly.  Before calling pil_interrupt
1189	 * the Error at TL>0 pending status is inspected.  If it is
1190	 * non-zero, then an error has occurred and it is handled.
1191	 * Otherwise control is transfered to pil_interrupt.  Note that if
1192	 * an error is detected pil_interrupt will not be called and
1193	 * overflow interrupts may be lost causing erroneous performance
1194	 * measurements.  However, error-recovery will have a detrimental
1195	 * effect on performance anyway.
1196	 */
1197	CPU_INDEX(%g1, %g4)
1198	set	ch_err_tl1_pending, %g4
1199	ldub	[%g1 + %g4], %g2
1200	brz	%g2, 1f
1201	  nop
1202
1203	/*
1204	 * We have a pending TL>0 error, clear the TL>0 pending status.
1205	 */
1206	stb	%g0, [%g1 + %g4]
1207
1208	/*
1209	 * Clear the softint.
1210	 */
1211	mov	1, %g5
1212	sll	%g5, PIL_15, %g5
1213	wr	%g5, CLEAR_SOFTINT
1214
1215	/*
1216	 * For Cheetah*, call cpu_tl1_error via systrap at PIL 15
1217	 * to process the Fast ECC/Cache Parity at TL>0 error.  Clear
1218	 * panic flag (%g2).
1219	 */
1220	set	cpu_tl1_error, %g1
1221	clr	%g2
1222	ba	sys_trap
1223	  mov	PIL_15, %g4
1224
12251:
1226	/*
1227	 * The logout is invalid.
1228	 *
1229	 * Call the default interrupt handler.
1230	 */
1231	sethi	%hi(pil_interrupt), %g1
1232	jmp	%g1 + %lo(pil_interrupt)
1233	  mov	PIL_15, %g4
1234
1235	SET_SIZE(ch_pil15_interrupt)
1236#endif
1237
1238
1239/*
1240 * Error Handling
1241 *
1242 * Cheetah provides error checking for all memory access paths between
1243 * the CPU, External Cache, Cheetah Data Switch and system bus. Error
1244 * information is logged in the AFSR, (also AFSR_EXT for Panther) and
1245 * AFAR and one of the following traps is generated (provided that it
1246 * is enabled in External Cache Error Enable Register) to handle that
1247 * error:
1248 * 1. trap 0x70: Precise trap
1249 *    tt0_fecc for errors at trap level(TL)>=0
1250 * 2. trap 0x0A and 0x32: Deferred trap
1251 *    async_err for errors at TL>=0
1252 * 3. trap 0x63: Disrupting trap
1253 *    ce_err for errors at TL=0
1254 *    (Note that trap 0x63 cannot happen at trap level > 0)
1255 *
1256 * Trap level one handlers panic the system except for the fast ecc
1257 * error handler which tries to recover from certain errors.
1258 */
1259
1260/*
1261 * FAST ECC TRAP STRATEGY:
1262 *
1263 * Software must handle single and multi bit errors which occur due to data
1264 * or instruction cache reads from the external cache. A single or multi bit
1265 * error occuring in one of these situations results in a precise trap.
1266 *
1267 * The basic flow of this trap handler is as follows:
1268 *
1269 * 1) Record the state and then turn off the Dcache and Icache.  The Dcache
1270 *    is disabled because bad data could have been installed.  The Icache is
1271 *    turned off because we want to capture the Icache line related to the
1272 *    AFAR.
1273 * 2) Disable trapping on CEEN/NCCEN errors during TL=0 processing.
1274 * 3) Park sibling core if caches are shared (to avoid race condition while
1275 *    accessing shared resources such as L3 data staging register during
1276 *    CPU logout.
1277 * 4) Read the AFAR and AFSR.
1278 * 5) If CPU logout structure is not being used, then:
1279 *    6) Clear all errors from the AFSR.
1280 *    7) Capture Ecache, Dcache and Icache lines in "CPU log out" structure.
1281 *    8) Flush Ecache then Flush Dcache and Icache and restore to previous
1282 *       state.
1283 *    9) Unpark sibling core if we parked it earlier.
1284 *    10) call cpu_fast_ecc_error via systrap at PIL 14 unless we're already
1285 *        running at PIL 15.
1286 * 6) Otherwise, if CPU logout structure is being used:
1287 *    7) Incriment the "logout busy count".
1288 *    8) Flush Ecache then Flush Dcache and Icache and restore to previous
1289 *       state.
1290 *    9) Unpark sibling core if we parked it earlier.
1291 *    10) Issue a retry since the other CPU error logging code will end up
1292 *       finding this error bit and logging information about it later.
1293 * 7) Alternatively (to 5 and 6 above), if the cpu_private struct is not
1294 *    yet initialized such that we can't even check the logout struct, then
1295 *    we place the clo_flags data into %g2 (sys_trap->have_win arg #1) and
1296 *    call cpu_fast_ecc_error via systrap. The clo_flags parameter is used
1297 *    to determine information such as TL, TT, CEEN and NCEEN settings, etc
1298 *    in the high level trap handler since we don't have access to detailed
1299 *    logout information in cases where the cpu_private struct is not yet
1300 *    initialized.
1301 *
1302 * We flush the E$ and D$ here on TL=1 code to prevent getting nested
1303 * Fast ECC traps in the TL=0 code.  If we get a Fast ECC event here in
1304 * the TL=1 code, we will go to the Fast ECC at TL>0 handler which,
1305 * since it is uses different code/data from this handler, has a better
1306 * chance of fixing things up than simply recursing through this code
1307 * again (this would probably cause an eventual kernel stack overflow).
1308 * If the Fast ECC at TL>0 handler encounters a Fast ECC error before it
1309 * can flush the E$ (or the error is a stuck-at bit), we will recurse in
1310 * the Fast ECC at TL>0 handler and eventually Red Mode.
1311 *
1312 * Note that for Cheetah (and only Cheetah), we use alias addresses for
1313 * flushing rather than ASI accesses (which don't exist on Cheetah).
1314 * Should we encounter a Fast ECC error within this handler on Cheetah,
1315 * there's a good chance it's within the ecache_flushaddr buffer (since
1316 * it's the largest piece of memory we touch in the handler and it is
1317 * usually kernel text/data).  For that reason the Fast ECC at TL>0
1318 * handler for Cheetah uses an alternate buffer: ecache_tl1_flushaddr.
1319 */
1320
1321/*
1322 * Cheetah ecc-protected E$ trap (Trap 70) at TL=0
1323 * tt0_fecc is replaced by fecc_err_instr in cpu_init_trap of the various
1324 * architecture-specific files.
1325 * NB: Must be 8 instructions or less to fit in trap table and code must
1326 *     be relocatable.
1327 */
1328
1329#if defined(lint)
1330
1331void
1332fecc_err_instr(void)
1333{}
1334
1335#else	/* lint */
1336
1337	ENTRY_NP(fecc_err_instr)
1338	membar	#Sync			! Cheetah requires membar #Sync
1339
1340	/*
1341	 * Save current DCU state.  Turn off the Dcache and Icache.
1342	 */
1343	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
1344	andn	%g1, DCU_DC + DCU_IC, %g4
1345	stxa	%g4, [%g0]ASI_DCU
1346	flush	%g0	/* flush required after changing the IC bit */
1347
1348	ASM_JMP(%g4, fast_ecc_err)
1349	SET_SIZE(fecc_err_instr)
1350
1351#endif	/* lint */
1352
1353
1354#if !(defined(JALAPENO) || defined(SERRANO))
1355
1356#if defined(lint)
1357
1358void
1359fast_ecc_err(void)
1360{}
1361
1362#else	/* lint */
1363
1364	.section ".text"
1365	.align	64
1366	ENTRY_NP(fast_ecc_err)
1367
1368	/*
1369	 * Turn off CEEN and NCEEN.
1370	 */
1371	ldxa	[%g0]ASI_ESTATE_ERR, %g3
1372	andn	%g3, EN_REG_NCEEN + EN_REG_CEEN, %g4
1373	stxa	%g4, [%g0]ASI_ESTATE_ERR
1374	membar	#Sync			! membar sync required
1375
1376	/*
1377	 * Check to see whether we need to park our sibling core
1378	 * before recording diagnostic information from caches
1379	 * which may be shared by both cores.
1380	 * We use %g1 to store information about whether or not
1381	 * we had to park the core (%g1 holds our DCUCR value and
1382	 * we only use bits from that register which are "reserved"
1383	 * to keep track of core parking) so that we know whether
1384	 * or not to unpark later. %g5 and %g4 are scratch registers.
1385	 */
1386	PARK_SIBLING_CORE(%g1, %g5, %g4)
1387
1388	/*
1389	 * Do the CPU log out capture.
1390	 *   %g3 = "failed?" return value.
1391	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
1392	 *         into this macro via %g4. Output only valid if cpu_private
1393	 *         struct has not been initialized.
1394	 *   CHPR_FECCTL0_LOGOUT = cpu logout structure offset input
1395	 *   %g4 = Trap information stored in the cpu logout flags field
1396	 *   %g5 = scr1
1397	 *   %g6 = scr2
1398	 *   %g3 = scr3
1399	 *   %g4 = scr4
1400	 */
1401	 /* store the CEEN and NCEEN values, TL=0 */
1402	and	%g3, EN_REG_CEEN + EN_REG_NCEEN, %g4
1403	set	CHPR_FECCTL0_LOGOUT, %g6
1404	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)
1405
1406	/*
1407	 * Flush the Ecache (and L2 cache for Panther) to get the error out
1408	 * of the Ecache.  If the UCC or UCU is on a dirty line, then the
1409	 * following flush will turn that into a WDC or WDU, respectively.
1410	 */
1411	PN_L2_FLUSHALL(%g4, %g5, %g6)
1412
1413	CPU_INDEX(%g4, %g5)
1414	mulx	%g4, CPU_NODE_SIZE, %g4
1415	set	cpunodes, %g5
1416	add	%g4, %g5, %g4
1417	ld	[%g4 + ECACHE_LINESIZE], %g5
1418	ld	[%g4 + ECACHE_SIZE], %g4
1419
1420	ASM_LDX(%g6, ecache_flushaddr)
1421	ECACHE_FLUSHALL(%g4, %g5, %g6, %g7)
1422
1423	/*
1424	 * Flush the Dcache.  Since bad data could have been installed in
1425	 * the Dcache we must flush it before re-enabling it.
1426	 */
1427	ASM_LD(%g5, dcache_size)
1428	ASM_LD(%g6, dcache_linesize)
1429	CH_DCACHE_FLUSHALL(%g5, %g6, %g7)
1430
1431	/*
1432	 * Flush the Icache.  Since we turned off the Icache to capture the
1433	 * Icache line it is now stale or corrupted and we must flush it
1434	 * before re-enabling it.
1435	 */
1436	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, fast_ecc_err_5);
1437	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
1438	ba,pt	%icc, 6f
1439	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
1440fast_ecc_err_5:
1441	ASM_LD(%g5, icache_size)
1442	ASM_LD(%g6, icache_linesize)
14436:
1444	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)
1445
1446	/*
1447	 * check to see whether we parked our sibling core at the start
1448	 * of this handler. If so, we need to unpark it here.
1449	 * We use DCUCR reserved bits (stored in %g1) to keep track of
1450	 * whether or not we need to unpark. %g5 and %g4 are scratch registers.
1451	 */
1452	UNPARK_SIBLING_CORE(%g1, %g5, %g4)
1453
1454	/*
1455	 * Restore the Dcache and Icache to the previous state.
1456	 */
1457	stxa	%g1, [%g0]ASI_DCU
1458	flush	%g0	/* flush required after changing the IC bit */
1459
1460	/*
1461	 * Make sure our CPU logout operation was successful.
1462	 */
1463	cmp	%g3, %g0
1464	be	8f
1465	  nop
1466
1467	/*
1468	 * If the logout structure had been busy, how many times have
1469	 * we tried to use it and failed (nesting count)? If we have
1470	 * already recursed a substantial number of times, then we can
1471	 * assume things are not going to get better by themselves and
1472	 * so it would be best to panic.
1473	 */
1474	cmp	%g3, CLO_NESTING_MAX
1475	blt	7f
1476	  nop
1477
1478        call ptl1_panic
1479          mov   PTL1_BAD_ECC, %g1
1480
14817:
1482	/*
1483	 * Otherwise, if the logout structure was busy but we have not
1484	 * nested more times than our maximum value, then we simply
1485	 * issue a retry. Our TL=0 trap handler code will check and
1486	 * clear the AFSR after it is done logging what is currently
1487	 * in the logout struct and handle this event at that time.
1488	 */
1489	retry
14908:
1491	/*
1492	 * Call cpu_fast_ecc_error via systrap at PIL 14 unless we're
1493	 * already at PIL 15.
1494	 */
1495	set	cpu_fast_ecc_error, %g1
1496	rdpr	%pil, %g4
1497	cmp	%g4, PIL_14
1498	ba	sys_trap
1499	  movl	%icc, PIL_14, %g4
1500
1501	SET_SIZE(fast_ecc_err)
1502
1503#endif	/* lint */
1504
1505#endif	/* !(JALAPENO || SERRANO) */
1506
1507
1508/*
1509 * Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy:
1510 *
1511 * The basic flow of this trap handler is as follows:
1512 *
1513 * 1) In the "trap 70" trap table code (fecc_err_tl1_instr), generate a
1514 *    software trap 0 ("ta 0") to buy an extra set of %tpc, etc. which we
1515 *    will use to save %g1 and %g2.
1516 * 2) At the software trap 0 at TL>0 trap table code (fecc_err_tl1_cont_instr),
1517 *    we save %g1+%g2 using %tpc, %tnpc + %tstate and jump to the fast ecc
1518 *    handler (using the just saved %g1).
1519 * 3) Turn off the Dcache if it was on and save the state of the Dcache
1520 *    (whether on or off) in Bit2 (CH_ERR_TSTATE_DC_ON) of %tstate.
1521 *    NB: we don't turn off the Icache because bad data is not installed nor
1522 *        will we be doing any diagnostic accesses.
1523 * 4) compute physical address of the per-cpu/per-tl save area using %g1+%g2
1524 * 5) Save %g1-%g7 into the per-cpu/per-tl save area (%g1 + %g2 from the
1525 *    %tpc, %tnpc, %tstate values previously saved).
1526 * 6) set %tl to %tl - 1.
1527 * 7) Save the appropriate flags and TPC in the ch_err_tl1_data structure.
1528 * 8) Save the value of CH_ERR_TSTATE_DC_ON in the ch_err_tl1_tmp field.
1529 * 9) For Cheetah and Jalapeno, read the AFAR and AFSR and clear.  For
1530 *    Cheetah+ (and later), read the shadow AFAR and AFSR but don't clear.
1531 *    Save the values in ch_err_tl1_data.  For Panther, read the shadow
1532 *    AFSR_EXT and save the value in ch_err_tl1_data.
1533 * 10) Disable CEEN/NCEEN to prevent any disrupting/deferred errors from
1534 *    being queued.  We'll report them via the AFSR/AFAR capture in step 13.
1535 * 11) Flush the Ecache.
1536 *    NB: the Ecache is flushed assuming the largest possible size with
1537 *        the smallest possible line size since access to the cpu_nodes may
1538 *        cause an unrecoverable DTLB miss.
1539 * 12) Reenable CEEN/NCEEN with the value saved from step 10.
1540 * 13) For Cheetah and Jalapeno, read the AFAR and AFSR and clear again.
1541 *    For Cheetah+ (and later), read the primary AFAR and AFSR and now clear.
1542 *    Save the read AFSR/AFAR values in ch_err_tl1_data.  For Panther,
1543 *    read and clear the primary AFSR_EXT and save it in ch_err_tl1_data.
1544 * 14) Flush and re-enable the Dcache if it was on at step 3.
1545 * 15) Do TRAPTRACE if enabled.
1546 * 16) Check if a UCU->WDU (or L3_UCU->WDU for Panther) happened, panic if so.
1547 * 17) Set the event pending flag in ch_err_tl1_pending[CPU]
1548 * 18) Cause a softint 15.  The pil15_interrupt handler will inspect the
1549 *    event pending flag and call cpu_tl1_error via systrap if set.
1550 * 19) Restore the registers from step 5 and issue retry.
1551 */
1552
1553/*
1554 * Cheetah ecc-protected E$ trap (Trap 70) at TL>0
1555 * tt1_fecc is replaced by fecc_err_tl1_instr in cpu_init_trap of the various
1556 * architecture-specific files.  This generates a "Software Trap 0" at TL>0,
1557 * which goes to fecc_err_tl1_cont_instr, and we continue the handling there.
1558 * NB: Must be 8 instructions or less to fit in trap table and code must
1559 *     be relocatable.
1560 */
1561
1562#if defined(lint)
1563
1564void
1565fecc_err_tl1_instr(void)
1566{}
1567
1568#else	/* lint */
1569
1570	ENTRY_NP(fecc_err_tl1_instr)
1571	CH_ERR_TL1_TRAPENTRY(SWTRAP_0);
1572	SET_SIZE(fecc_err_tl1_instr)
1573
1574#endif	/* lint */
1575
1576/*
1577 * Software trap 0 at TL>0.
1578 * tt1_swtrap0 is replaced by fecc_err_tl1_cont_instr in cpu_init_trap of
1579 * the various architecture-specific files.  This is used as a continuation
1580 * of the fast ecc handling where we've bought an extra TL level, so we can
1581 * use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
1582 * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
1583 * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
1584 * order two bits from %g1 and %g2 respectively).
1585 * NB: Must be 8 instructions or less to fit in trap table and code must
1586 *     be relocatable.
1587 */
1588#if defined(lint)
1589
1590void
1591fecc_err_tl1_cont_instr(void)
1592{}
1593
1594#else	/* lint */
1595
1596	ENTRY_NP(fecc_err_tl1_cont_instr)
1597	CH_ERR_TL1_SWTRAPENTRY(fast_ecc_tl1_err)
1598	SET_SIZE(fecc_err_tl1_cont_instr)
1599
1600#endif	/* lint */
1601
1602
1603#if defined(lint)
1604
1605void
1606ce_err(void)
1607{}
1608
1609#else	/* lint */
1610
1611/*
1612 * The ce_err function handles disrupting trap type 0x63 at TL=0.
1613 *
1614 * AFSR errors bits which cause this trap are:
1615 *	CE, EMC, EDU:ST, EDC, WDU, WDC, CPU, CPC, IVU, IVC
1616 *
1617 * NCEEN Bit of Cheetah External Cache Error Enable Register enables
1618 * the following AFSR disrupting traps: EDU:ST, WDU, CPU, IVU
1619 *
1620 * CEEN Bit of Cheetah External Cache Error Enable Register enables
1621 * the following AFSR disrupting traps: CE, EMC, EDC, WDC, CPC, IVC
1622 *
1623 * Cheetah+ also handles (No additional processing required):
1624 *    DUE, DTO, DBERR	(NCEEN controlled)
1625 *    THCE		(CEEN and ET_ECC_en controlled)
1626 *    TUE		(ET_ECC_en controlled)
1627 *
1628 * Panther further adds:
1629 *    IMU, L3_EDU, L3_WDU, L3_CPU		(NCEEN controlled)
1630 *    IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE	(CEEN controlled)
1631 *    TUE_SH, TUE		(NCEEN and L2_tag_ECC_en controlled)
1632 *    L3_TUE, L3_TUE_SH		(NCEEN and ET_ECC_en controlled)
1633 *    THCE			(CEEN and L2_tag_ECC_en controlled)
1634 *    L3_THCE			(CEEN and ET_ECC_en controlled)
1635 *
1636 * Steps:
1637 *	1. Disable hardware corrected disrupting errors only (CEEN)
1638 *	2. Park sibling core if caches are shared (to avoid race
1639 *	   condition while accessing shared resources such as L3
1640 *	   data staging register during CPU logout.
1641 *	3. If the CPU logout structure is not currently being used:
1642 *		4. Clear AFSR error bits
1643 *		5. Capture Ecache, Dcache and Icache lines associated
1644 *		   with AFAR.
1645 *		6. Unpark sibling core if we parked it earlier.
1646 *		7. call cpu_disrupting_error via sys_trap at PIL 14
1647 *		   unless we're already running at PIL 15.
1648 *	4. Otherwise, if the CPU logout structure is busy:
1649 *		5. Incriment "logout busy count" and place into %g3
1650 *		6. Unpark sibling core if we parked it earlier.
1651 *		7. Issue a retry since the other CPU error logging
1652 *		   code will end up finding this error bit and logging
1653 *		   information about it later.
1654 *	5. Alternatively (to 3 and 4 above), if the cpu_private struct is
1655 *         not yet initialized such that we can't even check the logout
1656 *         struct, then we place the clo_flags data into %g2
1657 *         (sys_trap->have_win arg #1) and call cpu_disrupting_error via
1658 *         systrap. The clo_flags parameter is used to determine information
1659 *         such as TL, TT, CEEN settings, etc in the high level trap
1660 *         handler since we don't have access to detailed logout information
1661 *         in cases where the cpu_private struct is not yet initialized.
1662 *
1663 * %g3: [ logout busy count ] - arg #2
1664 * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1
1665 */
1666
1667	.align	128
1668	ENTRY_NP(ce_err)
1669	membar	#Sync			! Cheetah requires membar #Sync
1670
1671	/*
1672	 * Disable trap on hardware corrected errors (CEEN) while at TL=0
1673	 * to prevent recursion.
1674	 */
1675	ldxa	[%g0]ASI_ESTATE_ERR, %g1
1676	bclr	EN_REG_CEEN, %g1
1677	stxa	%g1, [%g0]ASI_ESTATE_ERR
1678	membar	#Sync			! membar sync required
1679
1680	/*
1681	 * Save current DCU state.  Turn off Icache to allow capture of
1682	 * Icache data by DO_CPU_LOGOUT.
1683	 */
1684	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
1685	andn	%g1, DCU_IC, %g4
1686	stxa	%g4, [%g0]ASI_DCU
1687	flush	%g0	/* flush required after changing the IC bit */
1688
1689	/*
1690	 * Check to see whether we need to park our sibling core
1691	 * before recording diagnostic information from caches
1692	 * which may be shared by both cores.
1693	 * We use %g1 to store information about whether or not
1694	 * we had to park the core (%g1 holds our DCUCR value and
1695	 * we only use bits from that register which are "reserved"
1696	 * to keep track of core parking) so that we know whether
1697	 * or not to unpark later. %g5 and %g4 are scratch registers.
1698	 */
1699	PARK_SIBLING_CORE(%g1, %g5, %g4)
1700
1701	/*
1702	 * Do the CPU log out capture.
1703	 *   %g3 = "failed?" return value.
1704	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
1705	 *         into this macro via %g4. Output only valid if cpu_private
1706	 *         struct has not been initialized.
1707	 *   CHPR_CECC_LOGOUT = cpu logout structure offset input
1708	 *   %g4 = Trap information stored in the cpu logout flags field
1709	 *   %g5 = scr1
1710	 *   %g6 = scr2
1711	 *   %g3 = scr3
1712	 *   %g4 = scr4
1713	 */
1714	clr	%g4			! TL=0 bit in afsr
1715	set	CHPR_CECC_LOGOUT, %g6
1716	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)
1717
1718	/*
1719	 * Flush the Icache.  Since we turned off the Icache to capture the
1720	 * Icache line it is now stale or corrupted and we must flush it
1721	 * before re-enabling it.
1722	 */
1723	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, ce_err_1);
1724	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
1725	ba,pt	%icc, 2f
1726	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
1727ce_err_1:
1728	ASM_LD(%g5, icache_size)
1729	ASM_LD(%g6, icache_linesize)
17302:
1731	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)
1732
1733	/*
1734	 * check to see whether we parked our sibling core at the start
1735	 * of this handler. If so, we need to unpark it here.
1736	 * We use DCUCR reserved bits (stored in %g1) to keep track of
1737	 * whether or not we need to unpark. %g5 and %g4 are scratch registers.
1738	 */
1739	UNPARK_SIBLING_CORE(%g1, %g5, %g4)
1740
1741	/*
1742	 * Restore Icache to previous state.
1743	 */
1744	stxa	%g1, [%g0]ASI_DCU
1745	flush	%g0	/* flush required after changing the IC bit */
1746
1747	/*
1748	 * Make sure our CPU logout operation was successful.
1749	 */
1750	cmp	%g3, %g0
1751	be	4f
1752	  nop
1753
1754	/*
1755	 * If the logout structure had been busy, how many times have
1756	 * we tried to use it and failed (nesting count)? If we have
1757	 * already recursed a substantial number of times, then we can
1758	 * assume things are not going to get better by themselves and
1759	 * so it would be best to panic.
1760	 */
1761	cmp	%g3, CLO_NESTING_MAX
1762	blt	3f
1763	  nop
1764
1765        call ptl1_panic
1766          mov   PTL1_BAD_ECC, %g1
1767
17683:
1769	/*
1770	 * Otherwise, if the logout structure was busy but we have not
1771	 * nested more times than our maximum value, then we simply
1772	 * issue a retry. Our TL=0 trap handler code will check and
1773	 * clear the AFSR after it is done logging what is currently
1774	 * in the logout struct and handle this event at that time.
1775	 */
1776	retry
17774:
1778	/*
1779	 * Call cpu_disrupting_error via systrap at PIL 14 unless we're
1780	 * already at PIL 15.
1781	 */
1782	set	cpu_disrupting_error, %g1
1783	rdpr	%pil, %g4
1784	cmp	%g4, PIL_14
1785	ba	sys_trap
1786	  movl	%icc, PIL_14, %g4
1787	SET_SIZE(ce_err)
1788
1789#endif	/* lint */
1790
1791
1792#if defined(lint)
1793
1794/*
1795 * This trap cannot happen at TL>0 which means this routine will never
1796 * actually be called and so we treat this like a BAD TRAP panic.
1797 */
1798void
1799ce_err_tl1(void)
1800{}
1801
1802#else	/* lint */
1803
1804	.align	64
1805	ENTRY_NP(ce_err_tl1)
1806
1807        call ptl1_panic
1808          mov   PTL1_BAD_TRAP, %g1
1809
1810	SET_SIZE(ce_err_tl1)
1811
1812#endif	/* lint */
1813
1814
1815#if defined(lint)
1816
1817void
1818async_err(void)
1819{}
1820
1821#else	/* lint */
1822
1823/*
1824 * The async_err function handles deferred trap types 0xA
1825 * (instruction_access_error) and 0x32 (data_access_error) at TL>=0.
1826 *
1827 * AFSR errors bits which cause this trap are:
1828 *	UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR
1829 * On some platforms, EMU may causes cheetah to pull the error pin
1830 * never giving Solaris a chance to take a trap.
1831 *
1832 * NCEEN Bit of Cheetah External Cache Error Enable Register enables
1833 * the following AFSR deferred traps: UE, EMU, EDU:BLD, TO, BERR
1834 *
1835 * Steps:
1836 *	1. Disable CEEN and NCEEN errors to prevent recursive errors.
1837 *	2. Turn D$ off per Cheetah PRM P.5 Note 6, turn I$ off to capture
1838 *         I$ line in DO_CPU_LOGOUT.
1839 *	3. Park sibling core if caches are shared (to avoid race
1840 *	   condition while accessing shared resources such as L3
1841 *	   data staging register during CPU logout.
1842 *	4. If the CPU logout structure is not currently being used:
1843 *		5. Clear AFSR error bits
1844 *		6. Capture Ecache, Dcache and Icache lines associated
1845 *		   with AFAR.
1846 *		7. Unpark sibling core if we parked it earlier.
1847 *		8. call cpu_deferred_error via sys_trap.
1848 *	5. Otherwise, if the CPU logout structure is busy:
1849 *		6. Incriment "logout busy count"
1850 *		7. Unpark sibling core if we parked it earlier.
1851 *		8) Issue a retry since the other CPU error logging
1852 *		   code will end up finding this error bit and logging
1853 *		   information about it later.
1854 *      6. Alternatively (to 4 and 5 above), if the cpu_private struct is
1855 *         not yet initialized such that we can't even check the logout
1856 *         struct, then we place the clo_flags data into %g2
1857 *         (sys_trap->have_win arg #1) and call cpu_deferred_error via
1858 *         systrap. The clo_flags parameter is used to determine information
1859 *         such as TL, TT, CEEN settings, etc in the high level trap handler
1860 *         since we don't have access to detailed logout information in cases
1861 *         where the cpu_private struct is not yet initialized.
1862 *
1863 * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1
1864 * %g3: [ logout busy count ] - arg #2
1865 */
1866
1867	ENTRY_NP(async_err)
1868	membar	#Sync			! Cheetah requires membar #Sync
1869
1870	/*
1871	 * Disable CEEN and NCEEN.
1872	 */
1873	ldxa	[%g0]ASI_ESTATE_ERR, %g3
1874	andn	%g3, EN_REG_NCEEN + EN_REG_CEEN, %g4
1875	stxa	%g4, [%g0]ASI_ESTATE_ERR
1876	membar	#Sync			! membar sync required
1877
1878	/*
1879	 * Save current DCU state.
1880	 * Disable Icache to allow capture of Icache data by DO_CPU_LOGOUT.
1881	 * Do this regardless of whether this is a Data Access Error or
1882	 * Instruction Access Error Trap.
1883	 * Disable Dcache for both Data Access Error and Instruction Access
1884	 * Error per Cheetah PRM P.5 Note 6.
1885	 */
1886	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
1887	andn	%g1, DCU_IC + DCU_DC, %g4
1888	stxa	%g4, [%g0]ASI_DCU
1889	flush	%g0	/* flush required after changing the IC bit */
1890
1891	/*
1892	 * Check to see whether we need to park our sibling core
1893	 * before recording diagnostic information from caches
1894	 * which may be shared by both cores.
1895	 * We use %g1 to store information about whether or not
1896	 * we had to park the core (%g1 holds our DCUCR value and
1897	 * we only use bits from that register which are "reserved"
1898	 * to keep track of core parking) so that we know whether
1899	 * or not to unpark later. %g6 and %g4 are scratch registers.
1900	 */
1901	PARK_SIBLING_CORE(%g1, %g6, %g4)
1902
1903	/*
1904	 * Do the CPU logout capture.
1905	 *
1906	 *   %g3 = "failed?" return value.
1907	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
1908	 *         into this macro via %g4. Output only valid if cpu_private
1909	 *         struct has not been initialized.
1910	 *   CHPR_ASYNC_LOGOUT = cpu logout structure offset input
1911	 *   %g4 = Trap information stored in the cpu logout flags field
1912	 *   %g5 = scr1
1913	 *   %g6 = scr2
1914	 *   %g3 = scr3
1915	 *   %g4 = scr4
1916	 */
1917	andcc	%g5, T_TL1, %g0
1918	clr	%g6
1919	movnz	%xcc, 1, %g6			! set %g6 if T_TL1 set
1920	sllx	%g6, CLO_FLAGS_TL_SHIFT, %g6
1921	sllx	%g5, CLO_FLAGS_TT_SHIFT, %g4
1922	set	CLO_FLAGS_TT_MASK, %g2
1923	and	%g4, %g2, %g4			! ttype
1924	or	%g6, %g4, %g4			! TT and TL
1925	and	%g3, EN_REG_CEEN, %g3		! CEEN value
1926	or	%g3, %g4, %g4			! TT and TL and CEEN
1927	set	CHPR_ASYNC_LOGOUT, %g6
1928	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)
1929
1930	/*
1931	 * If the logout struct was busy, we may need to pass the
1932	 * TT, TL, and CEEN information to the TL=0 handler via
1933	 * systrap parameter so save it off here.
1934	 */
1935	cmp	%g3, %g0
1936	be	1f
1937	  nop
1938	sllx	%g4, 32, %g4
1939	or	%g4, %g3, %g3
19401:
1941	/*
1942	 * Flush the Icache.  Since we turned off the Icache to capture the
1943	 * Icache line it is now stale or corrupted and we must flush it
1944	 * before re-enabling it.
1945	 */
1946	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, async_err_1);
1947	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
1948	ba,pt	%icc, 2f
1949	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
1950async_err_1:
1951	ASM_LD(%g5, icache_size)
1952	ASM_LD(%g6, icache_linesize)
19532:
1954	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)
1955
1956	/*
1957	 * XXX - Don't we need to flush the Dcache before turning it back
1958	 *       on to avoid stale or corrupt data? Was this broken?
1959	 */
1960	/*
1961	 * Flush the Dcache before turning it back on since it may now
1962	 * contain stale or corrupt data.
1963	 */
1964	ASM_LD(%g5, dcache_size)
1965	ASM_LD(%g6, dcache_linesize)
1966	CH_DCACHE_FLUSHALL(%g5, %g6, %g7)
1967
1968	/*
1969	 * check to see whether we parked our sibling core at the start
1970	 * of this handler. If so, we need to unpark it here.
1971	 * We use DCUCR reserved bits (stored in %g1) to keep track of
1972	 * whether or not we need to unpark. %g5 and %g7 are scratch registers.
1973	 */
1974	UNPARK_SIBLING_CORE(%g1, %g5, %g7)
1975
1976	/*
1977	 * Restore Icache and Dcache to previous state.
1978	 */
1979	stxa	%g1, [%g0]ASI_DCU
1980	flush	%g0	/* flush required after changing the IC bit */
1981
1982	/*
1983	 * Make sure our CPU logout operation was successful.
1984	 */
1985	cmp	%g3, %g0
1986	be	4f
1987	  nop
1988
1989	/*
1990	 * If the logout structure had been busy, how many times have
1991	 * we tried to use it and failed (nesting count)? If we have
1992	 * already recursed a substantial number of times, then we can
1993	 * assume things are not going to get better by themselves and
1994	 * so it would be best to panic.
1995	 */
1996	cmp	%g3, CLO_NESTING_MAX
1997	blt	3f
1998	  nop
1999
2000        call ptl1_panic
2001          mov   PTL1_BAD_ECC, %g1
2002
20033:
2004	/*
2005	 * Otherwise, if the logout structure was busy but we have not
2006	 * nested more times than our maximum value, then we simply
2007	 * issue a retry. Our TL=0 trap handler code will check and
2008	 * clear the AFSR after it is done logging what is currently
2009	 * in the logout struct and handle this event at that time.
2010	 */
2011	retry
20124:
2013	RESET_USER_RTT_REGS(%g4, %g5, 5f)
20145:
2015	set	cpu_deferred_error, %g1
2016	ba	sys_trap
2017	  mov	PIL_15, %g4		! run at pil 15
2018	SET_SIZE(async_err)
2019
2020#endif	/* lint */
2021
2022#if defined(CPU_IMP_L1_CACHE_PARITY)
2023
2024/*
2025 * D$ parity error trap (trap 71) at TL=0.
2026 * tt0_dperr is replaced by dcache_parity_instr in cpu_init_trap of
2027 * the various architecture-specific files.  This merely sets up the
2028 * arguments for cpu_parity_error and calls it via sys_trap.
2029 * NB: Must be 8 instructions or less to fit in trap table and code must
2030 *     be relocatable.
2031 */
2032#if defined(lint)
2033
2034void
2035dcache_parity_instr(void)
2036{}
2037
2038#else	/* lint */
2039	ENTRY_NP(dcache_parity_instr)
2040	membar	#Sync			! Cheetah+ requires membar #Sync
2041	set	cpu_parity_error, %g1
2042	or	%g0, CH_ERR_DPE, %g2
2043	rdpr	%tpc, %g3
2044	sethi	%hi(sys_trap), %g7
2045	jmp	%g7 + %lo(sys_trap)
2046	  mov	PIL_15, %g4		! run at pil 15
2047	SET_SIZE(dcache_parity_instr)
2048
2049#endif	/* lint */
2050
2051
2052/*
2053 * D$ parity error trap (trap 71) at TL>0.
2054 * tt1_dperr is replaced by dcache_parity_tl1_instr in cpu_init_trap of
2055 * the various architecture-specific files.  This generates a "Software
2056 * Trap 1" at TL>0, which goes to dcache_parity_tl1_cont_instr, and we
2057 * continue the handling there.
2058 * NB: Must be 8 instructions or less to fit in trap table and code must
2059 *     be relocatable.
2060 */
2061#if defined(lint)
2062
2063void
2064dcache_parity_tl1_instr(void)
2065{}
2066
2067#else	/* lint */
2068	ENTRY_NP(dcache_parity_tl1_instr)
2069	CH_ERR_TL1_TRAPENTRY(SWTRAP_1);
2070	SET_SIZE(dcache_parity_tl1_instr)
2071
2072#endif	/* lint */
2073
2074
2075/*
2076 * Software trap 1 at TL>0.
2077 * tt1_swtrap1 is replaced by dcache_parity_tl1_cont_instr in cpu_init_trap
2078 * of the various architecture-specific files.  This is used as a continuation
2079 * of the dcache parity handling where we've bought an extra TL level, so we
2080 * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
2081 * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
2082 * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
2083 * order two bits from %g1 and %g2 respectively).
2084 * NB: Must be 8 instructions or less to fit in trap table and code must
2085 *     be relocatable.
2086 */
2087#if defined(lint)
2088
2089void
2090dcache_parity_tl1_cont_instr(void)
2091{}
2092
2093#else	/* lint */
2094	ENTRY_NP(dcache_parity_tl1_cont_instr)
2095	CH_ERR_TL1_SWTRAPENTRY(dcache_parity_tl1_err);
2096	SET_SIZE(dcache_parity_tl1_cont_instr)
2097
2098#endif	/* lint */
2099
2100/*
2101 * D$ parity error at TL>0 handler
2102 * We get here via trap 71 at TL>0->Software trap 1 at TL>0.  We enter
2103 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
2104 */
2105#if defined(lint)
2106
2107void
2108dcache_parity_tl1_err(void)
2109{}
2110
2111#else	/* lint */
2112
2113	ENTRY_NP(dcache_parity_tl1_err)
2114
2115	/*
2116	 * This macro saves all the %g registers in the ch_err_tl1_data
2117	 * structure, updates the ch_err_tl1_flags and saves the %tpc in
2118	 * ch_err_tl1_tpc.  At the end of this macro, %g1 will point to
2119	 * the ch_err_tl1_data structure and %g2 will have the original
2120	 * flags in the ch_err_tl1_data structure.  All %g registers
2121	 * except for %g1 and %g2 will be available.
2122	 */
2123	CH_ERR_TL1_ENTER(CH_ERR_DPE);
2124
2125#ifdef TRAPTRACE
2126	/*
2127	 * Get current trap trace entry physical pointer.
2128	 */
2129	CPU_INDEX(%g6, %g5)
2130	sll	%g6, TRAPTR_SIZE_SHIFT, %g6
2131	set	trap_trace_ctl, %g5
2132	add	%g6, %g5, %g6
2133	ld	[%g6 + TRAPTR_LIMIT], %g5
2134	tst	%g5
2135	be	%icc, dpe_tl1_skip_tt
2136	  nop
2137	ldx	[%g6 + TRAPTR_PBASE], %g5
2138	ld	[%g6 + TRAPTR_OFFSET], %g4
2139	add	%g5, %g4, %g5
2140
2141	/*
2142	 * Create trap trace entry.
2143	 */
2144	rd	%asi, %g7
2145	wr	%g0, TRAPTR_ASI, %asi
2146	rd	STICK, %g4
2147	stxa	%g4, [%g5 + TRAP_ENT_TICK]%asi
2148	rdpr	%tl, %g4
2149	stha	%g4, [%g5 + TRAP_ENT_TL]%asi
2150	rdpr	%tt, %g4
2151	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
2152	rdpr	%tpc, %g4
2153	stna	%g4, [%g5 + TRAP_ENT_TPC]%asi
2154	rdpr	%tstate, %g4
2155	stxa	%g4, [%g5 + TRAP_ENT_TSTATE]%asi
2156	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
2157	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
2158	stna	%g0, [%g5 + TRAP_ENT_F1]%asi
2159	stna	%g0, [%g5 + TRAP_ENT_F2]%asi
2160	stna	%g0, [%g5 + TRAP_ENT_F3]%asi
2161	stna	%g0, [%g5 + TRAP_ENT_F4]%asi
2162	wr	%g0, %g7, %asi
2163
2164	/*
2165	 * Advance trap trace pointer.
2166	 */
2167	ld	[%g6 + TRAPTR_OFFSET], %g5
2168	ld	[%g6 + TRAPTR_LIMIT], %g4
2169	st	%g5, [%g6 + TRAPTR_LAST_OFFSET]
2170	add	%g5, TRAP_ENT_SIZE, %g5
2171	sub	%g4, TRAP_ENT_SIZE, %g4
2172	cmp	%g5, %g4
2173	movge	%icc, 0, %g5
2174	st	%g5, [%g6 + TRAPTR_OFFSET]
2175dpe_tl1_skip_tt:
2176#endif	/* TRAPTRACE */
2177
2178	/*
2179	 * I$ and D$ are automatically turned off by HW when the CPU hits
2180	 * a dcache or icache parity error so we will just leave those two
2181	 * off for now to avoid repeating this trap.
2182	 * For Panther, however, since we trap on P$ data parity errors
2183	 * and HW does not automatically disable P$, we need to disable it
2184	 * here so that we don't encounter any recursive traps when we
2185	 * issue the retry.
2186	 */
2187	ldxa	[%g0]ASI_DCU, %g3
2188	mov	1, %g4
2189	sllx	%g4, DCU_PE_SHIFT, %g4
2190	andn	%g3, %g4, %g3
2191	stxa	%g3, [%g0]ASI_DCU
2192	membar	#Sync
2193
2194	/*
2195	 * We fall into this macro if we've successfully logged the error in
2196	 * the ch_err_tl1_data structure and want the PIL15 softint to pick
2197	 * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
2198	 * Restores the %g registers and issues retry.
2199	 */
2200	CH_ERR_TL1_EXIT;
2201	SET_SIZE(dcache_parity_tl1_err)
2202
2203#endif	/* lint */
2204
2205/*
2206 * I$ parity error trap (trap 72) at TL=0.
2207 * tt0_iperr is replaced by icache_parity_instr in cpu_init_trap of
2208 * the various architecture-specific files.  This merely sets up the
2209 * arguments for cpu_parity_error and calls it via sys_trap.
2210 * NB: Must be 8 instructions or less to fit in trap table and code must
2211 *     be relocatable.
2212 */
2213#if defined(lint)
2214
2215void
2216icache_parity_instr(void)
2217{}
2218
2219#else	/* lint */
2220
2221	ENTRY_NP(icache_parity_instr)
2222	membar	#Sync			! Cheetah+ requires membar #Sync
2223	set	cpu_parity_error, %g1
2224	or	%g0, CH_ERR_IPE, %g2
2225	rdpr	%tpc, %g3
2226	sethi	%hi(sys_trap), %g7
2227	jmp	%g7 + %lo(sys_trap)
2228	  mov	PIL_15, %g4		! run at pil 15
2229	SET_SIZE(icache_parity_instr)
2230
2231#endif	/* lint */
2232
2233/*
2234 * I$ parity error trap (trap 72) at TL>0.
2235 * tt1_iperr is replaced by icache_parity_tl1_instr in cpu_init_trap of
2236 * the various architecture-specific files.  This generates a "Software
2237 * Trap 2" at TL>0, which goes to icache_parity_tl1_cont_instr, and we
2238 * continue the handling there.
2239 * NB: Must be 8 instructions or less to fit in trap table and code must
2240 *     be relocatable.
2241 */
2242#if defined(lint)
2243
2244void
2245icache_parity_tl1_instr(void)
2246{}
2247
2248#else	/* lint */
2249	ENTRY_NP(icache_parity_tl1_instr)
2250	CH_ERR_TL1_TRAPENTRY(SWTRAP_2);
2251	SET_SIZE(icache_parity_tl1_instr)
2252
2253#endif	/* lint */
2254
2255/*
2256 * Software trap 2 at TL>0.
2257 * tt1_swtrap2 is replaced by icache_parity_tl1_cont_instr in cpu_init_trap
2258 * of the various architecture-specific files.  This is used as a continuation
2259 * of the icache parity handling where we've bought an extra TL level, so we
2260 * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
2261 * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
2262 * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
2263 * order two bits from %g1 and %g2 respectively).
2264 * NB: Must be 8 instructions or less to fit in trap table and code must
2265 *     be relocatable.
2266 */
2267#if defined(lint)
2268
2269void
2270icache_parity_tl1_cont_instr(void)
2271{}
2272
2273#else	/* lint */
2274	ENTRY_NP(icache_parity_tl1_cont_instr)
2275	CH_ERR_TL1_SWTRAPENTRY(icache_parity_tl1_err);
2276	SET_SIZE(icache_parity_tl1_cont_instr)
2277
2278#endif	/* lint */
2279
2280
2281/*
2282 * I$ parity error at TL>0 handler
2283 * We get here via trap 72 at TL>0->Software trap 2 at TL>0.  We enter
2284 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
2285 */
2286#if defined(lint)
2287
2288void
2289icache_parity_tl1_err(void)
2290{}
2291
2292#else	/* lint */
2293
2294	ENTRY_NP(icache_parity_tl1_err)
2295
2296	/*
2297	 * This macro saves all the %g registers in the ch_err_tl1_data
2298	 * structure, updates the ch_err_tl1_flags and saves the %tpc in
2299	 * ch_err_tl1_tpc.  At the end of this macro, %g1 will point to
2300	 * the ch_err_tl1_data structure and %g2 will have the original
2301	 * flags in the ch_err_tl1_data structure.  All %g registers
2302	 * except for %g1 and %g2 will be available.
2303	 */
2304	CH_ERR_TL1_ENTER(CH_ERR_IPE);
2305
2306#ifdef TRAPTRACE
2307	/*
2308	 * Get current trap trace entry physical pointer.
2309	 */
2310	CPU_INDEX(%g6, %g5)
2311	sll	%g6, TRAPTR_SIZE_SHIFT, %g6
2312	set	trap_trace_ctl, %g5
2313	add	%g6, %g5, %g6
2314	ld	[%g6 + TRAPTR_LIMIT], %g5
2315	tst	%g5
2316	be	%icc, ipe_tl1_skip_tt
2317	  nop
2318	ldx	[%g6 + TRAPTR_PBASE], %g5
2319	ld	[%g6 + TRAPTR_OFFSET], %g4
2320	add	%g5, %g4, %g5
2321
2322	/*
2323	 * Create trap trace entry.
2324	 */
2325	rd	%asi, %g7
2326	wr	%g0, TRAPTR_ASI, %asi
2327	rd	STICK, %g4
2328	stxa	%g4, [%g5 + TRAP_ENT_TICK]%asi
2329	rdpr	%tl, %g4
2330	stha	%g4, [%g5 + TRAP_ENT_TL]%asi
2331	rdpr	%tt, %g4
2332	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
2333	rdpr	%tpc, %g4
2334	stna	%g4, [%g5 + TRAP_ENT_TPC]%asi
2335	rdpr	%tstate, %g4
2336	stxa	%g4, [%g5 + TRAP_ENT_TSTATE]%asi
2337	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
2338	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
2339	stna	%g0, [%g5 + TRAP_ENT_F1]%asi
2340	stna	%g0, [%g5 + TRAP_ENT_F2]%asi
2341	stna	%g0, [%g5 + TRAP_ENT_F3]%asi
2342	stna	%g0, [%g5 + TRAP_ENT_F4]%asi
2343	wr	%g0, %g7, %asi
2344
2345	/*
2346	 * Advance trap trace pointer.
2347	 */
2348	ld	[%g6 + TRAPTR_OFFSET], %g5
2349	ld	[%g6 + TRAPTR_LIMIT], %g4
2350	st	%g5, [%g6 + TRAPTR_LAST_OFFSET]
2351	add	%g5, TRAP_ENT_SIZE, %g5
2352	sub	%g4, TRAP_ENT_SIZE, %g4
2353	cmp	%g5, %g4
2354	movge	%icc, 0, %g5
2355	st	%g5, [%g6 + TRAPTR_OFFSET]
2356ipe_tl1_skip_tt:
2357#endif	/* TRAPTRACE */
2358
2359	/*
2360	 * We fall into this macro if we've successfully logged the error in
2361	 * the ch_err_tl1_data structure and want the PIL15 softint to pick
2362	 * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
2363	 * Restores the %g registers and issues retry.
2364	 */
2365	CH_ERR_TL1_EXIT;
2366
2367	SET_SIZE(icache_parity_tl1_err)
2368
2369#endif	/* lint */
2370
2371#endif	/* CPU_IMP_L1_CACHE_PARITY */
2372
2373
2374/*
2375 * The itlb_rd_entry and dtlb_rd_entry functions return the tag portion of the
2376 * tte, the virtual address, and the ctxnum of the specified tlb entry.  They
2377 * should only be used in places where you have no choice but to look at the
2378 * tlb itself.
2379 *
2380 * Note: These two routines are required by the Estar "cpr" loadable module.
2381 */
2382
2383#if defined(lint)
2384
2385/* ARGSUSED */
2386void
2387itlb_rd_entry(uint_t entry, tte_t *tte, uint64_t *va_tag)
2388{}
2389
2390#else	/* lint */
2391
2392	ENTRY_NP(itlb_rd_entry)
2393	sllx	%o0, 3, %o0
2394	ldxa	[%o0]ASI_ITLB_ACCESS, %g1
2395	stx	%g1, [%o1]
2396	ldxa	[%o0]ASI_ITLB_TAGREAD, %g2
2397	set	TAGREAD_CTX_MASK, %o4
2398	andn	%g2, %o4, %o5
2399	retl
2400	  stx	%o5, [%o2]
2401	SET_SIZE(itlb_rd_entry)
2402
2403#endif	/* lint */
2404
2405
2406#if defined(lint)
2407
2408/* ARGSUSED */
2409void
2410dtlb_rd_entry(uint_t entry, tte_t *tte, uint64_t *va_tag)
2411{}
2412
2413#else	/* lint */
2414
2415	ENTRY_NP(dtlb_rd_entry)
2416	sllx	%o0, 3, %o0
2417	ldxa	[%o0]ASI_DTLB_ACCESS, %g1
2418	stx	%g1, [%o1]
2419	ldxa	[%o0]ASI_DTLB_TAGREAD, %g2
2420	set	TAGREAD_CTX_MASK, %o4
2421	andn	%g2, %o4, %o5
2422	retl
2423	  stx	%o5, [%o2]
2424	SET_SIZE(dtlb_rd_entry)
2425#endif	/* lint */
2426
2427
2428#if !(defined(JALAPENO) || defined(SERRANO))
2429
2430#if defined(lint)
2431
2432uint64_t
2433get_safari_config(void)
2434{ return (0); }
2435
2436#else	/* lint */
2437
2438	ENTRY(get_safari_config)
2439	ldxa	[%g0]ASI_SAFARI_CONFIG, %o0
2440	retl
2441	nop
2442	SET_SIZE(get_safari_config)
2443
2444#endif	/* lint */
2445
2446
2447#if defined(lint)
2448
2449/* ARGSUSED */
2450void
2451set_safari_config(uint64_t safari_config)
2452{}
2453
2454#else	/* lint */
2455
2456	ENTRY(set_safari_config)
2457	stxa	%o0, [%g0]ASI_SAFARI_CONFIG
2458	membar	#Sync
2459	retl
2460	nop
2461	SET_SIZE(set_safari_config)
2462
2463#endif	/* lint */
2464
2465#endif	/* !(JALAPENO || SERRANO) */
2466
2467
2468#if defined(lint)
2469
2470void
2471cpu_cleartickpnt(void)
2472{}
2473
2474#else	/* lint */
2475	/*
2476	 * Clear the NPT (non-privileged trap) bit in the %tick/%stick
2477	 * registers. In an effort to make the change in the
2478	 * tick/stick counter as consistent as possible, we disable
2479	 * all interrupts while we're changing the registers. We also
2480	 * ensure that the read and write instructions are in the same
2481	 * line in the instruction cache.
2482	 */
2483	ENTRY_NP(cpu_clearticknpt)
2484	rdpr	%pstate, %g1		/* save processor state */
2485	andn	%g1, PSTATE_IE, %g3	/* turn off */
2486	wrpr	%g0, %g3, %pstate	/*   interrupts */
2487	rdpr	%tick, %g2		/* get tick register */
2488	brgez,pn %g2, 1f		/* if NPT bit off, we're done */
2489	mov	1, %g3			/* create mask */
2490	sllx	%g3, 63, %g3		/*   for NPT bit */
2491	ba,a,pt	%xcc, 2f
2492	.align	8			/* Ensure rd/wr in same i$ line */
24932:
2494	rdpr	%tick, %g2		/* get tick register */
2495	wrpr	%g3, %g2, %tick		/* write tick register, */
2496					/*   clearing NPT bit   */
24971:
2498	rd	STICK, %g2		/* get stick register */
2499	brgez,pn %g2, 3f		/* if NPT bit off, we're done */
2500	mov	1, %g3			/* create mask */
2501	sllx	%g3, 63, %g3		/*   for NPT bit */
2502	ba,a,pt	%xcc, 4f
2503	.align	8			/* Ensure rd/wr in same i$ line */
25044:
2505	rd	STICK, %g2		/* get stick register */
2506	wr	%g3, %g2, STICK		/* write stick register, */
2507					/*   clearing NPT bit   */
25083:
2509	jmp	%g4 + 4
2510	wrpr	%g0, %g1, %pstate	/* restore processor state */
2511
2512	SET_SIZE(cpu_clearticknpt)
2513
2514#endif	/* lint */
2515
2516
2517#if defined(CPU_IMP_L1_CACHE_PARITY)
2518
2519#if defined(lint)
2520/*
2521 * correct_dcache_parity(size_t size, size_t linesize)
2522 *
2523 * Correct D$ data parity by zeroing the data and initializing microtag
2524 * for all indexes and all ways of the D$.
2525 *
2526 */
2527/* ARGSUSED */
2528void
2529correct_dcache_parity(size_t size, size_t linesize)
2530{}
2531
2532#else	/* lint */
2533
2534	ENTRY(correct_dcache_parity)
2535	/*
2536	 * Register Usage:
2537	 *
2538	 * %o0 = input D$ size
2539	 * %o1 = input D$ line size
2540	 * %o2 = scratch
2541	 * %o3 = scratch
2542	 * %o4 = scratch
2543	 */
2544
2545	sub	%o0, %o1, %o0			! init cache line address
2546
2547	/*
2548	 * For Panther CPUs, we also need to clear the data parity bits
2549	 * using DC_data_parity bit of the ASI_DCACHE_DATA register.
2550	 */
2551	GET_CPU_IMPL(%o3)
2552	cmp	%o3, PANTHER_IMPL
2553	bne	1f
2554	  clr	%o3				! zero for non-Panther
2555	mov	1, %o3
2556	sll	%o3, PN_DC_DATA_PARITY_BIT_SHIFT, %o3
2557
25581:
2559	/*
2560	 * Set utag = way since it must be unique within an index.
2561	 */
2562	srl	%o0, 14, %o2			! get cache way (DC_way)
2563	membar	#Sync				! required before ASI_DC_UTAG
2564	stxa	%o2, [%o0]ASI_DC_UTAG		! set D$ utag = cache way
2565	membar	#Sync				! required after ASI_DC_UTAG
2566
2567	/*
2568	 * Zero line of D$ data (and data parity bits for Panther)
2569	 */
2570	sub	%o1, 8, %o2
2571	or	%o0, %o3, %o4			! same address + DC_data_parity
25722:
2573	membar	#Sync				! required before ASI_DC_DATA
2574	stxa	%g0, [%o0 + %o2]ASI_DC_DATA	! zero 8 bytes of D$ data
2575	membar	#Sync				! required after ASI_DC_DATA
2576	/*
2577	 * We also clear the parity bits if this is a panther. For non-Panther
2578	 * CPUs, we simply end up clearing the $data register twice.
2579	 */
2580	stxa	%g0, [%o4 + %o2]ASI_DC_DATA
2581	membar	#Sync
2582
2583	subcc	%o2, 8, %o2
2584	bge	2b
2585	nop
2586
2587	subcc	%o0, %o1, %o0
2588	bge	1b
2589	nop
2590
2591	retl
2592	  nop
2593	SET_SIZE(correct_dcache_parity)
2594
2595#endif	/* lint */
2596
2597#endif	/* CPU_IMP_L1_CACHE_PARITY */
2598
2599
2600#if defined(lint)
2601/*
2602 *  Get timestamp (stick).
2603 */
2604/* ARGSUSED */
2605void
2606stick_timestamp(int64_t *ts)
2607{
2608}
2609
2610#else	/* lint */
2611
2612	ENTRY_NP(stick_timestamp)
2613	rd	STICK, %g1	! read stick reg
2614	sllx	%g1, 1, %g1
2615	srlx	%g1, 1, %g1	! clear npt bit
2616
2617	retl
2618	stx     %g1, [%o0]	! store the timestamp
2619	SET_SIZE(stick_timestamp)
2620
2621#endif	/* lint */
2622
2623
2624#if defined(lint)
2625/*
2626 * Set STICK adjusted by skew.
2627 */
2628/* ARGSUSED */
2629void
2630stick_adj(int64_t skew)
2631{
2632}
2633
2634#else	/* lint */
2635
2636	ENTRY_NP(stick_adj)
2637	rdpr	%pstate, %g1		! save processor state
2638	andn	%g1, PSTATE_IE, %g3
2639	ba	1f			! cache align stick adj
2640	wrpr	%g0, %g3, %pstate	! turn off interrupts
2641
2642	.align	16
26431:	nop
2644
2645	rd	STICK, %g4		! read stick reg
2646	add	%g4, %o0, %o1		! adjust stick with skew
2647	wr	%o1, %g0, STICK		! write stick reg
2648
2649	retl
2650	wrpr	%g1, %pstate		! restore processor state
2651	SET_SIZE(stick_adj)
2652
2653#endif	/* lint */
2654
2655#if defined(lint)
2656/*
2657 * Debugger-specific stick retrieval
2658 */
2659/*ARGSUSED*/
2660int
2661kdi_get_stick(uint64_t *stickp)
2662{
2663	return (0);
2664}
2665
2666#else	/* lint */
2667
2668	ENTRY_NP(kdi_get_stick)
2669	rd	STICK, %g1
2670	stx	%g1, [%o0]
2671	retl
2672	mov	%g0, %o0
2673	SET_SIZE(kdi_get_stick)
2674
2675#endif	/* lint */
2676
2677#if defined(lint)
2678/*
2679 * Invalidate the specified line from the D$.
2680 *
2681 * Register usage:
2682 *	%o0 - index for the invalidation, specifies DC_way and DC_addr
2683 *
2684 * ASI_DC_TAG, 0x47, is used in the following manner. A 64-bit value is
2685 * stored to a particular DC_way and DC_addr in ASI_DC_TAG.
2686 *
2687 * The format of the stored 64-bit value is:
2688 *
2689 *	+----------+--------+----------+
2690 *	| Reserved | DC_tag | DC_valid |
2691 *	+----------+--------+----------+
2692 *       63      31 30     1	      0
2693 *
2694 * DC_tag is the 30-bit physical tag of the associated line.
2695 * DC_valid is the 1-bit valid field for both the physical and snoop tags.
2696 *
2697 * The format of the 64-bit DC_way and DC_addr into ASI_DC_TAG is:
2698 *
2699 *	+----------+--------+----------+----------+
2700 *	| Reserved | DC_way | DC_addr  | Reserved |
2701 *	+----------+--------+----------+----------+
2702 *       63      16 15    14 13       5 4        0
2703 *
2704 * DC_way is a 2-bit index that selects one of the 4 ways.
2705 * DC_addr is a 9-bit index that selects one of 512 tag/valid fields.
2706 *
2707 * Setting the DC_valid bit to zero for the specified DC_way and
2708 * DC_addr index into the D$ results in an invalidation of a D$ line.
2709 */
2710/*ARGSUSED*/
2711void
2712dcache_inval_line(int index)
2713{
2714}
2715#else	/* lint */
2716	ENTRY(dcache_inval_line)
2717	sll	%o0, 5, %o0		! shift index into DC_way and DC_addr
2718	stxa	%g0, [%o0]ASI_DC_TAG	! zero the DC_valid and DC_tag bits
2719	membar	#Sync
2720	retl
2721	nop
2722	SET_SIZE(dcache_inval_line)
2723#endif	/* lint */
2724
2725#if defined(lint)
2726/*
2727 * Invalidate the entire I$
2728 *
2729 * Register usage:
2730 *	%o0 - specifies IC_way, IC_addr, IC_tag
2731 *	%o1 - scratch
2732 *	%o2 - used to save and restore DCU value
2733 *	%o3 - scratch
2734 *	%o5 - used to save and restore PSTATE
2735 *
2736 * Due to the behavior of the I$ control logic when accessing ASI_IC_TAG,
2737 * the I$ should be turned off. Accesses to ASI_IC_TAG may collide and
2738 * block out snoops and invalidates to the I$, causing I$ consistency
2739 * to be broken. Before turning on the I$, all I$ lines must be invalidated.
2740 *
2741 * ASI_IC_TAG, 0x67, is used in the following manner. A 64-bit value is
2742 * stored to a particular IC_way, IC_addr, IC_tag in ASI_IC_TAG. The
2743 * info below describes store (write) use of ASI_IC_TAG. Note that read
2744 * use of ASI_IC_TAG behaves differently.
2745 *
2746 * The format of the stored 64-bit value is:
2747 *
2748 *	+----------+--------+---------------+-----------+
2749 *	| Reserved | Valid  | IC_vpred<7:0> | Undefined |
2750 *	+----------+--------+---------------+-----------+
2751 *       63      55    54    53           46 45        0
2752 *
2753 * Valid is the 1-bit valid field for both the physical and snoop tags.
2754 * IC_vpred is the 8-bit LPB bits for 8 instructions starting at
2755 *	the 32-byte boundary aligned address specified by IC_addr.
2756 *
2757 * The format of the 64-bit IC_way, IC_addr, IC_tag into ASI_IC_TAG is:
2758 *
2759 *	+----------+--------+---------+--------+---------+
2760 *	| Reserved | IC_way | IC_addr | IC_tag |Reserved |
2761 *	+----------+--------+---------+--------+---------+
2762 *       63      16 15    14 13      5 4      3 2       0
2763 *
2764 * IC_way is a 2-bit index that selects one of the 4 ways.
2765 * IC_addr[13:6] is an 8-bit index that selects one of 256 valid fields.
2766 * IC_addr[5] is a "don't care" for a store.
2767 * IC_tag set to 2 specifies that the stored value is to be interpreted
2768 *	as containing Valid and IC_vpred as described above.
2769 *
2770 * Setting the Valid bit to zero for the specified IC_way and
2771 * IC_addr index into the I$ results in an invalidation of an I$ line.
2772 */
2773/*ARGSUSED*/
2774void
2775icache_inval_all(void)
2776{
2777}
2778#else	/* lint */
2779	ENTRY(icache_inval_all)
2780	rdpr	%pstate, %o5
2781	andn	%o5, PSTATE_IE, %o3
2782	wrpr	%g0, %o3, %pstate	! clear IE bit
2783
2784	GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, icache_inval_all_1);
2785	ld	[%o0 + CHPR_ICACHE_LINESIZE], %o1
2786	ba,pt	%icc, 2f
2787	  ld	[%o0 + CHPR_ICACHE_SIZE], %o0
2788icache_inval_all_1:
2789	ASM_LD(%o0, icache_size)
2790	ASM_LD(%o1, icache_linesize)
27912:
2792	CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4)
2793
2794	retl
2795	wrpr	%g0, %o5, %pstate	! restore earlier pstate
2796	SET_SIZE(icache_inval_all)
2797#endif	/* lint */
2798
2799
2800#if defined(lint)
2801/* ARGSUSED */
2802void
2803cache_scrubreq_tl1(uint64_t inum, uint64_t index)
2804{
2805}
2806
2807#else	/* lint */
2808/*
2809 * cache_scrubreq_tl1 is the crosstrap handler called on offlined cpus via a
2810 * crosstrap.  It atomically increments the outstanding request counter and,
2811 * if there was not already an outstanding request, branches to setsoftint_tl1
2812 * to enqueue an intr_req for the given inum.
2813 */
2814
2815	! Register usage:
2816	!
2817	! Arguments:
2818	! %g1 - inum
2819	! %g2 - index into chsm_outstanding array
2820	!
2821	! Internal:
2822	! %g2, %g3, %g5 - scratch
2823	! %g4 - ptr. to scrub_misc chsm_outstanding[index].
2824	! %g6 - setsoftint_tl1 address
2825
2826	ENTRY_NP(cache_scrubreq_tl1)
2827	mulx	%g2, CHSM_OUTSTANDING_INCR, %g2
2828	set	CHPR_SCRUB_MISC + CHSM_OUTSTANDING, %g3
2829	add	%g2, %g3, %g2
2830	GET_CPU_PRIVATE_PTR(%g2, %g4, %g5, 1f);
2831	ld	[%g4], %g2		! cpu's chsm_outstanding[index]
2832	!
2833	! no need to use atomic instructions for the following
2834	! increment - we're at tl1
2835	!
2836	add	%g2, 0x1, %g3
2837	brnz,pn	%g2, 1f			! no need to enqueue more intr_req
2838	  st	%g3, [%g4]		! delay - store incremented counter
2839	ASM_JMP(%g6, setsoftint_tl1)
2840	! not reached
28411:
2842	retry
2843	SET_SIZE(cache_scrubreq_tl1)
2844
2845#endif	/* lint */
2846
2847
2848#if defined(lint)
2849
2850/* ARGSUSED */
2851void
2852get_cpu_error_state(ch_cpu_errors_t *cpu_error_regs)
2853{}
2854
2855#else	/* lint */
2856
2857/*
2858 * Get the error state for the processor.
2859 * Note that this must not be used at TL>0
2860 */
2861	ENTRY(get_cpu_error_state)
2862#if defined(CHEETAH_PLUS)
2863	set	ASI_SHADOW_REG_VA, %o2
2864	ldxa	[%o2]ASI_AFSR, %o1		! shadow afsr reg
2865	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR]
2866	ldxa	[%o2]ASI_AFAR, %o1		! shadow afar reg
2867	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR]
2868	GET_CPU_IMPL(%o3)	! Only panther has AFSR_EXT registers
2869	cmp	%o3, PANTHER_IMPL
2870	bne,a	1f
2871	  stx	%g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT]	! zero for non-PN
2872	set	ASI_AFSR_EXT_VA, %o2
2873	ldxa	[%o2]ASI_AFSR, %o1		! afsr_ext reg
2874	stx	%o1, [%o0 + CH_CPU_ERRORS_AFSR_EXT]
2875	set	ASI_SHADOW_AFSR_EXT_VA, %o2
2876	ldxa	[%o2]ASI_AFSR, %o1		! shadow afsr_ext reg
2877	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT]
2878	b	2f
2879	  nop
28801:
2881	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT] ! zero for non-PN
28822:
2883#else	/* CHEETAH_PLUS */
2884	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR]
2885	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR]
2886	stx	%g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT]
2887	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT]
2888#endif	/* CHEETAH_PLUS */
2889#if defined(SERRANO)
2890	/*
2891	 * Serrano has an afar2 which captures the address on FRC/FRU errors.
2892	 * We save this in the afar2 of the register save area.
2893	 */
2894	set	ASI_MCU_AFAR2_VA, %o2
2895	ldxa	[%o2]ASI_MCU_CTRL, %o1
2896	stx	%o1, [%o0 + CH_CPU_ERRORS_AFAR2]
2897#endif	/* SERRANO */
2898	ldxa	[%g0]ASI_AFSR, %o1		! primary afsr reg
2899	stx	%o1, [%o0 + CH_CPU_ERRORS_AFSR]
2900	ldxa	[%g0]ASI_AFAR, %o1		! primary afar reg
2901	retl
2902	stx	%o1, [%o0 + CH_CPU_ERRORS_AFAR]
2903	SET_SIZE(get_cpu_error_state)
2904#endif	/* lint */
2905
2906#if defined(lint)
2907
2908/*
2909 * Check a page of memory for errors.
2910 *
2911 * Load each 64 byte block from physical memory.
2912 * Check AFSR after each load to see if an error
2913 * was caused. If so, log/scrub that error.
2914 *
2915 * Used to determine if a page contains
2916 * CEs when CEEN is disabled.
2917 */
2918/*ARGSUSED*/
2919void
2920cpu_check_block(caddr_t va, uint_t psz)
2921{}
2922
2923#else	/* lint */
2924
2925	ENTRY(cpu_check_block)
2926	!
2927	! get a new window with room for the error regs
2928	!
2929	save	%sp, -SA(MINFRAME + CH_CPU_ERROR_SIZE), %sp
2930	srl	%i1, 6, %l4		! clear top bits of psz
2931					! and divide by 64
2932	rd	%fprs, %l2		! store FP
2933	wr	%g0, FPRS_FEF, %fprs	! enable FP
29341:
2935	ldda	[%i0]ASI_BLK_P, %d0	! load a block
2936	membar	#Sync
2937	ldxa    [%g0]ASI_AFSR, %l3	! read afsr reg
2938	brz,a,pt %l3, 2f		! check for error
2939	nop
2940
2941	!
2942	! if error, read the error regs and log it
2943	!
2944	call	get_cpu_error_state
2945	add	%fp, STACK_BIAS - CH_CPU_ERROR_SIZE, %o0
2946
2947	!
2948	! cpu_ce_detected(ch_cpu_errors_t *, flag)
2949	!
2950	call	cpu_ce_detected		! log the error
2951	mov	CE_CEEN_TIMEOUT, %o1
29522:
2953	dec	%l4			! next 64-byte block
2954	brnz,a,pt  %l4, 1b
2955	add	%i0, 64, %i0		! increment block addr
2956
2957	wr	%l2, %g0, %fprs		! restore FP
2958	ret
2959	restore
2960
2961	SET_SIZE(cpu_check_block)
2962
2963#endif	/* lint */
2964
2965#if defined(lint)
2966
2967/*
2968 * Perform a cpu logout called from C.  This is used where we did not trap
2969 * for the error but still want to gather "what we can".  Caller must make
2970 * sure cpu private area exists and that the indicated logout area is free
2971 * for use, and that we are unable to migrate cpus.
2972 */
2973/*ARGSUSED*/
2974void
2975cpu_delayed_logout(uint64_t afar, ch_cpu_logout_t *clop)
2976{ }
2977
2978#else
2979	ENTRY(cpu_delayed_logout)
2980	rdpr	%pstate, %o2
2981	andn	%o2, PSTATE_IE, %o2
2982	wrpr	%g0, %o2, %pstate		! disable interrupts
2983	PARK_SIBLING_CORE(%o2, %o3, %o4)	! %o2 has DCU value
2984	add	%o1, CH_CLO_DATA + CH_CHD_EC_DATA, %o1
2985	rd	%asi, %g1
2986	wr	%g0, ASI_P, %asi
2987	GET_ECACHE_DTAGS(%o0, %o1, %o3, %o4, %o5)
2988	wr	%g1, %asi
2989	UNPARK_SIBLING_CORE(%o2, %o3, %o4)	! can use %o2 again
2990	rdpr	%pstate, %o2
2991	or	%o2, PSTATE_IE, %o2
2992	wrpr	%g0, %o2, %pstate
2993	retl
2994	  nop
2995	SET_SIZE(cpu_delayed_logout)
2996
2997#endif	/* lint */
2998
2999#if defined(lint)
3000
3001/*ARGSUSED*/
3002int
3003dtrace_blksuword32(uintptr_t addr, uint32_t *data, int tryagain)
3004{ return (0); }
3005
3006#else
3007
3008	ENTRY(dtrace_blksuword32)
3009	save	%sp, -SA(MINFRAME + 4), %sp
3010
3011	rdpr	%pstate, %l1
3012	andn	%l1, PSTATE_IE, %l2		! disable interrupts to
3013	wrpr	%g0, %l2, %pstate		! protect our FPU diddling
3014
3015	rd	%fprs, %l0
3016	andcc	%l0, FPRS_FEF, %g0
3017	bz,a,pt	%xcc, 1f			! if the fpu is disabled
3018	wr	%g0, FPRS_FEF, %fprs		! ... enable the fpu
3019
3020	st	%f0, [%fp + STACK_BIAS - 4]	! save %f0 to the stack
30211:
3022	set	0f, %l5
3023        /*
3024         * We're about to write a block full or either total garbage
3025         * (not kernel data, don't worry) or user floating-point data
3026         * (so it only _looks_ like garbage).
3027         */
3028	ld	[%i1], %f0			! modify the block
3029	membar	#Sync
3030	stn	%l5, [THREAD_REG + T_LOFAULT]	! set up the lofault handler
3031	stda	%d0, [%i0]ASI_BLK_COMMIT_S	! store the modified block
3032	membar	#Sync
3033	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler
3034
3035	bz,a,pt	%xcc, 1f
3036	wr	%g0, %l0, %fprs			! restore %fprs
3037
3038	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
30391:
3040
3041	wrpr	%g0, %l1, %pstate		! restore interrupts
3042
3043	ret
3044	restore	%g0, %g0, %o0
3045
30460:
3047	membar	#Sync
3048	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler
3049
3050	bz,a,pt	%xcc, 1f
3051	wr	%g0, %l0, %fprs			! restore %fprs
3052
3053	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
30541:
3055
3056	wrpr	%g0, %l1, %pstate		! restore interrupts
3057
3058	/*
3059	 * If tryagain is set (%i2) we tail-call dtrace_blksuword32_err()
3060	 * which deals with watchpoints. Otherwise, just return -1.
3061	 */
3062	brnz,pt	%i2, 1f
3063	nop
3064	ret
3065	restore	%g0, -1, %o0
30661:
3067	call	dtrace_blksuword32_err
3068	restore
3069
3070	SET_SIZE(dtrace_blksuword32)
3071
3072#endif /* lint */
3073
3074#ifdef	CHEETAHPLUS_ERRATUM_25
3075
3076#if	defined(lint)
3077/*
3078 * Claim a chunk of physical address space.
3079 */
3080/*ARGSUSED*/
3081void
3082claimlines(uint64_t pa, size_t sz, int stride)
3083{}
3084#else	/* lint */
3085	ENTRY(claimlines)
30861:
3087	subcc	%o1, %o2, %o1
3088	add	%o0, %o1, %o3
3089	bgeu,a,pt	%xcc, 1b
3090	casxa	[%o3]ASI_MEM, %g0, %g0
3091	membar  #Sync
3092	retl
3093	nop
3094	SET_SIZE(claimlines)
3095#endif	/* lint */
3096
3097#if	defined(lint)
3098/*
3099 * CPU feature initialization,
3100 * turn BPE off,
3101 * get device id.
3102 */
3103/*ARGSUSED*/
3104void
3105cpu_feature_init(void)
3106{}
3107#else	/* lint */
3108	ENTRY(cpu_feature_init)
3109	save	%sp, -SA(MINFRAME), %sp
3110	sethi	%hi(cheetah_bpe_off), %o0
3111	ld	[%o0 + %lo(cheetah_bpe_off)], %o0
3112	brz	%o0, 1f
3113	nop
3114	rd	ASR_DISPATCH_CONTROL, %o0
3115	andn	%o0, ASR_DISPATCH_CONTROL_BPE, %o0
3116	wr	%o0, 0, ASR_DISPATCH_CONTROL
31171:
3118	!
3119	! get the device_id and store the device_id
3120	! in the appropriate cpunodes structure
3121	! given the cpus index
3122	!
3123	CPU_INDEX(%o0, %o1)
3124	mulx %o0, CPU_NODE_SIZE, %o0
3125	set  cpunodes + DEVICE_ID, %o1
3126	ldxa [%g0] ASI_DEVICE_SERIAL_ID, %o2
3127	stx  %o2, [%o0 + %o1]
3128#ifdef	CHEETAHPLUS_ERRATUM_34
3129	!
3130	! apply Cheetah+ erratum 34 workaround
3131	!
3132	call itlb_erratum34_fixup
3133	  nop
3134#endif	/* CHEETAHPLUS_ERRATUM_34 */
3135	ret
3136	  restore
3137	SET_SIZE(cpu_feature_init)
3138#endif	/* lint */
3139
3140#if	defined(lint)
3141/*
3142 * Copy a tsb entry atomically, from src to dest.
3143 * src must be 128 bit aligned.
3144 */
3145/*ARGSUSED*/
3146void
3147copy_tsb_entry(uintptr_t src, uintptr_t dest)
3148{}
3149#else	/* lint */
3150	ENTRY(copy_tsb_entry)
3151	ldda	[%o0]ASI_NQUAD_LD, %o2		! %o2 = tag, %o3 = data
3152	stx	%o2, [%o1]
3153	stx	%o3, [%o1 + 8 ]
3154	retl
3155	nop
3156	SET_SIZE(copy_tsb_entry)
3157#endif	/* lint */
3158
3159#endif	/* CHEETAHPLUS_ERRATUM_25 */
3160
3161#ifdef	CHEETAHPLUS_ERRATUM_34
3162
3163#if	defined(lint)
3164
3165/*ARGSUSED*/
3166void
3167itlb_erratum34_fixup(void)
3168{}
3169
3170#else	/* lint */
3171
3172	!
3173	! In Cheetah+ erratum 34, under certain conditions an ITLB locked
3174	! index 0 TTE will erroneously be displaced when a new TTE is
3175	! loaded via ASI_ITLB_IN.  In order to avoid cheetah+ erratum 34,
3176	! locked index 0 TTEs must be relocated.
3177	!
3178	! NOTE: Care must be taken to avoid an ITLB miss in this routine.
3179	!
3180	ENTRY_NP(itlb_erratum34_fixup)
3181	rdpr	%pstate, %o3
3182#ifdef DEBUG
3183	andcc	%o3, PSTATE_IE, %g0		! If interrupts already
3184	bnz,pt %icc, 0f				!   disabled, panic
3185	  nop
3186	sethi	%hi(sfmmu_panic1), %o0
3187	call	panic
3188	 or	%o0, %lo(sfmmu_panic1), %o0
31890:
3190#endif /* DEBUG */
3191	wrpr	%o3, PSTATE_IE, %pstate		! Disable interrupts
3192	ldxa	[%g0]ASI_ITLB_ACCESS, %o1	! %o1 = entry 0 data
3193	ldxa	[%g0]ASI_ITLB_TAGREAD, %o2	! %o2 = entry 0 tag
3194
3195	cmp	%o1, %g0			! Is this entry valid?
3196	bge	%xcc, 1f
3197	  andcc	%o1, TTE_LCK_INT, %g0		! Is this entry locked?
3198	bnz	%icc, 2f
3199	  nop
32001:
3201	retl					! Nope, outta here...
3202	  wrpr	%g0, %o3, %pstate		! Enable interrupts
32032:
3204	sethi	%hi(FLUSH_ADDR), %o4
3205	stxa	%g0, [%o2]ASI_ITLB_DEMAP	! Flush this mapping
3206	flush	%o4				! Flush required for I-MMU
3207	!
3208	! Start search from index 1 up.  This is because the kernel force
3209	! loads its text page at index 15 in sfmmu_kernel_remap() and we
3210	! don't want our relocated entry evicted later.
3211	!
3212	! NOTE: We assume that we'll be successful in finding an unlocked
3213	! or invalid entry.  If that isn't the case there are bound to
3214	! bigger problems.
3215	!
3216	set	(1 << 3), %g3
32173:
3218	ldxa	[%g3]ASI_ITLB_ACCESS, %o4	! Load TTE from t16
3219	!
3220	! If this entry isn't valid, we'll choose to displace it (regardless
3221	! of the lock bit).
3222	!
3223	cmp	%o4, %g0			! TTE is > 0 iff not valid
3224	bge	%xcc, 4f			! If invalid, go displace
3225	  andcc	%o4, TTE_LCK_INT, %g0		! Check for lock bit
3226	bnz,a	%icc, 3b			! If locked, look at next
3227	  add	%g3, (1 << 3), %g3		!  entry
32284:
3229	!
3230	! We found an unlocked or invalid entry; we'll explicitly load
3231	! the former index 0 entry here.
3232	!
3233	sethi	%hi(FLUSH_ADDR), %o4
3234	set	MMU_TAG_ACCESS, %g4
3235	stxa	%o2, [%g4]ASI_IMMU
3236	stxa	%o1, [%g3]ASI_ITLB_ACCESS
3237	flush	%o4				! Flush required for I-MMU
3238	retl
3239	  wrpr	%g0, %o3, %pstate		! Enable interrupts
3240	SET_SIZE(itlb_erratum34_fixup)
3241
3242#endif	/* lint */
3243
3244#endif	/* CHEETAHPLUS_ERRATUM_34 */
3245
3246