xref: /illumos-gate/usr/src/uts/sparc/v9/ml/ddi_v9_asm.S (revision 5d9d9091f564c198a760790b0bfa72c44e17912b)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#include <sys/asi.h>
27#include <sys/asm_linkage.h>
28#include <sys/machthread.h>
29#include <sys/privregs.h>
30#include <sys/ontrap.h>
31#include <sys/dditypes.h>
32
33#include "assym.h"
34
35/*
36 * This file implements the following ddi common access
37 * functions:
38 *
39 *	ddi_get{8,16,32,64}
40 *	ddi_put{8,16,32,64}
41 *
42 * and the underlying "trivial" implementations
43 *
44 *      i_ddi_{get,put}{8,16,32,64}
45 *
46 * which assume that there is no need to check the access handle -
47 * byte swapping will be done by the mmu and the address is always
48 * accessible via ld/st instructions.
49 */
50
51/*
52 * The functionality of each of the ddi_get/put routines is performed by
53 * the respective indirect function defined in the access handle.  Use of
54 * the access handle functions provides compatibility across platforms for
55 * drivers.
56 *
57 * By default, the indirect access handle functions are initialized to the
58 * i_ddi_get/put routines to perform memory mapped IO.  If memory mapped IO
59 * is not possible or desired, the access handle must be intialized to another
60 * valid routine to perform the sepcified IO operation.
61 *
62 * The alignment and placement of the following functions have been optimized
63 * such that the implementation specific versions, i_ddi*, fall within the
64 * same cache-line of the generic versions, ddi_*.  This insures that an
65 * I-cache hit will occur thus minimizing the performance impact of using the
66 * access handle.
67 */
68
69	.align 32
70	ENTRY(ddi_get8)
71	ALTENTRY(ddi_io_get8)
72	ALTENTRY(ddi_mem_get8)
73	ldn      [%o0 + AHI_GET8], %g1   /* hdl->ahi_get8 access hndl */
74	jmpl    %g1, %g0                 /* jump to access handle routine */
75	nop
76	SET_SIZE(ddi_get8)
77	SET_SIZE(ddi_io_get8)
78	SET_SIZE(ddi_mem_get8)
79
80	.align 16
81	ENTRY(i_ddi_get8)
82	retl
83	ldub	[%o1], %o0
84	SET_SIZE(i_ddi_get8)
85
86	.align 32
87	ENTRY(ddi_get16)
88	ALTENTRY(ddi_io_get16)
89	ALTENTRY(ddi_mem_get16)
90	ldn      [%o0 + AHI_GET16], %g1   /* hdl->ahi_get16 access hndl */
91	jmpl    %g1, %g0                  /* jump to access handle routine */
92	nop
93	SET_SIZE(ddi_get16)
94	SET_SIZE(ddi_io_get16)
95	SET_SIZE(ddi_mem_get16)
96
97	.align 16
98	ENTRY(i_ddi_get16)
99	ALTENTRY(i_ddi_swap_get16)
100	retl
101	lduh	[%o1], %o0
102	SET_SIZE(i_ddi_get16)
103	SET_SIZE(i_ddi_swap_get16)
104
105	.align 32
106	ENTRY(ddi_get32)
107	ALTENTRY(ddi_io_get32)
108	ALTENTRY(ddi_mem_get32)
109	ldn      [%o0 + AHI_GET32], %g1   /* hdl->ahi_get32 access handle */
110	jmpl    %g1, %g0		  /* jump to access handle routine */
111	nop
112	SET_SIZE(ddi_get32)
113	SET_SIZE(ddi_io_get32)
114	SET_SIZE(ddi_mem_get32)
115
116	.align 16
117	ENTRY(i_ddi_get32)
118	ALTENTRY(i_ddi_swap_get32)
119	retl
120	ld	[%o1], %o0
121	SET_SIZE(i_ddi_get32)
122	SET_SIZE(i_ddi_swap_get32)
123
124	.align 32
125	ENTRY(ddi_get64)
126	ALTENTRY(ddi_io_get64)
127	ALTENTRY(ddi_mem_get64)
128	ldn      [%o0 + AHI_GET64], %g1   /* hdl->ahi_get64 access handle */
129	jmpl    %g1, %g0                  /* jump to access handle routine */
130	nop
131	SET_SIZE(ddi_get64)
132	SET_SIZE(ddi_io_get64)
133	SET_SIZE(ddi_mem_get64)
134
135	.align 16
136	ENTRY(i_ddi_get64)
137	ALTENTRY(i_ddi_swap_get64)
138	retl
139	ldx	[%o1], %o0
140	SET_SIZE(i_ddi_get64)
141	SET_SIZE(i_ddi_swap_get64)
142
143	.align 32
144	ENTRY(ddi_put8)
145	ALTENTRY(ddi_io_put8)
146	ALTENTRY(ddi_mem_put8)
147	ldn      [%o0 + AHI_PUT8], %g1   /* hdl->ahi_put8 access handle */
148	jmpl    %g1, %g0                 /* jump to access handle routine */
149	nop
150	SET_SIZE(ddi_put8)
151	SET_SIZE(ddi_io_put8)
152	SET_SIZE(ddi_mem_put8)
153
154	.align 16
155	ENTRY(i_ddi_put8)
156	retl
157	stub	%o2, [%o1]
158	SET_SIZE(i_ddi_put8)
159
160	.align 32
161	ENTRY(ddi_put16)
162	ALTENTRY(ddi_io_put16)
163	ALTENTRY(ddi_mem_put16)
164	ldn      [%o0 + AHI_PUT16], %g1   /* hdl->ahi_put16 access handle */
165	jmpl    %g1, %g0                  /* jump to access handle routine */
166	nop
167	SET_SIZE(ddi_put16)
168	SET_SIZE(ddi_io_put16)
169	SET_SIZE(ddi_mem_put16)
170
171	.align 16
172	ENTRY(i_ddi_put16)
173	ALTENTRY(i_ddi_swap_put16)
174	retl
175	stuh	%o2, [%o1]
176	SET_SIZE(i_ddi_put16)
177	SET_SIZE(i_ddi_swap_put16)
178
179	.align 32
180	ENTRY(ddi_put32)
181	ALTENTRY(ddi_io_put32)
182	ALTENTRY(ddi_mem_put32)
183	ldn      [%o0 + AHI_PUT32], %g1   /* hdl->ahi_put16 access handle */
184	jmpl    %g1, %g0                  /* jump to access handle routine */
185	nop
186	SET_SIZE(ddi_put32)
187	SET_SIZE(ddi_io_put32)
188	SET_SIZE(ddi_mem_put32)
189
190	.align 16
191	ENTRY(i_ddi_put32)
192	ALTENTRY(i_ddi_swap_put32)
193	retl
194	st	%o2, [%o1]
195	SET_SIZE(i_ddi_put32)
196	SET_SIZE(i_ddi_swap_put32)
197
198	.align 32
199	ENTRY(ddi_put64)
200	ALTENTRY(ddi_io_put64)
201	ALTENTRY(ddi_mem_put64)
202	ldn      [%o0 + AHI_PUT64], %g1   /* hdl->ahi_put64 access handle */
203	jmpl    %g1, %g0                  /* jump to access handle routine */
204	nop
205	SET_SIZE(ddi_put64)
206	SET_SIZE(ddi_io_put64)
207	SET_SIZE(ddi_mem_put64)
208
209	.align 16
210	ENTRY(i_ddi_put64)
211	ALTENTRY(i_ddi_swap_put64)
212	retl
213	stx	%o2, [%o1]
214	SET_SIZE(i_ddi_put64)
215	SET_SIZE(i_ddi_swap_put64)
216
217/*
218 * The ddi_io_rep_get/put routines don't take a flag argument like the "plain"
219 * and mem versions do.  This flag is used to determine whether or not the
220 * device address or port should be automatically incremented.  For IO space,
221 * the device port is never incremented and as such, the flag is always set
222 * to DDI_DEV_NO_AUTOINCR.
223 *
224 * This define processes the repetitive get functionality.  Automatic
225 * incrementing of the device address is determined by the flag field
226 * %o4.  If this is set for AUTOINCR, %o4 is updated with 1 for the
227 * subsequent increment in 2:.
228 *
229 * If this flag is not set for AUTOINCR, %o4 is update with a value of 0 thus
230 * making the increment operation a non-operation.
231 */
232
233#define DDI_REP_GET(n,s)			\
234	cmp	DDI_DEV_NO_AUTOINCR, %o4;	\
235	mov	%g0, %o4;			\
236	brz,pn	%o3, 1f;			\
237	movnz	%xcc, n, %o4;			\
2382:						\
239	dec	%o3;				\
240	ld##s	[%o2], %g4;			\
241	add	%o2, %o4, %o2;			\
242	st##s	%g4, [%o1];			\
243	brnz,pt	%o3, 2b;			\
244	add	%o1, n, %o1;			\
2451:
246
247	.align 32
248	ENTRY(ddi_rep_get8)
249	ALTENTRY(ddi_mem_rep_get8)
250	ldn      [%o0 + AHI_REP_GET8], %g1
251	jmpl    %g1, %g0
252	nop
253	SET_SIZE(ddi_rep_get8)
254	SET_SIZE(ddi_mem_rep_get8)
255
256	.align 16
257	ENTRY(i_ddi_rep_get8)
258	DDI_REP_GET(1,ub)
259	retl
260	nop
261	SET_SIZE(i_ddi_rep_get8)
262
263	.align 32
264	ENTRY(ddi_rep_get16)
265	ALTENTRY(ddi_mem_rep_get16)
266	ldn	[%o0 + AHI_REP_GET16], %g1
267	jmpl    %g1, %g0
268	nop
269	SET_SIZE(ddi_rep_get16)
270	SET_SIZE(ddi_mem_rep_get16)
271
272	.align 16
273	ENTRY(i_ddi_rep_get16)
274	ALTENTRY(i_ddi_swap_rep_get16)
275	DDI_REP_GET(2,uh)
276	retl
277	nop
278	SET_SIZE(i_ddi_rep_get16)
279	SET_SIZE(i_ddi_swap_rep_get16)
280
281	.align 32
282	ENTRY(ddi_rep_get32)
283	ALTENTRY(ddi_mem_rep_get32)
284	ldn      [%o0 + AHI_REP_GET32], %g1
285	jmpl    %g1, %g0
286	nop
287	SET_SIZE(ddi_rep_get32)
288	SET_SIZE(ddi_mem_rep_get32)
289
290	.align 16
291	ENTRY(i_ddi_rep_get32)
292	ALTENTRY(i_ddi_swap_rep_get32)
293	DDI_REP_GET(4,/**/)
294	retl
295	nop
296	SET_SIZE(i_ddi_rep_get32)
297	SET_SIZE(i_ddi_swap_rep_get32)
298
299	.align 32
300	ENTRY(ddi_rep_get64)
301	ALTENTRY(ddi_mem_rep_get64)
302	ldn      [%o0 + AHI_REP_GET64], %g1
303	jmpl    %g1, %g0
304	nop
305	SET_SIZE(ddi_rep_get64)
306	SET_SIZE(ddi_mem_rep_get64)
307
308	.align 16
309	ENTRY(i_ddi_rep_get64)
310	ALTENTRY(i_ddi_swap_rep_get64)
311	DDI_REP_GET(8,x)
312	retl
313	nop
314	SET_SIZE(i_ddi_rep_get64)
315	SET_SIZE(i_ddi_swap_rep_get64)
316
317/*
318 * This define processes the repetitive put functionality.  Automatic
319 * incrementing of the device address is determined by the flag field
320 * %o4.  If this is set for AUTOINCR, %o4 is updated with 1 for the
321 * subsequent increment in 2:.
322 *
323 * If this flag is not set for AUTOINCR, %o4 is update with a value of 0 thus
324 * making the increment operation a non-operation.
325 */
326#define DDI_REP_PUT(n,s)			\
327	cmp	DDI_DEV_NO_AUTOINCR, %o4;	\
328	mov	%g0, %o4;			\
329	brz,pn	%o3, 1f;			\
330	movnz	%xcc, n, %o4;			\
3312:						\
332	dec	%o3;				\
333	ld##s	[%o1], %g4;			\
334	add	%o1, n, %o1;			\
335	st##s	%g4, [%o2];			\
336	brnz,pt	%o3, 2b;			\
337	add	%o2, %o4, %o2;			\
3381:
339
340	.align 32
341	ENTRY(ddi_rep_put8)
342	ALTENTRY(ddi_mem_rep_put8)
343	ldn      [%o0 + AHI_REP_PUT8], %g1
344	jmpl    %g1, %g0
345	nop
346	SET_SIZE(ddi_rep_put8)
347	SET_SIZE(ddi_mem_rep_put8)
348
349	.align 16
350	ENTRY(i_ddi_rep_put8)
351	DDI_REP_PUT(1,ub)
352	retl
353	nop
354	SET_SIZE(i_ddi_rep_put8)
355
356	.align 32
357	ENTRY(ddi_rep_put16)
358	ALTENTRY(ddi_mem_rep_put16)
359	ldn      [%o0 + AHI_REP_PUT16], %g1
360	jmpl    %g1, %g0
361	nop
362	SET_SIZE(ddi_rep_put16)
363	SET_SIZE(ddi_mem_rep_put16)
364
365	.align 16
366	ENTRY(i_ddi_rep_put16)
367	ALTENTRY(i_ddi_swap_rep_put16)
368	DDI_REP_PUT(2,uh)
369	retl
370	nop
371	SET_SIZE(i_ddi_rep_put16)
372	SET_SIZE(i_ddi_swap_rep_put16)
373
374	.align 32
375	ENTRY(ddi_rep_put32)
376	ALTENTRY(ddi_mem_rep_put32)
377	ldn      [%o0 + AHI_REP_PUT32], %g1
378	jmpl    %g1, %g0
379	nop
380	SET_SIZE(ddi_rep_put32)
381	SET_SIZE(ddi_mem_rep_put32)
382
383	.align 16
384	ENTRY(i_ddi_rep_put32)
385	ALTENTRY(i_ddi_swap_rep_put32)
386	DDI_REP_PUT(4,/**/)
387	retl
388	nop
389	SET_SIZE(i_ddi_rep_put32)
390	SET_SIZE(i_ddi_swap_rep_put32)
391
392	.align 32
393	ENTRY(ddi_rep_put64)
394	ALTENTRY(ddi_mem_rep_put64)
395	ldn      [%o0 + AHI_REP_PUT64], %g1
396	jmpl    %g1, %g0
397	nop
398	SET_SIZE(ddi_rep_put64)
399	SET_SIZE(ddi_mem_rep_put64)
400
401	.align 16
402	ENTRY(i_ddi_rep_put64)
403	ALTENTRY(i_ddi_swap_rep_put64)
404	DDI_REP_PUT(8,x)
405	retl
406	nop
407	SET_SIZE(i_ddi_rep_put64)
408	SET_SIZE(i_ddi_swap_rep_put64)
409
410	.align 16
411	ENTRY(ddi_io_rep_get8)
412	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
413	ldn	[%o0 + AHI_REP_GET8], %g1
414	jmpl    %g1, %g0
415	nop
416	SET_SIZE(ddi_io_rep_get8)
417
418	.align 16
419	ENTRY(ddi_io_rep_get16)
420	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
421	ldn	[%o0 + AHI_REP_GET16], %g1
422	jmpl    %g1, %g0
423	nop
424	SET_SIZE(ddi_io_rep_get16)
425
426	.align 16
427	ENTRY(ddi_io_rep_get32)
428	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
429	ldn	[%o0 + AHI_REP_GET32], %g1
430	jmpl    %g1, %g0
431	nop
432	SET_SIZE(ddi_io_rep_get32)
433
434	.align 16
435	ENTRY(ddi_io_rep_get64)
436	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
437	ldn	[%o0 + AHI_REP_GET64], %g1
438	jmpl    %g1, %g0
439	nop
440	SET_SIZE(ddi_io_rep_get64)
441
442        .align 64
443	ENTRY(ddi_check_acc_handle)
444	save	%sp, -SA(WINDOWSIZE), %sp	! get a new window
445	ldn	[%i0 + AHI_FAULT_CHECK], %g1
446	jmpl	%g1, %o7
447	mov	%i0, %o0
448	brnz,a,pn %o0, 0f			! if (return_value != 0)
449	mov	-1, %o0				!	return (DDI_FAILURE)
4500:						! else	return (DDI_SUCCESS)
451	sra	%o0, 0, %i0
452	ret
453	restore
454	SET_SIZE(ddi_check_acc_handle)
455
456        .align 16
457        ENTRY(i_ddi_acc_fault_check)
458	retl
459	ld      [%o0 + AHI_FAULT], %o0
460        SET_SIZE(i_ddi_acc_fault_check)
461
462	.align 16
463	ENTRY(ddi_io_rep_put8)
464	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
465	ldn	[%o0 + AHI_REP_PUT8], %g1
466	jmpl    %g1, %g0
467	nop
468	SET_SIZE(ddi_io_rep_put8)
469
470	.align 16
471	ENTRY(ddi_io_rep_put16)
472	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
473	ldn	[%o0 + AHI_REP_PUT16], %g1
474	jmpl    %g1, %g0
475	nop
476	SET_SIZE(ddi_io_rep_put16)
477
478	.align 16
479	ENTRY(ddi_io_rep_put32)
480	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
481	ldn	[%o0 + AHI_REP_PUT32], %g1
482	jmpl    %g1, %g0
483	nop
484	SET_SIZE(ddi_io_rep_put32)
485
486	.align 16
487	ENTRY(ddi_io_rep_put64)
488	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
489	ldn	[%o0 + AHI_REP_PUT64], %g1
490	jmpl    %g1, %g0
491	nop
492	SET_SIZE(ddi_io_rep_put64)
493
494	ENTRY(do_peek)
495	rdpr	%pstate, %o3	! check ints
496	andcc	%o3, PSTATE_IE, %g0
497	bz,a	done
498	or	%g0, 1, %o0	! Return failure if ints are disabled
499	wrpr	%o3, PSTATE_IE, %pstate
500	cmp	%o0, 8		! 64-bit?
501	bne,a	.peek_int
502	cmp	%o0, 4		! 32-bit?
503	ldx	[%o1], %g1
504	ba	.peekdone
505	stx	%g1, [%o2]
506.peek_int:
507	bne,a	.peek_half
508	cmp	%o0, 2		! 16-bit?
509	lduw	[%o1], %g1
510	ba	.peekdone
511	stuw	%g1, [%o2]
512.peek_half:
513	bne,a	.peek_byte
514	ldub	[%o1], %g1	! 8-bit!
515	lduh	[%o1], %g1
516	ba	.peekdone
517	stuh	%g1, [%o2]
518.peek_byte:
519	stub	%g1, [%o2]
520.peekdone:
521	membar	#Sync		! Make sure the loads take
522	rdpr	%pstate, %o3	! check&enable ints
523	andcc	%o3, PSTATE_IE, %g0
524	bnz	1f
525	nop
526	wrpr	%o3, PSTATE_IE, %pstate
5271:
528	mov	%g0, %o0
529done:
530	retl
531	nop
532	SET_SIZE(do_peek)
533
534	ENTRY(do_poke)
535	cmp	%o0, 8		! 64 bit?
536	bne,a	.poke_int
537	cmp	%o0, 4		! 32-bit?
538	ldx	[%o2], %g1
539	ba	.pokedone
540	stx	%g1, [%o1]
541.poke_int:
542	bne,a	.poke_half
543	cmp	%o0, 2		! 16-bit?
544	lduw	[%o2], %g1
545	ba	.pokedone
546	stuw	%g1, [%o1]
547.poke_half:
548	bne,a	.poke_byte
549	ldub	[%o2], %g1	! 8-bit!
550	lduh	[%o2], %g1
551	ba	.pokedone
552	stuh	%g1, [%o1]
553.poke_byte:
554	stub	%g1, [%o1]
555.pokedone:
556	membar	#Sync
557	retl
558	mov	%g0, %o0
559	SET_SIZE(do_poke)
560
561
562/*
563 * The peek_fault() and poke_fault() routines below are used as on_trap()
564 * trampoline routines.  i_ddi_peek and i_ddi_poke execute do_peek and do_poke
565 * under on_trap protection (see <sys/ontrap.h>), but modify ot_trampoline to
566 * refer to the corresponding routine below.  If a trap occurs, the trap code
567 * will bounce back to the trampoline code, which will effectively cause
568 * do_peek or do_poke to return DDI_FAILURE, instead of longjmp'ing back to
569 * on_trap.  In the case of a peek, we may also need to re-enable interrupts.
570 */
571	.seg	".data"
572.peek_panic:
573	.asciz	"peek_fault: missing or invalid on_trap_data"
574.poke_panic:
575	.asciz	"poke_fault: missing or invalid on_trap_data"
576
577	ENTRY(peek_fault)
578	ldn	[THREAD_REG + T_ONTRAP], %o0	! %o0 = on_trap_data pointer
579	brz,pn	%o0, .peekfail			! if (%o0 == NULL) panic
580	nop
581	lduh	[%o0 + OT_PROT], %o1		! %o1 = %o0->ot_prot
582	andcc	%o1, OT_DATA_ACCESS, %g0	! if (!(%o1 & OT_DATA_ACCESS))
583	bz,pn	%icc, .peekfail			!     panic
584	rdpr	%pstate, %o3
585
586	andcc	%o3, PSTATE_IE, %g0		! enable interrupts
587	bnz	1f
588	nop
589	wrpr	%o3, PSTATE_IE, %pstate
5901:
591	retl
592	sub	%g0, 1, %o0			! return (DDI_FAILURE);
593.peekfail:
594	set	.peek_panic, %o0		! Load panic message
595	call	panic				! Panic if bad t_ontrap data
596	nop
597	SET_SIZE(peek_fault)
598
599
600	ENTRY(poke_fault)
601	ldn	[THREAD_REG + T_ONTRAP], %o0	! %o0 = on_trap_data pointer
602	brz,pn	%o0, .pokefail			! if (%o0 == NULL) panic
603	nop
604	lduh	[%o0 + OT_PROT], %o1		! %o1 = %o0->ot_prot
605	andcc	%o1, OT_DATA_ACCESS, %g0	! if (!(%o1 & OT_DATA_ACCESS))
606	bz,pn	%icc, .pokefail			!     panic
607	nop
608	retl
609	sub	%g0, 1, %o0			! return (DDI_FAILURE);
610.pokefail:
611	set	.poke_panic, %o0		! Load panic message
612	call	panic				! Panic if bad t_ontrap data
613	nop
614	SET_SIZE(poke_fault)
615
616
617/*
618 * IO Fault Services
619 *
620 * Support for protected IO accesses is implemented in the following
621 * functions.  A driver may request one of three protection mechanisms
622 * that enable the system to survive an access errors.  The protection
623 * mechansim is set-up during ddi_regs_map_setup time and may be one of:
624 *
625 *	DDI_DEFAULT_ACC	- no error protection requested.  We will
626 *			use the standard ddi_get/ddi_put operations
627 *			defined above.
628 *
629 *	DDI_FLAGERR - Driver requests that errors encountered will
630 *			be flagged by the system.  The driver is
631 *			responsible for checking the error status
632 *			of the access with a call to ddi_acc_err_get()
633 *			upon return of ddi_get or ddi_put.  To prevent
634 *			an access from causing a system we use internal
635 *			on_trap semantics.
636 *
637 *			The system, depending upon the error,
638 *			may or may not panic.
639 *
640 *	DDI_CAUTIOUS_ACC - Driver expects that the access may cause
641 *			an error to occur.  The system will return
642 *			an error status but will not generate an ereport.
643 *			The system will also ensure synchronous and
644 *			exclusive access to the IO space accessed by
645 *			the caller.
646 *
647 *			To prevent an access from causing a system panic,
648 *			we use on_trap semantics to catch the error and
649 *			set error status.
650 *
651 *	If a read access error is detected and DDI_CAUTIOUS_ACC or
652 *	DDI_FLAGERR_ACC	protection was requested, we will trampoline to the
653 *	error handler, i_ddi_trampoline.  i_ddi_trampoline will:
654 *		- check for proper protection semantics
655 *		- set the error status of the access handle to DDI_FM_NONFATAL
656 *		- re-enable interrupts if neccessary
657 *		- longjmp back to the initiating access function.
658
659 *	If a write access error is detected, an interrupt is typically
660 *	generated and claimed by a bus nexus responsible for the write
661 *	transaction.  The nexus error handler is expected to set the
662 *	error status and the IO initiating driver is expected to check
663 *	for a failed transaction via ddi_fm_acc_err_get().
664 *
665 */
666
667	.seg	".data"
668.acc_panic:
669	.asciz	"DDI access: missing or invalid on_trap_data"
670
671	ENTRY(i_ddi_caut_trampoline)
672	ldn	[THREAD_REG + T_ONTRAP], %o5    ! %o5 = curthread->t_ontrap
673	lduh	[%o5 + OT_PROT], %o1		! %o1 = %o0->ot_prot
674	andcc	%o1, OT_DATA_ACCESS, %g0	! if (!(%o1 & OT_DATA_ACCESS))
675	bz,pn	%icc, .cautaccfail		!     panic
676	rdpr	%pstate, %o3
677	andcc	%o3, PSTATE_IE, %g0		! enable interrupts
678	bnz	1f
679	nop
680	wrpr	%o3, PSTATE_IE, %pstate
6811:
682	ldn	[%o5 + OT_HANDLE], %o0		! %o0 = ot_handle
683	brz,pn	%o0, .cautaccfail		! if (ot_handle == NULL) panic
684	nop
685	ldn	[%o0 + AHI_ERR], %o4		! %o4 = hp->ahi_err
686	membar	#Sync
687	stx	%g0, [%o4 + ERR_ENA]		! ahi_err->err_ena = 0
688	mov	-2, %o0
689	st	%o0, [%o4 + ERR_STATUS]		! ahi_err->err_status = NONFATAL
690	b	longjmp				! longjmp back
691	add	%o5, OT_JMPBUF, %o0		! %o0 = &ot_jmpbuf
692.cautaccfail:
693	set	.acc_panic, %o0			! Load panic message
694	call	panic				! Panic if bad t_ontrap data
695	nop
696	SET_SIZE(i_ddi_caut_trampoline)
697
698/*
699 * DDI on_trap set-up functions,  i_ddi_ontrap() and i_ddinotrap() are used
700 * to protect * ddi_get accesses for DDI_CAUT_ACC.  i_ddi_ontrap() sets
701 * the jumpbuf (setjmp) that will return back to the access routine from
702 * i_ddi_trampoline().  DDI_NOPROTECT() clears the ontrap set-up.
703 */
704	ENTRY(i_ddi_ontrap)
705	ldn	[%o0 + AHI_ERR], %o4
706	ldn	[%o4 + ERR_ONTRAP],  %o4	! %o4 = hp->ahi_err->err_ontrap
707	ldn	[THREAD_REG + T_ONTRAP], %o5	! %o5 = curthread->t_ontrap
708	stn	%o5, [%o4 + OT_PREV]		! ot_prev = t_ontrap
709	membar	#Sync				! force error barrier
710	stn	%o4, [THREAD_REG + T_ONTRAP]	! t_ontrap = err_ontrap
711	b	setjmp
712	add	%o4, OT_JMPBUF, %o0
713	SET_SIZE(i_ddi_ontrap)
714
715	ENTRY(i_ddi_notrap)
716	membar	#Sync				! force error barrier
717	ldn	[%o0 + AHI_ERR], %o4
718	ldn	[%o4 + ERR_ONTRAP],  %o4	! %o4 = hp->ahi_err->err_ontrap
719	ldn	[%o4 + OT_PREV], %o4
720	retl
721	stn	%o4, [THREAD_REG + T_ONTRAP]	! restore curthread->t_ontrap
722	SET_SIZE(i_ddi_notrap)
723
724/*
725 * Internal on_trap set-up macros.  DDI_PROTECT() and DDI_NOPROTECT() are used
726 * to protect * ddi_get accesses for DDI_FLAGERR_ACC.  DDI_NOPROTECT() sets
727 * the jumpbuf that will return back to the access routine from
728 * i_ddi_protect_trampoline().  DDI_NOPROTECT() clears the ontrap set-up.
729 */
730	ENTRY(i_ddi_prot_trampoline)
731	ldn	[THREAD_REG + T_ONTRAP], %o5    ! %o5 = curthread->t_ontrap
732	lduh	[%o5 + OT_PROT], %o1		! %o1 = %o0->ot_prot
733	andcc	%o1, OT_DATA_ACCESS, %g0	! if (!(%o1 & OT_DATA_ACCESS))
734	bz,pn	%icc, .protaccfail		!     panic
735	rdpr	%pstate, %o3
736	andcc	%o3, PSTATE_IE, %g0		! enable interrupts
737	bnz	1f
738	nop
739	wrpr	%o3, PSTATE_IE, %pstate
7401:
741	ldn	[%o5 + OT_HANDLE], %o0		! %o0 = ot_handle
742	brz,pn	%o0, .protaccfail		! if (ot_handle == NULL) panic
743	nop
744	ldn	[%o0 + AHI_ERR], %o4		! %o4 = hp->ahi_err
745	stn	%g0, [%o4 + ERR_ENA]		! ahi_err->err_ena = 0
746	mov	-2, %o0
747	st	%o0, [%o4 + ERR_STATUS]		! ahi_err->err_status = NONFATAL
748	ldn	[%o5 + OT_PREV], %o0		! restore ontrap
749	membar	#Sync				! force error barrier
750	stn	%o0, [THREAD_REG + T_ONTRAP];
751	b	longjmp				! longjmp back
752	add	%o5, OT_JMPBUF, %o0		! %o0 = &ot_jmpbuf
753.protaccfail:
754	set	.acc_panic, %o0			! Load panic message
755	call	panic				! Panic if bad t_ontrap data
756	nop
757	SET_SIZE(i_ddi_prot_trampoline)
758
759#define	DDI_PROTECT()				\
760	ldn	[%o0 + AHI_ERR], %o4;		\
761	ldn	[%o4 + ERR_ONTRAP],  %o4;	\
762	ldn	[THREAD_REG + T_ONTRAP], %o5;	\
763	stn	%o5, [%o4 + OT_PREV];		\
764	membar	#Sync;				\
765	stn	%o4, [THREAD_REG + T_ONTRAP];	\
766	add     %o4, OT_JMPBUF, %o0;		\
767	stn	%o7, [%o0 + L_PC];		\
768	stn	%sp, [%o0 + L_SP];		\
769	clr	%o0;
770
771#define	DDI_NOPROTECT()				\
772	ldn	[THREAD_REG + T_ONTRAP], %o4;	\
773	ldn	[%o4 + OT_PREV], %o5;		\
774	membar	#Sync;				\
775	stn	%o5, [THREAD_REG + T_ONTRAP];
776
777/*
778 * DDI_FLAGERR_ACC specific get/put routines.
779 */
780	.align 16
781	ENTRY(i_ddi_prot_get8)
782	DDI_PROTECT()				! set ontrap protection
783	ldub	[%o1], %o2			! do the io access
784	DDI_NOPROTECT()				! remove protection & ret
785	retl
786	mov	%o2, %o0			! set return value
787	SET_SIZE(i_ddi_prot_get8)
788
789	.align 16
790	ENTRY(i_ddi_prot_get16)
791	DDI_PROTECT()				! set ontrap protection
792	lduh	[%o1], %o2			! do the io access
793	DDI_NOPROTECT()				! remove protection & ret
794	retl
795	mov	%o2, %o0			! set return value
796	SET_SIZE(i_ddi_prot_get16)
797
798	.align 16
799	ENTRY(i_ddi_prot_get32)
800	DDI_PROTECT()				! set ontrap protection
801	ld	[%o1], %o2			! do the io access
802	DDI_NOPROTECT()				! remove protection & ret
803	retl
804	mov	%o2, %o0			! set return value
805	SET_SIZE(i_ddi_prot_get32)
806
807	.align 16
808	ENTRY(i_ddi_prot_get64)
809	DDI_PROTECT()				! set ontrap protection
810	ldx	[%o1], %o2			! do the io access
811	DDI_NOPROTECT()				! remove protection & ret
812	retl
813	mov	%o2, %o0			! set return value
814	SET_SIZE(i_ddi_prot_get64)
815
816	.align 16
817	ENTRY(i_ddi_prot_put8)
818	stub	%o2, [%o1]			! do the io access
819	retl
820	membar	#Sync;
821	SET_SIZE(i_ddi_prot_put8)
822
823	.align 16
824	ENTRY(i_ddi_prot_put16)
825	stuh	%o2, [%o1]			! do the io access
826	retl
827	membar	#Sync;
828	SET_SIZE(i_ddi_prot_put16)
829
830	.align 16
831	ENTRY(i_ddi_prot_put32)
832	st	%o2, [%o1]			! do the io access
833	retl
834	membar	#Sync;
835	SET_SIZE(i_ddi_prot_put32)
836
837	.align 16
838	ENTRY(i_ddi_prot_put64)
839	stx	%o2, [%o1]			! do the io access
840	retl
841	membar	#Sync;
842	SET_SIZE(i_ddi_prot_put64)
843
844	.align 16
845	ENTRY(i_ddi_prot_rep_get8)
846	DDI_PROTECT()				! set ontrap protection
847	tst	%o0				! check access error
848	bnz,a	1f
849	nop
850	DDI_REP_GET(1,ub)
8511:
852	DDI_NOPROTECT()				! remove protection & ret
853	retl
854	nop
855	SET_SIZE(i_ddi_prot_rep_get8)
856
857	.align 16
858	ENTRY(i_ddi_prot_rep_get16)
859	DDI_PROTECT()				! set ontrap protection
860	tst	%o0				! check access error
861	bnz,a	1f
862	nop
863	DDI_REP_GET(2,uh)
8641:
865	DDI_NOPROTECT()				! remove protection & ret
866	retl
867	nop
868	SET_SIZE(i_ddi_prot_rep_get16)
869
870	.align 16
871	ENTRY(i_ddi_prot_rep_get32)
872	DDI_PROTECT()				! set ontrap protection
873	tst	%o0				! check access error
874	bnz,a	1f
875	nop
876	DDI_REP_GET(4,/**/)
8771:
878	DDI_NOPROTECT()				! remove protection & ret
879	retl
880	nop
881	SET_SIZE(i_ddi_prot_rep_get32)
882
883	.align 16
884	ENTRY(i_ddi_prot_rep_get64)
885	DDI_PROTECT()				! set ontrap protection
886	tst	%o0				! check access error
887	bnz,a	1f
888	nop
889	DDI_REP_GET(8,x)
8901:
891	DDI_NOPROTECT()				! remove protection & ret
892	retl
893	nop
894	SET_SIZE(i_ddi_prot_rep_get64)
895
896	.align 16
897	ENTRY(i_ddi_prot_rep_put8)
898	DDI_REP_PUT(1,ub)
899	retl
900	membar	#Sync;
901	SET_SIZE(i_ddi_prot_rep_put8)
902
903	.align 16
904	ENTRY(i_ddi_prot_rep_put16)
905	DDI_REP_PUT(2,uh)
906	retl
907	membar	#Sync;
908	SET_SIZE(i_ddi_prot_rep_put16)
909
910	.align 16
911	ENTRY(i_ddi_prot_rep_put32)
912	DDI_REP_PUT(4,/**/)
913	retl
914	membar	#Sync;
915	SET_SIZE(i_ddi_prot_rep_put32)
916
917	.align 16
918	ENTRY(i_ddi_prot_rep_put64)
919	DDI_REP_PUT(8,x)
920	retl
921	membar	#Sync;
922	SET_SIZE(i_ddi_prot_rep_put64)
923
924/*
925 * Common DDI_CAUTIOUS_ACC routine called from cautious access routines
926 * in ddi_impl.c
927 */
928	ENTRY(i_ddi_caut_get)
929	rdpr	%pstate, %o3	! check ints
930	andcc	%o3, PSTATE_IE, %g0
931	bz,a	cautdone
932	nop
933	wrpr	%o3, PSTATE_IE, %pstate
934	cmp	%o0, 8		! 64-bit?
935	bne,a	.get_int
936	cmp	%o0, 4		! 32-bit?
937	ldx	[%o1], %g1
938	ba	.getdone
939	stx	%g1, [%o2]
940.get_int:
941	bne,a	.get_half
942	cmp	%o0, 2		! 16-bit?
943	lduw	[%o1], %g1
944	ba	.getdone
945	stuw	%g1, [%o2]
946.get_half:
947	bne,a	.get_byte
948	ldub	[%o1], %g1	! 8-bit!
949	lduh	[%o1], %g1
950	ba	.getdone
951	stuh	%g1, [%o2]
952.get_byte:
953	stub	%g1, [%o2]
954.getdone:
955	rdpr	%pstate, %o3	! check&enable ints
956	andcc	%o3, PSTATE_IE, %g0
957	bnz,a	cautdone
958	nop
959	wrpr	%o3, PSTATE_IE, %pstate
960cautdone:
961	retl
962	nop
963	SET_SIZE(i_ddi_caut_get)
964