xref: /titanic_41/usr/src/uts/intel/ia32/ml/copy.s (revision ba2e4443695ee6a6f420a35cd4fc3d3346d22932)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27/*       Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.	*/
28/*       Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T		*/
29/*         All Rights Reserved						*/
30
31/*       Copyright (c) 1987, 1988 Microsoft Corporation			*/
32/*         All Rights Reserved						*/
33
34#pragma ident	"%Z%%M%	%I%	%E% SMI"
35
36#include <sys/errno.h>
37#include <sys/asm_linkage.h>
38
39#if defined(__lint)
40#include <sys/types.h>
41#include <sys/systm.h>
42#else	/* __lint */
43#include "assym.h"
44#endif	/* __lint */
45
46#define	KCOPY_MIN_SIZE	128	/* Must be >= 16 bytes */
47#define	XCOPY_MIN_SIZE	128	/* Must be >= 16 bytes */
48/*
49 * Non-temopral access (NTA) alignment requirement
50 */
51#define	NTA_ALIGN_SIZE	4	/* Must be at least 4-byte aligned */
52#define	NTA_ALIGN_MASK	_CONST(NTA_ALIGN_SIZE-1)
53#define	COUNT_ALIGN_SIZE	16	/* Must be at least 16-byte aligned */
54#define	COUNT_ALIGN_MASK	_CONST(COUNT_ALIGN_SIZE-1)
55
56/*
57 * Copy a block of storage, returning an error code if `from' or
58 * `to' takes a kernel pagefault which cannot be resolved.
59 * Returns errno value on pagefault error, 0 if all ok
60 */
61
62#if defined(__lint)
63
64/* ARGSUSED */
65int
66kcopy(const void *from, void *to, size_t count)
67{ return (0); }
68
69#else	/* __lint */
70
71	.globl	kernelbase
72
73#if defined(__amd64)
74
75	ENTRY(kcopy)
76	pushq	%rbp
77	movq	%rsp, %rbp
78#ifdef DEBUG
79	movq	kernelbase(%rip), %rax
80	cmpq	%rax, %rdi 		/* %rdi = from */
81	jb	0f
82	cmpq	%rax, %rsi		/* %rsi = to */
83	jnb	1f
840:	leaq	.kcopy_panic_msg(%rip), %rdi
85	xorl	%eax, %eax
86	call	panic
871:
88#endif
89	/*
90	 * pass lofault value as 4th argument to do_copy_fault
91	 */
92	leaq	_kcopy_copyerr(%rip), %rcx
93	movq	%gs:CPU_THREAD, %r9	/* %r9 = thread addr */
94
95do_copy_fault:
96	movq	T_LOFAULT(%r9), %r11	/* save the current lofault */
97	movq	%rcx, T_LOFAULT(%r9)	/* new lofault */
98
99	xchgq	%rdi, %rsi		/* %rsi = source, %rdi = destination */
100	movq	%rdx, %rcx		/* %rcx = count */
101	shrq	$3, %rcx		/* 8-byte word count */
102	rep
103	  smovq
104
105	movq	%rdx, %rcx
106	andq	$7, %rcx		/* bytes left over */
107	rep
108	  smovb
109	xorl	%eax, %eax		/* return 0 (success) */
110
111	/*
112	 * A fault during do_copy_fault is indicated through an errno value
113	 * in %rax and we iretq from the trap handler to here.
114	 */
115_kcopy_copyerr:
116	movq	%r11, T_LOFAULT(%r9)	/* restore original lofault */
117	leave
118	ret
119	SET_SIZE(kcopy)
120
121#elif defined(__i386)
122
123#define	ARG_FROM	8
124#define	ARG_TO		12
125#define	ARG_COUNT	16
126
127	ENTRY(kcopy)
128#ifdef DEBUG
129	pushl	%ebp
130	movl	%esp, %ebp
131	movl	kernelbase, %eax
132	cmpl	%eax, ARG_FROM(%ebp)
133	jb	0f
134	cmpl	%eax, ARG_TO(%ebp)
135	jnb	1f
1360:	pushl	$.kcopy_panic_msg
137	call	panic
1381:	popl	%ebp
139#endif
140	lea	_kcopy_copyerr, %eax	/* lofault value */
141	movl	%gs:CPU_THREAD, %edx
142
143do_copy_fault:
144	pushl	%ebp
145	movl	%esp, %ebp		/* setup stack frame */
146	pushl	%esi
147	pushl	%edi			/* save registers */
148
149	movl	T_LOFAULT(%edx), %edi
150	pushl	%edi			/* save the current lofault */
151	movl	%eax, T_LOFAULT(%edx)	/* new lofault */
152
153	movl	ARG_COUNT(%ebp), %ecx
154	movl	ARG_FROM(%ebp), %esi
155	movl	ARG_TO(%ebp), %edi
156	shrl	$2, %ecx		/* word count */
157	rep
158	  smovl
159	movl	ARG_COUNT(%ebp), %ecx
160	andl	$3, %ecx		/* bytes left over */
161	rep
162	  smovb
163	xorl	%eax, %eax
164
165	/*
166	 * A fault during do_copy_fault is indicated through an errno value
167	 * in %eax and we iret from the trap handler to here.
168	 */
169_kcopy_copyerr:
170	popl	%ecx
171	popl	%edi
172	movl	%ecx, T_LOFAULT(%edx)	/* restore the original lofault */
173	popl	%esi
174	popl	%ebp
175	ret
176	SET_SIZE(kcopy)
177
178#undef	ARG_FROM
179#undef	ARG_TO
180#undef	ARG_COUNT
181
182#endif	/* __i386 */
183#endif	/* __lint */
184
185#if defined(__lint)
186
187/*
188 * Copy a block of storage.  Similar to kcopy but uses non-temporal
189 * instructions.
190 */
191
192/* ARGSUSED */
193int
194kcopy_nta(const void *from, void *to, size_t count, int copy_cached)
195{ return (0); }
196
197#else	/* __lint */
198
199#if defined(__amd64)
200
201#define	COPY_LOOP_INIT(src, dst, cnt)	\
202	addq	cnt, src;			\
203	addq	cnt, dst;			\
204	shrq	$3, cnt;			\
205	neg	cnt
206
207	/* Copy 16 bytes per loop.  Uses %rax and %r8 */
208#define	COPY_LOOP_BODY(src, dst, cnt)	\
209	prefetchnta	0x100(src, cnt, 8);	\
210	movq	(src, cnt, 8), %rax;		\
211	movq	0x8(src, cnt, 8), %r8;		\
212	movnti	%rax, (dst, cnt, 8);		\
213	movnti	%r8, 0x8(dst, cnt, 8);		\
214	addq	$2, cnt
215
216	ENTRY(kcopy_nta)
217	pushq	%rbp
218	movq	%rsp, %rbp
219#ifdef DEBUG
220	movq	kernelbase(%rip), %rax
221	cmpq	%rax, %rdi 		/* %rdi = from */
222	jb	0f
223	cmpq	%rax, %rsi		/* %rsi = to */
224	jnb	1f
2250:	leaq	.kcopy_panic_msg(%rip), %rdi
226	xorl	%eax, %eax
227	call	panic
2281:
229#endif
230
231	movq	%gs:CPU_THREAD, %r9
232	cmpq	$0, %rcx		/* No non-temporal access? */
233	/*
234	 * pass lofault value as 4th argument to do_copy_fault
235	 */
236	leaq	_kcopy_nta_copyerr(%rip), %rcx	/* doesn't set rflags */
237	jnz	do_copy_fault		/* use regular access */
238	/*
239	 * Make sure cnt is >= KCOPY_MIN_SIZE
240	 */
241	cmpq	$KCOPY_MIN_SIZE, %rdx
242	jb	do_copy_fault
243
244	/*
245	 * Make sure src and dst are NTA_ALIGN_SIZE aligned,
246	 * count is COUNT_ALIGN_SIZE aligned.
247	 */
248	movq	%rdi, %r10
249	orq	%rsi, %r10
250	andq	$NTA_ALIGN_MASK, %r10
251	orq	%rdx, %r10
252	andq	$COUNT_ALIGN_MASK, %r10
253	jnz	do_copy_fault
254
255	ALTENTRY(do_copy_fault_nta)
256	movq    %gs:CPU_THREAD, %r9     /* %r9 = thread addr */
257	movq    T_LOFAULT(%r9), %r11    /* save the current lofault */
258	movq    %rcx, T_LOFAULT(%r9)    /* new lofault */
259
260	/*
261	 * COPY_LOOP_BODY uses %rax and %r8
262	 */
263	COPY_LOOP_INIT(%rdi, %rsi, %rdx)
2642:	COPY_LOOP_BODY(%rdi, %rsi, %rdx)
265	jnz	2b
266
267	mfence
268	xorl	%eax, %eax		/* return 0 (success) */
269
270_kcopy_nta_copyerr:
271	movq	%r11, T_LOFAULT(%r9)    /* restore original lofault */
272	leave
273	ret
274	SET_SIZE(do_copy_fault_nta)
275	SET_SIZE(kcopy_nta)
276
277#elif defined(__i386)
278
279#define	ARG_FROM	8
280#define	ARG_TO		12
281#define	ARG_COUNT	16
282
283#define	COPY_LOOP_INIT(src, dst, cnt)	\
284	addl	cnt, src;			\
285	addl	cnt, dst;			\
286	shrl	$3, cnt;			\
287	neg	cnt
288
289#define	COPY_LOOP_BODY(src, dst, cnt)	\
290	prefetchnta	0x100(src, cnt, 8);	\
291	movl	(src, cnt, 8), %esi;		\
292	movnti	%esi, (dst, cnt, 8);		\
293	movl	0x4(src, cnt, 8), %esi;		\
294	movnti	%esi, 0x4(dst, cnt, 8);		\
295	movl	0x8(src, cnt, 8), %esi;		\
296	movnti	%esi, 0x8(dst, cnt, 8);		\
297	movl	0xc(src, cnt, 8), %esi;		\
298	movnti	%esi, 0xc(dst, cnt, 8);		\
299	addl	$2, cnt
300
301	/*
302	 * kcopy_nta is not implemented for 32-bit as no performance
303	 * improvement was shown.  We simply jump directly to kcopy
304	 * and discard the 4 arguments.
305	 */
306	ENTRY(kcopy_nta)
307	jmp	kcopy
308
309	lea	_kcopy_nta_copyerr, %eax	/* lofault value */
310	ALTENTRY(do_copy_fault_nta)
311	pushl	%ebp
312	movl	%esp, %ebp		/* setup stack frame */
313	pushl	%esi
314	pushl	%edi
315
316	movl	%gs:CPU_THREAD, %edx
317	movl	T_LOFAULT(%edx), %edi
318	pushl	%edi			/* save the current lofault */
319	movl	%eax, T_LOFAULT(%edx)	/* new lofault */
320
321	/* COPY_LOOP_BODY needs to use %esi */
322	movl	ARG_COUNT(%ebp), %ecx
323	movl	ARG_FROM(%ebp), %edi
324	movl	ARG_TO(%ebp), %eax
325	COPY_LOOP_INIT(%edi, %eax, %ecx)
3261:	COPY_LOOP_BODY(%edi, %eax, %ecx)
327	jnz	1b
328	mfence
329
330	xorl	%eax, %eax
331_kcopy_nta_copyerr:
332	popl	%ecx
333	popl	%edi
334	movl	%ecx, T_LOFAULT(%edx)	/* restore the original lofault */
335	popl	%esi
336	leave
337	ret
338	SET_SIZE(do_copy_fault_nta)
339	SET_SIZE(kcopy_nta)
340
341#undef	ARG_FROM
342#undef	ARG_TO
343#undef	ARG_COUNT
344
345#endif	/* __i386 */
346#endif	/* __lint */
347
348#if defined(__lint)
349
350/* ARGSUSED */
351void
352bcopy(const void *from, void *to, size_t count)
353{}
354
355#else	/* __lint */
356
357#if defined(__amd64)
358
359	ENTRY(bcopy)
360#ifdef DEBUG
361	orq	%rdx, %rdx		/* %rdx = count */
362	jz	1f
363	movq	kernelbase(%rip), %rax
364	cmpq	%rax, %rdi		/* %rdi = from */
365	jb	0f
366	cmpq	%rax, %rsi		/* %rsi = to */
367	jnb	1f
3680:	leaq	.bcopy_panic_msg(%rip), %rdi
369	jmp	call_panic		/* setup stack and call panic */
3701:
371#endif
372do_copy:
373	xchgq	%rdi, %rsi		/* %rsi = source, %rdi = destination */
374	movq	%rdx, %rcx		/* %rcx = count */
375	shrq	$3, %rcx		/* 8-byte word count */
376	rep
377	  smovq
378
379	movq	%rdx, %rcx
380	andq	$7, %rcx		/* bytes left over */
381	rep
382	  smovb
383	ret
384
385#ifdef DEBUG
386	/*
387	 * Setup frame on the run-time stack. The end of the input argument
388	 * area must be aligned on a 16 byte boundary. The stack pointer %rsp,
389	 * always points to the end of the latest allocated stack frame.
390	 * panic(const char *format, ...) is a varargs function. When a
391	 * function taking variable arguments is called, %rax must be set
392	 * to eight times the number of floating point parameters passed
393	 * to the function in SSE registers.
394	 */
395call_panic:
396	pushq	%rbp			/* align stack properly */
397	movq	%rsp, %rbp
398	xorl	%eax, %eax		/* no variable arguments */
399	call	panic			/* %rdi = format string */
400#endif
401	SET_SIZE(bcopy)
402
403#elif defined(__i386)
404
405#define	ARG_FROM	4
406#define	ARG_TO		8
407#define	ARG_COUNT	12
408
409	ENTRY(bcopy)
410#ifdef DEBUG
411	movl	ARG_COUNT(%esp), %eax
412	orl	%eax, %eax
413	jz	1f
414	movl	kernelbase, %eax
415	cmpl	%eax, ARG_FROM(%esp)
416	jb	0f
417	cmpl	%eax, ARG_TO(%esp)
418	jnb	1f
4190:	pushl	%ebp
420	movl	%esp, %ebp
421	pushl	$.bcopy_panic_msg
422	call	panic
4231:
424#endif
425do_copy:
426	movl	%esi, %eax		/* save registers */
427	movl	%edi, %edx
428	movl	ARG_COUNT(%esp), %ecx
429	movl	ARG_FROM(%esp), %esi
430	movl	ARG_TO(%esp), %edi
431
432	shrl	$2, %ecx		/* word count */
433	rep
434	  smovl
435	movl	ARG_COUNT(%esp), %ecx
436	andl	$3, %ecx		/* bytes left over */
437	rep
438	  smovb
439	movl	%eax, %esi		/* restore registers */
440	movl	%edx, %edi
441	ret
442	SET_SIZE(bcopy)
443
444#undef	ARG_COUNT
445#undef	ARG_FROM
446#undef	ARG_TO
447
448#endif	/* __i386 */
449#endif	/* __lint */
450
451
452/*
453 * Zero a block of storage, returning an error code if we
454 * take a kernel pagefault which cannot be resolved.
455 * Returns errno value on pagefault error, 0 if all ok
456 */
457
458#if defined(__lint)
459
460/* ARGSUSED */
461int
462kzero(void *addr, size_t count)
463{ return (0); }
464
465#else	/* __lint */
466
467#if defined(__amd64)
468
469	ENTRY(kzero)
470#ifdef DEBUG
471        cmpq	kernelbase(%rip), %rdi	/* %rdi = addr */
472        jnb	0f
473        leaq	.kzero_panic_msg(%rip), %rdi
474	jmp	call_panic		/* setup stack and call panic */
4750:
476#endif
477	/*
478	 * pass lofault value as 3rd argument to do_zero_fault
479	 */
480	leaq	_kzeroerr(%rip), %rdx
481
482do_zero_fault:
483	movq	%gs:CPU_THREAD, %r9	/* %r9 = thread addr */
484	movq	T_LOFAULT(%r9), %r11	/* save the current lofault */
485	movq	%rdx, T_LOFAULT(%r9)	/* new lofault */
486
487	movq	%rsi, %rcx		/* get size in bytes */
488	shrq	$3, %rcx		/* count of 8-byte words to zero */
489	xorl	%eax, %eax		/* clear %rax; used in sstoq / sstob */
490	rep
491	  sstoq				/* %rcx = words to clear (%rax=0) */
492
493	movq	%rsi, %rcx
494	andq	$7, %rcx		/* bytes left over */
495	rep
496	  sstob				/* %rcx = residual bytes to clear */
497
498	/*
499	 * A fault during do_zero_fault is indicated through an errno value
500	 * in %rax when we iretq to here.
501	 */
502_kzeroerr:
503	movq	%r11, T_LOFAULT(%r9)	/* restore the original lofault */
504	ret
505	SET_SIZE(kzero)
506
507#elif defined(__i386)
508
509#define	ARG_ADDR	8
510#define	ARG_COUNT	12
511
512	ENTRY(kzero)
513#ifdef DEBUG
514	pushl	%ebp
515	movl	%esp, %ebp
516	movl	kernelbase, %eax
517        cmpl	%eax, ARG_ADDR(%ebp)
518        jnb	0f
519        pushl   $.kzero_panic_msg
520        call    panic
5210:	popl	%ebp
522#endif
523	lea	_kzeroerr, %eax		/* kzeroerr is lofault value */
524
525do_zero_fault:
526	pushl	%ebp			/* save stack base */
527	movl	%esp, %ebp		/* set new stack base */
528	pushl	%edi			/* save %edi */
529
530	mov	%gs:CPU_THREAD, %edx
531	movl	T_LOFAULT(%edx), %edi
532	pushl	%edi			/* save the current lofault */
533	movl	%eax, T_LOFAULT(%edx)	/* new lofault */
534
535	movl	ARG_COUNT(%ebp), %ecx	/* get size in bytes */
536	movl	ARG_ADDR(%ebp), %edi	/* %edi <- address of bytes to clear */
537	shrl	$2, %ecx		/* Count of double words to zero */
538	xorl	%eax, %eax		/* sstol val */
539	rep
540	  sstol			/* %ecx contains words to clear (%eax=0) */
541
542	movl	ARG_COUNT(%ebp), %ecx	/* get size in bytes */
543	andl	$3, %ecx		/* do mod 4 */
544	rep
545	  sstob			/* %ecx contains residual bytes to clear */
546
547	/*
548	 * A fault during do_zero_fault is indicated through an errno value
549	 * in %eax when we iret to here.
550	 */
551_kzeroerr:
552	popl	%edi
553	movl	%edi, T_LOFAULT(%edx)	/* restore the original lofault */
554	popl	%edi
555	popl	%ebp
556	ret
557	SET_SIZE(kzero)
558
559#undef	ARG_ADDR
560#undef	ARG_COUNT
561
562#endif	/* __i386 */
563#endif	/* __lint */
564
565/*
566 * Zero a block of storage.
567 */
568
569#if defined(__lint)
570
571/* ARGSUSED */
572void
573bzero(void *addr, size_t count)
574{}
575
576#else	/* __lint */
577
578#if defined(__amd64)
579
580	ENTRY(bzero)
581#ifdef DEBUG
582	cmpq	kernelbase(%rip), %rdi	/* %rdi = addr */
583	jnb	0f
584	leaq	.bzero_panic_msg(%rip), %rdi
585	jmp	call_panic		/* setup stack and call panic */
5860:
587#endif
588do_zero:
589	movq	%rsi, %rcx		/* get size in bytes */
590	shrq	$3, %rcx		/* count of 8-byte words to zero */
591	xorl	%eax, %eax		/* clear %rax; used in sstoq / sstob */
592	rep
593	  sstoq				/* %rcx = words to clear (%rax=0) */
594
595	movq	%rsi, %rcx
596	andq	$7, %rcx		/* bytes left over */
597	rep
598	  sstob				/* %rcx = residual bytes to clear */
599	ret
600	SET_SIZE(bzero)
601
602#elif defined(__i386)
603
604#define	ARG_ADDR	4
605#define	ARG_COUNT	8
606
607	ENTRY(bzero)
608#ifdef DEBUG
609	movl	kernelbase, %eax
610	cmpl	%eax, ARG_ADDR(%esp)
611	jnb	0f
612	pushl	%ebp
613	movl	%esp, %ebp
614	pushl	$.bzero_panic_msg
615	call	panic
6160:
617#endif
618do_zero:
619	movl	%edi, %edx
620	movl	ARG_COUNT(%esp), %ecx
621	movl	ARG_ADDR(%esp), %edi
622	shrl	$2, %ecx
623	xorl	%eax, %eax
624	rep
625	  sstol
626	movl	ARG_COUNT(%esp), %ecx
627	andl	$3, %ecx
628	rep
629	  sstob
630	movl	%edx, %edi
631	ret
632	SET_SIZE(bzero)
633
634#undef	ARG_ADDR
635#undef	ARG_COUNT
636
637#endif	/* __i386 */
638#endif	/* __lint */
639
640/*
641 * Transfer data to and from user space -
642 * Note that these routines can cause faults
643 * It is assumed that the kernel has nothing at
644 * less than KERNELBASE in the virtual address space.
645 *
646 * Note that copyin(9F) and copyout(9F) are part of the
647 * DDI/DKI which specifies that they return '-1' on "errors."
648 *
649 * Sigh.
650 *
651 * So there's two extremely similar routines - xcopyin_nta() and
652 * xcopyout_nta() which return the errno that we've faithfully computed.
653 * This allows other callers (e.g. uiomove(9F)) to work correctly.
654 * Given that these are used pretty heavily, we expand the calling
655 * sequences inline for all flavours (rather than making wrappers).
656 */
657
658/*
659 * Copy user data to kernel space.
660 */
661
662#if defined(__lint)
663
664/* ARGSUSED */
665int
666copyin(const void *uaddr, void *kaddr, size_t count)
667{ return (0); }
668
669#else	/* lint */
670
671#if defined(__amd64)
672
673	ENTRY(copyin)
674	pushq	%rbp
675	movq	%rsp, %rbp
676	subq	$32, %rsp
677
678	/*
679	 * save args in case we trap and need to rerun as a copyop
680	 */
681	movq	%rdi, (%rsp)
682	movq	%rsi, 0x8(%rsp)
683	movq	%rdx, 0x10(%rsp)
684
685	movq	kernelbase(%rip), %rax
686#ifdef DEBUG
687	cmpq	%rax, %rsi		/* %rsi = kaddr */
688	jnb	1f
689	leaq	.copyin_panic_msg(%rip), %rdi
690	xorl	%eax, %eax
691	call	panic
6921:
693#endif
694	/*
695	 * pass lofault value as 4th argument to do_copy_fault
696	 */
697	leaq	_copyin_err(%rip), %rcx
698
699	movq	%gs:CPU_THREAD, %r9
700	cmpq	%rax, %rdi		/* test uaddr < kernelbase */
701	jb	do_copy_fault
702	jmp	3f
703
704_copyin_err:
705	movq	%r11, T_LOFAULT(%r9)	/* restore original lofault */
7063:
707	movq	T_COPYOPS(%r9), %rax
708	cmpq	$0, %rax
709	jz	2f
710	/*
711	 * reload args for the copyop
712	 */
713	movq	(%rsp), %rdi
714	movq	0x8(%rsp), %rsi
715	movq	0x10(%rsp), %rdx
716	leave
717	jmp	*CP_COPYIN(%rax)
718
7192:	movl	$-1, %eax
720	leave
721	ret
722	SET_SIZE(copyin)
723
724#elif defined(__i386)
725
726#define	ARG_UADDR	4
727#define	ARG_KADDR	8
728
729	ENTRY(copyin)
730	movl	kernelbase, %ecx
731#ifdef DEBUG
732	cmpl	%ecx, ARG_KADDR(%esp)
733	jnb	1f
734	pushl	%ebp
735	movl	%esp, %ebp
736	pushl	$.copyin_panic_msg
737	call	panic
7381:
739#endif
740	lea	_copyin_err, %eax
741
742	movl	%gs:CPU_THREAD, %edx
743	cmpl	%ecx, ARG_UADDR(%esp)	/* test uaddr < kernelbase */
744	jb	do_copy_fault
745	jmp	3f
746
747_copyin_err:
748	popl	%ecx
749	popl	%edi
750	movl	%ecx, T_LOFAULT(%edx)	/* restore original lofault */
751	popl	%esi
752	popl	%ebp
7533:
754	movl	T_COPYOPS(%edx), %eax
755	cmpl	$0, %eax
756	jz	2f
757	jmp	*CP_COPYIN(%eax)
758
7592:	movl	$-1, %eax
760	ret
761	SET_SIZE(copyin)
762
763#undef	ARG_UADDR
764#undef	ARG_KADDR
765
766#endif	/* __i386 */
767#endif	/* __lint */
768
769#if defined(__lint)
770
771/* ARGSUSED */
772int
773xcopyin_nta(const void *uaddr, void *kaddr, size_t count, int copy_cached)
774{ return (0); }
775
776#else	/* __lint */
777
778#if defined(__amd64)
779
780	ENTRY(xcopyin_nta)
781	pushq	%rbp
782	movq	%rsp, %rbp
783	subq	$32, %rsp
784
785	/*
786	 * save args in case we trap and need to rerun as a copyop
787	 * %rcx is consumed in this routine so we don't need to save
788	 * it.
789	 */
790	movq	%rdi, (%rsp)
791	movq	%rsi, 0x8(%rsp)
792	movq	%rdx, 0x10(%rsp)
793
794	movq	kernelbase(%rip), %rax
795#ifdef DEBUG
796	cmpq	%rax, %rsi		/* %rsi = kaddr */
797	jnb	1f
798	leaq	.xcopyin_panic_msg(%rip), %rdi
799	xorl	%eax, %eax
800	call	panic
8011:
802#endif
803	movq	%gs:CPU_THREAD, %r9
804	cmpq	%rax, %rdi		/* test uaddr < kernelbase */
805	jae	4f
806	cmpq	$0, %rcx		/* No non-temporal access? */
807	/*
808	 * pass lofault value as 4th argument to do_copy_fault
809	 */
810	leaq	_xcopyin_err(%rip), %rcx	/* doesn't set rflags */
811	jnz	do_copy_fault		/* use regular access */
812	/*
813	 * Make sure cnt is >= XCOPY_MIN_SIZE bytes
814	 */
815	cmpq	$XCOPY_MIN_SIZE, %rdx
816	jb	do_copy_fault
817
818	/*
819	 * Make sure src and dst are NTA_ALIGN_SIZE aligned,
820	 * count is COUNT_ALIGN_SIZE aligned.
821	 */
822	movq	%rdi, %r10
823	orq	%rsi, %r10
824	andq	$NTA_ALIGN_MASK, %r10
825	orq	%rdx, %r10
826	andq	$COUNT_ALIGN_MASK, %r10
827	jnz	do_copy_fault
828	jmp	do_copy_fault_nta	/* use non-temporal access */
829
8304:
831	movl	$EFAULT, %eax
832	jmp	3f
833
834	/*
835	 * A fault during do_copy_fault or do_copy_fault_nta is
836	 * indicated through an errno value in %rax and we iret from the
837	 * trap handler to here.
838	 */
839_xcopyin_err:
840	movq	%r11, T_LOFAULT(%r9)	/* restore original lofault */
8413:
842	movq	T_COPYOPS(%r9), %r8
843	cmpq	$0, %r8
844	jz	2f
845
846	/*
847	 * reload args for the copyop
848	 */
849	movq	(%rsp), %rdi
850	movq	0x8(%rsp), %rsi
851	movq	0x10(%rsp), %rdx
852	leave
853	jmp	*CP_XCOPYIN(%r8)
854
8552:	leave
856	ret
857	SET_SIZE(xcopyin_nta)
858
859#elif defined(__i386)
860
861#define	ARG_UADDR	4
862#define	ARG_KADDR	8
863#define	ARG_COUNT	12
864#define	ARG_CACHED	16
865
866	.globl	use_sse_copy
867
868	ENTRY(xcopyin_nta)
869	movl	kernelbase, %ecx
870	lea	_xcopyin_err, %eax
871	movl	%gs:CPU_THREAD, %edx
872	cmpl	%ecx, ARG_UADDR(%esp)	/* test uaddr < kernelbase */
873	jae	4f
874
875	cmpl	$0, use_sse_copy	/* no sse support */
876	jz	do_copy_fault
877
878	cmpl	$0, ARG_CACHED(%esp)	/* copy_cached hint set? */
879	jnz	do_copy_fault
880
881	/*
882	 * Make sure cnt is >= XCOPY_MIN_SIZE bytes
883	 */
884	cmpl	$XCOPY_MIN_SIZE, ARG_COUNT(%esp)
885	jb	do_copy_fault
886
887	/*
888	 * Make sure src and dst are NTA_ALIGN_SIZE aligned,
889	 * count is COUNT_ALIGN_SIZE aligned.
890	 */
891	movl	ARG_UADDR(%esp), %ecx
892	orl	ARG_KADDR(%esp), %ecx
893	andl	$NTA_ALIGN_MASK, %ecx
894	orl	ARG_COUNT(%esp), %ecx
895	andl	$COUNT_ALIGN_MASK, %ecx
896	jnz	do_copy_fault
897
898	jmp	do_copy_fault_nta	/* use regular access */
899
9004:
901	movl	$EFAULT, %eax
902	jmp	3f
903
904	/*
905	 * A fault during do_copy_fault or do_copy_fault_nta is
906	 * indicated through an errno value in %eax and we iret from the
907	 * trap handler to here.
908	 */
909_xcopyin_err:
910	popl	%ecx
911	popl	%edi
912	movl	%ecx, T_LOFAULT(%edx)	/* restore original lofault */
913	popl	%esi
914	popl	%ebp
9153:
916	cmpl	$0, T_COPYOPS(%edx)
917	jz	2f
918	movl	T_COPYOPS(%edx), %eax
919	jmp	*CP_XCOPYIN(%eax)
920
9212:	rep; 	ret	/* use 2 byte return instruction when branch target */
922			/* AMD Software Optimization Guide - Section 6.2 */
923	SET_SIZE(xcopyin_nta)
924
925#undef	ARG_UADDR
926#undef	ARG_KADDR
927#undef	ARG_COUNT
928#undef	ARG_CACHED
929
930#endif	/* __i386 */
931#endif	/* __lint */
932
933/*
934 * Copy kernel data to user space.
935 */
936
937#if defined(__lint)
938
939/* ARGSUSED */
940int
941copyout(const void *kaddr, void *uaddr, size_t count)
942{ return (0); }
943
944#else	/* __lint */
945
946#if defined(__amd64)
947
948	ENTRY(copyout)
949	pushq	%rbp
950	movq	%rsp, %rbp
951	subq	$32, %rsp
952
953	/*
954	 * save args in case we trap and need to rerun as a copyop
955	 */
956	movq	%rdi, (%rsp)
957	movq	%rsi, 0x8(%rsp)
958	movq	%rdx, 0x10(%rsp)
959
960	movq	kernelbase(%rip), %rax
961#ifdef DEBUG
962	cmpq	%rax, %rdi		/* %rdi = kaddr */
963	jnb	1f
964	leaq	.copyout_panic_msg(%rip), %rdi
965	xorl	%eax, %eax
966	call	panic
9671:
968#endif
969	/*
970	 * pass lofault value as 4th argument to do_copy_fault
971	 */
972	leaq	_copyout_err(%rip), %rcx
973
974	movq	%gs:CPU_THREAD, %r9
975	cmpq	%rax, %rsi		/* test uaddr < kernelbase */
976	jb	do_copy_fault
977	jmp	3f
978
979_copyout_err:
980	movq	%r11, T_LOFAULT(%r9)	/* restore original lofault */
9813:
982	movq	T_COPYOPS(%r9), %rax
983	cmpq	$0, %rax
984	jz	2f
985
986	/*
987	 * reload args for the copyop
988	 */
989	movq	(%rsp), %rdi
990	movq	0x8(%rsp), %rsi
991	movq	0x10(%rsp), %rdx
992	leave
993	jmp	*CP_COPYOUT(%rax)
994
9952:	movl	$-1, %eax
996	leave
997	ret
998	SET_SIZE(copyout)
999
1000#elif defined(__i386)
1001
1002#define	ARG_KADDR	4
1003#define	ARG_UADDR	8
1004
1005	ENTRY(copyout)
1006	movl	kernelbase, %ecx
1007#ifdef DEBUG
1008	cmpl	%ecx, ARG_KADDR(%esp)
1009	jnb	1f
1010	pushl	%ebp
1011	movl	%esp, %ebp
1012	pushl	$.copyout_panic_msg
1013	call	panic
10141:
1015#endif
1016	lea	_copyout_err, %eax
1017	movl	%gs:CPU_THREAD, %edx
1018	cmpl	%ecx, ARG_UADDR(%esp)	/* test uaddr < kernelbase */
1019	jb	do_copy_fault
1020	jmp	3f
1021
1022_copyout_err:
1023	popl	%ecx
1024	popl	%edi
1025	movl	%ecx, T_LOFAULT(%edx)	/* restore original lofault */
1026	popl	%esi
1027	popl	%ebp
10283:
1029	movl	T_COPYOPS(%edx), %eax
1030	cmpl	$0, %eax
1031	jz	2f
1032	jmp	*CP_COPYOUT(%eax)
1033
10342:	movl	$-1, %eax
1035	ret
1036	SET_SIZE(copyout)
1037
1038#undef	ARG_UADDR
1039#undef	ARG_KADDR
1040
1041#endif	/* __i386 */
1042#endif	/* __lint */
1043
1044#if defined(__lint)
1045
1046/* ARGSUSED */
1047int
1048xcopyout_nta(const void *kaddr, void *uaddr, size_t count, int copy_cached)
1049{ return (0); }
1050
1051#else	/* __lint */
1052
1053#if defined(__amd64)
1054
1055	ENTRY(xcopyout_nta)
1056	pushq	%rbp
1057	movq	%rsp, %rbp
1058	subq	$32, %rsp
1059
1060	/*
1061	 * save args in case we trap and need to rerun as a copyop
1062	 */
1063	movq	%rdi, (%rsp)
1064	movq	%rsi, 0x8(%rsp)
1065	movq	%rdx, 0x10(%rsp)
1066
1067	movq	kernelbase(%rip), %rax
1068#ifdef DEBUG
1069	cmpq	%rax, %rdi		/* %rdi = kaddr */
1070	jnb	1f
1071	leaq	.xcopyout_panic_msg(%rip), %rdi
1072	xorl	%eax, %eax
1073	call	panic
10741:
1075#endif
1076	movq	%gs:CPU_THREAD, %r9
1077	cmpq	%rax, %rsi		/* test uaddr < kernelbase */
1078	jae	4f
1079
1080	cmpq	$0, %rcx		/* No non-temporal access? */
1081	/*
1082	 * pass lofault value as 4th argument to do_copy_fault
1083	 */
1084	leaq	_xcopyout_err(%rip), %rcx
1085	jnz	do_copy_fault
1086	/*
1087	 * Make sure cnt is >= XCOPY_MIN_SIZE bytes
1088	 */
1089	cmpq	$XCOPY_MIN_SIZE, %rdx
1090	jb	do_copy_fault
1091
1092	/*
1093	 * Make sure src and dst are NTA_ALIGN_SIZE aligned,
1094	 * count is COUNT_ALIGN_SIZE aligned.
1095	 */
1096	movq	%rdi, %r10
1097	orq	%rsi, %r10
1098	andq	$NTA_ALIGN_MASK, %r10
1099	orq	%rdx, %r10
1100	andq	$COUNT_ALIGN_MASK, %r10
1101	jnz	do_copy_fault
1102	jmp	do_copy_fault_nta
1103
11044:
1105	movl	$EFAULT, %eax
1106	jmp	3f
1107
1108	/*
1109	 * A fault during do_copy_fault or do_copy_fault_nta is
1110	 * indicated through an errno value in %rax and we iret from the
1111	 * trap handler to here.
1112	 */
1113_xcopyout_err:
1114	movq	%r11, T_LOFAULT(%r9)	/* restore original lofault */
11153:
1116	movq	T_COPYOPS(%r9), %r8
1117	cmpq	$0, %r8
1118	jz	2f
1119
1120	/*
1121	 * reload args for the copyop
1122	 */
1123	movq	(%rsp), %rdi
1124	movq	0x8(%rsp), %rsi
1125	movq	0x10(%rsp), %rdx
1126	leave
1127	jmp	*CP_XCOPYOUT(%r8)
1128
11292:	leave
1130	ret
1131	SET_SIZE(xcopyout_nta)
1132
1133#elif defined(__i386)
1134
1135#define	ARG_KADDR	4
1136#define	ARG_UADDR	8
1137#define	ARG_COUNT	12
1138#define	ARG_CACHED	16
1139
1140	ENTRY(xcopyout_nta)
1141	movl	kernelbase, %ecx
1142	lea	_xcopyout_err, %eax
1143	movl	%gs:CPU_THREAD, %edx
1144	cmpl	%ecx, ARG_UADDR(%esp)	/* test uaddr < kernelbase */
1145	jae	4f
1146
1147	cmpl	$0, use_sse_copy	/* no sse support */
1148	jz	do_copy_fault
1149
1150	cmpl	$0, ARG_CACHED(%esp)	/* copy_cached hint set? */
1151	jnz	do_copy_fault
1152
1153	/*
1154	 * Make sure cnt is >= XCOPY_MIN_SIZE bytes
1155	 */
1156	cmpl	$XCOPY_MIN_SIZE, %edx
1157	jb	do_copy_fault
1158
1159	/*
1160	 * Make sure src and dst are NTA_ALIGN_SIZE aligned,
1161	 * count is COUNT_ALIGN_SIZE aligned.
1162	 */
1163	movl	ARG_UADDR(%esp), %ecx
1164	orl	ARG_KADDR(%esp), %ecx
1165	andl	$NTA_ALIGN_MASK, %ecx
1166	orl	ARG_COUNT(%esp), %ecx
1167	andl	$COUNT_ALIGN_MASK, %ecx
1168	jnz	do_copy_fault
1169	jmp	do_copy_fault_nta
1170
11714:
1172	movl	$EFAULT, %eax
1173	jmp	3f
1174
1175	/*
1176	 * A fault during do_copy_fault or do_copy_fault_nta is
1177	 * indicated through an errno value in %eax and we iret from the
1178	 * trap handler to here.
1179	 */
1180_xcopyout_err:
1181	/ restore the original lofault
1182	popl	%ecx
1183	popl	%edi
1184	movl	%ecx, T_LOFAULT(%edx)	/ original lofault
1185	popl	%esi
1186	popl	%ebp
11873:
1188	cmpl	$0, T_COPYOPS(%edx)
1189	jz	2f
1190	movl	T_COPYOPS(%edx), %eax
1191	jmp	*CP_XCOPYOUT(%eax)
1192
11932:	rep;	ret	/* use 2 byte return instruction when branch target */
1194			/* AMD Software Optimization Guide - Section 6.2 */
1195	SET_SIZE(xcopyout_nta)
1196
1197#undef	ARG_UADDR
1198#undef	ARG_KADDR
1199#undef	ARG_COUNT
1200#undef	ARG_CACHED
1201
1202#endif	/* __i386 */
1203#endif	/* __lint */
1204
1205/*
1206 * Copy a null terminated string from one point to another in
1207 * the kernel address space.
1208 */
1209
1210#if defined(__lint)
1211
1212/* ARGSUSED */
1213int
1214copystr(const char *from, char *to, size_t maxlength, size_t *lencopied)
1215{ return (0); }
1216
1217#else	/* __lint */
1218
1219#if defined(__amd64)
1220
1221	ENTRY(copystr)
1222	pushq	%rbp
1223	movq	%rsp, %rbp
1224#ifdef DEBUG
1225	movq	kernelbase(%rip), %rax
1226	cmpq	%rax, %rdi		/* %rdi = from */
1227	jb	0f
1228	cmpq	%rax, %rsi		/* %rsi = to */
1229	jnb	1f
12300:	leaq	.copystr_panic_msg(%rip), %rdi
1231	xorl	%eax, %eax
1232	call	panic
12331:
1234#endif
1235	movq	%gs:CPU_THREAD, %r9
1236	movq	T_LOFAULT(%r9), %r8	/* pass current lofault value as */
1237					/* 5th argument to do_copystr */
1238do_copystr:
1239	movq	%gs:CPU_THREAD, %r9	/* %r9 = thread addr */
1240	movq    T_LOFAULT(%r9), %r11	/* save the current lofault */
1241	movq	%r8, T_LOFAULT(%r9)	/* new lofault */
1242
1243	movq	%rdx, %r8		/* save maxlength */
1244
1245	cmpq	$0, %rdx		/* %rdx = maxlength */
1246	je	copystr_enametoolong	/* maxlength == 0 */
1247
1248copystr_loop:
1249	decq	%r8
1250	movb	(%rdi), %al
1251	incq	%rdi
1252	movb	%al, (%rsi)
1253	incq	%rsi
1254	cmpb	$0, %al
1255	je	copystr_null		/* null char */
1256	cmpq	$0, %r8
1257	jne	copystr_loop
1258
1259copystr_enametoolong:
1260	movl	$ENAMETOOLONG, %eax
1261	jmp	copystr_out
1262
1263copystr_null:
1264	xorl	%eax, %eax		/* no error */
1265
1266copystr_out:
1267	cmpq	$0, %rcx		/* want length? */
1268	je	copystr_done		/* no */
1269	subq	%r8, %rdx		/* compute length and store it */
1270	movq	%rdx, (%rcx)
1271
1272copystr_done:
1273	movq	%r11, T_LOFAULT(%r9)	/* restore the original lofault */
1274	leave
1275	ret
1276	SET_SIZE(copystr)
1277
1278#elif defined(__i386)
1279
1280#define	ARG_FROM	8
1281#define	ARG_TO		12
1282#define	ARG_MAXLEN	16
1283#define	ARG_LENCOPIED	20
1284
1285	ENTRY(copystr)
1286#ifdef DEBUG
1287	pushl	%ebp
1288	movl	%esp, %ebp
1289	movl	kernelbase, %eax
1290	cmpl	%eax, ARG_FROM(%esp)
1291	jb	0f
1292	cmpl	%eax, ARG_TO(%esp)
1293	jnb	1f
12940:	pushl	$.copystr_panic_msg
1295	call	panic
12961:	popl	%ebp
1297#endif
1298	/* get the current lofault address */
1299	movl	%gs:CPU_THREAD, %eax
1300	movl	T_LOFAULT(%eax), %eax
1301do_copystr:
1302	pushl	%ebp			/* setup stack frame */
1303	movl	%esp, %ebp
1304	pushl	%ebx			/* save registers */
1305	pushl	%edi
1306
1307	movl	%gs:CPU_THREAD, %ebx
1308	movl	T_LOFAULT(%ebx), %edi
1309	pushl	%edi			/* save the current lofault */
1310	movl	%eax, T_LOFAULT(%ebx)	/* new lofault */
1311
1312	movl	ARG_MAXLEN(%ebp), %ecx
1313	cmpl	$0, %ecx
1314	je	copystr_enametoolong	/* maxlength == 0 */
1315
1316	movl	ARG_FROM(%ebp), %ebx	/* source address */
1317	movl	ARG_TO(%ebp), %edx	/* destination address */
1318
1319copystr_loop:
1320	decl	%ecx
1321	movb	(%ebx), %al
1322	incl	%ebx
1323	movb	%al, (%edx)
1324	incl	%edx
1325	cmpb	$0, %al
1326	je	copystr_null		/* null char */
1327	cmpl	$0, %ecx
1328	jne	copystr_loop
1329
1330copystr_enametoolong:
1331	movl	$ENAMETOOLONG, %eax
1332	jmp	copystr_out
1333
1334copystr_null:
1335	xorl	%eax, %eax		/* no error */
1336
1337copystr_out:
1338	cmpl	$0, ARG_LENCOPIED(%ebp)	/* want length? */
1339	je	copystr_done		/* no */
1340	movl	ARG_MAXLEN(%ebp), %edx
1341	subl	%ecx, %edx		/* compute length and store it */
1342	movl	ARG_LENCOPIED(%ebp), %ecx
1343	movl	%edx, (%ecx)
1344
1345copystr_done:
1346	popl	%edi
1347	movl	%gs:CPU_THREAD, %ebx
1348	movl	%edi, T_LOFAULT(%ebx)	/* restore the original lofault */
1349
1350	popl	%edi
1351	popl	%ebx
1352	popl	%ebp
1353	ret
1354	SET_SIZE(copystr)
1355
1356#undef	ARG_FROM
1357#undef	ARG_TO
1358#undef	ARG_MAXLEN
1359#undef	ARG_LENCOPIED
1360
1361#endif	/* __i386 */
1362#endif	/* __lint */
1363
1364/*
1365 * Copy a null terminated string from the user address space into
1366 * the kernel address space.
1367 */
1368
1369#if defined(__lint)
1370
1371/* ARGSUSED */
1372int
1373copyinstr(const char *uaddr, char *kaddr, size_t maxlength,
1374    size_t *lencopied)
1375{ return (0); }
1376
1377#else	/* __lint */
1378
1379#if defined(__amd64)
1380
1381	ENTRY(copyinstr)
1382	pushq	%rbp
1383	movq	%rsp, %rbp
1384	subq	$32, %rsp
1385
1386	/*
1387	 * save args in case we trap and need to rerun as a copyop
1388	 */
1389	movq	%rdi, (%rsp)
1390	movq	%rsi, 0x8(%rsp)
1391	movq	%rdx, 0x10(%rsp)
1392	movq	%rcx, 0x18(%rsp)
1393
1394	movq	kernelbase(%rip), %rax
1395#ifdef DEBUG
1396	cmpq	%rax, %rsi		/* %rsi = kaddr */
1397	jnb	1f
1398	leaq	.copyinstr_panic_msg(%rip), %rdi
1399	xorl	%eax, %eax
1400	call	panic
14011:
1402#endif
1403	/*
1404	 * pass lofault value as 5th argument to do_copystr
1405	 */
1406	leaq	_copyinstr_error(%rip), %r8
1407
1408	cmpq	%rax, %rdi		/* test uaddr < kernelbase */
1409	jb	do_copystr
1410	movq	%gs:CPU_THREAD, %r9
1411	jmp	3f
1412
1413_copyinstr_error:
1414	movq	%r11, T_LOFAULT(%r9)	/* restore original lofault */
14153:
1416	movq	T_COPYOPS(%r9), %rax
1417	cmpq	$0, %rax
1418	jz	2f
1419
1420	/*
1421	 * reload args for the copyop
1422	 */
1423	movq	(%rsp), %rdi
1424	movq	0x8(%rsp), %rsi
1425	movq	0x10(%rsp), %rdx
1426	movq	0x18(%rsp), %rcx
1427	leave
1428	jmp	*CP_COPYINSTR(%rax)
1429
14302:	movl	$EFAULT, %eax		/* return EFAULT */
1431	leave
1432	ret
1433	SET_SIZE(copyinstr)
1434
1435#elif defined(__i386)
1436
1437#define	ARG_UADDR	4
1438#define	ARG_KADDR	8
1439
1440	ENTRY(copyinstr)
1441	movl	kernelbase, %ecx
1442#ifdef DEBUG
1443	cmpl	%ecx, ARG_KADDR(%esp)
1444	jnb	1f
1445	pushl	%ebp
1446	movl	%esp, %ebp
1447	pushl	$.copyinstr_panic_msg
1448	call	panic
14491:
1450#endif
1451	lea	_copyinstr_error, %eax
1452	cmpl	%ecx, ARG_UADDR(%esp)	/* test uaddr < kernelbase */
1453	jb	do_copystr
1454	movl	%gs:CPU_THREAD, %edx
1455	jmp	3f
1456
1457_copyinstr_error:
1458	popl	%edi
1459	movl	%gs:CPU_THREAD, %edx
1460	movl	%edi, T_LOFAULT(%edx)	/* original lofault */
1461
1462	popl	%edi
1463	popl	%ebx
1464	popl	%ebp
14653:
1466	movl	T_COPYOPS(%edx), %eax
1467	cmpl	$0, %eax
1468	jz	2f
1469	jmp	*CP_COPYINSTR(%eax)
1470
14712:	movl	$EFAULT, %eax		/* return EFAULT */
1472	ret
1473	SET_SIZE(copyinstr)
1474
1475#undef	ARG_UADDR
1476#undef	ARG_KADDR
1477
1478#endif	/* __i386 */
1479#endif	/* __lint */
1480
1481/*
1482 * Copy a null terminated string from the kernel
1483 * address space to the user address space.
1484 */
1485
1486#if defined(__lint)
1487
1488/* ARGSUSED */
1489int
1490copyoutstr(const char *kaddr, char *uaddr, size_t maxlength,
1491    size_t *lencopied)
1492{ return (0); }
1493
1494#else	/* __lint */
1495
1496#if defined(__amd64)
1497
1498	ENTRY(copyoutstr)
1499	pushq	%rbp
1500	movq	%rsp, %rbp
1501	subq	$32, %rsp
1502
1503	/*
1504	 * save args in case we trap and need to rerun as a copyop
1505	 */
1506	movq	%rdi, (%rsp)
1507	movq	%rsi, 0x8(%rsp)
1508	movq	%rdx, 0x10(%rsp)
1509	movq	%rcx, 0x18(%rsp)
1510
1511	movq	kernelbase(%rip), %rax
1512#ifdef DEBUG
1513	cmpq	%rax, %rdi		/* %rdi = kaddr */
1514	jnb	1f
1515	leaq	.copyoutstr_panic_msg(%rip), %rdi
1516	jmp	call_panic		/* setup stack and call panic */
15171:
1518#endif
1519	/*
1520	 * pass lofault value as 5th argument to do_copystr
1521	 */
1522	leaq	_copyoutstr_error(%rip), %r8
1523
1524	cmpq	%rax, %rsi		/* test uaddr < kernelbase */
1525	jb	do_copystr
1526	movq	%gs:CPU_THREAD, %r9
1527	jmp	3f
1528
1529_copyoutstr_error:
1530	movq	%r11, T_LOFAULT(%r9)	/* restore the original lofault */
15313:
1532	movq	T_COPYOPS(%r9), %rax
1533	cmpq	$0, %rax
1534	jz	2f
1535
1536	/*
1537	 * reload args for the copyop
1538	 */
1539	movq	(%rsp), %rdi
1540	movq	0x8(%rsp), %rsi
1541	movq	0x10(%rsp), %rdx
1542	movq	0x18(%rsp), %rcx
1543	leave
1544	jmp	*CP_COPYOUTSTR(%rax)
1545
15462:	movl	$EFAULT, %eax		/* return EFAULT */
1547	leave
1548	ret
1549	SET_SIZE(copyoutstr)
1550
1551#elif defined(__i386)
1552
1553#define	ARG_KADDR	4
1554#define	ARG_UADDR	8
1555
1556	ENTRY(copyoutstr)
1557	movl	kernelbase, %ecx
1558#ifdef DEBUG
1559	cmpl	%ecx, ARG_KADDR(%esp)
1560	jnb	1f
1561	pushl	%ebp
1562	movl	%esp, %ebp
1563	pushl	$.copyoutstr_panic_msg
1564	call	panic
15651:
1566#endif
1567	lea	_copyoutstr_error, %eax
1568	cmpl	%ecx, ARG_UADDR(%esp)	/* test uaddr < kernelbase */
1569	jb	do_copystr
1570	movl	%gs:CPU_THREAD, %edx
1571	jmp	3f
1572
1573_copyoutstr_error:
1574	popl	%edi
1575	movl	%gs:CPU_THREAD, %edx
1576	movl	%edi, T_LOFAULT(%edx)	/* restore the original lofault */
1577
1578	popl	%edi
1579	popl	%ebx
1580	popl	%ebp
15813:
1582	movl	T_COPYOPS(%edx), %eax
1583	cmpl	$0, %eax
1584	jz	2f
1585	jmp	*CP_COPYOUTSTR(%eax)
1586
15872:	movl	$EFAULT, %eax		/* return EFAULT */
1588	ret
1589	SET_SIZE(copyoutstr)
1590
1591#undef	ARG_KADDR
1592#undef	ARG_UADDR
1593
1594#endif	/* __i386 */
1595#endif	/* __lint */
1596
1597/*
1598 * Since all of the fuword() variants are so similar, we have a macro to spit
1599 * them out.  This allows us to create DTrace-unobservable functions easily.
1600 */
1601
1602#if defined(__lint)
1603
1604#if defined(__amd64)
1605
1606/* ARGSUSED */
1607int
1608fuword64(const void *addr, uint64_t *dst)
1609{ return (0); }
1610
1611#endif
1612
1613/* ARGSUSED */
1614int
1615fuword32(const void *addr, uint32_t *dst)
1616{ return (0); }
1617
1618/* ARGSUSED */
1619int
1620fuword16(const void *addr, uint16_t *dst)
1621{ return (0); }
1622
1623/* ARGSUSED */
1624int
1625fuword8(const void *addr, uint8_t *dst)
1626{ return (0); }
1627
1628#else	/* __lint */
1629
1630#if defined(__amd64)
1631
1632/*
1633 * (Note that we don't save and reload the arguments here
1634 * because their values are not altered in the copy path)
1635 */
1636
1637#define	FUWORD(NAME, INSTR, REG, COPYOP)	\
1638	ENTRY(NAME)				\
1639	movq	%gs:CPU_THREAD, %r9;		\
1640	cmpq	kernelbase(%rip), %rdi;		\
1641	jae	1f;				\
1642	leaq	_flt_/**/NAME, %rdx;		\
1643	movq	%rdx, T_LOFAULT(%r9);		\
1644	INSTR	(%rdi), REG;			\
1645	movq	$0, T_LOFAULT(%r9);		\
1646	INSTR	REG, (%rsi);			\
1647	xorl	%eax, %eax;			\
1648	ret;					\
1649_flt_/**/NAME:					\
1650	movq	$0, T_LOFAULT(%r9);		\
16511:						\
1652	movq	T_COPYOPS(%r9), %rax;		\
1653	cmpq	$0, %rax;			\
1654	jz	2f;				\
1655	jmp	*COPYOP(%rax);			\
16562:						\
1657	movl	$-1, %eax;			\
1658	ret;					\
1659	SET_SIZE(NAME)
1660
1661	FUWORD(fuword64, movq, %rax, CP_FUWORD64)
1662	FUWORD(fuword32, movl, %eax, CP_FUWORD32)
1663	FUWORD(fuword16, movw, %ax, CP_FUWORD16)
1664	FUWORD(fuword8, movb, %al, CP_FUWORD8)
1665
1666#elif defined(__i386)
1667
1668#define	FUWORD(NAME, INSTR, REG, COPYOP)	\
1669	ENTRY(NAME)				\
1670	movl	%gs:CPU_THREAD, %ecx;		\
1671	movl	kernelbase, %eax;		\
1672	cmpl	%eax, 4(%esp);			\
1673	jae	1f;				\
1674	lea	_flt_/**/NAME, %edx;		\
1675	movl	%edx, T_LOFAULT(%ecx);		\
1676	movl	4(%esp), %eax;			\
1677	movl	8(%esp), %edx;			\
1678	INSTR	(%eax), REG;			\
1679	movl	$0, T_LOFAULT(%ecx);		\
1680	INSTR	REG, (%edx);			\
1681	xorl	%eax, %eax;			\
1682	ret;					\
1683_flt_/**/NAME:					\
1684	movl	$0, T_LOFAULT(%ecx);		\
16851:						\
1686	movl	T_COPYOPS(%ecx), %eax;		\
1687	cmpl	$0, %eax;			\
1688	jz	2f;				\
1689	jmp	*COPYOP(%eax);			\
16902:						\
1691	movl	$-1, %eax;			\
1692	ret;					\
1693	SET_SIZE(NAME)
1694
1695	FUWORD(fuword32, movl, %eax, CP_FUWORD32)
1696	FUWORD(fuword16, movw, %ax, CP_FUWORD16)
1697	FUWORD(fuword8, movb, %al, CP_FUWORD8)
1698
1699#endif	/* __i386 */
1700
1701#undef	FUWORD
1702
1703#endif	/* __lint */
1704
1705/*
1706 * Set user word.
1707 */
1708
1709#if defined(__lint)
1710
1711#if defined(__amd64)
1712
1713/* ARGSUSED */
1714int
1715suword64(void *addr, uint64_t value)
1716{ return (0); }
1717
1718#endif
1719
1720/* ARGSUSED */
1721int
1722suword32(void *addr, uint32_t value)
1723{ return (0); }
1724
1725/* ARGSUSED */
1726int
1727suword16(void *addr, uint16_t value)
1728{ return (0); }
1729
1730/* ARGSUSED */
1731int
1732suword8(void *addr, uint8_t value)
1733{ return (0); }
1734
1735#else	/* lint */
1736
1737#if defined(__amd64)
1738
1739/*
1740 * (Note that we don't save and reload the arguments here
1741 * because their values are not altered in the copy path)
1742 */
1743
1744#define	SUWORD(NAME, INSTR, REG, COPYOP)	\
1745	ENTRY(NAME)				\
1746	movq	%gs:CPU_THREAD, %r9;		\
1747	cmpq	kernelbase(%rip), %rdi;		\
1748	jae	1f;				\
1749	leaq	_flt_/**/NAME, %rdx;		\
1750	movq	%rdx, T_LOFAULT(%r9);		\
1751	INSTR	REG, (%rdi);			\
1752	movq	$0, T_LOFAULT(%r9);		\
1753	xorl	%eax, %eax;			\
1754	ret;					\
1755_flt_/**/NAME:					\
1756	movq	$0, T_LOFAULT(%r9);		\
17571:						\
1758	movq	T_COPYOPS(%r9), %rax;		\
1759	cmpq	$0, %rax;			\
1760	jz	3f;				\
1761	jmp	*COPYOP(%rax);			\
17623:						\
1763	movl	$-1, %eax;			\
1764	ret;					\
1765	SET_SIZE(NAME)
1766
1767	SUWORD(suword64, movq, %rsi, CP_SUWORD64)
1768	SUWORD(suword32, movl, %esi, CP_SUWORD32)
1769	SUWORD(suword16, movw, %si, CP_SUWORD16)
1770	SUWORD(suword8, movb, %sil, CP_SUWORD8)
1771
1772#elif defined(__i386)
1773
1774#define	SUWORD(NAME, INSTR, REG, COPYOP)	\
1775	ENTRY(NAME)				\
1776	movl	%gs:CPU_THREAD, %ecx;		\
1777	movl	kernelbase, %eax;		\
1778	cmpl	%eax, 4(%esp);			\
1779	jae	1f;				\
1780	lea	_flt_/**/NAME, %edx;		\
1781	movl	%edx, T_LOFAULT(%ecx);		\
1782	movl	4(%esp), %eax;			\
1783	movl	8(%esp), %edx;			\
1784	INSTR	REG, (%eax);			\
1785	movl	$0, T_LOFAULT(%ecx);		\
1786	xorl	%eax, %eax;			\
1787	ret;					\
1788_flt_/**/NAME:					\
1789	movl	$0, T_LOFAULT(%ecx);		\
17901:						\
1791	movl	T_COPYOPS(%ecx), %eax;		\
1792	cmpl	$0, %eax;			\
1793	jz	3f;				\
1794	movl	COPYOP(%eax), %ecx;		\
1795	jmp	*%ecx;				\
17963:						\
1797	movl	$-1, %eax;			\
1798	ret;					\
1799	SET_SIZE(NAME)
1800
1801	SUWORD(suword32, movl, %edx, CP_SUWORD32)
1802	SUWORD(suword16, movw, %dx, CP_SUWORD16)
1803	SUWORD(suword8, movb, %dl, CP_SUWORD8)
1804
1805#endif	/* __i386 */
1806
1807#undef	SUWORD
1808
1809#endif	/* __lint */
1810
1811#if defined(__lint)
1812
1813#if defined(__amd64)
1814
1815/*ARGSUSED*/
1816void
1817fuword64_noerr(const void *addr, uint64_t *dst)
1818{}
1819
1820#endif
1821
1822/*ARGSUSED*/
1823void
1824fuword32_noerr(const void *addr, uint32_t *dst)
1825{}
1826
1827/*ARGSUSED*/
1828void
1829fuword8_noerr(const void *addr, uint8_t *dst)
1830{}
1831
1832/*ARGSUSED*/
1833void
1834fuword16_noerr(const void *addr, uint16_t *dst)
1835{}
1836
1837#else   /* __lint */
1838
1839#if defined(__amd64)
1840
1841#define	FUWORD_NOERR(NAME, INSTR, REG)		\
1842	ENTRY(NAME)				\
1843	cmpq	kernelbase(%rip), %rdi;		\
1844	cmovnbq	kernelbase(%rip), %rdi;		\
1845	INSTR	(%rdi), REG;			\
1846	INSTR	REG, (%rsi);			\
1847	ret;					\
1848	SET_SIZE(NAME)
1849
1850	FUWORD_NOERR(fuword64_noerr, movq, %rax)
1851	FUWORD_NOERR(fuword32_noerr, movl, %eax)
1852	FUWORD_NOERR(fuword16_noerr, movw, %ax)
1853	FUWORD_NOERR(fuword8_noerr, movb, %al)
1854
1855#elif defined(__i386)
1856
1857#define	FUWORD_NOERR(NAME, INSTR, REG)		\
1858	ENTRY(NAME)				\
1859	movl	4(%esp), %eax;			\
1860	cmpl	kernelbase, %eax;		\
1861	jb	1f;				\
1862	movl	kernelbase, %eax;		\
18631:	movl	8(%esp), %edx;			\
1864	INSTR	(%eax), REG;			\
1865	INSTR	REG, (%edx);			\
1866	ret;					\
1867	SET_SIZE(NAME)
1868
1869	FUWORD_NOERR(fuword32_noerr, movl, %ecx)
1870	FUWORD_NOERR(fuword16_noerr, movw, %cx)
1871	FUWORD_NOERR(fuword8_noerr, movb, %cl)
1872
1873#endif	/* __i386 */
1874
1875#undef	FUWORD_NOERR
1876
1877#endif	/* __lint */
1878
1879#if defined(__lint)
1880
1881#if defined(__amd64)
1882
1883/*ARGSUSED*/
1884void
1885suword64_noerr(void *addr, uint64_t value)
1886{}
1887
1888#endif
1889
1890/*ARGSUSED*/
1891void
1892suword32_noerr(void *addr, uint32_t value)
1893{}
1894
1895/*ARGSUSED*/
1896void
1897suword16_noerr(void *addr, uint16_t value)
1898{}
1899
1900/*ARGSUSED*/
1901void
1902suword8_noerr(void *addr, uint8_t value)
1903{}
1904
1905#else	/* lint */
1906
1907#if defined(__amd64)
1908
1909#define	SUWORD_NOERR(NAME, INSTR, REG)		\
1910	ENTRY(NAME)				\
1911	cmpq	kernelbase(%rip), %rdi;		\
1912	cmovnbq	kernelbase(%rip), %rdi;		\
1913	INSTR	REG, (%rdi);			\
1914	ret;					\
1915	SET_SIZE(NAME)
1916
1917	SUWORD_NOERR(suword64_noerr, movq, %rsi)
1918	SUWORD_NOERR(suword32_noerr, movl, %esi)
1919	SUWORD_NOERR(suword16_noerr, movw, %si)
1920	SUWORD_NOERR(suword8_noerr, movb, %sil)
1921
1922#elif defined(__i386)
1923
1924#define	SUWORD_NOERR(NAME, INSTR, REG)		\
1925	ENTRY(NAME)				\
1926	movl	4(%esp), %eax;			\
1927	cmpl	kernelbase, %eax;		\
1928	jb	1f;				\
1929	movl	kernelbase, %eax;		\
19301:						\
1931	movl	8(%esp), %edx;			\
1932	INSTR	REG, (%eax);			\
1933	ret;					\
1934	SET_SIZE(NAME)
1935
1936	SUWORD_NOERR(suword32_noerr, movl, %edx)
1937	SUWORD_NOERR(suword16_noerr, movw, %dx)
1938	SUWORD_NOERR(suword8_noerr, movb, %dl)
1939
1940#endif	/* __i386 */
1941
1942#undef	SUWORD_NOERR
1943
1944#endif	/* lint */
1945
1946
1947#if defined(__lint)
1948
1949/*ARGSUSED*/
1950int
1951subyte(void *addr, uchar_t value)
1952{ return (0); }
1953
1954/*ARGSUSED*/
1955void
1956subyte_noerr(void *addr, uchar_t value)
1957{}
1958
1959/*ARGSUSED*/
1960int
1961fulword(const void *addr, ulong_t *valuep)
1962{ return (0); }
1963
1964/*ARGSUSED*/
1965void
1966fulword_noerr(const void *addr, ulong_t *valuep)
1967{}
1968
1969/*ARGSUSED*/
1970int
1971sulword(void *addr, ulong_t valuep)
1972{ return (0); }
1973
1974/*ARGSUSED*/
1975void
1976sulword_noerr(void *addr, ulong_t valuep)
1977{}
1978
1979#else
1980
1981	.weak	subyte
1982	subyte=suword8
1983	.weak	subyte_noerr
1984	subyte_noerr=suword8_noerr
1985
1986#if defined(__amd64)
1987
1988	.weak	fulword
1989	fulword=fuword64
1990	.weak	fulword_noerr
1991	fulword_noerr=fuword64_noerr
1992	.weak	sulword
1993	sulword=suword64
1994	.weak	sulword_noerr
1995	sulword_noerr=suword64_noerr
1996
1997#elif defined(__i386)
1998
1999	.weak	fulword
2000	fulword=fuword32
2001	.weak	fulword_noerr
2002	fulword_noerr=fuword32_noerr
2003	.weak	sulword
2004	sulword=suword32
2005	.weak	sulword_noerr
2006	sulword_noerr=suword32_noerr
2007
2008#endif /* __i386 */
2009
2010#endif /* __lint */
2011
2012#if defined(__lint)
2013
2014/*
2015 * Copy a block of storage - must not overlap (from + len <= to).
2016 * No fault handler installed (to be called under on_fault())
2017 */
2018
2019/* ARGSUSED */
2020void
2021copyout_noerr(const void *kfrom, void *uto, size_t count)
2022{}
2023
2024/* ARGSUSED */
2025void
2026copyin_noerr(const void *ufrom, void *kto, size_t count)
2027{}
2028
2029/*
2030 * Zero a block of storage in user space
2031 */
2032
2033/* ARGSUSED */
2034void
2035uzero(void *addr, size_t count)
2036{}
2037
2038/*
2039 * copy a block of storage in user space
2040 */
2041
2042/* ARGSUSED */
2043void
2044ucopy(const void *ufrom, void *uto, size_t ulength)
2045{}
2046
2047#else /* __lint */
2048
2049#if defined(__amd64)
2050
2051	ENTRY(copyin_noerr)
2052	movq	kernelbase(%rip), %rax
2053#ifdef DEBUG
2054	cmpq	%rax, %rsi		/* %rsi = kto */
2055	jae	1f
2056	leaq	.cpyin_ne_pmsg(%rip), %rdi
2057	jmp	call_panic		/* setup stack and call panic */
20581:
2059#endif
2060	cmpq	%rax, %rdi		/* ufrom < kernelbase */
2061	jb	do_copy
2062	movq	%rax, %rdi		/* force fault at kernelbase */
2063	jmp	do_copy
2064	SET_SIZE(copyin_noerr)
2065
2066	ENTRY(copyout_noerr)
2067	movq	kernelbase(%rip), %rax
2068#ifdef DEBUG
2069	cmpq	%rax, %rdi		/* %rdi = kfrom */
2070	jae	1f
2071	leaq	.cpyout_ne_pmsg(%rip), %rdi
2072	jmp	call_panic		/* setup stack and call panic */
20731:
2074#endif
2075	cmpq	%rax, %rsi		/* uto < kernelbase */
2076	jb	do_copy
2077	movq	%rax, %rsi		/* force fault at kernelbase */
2078	jmp	do_copy
2079	SET_SIZE(copyout_noerr)
2080
2081	ENTRY(uzero)
2082	movq	kernelbase(%rip), %rax
2083	cmpq	%rax, %rdi
2084	jb	do_zero
2085	movq	%rax, %rdi	/* force fault at kernelbase */
2086	jmp	do_zero
2087	SET_SIZE(uzero)
2088
2089	ENTRY(ucopy)
2090	movq	kernelbase(%rip), %rax
2091	cmpq	%rax, %rdi
2092	jb	1f
2093	movq	%rax, %rdi
20941:
2095	cmpq	%rax, %rsi
2096	jb	do_copy
2097	movq	%rax, %rsi
2098	jmp	do_copy
2099	SET_SIZE(ucopy)
2100
2101#elif defined(__i386)
2102
2103	ENTRY(copyin_noerr)
2104	movl	kernelbase, %eax
2105#ifdef DEBUG
2106	cmpl	%eax, 8(%esp)
2107	jae	1f
2108	pushl	$.cpyin_ne_pmsg
2109	call	panic
21101:
2111#endif
2112	cmpl	%eax, 4(%esp)
2113	jb	do_copy
2114	movl	%eax, 4(%esp)	/* force fault at kernelbase */
2115	jmp	do_copy
2116	SET_SIZE(copyin_noerr)
2117
2118	ENTRY(copyout_noerr)
2119	movl	kernelbase, %eax
2120#ifdef DEBUG
2121	cmpl	%eax, 4(%esp)
2122	jae	1f
2123	pushl	$.cpyout_ne_pmsg
2124	call	panic
21251:
2126#endif
2127	cmpl	%eax, 8(%esp)
2128	jb	do_copy
2129	movl	%eax, 8(%esp)	/* force fault at kernelbase */
2130	jmp	do_copy
2131	SET_SIZE(copyout_noerr)
2132
2133	ENTRY(uzero)
2134	movl	kernelbase, %eax
2135	cmpl	%eax, 4(%esp)
2136	jb	do_zero
2137	movl	%eax, 4(%esp)	/* force fault at kernelbase */
2138	jmp	do_zero
2139	SET_SIZE(uzero)
2140
2141	ENTRY(ucopy)
2142	movl	kernelbase, %eax
2143	cmpl	%eax, 4(%esp)
2144	jb	1f
2145	movl	%eax, 4(%esp)	/* force fault at kernelbase */
21461:
2147	cmpl	%eax, 8(%esp)
2148	jb	do_copy
2149	movl	%eax, 8(%esp)	/* force fault at kernelbase */
2150	jmp	do_copy
2151	SET_SIZE(ucopy)
2152
2153#endif	/* __i386 */
2154
2155#ifdef DEBUG
2156	.data
2157.kcopy_panic_msg:
2158	.string "kcopy: arguments below kernelbase"
2159.bcopy_panic_msg:
2160	.string "bcopy: arguments below kernelbase"
2161.kzero_panic_msg:
2162        .string "kzero: arguments below kernelbase"
2163.bzero_panic_msg:
2164	.string	"bzero: arguments below kernelbase"
2165.copyin_panic_msg:
2166	.string "copyin: kaddr argument below kernelbase"
2167.xcopyin_panic_msg:
2168	.string	"xcopyin: kaddr argument below kernelbase"
2169.copyout_panic_msg:
2170	.string "copyout: kaddr argument below kernelbase"
2171.xcopyout_panic_msg:
2172	.string	"xcopyout: kaddr argument below kernelbase"
2173.copystr_panic_msg:
2174	.string	"copystr: arguments in user space"
2175.copyinstr_panic_msg:
2176	.string	"copyinstr: kaddr argument not in kernel address space"
2177.copyoutstr_panic_msg:
2178	.string	"copyoutstr: kaddr argument not in kernel address space"
2179.cpyin_ne_pmsg:
2180	.string "copyin_noerr: argument not in kernel address space"
2181.cpyout_ne_pmsg:
2182	.string "copyout_noerr: argument not in kernel address space"
2183#endif
2184
2185#endif	/* __lint */
2186