xref: /illumos-gate/usr/src/lib/libc/capabilities/sun4v/common/memcpy.S (revision 5d9d9091f564c198a760790b0bfa72c44e17912b)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
 */

	.file	"memcpy.s"

/*
 * memcpy(s1, s2, len)
 *
 * Copy s2 to s1, always copy n bytes.
 * Note: this C code does not work for overlapped copies.
 *       Memmove() and bcopy() do.
 *
 * Added entry __align_cpy_1 is generally for use of the compilers.
 *
 * Fast assembler language version of the following C-program for memcpy
 * which represents the `standard' for the C-library.
 *
 *	void *
 *	memcpy(void *s, const void *s0, size_t n)
 *	{
 *		if (n != 0) {
 *		    char *s1 = s;
 *		    const char *s2 = s0;
 *		    do {
 *			*s1++ = *s2++;
 *		    } while (--n != 0);
 *		}
 *		return (s);
 *	}
 *
 *
 * N1 Flow :
 *
 * if (count < 17) {
 *	Do the byte copy
 *	Return destination address
 * }
 * if (count < 128) {
 *	Is source aligned on word boundary
 *	If no then align source on word boundary then goto .ald
 *	If yes goto .ald
 *	.ald:
 *		Is destination aligned on word boundary
 *		Depending on destination offset (last 2 bits of destination)
 *		copy data by shifting and merging.
 *		Copy residue bytes as byte copy
 *		Return destination address
 * } else {
 *	Align destination on block boundary
 *	Depending on the source offset (last 4 bits of source address) align
 *	the data and store to destination. Both the load and store are done
 *	using ASI_BLK_INIT_ST_QUAD_LDD_P.
 *	For remaining count copy as much data in 8-byte chunk from source to
 *	destination.
 *	Followed by trailing copy using byte copy.
 *	Return saved destination address
 * }
 *
 *
 * N2 Flow :
 * Flow :
 *
 * if (count < 128) {
 *   if count < 3
 *	copy bytes; exit with dst addr
 *   if src & dst aligned on word boundary but not long word boundary,
 *     copy with ldw/stw; branch to finish_up
 *   if src & dst aligned on long word boundary
 *     copy with ldx/stx; branch to finish_up
 *   if src & dst not aligned and length <= 14
 *     copy bytes; exit with dst addr
 *   move enough bytes to get src to word boundary
 *   if dst now on word boundary
 * move_words:
 *     copy words; branch to finish_up
 *   if dst now on half word boundary
 *     load words, shift half words, store words; branch to finish_up
 *   if dst on byte 1
 *     load words, shift 3 bytes, store words; branch to finish_up
 *   if dst on byte 3
 *     load words, shift 1 byte, store words; branch to finish_up
 * finish_up:
 *     copy bytes; exit with dst addr
 * } else {                                         More than 128 bytes
 *   move bytes until dst is on long word boundary
 *   if( src is on long word boundary ) {
 *     if (count < 512) {
 * finish_long:				           src/dst aligned on 8 bytes
 *       copy with ldx/stx in 8-way unrolled loop;
 *       copy final 0-63 bytes; exit with dst addr
 *     } else {                                 src/dst aligned; count > 512
 *       align dst on 64 byte boundary; use 8-way test for each of 8 possible
 *       src alignments relative to a 64 byte boundary to select the
 *       16-way unrolled loop to use for
 *       block load, fmovd, block-init-store, block-store, fmovd operations
 *       then go to finish_long.
 *     }
 *   } else {                                   src/dst not aligned on 8 bytes
 *     if src is word aligned and count < 512
 *       move words in 8-way unrolled loop
 *       move final 0-31 bytes; exit with dst addr
 *     if count < 512
 *       use alignaddr/faligndata combined with ldd/std in 8-way
 *       unrolled loop to move data.
 *       go to unalign_done
 *     else
 *       setup alignaddr for faligndata instructions
 *       align dst on 64 byte boundary; use 8-way test for each of 8 possible
 *       src alignments to nearest long word relative to 64 byte boundary to
 *       select the 8-way unrolled loop to use for
 *       block load, falign, fmovd, block-init-store, block-store loop
 *	 (only use block-init-store when src/dst on 8 byte boundaries.)
 * unalign_done:
 *       move remaining bytes for unaligned cases. exit with dst addr.
 * }
 *
 * Comment on N2 memmove and memcpy common code and block-store-init:
 *   In the man page for memmove, it specifies that copying will take place
 *   correctly between objects that overlap.  For memcpy, behavior is
 *   undefined for objects that overlap.
 *
 *   In rare cases, some multi-threaded applications may attempt to examine
 *   the copy destination buffer during the copy. Using the block-store-init
 *   instruction allows those applications to observe zeros in some
 *   cache lines of the destination buffer for narrow windows. But the
 *   the block-store-init provides memory throughput advantages for many
 *   common applications. To meet both needs, those applications which need
 *   the destination buffer to retain meaning during the copy should use
 *   memmove instead of memcpy.  The memmove version duplicates the memcpy
 *   algorithms except the memmove version does not use block-store-init
 *   in those cases where memcpy does use block-store-init. Otherwise, when
 *   memmove can determine the source and destination do not overlap,
 *   memmove shares the memcpy code.
 */

#include <sys/asm_linkage.h>
#include <sys/niagaraasi.h>
#include <sys/asi.h>
#include <sys/trap.h>

/* documented name for primary block initializing store */
#define	ASI_STBI_P	ASI_BLK_INIT_ST_QUAD_LDD_P

#define	BLOCK_SIZE	64
#define	FPRS_FEF	0x4

#define	SHORTCOPY	3
#define	SHORTCHECK	14
#define	SHORT_LONG	64	/* max copy for short longword-aligned case */
				/* must be at least 32 */
#define	SMALL_MAX	128
#define	MED_UMAX	512	/* max copy for medium un-aligned case */
#define	MED_WMAX	512	/* max copy for medium word-aligned case */
#define	MED_MAX		512	/* max copy for medium longword-aligned case */

#ifdef NIAGARA2_IMPL
#include <sys/sun4asi.h>

#else	/* NIAGARA2_IMPL */
/*
 * This define is to align data for the unaligned source cases.
 * The data1, data2 and data3 is merged into data1 and data2.
 * The data3 is preserved for next merge.
 */
#define	ALIGN_DATA(data1, data2, data3, lshift, rshift, tmp)	\
	sllx	data1, lshift, data1				;\
	srlx	data2, rshift, tmp				;\
	or	data1, tmp, data1				;\
	sllx	data2, lshift, data2				;\
	srlx	data3, rshift, tmp				;\
	or	data2, tmp, data2
/*
 * Align the data. Merge the data1 and data2 into data1.
 */
#define	ALIGN_DATA_EW(data1, data2, lshift, rshift, tmp)	\
	sllx	data1, lshift, data1				;\
	srlx	data2, rshift, tmp				;\
	or	data1, tmp, data1
#endif	/* NIAGARA2_IMPL */


	ANSI_PRAGMA_WEAK(memmove,function)
	ANSI_PRAGMA_WEAK(memcpy,function)

	ENTRY(memmove)
	cmp	%o1, %o0	! if from address is >= to use forward copy
	bgeu,pn	%ncc, .forcpy	! else use backward if ...
	sub	%o0, %o1, %o4	! get difference of two addresses
	cmp	%o2, %o4	! compare size and difference of addresses
	bleu,pn	%ncc, .forcpy	! if size is bigger, do overlapped copy
	add	%o1, %o2, %o5	! get to end of source space

	!
	! an overlapped copy that must be done "backwards"
	!
.chksize:
	cmp	%o2, 8			! less than 8 byte do byte copy
	blu,pt %ncc, 2f			! else continue

	! Now size is bigger than 8
.dbalign:
	add	%o0, %o2, %g1		! get to end of dest space
	andcc	%g1, 7, %o3		! %o3 has bytes till dst 8 bytes aligned
	bz,a,pn	%ncc, .dbbck		! if dst is not 8 byte aligned: align it
	andn	%o2, 7, %o3		! %o3 count is multiple of 8 bytes size
	sub	%o2, %o3, %o2		! update o2 with new count

1:	dec	%o5			! decrement source
	ldub	[%o5], %g1		! load one byte
	deccc	%o3			! decrement count
	bgu,pt	%ncc, 1b		! if not done keep copying
	stb	%g1, [%o5+%o4]		! store one byte into dest
	andncc	%o2, 7, %o3		! %o3 count is multiple of 8 bytes size
	bz,pn	%ncc, 2f		! if size < 8, move to byte copy

	! Now Destination is 8 byte aligned
.dbbck:
	andcc	%o5, 7, %o0		! %o0 has src offset
	bz,a,pn	%ncc, .dbcopybc		! if src is aligned to fast mem move
	sub	%o2, %o3, %o2		! Residue bytes in %o2

.cpy_dbwdbc:				! alignment of src is needed
	sub	%o2, 8, %o2		! set size one loop ahead
	sll	%o0, 3, %g1		! %g1 is left shift
	mov	64, %g5			! init %g5 to be 64
	sub	%g5, %g1, %g5		! %g5 right shift = (64 - left shift)
	sub	%o5, %o0, %o5		! align the src at 8 bytes.
	add	%o4, %o0, %o4		! increase difference between src & dst
	ldx	[%o5], %o1		! load first 8 bytes
	srlx	%o1, %g5, %o1
1:	sub	%o5, 8, %o5		! subtract 8 from src
	ldx	[%o5], %o0		! load 8 byte
	sllx	%o0, %g1, %o3		! shift loaded 8 bytes left into tmp reg
	or	%o1, %o3, %o3		! align data
	stx	%o3, [%o5+%o4]		! store 8 byte
	subcc	%o2, 8, %o2		! subtract 8 byte from size
	bg,pt	%ncc, 1b		! if size > 0 continue
	srlx	%o0, %g5, %o1		! move extra byte for the next use

	srl	%g1, 3, %o0		! retsote %o0 value for alignment
	add	%o5, %o0, %o5		! restore src alignment
	sub	%o4, %o0, %o4		! restore difference between src & dest

	ba	2f			! branch to the trailing byte copy
	add	%o2, 8, %o2		! restore size value

.dbcopybc:				! alignment of src is not needed
1:	sub	%o5, 8, %o5		! subtract from src
	ldx	[%o5], %g1		! load 8 bytes
	subcc	%o3, 8, %o3		! subtract from size
	bgu,pt	%ncc, 1b		! if size is bigger 0 continue
	stx	%g1, [%o5+%o4]		! store 8 bytes to destination

	ba	2f
	nop

.bcbyte:
1:	ldub	[%o5], %g1		! load one byte
	stb	%g1, [%o5+%o4]		! store one byte
2:	deccc	%o2			! decrement size
	bgeu,a,pt %ncc, 1b		! if size is >= 0 continue
	dec	%o5			! decrement from address

.exitbc:				! exit from backward copy
	retl
	add	%o5, %o4, %o0		! restore dest addr

#ifdef NIAGARA2_IMPL
	!
	! Check to see if memmove is large aligned copy
	! If so, use special version of copy that avoids
	! use of block store init
	!
.forcpy:
	cmp	%o2, SMALL_MAX		! check for not small case
	blt,pn	%ncc, .mv_short		! merge with memcpy
	mov	%o0, %g1		! save %o0
	neg	%o0, %o5
	andcc	%o5, 7, %o5		! bytes till DST 8 byte aligned
	brz,pt	%o5, .mv_dst_aligned_on_8

	! %o5 has the bytes to be written in partial store.
	sub	%o2, %o5, %o2
	sub	%o1, %o0, %o1		! %o1 gets the difference
7:					! dst aligning loop
	ldub	[%o1+%o0], %o4		! load one byte
	subcc	%o5, 1, %o5
	stb	%o4, [%o0]
	bgu,pt	%ncc, 7b
	add	%o0, 1, %o0		! advance dst
	add	%o1, %o0, %o1		! restore %o1
.mv_dst_aligned_on_8:
	andcc	%o1, 7, %o5
	brnz,pt	%o5, .src_dst_unaligned_on_8
	prefetch [%o1 + (1 * BLOCK_SIZE)], #one_read

.mv_src_dst_aligned_on_8:
	! check if we are copying MED_MAX or more bytes
	cmp	%o2, MED_MAX		! limit to store buffer size
	bleu,pt	%ncc, .medlong
	prefetch [%o1 + (2 * BLOCK_SIZE)], #one_read

/*
 * The following memmove code mimics the memcpy code for large aligned copies,
 * but does not use the ASI_STBI_P (block initializing store) performance
 * optimization. See memmove rationale section in documentation
 */
.mv_large_align8_copy:			! Src and dst share 8 byte alignment
	rd	%fprs, %g5		! check for unused fp
	! if fprs.fef == 0, set it.
	! Setting it when already set costs more than checking
	andcc	%g5, FPRS_FEF, %g5	! test FEF, fprs.du = fprs.dl = 0
	bz,a	%ncc, 1f
	wr	%g0, FPRS_FEF, %fprs	! fprs.fef = 1
1:
	! align dst to 64 byte boundary
	andcc	%o0, 0x3f, %o3		! %o3 == 0 means dst is 64 byte aligned
	brz,pn	%o3, .mv_aligned_on_64
	sub	%o3, 64, %o3		! %o3 has negative bytes to move
	add	%o2, %o3, %o2		! adjust remaining count
.mv_align_to_64:
	ldx	[%o1], %o4
	add	%o1, 8, %o1		! increment src ptr
	addcc	%o3, 8, %o3
	stx	%o4, [%o0]
	brnz,pt	%o3, .mv_align_to_64
	add	%o0, 8, %o0		! increment dst ptr

.mv_aligned_on_64:
	prefetch [%o1 + (3 * BLOCK_SIZE)], #one_read
	mov	%asi,%o4		! save %asi
	! Determine source alignment
	! to correct 8 byte offset
	andcc	%o1, 0x20, %o3
	brnz,pn	%o3, .mv_align_1
	mov	ASI_BLK_P, %asi		! setup %asi for block load/store
	andcc	%o1, 0x10, %o3
	brnz,pn	%o3, .mv_align_01
	nop
	andcc	%o1, 0x08, %o3
	brz,pn	%o3, .mv_align_000
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read
	ba	.mv_align_001
	nop
.mv_align_01:
	andcc	%o1, 0x08, %o3
	brnz,pn	%o3, .mv_align_011
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read
	ba	.mv_align_010
	nop
.mv_align_1:
	andcc	%o1, 0x10, %o3
	brnz,pn	%o3, .mv_align_11
	nop
	andcc	%o1, 0x08, %o3
	brnz,pn	%o3, .mv_align_101
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read
	ba	.mv_align_100
	nop
.mv_align_11:
	andcc	%o1, 0x08, %o3
	brz,pn	%o3, .mv_align_110
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

.mv_align_111:
! Alignment off by 8 bytes
	ldd	[%o1], %d0
	add	%o1, 8, %o1
	sub	%o2, 8, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.mv_align_111_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */
	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d2
	fmovd	%d18, %d4
	fmovd	%d20, %d6
	fmovd	%d22, %d8
	fmovd	%d24, %d10
	fmovd	%d26, %d12
	fmovd	%d28, %d14
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d0

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d2
	fmovd	%d18, %d4
	fmovd	%d20, %d6
	fmovd	%d22, %d8
	fmovd	%d24, %d10
	fmovd	%d26, %d12
	fmovd	%d28, %d14
	add	%o1, 128, %o1		! increment src
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d30, %d0
	bgt,pt	%ncc, .mv_align_111_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	ba	.remain_stuff
	add	%o0, 8, %o0
	! END OF mv_align_111

.mv_align_110:
! Alignment off by 16 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	add	%o1, 16, %o1
	sub	%o2, 16, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.mv_align_110_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */

	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d4
	fmovd	%d18, %d6
	fmovd	%d20, %d8
	fmovd	%d22, %d10
	fmovd	%d24, %d12
	fmovd	%d26, %d14
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d28, %d0
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d2

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d4
	fmovd	%d18, %d6
	fmovd	%d20, %d8
	fmovd	%d22, %d10
	fmovd	%d24, %d12
	fmovd	%d26, %d14
	add	%o1, 128, %o1		! increment src
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d28, %d0
	fmovd	%d30, %d2
	bgt,pt	%ncc, .mv_align_110_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	ba	.remain_stuff
	add	%o0, 16, %o0
	! END OF mv_align_110

.mv_align_101:
! Alignment off by 24 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	ldd	[%o1+16], %d4
	add	%o1, 24, %o1
	sub	%o2, 24, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.mv_align_101_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */

	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d6
	fmovd	%d18, %d8
	fmovd	%d20, %d10
	fmovd	%d22, %d12
	fmovd	%d24, %d14
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d26, %d0
	fmovd	%d28, %d2
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d4

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d6
	fmovd	%d18, %d8
	fmovd	%d20, %d10
	fmovd	%d22, %d12
	fmovd	%d24, %d14
	add	%o1, 128, %o1		! increment src
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d26, %d0
	fmovd	%d28, %d2
	fmovd	%d30, %d4
	bgt,pt	%ncc, .mv_align_101_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	std	%d4, [%o0+16]
	ba	.remain_stuff
	add	%o0, 24, %o0
	! END OF mv_align_101

.mv_align_100:
! Alignment off by 32 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	ldd	[%o1+16],%d4
	ldd	[%o1+24],%d6
	add	%o1, 32, %o1
	sub	%o2, 32, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.mv_align_100_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */
	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d8
	fmovd	%d18, %d10
	fmovd	%d20, %d12
	fmovd	%d22, %d14
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d24, %d0
	fmovd	%d26, %d2
	fmovd	%d28, %d4
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d6

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d8
	fmovd	%d18, %d10
	fmovd	%d20, %d12
	fmovd	%d22, %d14
	add	%o1, 128, %o1		! increment src
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d24, %d0
	fmovd	%d26, %d2
	fmovd	%d28, %d4
	fmovd	%d30, %d6
	bgt,pt	%ncc, .mv_align_100_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	std	%d4, [%o0+16]
	std	%d6, [%o0+24]
	ba	.remain_stuff
	add	%o0, 32, %o0
	! END OF mv_align_100

.mv_align_011:
! Alignment off by 40 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	ldd	[%o1+16], %d4
	ldd	[%o1+24], %d6
	ldd	[%o1+32], %d8
	add	%o1, 40, %o1
	sub	%o2, 40, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.mv_align_011_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */

	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d10
	fmovd	%d18, %d12
	fmovd	%d20, %d14
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d22, %d0
	fmovd	%d24, %d2
	fmovd	%d26, %d4
	fmovd	%d28, %d6
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d8

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d10
	fmovd	%d18, %d12
	fmovd	%d20, %d14
	add	%o1, 128, %o1		! increment src
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d22, %d0
	fmovd	%d24, %d2
	fmovd	%d26, %d4
	fmovd	%d28, %d6
	fmovd	%d30, %d8
	bgt,pt	%ncc, .mv_align_011_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	std	%d4, [%o0+16]
	std	%d6, [%o0+24]
	std	%d8, [%o0+32]
	ba	.remain_stuff
	add	%o0, 40, %o0
	! END OF mv_align_011

.mv_align_010:
! Alignment off by 48 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	ldd	[%o1+16], %d4
	ldd	[%o1+24], %d6
	ldd	[%o1+32], %d8
	ldd	[%o1+40], %d10
	add	%o1, 48, %o1
	sub	%o2, 48, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.mv_align_010_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */

	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d12
	fmovd	%d18, %d14
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d20, %d0
	fmovd	%d22, %d2
	fmovd	%d24, %d4
	fmovd	%d26, %d6
	fmovd	%d28, %d8
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d10

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d12
	fmovd	%d18, %d14
	add	%o1, 128, %o1	! increment src
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d20, %d0
	fmovd	%d22, %d2
	fmovd	%d24, %d4
	fmovd	%d26, %d6
	fmovd	%d28, %d8
	fmovd	%d30, %d10
	bgt,pt	%ncc, .mv_align_010_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	std	%d4, [%o0+16]
	std	%d6, [%o0+24]
	std	%d8, [%o0+32]
	std	%d10, [%o0+40]
	ba	.remain_stuff
	add	%o0, 48, %o0
	! END OF mv_align_010

.mv_align_001:
! Alignment off by 56 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	ldd	[%o1+16], %d4
	ldd	[%o1+24], %d6
	ldd	[%o1+32], %d8
	ldd	[%o1+40], %d10
	ldd	[%o1+48], %d12
	add	%o1, 56, %o1
	sub	%o2, 56, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.mv_align_001_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */

	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d14
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d18, %d0
	fmovd	%d20, %d2
	fmovd	%d22, %d4
	fmovd	%d24, %d6
	fmovd	%d26, %d8
	fmovd	%d28, %d10
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d12

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d14
	add	%o1, 128, %o1		! increment src
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d18, %d0
	fmovd	%d20, %d2
	fmovd	%d22, %d4
	fmovd	%d24, %d6
	fmovd	%d26, %d8
	fmovd	%d28, %d10
	fmovd	%d30, %d12
	bgt,pt	%ncc, .mv_align_001_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	std	%d4, [%o0+16]
	std	%d6, [%o0+24]
	std	%d8, [%o0+32]
	std	%d10, [%o0+40]
	std	%d12, [%o0+48]
	ba	.remain_stuff
	add	%o0, 56, %o0
	! END OF mv_align_001

.mv_align_000:
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.mv_align_000_loop:
	/* ---- copy line 1 of 2. ---- */
	subcc	%o5, 128, %o5
	ldda	[%o1]%asi,%d0
	stda	%d0,[%o0]%asi
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read

	/* ---- copy line 2 of 2. ---- */
	add	%o0, 64, %o0
	ldda	[%o1+64]%asi,%d0
	add	%o1, 128, %o1		! increment src
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! increment dst
	bgt,pt	%ncc, .mv_align_000_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read
	ba	.remain_stuff
	nop

	! END OF mv_align_000
#else	/* NIAGARA2_IMPL */
#endif	/* NIAGARA2_IMPL */

	SET_SIZE(memmove)

	ENTRY(memcpy)
	ENTRY(__align_cpy_1)
#ifdef NIAGARA2_IMPL
	cmp	%o2, SMALL_MAX		! check for not small case
	bgeu,pn	%ncc, .medium		! go to larger cases
	mov	%o0, %g1		! save %o0
.mv_short:
	cmp	%o2, SHORTCOPY		! check for really short case
	ble,pt	%ncc, .smallfin
	or	%o0, %o1, %o4		! prepare alignment check
	andcc	%o4, 0x3, %o5		! test for alignment
	bz,pt	%ncc, .smallword	! branch to word aligned case
	cmp	%o2, SHORTCHECK
	ble,pt	%ncc, .smallrest
	andcc	%o1, 0x3, %o5		! is src word aligned
	bz,pn	%ncc, .aldst
	cmp	%o5, 2			! is src half-word aligned
	be,pt	%ncc, .s2algn
	cmp	%o5, 3			! src is byte aligned
.s1algn:ldub	[%o1], %o3		! move 1 or 3 bytes to align it
	inc	1, %o1
	stb	%o3, [%o0]		! move a byte to align src
	inc	1, %o0
	bne,pt	%ncc, .s2algn
	dec	%o2
	b	.ald			! now go align dest
	andcc	%o0, 0x3, %o5

.s2algn:lduh	[%o1], %o3		! know src is 2 byte aligned
	inc	2, %o1
	srl	%o3, 8, %o4
	stb	%o4, [%o0]		! have to do bytes,
	stb	%o3, [%o0 + 1]		! don't know dst alignment
	inc	2, %o0
	dec	2, %o2

.aldst:	andcc	%o0, 0x3, %o5		! align the destination address
.ald:	bz,pn	%ncc, .w4cp
	cmp	%o5, 2
	be,pn	%ncc, .w2cp
	cmp	%o5, 3
.w3cp:	lduw	[%o1], %o4
	inc	4, %o1
	srl	%o4, 24, %o5
	stb	%o5, [%o0]
	bne,pt	%ncc, .w1cp
	inc	%o0
	dec	1, %o2
	andn	%o2, 3, %o3		! %o3 is aligned word count
	dec	4, %o3			! avoid reading beyond tail of src
	sub	%o1, %o0, %o1		! %o1 gets the difference

1:	sll	%o4, 8, %g5		! save residual bytes
	lduw	[%o1+%o0], %o4
	deccc	4, %o3
	srl	%o4, 24, %o5		! merge with residual
	or	%o5, %g5, %g5
	st	%g5, [%o0]
	bnz,pt	%ncc, 1b
	inc	4, %o0
	sub	%o1, 3, %o1		! used one byte of last word read
	and	%o2, 3, %o2
	b	7f
	inc	4, %o2

.w1cp:	srl	%o4, 8, %o5
	sth	%o5, [%o0]
	inc	2, %o0
	dec	3, %o2
	andn	%o2, 3, %o3		! %o3 is aligned word count
	dec	4, %o3			! avoid reading beyond tail of src
	sub	%o1, %o0, %o1		! %o1 gets the difference

2:	sll	%o4, 24, %g5		! save residual bytes
	lduw	[%o1+%o0], %o4
	deccc	4, %o3
	srl	%o4, 8, %o5		! merge with residual
	or	%o5, %g5, %g5
	st	%g5, [%o0]
	bnz,pt	%ncc, 2b
	inc	4, %o0
	sub	%o1, 1, %o1		! used three bytes of last word read
	and	%o2, 3, %o2
	b	7f
	inc	4, %o2

.w2cp:	lduw	[%o1], %o4
	inc	4, %o1
	srl	%o4, 16, %o5
	sth	%o5, [%o0]
	inc	2, %o0
	dec	2, %o2
	andn	%o2, 3, %o3		! %o3 is aligned word count
	dec	4, %o3			! avoid reading beyond tail of src
	sub	%o1, %o0, %o1		! %o1 gets the difference

3:	sll	%o4, 16, %g5		! save residual bytes
	lduw	[%o1+%o0], %o4
	deccc	4, %o3
	srl	%o4, 16, %o5		! merge with residual
	or	%o5, %g5, %g5
	st	%g5, [%o0]
	bnz,pt	%ncc, 3b
	inc	4, %o0
	sub	%o1, 2, %o1		! used two bytes of last word read
	and	%o2, 3, %o2
	b	7f
	inc	4, %o2

.w4cp:	andn	%o2, 3, %o3		! %o3 is aligned word count
	sub	%o1, %o0, %o1		! %o1 gets the difference

1:	lduw	[%o1+%o0], %o4		! read from address
	deccc	4, %o3			! decrement count
	st	%o4, [%o0]		! write at destination address
	bgu,pt	%ncc, 1b
	inc	4, %o0			! increment to address
	and	%o2, 3, %o2		! number of leftover bytes, if any

	! simple finish up byte copy, works with any alignment
7:
	add	%o1, %o0, %o1		! restore %o1
.smallrest:
	tst	%o2
	bz,pt	%ncc, .smallx
	cmp	%o2, 4
	blt,pt	%ncc, .smallleft3
	nop
	sub	%o2, 3, %o2
.smallnotalign4:
	ldub	[%o1], %o3		! read byte
	subcc	%o2, 4, %o2		! reduce count by 4
	stb	%o3, [%o0]		! write byte
	ldub	[%o1+1], %o3		! repeat for total of 4 bytes
	add	%o1, 4, %o1		! advance SRC by 4
	stb	%o3, [%o0+1]
	ldub	[%o1-2], %o3
	add	%o0, 4, %o0		! advance DST by 4
	stb	%o3, [%o0-2]
	ldub	[%o1-1], %o3
	bgu,pt	%ncc, .smallnotalign4	! loop til 3 or fewer bytes remain
	stb	%o3, [%o0-1]
	addcc	%o2, 3, %o2		! restore count
	bz,pt	%ncc, .smallx
.smallleft3:				! 1, 2, or 3 bytes remain
	subcc	%o2, 1, %o2
	ldub	[%o1], %o3		! load one byte
	bz,pt	%ncc, .smallx
	stb	%o3, [%o0]		! store one byte
	ldub	[%o1+1], %o3		! load second byte
	subcc	%o2, 1, %o2
	bz,pt	%ncc, .smallx
	stb	%o3, [%o0+1]		! store second byte
	ldub	[%o1+2], %o3		! load third byte
	stb	%o3, [%o0+2]		! store third byte
.smallx:
	retl
	mov	%g1, %o0		! restore %o0

.smallfin:
	tst	%o2
	bnz,pt	%ncc, .smallleft3
	nop
	retl
	mov	%g1, %o0		! restore %o0

	.align 16
.smallwords:
	lduw	[%o1], %o3		! read word
.smallwordx:
	subcc	%o2, 8, %o2		! update count
	stw	%o3, [%o0]		! write word
	add	%o1, 8, %o1		! update SRC
	lduw	[%o1-4], %o3		! read word
	add	%o0, 8, %o0		! update DST
	bgu,pt	%ncc, .smallwords	! loop until done
	stw	%o3, [%o0-4]		! write word
	addcc	%o2, 7, %o2		! restore count
	bz,pt	%ncc, .smallexit	! check for completion
	cmp	%o2, 4			! check for 4 or more bytes left
	blt	%ncc, .smallleft3	! if not, go to finish up
	nop
	lduw	[%o1], %o3
	add	%o1, 4, %o1
	subcc	%o2, 4, %o2
	add	%o0, 4, %o0
	bnz,pt	%ncc, .smallleft3
	stw	%o3, [%o0-4]
	retl
	mov	%g1, %o0		! restore %o0

! 8 or more bytes, src and dest start on word boundary
! %o4 contains or %o0, %o1; %o3 contains first four bytes of src
.smalllong:
	andcc	%o4, 0x7, %o5		! test for long alignment
	bnz,pt	%ncc, .smallwordx	! branch to word aligned case
	cmp	%o2, SHORT_LONG-7
	bge,a	%ncc, .medl64		! if we branch
	sub	%o2,56,%o2		! adjust %o2 to -31 off count
	sub	%o1, %o0, %o1		! %o1 gets the difference
.small_long_l:
	ldx	[%o1+%o0], %o3
	subcc	%o2, 8, %o2
	add	%o0, 8, %o0
	bgu,pt	%ncc, .small_long_l	! loop until done
	stx	%o3, [%o0-8]		! write word
	add	%o1, %o0, %o1		! restore %o1
	addcc	%o2, 7, %o2		! restore %o2 to correct count
	bz,pt	%ncc, .smallexit	! check for completion
	cmp	%o2, 4			! check for 4 or more bytes left
	blt,pt	%ncc, .smallleft3	! if not, go to finish up
	nop
	lduw	[%o1], %o3
	add	%o1, 4, %o1
	subcc	%o2, 4, %o2
	stw	%o3, [%o0]
	add	%o0, 4, %o0
	bnz,pt	%ncc, .smallleft3
	nop
	retl
	mov	%g1, %o0		! restore %o0

	.align 16
! src and dest start on word boundary
.smallword:
	subcc	%o2, 7, %o2		! adjust count
	bgu,pt	%ncc, .smalllong
	lduw	[%o1], %o3		! read word
	addcc	%o2, 3, %o2		! restore count
	bz,pt	%ncc, .smallexit
	stw	%o3, [%o0]		! write word
	deccc	%o2			! reduce count for cc test
	ldub	[%o1+4], %o3		! load one byte
	bz,pt	%ncc, .smallexit
	stb	%o3, [%o0+4]		! store one byte
	ldub	[%o1+5], %o3		! load second byte
	deccc	%o2
	bz,pt	%ncc, .smallexit
	stb	%o3, [%o0+5]		! store second byte
	ldub	[%o1+6], %o3		! load third byte
	stb	%o3, [%o0+6]		! store third byte
.smallexit:
	retl
	mov	%g1, %o0		! restore %o0

	.align 16
.medium:
	neg	%o0, %o5
	andcc	%o5, 7, %o5		! bytes till DST 8 byte aligned
	brz,pt	%o5, .dst_aligned_on_8

	! %o5 has the bytes to be written in partial store.
	sub	%o2, %o5, %o2
	sub	%o1, %o0, %o1		! %o1 gets the difference
7:					! dst aligning loop
	ldub	[%o1+%o0], %o4		! load one byte
	subcc	%o5, 1, %o5
	stb	%o4, [%o0]
	bgu,pt	%ncc, 7b
	add	%o0, 1, %o0		! advance dst
	add	%o1, %o0, %o1		! restore %o1
.dst_aligned_on_8:
	andcc	%o1, 7, %o5
	brnz,pt	%o5, .src_dst_unaligned_on_8
	prefetch [%o1 + (1 * BLOCK_SIZE)], #one_read

.src_dst_aligned_on_8:
	! check if we are copying MED_MAX or more bytes
	cmp	%o2, MED_MAX		! limit to store buffer size
	bgu,pt	%ncc, .large_align8_copy
	prefetch [%o1 + (2 * BLOCK_SIZE)], #one_read
/*
 * Special case for handling when src and dest are both long word aligned
 * and total data to move is less than MED_MAX bytes
 */
.medlong:
	subcc	%o2, 63, %o2		! adjust length to allow cc test
	ble,pt	%ncc, .medl63		! skip big loop if less than 64 bytes
.medl64:
	prefetch [%o1 + (3 * BLOCK_SIZE)], #one_read ! into the l2 cache
	ldx	[%o1], %o4		! load
	subcc	%o2, 64, %o2		! decrement length count
	stx	%o4, [%o0]		! and store
	ldx	[%o1+8], %o3		! a block of 64 bytes
	stx	%o3, [%o0+8]
	ldx	[%o1+16], %o4
	stx	%o4, [%o0+16]
	ldx	[%o1+24], %o3
	stx	%o3, [%o0+24]
	ldx	[%o1+32], %o4		! load
	stx	%o4, [%o0+32]		! and store
	ldx	[%o1+40], %o3		! a block of 64 bytes
	add	%o1, 64, %o1		! increase src ptr by 64
	stx	%o3, [%o0+40]
	ldx	[%o1-16], %o4
	add	%o0, 64, %o0		! increase dst ptr by 64
	stx	%o4, [%o0-16]
	ldx	[%o1-8], %o3
	bgu,pt	%ncc, .medl64		! repeat if at least 64 bytes left
	stx	%o3, [%o0-8]
.medl63:
	addcc	%o2, 32, %o2		! adjust remaining count
	ble,pt	%ncc, .medl31		! to skip if 31 or fewer bytes left
	nop
	ldx	[%o1], %o4		! load
	sub	%o2, 32, %o2		! decrement length count
	stx	%o4, [%o0]		! and store
	ldx	[%o1+8], %o3		! a block of 32 bytes
	add	%o1, 32, %o1		! increase src ptr by 32
	stx	%o3, [%o0+8]
	ldx	[%o1-16], %o4
	add	%o0, 32, %o0		! increase dst ptr by 32
	stx	%o4, [%o0-16]
	ldx	[%o1-8], %o3
	stx	%o3, [%o0-8]
.medl31:
	addcc	%o2, 16, %o2		! adjust remaining count
	ble,pt	%ncc, .medl15		! skip if 15 or fewer bytes left
	nop				!
	ldx	[%o1], %o4		! load and store 16 bytes
	add	%o1, 16, %o1		! increase src ptr by 16
	stx	%o4, [%o0]		!
	sub	%o2, 16, %o2		! decrease count by 16
	ldx	[%o1-8], %o3		!
	add	%o0, 16, %o0		! increase dst ptr by 16
	stx	%o3, [%o0-8]
.medl15:
	addcc	%o2, 15, %o2		! restore count
	bz,pt	%ncc, .smallexit	! exit if finished
	cmp	%o2, 8
	blt,pt	%ncc, .medw7		! skip if 7 or fewer bytes left
	tst	%o2
	ldx	[%o1], %o4		! load 8 bytes
	add	%o1, 8, %o1		! increase src ptr by 8
	add	%o0, 8, %o0		! increase dst ptr by 8
	subcc	%o2, 8, %o2		! decrease count by 8
	bnz,pt	%ncc, .medw7
	stx	%o4, [%o0-8]		! and store 8 bytes
	retl
	mov	%g1, %o0		! restore %o0

	.align 16
.src_dst_unaligned_on_8:
	! DST is 8-byte aligned, src is not
2:
	andcc	%o1, 0x3, %o5		! test word alignment
	bnz,pt	%ncc, .unalignsetup	! branch to skip if not word aligned
	prefetch [%o1 + (2 * BLOCK_SIZE)], #one_read

/*
 * Handle all cases where src and dest are aligned on word
 * boundaries. Use unrolled loops for better performance.
 * This option wins over standard large data move when
 * source and destination is in cache for medium
 * to short data moves.
 */
	cmp	%o2, MED_WMAX		! limit to store buffer size
	bge,pt	%ncc, .unalignrejoin	! otherwise rejoin main loop
	prefetch [%o1 + (3 * BLOCK_SIZE)], #one_read

	subcc	%o2, 31, %o2		! adjust length to allow cc test
					! for end of loop
	ble,pt	%ncc, .medw31		! skip big loop if less than 16
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read
.medw32:
	ld	[%o1], %o4		! move a block of 32 bytes
	stw	%o4, [%o0]
	ld	[%o1+4], %o3
	stw	%o3, [%o0+4]
	ld	[%o1+8], %o4
	stw	%o4, [%o0+8]
	ld	[%o1+12], %o3
	stw	%o3, [%o0+12]
	ld	[%o1+16], %o4
	subcc	%o2, 32, %o2		! decrement length count
	stw	%o4, [%o0+16]
	ld	[%o1+20], %o3
	add	%o1, 32, %o1		! increase src ptr by 32
	stw	%o3, [%o0+20]
	ld	[%o1-8], %o4
	add	%o0, 32, %o0		! increase dst ptr by 32
	stw	%o4, [%o0-8]
	ld	[%o1-4], %o3
	bgu,pt	%ncc, .medw32		! repeat if at least 32 bytes left
	stw	%o3, [%o0-4]
.medw31:
	addcc	%o2, 31, %o2		! restore count

	bz,pt	%ncc, .smallexit	! exit if finished
	nop
	cmp	%o2, 16
	blt,pt	%ncc, .medw15
	nop
	ld	[%o1], %o4		! move a block of 16 bytes
	subcc	%o2, 16, %o2		! decrement length count
	stw	%o4, [%o0]
	ld	[%o1+4], %o3
	add	%o1, 16, %o1		! increase src ptr by 16
	stw	%o3, [%o0+4]
	ld	[%o1-8], %o4
	add	%o0, 16, %o0		! increase dst ptr by 16
	stw	%o4, [%o0-8]
	ld	[%o1-4], %o3
	stw	%o3, [%o0-4]
.medw15:
	bz,pt	%ncc, .smallexit	! exit if finished
	cmp	%o2, 8
	blt,pt	%ncc, .medw7		! skip if 7 or fewer bytes left
	tst	%o2
	ld	[%o1], %o4		! load 4 bytes
	subcc	%o2, 8, %o2		! decrease count by 8
	stw	%o4, [%o0]		! and store 4 bytes
	add	%o1, 8, %o1		! increase src ptr by 8
	ld	[%o1-4], %o3		! load 4 bytes
	add	%o0, 8, %o0		! increase dst ptr by 8
	stw	%o3, [%o0-4]		! and store 4 bytes
	bz,pt	%ncc, .smallexit	! exit if finished
.medw7:					! count is ge 1, less than 8
	cmp	%o2, 4			! check for 4 bytes left
	blt,pt	%ncc, .smallleft3	! skip if 3 or fewer bytes left
	nop				!
	ld	[%o1], %o4		! load 4 bytes
	add	%o1, 4, %o1		! increase src ptr by 4
	add	%o0, 4, %o0		! increase dst ptr by 4
	subcc	%o2, 4, %o2		! decrease count by 4
	bnz	.smallleft3
	stw	%o4, [%o0-4]		! and store 4 bytes
	retl
	mov	%g1, %o0		! restore %o0

	.align	16
.large_align8_copy:			! Src and dst share 8 byte alignment
	rd	%fprs, %g5		! check for unused fp
	! if fprs.fef == 0, set it.
	! Setting it when already set costs more than checking
	andcc	%g5, FPRS_FEF, %g5	! test FEF, fprs.du = fprs.dl = 0
	bz,a	%ncc, 1f
	wr	%g0, FPRS_FEF, %fprs	! fprs.fef = 1
1:
	! align dst to 64 byte boundary
	andcc	%o0, 0x3f, %o3		! %o3 == 0 means dst is 64 byte aligned
	brz,pn	%o3, .aligned_to_64
	andcc	%o0, 8, %o3		! odd long words to move?
	brz,pt	%o3, .aligned_to_16
	nop
	ldx	[%o1], %o4
	sub	%o2, 8, %o2
	add	%o1, 8, %o1		! increment src ptr
	add	%o0, 8, %o0		! increment dst ptr
	stx	%o4, [%o0-8]
.aligned_to_16:
	andcc	%o0, 16, %o3		! pair of long words to move?
	brz,pt	%o3, .aligned_to_32
	nop
	ldx	[%o1], %o4
	sub	%o2, 16, %o2
	stx	%o4, [%o0]
	add	%o1, 16, %o1		! increment src ptr
	ldx	[%o1-8], %o4
	add	%o0, 16, %o0		! increment dst ptr
	stx	%o4, [%o0-8]
.aligned_to_32:
	andcc	%o0, 32, %o3		! four long words to move?
	brz,pt	%o3, .aligned_to_64
	nop
	ldx	[%o1], %o4
	sub	%o2, 32, %o2
	stx	%o4, [%o0]
	ldx	[%o1+8], %o4
	stx	%o4, [%o0+8]
	ldx	[%o1+16], %o4
	stx	%o4, [%o0+16]
	add	%o1, 32, %o1		! increment src ptr
	ldx	[%o1-8], %o4
	add	%o0, 32, %o0		! increment dst ptr
	stx	%o4, [%o0-8]
.aligned_to_64:
	prefetch [%o1 + (3 * BLOCK_SIZE)], #one_read
	mov	%asi,%o4		! save %asi
	! Determine source alignment
	! to correct 8 byte offset
	andcc	%o1, 0x20, %o3
	brnz,pn	%o3, .align_1
	mov	ASI_BLK_P, %asi		! setup %asi for block load/store
	andcc	%o1, 0x10, %o3
	brnz,pn	%o3, .align_01
	nop
	andcc	%o1, 0x08, %o3
	brz,pn	%o3, .align_000
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read
	ba	.align_001
	nop
.align_01:
	andcc	%o1, 0x08, %o3
	brnz,pn	%o3, .align_011
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read
	ba	.align_010
	nop
.align_1:
	andcc	%o1, 0x10, %o3
	brnz,pn	%o3, .align_11
	nop
	andcc	%o1, 0x08, %o3
	brnz,pn	%o3, .align_101
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read
	ba	.align_100
	nop
.align_11:
	andcc	%o1, 0x08, %o3
	brz,pn	%o3, .align_110
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

.align_111:
! Alignment off by 8 bytes
	ldd	[%o1], %d0
	add	%o1, 8, %o1
	sub	%o2, 8, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.align_111_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */
	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d2
	fmovd	%d18, %d4
	fmovd	%d20, %d6
	fmovd	%d22, %d8
	fmovd	%d24, %d10
	fmovd	%d26, %d12
	fmovd	%d28, %d14
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d0

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d2
	fmovd	%d18, %d4
	fmovd	%d20, %d6
	fmovd	%d22, %d8
	fmovd	%d24, %d10
	fmovd	%d26, %d12
	fmovd	%d28, %d14
	add	%o1, 128, %o1		! increment src
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d30, %d0
	bgt,pt	%ncc, .align_111_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	ba	.remain_stuff
	add	%o0, 8, %o0
	! END OF align_111

.align_110:
! Alignment off by 16 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	add	%o1, 16, %o1
	sub	%o2, 16, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.align_110_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */

	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d4
	fmovd	%d18, %d6
	fmovd	%d20, %d8
	fmovd	%d22, %d10
	fmovd	%d24, %d12
	fmovd	%d26, %d14
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d28, %d0
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d2

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d4
	fmovd	%d18, %d6
	fmovd	%d20, %d8
	fmovd	%d22, %d10
	fmovd	%d24, %d12
	fmovd	%d26, %d14
	add	%o1, 128, %o1		! increment src
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d28, %d0
	fmovd	%d30, %d2
	bgt,pt	%ncc, .align_110_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	ba	.remain_stuff
	add	%o0, 16, %o0
	! END OF align_110

.align_101:
! Alignment off by 24 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	ldd	[%o1+16], %d4
	add	%o1, 24, %o1
	sub	%o2, 24, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.align_101_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */

	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d6
	fmovd	%d18, %d8
	fmovd	%d20, %d10
	fmovd	%d22, %d12
	fmovd	%d24, %d14
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d26, %d0
	fmovd	%d28, %d2
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d4

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d6
	fmovd	%d18, %d8
	fmovd	%d20, %d10
	fmovd	%d22, %d12
	fmovd	%d24, %d14
	add	%o1, 128, %o1		! increment src
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d26, %d0
	fmovd	%d28, %d2
	fmovd	%d30, %d4
	bgt,pt	%ncc, .align_101_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	std	%d4, [%o0+16]
	ba	.remain_stuff
	add	%o0, 24, %o0
	! END OF align_101

.align_100:
! Alignment off by 32 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	ldd	[%o1+16],%d4
	ldd	[%o1+24],%d6
	add	%o1, 32, %o1
	sub	%o2, 32, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.align_100_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */
	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d8
	fmovd	%d18, %d10
	fmovd	%d20, %d12
	fmovd	%d22, %d14
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d24, %d0
	fmovd	%d26, %d2
	fmovd	%d28, %d4
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d6

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d8
	fmovd	%d18, %d10
	fmovd	%d20, %d12
	fmovd	%d22, %d14
	add	%o1, 128, %o1		! increment src
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d24, %d0
	fmovd	%d26, %d2
	fmovd	%d28, %d4
	fmovd	%d30, %d6
	bgt,pt	%ncc, .align_100_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	std	%d4, [%o0+16]
	std	%d6, [%o0+24]
	ba	.remain_stuff
	add	%o0, 32, %o0
	! END OF align_100

.align_011:
! Alignment off by 40 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	ldd	[%o1+16], %d4
	ldd	[%o1+24], %d6
	ldd	[%o1+32], %d8
	add	%o1, 40, %o1
	sub	%o2, 40, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.align_011_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */

	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d10
	fmovd	%d18, %d12
	fmovd	%d20, %d14
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d22, %d0
	fmovd	%d24, %d2
	fmovd	%d26, %d4
	fmovd	%d28, %d6
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d8

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d10
	fmovd	%d18, %d12
	fmovd	%d20, %d14
	add	%o1, 128, %o1		! increment src
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d22, %d0
	fmovd	%d24, %d2
	fmovd	%d26, %d4
	fmovd	%d28, %d6
	fmovd	%d30, %d8
	bgt,pt	%ncc, .align_011_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	std	%d4, [%o0+16]
	std	%d6, [%o0+24]
	std	%d8, [%o0+32]
	ba	.remain_stuff
	add	%o0, 40, %o0
	! END OF align_011

.align_010:
! Alignment off by 48 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	ldd	[%o1+16], %d4
	ldd	[%o1+24], %d6
	ldd	[%o1+32], %d8
	ldd	[%o1+40], %d10
	add	%o1, 48, %o1
	sub	%o2, 48, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.align_010_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */

	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d12
	fmovd	%d18, %d14
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d20, %d0
	fmovd	%d22, %d2
	fmovd	%d24, %d4
	fmovd	%d26, %d6
	fmovd	%d28, %d8
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d10

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d12
	fmovd	%d18, %d14
	add	%o1, 128, %o1	! increment src
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d20, %d0
	fmovd	%d22, %d2
	fmovd	%d24, %d4
	fmovd	%d26, %d6
	fmovd	%d28, %d8
	fmovd	%d30, %d10
	bgt,pt	%ncc, .align_010_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	std	%d4, [%o0+16]
	std	%d6, [%o0+24]
	std	%d8, [%o0+32]
	std	%d10, [%o0+40]
	ba	.remain_stuff
	add	%o0, 48, %o0
	! END OF align_010

.align_001:
! Alignment off by 56 bytes
	ldd	[%o1], %d0
	ldd	[%o1+8], %d2
	ldd	[%o1+16], %d4
	ldd	[%o1+24], %d6
	ldd	[%o1+32], %d8
	ldd	[%o1+40], %d10
	ldd	[%o1+48], %d12
	add	%o1, 56, %o1
	sub	%o2, 56, %o2
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.align_001_loop:
	subcc	%o5, 128, %o5
	/* ---- copy line 1 of 2. ---- */

	ldda	[%o1]%asi,%d16		! block load
	fmovd	%d16, %d14
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d18, %d0
	fmovd	%d20, %d2
	fmovd	%d22, %d4
	fmovd	%d24, %d6
	fmovd	%d26, %d8
	fmovd	%d28, %d10
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read
	fmovd	%d30, %d12

	/* ---- copy line 2 of 2. ---- */
	ldda	[%o1+64]%asi,%d16
	fmovd	%d16, %d14
	add	%o1, 128, %o1		! increment src
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! advance dst
	fmovd	%d18, %d0
	fmovd	%d20, %d2
	fmovd	%d22, %d4
	fmovd	%d24, %d6
	fmovd	%d26, %d8
	fmovd	%d28, %d10
	fmovd	%d30, %d12
	bgt,pt	%ncc, .align_001_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	std	%d0, [%o0]
	std	%d2, [%o0+8]
	std	%d4, [%o0+16]
	std	%d6, [%o0+24]
	std	%d8, [%o0+32]
	std	%d10, [%o0+40]
	std	%d12, [%o0+48]
	ba	.remain_stuff
	add	%o0, 56, %o0
	! END OF align_001

.align_000:
	andn	%o2, 0x7f, %o5		! %o5 is multiple of 2*block size
	and	%o2, 0x7f, %o2		! residue bytes in %o2
.align_000_loop:
	/* ---- copy line 1 of 2. ---- */
	subcc	%o5, 128, %o5
	ldda	[%o1]%asi,%d0
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	prefetch [%o1 + (5 * BLOCK_SIZE)], #one_read

	/* ---- copy line 2 of 2. ---- */
	add	%o0, 64, %o0
	ldda	[%o1+64]%asi,%d0
	add	%o1, 128, %o1		! increment src
	stxa	%g0,[%o0]ASI_STBI_P	! block initializing store
	stda	%d0,[%o0]%asi
	add	%o0, 64, %o0		! increment dst
	bgt,pt	%ncc, .align_000_loop
	prefetch [%o1 + (4 * BLOCK_SIZE)], #one_read

	! END OF align_000

.remain_stuff:
	mov	%o4, %asi		! restore %asi
	brnz	%g5, .medlong
	membar	#Sync
	ba	.medlong
	wr	%g5, %g0, %fprs

	.align 16
	! Dst is on 8 byte boundary; src is not; remaining count > SMALL_MAX
.unalignsetup:
	prefetch [%o1 + (3 * BLOCK_SIZE)], #one_read
.unalignrejoin:
	rd	%fprs, %g5		! check for unused fp
	! if fprs.fef == 0, set it.
	! Setting it when already set costs more than checking
	andcc	%g5, FPRS_FEF, %g5	! test FEF, fprs.du = fprs.dl = 0
	bz,a	%ncc, 1f
	wr	%g0, FPRS_FEF, %fprs	! fprs.fef = 1
1:
	cmp	%o2, MED_UMAX		! check for medium unaligned limit
	bge,pt	%ncc,.unalign_large
	nop
	andn	%o2, 0x3f, %o5		! %o5 is multiple of block size
	and	%o2, 0x3f, %o2		! residue bytes in %o2
	cmp	%o2, 8			! Insure we don't load beyond
	bgt	.unalign_adjust		! end of source buffer
	andn	%o1, 0x7, %o4		! %o4 has long word aligned src address
	add	%o2, 64, %o2		! adjust to leave loop
	sub	%o5, 64, %o5		! early if necessary
.unalign_adjust:
	alignaddr %o1, %g0, %g0		! generate %gsr
	add	%o1, %o5, %o1		! advance %o1 to after blocks
	ldd	[%o4], %d0
.unalign_loop:
	ldd	[%o4+8], %d2
	faligndata %d0, %d2, %d16
	ldd	[%o4+16], %d4
	std	%d16, [%o0]
	faligndata %d2, %d4, %d18
	ldd	[%o4+24], %d6
	std	%d18, [%o0+8]
	faligndata %d4, %d6, %d20
	ldd	[%o4+32], %d8
	std	%d20, [%o0+16]
	faligndata %d6, %d8, %d22
	ldd	[%o4+40], %d10
	std	%d22, [%o0+24]
	faligndata %d8, %d10, %d24
	ldd	[%o4+48], %d12
	std	%d24, [%o0+32]
	faligndata %d10, %d12, %d26
	ldd	[%o4+56], %d14
	std	%d26, [%o0+40]
	faligndata %d12, %d14, %d28
	ldd	[%o4+64], %d0
	std	%d28, [%o0+48]
	faligndata %d14, %d0, %d30
	add	%o4, BLOCK_SIZE, %o4
	std	%d30, [%o0+56]
	add	%o0, BLOCK_SIZE, %o0
	subcc	%o5, BLOCK_SIZE, %o5
	bgu,pt	%ncc, .unalign_loop
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	ba	.unalign_done
	nop

.unalign_large:
	andcc	%o0, 0x3f, %o3		! is dst 64-byte block aligned?
	bz	%ncc, .unalignsrc
	sub	%o3, 64, %o3		! %o3 will be multiple of 8
	neg	%o3			! bytes until dest is 64 byte aligned
	sub	%o2, %o3, %o2		! update cnt with bytes to be moved
	! Move bytes according to source alignment
	andcc	%o1, 0x1, %o5
	bnz	%ncc, .unalignbyte	! check for byte alignment
	nop
	andcc	%o1, 2, %o5		! check for half word alignment
	bnz	%ncc, .unalignhalf
	nop
	! Src is word aligned
.unalignword:
	ld	[%o1], %o4		! load 4 bytes
	stw	%o4, [%o0]		! and store 4 bytes
	ld	[%o1+4], %o4		! load 4 bytes
	add	%o1, 8, %o1		! increase src ptr by 8
	stw	%o4, [%o0+4]		! and store 4 bytes
	subcc	%o3, 8, %o3		! decrease count by 8
	bnz	%ncc, .unalignword
	add	%o0, 8, %o0		! increase dst ptr by 8
	ba	.unalignsrc
	nop

	! Src is half-word aligned
.unalignhalf:
	lduh	[%o1], %o4		! load 2 bytes
	sllx	%o4, 32, %o5		! shift left
	lduw	[%o1+2], %o4
	or	%o4, %o5, %o5
	sllx	%o5, 16, %o5
	lduh	[%o1+6], %o4
	or	%o4, %o5, %o5
	stx	%o5, [%o0]
	add	%o1, 8, %o1
	subcc	%o3, 8, %o3
	bnz	%ncc, .unalignhalf
	add	%o0, 8, %o0
	ba	.unalignsrc
	nop

	! Src is Byte aligned
.unalignbyte:
	sub	%o0, %o1, %o0		! share pointer advance
.unalignbyte_loop:
	ldub	[%o1], %o4
	sllx	%o4, 56, %o5
	lduh	[%o1+1], %o4
	sllx	%o4, 40, %o4
	or	%o4, %o5, %o5
	lduh	[%o1+3], %o4
	sllx	%o4, 24, %o4
	or	%o4, %o5, %o5
	lduh	[%o1+5], %o4
	sllx	%o4,  8, %o4
	or	%o4, %o5, %o5
	ldub	[%o1+7], %o4
	or	%o4, %o5, %o5
	stx	%o5, [%o0+%o1]
	subcc	%o3, 8, %o3
	bnz	%ncc, .unalignbyte_loop
	add	%o1, 8, %o1
	add	%o0,%o1, %o0 		! restore pointer

	! Destination is now block (64 byte aligned)
.unalignsrc:
	andn	%o2, 0x3f, %o5		! %o5 is multiple of block size
	and	%o2, 0x3f, %o2		! residue bytes in %o2
	add	%o2, 64, %o2		! Insure we don't load beyond
	sub	%o5, 64, %o5		! end of source buffer

	andn	%o1, 0x3f, %o4		! %o4 has block aligned src address
	prefetch [%o4 + (3 * BLOCK_SIZE)], #one_read
	alignaddr %o1, %g0, %g0		! generate %gsr
	add	%o1, %o5, %o1		! advance %o1 to after blocks
	!
	! Determine source alignment to correct 8 byte offset
	andcc	%o1, 0x20, %o3
	brnz,pn	%o3, .unalign_1
	nop
	andcc	%o1, 0x10, %o3
	brnz,pn	%o3, .unalign_01
	nop
	andcc	%o1, 0x08, %o3
	brz,a	%o3, .unalign_000
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	ba	.unalign_001
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
.unalign_01:
	andcc	%o1, 0x08, %o3
	brnz,a	%o3, .unalign_011
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	ba	.unalign_010
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
.unalign_1:
	andcc	%o1, 0x10, %o3
	brnz,pn	%o3, .unalign_11
	nop
	andcc	%o1, 0x08, %o3
	brnz,a	%o3, .unalign_101
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	ba	.unalign_100
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
.unalign_11:
	andcc	%o1, 0x08, %o3
	brz,pn	%o3, .unalign_110
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read

.unalign_111:
	ldd	[%o4+56], %d14
.unalign_111_loop:
	add	%o4, 64, %o4
	ldda	[%o4]ASI_BLK_P, %d16
	faligndata %d14, %d16, %d48
	faligndata %d16, %d18, %d50
	faligndata %d18, %d20, %d52
	faligndata %d20, %d22, %d54
	faligndata %d22, %d24, %d56
	faligndata %d24, %d26, %d58
	faligndata %d26, %d28, %d60
	faligndata %d28, %d30, %d62
	fmovd	%d30, %d14
	stda	%d48, [%o0]ASI_BLK_P
	subcc	%o5, 64, %o5
	add	%o0, 64, %o0
	bgu,pt	%ncc, .unalign_111_loop
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	ba	.unalign_done
	membar	#Sync

.unalign_110:
	ldd	[%o4+48], %d12
	ldd	[%o4+56], %d14
.unalign_110_loop:
	add	%o4, 64, %o4
	ldda	[%o4]ASI_BLK_P, %d16
	faligndata %d12, %d14, %d48
	faligndata %d14, %d16, %d50
	faligndata %d16, %d18, %d52
	faligndata %d18, %d20, %d54
	faligndata %d20, %d22, %d56
	faligndata %d22, %d24, %d58
	faligndata %d24, %d26, %d60
	faligndata %d26, %d28, %d62
	fmovd	%d28, %d12
	fmovd	%d30, %d14
	stda	%d48, [%o0]ASI_BLK_P
	subcc	%o5, 64, %o5
	add	%o0, 64, %o0
	bgu,pt	%ncc, .unalign_110_loop
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	ba	.unalign_done
	membar	#Sync

.unalign_101:
	ldd	[%o4+40], %d10
	ldd	[%o4+48], %d12
	ldd	[%o4+56], %d14
.unalign_101_loop:
	add	%o4, 64, %o4
	ldda	[%o4]ASI_BLK_P, %d16
	faligndata %d10, %d12, %d48
	faligndata %d12, %d14, %d50
	faligndata %d14, %d16, %d52
	faligndata %d16, %d18, %d54
	faligndata %d18, %d20, %d56
	faligndata %d20, %d22, %d58
	faligndata %d22, %d24, %d60
	faligndata %d24, %d26, %d62
	fmovd	%d26, %d10
	fmovd	%d28, %d12
	fmovd	%d30, %d14
	stda	%d48, [%o0]ASI_BLK_P
	subcc	%o5, 64, %o5
	add	%o0, 64, %o0
	bgu,pt	%ncc, .unalign_101_loop
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	ba	.unalign_done
	membar	#Sync

.unalign_100:
	ldd	[%o4+32], %d8
	ldd	[%o4+40], %d10
	ldd	[%o4+48], %d12
	ldd	[%o4+56], %d14
.unalign_100_loop:
	add	%o4, 64, %o4
	ldda	[%o4]ASI_BLK_P, %d16
	faligndata %d8, %d10, %d48
	faligndata %d10, %d12, %d50
	faligndata %d12, %d14, %d52
	faligndata %d14, %d16, %d54
	faligndata %d16, %d18, %d56
	faligndata %d18, %d20, %d58
	faligndata %d20, %d22, %d60
	faligndata %d22, %d24, %d62
	fmovd	%d24, %d8
	fmovd	%d26, %d10
	fmovd	%d28, %d12
	fmovd	%d30, %d14
	stda	%d48, [%o0]ASI_BLK_P
	subcc	%o5, 64, %o5
	add	%o0, 64, %o0
	bgu,pt	%ncc, .unalign_100_loop
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	ba	.unalign_done
	membar	#Sync

.unalign_011:
	ldd	[%o4+24], %d6
	ldd	[%o4+32], %d8
	ldd	[%o4+40], %d10
	ldd	[%o4+48], %d12
	ldd	[%o4+56], %d14
.unalign_011_loop:
	add	%o4, 64, %o4
	ldda	[%o4]ASI_BLK_P, %d16
	faligndata %d6, %d8, %d48
	faligndata %d8, %d10, %d50
	faligndata %d10, %d12, %d52
	faligndata %d12, %d14, %d54
	faligndata %d14, %d16, %d56
	faligndata %d16, %d18, %d58
	faligndata %d18, %d20, %d60
	faligndata %d20, %d22, %d62
	fmovd	%d22, %d6
	fmovd	%d24, %d8
	fmovd	%d26, %d10
	fmovd	%d28, %d12
	fmovd	%d30, %d14
	stda	%d48, [%o0]ASI_BLK_P
	subcc	%o5, 64, %o5
	add	%o0, 64, %o0
	bgu,pt	%ncc, .unalign_011_loop
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	ba	.unalign_done
	membar	#Sync

.unalign_010:
	ldd	[%o4+16], %d4
	ldd	[%o4+24], %d6
	ldd	[%o4+32], %d8
	ldd	[%o4+40], %d10
	ldd	[%o4+48], %d12
	ldd	[%o4+56], %d14
.unalign_010_loop:
	add	%o4, 64, %o4
	ldda	[%o4]ASI_BLK_P, %d16
	faligndata %d4, %d6, %d48
	faligndata %d6, %d8, %d50
	faligndata %d8, %d10, %d52
	faligndata %d10, %d12, %d54
	faligndata %d12, %d14, %d56
	faligndata %d14, %d16, %d58
	faligndata %d16, %d18, %d60
	faligndata %d18, %d20, %d62
	fmovd	%d20, %d4
	fmovd	%d22, %d6
	fmovd	%d24, %d8
	fmovd	%d26, %d10
	fmovd	%d28, %d12
	fmovd	%d30, %d14
	stda	%d48, [%o0]ASI_BLK_P
	subcc	%o5, 64, %o5
	add	%o0, 64, %o0
	bgu,pt	%ncc, .unalign_010_loop
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	ba	.unalign_done
	membar	#Sync

.unalign_001:
	ldd	[%o4+8], %d2
	ldd	[%o4+16], %d4
	ldd	[%o4+24], %d6
	ldd	[%o4+32], %d8
	ldd	[%o4+40], %d10
	ldd	[%o4+48], %d12
	ldd	[%o4+56], %d14
.unalign_001_loop:
	add	%o4, 64, %o4
	ldda	[%o4]ASI_BLK_P, %d16
	faligndata %d2, %d4, %d48
	faligndata %d4, %d6, %d50
	faligndata %d6, %d8, %d52
	faligndata %d8, %d10, %d54
	faligndata %d10, %d12, %d56
	faligndata %d12, %d14, %d58
	faligndata %d14, %d16, %d60
	faligndata %d16, %d18, %d62
	fmovd	%d18, %d2
	fmovd	%d20, %d4
	fmovd	%d22, %d6
	fmovd	%d24, %d8
	fmovd	%d26, %d10
	fmovd	%d28, %d12
	fmovd	%d30, %d14
	stda	%d48, [%o0]ASI_BLK_P
	subcc	%o5, 64, %o5
	add	%o0, 64, %o0
	bgu,pt	%ncc, .unalign_001_loop
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	ba	.unalign_done
	membar	#Sync

.unalign_000:
	ldda	[%o4]ASI_BLK_P, %d0
.unalign_000_loop:
	add	%o4, 64, %o4
	ldda	[%o4]ASI_BLK_P, %d16
	faligndata %d0, %d2, %d48
	faligndata %d2, %d4, %d50
	faligndata %d4, %d6, %d52
	faligndata %d6, %d8, %d54
	faligndata %d8, %d10, %d56
	faligndata %d10, %d12, %d58
	faligndata %d12, %d14, %d60
	faligndata %d14, %d16, %d62
	fmovd	%d16, %d0
	fmovd	%d18, %d2
	fmovd	%d20, %d4
	fmovd	%d22, %d6
	fmovd	%d24, %d8
	fmovd	%d26, %d10
	fmovd	%d28, %d12
	fmovd	%d30, %d14
	stda	%d48, [%o0]ASI_BLK_P
	subcc	%o5, 64, %o5
	add	%o0, 64, %o0
	bgu,pt	%ncc, .unalign_000_loop
	prefetch [%o4 + (4 * BLOCK_SIZE)], #one_read
	membar	#Sync

.unalign_done:
	! Handle trailing bytes, 64 to 127
	! Dest long word aligned, Src not long word aligned
	cmp	%o2, 15
	bleu	%ncc, .unalign_short

	andn	%o2, 0x7, %o5		! %o5 is multiple of 8
	and	%o2, 0x7, %o2		! residue bytes in %o2
	add	%o2, 8, %o2
	sub	%o5, 8, %o5		! insure we don't load past end of src
	andn	%o1, 0x7, %o4		! %o4 has long word aligned src address
	add	%o1, %o5, %o1		! advance %o1 to after multiple of 8
	ldd	[%o4], %d0		! fetch partial word
.unalign_by8:
	ldd	[%o4+8], %d2
	add	%o4, 8, %o4
	faligndata %d0, %d2, %d16
	subcc	%o5, 8, %o5
	std	%d16, [%o0]
	fmovd	%d2, %d0
	bgu,pt	%ncc, .unalign_by8
	add	%o0, 8, %o0

.unalign_short:
	brnz	%g5, .smallrest
	nop
	ba	.smallrest
	wr	%g5, %g0, %fprs
#else	/* NIAGARA2_IMPL */
.forcpy:
	mov	%o0, %g5		! save des address for return val
	cmp	%o2, 17			! for small counts copy bytes
	bleu,pt	%ncc, .dbytecp
	nop

	cmp	%o2, 0x80		! For lengths less than 128 bytes no
	bleu,pn	%ncc, .no_blkcpy	! copy using ASI_BLK_INIT_ST_QUAD_LDD_P

	/*
	 * Make sure that source and destination buffers are 64 bytes apart.
	 * If they are not, do not use ASI_BLK_INIT_ST_QUAD_LDD_P asi to copy
	 * the data.
	 */
	subcc	%o1, %o0, %o3
	blu	%ncc, .blkalgndst
	cmp	%o3, 0x40		! if src - dst >= 0x40
	bgeu,pt	%ncc, .blkalgndst	! then use ASI_BLK_INIT_ST_QUAD_LDD_P
.no_blkcpy:
	andcc	%o1, 3, %o5		! is src word aligned
	bz,pn	%ncc, .aldst
	cmp	%o5, 2			! is src half-word aligned
	be,pt	%ncc, .s2algn
	cmp	%o5, 3			! src is byte aligned
.s1algn:ldub	[%o1], %o3		! move 1 or 3 bytes to align it
	inc	1, %o1
	stb	%o3, [%g5]		! move a byte to align src
	inc	1, %g5
	bne,pt	%ncc, .s2algn
	dec	%o2
	b	.ald			! now go align dest
	andcc	%g5, 3, %o5

.s2algn:lduh	[%o1], %o3		! know src is 2 byte alinged
	inc	2, %o1
	srl	%o3, 8, %o4
	stb	%o4, [%g5]		! have to do bytes,
	stb	%o3, [%g5 + 1]		! don't know dst alingment
	inc	2, %g5
	dec	2, %o2

.aldst:	andcc	%g5, 3, %o5		! align the destination address
.ald:	bz,pn	%ncc, .w4cp
	cmp	%o5, 2
	bz,pn	%ncc, .w2cp
	cmp	%o5, 3
.w3cp:	lduw	[%o1], %o4
	inc	4, %o1
	srl	%o4, 24, %o5
	stb	%o5, [%g5]
	bne,pt	%ncc, .w1cp
	inc	%g5
	dec	1, %o2
	andn	%o2, 3, %o3		! o3 is aligned word count
	dec	4, %o3			! avoid reading beyond tail of src
	sub	%o1, %g5, %o1		! o1 gets the difference

1:	sll	%o4, 8, %g1		! save residual bytes
	lduw	[%o1+%g5], %o4
	deccc	4, %o3
	srl	%o4, 24, %o5		! merge with residual
	or	%o5, %g1, %g1
	st	%g1, [%g5]
	bnz,pt	%ncc, 1b
	inc	4, %g5
	sub	%o1, 3, %o1		! used one byte of last word read
	and	%o2, 3, %o2
	b	7f
	inc	4, %o2

.w1cp:	srl	%o4, 8, %o5
	sth	%o5, [%g5]
	inc	2, %g5
	dec	3, %o2
	andn	%o2, 3, %o3		! o3 is aligned word count
	dec	4, %o3			! avoid reading beyond tail of src
	sub	%o1, %g5, %o1		! o1 gets the difference

2:	sll	%o4, 24, %g1		! save residual bytes
	lduw	[%o1+%g5], %o4
	deccc	4, %o3
	srl	%o4, 8, %o5		! merge with residual
	or	%o5, %g1, %g1
	st	%g1, [%g5]
	bnz,pt	%ncc, 2b
	inc	4, %g5
	sub	%o1, 1, %o1		! used three bytes of last word read
	and	%o2, 3, %o2
	b	7f
	inc	4, %o2

.w2cp:	lduw	[%o1], %o4
	inc	4, %o1
	srl	%o4, 16, %o5
	sth	%o5, [%g5]
	inc	2, %g5
	dec	2, %o2
	andn	%o2, 3, %o3		! o3 is aligned word count
	dec	4, %o3			! avoid reading beyond tail of src
	sub	%o1, %g5, %o1		! o1 gets the difference

3:	sll	%o4, 16, %g1		! save residual bytes
	lduw	[%o1+%g5], %o4
	deccc	4, %o3
	srl	%o4, 16, %o5		! merge with residual
	or	%o5, %g1, %g1
	st	%g1, [%g5]
	bnz,pt	%ncc, 3b
	inc	4, %g5
	sub	%o1, 2, %o1		! used two bytes of last word read
	and	%o2, 3, %o2
	b	7f
	inc	4, %o2

.w4cp:	andn	%o2, 3, %o3		! o3 is aligned word count
	sub	%o1, %g5, %o1		! o1 gets the difference

1:	lduw	[%o1+%g5], %o4		! read from address
	deccc	4, %o3			! decrement count
	st	%o4, [%g5]		! write at destination address
	bgu,pt	%ncc, 1b
	inc	4, %g5			! increment to address
	b	7f
	and	%o2, 3, %o2		! number of leftover bytes, if any

	!
	! differenced byte copy, works with any alignment
	!
.dbytecp:
	b	7f
	sub	%o1, %g5, %o1		! o1 gets the difference

4:	stb	%o4, [%g5]		! write to address
	inc	%g5			! inc to address
7:	deccc	%o2			! decrement count
	bgeu,a,pt %ncc,4b		! loop till done
	ldub	[%o1+%g5], %o4		! read from address
	retl				! %o0 was preserved
	nop

.blkalgndst:
	save	%sp, -SA(MINFRAME), %sp

	! Block (64 bytes) align the destination.
	andcc	%i0, 0x3f, %i3		! is dst block aligned
	bz	%ncc, .chksrc		! dst already block aligned
	sub	%i3, 0x40, %i3
	neg	%i3			! bytes till dst 64 bytes aligned
	sub	%i2, %i3, %i2		! update i2 with new count

	! Based on source and destination alignment do
	! either 8 bytes, 4 bytes, 2 bytes or byte copy.

	! Is dst & src 8B aligned
	or	%i0, %i1, %o2
	andcc	%o2, 0x7, %g0
	bz	%ncc, .alewdcp
	nop

	! Is dst & src 4B aligned
	andcc	%o2, 0x3, %g0
	bz	%ncc, .alwdcp
	nop

	! Is dst & src 2B aligned
	andcc	%o2, 0x1, %g0
	bz	%ncc, .alhlfwdcp
	nop

	! 1B aligned
1:	ldub	[%i1], %o2
	stb	%o2, [%i0]
	inc	%i1
	deccc	%i3
	bgu,pt	%ncc, 1b
	inc	%i0

	ba	.chksrc
	nop

	! dst & src 4B aligned
.alwdcp:
	ld	[%i1], %o2
	st	%o2, [%i0]
	add	%i1, 0x4, %i1
	subcc	%i3, 0x4, %i3
	bgu,pt	%ncc, .alwdcp
	add	%i0, 0x4, %i0

	ba	.chksrc
	nop

	! dst & src 2B aligned
.alhlfwdcp:
	lduh	[%i1], %o2
	stuh	%o2, [%i0]
	add	%i1, 0x2, %i1
	subcc	%i3, 0x2, %i3
	bgu,pt	%ncc, .alhlfwdcp
	add	%i0, 0x2, %i0

	ba	.chksrc
	nop

	! dst & src 8B aligned
.alewdcp:
	ldx	[%i1], %o2
	stx	%o2, [%i0]
	add	%i1, 0x8, %i1
	subcc	%i3, 0x8, %i3
	bgu,pt	%ncc, .alewdcp
	add	%i0, 0x8, %i0

	! Now Destination is block (64 bytes) aligned
.chksrc:
	andn	%i2, 0x3f, %i3		! %i3 count is multiple of block size
	sub	%i2, %i3, %i2		! Residue bytes in %i2
	mov	ASI_BLK_INIT_ST_QUAD_LDD_P, %asi
	andcc	%i1, 0xf, %l1		! is src quadword aligned
	bz,pn	%ncc, .blkcpy		! src offset in %l1
	nop
	cmp	%l1, 0x8
	bgu	%ncc, .cpy_upper_double
	nop
	blu	%ncc, .cpy_lower_double
	nop

	! Falls through when source offset is equal to 8 i.e.
	! source is double word aligned.
	! In this case no shift/merge of data is required
	sub	%i1, %l1, %i1		! align the src at 16 bytes.
	andn	%i1, 0x3f, %o0		! %o0 has block aligned source
	prefetch [%o0+0x0], #one_read
	ldda	[%i1+0x0]%asi, %o2
loop0:
	ldda	[%i1+0x10]%asi, %o4
	prefetch [%o0+0x40], #one_read

	stxa	%o3, [%i0+0x0]%asi
	stxa	%o4, [%i0+0x8]%asi

	ldda	[%i1+0x20]%asi, %o2
	stxa	%o5, [%i0+0x10]%asi
	stxa	%o2, [%i0+0x18]%asi

	ldda	[%i1+0x30]%asi, %o4
	stxa	%o3, [%i0+0x20]%asi
	stxa	%o4, [%i0+0x28]%asi

	ldda	[%i1+0x40]%asi, %o2
	stxa	%o5, [%i0+0x30]%asi
	stxa	%o2, [%i0+0x38]%asi

	add	%o0, 0x40, %o0
	add	%i1, 0x40, %i1
	subcc	%i3, 0x40, %i3
	bgu,pt	%ncc, loop0
	add	%i0, 0x40, %i0
	ba	.blkdone
	add	%i1, %l1, %i1		! increment the source by src offset

.cpy_lower_double:
	sub	%i1, %l1, %i1		! align the src at 16 bytes.
	sll	%l1, 3, %l2		! %l2 left shift
	mov	0x40, %l3
	sub	%l3, %l2, %l3		! %l3 right shift = (64 - left shift)
	andn	%i1, 0x3f, %o0		! %o0 has block aligned source
	prefetch [%o0+0x0], #one_read
	ldda	[%i1+0x0]%asi, %o2	! partial data in %o2 and %o3 has
					! complete data
loop1:
	ldda	[%i1+0x10]%asi, %o4	! %o4 has partial data for this read.
	ALIGN_DATA(%o2, %o3, %o4, %l2, %l3, %g1)	! merge %o2, %o3 and %o4
							! into %o2 and %o3
	prefetch [%o0+0x40], #one_read
	stxa	%o2, [%i0+0x0]%asi
	stxa	%o3, [%i0+0x8]%asi

	ldda	[%i1+0x20]%asi, %o2
	ALIGN_DATA(%o4, %o5, %o2, %l2, %l3, %g1)	! merge %o2 with %o5 and
	stxa	%o4, [%i0+0x10]%asi			! %o4 from previous read
	stxa	%o5, [%i0+0x18]%asi			! into %o4 and %o5

	! Repeat the same for next 32 bytes.

	ldda	[%i1+0x30]%asi, %o4
	ALIGN_DATA(%o2, %o3, %o4, %l2, %l3, %g1)
	stxa	%o2, [%i0+0x20]%asi
	stxa	%o3, [%i0+0x28]%asi

	ldda	[%i1+0x40]%asi, %o2
	ALIGN_DATA(%o4, %o5, %o2, %l2, %l3, %g1)
	stxa	%o4, [%i0+0x30]%asi
	stxa	%o5, [%i0+0x38]%asi

	add	%o0, 0x40, %o0
	add	%i1, 0x40, %i1
	subcc	%i3, 0x40, %i3
	bgu,pt	%ncc, loop1
	add	%i0, 0x40, %i0
	ba	.blkdone
	add	%i1, %l1, %i1		! increment the source by src offset

.cpy_upper_double:
	sub	%i1, %l1, %i1		! align the src at 16 bytes.
	mov	0x8, %l2
	sub	%l1, %l2, %l2
	sll	%l2, 3, %l2		! %l2 left shift
	mov	0x40, %l3
	sub	%l3, %l2, %l3		! %l3 right shift = (64 - left shift)
	andn	%i1, 0x3f, %o0		! %o0 has block aligned source
	prefetch [%o0+0x0], #one_read
	ldda	[%i1+0x0]%asi, %o2	! partial data in %o3 for this read and
					! no data in %o2
loop2:
	ldda	[%i1+0x10]%asi, %o4	! %o4 has complete data and %o5 has
					! partial
	ALIGN_DATA(%o3, %o4, %o5, %l2, %l3, %g1)	! merge %o3, %o4 and %o5
							! into %o3 and %o4
	prefetch [%o0+0x40], #one_read
	stxa	%o3, [%i0+0x0]%asi
	stxa	%o4, [%i0+0x8]%asi

	ldda	[%i1+0x20]%asi, %o2
	ALIGN_DATA(%o5, %o2, %o3, %l2, %l3, %g1)	! merge %o2 and %o3 with
	stxa	%o5, [%i0+0x10]%asi			! %o5 from previous read
	stxa	%o2, [%i0+0x18]%asi			! into %o5 and %o2

	! Repeat the same for next 32 bytes.

	ldda	[%i1+0x30]%asi, %o4
	ALIGN_DATA(%o3, %o4, %o5, %l2, %l3, %g1)
	stxa	%o3, [%i0+0x20]%asi
	stxa	%o4, [%i0+0x28]%asi

	ldda	[%i1+0x40]%asi, %o2
	ALIGN_DATA(%o5, %o2, %o3, %l2, %l3, %g1)
	stxa	%o5, [%i0+0x30]%asi
	stxa	%o2, [%i0+0x38]%asi

	add	%o0, 0x40, %o0
	add	%i1, 0x40, %i1
	subcc	%i3, 0x40, %i3
	bgu,pt	%ncc, loop2
	add	%i0, 0x40, %i0
	ba	.blkdone
	add	%i1, %l1, %i1		! increment the source by src offset

	! Do fast copy using ASI_BLK_INIT_ST_QUAD_LDD_P
.blkcpy:
	andn	%i1, 0x3f, %o0		! %o0 has block aligned source
	prefetch [%o0+0x0], #one_read
1:
	prefetch [%o0+0x40], #one_read

	ldda	[%i1+0x0]%asi, %o2
	ldda	[%i1+0x10]%asi, %o4

	stxa	%o2, [%i0+0x0]%asi
	stxa	%o3, [%i0+0x8]%asi
	stxa	%o4, [%i0+0x10]%asi
	stxa	%o5, [%i0+0x18]%asi

	ldda	[%i1+0x20]%asi, %o2
	ldda	[%i1+0x30]%asi, %o4

	stxa	%o2, [%i0+0x20]%asi
	stxa	%o3, [%i0+0x28]%asi
	stxa	%o4, [%i0+0x30]%asi
	stxa	%o5, [%i0+0x38]%asi

	add	%o0, 0x40, %o0
	add	%i1, 0x40, %i1
	subcc	%i3, 0x40, %i3
	bgu,pt	%ncc, 1b
	add	%i0, 0x40, %i0

.blkdone:
	membar	#Sync

	mov	ASI_PNF, %asi		! restore %asi to default
					! ASI_PRIMARY_NOFAULT value
	tst	%i2
	bz,pt	%ncc, .blkexit
	nop

	! Handle trailing bytes
	cmp	%i2, 0x8
	blu,pt	%ncc, .residue
	nop

	! Can we do some 8B ops
	or	%i1, %i0, %o2
	andcc	%o2, 0x7, %g0
	bnz	%ncc, .last4
	nop

	! Do 8byte ops as long as possible
.last8:
	ldx	[%i1], %o2
	stx	%o2, [%i0]
	add	%i1, 0x8, %i1
	sub	%i2, 0x8, %i2
	cmp	%i2, 0x8
	bgu,pt	%ncc, .last8
	add	%i0, 0x8, %i0

	tst	%i2
	bz,pt	%ncc, .blkexit
	nop

	ba	.residue
	nop

.last4:
	! Can we do 4B ops
	andcc	%o2, 0x3, %g0
	bnz	%ncc, .last2
	nop
1:
	ld	[%i1], %o2
	st	%o2, [%i0]
	add	%i1, 0x4, %i1
	sub	%i2, 0x4, %i2
	cmp	%i2, 0x4
	bgu,pt	%ncc, 1b
	add	%i0, 0x4, %i0

	cmp	%i2, 0
	bz,pt	%ncc, .blkexit
	nop

	ba	.residue
	nop

.last2:
	! Can we do 2B ops
	andcc	%o2, 0x1, %g0
	bnz	%ncc, .residue
	nop

1:
	lduh	[%i1], %o2
	stuh	%o2, [%i0]
	add	%i1, 0x2, %i1
	sub	%i2, 0x2, %i2
	cmp	%i2, 0x2
	bgu,pt	%ncc, 1b
	add	%i0, 0x2, %i0

	cmp	%i2, 0
	bz,pt	%ncc, .blkexit
	nop

.residue:
	ldub	[%i1], %o2
	stb	%o2, [%i0]
	inc	%i1
	deccc	%i2
	bgu,pt	%ncc, .residue
	inc	%i0

.blkexit:

	ret
	restore	%g5, %g0, %o0

#endif	/* NIAGARA2_IMPL */
	SET_SIZE(memcpy)
	SET_SIZE(__align_cpy_1)