Lines Matching +full:64 +full:- +full:byte
42 * Pseudo-code to aid in understanding the control flow of the
62 * %o5 = curthread->t_lofault; ! save existing handler in %o5
64 * curthread->t_lofault = .sm_copyerr;
69 * %o5 = curthread->t_lofault; ! save existing handler in %o5
71 * curthread->t_lofault = .copyerr;
79 * %o5 = curthread->t_lofault; ! save existing handler in %o5
80 * curthread->t_lofault = .sm_copyerr;
84 * %o5 = curthread->t_lofault; ! save existing handler in %o5
85 * curthread->t_lofault = .copyerr;
102 * curthread->t_lofault = (%o5 & ~LOFAULT_SET); ! restore old t_lofault
106 * curthread->t_lofault = (%o5 & ~LOFAULT_SET); ! restore old t_lofault
127 * copy with ldx/stx in 4-way unrolled loop;
128 * copy final 0-31 bytes; go to short_exit
130 * if src is word aligned, ld/st words in 32-byte chunks
132 * into long word, store long words in 32-byte chunks
133 * if src is byte aligned, ld byte,half,word parts; pack into long
134 * word, store long words in 32-byte chunks
135 * move final 0-31 bytes according to src alignment; go to short_exit
144 * align dst on 64 byte boundary; use 8-way test for each of 8 possible
145 * src alignments relative to a 64 byte boundary to select the
146 * 16-way unrolled loop (128 bytes) to use for
147 * block load, fmovd, block-init-store, block-store, fmovd operations
152 * align dst on 64 byte boundary; use 8-way test for each of 8 possible
153 * src alignments to nearest long word relative to 64 byte boundary to
154 * select the 8-way unrolled loop (64 bytes) to use for
155 * block load, falign, fmovd, block-store loop
156 * (only use block-init-store when src/dst on 8 byte boundaries.)
170 * For N2/RF, block store places data in the L2 cache, so use or non-use
179 * Less then or equal this number of bytes we will always copy byte-for-byte
307 * to justify using FP/VIS-accelerated memory operations.
316 #define CACHE_LINE 64
317 #define VIS_BLOCKSIZE 64
320 * Size of stack frame in order to accomodate a 64-byte aligned
321 * floating-point register save area and 2 64-bit temp locations.
323 * block-aligned three block buffer in which to save we must reserve
326 * _______________________________________ <-- %fp + STACK_BIAS
331 * | this area is 4 * VIS_BLOCKSIZE. | <-- - SAVED_FPREGS_OFFSET
332 * |-------------------------------------|
333 * | 8 bytes to save %fprs | <-- - SAVED_FPRS_OFFSET
334 * |-------------------------------------|
335 * | 8 bytes to save %gsr | <-- - SAVED_GSR_OFFSET
336 * ---------------------------------------
377 * Used to save and restore in-use fp registers when we want to use FP.
381 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
382 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
391 /* membar #Sync - provided at copy completion */ ;\
392 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
393 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
429 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
558 save %sp, -SA(MINFRAME), %sp
585 * Copy a block of storage - must not overlap (from + len <= to).
608 andcc %o1, 1, %o3 ! is dest byte aligned
633 ldx [%o0-8], %o4
636 stx %o4, [%o1-8]
647 stx %o4, [%o1-8]
671 stw %o4, [%o1-4]
675 ldub [%o0], %o4 ! load one byte
677 stb %o4, [%o1] ! store one byte
678 ldub [%o0+1], %o4 ! load second byte
681 stb %o4, [%o1+1] ! store second byte
682 ldub [%o0+2], %o4 ! load third byte
684 stb %o4, [%o1+2] ! store third byte
692 ldub [%o0], %o4 ! read byte
694 stb %o4, [%o1] ! write byte
698 ldub [%o0-2], %o4
700 stb %o4, [%o1-2]
701 ldub [%o0-1], %o4
703 stb %o4, [%o1-1]
710 .bc_align: ! byte align test in prior branch delay
737 ! dst is 8-byte aligned, src is not
744 bnz,pt %ncc, .bc_med_byte ! go to byte move if not halfword
745 andcc %o0, 0x2, %o3 ! test which byte alignment
749 ldub [%o0], %o4 ! move a byte
757 ldub [%o0], %o4 ! move a half-word (src align unknown)
797 ld [%o0-4], %o4
800 stw %o4, [%o1-4]
810 ld [%o0-4], %o4
813 stw %o4, [%o1-4]
825 stw %o4, [%o1-4]
866 lduh [%o0-8], %o4
867 lduw [%o0-6], %o3
871 lduh [%o0-2], %o4
874 stx %o4, [%o1-8]
892 stx %o4, [%o1-8]
907 stw %o4, [%o1-4]
913 bnz,pt %ncc, .bc_medbh32a ! go to correct byte move
959 ldub [%o0-8], %o4
961 lduh [%o0-7], %o4
964 lduw [%o0-5], %o4
967 ldub [%o0-1], %o4
970 stx %o4, [%o1-8]
992 stx %o4, [%o1-8]
1010 stw %o4, [%o1-4]
1060 ldub [%o0-8], %o4
1062 lduw [%o0-7], %o4
1065 lduh [%o0-3], %o4
1068 ldub [%o0-1], %o4
1071 stx %o4, [%o1-8]
1093 stx %o4, [%o1-8]
1106 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
1146 andcc %i1, 1, %o3 ! is dest byte aligned
1163 ! align dst to 64 byte boundary
1164 andcc %i1, 0x3f, %o3 ! %o3 == 0 means dst is 64 byte aligned
1167 sub %o3, 64, %o3 ! %o3 has negative bytes to move
1176 stx %o4, [%i1-8]
1177 ! Dest is aligned on 16 bytes, src 8 byte aligned
1192 ! Dest is aligned on 64 bytes, src 8 byte aligned
1195 ! to correct 8 byte offset
1234 subcc %o3, 64, %o3
1244 add %i0, 64, %i0
1267 subcc %o3, 64, %o3
1276 add %i0, 64, %i0
1302 subcc %o3, 64, %o3
1310 add %i0, 64, %i0
1338 subcc %o3, 64, %o3
1345 add %i0, 64, %i0
1377 subcc %o3, 64, %o3
1383 add %i0, 64, %i0
1417 subcc %o3, 64, %o3
1422 add %i0, 64, %i0
1459 subcc %o3, 64, %o3
1463 add %i0, 64, %i0
1493 subcc %o3, 64, %o3
1496 add %i0, 64, %i0
1516 ldx [%i0-8], %o4
1519 stx %o4, [%i1-8]
1530 stx %o4, [%i1-8] !
1542 stw %o4, [%i1-4]
1548 ldub [%i0], %o4 ! move a byte
1556 ldub [%i0], %o4 ! move a half-word (src align unknown)
1584 ! Dst is on 8 byte boundary; src is not;
1586 andcc %i1, 0x3f, %o3 ! is dst 64-byte block aligned?
1588 sub %o3, 64, %o3 ! %o3 will be multiple of 8
1589 neg %o3 ! bytes until dest is 64 byte aligned
1593 bnz %ncc, .bc_unalnbyte ! check for byte alignment
1598 ! Src is word aligned, move bytes until dest 64 byte aligned
1611 ! Src is half-word aligned, move bytes until dest 64 byte aligned
1628 ! Src is Byte aligned, move bytes until dest 64 byte aligned
1651 ! Destination is now block (64 byte aligned), src is not 8 byte aligned
1655 add %i2, 64, %i2 ! Insure we don't load beyond
1656 sub %i3, 64, %i3 ! end of source buffer
1663 ! Determine source alignment to correct 8 byte offset
1692 add %o4, 64, %o4
1704 subcc %i3, 64, %i3
1705 add %i1, 64, %i1
1715 add %o4, 64, %o4
1728 subcc %i3, 64, %i3
1729 add %i1, 64, %i1
1740 add %o4, 64, %o4
1754 subcc %i3, 64, %i3
1755 add %i1, 64, %i1
1767 add %o4, 64, %o4
1782 subcc %i3, 64, %i3
1783 add %i1, 64, %i1
1796 add %o4, 64, %o4
1812 subcc %i3, 64, %i3
1813 add %i1, 64, %i1
1827 add %o4, 64, %o4
1844 subcc %i3, 64, %i3
1845 add %i1, 64, %i1
1860 add %o4, 64, %o4
1878 subcc %i3, 64, %i3
1879 add %i1, 64, %i1
1888 add %o4, 64, %o4
1907 subcc %i3, 64, %i3
1908 add %i1, 64, %i1
1913 ! Handle trailing bytes, 64 to 127
1968 ldub [%i0], %o3 ! read byte
1982 stw %o4, [%i1-4]
1989 ldub [%i0], %o4 ! load one byte
1991 stb %o4, [%i1] ! store one byte
1992 ldub [%i0+1], %o4 ! load second byte
1995 stb %o4, [%i1+1] ! store second byte
1996 ldub [%i0+2], %o4 ! load third byte
1997 stb %o4, [%i1+2] ! store third byte
2048 save %sp, -SA(MINFRAME), %sp
2070 * and (dest & ~63) - (src & ~63) can be 3 blocks even if
2071 * src = dest + (64 * 3) + 63.
2087 ! Block (64 bytes) align the destination.
2088 andcc %i0, 0x3f, %i3 ! is dst aligned on a 64 bytes
2091 neg %i3 ! bytes till dst 64 bytes aligned
2095 ! either 8 bytes, 4 bytes, 2 bytes or byte copy.
2157 ! Now Destination is block (64 bytes) aligned
2212 sub %o1, %o0, %o1 ! %o1 right shift = (64 - left shift)
2257 sub %o1, %o0, %o1 ! %o1 right shift = (64 - left shift)
2341 ! Do 8byte ops as long as possible
2435 ! i0 - src address, i1 - dest address, i2 - count
2436 ! i3, i4 - tmps for used generating complete word
2439 ! l1 size in bits of lower part of source word (LS = 32 - US)
2441 ! l3 size in bits of lower part of destination word (LD = 32 - UD)
2445 mov 32, %l5 ! load an oft-needed constant
2455 ldub [%i0], %i3 ! read a byte from source address
2461 sll %i4, 8, %i4 ! make room for next byte
2471 srl %i5, %l3, %i4 ! prepare to write a single byte
2472 stb %i4, [%i1] ! write a byte
2484 sll %i3, %l1, %i3 ! clear upper used byte(s)
2501 sll %i3, %l0, %i3 ! save remaining byte(s)
2516 srl %i4, %l0, %i3 ! prepare to write a single byte
2517 stb %i3, [%i1] ! write a byte
2546 srl %i3, %l0, %i4 ! upper src byte into lower dst byte
2547 stb %i4, [%i1] ! write a byte
2560 ldub [%i0], %i3 ! read a byte from source address
2566 sll %i4, 8, %i4 ! make room for next byte
2569 ! if from address unaligned for double-word moves,
2581 ! source and destination are now double-word aligned
2610 call .alignit ! go word-align it
2616 ! byte copy, works with any alignment
2623 ! differenced byte copy, works with any alignment
2624 ! assumes dest in %i1 and (source - dest) in %i0
2652 stb %o4, [%i1] ! write a byte
2658 ldub [%i0], %o4 ! read next byte
2681 2: cmp %o2, %o3 ! cmp size and abs(from - to)
2705 ldub [%o0 + %o2], %o3 ! get byte at end of src
2721 save %sp, -SA(MINFRAME), %sp
2723 ! %i0 - source address (arg)
2724 ! %i1 - destination address (arg)
2725 ! %i2 - length of region (not arg)
2775 * Transfer data to and from user space -
2781 * DDI/DKI which specifies that they return '-1' on "errors."
2785 * So there's two extremely similar routines - xcopyin() and xcopyout()
2811 * single byte aligned - 256 (hw_copy_limit_1)
2812 * two byte aligned - 512 (hw_copy_limit_2)
2813 * four byte aligned - 1024 (hw_copy_limit_4)
2814 * eight byte aligned - 1024 (hw_copy_limit_8)
2827 * --> We'll use this handler if we end up grabbing a window
2828 * --> before we use block initializing store and quad load ASIs
2831 * always do a byte for byte copy.
2839 * If we don't exceed one of the limits, we store -count in %o3,
2840 * we store the number of chunks (8, 4, 2 or 1 byte) operated
2848 * unless there's only one byte left. We load that as we're
2867 * kaddr - %g2
2868 * uaddr - %g3
2869 * count - %g4
2967 andcc %o1, 1, %o3 ! is dest byte aligned
2993 ldx [%o0-8], %o4
2996 stxa %o4, [%o1-8]%asi
3007 stxa %o4, [%o1-8]%asi
3028 stwa %o4, [%o1-4]%asi
3032 ldub [%o0], %o4 ! load one byte
3034 stba %o4, [%o1]%asi ! store one byte
3035 ldub [%o0+1], %o4 ! load second byte
3038 stba %o4, [%o1+1]%asi ! store second byte
3039 ldub [%o0+2], %o4 ! load third byte
3041 stba %o4, [%o1+2]%asi ! store third byte
3047 ldub [%o0], %o4 ! read byte
3049 stba %o4, [%o1]%asi ! write byte
3053 ldub [%o0-2], %o4
3055 stba %o4, [%o1-2]%asi
3056 ldub [%o0-1], %o4
3058 stba %o4, [%o1-1]%asi
3064 .co_align: ! byte align test in prior branch delay
3091 ! dst is 8-byte aligned, src is not
3098 bnz,pt %ncc, .co_med_byte ! go to byte move if not halfword
3099 andcc %o0, 0x2, %o3 ! test which byte alignment
3103 ldub [%o0], %o4 ! move a byte
3111 ldub [%o0], %o4 ! move a half-word (src align unknown)
3151 ld [%o0-4], %o4
3154 stwa %o4, [%o1-4]%asi
3164 ld [%o0-4], %o4
3167 stwa %o4, [%o1-4]%asi
3179 stwa %o4, [%o1-4]%asi
3222 lduh [%o0-8], %o4
3223 lduw [%o0-6], %o3
3227 lduh [%o0-2], %o4
3230 stxa %o4, [%o1-8]%asi
3248 stxa %o4, [%o1-8]%asi
3263 stwa %o4, [%o1-4]%asi
3271 bnz,pt %ncc, .co_medbh32a ! go to correct byte move
3317 ldub [%o0-8], %o4
3319 lduh [%o0-7], %o4
3322 lduw [%o0-5], %o4
3325 ldub [%o0-1], %o4
3328 stxa %o4, [%o1-8]%asi
3350 stxa %o4, [%o1-8]%asi
3368 stwa %o4, [%o1-4]%asi
3420 ldub [%o0-8], %o4
3422 lduw [%o0-7], %o4
3425 lduh [%o0-3], %o4
3428 ldub [%o0-1], %o4
3431 stxa %o4, [%o1-8]%asi
3453 stxa %o4, [%o1-8]%asi
3474 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
3489 andcc %i1, 1, %o3 ! is dest byte aligned
3502 ! align dst to 64 byte boundary
3503 andcc %i1, 0x3f, %o3 ! %o3 == 0 means dst is 64 byte aligned
3506 sub %o3, 64, %o3 ! %o3 has negative bytes to move
3516 ! Dest is aligned on 16 bytes, src 8 byte aligned
3526 ldx [%i0-8], %o4
3532 ! Dest is aligned on 64 bytes, src 8 byte aligned
3535 ! to correct 8 byte offset
3573 subcc %o3, 64, %o3
3583 add %i0, 64, %i0
3606 subcc %o3, 64, %o3
3615 add %i0, 64, %i0
3641 subcc %o3, 64, %o3
3649 add %i0, 64, %i0
3677 subcc %o3, 64, %o3
3684 add %i0, 64, %i0
3716 subcc %o3, 64, %o3
3722 add %i0, 64, %i0
3756 subcc %o3, 64, %o3
3761 add %i0, 64, %i0
3798 subcc %o3, 64, %o3
3802 add %i0, 64, %i0
3832 subcc %o3, 64, %o3
3835 add %i0, 64, %i0
3855 ldx [%i0-8], %o4
3858 stxa %o4, [%i1-8]%asi
3869 stxa %o4, [%i1-8]%asi
3881 stwa %o4, [%i1-4]%asi
3887 ldub [%i0], %o4 ! move a byte
3895 ldub [%i0], %o4 ! move a half-word (src align unknown)
3924 ! Dst is on 8 byte boundary; src is not;
3926 andcc %i1, 0x3f, %o3 ! is dst 64-byte block aligned?
3928 sub %o3, 64, %o3 ! %o3 will be multiple of 8
3929 neg %o3 ! bytes until dest is 64 byte aligned
3933 bnz %ncc, .co_unalnbyte ! check for byte alignment
3938 ! Src is word aligned, move bytes until dest 64 byte aligned
3951 ! Src is half-word aligned, move bytes until dest 64 byte aligned
3968 ! Src is Byte aligned, move bytes until dest 64 byte aligned
3991 ! Destination is now block (64 byte aligned), src is not 8 byte aligned
3995 add %i2, 64, %i2 ! Insure we don't load beyond
3996 sub %i3, 64, %i3 ! end of source buffer
4003 ! Determine source alignment to correct 8 byte offset
4032 add %o4, 64, %o4
4044 subcc %i3, 64, %i3
4045 add %i1, 64, %i1
4055 add %o4, 64, %o4
4068 subcc %i3, 64, %i3
4069 add %i1, 64, %i1
4080 add %o4, 64, %o4
4094 subcc %i3, 64, %i3
4095 add %i1, 64, %i1
4107 add %o4, 64, %o4
4122 subcc %i3, 64, %i3
4123 add %i1, 64, %i1
4136 add %o4, 64, %o4
4152 subcc %i3, 64, %i3
4153 add %i1, 64, %i1
4167 add %o4, 64, %o4
4184 subcc %i3, 64, %i3
4185 add %i1, 64, %i1
4200 add %o4, 64, %o4
4218 subcc %i3, 64, %i3
4219 add %i1, 64, %i1
4228 add %o4, 64, %o4
4247 subcc %i3, 64, %i3
4248 add %i1, 64, %i1
4253 ! Handle trailing bytes, 64 to 127
4308 ldub [%i0], %o3 ! read byte
4322 stwa %o4, [%i1-4]%asi
4330 ldub [%i0], %o4 ! load one byte
4332 stba %o4, [%i1]%asi ! store one byte
4333 ldub [%i0+1], %o4 ! load second byte
4336 stba %o4, [%i1+1]%asi ! store second byte
4337 ldub [%i0+2], %o4 ! load third byte
4338 stba %o4, [%i1+2]%asi ! store third byte
4363 mov -1, %o0
4404 ldub [%o0 + %o3], %o4 ! load first byte
4437 ! See if we're single byte aligned. If we are, check the
4438 ! limit for single byte copies. If we're smaller or equal,
4439 ! bounce to the byte for byte copy loop. Otherwise do it in
4446 ! Single byte aligned. Do we do it via HW or via
4447 ! byte for byte? Do a quick no memory reference
4457 ! Is HW copy on? If not, do everything byte for byte.
4463 ! If we're less than or equal to the single byte copy limit,
4475 ! 8 byte aligned?
4506 ! Housekeeping for copy loops. Uses same idea as in the byte for
4507 ! byte copy loop above.
4513 srl %o2, 3, %o2 ! Number of 8 byte chunks to copy
4515 ! 4 byte aligned?
4549 srl %o2, 2, %o2 ! Number of 4 byte chunks to copy
4551 ! We must be 2 byte aligned. Off we go.
4574 srl %o2, 1, %o2 ! Number of 2 byte chunks to copy
4579 ! and bounce back to a non-HW assisted copy. This dispatches those
4605 ! eight byte aligned copies end here.
4610 ! Something is left - do it byte for byte.
4613 ldub [%o0 + %o3], %o4 ! load next byte
4615 ! Four byte copy loop. %o2 is the number of 4 byte chunks to copy.
4626 ! four byte aligned copies end here.
4631 ! Something is left. Do it byte for byte.
4634 ldub [%o0 + %o3], %o4 ! load next byte
4636 ! two byte aligned copy loop. %o2 is the number of 2 byte chunks to
4652 ! Deal with the last byte
4672 save %sp, -SA(MINFRAME), %sp
4681 ! Block (64 bytes) align the destination.
4685 neg %i3 ! bytes till dst 64 bytes aligned
4689 ! either 8 bytes, 4 bytes, 2 bytes or byte copy.
4751 ! Now Destination is block (64 bytes) aligned
4759 bz,pn %xcc, .co_blkcpy ! src offset in %o2 (last 4-bits)
4814 sub %o1, %o0, %o1 ! %o1 right shift = (64 - left shift)
4868 sub %o1, %o0, %o1 ! %o1 right shift = (64 - left shift)
4969 ! Do 8byte ops as long as possible
5023 ! Copy the residue as byte copy
5047 mov -1, %o0
5079 add %o0, %o4, %o0 ! start w/last byte
5085 sub %o0, 2, %o0 ! get next byte
5127 andcc %o1, 1, %o3 ! is dest byte aligned
5152 ldxa [%o0-8]%asi, %o4
5155 stx %o4, [%o1-8]
5166 stx %o4, [%o1-8]
5187 stw %o4, [%o1-4]
5191 lduba [%o0]%asi, %o4 ! load one byte
5193 stb %o4, [%o1] ! store one byte
5194 lduba [%o0+1]%asi, %o4 ! load second byte
5197 stb %o4, [%o1+1] ! store second byte
5198 lduba [%o0+2]%asi, %o4 ! load third byte
5200 stb %o4, [%o1+2] ! store third byte
5206 lduba [%o0]%asi, %o4 ! read byte
5208 stb %o4, [%o1] ! write byte
5212 lduba [%o0-2]%asi, %o4
5214 stb %o4, [%o1-2]
5215 lduba [%o0-1]%asi, %o4
5217 stb %o4, [%o1-1]
5250 ! dst is 8-byte aligned, src is not
5257 bnz,pt %ncc, .ci_med_byte ! go to byte move if not halfword
5258 andcc %o0, 0x2, %o3 ! test which byte alignment
5262 lduba [%o0]%asi, %o4 ! move a byte
5270 lduba [%o0]%asi, %o4 ! move a half-word (src align unknown)
5310 lda [%o0-4]%asi, %o4
5313 stw %o4, [%o1-4]
5323 lda [%o0-4]%asi, %o4
5326 stw %o4, [%o1-4]
5338 stw %o4, [%o1-4]
5381 lduha [%o0-8]%asi, %o4
5382 lduwa [%o0-6]%asi, %o3
5386 lduha [%o0-2]%asi, %o4
5389 stx %o4, [%o1-8]
5407 stx %o4, [%o1-8]
5422 stw %o4, [%o1-4]
5430 bnz,pt %ncc, .ci_medbh32a ! go to correct byte move
5476 lduba [%o0-8]%asi, %o4
5478 lduha [%o0-7]%asi, %o4
5481 lduwa [%o0-5]%asi, %o4
5484 lduba [%o0-1]%asi, %o4
5487 stx %o4, [%o1-8]
5509 stx %o4, [%o1-8]
5527 stw %o4, [%o1-4]
5579 lduba [%o0-8]%asi, %o4
5581 lduwa [%o0-7]%asi, %o4
5584 lduha [%o0-3]%asi, %o4
5587 lduba [%o0-1]%asi, %o4
5590 stx %o4, [%o1-8]
5612 stx %o4, [%o1-8]
5636 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
5651 andcc %i1, 1, %o3 ! is dest byte aligned
5664 ! align dst to 64 byte boundary
5665 andcc %i1, 0x3f, %o3 ! %o3 == 0 means dst is 64 byte aligned
5668 sub %o3, 64, %o3 ! %o3 has negative bytes to move
5677 stx %o4, [%i1-8]
5678 ! Dest is aligned on 16 bytes, src 8 byte aligned
5688 ldxa [%i0-8]%asi, %o4
5693 ! Dest is aligned on 64 bytes, src 8 byte aligned
5696 ! to correct 8 byte offset
5734 subcc %o3, 64, %o3
5744 add %i0, 64, %i0
5767 subcc %o3, 64, %o3
5776 add %i0, 64, %i0
5802 subcc %o3, 64, %o3
5810 add %i0, 64, %i0
5838 subcc %o3, 64, %o3
5845 add %i0, 64, %i0
5877 subcc %o3, 64, %o3
5883 add %i0, 64, %i0
5917 subcc %o3, 64, %o3
5922 add %i0, 64, %i0
5959 subcc %o3, 64, %o3
5963 add %i0, 64, %i0
5993 subcc %o3, 64, %o3
5996 add %i0, 64, %i0
6016 ldxa [%i0-8]%asi, %o4
6019 stx %o4, [%i1-8]
6030 stx %o4, [%i1-8] !
6042 stw %o4, [%i1-4]
6048 lduba [%i0]%asi, %o4 ! move a byte
6056 lduba [%i0]%asi, %o4 ! move a half-word (src align unknown)
6085 ! Dst is on 8 byte boundary; src is not;
6087 andcc %i1, 0x3f, %o3 ! is dst 64-byte block aligned?
6089 sub %o3, 64, %o3 ! %o3 will be multiple of 8
6090 neg %o3 ! bytes until dest is 64 byte aligned
6094 bnz %ncc, .ci_unalnbyte ! check for byte alignment
6099 ! Src is word aligned, move bytes until dest 64 byte aligned
6112 ! Src is half-word aligned, move bytes until dest 64 byte aligned
6129 ! Src is Byte aligned, move bytes until dest 64 byte aligned
6152 ! Destination is now block (64 byte aligned), src is not 8 byte aligned
6156 add %i2, 64, %i2 ! Insure we don't load beyond
6157 sub %i3, 64, %i3 ! end of source buffer
6164 ! Determine source alignment to correct 8 byte offset
6193 add %o4, 64, %o4
6205 subcc %i3, 64, %i3
6206 add %i1, 64, %i1
6216 add %o4, 64, %o4
6229 subcc %i3, 64, %i3
6230 add %i1, 64, %i1
6241 add %o4, 64, %o4
6255 subcc %i3, 64, %i3
6256 add %i1, 64, %i1
6268 add %o4, 64, %o4
6283 subcc %i3, 64, %i3
6284 add %i1, 64, %i1
6297 add %o4, 64, %o4
6313 subcc %i3, 64, %i3
6314 add %i1, 64, %i1
6328 add %o4, 64, %o4
6345 subcc %i3, 64, %i3
6346 add %i1, 64, %i1
6361 add %o4, 64, %o4
6379 subcc %i3, 64, %i3
6380 add %i1, 64, %i1
6389 add %o4, 64, %o4
6408 subcc %i3, 64, %i3
6409 add %i1, 64, %i1
6414 ! Handle trailing bytes, 64 to 127
6469 lduba [%i0]%asi, %o3 ! read byte
6483 stw %o4, [%i1-4]
6491 lduba [%i0]%asi, %o4 ! load one byte
6493 stb %o4, [%i1] ! store one byte
6494 lduba [%i0+1]%asi, %o4 ! load second byte
6497 stb %o4, [%i1+1] ! store second byte
6498 lduba [%i0+2]%asi, %o4 ! load third byte
6499 stb %o4, [%i1+2] ! store third byte
6524 mov -1, %o0
6590 ! See if we're single byte aligned. If we are, check the
6591 ! limit for single byte copies. If we're smaller, or equal,
6592 ! bounce to the byte for byte copy loop. Otherwise do it in
6599 ! We're single byte aligned.
6604 ! Is HW copy on? If not do everything byte for byte.
6611 ! go to byte for byte.
6622 ! 8 byte aligned?
6627 ! We're eight byte aligned.
6643 ! Housekeeping for copy loops. Uses same idea as in the byte for
6644 ! byte copy loop above.
6650 srl %o2, 3, %o2 ! Number of 8 byte chunks to copy
6652 ! 4 byte aligned?
6673 ! Housekeeping for copy loops. Uses same idea as in the byte
6674 ! for byte copy loop above.
6680 srl %o2, 2, %o2 ! Number of 4 byte chunks to copy
6683 ! We're two byte aligned. Check for "smallness"
6706 ! Housekeeping for copy loops. Uses same idea as in the byte
6707 ! for byte copy loop above.
6714 srl %o2, 1, %o2 ! Number of 2 byte chunks to copy
6720 ! and bounce back to a non-hw assisted copy. This dispatches
6739 ! Eight byte aligned copies. A steal from the original .small_copyin
6740 ! with modifications. %o2 is number of 8 byte chunks to copy. When
6741 ! done, we examine %o3. If this is < 0, we have 1 - 7 bytes more
6752 ! End of copy loop. Most 8 byte aligned copies end here.
6757 ! Something is left. Do it byte for byte.
6762 ! 4 byte copy loop. %o2 is number of 4 byte chunks to copy.
6772 ! End of copy loop. Most 4 byte aligned copies end here.
6777 ! Something is left. Do it byte for byte.
6782 ! 2 byte aligned copy loop. %o2 is number of 2 byte chunks to
6793 ! End of copy loop. Most 2 byte aligned copies end here.
6798 ! Deal with the last byte
6818 save %sp, -SA(MINFRAME), %sp
6827 ! Block (64 bytes) align the destination.
6831 neg %i3 ! bytes till dst 64 bytes aligned
6835 ! either 8 bytes, 4 bytes, 2 bytes or byte copy.
6904 bz,pn %xcc, .ci_blkcpy ! src offset in %o2 (last 4-bits)
6961 sub %o1, %o0, %o1 ! %o1 right shift = (64 - left shift)
7018 sub %o1, %o0, %o1 ! %o1 right shift = (64 - left shift)
7121 ! Do 8byte ops as long as possible
7175 ! Copy the residue as byte copy
7198 mov -1, %o0
7229 add %o0, %o4, %o0 ! start w/last byte
7235 sub %o0, 2, %o0 ! get next byte
7253 * Copy a block of storage - must not overlap (from + len <= to).
7267 * Copy a block of storage - must not overlap (from + len <= to).
7291 .align 64
7295 * hwblkclr - clears block-aligned, block-multiple-sized regions that are
7302 ! %i0 - start address
7303 ! %i1 - length of region (multiple of 64)
7306 save %sp, -SA(MINFRAME), %sp
7308 ! Must be block-aligned
7318 ! ... and length must be a multiple of 64
7328 restore %g0, 1, %o0 ! return (1) - did not use block operations
7375 ! Check if more than 64 bytes to set
7440 * For less than 15 bytes stores, align the address on 4 byte boundary.
7441 * Then store as many 4-byte chunks, followed by trailing bytes.
7443 * For sizes greater than 15 bytes, align the address on 8 byte boundary.
7445 * store as many 8-bytes chunks to block align the address
7449 * Store as many 8-byte chunks, followed by trailing bytes.
7482 * uzero or bzero was called with t_lofault non-zero.
7510 ! We're here because %o5 was non-zero. It was non-zero
7548 andcc %o0, 7, %o3 ! is add aligned on a 8 byte bound
7550 sub %o3, 8, %o3 ! -(bytes till double aligned)
7580 sub %o3, 0x40, %o3 ! -(bytes till block aligned)
7583 ! Clear -(%o3) bytes till block aligned
7642 ! ... check if 64 bytes to set
7693 stba %g0, [%o0]%asi ! clear a byte
7695 inc %o0 ! next byte
7698 sta %g0, [%o0]%asi ! 4-byte clearing loop
7711 deccc %o1 ! byte clearing loop
7725 ! %o5 and if it is non-zero be sure to clear LOFAULT_SET