1 /* Extracted from GLIBC memcpy.c and memcopy.h, which is: 2 Copyright (C) 1991, 1992, 1993, 1997, 2004 Free Software Foundation, Inc. 3 This file is part of the GNU C Library. 4 Contributed by Torbjorn Granlund (tege@sics.se). 5 6 The GNU C Library is free software; you can redistribute it and/or 7 modify it under the terms of the GNU Lesser General Public 8 License as published by the Free Software Foundation; either 9 version 2.1 of the License, or (at your option) any later version. 10 11 The GNU C Library is distributed in the hope that it will be useful, 12 but WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 Lesser General Public License for more details. 15 16 You should have received a copy of the GNU Lesser General Public 17 License along with the GNU C Library; if not, see 18 <http://www.gnu.org/licenses/>. */ 19 20 #include <linux/types.h> 21 22 /* Type to use for aligned memory operations. 23 This should normally be the biggest type supported by a single load 24 and store. */ 25 #define op_t unsigned long int 26 #define OPSIZ (sizeof(op_t)) 27 28 /* Optimal type for storing bytes in registers. */ 29 #define reg_char char 30 31 #define MERGE(w0, sh_1, w1, sh_2) (((w0) >> (sh_1)) | ((w1) << (sh_2))) 32 33 /* Copy exactly NBYTES bytes from SRC_BP to DST_BP, 34 without any assumptions about alignment of the pointers. */ 35 #define BYTE_COPY_FWD(dst_bp, src_bp, nbytes) \ 36 do { \ 37 size_t __nbytes = (nbytes); \ 38 while (__nbytes > 0) { \ 39 unsigned char __x = ((unsigned char *) src_bp)[0]; \ 40 src_bp += 1; \ 41 __nbytes -= 1; \ 42 ((unsigned char *) dst_bp)[0] = __x; \ 43 dst_bp += 1; \ 44 } \ 45 } while (0) 46 47 /* Copy *up to* NBYTES bytes from SRC_BP to DST_BP, with 48 the assumption that DST_BP is aligned on an OPSIZ multiple. If 49 not all bytes could be easily copied, store remaining number of bytes 50 in NBYTES_LEFT, otherwise store 0. */ 51 /* extern void _wordcopy_fwd_aligned __P ((long int, long int, size_t)); */ 52 /* extern void _wordcopy_fwd_dest_aligned __P ((long int, long int, size_t)); */ 53 #define WORD_COPY_FWD(dst_bp, src_bp, nbytes_left, nbytes) \ 54 do { \ 55 if (src_bp % OPSIZ == 0) \ 56 _wordcopy_fwd_aligned(dst_bp, src_bp, (nbytes) / OPSIZ);\ 57 else \ 58 _wordcopy_fwd_dest_aligned(dst_bp, src_bp, (nbytes) / OPSIZ);\ 59 src_bp += (nbytes) & -OPSIZ; \ 60 dst_bp += (nbytes) & -OPSIZ; \ 61 (nbytes_left) = (nbytes) % OPSIZ; \ 62 } while (0) 63 64 65 /* Threshold value for when to enter the unrolled loops. */ 66 #define OP_T_THRES 16 67 68 /* _wordcopy_fwd_aligned -- Copy block beginning at SRCP to 69 block beginning at DSTP with LEN `op_t' words (not LEN bytes!). 70 Both SRCP and DSTP should be aligned for memory operations on `op_t's. */ 71 /* stream-lined (read x8 + write x8) */ 72 static void _wordcopy_fwd_aligned(long int dstp, long int srcp, size_t len) 73 { 74 while (len > 7) { 75 register op_t a0, a1, a2, a3, a4, a5, a6, a7; 76 77 a0 = ((op_t *) srcp)[0]; 78 a1 = ((op_t *) srcp)[1]; 79 a2 = ((op_t *) srcp)[2]; 80 a3 = ((op_t *) srcp)[3]; 81 a4 = ((op_t *) srcp)[4]; 82 a5 = ((op_t *) srcp)[5]; 83 a6 = ((op_t *) srcp)[6]; 84 a7 = ((op_t *) srcp)[7]; 85 ((op_t *) dstp)[0] = a0; 86 ((op_t *) dstp)[1] = a1; 87 ((op_t *) dstp)[2] = a2; 88 ((op_t *) dstp)[3] = a3; 89 ((op_t *) dstp)[4] = a4; 90 ((op_t *) dstp)[5] = a5; 91 ((op_t *) dstp)[6] = a6; 92 ((op_t *) dstp)[7] = a7; 93 94 srcp += 8 * OPSIZ; 95 dstp += 8 * OPSIZ; 96 len -= 8; 97 } 98 while (len > 0) { 99 *(op_t *)dstp = *(op_t *)srcp; 100 101 srcp += OPSIZ; 102 dstp += OPSIZ; 103 len -= 1; 104 } 105 } 106 107 /* _wordcopy_fwd_dest_aligned -- Copy block beginning at SRCP to 108 block beginning at DSTP with LEN `op_t' words (not LEN bytes!). 109 DSTP should be aligned for memory operations on `op_t's, but SRCP must 110 *not* be aligned. */ 111 /* stream-lined (read x4 + write x4) */ 112 static void _wordcopy_fwd_dest_aligned(long int dstp, long int srcp, 113 size_t len) 114 { 115 op_t ap; 116 int sh_1, sh_2; 117 118 /* Calculate how to shift a word read at the memory operation 119 aligned srcp to make it aligned for copy. */ 120 121 sh_1 = 8 * (srcp % OPSIZ); 122 sh_2 = 8 * OPSIZ - sh_1; 123 124 /* Make SRCP aligned by rounding it down to the beginning of the `op_t' 125 it points in the middle of. */ 126 srcp &= -OPSIZ; 127 ap = ((op_t *) srcp)[0]; 128 srcp += OPSIZ; 129 130 while (len > 3) { 131 op_t a0, a1, a2, a3; 132 133 a0 = ((op_t *) srcp)[0]; 134 a1 = ((op_t *) srcp)[1]; 135 a2 = ((op_t *) srcp)[2]; 136 a3 = ((op_t *) srcp)[3]; 137 ((op_t *) dstp)[0] = MERGE(ap, sh_1, a0, sh_2); 138 ((op_t *) dstp)[1] = MERGE(a0, sh_1, a1, sh_2); 139 ((op_t *) dstp)[2] = MERGE(a1, sh_1, a2, sh_2); 140 ((op_t *) dstp)[3] = MERGE(a2, sh_1, a3, sh_2); 141 142 ap = a3; 143 srcp += 4 * OPSIZ; 144 dstp += 4 * OPSIZ; 145 len -= 4; 146 } 147 while (len > 0) { 148 register op_t a0; 149 150 a0 = ((op_t *) srcp)[0]; 151 ((op_t *) dstp)[0] = MERGE(ap, sh_1, a0, sh_2); 152 153 ap = a0; 154 srcp += OPSIZ; 155 dstp += OPSIZ; 156 len -= 1; 157 } 158 } 159 160 void *memcpy(void *dstpp, const void *srcpp, size_t len) 161 { 162 unsigned long int dstp = (long int) dstpp; 163 unsigned long int srcp = (long int) srcpp; 164 165 /* Copy from the beginning to the end. */ 166 167 /* If there not too few bytes to copy, use word copy. */ 168 if (len >= OP_T_THRES) { 169 /* Copy just a few bytes to make DSTP aligned. */ 170 len -= (-dstp) % OPSIZ; 171 BYTE_COPY_FWD(dstp, srcp, (-dstp) % OPSIZ); 172 173 /* Copy whole pages from SRCP to DSTP by virtual address 174 manipulation, as much as possible. */ 175 176 /* PAGE_COPY_FWD_MAYBE (dstp, srcp, len, len); */ 177 178 /* Copy from SRCP to DSTP taking advantage of the known 179 alignment of DSTP. Number of bytes remaining is put in the 180 third argument, i.e. in LEN. This number may vary from 181 machine to machine. */ 182 183 WORD_COPY_FWD(dstp, srcp, len, len); 184 185 /* Fall out and copy the tail. */ 186 } 187 188 /* There are just a few bytes to copy. Use byte memory operations. */ 189 BYTE_COPY_FWD(dstp, srcp, len); 190 191 return dstpp; 192 } 193 194 void *memcpyb(void *dstpp, const void *srcpp, unsigned len) 195 { 196 unsigned long int dstp = (long int) dstpp; 197 unsigned long int srcp = (long int) srcpp; 198 199 BYTE_COPY_FWD(dstp, srcp, len); 200 201 return dstpp; 202 } 203