1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/string.h> 3 #include <linux/export.h> 4 5 #undef memcpy 6 #undef memset 7 #undef memmove 8 9 __visible void *memcpy(void *to, const void *from, size_t n) 10 { 11 return __memcpy(to, from, n); 12 } 13 EXPORT_SYMBOL(memcpy); 14 15 __visible void *memset(void *s, int c, size_t count) 16 { 17 return __memset(s, c, count); 18 } 19 EXPORT_SYMBOL(memset); 20 21 __visible void *memmove(void *dest, const void *src, size_t n) 22 { 23 int d0,d1,d2,d3,d4,d5; 24 char *ret = dest; 25 26 __asm__ __volatile__( 27 /* Handle more 16 bytes in loop */ 28 "cmp $0x10, %0\n\t" 29 "jb 1f\n\t" 30 31 /* Decide forward/backward copy mode */ 32 "cmp %2, %1\n\t" 33 "jb 2f\n\t" 34 35 /* 36 * movs instruction have many startup latency 37 * so we handle small size by general register. 38 */ 39 "cmp $680, %0\n\t" 40 "jb 3f\n\t" 41 /* 42 * movs instruction is only good for aligned case. 43 */ 44 "mov %1, %3\n\t" 45 "xor %2, %3\n\t" 46 "and $0xff, %3\n\t" 47 "jz 4f\n\t" 48 "3:\n\t" 49 "sub $0x10, %0\n\t" 50 51 /* 52 * We gobble 16 bytes forward in each loop. 53 */ 54 "3:\n\t" 55 "sub $0x10, %0\n\t" 56 "mov 0*4(%1), %3\n\t" 57 "mov 1*4(%1), %4\n\t" 58 "mov %3, 0*4(%2)\n\t" 59 "mov %4, 1*4(%2)\n\t" 60 "mov 2*4(%1), %3\n\t" 61 "mov 3*4(%1), %4\n\t" 62 "mov %3, 2*4(%2)\n\t" 63 "mov %4, 3*4(%2)\n\t" 64 "lea 0x10(%1), %1\n\t" 65 "lea 0x10(%2), %2\n\t" 66 "jae 3b\n\t" 67 "add $0x10, %0\n\t" 68 "jmp 1f\n\t" 69 70 /* 71 * Handle data forward by movs. 72 */ 73 ".p2align 4\n\t" 74 "4:\n\t" 75 "mov -4(%1, %0), %3\n\t" 76 "lea -4(%2, %0), %4\n\t" 77 "shr $2, %0\n\t" 78 "rep movsl\n\t" 79 "mov %3, (%4)\n\t" 80 "jmp 11f\n\t" 81 /* 82 * Handle data backward by movs. 83 */ 84 ".p2align 4\n\t" 85 "6:\n\t" 86 "mov (%1), %3\n\t" 87 "mov %2, %4\n\t" 88 "lea -4(%1, %0), %1\n\t" 89 "lea -4(%2, %0), %2\n\t" 90 "shr $2, %0\n\t" 91 "std\n\t" 92 "rep movsl\n\t" 93 "mov %3,(%4)\n\t" 94 "cld\n\t" 95 "jmp 11f\n\t" 96 97 /* 98 * Start to prepare for backward copy. 99 */ 100 ".p2align 4\n\t" 101 "2:\n\t" 102 "cmp $680, %0\n\t" 103 "jb 5f\n\t" 104 "mov %1, %3\n\t" 105 "xor %2, %3\n\t" 106 "and $0xff, %3\n\t" 107 "jz 6b\n\t" 108 109 /* 110 * Calculate copy position to tail. 111 */ 112 "5:\n\t" 113 "add %0, %1\n\t" 114 "add %0, %2\n\t" 115 "sub $0x10, %0\n\t" 116 117 /* 118 * We gobble 16 bytes backward in each loop. 119 */ 120 "7:\n\t" 121 "sub $0x10, %0\n\t" 122 123 "mov -1*4(%1), %3\n\t" 124 "mov -2*4(%1), %4\n\t" 125 "mov %3, -1*4(%2)\n\t" 126 "mov %4, -2*4(%2)\n\t" 127 "mov -3*4(%1), %3\n\t" 128 "mov -4*4(%1), %4\n\t" 129 "mov %3, -3*4(%2)\n\t" 130 "mov %4, -4*4(%2)\n\t" 131 "lea -0x10(%1), %1\n\t" 132 "lea -0x10(%2), %2\n\t" 133 "jae 7b\n\t" 134 /* 135 * Calculate copy position to head. 136 */ 137 "add $0x10, %0\n\t" 138 "sub %0, %1\n\t" 139 "sub %0, %2\n\t" 140 141 /* 142 * Move data from 8 bytes to 15 bytes. 143 */ 144 ".p2align 4\n\t" 145 "1:\n\t" 146 "cmp $8, %0\n\t" 147 "jb 8f\n\t" 148 "mov 0*4(%1), %3\n\t" 149 "mov 1*4(%1), %4\n\t" 150 "mov -2*4(%1, %0), %5\n\t" 151 "mov -1*4(%1, %0), %1\n\t" 152 153 "mov %3, 0*4(%2)\n\t" 154 "mov %4, 1*4(%2)\n\t" 155 "mov %5, -2*4(%2, %0)\n\t" 156 "mov %1, -1*4(%2, %0)\n\t" 157 "jmp 11f\n\t" 158 159 /* 160 * Move data from 4 bytes to 7 bytes. 161 */ 162 ".p2align 4\n\t" 163 "8:\n\t" 164 "cmp $4, %0\n\t" 165 "jb 9f\n\t" 166 "mov 0*4(%1), %3\n\t" 167 "mov -1*4(%1, %0), %4\n\t" 168 "mov %3, 0*4(%2)\n\t" 169 "mov %4, -1*4(%2, %0)\n\t" 170 "jmp 11f\n\t" 171 172 /* 173 * Move data from 2 bytes to 3 bytes. 174 */ 175 ".p2align 4\n\t" 176 "9:\n\t" 177 "cmp $2, %0\n\t" 178 "jb 10f\n\t" 179 "movw 0*2(%1), %%dx\n\t" 180 "movw -1*2(%1, %0), %%bx\n\t" 181 "movw %%dx, 0*2(%2)\n\t" 182 "movw %%bx, -1*2(%2, %0)\n\t" 183 "jmp 11f\n\t" 184 185 /* 186 * Move data for 1 byte. 187 */ 188 ".p2align 4\n\t" 189 "10:\n\t" 190 "cmp $1, %0\n\t" 191 "jb 11f\n\t" 192 "movb (%1), %%cl\n\t" 193 "movb %%cl, (%2)\n\t" 194 ".p2align 4\n\t" 195 "11:" 196 : "=&c" (d0), "=&S" (d1), "=&D" (d2), 197 "=r" (d3),"=r" (d4), "=r"(d5) 198 :"0" (n), 199 "1" (src), 200 "2" (dest) 201 :"memory"); 202 203 return ret; 204 205 } 206 EXPORT_SYMBOL(memmove); 207