1/*- 2 * Copyright (c) 2018 The FreeBSD Foundation 3 * 4 * This software was developed by Mateusz Guzik <mjg@FreeBSD.org> 5 * under sponsorship from the FreeBSD Foundation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29#include <machine/asm.h> 30/* 31 * Note: this routine was written with kernel use in mind (read: no simd), 32 * it is only present in userspace as a temporary measure until something 33 * better gets imported. 34 */ 35 36#define ALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ 37 38/* 39 * memmove(dst, src, cnt) 40 * rdi, rsi, rdx 41 */ 42 43/* 44 * Register state at entry is supposed to be as follows: 45 * rdi - destination 46 * rsi - source 47 * rdx - count 48 * 49 * The macro possibly clobbers the above and: rcx, r8, r9, 10 50 * It does not clobber rax nor r11. 51 */ 52.macro MEMMOVE erms overlap begin end 53 \begin 54 55 /* 56 * For sizes 0..32 all data is read before it is written, so there 57 * is no correctness issue with direction of copying. 58 */ 59 cmpq $32,%rcx 60 jbe 101632f 61 62.if \overlap == 1 63 movq %rdi,%r8 64 subq %rsi,%r8 65 cmpq %rcx,%r8 /* overlapping && src < dst? */ 66 jb 2f 67.endif 68 69 cmpq $256,%rcx 70 ja 1256f 71 72 ALIGN_TEXT 73103200: 74 movq (%rsi),%rdx 75 movq %rdx,(%rdi) 76 movq 8(%rsi),%rdx 77 movq %rdx,8(%rdi) 78 movq 16(%rsi),%rdx 79 movq %rdx,16(%rdi) 80 movq 24(%rsi),%rdx 81 movq %rdx,24(%rdi) 82 leaq 32(%rsi),%rsi 83 leaq 32(%rdi),%rdi 84 subq $32,%rcx 85 cmpq $32,%rcx 86 jae 103200b 87 cmpb $0,%cl 88 jne 101632f 89 \end 90 ret 91 ALIGN_TEXT 92101632: 93 cmpb $16,%cl 94 jl 100816f 95 movq (%rsi),%rdx 96 movq 8(%rsi),%r8 97 movq -16(%rsi,%rcx),%r9 98 movq -8(%rsi,%rcx),%r10 99 movq %rdx,(%rdi) 100 movq %r8,8(%rdi) 101 movq %r9,-16(%rdi,%rcx) 102 movq %r10,-8(%rdi,%rcx) 103 \end 104 ret 105 ALIGN_TEXT 106100816: 107 cmpb $8,%cl 108 jl 100408f 109 movq (%rsi),%rdx 110 movq -8(%rsi,%rcx),%r8 111 movq %rdx,(%rdi) 112 movq %r8,-8(%rdi,%rcx,) 113 \end 114 ret 115 ALIGN_TEXT 116100408: 117 cmpb $4,%cl 118 jl 100204f 119 movl (%rsi),%edx 120 movl -4(%rsi,%rcx),%r8d 121 movl %edx,(%rdi) 122 movl %r8d,-4(%rdi,%rcx) 123 \end 124 ret 125 ALIGN_TEXT 126100204: 127 cmpb $2,%cl 128 jl 100001f 129 movzwl (%rsi),%edx 130 movzwl -2(%rsi,%rcx),%r8d 131 movw %dx,(%rdi) 132 movw %r8w,-2(%rdi,%rcx) 133 \end 134 ret 135 ALIGN_TEXT 136100001: 137 cmpb $1,%cl 138 jl 100000f 139 movb (%rsi),%dl 140 movb %dl,(%rdi) 141100000: 142 \end 143 ret 144 145 ALIGN_TEXT 1461256: 147 testb $15,%dil 148 jnz 100f 149.if \erms == 1 150 rep 151 movsb 152.else 153 shrq $3,%rcx /* copy by 64-bit words */ 154 rep 155 movsq 156 movq %rdx,%rcx 157 andl $7,%ecx /* any bytes left? */ 158 jne 100408b 159.endif 160 \end 161 ret 162100: 163 movq (%rsi),%r8 164 movq 8(%rsi),%r9 165 movq %rdi,%r10 166 movq %rdi,%rcx 167 andq $15,%rcx 168 leaq -16(%rdx,%rcx),%rdx 169 neg %rcx 170 leaq 16(%rdi,%rcx),%rdi 171 leaq 16(%rsi,%rcx),%rsi 172 movq %rdx,%rcx 173.if \erms == 1 174 rep 175 movsb 176 movq %r8,(%r10) 177 movq %r9,8(%r10) 178.else 179 shrq $3,%rcx /* copy by 64-bit words */ 180 rep 181 movsq 182 movq %r8,(%r10) 183 movq %r9,8(%r10) 184 movq %rdx,%rcx 185 andl $7,%ecx /* any bytes left? */ 186 jne 100408b 187.endif 188 \end 189 ret 190 191.if \overlap == 1 192 /* 193 * Copy backwards. 194 */ 195 ALIGN_TEXT 1962: 197 cmpq $256,%rcx 198 ja 2256f 199 200 leaq -8(%rdi,%rcx),%rdi 201 leaq -8(%rsi,%rcx),%rsi 202 203 cmpq $32,%rcx 204 jb 2016f 205 206 ALIGN_TEXT 2072032: 208 movq (%rsi),%rdx 209 movq %rdx,(%rdi) 210 movq -8(%rsi),%rdx 211 movq %rdx,-8(%rdi) 212 movq -16(%rsi),%rdx 213 movq %rdx,-16(%rdi) 214 movq -24(%rsi),%rdx 215 movq %rdx,-24(%rdi) 216 leaq -32(%rsi),%rsi 217 leaq -32(%rdi),%rdi 218 subq $32,%rcx 219 cmpq $32,%rcx 220 jae 2032b 221 cmpb $0,%cl 222 jne 2016f 223 \end 224 ret 225 ALIGN_TEXT 2262016: 227 cmpb $16,%cl 228 jl 2008f 229 movq (%rsi),%rdx 230 movq %rdx,(%rdi) 231 movq -8(%rsi),%rdx 232 movq %rdx,-8(%rdi) 233 subb $16,%cl 234 jz 2000f 235 leaq -16(%rsi),%rsi 236 leaq -16(%rdi),%rdi 2372008: 238 cmpb $8,%cl 239 jl 2004f 240 movq (%rsi),%rdx 241 movq %rdx,(%rdi) 242 subb $8,%cl 243 jz 2000f 244 leaq -8(%rsi),%rsi 245 leaq -8(%rdi),%rdi 2462004: 247 cmpb $4,%cl 248 jl 2002f 249 movl 4(%rsi),%edx 250 movl %edx,4(%rdi) 251 subb $4,%cl 252 jz 2000f 253 leaq -4(%rsi),%rsi 254 leaq -4(%rdi),%rdi 2552002: 256 cmpb $2,%cl 257 jl 2001f 258 movw 6(%rsi),%dx 259 movw %dx,6(%rdi) 260 subb $2,%cl 261 jz 2000f 262 leaq -2(%rsi),%rsi 263 leaq -2(%rdi),%rdi 2642001: 265 cmpb $1,%cl 266 jl 2000f 267 movb 7(%rsi),%dl 268 movb %dl,7(%rdi) 2692000: 270 \end 271 ret 272 ALIGN_TEXT 2732256: 274 std 275 leaq -8(%rdi,%rcx),%rdi 276 leaq -8(%rsi,%rcx),%rsi 277 shrq $3,%rcx 278 rep 279 movsq 280 cld 281 movq %rdx,%rcx 282 andb $7,%cl 283 jne 2004b 284 \end 285 ret 286.endif 287.endm 288 289 290.macro MEMMOVE_BEGIN 291 movq %rdi,%rax 292 movq %rdx,%rcx 293.endm 294 295.macro MEMMOVE_END 296.endm 297 298#ifndef MEMCPY 299ENTRY(memmove) 300 MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END 301END(memmove) 302#else 303ENTRY(memcpy) 304 MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END 305END(memcpy) 306#endif 307 308 .section .note.GNU-stack,"",%progbits 309