1/* Copyright 2002 Andi Kleen */ 2 3#include <linux/linkage.h> 4#include <asm/errno.h> 5#include <asm/cpufeatures.h> 6#include <asm/alternative-asm.h> 7#include <asm/export.h> 8 9/* 10 * We build a jump to memcpy_orig by default which gets NOPped out on 11 * the majority of x86 CPUs which set REP_GOOD. In addition, CPUs which 12 * have the enhanced REP MOVSB/STOSB feature (ERMS), change those NOPs 13 * to a jmp to memcpy_erms which does the REP; MOVSB mem copy. 14 */ 15 16.weak memcpy 17 18/* 19 * memcpy - Copy a memory block. 20 * 21 * Input: 22 * rdi destination 23 * rsi source 24 * rdx count 25 * 26 * Output: 27 * rax original destination 28 */ 29ENTRY(__memcpy) 30ENTRY(memcpy) 31 ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \ 32 "jmp memcpy_erms", X86_FEATURE_ERMS 33 34 movq %rdi, %rax 35 movq %rdx, %rcx 36 shrq $3, %rcx 37 andl $7, %edx 38 rep movsq 39 movl %edx, %ecx 40 rep movsb 41 ret 42ENDPROC(memcpy) 43ENDPROC(__memcpy) 44EXPORT_SYMBOL(memcpy) 45EXPORT_SYMBOL(__memcpy) 46 47/* 48 * memcpy_erms() - enhanced fast string memcpy. This is faster and 49 * simpler than memcpy. Use memcpy_erms when possible. 50 */ 51ENTRY(memcpy_erms) 52 movq %rdi, %rax 53 movq %rdx, %rcx 54 rep movsb 55 ret 56ENDPROC(memcpy_erms) 57 58ENTRY(memcpy_orig) 59 movq %rdi, %rax 60 61 cmpq $0x20, %rdx 62 jb .Lhandle_tail 63 64 /* 65 * We check whether memory false dependence could occur, 66 * then jump to corresponding copy mode. 67 */ 68 cmp %dil, %sil 69 jl .Lcopy_backward 70 subq $0x20, %rdx 71.Lcopy_forward_loop: 72 subq $0x20, %rdx 73 74 /* 75 * Move in blocks of 4x8 bytes: 76 */ 77 movq 0*8(%rsi), %r8 78 movq 1*8(%rsi), %r9 79 movq 2*8(%rsi), %r10 80 movq 3*8(%rsi), %r11 81 leaq 4*8(%rsi), %rsi 82 83 movq %r8, 0*8(%rdi) 84 movq %r9, 1*8(%rdi) 85 movq %r10, 2*8(%rdi) 86 movq %r11, 3*8(%rdi) 87 leaq 4*8(%rdi), %rdi 88 jae .Lcopy_forward_loop 89 addl $0x20, %edx 90 jmp .Lhandle_tail 91 92.Lcopy_backward: 93 /* 94 * Calculate copy position to tail. 95 */ 96 addq %rdx, %rsi 97 addq %rdx, %rdi 98 subq $0x20, %rdx 99 /* 100 * At most 3 ALU operations in one cycle, 101 * so append NOPS in the same 16 bytes trunk. 102 */ 103 .p2align 4 104.Lcopy_backward_loop: 105 subq $0x20, %rdx 106 movq -1*8(%rsi), %r8 107 movq -2*8(%rsi), %r9 108 movq -3*8(%rsi), %r10 109 movq -4*8(%rsi), %r11 110 leaq -4*8(%rsi), %rsi 111 movq %r8, -1*8(%rdi) 112 movq %r9, -2*8(%rdi) 113 movq %r10, -3*8(%rdi) 114 movq %r11, -4*8(%rdi) 115 leaq -4*8(%rdi), %rdi 116 jae .Lcopy_backward_loop 117 118 /* 119 * Calculate copy position to head. 120 */ 121 addl $0x20, %edx 122 subq %rdx, %rsi 123 subq %rdx, %rdi 124.Lhandle_tail: 125 cmpl $16, %edx 126 jb .Lless_16bytes 127 128 /* 129 * Move data from 16 bytes to 31 bytes. 130 */ 131 movq 0*8(%rsi), %r8 132 movq 1*8(%rsi), %r9 133 movq -2*8(%rsi, %rdx), %r10 134 movq -1*8(%rsi, %rdx), %r11 135 movq %r8, 0*8(%rdi) 136 movq %r9, 1*8(%rdi) 137 movq %r10, -2*8(%rdi, %rdx) 138 movq %r11, -1*8(%rdi, %rdx) 139 retq 140 .p2align 4 141.Lless_16bytes: 142 cmpl $8, %edx 143 jb .Lless_8bytes 144 /* 145 * Move data from 8 bytes to 15 bytes. 146 */ 147 movq 0*8(%rsi), %r8 148 movq -1*8(%rsi, %rdx), %r9 149 movq %r8, 0*8(%rdi) 150 movq %r9, -1*8(%rdi, %rdx) 151 retq 152 .p2align 4 153.Lless_8bytes: 154 cmpl $4, %edx 155 jb .Lless_3bytes 156 157 /* 158 * Move data from 4 bytes to 7 bytes. 159 */ 160 movl (%rsi), %ecx 161 movl -4(%rsi, %rdx), %r8d 162 movl %ecx, (%rdi) 163 movl %r8d, -4(%rdi, %rdx) 164 retq 165 .p2align 4 166.Lless_3bytes: 167 subl $1, %edx 168 jb .Lend 169 /* 170 * Move data from 1 bytes to 3 bytes. 171 */ 172 movzbl (%rsi), %ecx 173 jz .Lstore_1byte 174 movzbq 1(%rsi), %r8 175 movzbq (%rsi, %rdx), %r9 176 movb %r8b, 1(%rdi) 177 movb %r9b, (%rdi, %rdx) 178.Lstore_1byte: 179 movb %cl, (%rdi) 180 181.Lend: 182 retq 183ENDPROC(memcpy_orig) 184 185#ifndef CONFIG_UML 186/* 187 * __memcpy_mcsafe - memory copy with machine check exception handling 188 * Note that we only catch machine checks when reading the source addresses. 189 * Writes to target are posted and don't generate machine checks. 190 */ 191ENTRY(__memcpy_mcsafe) 192 cmpl $8, %edx 193 /* Less than 8 bytes? Go to byte copy loop */ 194 jb .L_no_whole_words 195 196 /* Check for bad alignment of source */ 197 testl $7, %esi 198 /* Already aligned */ 199 jz .L_8byte_aligned 200 201 /* Copy one byte at a time until source is 8-byte aligned */ 202 movl %esi, %ecx 203 andl $7, %ecx 204 subl $8, %ecx 205 negl %ecx 206 subl %ecx, %edx 207.L_read_leading_bytes: 208 movb (%rsi), %al 209.L_write_leading_bytes: 210 movb %al, (%rdi) 211 incq %rsi 212 incq %rdi 213 decl %ecx 214 jnz .L_read_leading_bytes 215 216.L_8byte_aligned: 217 movl %edx, %ecx 218 andl $7, %edx 219 shrl $3, %ecx 220 jz .L_no_whole_words 221 222.L_read_words: 223 movq (%rsi), %r8 224.L_write_words: 225 movq %r8, (%rdi) 226 addq $8, %rsi 227 addq $8, %rdi 228 decl %ecx 229 jnz .L_read_words 230 231 /* Any trailing bytes? */ 232.L_no_whole_words: 233 andl %edx, %edx 234 jz .L_done_memcpy_trap 235 236 /* Copy trailing bytes */ 237 movl %edx, %ecx 238.L_read_trailing_bytes: 239 movb (%rsi), %al 240.L_write_trailing_bytes: 241 movb %al, (%rdi) 242 incq %rsi 243 incq %rdi 244 decl %ecx 245 jnz .L_read_trailing_bytes 246 247 /* Copy successful. Return zero */ 248.L_done_memcpy_trap: 249 xorq %rax, %rax 250 ret 251ENDPROC(__memcpy_mcsafe) 252EXPORT_SYMBOL_GPL(__memcpy_mcsafe) 253 254 .section .fixup, "ax" 255 /* 256 * Return number of bytes not copied for any failure. Note that 257 * there is no "tail" handling since the source buffer is 8-byte 258 * aligned and poison is cacheline aligned. 259 */ 260.E_read_words: 261 shll $3, %ecx 262.E_leading_bytes: 263 addl %edx, %ecx 264.E_trailing_bytes: 265 mov %ecx, %eax 266 ret 267 268 /* 269 * For write fault handling, given the destination is unaligned, 270 * we handle faults on multi-byte writes with a byte-by-byte 271 * copy up to the write-protected page. 272 */ 273.E_write_words: 274 shll $3, %ecx 275 addl %edx, %ecx 276 movl %ecx, %edx 277 jmp mcsafe_handle_tail 278 279 .previous 280 281 _ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes) 282 _ASM_EXTABLE_FAULT(.L_read_words, .E_read_words) 283 _ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes) 284 _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes) 285 _ASM_EXTABLE(.L_write_words, .E_write_words) 286 _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes) 287#endif 288