/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2009, Intel Corporation * All rights reserved. */ /* * str[n]cpy - copy [n] chars from second operand into first operand */ #include "SYS.h" #include "proc64_id.h" #define LABEL(s) .strcpy##s #ifdef USE_AS_STRNCPY ENTRY(strncpy) test %edx, %edx jz LABEL(strncpy_exitz) mov %rdx, %r8 #else ENTRY(strcpy) /* (char *, const char *) */ xor %rdx, %rdx #endif mov %esi, %ecx and $0xfffffffffffffff0, %rsi /* force rsi 16 byte align */ and $0xf, %rcx mov %rdi, %rax /* save destination address for return value */ pxor %xmm0, %xmm0 /* clear %xmm0 for null char checks */ pcmpeqb (%rsi), %xmm0 /* check 16 bytes in src for null */ pmovmskb %xmm0, %edx shr %cl, %edx /* adjust for offset from 16byte boundary */ test %edx, %edx /* edx will be 0 if chars are non-null */ jnz LABEL(less16bytes) /* null char found in first 16 bytes examined */ #ifdef USE_AS_STRNCPY /* * Check if the count is satisfied in first 16 bytes examined. */ lea -16(%r8, %rcx), %r11 cmp $0, %r11 jle LABEL(less16bytes) #endif mov %rcx, %r9 /* rsi alignment offset */ or %edi, %ecx and $0xf, %ecx lea -16(%r9), %r10 jz LABEL(ashr_0) /* src and dest are both 16 byte aligned */ neg %r10 /* max src bytes remaining in current dqword */ pxor %xmm0, %xmm0 /* clear %xmm0, may be polluted by unaligned operation */ pcmpeqb 16(%rsi), %xmm0 /* check next 16 bytes in src for a null */ pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(less32bytes) /* null char found in first 32 bytes examined */ #ifdef USE_AS_STRNCPY /* * If strncpy count <= 16 go to exit case */ sub $16, %r8 jbe LABEL(less32bytes_strncpy_truncation) #endif /* * At least 16 bytes to copy to destination string. Move them now. * Don't worry about alignment. */ mov (%rsi, %r9), %rdx mov %rdx, (%rdi) mov 8(%rsi, %r9), %rdx mov %rdx, 8(%rdi) /* * so far destination rdi may be aligned by 16, re-calculate rsi and * jump to corresponding src/dest relative offset case. * rcx is offset of rsi * rdx is offset of rdi */ and $0xfffffffffffffff0, %rdi /* force rdi 16 byte align */ mov %rax, %rdx /* rax contains orignal rdi */ xor %rdi, %rdx /* same effect as "and $0xf, %rdx" */ #ifdef USE_AS_STRNCPY /* * Will now do 16 byte aligned stores. Stores may overlap some bytes * (ie store twice) if destination was unaligned. Compensate here. */ add %rdx, %r8 /* compensate for overlap */ #endif add $16, %rdi /* next 16 bytes for dest */ /* * align src to 16-byte boundary. Could be up or down depending on * whether src offset - dest offset > 0 (up) or * src offset - dest offset < 0 (down). */ sub %rdx, %r9 /* src offset - dest offset */ lea 16(%r9, %rsi), %rsi mov %esi, %ecx /* for new src offset */ and $0xfffffffffffffff0, %rsi /* force rsi 16 byte align */ and $0xf, %ecx /* new src offset is 0 if rsi/rdi have same alignment */ jz LABEL(ashr_0) #ifdef USE_AS_STRNCPY xor %edx, %edx /* In case unaligned_exit is taken */ #endif /* * Jump to case corresponding to source/dest string relative offsets * Index = (16 + (src offset - dest offset)) % 16 */ lea -16(%rcx), %r10 mov %rcx, %r9 neg %r10 /* max src bytes remaining in current dqword */ lea LABEL(unaligned_table)(%rip), %r11 movslq (%r11, %rcx, 4), %rcx lea (%r11, %rcx), %rcx jmp *%rcx /* * ashr_0 handles the following cases: * src alignment offset = dest alignment offset */ .p2align 5 LABEL(ashr_0): #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_aligned) #endif movdqa (%rsi), %xmm1 /* fetch 16 bytes from src string */ movdqa %xmm1, (%rdi) /* store 16 bytes into dest string */ add $16, %rsi add $16, %rdi pcmpeqb (%rsi), %xmm0 /* check 16 bytes in src for a null */ pmovmskb %xmm0, %edx test %edx, %edx /* edx will be 0 if chars are non-null */ jnz LABEL(aligned_16bytes) /* exit tail */ LABEL(ashr_0_loop): #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_aligned) #endif movdqa (%rsi, %rcx), %xmm1 movdqa %xmm1, (%rdi, %rcx) add $16, %rcx pcmpeqb (%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(aligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_aligned) #endif movdqa (%rsi, %rcx), %xmm1 movdqa %xmm1, (%rdi, %rcx) add $16, %rcx pcmpeqb (%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(aligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_aligned) #endif movdqa (%rsi, %rcx), %xmm1 movdqa %xmm1, (%rdi, %rcx) add $16, %rcx pcmpeqb (%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(aligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_aligned) #endif movdqa (%rsi, %rcx), %xmm1 movdqa %xmm1, (%rdi, %rcx) add $16, %rcx pcmpeqb (%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jz LABEL(ashr_0_loop) jmp LABEL(aligned_exit) /* * ashr_15 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 15 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_15): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_15_use_sse2) .p2align 4 LABEL(ashr_15_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $15, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0f movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $15, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0f movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_15_use_ssse3) .p2align 4 LABEL(ashr_15_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $15, %xmm2 pslldq $1, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $15, %xmm2 pslldq $1, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_15_use_sse2) /* * ashr_14 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 14 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_14): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_14_use_sse2) .p2align 4 LABEL(ashr_14_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $14, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0e movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $14, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0e movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_14_use_ssse3) .p2align 4 LABEL(ashr_14_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $14, %xmm2 pslldq $2, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $14, %xmm2 pslldq $2, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_14_use_sse2) /* * ashr_13 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 13 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_13): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_13_use_sse2) .p2align 4 LABEL(ashr_13_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $13, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0d movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $13, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0d movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_13_use_ssse3) .p2align 4 LABEL(ashr_13_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $13, %xmm2 pslldq $3, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $13, %xmm2 pslldq $3, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_13_use_sse2) /* * ashr_12 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 12 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_12): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_12_use_sse2) .p2align 4 LABEL(ashr_12_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $12, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0c movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $12, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0c movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_12_use_ssse3) .p2align 4 LABEL(ashr_12_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $12, %xmm2 pslldq $4, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $12, %xmm2 pslldq $4, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_12_use_sse2) /* * ashr_11 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 11 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_11): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_11_use_sse2) .p2align 4 LABEL(ashr_11_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $11, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0b movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $11, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0b movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_11_use_ssse3) .p2align 4 LABEL(ashr_11_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $11, %xmm2 pslldq $5, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $11, %xmm2 pslldq $5, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_11_use_sse2) /* * ashr_10 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 10 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_10): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_10_use_sse2) .p2align 4 LABEL(ashr_10_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $10, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0a movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $10, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x0a movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_10_use_ssse3) .p2align 4 LABEL(ashr_10_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $10, %xmm2 pslldq $6, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $10, %xmm2 pslldq $6, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_10_use_sse2) /* * ashr_9 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 9 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_9): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_9_use_sse2) .p2align 4 LABEL(ashr_9_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $9, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x09 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $9, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x09 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_9_use_ssse3) .p2align 4 LABEL(ashr_9_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $9, %xmm2 pslldq $7, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $9, %xmm2 pslldq $7, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_9_use_sse2) /* * ashr_8 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 8 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_8): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_8_use_sse2) .p2align 4 LABEL(ashr_8_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $8, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x08 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $8, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x08 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_8_use_ssse3) .p2align 4 LABEL(ashr_8_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $8, %xmm2 pslldq $8, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $8, %xmm2 pslldq $8, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_8_use_sse2) /* * ashr_7 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 7 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_7): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_7_use_sse2) .p2align 4 LABEL(ashr_7_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $7, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x07 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $7, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x07 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_7_use_ssse3) .p2align 4 LABEL(ashr_7_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $7, %xmm2 pslldq $9, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $7, %xmm2 pslldq $9, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_7_use_sse2) /* * ashr_6 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 6 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_6): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_6_use_sse2) .p2align 4 LABEL(ashr_6_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $6, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x06 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $6, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x06 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_6_use_ssse3) .p2align 4 LABEL(ashr_6_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $6, %xmm2 pslldq $10, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $6, %xmm2 pslldq $10, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_6_use_sse2) /* * ashr_5 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 5 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_5): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_5_use_sse2) .p2align 4 LABEL(ashr_5_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $5, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x05 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $5, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x05 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_5_use_ssse3) .p2align 4 LABEL(ashr_5_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $5, %xmm2 pslldq $11, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $5, %xmm2 pslldq $11, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_5_use_sse2) /* * ashr_4 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 4 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_4): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_4_use_sse2) .p2align 4 LABEL(ashr_4_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $4, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x04 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $4, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x04 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_4_use_ssse3) .p2align 4 LABEL(ashr_4_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $4, %xmm2 pslldq $12, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $4, %xmm2 pslldq $12, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_4_use_sse2) /* * ashr_3 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 3 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_3): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_3_use_sse2) .p2align 4 LABEL(ashr_3_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $3, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x03 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $3, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x03 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_3_use_ssse3) .p2align 4 LABEL(ashr_3_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $3, %xmm2 pslldq $13, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $3, %xmm2 pslldq $13, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_3_use_sse2) /* * ashr_2 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 2 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_2): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_2_use_sse2) .p2align 4 LABEL(ashr_2_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $2, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x02 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $2, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x02 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_2_use_ssse3) .p2align 4 LABEL(ashr_2_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $2, %xmm2 pslldq $14, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $2, %xmm2 pslldq $14, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_2_use_sse2) /* * ashr_1 handles the following cases: * (16 + (src offset - dest offset)) % 16 = 1 * * Based on above operation, start from (%r9 + rsi) to the left of this cache * bank, there is no null byte. */ .p2align 4 LABEL(ashr_1): xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ jz LABEL(ashr_1_use_sse2) .p2align 4 LABEL(ashr_1_use_ssse3): movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $1, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x01 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif movdqa 16(%rsi, %rcx), %xmm3 pcmpeqb %xmm3, %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif #palignr $1, (%rsi, %rcx), %xmm3 .byte 0x66, 0x0F, 0x3A ,0x0F .byte 0x1c, 0x0e, 0x01 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_1_use_ssse3) .p2align 4 LABEL(ashr_1_use_sse2): pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $1, %xmm2 pslldq $15, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif pcmpeqb 16(%rsi, %rcx), %xmm0 pmovmskb %xmm0, %edx test %edx, %edx jnz LABEL(unaligned_exit) #ifdef USE_AS_STRNCPY sub $16, %r8 jbe LABEL(strncpy_truncation_unaligned) #endif movdqa 16(%rsi, %rcx), %xmm3 movdqa (%rsi, %rcx), %xmm2 psrldq $1, %xmm2 pslldq $15, %xmm3 por %xmm2, %xmm3 movdqa %xmm3, (%rdi, %rcx) add $16, %rcx #ifdef USE_AS_STRNCPY cmp %r10, %r8 jbe LABEL(unaligned_exit) #endif jmp LABEL(ashr_1_use_sse2) /* * Exit tail code: * Up to 32 bytes are copied in the case of strcpy. */ .p2align 4 LABEL(less32bytes): xor %ecx, %ecx LABEL(unaligned_exit): add %r9, %rsi /* r9 holds offset of rsi */ mov %rcx, %r9 mov %r10, %rcx shl %cl, %edx /* after shl, calculate the exact number to be filled */ mov %r9, %rcx .p2align 4 LABEL(aligned_exit): add %rcx, %rdi /* locate exact address for rdi */ LABEL(less16bytes): add %rcx, %rsi /* locate exact address for rsi */ LABEL(aligned_16bytes): #ifdef USE_AS_STRNCPY /* * Null found in 16bytes checked. Set bit in bitmask corresponding to * the strncpy count argument. We will copy to the null (inclusive) * or count whichever comes first. */ mov $1, %r9d lea -1(%r8), %rcx shl %cl, %r9d cmp $32, %r8 ja LABEL(strncpy_tail) or %r9d, %edx LABEL(strncpy_tail): #endif /* * Check to see if BSF is fast on this processor. If not, use a * different exit tail. */ testb $USE_BSF, .memops_method(%rip) jz LABEL(AMD_exit) bsf %rdx, %rcx /* Find byte with null char */ lea LABEL(tail_table)(%rip), %r11 movslq (%r11, %rcx, 4), %rcx lea (%r11, %rcx), %rcx jmp *%rcx #ifdef USE_AS_STRNCPY /* * Count reached before null found. */ .p2align 4 LABEL(less32bytes_strncpy_truncation): xor %ecx, %ecx LABEL(strncpy_truncation_unaligned): add %r9, %rsi /* next src char to copy */ LABEL(strncpy_truncation_aligned): add %rcx, %rdi add %rcx, %rsi add $16, %r8 /* compensation */ lea -1(%r8), %rcx lea LABEL(tail_table)(%rip), %r11 movslq (%r11, %rcx, 4), %rcx lea (%r11, %rcx), %rcx jmp *%rcx .p2align 4 LABEL(strncpy_exitz): mov %rdi, %rax ret #endif .p2align 4 LABEL(AMD_exit): test %dl, %dl jz LABEL(AMD_exit_more_8) test $0x01, %dl jnz LABEL(tail_0) test $0x02, %dl jnz LABEL(tail_1) test $0x04, %dl jnz LABEL(tail_2) test $0x08, %dl jnz LABEL(tail_3) test $0x10, %dl jnz LABEL(tail_4) test $0x20, %dl jnz LABEL(tail_5) test $0x40, %dl jnz LABEL(tail_6) .p2align 4 LABEL(tail_7): /* 8 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) #ifdef USE_AS_STRNCPY mov $8, %cl sub $8, %r8 jnz LABEL(strncpy_fill_tail) #endif ret #ifdef USE_AS_STRNCPY /* * Null terminated src string shorter than count. Fill the rest of the * destination with null chars. */ .p2align 4 LABEL(strncpy_fill_tail): mov %rax, %rdx movzx %cl, %rax mov %r8, %rcx add %rax, %rdi xor %eax, %eax shr $3, %ecx jz LABEL(strncpy_fill_less_8) rep stosq LABEL(strncpy_fill_less_8): mov %r8, %rcx and $7, %rcx jz LABEL(strncpy_fill_return) LABEL(strncpy_fill_less_7): sub $1, %ecx mov %al, (%rdi, %rcx) jnz LABEL(strncpy_fill_less_7) LABEL(strncpy_fill_return): mov %rdx, %rax ret #endif .p2align 4 LABEL(tail_0): /* 1 byte */ mov (%rsi), %cl mov %cl, (%rdi) #ifdef USE_AS_STRNCPY mov $1, %cl sub $1, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_1): /* 2 bytes */ mov (%rsi), %cx mov %cx, (%rdi) #ifdef USE_AS_STRNCPY mov $2, %cl sub $2, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_2): /* 3 bytes */ mov (%rsi), %cx mov %cx, (%rdi) mov 1(%rsi), %cx mov %cx, 1(%rdi) #ifdef USE_AS_STRNCPY mov $3, %cl sub $3, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_3): /* 4 bytes */ mov (%rsi), %ecx mov %ecx, (%rdi) #ifdef USE_AS_STRNCPY mov $4, %cl sub $4, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_4): /* 5 bytes */ mov (%rsi), %ecx mov %ecx, (%rdi) mov 1(%rsi), %edx mov %edx, 1(%rdi) #ifdef USE_AS_STRNCPY mov $5, %cl sub $5, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_5): /* 6 bytes */ mov (%rsi), %ecx mov %ecx, (%rdi) mov 2(%rsi), %edx mov %edx, 2(%rdi) #ifdef USE_AS_STRNCPY mov $6, %cl sub $6, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_6): /* 7 bytes */ mov (%rsi), %ecx mov %ecx, (%rdi) mov 3(%rsi), %edx mov %edx,3(%rdi) #ifdef USE_AS_STRNCPY mov $7, %cl sub $7, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_8): /* 9 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 5(%rsi), %edx mov %edx, 5(%rdi) #ifdef USE_AS_STRNCPY mov $9, %cl sub $9, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(AMD_exit_more_8): test %dh, %dh jz LABEL(AMD_exit_more_16) test $0x01, %dh jnz LABEL(tail_8) test $0x02, %dh jnz LABEL(tail_9) test $0x04, %dh jnz LABEL(tail_10) test $0x08, %dh jnz LABEL(tail_11) test $0x10, %dh jnz LABEL(tail_12) test $0x20, %dh jnz LABEL(tail_13) test $0x40, %dh jnz LABEL(tail_14) .p2align 4 LABEL(tail_15): /* 16 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) #ifdef USE_AS_STRNCPY mov $16, %cl sub $16, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_9): /* 10 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 6(%rsi), %edx mov %edx, 6(%rdi) #ifdef USE_AS_STRNCPY mov $10, %cl sub $10, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_10): /* 11 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 7(%rsi), %edx mov %edx, 7(%rdi) #ifdef USE_AS_STRNCPY mov $11, %cl sub $11, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_11): /* 12 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %edx mov %edx, 8(%rdi) #ifdef USE_AS_STRNCPY mov $12, %cl sub $12, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_12): /* 13 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 5(%rsi), %rcx mov %rcx, 5(%rdi) #ifdef USE_AS_STRNCPY mov $13, %cl sub $13, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_13): /* 14 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 6(%rsi), %rcx mov %rcx, 6(%rdi) #ifdef USE_AS_STRNCPY mov $14, %cl sub $14, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_14): /* 15 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 7(%rsi), %rcx mov %rcx, 7(%rdi) #ifdef USE_AS_STRNCPY mov $15, %cl sub $15, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(AMD_exit_more_16): shr $16, %edx test %dl, %dl jz LABEL(AMD_exit_more_24) test $0x01, %dl jnz LABEL(tail_16) test $0x02, %dl jnz LABEL(tail_17) test $0x04, %dl jnz LABEL(tail_18) test $0x08, %dl jnz LABEL(tail_19) test $0x10, %dl jnz LABEL(tail_20) test $0x20, %dl jnz LABEL(tail_21) test $0x40, %dl jnz LABEL(tail_22) .p2align 4 LABEL(tail_23): /* 24 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %rcx mov %rcx, 16(%rdi) #ifdef USE_AS_STRNCPY mov $24, %cl sub $24, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_16): /* 17 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %cl mov %cl, 16(%rdi) #ifdef USE_AS_STRNCPY mov $17, %cl sub $17, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_17): /* 18 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %cx mov %cx, 16(%rdi) #ifdef USE_AS_STRNCPY mov $18, %cl sub $18, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_18): /* 19 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 15(%rsi), %ecx mov %ecx,15(%rdi) #ifdef USE_AS_STRNCPY mov $19, %cl sub $19, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_19): /* 20 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %ecx mov %ecx, 16(%rdi) #ifdef USE_AS_STRNCPY mov $20, %cl sub $20, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_20): /* 21 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 13(%rsi), %rcx mov %rcx, 13(%rdi) #ifdef USE_AS_STRNCPY mov $21, %cl sub $21, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_21): /* 22 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 14(%rsi), %rcx mov %rcx, 14(%rdi) #ifdef USE_AS_STRNCPY mov $22, %cl sub $22, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_22): /* 23 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 15(%rsi), %rcx mov %rcx, 15(%rdi) #ifdef USE_AS_STRNCPY mov $23, %cl sub $23, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(AMD_exit_more_24): test $0x01, %dh jnz LABEL(tail_24) test $0x02, %dh jnz LABEL(tail_25) test $0x04, %dh jnz LABEL(tail_26) test $0x08, %dh jnz LABEL(tail_27) test $0x10, %dh jnz LABEL(tail_28) test $0x20, %dh jnz LABEL(tail_29) test $0x40, %dh jnz LABEL(tail_30) .p2align 4 LABEL(tail_31): /* 32 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %rcx mov %rcx, 16(%rdi) mov 24(%rsi), %rdx mov %rdx, 24(%rdi) #ifdef USE_AS_STRNCPY mov $32, %cl sub $32, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_24): /* 25 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %rcx mov %rcx, 16(%rdi) mov 21(%rsi), %edx mov %edx, 21(%rdi) #ifdef USE_AS_STRNCPY mov $25, %cl sub $25, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_25): /* 26 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %rcx mov %rcx, 16(%rdi) mov 22(%rsi), %edx mov %edx, 22(%rdi) #ifdef USE_AS_STRNCPY mov $26, %cl sub $26, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_26): /* 27 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %rcx mov %rcx, 16(%rdi) mov 23(%rsi), %edx mov %edx, 23(%rdi) #ifdef USE_AS_STRNCPY mov $27, %cl sub $27, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_27): /* 28 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %rcx mov %rcx, 16(%rdi) mov 24(%rsi), %edx mov %edx, 24(%rdi) #ifdef USE_AS_STRNCPY mov $28, %cl sub $28, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_28): /* 29 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %rcx mov %rcx, 16(%rdi) mov 21(%rsi), %rdx mov %rdx, 21(%rdi) #ifdef USE_AS_STRNCPY mov $29, %cl sub $29, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_29): /* 30 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %rcx mov %rcx, 16(%rdi) mov 22(%rsi), %rdx mov %rdx, 22(%rdi) #ifdef USE_AS_STRNCPY mov $30, %cl sub $30, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .p2align 4 LABEL(tail_30): /* 31 bytes */ mov (%rsi), %rcx mov %rcx, (%rdi) mov 8(%rsi), %rdx mov %rdx, 8(%rdi) mov 16(%rsi), %rcx mov %rcx, 16(%rdi) mov 23(%rsi), %rdx mov %rdx, 23(%rdi) #ifdef USE_AS_STRNCPY mov $31, %cl sub $31, %r8 jnz LABEL(strncpy_fill_tail) #endif ret .pushsection .rodata .p2align 4 LABEL(tail_table): .int LABEL(tail_0) - LABEL(tail_table) /* 1 byte */ .int LABEL(tail_1) - LABEL(tail_table) .int LABEL(tail_2) - LABEL(tail_table) .int LABEL(tail_3) - LABEL(tail_table) .int LABEL(tail_4) - LABEL(tail_table) .int LABEL(tail_5) - LABEL(tail_table) .int LABEL(tail_6) - LABEL(tail_table) .int LABEL(tail_7) - LABEL(tail_table) .int LABEL(tail_8) - LABEL(tail_table) .int LABEL(tail_9) - LABEL(tail_table) .int LABEL(tail_10) - LABEL(tail_table) .int LABEL(tail_11) - LABEL(tail_table) .int LABEL(tail_12) - LABEL(tail_table) .int LABEL(tail_13) - LABEL(tail_table) .int LABEL(tail_14) - LABEL(tail_table) .int LABEL(tail_15) - LABEL(tail_table) .int LABEL(tail_16) - LABEL(tail_table) .int LABEL(tail_17) - LABEL(tail_table) .int LABEL(tail_18) - LABEL(tail_table) .int LABEL(tail_19) - LABEL(tail_table) .int LABEL(tail_20) - LABEL(tail_table) .int LABEL(tail_21) - LABEL(tail_table) .int LABEL(tail_22) - LABEL(tail_table) .int LABEL(tail_23) - LABEL(tail_table) .int LABEL(tail_24) - LABEL(tail_table) .int LABEL(tail_25) - LABEL(tail_table) .int LABEL(tail_26) - LABEL(tail_table) .int LABEL(tail_27) - LABEL(tail_table) .int LABEL(tail_28) - LABEL(tail_table) .int LABEL(tail_29) - LABEL(tail_table) .int LABEL(tail_30) - LABEL(tail_table) .int LABEL(tail_31) - LABEL(tail_table) /* 32 bytes */ .p2align 4 LABEL(unaligned_table): .int LABEL(ashr_0) - LABEL(unaligned_table) .int LABEL(ashr_1) - LABEL(unaligned_table) .int LABEL(ashr_2) - LABEL(unaligned_table) .int LABEL(ashr_3) - LABEL(unaligned_table) .int LABEL(ashr_4) - LABEL(unaligned_table) .int LABEL(ashr_5) - LABEL(unaligned_table) .int LABEL(ashr_6) - LABEL(unaligned_table) .int LABEL(ashr_7) - LABEL(unaligned_table) .int LABEL(ashr_8) - LABEL(unaligned_table) .int LABEL(ashr_9) - LABEL(unaligned_table) .int LABEL(ashr_10) - LABEL(unaligned_table) .int LABEL(ashr_11) - LABEL(unaligned_table) .int LABEL(ashr_12) - LABEL(unaligned_table) .int LABEL(ashr_13) - LABEL(unaligned_table) .int LABEL(ashr_14) - LABEL(unaligned_table) .int LABEL(ashr_15) - LABEL(unaligned_table) .popsection #ifdef USE_AS_STRNCPY SET_SIZE(strncpy) #else SET_SIZE(strcpy) /* (char *, const char *) */ #endif