1/*- 2 * Copyright (c) 2018 Instituto de Pesquisas Eldorado 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. Neither the name of the author nor the names of its contributors may 14 * be used to endorse or promote products derived from this software 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 */ 29 30#include <machine/asm.h> 31__FBSDID("$FreeBSD$"); 32 33#ifndef FN_NAME 34#define FN_NAME __memcpy 35WEAK_REFERENCE(__memcpy, memcpy); 36#define BLOCK_BITS 4 37#endif 38 39#define BLOCK_BYTES (1 << BLOCK_BITS) 40#define BLOCK_MASK (BLOCK_BYTES - 1) 41 42/* Minimum 8 byte alignment, to avoid cache-inhibited alignment faults. */ 43#ifndef ALIGN_MASK 44#define ALIGN_MASK 0x7 45#endif 46 47/* 48 * r3: dst 49 * r4: src 50 * r5: len 51 */ 52ENTRY(FN_NAME) 53 cmpdi %r5, 0 /* len == 0? nothing to do */ 54 beqlr- 55 56 /* If src and dst are relatively misaligned, do byte copies. */ 57 andi. %r8, %r3, ALIGN_MASK 58 andi. %r7, %r4, ALIGN_MASK 59 cmpd %r8, %r7 60 mr %r7, %r5 61 mr %r8, %r3 /* save dst */ 62 bne .Lcopy_remaining_fix_index_byte 63 64 /* align src */ 65.Lalignment_loop: 66 lbz %r6, 0(%r4) 67 stb %r6, 0(%r3) 68 addi %r3, %r3, 1 69 addi %r4, %r4, 1 70 addi %r5, %r5, -1 71 cmpdi %r5, 0 72 beq .Lexit 73 andi. %r0, %r4, BLOCK_MASK 74 bne .Lalignment_loop 75 76 /* r7 = remaining, non-block, bytes */ 77 andi. %r7, %r5, BLOCK_MASK 78 79 /* Check if there are blocks of BLOCK_BYTES to be copied */ 80 xor. %r5, %r5, %r7 81 beq .Lcopy_remaining_fix_index_byte 82 83#ifdef FN_COPY_LOOP 84FN_COPY_LOOP 85#else 86 /* Setup to copy word with ldu and stdu */ 87 ld %r6, 0(%r4) 88 ld %r9, 8(%r4) 89 std %r6, 0(%r3) 90 std %r9, 8(%r3) 91 addi %r5, %r5, -BLOCK_BYTES 92 cmpd %r5, 0 93 beq .Lcopy_remaining_fix_index_word 94 95 srdi %r5, %r5, BLOCK_BITS 96 mtctr %r5 97.Lcopy_word: 98 ldu %r6, 16(%r4) 99 ld %r9, 8(%r4) 100 stdu %r6, 16(%r3) 101 std %r9, 8(%r3) 102 bdnz .Lcopy_word 103 104.Lcopy_remaining_fix_index_word: 105 /* Check if there are remaining bytes */ 106 cmpd %r7, 0 107 beq .Lexit 108 addi %r3, %r3, BLOCK_MASK 109 addi %r4, %r4, BLOCK_MASK 110 b .Lcopy_remaining 111#endif 112 113.Lcopy_remaining_fix_index_byte: 114 addi %r4, %r4, -1 115 addi %r3, %r3, -1 116 117 /* Copy remaining bytes */ 118.Lcopy_remaining: 119 mtctr %r7 120.Lcopy_remaining_loop: 121 lbzu %r6, 1(%r4) 122 stbu %r6, 1(%r3) 123 bdnz .Lcopy_remaining_loop 124 125.Lexit: 126 /* Restore dst */ 127 mr %r3, %r8 128 blr 129 130END(FN_NAME) 131 132 .section .note.GNU-stack,"",%progbits 133 134