1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * Copyright (C) 2002 Paul Mackerras, IBM Corp. 4 */ 5#include <linux/export.h> 6#include <asm/processor.h> 7#include <asm/ppc_asm.h> 8#include <asm/asm-compat.h> 9#include <asm/feature-fixups.h> 10 11#ifndef SELFTEST_CASE 12/* For big-endian, 0 == most CPUs, 1 == POWER6, 2 == Cell */ 13#define SELFTEST_CASE 0 14#endif 15 16 .align 7 17_GLOBAL_TOC(memcpy) 18BEGIN_FTR_SECTION 19#ifdef __LITTLE_ENDIAN__ 20 cmpdi cr7,r5,0 21#else 22 std r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* save destination pointer for return value */ 23#endif 24FTR_SECTION_ELSE 25#ifdef CONFIG_PPC_BOOK3S_64 26 b memcpy_power7 27#endif 28ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY) 29#ifdef __LITTLE_ENDIAN__ 30 /* dumb little-endian memcpy that will get replaced at runtime */ 31 addi r9,r3,-1 32 addi r4,r4,-1 33 beqlr cr7 34 mtctr r5 351: lbzu r10,1(r4) 36 stbu r10,1(r9) 37 bdnz 1b 38 blr 39#else 40 PPC_MTOCRF(0x01,r5) 41 cmpldi cr1,r5,16 42 neg r6,r3 # LS 3 bits = # bytes to 8-byte dest bdry 43 andi. r6,r6,7 44 dcbt 0,r4 45 blt cr1,.Lshort_copy 46/* Below we want to nop out the bne if we're on a CPU that has the 47 CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit 48 cleared. 49 At the time of writing the only CPU that has this combination of bits 50 set is Power6. */ 51test_feature = (SELFTEST_CASE == 1) 52BEGIN_FTR_SECTION 53 nop 54FTR_SECTION_ELSE 55 bne .Ldst_unaligned 56ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \ 57 CPU_FTR_UNALIGNED_LD_STD) 58.Ldst_aligned: 59 addi r3,r3,-16 60test_feature = (SELFTEST_CASE == 0) 61BEGIN_FTR_SECTION 62 andi. r0,r4,7 63 bne .Lsrc_unaligned 64END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) 65 srdi r7,r5,4 66 ld r9,0(r4) 67 addi r4,r4,-8 68 mtctr r7 69 andi. r5,r5,7 70 bf cr7*4+0,2f 71 addi r3,r3,8 72 addi r4,r4,8 73 mr r8,r9 74 blt cr1,3f 751: ld r9,8(r4) 76 std r8,8(r3) 772: ldu r8,16(r4) 78 stdu r9,16(r3) 79 bdnz 1b 803: std r8,8(r3) 81 beq 3f 82 addi r3,r3,16 83.Ldo_tail: 84 bf cr7*4+1,1f 85 lwz r9,8(r4) 86 addi r4,r4,4 87 stw r9,0(r3) 88 addi r3,r3,4 891: bf cr7*4+2,2f 90 lhz r9,8(r4) 91 addi r4,r4,2 92 sth r9,0(r3) 93 addi r3,r3,2 942: bf cr7*4+3,3f 95 lbz r9,8(r4) 96 stb r9,0(r3) 973: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */ 98 blr 99 100.Lsrc_unaligned: 101 srdi r6,r5,3 102 addi r5,r5,-16 103 subf r4,r0,r4 104 srdi r7,r5,4 105 sldi r10,r0,3 106 cmpdi cr6,r6,3 107 andi. r5,r5,7 108 mtctr r7 109 subfic r11,r10,64 110 add r5,r5,r0 111 112 bt cr7*4+0,0f 113 114 ld r9,0(r4) # 3+2n loads, 2+2n stores 115 ld r0,8(r4) 116 sld r6,r9,r10 117 ldu r9,16(r4) 118 srd r7,r0,r11 119 sld r8,r0,r10 120 or r7,r7,r6 121 blt cr6,4f 122 ld r0,8(r4) 123 # s1<< in r8, d0=(s0<<|s1>>) in r7, s3 in r0, s2 in r9, nix in r6 & r12 124 b 2f 125 1260: ld r0,0(r4) # 4+2n loads, 3+2n stores 127 ldu r9,8(r4) 128 sld r8,r0,r10 129 addi r3,r3,-8 130 blt cr6,5f 131 ld r0,8(r4) 132 srd r12,r9,r11 133 sld r6,r9,r10 134 ldu r9,16(r4) 135 or r12,r8,r12 136 srd r7,r0,r11 137 sld r8,r0,r10 138 addi r3,r3,16 139 beq cr6,3f 140 141 # d0=(s0<<|s1>>) in r12, s1<< in r6, s2>> in r7, s2<< in r8, s3 in r9 1421: or r7,r7,r6 143 ld r0,8(r4) 144 std r12,8(r3) 1452: srd r12,r9,r11 146 sld r6,r9,r10 147 ldu r9,16(r4) 148 or r12,r8,r12 149 stdu r7,16(r3) 150 srd r7,r0,r11 151 sld r8,r0,r10 152 bdnz 1b 153 1543: std r12,8(r3) 155 or r7,r7,r6 1564: std r7,16(r3) 1575: srd r12,r9,r11 158 or r12,r8,r12 159 std r12,24(r3) 160 beq 4f 161 cmpwi cr1,r5,8 162 addi r3,r3,32 163 sld r9,r9,r10 164 ble cr1,6f 165 ld r0,8(r4) 166 srd r7,r0,r11 167 or r9,r7,r9 1686: 169 bf cr7*4+1,1f 170 rotldi r9,r9,32 171 stw r9,0(r3) 172 addi r3,r3,4 1731: bf cr7*4+2,2f 174 rotldi r9,r9,16 175 sth r9,0(r3) 176 addi r3,r3,2 1772: bf cr7*4+3,3f 178 rotldi r9,r9,8 179 stb r9,0(r3) 1803: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */ 181 blr 182 183.Ldst_unaligned: 184 PPC_MTOCRF(0x01,r6) # put #bytes to 8B bdry into cr7 185 subf r5,r6,r5 186 li r7,0 187 cmpldi cr1,r5,16 188 bf cr7*4+3,1f 189 lbz r0,0(r4) 190 stb r0,0(r3) 191 addi r7,r7,1 1921: bf cr7*4+2,2f 193 lhzx r0,r7,r4 194 sthx r0,r7,r3 195 addi r7,r7,2 1962: bf cr7*4+1,3f 197 lwzx r0,r7,r4 198 stwx r0,r7,r3 1993: PPC_MTOCRF(0x01,r5) 200 add r4,r6,r4 201 add r3,r6,r3 202 b .Ldst_aligned 203 204.Lshort_copy: 205 bf cr7*4+0,1f 206 lwz r0,0(r4) 207 lwz r9,4(r4) 208 addi r4,r4,8 209 stw r0,0(r3) 210 stw r9,4(r3) 211 addi r3,r3,8 2121: bf cr7*4+1,2f 213 lwz r0,0(r4) 214 addi r4,r4,4 215 stw r0,0(r3) 216 addi r3,r3,4 2172: bf cr7*4+2,3f 218 lhz r0,0(r4) 219 addi r4,r4,2 220 sth r0,0(r3) 221 addi r3,r3,2 2223: bf cr7*4+3,4f 223 lbz r0,0(r4) 224 stb r0,0(r3) 2254: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */ 226 blr 227#endif 228EXPORT_SYMBOL(memcpy) 229