1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * This file contains assembly-language implementations 4 * of IP-style 1's complement checksum routines. 5 * 6 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 7 * 8 * Severely hacked about by Paul Mackerras (paulus@cs.anu.edu.au). 9 */ 10 11#include <linux/sys.h> 12#include <asm/processor.h> 13#include <asm/cache.h> 14#include <asm/errno.h> 15#include <asm/ppc_asm.h> 16#include <asm/export.h> 17 18 .text 19 20/* 21 * computes the checksum of a memory block at buff, length len, 22 * and adds in "sum" (32-bit) 23 * 24 * __csum_partial(buff, len, sum) 25 */ 26_GLOBAL(__csum_partial) 27 subi r3,r3,4 28 srawi. r6,r4,2 /* Divide len by 4 and also clear carry */ 29 beq 3f /* if we're doing < 4 bytes */ 30 andi. r0,r3,2 /* Align buffer to longword boundary */ 31 beq+ 1f 32 lhz r0,4(r3) /* do 2 bytes to get aligned */ 33 subi r4,r4,2 34 addi r3,r3,2 35 srwi. r6,r4,2 /* # words to do */ 36 adde r5,r5,r0 37 beq 3f 381: andi. r6,r6,3 /* Prepare to handle words 4 by 4 */ 39 beq 21f 40 mtctr r6 412: lwzu r0,4(r3) 42 adde r5,r5,r0 43 bdnz 2b 4421: srwi. r6,r4,4 /* # blocks of 4 words to do */ 45 beq 3f 46 lwz r0,4(r3) 47 mtctr r6 48 lwz r6,8(r3) 49 adde r5,r5,r0 50 lwz r7,12(r3) 51 adde r5,r5,r6 52 lwzu r8,16(r3) 53 adde r5,r5,r7 54 bdz 23f 5522: lwz r0,4(r3) 56 adde r5,r5,r8 57 lwz r6,8(r3) 58 adde r5,r5,r0 59 lwz r7,12(r3) 60 adde r5,r5,r6 61 lwzu r8,16(r3) 62 adde r5,r5,r7 63 bdnz 22b 6423: adde r5,r5,r8 653: andi. r0,r4,2 66 beq+ 4f 67 lhz r0,4(r3) 68 addi r3,r3,2 69 adde r5,r5,r0 704: andi. r0,r4,1 71 beq+ 5f 72 lbz r0,4(r3) 73 slwi r0,r0,8 /* Upper byte of word */ 74 adde r5,r5,r0 755: addze r3,r5 /* add in final carry */ 76 blr 77EXPORT_SYMBOL(__csum_partial) 78 79/* 80 * Computes the checksum of a memory block at src, length len, 81 * and adds in 0xffffffff, while copying the block to dst. 82 * If an access exception occurs it returns zero. 83 * 84 * csum_partial_copy_generic(src, dst, len) 85 */ 86#define CSUM_COPY_16_BYTES_WITHEX(n) \ 878 ## n ## 0: \ 88 lwz r7,4(r4); \ 898 ## n ## 1: \ 90 lwz r8,8(r4); \ 918 ## n ## 2: \ 92 lwz r9,12(r4); \ 938 ## n ## 3: \ 94 lwzu r10,16(r4); \ 958 ## n ## 4: \ 96 stw r7,4(r6); \ 97 adde r12,r12,r7; \ 988 ## n ## 5: \ 99 stw r8,8(r6); \ 100 adde r12,r12,r8; \ 1018 ## n ## 6: \ 102 stw r9,12(r6); \ 103 adde r12,r12,r9; \ 1048 ## n ## 7: \ 105 stwu r10,16(r6); \ 106 adde r12,r12,r10 107 108#define CSUM_COPY_16_BYTES_EXCODE(n) \ 109 EX_TABLE(8 ## n ## 0b, fault); \ 110 EX_TABLE(8 ## n ## 1b, fault); \ 111 EX_TABLE(8 ## n ## 2b, fault); \ 112 EX_TABLE(8 ## n ## 3b, fault); \ 113 EX_TABLE(8 ## n ## 4b, fault); \ 114 EX_TABLE(8 ## n ## 5b, fault); \ 115 EX_TABLE(8 ## n ## 6b, fault); \ 116 EX_TABLE(8 ## n ## 7b, fault); 117 118 .text 119 .stabs "arch/powerpc/lib/",N_SO,0,0,0f 120 .stabs "checksum_32.S",N_SO,0,0,0f 1210: 122 123CACHELINE_BYTES = L1_CACHE_BYTES 124LG_CACHELINE_BYTES = L1_CACHE_SHIFT 125CACHELINE_MASK = (L1_CACHE_BYTES-1) 126 127_GLOBAL(csum_partial_copy_generic) 128 li r12,-1 129 addic r0,r0,0 /* clear carry */ 130 addi r6,r4,-4 131 neg r0,r4 132 addi r4,r3,-4 133 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */ 134 crset 4*cr7+eq 135 beq 58f 136 137 cmplw 0,r5,r0 /* is this more than total to do? */ 138 blt 63f /* if not much to do */ 139 rlwinm r7,r6,3,0x8 140 rlwnm r12,r12,r7,0,31 /* odd destination address: rotate one byte */ 141 cmplwi cr7,r7,0 /* is destination address even ? */ 142 andi. r8,r0,3 /* get it word-aligned first */ 143 mtctr r8 144 beq+ 61f 145 li r3,0 14670: lbz r9,4(r4) /* do some bytes */ 147 addi r4,r4,1 148 slwi r3,r3,8 149 rlwimi r3,r9,0,24,31 15071: stb r9,4(r6) 151 addi r6,r6,1 152 bdnz 70b 153 adde r12,r12,r3 15461: subf r5,r0,r5 155 srwi. r0,r0,2 156 mtctr r0 157 beq 58f 15872: lwzu r9,4(r4) /* do some words */ 159 adde r12,r12,r9 16073: stwu r9,4(r6) 161 bdnz 72b 162 16358: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */ 164 clrlwi r5,r5,32-LG_CACHELINE_BYTES 165 li r11,4 166 beq 63f 167 168 /* Here we decide how far ahead to prefetch the source */ 169 li r3,4 170 cmpwi r0,1 171 li r7,0 172 ble 114f 173 li r7,1 174#if MAX_COPY_PREFETCH > 1 175 /* Heuristically, for large transfers we prefetch 176 MAX_COPY_PREFETCH cachelines ahead. For small transfers 177 we prefetch 1 cacheline ahead. */ 178 cmpwi r0,MAX_COPY_PREFETCH 179 ble 112f 180 li r7,MAX_COPY_PREFETCH 181112: mtctr r7 182111: dcbt r3,r4 183 addi r3,r3,CACHELINE_BYTES 184 bdnz 111b 185#else 186 dcbt r3,r4 187 addi r3,r3,CACHELINE_BYTES 188#endif /* MAX_COPY_PREFETCH > 1 */ 189 190114: subf r8,r7,r0 191 mr r0,r7 192 mtctr r8 193 19453: dcbt r3,r4 19554: dcbz r11,r6 196/* the main body of the cacheline loop */ 197 CSUM_COPY_16_BYTES_WITHEX(0) 198#if L1_CACHE_BYTES >= 32 199 CSUM_COPY_16_BYTES_WITHEX(1) 200#if L1_CACHE_BYTES >= 64 201 CSUM_COPY_16_BYTES_WITHEX(2) 202 CSUM_COPY_16_BYTES_WITHEX(3) 203#if L1_CACHE_BYTES >= 128 204 CSUM_COPY_16_BYTES_WITHEX(4) 205 CSUM_COPY_16_BYTES_WITHEX(5) 206 CSUM_COPY_16_BYTES_WITHEX(6) 207 CSUM_COPY_16_BYTES_WITHEX(7) 208#endif 209#endif 210#endif 211 bdnz 53b 212 cmpwi r0,0 213 li r3,4 214 li r7,0 215 bne 114b 216 21763: srwi. r0,r5,2 218 mtctr r0 219 beq 64f 22030: lwzu r0,4(r4) 221 adde r12,r12,r0 22231: stwu r0,4(r6) 223 bdnz 30b 224 22564: andi. r0,r5,2 226 beq+ 65f 22740: lhz r0,4(r4) 228 addi r4,r4,2 22941: sth r0,4(r6) 230 adde r12,r12,r0 231 addi r6,r6,2 23265: andi. r0,r5,1 233 beq+ 66f 23450: lbz r0,4(r4) 23551: stb r0,4(r6) 236 slwi r0,r0,8 237 adde r12,r12,r0 23866: addze r3,r12 239 beqlr+ cr7 240 rlwinm r3,r3,8,0,31 /* odd destination address: rotate one byte */ 241 blr 242 243fault: 244 li r3,0 245 blr 246 247 EX_TABLE(70b, fault); 248 EX_TABLE(71b, fault); 249 EX_TABLE(72b, fault); 250 EX_TABLE(73b, fault); 251 EX_TABLE(54b, fault); 252 253/* 254 * this stuff handles faults in the cacheline loop and branches to either 255 * fault (if in read part) or fault (if in write part) 256 */ 257 CSUM_COPY_16_BYTES_EXCODE(0) 258#if L1_CACHE_BYTES >= 32 259 CSUM_COPY_16_BYTES_EXCODE(1) 260#if L1_CACHE_BYTES >= 64 261 CSUM_COPY_16_BYTES_EXCODE(2) 262 CSUM_COPY_16_BYTES_EXCODE(3) 263#if L1_CACHE_BYTES >= 128 264 CSUM_COPY_16_BYTES_EXCODE(4) 265 CSUM_COPY_16_BYTES_EXCODE(5) 266 CSUM_COPY_16_BYTES_EXCODE(6) 267 CSUM_COPY_16_BYTES_EXCODE(7) 268#endif 269#endif 270#endif 271 272 EX_TABLE(30b, fault); 273 EX_TABLE(31b, fault); 274 EX_TABLE(40b, fault); 275 EX_TABLE(41b, fault); 276 EX_TABLE(50b, fault); 277 EX_TABLE(51b, fault); 278 279EXPORT_SYMBOL(csum_partial_copy_generic) 280 281/* 282 * __sum16 csum_ipv6_magic(const struct in6_addr *saddr, 283 * const struct in6_addr *daddr, 284 * __u32 len, __u8 proto, __wsum sum) 285 */ 286 287_GLOBAL(csum_ipv6_magic) 288 lwz r8, 0(r3) 289 lwz r9, 4(r3) 290 addc r0, r7, r8 291 lwz r10, 8(r3) 292 adde r0, r0, r9 293 lwz r11, 12(r3) 294 adde r0, r0, r10 295 lwz r8, 0(r4) 296 adde r0, r0, r11 297 lwz r9, 4(r4) 298 adde r0, r0, r8 299 lwz r10, 8(r4) 300 adde r0, r0, r9 301 lwz r11, 12(r4) 302 adde r0, r0, r10 303 add r5, r5, r6 /* assumption: len + proto doesn't carry */ 304 adde r0, r0, r11 305 adde r0, r0, r5 306 addze r0, r0 307 rotlwi r3, r0, 16 308 add r3, r0, r3 309 not r3, r3 310 rlwinm r3, r3, 16, 16, 31 311 blr 312EXPORT_SYMBOL(csum_ipv6_magic) 313