1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/export.h> 4 #include <asm/checksum.h> 5 #include <asm/fpu.h> 6 7 /* 8 * Computes the checksum of a memory block at src, length len, 9 * and adds in "sum" (32-bit). If copy is true copies to dst. 10 * 11 * Returns a 32-bit number suitable for feeding into itself 12 * or csum_tcpudp_magic. 13 * 14 * This function must be called with even lengths, except 15 * for the last fragment, which may be odd. 16 * 17 * It's best to have src and dst aligned on a 64-bit boundary. 18 */ 19 static __always_inline __wsum csum_copy(void *dst, const void *src, int len, __wsum sum, bool copy) 20 { 21 DECLARE_KERNEL_FPU_ONSTACK8(vxstate); 22 23 if (!cpu_has_vx()) { 24 if (copy) 25 memcpy(dst, src, len); 26 return cksm(dst, len, sum); 27 } 28 kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23); 29 fpu_vlvgf(16, (__force u32)sum, 1); 30 fpu_vzero(17); 31 fpu_vzero(18); 32 fpu_vzero(19); 33 while (len >= 64) { 34 fpu_vlm(20, 23, src); 35 if (copy) { 36 fpu_vstm(20, 23, dst); 37 dst += 64; 38 } 39 fpu_vcksm(16, 20, 16); 40 fpu_vcksm(17, 21, 17); 41 fpu_vcksm(18, 22, 18); 42 fpu_vcksm(19, 23, 19); 43 src += 64; 44 len -= 64; 45 } 46 while (len >= 32) { 47 fpu_vlm(20, 21, src); 48 if (copy) { 49 fpu_vstm(20, 21, dst); 50 dst += 32; 51 } 52 fpu_vcksm(16, 20, 16); 53 fpu_vcksm(17, 21, 17); 54 src += 32; 55 len -= 32; 56 } 57 while (len >= 16) { 58 fpu_vl(20, src); 59 if (copy) { 60 fpu_vst(20, dst); 61 dst += 16; 62 } 63 fpu_vcksm(16, 20, 16); 64 src += 16; 65 len -= 16; 66 } 67 if (len) { 68 fpu_vll(20, len - 1, src); 69 if (copy) 70 fpu_vstl(20, len - 1, dst); 71 fpu_vcksm(16, 20, 16); 72 } 73 fpu_vcksm(18, 19, 18); 74 fpu_vcksm(16, 17, 16); 75 fpu_vcksm(16, 18, 16); 76 sum = (__force __wsum)fpu_vlgvf(16, 1); 77 kernel_fpu_end(&vxstate, KERNEL_VXR_V16V23); 78 return sum; 79 } 80 81 __wsum csum_partial(const void *buff, int len, __wsum sum) 82 { 83 return csum_copy(NULL, buff, len, sum, false); 84 } 85 EXPORT_SYMBOL(csum_partial); 86 87 __wsum csum_partial_copy_nocheck(const void *src, void *dst, int len) 88 { 89 return csum_copy(dst, src, len, 0, true); 90 } 91 EXPORT_SYMBOL(csum_partial_copy_nocheck); 92