1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Checksum library 4 * 5 * Influenced by arch/arm64/lib/csum.c 6 * Copyright (C) 2023-2024 Rivos Inc. 7 */ 8 #include <linux/bitops.h> 9 #include <linux/compiler.h> 10 #include <linux/jump_label.h> 11 #include <linux/kasan-checks.h> 12 #include <linux/kernel.h> 13 14 #include <asm/cpufeature.h> 15 16 #include <net/checksum.h> 17 18 /* Default version is sufficient for 32 bit */ 19 #ifndef CONFIG_32BIT 20 __sum16 csum_ipv6_magic(const struct in6_addr *saddr, 21 const struct in6_addr *daddr, 22 __u32 len, __u8 proto, __wsum csum) 23 { 24 unsigned int ulen, uproto; 25 unsigned long sum = (__force unsigned long)csum; 26 27 sum += (__force unsigned long)saddr->s6_addr32[0]; 28 sum += (__force unsigned long)saddr->s6_addr32[1]; 29 sum += (__force unsigned long)saddr->s6_addr32[2]; 30 sum += (__force unsigned long)saddr->s6_addr32[3]; 31 32 sum += (__force unsigned long)daddr->s6_addr32[0]; 33 sum += (__force unsigned long)daddr->s6_addr32[1]; 34 sum += (__force unsigned long)daddr->s6_addr32[2]; 35 sum += (__force unsigned long)daddr->s6_addr32[3]; 36 37 ulen = (__force unsigned int)htonl((unsigned int)len); 38 sum += ulen; 39 40 uproto = (__force unsigned int)htonl(proto); 41 sum += uproto; 42 43 if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { 44 unsigned long fold_temp; 45 46 /* 47 * Zbb is likely available when the kernel is compiled with Zbb 48 * support, so nop when Zbb is available and jump when Zbb is 49 * not available. 50 */ 51 asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, 52 RISCV_ISA_EXT_ZBB, 1) 53 : 54 : 55 : 56 : no_zbb); 57 asm(".option push \n\ 58 .option arch,+zbb \n\ 59 rori %[fold_temp], %[sum], 32 \n\ 60 add %[sum], %[fold_temp], %[sum] \n\ 61 srli %[sum], %[sum], 32 \n\ 62 not %[fold_temp], %[sum] \n\ 63 roriw %[sum], %[sum], 16 \n\ 64 subw %[sum], %[fold_temp], %[sum] \n\ 65 .option pop" 66 : [sum] "+r" (sum), [fold_temp] "=&r" (fold_temp)); 67 return (__force __sum16)(sum >> 16); 68 } 69 no_zbb: 70 sum += ror64(sum, 32); 71 sum >>= 32; 72 return csum_fold((__force __wsum)sum); 73 } 74 EXPORT_SYMBOL(csum_ipv6_magic); 75 #endif /* !CONFIG_32BIT */ 76 77 #ifdef CONFIG_32BIT 78 #define OFFSET_MASK 3 79 #elif CONFIG_64BIT 80 #define OFFSET_MASK 7 81 #endif 82 83 static inline __no_sanitize_address unsigned long 84 do_csum_common(const unsigned long *ptr, const unsigned long *end, 85 unsigned long data) 86 { 87 unsigned int shift; 88 unsigned long csum = 0, carry = 0; 89 90 /* 91 * Do 32-bit reads on RV32 and 64-bit reads otherwise. This should be 92 * faster than doing 32-bit reads on architectures that support larger 93 * reads. 94 */ 95 while (ptr < end) { 96 csum += data; 97 carry += csum < data; 98 data = *(ptr++); 99 } 100 101 /* 102 * Perform alignment (and over-read) bytes on the tail if any bytes 103 * leftover. 104 */ 105 shift = ((long)ptr - (long)end) * 8; 106 #ifdef __LITTLE_ENDIAN 107 data = (data << shift) >> shift; 108 #else 109 data = (data >> shift) << shift; 110 #endif 111 csum += data; 112 carry += csum < data; 113 csum += carry; 114 csum += csum < carry; 115 116 return csum; 117 } 118 119 /* 120 * Algorithm accounts for buff being misaligned. 121 * If buff is not aligned, will over-read bytes but not use the bytes that it 122 * shouldn't. The same thing will occur on the tail-end of the read. 123 */ 124 static inline __no_sanitize_address unsigned int 125 do_csum_with_alignment(const unsigned char *buff, int len) 126 { 127 unsigned int offset, shift; 128 unsigned long csum, data; 129 const unsigned long *ptr, *end; 130 131 /* 132 * Align address to closest word (double word on rv64) that comes before 133 * buff. This should always be in the same page and cache line. 134 * Directly call KASAN with the alignment we will be using. 135 */ 136 offset = (unsigned long)buff & OFFSET_MASK; 137 kasan_check_read(buff, len); 138 ptr = (const unsigned long *)(buff - offset); 139 140 /* 141 * Clear the most significant bytes that were over-read if buff was not 142 * aligned. 143 */ 144 shift = offset * 8; 145 data = *(ptr++); 146 #ifdef __LITTLE_ENDIAN 147 data = (data >> shift) << shift; 148 #else 149 data = (data << shift) >> shift; 150 #endif 151 end = (const unsigned long *)(buff + len); 152 csum = do_csum_common(ptr, end, data); 153 154 #ifdef CC_HAS_ASM_GOTO_TIED_OUTPUT 155 if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { 156 unsigned long fold_temp; 157 158 /* 159 * Zbb is likely available when the kernel is compiled with Zbb 160 * support, so nop when Zbb is available and jump when Zbb is 161 * not available. 162 */ 163 asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, 164 RISCV_ISA_EXT_ZBB, 1) 165 : 166 : 167 : 168 : no_zbb); 169 170 #ifdef CONFIG_32BIT 171 asm_goto_output(".option push \n\ 172 .option arch,+zbb \n\ 173 rori %[fold_temp], %[csum], 16 \n\ 174 andi %[offset], %[offset], 1 \n\ 175 add %[csum], %[fold_temp], %[csum] \n\ 176 beq %[offset], zero, %l[end] \n\ 177 rev8 %[csum], %[csum] \n\ 178 .option pop" 179 : [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp) 180 : [offset] "r" (offset) 181 : 182 : end); 183 184 return (unsigned short)csum; 185 #else /* !CONFIG_32BIT */ 186 asm_goto_output(".option push \n\ 187 .option arch,+zbb \n\ 188 rori %[fold_temp], %[csum], 32 \n\ 189 add %[csum], %[fold_temp], %[csum] \n\ 190 srli %[csum], %[csum], 32 \n\ 191 roriw %[fold_temp], %[csum], 16 \n\ 192 addw %[csum], %[fold_temp], %[csum] \n\ 193 andi %[offset], %[offset], 1 \n\ 194 beq %[offset], zero, %l[end] \n\ 195 rev8 %[csum], %[csum] \n\ 196 .option pop" 197 : [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp) 198 : [offset] "r" (offset) 199 : 200 : end); 201 202 return (csum << 16) >> 48; 203 #endif /* !CONFIG_32BIT */ 204 end: 205 return csum >> 16; 206 } 207 no_zbb: 208 #endif /* CC_HAS_ASM_GOTO_TIED_OUTPUT */ 209 #ifndef CONFIG_32BIT 210 csum += ror64(csum, 32); 211 csum >>= 32; 212 #endif 213 csum = (u32)csum + ror32((u32)csum, 16); 214 if (offset & 1) 215 return (u16)swab32(csum); 216 return csum >> 16; 217 } 218 219 /* 220 * Does not perform alignment, should only be used if machine has fast 221 * misaligned accesses, or when buff is known to be aligned. 222 */ 223 static inline __no_sanitize_address unsigned int 224 do_csum_no_alignment(const unsigned char *buff, int len) 225 { 226 unsigned long csum, data; 227 const unsigned long *ptr, *end; 228 229 ptr = (const unsigned long *)(buff); 230 data = *(ptr++); 231 232 kasan_check_read(buff, len); 233 234 end = (const unsigned long *)(buff + len); 235 csum = do_csum_common(ptr, end, data); 236 237 if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { 238 unsigned long fold_temp; 239 240 /* 241 * Zbb is likely available when the kernel is compiled with Zbb 242 * support, so nop when Zbb is available and jump when Zbb is 243 * not available. 244 */ 245 asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, 246 RISCV_ISA_EXT_ZBB, 1) 247 : 248 : 249 : 250 : no_zbb); 251 252 #ifdef CONFIG_32BIT 253 asm (".option push \n\ 254 .option arch,+zbb \n\ 255 rori %[fold_temp], %[csum], 16 \n\ 256 add %[csum], %[fold_temp], %[csum] \n\ 257 .option pop" 258 : [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp) 259 : 260 : ); 261 262 #else /* !CONFIG_32BIT */ 263 asm (".option push \n\ 264 .option arch,+zbb \n\ 265 rori %[fold_temp], %[csum], 32 \n\ 266 add %[csum], %[fold_temp], %[csum] \n\ 267 srli %[csum], %[csum], 32 \n\ 268 roriw %[fold_temp], %[csum], 16 \n\ 269 addw %[csum], %[fold_temp], %[csum] \n\ 270 .option pop" 271 : [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp) 272 : 273 : ); 274 #endif /* !CONFIG_32BIT */ 275 return csum >> 16; 276 } 277 no_zbb: 278 #ifndef CONFIG_32BIT 279 csum += ror64(csum, 32); 280 csum >>= 32; 281 #endif 282 csum = (u32)csum + ror32((u32)csum, 16); 283 return csum >> 16; 284 } 285 286 /* 287 * Perform a checksum on an arbitrary memory address. 288 * Will do a light-weight address alignment if buff is misaligned, unless 289 * cpu supports fast misaligned accesses. 290 */ 291 unsigned int do_csum(const unsigned char *buff, int len) 292 { 293 if (unlikely(len <= 0)) 294 return 0; 295 296 /* 297 * Significant performance gains can be seen by not doing alignment 298 * on machines with fast misaligned accesses. 299 * 300 * There is some duplicate code between the "with_alignment" and 301 * "no_alignment" implmentations, but the overlap is too awkward to be 302 * able to fit in one function without introducing multiple static 303 * branches. The largest chunk of overlap was delegated into the 304 * do_csum_common function. 305 */ 306 if (has_fast_unaligned_accesses() || (((unsigned long)buff & OFFSET_MASK) == 0)) 307 return do_csum_no_alignment(buff, len); 308 309 return do_csum_with_alignment(buff, len); 310 } 311