1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * CRC32 and CRC32C using LoongArch crc* instructions 4 * 5 * Module based on mips/crypto/crc32-mips.c 6 * 7 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org> 8 * Copyright (C) 2018 MIPS Tech, LLC 9 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited 10 */ 11 12 #include <asm/cpu-features.h> 13 #include <linux/unaligned.h> 14 15 #define _CRC32(crc, value, size, type) \ 16 do { \ 17 __asm__ __volatile__( \ 18 #type ".w." #size ".w" " %0, %1, %0\n\t"\ 19 : "+r" (crc) \ 20 : "r" (value) \ 21 : "memory"); \ 22 } while (0) 23 24 #define CRC32(crc, value, size) _CRC32(crc, value, size, crc) 25 #define CRC32C(crc, value, size) _CRC32(crc, value, size, crcc) 26 27 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 28 29 static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 30 { 31 if (!static_branch_likely(&have_crc32)) 32 return crc32_le_base(crc, p, len); 33 34 while (len >= sizeof(u64)) { 35 u64 value = get_unaligned_le64(p); 36 37 CRC32(crc, value, d); 38 p += sizeof(u64); 39 len -= sizeof(u64); 40 } 41 42 if (len & sizeof(u32)) { 43 u32 value = get_unaligned_le32(p); 44 45 CRC32(crc, value, w); 46 p += sizeof(u32); 47 } 48 49 if (len & sizeof(u16)) { 50 u16 value = get_unaligned_le16(p); 51 52 CRC32(crc, value, h); 53 p += sizeof(u16); 54 } 55 56 if (len & sizeof(u8)) { 57 u8 value = *p++; 58 59 CRC32(crc, value, b); 60 } 61 62 return crc; 63 } 64 65 static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 66 { 67 if (!static_branch_likely(&have_crc32)) 68 return crc32c_base(crc, p, len); 69 70 while (len >= sizeof(u64)) { 71 u64 value = get_unaligned_le64(p); 72 73 CRC32C(crc, value, d); 74 p += sizeof(u64); 75 len -= sizeof(u64); 76 } 77 78 if (len & sizeof(u32)) { 79 u32 value = get_unaligned_le32(p); 80 81 CRC32C(crc, value, w); 82 p += sizeof(u32); 83 } 84 85 if (len & sizeof(u16)) { 86 u16 value = get_unaligned_le16(p); 87 88 CRC32C(crc, value, h); 89 p += sizeof(u16); 90 } 91 92 if (len & sizeof(u8)) { 93 u8 value = *p++; 94 95 CRC32C(crc, value, b); 96 } 97 98 return crc; 99 } 100 101 #define crc32_be_arch crc32_be_base /* not implemented on this arch */ 102 103 #define crc32_mod_init_arch crc32_mod_init_arch 104 static inline void crc32_mod_init_arch(void) 105 { 106 if (cpu_has_crc32) 107 static_branch_enable(&have_crc32); 108 } 109 110 static inline u32 crc32_optimizations_arch(void) 111 { 112 if (static_key_enabled(&have_crc32)) 113 return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; 114 return 0; 115 } 116