1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * crc32-mips.c - CRC32 and CRC32C using optional MIPSr6 instructions 4 * 5 * Module based on arm64/crypto/crc32-arm.c 6 * 7 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org> 8 * Copyright (C) 2018 MIPS Tech, LLC 9 */ 10 11 #include <linux/cpufeature.h> 12 #include <asm/mipsregs.h> 13 #include <linux/unaligned.h> 14 15 #ifndef TOOLCHAIN_SUPPORTS_CRC 16 #define _ASM_SET_CRC(OP, SZ, TYPE) \ 17 _ASM_MACRO_3R(OP, rt, rs, rt2, \ 18 ".ifnc \\rt, \\rt2\n\t" \ 19 ".error \"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \ 20 ".endif\n\t" \ 21 _ASM_INSN_IF_MIPS(0x7c00000f | (__rt << 16) | (__rs << 21) | \ 22 ((SZ) << 6) | ((TYPE) << 8)) \ 23 _ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) | \ 24 ((SZ) << 14) | ((TYPE) << 3))) 25 #define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t" 26 #else /* !TOOLCHAIN_SUPPORTS_CRC */ 27 #define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t" 28 #define _ASM_UNSET_CRC(op, SZ, TYPE) 29 #endif 30 31 #define __CRC32(crc, value, op, SZ, TYPE) \ 32 do { \ 33 __asm__ __volatile__( \ 34 ".set push\n\t" \ 35 _ASM_SET_CRC(op, SZ, TYPE) \ 36 #op " %0, %1, %0\n\t" \ 37 _ASM_UNSET_CRC(op, SZ, TYPE) \ 38 ".set pop" \ 39 : "+r" (crc) \ 40 : "r" (value)); \ 41 } while (0) 42 43 #define _CRC32_crc32b(crc, value) __CRC32(crc, value, crc32b, 0, 0) 44 #define _CRC32_crc32h(crc, value) __CRC32(crc, value, crc32h, 1, 0) 45 #define _CRC32_crc32w(crc, value) __CRC32(crc, value, crc32w, 2, 0) 46 #define _CRC32_crc32d(crc, value) __CRC32(crc, value, crc32d, 3, 0) 47 #define _CRC32_crc32cb(crc, value) __CRC32(crc, value, crc32cb, 0, 1) 48 #define _CRC32_crc32ch(crc, value) __CRC32(crc, value, crc32ch, 1, 1) 49 #define _CRC32_crc32cw(crc, value) __CRC32(crc, value, crc32cw, 2, 1) 50 #define _CRC32_crc32cd(crc, value) __CRC32(crc, value, crc32cd, 3, 1) 51 52 #define _CRC32(crc, value, size, op) \ 53 _CRC32_##op##size(crc, value) 54 55 #define CRC32(crc, value, size) \ 56 _CRC32(crc, value, size, crc32) 57 58 #define CRC32C(crc, value, size) \ 59 _CRC32(crc, value, size, crc32c) 60 61 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 62 63 static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 64 { 65 if (!static_branch_likely(&have_crc32)) 66 return crc32_le_base(crc, p, len); 67 68 if (IS_ENABLED(CONFIG_64BIT)) { 69 for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) { 70 u64 value = get_unaligned_le64(p); 71 72 CRC32(crc, value, d); 73 } 74 75 if (len & sizeof(u32)) { 76 u32 value = get_unaligned_le32(p); 77 78 CRC32(crc, value, w); 79 p += sizeof(u32); 80 } 81 } else { 82 for (; len >= sizeof(u32); len -= sizeof(u32)) { 83 u32 value = get_unaligned_le32(p); 84 85 CRC32(crc, value, w); 86 p += sizeof(u32); 87 } 88 } 89 90 if (len & sizeof(u16)) { 91 u16 value = get_unaligned_le16(p); 92 93 CRC32(crc, value, h); 94 p += sizeof(u16); 95 } 96 97 if (len & sizeof(u8)) { 98 u8 value = *p++; 99 100 CRC32(crc, value, b); 101 } 102 103 return crc; 104 } 105 106 static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 107 { 108 if (!static_branch_likely(&have_crc32)) 109 return crc32c_base(crc, p, len); 110 111 if (IS_ENABLED(CONFIG_64BIT)) { 112 for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) { 113 u64 value = get_unaligned_le64(p); 114 115 CRC32C(crc, value, d); 116 } 117 118 if (len & sizeof(u32)) { 119 u32 value = get_unaligned_le32(p); 120 121 CRC32C(crc, value, w); 122 p += sizeof(u32); 123 } 124 } else { 125 for (; len >= sizeof(u32); len -= sizeof(u32)) { 126 u32 value = get_unaligned_le32(p); 127 128 CRC32C(crc, value, w); 129 p += sizeof(u32); 130 } 131 } 132 133 if (len & sizeof(u16)) { 134 u16 value = get_unaligned_le16(p); 135 136 CRC32C(crc, value, h); 137 p += sizeof(u16); 138 } 139 140 if (len & sizeof(u8)) { 141 u8 value = *p++; 142 143 CRC32C(crc, value, b); 144 } 145 return crc; 146 } 147 148 #define crc32_be_arch crc32_be_base /* not implemented on this arch */ 149 150 #define crc32_mod_init_arch crc32_mod_init_arch 151 static inline void crc32_mod_init_arch(void) 152 { 153 if (cpu_have_feature(cpu_feature(MIPS_CRC32))) 154 static_branch_enable(&have_crc32); 155 } 156 157 static inline u32 crc32_optimizations_arch(void) 158 { 159 if (static_key_enabled(&have_crc32)) 160 return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; 161 return 0; 162 } 163