1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal. 4 * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE) 5 * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at: 6 * http://www.intel.com/products/processor/manuals/ 7 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual 8 * Volume 2A: Instruction Set Reference, A-M 9 * 10 * Copyright (C) 2008 Intel Corporation 11 * Authors: Austin Zhang <austin_zhang@linux.intel.com> 12 * Kent Liu <kent.liu@intel.com> 13 */ 14 #include <linux/init.h> 15 #include <linux/module.h> 16 #include <linux/string.h> 17 #include <linux/kernel.h> 18 #include <crypto/internal/hash.h> 19 #include <crypto/internal/simd.h> 20 21 #include <asm/cpufeatures.h> 22 #include <asm/cpu_device_id.h> 23 #include <asm/simd.h> 24 25 #define CHKSUM_BLOCK_SIZE 1 26 #define CHKSUM_DIGEST_SIZE 4 27 28 #define SCALE_F sizeof(unsigned long) 29 30 #ifdef CONFIG_X86_64 31 #define REX_PRE "0x48, " 32 #else 33 #define REX_PRE 34 #endif 35 36 #ifdef CONFIG_X86_64 37 /* 38 * use carryless multiply version of crc32c when buffer 39 * size is >= 512 to account 40 * for fpu state save/restore overhead. 41 */ 42 #define CRC32C_PCL_BREAKEVEN 512 43 44 asmlinkage unsigned int crc_pcl(const u8 *buffer, int len, 45 unsigned int crc_init); 46 #endif /* CONFIG_X86_64 */ 47 48 static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length) 49 { 50 while (length--) { 51 __asm__ __volatile__( 52 ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1" 53 :"=S"(crc) 54 :"0"(crc), "c"(*data) 55 ); 56 data++; 57 } 58 59 return crc; 60 } 61 62 static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len) 63 { 64 unsigned int iquotient = len / SCALE_F; 65 unsigned int iremainder = len % SCALE_F; 66 unsigned long *ptmp = (unsigned long *)p; 67 68 while (iquotient--) { 69 __asm__ __volatile__( 70 ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;" 71 :"=S"(crc) 72 :"0"(crc), "c"(*ptmp) 73 ); 74 ptmp++; 75 } 76 77 if (iremainder) 78 crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp, 79 iremainder); 80 81 return crc; 82 } 83 84 /* 85 * Setting the seed allows arbitrary accumulators and flexible XOR policy 86 * If your algorithm starts with ~0, then XOR with ~0 before you set 87 * the seed. 88 */ 89 static int crc32c_intel_setkey(struct crypto_shash *hash, const u8 *key, 90 unsigned int keylen) 91 { 92 u32 *mctx = crypto_shash_ctx(hash); 93 94 if (keylen != sizeof(u32)) { 95 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); 96 return -EINVAL; 97 } 98 *mctx = le32_to_cpup((__le32 *)key); 99 return 0; 100 } 101 102 static int crc32c_intel_init(struct shash_desc *desc) 103 { 104 u32 *mctx = crypto_shash_ctx(desc->tfm); 105 u32 *crcp = shash_desc_ctx(desc); 106 107 *crcp = *mctx; 108 109 return 0; 110 } 111 112 static int crc32c_intel_update(struct shash_desc *desc, const u8 *data, 113 unsigned int len) 114 { 115 u32 *crcp = shash_desc_ctx(desc); 116 117 *crcp = crc32c_intel_le_hw(*crcp, data, len); 118 return 0; 119 } 120 121 static int __crc32c_intel_finup(u32 *crcp, const u8 *data, unsigned int len, 122 u8 *out) 123 { 124 *(__le32 *)out = ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len)); 125 return 0; 126 } 127 128 static int crc32c_intel_finup(struct shash_desc *desc, const u8 *data, 129 unsigned int len, u8 *out) 130 { 131 return __crc32c_intel_finup(shash_desc_ctx(desc), data, len, out); 132 } 133 134 static int crc32c_intel_final(struct shash_desc *desc, u8 *out) 135 { 136 u32 *crcp = shash_desc_ctx(desc); 137 138 *(__le32 *)out = ~cpu_to_le32p(crcp); 139 return 0; 140 } 141 142 static int crc32c_intel_digest(struct shash_desc *desc, const u8 *data, 143 unsigned int len, u8 *out) 144 { 145 return __crc32c_intel_finup(crypto_shash_ctx(desc->tfm), data, len, 146 out); 147 } 148 149 static int crc32c_intel_cra_init(struct crypto_tfm *tfm) 150 { 151 u32 *key = crypto_tfm_ctx(tfm); 152 153 *key = ~0; 154 155 return 0; 156 } 157 158 #ifdef CONFIG_X86_64 159 static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data, 160 unsigned int len) 161 { 162 u32 *crcp = shash_desc_ctx(desc); 163 164 /* 165 * use faster PCL version if datasize is large enough to 166 * overcome kernel fpu state save/restore overhead 167 */ 168 if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) { 169 kernel_fpu_begin(); 170 *crcp = crc_pcl(data, len, *crcp); 171 kernel_fpu_end(); 172 } else 173 *crcp = crc32c_intel_le_hw(*crcp, data, len); 174 return 0; 175 } 176 177 static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len, 178 u8 *out) 179 { 180 if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) { 181 kernel_fpu_begin(); 182 *(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp)); 183 kernel_fpu_end(); 184 } else 185 *(__le32 *)out = 186 ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len)); 187 return 0; 188 } 189 190 static int crc32c_pcl_intel_finup(struct shash_desc *desc, const u8 *data, 191 unsigned int len, u8 *out) 192 { 193 return __crc32c_pcl_intel_finup(shash_desc_ctx(desc), data, len, out); 194 } 195 196 static int crc32c_pcl_intel_digest(struct shash_desc *desc, const u8 *data, 197 unsigned int len, u8 *out) 198 { 199 return __crc32c_pcl_intel_finup(crypto_shash_ctx(desc->tfm), data, len, 200 out); 201 } 202 #endif /* CONFIG_X86_64 */ 203 204 static struct shash_alg alg = { 205 .setkey = crc32c_intel_setkey, 206 .init = crc32c_intel_init, 207 .update = crc32c_intel_update, 208 .final = crc32c_intel_final, 209 .finup = crc32c_intel_finup, 210 .digest = crc32c_intel_digest, 211 .descsize = sizeof(u32), 212 .digestsize = CHKSUM_DIGEST_SIZE, 213 .base = { 214 .cra_name = "crc32c", 215 .cra_driver_name = "crc32c-intel", 216 .cra_priority = 200, 217 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 218 .cra_blocksize = CHKSUM_BLOCK_SIZE, 219 .cra_ctxsize = sizeof(u32), 220 .cra_module = THIS_MODULE, 221 .cra_init = crc32c_intel_cra_init, 222 } 223 }; 224 225 static const struct x86_cpu_id crc32c_cpu_id[] = { 226 X86_FEATURE_MATCH(X86_FEATURE_XMM4_2), 227 {} 228 }; 229 MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id); 230 231 static int __init crc32c_intel_mod_init(void) 232 { 233 if (!x86_match_cpu(crc32c_cpu_id)) 234 return -ENODEV; 235 #ifdef CONFIG_X86_64 236 if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { 237 alg.update = crc32c_pcl_intel_update; 238 alg.finup = crc32c_pcl_intel_finup; 239 alg.digest = crc32c_pcl_intel_digest; 240 } 241 #endif 242 return crypto_register_shash(&alg); 243 } 244 245 static void __exit crc32c_intel_mod_fini(void) 246 { 247 crypto_unregister_shash(&alg); 248 } 249 250 module_init(crc32c_intel_mod_init); 251 module_exit(crc32c_intel_mod_fini); 252 253 MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>"); 254 MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware."); 255 MODULE_LICENSE("GPL"); 256 257 MODULE_ALIAS_CRYPTO("crc32c"); 258 MODULE_ALIAS_CRYPTO("crc32c-intel"); 259