1 /* 2 * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal. 3 * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE) 4 * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at: 5 * http://www.intel.com/products/processor/manuals/ 6 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual 7 * Volume 2A: Instruction Set Reference, A-M 8 * 9 * Copyright (C) 2008 Intel Corporation 10 * Authors: Austin Zhang <austin_zhang@linux.intel.com> 11 * Kent Liu <kent.liu@intel.com> 12 * 13 * This program is free software; you can redistribute it and/or modify it 14 * under the terms and conditions of the GNU General Public License, 15 * version 2, as published by the Free Software Foundation. 16 * 17 * This program is distributed in the hope it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 20 * more details. 21 * 22 * You should have received a copy of the GNU General Public License along with 23 * this program; if not, write to the Free Software Foundation, Inc., 24 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 25 * 26 */ 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/string.h> 30 #include <linux/kernel.h> 31 #include <crypto/internal/hash.h> 32 33 #include <asm/cpufeature.h> 34 #include <asm/cpu_device_id.h> 35 #include <asm/i387.h> 36 #include <asm/fpu-internal.h> 37 38 #define CHKSUM_BLOCK_SIZE 1 39 #define CHKSUM_DIGEST_SIZE 4 40 41 #define SCALE_F sizeof(unsigned long) 42 43 #ifdef CONFIG_X86_64 44 #define REX_PRE "0x48, " 45 #else 46 #define REX_PRE 47 #endif 48 49 #ifdef CONFIG_X86_64 50 /* 51 * use carryless multiply version of crc32c when buffer 52 * size is >= 512 (when eager fpu is enabled) or 53 * >= 1024 (when eager fpu is disabled) to account 54 * for fpu state save/restore overhead. 55 */ 56 #define CRC32C_PCL_BREAKEVEN_EAGERFPU 512 57 #define CRC32C_PCL_BREAKEVEN_NOEAGERFPU 1024 58 59 asmlinkage unsigned int crc_pcl(const u8 *buffer, int len, 60 unsigned int crc_init); 61 static int crc32c_pcl_breakeven = CRC32C_PCL_BREAKEVEN_EAGERFPU; 62 #if defined(X86_FEATURE_EAGER_FPU) 63 #define set_pcl_breakeven_point() \ 64 do { \ 65 if (!use_eager_fpu()) \ 66 crc32c_pcl_breakeven = CRC32C_PCL_BREAKEVEN_NOEAGERFPU; \ 67 } while (0) 68 #else 69 #define set_pcl_breakeven_point() \ 70 (crc32c_pcl_breakeven = CRC32C_PCL_BREAKEVEN_NOEAGERFPU) 71 #endif 72 #endif /* CONFIG_X86_64 */ 73 74 static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length) 75 { 76 while (length--) { 77 __asm__ __volatile__( 78 ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1" 79 :"=S"(crc) 80 :"0"(crc), "c"(*data) 81 ); 82 data++; 83 } 84 85 return crc; 86 } 87 88 static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len) 89 { 90 unsigned int iquotient = len / SCALE_F; 91 unsigned int iremainder = len % SCALE_F; 92 unsigned long *ptmp = (unsigned long *)p; 93 94 while (iquotient--) { 95 __asm__ __volatile__( 96 ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;" 97 :"=S"(crc) 98 :"0"(crc), "c"(*ptmp) 99 ); 100 ptmp++; 101 } 102 103 if (iremainder) 104 crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp, 105 iremainder); 106 107 return crc; 108 } 109 110 /* 111 * Setting the seed allows arbitrary accumulators and flexible XOR policy 112 * If your algorithm starts with ~0, then XOR with ~0 before you set 113 * the seed. 114 */ 115 static int crc32c_intel_setkey(struct crypto_shash *hash, const u8 *key, 116 unsigned int keylen) 117 { 118 u32 *mctx = crypto_shash_ctx(hash); 119 120 if (keylen != sizeof(u32)) { 121 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); 122 return -EINVAL; 123 } 124 *mctx = le32_to_cpup((__le32 *)key); 125 return 0; 126 } 127 128 static int crc32c_intel_init(struct shash_desc *desc) 129 { 130 u32 *mctx = crypto_shash_ctx(desc->tfm); 131 u32 *crcp = shash_desc_ctx(desc); 132 133 *crcp = *mctx; 134 135 return 0; 136 } 137 138 static int crc32c_intel_update(struct shash_desc *desc, const u8 *data, 139 unsigned int len) 140 { 141 u32 *crcp = shash_desc_ctx(desc); 142 143 *crcp = crc32c_intel_le_hw(*crcp, data, len); 144 return 0; 145 } 146 147 static int __crc32c_intel_finup(u32 *crcp, const u8 *data, unsigned int len, 148 u8 *out) 149 { 150 *(__le32 *)out = ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len)); 151 return 0; 152 } 153 154 static int crc32c_intel_finup(struct shash_desc *desc, const u8 *data, 155 unsigned int len, u8 *out) 156 { 157 return __crc32c_intel_finup(shash_desc_ctx(desc), data, len, out); 158 } 159 160 static int crc32c_intel_final(struct shash_desc *desc, u8 *out) 161 { 162 u32 *crcp = shash_desc_ctx(desc); 163 164 *(__le32 *)out = ~cpu_to_le32p(crcp); 165 return 0; 166 } 167 168 static int crc32c_intel_digest(struct shash_desc *desc, const u8 *data, 169 unsigned int len, u8 *out) 170 { 171 return __crc32c_intel_finup(crypto_shash_ctx(desc->tfm), data, len, 172 out); 173 } 174 175 static int crc32c_intel_cra_init(struct crypto_tfm *tfm) 176 { 177 u32 *key = crypto_tfm_ctx(tfm); 178 179 *key = ~0; 180 181 return 0; 182 } 183 184 #ifdef CONFIG_X86_64 185 static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data, 186 unsigned int len) 187 { 188 u32 *crcp = shash_desc_ctx(desc); 189 190 /* 191 * use faster PCL version if datasize is large enough to 192 * overcome kernel fpu state save/restore overhead 193 */ 194 if (len >= crc32c_pcl_breakeven && irq_fpu_usable()) { 195 kernel_fpu_begin(); 196 *crcp = crc_pcl(data, len, *crcp); 197 kernel_fpu_end(); 198 } else 199 *crcp = crc32c_intel_le_hw(*crcp, data, len); 200 return 0; 201 } 202 203 static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len, 204 u8 *out) 205 { 206 if (len >= crc32c_pcl_breakeven && irq_fpu_usable()) { 207 kernel_fpu_begin(); 208 *(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp)); 209 kernel_fpu_end(); 210 } else 211 *(__le32 *)out = 212 ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len)); 213 return 0; 214 } 215 216 static int crc32c_pcl_intel_finup(struct shash_desc *desc, const u8 *data, 217 unsigned int len, u8 *out) 218 { 219 return __crc32c_pcl_intel_finup(shash_desc_ctx(desc), data, len, out); 220 } 221 222 static int crc32c_pcl_intel_digest(struct shash_desc *desc, const u8 *data, 223 unsigned int len, u8 *out) 224 { 225 return __crc32c_pcl_intel_finup(crypto_shash_ctx(desc->tfm), data, len, 226 out); 227 } 228 #endif /* CONFIG_X86_64 */ 229 230 static struct shash_alg alg = { 231 .setkey = crc32c_intel_setkey, 232 .init = crc32c_intel_init, 233 .update = crc32c_intel_update, 234 .final = crc32c_intel_final, 235 .finup = crc32c_intel_finup, 236 .digest = crc32c_intel_digest, 237 .descsize = sizeof(u32), 238 .digestsize = CHKSUM_DIGEST_SIZE, 239 .base = { 240 .cra_name = "crc32c", 241 .cra_driver_name = "crc32c-intel", 242 .cra_priority = 200, 243 .cra_blocksize = CHKSUM_BLOCK_SIZE, 244 .cra_ctxsize = sizeof(u32), 245 .cra_module = THIS_MODULE, 246 .cra_init = crc32c_intel_cra_init, 247 } 248 }; 249 250 static const struct x86_cpu_id crc32c_cpu_id[] = { 251 X86_FEATURE_MATCH(X86_FEATURE_XMM4_2), 252 {} 253 }; 254 MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id); 255 256 static int __init crc32c_intel_mod_init(void) 257 { 258 if (!x86_match_cpu(crc32c_cpu_id)) 259 return -ENODEV; 260 #ifdef CONFIG_X86_64 261 if (cpu_has_pclmulqdq) { 262 alg.update = crc32c_pcl_intel_update; 263 alg.finup = crc32c_pcl_intel_finup; 264 alg.digest = crc32c_pcl_intel_digest; 265 set_pcl_breakeven_point(); 266 } 267 #endif 268 return crypto_register_shash(&alg); 269 } 270 271 static void __exit crc32c_intel_mod_fini(void) 272 { 273 crypto_unregister_shash(&alg); 274 } 275 276 module_init(crc32c_intel_mod_init); 277 module_exit(crc32c_intel_mod_fini); 278 279 MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>"); 280 MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware."); 281 MODULE_LICENSE("GPL"); 282 283 MODULE_ALIAS("crc32c"); 284 MODULE_ALIAS("crc32c-intel"); 285