1 /* 2 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions 3 * 4 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/cpufeature.h> 12 #include <linux/crc-t10dif.h> 13 #include <linux/init.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/string.h> 17 18 #include <crypto/internal/hash.h> 19 20 #include <asm/neon.h> 21 #include <asm/simd.h> 22 23 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 24 25 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 buf[], u64 len); 26 asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 buf[], u64 len); 27 28 static u16 (*crc_t10dif_pmull)(u16 init_crc, const u8 buf[], u64 len); 29 30 static int crct10dif_init(struct shash_desc *desc) 31 { 32 u16 *crc = shash_desc_ctx(desc); 33 34 *crc = 0; 35 return 0; 36 } 37 38 static int crct10dif_update(struct shash_desc *desc, const u8 *data, 39 unsigned int length) 40 { 41 u16 *crc = shash_desc_ctx(desc); 42 unsigned int l; 43 44 if (unlikely((u64)data % CRC_T10DIF_PMULL_CHUNK_SIZE)) { 45 l = min_t(u32, length, CRC_T10DIF_PMULL_CHUNK_SIZE - 46 ((u64)data % CRC_T10DIF_PMULL_CHUNK_SIZE)); 47 48 *crc = crc_t10dif_generic(*crc, data, l); 49 50 length -= l; 51 data += l; 52 } 53 54 if (length > 0) { 55 if (may_use_simd()) { 56 kernel_neon_begin(); 57 *crc = crc_t10dif_pmull(*crc, data, length); 58 kernel_neon_end(); 59 } else { 60 *crc = crc_t10dif_generic(*crc, data, length); 61 } 62 } 63 64 return 0; 65 } 66 67 static int crct10dif_final(struct shash_desc *desc, u8 *out) 68 { 69 u16 *crc = shash_desc_ctx(desc); 70 71 *(u16 *)out = *crc; 72 return 0; 73 } 74 75 static struct shash_alg crc_t10dif_alg = { 76 .digestsize = CRC_T10DIF_DIGEST_SIZE, 77 .init = crct10dif_init, 78 .update = crct10dif_update, 79 .final = crct10dif_final, 80 .descsize = CRC_T10DIF_DIGEST_SIZE, 81 82 .base.cra_name = "crct10dif", 83 .base.cra_driver_name = "crct10dif-arm64-ce", 84 .base.cra_priority = 200, 85 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 86 .base.cra_module = THIS_MODULE, 87 }; 88 89 static int __init crc_t10dif_mod_init(void) 90 { 91 if (elf_hwcap & HWCAP_PMULL) 92 crc_t10dif_pmull = crc_t10dif_pmull_p64; 93 else 94 crc_t10dif_pmull = crc_t10dif_pmull_p8; 95 96 return crypto_register_shash(&crc_t10dif_alg); 97 } 98 99 static void __exit crc_t10dif_mod_exit(void) 100 { 101 crypto_unregister_shash(&crc_t10dif_alg); 102 } 103 104 module_cpu_feature_match(ASIMD, crc_t10dif_mod_init); 105 module_exit(crc_t10dif_mod_exit); 106 107 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 108 MODULE_LICENSE("GPL v2"); 109 MODULE_ALIAS_CRYPTO("crct10dif"); 110 MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce"); 111