1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/crc-t10dif.h> 9 #include <linux/init.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/string.h> 13 14 #include <crypto/internal/hash.h> 15 #include <crypto/internal/simd.h> 16 17 #include <asm/neon.h> 18 #include <asm/simd.h> 19 20 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 21 22 asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 *buf, size_t len); 23 24 static int crct10dif_init(struct shash_desc *desc) 25 { 26 u16 *crc = shash_desc_ctx(desc); 27 28 *crc = 0; 29 return 0; 30 } 31 32 static int crct10dif_update(struct shash_desc *desc, const u8 *data, 33 unsigned int length) 34 { 35 u16 *crc = shash_desc_ctx(desc); 36 37 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { 38 kernel_neon_begin(); 39 *crc = crc_t10dif_pmull(*crc, data, length); 40 kernel_neon_end(); 41 } else { 42 *crc = crc_t10dif_generic(*crc, data, length); 43 } 44 45 return 0; 46 } 47 48 static int crct10dif_final(struct shash_desc *desc, u8 *out) 49 { 50 u16 *crc = shash_desc_ctx(desc); 51 52 *(u16 *)out = *crc; 53 return 0; 54 } 55 56 static struct shash_alg crc_t10dif_alg = { 57 .digestsize = CRC_T10DIF_DIGEST_SIZE, 58 .init = crct10dif_init, 59 .update = crct10dif_update, 60 .final = crct10dif_final, 61 .descsize = CRC_T10DIF_DIGEST_SIZE, 62 63 .base.cra_name = "crct10dif", 64 .base.cra_driver_name = "crct10dif-arm-ce", 65 .base.cra_priority = 200, 66 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 67 .base.cra_module = THIS_MODULE, 68 }; 69 70 static int __init crc_t10dif_mod_init(void) 71 { 72 if (!(elf_hwcap2 & HWCAP2_PMULL)) 73 return -ENODEV; 74 75 return crypto_register_shash(&crc_t10dif_alg); 76 } 77 78 static void __exit crc_t10dif_mod_exit(void) 79 { 80 crypto_unregister_shash(&crc_t10dif_alg); 81 } 82 83 module_init(crc_t10dif_mod_init); 84 module_exit(crc_t10dif_mod_exit); 85 86 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 87 MODULE_LICENSE("GPL v2"); 88 MODULE_ALIAS_CRYPTO("crct10dif"); 89