1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/cpufeature.h> 9 #include <linux/crc-t10dif.h> 10 #include <linux/init.h> 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/string.h> 14 15 #include <crypto/internal/hash.h> 16 #include <crypto/internal/simd.h> 17 18 #include <asm/neon.h> 19 #include <asm/simd.h> 20 21 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 22 23 asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len); 24 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); 25 26 static int crct10dif_init(struct shash_desc *desc) 27 { 28 u16 *crc = shash_desc_ctx(desc); 29 30 *crc = 0; 31 return 0; 32 } 33 34 static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data, 35 unsigned int length) 36 { 37 u16 *crc = shash_desc_ctx(desc); 38 39 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { 40 kernel_neon_begin(); 41 *crc = crc_t10dif_pmull_p8(*crc, data, length); 42 kernel_neon_end(); 43 } else { 44 *crc = crc_t10dif_generic(*crc, data, length); 45 } 46 47 return 0; 48 } 49 50 static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data, 51 unsigned int length) 52 { 53 u16 *crc = shash_desc_ctx(desc); 54 55 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { 56 kernel_neon_begin(); 57 *crc = crc_t10dif_pmull_p64(*crc, data, length); 58 kernel_neon_end(); 59 } else { 60 *crc = crc_t10dif_generic(*crc, data, length); 61 } 62 63 return 0; 64 } 65 66 static int crct10dif_final(struct shash_desc *desc, u8 *out) 67 { 68 u16 *crc = shash_desc_ctx(desc); 69 70 *(u16 *)out = *crc; 71 return 0; 72 } 73 74 static struct shash_alg crc_t10dif_alg[] = {{ 75 .digestsize = CRC_T10DIF_DIGEST_SIZE, 76 .init = crct10dif_init, 77 .update = crct10dif_update_pmull_p8, 78 .final = crct10dif_final, 79 .descsize = CRC_T10DIF_DIGEST_SIZE, 80 81 .base.cra_name = "crct10dif", 82 .base.cra_driver_name = "crct10dif-arm64-neon", 83 .base.cra_priority = 100, 84 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 85 .base.cra_module = THIS_MODULE, 86 }, { 87 .digestsize = CRC_T10DIF_DIGEST_SIZE, 88 .init = crct10dif_init, 89 .update = crct10dif_update_pmull_p64, 90 .final = crct10dif_final, 91 .descsize = CRC_T10DIF_DIGEST_SIZE, 92 93 .base.cra_name = "crct10dif", 94 .base.cra_driver_name = "crct10dif-arm64-ce", 95 .base.cra_priority = 200, 96 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 97 .base.cra_module = THIS_MODULE, 98 }}; 99 100 static int __init crc_t10dif_mod_init(void) 101 { 102 if (cpu_have_named_feature(PMULL)) 103 return crypto_register_shashes(crc_t10dif_alg, 104 ARRAY_SIZE(crc_t10dif_alg)); 105 else 106 /* only register the first array element */ 107 return crypto_register_shash(crc_t10dif_alg); 108 } 109 110 static void __exit crc_t10dif_mod_exit(void) 111 { 112 if (cpu_have_named_feature(PMULL)) 113 crypto_unregister_shashes(crc_t10dif_alg, 114 ARRAY_SIZE(crc_t10dif_alg)); 115 else 116 crypto_unregister_shash(crc_t10dif_alg); 117 } 118 119 module_cpu_feature_match(ASIMD, crc_t10dif_mod_init); 120 module_exit(crc_t10dif_mod_exit); 121 122 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 123 MODULE_LICENSE("GPL v2"); 124 MODULE_ALIAS_CRYPTO("crct10dif"); 125 MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce"); 126