1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/cpufeature.h> 9 #include <linux/crc-t10dif.h> 10 #include <linux/init.h> 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/string.h> 14 15 #include <crypto/internal/hash.h> 16 #include <crypto/internal/simd.h> 17 18 #include <asm/neon.h> 19 #include <asm/simd.h> 20 21 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 22 23 asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len, 24 u8 out[16]); 25 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); 26 27 static int crct10dif_init(struct shash_desc *desc) 28 { 29 u16 *crc = shash_desc_ctx(desc); 30 31 *crc = 0; 32 return 0; 33 } 34 35 static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data, 36 unsigned int length) 37 { 38 u16 *crcp = shash_desc_ctx(desc); 39 u16 crc = *crcp; 40 u8 buf[16]; 41 42 if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { 43 kernel_neon_begin(); 44 crc_t10dif_pmull_p8(crc, data, length, buf); 45 kernel_neon_end(); 46 47 crc = 0; 48 data = buf; 49 length = sizeof(buf); 50 } 51 52 *crcp = crc_t10dif_generic(crc, data, length); 53 return 0; 54 } 55 56 static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data, 57 unsigned int length) 58 { 59 u16 *crc = shash_desc_ctx(desc); 60 61 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { 62 kernel_neon_begin(); 63 *crc = crc_t10dif_pmull_p64(*crc, data, length); 64 kernel_neon_end(); 65 } else { 66 *crc = crc_t10dif_generic(*crc, data, length); 67 } 68 69 return 0; 70 } 71 72 static int crct10dif_final(struct shash_desc *desc, u8 *out) 73 { 74 u16 *crc = shash_desc_ctx(desc); 75 76 *(u16 *)out = *crc; 77 return 0; 78 } 79 80 static struct shash_alg crc_t10dif_alg[] = {{ 81 .digestsize = CRC_T10DIF_DIGEST_SIZE, 82 .init = crct10dif_init, 83 .update = crct10dif_update_pmull_p8, 84 .final = crct10dif_final, 85 .descsize = CRC_T10DIF_DIGEST_SIZE, 86 87 .base.cra_name = "crct10dif", 88 .base.cra_driver_name = "crct10dif-arm64-neon", 89 .base.cra_priority = 150, 90 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 91 .base.cra_module = THIS_MODULE, 92 }, { 93 .digestsize = CRC_T10DIF_DIGEST_SIZE, 94 .init = crct10dif_init, 95 .update = crct10dif_update_pmull_p64, 96 .final = crct10dif_final, 97 .descsize = CRC_T10DIF_DIGEST_SIZE, 98 99 .base.cra_name = "crct10dif", 100 .base.cra_driver_name = "crct10dif-arm64-ce", 101 .base.cra_priority = 200, 102 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 103 .base.cra_module = THIS_MODULE, 104 }}; 105 106 static int __init crc_t10dif_mod_init(void) 107 { 108 if (cpu_have_named_feature(PMULL)) 109 return crypto_register_shashes(crc_t10dif_alg, 110 ARRAY_SIZE(crc_t10dif_alg)); 111 else 112 /* only register the first array element */ 113 return crypto_register_shash(crc_t10dif_alg); 114 } 115 116 static void __exit crc_t10dif_mod_exit(void) 117 { 118 if (cpu_have_named_feature(PMULL)) 119 crypto_unregister_shashes(crc_t10dif_alg, 120 ARRAY_SIZE(crc_t10dif_alg)); 121 else 122 crypto_unregister_shash(crc_t10dif_alg); 123 } 124 125 module_cpu_feature_match(ASIMD, crc_t10dif_mod_init); 126 module_exit(crc_t10dif_mod_exit); 127 128 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 129 MODULE_DESCRIPTION("CRC-T10DIF using arm64 NEON and Crypto Extensions"); 130 MODULE_LICENSE("GPL v2"); 131 MODULE_ALIAS_CRYPTO("crct10dif"); 132 MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce"); 133