1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/cpufeature.h> 9 #include <linux/crc-t10dif.h> 10 #include <linux/init.h> 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/string.h> 14 15 #include <crypto/internal/hash.h> 16 #include <crypto/internal/simd.h> 17 18 #include <asm/neon.h> 19 #include <asm/simd.h> 20 21 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 22 23 asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len); 24 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); 25 26 static int crct10dif_init(struct shash_desc *desc) 27 { 28 u16 *crc = shash_desc_ctx(desc); 29 30 *crc = 0; 31 return 0; 32 } 33 34 static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data, 35 unsigned int length) 36 { 37 u16 *crc = shash_desc_ctx(desc); 38 39 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { 40 do { 41 unsigned int chunk = length; 42 43 if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE) 44 chunk = SZ_4K; 45 46 kernel_neon_begin(); 47 *crc = crc_t10dif_pmull_p8(*crc, data, chunk); 48 kernel_neon_end(); 49 data += chunk; 50 length -= chunk; 51 } while (length); 52 } else { 53 *crc = crc_t10dif_generic(*crc, data, length); 54 } 55 56 return 0; 57 } 58 59 static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data, 60 unsigned int length) 61 { 62 u16 *crc = shash_desc_ctx(desc); 63 64 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { 65 do { 66 unsigned int chunk = length; 67 68 if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE) 69 chunk = SZ_4K; 70 71 kernel_neon_begin(); 72 *crc = crc_t10dif_pmull_p64(*crc, data, chunk); 73 kernel_neon_end(); 74 data += chunk; 75 length -= chunk; 76 } while (length); 77 } else { 78 *crc = crc_t10dif_generic(*crc, data, length); 79 } 80 81 return 0; 82 } 83 84 static int crct10dif_final(struct shash_desc *desc, u8 *out) 85 { 86 u16 *crc = shash_desc_ctx(desc); 87 88 *(u16 *)out = *crc; 89 return 0; 90 } 91 92 static struct shash_alg crc_t10dif_alg[] = {{ 93 .digestsize = CRC_T10DIF_DIGEST_SIZE, 94 .init = crct10dif_init, 95 .update = crct10dif_update_pmull_p8, 96 .final = crct10dif_final, 97 .descsize = CRC_T10DIF_DIGEST_SIZE, 98 99 .base.cra_name = "crct10dif", 100 .base.cra_driver_name = "crct10dif-arm64-neon", 101 .base.cra_priority = 100, 102 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 103 .base.cra_module = THIS_MODULE, 104 }, { 105 .digestsize = CRC_T10DIF_DIGEST_SIZE, 106 .init = crct10dif_init, 107 .update = crct10dif_update_pmull_p64, 108 .final = crct10dif_final, 109 .descsize = CRC_T10DIF_DIGEST_SIZE, 110 111 .base.cra_name = "crct10dif", 112 .base.cra_driver_name = "crct10dif-arm64-ce", 113 .base.cra_priority = 200, 114 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 115 .base.cra_module = THIS_MODULE, 116 }}; 117 118 static int __init crc_t10dif_mod_init(void) 119 { 120 if (cpu_have_named_feature(PMULL)) 121 return crypto_register_shashes(crc_t10dif_alg, 122 ARRAY_SIZE(crc_t10dif_alg)); 123 else 124 /* only register the first array element */ 125 return crypto_register_shash(crc_t10dif_alg); 126 } 127 128 static void __exit crc_t10dif_mod_exit(void) 129 { 130 if (cpu_have_named_feature(PMULL)) 131 crypto_unregister_shashes(crc_t10dif_alg, 132 ARRAY_SIZE(crc_t10dif_alg)); 133 else 134 crypto_unregister_shash(crc_t10dif_alg); 135 } 136 137 module_cpu_feature_match(ASIMD, crc_t10dif_mod_init); 138 module_exit(crc_t10dif_mod_exit); 139 140 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 141 MODULE_LICENSE("GPL v2"); 142 MODULE_ALIAS_CRYPTO("crct10dif"); 143 MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce"); 144