1 /* 2 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions 3 * 4 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/cpufeature.h> 12 #include <linux/crc-t10dif.h> 13 #include <linux/init.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/string.h> 17 18 #include <crypto/internal/hash.h> 19 #include <crypto/internal/simd.h> 20 21 #include <asm/neon.h> 22 #include <asm/simd.h> 23 24 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 25 26 asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len); 27 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); 28 29 static int crct10dif_init(struct shash_desc *desc) 30 { 31 u16 *crc = shash_desc_ctx(desc); 32 33 *crc = 0; 34 return 0; 35 } 36 37 static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data, 38 unsigned int length) 39 { 40 u16 *crc = shash_desc_ctx(desc); 41 42 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { 43 kernel_neon_begin(); 44 *crc = crc_t10dif_pmull_p8(*crc, data, length); 45 kernel_neon_end(); 46 } else { 47 *crc = crc_t10dif_generic(*crc, data, length); 48 } 49 50 return 0; 51 } 52 53 static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data, 54 unsigned int length) 55 { 56 u16 *crc = shash_desc_ctx(desc); 57 58 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { 59 kernel_neon_begin(); 60 *crc = crc_t10dif_pmull_p64(*crc, data, length); 61 kernel_neon_end(); 62 } else { 63 *crc = crc_t10dif_generic(*crc, data, length); 64 } 65 66 return 0; 67 } 68 69 static int crct10dif_final(struct shash_desc *desc, u8 *out) 70 { 71 u16 *crc = shash_desc_ctx(desc); 72 73 *(u16 *)out = *crc; 74 return 0; 75 } 76 77 static struct shash_alg crc_t10dif_alg[] = {{ 78 .digestsize = CRC_T10DIF_DIGEST_SIZE, 79 .init = crct10dif_init, 80 .update = crct10dif_update_pmull_p8, 81 .final = crct10dif_final, 82 .descsize = CRC_T10DIF_DIGEST_SIZE, 83 84 .base.cra_name = "crct10dif", 85 .base.cra_driver_name = "crct10dif-arm64-neon", 86 .base.cra_priority = 100, 87 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 88 .base.cra_module = THIS_MODULE, 89 }, { 90 .digestsize = CRC_T10DIF_DIGEST_SIZE, 91 .init = crct10dif_init, 92 .update = crct10dif_update_pmull_p64, 93 .final = crct10dif_final, 94 .descsize = CRC_T10DIF_DIGEST_SIZE, 95 96 .base.cra_name = "crct10dif", 97 .base.cra_driver_name = "crct10dif-arm64-ce", 98 .base.cra_priority = 200, 99 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 100 .base.cra_module = THIS_MODULE, 101 }}; 102 103 static int __init crc_t10dif_mod_init(void) 104 { 105 if (cpu_have_named_feature(PMULL)) 106 return crypto_register_shashes(crc_t10dif_alg, 107 ARRAY_SIZE(crc_t10dif_alg)); 108 else 109 /* only register the first array element */ 110 return crypto_register_shash(crc_t10dif_alg); 111 } 112 113 static void __exit crc_t10dif_mod_exit(void) 114 { 115 if (cpu_have_named_feature(PMULL)) 116 crypto_unregister_shashes(crc_t10dif_alg, 117 ARRAY_SIZE(crc_t10dif_alg)); 118 else 119 crypto_unregister_shash(crc_t10dif_alg); 120 } 121 122 module_cpu_feature_match(ASIMD, crc_t10dif_mod_init); 123 module_exit(crc_t10dif_mod_exit); 124 125 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 126 MODULE_LICENSE("GPL v2"); 127 MODULE_ALIAS_CRYPTO("crct10dif"); 128 MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce"); 129