1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/crc-t10dif.h> 9 #include <linux/init.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/string.h> 13 14 #include <crypto/internal/hash.h> 15 #include <crypto/internal/simd.h> 16 17 #include <asm/neon.h> 18 #include <asm/simd.h> 19 20 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 21 22 asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); 23 asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, 24 u8 out[16]); 25 26 static int crct10dif_init(struct shash_desc *desc) 27 { 28 u16 *crc = shash_desc_ctx(desc); 29 30 *crc = 0; 31 return 0; 32 } 33 34 static int crct10dif_update_ce(struct shash_desc *desc, const u8 *data, 35 unsigned int length) 36 { 37 u16 *crc = shash_desc_ctx(desc); 38 39 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { 40 kernel_neon_begin(); 41 *crc = crc_t10dif_pmull64(*crc, data, length); 42 kernel_neon_end(); 43 } else { 44 *crc = crc_t10dif_generic(*crc, data, length); 45 } 46 47 return 0; 48 } 49 50 static int crct10dif_update_neon(struct shash_desc *desc, const u8 *data, 51 unsigned int length) 52 { 53 u16 *crcp = shash_desc_ctx(desc); 54 u8 buf[16] __aligned(16); 55 u16 crc = *crcp; 56 57 if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { 58 kernel_neon_begin(); 59 crc_t10dif_pmull8(crc, data, length, buf); 60 kernel_neon_end(); 61 62 crc = 0; 63 data = buf; 64 length = sizeof(buf); 65 } 66 67 *crcp = crc_t10dif_generic(crc, data, length); 68 return 0; 69 } 70 71 static int crct10dif_final(struct shash_desc *desc, u8 *out) 72 { 73 u16 *crc = shash_desc_ctx(desc); 74 75 *(u16 *)out = *crc; 76 return 0; 77 } 78 79 static struct shash_alg algs[] = {{ 80 .digestsize = CRC_T10DIF_DIGEST_SIZE, 81 .init = crct10dif_init, 82 .update = crct10dif_update_neon, 83 .final = crct10dif_final, 84 .descsize = CRC_T10DIF_DIGEST_SIZE, 85 86 .base.cra_name = "crct10dif", 87 .base.cra_driver_name = "crct10dif-arm-neon", 88 .base.cra_priority = 150, 89 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 90 .base.cra_module = THIS_MODULE, 91 }, { 92 .digestsize = CRC_T10DIF_DIGEST_SIZE, 93 .init = crct10dif_init, 94 .update = crct10dif_update_ce, 95 .final = crct10dif_final, 96 .descsize = CRC_T10DIF_DIGEST_SIZE, 97 98 .base.cra_name = "crct10dif", 99 .base.cra_driver_name = "crct10dif-arm-ce", 100 .base.cra_priority = 200, 101 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 102 .base.cra_module = THIS_MODULE, 103 }}; 104 105 static int __init crc_t10dif_mod_init(void) 106 { 107 if (!(elf_hwcap & HWCAP_NEON)) 108 return -ENODEV; 109 110 return crypto_register_shashes(algs, 1 + !!(elf_hwcap2 & HWCAP2_PMULL)); 111 } 112 113 static void __exit crc_t10dif_mod_exit(void) 114 { 115 crypto_unregister_shashes(algs, 1 + !!(elf_hwcap2 & HWCAP2_PMULL)); 116 } 117 118 module_init(crc_t10dif_mod_init); 119 module_exit(crc_t10dif_mod_exit); 120 121 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 122 MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions"); 123 MODULE_LICENSE("GPL v2"); 124 MODULE_ALIAS_CRYPTO("crct10dif"); 125