1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/cpufeature.h> 9 #include <linux/crc-t10dif.h> 10 #include <linux/init.h> 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/string.h> 14 15 #include <crypto/internal/simd.h> 16 17 #include <asm/neon.h> 18 #include <asm/simd.h> 19 20 static DEFINE_STATIC_KEY_FALSE(have_asimd); 21 static DEFINE_STATIC_KEY_FALSE(have_pmull); 22 23 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 24 25 asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len, 26 u8 out[16]); 27 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); 28 29 u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) 30 { 31 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { 32 if (static_branch_likely(&have_pmull)) { 33 if (crypto_simd_usable()) { 34 kernel_neon_begin(); 35 crc = crc_t10dif_pmull_p64(crc, data, length); 36 kernel_neon_end(); 37 return crc; 38 } 39 } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && 40 static_branch_likely(&have_asimd) && 41 crypto_simd_usable()) { 42 u8 buf[16]; 43 44 kernel_neon_begin(); 45 crc_t10dif_pmull_p8(crc, data, length, buf); 46 kernel_neon_end(); 47 48 crc = 0; 49 data = buf; 50 length = sizeof(buf); 51 } 52 } 53 return crc_t10dif_generic(crc, data, length); 54 } 55 EXPORT_SYMBOL(crc_t10dif_arch); 56 57 static int __init crc_t10dif_arm64_init(void) 58 { 59 if (cpu_have_named_feature(ASIMD)) { 60 static_branch_enable(&have_asimd); 61 if (cpu_have_named_feature(PMULL)) 62 static_branch_enable(&have_pmull); 63 } 64 return 0; 65 } 66 arch_initcall(crc_t10dif_arm64_init); 67 68 static void __exit crc_t10dif_arm64_exit(void) 69 { 70 } 71 module_exit(crc_t10dif_arm64_exit); 72 73 bool crc_t10dif_is_optimized(void) 74 { 75 return static_key_enabled(&have_asimd); 76 } 77 EXPORT_SYMBOL(crc_t10dif_is_optimized); 78 79 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 80 MODULE_DESCRIPTION("CRC-T10DIF using arm64 NEON and Crypto Extensions"); 81 MODULE_LICENSE("GPL v2"); 82