1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/crc-t10dif.h> 9 #include <linux/init.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/string.h> 13 14 #include <crypto/internal/simd.h> 15 16 #include <asm/neon.h> 17 #include <asm/simd.h> 18 19 static DEFINE_STATIC_KEY_FALSE(have_neon); 20 static DEFINE_STATIC_KEY_FALSE(have_pmull); 21 22 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 23 24 asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); 25 asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, 26 u8 out[16]); 27 28 u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) 29 { 30 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { 31 if (static_branch_likely(&have_pmull)) { 32 if (crypto_simd_usable()) { 33 kernel_neon_begin(); 34 crc = crc_t10dif_pmull64(crc, data, length); 35 kernel_neon_end(); 36 return crc; 37 } 38 } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && 39 static_branch_likely(&have_neon) && 40 crypto_simd_usable()) { 41 u8 buf[16] __aligned(16); 42 43 kernel_neon_begin(); 44 crc_t10dif_pmull8(crc, data, length, buf); 45 kernel_neon_end(); 46 47 crc = 0; 48 data = buf; 49 length = sizeof(buf); 50 } 51 } 52 return crc_t10dif_generic(crc, data, length); 53 } 54 EXPORT_SYMBOL(crc_t10dif_arch); 55 56 static int __init crc_t10dif_arm_init(void) 57 { 58 if (elf_hwcap & HWCAP_NEON) { 59 static_branch_enable(&have_neon); 60 if (elf_hwcap2 & HWCAP2_PMULL) 61 static_branch_enable(&have_pmull); 62 } 63 return 0; 64 } 65 arch_initcall(crc_t10dif_arm_init); 66 67 static void __exit crc_t10dif_arm_exit(void) 68 { 69 } 70 module_exit(crc_t10dif_arm_exit); 71 72 bool crc_t10dif_is_optimized(void) 73 { 74 return static_key_enabled(&have_neon); 75 } 76 EXPORT_SYMBOL(crc_t10dif_is_optimized); 77 78 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 79 MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions"); 80 MODULE_LICENSE("GPL v2"); 81