1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/cpufeature.h> 9 10 #include <crypto/internal/simd.h> 11 12 #include <asm/neon.h> 13 #include <asm/simd.h> 14 15 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd); 16 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 17 18 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 19 20 asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len, 21 u8 out[16]); 22 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); 23 24 static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) 25 { 26 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { 27 if (static_branch_likely(&have_pmull)) { 28 if (crypto_simd_usable()) { 29 kernel_neon_begin(); 30 crc = crc_t10dif_pmull_p64(crc, data, length); 31 kernel_neon_end(); 32 return crc; 33 } 34 } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && 35 static_branch_likely(&have_asimd) && 36 crypto_simd_usable()) { 37 u8 buf[16]; 38 39 kernel_neon_begin(); 40 crc_t10dif_pmull_p8(crc, data, length, buf); 41 kernel_neon_end(); 42 43 return crc_t10dif_generic(0, buf, sizeof(buf)); 44 } 45 } 46 return crc_t10dif_generic(crc, data, length); 47 } 48 49 #define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch 50 static inline void crc_t10dif_mod_init_arch(void) 51 { 52 if (cpu_have_named_feature(ASIMD)) { 53 static_branch_enable(&have_asimd); 54 if (cpu_have_named_feature(PMULL)) 55 static_branch_enable(&have_pmull); 56 } 57 } 58