1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/neon.h> 9 #include <asm/simd.h> 10 11 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 13 14 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 15 16 asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); 17 asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, 18 u8 out[16]); 19 20 static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) 21 { 22 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { 23 if (static_branch_likely(&have_pmull)) { 24 if (likely(may_use_simd())) { 25 kernel_neon_begin(); 26 crc = crc_t10dif_pmull64(crc, data, length); 27 kernel_neon_end(); 28 return crc; 29 } 30 } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && 31 static_branch_likely(&have_neon) && 32 likely(may_use_simd())) { 33 u8 buf[16] __aligned(16); 34 35 kernel_neon_begin(); 36 crc_t10dif_pmull8(crc, data, length, buf); 37 kernel_neon_end(); 38 39 return crc_t10dif_generic(0, buf, sizeof(buf)); 40 } 41 } 42 return crc_t10dif_generic(crc, data, length); 43 } 44 45 #define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch 46 static void crc_t10dif_mod_init_arch(void) 47 { 48 if (elf_hwcap & HWCAP_NEON) { 49 static_branch_enable(&have_neon); 50 if (elf_hwcap2 & HWCAP2_PMULL) 51 static_branch_enable(&have_pmull); 52 } 53 } 54