1*530b304fSEric Biggers // SPDX-License-Identifier: GPL-2.0-only
2*530b304fSEric Biggers /*
3*530b304fSEric Biggers * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
4*530b304fSEric Biggers *
5*530b304fSEric Biggers * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6*530b304fSEric Biggers */
7*530b304fSEric Biggers
8*530b304fSEric Biggers #include <crypto/internal/simd.h>
9*530b304fSEric Biggers
10*530b304fSEric Biggers #include <asm/neon.h>
11*530b304fSEric Biggers #include <asm/simd.h>
12*530b304fSEric Biggers
13*530b304fSEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
14*530b304fSEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
15*530b304fSEric Biggers
16*530b304fSEric Biggers #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
17*530b304fSEric Biggers
18*530b304fSEric Biggers asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len);
19*530b304fSEric Biggers asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len,
20*530b304fSEric Biggers u8 out[16]);
21*530b304fSEric Biggers
crc_t10dif_arch(u16 crc,const u8 * data,size_t length)22*530b304fSEric Biggers static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
23*530b304fSEric Biggers {
24*530b304fSEric Biggers if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
25*530b304fSEric Biggers if (static_branch_likely(&have_pmull)) {
26*530b304fSEric Biggers if (crypto_simd_usable()) {
27*530b304fSEric Biggers kernel_neon_begin();
28*530b304fSEric Biggers crc = crc_t10dif_pmull64(crc, data, length);
29*530b304fSEric Biggers kernel_neon_end();
30*530b304fSEric Biggers return crc;
31*530b304fSEric Biggers }
32*530b304fSEric Biggers } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&
33*530b304fSEric Biggers static_branch_likely(&have_neon) &&
34*530b304fSEric Biggers crypto_simd_usable()) {
35*530b304fSEric Biggers u8 buf[16] __aligned(16);
36*530b304fSEric Biggers
37*530b304fSEric Biggers kernel_neon_begin();
38*530b304fSEric Biggers crc_t10dif_pmull8(crc, data, length, buf);
39*530b304fSEric Biggers kernel_neon_end();
40*530b304fSEric Biggers
41*530b304fSEric Biggers return crc_t10dif_generic(0, buf, sizeof(buf));
42*530b304fSEric Biggers }
43*530b304fSEric Biggers }
44*530b304fSEric Biggers return crc_t10dif_generic(crc, data, length);
45*530b304fSEric Biggers }
46*530b304fSEric Biggers
47*530b304fSEric Biggers #define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
crc_t10dif_mod_init_arch(void)48*530b304fSEric Biggers static inline void crc_t10dif_mod_init_arch(void)
49*530b304fSEric Biggers {
50*530b304fSEric Biggers if (elf_hwcap & HWCAP_NEON) {
51*530b304fSEric Biggers static_branch_enable(&have_neon);
52*530b304fSEric Biggers if (elf_hwcap2 & HWCAP2_PMULL)
53*530b304fSEric Biggers static_branch_enable(&have_pmull);
54*530b304fSEric Biggers }
55*530b304fSEric Biggers }
56