1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
4 *
5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8 #include <crypto/internal/simd.h>
9
10 #include <asm/neon.h>
11 #include <asm/simd.h>
12
13 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
14 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
15
16 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
17
18 asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len);
19 asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len,
20 u8 out[16]);
21
crc_t10dif_arch(u16 crc,const u8 * data,size_t length)22 static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
23 {
24 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
25 if (static_branch_likely(&have_pmull)) {
26 if (crypto_simd_usable()) {
27 kernel_neon_begin();
28 crc = crc_t10dif_pmull64(crc, data, length);
29 kernel_neon_end();
30 return crc;
31 }
32 } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&
33 static_branch_likely(&have_neon) &&
34 crypto_simd_usable()) {
35 u8 buf[16] __aligned(16);
36
37 kernel_neon_begin();
38 crc_t10dif_pmull8(crc, data, length, buf);
39 kernel_neon_end();
40
41 return crc_t10dif_generic(0, buf, sizeof(buf));
42 }
43 }
44 return crc_t10dif_generic(crc, data, length);
45 }
46
47 #define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
crc_t10dif_mod_init_arch(void)48 static inline void crc_t10dif_mod_init_arch(void)
49 {
50 if (elf_hwcap & HWCAP_NEON) {
51 static_branch_enable(&have_neon);
52 if (elf_hwcap2 & HWCAP2_PMULL)
53 static_branch_enable(&have_pmull);
54 }
55 }
56