1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions
4 *
5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8 #include <linux/cpufeature.h>
9
10 #include <asm/hwcap.h>
11 #include <asm/neon.h>
12 #include <asm/simd.h>
13
14 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
15 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
16
17 #define PMULL_MIN_LEN 64 /* min size of buffer for pmull functions */
18
19 asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc);
20 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
21
22 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
23 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
24
crc32_le_scalar(u32 crc,const u8 * p,size_t len)25 static inline u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len)
26 {
27 if (static_branch_likely(&have_crc32))
28 return crc32_armv8_le(crc, p, len);
29 return crc32_le_base(crc, p, len);
30 }
31
crc32_le_arch(u32 crc,const u8 * p,size_t len)32 static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
33 {
34 if (len >= PMULL_MIN_LEN + 15 &&
35 static_branch_likely(&have_pmull) && likely(may_use_simd())) {
36 size_t n = -(uintptr_t)p & 15;
37
38 /* align p to 16-byte boundary */
39 if (n) {
40 crc = crc32_le_scalar(crc, p, n);
41 p += n;
42 len -= n;
43 }
44 n = round_down(len, 16);
45 kernel_neon_begin();
46 crc = crc32_pmull_le(p, n, crc);
47 kernel_neon_end();
48 p += n;
49 len -= n;
50 }
51 return crc32_le_scalar(crc, p, len);
52 }
53
crc32c_scalar(u32 crc,const u8 * p,size_t len)54 static inline u32 crc32c_scalar(u32 crc, const u8 *p, size_t len)
55 {
56 if (static_branch_likely(&have_crc32))
57 return crc32c_armv8_le(crc, p, len);
58 return crc32c_base(crc, p, len);
59 }
60
crc32c_arch(u32 crc,const u8 * p,size_t len)61 static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
62 {
63 if (len >= PMULL_MIN_LEN + 15 &&
64 static_branch_likely(&have_pmull) && likely(may_use_simd())) {
65 size_t n = -(uintptr_t)p & 15;
66
67 /* align p to 16-byte boundary */
68 if (n) {
69 crc = crc32c_scalar(crc, p, n);
70 p += n;
71 len -= n;
72 }
73 n = round_down(len, 16);
74 kernel_neon_begin();
75 crc = crc32c_pmull_le(p, n, crc);
76 kernel_neon_end();
77 p += n;
78 len -= n;
79 }
80 return crc32c_scalar(crc, p, len);
81 }
82
83 #define crc32_be_arch crc32_be_base /* not implemented on this arch */
84
85 #define crc32_mod_init_arch crc32_mod_init_arch
crc32_mod_init_arch(void)86 static void crc32_mod_init_arch(void)
87 {
88 if (elf_hwcap2 & HWCAP2_CRC32)
89 static_branch_enable(&have_crc32);
90 if (elf_hwcap2 & HWCAP2_PMULL)
91 static_branch_enable(&have_pmull);
92 }
93
crc32_optimizations_arch(void)94 static inline u32 crc32_optimizations_arch(void)
95 {
96 if (elf_hwcap2 & (HWCAP2_CRC32 | HWCAP2_PMULL))
97 return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
98 return 0;
99 }
100