1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * BLAKE2b digest algorithm, NEON accelerated
4 *
5 * Copyright 2020 Google LLC
6 */
7
8 #include <asm/neon.h>
9 #include <asm/simd.h>
10
11 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
12
13 asmlinkage void blake2b_compress_neon(struct blake2b_ctx *ctx,
14 const u8 *data, size_t nblocks, u32 inc);
15
blake2b_compress(struct blake2b_ctx * ctx,const u8 * data,size_t nblocks,u32 inc)16 static void blake2b_compress(struct blake2b_ctx *ctx,
17 const u8 *data, size_t nblocks, u32 inc)
18 {
19 if (!static_branch_likely(&have_neon) || !may_use_simd()) {
20 blake2b_compress_generic(ctx, data, nblocks, inc);
21 return;
22 }
23 do {
24 const size_t blocks = min_t(size_t, nblocks,
25 SZ_4K / BLAKE2B_BLOCK_SIZE);
26
27 scoped_ksimd()
28 blake2b_compress_neon(ctx, data, blocks, inc);
29
30 data += blocks * BLAKE2B_BLOCK_SIZE;
31 nblocks -= blocks;
32 } while (nblocks);
33 }
34
35 #define blake2b_mod_init_arch blake2b_mod_init_arch
blake2b_mod_init_arch(void)36 static void blake2b_mod_init_arch(void)
37 {
38 if (elf_hwcap & HWCAP_NEON)
39 static_branch_enable(&have_neon);
40 }
41