1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * BLAKE2b digest algorithm, NEON accelerated 4 * 5 * Copyright 2020 Google LLC 6 */ 7 8 #include <asm/neon.h> 9 #include <asm/simd.h> 10 11 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 12 13 asmlinkage void blake2b_compress_neon(struct blake2b_ctx *ctx, 14 const u8 *data, size_t nblocks, u32 inc); 15 16 static void blake2b_compress(struct blake2b_ctx *ctx, 17 const u8 *data, size_t nblocks, u32 inc) 18 { 19 if (!static_branch_likely(&have_neon) || !may_use_simd()) { 20 blake2b_compress_generic(ctx, data, nblocks, inc); 21 return; 22 } 23 do { 24 const size_t blocks = min_t(size_t, nblocks, 25 SZ_4K / BLAKE2B_BLOCK_SIZE); 26 27 kernel_neon_begin(); 28 blake2b_compress_neon(ctx, data, blocks, inc); 29 kernel_neon_end(); 30 31 data += blocks * BLAKE2B_BLOCK_SIZE; 32 nblocks -= blocks; 33 } while (nblocks); 34 } 35 36 #define blake2b_mod_init_arch blake2b_mod_init_arch 37 static void blake2b_mod_init_arch(void) 38 { 39 if (elf_hwcap & HWCAP_NEON) 40 static_branch_enable(&have_neon); 41 } 42