1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * BLAKE2b digest algorithm, NEON accelerated 4 * 5 * Copyright 2020 Google LLC 6 */ 7 8 #include <asm/neon.h> 9 #include <asm/simd.h> 10 11 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 12 13 asmlinkage void blake2b_compress_neon(struct blake2b_ctx *ctx, 14 const u8 *data, size_t nblocks, u32 inc); 15 16 static void blake2b_compress(struct blake2b_ctx *ctx, 17 const u8 *data, size_t nblocks, u32 inc) 18 { 19 if (!static_branch_likely(&have_neon) || !may_use_simd()) { 20 blake2b_compress_generic(ctx, data, nblocks, inc); 21 return; 22 } 23 do { 24 const size_t blocks = min_t(size_t, nblocks, 25 SZ_4K / BLAKE2B_BLOCK_SIZE); 26 27 scoped_ksimd() 28 blake2b_compress_neon(ctx, data, blocks, inc); 29 30 data += blocks * BLAKE2B_BLOCK_SIZE; 31 nblocks -= blocks; 32 } while (nblocks); 33 } 34 35 #define blake2b_mod_init_arch blake2b_mod_init_arch 36 static void blake2b_mod_init_arch(void) 37 { 38 if (elf_hwcap & HWCAP_NEON) 39 static_branch_enable(&have_neon); 40 } 41