1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * GHASH, arm optimized 4 * 5 * Copyright 2026 Google LLC 6 */ 7 8 #include <asm/hwcap.h> 9 #include <asm/neon.h> 10 #include <asm/simd.h> 11 12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 13 14 void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, 15 const u8 *src, const struct polyval_elem *h); 16 17 #define ghash_blocks_arch ghash_blocks_arch 18 static void ghash_blocks_arch(struct polyval_elem *acc, 19 const struct ghash_key *key, 20 const u8 *data, size_t nblocks) 21 { 22 if (static_branch_likely(&have_neon) && may_use_simd()) { 23 do { 24 /* Allow rescheduling every 4 KiB. */ 25 size_t n = 26 min_t(size_t, nblocks, 4096 / GHASH_BLOCK_SIZE); 27 28 scoped_ksimd() 29 pmull_ghash_update_p8(n, acc, data, &key->h); 30 data += n * GHASH_BLOCK_SIZE; 31 nblocks -= n; 32 } while (nblocks); 33 } else { 34 ghash_blocks_generic(acc, &key->h, data, nblocks); 35 } 36 } 37 38 #define gf128hash_mod_init_arch gf128hash_mod_init_arch 39 static void gf128hash_mod_init_arch(void) 40 { 41 if (elf_hwcap & HWCAP_NEON) 42 static_branch_enable(&have_neon); 43 } 44