1*71e59795SEric Biggers /* SPDX-License-Identifier: GPL-2.0-or-later */ 2*71e59795SEric Biggers /* 3*71e59795SEric Biggers * GHASH, arm optimized 4*71e59795SEric Biggers * 5*71e59795SEric Biggers * Copyright 2026 Google LLC 6*71e59795SEric Biggers */ 7*71e59795SEric Biggers 8*71e59795SEric Biggers #include <asm/hwcap.h> 9*71e59795SEric Biggers #include <asm/neon.h> 10*71e59795SEric Biggers #include <asm/simd.h> 11*71e59795SEric Biggers 12*71e59795SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 13*71e59795SEric Biggers 14*71e59795SEric Biggers void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, 15*71e59795SEric Biggers const u8 *src, const struct polyval_elem *h); 16*71e59795SEric Biggers 17*71e59795SEric Biggers #define ghash_blocks_arch ghash_blocks_arch 18*71e59795SEric Biggers static void ghash_blocks_arch(struct polyval_elem *acc, 19*71e59795SEric Biggers const struct ghash_key *key, 20*71e59795SEric Biggers const u8 *data, size_t nblocks) 21*71e59795SEric Biggers { 22*71e59795SEric Biggers if (static_branch_likely(&have_neon) && may_use_simd()) { 23*71e59795SEric Biggers do { 24*71e59795SEric Biggers /* Allow rescheduling every 4 KiB. */ 25*71e59795SEric Biggers size_t n = 26*71e59795SEric Biggers min_t(size_t, nblocks, 4096 / GHASH_BLOCK_SIZE); 27*71e59795SEric Biggers 28*71e59795SEric Biggers scoped_ksimd() 29*71e59795SEric Biggers pmull_ghash_update_p8(n, acc, data, &key->h); 30*71e59795SEric Biggers data += n * GHASH_BLOCK_SIZE; 31*71e59795SEric Biggers nblocks -= n; 32*71e59795SEric Biggers } while (nblocks); 33*71e59795SEric Biggers } else { 34*71e59795SEric Biggers ghash_blocks_generic(acc, &key->h, data, nblocks); 35*71e59795SEric Biggers } 36*71e59795SEric Biggers } 37*71e59795SEric Biggers 38*71e59795SEric Biggers #define gf128hash_mod_init_arch gf128hash_mod_init_arch 39*71e59795SEric Biggers static void gf128hash_mod_init_arch(void) 40*71e59795SEric Biggers { 41*71e59795SEric Biggers if (elf_hwcap & HWCAP_NEON) 42*71e59795SEric Biggers static_branch_enable(&have_neon); 43*71e59795SEric Biggers } 44