1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * GHASH, RISC-V optimized 4 * 5 * Copyright (C) 2023 VRULL GmbH 6 * Copyright (C) 2023 SiFive, Inc. 7 * Copyright 2026 Google LLC 8 */ 9 10 #include <asm/simd.h> 11 #include <asm/vector.h> 12 13 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg); 14 15 asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], 16 const u8 key[GHASH_BLOCK_SIZE], 17 const u8 *data, size_t nblocks); 18 19 #define ghash_preparekey_arch ghash_preparekey_arch 20 static void ghash_preparekey_arch(struct ghash_key *key, 21 const u8 raw_key[GHASH_BLOCK_SIZE]) 22 { 23 /* Save key in POLYVAL format for fallback */ 24 ghash_key_to_polyval(raw_key, &key->h); 25 26 /* Save key in GHASH format for zvkg */ 27 memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); 28 } 29 30 #define ghash_blocks_arch ghash_blocks_arch 31 static void ghash_blocks_arch(struct polyval_elem *acc, 32 const struct ghash_key *key, 33 const u8 *data, size_t nblocks) 34 { 35 if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) { 36 u8 ghash_acc[GHASH_BLOCK_SIZE]; 37 38 polyval_acc_to_ghash(acc, ghash_acc); 39 40 kernel_vector_begin(); 41 ghash_zvkg(ghash_acc, key->h_raw, data, nblocks); 42 kernel_vector_end(); 43 44 ghash_acc_to_polyval(ghash_acc, acc); 45 memzero_explicit(ghash_acc, sizeof(ghash_acc)); 46 } else { 47 ghash_blocks_generic(acc, &key->h, data, nblocks); 48 } 49 } 50 51 #define gf128hash_mod_init_arch gf128hash_mod_init_arch 52 static void gf128hash_mod_init_arch(void) 53 { 54 if (riscv_isa_extension_available(NULL, ZVKG) && 55 riscv_vector_vlen() >= 128) 56 static_branch_enable(&have_zvkg); 57 } 58