1*af413d71SEric Biggers /* SPDX-License-Identifier: GPL-2.0-only */ 2*af413d71SEric Biggers /* 3*af413d71SEric Biggers * GHASH, RISC-V optimized 4*af413d71SEric Biggers * 5*af413d71SEric Biggers * Copyright (C) 2023 VRULL GmbH 6*af413d71SEric Biggers * Copyright (C) 2023 SiFive, Inc. 7*af413d71SEric Biggers * Copyright 2026 Google LLC 8*af413d71SEric Biggers */ 9*af413d71SEric Biggers 10*af413d71SEric Biggers #include <asm/simd.h> 11*af413d71SEric Biggers #include <asm/vector.h> 12*af413d71SEric Biggers 13*af413d71SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg); 14*af413d71SEric Biggers 15*af413d71SEric Biggers asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], 16*af413d71SEric Biggers const u8 key[GHASH_BLOCK_SIZE], 17*af413d71SEric Biggers const u8 *data, size_t nblocks); 18*af413d71SEric Biggers 19*af413d71SEric Biggers #define ghash_preparekey_arch ghash_preparekey_arch 20*af413d71SEric Biggers static void ghash_preparekey_arch(struct ghash_key *key, 21*af413d71SEric Biggers const u8 raw_key[GHASH_BLOCK_SIZE]) 22*af413d71SEric Biggers { 23*af413d71SEric Biggers /* Save key in POLYVAL format for fallback */ 24*af413d71SEric Biggers ghash_key_to_polyval(raw_key, &key->h); 25*af413d71SEric Biggers 26*af413d71SEric Biggers /* Save key in GHASH format for zvkg */ 27*af413d71SEric Biggers memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); 28*af413d71SEric Biggers } 29*af413d71SEric Biggers 30*af413d71SEric Biggers #define ghash_blocks_arch ghash_blocks_arch 31*af413d71SEric Biggers static void ghash_blocks_arch(struct polyval_elem *acc, 32*af413d71SEric Biggers const struct ghash_key *key, 33*af413d71SEric Biggers const u8 *data, size_t nblocks) 34*af413d71SEric Biggers { 35*af413d71SEric Biggers if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) { 36*af413d71SEric Biggers u8 ghash_acc[GHASH_BLOCK_SIZE]; 37*af413d71SEric Biggers 38*af413d71SEric Biggers polyval_acc_to_ghash(acc, ghash_acc); 39*af413d71SEric Biggers 40*af413d71SEric Biggers kernel_vector_begin(); 41*af413d71SEric Biggers ghash_zvkg(ghash_acc, key->h_raw, data, nblocks); 42*af413d71SEric Biggers kernel_vector_end(); 43*af413d71SEric Biggers 44*af413d71SEric Biggers ghash_acc_to_polyval(ghash_acc, acc); 45*af413d71SEric Biggers memzero_explicit(ghash_acc, sizeof(ghash_acc)); 46*af413d71SEric Biggers } else { 47*af413d71SEric Biggers ghash_blocks_generic(acc, &key->h, data, nblocks); 48*af413d71SEric Biggers } 49*af413d71SEric Biggers } 50*af413d71SEric Biggers 51*af413d71SEric Biggers #define gf128hash_mod_init_arch gf128hash_mod_init_arch 52*af413d71SEric Biggers static void gf128hash_mod_init_arch(void) 53*af413d71SEric Biggers { 54*af413d71SEric Biggers if (riscv_isa_extension_available(NULL, ZVKG) && 55*af413d71SEric Biggers riscv_vector_vlen() >= 128) 56*af413d71SEric Biggers static_branch_enable(&have_zvkg); 57*af413d71SEric Biggers } 58