1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * GHASH optimized using the CP Assist for Cryptographic Functions (CPACF) 4 * 5 * Copyright 2026 Google LLC 6 */ 7 #include <asm/cpacf.h> 8 #include <linux/cpufeature.h> 9 10 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_ghash); 11 12 #define ghash_preparekey_arch ghash_preparekey_arch 13 static void ghash_preparekey_arch(struct ghash_key *key, 14 const u8 raw_key[GHASH_BLOCK_SIZE]) 15 { 16 /* Save key in POLYVAL format for fallback */ 17 ghash_key_to_polyval(raw_key, &key->h); 18 19 /* Save key in GHASH format for CPACF_KIMD_GHASH */ 20 memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); 21 } 22 23 #define ghash_blocks_arch ghash_blocks_arch 24 static void ghash_blocks_arch(struct polyval_elem *acc, 25 const struct ghash_key *key, 26 const u8 *data, size_t nblocks) 27 { 28 if (static_branch_likely(&have_cpacf_ghash)) { 29 /* 30 * CPACF_KIMD_GHASH requires the accumulator and key in a single 31 * buffer, each using the GHASH convention. 32 */ 33 u8 ctx[2][GHASH_BLOCK_SIZE] __aligned(8); 34 35 polyval_acc_to_ghash(acc, ctx[0]); 36 memcpy(ctx[1], key->h_raw, GHASH_BLOCK_SIZE); 37 38 cpacf_kimd(CPACF_KIMD_GHASH, ctx, data, 39 nblocks * GHASH_BLOCK_SIZE); 40 41 ghash_acc_to_polyval(ctx[0], acc); 42 memzero_explicit(ctx, sizeof(ctx)); 43 } else { 44 ghash_blocks_generic(acc, &key->h, data, nblocks); 45 } 46 } 47 48 #define gf128hash_mod_init_arch gf128hash_mod_init_arch 49 static void gf128hash_mod_init_arch(void) 50 { 51 if (cpu_have_feature(S390_CPU_FEATURE_MSA) && 52 cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH)) 53 static_branch_enable(&have_cpacf_ghash); 54 } 55