1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * SHA-3 optimized using the CP Assist for Cryptographic Functions (CPACF) 4 * 5 * Copyright 2025 Google LLC 6 */ 7 #include <asm/cpacf.h> 8 #include <linux/cpufeature.h> 9 10 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3); 11 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3_init_optim); 12 13 static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data, 14 size_t nblocks, size_t block_size) 15 { 16 if (static_branch_likely(&have_sha3)) { 17 /* 18 * Note that KIMD assumes little-endian order of the state 19 * words. sha3_state already uses that order, though, so 20 * there's no need for a byteswap. 21 */ 22 switch (block_size) { 23 case SHA3_224_BLOCK_SIZE: 24 cpacf_kimd(CPACF_KIMD_SHA3_224, state, 25 data, nblocks * block_size); 26 return; 27 case SHA3_256_BLOCK_SIZE: 28 /* 29 * This case handles both SHA3-256 and SHAKE256, since 30 * they have the same block size. 31 */ 32 cpacf_kimd(CPACF_KIMD_SHA3_256, state, 33 data, nblocks * block_size); 34 return; 35 case SHA3_384_BLOCK_SIZE: 36 cpacf_kimd(CPACF_KIMD_SHA3_384, state, 37 data, nblocks * block_size); 38 return; 39 case SHA3_512_BLOCK_SIZE: 40 cpacf_kimd(CPACF_KIMD_SHA3_512, state, 41 data, nblocks * block_size); 42 return; 43 } 44 } 45 sha3_absorb_blocks_generic(state, data, nblocks, block_size); 46 } 47 48 static void sha3_keccakf(struct sha3_state *state) 49 { 50 if (static_branch_likely(&have_sha3)) { 51 /* 52 * Passing zeroes into any of CPACF_KIMD_SHA3_* gives the plain 53 * Keccak-f permutation, which is what we want here. Use 54 * SHA3-512 since it has the smallest block size. 55 */ 56 static const u8 zeroes[SHA3_512_BLOCK_SIZE]; 57 58 cpacf_kimd(CPACF_KIMD_SHA3_512, state, zeroes, sizeof(zeroes)); 59 } else { 60 sha3_keccakf_generic(state); 61 } 62 } 63 64 static inline bool s390_sha3(int func, const u8 *in, size_t in_len, 65 u8 *out, size_t out_len) 66 { 67 struct sha3_state state; 68 69 if (!static_branch_likely(&have_sha3)) 70 return false; 71 72 if (static_branch_likely(&have_sha3_init_optim)) 73 func |= CPACF_KLMD_NIP | CPACF_KLMD_DUFOP; 74 else 75 memset(&state, 0, sizeof(state)); 76 77 cpacf_klmd(func, &state, in, in_len); 78 79 if (static_branch_likely(&have_sha3_init_optim)) 80 kmsan_unpoison_memory(&state, out_len); 81 82 memcpy(out, &state, out_len); 83 memzero_explicit(&state, sizeof(state)); 84 return true; 85 } 86 87 #define sha3_224_arch sha3_224_arch 88 static bool sha3_224_arch(const u8 *in, size_t in_len, 89 u8 out[SHA3_224_DIGEST_SIZE]) 90 { 91 return s390_sha3(CPACF_KLMD_SHA3_224, in, in_len, 92 out, SHA3_224_DIGEST_SIZE); 93 } 94 95 #define sha3_256_arch sha3_256_arch 96 static bool sha3_256_arch(const u8 *in, size_t in_len, 97 u8 out[SHA3_256_DIGEST_SIZE]) 98 { 99 return s390_sha3(CPACF_KLMD_SHA3_256, in, in_len, 100 out, SHA3_256_DIGEST_SIZE); 101 } 102 103 #define sha3_384_arch sha3_384_arch 104 static bool sha3_384_arch(const u8 *in, size_t in_len, 105 u8 out[SHA3_384_DIGEST_SIZE]) 106 { 107 return s390_sha3(CPACF_KLMD_SHA3_384, in, in_len, 108 out, SHA3_384_DIGEST_SIZE); 109 } 110 111 #define sha3_512_arch sha3_512_arch 112 static bool sha3_512_arch(const u8 *in, size_t in_len, 113 u8 out[SHA3_512_DIGEST_SIZE]) 114 { 115 return s390_sha3(CPACF_KLMD_SHA3_512, in, in_len, 116 out, SHA3_512_DIGEST_SIZE); 117 } 118 119 #define sha3_mod_init_arch sha3_mod_init_arch 120 static void sha3_mod_init_arch(void) 121 { 122 int num_present = 0; 123 int num_possible = 0; 124 125 if (!cpu_have_feature(S390_CPU_FEATURE_MSA)) 126 return; 127 /* 128 * Since all the SHA-3 functions are in Message-Security-Assist 129 * Extension 6, just treat them as all or nothing. This way we need 130 * only one static_key. 131 */ 132 #define QUERY(opcode, func) \ 133 ({ num_present += !!cpacf_query_func(opcode, func); num_possible++; }) 134 QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_224); 135 QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_256); 136 QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_384); 137 QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_512); 138 QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_224); 139 QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_256); 140 QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_384); 141 QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_512); 142 #undef QUERY 143 144 if (num_present == num_possible) { 145 static_branch_enable(&have_sha3); 146 if (test_facility(86)) 147 static_branch_enable(&have_sha3_init_optim); 148 } else if (num_present != 0) { 149 pr_warn("Unsupported combination of SHA-3 facilities\n"); 150 } 151 } 152