1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * SHA-3 optimized using the CP Assist for Cryptographic Functions (CPACF)
4 *
5 * Copyright 2025 Google LLC
6 */
7 #include <asm/cpacf.h>
8 #include <linux/cpufeature.h>
9
10 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3);
11 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3_init_optim);
12
sha3_absorb_blocks(struct sha3_state * state,const u8 * data,size_t nblocks,size_t block_size)13 static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data,
14 size_t nblocks, size_t block_size)
15 {
16 if (static_branch_likely(&have_sha3)) {
17 /*
18 * Note that KIMD assumes little-endian order of the state
19 * words. sha3_state already uses that order, though, so
20 * there's no need for a byteswap.
21 */
22 switch (block_size) {
23 case SHA3_224_BLOCK_SIZE:
24 cpacf_kimd(CPACF_KIMD_SHA3_224, state,
25 data, nblocks * block_size);
26 return;
27 case SHA3_256_BLOCK_SIZE:
28 /*
29 * This case handles both SHA3-256 and SHAKE256, since
30 * they have the same block size.
31 */
32 cpacf_kimd(CPACF_KIMD_SHA3_256, state,
33 data, nblocks * block_size);
34 return;
35 case SHA3_384_BLOCK_SIZE:
36 cpacf_kimd(CPACF_KIMD_SHA3_384, state,
37 data, nblocks * block_size);
38 return;
39 case SHA3_512_BLOCK_SIZE:
40 cpacf_kimd(CPACF_KIMD_SHA3_512, state,
41 data, nblocks * block_size);
42 return;
43 }
44 }
45 sha3_absorb_blocks_generic(state, data, nblocks, block_size);
46 }
47
sha3_keccakf(struct sha3_state * state)48 static void sha3_keccakf(struct sha3_state *state)
49 {
50 if (static_branch_likely(&have_sha3)) {
51 /*
52 * Passing zeroes into any of CPACF_KIMD_SHA3_* gives the plain
53 * Keccak-f permutation, which is what we want here. Use
54 * SHA3-512 since it has the smallest block size.
55 */
56 static const u8 zeroes[SHA3_512_BLOCK_SIZE];
57
58 cpacf_kimd(CPACF_KIMD_SHA3_512, state, zeroes, sizeof(zeroes));
59 } else {
60 sha3_keccakf_generic(state);
61 }
62 }
63
s390_sha3(int func,const u8 * in,size_t in_len,u8 * out,size_t out_len)64 static inline bool s390_sha3(int func, const u8 *in, size_t in_len,
65 u8 *out, size_t out_len)
66 {
67 struct sha3_state state;
68
69 if (!static_branch_likely(&have_sha3))
70 return false;
71
72 if (static_branch_likely(&have_sha3_init_optim))
73 func |= CPACF_KLMD_NIP | CPACF_KLMD_DUFOP;
74 else
75 memset(&state, 0, sizeof(state));
76
77 cpacf_klmd(func, &state, in, in_len);
78
79 if (static_branch_likely(&have_sha3_init_optim))
80 kmsan_unpoison_memory(&state, out_len);
81
82 memcpy(out, &state, out_len);
83 memzero_explicit(&state, sizeof(state));
84 return true;
85 }
86
87 #define sha3_224_arch sha3_224_arch
sha3_224_arch(const u8 * in,size_t in_len,u8 out[SHA3_224_DIGEST_SIZE])88 static bool sha3_224_arch(const u8 *in, size_t in_len,
89 u8 out[SHA3_224_DIGEST_SIZE])
90 {
91 return s390_sha3(CPACF_KLMD_SHA3_224, in, in_len,
92 out, SHA3_224_DIGEST_SIZE);
93 }
94
95 #define sha3_256_arch sha3_256_arch
sha3_256_arch(const u8 * in,size_t in_len,u8 out[SHA3_256_DIGEST_SIZE])96 static bool sha3_256_arch(const u8 *in, size_t in_len,
97 u8 out[SHA3_256_DIGEST_SIZE])
98 {
99 return s390_sha3(CPACF_KLMD_SHA3_256, in, in_len,
100 out, SHA3_256_DIGEST_SIZE);
101 }
102
103 #define sha3_384_arch sha3_384_arch
sha3_384_arch(const u8 * in,size_t in_len,u8 out[SHA3_384_DIGEST_SIZE])104 static bool sha3_384_arch(const u8 *in, size_t in_len,
105 u8 out[SHA3_384_DIGEST_SIZE])
106 {
107 return s390_sha3(CPACF_KLMD_SHA3_384, in, in_len,
108 out, SHA3_384_DIGEST_SIZE);
109 }
110
111 #define sha3_512_arch sha3_512_arch
sha3_512_arch(const u8 * in,size_t in_len,u8 out[SHA3_512_DIGEST_SIZE])112 static bool sha3_512_arch(const u8 *in, size_t in_len,
113 u8 out[SHA3_512_DIGEST_SIZE])
114 {
115 return s390_sha3(CPACF_KLMD_SHA3_512, in, in_len,
116 out, SHA3_512_DIGEST_SIZE);
117 }
118
119 #define sha3_mod_init_arch sha3_mod_init_arch
sha3_mod_init_arch(void)120 static void sha3_mod_init_arch(void)
121 {
122 int num_present = 0;
123 int num_possible = 0;
124
125 if (!cpu_have_feature(S390_CPU_FEATURE_MSA))
126 return;
127 /*
128 * Since all the SHA-3 functions are in Message-Security-Assist
129 * Extension 6, just treat them as all or nothing. This way we need
130 * only one static_key.
131 */
132 #define QUERY(opcode, func) \
133 ({ num_present += !!cpacf_query_func(opcode, func); num_possible++; })
134 QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_224);
135 QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_256);
136 QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_384);
137 QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_512);
138 QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_224);
139 QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_256);
140 QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_384);
141 QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_512);
142 #undef QUERY
143
144 if (num_present == num_possible) {
145 static_branch_enable(&have_sha3);
146 if (test_facility(86))
147 static_branch_enable(&have_sha3_init_optim);
148 } else if (num_present != 0) {
149 pr_warn("Unsupported combination of SHA-3 facilities\n");
150 }
151 }
152