1 /* SPDX-License-Identifier: GPL-2.0-only */
2
3 #ifndef _CRYPTO_INTERNAL_SHA2_H
4 #define _CRYPTO_INTERNAL_SHA2_H
5
6 #include <crypto/internal/simd.h>
7 #include <crypto/sha2.h>
8 #include <linux/compiler_attributes.h>
9 #include <linux/string.h>
10 #include <linux/types.h>
11 #include <linux/unaligned.h>
12
13 #if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256)
14 bool sha256_is_arch_optimized(void);
15 #else
sha256_is_arch_optimized(void)16 static inline bool sha256_is_arch_optimized(void)
17 {
18 return false;
19 }
20 #endif
21 void sha256_blocks_generic(u32 state[SHA256_STATE_WORDS],
22 const u8 *data, size_t nblocks);
23 void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
24 const u8 *data, size_t nblocks);
25 void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
26 const u8 *data, size_t nblocks);
27
sha256_choose_blocks(u32 state[SHA256_STATE_WORDS],const u8 * data,size_t nblocks,bool force_generic,bool force_simd)28 static __always_inline void sha256_choose_blocks(
29 u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks,
30 bool force_generic, bool force_simd)
31 {
32 if (!IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) || force_generic)
33 sha256_blocks_generic(state, data, nblocks);
34 else if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD) &&
35 (force_simd || crypto_simd_usable()))
36 sha256_blocks_simd(state, data, nblocks);
37 else
38 sha256_blocks_arch(state, data, nblocks);
39 }
40
sha256_finup(struct crypto_sha256_state * sctx,u8 buf[SHA256_BLOCK_SIZE],size_t len,u8 out[SHA256_DIGEST_SIZE],size_t digest_size,bool force_generic,bool force_simd)41 static __always_inline void sha256_finup(
42 struct crypto_sha256_state *sctx, u8 buf[SHA256_BLOCK_SIZE],
43 size_t len, u8 out[SHA256_DIGEST_SIZE], size_t digest_size,
44 bool force_generic, bool force_simd)
45 {
46 const size_t bit_offset = SHA256_BLOCK_SIZE - 8;
47 __be64 *bits = (__be64 *)&buf[bit_offset];
48 int i;
49
50 buf[len++] = 0x80;
51 if (len > bit_offset) {
52 memset(&buf[len], 0, SHA256_BLOCK_SIZE - len);
53 sha256_choose_blocks(sctx->state, buf, 1, force_generic,
54 force_simd);
55 len = 0;
56 }
57
58 memset(&buf[len], 0, bit_offset - len);
59 *bits = cpu_to_be64(sctx->count << 3);
60 sha256_choose_blocks(sctx->state, buf, 1, force_generic, force_simd);
61
62 for (i = 0; i < digest_size; i += 4)
63 put_unaligned_be32(sctx->state[i / 4], out + i);
64 }
65
66 #endif /* _CRYPTO_INTERNAL_SHA2_H */
67