1*13cecc52SEric Biggers /* SPDX-License-Identifier: GPL-2.0-only */
2*13cecc52SEric Biggers /*
3*13cecc52SEric Biggers * ChaCha stream cipher (RISC-V optimized)
4*13cecc52SEric Biggers *
5*13cecc52SEric Biggers * Copyright (C) 2023 SiFive, Inc.
6*13cecc52SEric Biggers * Author: Jerry Shih <jerry.shih@sifive.com>
7*13cecc52SEric Biggers */
8*13cecc52SEric Biggers
9*13cecc52SEric Biggers #include <asm/simd.h>
10*13cecc52SEric Biggers #include <asm/vector.h>
11*13cecc52SEric Biggers #include <crypto/internal/simd.h>
12*13cecc52SEric Biggers #include <linux/linkage.h>
13*13cecc52SEric Biggers
14*13cecc52SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_zvkb);
15*13cecc52SEric Biggers
16*13cecc52SEric Biggers asmlinkage void chacha_zvkb(struct chacha_state *state, const u8 *in, u8 *out,
17*13cecc52SEric Biggers size_t nblocks, int nrounds);
18*13cecc52SEric Biggers
19*13cecc52SEric Biggers #define hchacha_block_arch hchacha_block_generic /* not implemented yet */
20*13cecc52SEric Biggers
chacha_crypt_arch(struct chacha_state * state,u8 * dst,const u8 * src,unsigned int bytes,int nrounds)21*13cecc52SEric Biggers static void chacha_crypt_arch(struct chacha_state *state, u8 *dst,
22*13cecc52SEric Biggers const u8 *src, unsigned int bytes, int nrounds)
23*13cecc52SEric Biggers {
24*13cecc52SEric Biggers u8 block_buffer[CHACHA_BLOCK_SIZE];
25*13cecc52SEric Biggers unsigned int full_blocks = bytes / CHACHA_BLOCK_SIZE;
26*13cecc52SEric Biggers unsigned int tail_bytes = bytes % CHACHA_BLOCK_SIZE;
27*13cecc52SEric Biggers
28*13cecc52SEric Biggers if (!static_branch_likely(&use_zvkb) || !crypto_simd_usable())
29*13cecc52SEric Biggers return chacha_crypt_generic(state, dst, src, bytes, nrounds);
30*13cecc52SEric Biggers
31*13cecc52SEric Biggers kernel_vector_begin();
32*13cecc52SEric Biggers if (full_blocks) {
33*13cecc52SEric Biggers chacha_zvkb(state, src, dst, full_blocks, nrounds);
34*13cecc52SEric Biggers src += full_blocks * CHACHA_BLOCK_SIZE;
35*13cecc52SEric Biggers dst += full_blocks * CHACHA_BLOCK_SIZE;
36*13cecc52SEric Biggers }
37*13cecc52SEric Biggers if (tail_bytes) {
38*13cecc52SEric Biggers memcpy(block_buffer, src, tail_bytes);
39*13cecc52SEric Biggers chacha_zvkb(state, block_buffer, block_buffer, 1, nrounds);
40*13cecc52SEric Biggers memcpy(dst, block_buffer, tail_bytes);
41*13cecc52SEric Biggers }
42*13cecc52SEric Biggers kernel_vector_end();
43*13cecc52SEric Biggers }
44*13cecc52SEric Biggers
45*13cecc52SEric Biggers #define chacha_mod_init_arch chacha_mod_init_arch
chacha_mod_init_arch(void)46*13cecc52SEric Biggers static void chacha_mod_init_arch(void)
47*13cecc52SEric Biggers {
48*13cecc52SEric Biggers if (riscv_isa_extension_available(NULL, ZVKB) &&
49*13cecc52SEric Biggers riscv_vector_vlen() >= 128)
50*13cecc52SEric Biggers static_branch_enable(&use_zvkb);
51*13cecc52SEric Biggers }
52