1*bb546688SJerry Shih // SPDX-License-Identifier: GPL-2.0-only
2*bb546688SJerry Shih /*
3*bb546688SJerry Shih * ChaCha20 using the RISC-V vector crypto extensions
4*bb546688SJerry Shih *
5*bb546688SJerry Shih * Copyright (C) 2023 SiFive, Inc.
6*bb546688SJerry Shih * Author: Jerry Shih <jerry.shih@sifive.com>
7*bb546688SJerry Shih */
8*bb546688SJerry Shih
9*bb546688SJerry Shih #include <asm/simd.h>
10*bb546688SJerry Shih #include <asm/vector.h>
11*bb546688SJerry Shih #include <crypto/internal/chacha.h>
12*bb546688SJerry Shih #include <crypto/internal/skcipher.h>
13*bb546688SJerry Shih #include <linux/linkage.h>
14*bb546688SJerry Shih #include <linux/module.h>
15*bb546688SJerry Shih
16*bb546688SJerry Shih asmlinkage void chacha20_zvkb(const u32 key[8], const u8 *in, u8 *out,
17*bb546688SJerry Shih size_t len, const u32 iv[4]);
18*bb546688SJerry Shih
riscv64_chacha20_crypt(struct skcipher_request * req)19*bb546688SJerry Shih static int riscv64_chacha20_crypt(struct skcipher_request *req)
20*bb546688SJerry Shih {
21*bb546688SJerry Shih u32 iv[CHACHA_IV_SIZE / sizeof(u32)];
22*bb546688SJerry Shih u8 block_buffer[CHACHA_BLOCK_SIZE];
23*bb546688SJerry Shih struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
24*bb546688SJerry Shih const struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
25*bb546688SJerry Shih struct skcipher_walk walk;
26*bb546688SJerry Shih unsigned int nbytes;
27*bb546688SJerry Shih unsigned int tail_bytes;
28*bb546688SJerry Shih int err;
29*bb546688SJerry Shih
30*bb546688SJerry Shih iv[0] = get_unaligned_le32(req->iv);
31*bb546688SJerry Shih iv[1] = get_unaligned_le32(req->iv + 4);
32*bb546688SJerry Shih iv[2] = get_unaligned_le32(req->iv + 8);
33*bb546688SJerry Shih iv[3] = get_unaligned_le32(req->iv + 12);
34*bb546688SJerry Shih
35*bb546688SJerry Shih err = skcipher_walk_virt(&walk, req, false);
36*bb546688SJerry Shih while (walk.nbytes) {
37*bb546688SJerry Shih nbytes = walk.nbytes & ~(CHACHA_BLOCK_SIZE - 1);
38*bb546688SJerry Shih tail_bytes = walk.nbytes & (CHACHA_BLOCK_SIZE - 1);
39*bb546688SJerry Shih kernel_vector_begin();
40*bb546688SJerry Shih if (nbytes) {
41*bb546688SJerry Shih chacha20_zvkb(ctx->key, walk.src.virt.addr,
42*bb546688SJerry Shih walk.dst.virt.addr, nbytes, iv);
43*bb546688SJerry Shih iv[0] += nbytes / CHACHA_BLOCK_SIZE;
44*bb546688SJerry Shih }
45*bb546688SJerry Shih if (walk.nbytes == walk.total && tail_bytes > 0) {
46*bb546688SJerry Shih memcpy(block_buffer, walk.src.virt.addr + nbytes,
47*bb546688SJerry Shih tail_bytes);
48*bb546688SJerry Shih chacha20_zvkb(ctx->key, block_buffer, block_buffer,
49*bb546688SJerry Shih CHACHA_BLOCK_SIZE, iv);
50*bb546688SJerry Shih memcpy(walk.dst.virt.addr + nbytes, block_buffer,
51*bb546688SJerry Shih tail_bytes);
52*bb546688SJerry Shih tail_bytes = 0;
53*bb546688SJerry Shih }
54*bb546688SJerry Shih kernel_vector_end();
55*bb546688SJerry Shih
56*bb546688SJerry Shih err = skcipher_walk_done(&walk, tail_bytes);
57*bb546688SJerry Shih }
58*bb546688SJerry Shih
59*bb546688SJerry Shih return err;
60*bb546688SJerry Shih }
61*bb546688SJerry Shih
62*bb546688SJerry Shih static struct skcipher_alg riscv64_chacha_alg = {
63*bb546688SJerry Shih .setkey = chacha20_setkey,
64*bb546688SJerry Shih .encrypt = riscv64_chacha20_crypt,
65*bb546688SJerry Shih .decrypt = riscv64_chacha20_crypt,
66*bb546688SJerry Shih .min_keysize = CHACHA_KEY_SIZE,
67*bb546688SJerry Shih .max_keysize = CHACHA_KEY_SIZE,
68*bb546688SJerry Shih .ivsize = CHACHA_IV_SIZE,
69*bb546688SJerry Shih .chunksize = CHACHA_BLOCK_SIZE,
70*bb546688SJerry Shih .walksize = 4 * CHACHA_BLOCK_SIZE,
71*bb546688SJerry Shih .base = {
72*bb546688SJerry Shih .cra_blocksize = 1,
73*bb546688SJerry Shih .cra_ctxsize = sizeof(struct chacha_ctx),
74*bb546688SJerry Shih .cra_priority = 300,
75*bb546688SJerry Shih .cra_name = "chacha20",
76*bb546688SJerry Shih .cra_driver_name = "chacha20-riscv64-zvkb",
77*bb546688SJerry Shih .cra_module = THIS_MODULE,
78*bb546688SJerry Shih },
79*bb546688SJerry Shih };
80*bb546688SJerry Shih
riscv64_chacha_mod_init(void)81*bb546688SJerry Shih static int __init riscv64_chacha_mod_init(void)
82*bb546688SJerry Shih {
83*bb546688SJerry Shih if (riscv_isa_extension_available(NULL, ZVKB) &&
84*bb546688SJerry Shih riscv_vector_vlen() >= 128)
85*bb546688SJerry Shih return crypto_register_skcipher(&riscv64_chacha_alg);
86*bb546688SJerry Shih
87*bb546688SJerry Shih return -ENODEV;
88*bb546688SJerry Shih }
89*bb546688SJerry Shih
riscv64_chacha_mod_exit(void)90*bb546688SJerry Shih static void __exit riscv64_chacha_mod_exit(void)
91*bb546688SJerry Shih {
92*bb546688SJerry Shih crypto_unregister_skcipher(&riscv64_chacha_alg);
93*bb546688SJerry Shih }
94*bb546688SJerry Shih
95*bb546688SJerry Shih module_init(riscv64_chacha_mod_init);
96*bb546688SJerry Shih module_exit(riscv64_chacha_mod_exit);
97*bb546688SJerry Shih
98*bb546688SJerry Shih MODULE_DESCRIPTION("ChaCha20 (RISC-V accelerated)");
99*bb546688SJerry Shih MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>");
100*bb546688SJerry Shih MODULE_LICENSE("GPL");
101*bb546688SJerry Shih MODULE_ALIAS_CRYPTO("chacha20");
102