xref: /linux/arch/riscv/crypto/ghash-riscv64-glue.c (revision 40ccd6aa3e2e05be93394e3cd560c718dedfcc77)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * GHASH using the RISC-V vector crypto extensions
4  *
5  * Copyright (C) 2023 VRULL GmbH
6  * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
7  *
8  * Copyright (C) 2023 SiFive, Inc.
9  * Author: Jerry Shih <jerry.shih@sifive.com>
10  */
11 
12 #include <asm/simd.h>
13 #include <asm/vector.h>
14 #include <crypto/ghash.h>
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/simd.h>
17 #include <linux/linkage.h>
18 #include <linux/module.h>
19 
20 asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
21 			   size_t len);
22 
23 struct riscv64_ghash_tfm_ctx {
24 	be128 key;
25 };
26 
27 struct riscv64_ghash_desc_ctx {
28 	be128 accumulator;
29 	u8 buffer[GHASH_BLOCK_SIZE];
30 	u32 bytes;
31 };
32 
33 static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
34 				unsigned int keylen)
35 {
36 	struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm);
37 
38 	if (keylen != GHASH_BLOCK_SIZE)
39 		return -EINVAL;
40 
41 	memcpy(&tctx->key, key, GHASH_BLOCK_SIZE);
42 
43 	return 0;
44 }
45 
46 static int riscv64_ghash_init(struct shash_desc *desc)
47 {
48 	struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
49 
50 	*dctx = (struct riscv64_ghash_desc_ctx){};
51 
52 	return 0;
53 }
54 
55 static inline void
56 riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx,
57 		     struct riscv64_ghash_desc_ctx *dctx,
58 		     const u8 *src, size_t srclen)
59 {
60 	/* The srclen is nonzero and a multiple of 16. */
61 	if (crypto_simd_usable()) {
62 		kernel_vector_begin();
63 		ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen);
64 		kernel_vector_end();
65 	} else {
66 		do {
67 			crypto_xor((u8 *)&dctx->accumulator, src,
68 				   GHASH_BLOCK_SIZE);
69 			gf128mul_lle(&dctx->accumulator, &tctx->key);
70 			src += GHASH_BLOCK_SIZE;
71 			srclen -= GHASH_BLOCK_SIZE;
72 		} while (srclen);
73 	}
74 }
75 
76 static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src,
77 				unsigned int srclen)
78 {
79 	const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
80 	struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
81 	unsigned int len;
82 
83 	if (dctx->bytes) {
84 		if (dctx->bytes + srclen < GHASH_BLOCK_SIZE) {
85 			memcpy(dctx->buffer + dctx->bytes, src, srclen);
86 			dctx->bytes += srclen;
87 			return 0;
88 		}
89 		memcpy(dctx->buffer + dctx->bytes, src,
90 		       GHASH_BLOCK_SIZE - dctx->bytes);
91 		riscv64_ghash_blocks(tctx, dctx, dctx->buffer,
92 				     GHASH_BLOCK_SIZE);
93 		src += GHASH_BLOCK_SIZE - dctx->bytes;
94 		srclen -= GHASH_BLOCK_SIZE - dctx->bytes;
95 		dctx->bytes = 0;
96 	}
97 
98 	len = round_down(srclen, GHASH_BLOCK_SIZE);
99 	if (len) {
100 		riscv64_ghash_blocks(tctx, dctx, src, len);
101 		src += len;
102 		srclen -= len;
103 	}
104 
105 	if (srclen) {
106 		memcpy(dctx->buffer, src, srclen);
107 		dctx->bytes = srclen;
108 	}
109 
110 	return 0;
111 }
112 
113 static int riscv64_ghash_final(struct shash_desc *desc, u8 *out)
114 {
115 	const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
116 	struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
117 	int i;
118 
119 	if (dctx->bytes) {
120 		for (i = dctx->bytes; i < GHASH_BLOCK_SIZE; i++)
121 			dctx->buffer[i] = 0;
122 
123 		riscv64_ghash_blocks(tctx, dctx, dctx->buffer,
124 				     GHASH_BLOCK_SIZE);
125 	}
126 
127 	memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE);
128 	return 0;
129 }
130 
131 static struct shash_alg riscv64_ghash_alg = {
132 	.init = riscv64_ghash_init,
133 	.update = riscv64_ghash_update,
134 	.final = riscv64_ghash_final,
135 	.setkey = riscv64_ghash_setkey,
136 	.descsize = sizeof(struct riscv64_ghash_desc_ctx),
137 	.digestsize = GHASH_DIGEST_SIZE,
138 	.base = {
139 		.cra_blocksize = GHASH_BLOCK_SIZE,
140 		.cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx),
141 		.cra_priority = 300,
142 		.cra_name = "ghash",
143 		.cra_driver_name = "ghash-riscv64-zvkg",
144 		.cra_module = THIS_MODULE,
145 	},
146 };
147 
148 static int __init riscv64_ghash_mod_init(void)
149 {
150 	if (riscv_isa_extension_available(NULL, ZVKG) &&
151 	    riscv_vector_vlen() >= 128)
152 		return crypto_register_shash(&riscv64_ghash_alg);
153 
154 	return -ENODEV;
155 }
156 
157 static void __exit riscv64_ghash_mod_exit(void)
158 {
159 	crypto_unregister_shash(&riscv64_ghash_alg);
160 }
161 
162 module_init(riscv64_ghash_mod_init);
163 module_exit(riscv64_ghash_mod_exit);
164 
165 MODULE_DESCRIPTION("GHASH (RISC-V accelerated)");
166 MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
167 MODULE_LICENSE("GPL");
168 MODULE_ALIAS_CRYPTO("ghash");
169