1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * OpenSSL/Cryptogams accelerated Poly1305 transform for MIPS
4 *
5 * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
6 */
7
8 #include <linux/unaligned.h>
9 #include <crypto/algapi.h>
10 #include <crypto/internal/hash.h>
11 #include <crypto/internal/poly1305.h>
12 #include <linux/cpufeature.h>
13 #include <linux/crypto.h>
14 #include <linux/module.h>
15
16 asmlinkage void poly1305_init_mips(void *state, const u8 *key);
17 asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit);
18 asmlinkage void poly1305_emit_mips(void *state, u8 *digest, const u32 *nonce);
19
poly1305_init_arch(struct poly1305_desc_ctx * dctx,const u8 key[POLY1305_KEY_SIZE])20 void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
21 {
22 poly1305_init_mips(&dctx->h, key);
23 dctx->s[0] = get_unaligned_le32(key + 16);
24 dctx->s[1] = get_unaligned_le32(key + 20);
25 dctx->s[2] = get_unaligned_le32(key + 24);
26 dctx->s[3] = get_unaligned_le32(key + 28);
27 dctx->buflen = 0;
28 }
29 EXPORT_SYMBOL(poly1305_init_arch);
30
mips_poly1305_init(struct shash_desc * desc)31 static int mips_poly1305_init(struct shash_desc *desc)
32 {
33 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
34
35 dctx->buflen = 0;
36 dctx->rset = 0;
37 dctx->sset = false;
38
39 return 0;
40 }
41
mips_poly1305_blocks(struct poly1305_desc_ctx * dctx,const u8 * src,u32 len,u32 hibit)42 static void mips_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src,
43 u32 len, u32 hibit)
44 {
45 if (unlikely(!dctx->sset)) {
46 if (!dctx->rset) {
47 poly1305_init_mips(&dctx->h, src);
48 src += POLY1305_BLOCK_SIZE;
49 len -= POLY1305_BLOCK_SIZE;
50 dctx->rset = 1;
51 }
52 if (len >= POLY1305_BLOCK_SIZE) {
53 dctx->s[0] = get_unaligned_le32(src + 0);
54 dctx->s[1] = get_unaligned_le32(src + 4);
55 dctx->s[2] = get_unaligned_le32(src + 8);
56 dctx->s[3] = get_unaligned_le32(src + 12);
57 src += POLY1305_BLOCK_SIZE;
58 len -= POLY1305_BLOCK_SIZE;
59 dctx->sset = true;
60 }
61 if (len < POLY1305_BLOCK_SIZE)
62 return;
63 }
64
65 len &= ~(POLY1305_BLOCK_SIZE - 1);
66
67 poly1305_blocks_mips(&dctx->h, src, len, hibit);
68 }
69
mips_poly1305_update(struct shash_desc * desc,const u8 * src,unsigned int len)70 static int mips_poly1305_update(struct shash_desc *desc, const u8 *src,
71 unsigned int len)
72 {
73 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
74
75 if (unlikely(dctx->buflen)) {
76 u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen);
77
78 memcpy(dctx->buf + dctx->buflen, src, bytes);
79 src += bytes;
80 len -= bytes;
81 dctx->buflen += bytes;
82
83 if (dctx->buflen == POLY1305_BLOCK_SIZE) {
84 mips_poly1305_blocks(dctx, dctx->buf, POLY1305_BLOCK_SIZE, 1);
85 dctx->buflen = 0;
86 }
87 }
88
89 if (likely(len >= POLY1305_BLOCK_SIZE)) {
90 mips_poly1305_blocks(dctx, src, len, 1);
91 src += round_down(len, POLY1305_BLOCK_SIZE);
92 len %= POLY1305_BLOCK_SIZE;
93 }
94
95 if (unlikely(len)) {
96 dctx->buflen = len;
97 memcpy(dctx->buf, src, len);
98 }
99 return 0;
100 }
101
poly1305_update_arch(struct poly1305_desc_ctx * dctx,const u8 * src,unsigned int nbytes)102 void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
103 unsigned int nbytes)
104 {
105 if (unlikely(dctx->buflen)) {
106 u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);
107
108 memcpy(dctx->buf + dctx->buflen, src, bytes);
109 src += bytes;
110 nbytes -= bytes;
111 dctx->buflen += bytes;
112
113 if (dctx->buflen == POLY1305_BLOCK_SIZE) {
114 poly1305_blocks_mips(&dctx->h, dctx->buf,
115 POLY1305_BLOCK_SIZE, 1);
116 dctx->buflen = 0;
117 }
118 }
119
120 if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
121 unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE);
122
123 poly1305_blocks_mips(&dctx->h, src, len, 1);
124 src += len;
125 nbytes %= POLY1305_BLOCK_SIZE;
126 }
127
128 if (unlikely(nbytes)) {
129 dctx->buflen = nbytes;
130 memcpy(dctx->buf, src, nbytes);
131 }
132 }
133 EXPORT_SYMBOL(poly1305_update_arch);
134
poly1305_final_arch(struct poly1305_desc_ctx * dctx,u8 * dst)135 void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
136 {
137 if (unlikely(dctx->buflen)) {
138 dctx->buf[dctx->buflen++] = 1;
139 memset(dctx->buf + dctx->buflen, 0,
140 POLY1305_BLOCK_SIZE - dctx->buflen);
141 poly1305_blocks_mips(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
142 }
143
144 poly1305_emit_mips(&dctx->h, dst, dctx->s);
145 *dctx = (struct poly1305_desc_ctx){};
146 }
147 EXPORT_SYMBOL(poly1305_final_arch);
148
mips_poly1305_final(struct shash_desc * desc,u8 * dst)149 static int mips_poly1305_final(struct shash_desc *desc, u8 *dst)
150 {
151 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
152
153 if (unlikely(!dctx->sset))
154 return -ENOKEY;
155
156 poly1305_final_arch(dctx, dst);
157 return 0;
158 }
159
160 static struct shash_alg mips_poly1305_alg = {
161 .init = mips_poly1305_init,
162 .update = mips_poly1305_update,
163 .final = mips_poly1305_final,
164 .digestsize = POLY1305_DIGEST_SIZE,
165 .descsize = sizeof(struct poly1305_desc_ctx),
166
167 .base.cra_name = "poly1305",
168 .base.cra_driver_name = "poly1305-mips",
169 .base.cra_priority = 200,
170 .base.cra_blocksize = POLY1305_BLOCK_SIZE,
171 .base.cra_module = THIS_MODULE,
172 };
173
mips_poly1305_mod_init(void)174 static int __init mips_poly1305_mod_init(void)
175 {
176 return IS_REACHABLE(CONFIG_CRYPTO_HASH) ?
177 crypto_register_shash(&mips_poly1305_alg) : 0;
178 }
179
mips_poly1305_mod_exit(void)180 static void __exit mips_poly1305_mod_exit(void)
181 {
182 if (IS_REACHABLE(CONFIG_CRYPTO_HASH))
183 crypto_unregister_shash(&mips_poly1305_alg);
184 }
185
186 module_init(mips_poly1305_mod_init);
187 module_exit(mips_poly1305_mod_exit);
188
189 MODULE_DESCRIPTION("Poly1305 transform (MIPS accelerated");
190 MODULE_LICENSE("GPL v2");
191 MODULE_ALIAS_CRYPTO("poly1305");
192 MODULE_ALIAS_CRYPTO("poly1305-mips");
193