1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * amlogic-cipher.c - hardware cryptographic offloader for Amlogic GXL SoC
4 *
5 * Copyright (C) 2018-2019 Corentin LABBE <clabbe@baylibre.com>
6 *
7 * This file add support for AES cipher with 128,192,256 bits keysize in
8 * CBC and ECB mode.
9 */
10
11 #include <linux/crypto.h>
12 #include <linux/delay.h>
13 #include <linux/io.h>
14 #include <crypto/scatterwalk.h>
15 #include <linux/scatterlist.h>
16 #include <linux/dma-mapping.h>
17 #include <crypto/internal/skcipher.h>
18 #include "amlogic-gxl.h"
19
get_engine_number(struct meson_dev * mc)20 static int get_engine_number(struct meson_dev *mc)
21 {
22 return atomic_inc_return(&mc->flow) % MAXFLOW;
23 }
24
meson_cipher_need_fallback(struct skcipher_request * areq)25 static bool meson_cipher_need_fallback(struct skcipher_request *areq)
26 {
27 struct scatterlist *src_sg = areq->src;
28 struct scatterlist *dst_sg = areq->dst;
29
30 if (areq->cryptlen == 0)
31 return true;
32
33 if (sg_nents(src_sg) != sg_nents(dst_sg))
34 return true;
35
36 /* KEY/IV descriptors use 3 desc */
37 if (sg_nents(src_sg) > MAXDESC - 3 || sg_nents(dst_sg) > MAXDESC - 3)
38 return true;
39
40 while (src_sg && dst_sg) {
41 if ((src_sg->length % 16) != 0)
42 return true;
43 if ((dst_sg->length % 16) != 0)
44 return true;
45 if (src_sg->length != dst_sg->length)
46 return true;
47 if (!IS_ALIGNED(src_sg->offset, sizeof(u32)))
48 return true;
49 if (!IS_ALIGNED(dst_sg->offset, sizeof(u32)))
50 return true;
51 src_sg = sg_next(src_sg);
52 dst_sg = sg_next(dst_sg);
53 }
54
55 return false;
56 }
57
meson_cipher_do_fallback(struct skcipher_request * areq)58 static int meson_cipher_do_fallback(struct skcipher_request *areq)
59 {
60 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
61 struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
62 struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
63 int err;
64 #ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG
65 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
66 struct meson_alg_template *algt;
67
68 algt = container_of(alg, struct meson_alg_template, alg.skcipher.base);
69 algt->stat_fb++;
70 #endif
71 skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
72 skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
73 areq->base.complete, areq->base.data);
74 skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
75 areq->cryptlen, areq->iv);
76
77 if (rctx->op_dir == MESON_DECRYPT)
78 err = crypto_skcipher_decrypt(&rctx->fallback_req);
79 else
80 err = crypto_skcipher_encrypt(&rctx->fallback_req);
81 return err;
82 }
83
meson_cipher(struct skcipher_request * areq)84 static int meson_cipher(struct skcipher_request *areq)
85 {
86 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
87 struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
88 struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
89 struct meson_dev *mc = op->mc;
90 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
91 struct meson_alg_template *algt;
92 int flow = rctx->flow;
93 unsigned int todo, eat, len;
94 struct scatterlist *src_sg = areq->src;
95 struct scatterlist *dst_sg = areq->dst;
96 struct meson_desc *desc;
97 int nr_sgs, nr_sgd;
98 int i, err = 0;
99 unsigned int keyivlen, ivsize, offset, tloffset;
100 dma_addr_t phykeyiv;
101 void *backup_iv = NULL, *bkeyiv;
102 u32 v;
103
104 algt = container_of(alg, struct meson_alg_template, alg.skcipher.base);
105
106 dev_dbg(mc->dev, "%s %s %u %x IV(%u) key=%u flow=%d\n", __func__,
107 crypto_tfm_alg_name(areq->base.tfm),
108 areq->cryptlen,
109 rctx->op_dir, crypto_skcipher_ivsize(tfm),
110 op->keylen, flow);
111
112 #ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG
113 algt->stat_req++;
114 mc->chanlist[flow].stat_req++;
115 #endif
116
117 /*
118 * The hardware expect a list of meson_desc structures.
119 * The 2 first structures store key
120 * The third stores IV
121 */
122 bkeyiv = kzalloc(48, GFP_KERNEL | GFP_DMA);
123 if (!bkeyiv)
124 return -ENOMEM;
125
126 memcpy(bkeyiv, op->key, op->keylen);
127 keyivlen = op->keylen;
128
129 ivsize = crypto_skcipher_ivsize(tfm);
130 if (areq->iv && ivsize > 0) {
131 if (ivsize > areq->cryptlen) {
132 dev_err(mc->dev, "invalid ivsize=%d vs len=%d\n", ivsize, areq->cryptlen);
133 err = -EINVAL;
134 goto theend;
135 }
136 memcpy(bkeyiv + 32, areq->iv, ivsize);
137 keyivlen = 48;
138 if (rctx->op_dir == MESON_DECRYPT) {
139 backup_iv = kzalloc(ivsize, GFP_KERNEL);
140 if (!backup_iv) {
141 err = -ENOMEM;
142 goto theend;
143 }
144 offset = areq->cryptlen - ivsize;
145 scatterwalk_map_and_copy(backup_iv, areq->src, offset,
146 ivsize, 0);
147 }
148 }
149 if (keyivlen == 24)
150 keyivlen = 32;
151
152 phykeyiv = dma_map_single(mc->dev, bkeyiv, keyivlen,
153 DMA_TO_DEVICE);
154 err = dma_mapping_error(mc->dev, phykeyiv);
155 if (err) {
156 dev_err(mc->dev, "Cannot DMA MAP KEY IV\n");
157 goto theend;
158 }
159
160 tloffset = 0;
161 eat = 0;
162 i = 0;
163 while (keyivlen > eat) {
164 desc = &mc->chanlist[flow].tl[tloffset];
165 memset(desc, 0, sizeof(struct meson_desc));
166 todo = min(keyivlen - eat, 16u);
167 desc->t_src = cpu_to_le32(phykeyiv + i * 16);
168 desc->t_dst = cpu_to_le32(i * 16);
169 v = (MODE_KEY << 20) | DESC_OWN | 16;
170 desc->t_status = cpu_to_le32(v);
171
172 eat += todo;
173 i++;
174 tloffset++;
175 }
176
177 if (areq->src == areq->dst) {
178 nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src),
179 DMA_BIDIRECTIONAL);
180 if (!nr_sgs) {
181 dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs);
182 err = -EINVAL;
183 goto theend;
184 }
185 nr_sgd = nr_sgs;
186 } else {
187 nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src),
188 DMA_TO_DEVICE);
189 if (!nr_sgs || nr_sgs > MAXDESC - 3) {
190 dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs);
191 err = -EINVAL;
192 goto theend;
193 }
194 nr_sgd = dma_map_sg(mc->dev, areq->dst, sg_nents(areq->dst),
195 DMA_FROM_DEVICE);
196 if (!nr_sgd || nr_sgd > MAXDESC - 3) {
197 dev_err(mc->dev, "Invalid SG count %d\n", nr_sgd);
198 err = -EINVAL;
199 goto theend;
200 }
201 }
202
203 src_sg = areq->src;
204 dst_sg = areq->dst;
205 len = areq->cryptlen;
206 while (src_sg) {
207 desc = &mc->chanlist[flow].tl[tloffset];
208 memset(desc, 0, sizeof(struct meson_desc));
209
210 desc->t_src = cpu_to_le32(sg_dma_address(src_sg));
211 desc->t_dst = cpu_to_le32(sg_dma_address(dst_sg));
212 todo = min(len, sg_dma_len(src_sg));
213 v = (op->keymode << 20) | DESC_OWN | todo | (algt->blockmode << 26);
214 if (rctx->op_dir)
215 v |= DESC_ENCRYPTION;
216 len -= todo;
217
218 if (!sg_next(src_sg))
219 v |= DESC_LAST;
220 desc->t_status = cpu_to_le32(v);
221 tloffset++;
222 src_sg = sg_next(src_sg);
223 dst_sg = sg_next(dst_sg);
224 }
225
226 reinit_completion(&mc->chanlist[flow].complete);
227 mc->chanlist[flow].status = 0;
228 writel(mc->chanlist[flow].t_phy | 2, mc->base + (flow << 2));
229 wait_for_completion_interruptible_timeout(&mc->chanlist[flow].complete,
230 msecs_to_jiffies(500));
231 if (mc->chanlist[flow].status == 0) {
232 dev_err(mc->dev, "DMA timeout for flow %d\n", flow);
233 err = -EINVAL;
234 }
235
236 dma_unmap_single(mc->dev, phykeyiv, keyivlen, DMA_TO_DEVICE);
237
238 if (areq->src == areq->dst) {
239 dma_unmap_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_BIDIRECTIONAL);
240 } else {
241 dma_unmap_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE);
242 dma_unmap_sg(mc->dev, areq->dst, sg_nents(areq->dst), DMA_FROM_DEVICE);
243 }
244
245 if (areq->iv && ivsize > 0) {
246 if (rctx->op_dir == MESON_DECRYPT) {
247 memcpy(areq->iv, backup_iv, ivsize);
248 } else {
249 scatterwalk_map_and_copy(areq->iv, areq->dst,
250 areq->cryptlen - ivsize,
251 ivsize, 0);
252 }
253 }
254 theend:
255 kfree_sensitive(bkeyiv);
256 kfree_sensitive(backup_iv);
257
258 return err;
259 }
260
meson_handle_cipher_request(struct crypto_engine * engine,void * areq)261 int meson_handle_cipher_request(struct crypto_engine *engine, void *areq)
262 {
263 int err;
264 struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
265
266 err = meson_cipher(breq);
267 local_bh_disable();
268 crypto_finalize_skcipher_request(engine, breq, err);
269 local_bh_enable();
270
271 return 0;
272 }
273
meson_skdecrypt(struct skcipher_request * areq)274 int meson_skdecrypt(struct skcipher_request *areq)
275 {
276 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
277 struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
278 struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
279 struct crypto_engine *engine;
280 int e;
281
282 rctx->op_dir = MESON_DECRYPT;
283 if (meson_cipher_need_fallback(areq))
284 return meson_cipher_do_fallback(areq);
285 e = get_engine_number(op->mc);
286 engine = op->mc->chanlist[e].engine;
287 rctx->flow = e;
288
289 return crypto_transfer_skcipher_request_to_engine(engine, areq);
290 }
291
meson_skencrypt(struct skcipher_request * areq)292 int meson_skencrypt(struct skcipher_request *areq)
293 {
294 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
295 struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
296 struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
297 struct crypto_engine *engine;
298 int e;
299
300 rctx->op_dir = MESON_ENCRYPT;
301 if (meson_cipher_need_fallback(areq))
302 return meson_cipher_do_fallback(areq);
303 e = get_engine_number(op->mc);
304 engine = op->mc->chanlist[e].engine;
305 rctx->flow = e;
306
307 return crypto_transfer_skcipher_request_to_engine(engine, areq);
308 }
309
meson_cipher_init(struct crypto_tfm * tfm)310 int meson_cipher_init(struct crypto_tfm *tfm)
311 {
312 struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
313 struct meson_alg_template *algt;
314 const char *name = crypto_tfm_alg_name(tfm);
315 struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm);
316 struct skcipher_alg *alg = crypto_skcipher_alg(sktfm);
317
318 memset(op, 0, sizeof(struct meson_cipher_tfm_ctx));
319
320 algt = container_of(alg, struct meson_alg_template, alg.skcipher.base);
321 op->mc = algt->mc;
322
323 op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
324 if (IS_ERR(op->fallback_tfm)) {
325 dev_err(op->mc->dev, "ERROR: Cannot allocate fallback for %s %ld\n",
326 name, PTR_ERR(op->fallback_tfm));
327 return PTR_ERR(op->fallback_tfm);
328 }
329
330 crypto_skcipher_set_reqsize(sktfm, sizeof(struct meson_cipher_req_ctx) +
331 crypto_skcipher_reqsize(op->fallback_tfm));
332
333 return 0;
334 }
335
meson_cipher_exit(struct crypto_tfm * tfm)336 void meson_cipher_exit(struct crypto_tfm *tfm)
337 {
338 struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
339
340 kfree_sensitive(op->key);
341 crypto_free_skcipher(op->fallback_tfm);
342 }
343
meson_aes_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)344 int meson_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
345 unsigned int keylen)
346 {
347 struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
348 struct meson_dev *mc = op->mc;
349
350 switch (keylen) {
351 case 128 / 8:
352 op->keymode = MODE_AES_128;
353 break;
354 case 192 / 8:
355 op->keymode = MODE_AES_192;
356 break;
357 case 256 / 8:
358 op->keymode = MODE_AES_256;
359 break;
360 default:
361 dev_dbg(mc->dev, "ERROR: Invalid keylen %u\n", keylen);
362 return -EINVAL;
363 }
364 kfree_sensitive(op->key);
365 op->keylen = keylen;
366 op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
367 if (!op->key)
368 return -ENOMEM;
369
370 return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);
371 }
372