1 // SPDX-License-Identifier: GPL-2.0-only
2 // SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 /*
4 * Crypto driver to handle block cipher algorithms using NVIDIA Security Engine.
5 */
6
7 #include <linux/bottom_half.h>
8 #include <linux/clk.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/module.h>
11 #include <linux/of_device.h>
12 #include <linux/platform_device.h>
13
14 #include <crypto/aead.h>
15 #include <crypto/aes.h>
16 #include <crypto/engine.h>
17 #include <crypto/gcm.h>
18 #include <crypto/scatterwalk.h>
19 #include <crypto/xts.h>
20 #include <crypto/internal/aead.h>
21 #include <crypto/internal/hash.h>
22 #include <crypto/internal/skcipher.h>
23
24 #include "tegra-se.h"
25
26 struct tegra_aes_ctx {
27 struct tegra_se *se;
28 u32 alg;
29 u32 ivsize;
30 u32 key1_id;
31 u32 key2_id;
32 u32 keylen;
33 u8 key1[AES_MAX_KEY_SIZE];
34 u8 key2[AES_MAX_KEY_SIZE];
35 };
36
37 struct tegra_aes_reqctx {
38 struct tegra_se_datbuf datbuf;
39 bool encrypt;
40 u32 config;
41 u32 crypto_config;
42 u32 len;
43 u32 *iv;
44 };
45
46 struct tegra_aead_ctx {
47 struct tegra_se *se;
48 u32 alg;
49 u32 key_id;
50 u32 keylen;
51 u8 key[AES_MAX_KEY_SIZE];
52 };
53
54 struct tegra_aead_reqctx {
55 struct tegra_se_datbuf inbuf;
56 struct tegra_se_datbuf outbuf;
57 struct scatterlist *src_sg;
58 struct scatterlist *dst_sg;
59 unsigned int assoclen;
60 unsigned int cryptlen;
61 unsigned int authsize;
62 bool encrypt;
63 u32 crypto_config;
64 u32 config;
65 u32 key_id;
66 u32 iv[4];
67 u8 authdata[16];
68 };
69
70 struct tegra_cmac_ctx {
71 struct tegra_se *se;
72 unsigned int alg;
73 u32 key_id;
74 u32 keylen;
75 u8 key[AES_MAX_KEY_SIZE];
76 struct crypto_shash *fallback_tfm;
77 };
78
79 struct tegra_cmac_reqctx {
80 struct scatterlist *src_sg;
81 struct tegra_se_datbuf datbuf;
82 struct tegra_se_datbuf residue;
83 unsigned int total_len;
84 unsigned int blk_size;
85 unsigned int task;
86 u32 crypto_config;
87 u32 config;
88 u32 key_id;
89 u32 *iv;
90 u32 result[CMAC_RESULT_REG_COUNT];
91 };
92
93 /* increment counter (128-bit int) */
ctr_iv_inc(__u8 * counter,__u8 bits,__u32 nums)94 static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums)
95 {
96 do {
97 --bits;
98 nums += counter[bits];
99 counter[bits] = nums & 0xff;
100 nums >>= 8;
101 } while (bits && nums);
102 }
103
tegra_cbc_iv_copyback(struct skcipher_request * req,struct tegra_aes_ctx * ctx)104 static void tegra_cbc_iv_copyback(struct skcipher_request *req, struct tegra_aes_ctx *ctx)
105 {
106 struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req);
107 unsigned int offset;
108
109 offset = req->cryptlen - ctx->ivsize;
110
111 if (rctx->encrypt)
112 memcpy(req->iv, rctx->datbuf.buf + offset, ctx->ivsize);
113 else
114 scatterwalk_map_and_copy(req->iv, req->src, offset, ctx->ivsize, 0);
115 }
116
tegra_aes_update_iv(struct skcipher_request * req,struct tegra_aes_ctx * ctx)117 static void tegra_aes_update_iv(struct skcipher_request *req, struct tegra_aes_ctx *ctx)
118 {
119 int num;
120
121 if (ctx->alg == SE_ALG_CBC) {
122 tegra_cbc_iv_copyback(req, ctx);
123 } else if (ctx->alg == SE_ALG_CTR) {
124 num = req->cryptlen / ctx->ivsize;
125 if (req->cryptlen % ctx->ivsize)
126 num++;
127
128 ctr_iv_inc(req->iv, ctx->ivsize, num);
129 }
130 }
131
tegra234_aes_crypto_cfg(u32 alg,bool encrypt)132 static int tegra234_aes_crypto_cfg(u32 alg, bool encrypt)
133 {
134 switch (alg) {
135 case SE_ALG_CMAC:
136 case SE_ALG_GMAC:
137 case SE_ALG_GCM:
138 case SE_ALG_GCM_FINAL:
139 return 0;
140 case SE_ALG_CBC:
141 if (encrypt)
142 return SE_CRYPTO_CFG_CBC_ENCRYPT;
143 else
144 return SE_CRYPTO_CFG_CBC_DECRYPT;
145 case SE_ALG_ECB:
146 if (encrypt)
147 return SE_CRYPTO_CFG_ECB_ENCRYPT;
148 else
149 return SE_CRYPTO_CFG_ECB_DECRYPT;
150 case SE_ALG_XTS:
151 if (encrypt)
152 return SE_CRYPTO_CFG_XTS_ENCRYPT;
153 else
154 return SE_CRYPTO_CFG_XTS_DECRYPT;
155
156 case SE_ALG_CTR:
157 return SE_CRYPTO_CFG_CTR;
158 case SE_ALG_CBC_MAC:
159 return SE_CRYPTO_CFG_CBC_MAC;
160
161 default:
162 break;
163 }
164
165 return -EINVAL;
166 }
167
tegra234_aes_cfg(u32 alg,bool encrypt)168 static int tegra234_aes_cfg(u32 alg, bool encrypt)
169 {
170 switch (alg) {
171 case SE_ALG_CBC:
172 case SE_ALG_ECB:
173 case SE_ALG_XTS:
174 case SE_ALG_CTR:
175 if (encrypt)
176 return SE_CFG_AES_ENCRYPT;
177 else
178 return SE_CFG_AES_DECRYPT;
179
180 case SE_ALG_GMAC:
181 if (encrypt)
182 return SE_CFG_GMAC_ENCRYPT;
183 else
184 return SE_CFG_GMAC_DECRYPT;
185
186 case SE_ALG_GCM:
187 if (encrypt)
188 return SE_CFG_GCM_ENCRYPT;
189 else
190 return SE_CFG_GCM_DECRYPT;
191
192 case SE_ALG_GCM_FINAL:
193 if (encrypt)
194 return SE_CFG_GCM_FINAL_ENCRYPT;
195 else
196 return SE_CFG_GCM_FINAL_DECRYPT;
197
198 case SE_ALG_CMAC:
199 return SE_CFG_CMAC;
200
201 case SE_ALG_CBC_MAC:
202 return SE_AES_ENC_ALG_AES_ENC |
203 SE_AES_DST_HASH_REG;
204 }
205 return -EINVAL;
206 }
207
tegra_aes_prep_cmd(struct tegra_aes_ctx * ctx,struct tegra_aes_reqctx * rctx)208 static unsigned int tegra_aes_prep_cmd(struct tegra_aes_ctx *ctx,
209 struct tegra_aes_reqctx *rctx)
210 {
211 unsigned int data_count, res_bits, i = 0, j;
212 struct tegra_se *se = ctx->se;
213 u32 *cpuvaddr = se->cmdbuf->addr;
214 dma_addr_t addr = rctx->datbuf.addr;
215
216 data_count = rctx->len / AES_BLOCK_SIZE;
217 res_bits = (rctx->len % AES_BLOCK_SIZE) * 8;
218
219 /*
220 * Hardware processes data_count + 1 blocks.
221 * Reduce 1 block if there is no residue
222 */
223 if (!res_bits)
224 data_count--;
225
226 if (rctx->iv) {
227 cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT);
228 cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr);
229 for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++)
230 cpuvaddr[i++] = rctx->iv[j];
231 }
232
233 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1);
234 cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) |
235 SE_LAST_BLOCK_RES_BITS(res_bits);
236
237 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6);
238 cpuvaddr[i++] = rctx->config;
239 cpuvaddr[i++] = rctx->crypto_config;
240
241 /* Source address setting */
242 cpuvaddr[i++] = lower_32_bits(addr);
243 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(addr)) | SE_ADDR_HI_SZ(rctx->len);
244
245 /* Destination address setting */
246 cpuvaddr[i++] = lower_32_bits(addr);
247 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(addr)) |
248 SE_ADDR_HI_SZ(rctx->len);
249
250 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
251 cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_LASTBUF |
252 SE_AES_OP_START;
253
254 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
255 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
256 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
257
258 dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config);
259
260 return i;
261 }
262
tegra_aes_do_one_req(struct crypto_engine * engine,void * areq)263 static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq)
264 {
265 struct skcipher_request *req = container_of(areq, struct skcipher_request, base);
266 struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
267 struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req);
268 struct tegra_se *se = ctx->se;
269 unsigned int cmdlen, key1_id, key2_id;
270 int ret;
271
272 rctx->iv = (ctx->alg == SE_ALG_ECB) ? NULL : (u32 *)req->iv;
273 rctx->len = req->cryptlen;
274 key1_id = ctx->key1_id;
275 key2_id = ctx->key2_id;
276
277 /* Pad input to AES Block size */
278 if (ctx->alg != SE_ALG_XTS) {
279 if (rctx->len % AES_BLOCK_SIZE)
280 rctx->len += AES_BLOCK_SIZE - (rctx->len % AES_BLOCK_SIZE);
281 }
282
283 rctx->datbuf.size = rctx->len;
284 rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size,
285 &rctx->datbuf.addr, GFP_KERNEL);
286 if (!rctx->datbuf.buf) {
287 ret = -ENOMEM;
288 goto out_finalize;
289 }
290
291 scatterwalk_map_and_copy(rctx->datbuf.buf, req->src, 0, req->cryptlen, 0);
292
293 rctx->config = tegra234_aes_cfg(ctx->alg, rctx->encrypt);
294 rctx->crypto_config = tegra234_aes_crypto_cfg(ctx->alg, rctx->encrypt);
295
296 if (!key1_id) {
297 ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key1,
298 ctx->keylen, ctx->alg, &key1_id);
299 if (ret)
300 goto out;
301 }
302
303 rctx->crypto_config |= SE_AES_KEY_INDEX(key1_id);
304
305 if (ctx->alg == SE_ALG_XTS) {
306 if (!key2_id) {
307 ret = tegra_key_submit_reserved_xts(ctx->se, ctx->key2,
308 ctx->keylen, ctx->alg, &key2_id);
309 if (ret)
310 goto out;
311 }
312
313 rctx->crypto_config |= SE_AES_KEY2_INDEX(key2_id);
314 }
315
316 /* Prepare the command and submit for execution */
317 cmdlen = tegra_aes_prep_cmd(ctx, rctx);
318 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen);
319
320 /* Copy the result */
321 tegra_aes_update_iv(req, ctx);
322 scatterwalk_map_and_copy(rctx->datbuf.buf, req->dst, 0, req->cryptlen, 1);
323
324 out:
325 /* Free the buffer */
326 dma_free_coherent(ctx->se->dev, rctx->datbuf.size,
327 rctx->datbuf.buf, rctx->datbuf.addr);
328
329 if (tegra_key_is_reserved(key1_id))
330 tegra_key_invalidate_reserved(ctx->se, key1_id, ctx->alg);
331
332 if (tegra_key_is_reserved(key2_id))
333 tegra_key_invalidate_reserved(ctx->se, key2_id, ctx->alg);
334
335 out_finalize:
336 local_bh_disable();
337 crypto_finalize_skcipher_request(se->engine, req, ret);
338 local_bh_enable();
339
340 return 0;
341 }
342
tegra_aes_cra_init(struct crypto_skcipher * tfm)343 static int tegra_aes_cra_init(struct crypto_skcipher *tfm)
344 {
345 struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
346 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
347 struct tegra_se_alg *se_alg;
348 const char *algname;
349 int ret;
350
351 se_alg = container_of(alg, struct tegra_se_alg, alg.skcipher.base);
352
353 crypto_skcipher_set_reqsize(tfm, sizeof(struct tegra_aes_reqctx));
354
355 ctx->ivsize = crypto_skcipher_ivsize(tfm);
356 ctx->se = se_alg->se_dev;
357 ctx->key1_id = 0;
358 ctx->key2_id = 0;
359 ctx->keylen = 0;
360
361 algname = crypto_tfm_alg_name(&tfm->base);
362 ret = se_algname_to_algid(algname);
363 if (ret < 0) {
364 dev_err(ctx->se->dev, "invalid algorithm\n");
365 return ret;
366 }
367
368 ctx->alg = ret;
369
370 return 0;
371 }
372
tegra_aes_cra_exit(struct crypto_skcipher * tfm)373 static void tegra_aes_cra_exit(struct crypto_skcipher *tfm)
374 {
375 struct tegra_aes_ctx *ctx = crypto_tfm_ctx(&tfm->base);
376
377 if (ctx->key1_id)
378 tegra_key_invalidate(ctx->se, ctx->key1_id, ctx->alg);
379
380 if (ctx->key2_id)
381 tegra_key_invalidate(ctx->se, ctx->key2_id, ctx->alg);
382 }
383
tegra_aes_setkey(struct crypto_skcipher * tfm,const u8 * key,u32 keylen)384 static int tegra_aes_setkey(struct crypto_skcipher *tfm,
385 const u8 *key, u32 keylen)
386 {
387 struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
388 int ret;
389
390 if (aes_check_keylen(keylen)) {
391 dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen);
392 return -EINVAL;
393 }
394
395 ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key1_id);
396 if (ret) {
397 ctx->keylen = keylen;
398 memcpy(ctx->key1, key, keylen);
399 }
400
401 return 0;
402 }
403
tegra_xts_setkey(struct crypto_skcipher * tfm,const u8 * key,u32 keylen)404 static int tegra_xts_setkey(struct crypto_skcipher *tfm,
405 const u8 *key, u32 keylen)
406 {
407 struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
408 u32 len = keylen / 2;
409 int ret;
410
411 ret = xts_verify_key(tfm, key, keylen);
412 if (ret || aes_check_keylen(len)) {
413 dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen);
414 return -EINVAL;
415 }
416
417 ret = tegra_key_submit(ctx->se, key, len,
418 ctx->alg, &ctx->key1_id);
419 if (ret) {
420 ctx->keylen = len;
421 memcpy(ctx->key1, key, len);
422 }
423
424 ret = tegra_key_submit(ctx->se, key + len, len,
425 ctx->alg, &ctx->key2_id);
426 if (ret) {
427 ctx->keylen = len;
428 memcpy(ctx->key2, key + len, len);
429 }
430
431 return 0;
432 }
433
tegra_aes_kac_manifest(u32 user,u32 alg,u32 keylen)434 static int tegra_aes_kac_manifest(u32 user, u32 alg, u32 keylen)
435 {
436 int manifest;
437
438 manifest = SE_KAC_USER_NS;
439
440 switch (alg) {
441 case SE_ALG_CBC:
442 case SE_ALG_ECB:
443 case SE_ALG_CTR:
444 manifest |= SE_KAC_ENC;
445 break;
446 case SE_ALG_XTS:
447 manifest |= SE_KAC_XTS;
448 break;
449 case SE_ALG_GCM:
450 manifest |= SE_KAC_GCM;
451 break;
452 case SE_ALG_CMAC:
453 manifest |= SE_KAC_CMAC;
454 break;
455 case SE_ALG_CBC_MAC:
456 manifest |= SE_KAC_ENC;
457 break;
458 default:
459 return -EINVAL;
460 }
461
462 switch (keylen) {
463 case AES_KEYSIZE_128:
464 manifest |= SE_KAC_SIZE_128;
465 break;
466 case AES_KEYSIZE_192:
467 manifest |= SE_KAC_SIZE_192;
468 break;
469 case AES_KEYSIZE_256:
470 manifest |= SE_KAC_SIZE_256;
471 break;
472 default:
473 return -EINVAL;
474 }
475
476 return manifest;
477 }
478
tegra_aes_crypt(struct skcipher_request * req,bool encrypt)479 static int tegra_aes_crypt(struct skcipher_request *req, bool encrypt)
480
481 {
482 struct crypto_skcipher *tfm;
483 struct tegra_aes_ctx *ctx;
484 struct tegra_aes_reqctx *rctx;
485
486 tfm = crypto_skcipher_reqtfm(req);
487 ctx = crypto_skcipher_ctx(tfm);
488 rctx = skcipher_request_ctx(req);
489
490 if (ctx->alg != SE_ALG_XTS) {
491 if (!IS_ALIGNED(req->cryptlen, crypto_skcipher_blocksize(tfm))) {
492 dev_dbg(ctx->se->dev, "invalid length (%d)", req->cryptlen);
493 return -EINVAL;
494 }
495 } else if (req->cryptlen < XTS_BLOCK_SIZE) {
496 dev_dbg(ctx->se->dev, "invalid length (%d)", req->cryptlen);
497 return -EINVAL;
498 }
499
500 if (!req->cryptlen)
501 return 0;
502
503 rctx->encrypt = encrypt;
504
505 return crypto_transfer_skcipher_request_to_engine(ctx->se->engine, req);
506 }
507
tegra_aes_encrypt(struct skcipher_request * req)508 static int tegra_aes_encrypt(struct skcipher_request *req)
509 {
510 return tegra_aes_crypt(req, true);
511 }
512
tegra_aes_decrypt(struct skcipher_request * req)513 static int tegra_aes_decrypt(struct skcipher_request *req)
514 {
515 return tegra_aes_crypt(req, false);
516 }
517
518 static struct tegra_se_alg tegra_aes_algs[] = {
519 {
520 .alg.skcipher.op.do_one_request = tegra_aes_do_one_req,
521 .alg.skcipher.base = {
522 .init = tegra_aes_cra_init,
523 .exit = tegra_aes_cra_exit,
524 .setkey = tegra_aes_setkey,
525 .encrypt = tegra_aes_encrypt,
526 .decrypt = tegra_aes_decrypt,
527 .min_keysize = AES_MIN_KEY_SIZE,
528 .max_keysize = AES_MAX_KEY_SIZE,
529 .ivsize = AES_BLOCK_SIZE,
530 .base = {
531 .cra_name = "cbc(aes)",
532 .cra_driver_name = "cbc-aes-tegra",
533 .cra_priority = 500,
534 .cra_flags = CRYPTO_ALG_ASYNC,
535 .cra_blocksize = AES_BLOCK_SIZE,
536 .cra_ctxsize = sizeof(struct tegra_aes_ctx),
537 .cra_alignmask = 0xf,
538 .cra_module = THIS_MODULE,
539 },
540 }
541 }, {
542 .alg.skcipher.op.do_one_request = tegra_aes_do_one_req,
543 .alg.skcipher.base = {
544 .init = tegra_aes_cra_init,
545 .exit = tegra_aes_cra_exit,
546 .setkey = tegra_aes_setkey,
547 .encrypt = tegra_aes_encrypt,
548 .decrypt = tegra_aes_decrypt,
549 .min_keysize = AES_MIN_KEY_SIZE,
550 .max_keysize = AES_MAX_KEY_SIZE,
551 .base = {
552 .cra_name = "ecb(aes)",
553 .cra_driver_name = "ecb-aes-tegra",
554 .cra_priority = 500,
555 .cra_flags = CRYPTO_ALG_ASYNC,
556 .cra_blocksize = AES_BLOCK_SIZE,
557 .cra_ctxsize = sizeof(struct tegra_aes_ctx),
558 .cra_alignmask = 0xf,
559 .cra_module = THIS_MODULE,
560 },
561 }
562 }, {
563 .alg.skcipher.op.do_one_request = tegra_aes_do_one_req,
564 .alg.skcipher.base = {
565 .init = tegra_aes_cra_init,
566 .exit = tegra_aes_cra_exit,
567 .setkey = tegra_aes_setkey,
568 .encrypt = tegra_aes_encrypt,
569 .decrypt = tegra_aes_decrypt,
570 .min_keysize = AES_MIN_KEY_SIZE,
571 .max_keysize = AES_MAX_KEY_SIZE,
572 .ivsize = AES_BLOCK_SIZE,
573 .base = {
574 .cra_name = "ctr(aes)",
575 .cra_driver_name = "ctr-aes-tegra",
576 .cra_priority = 500,
577 .cra_flags = CRYPTO_ALG_ASYNC,
578 .cra_blocksize = 1,
579 .cra_ctxsize = sizeof(struct tegra_aes_ctx),
580 .cra_alignmask = 0xf,
581 .cra_module = THIS_MODULE,
582 },
583 }
584 }, {
585 .alg.skcipher.op.do_one_request = tegra_aes_do_one_req,
586 .alg.skcipher.base = {
587 .init = tegra_aes_cra_init,
588 .exit = tegra_aes_cra_exit,
589 .setkey = tegra_xts_setkey,
590 .encrypt = tegra_aes_encrypt,
591 .decrypt = tegra_aes_decrypt,
592 .min_keysize = 2 * AES_MIN_KEY_SIZE,
593 .max_keysize = 2 * AES_MAX_KEY_SIZE,
594 .ivsize = AES_BLOCK_SIZE,
595 .base = {
596 .cra_name = "xts(aes)",
597 .cra_driver_name = "xts-aes-tegra",
598 .cra_priority = 500,
599 .cra_flags = CRYPTO_ALG_ASYNC,
600 .cra_blocksize = AES_BLOCK_SIZE,
601 .cra_ctxsize = sizeof(struct tegra_aes_ctx),
602 .cra_alignmask = (__alignof__(u64) - 1),
603 .cra_module = THIS_MODULE,
604 },
605 }
606 },
607 };
608
tegra_gmac_prep_cmd(struct tegra_aead_ctx * ctx,struct tegra_aead_reqctx * rctx)609 static unsigned int tegra_gmac_prep_cmd(struct tegra_aead_ctx *ctx,
610 struct tegra_aead_reqctx *rctx)
611 {
612 unsigned int data_count, res_bits, i = 0;
613 struct tegra_se *se = ctx->se;
614 u32 *cpuvaddr = se->cmdbuf->addr;
615
616 data_count = (rctx->assoclen / AES_BLOCK_SIZE);
617 res_bits = (rctx->assoclen % AES_BLOCK_SIZE) * 8;
618
619 /*
620 * Hardware processes data_count + 1 blocks.
621 * Reduce 1 block if there is no residue
622 */
623 if (!res_bits)
624 data_count--;
625
626 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1);
627 cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) |
628 SE_LAST_BLOCK_RES_BITS(res_bits);
629
630 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 4);
631 cpuvaddr[i++] = rctx->config;
632 cpuvaddr[i++] = rctx->crypto_config;
633 cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr);
634 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) |
635 SE_ADDR_HI_SZ(rctx->assoclen);
636
637 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
638 cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL |
639 SE_AES_OP_INIT | SE_AES_OP_LASTBUF |
640 SE_AES_OP_START;
641
642 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
643 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
644 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
645
646 return i;
647 }
648
tegra_gcm_crypt_prep_cmd(struct tegra_aead_ctx * ctx,struct tegra_aead_reqctx * rctx)649 static unsigned int tegra_gcm_crypt_prep_cmd(struct tegra_aead_ctx *ctx,
650 struct tegra_aead_reqctx *rctx)
651 {
652 unsigned int data_count, res_bits, i = 0, j;
653 struct tegra_se *se = ctx->se;
654 u32 *cpuvaddr = se->cmdbuf->addr, op;
655
656 data_count = (rctx->cryptlen / AES_BLOCK_SIZE);
657 res_bits = (rctx->cryptlen % AES_BLOCK_SIZE) * 8;
658 op = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL |
659 SE_AES_OP_LASTBUF | SE_AES_OP_START;
660
661 /*
662 * If there is no assoc data,
663 * this will be the init command
664 */
665 if (!rctx->assoclen)
666 op |= SE_AES_OP_INIT;
667
668 /*
669 * Hardware processes data_count + 1 blocks.
670 * Reduce 1 block if there is no residue
671 */
672 if (!res_bits)
673 data_count--;
674
675 cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT);
676 cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr);
677 for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++)
678 cpuvaddr[i++] = rctx->iv[j];
679
680 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1);
681 cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) |
682 SE_LAST_BLOCK_RES_BITS(res_bits);
683
684 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6);
685 cpuvaddr[i++] = rctx->config;
686 cpuvaddr[i++] = rctx->crypto_config;
687
688 /* Source Address */
689 cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr);
690 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) |
691 SE_ADDR_HI_SZ(rctx->cryptlen);
692
693 /* Destination Address */
694 cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr);
695 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) |
696 SE_ADDR_HI_SZ(rctx->cryptlen);
697
698 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
699 cpuvaddr[i++] = op;
700
701 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
702 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
703 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
704
705 dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config);
706 return i;
707 }
708
tegra_gcm_prep_final_cmd(struct tegra_se * se,u32 * cpuvaddr,struct tegra_aead_reqctx * rctx)709 static int tegra_gcm_prep_final_cmd(struct tegra_se *se, u32 *cpuvaddr,
710 struct tegra_aead_reqctx *rctx)
711 {
712 unsigned int i = 0, j;
713 u32 op;
714
715 op = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL |
716 SE_AES_OP_LASTBUF | SE_AES_OP_START;
717
718 /*
719 * Set init for zero sized vector
720 */
721 if (!rctx->assoclen && !rctx->cryptlen)
722 op |= SE_AES_OP_INIT;
723
724 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->aad_len, 2);
725 cpuvaddr[i++] = rctx->assoclen * 8;
726 cpuvaddr[i++] = 0;
727
728 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->cryp_msg_len, 2);
729 cpuvaddr[i++] = rctx->cryptlen * 8;
730 cpuvaddr[i++] = 0;
731
732 cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT);
733 cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr);
734 for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++)
735 cpuvaddr[i++] = rctx->iv[j];
736
737 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6);
738 cpuvaddr[i++] = rctx->config;
739 cpuvaddr[i++] = rctx->crypto_config;
740 cpuvaddr[i++] = 0;
741 cpuvaddr[i++] = 0;
742
743 /* Destination Address */
744 cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr);
745 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) |
746 SE_ADDR_HI_SZ(0x10); /* HW always generates 128-bit tag */
747
748 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
749 cpuvaddr[i++] = op;
750
751 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
752 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
753 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
754
755 dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config);
756
757 return i;
758 }
759
tegra_gcm_do_gmac(struct tegra_aead_ctx * ctx,struct tegra_aead_reqctx * rctx)760 static int tegra_gcm_do_gmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
761 {
762 struct tegra_se *se = ctx->se;
763 unsigned int cmdlen;
764
765 scatterwalk_map_and_copy(rctx->inbuf.buf,
766 rctx->src_sg, 0, rctx->assoclen, 0);
767
768 rctx->config = tegra234_aes_cfg(SE_ALG_GMAC, rctx->encrypt);
769 rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GMAC, rctx->encrypt) |
770 SE_AES_KEY_INDEX(rctx->key_id);
771
772 cmdlen = tegra_gmac_prep_cmd(ctx, rctx);
773
774 return tegra_se_host1x_submit(se, se->cmdbuf, cmdlen);
775 }
776
tegra_gcm_do_crypt(struct tegra_aead_ctx * ctx,struct tegra_aead_reqctx * rctx)777 static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
778 {
779 struct tegra_se *se = ctx->se;
780 int cmdlen, ret;
781
782 scatterwalk_map_and_copy(rctx->inbuf.buf, rctx->src_sg,
783 rctx->assoclen, rctx->cryptlen, 0);
784
785 rctx->config = tegra234_aes_cfg(SE_ALG_GCM, rctx->encrypt);
786 rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM, rctx->encrypt) |
787 SE_AES_KEY_INDEX(rctx->key_id);
788
789 /* Prepare command and submit */
790 cmdlen = tegra_gcm_crypt_prep_cmd(ctx, rctx);
791 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen);
792 if (ret)
793 return ret;
794
795 /* Copy the result */
796 scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg,
797 rctx->assoclen, rctx->cryptlen, 1);
798
799 return 0;
800 }
801
tegra_gcm_do_final(struct tegra_aead_ctx * ctx,struct tegra_aead_reqctx * rctx)802 static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
803 {
804 struct tegra_se *se = ctx->se;
805 u32 *cpuvaddr = se->cmdbuf->addr;
806 int cmdlen, ret, offset;
807
808 rctx->config = tegra234_aes_cfg(SE_ALG_GCM_FINAL, rctx->encrypt);
809 rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM_FINAL, rctx->encrypt) |
810 SE_AES_KEY_INDEX(rctx->key_id);
811
812 /* Prepare command and submit */
813 cmdlen = tegra_gcm_prep_final_cmd(se, cpuvaddr, rctx);
814 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen);
815 if (ret)
816 return ret;
817
818 if (rctx->encrypt) {
819 /* Copy the result */
820 offset = rctx->assoclen + rctx->cryptlen;
821 scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg,
822 offset, rctx->authsize, 1);
823 }
824
825 return 0;
826 }
827
tegra_gcm_do_verify(struct tegra_se * se,struct tegra_aead_reqctx * rctx)828 static int tegra_gcm_do_verify(struct tegra_se *se, struct tegra_aead_reqctx *rctx)
829 {
830 unsigned int offset;
831 u8 mac[16];
832
833 offset = rctx->assoclen + rctx->cryptlen;
834 scatterwalk_map_and_copy(mac, rctx->src_sg, offset, rctx->authsize, 0);
835
836 if (crypto_memneq(rctx->outbuf.buf, mac, rctx->authsize))
837 return -EBADMSG;
838
839 return 0;
840 }
841
tegra_ccm_check_iv(const u8 * iv)842 static inline int tegra_ccm_check_iv(const u8 *iv)
843 {
844 /* iv[0] gives value of q-1
845 * 2 <= q <= 8 as per NIST 800-38C notation
846 * 2 <= L <= 8, so 1 <= L' <= 7. as per rfc 3610 notation
847 */
848 if (iv[0] < 1 || iv[0] > 7) {
849 pr_debug("ccm_check_iv failed %d\n", iv[0]);
850 return -EINVAL;
851 }
852
853 return 0;
854 }
855
tegra_cbcmac_prep_cmd(struct tegra_aead_ctx * ctx,struct tegra_aead_reqctx * rctx)856 static unsigned int tegra_cbcmac_prep_cmd(struct tegra_aead_ctx *ctx,
857 struct tegra_aead_reqctx *rctx)
858 {
859 unsigned int data_count, i = 0;
860 struct tegra_se *se = ctx->se;
861 u32 *cpuvaddr = se->cmdbuf->addr;
862
863 data_count = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1;
864
865 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1);
866 cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count);
867
868 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6);
869 cpuvaddr[i++] = rctx->config;
870 cpuvaddr[i++] = rctx->crypto_config;
871
872 cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr);
873 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) |
874 SE_ADDR_HI_SZ(rctx->inbuf.size);
875
876 cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr);
877 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) |
878 SE_ADDR_HI_SZ(0x10); /* HW always generates 128 bit tag */
879
880 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
881 cpuvaddr[i++] = SE_AES_OP_WRSTALL |
882 SE_AES_OP_LASTBUF | SE_AES_OP_START;
883
884 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
885 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
886 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
887
888 return i;
889 }
890
tegra_ctr_prep_cmd(struct tegra_aead_ctx * ctx,struct tegra_aead_reqctx * rctx)891 static unsigned int tegra_ctr_prep_cmd(struct tegra_aead_ctx *ctx,
892 struct tegra_aead_reqctx *rctx)
893 {
894 unsigned int i = 0, j;
895 struct tegra_se *se = ctx->se;
896 u32 *cpuvaddr = se->cmdbuf->addr;
897
898 cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT);
899 cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr);
900 for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++)
901 cpuvaddr[i++] = rctx->iv[j];
902
903 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1);
904 cpuvaddr[i++] = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1;
905 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6);
906 cpuvaddr[i++] = rctx->config;
907 cpuvaddr[i++] = rctx->crypto_config;
908
909 /* Source address setting */
910 cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr);
911 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) |
912 SE_ADDR_HI_SZ(rctx->inbuf.size);
913
914 /* Destination address setting */
915 cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr);
916 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) |
917 SE_ADDR_HI_SZ(rctx->inbuf.size);
918
919 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
920 cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_LASTBUF |
921 SE_AES_OP_START;
922
923 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
924 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
925 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
926
927 dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n",
928 rctx->config, rctx->crypto_config);
929
930 return i;
931 }
932
tegra_ccm_do_cbcmac(struct tegra_aead_ctx * ctx,struct tegra_aead_reqctx * rctx)933 static int tegra_ccm_do_cbcmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
934 {
935 struct tegra_se *se = ctx->se;
936 int cmdlen;
937
938 rctx->config = tegra234_aes_cfg(SE_ALG_CBC_MAC, rctx->encrypt);
939 rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CBC_MAC,
940 rctx->encrypt) |
941 SE_AES_KEY_INDEX(rctx->key_id);
942
943 /* Prepare command and submit */
944 cmdlen = tegra_cbcmac_prep_cmd(ctx, rctx);
945
946 return tegra_se_host1x_submit(se, se->cmdbuf, cmdlen);
947 }
948
tegra_ccm_set_msg_len(u8 * block,unsigned int msglen,int csize)949 static int tegra_ccm_set_msg_len(u8 *block, unsigned int msglen, int csize)
950 {
951 __be32 data;
952
953 memset(block, 0, csize);
954 block += csize;
955
956 if (csize >= 4)
957 csize = 4;
958 else if (msglen > (1 << (8 * csize)))
959 return -EOVERFLOW;
960
961 data = cpu_to_be32(msglen);
962 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
963
964 return 0;
965 }
966
tegra_ccm_format_nonce(struct tegra_aead_reqctx * rctx,u8 * nonce)967 static int tegra_ccm_format_nonce(struct tegra_aead_reqctx *rctx, u8 *nonce)
968 {
969 unsigned int q, t;
970 u8 *q_ptr, *iv = (u8 *)rctx->iv;
971
972 memcpy(nonce, rctx->iv, 16);
973
974 /*** 1. Prepare Flags Octet ***/
975
976 /* Encode t (mac length) */
977 t = rctx->authsize;
978 nonce[0] |= (((t - 2) / 2) << 3);
979
980 /* Adata */
981 if (rctx->assoclen)
982 nonce[0] |= (1 << 6);
983
984 /*** Encode Q - message length ***/
985 q = iv[0] + 1;
986 q_ptr = nonce + 16 - q;
987
988 return tegra_ccm_set_msg_len(q_ptr, rctx->cryptlen, q);
989 }
990
tegra_ccm_format_adata(u8 * adata,unsigned int a)991 static int tegra_ccm_format_adata(u8 *adata, unsigned int a)
992 {
993 int len = 0;
994
995 /* add control info for associated data
996 * RFC 3610 and NIST Special Publication 800-38C
997 */
998 if (a < 65280) {
999 *(__be16 *)adata = cpu_to_be16(a);
1000 len = 2;
1001 } else {
1002 *(__be16 *)adata = cpu_to_be16(0xfffe);
1003 *(__be32 *)&adata[2] = cpu_to_be32(a);
1004 len = 6;
1005 }
1006
1007 return len;
1008 }
1009
tegra_ccm_add_padding(u8 * buf,unsigned int len)1010 static int tegra_ccm_add_padding(u8 *buf, unsigned int len)
1011 {
1012 unsigned int padlen = 16 - (len % 16);
1013 u8 padding[16] = {0};
1014
1015 if (padlen == 16)
1016 return 0;
1017
1018 memcpy(buf, padding, padlen);
1019
1020 return padlen;
1021 }
1022
tegra_ccm_format_blocks(struct tegra_aead_reqctx * rctx)1023 static int tegra_ccm_format_blocks(struct tegra_aead_reqctx *rctx)
1024 {
1025 unsigned int alen = 0, offset = 0;
1026 u8 nonce[16], adata[16];
1027 int ret;
1028
1029 ret = tegra_ccm_format_nonce(rctx, nonce);
1030 if (ret)
1031 return ret;
1032
1033 memcpy(rctx->inbuf.buf, nonce, 16);
1034 offset = 16;
1035
1036 if (rctx->assoclen) {
1037 alen = tegra_ccm_format_adata(adata, rctx->assoclen);
1038 memcpy(rctx->inbuf.buf + offset, adata, alen);
1039 offset += alen;
1040
1041 scatterwalk_map_and_copy(rctx->inbuf.buf + offset,
1042 rctx->src_sg, 0, rctx->assoclen, 0);
1043
1044 offset += rctx->assoclen;
1045 offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset,
1046 rctx->assoclen + alen);
1047 }
1048
1049 return offset;
1050 }
1051
tegra_ccm_mac_result(struct tegra_se * se,struct tegra_aead_reqctx * rctx)1052 static int tegra_ccm_mac_result(struct tegra_se *se, struct tegra_aead_reqctx *rctx)
1053 {
1054 u32 result[16];
1055 int i, ret;
1056
1057 /* Read and clear Result */
1058 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++)
1059 result[i] = readl(se->base + se->hw->regs->result + (i * 4));
1060
1061 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++)
1062 writel(0, se->base + se->hw->regs->result + (i * 4));
1063
1064 if (rctx->encrypt) {
1065 memcpy(rctx->authdata, result, rctx->authsize);
1066 } else {
1067 ret = crypto_memneq(rctx->authdata, result, rctx->authsize);
1068 if (ret)
1069 return -EBADMSG;
1070 }
1071
1072 return 0;
1073 }
1074
tegra_ccm_ctr_result(struct tegra_se * se,struct tegra_aead_reqctx * rctx)1075 static int tegra_ccm_ctr_result(struct tegra_se *se, struct tegra_aead_reqctx *rctx)
1076 {
1077 /* Copy result */
1078 scatterwalk_map_and_copy(rctx->outbuf.buf + 16, rctx->dst_sg,
1079 rctx->assoclen, rctx->cryptlen, 1);
1080
1081 if (rctx->encrypt)
1082 scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg,
1083 rctx->assoclen + rctx->cryptlen,
1084 rctx->authsize, 1);
1085 else
1086 memcpy(rctx->authdata, rctx->outbuf.buf, rctx->authsize);
1087
1088 return 0;
1089 }
1090
tegra_ccm_compute_auth(struct tegra_aead_ctx * ctx,struct tegra_aead_reqctx * rctx)1091 static int tegra_ccm_compute_auth(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
1092 {
1093 struct tegra_se *se = ctx->se;
1094 struct scatterlist *sg;
1095 int offset, ret;
1096
1097 offset = tegra_ccm_format_blocks(rctx);
1098 if (offset < 0)
1099 return -EINVAL;
1100
1101 /* Copy plain text to the buffer */
1102 sg = rctx->encrypt ? rctx->src_sg : rctx->dst_sg;
1103
1104 scatterwalk_map_and_copy(rctx->inbuf.buf + offset,
1105 sg, rctx->assoclen,
1106 rctx->cryptlen, 0);
1107 offset += rctx->cryptlen;
1108 offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->cryptlen);
1109
1110 rctx->inbuf.size = offset;
1111
1112 ret = tegra_ccm_do_cbcmac(ctx, rctx);
1113 if (ret)
1114 return ret;
1115
1116 return tegra_ccm_mac_result(se, rctx);
1117 }
1118
tegra_ccm_do_ctr(struct tegra_aead_ctx * ctx,struct tegra_aead_reqctx * rctx)1119 static int tegra_ccm_do_ctr(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
1120 {
1121 struct tegra_se *se = ctx->se;
1122 unsigned int cmdlen, offset = 0;
1123 struct scatterlist *sg = rctx->src_sg;
1124 int ret;
1125
1126 rctx->config = tegra234_aes_cfg(SE_ALG_CTR, rctx->encrypt);
1127 rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CTR, rctx->encrypt) |
1128 SE_AES_KEY_INDEX(rctx->key_id);
1129
1130 /* Copy authdata in the top of buffer for encryption/decryption */
1131 if (rctx->encrypt)
1132 memcpy(rctx->inbuf.buf, rctx->authdata, rctx->authsize);
1133 else
1134 scatterwalk_map_and_copy(rctx->inbuf.buf, sg,
1135 rctx->assoclen + rctx->cryptlen,
1136 rctx->authsize, 0);
1137
1138 offset += rctx->authsize;
1139 offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->authsize);
1140
1141 /* If there is no cryptlen, proceed to submit the task */
1142 if (rctx->cryptlen) {
1143 scatterwalk_map_and_copy(rctx->inbuf.buf + offset, sg,
1144 rctx->assoclen, rctx->cryptlen, 0);
1145 offset += rctx->cryptlen;
1146 offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->cryptlen);
1147 }
1148
1149 rctx->inbuf.size = offset;
1150
1151 /* Prepare command and submit */
1152 cmdlen = tegra_ctr_prep_cmd(ctx, rctx);
1153 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen);
1154 if (ret)
1155 return ret;
1156
1157 return tegra_ccm_ctr_result(se, rctx);
1158 }
1159
tegra_ccm_crypt_init(struct aead_request * req,struct tegra_se * se,struct tegra_aead_reqctx * rctx)1160 static int tegra_ccm_crypt_init(struct aead_request *req, struct tegra_se *se,
1161 struct tegra_aead_reqctx *rctx)
1162 {
1163 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1164 u8 *iv = (u8 *)rctx->iv;
1165 int ret, i;
1166
1167 rctx->src_sg = req->src;
1168 rctx->dst_sg = req->dst;
1169 rctx->assoclen = req->assoclen;
1170 rctx->authsize = crypto_aead_authsize(tfm);
1171
1172 if (rctx->encrypt)
1173 rctx->cryptlen = req->cryptlen;
1174 else
1175 rctx->cryptlen = req->cryptlen - rctx->authsize;
1176
1177 memcpy(iv, req->iv, 16);
1178
1179 ret = tegra_ccm_check_iv(iv);
1180 if (ret)
1181 return ret;
1182
1183 /* Note: rfc 3610 and NIST 800-38C require counter (ctr_0) of
1184 * zero to encrypt auth tag.
1185 * req->iv has the formatted ctr_0 (i.e. Flags || N || 0).
1186 */
1187 memset(iv + 15 - iv[0], 0, iv[0] + 1);
1188
1189 /* Clear any previous result */
1190 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++)
1191 writel(0, se->base + se->hw->regs->result + (i * 4));
1192
1193 return 0;
1194 }
1195
tegra_ccm_do_one_req(struct crypto_engine * engine,void * areq)1196 static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq)
1197 {
1198 struct aead_request *req = container_of(areq, struct aead_request, base);
1199 struct tegra_aead_reqctx *rctx = aead_request_ctx(req);
1200 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1201 struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm);
1202 struct tegra_se *se = ctx->se;
1203 unsigned int bufsize;
1204 int ret;
1205
1206 ret = tegra_ccm_crypt_init(req, se, rctx);
1207 if (ret)
1208 goto out_finalize;
1209
1210 rctx->key_id = ctx->key_id;
1211
1212 /* Allocate buffers required */
1213 bufsize = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100;
1214 rctx->inbuf.size = bufsize;
1215 rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize,
1216 &rctx->inbuf.addr, GFP_KERNEL);
1217 ret = -ENOMEM;
1218 if (!rctx->inbuf.buf)
1219 goto out_finalize;
1220
1221 rctx->outbuf.size = bufsize;
1222 rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize,
1223 &rctx->outbuf.addr, GFP_KERNEL);
1224 if (!rctx->outbuf.buf)
1225 goto out_free_inbuf;
1226
1227 if (!ctx->key_id) {
1228 ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key,
1229 ctx->keylen, ctx->alg, &rctx->key_id);
1230 if (ret)
1231 goto out;
1232 }
1233
1234 if (rctx->encrypt) {
1235 /* CBC MAC Operation */
1236 ret = tegra_ccm_compute_auth(ctx, rctx);
1237 if (ret)
1238 goto out;
1239
1240 /* CTR operation */
1241 ret = tegra_ccm_do_ctr(ctx, rctx);
1242 if (ret)
1243 goto out;
1244 } else {
1245 /* CTR operation */
1246 ret = tegra_ccm_do_ctr(ctx, rctx);
1247 if (ret)
1248 goto out;
1249
1250 /* CBC MAC Operation */
1251 ret = tegra_ccm_compute_auth(ctx, rctx);
1252 if (ret)
1253 goto out;
1254 }
1255
1256 out:
1257 dma_free_coherent(ctx->se->dev, bufsize,
1258 rctx->outbuf.buf, rctx->outbuf.addr);
1259
1260 out_free_inbuf:
1261 dma_free_coherent(ctx->se->dev, bufsize,
1262 rctx->inbuf.buf, rctx->inbuf.addr);
1263
1264 if (tegra_key_is_reserved(rctx->key_id))
1265 tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg);
1266
1267 out_finalize:
1268 local_bh_disable();
1269 crypto_finalize_aead_request(ctx->se->engine, req, ret);
1270 local_bh_enable();
1271
1272 return 0;
1273 }
1274
tegra_gcm_do_one_req(struct crypto_engine * engine,void * areq)1275 static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq)
1276 {
1277 struct aead_request *req = container_of(areq, struct aead_request, base);
1278 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1279 struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm);
1280 struct tegra_aead_reqctx *rctx = aead_request_ctx(req);
1281 unsigned int bufsize;
1282 int ret;
1283
1284 rctx->src_sg = req->src;
1285 rctx->dst_sg = req->dst;
1286 rctx->assoclen = req->assoclen;
1287 rctx->authsize = crypto_aead_authsize(tfm);
1288
1289 if (rctx->encrypt)
1290 rctx->cryptlen = req->cryptlen;
1291 else
1292 rctx->cryptlen = req->cryptlen - rctx->authsize;
1293
1294 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
1295 rctx->iv[3] = (1 << 24);
1296
1297 rctx->key_id = ctx->key_id;
1298
1299 /* Allocate buffers required */
1300 bufsize = rctx->assoclen + rctx->authsize + rctx->cryptlen;
1301 rctx->inbuf.size = bufsize;
1302 rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize,
1303 &rctx->inbuf.addr, GFP_KERNEL);
1304 if (!rctx->inbuf.buf) {
1305 ret = -ENOMEM;
1306 goto out_finalize;
1307 }
1308
1309 rctx->outbuf.size = bufsize;
1310 rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize,
1311 &rctx->outbuf.addr, GFP_KERNEL);
1312 if (!rctx->outbuf.buf) {
1313 ret = -ENOMEM;
1314 goto out_free_inbuf;
1315 }
1316
1317 if (!ctx->key_id) {
1318 ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key,
1319 ctx->keylen, ctx->alg, &rctx->key_id);
1320 if (ret)
1321 goto out;
1322 }
1323
1324 /* If there is associated data perform GMAC operation */
1325 if (rctx->assoclen) {
1326 ret = tegra_gcm_do_gmac(ctx, rctx);
1327 if (ret)
1328 goto out;
1329 }
1330
1331 /* GCM Encryption/Decryption operation */
1332 if (rctx->cryptlen) {
1333 ret = tegra_gcm_do_crypt(ctx, rctx);
1334 if (ret)
1335 goto out;
1336 }
1337
1338 /* GCM_FINAL operation */
1339 ret = tegra_gcm_do_final(ctx, rctx);
1340 if (ret)
1341 goto out;
1342
1343 if (!rctx->encrypt)
1344 ret = tegra_gcm_do_verify(ctx->se, rctx);
1345
1346 out:
1347 dma_free_coherent(ctx->se->dev, bufsize,
1348 rctx->outbuf.buf, rctx->outbuf.addr);
1349
1350 out_free_inbuf:
1351 dma_free_coherent(ctx->se->dev, bufsize,
1352 rctx->inbuf.buf, rctx->inbuf.addr);
1353
1354 if (tegra_key_is_reserved(rctx->key_id))
1355 tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg);
1356
1357 out_finalize:
1358 local_bh_disable();
1359 crypto_finalize_aead_request(ctx->se->engine, req, ret);
1360 local_bh_enable();
1361
1362 return 0;
1363 }
1364
tegra_aead_cra_init(struct crypto_aead * tfm)1365 static int tegra_aead_cra_init(struct crypto_aead *tfm)
1366 {
1367 struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm);
1368 struct aead_alg *alg = crypto_aead_alg(tfm);
1369 struct tegra_se_alg *se_alg;
1370 const char *algname;
1371 int ret;
1372
1373 algname = crypto_tfm_alg_name(&tfm->base);
1374
1375 se_alg = container_of(alg, struct tegra_se_alg, alg.aead.base);
1376
1377 crypto_aead_set_reqsize(tfm, sizeof(struct tegra_aead_reqctx));
1378
1379 ctx->se = se_alg->se_dev;
1380 ctx->key_id = 0;
1381 ctx->keylen = 0;
1382
1383 ret = se_algname_to_algid(algname);
1384 if (ret < 0) {
1385 dev_err(ctx->se->dev, "invalid algorithm\n");
1386 return ret;
1387 }
1388
1389 ctx->alg = ret;
1390
1391 return 0;
1392 }
1393
tegra_ccm_setauthsize(struct crypto_aead * tfm,unsigned int authsize)1394 static int tegra_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
1395 {
1396 switch (authsize) {
1397 case 4:
1398 case 6:
1399 case 8:
1400 case 10:
1401 case 12:
1402 case 14:
1403 case 16:
1404 return 0;
1405 default:
1406 return -EINVAL;
1407 }
1408 }
1409
tegra_gcm_setauthsize(struct crypto_aead * tfm,unsigned int authsize)1410 static int tegra_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
1411 {
1412 return crypto_gcm_check_authsize(authsize);
1413 }
1414
tegra_aead_cra_exit(struct crypto_aead * tfm)1415 static void tegra_aead_cra_exit(struct crypto_aead *tfm)
1416 {
1417 struct tegra_aead_ctx *ctx = crypto_tfm_ctx(&tfm->base);
1418
1419 if (ctx->key_id)
1420 tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg);
1421 }
1422
tegra_aead_crypt(struct aead_request * req,bool encrypt)1423 static int tegra_aead_crypt(struct aead_request *req, bool encrypt)
1424 {
1425 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1426 struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm);
1427 struct tegra_aead_reqctx *rctx = aead_request_ctx(req);
1428
1429 rctx->encrypt = encrypt;
1430
1431 return crypto_transfer_aead_request_to_engine(ctx->se->engine, req);
1432 }
1433
tegra_aead_encrypt(struct aead_request * req)1434 static int tegra_aead_encrypt(struct aead_request *req)
1435 {
1436 return tegra_aead_crypt(req, true);
1437 }
1438
tegra_aead_decrypt(struct aead_request * req)1439 static int tegra_aead_decrypt(struct aead_request *req)
1440 {
1441 return tegra_aead_crypt(req, false);
1442 }
1443
tegra_aead_setkey(struct crypto_aead * tfm,const u8 * key,u32 keylen)1444 static int tegra_aead_setkey(struct crypto_aead *tfm,
1445 const u8 *key, u32 keylen)
1446 {
1447 struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm);
1448 int ret;
1449
1450 if (aes_check_keylen(keylen)) {
1451 dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen);
1452 return -EINVAL;
1453 }
1454
1455 ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id);
1456 if (ret) {
1457 ctx->keylen = keylen;
1458 memcpy(ctx->key, key, keylen);
1459 }
1460
1461 return 0;
1462 }
1463
tegra_cmac_prep_cmd(struct tegra_cmac_ctx * ctx,struct tegra_cmac_reqctx * rctx)1464 static unsigned int tegra_cmac_prep_cmd(struct tegra_cmac_ctx *ctx,
1465 struct tegra_cmac_reqctx *rctx)
1466 {
1467 unsigned int data_count, res_bits = 0, i = 0, j;
1468 struct tegra_se *se = ctx->se;
1469 u32 *cpuvaddr = se->cmdbuf->addr, op;
1470
1471 data_count = (rctx->datbuf.size / AES_BLOCK_SIZE);
1472
1473 op = SE_AES_OP_WRSTALL | SE_AES_OP_START | SE_AES_OP_LASTBUF;
1474
1475 if (!(rctx->task & SHA_UPDATE)) {
1476 op |= SE_AES_OP_FINAL;
1477 res_bits = (rctx->datbuf.size % AES_BLOCK_SIZE) * 8;
1478 }
1479
1480 if (!res_bits && data_count)
1481 data_count--;
1482
1483 if (rctx->task & SHA_FIRST) {
1484 rctx->task &= ~SHA_FIRST;
1485
1486 cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT);
1487 cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr);
1488 /* Load 0 IV */
1489 for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++)
1490 cpuvaddr[i++] = 0;
1491 }
1492
1493 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1);
1494 cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) |
1495 SE_LAST_BLOCK_RES_BITS(res_bits);
1496
1497 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6);
1498 cpuvaddr[i++] = rctx->config;
1499 cpuvaddr[i++] = rctx->crypto_config;
1500
1501 /* Source Address */
1502 cpuvaddr[i++] = lower_32_bits(rctx->datbuf.addr);
1503 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->datbuf.addr)) |
1504 SE_ADDR_HI_SZ(rctx->datbuf.size);
1505 cpuvaddr[i++] = 0;
1506 cpuvaddr[i++] = SE_ADDR_HI_SZ(AES_BLOCK_SIZE);
1507
1508 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
1509 cpuvaddr[i++] = op;
1510
1511 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
1512 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
1513 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
1514
1515 return i;
1516 }
1517
tegra_cmac_copy_result(struct tegra_se * se,struct tegra_cmac_reqctx * rctx)1518 static void tegra_cmac_copy_result(struct tegra_se *se, struct tegra_cmac_reqctx *rctx)
1519 {
1520 int i;
1521
1522 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++)
1523 rctx->result[i] = readl(se->base + se->hw->regs->result + (i * 4));
1524 }
1525
tegra_cmac_paste_result(struct tegra_se * se,struct tegra_cmac_reqctx * rctx)1526 static void tegra_cmac_paste_result(struct tegra_se *se, struct tegra_cmac_reqctx *rctx)
1527 {
1528 int i;
1529
1530 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++)
1531 writel(rctx->result[i],
1532 se->base + se->hw->regs->result + (i * 4));
1533 }
1534
tegra_cmac_do_init(struct ahash_request * req)1535 static int tegra_cmac_do_init(struct ahash_request *req)
1536 {
1537 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req);
1538 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1539 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm);
1540 struct tegra_se *se = ctx->se;
1541 int i;
1542
1543 rctx->total_len = 0;
1544 rctx->datbuf.size = 0;
1545 rctx->residue.size = 0;
1546 rctx->key_id = ctx->key_id;
1547 rctx->task |= SHA_FIRST;
1548 rctx->blk_size = crypto_ahash_blocksize(tfm);
1549
1550 rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size * 2,
1551 &rctx->residue.addr, GFP_KERNEL);
1552 if (!rctx->residue.buf)
1553 return -ENOMEM;
1554
1555 rctx->residue.size = 0;
1556
1557 /* Clear any previous result */
1558 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++)
1559 writel(0, se->base + se->hw->regs->result + (i * 4));
1560
1561 return 0;
1562 }
1563
tegra_cmac_do_update(struct ahash_request * req)1564 static int tegra_cmac_do_update(struct ahash_request *req)
1565 {
1566 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req);
1567 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1568 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm);
1569 struct tegra_se *se = ctx->se;
1570 unsigned int nblks, nresidue, cmdlen;
1571 int ret;
1572
1573 if (!req->nbytes)
1574 return 0;
1575
1576 nresidue = (req->nbytes + rctx->residue.size) % rctx->blk_size;
1577 nblks = (req->nbytes + rctx->residue.size) / rctx->blk_size;
1578
1579 /*
1580 * Reserve the last block as residue during final() to process.
1581 */
1582 if (!nresidue && nblks) {
1583 nresidue += rctx->blk_size;
1584 nblks--;
1585 }
1586
1587 rctx->src_sg = req->src;
1588 rctx->datbuf.size = (req->nbytes + rctx->residue.size) - nresidue;
1589 rctx->total_len += rctx->datbuf.size;
1590 rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0);
1591 rctx->crypto_config = SE_AES_KEY_INDEX(rctx->key_id);
1592
1593 /*
1594 * Keep one block and residue bytes in residue and
1595 * return. The bytes will be processed in final()
1596 */
1597 if (nblks < 1) {
1598 scatterwalk_map_and_copy(rctx->residue.buf + rctx->residue.size,
1599 rctx->src_sg, 0, req->nbytes, 0);
1600
1601 rctx->residue.size += req->nbytes;
1602 return 0;
1603 }
1604
1605 rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size,
1606 &rctx->datbuf.addr, GFP_KERNEL);
1607 if (!rctx->datbuf.buf)
1608 return -ENOMEM;
1609
1610 /* Copy the previous residue first */
1611 if (rctx->residue.size)
1612 memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size);
1613
1614 scatterwalk_map_and_copy(rctx->datbuf.buf + rctx->residue.size,
1615 rctx->src_sg, 0, req->nbytes - nresidue, 0);
1616
1617 scatterwalk_map_and_copy(rctx->residue.buf, rctx->src_sg,
1618 req->nbytes - nresidue, nresidue, 0);
1619
1620 /* Update residue value with the residue after current block */
1621 rctx->residue.size = nresidue;
1622
1623 /*
1624 * If this is not the first task, paste the previous copied
1625 * intermediate results to the registers so that it gets picked up.
1626 */
1627 if (!(rctx->task & SHA_FIRST))
1628 tegra_cmac_paste_result(ctx->se, rctx);
1629
1630 cmdlen = tegra_cmac_prep_cmd(ctx, rctx);
1631 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen);
1632
1633 tegra_cmac_copy_result(ctx->se, rctx);
1634
1635 dma_free_coherent(ctx->se->dev, rctx->datbuf.size,
1636 rctx->datbuf.buf, rctx->datbuf.addr);
1637
1638 return ret;
1639 }
1640
tegra_cmac_do_final(struct ahash_request * req)1641 static int tegra_cmac_do_final(struct ahash_request *req)
1642 {
1643 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req);
1644 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1645 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm);
1646 struct tegra_se *se = ctx->se;
1647 u32 *result = (u32 *)req->result;
1648 int ret = 0, i, cmdlen;
1649
1650 if (!req->nbytes && !rctx->total_len && ctx->fallback_tfm) {
1651 return crypto_shash_tfm_digest(ctx->fallback_tfm,
1652 NULL, 0, req->result);
1653 }
1654
1655 if (rctx->residue.size) {
1656 rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->residue.size,
1657 &rctx->datbuf.addr, GFP_KERNEL);
1658 if (!rctx->datbuf.buf) {
1659 ret = -ENOMEM;
1660 goto out_free;
1661 }
1662
1663 memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size);
1664 }
1665
1666 rctx->datbuf.size = rctx->residue.size;
1667 rctx->total_len += rctx->residue.size;
1668 rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0);
1669
1670 /*
1671 * If this is not the first task, paste the previous copied
1672 * intermediate results to the registers so that it gets picked up.
1673 */
1674 if (!(rctx->task & SHA_FIRST))
1675 tegra_cmac_paste_result(ctx->se, rctx);
1676
1677 /* Prepare command and submit */
1678 cmdlen = tegra_cmac_prep_cmd(ctx, rctx);
1679 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen);
1680 if (ret)
1681 goto out;
1682
1683 /* Read and clear Result register */
1684 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++)
1685 result[i] = readl(se->base + se->hw->regs->result + (i * 4));
1686
1687 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++)
1688 writel(0, se->base + se->hw->regs->result + (i * 4));
1689
1690 out:
1691 if (rctx->residue.size)
1692 dma_free_coherent(se->dev, rctx->datbuf.size,
1693 rctx->datbuf.buf, rctx->datbuf.addr);
1694 out_free:
1695 dma_free_coherent(se->dev, crypto_ahash_blocksize(tfm) * 2,
1696 rctx->residue.buf, rctx->residue.addr);
1697 return ret;
1698 }
1699
tegra_cmac_do_one_req(struct crypto_engine * engine,void * areq)1700 static int tegra_cmac_do_one_req(struct crypto_engine *engine, void *areq)
1701 {
1702 struct ahash_request *req = ahash_request_cast(areq);
1703 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req);
1704 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1705 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm);
1706 struct tegra_se *se = ctx->se;
1707 int ret = 0;
1708
1709 if (rctx->task & SHA_INIT) {
1710 ret = tegra_cmac_do_init(req);
1711 if (ret)
1712 goto out;
1713
1714 rctx->task &= ~SHA_INIT;
1715 }
1716
1717 if (!ctx->key_id) {
1718 ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key,
1719 ctx->keylen, ctx->alg, &rctx->key_id);
1720 if (ret)
1721 goto out;
1722 }
1723
1724 if (rctx->task & SHA_UPDATE) {
1725 ret = tegra_cmac_do_update(req);
1726 if (ret)
1727 goto out;
1728
1729 rctx->task &= ~SHA_UPDATE;
1730 }
1731
1732 if (rctx->task & SHA_FINAL) {
1733 ret = tegra_cmac_do_final(req);
1734 if (ret)
1735 goto out;
1736
1737 rctx->task &= ~SHA_FINAL;
1738 }
1739 out:
1740 if (tegra_key_is_reserved(rctx->key_id))
1741 tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg);
1742
1743 local_bh_disable();
1744 crypto_finalize_hash_request(se->engine, req, ret);
1745 local_bh_enable();
1746
1747 return 0;
1748 }
1749
tegra_cmac_init_fallback(struct crypto_ahash * tfm,struct tegra_cmac_ctx * ctx,const char * algname)1750 static void tegra_cmac_init_fallback(struct crypto_ahash *tfm, struct tegra_cmac_ctx *ctx,
1751 const char *algname)
1752 {
1753 unsigned int statesize;
1754
1755 ctx->fallback_tfm = crypto_alloc_shash(algname, 0, CRYPTO_ALG_NEED_FALLBACK);
1756
1757 if (IS_ERR(ctx->fallback_tfm)) {
1758 dev_warn(ctx->se->dev, "failed to allocate fallback for %s\n", algname);
1759 ctx->fallback_tfm = NULL;
1760 return;
1761 }
1762
1763 statesize = crypto_shash_statesize(ctx->fallback_tfm);
1764
1765 if (statesize > sizeof(struct tegra_cmac_reqctx))
1766 crypto_ahash_set_statesize(tfm, statesize);
1767 }
1768
tegra_cmac_cra_init(struct crypto_tfm * tfm)1769 static int tegra_cmac_cra_init(struct crypto_tfm *tfm)
1770 {
1771 struct tegra_cmac_ctx *ctx = crypto_tfm_ctx(tfm);
1772 struct crypto_ahash *ahash_tfm = __crypto_ahash_cast(tfm);
1773 struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg);
1774 struct tegra_se_alg *se_alg;
1775 const char *algname;
1776 int ret;
1777
1778 algname = crypto_tfm_alg_name(tfm);
1779 se_alg = container_of(alg, struct tegra_se_alg, alg.ahash.base);
1780
1781 crypto_ahash_set_reqsize(ahash_tfm, sizeof(struct tegra_cmac_reqctx));
1782
1783 ctx->se = se_alg->se_dev;
1784 ctx->key_id = 0;
1785 ctx->keylen = 0;
1786
1787 ret = se_algname_to_algid(algname);
1788 if (ret < 0) {
1789 dev_err(ctx->se->dev, "invalid algorithm\n");
1790 return ret;
1791 }
1792
1793 ctx->alg = ret;
1794
1795 tegra_cmac_init_fallback(ahash_tfm, ctx, algname);
1796
1797 return 0;
1798 }
1799
tegra_cmac_cra_exit(struct crypto_tfm * tfm)1800 static void tegra_cmac_cra_exit(struct crypto_tfm *tfm)
1801 {
1802 struct tegra_cmac_ctx *ctx = crypto_tfm_ctx(tfm);
1803
1804 if (ctx->fallback_tfm)
1805 crypto_free_shash(ctx->fallback_tfm);
1806
1807 tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg);
1808 }
1809
tegra_cmac_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1810 static int tegra_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
1811 unsigned int keylen)
1812 {
1813 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm);
1814 int ret;
1815
1816 if (aes_check_keylen(keylen)) {
1817 dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen);
1818 return -EINVAL;
1819 }
1820
1821 if (ctx->fallback_tfm)
1822 crypto_shash_setkey(ctx->fallback_tfm, key, keylen);
1823
1824 ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id);
1825 if (ret) {
1826 ctx->keylen = keylen;
1827 memcpy(ctx->key, key, keylen);
1828 }
1829
1830 return 0;
1831 }
1832
tegra_cmac_init(struct ahash_request * req)1833 static int tegra_cmac_init(struct ahash_request *req)
1834 {
1835 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1836 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm);
1837 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req);
1838
1839 rctx->task = SHA_INIT;
1840
1841 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req);
1842 }
1843
tegra_cmac_update(struct ahash_request * req)1844 static int tegra_cmac_update(struct ahash_request *req)
1845 {
1846 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1847 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm);
1848 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req);
1849
1850 rctx->task |= SHA_UPDATE;
1851
1852 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req);
1853 }
1854
tegra_cmac_final(struct ahash_request * req)1855 static int tegra_cmac_final(struct ahash_request *req)
1856 {
1857 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1858 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm);
1859 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req);
1860
1861 rctx->task |= SHA_FINAL;
1862
1863 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req);
1864 }
1865
tegra_cmac_finup(struct ahash_request * req)1866 static int tegra_cmac_finup(struct ahash_request *req)
1867 {
1868 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1869 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm);
1870 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req);
1871
1872 rctx->task |= SHA_UPDATE | SHA_FINAL;
1873
1874 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req);
1875 }
1876
tegra_cmac_digest(struct ahash_request * req)1877 static int tegra_cmac_digest(struct ahash_request *req)
1878 {
1879 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1880 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm);
1881 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req);
1882
1883 rctx->task |= SHA_INIT | SHA_UPDATE | SHA_FINAL;
1884
1885 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req);
1886 }
1887
tegra_cmac_export(struct ahash_request * req,void * out)1888 static int tegra_cmac_export(struct ahash_request *req, void *out)
1889 {
1890 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req);
1891
1892 memcpy(out, rctx, sizeof(*rctx));
1893
1894 return 0;
1895 }
1896
tegra_cmac_import(struct ahash_request * req,const void * in)1897 static int tegra_cmac_import(struct ahash_request *req, const void *in)
1898 {
1899 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req);
1900
1901 memcpy(rctx, in, sizeof(*rctx));
1902
1903 return 0;
1904 }
1905
1906 static struct tegra_se_alg tegra_aead_algs[] = {
1907 {
1908 .alg.aead.op.do_one_request = tegra_gcm_do_one_req,
1909 .alg.aead.base = {
1910 .init = tegra_aead_cra_init,
1911 .exit = tegra_aead_cra_exit,
1912 .setkey = tegra_aead_setkey,
1913 .setauthsize = tegra_gcm_setauthsize,
1914 .encrypt = tegra_aead_encrypt,
1915 .decrypt = tegra_aead_decrypt,
1916 .maxauthsize = AES_BLOCK_SIZE,
1917 .ivsize = GCM_AES_IV_SIZE,
1918 .base = {
1919 .cra_name = "gcm(aes)",
1920 .cra_driver_name = "gcm-aes-tegra",
1921 .cra_priority = 500,
1922 .cra_flags = CRYPTO_ALG_ASYNC,
1923 .cra_blocksize = 1,
1924 .cra_ctxsize = sizeof(struct tegra_aead_ctx),
1925 .cra_alignmask = 0xf,
1926 .cra_module = THIS_MODULE,
1927 },
1928 }
1929 }, {
1930 .alg.aead.op.do_one_request = tegra_ccm_do_one_req,
1931 .alg.aead.base = {
1932 .init = tegra_aead_cra_init,
1933 .exit = tegra_aead_cra_exit,
1934 .setkey = tegra_aead_setkey,
1935 .setauthsize = tegra_ccm_setauthsize,
1936 .encrypt = tegra_aead_encrypt,
1937 .decrypt = tegra_aead_decrypt,
1938 .maxauthsize = AES_BLOCK_SIZE,
1939 .ivsize = AES_BLOCK_SIZE,
1940 .chunksize = AES_BLOCK_SIZE,
1941 .base = {
1942 .cra_name = "ccm(aes)",
1943 .cra_driver_name = "ccm-aes-tegra",
1944 .cra_priority = 500,
1945 .cra_flags = CRYPTO_ALG_ASYNC,
1946 .cra_blocksize = 1,
1947 .cra_ctxsize = sizeof(struct tegra_aead_ctx),
1948 .cra_alignmask = 0xf,
1949 .cra_module = THIS_MODULE,
1950 },
1951 }
1952 }
1953 };
1954
1955 static struct tegra_se_alg tegra_cmac_algs[] = {
1956 {
1957 .alg.ahash.op.do_one_request = tegra_cmac_do_one_req,
1958 .alg.ahash.base = {
1959 .init = tegra_cmac_init,
1960 .setkey = tegra_cmac_setkey,
1961 .update = tegra_cmac_update,
1962 .final = tegra_cmac_final,
1963 .finup = tegra_cmac_finup,
1964 .digest = tegra_cmac_digest,
1965 .export = tegra_cmac_export,
1966 .import = tegra_cmac_import,
1967 .halg.digestsize = AES_BLOCK_SIZE,
1968 .halg.statesize = sizeof(struct tegra_cmac_reqctx),
1969 .halg.base = {
1970 .cra_name = "cmac(aes)",
1971 .cra_driver_name = "tegra-se-cmac",
1972 .cra_priority = 300,
1973 .cra_flags = CRYPTO_ALG_ASYNC,
1974 .cra_blocksize = AES_BLOCK_SIZE,
1975 .cra_ctxsize = sizeof(struct tegra_cmac_ctx),
1976 .cra_alignmask = 0,
1977 .cra_module = THIS_MODULE,
1978 .cra_init = tegra_cmac_cra_init,
1979 .cra_exit = tegra_cmac_cra_exit,
1980 }
1981 }
1982 }
1983 };
1984
tegra_init_aes(struct tegra_se * se)1985 int tegra_init_aes(struct tegra_se *se)
1986 {
1987 struct aead_engine_alg *aead_alg;
1988 struct ahash_engine_alg *ahash_alg;
1989 struct skcipher_engine_alg *sk_alg;
1990 int i, ret;
1991
1992 se->manifest = tegra_aes_kac_manifest;
1993
1994 for (i = 0; i < ARRAY_SIZE(tegra_aes_algs); i++) {
1995 sk_alg = &tegra_aes_algs[i].alg.skcipher;
1996 tegra_aes_algs[i].se_dev = se;
1997
1998 ret = crypto_engine_register_skcipher(sk_alg);
1999 if (ret) {
2000 dev_err(se->dev, "failed to register %s\n",
2001 sk_alg->base.base.cra_name);
2002 goto err_aes;
2003 }
2004 }
2005
2006 for (i = 0; i < ARRAY_SIZE(tegra_aead_algs); i++) {
2007 aead_alg = &tegra_aead_algs[i].alg.aead;
2008 tegra_aead_algs[i].se_dev = se;
2009
2010 ret = crypto_engine_register_aead(aead_alg);
2011 if (ret) {
2012 dev_err(se->dev, "failed to register %s\n",
2013 aead_alg->base.base.cra_name);
2014 goto err_aead;
2015 }
2016 }
2017
2018 for (i = 0; i < ARRAY_SIZE(tegra_cmac_algs); i++) {
2019 ahash_alg = &tegra_cmac_algs[i].alg.ahash;
2020 tegra_cmac_algs[i].se_dev = se;
2021
2022 ret = crypto_engine_register_ahash(ahash_alg);
2023 if (ret) {
2024 dev_err(se->dev, "failed to register %s\n",
2025 ahash_alg->base.halg.base.cra_name);
2026 goto err_cmac;
2027 }
2028 }
2029
2030 return 0;
2031
2032 err_cmac:
2033 while (i--)
2034 crypto_engine_unregister_ahash(&tegra_cmac_algs[i].alg.ahash);
2035
2036 i = ARRAY_SIZE(tegra_aead_algs);
2037 err_aead:
2038 while (i--)
2039 crypto_engine_unregister_aead(&tegra_aead_algs[i].alg.aead);
2040
2041 i = ARRAY_SIZE(tegra_aes_algs);
2042 err_aes:
2043 while (i--)
2044 crypto_engine_unregister_skcipher(&tegra_aes_algs[i].alg.skcipher);
2045
2046 return ret;
2047 }
2048
tegra_deinit_aes(struct tegra_se * se)2049 void tegra_deinit_aes(struct tegra_se *se)
2050 {
2051 int i;
2052
2053 for (i = 0; i < ARRAY_SIZE(tegra_aes_algs); i++)
2054 crypto_engine_unregister_skcipher(&tegra_aes_algs[i].alg.skcipher);
2055
2056 for (i = 0; i < ARRAY_SIZE(tegra_aead_algs); i++)
2057 crypto_engine_unregister_aead(&tegra_aead_algs[i].alg.aead);
2058
2059 for (i = 0; i < ARRAY_SIZE(tegra_cmac_algs); i++)
2060 crypto_engine_unregister_ahash(&tegra_cmac_algs[i].alg.ahash);
2061 }
2062