1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2019 - 2021
4 *
5 * Richard van Schagen <vschagen@icloud.com>
6 * Christian Marangi <ansuelsmth@gmail.com>
7 */
8
9 #include <crypto/aes.h>
10 #include <crypto/ctr.h>
11 #include <crypto/hmac.h>
12 #include <crypto/sha1.h>
13 #include <crypto/sha2.h>
14 #include <linux/kernel.h>
15 #include <linux/delay.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/scatterlist.h>
18
19 #include "eip93-cipher.h"
20 #include "eip93-hash.h"
21 #include "eip93-common.h"
22 #include "eip93-main.h"
23 #include "eip93-regs.h"
24
eip93_parse_ctrl_stat_err(struct eip93_device * eip93,int err)25 int eip93_parse_ctrl_stat_err(struct eip93_device *eip93, int err)
26 {
27 u32 ext_err;
28
29 if (!err)
30 return 0;
31
32 switch (err & ~EIP93_PE_CTRL_PE_EXT_ERR_CODE) {
33 case EIP93_PE_CTRL_PE_AUTH_ERR:
34 case EIP93_PE_CTRL_PE_PAD_ERR:
35 return -EBADMSG;
36 /* let software handle anti-replay errors */
37 case EIP93_PE_CTRL_PE_SEQNUM_ERR:
38 return 0;
39 case EIP93_PE_CTRL_PE_EXT_ERR:
40 break;
41 default:
42 dev_err(eip93->dev, "Unhandled error 0x%08x\n", err);
43 return -EINVAL;
44 }
45
46 /* Parse additional ext errors */
47 ext_err = FIELD_GET(EIP93_PE_CTRL_PE_EXT_ERR_CODE, err);
48 switch (ext_err) {
49 case EIP93_PE_CTRL_PE_EXT_ERR_BUS:
50 case EIP93_PE_CTRL_PE_EXT_ERR_PROCESSING:
51 return -EIO;
52 case EIP93_PE_CTRL_PE_EXT_ERR_DESC_OWNER:
53 return -EACCES;
54 case EIP93_PE_CTRL_PE_EXT_ERR_INVALID_CRYPTO_OP:
55 case EIP93_PE_CTRL_PE_EXT_ERR_INVALID_CRYPTO_ALGO:
56 case EIP93_PE_CTRL_PE_EXT_ERR_SPI:
57 return -EINVAL;
58 case EIP93_PE_CTRL_PE_EXT_ERR_ZERO_LENGTH:
59 case EIP93_PE_CTRL_PE_EXT_ERR_INVALID_PK_LENGTH:
60 case EIP93_PE_CTRL_PE_EXT_ERR_BLOCK_SIZE_ERR:
61 return -EBADMSG;
62 default:
63 dev_err(eip93->dev, "Unhandled ext error 0x%08x\n", ext_err);
64 return -EINVAL;
65 }
66 }
67
eip93_ring_next_wptr(struct eip93_device * eip93,struct eip93_desc_ring * ring)68 static void *eip93_ring_next_wptr(struct eip93_device *eip93,
69 struct eip93_desc_ring *ring)
70 {
71 void *ptr = ring->write;
72
73 if ((ring->write == ring->read - ring->offset) ||
74 (ring->read == ring->base && ring->write == ring->base_end))
75 return ERR_PTR(-ENOMEM);
76
77 if (ring->write == ring->base_end)
78 ring->write = ring->base;
79 else
80 ring->write += ring->offset;
81
82 return ptr;
83 }
84
eip93_ring_next_rptr(struct eip93_device * eip93,struct eip93_desc_ring * ring)85 static void *eip93_ring_next_rptr(struct eip93_device *eip93,
86 struct eip93_desc_ring *ring)
87 {
88 void *ptr = ring->read;
89
90 if (ring->write == ring->read)
91 return ERR_PTR(-ENOENT);
92
93 if (ring->read == ring->base_end)
94 ring->read = ring->base;
95 else
96 ring->read += ring->offset;
97
98 return ptr;
99 }
100
eip93_put_descriptor(struct eip93_device * eip93,struct eip93_descriptor * desc)101 int eip93_put_descriptor(struct eip93_device *eip93,
102 struct eip93_descriptor *desc)
103 {
104 struct eip93_descriptor *cdesc;
105 struct eip93_descriptor *rdesc;
106
107 rdesc = eip93_ring_next_wptr(eip93, &eip93->ring->rdr);
108 if (IS_ERR(rdesc))
109 return -ENOENT;
110
111 cdesc = eip93_ring_next_wptr(eip93, &eip93->ring->cdr);
112 if (IS_ERR(cdesc))
113 return -ENOENT;
114
115 memset(rdesc, 0, sizeof(struct eip93_descriptor));
116
117 memcpy(cdesc, desc, sizeof(struct eip93_descriptor));
118
119 return 0;
120 }
121
eip93_get_descriptor(struct eip93_device * eip93)122 void *eip93_get_descriptor(struct eip93_device *eip93)
123 {
124 struct eip93_descriptor *cdesc;
125 void *ptr;
126
127 cdesc = eip93_ring_next_rptr(eip93, &eip93->ring->cdr);
128 if (IS_ERR(cdesc))
129 return ERR_PTR(-ENOENT);
130
131 memset(cdesc, 0, sizeof(struct eip93_descriptor));
132
133 ptr = eip93_ring_next_rptr(eip93, &eip93->ring->rdr);
134 if (IS_ERR(ptr))
135 return ERR_PTR(-ENOENT);
136
137 return ptr;
138 }
139
eip93_free_sg_copy(const int len,struct scatterlist ** sg)140 static void eip93_free_sg_copy(const int len, struct scatterlist **sg)
141 {
142 if (!*sg || !len)
143 return;
144
145 free_pages((unsigned long)sg_virt(*sg), get_order(len));
146 kfree(*sg);
147 *sg = NULL;
148 }
149
eip93_make_sg_copy(struct scatterlist * src,struct scatterlist ** dst,const u32 len,const bool copy)150 static int eip93_make_sg_copy(struct scatterlist *src, struct scatterlist **dst,
151 const u32 len, const bool copy)
152 {
153 void *pages;
154
155 *dst = kmalloc_obj(**dst);
156 if (!*dst)
157 return -ENOMEM;
158
159 pages = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA,
160 get_order(len));
161 if (!pages) {
162 kfree(*dst);
163 *dst = NULL;
164 return -ENOMEM;
165 }
166
167 sg_init_table(*dst, 1);
168 sg_set_buf(*dst, pages, len);
169
170 /* copy only as requested */
171 if (copy)
172 sg_copy_to_buffer(src, sg_nents(src), pages, len);
173
174 return 0;
175 }
176
eip93_is_sg_aligned(struct scatterlist * sg,u32 len,const int blksize)177 static bool eip93_is_sg_aligned(struct scatterlist *sg, u32 len,
178 const int blksize)
179 {
180 int nents;
181
182 for (nents = 0; sg; sg = sg_next(sg), ++nents) {
183 if (!IS_ALIGNED(sg->offset, 4))
184 return false;
185
186 if (len <= sg->length) {
187 if (!IS_ALIGNED(len, blksize))
188 return false;
189
190 return true;
191 }
192
193 if (!IS_ALIGNED(sg->length, blksize))
194 return false;
195
196 len -= sg->length;
197 }
198 return false;
199 }
200
check_valid_request(struct eip93_cipher_reqctx * rctx)201 int check_valid_request(struct eip93_cipher_reqctx *rctx)
202 {
203 struct scatterlist *src = rctx->sg_src;
204 struct scatterlist *dst = rctx->sg_dst;
205 u32 textsize = rctx->textsize;
206 u32 authsize = rctx->authsize;
207 u32 blksize = rctx->blksize;
208 u32 totlen_src = rctx->assoclen + rctx->textsize;
209 u32 totlen_dst = rctx->assoclen + rctx->textsize;
210 u32 copy_len;
211 bool src_align, dst_align;
212 int src_nents, dst_nents;
213 int err = -EINVAL;
214
215 if (!IS_CTR(rctx->flags)) {
216 if (!IS_ALIGNED(textsize, blksize))
217 return err;
218 }
219
220 if (authsize) {
221 if (IS_ENCRYPT(rctx->flags))
222 totlen_dst += authsize;
223 else
224 totlen_src += authsize;
225 }
226
227 src_nents = sg_nents_for_len(src, totlen_src);
228 if (src_nents < 0)
229 return src_nents;
230
231 dst_nents = sg_nents_for_len(dst, totlen_dst);
232 if (dst_nents < 0)
233 return dst_nents;
234
235 if (src == dst) {
236 src_nents = max(src_nents, dst_nents);
237 dst_nents = src_nents;
238 if (unlikely((totlen_src || totlen_dst) && !src_nents))
239 return err;
240
241 } else {
242 if (unlikely(totlen_src && !src_nents))
243 return err;
244
245 if (unlikely(totlen_dst && !dst_nents))
246 return err;
247 }
248
249 if (authsize) {
250 if (dst_nents == 1 && src_nents == 1) {
251 src_align = eip93_is_sg_aligned(src, totlen_src, blksize);
252 if (src == dst)
253 dst_align = src_align;
254 else
255 dst_align = eip93_is_sg_aligned(dst, totlen_dst, blksize);
256 } else {
257 src_align = false;
258 dst_align = false;
259 }
260 } else {
261 src_align = eip93_is_sg_aligned(src, totlen_src, blksize);
262 if (src == dst)
263 dst_align = src_align;
264 else
265 dst_align = eip93_is_sg_aligned(dst, totlen_dst, blksize);
266 }
267
268 copy_len = max(totlen_src, totlen_dst);
269 if (!src_align) {
270 err = eip93_make_sg_copy(src, &rctx->sg_src, copy_len, true);
271 if (err)
272 return err;
273 }
274
275 if (!dst_align) {
276 err = eip93_make_sg_copy(dst, &rctx->sg_dst, copy_len, false);
277 if (err)
278 return err;
279 }
280
281 src_nents = sg_nents_for_len(rctx->sg_src, totlen_src);
282 if (src_nents < 0)
283 return src_nents;
284
285 dst_nents = sg_nents_for_len(rctx->sg_dst, totlen_dst);
286 if (dst_nents < 0)
287 return dst_nents;
288
289 rctx->src_nents = src_nents;
290 rctx->dst_nents = dst_nents;
291
292 return 0;
293 }
294
295 /*
296 * Set sa_record function:
297 * Even sa_record is set to "0", keep " = 0" for readability.
298 */
eip93_set_sa_record(struct sa_record * sa_record,const unsigned int keylen,const u32 flags)299 void eip93_set_sa_record(struct sa_record *sa_record, const unsigned int keylen,
300 const u32 flags)
301 {
302 /* Reset cmd word */
303 sa_record->sa_cmd0_word = 0;
304 sa_record->sa_cmd1_word = 0;
305
306 sa_record->sa_cmd0_word |= EIP93_SA_CMD_IV_FROM_STATE;
307 if (!IS_ECB(flags))
308 sa_record->sa_cmd0_word |= EIP93_SA_CMD_SAVE_IV;
309
310 sa_record->sa_cmd0_word |= EIP93_SA_CMD_OP_BASIC;
311
312 switch ((flags & EIP93_ALG_MASK)) {
313 case EIP93_ALG_AES:
314 sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_AES;
315 sa_record->sa_cmd1_word |= FIELD_PREP(EIP93_SA_CMD_AES_KEY_LENGTH,
316 keylen >> 3);
317 break;
318 case EIP93_ALG_3DES:
319 sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_3DES;
320 break;
321 case EIP93_ALG_DES:
322 sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_DES;
323 break;
324 default:
325 sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_NULL;
326 }
327
328 switch ((flags & EIP93_HASH_MASK)) {
329 case EIP93_HASH_SHA256:
330 sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_SHA256;
331 break;
332 case EIP93_HASH_SHA224:
333 sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_SHA224;
334 break;
335 case EIP93_HASH_SHA1:
336 sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_SHA1;
337 break;
338 case EIP93_HASH_MD5:
339 sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_MD5;
340 break;
341 default:
342 sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_NULL;
343 }
344
345 sa_record->sa_cmd0_word |= EIP93_SA_CMD_PAD_ZERO;
346
347 switch ((flags & EIP93_MODE_MASK)) {
348 case EIP93_MODE_CBC:
349 sa_record->sa_cmd1_word |= EIP93_SA_CMD_CHIPER_MODE_CBC;
350 break;
351 case EIP93_MODE_CTR:
352 sa_record->sa_cmd1_word |= EIP93_SA_CMD_CHIPER_MODE_CTR;
353 break;
354 case EIP93_MODE_ECB:
355 sa_record->sa_cmd1_word |= EIP93_SA_CMD_CHIPER_MODE_ECB;
356 break;
357 }
358
359 sa_record->sa_cmd0_word |= EIP93_SA_CMD_DIGEST_3WORD;
360 if (IS_HASH(flags)) {
361 sa_record->sa_cmd1_word |= EIP93_SA_CMD_COPY_PAD;
362 sa_record->sa_cmd1_word |= EIP93_SA_CMD_COPY_DIGEST;
363 }
364
365 if (IS_HMAC(flags)) {
366 sa_record->sa_cmd1_word |= EIP93_SA_CMD_HMAC;
367 sa_record->sa_cmd1_word |= EIP93_SA_CMD_COPY_HEADER;
368 }
369
370 sa_record->sa_spi = 0x0;
371 sa_record->sa_seqmum_mask[0] = 0xFFFFFFFF;
372 sa_record->sa_seqmum_mask[1] = 0x0;
373 }
374
375 /*
376 * Poor mans Scatter/gather function:
377 * Create a Descriptor for every segment to avoid copying buffers.
378 * For performance better to wait for hardware to perform multiple DMA
379 */
eip93_scatter_combine(struct eip93_device * eip93,struct eip93_cipher_reqctx * rctx,u32 datalen,u32 split,int offsetin)380 static int eip93_scatter_combine(struct eip93_device *eip93,
381 struct eip93_cipher_reqctx *rctx,
382 u32 datalen, u32 split, int offsetin)
383 {
384 struct eip93_descriptor *cdesc = rctx->cdesc;
385 struct scatterlist *sgsrc = rctx->sg_src;
386 struct scatterlist *sgdst = rctx->sg_dst;
387 unsigned int remainin = sg_dma_len(sgsrc);
388 unsigned int remainout = sg_dma_len(sgdst);
389 dma_addr_t saddr = sg_dma_address(sgsrc);
390 dma_addr_t daddr = sg_dma_address(sgdst);
391 dma_addr_t state_addr;
392 u32 src_addr, dst_addr, len, n;
393 bool nextin = false;
394 bool nextout = false;
395 int offsetout = 0;
396 int err;
397
398 if (IS_ECB(rctx->flags))
399 rctx->sa_state_base = 0;
400
401 if (split < datalen) {
402 state_addr = rctx->sa_state_ctr_base;
403 n = split;
404 } else {
405 state_addr = rctx->sa_state_base;
406 n = datalen;
407 }
408
409 do {
410 if (nextin) {
411 sgsrc = sg_next(sgsrc);
412 remainin = sg_dma_len(sgsrc);
413 if (remainin == 0)
414 continue;
415
416 saddr = sg_dma_address(sgsrc);
417 offsetin = 0;
418 nextin = false;
419 }
420
421 if (nextout) {
422 sgdst = sg_next(sgdst);
423 remainout = sg_dma_len(sgdst);
424 if (remainout == 0)
425 continue;
426
427 daddr = sg_dma_address(sgdst);
428 offsetout = 0;
429 nextout = false;
430 }
431 src_addr = saddr + offsetin;
432 dst_addr = daddr + offsetout;
433
434 if (remainin == remainout) {
435 len = remainin;
436 if (len > n) {
437 len = n;
438 remainin -= n;
439 remainout -= n;
440 offsetin += n;
441 offsetout += n;
442 } else {
443 nextin = true;
444 nextout = true;
445 }
446 } else if (remainin < remainout) {
447 len = remainin;
448 if (len > n) {
449 len = n;
450 remainin -= n;
451 remainout -= n;
452 offsetin += n;
453 offsetout += n;
454 } else {
455 offsetout += len;
456 remainout -= len;
457 nextin = true;
458 }
459 } else {
460 len = remainout;
461 if (len > n) {
462 len = n;
463 remainin -= n;
464 remainout -= n;
465 offsetin += n;
466 offsetout += n;
467 } else {
468 offsetin += len;
469 remainin -= len;
470 nextout = true;
471 }
472 }
473 n -= len;
474
475 cdesc->src_addr = src_addr;
476 cdesc->dst_addr = dst_addr;
477 cdesc->state_addr = state_addr;
478 cdesc->pe_length_word = FIELD_PREP(EIP93_PE_LENGTH_HOST_PE_READY,
479 EIP93_PE_LENGTH_HOST_READY);
480 cdesc->pe_length_word |= FIELD_PREP(EIP93_PE_LENGTH_LENGTH, len);
481
482 if (n == 0) {
483 n = datalen - split;
484 split = datalen;
485 state_addr = rctx->sa_state_base;
486 }
487
488 if (n == 0)
489 cdesc->user_id |= FIELD_PREP(EIP93_PE_USER_ID_DESC_FLAGS,
490 EIP93_DESC_LAST);
491
492 /*
493 * Loop - Delay - No need to rollback
494 * Maybe refine by slowing down at EIP93_RING_BUSY
495 */
496 again:
497 scoped_guard(spinlock_irqsave, &eip93->ring->write_lock)
498 err = eip93_put_descriptor(eip93, cdesc);
499 if (err) {
500 usleep_range(EIP93_RING_BUSY_DELAY,
501 EIP93_RING_BUSY_DELAY * 2);
502 goto again;
503 }
504 /* Writing new descriptor count starts DMA action */
505 writel(1, eip93->base + EIP93_REG_PE_CD_COUNT);
506 } while (n);
507
508 return -EINPROGRESS;
509 }
510
eip93_send_req(struct crypto_async_request * async,const u8 * reqiv,struct eip93_cipher_reqctx * rctx)511 int eip93_send_req(struct crypto_async_request *async,
512 const u8 *reqiv, struct eip93_cipher_reqctx *rctx)
513 {
514 struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(async->tfm);
515 struct eip93_device *eip93 = ctx->eip93;
516 struct scatterlist *src = rctx->sg_src;
517 struct scatterlist *dst = rctx->sg_dst;
518 struct sa_state *sa_state;
519 struct eip93_descriptor cdesc;
520 u32 flags = rctx->flags;
521 int offsetin = 0, err;
522 u32 datalen = rctx->assoclen + rctx->textsize;
523 u32 split = datalen;
524 u32 start, end, ctr, blocks;
525 u32 iv[AES_BLOCK_SIZE / sizeof(u32)];
526 int crypto_async_idr;
527
528 rctx->sa_state_ctr = NULL;
529 rctx->sa_state = NULL;
530
531 if (IS_ECB(flags))
532 goto skip_iv;
533
534 memcpy(iv, reqiv, rctx->ivsize);
535
536 rctx->sa_state = kzalloc_obj(*rctx->sa_state);
537 if (!rctx->sa_state)
538 return -ENOMEM;
539
540 sa_state = rctx->sa_state;
541
542 memcpy(sa_state->state_iv, iv, rctx->ivsize);
543 if (IS_RFC3686(flags)) {
544 sa_state->state_iv[0] = ctx->sa_nonce;
545 sa_state->state_iv[1] = iv[0];
546 sa_state->state_iv[2] = iv[1];
547 sa_state->state_iv[3] = (u32 __force)cpu_to_be32(0x1);
548 } else if (!IS_HMAC(flags) && IS_CTR(flags)) {
549 /* Compute data length. */
550 blocks = DIV_ROUND_UP(rctx->textsize, AES_BLOCK_SIZE);
551 ctr = be32_to_cpu((__be32 __force)iv[3]);
552 /* Check 32bit counter overflow. */
553 start = ctr;
554 end = start + blocks - 1;
555 if (end < start) {
556 split = AES_BLOCK_SIZE * -start;
557 /*
558 * Increment the counter manually to cope with
559 * the hardware counter overflow.
560 */
561 iv[3] = 0xffffffff;
562 crypto_inc((u8 *)iv, AES_BLOCK_SIZE);
563
564 rctx->sa_state_ctr = kzalloc_obj(*rctx->sa_state_ctr);
565 if (!rctx->sa_state_ctr) {
566 err = -ENOMEM;
567 goto free_sa_state;
568 }
569
570 memcpy(rctx->sa_state_ctr->state_iv, reqiv, rctx->ivsize);
571 memcpy(sa_state->state_iv, iv, rctx->ivsize);
572
573 rctx->sa_state_ctr_base = dma_map_single(eip93->dev, rctx->sa_state_ctr,
574 sizeof(*rctx->sa_state_ctr),
575 DMA_TO_DEVICE);
576 err = dma_mapping_error(eip93->dev, rctx->sa_state_ctr_base);
577 if (err)
578 goto free_sa_state_ctr;
579 }
580 }
581
582 rctx->sa_state_base = dma_map_single(eip93->dev, rctx->sa_state,
583 sizeof(*rctx->sa_state), DMA_TO_DEVICE);
584 err = dma_mapping_error(eip93->dev, rctx->sa_state_base);
585 if (err)
586 goto free_sa_state_ctr_dma;
587
588 skip_iv:
589
590 cdesc.pe_ctrl_stat_word = FIELD_PREP(EIP93_PE_CTRL_PE_READY_DES_TRING_OWN,
591 EIP93_PE_CTRL_HOST_READY);
592 cdesc.sa_addr = rctx->sa_record_base;
593 cdesc.arc4_addr = 0;
594
595 scoped_guard(spinlock_bh, &eip93->ring->idr_lock)
596 crypto_async_idr = idr_alloc(&eip93->ring->crypto_async_idr, async, 0,
597 EIP93_RING_NUM - 1, GFP_ATOMIC);
598
599 cdesc.user_id = FIELD_PREP(EIP93_PE_USER_ID_CRYPTO_IDR, (u16)crypto_async_idr) |
600 FIELD_PREP(EIP93_PE_USER_ID_DESC_FLAGS, rctx->desc_flags);
601
602 rctx->cdesc = &cdesc;
603
604 /* map DMA_BIDIRECTIONAL to invalidate cache on destination
605 * implies __dma_cache_wback_inv
606 */
607 if (!dma_map_sg(eip93->dev, dst, rctx->dst_nents, DMA_BIDIRECTIONAL)) {
608 err = -ENOMEM;
609 goto free_sa_state_ctr_dma;
610 }
611
612 if (src != dst &&
613 !dma_map_sg(eip93->dev, src, rctx->src_nents, DMA_TO_DEVICE)) {
614 err = -ENOMEM;
615 goto free_sg_dma;
616 }
617
618 return eip93_scatter_combine(eip93, rctx, datalen, split, offsetin);
619
620 free_sg_dma:
621 dma_unmap_sg(eip93->dev, dst, rctx->dst_nents, DMA_BIDIRECTIONAL);
622 free_sa_state_ctr_dma:
623 if (rctx->sa_state_ctr)
624 dma_unmap_single(eip93->dev, rctx->sa_state_ctr_base,
625 sizeof(*rctx->sa_state_ctr),
626 DMA_TO_DEVICE);
627 free_sa_state_ctr:
628 kfree(rctx->sa_state_ctr);
629 if (rctx->sa_state)
630 dma_unmap_single(eip93->dev, rctx->sa_state_base,
631 sizeof(*rctx->sa_state),
632 DMA_TO_DEVICE);
633 free_sa_state:
634 kfree(rctx->sa_state);
635
636 return err;
637 }
638
eip93_unmap_dma(struct eip93_device * eip93,struct eip93_cipher_reqctx * rctx,struct scatterlist * reqsrc,struct scatterlist * reqdst)639 void eip93_unmap_dma(struct eip93_device *eip93, struct eip93_cipher_reqctx *rctx,
640 struct scatterlist *reqsrc, struct scatterlist *reqdst)
641 {
642 u32 len = rctx->assoclen + rctx->textsize;
643 u32 authsize = rctx->authsize;
644 u32 flags = rctx->flags;
645 u32 *otag;
646 int i;
647
648 if (rctx->sg_src == rctx->sg_dst) {
649 dma_unmap_sg(eip93->dev, rctx->sg_dst, rctx->dst_nents,
650 DMA_BIDIRECTIONAL);
651 goto process_tag;
652 }
653
654 dma_unmap_sg(eip93->dev, rctx->sg_src, rctx->src_nents,
655 DMA_TO_DEVICE);
656
657 if (rctx->sg_src != reqsrc)
658 eip93_free_sg_copy(len + rctx->authsize, &rctx->sg_src);
659
660 dma_unmap_sg(eip93->dev, rctx->sg_dst, rctx->dst_nents,
661 DMA_BIDIRECTIONAL);
662
663 /* SHA tags need conversion from net-to-host */
664 process_tag:
665 if (IS_DECRYPT(flags))
666 authsize = 0;
667
668 if (authsize) {
669 if (!IS_HASH_MD5(flags)) {
670 otag = sg_virt(rctx->sg_dst) + len;
671 for (i = 0; i < (authsize / 4); i++)
672 otag[i] = be32_to_cpu((__be32 __force)otag[i]);
673 }
674 }
675
676 if (rctx->sg_dst != reqdst) {
677 sg_copy_from_buffer(reqdst, sg_nents(reqdst),
678 sg_virt(rctx->sg_dst), len + authsize);
679 eip93_free_sg_copy(len + rctx->authsize, &rctx->sg_dst);
680 }
681 }
682
eip93_handle_result(struct eip93_device * eip93,struct eip93_cipher_reqctx * rctx,u8 * reqiv)683 void eip93_handle_result(struct eip93_device *eip93, struct eip93_cipher_reqctx *rctx,
684 u8 *reqiv)
685 {
686 if (rctx->sa_state_ctr)
687 dma_unmap_single(eip93->dev, rctx->sa_state_ctr_base,
688 sizeof(*rctx->sa_state_ctr),
689 DMA_FROM_DEVICE);
690
691 if (rctx->sa_state)
692 dma_unmap_single(eip93->dev, rctx->sa_state_base,
693 sizeof(*rctx->sa_state),
694 DMA_FROM_DEVICE);
695
696 if (!IS_ECB(rctx->flags))
697 memcpy(reqiv, rctx->sa_state->state_iv, rctx->ivsize);
698
699 kfree(rctx->sa_state_ctr);
700 kfree(rctx->sa_state);
701 }
702
eip93_hmac_setkey(u32 ctx_flags,const u8 * key,unsigned int keylen,unsigned int hashlen,u8 * dest_ipad,u8 * dest_opad,bool skip_ipad)703 int eip93_hmac_setkey(u32 ctx_flags, const u8 *key, unsigned int keylen,
704 unsigned int hashlen, u8 *dest_ipad, u8 *dest_opad,
705 bool skip_ipad)
706 {
707 u8 ipad[SHA256_BLOCK_SIZE], opad[SHA256_BLOCK_SIZE];
708 struct crypto_ahash *ahash_tfm;
709 struct eip93_hash_reqctx *rctx;
710 struct ahash_request *req;
711 DECLARE_CRYPTO_WAIT(wait);
712 struct scatterlist sg[1];
713 const char *alg_name;
714 int i, ret;
715
716 switch (ctx_flags & EIP93_HASH_MASK) {
717 case EIP93_HASH_SHA256:
718 alg_name = "sha256-eip93";
719 break;
720 case EIP93_HASH_SHA224:
721 alg_name = "sha224-eip93";
722 break;
723 case EIP93_HASH_SHA1:
724 alg_name = "sha1-eip93";
725 break;
726 case EIP93_HASH_MD5:
727 alg_name = "md5-eip93";
728 break;
729 default: /* Impossible */
730 return -EINVAL;
731 }
732
733 ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0);
734 if (IS_ERR(ahash_tfm))
735 return PTR_ERR(ahash_tfm);
736
737 req = ahash_request_alloc(ahash_tfm, GFP_ATOMIC);
738 if (!req) {
739 ret = -ENOMEM;
740 goto err_ahash;
741 }
742
743 rctx = ahash_request_ctx_dma(req);
744 crypto_init_wait(&wait);
745 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
746 crypto_req_done, &wait);
747
748 /* Hash the key if > SHA256_BLOCK_SIZE */
749 if (keylen > SHA256_BLOCK_SIZE) {
750 sg_init_one(&sg[0], key, keylen);
751
752 ahash_request_set_crypt(req, sg, ipad, keylen);
753 ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
754 if (ret)
755 goto err_req;
756
757 keylen = hashlen;
758 } else {
759 memcpy(ipad, key, keylen);
760 }
761
762 /* Copy to opad */
763 memset(ipad + keylen, 0, SHA256_BLOCK_SIZE - keylen);
764 memcpy(opad, ipad, SHA256_BLOCK_SIZE);
765
766 /* Pad with HMAC constants */
767 for (i = 0; i < SHA256_BLOCK_SIZE; i++) {
768 ipad[i] ^= HMAC_IPAD_VALUE;
769 opad[i] ^= HMAC_OPAD_VALUE;
770 }
771
772 if (skip_ipad) {
773 memcpy(dest_ipad, ipad, SHA256_BLOCK_SIZE);
774 } else {
775 /* Hash ipad */
776 sg_init_one(&sg[0], ipad, SHA256_BLOCK_SIZE);
777 ahash_request_set_crypt(req, sg, dest_ipad, SHA256_BLOCK_SIZE);
778 ret = crypto_ahash_init(req);
779 if (ret)
780 goto err_req;
781
782 /* Disable HASH_FINALIZE for ipad hash */
783 rctx->partial_hash = true;
784
785 ret = crypto_wait_req(crypto_ahash_finup(req), &wait);
786 if (ret)
787 goto err_req;
788 }
789
790 /* Hash opad */
791 sg_init_one(&sg[0], opad, SHA256_BLOCK_SIZE);
792 ahash_request_set_crypt(req, sg, dest_opad, SHA256_BLOCK_SIZE);
793 ret = crypto_ahash_init(req);
794 if (ret)
795 goto err_req;
796
797 /* Disable HASH_FINALIZE for opad hash */
798 rctx->partial_hash = true;
799
800 ret = crypto_wait_req(crypto_ahash_finup(req), &wait);
801 if (ret)
802 goto err_req;
803
804 if (!IS_HASH_MD5(ctx_flags)) {
805 for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(u32); i++) {
806 u32 *ipad_hash = (u32 *)dest_ipad;
807 u32 *opad_hash = (u32 *)dest_opad;
808
809 if (!skip_ipad)
810 ipad_hash[i] = (u32 __force)cpu_to_be32(ipad_hash[i]);
811 opad_hash[i] = (u32 __force)cpu_to_be32(opad_hash[i]);
812 }
813 }
814
815 err_req:
816 ahash_request_free(req);
817 err_ahash:
818 crypto_free_ahash(ahash_tfm);
819
820 return ret;
821 }
822