1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2014 Imagination Technologies
4 * Authors: Will Thomas, James Hartley
5 *
6 * Interface structure taken from omap-sham driver
7 */
8
9 #include <linux/clk.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmaengine.h>
12 #include <linux/interrupt.h>
13 #include <linux/io.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/platform_device.h>
17 #include <linux/scatterlist.h>
18
19 #include <crypto/internal/hash.h>
20 #include <crypto/md5.h>
21 #include <crypto/sha1.h>
22 #include <crypto/sha2.h>
23
24 #define CR_RESET 0
25 #define CR_RESET_SET 1
26 #define CR_RESET_UNSET 0
27
28 #define CR_MESSAGE_LENGTH_H 0x4
29 #define CR_MESSAGE_LENGTH_L 0x8
30
31 #define CR_CONTROL 0xc
32 #define CR_CONTROL_BYTE_ORDER_3210 0
33 #define CR_CONTROL_BYTE_ORDER_0123 1
34 #define CR_CONTROL_BYTE_ORDER_2310 2
35 #define CR_CONTROL_BYTE_ORDER_1032 3
36 #define CR_CONTROL_BYTE_ORDER_SHIFT 8
37 #define CR_CONTROL_ALGO_MD5 0
38 #define CR_CONTROL_ALGO_SHA1 1
39 #define CR_CONTROL_ALGO_SHA224 2
40 #define CR_CONTROL_ALGO_SHA256 3
41
42 #define CR_INTSTAT 0x10
43 #define CR_INTENAB 0x14
44 #define CR_INTCLEAR 0x18
45 #define CR_INT_RESULTS_AVAILABLE BIT(0)
46 #define CR_INT_NEW_RESULTS_SET BIT(1)
47 #define CR_INT_RESULT_READ_ERR BIT(2)
48 #define CR_INT_MESSAGE_WRITE_ERROR BIT(3)
49 #define CR_INT_STATUS BIT(8)
50
51 #define CR_RESULT_QUEUE 0x1c
52 #define CR_RSD0 0x40
53 #define CR_CORE_REV 0x50
54 #define CR_CORE_DES1 0x60
55 #define CR_CORE_DES2 0x70
56
57 #define DRIVER_FLAGS_BUSY BIT(0)
58 #define DRIVER_FLAGS_FINAL BIT(1)
59 #define DRIVER_FLAGS_DMA_ACTIVE BIT(2)
60 #define DRIVER_FLAGS_OUTPUT_READY BIT(3)
61 #define DRIVER_FLAGS_INIT BIT(4)
62 #define DRIVER_FLAGS_CPU BIT(5)
63 #define DRIVER_FLAGS_DMA_READY BIT(6)
64 #define DRIVER_FLAGS_ERROR BIT(7)
65 #define DRIVER_FLAGS_SG BIT(8)
66 #define DRIVER_FLAGS_SHA1 BIT(18)
67 #define DRIVER_FLAGS_SHA224 BIT(19)
68 #define DRIVER_FLAGS_SHA256 BIT(20)
69 #define DRIVER_FLAGS_MD5 BIT(21)
70
71 #define IMG_HASH_QUEUE_LENGTH 20
72 #define IMG_HASH_DMA_BURST 4
73 #define IMG_HASH_DMA_THRESHOLD 64
74
75 #ifdef __LITTLE_ENDIAN
76 #define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_3210
77 #else
78 #define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_0123
79 #endif
80
81 struct img_hash_dev;
82
83 struct img_hash_request_ctx {
84 struct img_hash_dev *hdev;
85 u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32));
86 unsigned long flags;
87 size_t digsize;
88
89 dma_addr_t dma_addr;
90 size_t dma_ct;
91
92 /* sg root */
93 struct scatterlist *sgfirst;
94 /* walk state */
95 struct scatterlist *sg;
96 size_t nents;
97 size_t offset;
98 unsigned int total;
99 size_t sent;
100
101 unsigned long op;
102
103 size_t bufcnt;
104 struct ahash_request fallback_req;
105
106 /* Zero length buffer must remain last member of struct */
107 u8 buffer[] __aligned(sizeof(u32));
108 };
109
110 struct img_hash_ctx {
111 struct img_hash_dev *hdev;
112 unsigned long flags;
113 struct crypto_ahash *fallback;
114 };
115
116 struct img_hash_dev {
117 struct list_head list;
118 struct device *dev;
119 struct clk *hash_clk;
120 struct clk *sys_clk;
121 void __iomem *io_base;
122
123 phys_addr_t bus_addr;
124 void __iomem *cpu_addr;
125
126 spinlock_t lock;
127 int err;
128 struct tasklet_struct done_task;
129 struct tasklet_struct dma_task;
130
131 unsigned long flags;
132 struct crypto_queue queue;
133 struct ahash_request *req;
134
135 struct dma_chan *dma_lch;
136 };
137
138 struct img_hash_drv {
139 struct list_head dev_list;
140 spinlock_t lock;
141 };
142
143 static struct img_hash_drv img_hash = {
144 .dev_list = LIST_HEAD_INIT(img_hash.dev_list),
145 .lock = __SPIN_LOCK_UNLOCKED(img_hash.lock),
146 };
147
img_hash_read(struct img_hash_dev * hdev,u32 offset)148 static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset)
149 {
150 return readl_relaxed(hdev->io_base + offset);
151 }
152
img_hash_write(struct img_hash_dev * hdev,u32 offset,u32 value)153 static inline void img_hash_write(struct img_hash_dev *hdev,
154 u32 offset, u32 value)
155 {
156 writel_relaxed(value, hdev->io_base + offset);
157 }
158
img_hash_read_result_queue(struct img_hash_dev * hdev)159 static inline __be32 img_hash_read_result_queue(struct img_hash_dev *hdev)
160 {
161 return cpu_to_be32(img_hash_read(hdev, CR_RESULT_QUEUE));
162 }
163
img_hash_start(struct img_hash_dev * hdev,bool dma)164 static void img_hash_start(struct img_hash_dev *hdev, bool dma)
165 {
166 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
167 u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT;
168
169 if (ctx->flags & DRIVER_FLAGS_MD5)
170 cr |= CR_CONTROL_ALGO_MD5;
171 else if (ctx->flags & DRIVER_FLAGS_SHA1)
172 cr |= CR_CONTROL_ALGO_SHA1;
173 else if (ctx->flags & DRIVER_FLAGS_SHA224)
174 cr |= CR_CONTROL_ALGO_SHA224;
175 else if (ctx->flags & DRIVER_FLAGS_SHA256)
176 cr |= CR_CONTROL_ALGO_SHA256;
177 dev_dbg(hdev->dev, "Starting hash process\n");
178 img_hash_write(hdev, CR_CONTROL, cr);
179
180 /*
181 * The hardware block requires two cycles between writing the control
182 * register and writing the first word of data in non DMA mode, to
183 * ensure the first data write is not grouped in burst with the control
184 * register write a read is issued to 'flush' the bus.
185 */
186 if (!dma)
187 img_hash_read(hdev, CR_CONTROL);
188 }
189
img_hash_xmit_cpu(struct img_hash_dev * hdev,const u8 * buf,size_t length,int final)190 static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf,
191 size_t length, int final)
192 {
193 u32 count, len32;
194 const u32 *buffer = (const u32 *)buf;
195
196 dev_dbg(hdev->dev, "xmit_cpu: length: %zu bytes\n", length);
197
198 if (final)
199 hdev->flags |= DRIVER_FLAGS_FINAL;
200
201 len32 = DIV_ROUND_UP(length, sizeof(u32));
202
203 for (count = 0; count < len32; count++)
204 writel_relaxed(buffer[count], hdev->cpu_addr);
205
206 return -EINPROGRESS;
207 }
208
img_hash_dma_callback(void * data)209 static void img_hash_dma_callback(void *data)
210 {
211 struct img_hash_dev *hdev = data;
212 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
213
214 if (ctx->bufcnt) {
215 img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0);
216 ctx->bufcnt = 0;
217 }
218 if (ctx->sg)
219 tasklet_schedule(&hdev->dma_task);
220 }
221
img_hash_xmit_dma(struct img_hash_dev * hdev,struct scatterlist * sg)222 static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg)
223 {
224 struct dma_async_tx_descriptor *desc;
225 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
226
227 ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
228 if (ctx->dma_ct == 0) {
229 dev_err(hdev->dev, "Invalid DMA sg\n");
230 hdev->err = -EINVAL;
231 return -EINVAL;
232 }
233
234 desc = dmaengine_prep_slave_sg(hdev->dma_lch,
235 sg,
236 ctx->dma_ct,
237 DMA_MEM_TO_DEV,
238 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
239 if (!desc) {
240 dev_err(hdev->dev, "Null DMA descriptor\n");
241 hdev->err = -EINVAL;
242 dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
243 return -EINVAL;
244 }
245 desc->callback = img_hash_dma_callback;
246 desc->callback_param = hdev;
247 dmaengine_submit(desc);
248 dma_async_issue_pending(hdev->dma_lch);
249
250 return 0;
251 }
252
img_hash_write_via_cpu(struct img_hash_dev * hdev)253 static int img_hash_write_via_cpu(struct img_hash_dev *hdev)
254 {
255 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
256
257 ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg),
258 ctx->buffer, hdev->req->nbytes);
259
260 ctx->total = hdev->req->nbytes;
261 ctx->bufcnt = 0;
262
263 hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL);
264
265 img_hash_start(hdev, false);
266
267 return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1);
268 }
269
img_hash_finish(struct ahash_request * req)270 static int img_hash_finish(struct ahash_request *req)
271 {
272 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
273
274 if (!req->result)
275 return -EINVAL;
276
277 memcpy(req->result, ctx->digest, ctx->digsize);
278
279 return 0;
280 }
281
img_hash_copy_hash(struct ahash_request * req)282 static void img_hash_copy_hash(struct ahash_request *req)
283 {
284 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
285 __be32 *hash = (__be32 *)ctx->digest;
286 int i;
287
288 for (i = (ctx->digsize / sizeof(*hash)) - 1; i >= 0; i--)
289 hash[i] = img_hash_read_result_queue(ctx->hdev);
290 }
291
img_hash_finish_req(struct ahash_request * req,int err)292 static void img_hash_finish_req(struct ahash_request *req, int err)
293 {
294 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
295 struct img_hash_dev *hdev = ctx->hdev;
296
297 if (!err) {
298 img_hash_copy_hash(req);
299 if (DRIVER_FLAGS_FINAL & hdev->flags)
300 err = img_hash_finish(req);
301 } else {
302 dev_warn(hdev->dev, "Hash failed with error %d\n", err);
303 ctx->flags |= DRIVER_FLAGS_ERROR;
304 }
305
306 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY |
307 DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL);
308
309 if (req->base.complete)
310 ahash_request_complete(req, err);
311 }
312
img_hash_write_via_dma(struct img_hash_dev * hdev)313 static int img_hash_write_via_dma(struct img_hash_dev *hdev)
314 {
315 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
316
317 img_hash_start(hdev, true);
318
319 dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total);
320
321 if (!ctx->total)
322 hdev->flags |= DRIVER_FLAGS_FINAL;
323
324 hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL;
325
326 tasklet_schedule(&hdev->dma_task);
327
328 return -EINPROGRESS;
329 }
330
img_hash_dma_init(struct img_hash_dev * hdev)331 static int img_hash_dma_init(struct img_hash_dev *hdev)
332 {
333 struct dma_slave_config dma_conf;
334 int err;
335
336 hdev->dma_lch = dma_request_chan(hdev->dev, "tx");
337 if (IS_ERR(hdev->dma_lch)) {
338 dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n");
339 return PTR_ERR(hdev->dma_lch);
340 }
341 dma_conf.direction = DMA_MEM_TO_DEV;
342 dma_conf.dst_addr = hdev->bus_addr;
343 dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
344 dma_conf.dst_maxburst = IMG_HASH_DMA_BURST;
345 dma_conf.device_fc = false;
346
347 err = dmaengine_slave_config(hdev->dma_lch, &dma_conf);
348 if (err) {
349 dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
350 dma_release_channel(hdev->dma_lch);
351 return err;
352 }
353
354 return 0;
355 }
356
img_hash_dma_task(unsigned long d)357 static void img_hash_dma_task(unsigned long d)
358 {
359 struct img_hash_dev *hdev = (struct img_hash_dev *)d;
360 struct img_hash_request_ctx *ctx;
361 u8 *addr;
362 size_t nbytes, bleft, wsend, len, tbc;
363 struct scatterlist tsg;
364
365 if (!hdev->req)
366 return;
367
368 ctx = ahash_request_ctx(hdev->req);
369 if (!ctx->sg)
370 return;
371
372 addr = sg_virt(ctx->sg);
373 nbytes = ctx->sg->length - ctx->offset;
374
375 /*
376 * The hash accelerator does not support a data valid mask. This means
377 * that if each dma (i.e. per page) is not a multiple of 4 bytes, the
378 * padding bytes in the last word written by that dma would erroneously
379 * be included in the hash. To avoid this we round down the transfer,
380 * and add the excess to the start of the next dma. It does not matter
381 * that the final dma may not be a multiple of 4 bytes as the hashing
382 * block is programmed to accept the correct number of bytes.
383 */
384
385 bleft = nbytes % 4;
386 wsend = (nbytes / 4);
387
388 if (wsend) {
389 sg_init_one(&tsg, addr + ctx->offset, wsend * 4);
390 if (img_hash_xmit_dma(hdev, &tsg)) {
391 dev_err(hdev->dev, "DMA failed, falling back to CPU");
392 ctx->flags |= DRIVER_FLAGS_CPU;
393 hdev->err = 0;
394 img_hash_xmit_cpu(hdev, addr + ctx->offset,
395 wsend * 4, 0);
396 ctx->sent += wsend * 4;
397 wsend = 0;
398 } else {
399 ctx->sent += wsend * 4;
400 }
401 }
402
403 if (bleft) {
404 ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
405 ctx->buffer, bleft, ctx->sent);
406 tbc = 0;
407 ctx->sg = sg_next(ctx->sg);
408 while (ctx->sg && (ctx->bufcnt < 4)) {
409 len = ctx->sg->length;
410 if (likely(len > (4 - ctx->bufcnt)))
411 len = 4 - ctx->bufcnt;
412 tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
413 ctx->buffer + ctx->bufcnt, len,
414 ctx->sent + ctx->bufcnt);
415 ctx->bufcnt += tbc;
416 if (tbc >= ctx->sg->length) {
417 ctx->sg = sg_next(ctx->sg);
418 tbc = 0;
419 }
420 }
421
422 ctx->sent += ctx->bufcnt;
423 ctx->offset = tbc;
424
425 if (!wsend)
426 img_hash_dma_callback(hdev);
427 } else {
428 ctx->offset = 0;
429 ctx->sg = sg_next(ctx->sg);
430 }
431 }
432
img_hash_write_via_dma_stop(struct img_hash_dev * hdev)433 static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev)
434 {
435 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
436
437 if (ctx->flags & DRIVER_FLAGS_SG)
438 dma_unmap_sg(hdev->dev, ctx->sg, 1, DMA_TO_DEVICE);
439
440 return 0;
441 }
442
img_hash_process_data(struct img_hash_dev * hdev)443 static int img_hash_process_data(struct img_hash_dev *hdev)
444 {
445 struct ahash_request *req = hdev->req;
446 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
447 int err = 0;
448
449 ctx->bufcnt = 0;
450
451 if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) {
452 dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n",
453 req->nbytes);
454 err = img_hash_write_via_dma(hdev);
455 } else {
456 dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n",
457 req->nbytes);
458 err = img_hash_write_via_cpu(hdev);
459 }
460 return err;
461 }
462
img_hash_hw_init(struct img_hash_dev * hdev)463 static int img_hash_hw_init(struct img_hash_dev *hdev)
464 {
465 unsigned long long nbits;
466 u32 u, l;
467
468 img_hash_write(hdev, CR_RESET, CR_RESET_SET);
469 img_hash_write(hdev, CR_RESET, CR_RESET_UNSET);
470 img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET);
471
472 nbits = (u64)hdev->req->nbytes << 3;
473 u = nbits >> 32;
474 l = nbits;
475 img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u);
476 img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l);
477
478 if (!(DRIVER_FLAGS_INIT & hdev->flags)) {
479 hdev->flags |= DRIVER_FLAGS_INIT;
480 hdev->err = 0;
481 }
482 dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits);
483 return 0;
484 }
485
img_hash_init(struct ahash_request * req)486 static int img_hash_init(struct ahash_request *req)
487 {
488 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
489 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
490 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
491
492 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
493 ahash_request_set_callback(&rctx->fallback_req,
494 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
495 req->base.complete, req->base.data);
496
497 return crypto_ahash_init(&rctx->fallback_req);
498 }
499
img_hash_handle_queue(struct img_hash_dev * hdev,struct ahash_request * req)500 static int img_hash_handle_queue(struct img_hash_dev *hdev,
501 struct ahash_request *req)
502 {
503 struct crypto_async_request *async_req, *backlog;
504 struct img_hash_request_ctx *ctx;
505 unsigned long flags;
506 int err = 0, res = 0;
507
508 spin_lock_irqsave(&hdev->lock, flags);
509
510 if (req)
511 res = ahash_enqueue_request(&hdev->queue, req);
512
513 if (DRIVER_FLAGS_BUSY & hdev->flags) {
514 spin_unlock_irqrestore(&hdev->lock, flags);
515 return res;
516 }
517
518 backlog = crypto_get_backlog(&hdev->queue);
519 async_req = crypto_dequeue_request(&hdev->queue);
520 if (async_req)
521 hdev->flags |= DRIVER_FLAGS_BUSY;
522
523 spin_unlock_irqrestore(&hdev->lock, flags);
524
525 if (!async_req)
526 return res;
527
528 if (backlog)
529 crypto_request_complete(backlog, -EINPROGRESS);
530
531 req = ahash_request_cast(async_req);
532 hdev->req = req;
533
534 ctx = ahash_request_ctx(req);
535
536 dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n",
537 ctx->op, req->nbytes);
538
539 err = img_hash_hw_init(hdev);
540
541 if (!err)
542 err = img_hash_process_data(hdev);
543
544 if (err != -EINPROGRESS) {
545 /* done_task will not finish so do it here */
546 img_hash_finish_req(req, err);
547 }
548 return res;
549 }
550
img_hash_update(struct ahash_request * req)551 static int img_hash_update(struct ahash_request *req)
552 {
553 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
554 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
555 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
556
557 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
558 ahash_request_set_callback(&rctx->fallback_req,
559 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
560 req->base.complete, req->base.data);
561 ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes);
562
563 return crypto_ahash_update(&rctx->fallback_req);
564 }
565
img_hash_final(struct ahash_request * req)566 static int img_hash_final(struct ahash_request *req)
567 {
568 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
569 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
570 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
571
572 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
573 ahash_request_set_callback(&rctx->fallback_req,
574 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
575 req->base.complete, req->base.data);
576 ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0);
577
578 return crypto_ahash_final(&rctx->fallback_req);
579 }
580
img_hash_finup(struct ahash_request * req)581 static int img_hash_finup(struct ahash_request *req)
582 {
583 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
584 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
585 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
586
587 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
588 ahash_request_set_callback(&rctx->fallback_req,
589 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
590 req->base.complete, req->base.data);
591 ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result,
592 req->nbytes);
593
594
595 return crypto_ahash_finup(&rctx->fallback_req);
596 }
597
img_hash_import(struct ahash_request * req,const void * in)598 static int img_hash_import(struct ahash_request *req, const void *in)
599 {
600 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
601 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
602 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
603
604 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
605 ahash_request_set_callback(&rctx->fallback_req,
606 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
607 req->base.complete, req->base.data);
608
609 return crypto_ahash_import(&rctx->fallback_req, in);
610 }
611
img_hash_export(struct ahash_request * req,void * out)612 static int img_hash_export(struct ahash_request *req, void *out)
613 {
614 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
615 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
616 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
617
618 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
619 ahash_request_set_callback(&rctx->fallback_req,
620 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
621 req->base.complete, req->base.data);
622
623 return crypto_ahash_export(&rctx->fallback_req, out);
624 }
625
img_hash_digest(struct ahash_request * req)626 static int img_hash_digest(struct ahash_request *req)
627 {
628 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
629 struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm);
630 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
631
632 spin_lock(&img_hash.lock);
633 if (!tctx->hdev)
634 tctx->hdev = list_first_entry_or_null(&img_hash.dev_list,
635 struct img_hash_dev, list);
636 ctx->hdev = tctx->hdev;
637 spin_unlock(&img_hash.lock);
638
639 ctx->flags = 0;
640 ctx->digsize = crypto_ahash_digestsize(tfm);
641
642 switch (ctx->digsize) {
643 case SHA1_DIGEST_SIZE:
644 ctx->flags |= DRIVER_FLAGS_SHA1;
645 break;
646 case SHA256_DIGEST_SIZE:
647 ctx->flags |= DRIVER_FLAGS_SHA256;
648 break;
649 case SHA224_DIGEST_SIZE:
650 ctx->flags |= DRIVER_FLAGS_SHA224;
651 break;
652 case MD5_DIGEST_SIZE:
653 ctx->flags |= DRIVER_FLAGS_MD5;
654 break;
655 default:
656 return -EINVAL;
657 }
658
659 ctx->bufcnt = 0;
660 ctx->offset = 0;
661 ctx->sent = 0;
662 ctx->total = req->nbytes;
663 ctx->sg = req->src;
664 ctx->sgfirst = req->src;
665 ctx->nents = sg_nents(ctx->sg);
666
667 return img_hash_handle_queue(ctx->hdev, req);
668 }
669
img_hash_cra_init(struct crypto_tfm * tfm,const char * alg_name)670 static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name)
671 {
672 struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm);
673
674 ctx->fallback = crypto_alloc_ahash(alg_name, 0,
675 CRYPTO_ALG_NEED_FALLBACK);
676 if (IS_ERR(ctx->fallback)) {
677 pr_err("img_hash: Could not load fallback driver.\n");
678 return PTR_ERR(ctx->fallback);
679 }
680 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
681 sizeof(struct img_hash_request_ctx) +
682 crypto_ahash_reqsize(ctx->fallback) +
683 IMG_HASH_DMA_THRESHOLD);
684
685 return 0;
686 }
687
img_hash_cra_md5_init(struct crypto_tfm * tfm)688 static int img_hash_cra_md5_init(struct crypto_tfm *tfm)
689 {
690 return img_hash_cra_init(tfm, "md5-lib");
691 }
692
img_hash_cra_sha1_init(struct crypto_tfm * tfm)693 static int img_hash_cra_sha1_init(struct crypto_tfm *tfm)
694 {
695 return img_hash_cra_init(tfm, "sha1-lib");
696 }
697
img_hash_cra_sha224_init(struct crypto_tfm * tfm)698 static int img_hash_cra_sha224_init(struct crypto_tfm *tfm)
699 {
700 return img_hash_cra_init(tfm, "sha224-lib");
701 }
702
img_hash_cra_sha256_init(struct crypto_tfm * tfm)703 static int img_hash_cra_sha256_init(struct crypto_tfm *tfm)
704 {
705 return img_hash_cra_init(tfm, "sha256-lib");
706 }
707
img_hash_cra_exit(struct crypto_tfm * tfm)708 static void img_hash_cra_exit(struct crypto_tfm *tfm)
709 {
710 struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm);
711
712 crypto_free_ahash(tctx->fallback);
713 }
714
img_irq_handler(int irq,void * dev_id)715 static irqreturn_t img_irq_handler(int irq, void *dev_id)
716 {
717 struct img_hash_dev *hdev = dev_id;
718 u32 reg;
719
720 reg = img_hash_read(hdev, CR_INTSTAT);
721 img_hash_write(hdev, CR_INTCLEAR, reg);
722
723 if (reg & CR_INT_NEW_RESULTS_SET) {
724 dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n");
725 if (DRIVER_FLAGS_BUSY & hdev->flags) {
726 hdev->flags |= DRIVER_FLAGS_OUTPUT_READY;
727 if (!(DRIVER_FLAGS_CPU & hdev->flags))
728 hdev->flags |= DRIVER_FLAGS_DMA_READY;
729 tasklet_schedule(&hdev->done_task);
730 } else {
731 dev_warn(hdev->dev,
732 "HASH interrupt when no active requests.\n");
733 }
734 } else if (reg & CR_INT_RESULTS_AVAILABLE) {
735 dev_warn(hdev->dev,
736 "IRQ triggered before the hash had completed\n");
737 } else if (reg & CR_INT_RESULT_READ_ERR) {
738 dev_warn(hdev->dev,
739 "Attempt to read from an empty result queue\n");
740 } else if (reg & CR_INT_MESSAGE_WRITE_ERROR) {
741 dev_warn(hdev->dev,
742 "Data written before the hardware was configured\n");
743 }
744 return IRQ_HANDLED;
745 }
746
747 static struct ahash_alg img_algs[] = {
748 {
749 .init = img_hash_init,
750 .update = img_hash_update,
751 .final = img_hash_final,
752 .finup = img_hash_finup,
753 .export = img_hash_export,
754 .import = img_hash_import,
755 .digest = img_hash_digest,
756 .halg = {
757 .digestsize = MD5_DIGEST_SIZE,
758 .statesize = sizeof(struct md5_state),
759 .base = {
760 .cra_name = "md5",
761 .cra_driver_name = "img-md5",
762 .cra_priority = 300,
763 .cra_flags =
764 CRYPTO_ALG_ASYNC |
765 CRYPTO_ALG_NEED_FALLBACK,
766 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
767 .cra_ctxsize = sizeof(struct img_hash_ctx),
768 .cra_init = img_hash_cra_md5_init,
769 .cra_exit = img_hash_cra_exit,
770 .cra_module = THIS_MODULE,
771 }
772 }
773 },
774 {
775 .init = img_hash_init,
776 .update = img_hash_update,
777 .final = img_hash_final,
778 .finup = img_hash_finup,
779 .export = img_hash_export,
780 .import = img_hash_import,
781 .digest = img_hash_digest,
782 .halg = {
783 .digestsize = SHA1_DIGEST_SIZE,
784 .statesize = sizeof(struct sha1_state),
785 .base = {
786 .cra_name = "sha1",
787 .cra_driver_name = "img-sha1",
788 .cra_priority = 300,
789 .cra_flags =
790 CRYPTO_ALG_ASYNC |
791 CRYPTO_ALG_NEED_FALLBACK,
792 .cra_blocksize = SHA1_BLOCK_SIZE,
793 .cra_ctxsize = sizeof(struct img_hash_ctx),
794 .cra_init = img_hash_cra_sha1_init,
795 .cra_exit = img_hash_cra_exit,
796 .cra_module = THIS_MODULE,
797 }
798 }
799 },
800 {
801 .init = img_hash_init,
802 .update = img_hash_update,
803 .final = img_hash_final,
804 .finup = img_hash_finup,
805 .export = img_hash_export,
806 .import = img_hash_import,
807 .digest = img_hash_digest,
808 .halg = {
809 .digestsize = SHA224_DIGEST_SIZE,
810 .statesize = sizeof(struct sha256_state),
811 .base = {
812 .cra_name = "sha224",
813 .cra_driver_name = "img-sha224",
814 .cra_priority = 300,
815 .cra_flags =
816 CRYPTO_ALG_ASYNC |
817 CRYPTO_ALG_NEED_FALLBACK,
818 .cra_blocksize = SHA224_BLOCK_SIZE,
819 .cra_ctxsize = sizeof(struct img_hash_ctx),
820 .cra_init = img_hash_cra_sha224_init,
821 .cra_exit = img_hash_cra_exit,
822 .cra_module = THIS_MODULE,
823 }
824 }
825 },
826 {
827 .init = img_hash_init,
828 .update = img_hash_update,
829 .final = img_hash_final,
830 .finup = img_hash_finup,
831 .export = img_hash_export,
832 .import = img_hash_import,
833 .digest = img_hash_digest,
834 .halg = {
835 .digestsize = SHA256_DIGEST_SIZE,
836 .statesize = sizeof(struct sha256_state),
837 .base = {
838 .cra_name = "sha256",
839 .cra_driver_name = "img-sha256",
840 .cra_priority = 300,
841 .cra_flags =
842 CRYPTO_ALG_ASYNC |
843 CRYPTO_ALG_NEED_FALLBACK,
844 .cra_blocksize = SHA256_BLOCK_SIZE,
845 .cra_ctxsize = sizeof(struct img_hash_ctx),
846 .cra_init = img_hash_cra_sha256_init,
847 .cra_exit = img_hash_cra_exit,
848 .cra_module = THIS_MODULE,
849 }
850 }
851 }
852 };
853
img_register_algs(struct img_hash_dev * hdev)854 static int img_register_algs(struct img_hash_dev *hdev)
855 {
856 int i, err;
857
858 for (i = 0; i < ARRAY_SIZE(img_algs); i++) {
859 err = crypto_register_ahash(&img_algs[i]);
860 if (err) {
861 crypto_unregister_ahashes(img_algs, i);
862 return err;
863 }
864 }
865
866 return 0;
867 }
868
img_unregister_algs(struct img_hash_dev * hdev)869 static void img_unregister_algs(struct img_hash_dev *hdev)
870 {
871 crypto_unregister_ahashes(img_algs, ARRAY_SIZE(img_algs));
872 }
873
img_hash_done_task(unsigned long data)874 static void img_hash_done_task(unsigned long data)
875 {
876 struct img_hash_dev *hdev = (struct img_hash_dev *)data;
877 int err = 0;
878
879 if (hdev->err == -EINVAL) {
880 err = hdev->err;
881 goto finish;
882 }
883
884 if (!(DRIVER_FLAGS_BUSY & hdev->flags)) {
885 img_hash_handle_queue(hdev, NULL);
886 return;
887 }
888
889 if (DRIVER_FLAGS_CPU & hdev->flags) {
890 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
891 hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY;
892 goto finish;
893 }
894 } else if (DRIVER_FLAGS_DMA_READY & hdev->flags) {
895 if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) {
896 hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE;
897 img_hash_write_via_dma_stop(hdev);
898 if (hdev->err) {
899 err = hdev->err;
900 goto finish;
901 }
902 }
903 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
904 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY |
905 DRIVER_FLAGS_OUTPUT_READY);
906 goto finish;
907 }
908 }
909 return;
910
911 finish:
912 img_hash_finish_req(hdev->req, err);
913 }
914
915 static const struct of_device_id img_hash_match[] __maybe_unused = {
916 { .compatible = "img,hash-accelerator" },
917 {}
918 };
919 MODULE_DEVICE_TABLE(of, img_hash_match);
920
img_hash_probe(struct platform_device * pdev)921 static int img_hash_probe(struct platform_device *pdev)
922 {
923 struct img_hash_dev *hdev;
924 struct device *dev = &pdev->dev;
925 struct resource *hash_res;
926 int irq;
927 int err;
928
929 hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
930 if (hdev == NULL)
931 return -ENOMEM;
932
933 spin_lock_init(&hdev->lock);
934
935 hdev->dev = dev;
936
937 platform_set_drvdata(pdev, hdev);
938
939 INIT_LIST_HEAD(&hdev->list);
940
941 tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev);
942 tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev);
943
944 crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH);
945
946 /* Register bank */
947 hdev->io_base = devm_platform_ioremap_resource(pdev, 0);
948 if (IS_ERR(hdev->io_base)) {
949 err = PTR_ERR(hdev->io_base);
950 goto res_err;
951 }
952
953 /* Write port (DMA or CPU) */
954 hdev->cpu_addr = devm_platform_get_and_ioremap_resource(pdev, 1, &hash_res);
955 if (IS_ERR(hdev->cpu_addr)) {
956 err = PTR_ERR(hdev->cpu_addr);
957 goto res_err;
958 }
959 hdev->bus_addr = hash_res->start;
960
961 irq = platform_get_irq(pdev, 0);
962 if (irq < 0) {
963 err = irq;
964 goto res_err;
965 }
966
967 err = devm_request_irq(dev, irq, img_irq_handler, 0,
968 dev_name(dev), hdev);
969 if (err)
970 goto res_err;
971 dev_dbg(dev, "using IRQ channel %d\n", irq);
972
973 hdev->hash_clk = devm_clk_get_enabled(&pdev->dev, "hash");
974 if (IS_ERR(hdev->hash_clk)) {
975 dev_err(dev, "clock initialization failed.\n");
976 err = PTR_ERR(hdev->hash_clk);
977 goto res_err;
978 }
979
980 hdev->sys_clk = devm_clk_get_enabled(&pdev->dev, "sys");
981 if (IS_ERR(hdev->sys_clk)) {
982 dev_err(dev, "clock initialization failed.\n");
983 err = PTR_ERR(hdev->sys_clk);
984 goto res_err;
985 }
986
987 err = img_hash_dma_init(hdev);
988 if (err)
989 goto res_err;
990
991 dev_dbg(dev, "using %s for DMA transfers\n",
992 dma_chan_name(hdev->dma_lch));
993
994 spin_lock(&img_hash.lock);
995 list_add_tail(&hdev->list, &img_hash.dev_list);
996 spin_unlock(&img_hash.lock);
997
998 err = img_register_algs(hdev);
999 if (err)
1000 goto err_algs;
1001 dev_info(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n");
1002
1003 return 0;
1004
1005 err_algs:
1006 spin_lock(&img_hash.lock);
1007 list_del(&hdev->list);
1008 spin_unlock(&img_hash.lock);
1009 dma_release_channel(hdev->dma_lch);
1010 res_err:
1011 tasklet_kill(&hdev->done_task);
1012 tasklet_kill(&hdev->dma_task);
1013
1014 return err;
1015 }
1016
img_hash_remove(struct platform_device * pdev)1017 static void img_hash_remove(struct platform_device *pdev)
1018 {
1019 struct img_hash_dev *hdev;
1020
1021 hdev = platform_get_drvdata(pdev);
1022 spin_lock(&img_hash.lock);
1023 list_del(&hdev->list);
1024 spin_unlock(&img_hash.lock);
1025
1026 img_unregister_algs(hdev);
1027
1028 tasklet_kill(&hdev->done_task);
1029 tasklet_kill(&hdev->dma_task);
1030
1031 dma_release_channel(hdev->dma_lch);
1032 }
1033
1034 #ifdef CONFIG_PM_SLEEP
img_hash_suspend(struct device * dev)1035 static int img_hash_suspend(struct device *dev)
1036 {
1037 struct img_hash_dev *hdev = dev_get_drvdata(dev);
1038
1039 clk_disable_unprepare(hdev->hash_clk);
1040 clk_disable_unprepare(hdev->sys_clk);
1041
1042 return 0;
1043 }
1044
img_hash_resume(struct device * dev)1045 static int img_hash_resume(struct device *dev)
1046 {
1047 struct img_hash_dev *hdev = dev_get_drvdata(dev);
1048 int ret;
1049
1050 ret = clk_prepare_enable(hdev->hash_clk);
1051 if (ret)
1052 return ret;
1053
1054 ret = clk_prepare_enable(hdev->sys_clk);
1055 if (ret) {
1056 clk_disable_unprepare(hdev->hash_clk);
1057 return ret;
1058 }
1059
1060 return 0;
1061 }
1062 #endif /* CONFIG_PM_SLEEP */
1063
1064 static const struct dev_pm_ops img_hash_pm_ops = {
1065 SET_SYSTEM_SLEEP_PM_OPS(img_hash_suspend, img_hash_resume)
1066 };
1067
1068 static struct platform_driver img_hash_driver = {
1069 .probe = img_hash_probe,
1070 .remove = img_hash_remove,
1071 .driver = {
1072 .name = "img-hash-accelerator",
1073 .pm = &img_hash_pm_ops,
1074 .of_match_table = img_hash_match,
1075 }
1076 };
1077 module_platform_driver(img_hash_driver);
1078
1079 MODULE_LICENSE("GPL v2");
1080 MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver");
1081 MODULE_AUTHOR("Will Thomas.");
1082 MODULE_AUTHOR("James Hartley <james.hartley@imgtec.com>");
1083