1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Intel Keem Bay OCS HCU Crypto Driver.
4 *
5 * Copyright (C) 2018-2020 Intel Corporation
6 */
7
8 #include <crypto/engine.h>
9 #include <crypto/hmac.h>
10 #include <crypto/internal/hash.h>
11 #include <crypto/scatterwalk.h>
12 #include <crypto/sha2.h>
13 #include <crypto/sm3.h>
14 #include <linux/completion.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/err.h>
17 #include <linux/interrupt.h>
18 #include <linux/kernel.h>
19 #include <linux/module.h>
20 #include <linux/platform_device.h>
21 #include <linux/string.h>
22
23 #include "ocs-hcu.h"
24
25 #define DRV_NAME "keembay-ocs-hcu"
26
27 /* Flag marking a final request. */
28 #define REQ_FINAL BIT(0)
29 /* Flag marking a HMAC request. */
30 #define REQ_FLAGS_HMAC BIT(1)
31 /* Flag set when HW HMAC is being used. */
32 #define REQ_FLAGS_HMAC_HW BIT(2)
33 /* Flag set when SW HMAC is being used. */
34 #define REQ_FLAGS_HMAC_SW BIT(3)
35
36 /**
37 * struct ocs_hcu_ctx: OCS HCU Transform context.
38 * @hcu_dev: The OCS HCU device used by the transformation.
39 * @key: The key (used only for HMAC transformations).
40 * @key_len: The length of the key.
41 * @is_sm3_tfm: Whether or not this is an SM3 transformation.
42 * @is_hmac_tfm: Whether or not this is a HMAC transformation.
43 */
44 struct ocs_hcu_ctx {
45 struct ocs_hcu_dev *hcu_dev;
46 u8 key[SHA512_BLOCK_SIZE];
47 size_t key_len;
48 bool is_sm3_tfm;
49 bool is_hmac_tfm;
50 };
51
52 /**
53 * struct ocs_hcu_rctx - Context for the request.
54 * @hcu_dev: OCS HCU device to be used to service the request.
55 * @flags: Flags tracking request status.
56 * @algo: Algorithm to use for the request.
57 * @blk_sz: Block size of the transformation / request.
58 * @dig_sz: Digest size of the transformation / request.
59 * @dma_list: OCS DMA linked list.
60 * @hash_ctx: OCS HCU hashing context.
61 * @buffer: Buffer to store: partial block of data and SW HMAC
62 * artifacts (ipad, opad, etc.).
63 * @buf_cnt: Number of bytes currently stored in the buffer.
64 * @buf_dma_addr: The DMA address of @buffer (when mapped).
65 * @buf_dma_count: The number of bytes in @buffer currently DMA-mapped.
66 * @sg: Head of the scatterlist entries containing data.
67 * @sg_data_total: Total data in the SG list at any time.
68 * @sg_data_offset: Offset into the data of the current individual SG node.
69 * @sg_dma_nents: Number of sg entries mapped in dma_list.
70 * @nents: Number of entries in the scatterlist.
71 */
72 struct ocs_hcu_rctx {
73 struct ocs_hcu_dev *hcu_dev;
74 u32 flags;
75 enum ocs_hcu_algo algo;
76 size_t blk_sz;
77 size_t dig_sz;
78 struct ocs_hcu_dma_list *dma_list;
79 struct ocs_hcu_hash_ctx hash_ctx;
80 /*
81 * Buffer is double the block size because we need space for SW HMAC
82 * artifacts, i.e:
83 * - ipad (1 block) + a possible partial block of data.
84 * - opad (1 block) + digest of H(k ^ ipad || m)
85 */
86 u8 buffer[2 * SHA512_BLOCK_SIZE];
87 size_t buf_cnt;
88 dma_addr_t buf_dma_addr;
89 size_t buf_dma_count;
90 struct scatterlist *sg;
91 unsigned int sg_data_total;
92 unsigned int sg_data_offset;
93 unsigned int sg_dma_nents;
94 unsigned int nents;
95 };
96
97 /**
98 * struct ocs_hcu_drv - Driver data
99 * @dev_list: The list of HCU devices.
100 * @lock: The lock protecting dev_list.
101 */
102 struct ocs_hcu_drv {
103 struct list_head dev_list;
104 spinlock_t lock; /* Protects dev_list. */
105 };
106
107 static struct ocs_hcu_drv ocs_hcu = {
108 .dev_list = LIST_HEAD_INIT(ocs_hcu.dev_list),
109 .lock = __SPIN_LOCK_UNLOCKED(ocs_hcu.lock),
110 };
111
112 /*
113 * Return the total amount of data in the request; that is: the data in the
114 * request buffer + the data in the sg list.
115 */
kmb_get_total_data(struct ocs_hcu_rctx * rctx)116 static inline unsigned int kmb_get_total_data(struct ocs_hcu_rctx *rctx)
117 {
118 return rctx->sg_data_total + rctx->buf_cnt;
119 }
120
121 /* Move remaining content of scatter-gather list to context buffer. */
flush_sg_to_ocs_buffer(struct ocs_hcu_rctx * rctx)122 static int flush_sg_to_ocs_buffer(struct ocs_hcu_rctx *rctx)
123 {
124 size_t count;
125
126 if (rctx->sg_data_total > (sizeof(rctx->buffer) - rctx->buf_cnt)) {
127 WARN(1, "%s: sg data does not fit in buffer\n", __func__);
128 return -EINVAL;
129 }
130
131 while (rctx->sg_data_total) {
132 if (!rctx->sg) {
133 WARN(1, "%s: unexpected NULL sg\n", __func__);
134 return -EINVAL;
135 }
136 /*
137 * If current sg has been fully processed, skip to the next
138 * one.
139 */
140 if (rctx->sg_data_offset == rctx->sg->length) {
141 rctx->sg = sg_next(rctx->sg);
142 rctx->sg_data_offset = 0;
143 continue;
144 }
145 /*
146 * Determine the maximum data available to copy from the node.
147 * Minimum of the length left in the sg node, or the total data
148 * in the request.
149 */
150 count = min(rctx->sg->length - rctx->sg_data_offset,
151 rctx->sg_data_total);
152 /* Copy from scatter-list entry to context buffer. */
153 scatterwalk_map_and_copy(&rctx->buffer[rctx->buf_cnt],
154 rctx->sg, rctx->sg_data_offset,
155 count, 0);
156
157 rctx->sg_data_offset += count;
158 rctx->sg_data_total -= count;
159 rctx->buf_cnt += count;
160 }
161
162 return 0;
163 }
164
kmb_ocs_hcu_find_dev(struct ahash_request * req)165 static struct ocs_hcu_dev *kmb_ocs_hcu_find_dev(struct ahash_request *req)
166 {
167 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
168 struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm);
169
170 /* If the HCU device for the request was previously set, return it. */
171 if (tctx->hcu_dev)
172 return tctx->hcu_dev;
173
174 /*
175 * Otherwise, get the first HCU device available (there should be one
176 * and only one device).
177 */
178 spin_lock_bh(&ocs_hcu.lock);
179 tctx->hcu_dev = list_first_entry_or_null(&ocs_hcu.dev_list,
180 struct ocs_hcu_dev,
181 list);
182 spin_unlock_bh(&ocs_hcu.lock);
183
184 return tctx->hcu_dev;
185 }
186
187 /* Free OCS DMA linked list and DMA-able context buffer. */
kmb_ocs_hcu_dma_cleanup(struct ahash_request * req,struct ocs_hcu_rctx * rctx)188 static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req,
189 struct ocs_hcu_rctx *rctx)
190 {
191 struct ocs_hcu_dev *hcu_dev = rctx->hcu_dev;
192 struct device *dev = hcu_dev->dev;
193
194 /* Unmap rctx->buffer (if mapped). */
195 if (rctx->buf_dma_count) {
196 dma_unmap_single(dev, rctx->buf_dma_addr, rctx->buf_dma_count,
197 DMA_TO_DEVICE);
198 rctx->buf_dma_count = 0;
199 }
200
201 /* Unmap req->src (if mapped). */
202 if (rctx->sg_dma_nents) {
203 dma_unmap_sg(dev, req->src, rctx->nents, DMA_TO_DEVICE);
204 rctx->sg_dma_nents = 0;
205 }
206
207 /* Free dma_list (if allocated). */
208 if (rctx->dma_list) {
209 ocs_hcu_dma_list_free(hcu_dev, rctx->dma_list);
210 rctx->dma_list = NULL;
211 }
212 }
213
214 /*
215 * Prepare for DMA operation:
216 * - DMA-map request context buffer (if needed)
217 * - DMA-map SG list (only the entries to be processed, see note below)
218 * - Allocate OCS HCU DMA linked list (number of elements = SG entries to
219 * process + context buffer (if not empty)).
220 * - Add DMA-mapped request context buffer to OCS HCU DMA list.
221 * - Add SG entries to DMA list.
222 *
223 * Note: if this is a final request, we process all the data in the SG list,
224 * otherwise we can only process up to the maximum amount of block-aligned data
225 * (the remainder will be put into the context buffer and processed in the next
226 * request).
227 */
kmb_ocs_dma_prepare(struct ahash_request * req)228 static int kmb_ocs_dma_prepare(struct ahash_request *req)
229 {
230 struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
231 struct device *dev = rctx->hcu_dev->dev;
232 unsigned int remainder = 0;
233 unsigned int total;
234 int nents;
235 size_t count;
236 int rc;
237 int i;
238
239 /* This function should be called only when there is data to process. */
240 total = kmb_get_total_data(rctx);
241 if (!total)
242 return -EINVAL;
243
244 /*
245 * If this is not a final DMA (terminated DMA), the data passed to the
246 * HCU must be aligned to the block size; compute the remainder data to
247 * be processed in the next request.
248 */
249 if (!(rctx->flags & REQ_FINAL))
250 remainder = total % rctx->blk_sz;
251
252 /* Determine the number of scatter gather list entries to process. */
253 nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder);
254
255 if (nents < 0)
256 return nents;
257
258 /* If there are entries to process, map them. */
259 if (nents) {
260 rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents,
261 DMA_TO_DEVICE);
262 if (!rctx->sg_dma_nents) {
263 dev_err(dev, "Failed to MAP SG\n");
264 rc = -ENOMEM;
265 goto cleanup;
266 }
267
268 /* Save the value of nents to pass to dma_unmap_sg. */
269 rctx->nents = nents;
270
271 /*
272 * The value returned by dma_map_sg() can be < nents; so update
273 * nents accordingly.
274 */
275 nents = rctx->sg_dma_nents;
276 }
277
278 /*
279 * If context buffer is not empty, map it and add extra DMA entry for
280 * it.
281 */
282 if (rctx->buf_cnt) {
283 rctx->buf_dma_addr = dma_map_single(dev, rctx->buffer,
284 rctx->buf_cnt,
285 DMA_TO_DEVICE);
286 if (dma_mapping_error(dev, rctx->buf_dma_addr)) {
287 dev_err(dev, "Failed to map request context buffer\n");
288 rc = -ENOMEM;
289 goto cleanup;
290 }
291 rctx->buf_dma_count = rctx->buf_cnt;
292 /* Increase number of dma entries. */
293 nents++;
294 }
295
296 /* Allocate OCS HCU DMA list. */
297 rctx->dma_list = ocs_hcu_dma_list_alloc(rctx->hcu_dev, nents);
298 if (!rctx->dma_list) {
299 rc = -ENOMEM;
300 goto cleanup;
301 }
302
303 /* Add request context buffer (if previously DMA-mapped) */
304 if (rctx->buf_dma_count) {
305 rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, rctx->dma_list,
306 rctx->buf_dma_addr,
307 rctx->buf_dma_count);
308 if (rc)
309 goto cleanup;
310 }
311
312 /* Add the SG nodes to be processed to the DMA linked list. */
313 for_each_sg(req->src, rctx->sg, rctx->sg_dma_nents, i) {
314 /*
315 * The number of bytes to add to the list entry is the minimum
316 * between:
317 * - The DMA length of the SG entry.
318 * - The data left to be processed.
319 */
320 count = min(rctx->sg_data_total - remainder,
321 sg_dma_len(rctx->sg) - rctx->sg_data_offset);
322 /*
323 * Do not create a zero length DMA descriptor. Check in case of
324 * zero length SG node.
325 */
326 if (count == 0)
327 continue;
328 /* Add sg to HCU DMA list. */
329 rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev,
330 rctx->dma_list,
331 rctx->sg->dma_address,
332 count);
333 if (rc)
334 goto cleanup;
335
336 /* Update amount of data remaining in SG list. */
337 rctx->sg_data_total -= count;
338
339 /*
340 * If remaining data is equal to remainder (note: 'less than'
341 * case should never happen in practice), we are done: update
342 * offset and exit the loop.
343 */
344 if (rctx->sg_data_total <= remainder) {
345 WARN_ON(rctx->sg_data_total < remainder);
346 rctx->sg_data_offset += count;
347 break;
348 }
349
350 /*
351 * If we get here is because we need to process the next sg in
352 * the list; set offset within the sg to 0.
353 */
354 rctx->sg_data_offset = 0;
355 }
356
357 return 0;
358 cleanup:
359 dev_err(dev, "Failed to prepare DMA.\n");
360 kmb_ocs_hcu_dma_cleanup(req, rctx);
361
362 return rc;
363 }
364
kmb_ocs_hcu_secure_cleanup(struct ahash_request * req)365 static void kmb_ocs_hcu_secure_cleanup(struct ahash_request *req)
366 {
367 struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
368
369 /* Clear buffer of any data. */
370 memzero_explicit(rctx->buffer, sizeof(rctx->buffer));
371 }
372
kmb_ocs_hcu_handle_queue(struct ahash_request * req)373 static int kmb_ocs_hcu_handle_queue(struct ahash_request *req)
374 {
375 struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
376
377 if (!hcu_dev)
378 return -ENOENT;
379
380 return crypto_transfer_hash_request_to_engine(hcu_dev->engine, req);
381 }
382
prepare_ipad(struct ahash_request * req)383 static int prepare_ipad(struct ahash_request *req)
384 {
385 struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
386 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
387 struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
388 int i;
389
390 WARN(rctx->buf_cnt, "%s: Context buffer is not empty\n", __func__);
391 WARN(!(rctx->flags & REQ_FLAGS_HMAC_SW),
392 "%s: HMAC_SW flag is not set\n", __func__);
393 /*
394 * Key length must be equal to block size. If key is shorter,
395 * we pad it with zero (note: key cannot be longer, since
396 * longer keys are hashed by kmb_ocs_hcu_setkey()).
397 */
398 if (ctx->key_len > rctx->blk_sz) {
399 WARN(1, "%s: Invalid key length in tfm context\n", __func__);
400 return -EINVAL;
401 }
402 memzero_explicit(&ctx->key[ctx->key_len],
403 rctx->blk_sz - ctx->key_len);
404 ctx->key_len = rctx->blk_sz;
405 /*
406 * Prepare IPAD for HMAC. Only done for first block.
407 * HMAC(k,m) = H(k ^ opad || H(k ^ ipad || m))
408 * k ^ ipad will be first hashed block.
409 * k ^ opad will be calculated in the final request.
410 * Only needed if not using HW HMAC.
411 */
412 for (i = 0; i < rctx->blk_sz; i++)
413 rctx->buffer[i] = ctx->key[i] ^ HMAC_IPAD_VALUE;
414 rctx->buf_cnt = rctx->blk_sz;
415
416 return 0;
417 }
418
kmb_ocs_hcu_do_one_request(struct crypto_engine * engine,void * areq)419 static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq)
420 {
421 struct ahash_request *req = container_of(areq, struct ahash_request,
422 base);
423 struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
424 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
425 struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
426 struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm);
427 int rc;
428 int i;
429
430 if (!hcu_dev) {
431 rc = -ENOENT;
432 goto error;
433 }
434
435 /*
436 * If hardware HMAC flag is set, perform HMAC in hardware.
437 *
438 * NOTE: this flag implies REQ_FINAL && kmb_get_total_data(rctx)
439 */
440 if (rctx->flags & REQ_FLAGS_HMAC_HW) {
441 /* Map input data into the HCU DMA linked list. */
442 rc = kmb_ocs_dma_prepare(req);
443 if (rc)
444 goto error;
445
446 rc = ocs_hcu_hmac(hcu_dev, rctx->algo, tctx->key, tctx->key_len,
447 rctx->dma_list, req->result, rctx->dig_sz);
448
449 /* Unmap data and free DMA list regardless of return code. */
450 kmb_ocs_hcu_dma_cleanup(req, rctx);
451
452 /* Process previous return code. */
453 if (rc)
454 goto error;
455
456 goto done;
457 }
458
459 /* Handle update request case. */
460 if (!(rctx->flags & REQ_FINAL)) {
461 /* Update should always have input data. */
462 if (!kmb_get_total_data(rctx))
463 return -EINVAL;
464
465 /* Map input data into the HCU DMA linked list. */
466 rc = kmb_ocs_dma_prepare(req);
467 if (rc)
468 goto error;
469
470 /* Do hashing step. */
471 rc = ocs_hcu_hash_update(hcu_dev, &rctx->hash_ctx,
472 rctx->dma_list);
473
474 /* Unmap data and free DMA list regardless of return code. */
475 kmb_ocs_hcu_dma_cleanup(req, rctx);
476
477 /* Process previous return code. */
478 if (rc)
479 goto error;
480
481 /*
482 * Reset request buffer count (data in the buffer was just
483 * processed).
484 */
485 rctx->buf_cnt = 0;
486 /*
487 * Move remaining sg data into the request buffer, so that it
488 * will be processed during the next request.
489 *
490 * NOTE: we have remaining data if kmb_get_total_data() was not
491 * a multiple of block size.
492 */
493 rc = flush_sg_to_ocs_buffer(rctx);
494 if (rc)
495 goto error;
496
497 goto done;
498 }
499
500 /* If we get here, this is a final request. */
501
502 /* If there is data to process, use finup. */
503 if (kmb_get_total_data(rctx)) {
504 /* Map input data into the HCU DMA linked list. */
505 rc = kmb_ocs_dma_prepare(req);
506 if (rc)
507 goto error;
508
509 /* Do hashing step. */
510 rc = ocs_hcu_hash_finup(hcu_dev, &rctx->hash_ctx,
511 rctx->dma_list,
512 req->result, rctx->dig_sz);
513 /* Free DMA list regardless of return code. */
514 kmb_ocs_hcu_dma_cleanup(req, rctx);
515
516 /* Process previous return code. */
517 if (rc)
518 goto error;
519
520 } else { /* Otherwise (if we have no data), use final. */
521 rc = ocs_hcu_hash_final(hcu_dev, &rctx->hash_ctx, req->result,
522 rctx->dig_sz);
523 if (rc)
524 goto error;
525 }
526
527 /*
528 * If we are finalizing a SW HMAC request, we just computed the result
529 * of: H(k ^ ipad || m).
530 *
531 * We now need to complete the HMAC calculation with the OPAD step,
532 * that is, we need to compute H(k ^ opad || digest), where digest is
533 * the digest we just obtained, i.e., H(k ^ ipad || m).
534 */
535 if (rctx->flags & REQ_FLAGS_HMAC_SW) {
536 /*
537 * Compute k ^ opad and store it in the request buffer (which
538 * is not used anymore at this point).
539 * Note: key has been padded / hashed already (so keylen ==
540 * blksz) .
541 */
542 WARN_ON(tctx->key_len != rctx->blk_sz);
543 for (i = 0; i < rctx->blk_sz; i++)
544 rctx->buffer[i] = tctx->key[i] ^ HMAC_OPAD_VALUE;
545 /* Now append the digest to the rest of the buffer. */
546 for (i = 0; (i < rctx->dig_sz); i++)
547 rctx->buffer[rctx->blk_sz + i] = req->result[i];
548
549 /* Now hash the buffer to obtain the final HMAC. */
550 rc = ocs_hcu_digest(hcu_dev, rctx->algo, rctx->buffer,
551 rctx->blk_sz + rctx->dig_sz, req->result,
552 rctx->dig_sz);
553 if (rc)
554 goto error;
555 }
556
557 /* Perform secure clean-up. */
558 kmb_ocs_hcu_secure_cleanup(req);
559 done:
560 crypto_finalize_hash_request(hcu_dev->engine, req, 0);
561
562 return 0;
563
564 error:
565 kmb_ocs_hcu_secure_cleanup(req);
566 return rc;
567 }
568
kmb_ocs_hcu_init(struct ahash_request * req)569 static int kmb_ocs_hcu_init(struct ahash_request *req)
570 {
571 struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
572 struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
573 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
574 struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
575
576 if (!hcu_dev)
577 return -ENOENT;
578
579 /* Initialize entire request context to zero. */
580 memset(rctx, 0, sizeof(*rctx));
581
582 rctx->hcu_dev = hcu_dev;
583 rctx->dig_sz = crypto_ahash_digestsize(tfm);
584
585 switch (rctx->dig_sz) {
586 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224
587 case SHA224_DIGEST_SIZE:
588 rctx->blk_sz = SHA224_BLOCK_SIZE;
589 rctx->algo = OCS_HCU_ALGO_SHA224;
590 break;
591 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */
592 case SHA256_DIGEST_SIZE:
593 rctx->blk_sz = SHA256_BLOCK_SIZE;
594 /*
595 * SHA256 and SM3 have the same digest size: use info from tfm
596 * context to find out which one we should use.
597 */
598 rctx->algo = ctx->is_sm3_tfm ? OCS_HCU_ALGO_SM3 :
599 OCS_HCU_ALGO_SHA256;
600 break;
601 case SHA384_DIGEST_SIZE:
602 rctx->blk_sz = SHA384_BLOCK_SIZE;
603 rctx->algo = OCS_HCU_ALGO_SHA384;
604 break;
605 case SHA512_DIGEST_SIZE:
606 rctx->blk_sz = SHA512_BLOCK_SIZE;
607 rctx->algo = OCS_HCU_ALGO_SHA512;
608 break;
609 default:
610 return -EINVAL;
611 }
612
613 /* Initialize intermediate data. */
614 ocs_hcu_hash_init(&rctx->hash_ctx, rctx->algo);
615
616 /* If this a HMAC request, set HMAC flag. */
617 if (ctx->is_hmac_tfm)
618 rctx->flags |= REQ_FLAGS_HMAC;
619
620 return 0;
621 }
622
kmb_ocs_hcu_update(struct ahash_request * req)623 static int kmb_ocs_hcu_update(struct ahash_request *req)
624 {
625 struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
626 int rc;
627
628 if (!req->nbytes)
629 return 0;
630
631 rctx->sg_data_total = req->nbytes;
632 rctx->sg_data_offset = 0;
633 rctx->sg = req->src;
634
635 /*
636 * If we are doing HMAC, then we must use SW-assisted HMAC, since HW
637 * HMAC does not support context switching (there it can only be used
638 * with finup() or digest()).
639 */
640 if (rctx->flags & REQ_FLAGS_HMAC &&
641 !(rctx->flags & REQ_FLAGS_HMAC_SW)) {
642 rctx->flags |= REQ_FLAGS_HMAC_SW;
643 rc = prepare_ipad(req);
644 if (rc)
645 return rc;
646 }
647
648 /*
649 * If remaining sg_data fits into ctx buffer, just copy it there; we'll
650 * process it at the next update() or final().
651 */
652 if (rctx->sg_data_total <= (sizeof(rctx->buffer) - rctx->buf_cnt))
653 return flush_sg_to_ocs_buffer(rctx);
654
655 return kmb_ocs_hcu_handle_queue(req);
656 }
657
658 /* Common logic for kmb_ocs_hcu_final() and kmb_ocs_hcu_finup(). */
kmb_ocs_hcu_fin_common(struct ahash_request * req)659 static int kmb_ocs_hcu_fin_common(struct ahash_request *req)
660 {
661 struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
662 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
663 struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
664 int rc;
665
666 rctx->flags |= REQ_FINAL;
667
668 /*
669 * If this is a HMAC request and, so far, we didn't have to switch to
670 * SW HMAC, check if we can use HW HMAC.
671 */
672 if (rctx->flags & REQ_FLAGS_HMAC &&
673 !(rctx->flags & REQ_FLAGS_HMAC_SW)) {
674 /*
675 * If we are here, it means we never processed any data so far,
676 * so we can use HW HMAC, but only if there is some data to
677 * process (since OCS HW MAC does not support zero-length
678 * messages) and the key length is supported by the hardware
679 * (OCS HCU HW only supports length <= 64); if HW HMAC cannot
680 * be used, fall back to SW-assisted HMAC.
681 */
682 if (kmb_get_total_data(rctx) &&
683 ctx->key_len <= OCS_HCU_HW_KEY_LEN) {
684 rctx->flags |= REQ_FLAGS_HMAC_HW;
685 } else {
686 rctx->flags |= REQ_FLAGS_HMAC_SW;
687 rc = prepare_ipad(req);
688 if (rc)
689 return rc;
690 }
691 }
692
693 return kmb_ocs_hcu_handle_queue(req);
694 }
695
kmb_ocs_hcu_final(struct ahash_request * req)696 static int kmb_ocs_hcu_final(struct ahash_request *req)
697 {
698 struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
699
700 rctx->sg_data_total = 0;
701 rctx->sg_data_offset = 0;
702 rctx->sg = NULL;
703
704 return kmb_ocs_hcu_fin_common(req);
705 }
706
kmb_ocs_hcu_finup(struct ahash_request * req)707 static int kmb_ocs_hcu_finup(struct ahash_request *req)
708 {
709 struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
710
711 rctx->sg_data_total = req->nbytes;
712 rctx->sg_data_offset = 0;
713 rctx->sg = req->src;
714
715 return kmb_ocs_hcu_fin_common(req);
716 }
717
kmb_ocs_hcu_digest(struct ahash_request * req)718 static int kmb_ocs_hcu_digest(struct ahash_request *req)
719 {
720 int rc = 0;
721 struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
722
723 if (!hcu_dev)
724 return -ENOENT;
725
726 rc = kmb_ocs_hcu_init(req);
727 if (rc)
728 return rc;
729
730 rc = kmb_ocs_hcu_finup(req);
731
732 return rc;
733 }
734
kmb_ocs_hcu_export(struct ahash_request * req,void * out)735 static int kmb_ocs_hcu_export(struct ahash_request *req, void *out)
736 {
737 struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
738
739 /* Intermediate data is always stored and applied per request. */
740 memcpy(out, rctx, sizeof(*rctx));
741
742 return 0;
743 }
744
kmb_ocs_hcu_import(struct ahash_request * req,const void * in)745 static int kmb_ocs_hcu_import(struct ahash_request *req, const void *in)
746 {
747 struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
748
749 /* Intermediate data is always stored and applied per request. */
750 memcpy(rctx, in, sizeof(*rctx));
751
752 return 0;
753 }
754
kmb_ocs_hcu_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)755 static int kmb_ocs_hcu_setkey(struct crypto_ahash *tfm, const u8 *key,
756 unsigned int keylen)
757 {
758 unsigned int digestsize = crypto_ahash_digestsize(tfm);
759 struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
760 size_t blk_sz = crypto_ahash_blocksize(tfm);
761 struct crypto_ahash *ahash_tfm;
762 struct ahash_request *req;
763 struct crypto_wait wait;
764 struct scatterlist sg;
765 const char *alg_name;
766 int rc;
767
768 /*
769 * Key length must be equal to block size:
770 * - If key is shorter, we are done for now (the key will be padded
771 * later on); this is to maximize the use of HW HMAC (which works
772 * only for keys <= 64 bytes).
773 * - If key is longer, we hash it.
774 */
775 if (keylen <= blk_sz) {
776 memcpy(ctx->key, key, keylen);
777 ctx->key_len = keylen;
778 return 0;
779 }
780
781 switch (digestsize) {
782 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224
783 case SHA224_DIGEST_SIZE:
784 alg_name = "sha224-keembay-ocs";
785 break;
786 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */
787 case SHA256_DIGEST_SIZE:
788 alg_name = ctx->is_sm3_tfm ? "sm3-keembay-ocs" :
789 "sha256-keembay-ocs";
790 break;
791 case SHA384_DIGEST_SIZE:
792 alg_name = "sha384-keembay-ocs";
793 break;
794 case SHA512_DIGEST_SIZE:
795 alg_name = "sha512-keembay-ocs";
796 break;
797 default:
798 return -EINVAL;
799 }
800
801 ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0);
802 if (IS_ERR(ahash_tfm))
803 return PTR_ERR(ahash_tfm);
804
805 req = ahash_request_alloc(ahash_tfm, GFP_KERNEL);
806 if (!req) {
807 rc = -ENOMEM;
808 goto err_free_ahash;
809 }
810
811 crypto_init_wait(&wait);
812 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
813 crypto_req_done, &wait);
814 crypto_ahash_clear_flags(ahash_tfm, ~0);
815
816 sg_init_one(&sg, key, keylen);
817 ahash_request_set_crypt(req, &sg, ctx->key, keylen);
818
819 rc = crypto_wait_req(crypto_ahash_digest(req), &wait);
820 if (rc == 0)
821 ctx->key_len = digestsize;
822
823 ahash_request_free(req);
824 err_free_ahash:
825 crypto_free_ahash(ahash_tfm);
826
827 return rc;
828 }
829
830 /* Set request size and initialize tfm context. */
__cra_init(struct crypto_tfm * tfm,struct ocs_hcu_ctx * ctx)831 static void __cra_init(struct crypto_tfm *tfm, struct ocs_hcu_ctx *ctx)
832 {
833 crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm),
834 sizeof(struct ocs_hcu_rctx));
835 }
836
kmb_ocs_hcu_sha_cra_init(struct crypto_tfm * tfm)837 static int kmb_ocs_hcu_sha_cra_init(struct crypto_tfm *tfm)
838 {
839 struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
840
841 __cra_init(tfm, ctx);
842
843 return 0;
844 }
845
kmb_ocs_hcu_sm3_cra_init(struct crypto_tfm * tfm)846 static int kmb_ocs_hcu_sm3_cra_init(struct crypto_tfm *tfm)
847 {
848 struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
849
850 __cra_init(tfm, ctx);
851
852 ctx->is_sm3_tfm = true;
853
854 return 0;
855 }
856
kmb_ocs_hcu_hmac_sm3_cra_init(struct crypto_tfm * tfm)857 static int kmb_ocs_hcu_hmac_sm3_cra_init(struct crypto_tfm *tfm)
858 {
859 struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
860
861 __cra_init(tfm, ctx);
862
863 ctx->is_sm3_tfm = true;
864 ctx->is_hmac_tfm = true;
865
866 return 0;
867 }
868
kmb_ocs_hcu_hmac_cra_init(struct crypto_tfm * tfm)869 static int kmb_ocs_hcu_hmac_cra_init(struct crypto_tfm *tfm)
870 {
871 struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
872
873 __cra_init(tfm, ctx);
874
875 ctx->is_hmac_tfm = true;
876
877 return 0;
878 }
879
880 /* Function called when 'tfm' is de-initialized. */
kmb_ocs_hcu_hmac_cra_exit(struct crypto_tfm * tfm)881 static void kmb_ocs_hcu_hmac_cra_exit(struct crypto_tfm *tfm)
882 {
883 struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
884
885 /* Clear the key. */
886 memzero_explicit(ctx->key, sizeof(ctx->key));
887 }
888
889 static struct ahash_engine_alg ocs_hcu_algs[] = {
890 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224
891 {
892 .base.init = kmb_ocs_hcu_init,
893 .base.update = kmb_ocs_hcu_update,
894 .base.final = kmb_ocs_hcu_final,
895 .base.finup = kmb_ocs_hcu_finup,
896 .base.digest = kmb_ocs_hcu_digest,
897 .base.export = kmb_ocs_hcu_export,
898 .base.import = kmb_ocs_hcu_import,
899 .base.halg = {
900 .digestsize = SHA224_DIGEST_SIZE,
901 .statesize = sizeof(struct ocs_hcu_rctx),
902 .base = {
903 .cra_name = "sha224",
904 .cra_driver_name = "sha224-keembay-ocs",
905 .cra_priority = 255,
906 .cra_flags = CRYPTO_ALG_ASYNC,
907 .cra_blocksize = SHA224_BLOCK_SIZE,
908 .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
909 .cra_alignmask = 0,
910 .cra_module = THIS_MODULE,
911 .cra_init = kmb_ocs_hcu_sha_cra_init,
912 }
913 },
914 .op.do_one_request = kmb_ocs_hcu_do_one_request,
915 },
916 {
917 .base.init = kmb_ocs_hcu_init,
918 .base.update = kmb_ocs_hcu_update,
919 .base.final = kmb_ocs_hcu_final,
920 .base.finup = kmb_ocs_hcu_finup,
921 .base.digest = kmb_ocs_hcu_digest,
922 .base.export = kmb_ocs_hcu_export,
923 .base.import = kmb_ocs_hcu_import,
924 .base.setkey = kmb_ocs_hcu_setkey,
925 .base.halg = {
926 .digestsize = SHA224_DIGEST_SIZE,
927 .statesize = sizeof(struct ocs_hcu_rctx),
928 .base = {
929 .cra_name = "hmac(sha224)",
930 .cra_driver_name = "hmac-sha224-keembay-ocs",
931 .cra_priority = 255,
932 .cra_flags = CRYPTO_ALG_ASYNC,
933 .cra_blocksize = SHA224_BLOCK_SIZE,
934 .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
935 .cra_alignmask = 0,
936 .cra_module = THIS_MODULE,
937 .cra_init = kmb_ocs_hcu_hmac_cra_init,
938 .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
939 }
940 },
941 .op.do_one_request = kmb_ocs_hcu_do_one_request,
942 },
943 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */
944 {
945 .base.init = kmb_ocs_hcu_init,
946 .base.update = kmb_ocs_hcu_update,
947 .base.final = kmb_ocs_hcu_final,
948 .base.finup = kmb_ocs_hcu_finup,
949 .base.digest = kmb_ocs_hcu_digest,
950 .base.export = kmb_ocs_hcu_export,
951 .base.import = kmb_ocs_hcu_import,
952 .base.halg = {
953 .digestsize = SHA256_DIGEST_SIZE,
954 .statesize = sizeof(struct ocs_hcu_rctx),
955 .base = {
956 .cra_name = "sha256",
957 .cra_driver_name = "sha256-keembay-ocs",
958 .cra_priority = 255,
959 .cra_flags = CRYPTO_ALG_ASYNC,
960 .cra_blocksize = SHA256_BLOCK_SIZE,
961 .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
962 .cra_alignmask = 0,
963 .cra_module = THIS_MODULE,
964 .cra_init = kmb_ocs_hcu_sha_cra_init,
965 }
966 },
967 .op.do_one_request = kmb_ocs_hcu_do_one_request,
968 },
969 {
970 .base.init = kmb_ocs_hcu_init,
971 .base.update = kmb_ocs_hcu_update,
972 .base.final = kmb_ocs_hcu_final,
973 .base.finup = kmb_ocs_hcu_finup,
974 .base.digest = kmb_ocs_hcu_digest,
975 .base.export = kmb_ocs_hcu_export,
976 .base.import = kmb_ocs_hcu_import,
977 .base.setkey = kmb_ocs_hcu_setkey,
978 .base.halg = {
979 .digestsize = SHA256_DIGEST_SIZE,
980 .statesize = sizeof(struct ocs_hcu_rctx),
981 .base = {
982 .cra_name = "hmac(sha256)",
983 .cra_driver_name = "hmac-sha256-keembay-ocs",
984 .cra_priority = 255,
985 .cra_flags = CRYPTO_ALG_ASYNC,
986 .cra_blocksize = SHA256_BLOCK_SIZE,
987 .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
988 .cra_alignmask = 0,
989 .cra_module = THIS_MODULE,
990 .cra_init = kmb_ocs_hcu_hmac_cra_init,
991 .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
992 }
993 },
994 .op.do_one_request = kmb_ocs_hcu_do_one_request,
995 },
996 {
997 .base.init = kmb_ocs_hcu_init,
998 .base.update = kmb_ocs_hcu_update,
999 .base.final = kmb_ocs_hcu_final,
1000 .base.finup = kmb_ocs_hcu_finup,
1001 .base.digest = kmb_ocs_hcu_digest,
1002 .base.export = kmb_ocs_hcu_export,
1003 .base.import = kmb_ocs_hcu_import,
1004 .base.halg = {
1005 .digestsize = SM3_DIGEST_SIZE,
1006 .statesize = sizeof(struct ocs_hcu_rctx),
1007 .base = {
1008 .cra_name = "sm3",
1009 .cra_driver_name = "sm3-keembay-ocs",
1010 .cra_priority = 255,
1011 .cra_flags = CRYPTO_ALG_ASYNC,
1012 .cra_blocksize = SM3_BLOCK_SIZE,
1013 .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
1014 .cra_alignmask = 0,
1015 .cra_module = THIS_MODULE,
1016 .cra_init = kmb_ocs_hcu_sm3_cra_init,
1017 }
1018 },
1019 .op.do_one_request = kmb_ocs_hcu_do_one_request,
1020 },
1021 {
1022 .base.init = kmb_ocs_hcu_init,
1023 .base.update = kmb_ocs_hcu_update,
1024 .base.final = kmb_ocs_hcu_final,
1025 .base.finup = kmb_ocs_hcu_finup,
1026 .base.digest = kmb_ocs_hcu_digest,
1027 .base.export = kmb_ocs_hcu_export,
1028 .base.import = kmb_ocs_hcu_import,
1029 .base.setkey = kmb_ocs_hcu_setkey,
1030 .base.halg = {
1031 .digestsize = SM3_DIGEST_SIZE,
1032 .statesize = sizeof(struct ocs_hcu_rctx),
1033 .base = {
1034 .cra_name = "hmac(sm3)",
1035 .cra_driver_name = "hmac-sm3-keembay-ocs",
1036 .cra_priority = 255,
1037 .cra_flags = CRYPTO_ALG_ASYNC,
1038 .cra_blocksize = SM3_BLOCK_SIZE,
1039 .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
1040 .cra_alignmask = 0,
1041 .cra_module = THIS_MODULE,
1042 .cra_init = kmb_ocs_hcu_hmac_sm3_cra_init,
1043 .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
1044 }
1045 },
1046 .op.do_one_request = kmb_ocs_hcu_do_one_request,
1047 },
1048 {
1049 .base.init = kmb_ocs_hcu_init,
1050 .base.update = kmb_ocs_hcu_update,
1051 .base.final = kmb_ocs_hcu_final,
1052 .base.finup = kmb_ocs_hcu_finup,
1053 .base.digest = kmb_ocs_hcu_digest,
1054 .base.export = kmb_ocs_hcu_export,
1055 .base.import = kmb_ocs_hcu_import,
1056 .base.halg = {
1057 .digestsize = SHA384_DIGEST_SIZE,
1058 .statesize = sizeof(struct ocs_hcu_rctx),
1059 .base = {
1060 .cra_name = "sha384",
1061 .cra_driver_name = "sha384-keembay-ocs",
1062 .cra_priority = 255,
1063 .cra_flags = CRYPTO_ALG_ASYNC,
1064 .cra_blocksize = SHA384_BLOCK_SIZE,
1065 .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
1066 .cra_alignmask = 0,
1067 .cra_module = THIS_MODULE,
1068 .cra_init = kmb_ocs_hcu_sha_cra_init,
1069 }
1070 },
1071 .op.do_one_request = kmb_ocs_hcu_do_one_request,
1072 },
1073 {
1074 .base.init = kmb_ocs_hcu_init,
1075 .base.update = kmb_ocs_hcu_update,
1076 .base.final = kmb_ocs_hcu_final,
1077 .base.finup = kmb_ocs_hcu_finup,
1078 .base.digest = kmb_ocs_hcu_digest,
1079 .base.export = kmb_ocs_hcu_export,
1080 .base.import = kmb_ocs_hcu_import,
1081 .base.setkey = kmb_ocs_hcu_setkey,
1082 .base.halg = {
1083 .digestsize = SHA384_DIGEST_SIZE,
1084 .statesize = sizeof(struct ocs_hcu_rctx),
1085 .base = {
1086 .cra_name = "hmac(sha384)",
1087 .cra_driver_name = "hmac-sha384-keembay-ocs",
1088 .cra_priority = 255,
1089 .cra_flags = CRYPTO_ALG_ASYNC,
1090 .cra_blocksize = SHA384_BLOCK_SIZE,
1091 .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
1092 .cra_alignmask = 0,
1093 .cra_module = THIS_MODULE,
1094 .cra_init = kmb_ocs_hcu_hmac_cra_init,
1095 .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
1096 }
1097 },
1098 .op.do_one_request = kmb_ocs_hcu_do_one_request,
1099 },
1100 {
1101 .base.init = kmb_ocs_hcu_init,
1102 .base.update = kmb_ocs_hcu_update,
1103 .base.final = kmb_ocs_hcu_final,
1104 .base.finup = kmb_ocs_hcu_finup,
1105 .base.digest = kmb_ocs_hcu_digest,
1106 .base.export = kmb_ocs_hcu_export,
1107 .base.import = kmb_ocs_hcu_import,
1108 .base.halg = {
1109 .digestsize = SHA512_DIGEST_SIZE,
1110 .statesize = sizeof(struct ocs_hcu_rctx),
1111 .base = {
1112 .cra_name = "sha512",
1113 .cra_driver_name = "sha512-keembay-ocs",
1114 .cra_priority = 255,
1115 .cra_flags = CRYPTO_ALG_ASYNC,
1116 .cra_blocksize = SHA512_BLOCK_SIZE,
1117 .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
1118 .cra_alignmask = 0,
1119 .cra_module = THIS_MODULE,
1120 .cra_init = kmb_ocs_hcu_sha_cra_init,
1121 }
1122 },
1123 .op.do_one_request = kmb_ocs_hcu_do_one_request,
1124 },
1125 {
1126 .base.init = kmb_ocs_hcu_init,
1127 .base.update = kmb_ocs_hcu_update,
1128 .base.final = kmb_ocs_hcu_final,
1129 .base.finup = kmb_ocs_hcu_finup,
1130 .base.digest = kmb_ocs_hcu_digest,
1131 .base.export = kmb_ocs_hcu_export,
1132 .base.import = kmb_ocs_hcu_import,
1133 .base.setkey = kmb_ocs_hcu_setkey,
1134 .base.halg = {
1135 .digestsize = SHA512_DIGEST_SIZE,
1136 .statesize = sizeof(struct ocs_hcu_rctx),
1137 .base = {
1138 .cra_name = "hmac(sha512)",
1139 .cra_driver_name = "hmac-sha512-keembay-ocs",
1140 .cra_priority = 255,
1141 .cra_flags = CRYPTO_ALG_ASYNC,
1142 .cra_blocksize = SHA512_BLOCK_SIZE,
1143 .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
1144 .cra_alignmask = 0,
1145 .cra_module = THIS_MODULE,
1146 .cra_init = kmb_ocs_hcu_hmac_cra_init,
1147 .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
1148 }
1149 },
1150 .op.do_one_request = kmb_ocs_hcu_do_one_request,
1151 },
1152 };
1153
1154 /* Device tree driver match. */
1155 static const struct of_device_id kmb_ocs_hcu_of_match[] = {
1156 {
1157 .compatible = "intel,keembay-ocs-hcu",
1158 },
1159 {}
1160 };
1161 MODULE_DEVICE_TABLE(of, kmb_ocs_hcu_of_match);
1162
kmb_ocs_hcu_remove(struct platform_device * pdev)1163 static void kmb_ocs_hcu_remove(struct platform_device *pdev)
1164 {
1165 struct ocs_hcu_dev *hcu_dev = platform_get_drvdata(pdev);
1166
1167 crypto_engine_unregister_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs));
1168
1169 crypto_engine_exit(hcu_dev->engine);
1170
1171 spin_lock_bh(&ocs_hcu.lock);
1172 list_del(&hcu_dev->list);
1173 spin_unlock_bh(&ocs_hcu.lock);
1174 }
1175
kmb_ocs_hcu_probe(struct platform_device * pdev)1176 static int kmb_ocs_hcu_probe(struct platform_device *pdev)
1177 {
1178 struct device *dev = &pdev->dev;
1179 struct ocs_hcu_dev *hcu_dev;
1180 int rc;
1181
1182 hcu_dev = devm_kzalloc(dev, sizeof(*hcu_dev), GFP_KERNEL);
1183 if (!hcu_dev)
1184 return -ENOMEM;
1185
1186 hcu_dev->dev = dev;
1187
1188 platform_set_drvdata(pdev, hcu_dev);
1189 rc = dma_set_mask_and_coherent(&pdev->dev, OCS_HCU_DMA_BIT_MASK);
1190 if (rc)
1191 return rc;
1192
1193 hcu_dev->io_base = devm_platform_ioremap_resource(pdev, 0);
1194 if (IS_ERR(hcu_dev->io_base))
1195 return PTR_ERR(hcu_dev->io_base);
1196
1197 init_completion(&hcu_dev->irq_done);
1198
1199 /* Get and request IRQ. */
1200 hcu_dev->irq = platform_get_irq(pdev, 0);
1201 if (hcu_dev->irq < 0)
1202 return hcu_dev->irq;
1203
1204 rc = devm_request_threaded_irq(&pdev->dev, hcu_dev->irq,
1205 ocs_hcu_irq_handler, NULL, 0,
1206 "keembay-ocs-hcu", hcu_dev);
1207 if (rc < 0)
1208 return rc;
1209
1210 INIT_LIST_HEAD(&hcu_dev->list);
1211
1212 spin_lock_bh(&ocs_hcu.lock);
1213 list_add_tail(&hcu_dev->list, &ocs_hcu.dev_list);
1214 spin_unlock_bh(&ocs_hcu.lock);
1215
1216 /* Initialize crypto engine */
1217 hcu_dev->engine = crypto_engine_alloc_init(dev, 1);
1218 if (!hcu_dev->engine) {
1219 rc = -ENOMEM;
1220 goto list_del;
1221 }
1222
1223 rc = crypto_engine_start(hcu_dev->engine);
1224 if (rc) {
1225 dev_err(dev, "Could not start engine.\n");
1226 goto cleanup;
1227 }
1228
1229 /* Security infrastructure guarantees OCS clock is enabled. */
1230
1231 rc = crypto_engine_register_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs));
1232 if (rc) {
1233 dev_err(dev, "Could not register algorithms.\n");
1234 goto cleanup;
1235 }
1236
1237 return 0;
1238
1239 cleanup:
1240 crypto_engine_exit(hcu_dev->engine);
1241 list_del:
1242 spin_lock_bh(&ocs_hcu.lock);
1243 list_del(&hcu_dev->list);
1244 spin_unlock_bh(&ocs_hcu.lock);
1245
1246 return rc;
1247 }
1248
1249 /* The OCS driver is a platform device. */
1250 static struct platform_driver kmb_ocs_hcu_driver = {
1251 .probe = kmb_ocs_hcu_probe,
1252 .remove = kmb_ocs_hcu_remove,
1253 .driver = {
1254 .name = DRV_NAME,
1255 .of_match_table = kmb_ocs_hcu_of_match,
1256 },
1257 };
1258
1259 module_platform_driver(kmb_ocs_hcu_driver);
1260
1261 MODULE_LICENSE("GPL");
1262