1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2014 Imagination Technologies 4 * Authors: Will Thomas, James Hartley 5 * 6 * Interface structure taken from omap-sham driver 7 */ 8 9 #include <linux/clk.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmaengine.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/platform_device.h> 17 #include <linux/scatterlist.h> 18 19 #include <crypto/internal/hash.h> 20 #include <crypto/md5.h> 21 #include <crypto/sha1.h> 22 #include <crypto/sha2.h> 23 24 #define CR_RESET 0 25 #define CR_RESET_SET 1 26 #define CR_RESET_UNSET 0 27 28 #define CR_MESSAGE_LENGTH_H 0x4 29 #define CR_MESSAGE_LENGTH_L 0x8 30 31 #define CR_CONTROL 0xc 32 #define CR_CONTROL_BYTE_ORDER_3210 0 33 #define CR_CONTROL_BYTE_ORDER_0123 1 34 #define CR_CONTROL_BYTE_ORDER_2310 2 35 #define CR_CONTROL_BYTE_ORDER_1032 3 36 #define CR_CONTROL_BYTE_ORDER_SHIFT 8 37 #define CR_CONTROL_ALGO_MD5 0 38 #define CR_CONTROL_ALGO_SHA1 1 39 #define CR_CONTROL_ALGO_SHA224 2 40 #define CR_CONTROL_ALGO_SHA256 3 41 42 #define CR_INTSTAT 0x10 43 #define CR_INTENAB 0x14 44 #define CR_INTCLEAR 0x18 45 #define CR_INT_RESULTS_AVAILABLE BIT(0) 46 #define CR_INT_NEW_RESULTS_SET BIT(1) 47 #define CR_INT_RESULT_READ_ERR BIT(2) 48 #define CR_INT_MESSAGE_WRITE_ERROR BIT(3) 49 #define CR_INT_STATUS BIT(8) 50 51 #define CR_RESULT_QUEUE 0x1c 52 #define CR_RSD0 0x40 53 #define CR_CORE_REV 0x50 54 #define CR_CORE_DES1 0x60 55 #define CR_CORE_DES2 0x70 56 57 #define DRIVER_FLAGS_BUSY BIT(0) 58 #define DRIVER_FLAGS_FINAL BIT(1) 59 #define DRIVER_FLAGS_DMA_ACTIVE BIT(2) 60 #define DRIVER_FLAGS_OUTPUT_READY BIT(3) 61 #define DRIVER_FLAGS_INIT BIT(4) 62 #define DRIVER_FLAGS_CPU BIT(5) 63 #define DRIVER_FLAGS_DMA_READY BIT(6) 64 #define DRIVER_FLAGS_ERROR BIT(7) 65 #define DRIVER_FLAGS_SG BIT(8) 66 #define DRIVER_FLAGS_SHA1 BIT(18) 67 #define DRIVER_FLAGS_SHA224 BIT(19) 68 #define DRIVER_FLAGS_SHA256 BIT(20) 69 #define DRIVER_FLAGS_MD5 BIT(21) 70 71 #define IMG_HASH_QUEUE_LENGTH 20 72 #define IMG_HASH_DMA_BURST 4 73 #define IMG_HASH_DMA_THRESHOLD 64 74 75 #ifdef __LITTLE_ENDIAN 76 #define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_3210 77 #else 78 #define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_0123 79 #endif 80 81 struct img_hash_dev; 82 83 struct img_hash_request_ctx { 84 struct img_hash_dev *hdev; 85 u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32)); 86 unsigned long flags; 87 size_t digsize; 88 89 dma_addr_t dma_addr; 90 size_t dma_ct; 91 92 /* sg root */ 93 struct scatterlist *sgfirst; 94 /* walk state */ 95 struct scatterlist *sg; 96 size_t nents; 97 size_t offset; 98 unsigned int total; 99 size_t sent; 100 101 unsigned long op; 102 103 size_t bufcnt; 104 struct ahash_request fallback_req; 105 106 /* Zero length buffer must remain last member of struct */ 107 u8 buffer[] __aligned(sizeof(u32)); 108 }; 109 110 struct img_hash_ctx { 111 struct img_hash_dev *hdev; 112 unsigned long flags; 113 struct crypto_ahash *fallback; 114 }; 115 116 struct img_hash_dev { 117 struct list_head list; 118 struct device *dev; 119 struct clk *hash_clk; 120 struct clk *sys_clk; 121 void __iomem *io_base; 122 123 phys_addr_t bus_addr; 124 void __iomem *cpu_addr; 125 126 spinlock_t lock; 127 int err; 128 struct tasklet_struct done_task; 129 struct tasklet_struct dma_task; 130 131 unsigned long flags; 132 struct crypto_queue queue; 133 struct ahash_request *req; 134 135 struct dma_chan *dma_lch; 136 }; 137 138 struct img_hash_drv { 139 struct list_head dev_list; 140 spinlock_t lock; 141 }; 142 143 static struct img_hash_drv img_hash = { 144 .dev_list = LIST_HEAD_INIT(img_hash.dev_list), 145 .lock = __SPIN_LOCK_UNLOCKED(img_hash.lock), 146 }; 147 148 static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset) 149 { 150 return readl_relaxed(hdev->io_base + offset); 151 } 152 153 static inline void img_hash_write(struct img_hash_dev *hdev, 154 u32 offset, u32 value) 155 { 156 writel_relaxed(value, hdev->io_base + offset); 157 } 158 159 static inline __be32 img_hash_read_result_queue(struct img_hash_dev *hdev) 160 { 161 return cpu_to_be32(img_hash_read(hdev, CR_RESULT_QUEUE)); 162 } 163 164 static void img_hash_start(struct img_hash_dev *hdev, bool dma) 165 { 166 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 167 u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT; 168 169 if (ctx->flags & DRIVER_FLAGS_MD5) 170 cr |= CR_CONTROL_ALGO_MD5; 171 else if (ctx->flags & DRIVER_FLAGS_SHA1) 172 cr |= CR_CONTROL_ALGO_SHA1; 173 else if (ctx->flags & DRIVER_FLAGS_SHA224) 174 cr |= CR_CONTROL_ALGO_SHA224; 175 else if (ctx->flags & DRIVER_FLAGS_SHA256) 176 cr |= CR_CONTROL_ALGO_SHA256; 177 dev_dbg(hdev->dev, "Starting hash process\n"); 178 img_hash_write(hdev, CR_CONTROL, cr); 179 180 /* 181 * The hardware block requires two cycles between writing the control 182 * register and writing the first word of data in non DMA mode, to 183 * ensure the first data write is not grouped in burst with the control 184 * register write a read is issued to 'flush' the bus. 185 */ 186 if (!dma) 187 img_hash_read(hdev, CR_CONTROL); 188 } 189 190 static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf, 191 size_t length, int final) 192 { 193 u32 count, len32; 194 const u32 *buffer = (const u32 *)buf; 195 196 dev_dbg(hdev->dev, "xmit_cpu: length: %zu bytes\n", length); 197 198 if (final) 199 hdev->flags |= DRIVER_FLAGS_FINAL; 200 201 len32 = DIV_ROUND_UP(length, sizeof(u32)); 202 203 for (count = 0; count < len32; count++) 204 writel_relaxed(buffer[count], hdev->cpu_addr); 205 206 return -EINPROGRESS; 207 } 208 209 static void img_hash_dma_callback(void *data) 210 { 211 struct img_hash_dev *hdev = data; 212 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 213 214 if (ctx->bufcnt) { 215 img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0); 216 ctx->bufcnt = 0; 217 } 218 if (ctx->sg) 219 tasklet_schedule(&hdev->dma_task); 220 } 221 222 static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg) 223 { 224 struct dma_async_tx_descriptor *desc; 225 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 226 227 ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_TO_DEVICE); 228 if (ctx->dma_ct == 0) { 229 dev_err(hdev->dev, "Invalid DMA sg\n"); 230 hdev->err = -EINVAL; 231 return -EINVAL; 232 } 233 234 desc = dmaengine_prep_slave_sg(hdev->dma_lch, 235 sg, 236 ctx->dma_ct, 237 DMA_MEM_TO_DEV, 238 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 239 if (!desc) { 240 dev_err(hdev->dev, "Null DMA descriptor\n"); 241 hdev->err = -EINVAL; 242 dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE); 243 return -EINVAL; 244 } 245 desc->callback = img_hash_dma_callback; 246 desc->callback_param = hdev; 247 dmaengine_submit(desc); 248 dma_async_issue_pending(hdev->dma_lch); 249 250 return 0; 251 } 252 253 static int img_hash_write_via_cpu(struct img_hash_dev *hdev) 254 { 255 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 256 257 ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg), 258 ctx->buffer, hdev->req->nbytes); 259 260 ctx->total = hdev->req->nbytes; 261 ctx->bufcnt = 0; 262 263 hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL); 264 265 img_hash_start(hdev, false); 266 267 return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1); 268 } 269 270 static int img_hash_finish(struct ahash_request *req) 271 { 272 struct img_hash_request_ctx *ctx = ahash_request_ctx(req); 273 274 if (!req->result) 275 return -EINVAL; 276 277 memcpy(req->result, ctx->digest, ctx->digsize); 278 279 return 0; 280 } 281 282 static void img_hash_copy_hash(struct ahash_request *req) 283 { 284 struct img_hash_request_ctx *ctx = ahash_request_ctx(req); 285 __be32 *hash = (__be32 *)ctx->digest; 286 int i; 287 288 for (i = (ctx->digsize / sizeof(*hash)) - 1; i >= 0; i--) 289 hash[i] = img_hash_read_result_queue(ctx->hdev); 290 } 291 292 static void img_hash_finish_req(struct ahash_request *req, int err) 293 { 294 struct img_hash_request_ctx *ctx = ahash_request_ctx(req); 295 struct img_hash_dev *hdev = ctx->hdev; 296 297 if (!err) { 298 img_hash_copy_hash(req); 299 if (DRIVER_FLAGS_FINAL & hdev->flags) 300 err = img_hash_finish(req); 301 } else { 302 dev_warn(hdev->dev, "Hash failed with error %d\n", err); 303 ctx->flags |= DRIVER_FLAGS_ERROR; 304 } 305 306 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY | 307 DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL); 308 309 if (req->base.complete) 310 ahash_request_complete(req, err); 311 } 312 313 static int img_hash_write_via_dma(struct img_hash_dev *hdev) 314 { 315 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 316 317 img_hash_start(hdev, true); 318 319 dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total); 320 321 if (!ctx->total) 322 hdev->flags |= DRIVER_FLAGS_FINAL; 323 324 hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL; 325 326 tasklet_schedule(&hdev->dma_task); 327 328 return -EINPROGRESS; 329 } 330 331 static int img_hash_dma_init(struct img_hash_dev *hdev) 332 { 333 struct dma_slave_config dma_conf; 334 int err; 335 336 hdev->dma_lch = dma_request_chan(hdev->dev, "tx"); 337 if (IS_ERR(hdev->dma_lch)) { 338 dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n"); 339 return PTR_ERR(hdev->dma_lch); 340 } 341 dma_conf.direction = DMA_MEM_TO_DEV; 342 dma_conf.dst_addr = hdev->bus_addr; 343 dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 344 dma_conf.dst_maxburst = IMG_HASH_DMA_BURST; 345 dma_conf.device_fc = false; 346 347 err = dmaengine_slave_config(hdev->dma_lch, &dma_conf); 348 if (err) { 349 dev_err(hdev->dev, "Couldn't configure DMA slave.\n"); 350 dma_release_channel(hdev->dma_lch); 351 return err; 352 } 353 354 return 0; 355 } 356 357 static void img_hash_dma_task(unsigned long d) 358 { 359 struct img_hash_dev *hdev = (struct img_hash_dev *)d; 360 struct img_hash_request_ctx *ctx; 361 u8 *addr; 362 size_t nbytes, bleft, wsend, len, tbc; 363 struct scatterlist tsg; 364 365 if (!hdev->req) 366 return; 367 368 ctx = ahash_request_ctx(hdev->req); 369 if (!ctx->sg) 370 return; 371 372 addr = sg_virt(ctx->sg); 373 nbytes = ctx->sg->length - ctx->offset; 374 375 /* 376 * The hash accelerator does not support a data valid mask. This means 377 * that if each dma (i.e. per page) is not a multiple of 4 bytes, the 378 * padding bytes in the last word written by that dma would erroneously 379 * be included in the hash. To avoid this we round down the transfer, 380 * and add the excess to the start of the next dma. It does not matter 381 * that the final dma may not be a multiple of 4 bytes as the hashing 382 * block is programmed to accept the correct number of bytes. 383 */ 384 385 bleft = nbytes % 4; 386 wsend = (nbytes / 4); 387 388 if (wsend) { 389 sg_init_one(&tsg, addr + ctx->offset, wsend * 4); 390 if (img_hash_xmit_dma(hdev, &tsg)) { 391 dev_err(hdev->dev, "DMA failed, falling back to CPU"); 392 ctx->flags |= DRIVER_FLAGS_CPU; 393 hdev->err = 0; 394 img_hash_xmit_cpu(hdev, addr + ctx->offset, 395 wsend * 4, 0); 396 ctx->sent += wsend * 4; 397 wsend = 0; 398 } else { 399 ctx->sent += wsend * 4; 400 } 401 } 402 403 if (bleft) { 404 ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents, 405 ctx->buffer, bleft, ctx->sent); 406 tbc = 0; 407 ctx->sg = sg_next(ctx->sg); 408 while (ctx->sg && (ctx->bufcnt < 4)) { 409 len = ctx->sg->length; 410 if (likely(len > (4 - ctx->bufcnt))) 411 len = 4 - ctx->bufcnt; 412 tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents, 413 ctx->buffer + ctx->bufcnt, len, 414 ctx->sent + ctx->bufcnt); 415 ctx->bufcnt += tbc; 416 if (tbc >= ctx->sg->length) { 417 ctx->sg = sg_next(ctx->sg); 418 tbc = 0; 419 } 420 } 421 422 ctx->sent += ctx->bufcnt; 423 ctx->offset = tbc; 424 425 if (!wsend) 426 img_hash_dma_callback(hdev); 427 } else { 428 ctx->offset = 0; 429 ctx->sg = sg_next(ctx->sg); 430 } 431 } 432 433 static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev) 434 { 435 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 436 437 if (ctx->flags & DRIVER_FLAGS_SG) 438 dma_unmap_sg(hdev->dev, ctx->sg, 1, DMA_TO_DEVICE); 439 440 return 0; 441 } 442 443 static int img_hash_process_data(struct img_hash_dev *hdev) 444 { 445 struct ahash_request *req = hdev->req; 446 struct img_hash_request_ctx *ctx = ahash_request_ctx(req); 447 int err = 0; 448 449 ctx->bufcnt = 0; 450 451 if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) { 452 dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n", 453 req->nbytes); 454 err = img_hash_write_via_dma(hdev); 455 } else { 456 dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n", 457 req->nbytes); 458 err = img_hash_write_via_cpu(hdev); 459 } 460 return err; 461 } 462 463 static int img_hash_hw_init(struct img_hash_dev *hdev) 464 { 465 unsigned long long nbits; 466 u32 u, l; 467 468 img_hash_write(hdev, CR_RESET, CR_RESET_SET); 469 img_hash_write(hdev, CR_RESET, CR_RESET_UNSET); 470 img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET); 471 472 nbits = (u64)hdev->req->nbytes << 3; 473 u = nbits >> 32; 474 l = nbits; 475 img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u); 476 img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l); 477 478 if (!(DRIVER_FLAGS_INIT & hdev->flags)) { 479 hdev->flags |= DRIVER_FLAGS_INIT; 480 hdev->err = 0; 481 } 482 dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits); 483 return 0; 484 } 485 486 static int img_hash_init(struct ahash_request *req) 487 { 488 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 489 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 490 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 491 492 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 493 ahash_request_set_callback(&rctx->fallback_req, 494 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 495 req->base.complete, req->base.data); 496 497 return crypto_ahash_init(&rctx->fallback_req); 498 } 499 500 static int img_hash_handle_queue(struct img_hash_dev *hdev, 501 struct ahash_request *req) 502 { 503 struct crypto_async_request *async_req, *backlog; 504 struct img_hash_request_ctx *ctx; 505 unsigned long flags; 506 int err = 0, res = 0; 507 508 spin_lock_irqsave(&hdev->lock, flags); 509 510 if (req) 511 res = ahash_enqueue_request(&hdev->queue, req); 512 513 if (DRIVER_FLAGS_BUSY & hdev->flags) { 514 spin_unlock_irqrestore(&hdev->lock, flags); 515 return res; 516 } 517 518 backlog = crypto_get_backlog(&hdev->queue); 519 async_req = crypto_dequeue_request(&hdev->queue); 520 if (async_req) 521 hdev->flags |= DRIVER_FLAGS_BUSY; 522 523 spin_unlock_irqrestore(&hdev->lock, flags); 524 525 if (!async_req) 526 return res; 527 528 if (backlog) 529 crypto_request_complete(backlog, -EINPROGRESS); 530 531 req = ahash_request_cast(async_req); 532 hdev->req = req; 533 534 ctx = ahash_request_ctx(req); 535 536 dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n", 537 ctx->op, req->nbytes); 538 539 err = img_hash_hw_init(hdev); 540 541 if (!err) 542 err = img_hash_process_data(hdev); 543 544 if (err != -EINPROGRESS) { 545 /* done_task will not finish so do it here */ 546 img_hash_finish_req(req, err); 547 } 548 return res; 549 } 550 551 static int img_hash_update(struct ahash_request *req) 552 { 553 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 554 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 555 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 556 557 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 558 ahash_request_set_callback(&rctx->fallback_req, 559 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 560 req->base.complete, req->base.data); 561 ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); 562 563 return crypto_ahash_update(&rctx->fallback_req); 564 } 565 566 static int img_hash_final(struct ahash_request *req) 567 { 568 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 569 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 570 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 571 572 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 573 ahash_request_set_callback(&rctx->fallback_req, 574 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 575 req->base.complete, req->base.data); 576 ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); 577 578 return crypto_ahash_final(&rctx->fallback_req); 579 } 580 581 static int img_hash_finup(struct ahash_request *req) 582 { 583 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 584 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 585 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 586 587 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 588 ahash_request_set_callback(&rctx->fallback_req, 589 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 590 req->base.complete, req->base.data); 591 ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, 592 req->nbytes); 593 594 595 return crypto_ahash_finup(&rctx->fallback_req); 596 } 597 598 static int img_hash_import(struct ahash_request *req, const void *in) 599 { 600 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 601 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 602 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 603 604 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 605 ahash_request_set_callback(&rctx->fallback_req, 606 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 607 req->base.complete, req->base.data); 608 609 return crypto_ahash_import(&rctx->fallback_req, in); 610 } 611 612 static int img_hash_export(struct ahash_request *req, void *out) 613 { 614 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 615 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 616 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 617 618 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 619 ahash_request_set_callback(&rctx->fallback_req, 620 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 621 req->base.complete, req->base.data); 622 623 return crypto_ahash_export(&rctx->fallback_req, out); 624 } 625 626 static int img_hash_digest(struct ahash_request *req) 627 { 628 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 629 struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm); 630 struct img_hash_request_ctx *ctx = ahash_request_ctx(req); 631 632 spin_lock(&img_hash.lock); 633 if (!tctx->hdev) 634 tctx->hdev = list_first_entry_or_null(&img_hash.dev_list, 635 struct img_hash_dev, list); 636 ctx->hdev = tctx->hdev; 637 spin_unlock(&img_hash.lock); 638 639 ctx->flags = 0; 640 ctx->digsize = crypto_ahash_digestsize(tfm); 641 642 switch (ctx->digsize) { 643 case SHA1_DIGEST_SIZE: 644 ctx->flags |= DRIVER_FLAGS_SHA1; 645 break; 646 case SHA256_DIGEST_SIZE: 647 ctx->flags |= DRIVER_FLAGS_SHA256; 648 break; 649 case SHA224_DIGEST_SIZE: 650 ctx->flags |= DRIVER_FLAGS_SHA224; 651 break; 652 case MD5_DIGEST_SIZE: 653 ctx->flags |= DRIVER_FLAGS_MD5; 654 break; 655 default: 656 return -EINVAL; 657 } 658 659 ctx->bufcnt = 0; 660 ctx->offset = 0; 661 ctx->sent = 0; 662 ctx->total = req->nbytes; 663 ctx->sg = req->src; 664 ctx->sgfirst = req->src; 665 ctx->nents = sg_nents(ctx->sg); 666 667 return img_hash_handle_queue(ctx->hdev, req); 668 } 669 670 static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name) 671 { 672 struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm); 673 674 ctx->fallback = crypto_alloc_ahash(alg_name, 0, 675 CRYPTO_ALG_NEED_FALLBACK); 676 if (IS_ERR(ctx->fallback)) { 677 pr_err("img_hash: Could not load fallback driver.\n"); 678 return PTR_ERR(ctx->fallback); 679 } 680 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 681 sizeof(struct img_hash_request_ctx) + 682 crypto_ahash_reqsize(ctx->fallback) + 683 IMG_HASH_DMA_THRESHOLD); 684 685 return 0; 686 } 687 688 static int img_hash_cra_md5_init(struct crypto_tfm *tfm) 689 { 690 return img_hash_cra_init(tfm, "md5-lib"); 691 } 692 693 static int img_hash_cra_sha1_init(struct crypto_tfm *tfm) 694 { 695 return img_hash_cra_init(tfm, "sha1-lib"); 696 } 697 698 static int img_hash_cra_sha224_init(struct crypto_tfm *tfm) 699 { 700 return img_hash_cra_init(tfm, "sha224-lib"); 701 } 702 703 static int img_hash_cra_sha256_init(struct crypto_tfm *tfm) 704 { 705 return img_hash_cra_init(tfm, "sha256-lib"); 706 } 707 708 static void img_hash_cra_exit(struct crypto_tfm *tfm) 709 { 710 struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm); 711 712 crypto_free_ahash(tctx->fallback); 713 } 714 715 static irqreturn_t img_irq_handler(int irq, void *dev_id) 716 { 717 struct img_hash_dev *hdev = dev_id; 718 u32 reg; 719 720 reg = img_hash_read(hdev, CR_INTSTAT); 721 img_hash_write(hdev, CR_INTCLEAR, reg); 722 723 if (reg & CR_INT_NEW_RESULTS_SET) { 724 dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n"); 725 if (DRIVER_FLAGS_BUSY & hdev->flags) { 726 hdev->flags |= DRIVER_FLAGS_OUTPUT_READY; 727 if (!(DRIVER_FLAGS_CPU & hdev->flags)) 728 hdev->flags |= DRIVER_FLAGS_DMA_READY; 729 tasklet_schedule(&hdev->done_task); 730 } else { 731 dev_warn(hdev->dev, 732 "HASH interrupt when no active requests.\n"); 733 } 734 } else if (reg & CR_INT_RESULTS_AVAILABLE) { 735 dev_warn(hdev->dev, 736 "IRQ triggered before the hash had completed\n"); 737 } else if (reg & CR_INT_RESULT_READ_ERR) { 738 dev_warn(hdev->dev, 739 "Attempt to read from an empty result queue\n"); 740 } else if (reg & CR_INT_MESSAGE_WRITE_ERROR) { 741 dev_warn(hdev->dev, 742 "Data written before the hardware was configured\n"); 743 } 744 return IRQ_HANDLED; 745 } 746 747 static struct ahash_alg img_algs[] = { 748 { 749 .init = img_hash_init, 750 .update = img_hash_update, 751 .final = img_hash_final, 752 .finup = img_hash_finup, 753 .export = img_hash_export, 754 .import = img_hash_import, 755 .digest = img_hash_digest, 756 .halg = { 757 .digestsize = MD5_DIGEST_SIZE, 758 .statesize = sizeof(struct md5_state), 759 .base = { 760 .cra_name = "md5", 761 .cra_driver_name = "img-md5", 762 .cra_priority = 300, 763 .cra_flags = 764 CRYPTO_ALG_ASYNC | 765 CRYPTO_ALG_NEED_FALLBACK, 766 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 767 .cra_ctxsize = sizeof(struct img_hash_ctx), 768 .cra_init = img_hash_cra_md5_init, 769 .cra_exit = img_hash_cra_exit, 770 .cra_module = THIS_MODULE, 771 } 772 } 773 }, 774 { 775 .init = img_hash_init, 776 .update = img_hash_update, 777 .final = img_hash_final, 778 .finup = img_hash_finup, 779 .export = img_hash_export, 780 .import = img_hash_import, 781 .digest = img_hash_digest, 782 .halg = { 783 .digestsize = SHA1_DIGEST_SIZE, 784 .statesize = sizeof(struct sha1_state), 785 .base = { 786 .cra_name = "sha1", 787 .cra_driver_name = "img-sha1", 788 .cra_priority = 300, 789 .cra_flags = 790 CRYPTO_ALG_ASYNC | 791 CRYPTO_ALG_NEED_FALLBACK, 792 .cra_blocksize = SHA1_BLOCK_SIZE, 793 .cra_ctxsize = sizeof(struct img_hash_ctx), 794 .cra_init = img_hash_cra_sha1_init, 795 .cra_exit = img_hash_cra_exit, 796 .cra_module = THIS_MODULE, 797 } 798 } 799 }, 800 { 801 .init = img_hash_init, 802 .update = img_hash_update, 803 .final = img_hash_final, 804 .finup = img_hash_finup, 805 .export = img_hash_export, 806 .import = img_hash_import, 807 .digest = img_hash_digest, 808 .halg = { 809 .digestsize = SHA224_DIGEST_SIZE, 810 .statesize = sizeof(struct sha256_state), 811 .base = { 812 .cra_name = "sha224", 813 .cra_driver_name = "img-sha224", 814 .cra_priority = 300, 815 .cra_flags = 816 CRYPTO_ALG_ASYNC | 817 CRYPTO_ALG_NEED_FALLBACK, 818 .cra_blocksize = SHA224_BLOCK_SIZE, 819 .cra_ctxsize = sizeof(struct img_hash_ctx), 820 .cra_init = img_hash_cra_sha224_init, 821 .cra_exit = img_hash_cra_exit, 822 .cra_module = THIS_MODULE, 823 } 824 } 825 }, 826 { 827 .init = img_hash_init, 828 .update = img_hash_update, 829 .final = img_hash_final, 830 .finup = img_hash_finup, 831 .export = img_hash_export, 832 .import = img_hash_import, 833 .digest = img_hash_digest, 834 .halg = { 835 .digestsize = SHA256_DIGEST_SIZE, 836 .statesize = sizeof(struct sha256_state), 837 .base = { 838 .cra_name = "sha256", 839 .cra_driver_name = "img-sha256", 840 .cra_priority = 300, 841 .cra_flags = 842 CRYPTO_ALG_ASYNC | 843 CRYPTO_ALG_NEED_FALLBACK, 844 .cra_blocksize = SHA256_BLOCK_SIZE, 845 .cra_ctxsize = sizeof(struct img_hash_ctx), 846 .cra_init = img_hash_cra_sha256_init, 847 .cra_exit = img_hash_cra_exit, 848 .cra_module = THIS_MODULE, 849 } 850 } 851 } 852 }; 853 854 static int img_register_algs(struct img_hash_dev *hdev) 855 { 856 int i, err; 857 858 for (i = 0; i < ARRAY_SIZE(img_algs); i++) { 859 err = crypto_register_ahash(&img_algs[i]); 860 if (err) { 861 crypto_unregister_ahashes(img_algs, i); 862 return err; 863 } 864 } 865 866 return 0; 867 } 868 869 static void img_unregister_algs(struct img_hash_dev *hdev) 870 { 871 crypto_unregister_ahashes(img_algs, ARRAY_SIZE(img_algs)); 872 } 873 874 static void img_hash_done_task(unsigned long data) 875 { 876 struct img_hash_dev *hdev = (struct img_hash_dev *)data; 877 int err = 0; 878 879 if (hdev->err == -EINVAL) { 880 err = hdev->err; 881 goto finish; 882 } 883 884 if (!(DRIVER_FLAGS_BUSY & hdev->flags)) { 885 img_hash_handle_queue(hdev, NULL); 886 return; 887 } 888 889 if (DRIVER_FLAGS_CPU & hdev->flags) { 890 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) { 891 hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY; 892 goto finish; 893 } 894 } else if (DRIVER_FLAGS_DMA_READY & hdev->flags) { 895 if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) { 896 hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE; 897 img_hash_write_via_dma_stop(hdev); 898 if (hdev->err) { 899 err = hdev->err; 900 goto finish; 901 } 902 } 903 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) { 904 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | 905 DRIVER_FLAGS_OUTPUT_READY); 906 goto finish; 907 } 908 } 909 return; 910 911 finish: 912 img_hash_finish_req(hdev->req, err); 913 } 914 915 static const struct of_device_id img_hash_match[] __maybe_unused = { 916 { .compatible = "img,hash-accelerator" }, 917 {} 918 }; 919 MODULE_DEVICE_TABLE(of, img_hash_match); 920 921 static int img_hash_probe(struct platform_device *pdev) 922 { 923 struct img_hash_dev *hdev; 924 struct device *dev = &pdev->dev; 925 struct resource *hash_res; 926 int irq; 927 int err; 928 929 hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL); 930 if (hdev == NULL) 931 return -ENOMEM; 932 933 spin_lock_init(&hdev->lock); 934 935 hdev->dev = dev; 936 937 platform_set_drvdata(pdev, hdev); 938 939 INIT_LIST_HEAD(&hdev->list); 940 941 tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev); 942 tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev); 943 944 crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH); 945 946 /* Register bank */ 947 hdev->io_base = devm_platform_ioremap_resource(pdev, 0); 948 if (IS_ERR(hdev->io_base)) { 949 err = PTR_ERR(hdev->io_base); 950 goto res_err; 951 } 952 953 /* Write port (DMA or CPU) */ 954 hdev->cpu_addr = devm_platform_get_and_ioremap_resource(pdev, 1, &hash_res); 955 if (IS_ERR(hdev->cpu_addr)) { 956 err = PTR_ERR(hdev->cpu_addr); 957 goto res_err; 958 } 959 hdev->bus_addr = hash_res->start; 960 961 irq = platform_get_irq(pdev, 0); 962 if (irq < 0) { 963 err = irq; 964 goto res_err; 965 } 966 967 err = devm_request_irq(dev, irq, img_irq_handler, 0, 968 dev_name(dev), hdev); 969 if (err) { 970 dev_err(dev, "unable to request irq\n"); 971 goto res_err; 972 } 973 dev_dbg(dev, "using IRQ channel %d\n", irq); 974 975 hdev->hash_clk = devm_clk_get_enabled(&pdev->dev, "hash"); 976 if (IS_ERR(hdev->hash_clk)) { 977 dev_err(dev, "clock initialization failed.\n"); 978 err = PTR_ERR(hdev->hash_clk); 979 goto res_err; 980 } 981 982 hdev->sys_clk = devm_clk_get_enabled(&pdev->dev, "sys"); 983 if (IS_ERR(hdev->sys_clk)) { 984 dev_err(dev, "clock initialization failed.\n"); 985 err = PTR_ERR(hdev->sys_clk); 986 goto res_err; 987 } 988 989 err = img_hash_dma_init(hdev); 990 if (err) 991 goto res_err; 992 993 dev_dbg(dev, "using %s for DMA transfers\n", 994 dma_chan_name(hdev->dma_lch)); 995 996 spin_lock(&img_hash.lock); 997 list_add_tail(&hdev->list, &img_hash.dev_list); 998 spin_unlock(&img_hash.lock); 999 1000 err = img_register_algs(hdev); 1001 if (err) 1002 goto err_algs; 1003 dev_info(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n"); 1004 1005 return 0; 1006 1007 err_algs: 1008 spin_lock(&img_hash.lock); 1009 list_del(&hdev->list); 1010 spin_unlock(&img_hash.lock); 1011 dma_release_channel(hdev->dma_lch); 1012 res_err: 1013 tasklet_kill(&hdev->done_task); 1014 tasklet_kill(&hdev->dma_task); 1015 1016 return err; 1017 } 1018 1019 static void img_hash_remove(struct platform_device *pdev) 1020 { 1021 struct img_hash_dev *hdev; 1022 1023 hdev = platform_get_drvdata(pdev); 1024 spin_lock(&img_hash.lock); 1025 list_del(&hdev->list); 1026 spin_unlock(&img_hash.lock); 1027 1028 img_unregister_algs(hdev); 1029 1030 tasklet_kill(&hdev->done_task); 1031 tasklet_kill(&hdev->dma_task); 1032 1033 dma_release_channel(hdev->dma_lch); 1034 } 1035 1036 #ifdef CONFIG_PM_SLEEP 1037 static int img_hash_suspend(struct device *dev) 1038 { 1039 struct img_hash_dev *hdev = dev_get_drvdata(dev); 1040 1041 clk_disable_unprepare(hdev->hash_clk); 1042 clk_disable_unprepare(hdev->sys_clk); 1043 1044 return 0; 1045 } 1046 1047 static int img_hash_resume(struct device *dev) 1048 { 1049 struct img_hash_dev *hdev = dev_get_drvdata(dev); 1050 int ret; 1051 1052 ret = clk_prepare_enable(hdev->hash_clk); 1053 if (ret) 1054 return ret; 1055 1056 ret = clk_prepare_enable(hdev->sys_clk); 1057 if (ret) { 1058 clk_disable_unprepare(hdev->hash_clk); 1059 return ret; 1060 } 1061 1062 return 0; 1063 } 1064 #endif /* CONFIG_PM_SLEEP */ 1065 1066 static const struct dev_pm_ops img_hash_pm_ops = { 1067 SET_SYSTEM_SLEEP_PM_OPS(img_hash_suspend, img_hash_resume) 1068 }; 1069 1070 static struct platform_driver img_hash_driver = { 1071 .probe = img_hash_probe, 1072 .remove = img_hash_remove, 1073 .driver = { 1074 .name = "img-hash-accelerator", 1075 .pm = &img_hash_pm_ops, 1076 .of_match_table = img_hash_match, 1077 } 1078 }; 1079 module_platform_driver(img_hash_driver); 1080 1081 MODULE_LICENSE("GPL v2"); 1082 MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver"); 1083 MODULE_AUTHOR("Will Thomas."); 1084 MODULE_AUTHOR("James Hartley <james.hartley@imgtec.com>"); 1085