1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Cryptographic API. 4 * 5 * Support for ATMEL DES/TDES HW acceleration. 6 * 7 * Copyright (c) 2012 Eukréa Electromatique - ATMEL 8 * Author: Nicolas Royer <nicolas@eukrea.com> 9 * 10 * Some ideas are from omap-aes.c drivers. 11 */ 12 13 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/slab.h> 17 #include <linux/err.h> 18 #include <linux/clk.h> 19 #include <linux/io.h> 20 #include <linux/hw_random.h> 21 #include <linux/platform_device.h> 22 23 #include <linux/device.h> 24 #include <linux/dmaengine.h> 25 #include <linux/init.h> 26 #include <linux/errno.h> 27 #include <linux/interrupt.h> 28 #include <linux/irq.h> 29 #include <linux/scatterlist.h> 30 #include <linux/dma-mapping.h> 31 #include <linux/delay.h> 32 #include <linux/crypto.h> 33 #include <crypto/scatterwalk.h> 34 #include <crypto/algapi.h> 35 #include <crypto/internal/des.h> 36 #include <crypto/internal/skcipher.h> 37 #include "atmel-tdes-regs.h" 38 39 #define ATMEL_TDES_PRIORITY 300 40 41 /* TDES flags */ 42 /* Reserve bits [17:16], [13:12], [2:0] for AES Mode Register */ 43 #define TDES_FLAGS_ENCRYPT TDES_MR_CYPHER_ENC 44 #define TDES_FLAGS_OPMODE_MASK (TDES_MR_OPMOD_MASK | TDES_MR_CFBS_MASK) 45 #define TDES_FLAGS_ECB TDES_MR_OPMOD_ECB 46 #define TDES_FLAGS_CBC TDES_MR_OPMOD_CBC 47 48 #define TDES_FLAGS_MODE_MASK (TDES_FLAGS_OPMODE_MASK | TDES_FLAGS_ENCRYPT) 49 50 #define TDES_FLAGS_INIT BIT(3) 51 #define TDES_FLAGS_FAST BIT(4) 52 #define TDES_FLAGS_BUSY BIT(5) 53 #define TDES_FLAGS_DMA BIT(6) 54 55 #define ATMEL_TDES_QUEUE_LENGTH 50 56 57 struct atmel_tdes_caps { 58 bool has_dma; 59 }; 60 61 struct atmel_tdes_dev; 62 63 struct atmel_tdes_ctx { 64 struct atmel_tdes_dev *dd; 65 66 int keylen; 67 u32 key[DES3_EDE_KEY_SIZE / sizeof(u32)]; 68 unsigned long flags; 69 70 u16 block_size; 71 }; 72 73 struct atmel_tdes_reqctx { 74 unsigned long mode; 75 u8 lastc[DES_BLOCK_SIZE]; 76 }; 77 78 struct atmel_tdes_dma { 79 struct dma_chan *chan; 80 struct dma_slave_config dma_conf; 81 }; 82 83 struct atmel_tdes_dev { 84 struct list_head list; 85 unsigned long phys_base; 86 void __iomem *io_base; 87 88 struct atmel_tdes_ctx *ctx; 89 struct device *dev; 90 struct clk *iclk; 91 int irq; 92 93 unsigned long flags; 94 95 spinlock_t lock; 96 struct crypto_queue queue; 97 98 struct tasklet_struct done_task; 99 struct tasklet_struct queue_task; 100 101 struct skcipher_request *req; 102 size_t total; 103 104 struct scatterlist *in_sg; 105 unsigned int nb_in_sg; 106 size_t in_offset; 107 struct scatterlist *out_sg; 108 unsigned int nb_out_sg; 109 size_t out_offset; 110 111 size_t buflen; 112 size_t dma_size; 113 114 void *buf_in; 115 int dma_in; 116 dma_addr_t dma_addr_in; 117 struct atmel_tdes_dma dma_lch_in; 118 119 void *buf_out; 120 int dma_out; 121 dma_addr_t dma_addr_out; 122 struct atmel_tdes_dma dma_lch_out; 123 124 struct atmel_tdes_caps caps; 125 126 u32 hw_version; 127 }; 128 129 struct atmel_tdes_drv { 130 struct list_head dev_list; 131 spinlock_t lock; 132 }; 133 134 static struct atmel_tdes_drv atmel_tdes = { 135 .dev_list = LIST_HEAD_INIT(atmel_tdes.dev_list), 136 .lock = __SPIN_LOCK_UNLOCKED(atmel_tdes.lock), 137 }; 138 139 static int atmel_tdes_sg_copy(struct scatterlist **sg, size_t *offset, 140 void *buf, size_t buflen, size_t total, int out) 141 { 142 size_t count, off = 0; 143 144 while (buflen && total) { 145 count = min((*sg)->length - *offset, total); 146 count = min(count, buflen); 147 148 if (!count) 149 return off; 150 151 scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out); 152 153 off += count; 154 buflen -= count; 155 *offset += count; 156 total -= count; 157 158 if (*offset == (*sg)->length) { 159 *sg = sg_next(*sg); 160 if (*sg) 161 *offset = 0; 162 else 163 total = 0; 164 } 165 } 166 167 return off; 168 } 169 170 static inline u32 atmel_tdes_read(struct atmel_tdes_dev *dd, u32 offset) 171 { 172 return readl_relaxed(dd->io_base + offset); 173 } 174 175 static inline void atmel_tdes_write(struct atmel_tdes_dev *dd, 176 u32 offset, u32 value) 177 { 178 writel_relaxed(value, dd->io_base + offset); 179 } 180 181 static void atmel_tdes_write_n(struct atmel_tdes_dev *dd, u32 offset, 182 const u32 *value, int count) 183 { 184 for (; count--; value++, offset += 4) 185 atmel_tdes_write(dd, offset, *value); 186 } 187 188 static struct atmel_tdes_dev *atmel_tdes_dev_alloc(void) 189 { 190 struct atmel_tdes_dev *tdes_dd; 191 192 spin_lock_bh(&atmel_tdes.lock); 193 /* One TDES IP per SoC. */ 194 tdes_dd = list_first_entry_or_null(&atmel_tdes.dev_list, 195 struct atmel_tdes_dev, list); 196 spin_unlock_bh(&atmel_tdes.lock); 197 return tdes_dd; 198 } 199 200 static int atmel_tdes_hw_init(struct atmel_tdes_dev *dd) 201 { 202 int err; 203 204 err = clk_prepare_enable(dd->iclk); 205 if (err) 206 return err; 207 208 if (!(dd->flags & TDES_FLAGS_INIT)) { 209 atmel_tdes_write(dd, TDES_CR, TDES_CR_SWRST); 210 dd->flags |= TDES_FLAGS_INIT; 211 } 212 213 return 0; 214 } 215 216 static inline unsigned int atmel_tdes_get_version(struct atmel_tdes_dev *dd) 217 { 218 return atmel_tdes_read(dd, TDES_HW_VERSION) & 0x00000fff; 219 } 220 221 static int atmel_tdes_hw_version_init(struct atmel_tdes_dev *dd) 222 { 223 int err; 224 225 err = atmel_tdes_hw_init(dd); 226 if (err) 227 return err; 228 229 dd->hw_version = atmel_tdes_get_version(dd); 230 231 dev_info(dd->dev, 232 "version: 0x%x\n", dd->hw_version); 233 234 clk_disable_unprepare(dd->iclk); 235 236 return 0; 237 } 238 239 static void atmel_tdes_dma_callback(void *data) 240 { 241 struct atmel_tdes_dev *dd = data; 242 243 /* dma_lch_out - completed */ 244 tasklet_schedule(&dd->done_task); 245 } 246 247 static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd) 248 { 249 int err; 250 u32 valmr = TDES_MR_SMOD_PDC; 251 252 err = atmel_tdes_hw_init(dd); 253 254 if (err) 255 return err; 256 257 if (!dd->caps.has_dma) 258 atmel_tdes_write(dd, TDES_PTCR, 259 TDES_PTCR_TXTDIS | TDES_PTCR_RXTDIS); 260 261 /* MR register must be set before IV registers */ 262 if (dd->ctx->keylen > (DES_KEY_SIZE << 1)) { 263 valmr |= TDES_MR_KEYMOD_3KEY; 264 valmr |= TDES_MR_TDESMOD_TDES; 265 } else if (dd->ctx->keylen > DES_KEY_SIZE) { 266 valmr |= TDES_MR_KEYMOD_2KEY; 267 valmr |= TDES_MR_TDESMOD_TDES; 268 } else { 269 valmr |= TDES_MR_TDESMOD_DES; 270 } 271 272 valmr |= dd->flags & TDES_FLAGS_MODE_MASK; 273 274 atmel_tdes_write(dd, TDES_MR, valmr); 275 276 atmel_tdes_write_n(dd, TDES_KEY1W1R, dd->ctx->key, 277 dd->ctx->keylen >> 2); 278 279 if (dd->req->iv && (valmr & TDES_MR_OPMOD_MASK) != TDES_MR_OPMOD_ECB) 280 atmel_tdes_write_n(dd, TDES_IV1R, (void *)dd->req->iv, 2); 281 282 return 0; 283 } 284 285 static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd) 286 { 287 int err = 0; 288 size_t count; 289 290 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS); 291 292 if (dd->flags & TDES_FLAGS_FAST) { 293 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 294 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 295 } else { 296 dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, 297 dd->dma_size, DMA_FROM_DEVICE); 298 299 /* copy data */ 300 count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset, 301 dd->buf_out, dd->buflen, dd->dma_size, 1); 302 if (count != dd->dma_size) { 303 err = -EINVAL; 304 dev_dbg(dd->dev, "not all data converted: %zu\n", count); 305 } 306 } 307 308 return err; 309 } 310 311 static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd) 312 { 313 int err = -ENOMEM; 314 315 dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0); 316 dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0); 317 dd->buflen = PAGE_SIZE; 318 dd->buflen &= ~(DES_BLOCK_SIZE - 1); 319 320 if (!dd->buf_in || !dd->buf_out) { 321 dev_dbg(dd->dev, "unable to alloc pages.\n"); 322 goto err_alloc; 323 } 324 325 /* MAP here */ 326 dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, 327 dd->buflen, DMA_TO_DEVICE); 328 err = dma_mapping_error(dd->dev, dd->dma_addr_in); 329 if (err) { 330 dev_dbg(dd->dev, "dma %zd bytes error\n", dd->buflen); 331 goto err_map_in; 332 } 333 334 dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, 335 dd->buflen, DMA_FROM_DEVICE); 336 err = dma_mapping_error(dd->dev, dd->dma_addr_out); 337 if (err) { 338 dev_dbg(dd->dev, "dma %zd bytes error\n", dd->buflen); 339 goto err_map_out; 340 } 341 342 return 0; 343 344 err_map_out: 345 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, 346 DMA_TO_DEVICE); 347 err_map_in: 348 err_alloc: 349 free_page((unsigned long)dd->buf_out); 350 free_page((unsigned long)dd->buf_in); 351 return err; 352 } 353 354 static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd) 355 { 356 dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, 357 DMA_FROM_DEVICE); 358 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, 359 DMA_TO_DEVICE); 360 free_page((unsigned long)dd->buf_out); 361 free_page((unsigned long)dd->buf_in); 362 } 363 364 static int atmel_tdes_crypt_pdc(struct atmel_tdes_dev *dd, 365 dma_addr_t dma_addr_in, 366 dma_addr_t dma_addr_out, int length) 367 { 368 int len32; 369 370 dd->dma_size = length; 371 372 if (!(dd->flags & TDES_FLAGS_FAST)) { 373 dma_sync_single_for_device(dd->dev, dma_addr_in, length, 374 DMA_TO_DEVICE); 375 } 376 377 len32 = DIV_ROUND_UP(length, sizeof(u32)); 378 379 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS); 380 atmel_tdes_write(dd, TDES_TPR, dma_addr_in); 381 atmel_tdes_write(dd, TDES_TCR, len32); 382 atmel_tdes_write(dd, TDES_RPR, dma_addr_out); 383 atmel_tdes_write(dd, TDES_RCR, len32); 384 385 /* Enable Interrupt */ 386 atmel_tdes_write(dd, TDES_IER, TDES_INT_ENDRX); 387 388 /* Start DMA transfer */ 389 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTEN | TDES_PTCR_RXTEN); 390 391 return 0; 392 } 393 394 static int atmel_tdes_crypt_dma(struct atmel_tdes_dev *dd, 395 dma_addr_t dma_addr_in, 396 dma_addr_t dma_addr_out, int length) 397 { 398 struct scatterlist sg[2]; 399 struct dma_async_tx_descriptor *in_desc, *out_desc; 400 enum dma_slave_buswidth addr_width; 401 402 dd->dma_size = length; 403 404 if (!(dd->flags & TDES_FLAGS_FAST)) { 405 dma_sync_single_for_device(dd->dev, dma_addr_in, length, 406 DMA_TO_DEVICE); 407 } 408 409 addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 410 411 dd->dma_lch_in.dma_conf.dst_addr_width = addr_width; 412 dd->dma_lch_out.dma_conf.src_addr_width = addr_width; 413 414 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); 415 dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf); 416 417 dd->flags |= TDES_FLAGS_DMA; 418 419 sg_init_table(&sg[0], 1); 420 sg_dma_address(&sg[0]) = dma_addr_in; 421 sg_dma_len(&sg[0]) = length; 422 423 sg_init_table(&sg[1], 1); 424 sg_dma_address(&sg[1]) = dma_addr_out; 425 sg_dma_len(&sg[1]) = length; 426 427 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0], 428 1, DMA_MEM_TO_DEV, 429 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 430 if (!in_desc) 431 return -EINVAL; 432 433 out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1], 434 1, DMA_DEV_TO_MEM, 435 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 436 if (!out_desc) 437 return -EINVAL; 438 439 out_desc->callback = atmel_tdes_dma_callback; 440 out_desc->callback_param = dd; 441 442 dmaengine_submit(out_desc); 443 dma_async_issue_pending(dd->dma_lch_out.chan); 444 445 dmaengine_submit(in_desc); 446 dma_async_issue_pending(dd->dma_lch_in.chan); 447 448 return 0; 449 } 450 451 static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd) 452 { 453 int err, fast = 0, in, out; 454 size_t count; 455 dma_addr_t addr_in, addr_out; 456 457 if ((!dd->in_offset) && (!dd->out_offset)) { 458 /* check for alignment */ 459 in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)) && 460 IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size); 461 out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)) && 462 IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size); 463 fast = in && out; 464 465 if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg)) 466 fast = 0; 467 } 468 469 470 if (fast) { 471 count = min_t(size_t, dd->total, sg_dma_len(dd->in_sg)); 472 count = min_t(size_t, count, sg_dma_len(dd->out_sg)); 473 474 err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 475 if (!err) { 476 dev_dbg(dd->dev, "dma_map_sg() error\n"); 477 return -EINVAL; 478 } 479 480 err = dma_map_sg(dd->dev, dd->out_sg, 1, 481 DMA_FROM_DEVICE); 482 if (!err) { 483 dev_dbg(dd->dev, "dma_map_sg() error\n"); 484 dma_unmap_sg(dd->dev, dd->in_sg, 1, 485 DMA_TO_DEVICE); 486 return -EINVAL; 487 } 488 489 addr_in = sg_dma_address(dd->in_sg); 490 addr_out = sg_dma_address(dd->out_sg); 491 492 dd->flags |= TDES_FLAGS_FAST; 493 494 } else { 495 /* use cache buffers */ 496 count = atmel_tdes_sg_copy(&dd->in_sg, &dd->in_offset, 497 dd->buf_in, dd->buflen, dd->total, 0); 498 499 addr_in = dd->dma_addr_in; 500 addr_out = dd->dma_addr_out; 501 502 dd->flags &= ~TDES_FLAGS_FAST; 503 } 504 505 dd->total -= count; 506 507 if (dd->caps.has_dma) 508 err = atmel_tdes_crypt_dma(dd, addr_in, addr_out, count); 509 else 510 err = atmel_tdes_crypt_pdc(dd, addr_in, addr_out, count); 511 512 if (err && (dd->flags & TDES_FLAGS_FAST)) { 513 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 514 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 515 } 516 517 return err; 518 } 519 520 static void 521 atmel_tdes_set_iv_as_last_ciphertext_block(struct atmel_tdes_dev *dd) 522 { 523 struct skcipher_request *req = dd->req; 524 struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); 525 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 526 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 527 528 if (req->cryptlen < ivsize) 529 return; 530 531 if (rctx->mode & TDES_FLAGS_ENCRYPT) 532 scatterwalk_map_and_copy(req->iv, req->dst, 533 req->cryptlen - ivsize, ivsize, 0); 534 else 535 memcpy(req->iv, rctx->lastc, ivsize); 536 537 } 538 539 static void atmel_tdes_finish_req(struct atmel_tdes_dev *dd, int err) 540 { 541 struct skcipher_request *req = dd->req; 542 struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); 543 544 clk_disable_unprepare(dd->iclk); 545 546 dd->flags &= ~TDES_FLAGS_BUSY; 547 548 if (!err && (rctx->mode & TDES_FLAGS_OPMODE_MASK) != TDES_FLAGS_ECB) 549 atmel_tdes_set_iv_as_last_ciphertext_block(dd); 550 551 skcipher_request_complete(req, err); 552 } 553 554 static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd, 555 struct skcipher_request *req) 556 { 557 struct crypto_async_request *async_req, *backlog; 558 struct atmel_tdes_ctx *ctx; 559 struct atmel_tdes_reqctx *rctx; 560 unsigned long flags; 561 int err, ret = 0; 562 563 spin_lock_irqsave(&dd->lock, flags); 564 if (req) 565 ret = crypto_enqueue_request(&dd->queue, &req->base); 566 if (dd->flags & TDES_FLAGS_BUSY) { 567 spin_unlock_irqrestore(&dd->lock, flags); 568 return ret; 569 } 570 backlog = crypto_get_backlog(&dd->queue); 571 async_req = crypto_dequeue_request(&dd->queue); 572 if (async_req) 573 dd->flags |= TDES_FLAGS_BUSY; 574 spin_unlock_irqrestore(&dd->lock, flags); 575 576 if (!async_req) 577 return ret; 578 579 if (backlog) 580 crypto_request_complete(backlog, -EINPROGRESS); 581 582 req = skcipher_request_cast(async_req); 583 584 /* assign new request to device */ 585 dd->req = req; 586 dd->total = req->cryptlen; 587 dd->in_offset = 0; 588 dd->in_sg = req->src; 589 dd->out_offset = 0; 590 dd->out_sg = req->dst; 591 592 rctx = skcipher_request_ctx(req); 593 ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 594 rctx->mode &= TDES_FLAGS_MODE_MASK; 595 dd->flags = (dd->flags & ~TDES_FLAGS_MODE_MASK) | rctx->mode; 596 dd->ctx = ctx; 597 598 err = atmel_tdes_write_ctrl(dd); 599 if (!err) 600 err = atmel_tdes_crypt_start(dd); 601 if (err) { 602 /* des_task will not finish it, so do it here */ 603 atmel_tdes_finish_req(dd, err); 604 tasklet_schedule(&dd->queue_task); 605 } 606 607 return ret; 608 } 609 610 static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd) 611 { 612 int err = -EINVAL; 613 size_t count; 614 615 if (dd->flags & TDES_FLAGS_DMA) { 616 err = 0; 617 if (dd->flags & TDES_FLAGS_FAST) { 618 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 619 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 620 } else { 621 dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, 622 dd->dma_size, DMA_FROM_DEVICE); 623 624 /* copy data */ 625 count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset, 626 dd->buf_out, dd->buflen, dd->dma_size, 1); 627 if (count != dd->dma_size) { 628 err = -EINVAL; 629 dev_dbg(dd->dev, "not all data converted: %zu\n", count); 630 } 631 } 632 } 633 return err; 634 } 635 636 static int atmel_tdes_crypt(struct skcipher_request *req, unsigned long mode) 637 { 638 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 639 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(skcipher); 640 struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); 641 struct device *dev = ctx->dd->dev; 642 643 if (!req->cryptlen) 644 return 0; 645 646 if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) { 647 dev_dbg(dev, "request size is not exact amount of DES blocks\n"); 648 return -EINVAL; 649 } 650 ctx->block_size = DES_BLOCK_SIZE; 651 652 rctx->mode = mode; 653 654 if ((mode & TDES_FLAGS_OPMODE_MASK) != TDES_FLAGS_ECB && 655 !(mode & TDES_FLAGS_ENCRYPT)) { 656 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 657 658 if (req->cryptlen >= ivsize) 659 scatterwalk_map_and_copy(rctx->lastc, req->src, 660 req->cryptlen - ivsize, 661 ivsize, 0); 662 } 663 664 return atmel_tdes_handle_queue(ctx->dd, req); 665 } 666 667 static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd) 668 { 669 int ret; 670 671 /* Try to grab 2 DMA channels */ 672 dd->dma_lch_in.chan = dma_request_chan(dd->dev, "tx"); 673 if (IS_ERR(dd->dma_lch_in.chan)) { 674 ret = PTR_ERR(dd->dma_lch_in.chan); 675 goto err_dma_in; 676 } 677 678 dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + 679 TDES_IDATA1R; 680 dd->dma_lch_in.dma_conf.src_maxburst = 1; 681 dd->dma_lch_in.dma_conf.src_addr_width = 682 DMA_SLAVE_BUSWIDTH_4_BYTES; 683 dd->dma_lch_in.dma_conf.dst_maxburst = 1; 684 dd->dma_lch_in.dma_conf.dst_addr_width = 685 DMA_SLAVE_BUSWIDTH_4_BYTES; 686 dd->dma_lch_in.dma_conf.device_fc = false; 687 688 dd->dma_lch_out.chan = dma_request_chan(dd->dev, "rx"); 689 if (IS_ERR(dd->dma_lch_out.chan)) { 690 ret = PTR_ERR(dd->dma_lch_out.chan); 691 goto err_dma_out; 692 } 693 694 dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + 695 TDES_ODATA1R; 696 dd->dma_lch_out.dma_conf.src_maxburst = 1; 697 dd->dma_lch_out.dma_conf.src_addr_width = 698 DMA_SLAVE_BUSWIDTH_4_BYTES; 699 dd->dma_lch_out.dma_conf.dst_maxburst = 1; 700 dd->dma_lch_out.dma_conf.dst_addr_width = 701 DMA_SLAVE_BUSWIDTH_4_BYTES; 702 dd->dma_lch_out.dma_conf.device_fc = false; 703 704 return 0; 705 706 err_dma_out: 707 dma_release_channel(dd->dma_lch_in.chan); 708 err_dma_in: 709 dev_err(dd->dev, "no DMA channel available\n"); 710 return ret; 711 } 712 713 static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd) 714 { 715 dma_release_channel(dd->dma_lch_in.chan); 716 dma_release_channel(dd->dma_lch_out.chan); 717 } 718 719 static int atmel_des_setkey(struct crypto_skcipher *tfm, const u8 *key, 720 unsigned int keylen) 721 { 722 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); 723 int err; 724 725 err = verify_skcipher_des_key(tfm, key); 726 if (err) 727 return err; 728 729 memcpy(ctx->key, key, keylen); 730 ctx->keylen = keylen; 731 732 return 0; 733 } 734 735 static int atmel_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key, 736 unsigned int keylen) 737 { 738 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); 739 int err; 740 741 err = verify_skcipher_des3_key(tfm, key); 742 if (err) 743 return err; 744 745 memcpy(ctx->key, key, keylen); 746 ctx->keylen = keylen; 747 748 return 0; 749 } 750 751 static int atmel_tdes_ecb_encrypt(struct skcipher_request *req) 752 { 753 return atmel_tdes_crypt(req, TDES_FLAGS_ECB | TDES_FLAGS_ENCRYPT); 754 } 755 756 static int atmel_tdes_ecb_decrypt(struct skcipher_request *req) 757 { 758 return atmel_tdes_crypt(req, TDES_FLAGS_ECB); 759 } 760 761 static int atmel_tdes_cbc_encrypt(struct skcipher_request *req) 762 { 763 return atmel_tdes_crypt(req, TDES_FLAGS_CBC | TDES_FLAGS_ENCRYPT); 764 } 765 766 static int atmel_tdes_cbc_decrypt(struct skcipher_request *req) 767 { 768 return atmel_tdes_crypt(req, TDES_FLAGS_CBC); 769 } 770 771 static int atmel_tdes_init_tfm(struct crypto_skcipher *tfm) 772 { 773 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); 774 775 ctx->dd = atmel_tdes_dev_alloc(); 776 if (!ctx->dd) 777 return -ENODEV; 778 779 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_tdes_reqctx)); 780 781 return 0; 782 } 783 784 static void atmel_tdes_skcipher_alg_init(struct skcipher_alg *alg) 785 { 786 alg->base.cra_priority = ATMEL_TDES_PRIORITY; 787 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 788 alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx); 789 alg->base.cra_module = THIS_MODULE; 790 791 alg->init = atmel_tdes_init_tfm; 792 } 793 794 static struct skcipher_alg tdes_algs[] = { 795 { 796 .base.cra_name = "ecb(des)", 797 .base.cra_driver_name = "atmel-ecb-des", 798 .base.cra_blocksize = DES_BLOCK_SIZE, 799 .base.cra_alignmask = 0x7, 800 801 .min_keysize = DES_KEY_SIZE, 802 .max_keysize = DES_KEY_SIZE, 803 .setkey = atmel_des_setkey, 804 .encrypt = atmel_tdes_ecb_encrypt, 805 .decrypt = atmel_tdes_ecb_decrypt, 806 }, 807 { 808 .base.cra_name = "cbc(des)", 809 .base.cra_driver_name = "atmel-cbc-des", 810 .base.cra_blocksize = DES_BLOCK_SIZE, 811 .base.cra_alignmask = 0x7, 812 813 .min_keysize = DES_KEY_SIZE, 814 .max_keysize = DES_KEY_SIZE, 815 .ivsize = DES_BLOCK_SIZE, 816 .setkey = atmel_des_setkey, 817 .encrypt = atmel_tdes_cbc_encrypt, 818 .decrypt = atmel_tdes_cbc_decrypt, 819 }, 820 { 821 .base.cra_name = "ecb(des3_ede)", 822 .base.cra_driver_name = "atmel-ecb-tdes", 823 .base.cra_blocksize = DES_BLOCK_SIZE, 824 .base.cra_alignmask = 0x7, 825 826 .min_keysize = DES3_EDE_KEY_SIZE, 827 .max_keysize = DES3_EDE_KEY_SIZE, 828 .setkey = atmel_tdes_setkey, 829 .encrypt = atmel_tdes_ecb_encrypt, 830 .decrypt = atmel_tdes_ecb_decrypt, 831 }, 832 { 833 .base.cra_name = "cbc(des3_ede)", 834 .base.cra_driver_name = "atmel-cbc-tdes", 835 .base.cra_blocksize = DES_BLOCK_SIZE, 836 .base.cra_alignmask = 0x7, 837 838 .min_keysize = DES3_EDE_KEY_SIZE, 839 .max_keysize = DES3_EDE_KEY_SIZE, 840 .setkey = atmel_tdes_setkey, 841 .encrypt = atmel_tdes_cbc_encrypt, 842 .decrypt = atmel_tdes_cbc_decrypt, 843 .ivsize = DES_BLOCK_SIZE, 844 }, 845 }; 846 847 static void atmel_tdes_queue_task(unsigned long data) 848 { 849 struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *)data; 850 851 atmel_tdes_handle_queue(dd, NULL); 852 } 853 854 static void atmel_tdes_done_task(unsigned long data) 855 { 856 struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *) data; 857 int err; 858 859 if (!(dd->flags & TDES_FLAGS_DMA)) 860 err = atmel_tdes_crypt_pdc_stop(dd); 861 else 862 err = atmel_tdes_crypt_dma_stop(dd); 863 864 if (dd->total && !err) { 865 if (dd->flags & TDES_FLAGS_FAST) { 866 dd->in_sg = sg_next(dd->in_sg); 867 dd->out_sg = sg_next(dd->out_sg); 868 if (!dd->in_sg || !dd->out_sg) 869 err = -EINVAL; 870 } 871 if (!err) 872 err = atmel_tdes_crypt_start(dd); 873 if (!err) 874 return; /* DMA started. Not finishing. */ 875 } 876 877 atmel_tdes_finish_req(dd, err); 878 atmel_tdes_handle_queue(dd, NULL); 879 } 880 881 static irqreturn_t atmel_tdes_irq(int irq, void *dev_id) 882 { 883 struct atmel_tdes_dev *tdes_dd = dev_id; 884 u32 reg; 885 886 reg = atmel_tdes_read(tdes_dd, TDES_ISR); 887 if (reg & atmel_tdes_read(tdes_dd, TDES_IMR)) { 888 atmel_tdes_write(tdes_dd, TDES_IDR, reg); 889 if (TDES_FLAGS_BUSY & tdes_dd->flags) 890 tasklet_schedule(&tdes_dd->done_task); 891 else 892 dev_warn(tdes_dd->dev, "TDES interrupt when no active requests.\n"); 893 return IRQ_HANDLED; 894 } 895 896 return IRQ_NONE; 897 } 898 899 static int atmel_tdes_register_algs(struct atmel_tdes_dev *dd) 900 { 901 int err, i; 902 903 for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) { 904 atmel_tdes_skcipher_alg_init(&tdes_algs[i]); 905 906 err = crypto_register_skcipher(&tdes_algs[i]); 907 if (err) { 908 crypto_unregister_skciphers(tdes_algs, i); 909 return err; 910 } 911 } 912 913 return 0; 914 } 915 916 static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd) 917 { 918 dd->caps.has_dma = 0; 919 920 /* keep only major version number */ 921 switch (dd->hw_version & 0xf00) { 922 case 0x800: 923 case 0x700: 924 dd->caps.has_dma = 1; 925 break; 926 case 0x600: 927 break; 928 default: 929 dev_warn(dd->dev, 930 "Unmanaged tdes version, set minimum capabilities\n"); 931 break; 932 } 933 } 934 935 static const struct of_device_id atmel_tdes_dt_ids[] = { 936 { .compatible = "atmel,at91sam9g46-tdes" }, 937 { /* sentinel */ } 938 }; 939 MODULE_DEVICE_TABLE(of, atmel_tdes_dt_ids); 940 941 static int atmel_tdes_probe(struct platform_device *pdev) 942 { 943 struct atmel_tdes_dev *tdes_dd; 944 struct device *dev = &pdev->dev; 945 struct resource *tdes_res; 946 int err; 947 948 tdes_dd = devm_kmalloc(&pdev->dev, sizeof(*tdes_dd), GFP_KERNEL); 949 if (!tdes_dd) 950 return -ENOMEM; 951 952 tdes_dd->dev = dev; 953 954 platform_set_drvdata(pdev, tdes_dd); 955 956 INIT_LIST_HEAD(&tdes_dd->list); 957 spin_lock_init(&tdes_dd->lock); 958 959 tasklet_init(&tdes_dd->done_task, atmel_tdes_done_task, 960 (unsigned long)tdes_dd); 961 tasklet_init(&tdes_dd->queue_task, atmel_tdes_queue_task, 962 (unsigned long)tdes_dd); 963 964 crypto_init_queue(&tdes_dd->queue, ATMEL_TDES_QUEUE_LENGTH); 965 966 tdes_dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &tdes_res); 967 if (IS_ERR(tdes_dd->io_base)) { 968 err = PTR_ERR(tdes_dd->io_base); 969 goto err_tasklet_kill; 970 } 971 tdes_dd->phys_base = tdes_res->start; 972 973 /* Get the IRQ */ 974 tdes_dd->irq = platform_get_irq(pdev, 0); 975 if (tdes_dd->irq < 0) { 976 err = tdes_dd->irq; 977 goto err_tasklet_kill; 978 } 979 980 err = devm_request_irq(&pdev->dev, tdes_dd->irq, atmel_tdes_irq, 981 IRQF_SHARED, "atmel-tdes", tdes_dd); 982 if (err) { 983 dev_err(dev, "unable to request tdes irq.\n"); 984 goto err_tasklet_kill; 985 } 986 987 /* Initializing the clock */ 988 tdes_dd->iclk = devm_clk_get(&pdev->dev, "tdes_clk"); 989 if (IS_ERR(tdes_dd->iclk)) { 990 dev_err(dev, "clock initialization failed.\n"); 991 err = PTR_ERR(tdes_dd->iclk); 992 goto err_tasklet_kill; 993 } 994 995 err = atmel_tdes_hw_version_init(tdes_dd); 996 if (err) 997 goto err_tasklet_kill; 998 999 atmel_tdes_get_cap(tdes_dd); 1000 1001 err = atmel_tdes_buff_init(tdes_dd); 1002 if (err) 1003 goto err_tasklet_kill; 1004 1005 if (tdes_dd->caps.has_dma) { 1006 err = atmel_tdes_dma_init(tdes_dd); 1007 if (err) 1008 goto err_buff_cleanup; 1009 1010 dev_info(dev, "using %s, %s for DMA transfers\n", 1011 dma_chan_name(tdes_dd->dma_lch_in.chan), 1012 dma_chan_name(tdes_dd->dma_lch_out.chan)); 1013 } 1014 1015 spin_lock(&atmel_tdes.lock); 1016 list_add_tail(&tdes_dd->list, &atmel_tdes.dev_list); 1017 spin_unlock(&atmel_tdes.lock); 1018 1019 err = atmel_tdes_register_algs(tdes_dd); 1020 if (err) 1021 goto err_algs; 1022 1023 dev_info(dev, "Atmel DES/TDES\n"); 1024 1025 return 0; 1026 1027 err_algs: 1028 spin_lock(&atmel_tdes.lock); 1029 list_del(&tdes_dd->list); 1030 spin_unlock(&atmel_tdes.lock); 1031 if (tdes_dd->caps.has_dma) 1032 atmel_tdes_dma_cleanup(tdes_dd); 1033 err_buff_cleanup: 1034 atmel_tdes_buff_cleanup(tdes_dd); 1035 err_tasklet_kill: 1036 tasklet_kill(&tdes_dd->done_task); 1037 tasklet_kill(&tdes_dd->queue_task); 1038 1039 return err; 1040 } 1041 1042 static void atmel_tdes_remove(struct platform_device *pdev) 1043 { 1044 struct atmel_tdes_dev *tdes_dd = platform_get_drvdata(pdev); 1045 1046 spin_lock(&atmel_tdes.lock); 1047 list_del(&tdes_dd->list); 1048 spin_unlock(&atmel_tdes.lock); 1049 1050 crypto_unregister_skciphers(tdes_algs, ARRAY_SIZE(tdes_algs)); 1051 1052 tasklet_kill(&tdes_dd->done_task); 1053 tasklet_kill(&tdes_dd->queue_task); 1054 1055 if (tdes_dd->caps.has_dma) 1056 atmel_tdes_dma_cleanup(tdes_dd); 1057 1058 atmel_tdes_buff_cleanup(tdes_dd); 1059 } 1060 1061 static struct platform_driver atmel_tdes_driver = { 1062 .probe = atmel_tdes_probe, 1063 .remove = atmel_tdes_remove, 1064 .driver = { 1065 .name = "atmel_tdes", 1066 .of_match_table = atmel_tdes_dt_ids, 1067 }, 1068 }; 1069 1070 module_platform_driver(atmel_tdes_driver); 1071 1072 MODULE_DESCRIPTION("Atmel DES/TDES hw acceleration support."); 1073 MODULE_LICENSE("GPL v2"); 1074 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 1075