1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Cryptographic API. 4 * 5 * Support for ATMEL DES/TDES HW acceleration. 6 * 7 * Copyright (c) 2012 Eukréa Electromatique - ATMEL 8 * Author: Nicolas Royer <nicolas@eukrea.com> 9 * 10 * Some ideas are from omap-aes.c drivers. 11 */ 12 13 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/slab.h> 17 #include <linux/err.h> 18 #include <linux/clk.h> 19 #include <linux/io.h> 20 #include <linux/hw_random.h> 21 #include <linux/platform_device.h> 22 23 #include <linux/device.h> 24 #include <linux/dmaengine.h> 25 #include <linux/init.h> 26 #include <linux/errno.h> 27 #include <linux/interrupt.h> 28 #include <linux/irq.h> 29 #include <linux/scatterlist.h> 30 #include <linux/dma-mapping.h> 31 #include <linux/delay.h> 32 #include <linux/crypto.h> 33 #include <crypto/scatterwalk.h> 34 #include <crypto/algapi.h> 35 #include <crypto/internal/des.h> 36 #include <crypto/internal/skcipher.h> 37 #include "atmel-tdes-regs.h" 38 39 #define ATMEL_TDES_PRIORITY 300 40 41 /* TDES flags */ 42 /* Reserve bits [17:16], [13:12], [2:0] for AES Mode Register */ 43 #define TDES_FLAGS_ENCRYPT TDES_MR_CYPHER_ENC 44 #define TDES_FLAGS_OPMODE_MASK (TDES_MR_OPMOD_MASK | TDES_MR_CFBS_MASK) 45 #define TDES_FLAGS_ECB TDES_MR_OPMOD_ECB 46 #define TDES_FLAGS_CBC TDES_MR_OPMOD_CBC 47 48 #define TDES_FLAGS_MODE_MASK (TDES_FLAGS_OPMODE_MASK | TDES_FLAGS_ENCRYPT) 49 50 #define TDES_FLAGS_INIT BIT(3) 51 #define TDES_FLAGS_FAST BIT(4) 52 #define TDES_FLAGS_BUSY BIT(5) 53 #define TDES_FLAGS_DMA BIT(6) 54 55 #define ATMEL_TDES_QUEUE_LENGTH 50 56 57 struct atmel_tdes_caps { 58 bool has_dma; 59 }; 60 61 struct atmel_tdes_dev; 62 63 struct atmel_tdes_ctx { 64 struct atmel_tdes_dev *dd; 65 66 int keylen; 67 u32 key[DES3_EDE_KEY_SIZE / sizeof(u32)]; 68 unsigned long flags; 69 70 u16 block_size; 71 }; 72 73 struct atmel_tdes_reqctx { 74 unsigned long mode; 75 u8 lastc[DES_BLOCK_SIZE]; 76 }; 77 78 struct atmel_tdes_dma { 79 struct dma_chan *chan; 80 struct dma_slave_config dma_conf; 81 }; 82 83 struct atmel_tdes_dev { 84 struct list_head list; 85 unsigned long phys_base; 86 void __iomem *io_base; 87 88 struct atmel_tdes_ctx *ctx; 89 struct device *dev; 90 struct clk *iclk; 91 int irq; 92 93 unsigned long flags; 94 95 spinlock_t lock; 96 struct crypto_queue queue; 97 98 struct tasklet_struct done_task; 99 struct tasklet_struct queue_task; 100 101 struct skcipher_request *req; 102 size_t total; 103 104 struct scatterlist *in_sg; 105 unsigned int nb_in_sg; 106 size_t in_offset; 107 struct scatterlist *out_sg; 108 unsigned int nb_out_sg; 109 size_t out_offset; 110 111 size_t buflen; 112 size_t dma_size; 113 114 void *buf_in; 115 int dma_in; 116 dma_addr_t dma_addr_in; 117 struct atmel_tdes_dma dma_lch_in; 118 119 void *buf_out; 120 int dma_out; 121 dma_addr_t dma_addr_out; 122 struct atmel_tdes_dma dma_lch_out; 123 124 struct atmel_tdes_caps caps; 125 126 u32 hw_version; 127 }; 128 129 struct atmel_tdes_drv { 130 struct list_head dev_list; 131 spinlock_t lock; 132 }; 133 134 static struct atmel_tdes_drv atmel_tdes = { 135 .dev_list = LIST_HEAD_INIT(atmel_tdes.dev_list), 136 .lock = __SPIN_LOCK_UNLOCKED(atmel_tdes.lock), 137 }; 138 139 static int atmel_tdes_sg_copy(struct scatterlist **sg, size_t *offset, 140 void *buf, size_t buflen, size_t total, int out) 141 { 142 size_t count, off = 0; 143 144 while (buflen && total) { 145 count = min((*sg)->length - *offset, total); 146 count = min(count, buflen); 147 148 if (!count) 149 return off; 150 151 scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out); 152 153 off += count; 154 buflen -= count; 155 *offset += count; 156 total -= count; 157 158 if (*offset == (*sg)->length) { 159 *sg = sg_next(*sg); 160 if (*sg) 161 *offset = 0; 162 else 163 total = 0; 164 } 165 } 166 167 return off; 168 } 169 170 static inline u32 atmel_tdes_read(struct atmel_tdes_dev *dd, u32 offset) 171 { 172 return readl_relaxed(dd->io_base + offset); 173 } 174 175 static inline void atmel_tdes_write(struct atmel_tdes_dev *dd, 176 u32 offset, u32 value) 177 { 178 writel_relaxed(value, dd->io_base + offset); 179 } 180 181 static void atmel_tdes_write_n(struct atmel_tdes_dev *dd, u32 offset, 182 const u32 *value, int count) 183 { 184 for (; count--; value++, offset += 4) 185 atmel_tdes_write(dd, offset, *value); 186 } 187 188 static struct atmel_tdes_dev *atmel_tdes_dev_alloc(void) 189 { 190 struct atmel_tdes_dev *tdes_dd; 191 192 spin_lock_bh(&atmel_tdes.lock); 193 /* One TDES IP per SoC. */ 194 tdes_dd = list_first_entry_or_null(&atmel_tdes.dev_list, 195 struct atmel_tdes_dev, list); 196 spin_unlock_bh(&atmel_tdes.lock); 197 return tdes_dd; 198 } 199 200 static int atmel_tdes_hw_init(struct atmel_tdes_dev *dd) 201 { 202 int err; 203 204 err = clk_prepare_enable(dd->iclk); 205 if (err) 206 return err; 207 208 if (!(dd->flags & TDES_FLAGS_INIT)) { 209 atmel_tdes_write(dd, TDES_CR, TDES_CR_SWRST); 210 dd->flags |= TDES_FLAGS_INIT; 211 } 212 213 return 0; 214 } 215 216 static inline unsigned int atmel_tdes_get_version(struct atmel_tdes_dev *dd) 217 { 218 return atmel_tdes_read(dd, TDES_HW_VERSION) & 0x00000fff; 219 } 220 221 static int atmel_tdes_hw_version_init(struct atmel_tdes_dev *dd) 222 { 223 int err; 224 225 err = atmel_tdes_hw_init(dd); 226 if (err) 227 return err; 228 229 dd->hw_version = atmel_tdes_get_version(dd); 230 231 dev_info(dd->dev, 232 "version: 0x%x\n", dd->hw_version); 233 234 clk_disable_unprepare(dd->iclk); 235 236 return 0; 237 } 238 239 static void atmel_tdes_dma_callback(void *data) 240 { 241 struct atmel_tdes_dev *dd = data; 242 243 /* dma_lch_out - completed */ 244 tasklet_schedule(&dd->done_task); 245 } 246 247 static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd) 248 { 249 int err; 250 u32 valmr = TDES_MR_SMOD_PDC; 251 252 err = atmel_tdes_hw_init(dd); 253 254 if (err) 255 return err; 256 257 if (!dd->caps.has_dma) 258 atmel_tdes_write(dd, TDES_PTCR, 259 TDES_PTCR_TXTDIS | TDES_PTCR_RXTDIS); 260 261 /* MR register must be set before IV registers */ 262 if (dd->ctx->keylen > (DES_KEY_SIZE << 1)) { 263 valmr |= TDES_MR_KEYMOD_3KEY; 264 valmr |= TDES_MR_TDESMOD_TDES; 265 } else if (dd->ctx->keylen > DES_KEY_SIZE) { 266 valmr |= TDES_MR_KEYMOD_2KEY; 267 valmr |= TDES_MR_TDESMOD_TDES; 268 } else { 269 valmr |= TDES_MR_TDESMOD_DES; 270 } 271 272 valmr |= dd->flags & TDES_FLAGS_MODE_MASK; 273 274 atmel_tdes_write(dd, TDES_MR, valmr); 275 276 atmel_tdes_write_n(dd, TDES_KEY1W1R, dd->ctx->key, 277 dd->ctx->keylen >> 2); 278 279 if (dd->req->iv && (valmr & TDES_MR_OPMOD_MASK) != TDES_MR_OPMOD_ECB) 280 atmel_tdes_write_n(dd, TDES_IV1R, (void *)dd->req->iv, 2); 281 282 return 0; 283 } 284 285 static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd) 286 { 287 size_t count; 288 289 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS); 290 291 if (dd->flags & TDES_FLAGS_FAST) { 292 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 293 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 294 } else { 295 dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, 296 dd->dma_size, DMA_FROM_DEVICE); 297 298 count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset, 299 dd->buf_out, dd->buflen, dd->dma_size, 1); 300 if (count != dd->dma_size) { 301 dev_dbg(dd->dev, "not all data converted: %zu\n", count); 302 return -EINVAL; 303 } 304 } 305 306 return 0; 307 } 308 309 static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd) 310 { 311 int err = -ENOMEM; 312 313 dd->buf_in = (void *)__get_free_page(GFP_KERNEL); 314 dd->buf_out = (void *)__get_free_page(GFP_KERNEL); 315 dd->buflen = PAGE_SIZE; 316 dd->buflen &= ~(DES_BLOCK_SIZE - 1); 317 318 if (!dd->buf_in || !dd->buf_out) { 319 dev_dbg(dd->dev, "unable to alloc pages.\n"); 320 goto err_alloc; 321 } 322 323 /* MAP here */ 324 dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, 325 dd->buflen, DMA_TO_DEVICE); 326 err = dma_mapping_error(dd->dev, dd->dma_addr_in); 327 if (err) { 328 dev_dbg(dd->dev, "dma %zd bytes error\n", dd->buflen); 329 goto err_map_in; 330 } 331 332 dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, 333 dd->buflen, DMA_FROM_DEVICE); 334 err = dma_mapping_error(dd->dev, dd->dma_addr_out); 335 if (err) { 336 dev_dbg(dd->dev, "dma %zd bytes error\n", dd->buflen); 337 goto err_map_out; 338 } 339 340 return 0; 341 342 err_map_out: 343 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, 344 DMA_TO_DEVICE); 345 err_map_in: 346 err_alloc: 347 free_page((unsigned long)dd->buf_out); 348 free_page((unsigned long)dd->buf_in); 349 return err; 350 } 351 352 static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd) 353 { 354 dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, 355 DMA_FROM_DEVICE); 356 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, 357 DMA_TO_DEVICE); 358 free_page((unsigned long)dd->buf_out); 359 free_page((unsigned long)dd->buf_in); 360 } 361 362 static int atmel_tdes_crypt_pdc(struct atmel_tdes_dev *dd, 363 dma_addr_t dma_addr_in, 364 dma_addr_t dma_addr_out, int length) 365 { 366 int len32; 367 368 dd->dma_size = length; 369 370 if (!(dd->flags & TDES_FLAGS_FAST)) { 371 dma_sync_single_for_device(dd->dev, dma_addr_in, length, 372 DMA_TO_DEVICE); 373 } 374 375 len32 = DIV_ROUND_UP(length, sizeof(u32)); 376 377 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS); 378 atmel_tdes_write(dd, TDES_TPR, dma_addr_in); 379 atmel_tdes_write(dd, TDES_TCR, len32); 380 atmel_tdes_write(dd, TDES_RPR, dma_addr_out); 381 atmel_tdes_write(dd, TDES_RCR, len32); 382 383 /* Enable Interrupt */ 384 atmel_tdes_write(dd, TDES_IER, TDES_INT_ENDRX); 385 386 /* Start DMA transfer */ 387 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTEN | TDES_PTCR_RXTEN); 388 389 return 0; 390 } 391 392 static int atmel_tdes_crypt_dma(struct atmel_tdes_dev *dd, 393 dma_addr_t dma_addr_in, 394 dma_addr_t dma_addr_out, int length) 395 { 396 struct scatterlist sg[2]; 397 struct dma_async_tx_descriptor *in_desc, *out_desc; 398 enum dma_slave_buswidth addr_width; 399 400 dd->dma_size = length; 401 402 if (!(dd->flags & TDES_FLAGS_FAST)) { 403 dma_sync_single_for_device(dd->dev, dma_addr_in, length, 404 DMA_TO_DEVICE); 405 } 406 407 addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 408 409 dd->dma_lch_in.dma_conf.dst_addr_width = addr_width; 410 dd->dma_lch_out.dma_conf.src_addr_width = addr_width; 411 412 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); 413 dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf); 414 415 dd->flags |= TDES_FLAGS_DMA; 416 417 sg_init_table(&sg[0], 1); 418 sg_dma_address(&sg[0]) = dma_addr_in; 419 sg_dma_len(&sg[0]) = length; 420 421 sg_init_table(&sg[1], 1); 422 sg_dma_address(&sg[1]) = dma_addr_out; 423 sg_dma_len(&sg[1]) = length; 424 425 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0], 426 1, DMA_MEM_TO_DEV, 427 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 428 if (!in_desc) 429 return -EINVAL; 430 431 out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1], 432 1, DMA_DEV_TO_MEM, 433 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 434 if (!out_desc) 435 return -EINVAL; 436 437 out_desc->callback = atmel_tdes_dma_callback; 438 out_desc->callback_param = dd; 439 440 dmaengine_submit(out_desc); 441 dma_async_issue_pending(dd->dma_lch_out.chan); 442 443 dmaengine_submit(in_desc); 444 dma_async_issue_pending(dd->dma_lch_in.chan); 445 446 return 0; 447 } 448 449 static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd) 450 { 451 bool fast; 452 int err; 453 size_t count; 454 dma_addr_t addr_in, addr_out; 455 456 fast = !dd->in_offset && !dd->out_offset && 457 dd->in_sg->length == dd->out_sg->length && 458 IS_ALIGNED(dd->in_sg->offset, sizeof(u32)) && 459 IS_ALIGNED(dd->out_sg->offset, sizeof(u32)) && 460 IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size); 461 462 if (fast) { 463 count = min(dd->total, dd->in_sg->length); 464 465 err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 466 if (!err) { 467 dev_dbg(dd->dev, "dma_map_sg() error\n"); 468 return -EINVAL; 469 } 470 471 err = dma_map_sg(dd->dev, dd->out_sg, 1, 472 DMA_FROM_DEVICE); 473 if (!err) { 474 dev_dbg(dd->dev, "dma_map_sg() error\n"); 475 dma_unmap_sg(dd->dev, dd->in_sg, 1, 476 DMA_TO_DEVICE); 477 return -EINVAL; 478 } 479 480 addr_in = sg_dma_address(dd->in_sg); 481 addr_out = sg_dma_address(dd->out_sg); 482 483 dd->flags |= TDES_FLAGS_FAST; 484 485 } else { 486 /* use cache buffers */ 487 count = atmel_tdes_sg_copy(&dd->in_sg, &dd->in_offset, 488 dd->buf_in, dd->buflen, dd->total, 0); 489 490 addr_in = dd->dma_addr_in; 491 addr_out = dd->dma_addr_out; 492 493 dd->flags &= ~TDES_FLAGS_FAST; 494 } 495 496 dd->total -= count; 497 498 if (dd->caps.has_dma) 499 err = atmel_tdes_crypt_dma(dd, addr_in, addr_out, count); 500 else 501 err = atmel_tdes_crypt_pdc(dd, addr_in, addr_out, count); 502 503 if (err && (dd->flags & TDES_FLAGS_FAST)) { 504 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 505 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 506 } 507 508 return err; 509 } 510 511 static void 512 atmel_tdes_set_iv_as_last_ciphertext_block(struct atmel_tdes_dev *dd) 513 { 514 struct skcipher_request *req = dd->req; 515 struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); 516 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 517 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 518 519 if (req->cryptlen < ivsize) 520 return; 521 522 if (rctx->mode & TDES_FLAGS_ENCRYPT) 523 scatterwalk_map_and_copy(req->iv, req->dst, 524 req->cryptlen - ivsize, ivsize, 0); 525 else 526 memcpy(req->iv, rctx->lastc, ivsize); 527 528 } 529 530 static void atmel_tdes_finish_req(struct atmel_tdes_dev *dd, int err) 531 { 532 struct skcipher_request *req = dd->req; 533 struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); 534 535 clk_disable_unprepare(dd->iclk); 536 537 dd->flags &= ~TDES_FLAGS_BUSY; 538 539 if (!err && (rctx->mode & TDES_FLAGS_OPMODE_MASK) != TDES_FLAGS_ECB) 540 atmel_tdes_set_iv_as_last_ciphertext_block(dd); 541 542 skcipher_request_complete(req, err); 543 } 544 545 static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd, 546 struct skcipher_request *req) 547 { 548 struct crypto_async_request *async_req, *backlog; 549 struct atmel_tdes_ctx *ctx; 550 struct atmel_tdes_reqctx *rctx; 551 unsigned long flags; 552 int err, ret = 0; 553 554 spin_lock_irqsave(&dd->lock, flags); 555 if (req) 556 ret = crypto_enqueue_request(&dd->queue, &req->base); 557 if (dd->flags & TDES_FLAGS_BUSY) { 558 spin_unlock_irqrestore(&dd->lock, flags); 559 return ret; 560 } 561 backlog = crypto_get_backlog(&dd->queue); 562 async_req = crypto_dequeue_request(&dd->queue); 563 if (async_req) 564 dd->flags |= TDES_FLAGS_BUSY; 565 spin_unlock_irqrestore(&dd->lock, flags); 566 567 if (!async_req) 568 return ret; 569 570 if (backlog) 571 crypto_request_complete(backlog, -EINPROGRESS); 572 573 req = skcipher_request_cast(async_req); 574 575 /* assign new request to device */ 576 dd->req = req; 577 dd->total = req->cryptlen; 578 dd->in_offset = 0; 579 dd->in_sg = req->src; 580 dd->out_offset = 0; 581 dd->out_sg = req->dst; 582 583 rctx = skcipher_request_ctx(req); 584 ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 585 rctx->mode &= TDES_FLAGS_MODE_MASK; 586 dd->flags = (dd->flags & ~TDES_FLAGS_MODE_MASK) | rctx->mode; 587 dd->ctx = ctx; 588 589 err = atmel_tdes_write_ctrl(dd); 590 if (!err) 591 err = atmel_tdes_crypt_start(dd); 592 if (err) { 593 /* des_task will not finish it, so do it here */ 594 atmel_tdes_finish_req(dd, err); 595 tasklet_schedule(&dd->queue_task); 596 } 597 598 return ret; 599 } 600 601 static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd) 602 { 603 size_t count; 604 605 if (dd->flags & TDES_FLAGS_FAST) { 606 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 607 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 608 } else { 609 dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, dd->dma_size, 610 DMA_FROM_DEVICE); 611 612 count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset, 613 dd->buf_out, dd->buflen, 614 dd->dma_size, 1); 615 if (count != dd->dma_size) { 616 dev_dbg(dd->dev, "not all data converted: %zu\n", count); 617 return -EINVAL; 618 } 619 } 620 621 return 0; 622 } 623 624 static int atmel_tdes_crypt(struct skcipher_request *req, unsigned long mode) 625 { 626 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 627 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(skcipher); 628 struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); 629 struct device *dev = ctx->dd->dev; 630 631 if (!req->cryptlen) 632 return 0; 633 634 if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) { 635 dev_dbg(dev, "request size is not exact amount of DES blocks\n"); 636 return -EINVAL; 637 } 638 ctx->block_size = DES_BLOCK_SIZE; 639 640 rctx->mode = mode; 641 642 if ((mode & TDES_FLAGS_OPMODE_MASK) != TDES_FLAGS_ECB && 643 !(mode & TDES_FLAGS_ENCRYPT)) { 644 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 645 646 if (req->cryptlen >= ivsize) 647 scatterwalk_map_and_copy(rctx->lastc, req->src, 648 req->cryptlen - ivsize, 649 ivsize, 0); 650 } 651 652 return atmel_tdes_handle_queue(ctx->dd, req); 653 } 654 655 static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd) 656 { 657 int ret; 658 659 /* Try to grab 2 DMA channels */ 660 dd->dma_lch_in.chan = dma_request_chan(dd->dev, "tx"); 661 if (IS_ERR(dd->dma_lch_in.chan)) { 662 ret = PTR_ERR(dd->dma_lch_in.chan); 663 goto err_dma_in; 664 } 665 666 dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + 667 TDES_IDATA1R; 668 dd->dma_lch_in.dma_conf.src_maxburst = 1; 669 dd->dma_lch_in.dma_conf.src_addr_width = 670 DMA_SLAVE_BUSWIDTH_4_BYTES; 671 dd->dma_lch_in.dma_conf.dst_maxburst = 1; 672 dd->dma_lch_in.dma_conf.dst_addr_width = 673 DMA_SLAVE_BUSWIDTH_4_BYTES; 674 dd->dma_lch_in.dma_conf.device_fc = false; 675 676 dd->dma_lch_out.chan = dma_request_chan(dd->dev, "rx"); 677 if (IS_ERR(dd->dma_lch_out.chan)) { 678 ret = PTR_ERR(dd->dma_lch_out.chan); 679 goto err_dma_out; 680 } 681 682 dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + 683 TDES_ODATA1R; 684 dd->dma_lch_out.dma_conf.src_maxburst = 1; 685 dd->dma_lch_out.dma_conf.src_addr_width = 686 DMA_SLAVE_BUSWIDTH_4_BYTES; 687 dd->dma_lch_out.dma_conf.dst_maxburst = 1; 688 dd->dma_lch_out.dma_conf.dst_addr_width = 689 DMA_SLAVE_BUSWIDTH_4_BYTES; 690 dd->dma_lch_out.dma_conf.device_fc = false; 691 692 return 0; 693 694 err_dma_out: 695 dma_release_channel(dd->dma_lch_in.chan); 696 err_dma_in: 697 dev_err(dd->dev, "no DMA channel available\n"); 698 return ret; 699 } 700 701 static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd) 702 { 703 dma_release_channel(dd->dma_lch_in.chan); 704 dma_release_channel(dd->dma_lch_out.chan); 705 } 706 707 static int atmel_des_setkey(struct crypto_skcipher *tfm, const u8 *key, 708 unsigned int keylen) 709 { 710 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); 711 int err; 712 713 err = verify_skcipher_des_key(tfm, key); 714 if (err) 715 return err; 716 717 memcpy(ctx->key, key, keylen); 718 ctx->keylen = keylen; 719 720 return 0; 721 } 722 723 static int atmel_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key, 724 unsigned int keylen) 725 { 726 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); 727 int err; 728 729 err = verify_skcipher_des3_key(tfm, key); 730 if (err) 731 return err; 732 733 memcpy(ctx->key, key, keylen); 734 ctx->keylen = keylen; 735 736 return 0; 737 } 738 739 static int atmel_tdes_ecb_encrypt(struct skcipher_request *req) 740 { 741 return atmel_tdes_crypt(req, TDES_FLAGS_ECB | TDES_FLAGS_ENCRYPT); 742 } 743 744 static int atmel_tdes_ecb_decrypt(struct skcipher_request *req) 745 { 746 return atmel_tdes_crypt(req, TDES_FLAGS_ECB); 747 } 748 749 static int atmel_tdes_cbc_encrypt(struct skcipher_request *req) 750 { 751 return atmel_tdes_crypt(req, TDES_FLAGS_CBC | TDES_FLAGS_ENCRYPT); 752 } 753 754 static int atmel_tdes_cbc_decrypt(struct skcipher_request *req) 755 { 756 return atmel_tdes_crypt(req, TDES_FLAGS_CBC); 757 } 758 759 static int atmel_tdes_init_tfm(struct crypto_skcipher *tfm) 760 { 761 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); 762 763 ctx->dd = atmel_tdes_dev_alloc(); 764 if (!ctx->dd) 765 return -ENODEV; 766 767 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_tdes_reqctx)); 768 769 return 0; 770 } 771 772 static void atmel_tdes_skcipher_alg_init(struct skcipher_alg *alg) 773 { 774 alg->base.cra_priority = ATMEL_TDES_PRIORITY; 775 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 776 alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx); 777 alg->base.cra_module = THIS_MODULE; 778 779 alg->init = atmel_tdes_init_tfm; 780 } 781 782 static struct skcipher_alg tdes_algs[] = { 783 { 784 .base.cra_name = "ecb(des)", 785 .base.cra_driver_name = "atmel-ecb-des", 786 .base.cra_blocksize = DES_BLOCK_SIZE, 787 .base.cra_alignmask = 0x7, 788 789 .min_keysize = DES_KEY_SIZE, 790 .max_keysize = DES_KEY_SIZE, 791 .setkey = atmel_des_setkey, 792 .encrypt = atmel_tdes_ecb_encrypt, 793 .decrypt = atmel_tdes_ecb_decrypt, 794 }, 795 { 796 .base.cra_name = "cbc(des)", 797 .base.cra_driver_name = "atmel-cbc-des", 798 .base.cra_blocksize = DES_BLOCK_SIZE, 799 .base.cra_alignmask = 0x7, 800 801 .min_keysize = DES_KEY_SIZE, 802 .max_keysize = DES_KEY_SIZE, 803 .ivsize = DES_BLOCK_SIZE, 804 .setkey = atmel_des_setkey, 805 .encrypt = atmel_tdes_cbc_encrypt, 806 .decrypt = atmel_tdes_cbc_decrypt, 807 }, 808 { 809 .base.cra_name = "ecb(des3_ede)", 810 .base.cra_driver_name = "atmel-ecb-tdes", 811 .base.cra_blocksize = DES_BLOCK_SIZE, 812 .base.cra_alignmask = 0x7, 813 814 .min_keysize = DES3_EDE_KEY_SIZE, 815 .max_keysize = DES3_EDE_KEY_SIZE, 816 .setkey = atmel_tdes_setkey, 817 .encrypt = atmel_tdes_ecb_encrypt, 818 .decrypt = atmel_tdes_ecb_decrypt, 819 }, 820 { 821 .base.cra_name = "cbc(des3_ede)", 822 .base.cra_driver_name = "atmel-cbc-tdes", 823 .base.cra_blocksize = DES_BLOCK_SIZE, 824 .base.cra_alignmask = 0x7, 825 826 .min_keysize = DES3_EDE_KEY_SIZE, 827 .max_keysize = DES3_EDE_KEY_SIZE, 828 .setkey = atmel_tdes_setkey, 829 .encrypt = atmel_tdes_cbc_encrypt, 830 .decrypt = atmel_tdes_cbc_decrypt, 831 .ivsize = DES_BLOCK_SIZE, 832 }, 833 }; 834 835 static void atmel_tdes_queue_task(unsigned long data) 836 { 837 struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *)data; 838 839 atmel_tdes_handle_queue(dd, NULL); 840 } 841 842 static void atmel_tdes_done_task(unsigned long data) 843 { 844 struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *) data; 845 int err; 846 847 if (!(dd->flags & TDES_FLAGS_DMA)) 848 err = atmel_tdes_crypt_pdc_stop(dd); 849 else 850 err = atmel_tdes_crypt_dma_stop(dd); 851 852 if (dd->total && !err) { 853 if (dd->flags & TDES_FLAGS_FAST) { 854 dd->in_sg = sg_next(dd->in_sg); 855 dd->out_sg = sg_next(dd->out_sg); 856 if (!dd->in_sg || !dd->out_sg) 857 err = -EINVAL; 858 } 859 if (!err) 860 err = atmel_tdes_crypt_start(dd); 861 if (!err) 862 return; /* DMA started. Not finishing. */ 863 } 864 865 atmel_tdes_finish_req(dd, err); 866 atmel_tdes_handle_queue(dd, NULL); 867 } 868 869 static irqreturn_t atmel_tdes_irq(int irq, void *dev_id) 870 { 871 struct atmel_tdes_dev *tdes_dd = dev_id; 872 u32 reg; 873 874 reg = atmel_tdes_read(tdes_dd, TDES_ISR); 875 if (reg & atmel_tdes_read(tdes_dd, TDES_IMR)) { 876 atmel_tdes_write(tdes_dd, TDES_IDR, reg); 877 if (TDES_FLAGS_BUSY & tdes_dd->flags) 878 tasklet_schedule(&tdes_dd->done_task); 879 else 880 dev_warn(tdes_dd->dev, "TDES interrupt when no active requests.\n"); 881 return IRQ_HANDLED; 882 } 883 884 return IRQ_NONE; 885 } 886 887 static int atmel_tdes_register_algs(struct atmel_tdes_dev *dd) 888 { 889 int err, i; 890 891 for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) { 892 atmel_tdes_skcipher_alg_init(&tdes_algs[i]); 893 894 err = crypto_register_skcipher(&tdes_algs[i]); 895 if (err) { 896 crypto_unregister_skciphers(tdes_algs, i); 897 return err; 898 } 899 } 900 901 return 0; 902 } 903 904 static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd) 905 { 906 dd->caps.has_dma = 0; 907 908 /* keep only major version number */ 909 switch (dd->hw_version & 0xf00) { 910 case 0x800: 911 case 0x700: 912 dd->caps.has_dma = 1; 913 break; 914 case 0x600: 915 break; 916 default: 917 dev_warn(dd->dev, 918 "Unmanaged tdes version, set minimum capabilities\n"); 919 break; 920 } 921 } 922 923 static const struct of_device_id atmel_tdes_dt_ids[] = { 924 { .compatible = "atmel,at91sam9g46-tdes" }, 925 { /* sentinel */ } 926 }; 927 MODULE_DEVICE_TABLE(of, atmel_tdes_dt_ids); 928 929 static int atmel_tdes_probe(struct platform_device *pdev) 930 { 931 struct atmel_tdes_dev *tdes_dd; 932 struct device *dev = &pdev->dev; 933 struct resource *tdes_res; 934 int err; 935 936 tdes_dd = devm_kmalloc(&pdev->dev, sizeof(*tdes_dd), GFP_KERNEL); 937 if (!tdes_dd) 938 return -ENOMEM; 939 940 tdes_dd->dev = dev; 941 942 platform_set_drvdata(pdev, tdes_dd); 943 944 INIT_LIST_HEAD(&tdes_dd->list); 945 spin_lock_init(&tdes_dd->lock); 946 947 tasklet_init(&tdes_dd->done_task, atmel_tdes_done_task, 948 (unsigned long)tdes_dd); 949 tasklet_init(&tdes_dd->queue_task, atmel_tdes_queue_task, 950 (unsigned long)tdes_dd); 951 952 crypto_init_queue(&tdes_dd->queue, ATMEL_TDES_QUEUE_LENGTH); 953 954 tdes_dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &tdes_res); 955 if (IS_ERR(tdes_dd->io_base)) { 956 err = PTR_ERR(tdes_dd->io_base); 957 goto err_tasklet_kill; 958 } 959 tdes_dd->phys_base = tdes_res->start; 960 961 /* Get the IRQ */ 962 tdes_dd->irq = platform_get_irq(pdev, 0); 963 if (tdes_dd->irq < 0) { 964 err = tdes_dd->irq; 965 goto err_tasklet_kill; 966 } 967 968 err = devm_request_irq(&pdev->dev, tdes_dd->irq, atmel_tdes_irq, 969 IRQF_SHARED, "atmel-tdes", tdes_dd); 970 if (err) 971 goto err_tasklet_kill; 972 973 /* Initializing the clock */ 974 tdes_dd->iclk = devm_clk_get(&pdev->dev, "tdes_clk"); 975 if (IS_ERR(tdes_dd->iclk)) { 976 dev_err(dev, "clock initialization failed.\n"); 977 err = PTR_ERR(tdes_dd->iclk); 978 goto err_tasklet_kill; 979 } 980 981 err = atmel_tdes_hw_version_init(tdes_dd); 982 if (err) 983 goto err_tasklet_kill; 984 985 atmel_tdes_get_cap(tdes_dd); 986 987 err = atmel_tdes_buff_init(tdes_dd); 988 if (err) 989 goto err_tasklet_kill; 990 991 if (tdes_dd->caps.has_dma) { 992 err = atmel_tdes_dma_init(tdes_dd); 993 if (err) 994 goto err_buff_cleanup; 995 996 dev_info(dev, "using %s, %s for DMA transfers\n", 997 dma_chan_name(tdes_dd->dma_lch_in.chan), 998 dma_chan_name(tdes_dd->dma_lch_out.chan)); 999 } 1000 1001 spin_lock(&atmel_tdes.lock); 1002 list_add_tail(&tdes_dd->list, &atmel_tdes.dev_list); 1003 spin_unlock(&atmel_tdes.lock); 1004 1005 err = atmel_tdes_register_algs(tdes_dd); 1006 if (err) 1007 goto err_algs; 1008 1009 dev_info(dev, "Atmel DES/TDES\n"); 1010 1011 return 0; 1012 1013 err_algs: 1014 spin_lock(&atmel_tdes.lock); 1015 list_del(&tdes_dd->list); 1016 spin_unlock(&atmel_tdes.lock); 1017 if (tdes_dd->caps.has_dma) 1018 atmel_tdes_dma_cleanup(tdes_dd); 1019 err_buff_cleanup: 1020 atmel_tdes_buff_cleanup(tdes_dd); 1021 err_tasklet_kill: 1022 tasklet_kill(&tdes_dd->done_task); 1023 tasklet_kill(&tdes_dd->queue_task); 1024 1025 return err; 1026 } 1027 1028 static void atmel_tdes_remove(struct platform_device *pdev) 1029 { 1030 struct atmel_tdes_dev *tdes_dd = platform_get_drvdata(pdev); 1031 1032 spin_lock(&atmel_tdes.lock); 1033 list_del(&tdes_dd->list); 1034 spin_unlock(&atmel_tdes.lock); 1035 1036 crypto_unregister_skciphers(tdes_algs, ARRAY_SIZE(tdes_algs)); 1037 1038 tasklet_kill(&tdes_dd->done_task); 1039 tasklet_kill(&tdes_dd->queue_task); 1040 1041 if (tdes_dd->caps.has_dma) 1042 atmel_tdes_dma_cleanup(tdes_dd); 1043 1044 atmel_tdes_buff_cleanup(tdes_dd); 1045 } 1046 1047 static struct platform_driver atmel_tdes_driver = { 1048 .probe = atmel_tdes_probe, 1049 .remove = atmel_tdes_remove, 1050 .driver = { 1051 .name = "atmel_tdes", 1052 .of_match_table = atmel_tdes_dt_ids, 1053 }, 1054 }; 1055 1056 module_platform_driver(atmel_tdes_driver); 1057 1058 MODULE_DESCRIPTION("Atmel DES/TDES hw acceleration support."); 1059 MODULE_LICENSE("GPL v2"); 1060 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 1061