1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Cryptographic API. 4 * 5 * Support for ATMEL DES/TDES HW acceleration. 6 * 7 * Copyright (c) 2012 Eukréa Electromatique - ATMEL 8 * Author: Nicolas Royer <nicolas@eukrea.com> 9 * 10 * Some ideas are from omap-aes.c drivers. 11 */ 12 13 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/slab.h> 17 #include <linux/err.h> 18 #include <linux/clk.h> 19 #include <linux/io.h> 20 #include <linux/hw_random.h> 21 #include <linux/platform_device.h> 22 23 #include <linux/device.h> 24 #include <linux/dmaengine.h> 25 #include <linux/init.h> 26 #include <linux/errno.h> 27 #include <linux/interrupt.h> 28 #include <linux/irq.h> 29 #include <linux/scatterlist.h> 30 #include <linux/dma-mapping.h> 31 #include <linux/delay.h> 32 #include <linux/crypto.h> 33 #include <crypto/scatterwalk.h> 34 #include <crypto/algapi.h> 35 #include <crypto/internal/des.h> 36 #include <crypto/internal/skcipher.h> 37 #include "atmel-tdes-regs.h" 38 39 #define ATMEL_TDES_PRIORITY 300 40 41 /* TDES flags */ 42 /* Reserve bits [17:16], [13:12], [2:0] for AES Mode Register */ 43 #define TDES_FLAGS_ENCRYPT TDES_MR_CYPHER_ENC 44 #define TDES_FLAGS_OPMODE_MASK (TDES_MR_OPMOD_MASK | TDES_MR_CFBS_MASK) 45 #define TDES_FLAGS_ECB TDES_MR_OPMOD_ECB 46 #define TDES_FLAGS_CBC TDES_MR_OPMOD_CBC 47 48 #define TDES_FLAGS_MODE_MASK (TDES_FLAGS_OPMODE_MASK | TDES_FLAGS_ENCRYPT) 49 50 #define TDES_FLAGS_INIT BIT(3) 51 #define TDES_FLAGS_FAST BIT(4) 52 #define TDES_FLAGS_BUSY BIT(5) 53 #define TDES_FLAGS_DMA BIT(6) 54 55 #define ATMEL_TDES_QUEUE_LENGTH 50 56 57 struct atmel_tdes_caps { 58 bool has_dma; 59 }; 60 61 struct atmel_tdes_dev; 62 63 struct atmel_tdes_ctx { 64 struct atmel_tdes_dev *dd; 65 66 int keylen; 67 u32 key[DES3_EDE_KEY_SIZE / sizeof(u32)]; 68 unsigned long flags; 69 70 u16 block_size; 71 }; 72 73 struct atmel_tdes_reqctx { 74 unsigned long mode; 75 u8 lastc[DES_BLOCK_SIZE]; 76 }; 77 78 struct atmel_tdes_dma { 79 struct dma_chan *chan; 80 struct dma_slave_config dma_conf; 81 }; 82 83 struct atmel_tdes_dev { 84 struct list_head list; 85 unsigned long phys_base; 86 void __iomem *io_base; 87 88 struct atmel_tdes_ctx *ctx; 89 struct device *dev; 90 struct clk *iclk; 91 int irq; 92 93 unsigned long flags; 94 95 spinlock_t lock; 96 struct crypto_queue queue; 97 98 struct tasklet_struct done_task; 99 struct tasklet_struct queue_task; 100 101 struct skcipher_request *req; 102 size_t total; 103 104 struct scatterlist *in_sg; 105 unsigned int nb_in_sg; 106 size_t in_offset; 107 struct scatterlist *out_sg; 108 unsigned int nb_out_sg; 109 size_t out_offset; 110 111 size_t buflen; 112 size_t dma_size; 113 114 void *buf_in; 115 int dma_in; 116 dma_addr_t dma_addr_in; 117 struct atmel_tdes_dma dma_lch_in; 118 119 void *buf_out; 120 int dma_out; 121 dma_addr_t dma_addr_out; 122 struct atmel_tdes_dma dma_lch_out; 123 124 struct atmel_tdes_caps caps; 125 126 u32 hw_version; 127 }; 128 129 struct atmel_tdes_drv { 130 struct list_head dev_list; 131 spinlock_t lock; 132 }; 133 134 static struct atmel_tdes_drv atmel_tdes = { 135 .dev_list = LIST_HEAD_INIT(atmel_tdes.dev_list), 136 .lock = __SPIN_LOCK_UNLOCKED(atmel_tdes.lock), 137 }; 138 139 static int atmel_tdes_sg_copy(struct scatterlist **sg, size_t *offset, 140 void *buf, size_t buflen, size_t total, int out) 141 { 142 size_t count, off = 0; 143 144 while (buflen && total) { 145 count = min((*sg)->length - *offset, total); 146 count = min(count, buflen); 147 148 if (!count) 149 return off; 150 151 scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out); 152 153 off += count; 154 buflen -= count; 155 *offset += count; 156 total -= count; 157 158 if (*offset == (*sg)->length) { 159 *sg = sg_next(*sg); 160 if (*sg) 161 *offset = 0; 162 else 163 total = 0; 164 } 165 } 166 167 return off; 168 } 169 170 static inline u32 atmel_tdes_read(struct atmel_tdes_dev *dd, u32 offset) 171 { 172 return readl_relaxed(dd->io_base + offset); 173 } 174 175 static inline void atmel_tdes_write(struct atmel_tdes_dev *dd, 176 u32 offset, u32 value) 177 { 178 writel_relaxed(value, dd->io_base + offset); 179 } 180 181 static void atmel_tdes_write_n(struct atmel_tdes_dev *dd, u32 offset, 182 const u32 *value, int count) 183 { 184 for (; count--; value++, offset += 4) 185 atmel_tdes_write(dd, offset, *value); 186 } 187 188 static struct atmel_tdes_dev *atmel_tdes_dev_alloc(void) 189 { 190 struct atmel_tdes_dev *tdes_dd; 191 192 spin_lock_bh(&atmel_tdes.lock); 193 /* One TDES IP per SoC. */ 194 tdes_dd = list_first_entry_or_null(&atmel_tdes.dev_list, 195 struct atmel_tdes_dev, list); 196 spin_unlock_bh(&atmel_tdes.lock); 197 return tdes_dd; 198 } 199 200 static int atmel_tdes_hw_init(struct atmel_tdes_dev *dd) 201 { 202 int err; 203 204 err = clk_prepare_enable(dd->iclk); 205 if (err) 206 return err; 207 208 if (!(dd->flags & TDES_FLAGS_INIT)) { 209 atmel_tdes_write(dd, TDES_CR, TDES_CR_SWRST); 210 dd->flags |= TDES_FLAGS_INIT; 211 } 212 213 return 0; 214 } 215 216 static inline unsigned int atmel_tdes_get_version(struct atmel_tdes_dev *dd) 217 { 218 return atmel_tdes_read(dd, TDES_HW_VERSION) & 0x00000fff; 219 } 220 221 static int atmel_tdes_hw_version_init(struct atmel_tdes_dev *dd) 222 { 223 int err; 224 225 err = atmel_tdes_hw_init(dd); 226 if (err) 227 return err; 228 229 dd->hw_version = atmel_tdes_get_version(dd); 230 231 dev_info(dd->dev, 232 "version: 0x%x\n", dd->hw_version); 233 234 clk_disable_unprepare(dd->iclk); 235 236 return 0; 237 } 238 239 static void atmel_tdes_dma_callback(void *data) 240 { 241 struct atmel_tdes_dev *dd = data; 242 243 /* dma_lch_out - completed */ 244 tasklet_schedule(&dd->done_task); 245 } 246 247 static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd) 248 { 249 int err; 250 u32 valmr = TDES_MR_SMOD_PDC; 251 252 err = atmel_tdes_hw_init(dd); 253 254 if (err) 255 return err; 256 257 if (!dd->caps.has_dma) 258 atmel_tdes_write(dd, TDES_PTCR, 259 TDES_PTCR_TXTDIS | TDES_PTCR_RXTDIS); 260 261 /* MR register must be set before IV registers */ 262 if (dd->ctx->keylen > (DES_KEY_SIZE << 1)) { 263 valmr |= TDES_MR_KEYMOD_3KEY; 264 valmr |= TDES_MR_TDESMOD_TDES; 265 } else if (dd->ctx->keylen > DES_KEY_SIZE) { 266 valmr |= TDES_MR_KEYMOD_2KEY; 267 valmr |= TDES_MR_TDESMOD_TDES; 268 } else { 269 valmr |= TDES_MR_TDESMOD_DES; 270 } 271 272 valmr |= dd->flags & TDES_FLAGS_MODE_MASK; 273 274 atmel_tdes_write(dd, TDES_MR, valmr); 275 276 atmel_tdes_write_n(dd, TDES_KEY1W1R, dd->ctx->key, 277 dd->ctx->keylen >> 2); 278 279 if (dd->req->iv && (valmr & TDES_MR_OPMOD_MASK) != TDES_MR_OPMOD_ECB) 280 atmel_tdes_write_n(dd, TDES_IV1R, (void *)dd->req->iv, 2); 281 282 return 0; 283 } 284 285 static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd) 286 { 287 size_t count; 288 289 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS); 290 291 if (dd->flags & TDES_FLAGS_FAST) { 292 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 293 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 294 } else { 295 dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, 296 dd->dma_size, DMA_FROM_DEVICE); 297 298 count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset, 299 dd->buf_out, dd->buflen, dd->dma_size, 1); 300 if (count != dd->dma_size) { 301 dev_dbg(dd->dev, "not all data converted: %zu\n", count); 302 return -EINVAL; 303 } 304 } 305 306 return 0; 307 } 308 309 static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd) 310 { 311 dd->buf_in = (void *)__get_free_page(GFP_KERNEL); 312 dd->buf_out = (void *)__get_free_page(GFP_KERNEL); 313 dd->buflen = PAGE_SIZE & ~(DES_BLOCK_SIZE - 1); 314 315 if (!dd->buf_in || !dd->buf_out) { 316 dev_err(dd->dev, "failed to allocate DMA buffers\n"); 317 goto err_alloc; 318 } 319 320 dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen, DMA_TO_DEVICE); 321 if (dma_mapping_error(dd->dev, dd->dma_addr_in)) 322 goto err_map; 323 324 dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen, DMA_FROM_DEVICE); 325 if (dma_mapping_error(dd->dev, dd->dma_addr_out)) { 326 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); 327 goto err_map; 328 } 329 330 return 0; 331 332 err_map: 333 dev_err(dd->dev, "failed to map %zu bytes for DMA\n", dd->buflen); 334 err_alloc: 335 free_page((unsigned long)dd->buf_out); 336 free_page((unsigned long)dd->buf_in); 337 return -ENOMEM; 338 } 339 340 static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd) 341 { 342 dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, 343 DMA_FROM_DEVICE); 344 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, 345 DMA_TO_DEVICE); 346 free_page((unsigned long)dd->buf_out); 347 free_page((unsigned long)dd->buf_in); 348 } 349 350 static int atmel_tdes_crypt_pdc(struct atmel_tdes_dev *dd, 351 dma_addr_t dma_addr_in, 352 dma_addr_t dma_addr_out) 353 { 354 u32 len32 = DIV_ROUND_UP(dd->dma_size, sizeof(u32)); 355 356 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS); 357 atmel_tdes_write(dd, TDES_TPR, dma_addr_in); 358 atmel_tdes_write(dd, TDES_TCR, len32); 359 atmel_tdes_write(dd, TDES_RPR, dma_addr_out); 360 atmel_tdes_write(dd, TDES_RCR, len32); 361 362 /* Enable Interrupt */ 363 atmel_tdes_write(dd, TDES_IER, TDES_INT_ENDRX); 364 365 /* Start DMA transfer */ 366 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTEN | TDES_PTCR_RXTEN); 367 368 return 0; 369 } 370 371 static int atmel_tdes_crypt_dma(struct atmel_tdes_dev *dd, 372 dma_addr_t dma_addr_in, 373 dma_addr_t dma_addr_out) 374 { 375 struct scatterlist sg_in, sg_out; 376 struct dma_async_tx_descriptor *in_desc, *out_desc; 377 378 dd->dma_lch_in.dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 379 dd->dma_lch_out.dma_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 380 381 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); 382 dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf); 383 384 dd->flags |= TDES_FLAGS_DMA; 385 386 sg_init_table(&sg_in, 1); 387 sg_dma_address(&sg_in) = dma_addr_in; 388 sg_dma_len(&sg_in) = dd->dma_size; 389 390 sg_init_table(&sg_out, 1); 391 sg_dma_address(&sg_out) = dma_addr_out; 392 sg_dma_len(&sg_out) = dd->dma_size; 393 394 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg_in, 1, DMA_MEM_TO_DEV, 395 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 396 if (!in_desc) 397 return -EINVAL; 398 399 out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg_out, 1, DMA_DEV_TO_MEM, 400 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 401 if (!out_desc) 402 return -EINVAL; 403 404 out_desc->callback = atmel_tdes_dma_callback; 405 out_desc->callback_param = dd; 406 407 dmaengine_submit(out_desc); 408 dma_async_issue_pending(dd->dma_lch_out.chan); 409 410 dmaengine_submit(in_desc); 411 dma_async_issue_pending(dd->dma_lch_in.chan); 412 413 return 0; 414 } 415 416 static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd) 417 { 418 bool fast; 419 int err; 420 size_t count; 421 dma_addr_t addr_in, addr_out; 422 423 fast = !dd->in_offset && !dd->out_offset && 424 dd->in_sg->length == dd->out_sg->length && 425 IS_ALIGNED(dd->in_sg->offset, sizeof(u32)) && 426 IS_ALIGNED(dd->out_sg->offset, sizeof(u32)) && 427 IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size); 428 429 if (fast) { 430 count = min(dd->total, dd->in_sg->length); 431 432 err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 433 if (!err) { 434 dev_dbg(dd->dev, "dma_map_sg() error\n"); 435 return -EINVAL; 436 } 437 438 err = dma_map_sg(dd->dev, dd->out_sg, 1, 439 DMA_FROM_DEVICE); 440 if (!err) { 441 dev_dbg(dd->dev, "dma_map_sg() error\n"); 442 dma_unmap_sg(dd->dev, dd->in_sg, 1, 443 DMA_TO_DEVICE); 444 return -EINVAL; 445 } 446 447 addr_in = sg_dma_address(dd->in_sg); 448 addr_out = sg_dma_address(dd->out_sg); 449 450 dd->flags |= TDES_FLAGS_FAST; 451 452 } else { 453 /* use cache buffers */ 454 count = atmel_tdes_sg_copy(&dd->in_sg, &dd->in_offset, 455 dd->buf_in, dd->buflen, dd->total, 0); 456 457 addr_in = dd->dma_addr_in; 458 addr_out = dd->dma_addr_out; 459 460 dd->flags &= ~TDES_FLAGS_FAST; 461 } 462 463 dd->total -= count; 464 dd->dma_size = count; 465 466 if (!fast) { 467 dma_sync_single_for_device(dd->dev, addr_in, dd->dma_size, DMA_TO_DEVICE); 468 dma_sync_single_for_device(dd->dev, addr_out, dd->dma_size, DMA_FROM_DEVICE); 469 } 470 471 if (dd->caps.has_dma) 472 err = atmel_tdes_crypt_dma(dd, addr_in, addr_out); 473 else 474 err = atmel_tdes_crypt_pdc(dd, addr_in, addr_out); 475 476 if (err && fast) { 477 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 478 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 479 } 480 481 return err; 482 } 483 484 static void 485 atmel_tdes_set_iv_as_last_ciphertext_block(struct atmel_tdes_dev *dd) 486 { 487 struct skcipher_request *req = dd->req; 488 struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); 489 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 490 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 491 492 if (req->cryptlen < ivsize) 493 return; 494 495 if (rctx->mode & TDES_FLAGS_ENCRYPT) 496 scatterwalk_map_and_copy(req->iv, req->dst, 497 req->cryptlen - ivsize, ivsize, 0); 498 else 499 memcpy(req->iv, rctx->lastc, ivsize); 500 501 } 502 503 static void atmel_tdes_finish_req(struct atmel_tdes_dev *dd, int err) 504 { 505 struct skcipher_request *req = dd->req; 506 struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); 507 508 clk_disable_unprepare(dd->iclk); 509 510 dd->flags &= ~TDES_FLAGS_BUSY; 511 512 if (!err && (rctx->mode & TDES_FLAGS_OPMODE_MASK) != TDES_FLAGS_ECB) 513 atmel_tdes_set_iv_as_last_ciphertext_block(dd); 514 515 skcipher_request_complete(req, err); 516 } 517 518 static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd, 519 struct skcipher_request *req) 520 { 521 struct crypto_async_request *async_req, *backlog; 522 struct atmel_tdes_ctx *ctx; 523 struct atmel_tdes_reqctx *rctx; 524 unsigned long flags; 525 int err, ret = 0; 526 527 spin_lock_irqsave(&dd->lock, flags); 528 if (req) 529 ret = crypto_enqueue_request(&dd->queue, &req->base); 530 if (dd->flags & TDES_FLAGS_BUSY) { 531 spin_unlock_irqrestore(&dd->lock, flags); 532 return ret; 533 } 534 backlog = crypto_get_backlog(&dd->queue); 535 async_req = crypto_dequeue_request(&dd->queue); 536 if (async_req) 537 dd->flags |= TDES_FLAGS_BUSY; 538 spin_unlock_irqrestore(&dd->lock, flags); 539 540 if (!async_req) 541 return ret; 542 543 if (backlog) 544 crypto_request_complete(backlog, -EINPROGRESS); 545 546 req = skcipher_request_cast(async_req); 547 548 /* assign new request to device */ 549 dd->req = req; 550 dd->total = req->cryptlen; 551 dd->in_offset = 0; 552 dd->in_sg = req->src; 553 dd->out_offset = 0; 554 dd->out_sg = req->dst; 555 556 rctx = skcipher_request_ctx(req); 557 ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 558 rctx->mode &= TDES_FLAGS_MODE_MASK; 559 dd->flags = (dd->flags & ~TDES_FLAGS_MODE_MASK) | rctx->mode; 560 dd->ctx = ctx; 561 562 err = atmel_tdes_write_ctrl(dd); 563 if (!err) 564 err = atmel_tdes_crypt_start(dd); 565 if (err) { 566 /* des_task will not finish it, so do it here */ 567 atmel_tdes_finish_req(dd, err); 568 tasklet_schedule(&dd->queue_task); 569 } 570 571 return ret; 572 } 573 574 static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd) 575 { 576 size_t count; 577 578 if (dd->flags & TDES_FLAGS_FAST) { 579 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 580 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 581 } else { 582 dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, dd->dma_size, 583 DMA_FROM_DEVICE); 584 585 count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset, 586 dd->buf_out, dd->buflen, 587 dd->dma_size, 1); 588 if (count != dd->dma_size) { 589 dev_dbg(dd->dev, "not all data converted: %zu\n", count); 590 return -EINVAL; 591 } 592 } 593 594 return 0; 595 } 596 597 static int atmel_tdes_crypt(struct skcipher_request *req, unsigned long mode) 598 { 599 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 600 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(skcipher); 601 struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); 602 struct device *dev = ctx->dd->dev; 603 604 if (!req->cryptlen) 605 return 0; 606 607 if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) { 608 dev_dbg(dev, "request size is not exact amount of DES blocks\n"); 609 return -EINVAL; 610 } 611 ctx->block_size = DES_BLOCK_SIZE; 612 613 rctx->mode = mode; 614 615 if ((mode & TDES_FLAGS_OPMODE_MASK) != TDES_FLAGS_ECB && 616 !(mode & TDES_FLAGS_ENCRYPT)) { 617 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 618 619 if (req->cryptlen >= ivsize) 620 scatterwalk_map_and_copy(rctx->lastc, req->src, 621 req->cryptlen - ivsize, 622 ivsize, 0); 623 } 624 625 return atmel_tdes_handle_queue(ctx->dd, req); 626 } 627 628 static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd) 629 { 630 int ret; 631 632 /* Try to grab 2 DMA channels */ 633 dd->dma_lch_in.chan = dma_request_chan(dd->dev, "tx"); 634 if (IS_ERR(dd->dma_lch_in.chan)) { 635 ret = PTR_ERR(dd->dma_lch_in.chan); 636 goto err_dma; 637 } 638 639 dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + 640 TDES_IDATA1R; 641 dd->dma_lch_in.dma_conf.src_maxburst = 1; 642 dd->dma_lch_in.dma_conf.src_addr_width = 643 DMA_SLAVE_BUSWIDTH_4_BYTES; 644 dd->dma_lch_in.dma_conf.dst_maxburst = 1; 645 dd->dma_lch_in.dma_conf.dst_addr_width = 646 DMA_SLAVE_BUSWIDTH_4_BYTES; 647 dd->dma_lch_in.dma_conf.device_fc = false; 648 649 dd->dma_lch_out.chan = dma_request_chan(dd->dev, "rx"); 650 if (IS_ERR(dd->dma_lch_out.chan)) { 651 ret = PTR_ERR(dd->dma_lch_out.chan); 652 dma_release_channel(dd->dma_lch_in.chan); 653 goto err_dma; 654 } 655 656 dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + 657 TDES_ODATA1R; 658 dd->dma_lch_out.dma_conf.src_maxburst = 1; 659 dd->dma_lch_out.dma_conf.src_addr_width = 660 DMA_SLAVE_BUSWIDTH_4_BYTES; 661 dd->dma_lch_out.dma_conf.dst_maxburst = 1; 662 dd->dma_lch_out.dma_conf.dst_addr_width = 663 DMA_SLAVE_BUSWIDTH_4_BYTES; 664 dd->dma_lch_out.dma_conf.device_fc = false; 665 666 return 0; 667 668 err_dma: 669 dev_err(dd->dev, "no DMA channel available\n"); 670 return ret; 671 } 672 673 static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd) 674 { 675 dma_release_channel(dd->dma_lch_in.chan); 676 dma_release_channel(dd->dma_lch_out.chan); 677 } 678 679 static int atmel_des_setkey(struct crypto_skcipher *tfm, const u8 *key, 680 unsigned int keylen) 681 { 682 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); 683 int err; 684 685 err = verify_skcipher_des_key(tfm, key); 686 if (err) 687 return err; 688 689 memcpy(ctx->key, key, keylen); 690 ctx->keylen = keylen; 691 692 return 0; 693 } 694 695 static int atmel_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key, 696 unsigned int keylen) 697 { 698 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); 699 int err; 700 701 err = verify_skcipher_des3_key(tfm, key); 702 if (err) 703 return err; 704 705 memcpy(ctx->key, key, keylen); 706 ctx->keylen = keylen; 707 708 return 0; 709 } 710 711 static int atmel_tdes_ecb_encrypt(struct skcipher_request *req) 712 { 713 return atmel_tdes_crypt(req, TDES_FLAGS_ECB | TDES_FLAGS_ENCRYPT); 714 } 715 716 static int atmel_tdes_ecb_decrypt(struct skcipher_request *req) 717 { 718 return atmel_tdes_crypt(req, TDES_FLAGS_ECB); 719 } 720 721 static int atmel_tdes_cbc_encrypt(struct skcipher_request *req) 722 { 723 return atmel_tdes_crypt(req, TDES_FLAGS_CBC | TDES_FLAGS_ENCRYPT); 724 } 725 726 static int atmel_tdes_cbc_decrypt(struct skcipher_request *req) 727 { 728 return atmel_tdes_crypt(req, TDES_FLAGS_CBC); 729 } 730 731 static int atmel_tdes_init_tfm(struct crypto_skcipher *tfm) 732 { 733 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); 734 735 ctx->dd = atmel_tdes_dev_alloc(); 736 if (!ctx->dd) 737 return -ENODEV; 738 739 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_tdes_reqctx)); 740 741 return 0; 742 } 743 744 static void atmel_tdes_skcipher_alg_init(struct skcipher_alg *alg) 745 { 746 alg->base.cra_priority = ATMEL_TDES_PRIORITY; 747 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 748 alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx); 749 alg->base.cra_module = THIS_MODULE; 750 751 alg->init = atmel_tdes_init_tfm; 752 } 753 754 static struct skcipher_alg tdes_algs[] = { 755 { 756 .base.cra_name = "ecb(des)", 757 .base.cra_driver_name = "atmel-ecb-des", 758 .base.cra_blocksize = DES_BLOCK_SIZE, 759 .base.cra_alignmask = 0x7, 760 761 .min_keysize = DES_KEY_SIZE, 762 .max_keysize = DES_KEY_SIZE, 763 .setkey = atmel_des_setkey, 764 .encrypt = atmel_tdes_ecb_encrypt, 765 .decrypt = atmel_tdes_ecb_decrypt, 766 }, 767 { 768 .base.cra_name = "cbc(des)", 769 .base.cra_driver_name = "atmel-cbc-des", 770 .base.cra_blocksize = DES_BLOCK_SIZE, 771 .base.cra_alignmask = 0x7, 772 773 .min_keysize = DES_KEY_SIZE, 774 .max_keysize = DES_KEY_SIZE, 775 .ivsize = DES_BLOCK_SIZE, 776 .setkey = atmel_des_setkey, 777 .encrypt = atmel_tdes_cbc_encrypt, 778 .decrypt = atmel_tdes_cbc_decrypt, 779 }, 780 { 781 .base.cra_name = "ecb(des3_ede)", 782 .base.cra_driver_name = "atmel-ecb-tdes", 783 .base.cra_blocksize = DES_BLOCK_SIZE, 784 .base.cra_alignmask = 0x7, 785 786 .min_keysize = DES3_EDE_KEY_SIZE, 787 .max_keysize = DES3_EDE_KEY_SIZE, 788 .setkey = atmel_tdes_setkey, 789 .encrypt = atmel_tdes_ecb_encrypt, 790 .decrypt = atmel_tdes_ecb_decrypt, 791 }, 792 { 793 .base.cra_name = "cbc(des3_ede)", 794 .base.cra_driver_name = "atmel-cbc-tdes", 795 .base.cra_blocksize = DES_BLOCK_SIZE, 796 .base.cra_alignmask = 0x7, 797 798 .min_keysize = DES3_EDE_KEY_SIZE, 799 .max_keysize = DES3_EDE_KEY_SIZE, 800 .setkey = atmel_tdes_setkey, 801 .encrypt = atmel_tdes_cbc_encrypt, 802 .decrypt = atmel_tdes_cbc_decrypt, 803 .ivsize = DES_BLOCK_SIZE, 804 }, 805 }; 806 807 static void atmel_tdes_queue_task(unsigned long data) 808 { 809 struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *)data; 810 811 atmel_tdes_handle_queue(dd, NULL); 812 } 813 814 static void atmel_tdes_done_task(unsigned long data) 815 { 816 struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *) data; 817 int err; 818 819 if (!(dd->flags & TDES_FLAGS_DMA)) 820 err = atmel_tdes_crypt_pdc_stop(dd); 821 else 822 err = atmel_tdes_crypt_dma_stop(dd); 823 824 if (dd->total && !err) { 825 if (dd->flags & TDES_FLAGS_FAST) { 826 dd->in_sg = sg_next(dd->in_sg); 827 dd->out_sg = sg_next(dd->out_sg); 828 if (!dd->in_sg || !dd->out_sg) 829 err = -EINVAL; 830 } 831 if (!err) 832 err = atmel_tdes_crypt_start(dd); 833 if (!err) 834 return; /* DMA started. Not finishing. */ 835 } 836 837 atmel_tdes_finish_req(dd, err); 838 atmel_tdes_handle_queue(dd, NULL); 839 } 840 841 static irqreturn_t atmel_tdes_irq(int irq, void *dev_id) 842 { 843 struct atmel_tdes_dev *tdes_dd = dev_id; 844 u32 reg; 845 846 reg = atmel_tdes_read(tdes_dd, TDES_ISR); 847 if (reg & atmel_tdes_read(tdes_dd, TDES_IMR)) { 848 atmel_tdes_write(tdes_dd, TDES_IDR, reg); 849 if (TDES_FLAGS_BUSY & tdes_dd->flags) 850 tasklet_schedule(&tdes_dd->done_task); 851 else 852 dev_warn(tdes_dd->dev, "TDES interrupt when no active requests.\n"); 853 return IRQ_HANDLED; 854 } 855 856 return IRQ_NONE; 857 } 858 859 static int atmel_tdes_register_algs(struct atmel_tdes_dev *dd) 860 { 861 int err, i; 862 863 for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) { 864 atmel_tdes_skcipher_alg_init(&tdes_algs[i]); 865 866 err = crypto_register_skcipher(&tdes_algs[i]); 867 if (err) { 868 crypto_unregister_skciphers(tdes_algs, i); 869 return err; 870 } 871 } 872 873 return 0; 874 } 875 876 static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd) 877 { 878 dd->caps.has_dma = 0; 879 880 /* keep only major version number */ 881 switch (dd->hw_version & 0xf00) { 882 case 0x800: 883 case 0x700: 884 dd->caps.has_dma = 1; 885 break; 886 case 0x600: 887 break; 888 default: 889 dev_warn(dd->dev, 890 "Unmanaged tdes version, set minimum capabilities\n"); 891 break; 892 } 893 } 894 895 static const struct of_device_id atmel_tdes_dt_ids[] = { 896 { .compatible = "atmel,at91sam9g46-tdes" }, 897 { /* sentinel */ } 898 }; 899 MODULE_DEVICE_TABLE(of, atmel_tdes_dt_ids); 900 901 static int atmel_tdes_probe(struct platform_device *pdev) 902 { 903 struct atmel_tdes_dev *tdes_dd; 904 struct device *dev = &pdev->dev; 905 struct resource *tdes_res; 906 int err; 907 908 tdes_dd = devm_kzalloc(&pdev->dev, sizeof(*tdes_dd), GFP_KERNEL); 909 if (!tdes_dd) 910 return -ENOMEM; 911 912 tdes_dd->dev = dev; 913 914 platform_set_drvdata(pdev, tdes_dd); 915 916 INIT_LIST_HEAD(&tdes_dd->list); 917 spin_lock_init(&tdes_dd->lock); 918 919 tasklet_init(&tdes_dd->done_task, atmel_tdes_done_task, 920 (unsigned long)tdes_dd); 921 tasklet_init(&tdes_dd->queue_task, atmel_tdes_queue_task, 922 (unsigned long)tdes_dd); 923 924 crypto_init_queue(&tdes_dd->queue, ATMEL_TDES_QUEUE_LENGTH); 925 926 tdes_dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &tdes_res); 927 if (IS_ERR(tdes_dd->io_base)) { 928 err = PTR_ERR(tdes_dd->io_base); 929 goto err_tasklet_kill; 930 } 931 tdes_dd->phys_base = tdes_res->start; 932 933 /* Get the IRQ */ 934 tdes_dd->irq = platform_get_irq(pdev, 0); 935 if (tdes_dd->irq < 0) { 936 err = tdes_dd->irq; 937 goto err_tasklet_kill; 938 } 939 940 err = devm_request_irq(&pdev->dev, tdes_dd->irq, atmel_tdes_irq, 941 IRQF_SHARED, "atmel-tdes", tdes_dd); 942 if (err) 943 goto err_tasklet_kill; 944 945 /* Initializing the clock */ 946 tdes_dd->iclk = devm_clk_get(&pdev->dev, "tdes_clk"); 947 if (IS_ERR(tdes_dd->iclk)) { 948 dev_err(dev, "clock initialization failed.\n"); 949 err = PTR_ERR(tdes_dd->iclk); 950 goto err_tasklet_kill; 951 } 952 953 err = atmel_tdes_hw_version_init(tdes_dd); 954 if (err) 955 goto err_tasklet_kill; 956 957 atmel_tdes_get_cap(tdes_dd); 958 959 err = atmel_tdes_buff_init(tdes_dd); 960 if (err) 961 goto err_tasklet_kill; 962 963 if (tdes_dd->caps.has_dma) { 964 err = atmel_tdes_dma_init(tdes_dd); 965 if (err) 966 goto err_buff_cleanup; 967 968 dev_info(dev, "using %s, %s for DMA transfers\n", 969 dma_chan_name(tdes_dd->dma_lch_in.chan), 970 dma_chan_name(tdes_dd->dma_lch_out.chan)); 971 } 972 973 spin_lock(&atmel_tdes.lock); 974 list_add_tail(&tdes_dd->list, &atmel_tdes.dev_list); 975 spin_unlock(&atmel_tdes.lock); 976 977 err = atmel_tdes_register_algs(tdes_dd); 978 if (err) 979 goto err_algs; 980 981 dev_info(dev, "Atmel DES/TDES\n"); 982 983 return 0; 984 985 err_algs: 986 spin_lock(&atmel_tdes.lock); 987 list_del(&tdes_dd->list); 988 spin_unlock(&atmel_tdes.lock); 989 if (tdes_dd->caps.has_dma) 990 atmel_tdes_dma_cleanup(tdes_dd); 991 err_buff_cleanup: 992 atmel_tdes_buff_cleanup(tdes_dd); 993 err_tasklet_kill: 994 tasklet_kill(&tdes_dd->done_task); 995 tasklet_kill(&tdes_dd->queue_task); 996 997 return err; 998 } 999 1000 static void atmel_tdes_remove(struct platform_device *pdev) 1001 { 1002 struct atmel_tdes_dev *tdes_dd = platform_get_drvdata(pdev); 1003 1004 spin_lock(&atmel_tdes.lock); 1005 list_del(&tdes_dd->list); 1006 spin_unlock(&atmel_tdes.lock); 1007 1008 crypto_unregister_skciphers(tdes_algs, ARRAY_SIZE(tdes_algs)); 1009 1010 tasklet_kill(&tdes_dd->done_task); 1011 tasklet_kill(&tdes_dd->queue_task); 1012 1013 if (tdes_dd->caps.has_dma) 1014 atmel_tdes_dma_cleanup(tdes_dd); 1015 1016 atmel_tdes_buff_cleanup(tdes_dd); 1017 } 1018 1019 static struct platform_driver atmel_tdes_driver = { 1020 .probe = atmel_tdes_probe, 1021 .remove = atmel_tdes_remove, 1022 .driver = { 1023 .name = "atmel_tdes", 1024 .of_match_table = atmel_tdes_dt_ids, 1025 }, 1026 }; 1027 1028 module_platform_driver(atmel_tdes_driver); 1029 1030 MODULE_DESCRIPTION("Atmel DES/TDES hw acceleration support."); 1031 MODULE_LICENSE("GPL v2"); 1032 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 1033