1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Cryptographic API. 4 * 5 * Support for ATMEL AES HW acceleration. 6 * 7 * Copyright (c) 2012 Eukréa Electromatique - ATMEL 8 * Author: Nicolas Royer <nicolas@eukrea.com> 9 * 10 * Some ideas are from omap-aes.c driver. 11 */ 12 13 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/slab.h> 17 #include <linux/err.h> 18 #include <linux/clk.h> 19 #include <linux/io.h> 20 #include <linux/hw_random.h> 21 #include <linux/platform_device.h> 22 23 #include <linux/device.h> 24 #include <linux/dmaengine.h> 25 #include <linux/init.h> 26 #include <linux/errno.h> 27 #include <linux/interrupt.h> 28 #include <linux/irq.h> 29 #include <linux/scatterlist.h> 30 #include <linux/dma-mapping.h> 31 #include <linux/delay.h> 32 #include <linux/crypto.h> 33 #include <crypto/scatterwalk.h> 34 #include <crypto/algapi.h> 35 #include <crypto/aes.h> 36 #include <crypto/gcm.h> 37 #include <crypto/xts.h> 38 #include <crypto/internal/aead.h> 39 #include <crypto/internal/skcipher.h> 40 #include "atmel-aes-regs.h" 41 #include "atmel-authenc.h" 42 43 #define ATMEL_AES_PRIORITY 300 44 45 #define ATMEL_AES_BUFFER_ORDER 2 46 #define ATMEL_AES_BUFFER_SIZE (PAGE_SIZE << ATMEL_AES_BUFFER_ORDER) 47 48 #define SIZE_IN_WORDS(x) ((x) >> 2) 49 50 /* AES flags */ 51 /* Reserve bits [18:16] [14:12] [1:0] for mode (same as for AES_MR) */ 52 #define AES_FLAGS_ENCRYPT AES_MR_CYPHER_ENC 53 #define AES_FLAGS_GTAGEN AES_MR_GTAGEN 54 #define AES_FLAGS_OPMODE_MASK (AES_MR_OPMOD_MASK | AES_MR_CFBS_MASK) 55 #define AES_FLAGS_ECB AES_MR_OPMOD_ECB 56 #define AES_FLAGS_CBC AES_MR_OPMOD_CBC 57 #define AES_FLAGS_CTR AES_MR_OPMOD_CTR 58 #define AES_FLAGS_GCM AES_MR_OPMOD_GCM 59 #define AES_FLAGS_XTS AES_MR_OPMOD_XTS 60 61 #define AES_FLAGS_MODE_MASK (AES_FLAGS_OPMODE_MASK | \ 62 AES_FLAGS_ENCRYPT | \ 63 AES_FLAGS_GTAGEN) 64 65 #define AES_FLAGS_BUSY BIT(3) 66 #define AES_FLAGS_DUMP_REG BIT(4) 67 #define AES_FLAGS_OWN_SHA BIT(5) 68 69 #define AES_FLAGS_PERSISTENT AES_FLAGS_BUSY 70 71 #define ATMEL_AES_QUEUE_LENGTH 50 72 73 #define ATMEL_AES_DMA_THRESHOLD 256 74 75 76 struct atmel_aes_caps { 77 bool has_dualbuff; 78 bool has_gcm; 79 bool has_xts; 80 bool has_authenc; 81 u32 max_burst_size; 82 }; 83 84 struct atmel_aes_dev; 85 86 87 typedef int (*atmel_aes_fn_t)(struct atmel_aes_dev *); 88 89 90 struct atmel_aes_base_ctx { 91 struct atmel_aes_dev *dd; 92 atmel_aes_fn_t start; 93 int keylen; 94 u32 key[AES_KEYSIZE_256 / sizeof(u32)]; 95 u16 block_size; 96 bool is_aead; 97 }; 98 99 struct atmel_aes_ctx { 100 struct atmel_aes_base_ctx base; 101 }; 102 103 struct atmel_aes_ctr_ctx { 104 struct atmel_aes_base_ctx base; 105 106 __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; 107 size_t offset; 108 struct scatterlist src[2]; 109 struct scatterlist dst[2]; 110 u32 blocks; 111 }; 112 113 struct atmel_aes_gcm_ctx { 114 struct atmel_aes_base_ctx base; 115 116 struct scatterlist src[2]; 117 struct scatterlist dst[2]; 118 119 __be32 j0[AES_BLOCK_SIZE / sizeof(u32)]; 120 u32 tag[AES_BLOCK_SIZE / sizeof(u32)]; 121 __be32 ghash[AES_BLOCK_SIZE / sizeof(u32)]; 122 size_t textlen; 123 124 const __be32 *ghash_in; 125 __be32 *ghash_out; 126 atmel_aes_fn_t ghash_resume; 127 }; 128 129 struct atmel_aes_xts_ctx { 130 struct atmel_aes_base_ctx base; 131 132 u32 key2[AES_KEYSIZE_256 / sizeof(u32)]; 133 struct crypto_skcipher *fallback_tfm; 134 }; 135 136 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 137 struct atmel_aes_authenc_ctx { 138 struct atmel_aes_base_ctx base; 139 struct atmel_sha_authenc_ctx *auth; 140 }; 141 #endif 142 143 struct atmel_aes_reqctx { 144 unsigned long mode; 145 u8 lastc[AES_BLOCK_SIZE]; 146 struct skcipher_request fallback_req; 147 }; 148 149 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 150 struct atmel_aes_authenc_reqctx { 151 struct atmel_aes_reqctx base; 152 153 struct scatterlist src[2]; 154 struct scatterlist dst[2]; 155 size_t textlen; 156 u32 digest[SHA512_DIGEST_SIZE / sizeof(u32)]; 157 158 /* auth_req MUST be place last. */ 159 struct ahash_request auth_req; 160 }; 161 #endif 162 163 struct atmel_aes_dma { 164 struct dma_chan *chan; 165 struct scatterlist *sg; 166 int nents; 167 unsigned int remainder; 168 unsigned int sg_len; 169 }; 170 171 struct atmel_aes_dev { 172 struct list_head list; 173 unsigned long phys_base; 174 void __iomem *io_base; 175 176 struct crypto_async_request *areq; 177 struct atmel_aes_base_ctx *ctx; 178 179 bool is_async; 180 atmel_aes_fn_t resume; 181 atmel_aes_fn_t cpu_transfer_complete; 182 183 struct device *dev; 184 struct clk *iclk; 185 int irq; 186 187 unsigned long flags; 188 189 spinlock_t lock; 190 struct crypto_queue queue; 191 192 struct tasklet_struct done_task; 193 struct tasklet_struct queue_task; 194 195 size_t total; 196 size_t datalen; 197 u32 *data; 198 199 struct atmel_aes_dma src; 200 struct atmel_aes_dma dst; 201 202 size_t buflen; 203 void *buf; 204 struct scatterlist aligned_sg; 205 struct scatterlist *real_dst; 206 207 struct atmel_aes_caps caps; 208 209 u32 hw_version; 210 }; 211 212 struct atmel_aes_drv { 213 struct list_head dev_list; 214 spinlock_t lock; 215 }; 216 217 static struct atmel_aes_drv atmel_aes = { 218 .dev_list = LIST_HEAD_INIT(atmel_aes.dev_list), 219 .lock = __SPIN_LOCK_UNLOCKED(atmel_aes.lock), 220 }; 221 222 #ifdef VERBOSE_DEBUG 223 static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz) 224 { 225 switch (offset) { 226 case AES_CR: 227 return "CR"; 228 229 case AES_MR: 230 return "MR"; 231 232 case AES_ISR: 233 return "ISR"; 234 235 case AES_IMR: 236 return "IMR"; 237 238 case AES_IER: 239 return "IER"; 240 241 case AES_IDR: 242 return "IDR"; 243 244 case AES_KEYWR(0): 245 case AES_KEYWR(1): 246 case AES_KEYWR(2): 247 case AES_KEYWR(3): 248 case AES_KEYWR(4): 249 case AES_KEYWR(5): 250 case AES_KEYWR(6): 251 case AES_KEYWR(7): 252 snprintf(tmp, sz, "KEYWR[%u]", (offset - AES_KEYWR(0)) >> 2); 253 break; 254 255 case AES_IDATAR(0): 256 case AES_IDATAR(1): 257 case AES_IDATAR(2): 258 case AES_IDATAR(3): 259 snprintf(tmp, sz, "IDATAR[%u]", (offset - AES_IDATAR(0)) >> 2); 260 break; 261 262 case AES_ODATAR(0): 263 case AES_ODATAR(1): 264 case AES_ODATAR(2): 265 case AES_ODATAR(3): 266 snprintf(tmp, sz, "ODATAR[%u]", (offset - AES_ODATAR(0)) >> 2); 267 break; 268 269 case AES_IVR(0): 270 case AES_IVR(1): 271 case AES_IVR(2): 272 case AES_IVR(3): 273 snprintf(tmp, sz, "IVR[%u]", (offset - AES_IVR(0)) >> 2); 274 break; 275 276 case AES_AADLENR: 277 return "AADLENR"; 278 279 case AES_CLENR: 280 return "CLENR"; 281 282 case AES_GHASHR(0): 283 case AES_GHASHR(1): 284 case AES_GHASHR(2): 285 case AES_GHASHR(3): 286 snprintf(tmp, sz, "GHASHR[%u]", (offset - AES_GHASHR(0)) >> 2); 287 break; 288 289 case AES_TAGR(0): 290 case AES_TAGR(1): 291 case AES_TAGR(2): 292 case AES_TAGR(3): 293 snprintf(tmp, sz, "TAGR[%u]", (offset - AES_TAGR(0)) >> 2); 294 break; 295 296 case AES_CTRR: 297 return "CTRR"; 298 299 case AES_GCMHR(0): 300 case AES_GCMHR(1): 301 case AES_GCMHR(2): 302 case AES_GCMHR(3): 303 snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2); 304 break; 305 306 case AES_EMR: 307 return "EMR"; 308 309 case AES_TWR(0): 310 case AES_TWR(1): 311 case AES_TWR(2): 312 case AES_TWR(3): 313 snprintf(tmp, sz, "TWR[%u]", (offset - AES_TWR(0)) >> 2); 314 break; 315 316 case AES_ALPHAR(0): 317 case AES_ALPHAR(1): 318 case AES_ALPHAR(2): 319 case AES_ALPHAR(3): 320 snprintf(tmp, sz, "ALPHAR[%u]", (offset - AES_ALPHAR(0)) >> 2); 321 break; 322 323 default: 324 snprintf(tmp, sz, "0x%02x", offset); 325 break; 326 } 327 328 return tmp; 329 } 330 #endif /* VERBOSE_DEBUG */ 331 332 /* Shared functions */ 333 334 static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset) 335 { 336 u32 value = readl_relaxed(dd->io_base + offset); 337 338 #ifdef VERBOSE_DEBUG 339 if (dd->flags & AES_FLAGS_DUMP_REG) { 340 char tmp[16]; 341 342 dev_vdbg(dd->dev, "read 0x%08x from %s\n", value, 343 atmel_aes_reg_name(offset, tmp, sizeof(tmp))); 344 } 345 #endif /* VERBOSE_DEBUG */ 346 347 return value; 348 } 349 350 static inline void atmel_aes_write(struct atmel_aes_dev *dd, 351 u32 offset, u32 value) 352 { 353 #ifdef VERBOSE_DEBUG 354 if (dd->flags & AES_FLAGS_DUMP_REG) { 355 char tmp[16]; 356 357 dev_vdbg(dd->dev, "write 0x%08x into %s\n", value, 358 atmel_aes_reg_name(offset, tmp, sizeof(tmp))); 359 } 360 #endif /* VERBOSE_DEBUG */ 361 362 writel_relaxed(value, dd->io_base + offset); 363 } 364 365 static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset, 366 u32 *value, int count) 367 { 368 for (; count--; value++, offset += 4) 369 *value = atmel_aes_read(dd, offset); 370 } 371 372 static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset, 373 const u32 *value, int count) 374 { 375 for (; count--; value++, offset += 4) 376 atmel_aes_write(dd, offset, *value); 377 } 378 379 static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset, 380 void *value) 381 { 382 atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); 383 } 384 385 static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset, 386 const void *value) 387 { 388 atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); 389 } 390 391 static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd, 392 atmel_aes_fn_t resume) 393 { 394 u32 isr = atmel_aes_read(dd, AES_ISR); 395 396 if (unlikely(isr & AES_INT_DATARDY)) 397 return resume(dd); 398 399 dd->resume = resume; 400 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 401 return -EINPROGRESS; 402 } 403 404 static inline size_t atmel_aes_padlen(size_t len, size_t block_size) 405 { 406 len &= block_size - 1; 407 return len ? block_size - len : 0; 408 } 409 410 static struct atmel_aes_dev *atmel_aes_dev_alloc(struct atmel_aes_base_ctx *ctx) 411 { 412 struct atmel_aes_dev *aes_dd; 413 414 spin_lock_bh(&atmel_aes.lock); 415 /* One AES IP per SoC. */ 416 aes_dd = list_first_entry_or_null(&atmel_aes.dev_list, 417 struct atmel_aes_dev, list); 418 spin_unlock_bh(&atmel_aes.lock); 419 return aes_dd; 420 } 421 422 static int atmel_aes_hw_init(struct atmel_aes_dev *dd) 423 { 424 int err; 425 426 err = clk_enable(dd->iclk); 427 if (err) 428 return err; 429 430 atmel_aes_write(dd, AES_CR, AES_CR_SWRST); 431 atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); 432 433 return 0; 434 } 435 436 static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd) 437 { 438 return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff; 439 } 440 441 static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd) 442 { 443 int err; 444 445 err = atmel_aes_hw_init(dd); 446 if (err) 447 return err; 448 449 dd->hw_version = atmel_aes_get_version(dd); 450 451 dev_info(dd->dev, "version: 0x%x\n", dd->hw_version); 452 453 clk_disable(dd->iclk); 454 return 0; 455 } 456 457 static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd, 458 const struct atmel_aes_reqctx *rctx) 459 { 460 /* Clear all but persistent flags and set request flags. */ 461 dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode; 462 } 463 464 static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd) 465 { 466 return (dd->flags & AES_FLAGS_ENCRYPT); 467 } 468 469 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 470 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err); 471 #endif 472 473 static void atmel_aes_set_iv_as_last_ciphertext_block(struct atmel_aes_dev *dd) 474 { 475 struct skcipher_request *req = skcipher_request_cast(dd->areq); 476 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 477 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 478 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 479 480 if (req->cryptlen < ivsize) 481 return; 482 483 if (rctx->mode & AES_FLAGS_ENCRYPT) 484 scatterwalk_map_and_copy(req->iv, req->dst, 485 req->cryptlen - ivsize, ivsize, 0); 486 else 487 memcpy(req->iv, rctx->lastc, ivsize); 488 } 489 490 static inline struct atmel_aes_ctr_ctx * 491 atmel_aes_ctr_ctx_cast(struct atmel_aes_base_ctx *ctx) 492 { 493 return container_of(ctx, struct atmel_aes_ctr_ctx, base); 494 } 495 496 static void atmel_aes_ctr_update_req_iv(struct atmel_aes_dev *dd) 497 { 498 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); 499 struct skcipher_request *req = skcipher_request_cast(dd->areq); 500 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 501 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 502 int i; 503 504 /* 505 * The CTR transfer works in fragments of data of maximum 1 MByte 506 * because of the 16 bit CTR counter embedded in the IP. When reaching 507 * here, ctx->blocks contains the number of blocks of the last fragment 508 * processed, there is no need to explicit cast it to u16. 509 */ 510 for (i = 0; i < ctx->blocks; i++) 511 crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE); 512 513 memcpy(req->iv, ctx->iv, ivsize); 514 } 515 516 static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) 517 { 518 struct skcipher_request *req = skcipher_request_cast(dd->areq); 519 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 520 521 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 522 if (dd->ctx->is_aead) 523 atmel_aes_authenc_complete(dd, err); 524 #endif 525 526 clk_disable(dd->iclk); 527 dd->flags &= ~AES_FLAGS_BUSY; 528 529 if (!err && !dd->ctx->is_aead && 530 (rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_ECB) { 531 if ((rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_CTR) 532 atmel_aes_set_iv_as_last_ciphertext_block(dd); 533 else 534 atmel_aes_ctr_update_req_iv(dd); 535 } 536 537 if (dd->is_async) 538 crypto_request_complete(dd->areq, err); 539 540 tasklet_schedule(&dd->queue_task); 541 542 return err; 543 } 544 545 static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma, 546 const __be32 *iv, const u32 *key, int keylen) 547 { 548 u32 valmr = 0; 549 550 /* MR register must be set before IV registers */ 551 if (keylen == AES_KEYSIZE_128) 552 valmr |= AES_MR_KEYSIZE_128; 553 else if (keylen == AES_KEYSIZE_192) 554 valmr |= AES_MR_KEYSIZE_192; 555 else 556 valmr |= AES_MR_KEYSIZE_256; 557 558 valmr |= dd->flags & AES_FLAGS_MODE_MASK; 559 560 if (use_dma) { 561 valmr |= AES_MR_SMOD_IDATAR0; 562 if (dd->caps.has_dualbuff) 563 valmr |= AES_MR_DUALBUFF; 564 } else { 565 valmr |= AES_MR_SMOD_AUTO; 566 } 567 568 atmel_aes_write(dd, AES_MR, valmr); 569 570 atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen)); 571 572 if (iv && (valmr & AES_MR_OPMOD_MASK) != AES_MR_OPMOD_ECB) 573 atmel_aes_write_block(dd, AES_IVR(0), iv); 574 } 575 576 static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma, 577 const __be32 *iv) 578 579 { 580 atmel_aes_write_ctrl_key(dd, use_dma, iv, 581 dd->ctx->key, dd->ctx->keylen); 582 } 583 584 /* CPU transfer */ 585 586 static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd) 587 { 588 int err = 0; 589 u32 isr; 590 591 for (;;) { 592 atmel_aes_read_block(dd, AES_ODATAR(0), dd->data); 593 dd->data += 4; 594 dd->datalen -= AES_BLOCK_SIZE; 595 596 if (dd->datalen < AES_BLOCK_SIZE) 597 break; 598 599 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 600 601 isr = atmel_aes_read(dd, AES_ISR); 602 if (!(isr & AES_INT_DATARDY)) { 603 dd->resume = atmel_aes_cpu_transfer; 604 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 605 return -EINPROGRESS; 606 } 607 } 608 609 if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), 610 dd->buf, dd->total)) 611 err = -EINVAL; 612 613 if (err) 614 return atmel_aes_complete(dd, err); 615 616 return dd->cpu_transfer_complete(dd); 617 } 618 619 static int atmel_aes_cpu_start(struct atmel_aes_dev *dd, 620 struct scatterlist *src, 621 struct scatterlist *dst, 622 size_t len, 623 atmel_aes_fn_t resume) 624 { 625 size_t padlen = atmel_aes_padlen(len, AES_BLOCK_SIZE); 626 627 if (unlikely(len == 0)) 628 return -EINVAL; 629 630 sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); 631 632 dd->total = len; 633 dd->real_dst = dst; 634 dd->cpu_transfer_complete = resume; 635 dd->datalen = len + padlen; 636 dd->data = (u32 *)dd->buf; 637 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 638 return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer); 639 } 640 641 642 /* DMA transfer */ 643 644 static void atmel_aes_dma_callback(void *data); 645 646 static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd, 647 struct scatterlist *sg, 648 size_t len, 649 struct atmel_aes_dma *dma) 650 { 651 int nents; 652 653 if (!IS_ALIGNED(len, dd->ctx->block_size)) 654 return false; 655 656 for (nents = 0; sg; sg = sg_next(sg), ++nents) { 657 if (!IS_ALIGNED(sg->offset, sizeof(u32))) 658 return false; 659 660 if (len <= sg->length) { 661 if (!IS_ALIGNED(len, dd->ctx->block_size)) 662 return false; 663 664 dma->nents = nents+1; 665 dma->remainder = sg->length - len; 666 sg->length = len; 667 return true; 668 } 669 670 if (!IS_ALIGNED(sg->length, dd->ctx->block_size)) 671 return false; 672 673 len -= sg->length; 674 } 675 676 return false; 677 } 678 679 static inline void atmel_aes_restore_sg(const struct atmel_aes_dma *dma) 680 { 681 struct scatterlist *sg = dma->sg; 682 int nents = dma->nents; 683 684 if (!dma->remainder) 685 return; 686 687 while (--nents > 0 && sg) 688 sg = sg_next(sg); 689 690 if (!sg) 691 return; 692 693 sg->length += dma->remainder; 694 } 695 696 static int atmel_aes_map(struct atmel_aes_dev *dd, 697 struct scatterlist *src, 698 struct scatterlist *dst, 699 size_t len) 700 { 701 bool src_aligned, dst_aligned; 702 size_t padlen; 703 704 dd->total = len; 705 dd->src.sg = src; 706 dd->dst.sg = dst; 707 dd->real_dst = dst; 708 709 src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src); 710 if (src == dst) 711 dst_aligned = src_aligned; 712 else 713 dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst); 714 if (!src_aligned || !dst_aligned) { 715 padlen = atmel_aes_padlen(len, dd->ctx->block_size); 716 717 if (dd->buflen < len + padlen) 718 return -ENOMEM; 719 720 if (!src_aligned) { 721 sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); 722 dd->src.sg = &dd->aligned_sg; 723 dd->src.nents = 1; 724 dd->src.remainder = 0; 725 } 726 727 if (!dst_aligned) { 728 dd->dst.sg = &dd->aligned_sg; 729 dd->dst.nents = 1; 730 dd->dst.remainder = 0; 731 } 732 733 sg_init_table(&dd->aligned_sg, 1); 734 sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen); 735 } 736 737 if (dd->src.sg == dd->dst.sg) { 738 dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, 739 DMA_BIDIRECTIONAL); 740 dd->dst.sg_len = dd->src.sg_len; 741 if (!dd->src.sg_len) 742 return -EFAULT; 743 } else { 744 dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, 745 DMA_TO_DEVICE); 746 if (!dd->src.sg_len) 747 return -EFAULT; 748 749 dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents, 750 DMA_FROM_DEVICE); 751 if (!dd->dst.sg_len) { 752 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 753 DMA_TO_DEVICE); 754 return -EFAULT; 755 } 756 } 757 758 return 0; 759 } 760 761 static void atmel_aes_unmap(struct atmel_aes_dev *dd) 762 { 763 if (dd->src.sg == dd->dst.sg) { 764 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 765 DMA_BIDIRECTIONAL); 766 767 if (dd->src.sg != &dd->aligned_sg) 768 atmel_aes_restore_sg(&dd->src); 769 } else { 770 dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents, 771 DMA_FROM_DEVICE); 772 773 if (dd->dst.sg != &dd->aligned_sg) 774 atmel_aes_restore_sg(&dd->dst); 775 776 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 777 DMA_TO_DEVICE); 778 779 if (dd->src.sg != &dd->aligned_sg) 780 atmel_aes_restore_sg(&dd->src); 781 } 782 783 if (dd->dst.sg == &dd->aligned_sg) 784 sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), 785 dd->buf, dd->total); 786 } 787 788 static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd, 789 enum dma_slave_buswidth addr_width, 790 enum dma_transfer_direction dir, 791 u32 maxburst) 792 { 793 struct dma_async_tx_descriptor *desc; 794 struct dma_slave_config config; 795 dma_async_tx_callback callback; 796 struct atmel_aes_dma *dma; 797 int err; 798 799 memset(&config, 0, sizeof(config)); 800 config.src_addr_width = addr_width; 801 config.dst_addr_width = addr_width; 802 config.src_maxburst = maxburst; 803 config.dst_maxburst = maxburst; 804 805 switch (dir) { 806 case DMA_MEM_TO_DEV: 807 dma = &dd->src; 808 callback = NULL; 809 config.dst_addr = dd->phys_base + AES_IDATAR(0); 810 break; 811 812 case DMA_DEV_TO_MEM: 813 dma = &dd->dst; 814 callback = atmel_aes_dma_callback; 815 config.src_addr = dd->phys_base + AES_ODATAR(0); 816 break; 817 818 default: 819 return -EINVAL; 820 } 821 822 err = dmaengine_slave_config(dma->chan, &config); 823 if (err) 824 return err; 825 826 desc = dmaengine_prep_slave_sg(dma->chan, dma->sg, dma->sg_len, dir, 827 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 828 if (!desc) 829 return -ENOMEM; 830 831 desc->callback = callback; 832 desc->callback_param = dd; 833 dmaengine_submit(desc); 834 dma_async_issue_pending(dma->chan); 835 836 return 0; 837 } 838 839 static int atmel_aes_dma_start(struct atmel_aes_dev *dd, 840 struct scatterlist *src, 841 struct scatterlist *dst, 842 size_t len, 843 atmel_aes_fn_t resume) 844 { 845 enum dma_slave_buswidth addr_width; 846 u32 maxburst; 847 int err; 848 849 switch (dd->ctx->block_size) { 850 case AES_BLOCK_SIZE: 851 addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 852 maxburst = dd->caps.max_burst_size; 853 break; 854 855 default: 856 err = -EINVAL; 857 goto exit; 858 } 859 860 err = atmel_aes_map(dd, src, dst, len); 861 if (err) 862 goto exit; 863 864 dd->resume = resume; 865 866 /* Set output DMA transfer first */ 867 err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM, 868 maxburst); 869 if (err) 870 goto unmap; 871 872 /* Then set input DMA transfer */ 873 err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV, 874 maxburst); 875 if (err) 876 goto output_transfer_stop; 877 878 return -EINPROGRESS; 879 880 output_transfer_stop: 881 dmaengine_terminate_sync(dd->dst.chan); 882 unmap: 883 atmel_aes_unmap(dd); 884 exit: 885 return atmel_aes_complete(dd, err); 886 } 887 888 static void atmel_aes_dma_callback(void *data) 889 { 890 struct atmel_aes_dev *dd = data; 891 892 atmel_aes_unmap(dd); 893 dd->is_async = true; 894 (void)dd->resume(dd); 895 } 896 897 static int atmel_aes_handle_queue(struct atmel_aes_dev *dd, 898 struct crypto_async_request *new_areq) 899 { 900 struct crypto_async_request *areq, *backlog; 901 struct atmel_aes_base_ctx *ctx; 902 unsigned long flags; 903 bool start_async; 904 int err, ret = 0; 905 906 spin_lock_irqsave(&dd->lock, flags); 907 if (new_areq) 908 ret = crypto_enqueue_request(&dd->queue, new_areq); 909 if (dd->flags & AES_FLAGS_BUSY) { 910 spin_unlock_irqrestore(&dd->lock, flags); 911 return ret; 912 } 913 backlog = crypto_get_backlog(&dd->queue); 914 areq = crypto_dequeue_request(&dd->queue); 915 if (areq) 916 dd->flags |= AES_FLAGS_BUSY; 917 spin_unlock_irqrestore(&dd->lock, flags); 918 919 if (!areq) 920 return ret; 921 922 if (backlog) 923 crypto_request_complete(backlog, -EINPROGRESS); 924 925 ctx = crypto_tfm_ctx(areq->tfm); 926 927 dd->areq = areq; 928 dd->ctx = ctx; 929 start_async = (areq != new_areq); 930 dd->is_async = start_async; 931 932 /* WARNING: ctx->start() MAY change dd->is_async. */ 933 err = ctx->start(dd); 934 return (start_async) ? ret : err; 935 } 936 937 938 /* AES async block ciphers */ 939 940 static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd) 941 { 942 return atmel_aes_complete(dd, 0); 943 } 944 945 static int atmel_aes_start(struct atmel_aes_dev *dd) 946 { 947 struct skcipher_request *req = skcipher_request_cast(dd->areq); 948 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 949 bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD || 950 dd->ctx->block_size != AES_BLOCK_SIZE); 951 int err; 952 953 atmel_aes_set_mode(dd, rctx); 954 955 err = atmel_aes_hw_init(dd); 956 if (err) 957 return atmel_aes_complete(dd, err); 958 959 atmel_aes_write_ctrl(dd, use_dma, (void *)req->iv); 960 if (use_dma) 961 return atmel_aes_dma_start(dd, req->src, req->dst, 962 req->cryptlen, 963 atmel_aes_transfer_complete); 964 965 return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen, 966 atmel_aes_transfer_complete); 967 } 968 969 static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd) 970 { 971 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); 972 struct skcipher_request *req = skcipher_request_cast(dd->areq); 973 struct scatterlist *src, *dst; 974 size_t datalen; 975 u32 ctr; 976 u16 start, end; 977 bool use_dma, fragmented = false; 978 979 /* Check for transfer completion. */ 980 ctx->offset += dd->total; 981 if (ctx->offset >= req->cryptlen) 982 return atmel_aes_transfer_complete(dd); 983 984 /* Compute data length. */ 985 datalen = req->cryptlen - ctx->offset; 986 ctx->blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE); 987 ctr = be32_to_cpu(ctx->iv[3]); 988 989 /* Check 16bit counter overflow. */ 990 start = ctr & 0xffff; 991 end = start + ctx->blocks - 1; 992 993 if (ctx->blocks >> 16 || end < start) { 994 ctr |= 0xffff; 995 datalen = AES_BLOCK_SIZE * (0x10000 - start); 996 fragmented = true; 997 } 998 999 use_dma = (datalen >= ATMEL_AES_DMA_THRESHOLD); 1000 1001 /* Jump to offset. */ 1002 src = scatterwalk_ffwd(ctx->src, req->src, ctx->offset); 1003 dst = ((req->src == req->dst) ? src : 1004 scatterwalk_ffwd(ctx->dst, req->dst, ctx->offset)); 1005 1006 /* Configure hardware. */ 1007 atmel_aes_write_ctrl(dd, use_dma, ctx->iv); 1008 if (unlikely(fragmented)) { 1009 /* 1010 * Increment the counter manually to cope with the hardware 1011 * counter overflow. 1012 */ 1013 ctx->iv[3] = cpu_to_be32(ctr); 1014 crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE); 1015 } 1016 1017 if (use_dma) 1018 return atmel_aes_dma_start(dd, src, dst, datalen, 1019 atmel_aes_ctr_transfer); 1020 1021 return atmel_aes_cpu_start(dd, src, dst, datalen, 1022 atmel_aes_ctr_transfer); 1023 } 1024 1025 static int atmel_aes_ctr_start(struct atmel_aes_dev *dd) 1026 { 1027 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); 1028 struct skcipher_request *req = skcipher_request_cast(dd->areq); 1029 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 1030 int err; 1031 1032 atmel_aes_set_mode(dd, rctx); 1033 1034 err = atmel_aes_hw_init(dd); 1035 if (err) 1036 return atmel_aes_complete(dd, err); 1037 1038 memcpy(ctx->iv, req->iv, AES_BLOCK_SIZE); 1039 ctx->offset = 0; 1040 dd->total = 0; 1041 return atmel_aes_ctr_transfer(dd); 1042 } 1043 1044 static int atmel_aes_xts_fallback(struct skcipher_request *req, bool enc) 1045 { 1046 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 1047 struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx( 1048 crypto_skcipher_reqtfm(req)); 1049 1050 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 1051 skcipher_request_set_callback(&rctx->fallback_req, req->base.flags, 1052 req->base.complete, req->base.data); 1053 skcipher_request_set_crypt(&rctx->fallback_req, req->src, req->dst, 1054 req->cryptlen, req->iv); 1055 1056 return enc ? crypto_skcipher_encrypt(&rctx->fallback_req) : 1057 crypto_skcipher_decrypt(&rctx->fallback_req); 1058 } 1059 1060 static int atmel_aes_crypt(struct skcipher_request *req, unsigned long mode) 1061 { 1062 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1063 struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(skcipher); 1064 struct atmel_aes_reqctx *rctx; 1065 u32 opmode = mode & AES_FLAGS_OPMODE_MASK; 1066 1067 if (opmode == AES_FLAGS_XTS) { 1068 if (req->cryptlen < XTS_BLOCK_SIZE) 1069 return -EINVAL; 1070 1071 if (!IS_ALIGNED(req->cryptlen, XTS_BLOCK_SIZE)) 1072 return atmel_aes_xts_fallback(req, 1073 mode & AES_FLAGS_ENCRYPT); 1074 } 1075 1076 /* 1077 * ECB, CBC or CTR mode require the plaintext and ciphertext 1078 * to have a positve integer length. 1079 */ 1080 if (!req->cryptlen && opmode != AES_FLAGS_XTS) 1081 return 0; 1082 1083 if ((opmode == AES_FLAGS_ECB || opmode == AES_FLAGS_CBC) && 1084 !IS_ALIGNED(req->cryptlen, crypto_skcipher_blocksize(skcipher))) 1085 return -EINVAL; 1086 1087 ctx->block_size = AES_BLOCK_SIZE; 1088 ctx->is_aead = false; 1089 1090 rctx = skcipher_request_ctx(req); 1091 rctx->mode = mode; 1092 1093 if (opmode != AES_FLAGS_ECB && 1094 !(mode & AES_FLAGS_ENCRYPT)) { 1095 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1096 1097 if (req->cryptlen >= ivsize) 1098 scatterwalk_map_and_copy(rctx->lastc, req->src, 1099 req->cryptlen - ivsize, 1100 ivsize, 0); 1101 } 1102 1103 return atmel_aes_handle_queue(ctx->dd, &req->base); 1104 } 1105 1106 static int atmel_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 1107 unsigned int keylen) 1108 { 1109 struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(tfm); 1110 1111 if (keylen != AES_KEYSIZE_128 && 1112 keylen != AES_KEYSIZE_192 && 1113 keylen != AES_KEYSIZE_256) 1114 return -EINVAL; 1115 1116 memcpy(ctx->key, key, keylen); 1117 ctx->keylen = keylen; 1118 1119 return 0; 1120 } 1121 1122 static int atmel_aes_ecb_encrypt(struct skcipher_request *req) 1123 { 1124 return atmel_aes_crypt(req, AES_FLAGS_ECB | AES_FLAGS_ENCRYPT); 1125 } 1126 1127 static int atmel_aes_ecb_decrypt(struct skcipher_request *req) 1128 { 1129 return atmel_aes_crypt(req, AES_FLAGS_ECB); 1130 } 1131 1132 static int atmel_aes_cbc_encrypt(struct skcipher_request *req) 1133 { 1134 return atmel_aes_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT); 1135 } 1136 1137 static int atmel_aes_cbc_decrypt(struct skcipher_request *req) 1138 { 1139 return atmel_aes_crypt(req, AES_FLAGS_CBC); 1140 } 1141 1142 static int atmel_aes_ctr_encrypt(struct skcipher_request *req) 1143 { 1144 return atmel_aes_crypt(req, AES_FLAGS_CTR | AES_FLAGS_ENCRYPT); 1145 } 1146 1147 static int atmel_aes_ctr_decrypt(struct skcipher_request *req) 1148 { 1149 return atmel_aes_crypt(req, AES_FLAGS_CTR); 1150 } 1151 1152 static int atmel_aes_init_tfm(struct crypto_skcipher *tfm) 1153 { 1154 struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 1155 struct atmel_aes_dev *dd; 1156 1157 dd = atmel_aes_dev_alloc(&ctx->base); 1158 if (!dd) 1159 return -ENODEV; 1160 1161 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); 1162 ctx->base.dd = dd; 1163 ctx->base.start = atmel_aes_start; 1164 1165 return 0; 1166 } 1167 1168 static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm) 1169 { 1170 struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 1171 struct atmel_aes_dev *dd; 1172 1173 dd = atmel_aes_dev_alloc(&ctx->base); 1174 if (!dd) 1175 return -ENODEV; 1176 1177 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); 1178 ctx->base.dd = dd; 1179 ctx->base.start = atmel_aes_ctr_start; 1180 1181 return 0; 1182 } 1183 1184 static struct skcipher_alg aes_algs[] = { 1185 { 1186 .base.cra_name = "ecb(aes)", 1187 .base.cra_driver_name = "atmel-ecb-aes", 1188 .base.cra_blocksize = AES_BLOCK_SIZE, 1189 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1190 1191 .init = atmel_aes_init_tfm, 1192 .min_keysize = AES_MIN_KEY_SIZE, 1193 .max_keysize = AES_MAX_KEY_SIZE, 1194 .setkey = atmel_aes_setkey, 1195 .encrypt = atmel_aes_ecb_encrypt, 1196 .decrypt = atmel_aes_ecb_decrypt, 1197 }, 1198 { 1199 .base.cra_name = "cbc(aes)", 1200 .base.cra_driver_name = "atmel-cbc-aes", 1201 .base.cra_blocksize = AES_BLOCK_SIZE, 1202 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1203 1204 .init = atmel_aes_init_tfm, 1205 .min_keysize = AES_MIN_KEY_SIZE, 1206 .max_keysize = AES_MAX_KEY_SIZE, 1207 .setkey = atmel_aes_setkey, 1208 .encrypt = atmel_aes_cbc_encrypt, 1209 .decrypt = atmel_aes_cbc_decrypt, 1210 .ivsize = AES_BLOCK_SIZE, 1211 }, 1212 { 1213 .base.cra_name = "ctr(aes)", 1214 .base.cra_driver_name = "atmel-ctr-aes", 1215 .base.cra_blocksize = 1, 1216 .base.cra_ctxsize = sizeof(struct atmel_aes_ctr_ctx), 1217 1218 .init = atmel_aes_ctr_init_tfm, 1219 .min_keysize = AES_MIN_KEY_SIZE, 1220 .max_keysize = AES_MAX_KEY_SIZE, 1221 .setkey = atmel_aes_setkey, 1222 .encrypt = atmel_aes_ctr_encrypt, 1223 .decrypt = atmel_aes_ctr_decrypt, 1224 .ivsize = AES_BLOCK_SIZE, 1225 }, 1226 }; 1227 1228 1229 /* gcm aead functions */ 1230 1231 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, 1232 const u32 *data, size_t datalen, 1233 const __be32 *ghash_in, __be32 *ghash_out, 1234 atmel_aes_fn_t resume); 1235 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd); 1236 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd); 1237 1238 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd); 1239 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd); 1240 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd); 1241 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd); 1242 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd); 1243 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd); 1244 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd); 1245 1246 static inline struct atmel_aes_gcm_ctx * 1247 atmel_aes_gcm_ctx_cast(struct atmel_aes_base_ctx *ctx) 1248 { 1249 return container_of(ctx, struct atmel_aes_gcm_ctx, base); 1250 } 1251 1252 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, 1253 const u32 *data, size_t datalen, 1254 const __be32 *ghash_in, __be32 *ghash_out, 1255 atmel_aes_fn_t resume) 1256 { 1257 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1258 1259 dd->data = (u32 *)data; 1260 dd->datalen = datalen; 1261 ctx->ghash_in = ghash_in; 1262 ctx->ghash_out = ghash_out; 1263 ctx->ghash_resume = resume; 1264 1265 atmel_aes_write_ctrl(dd, false, NULL); 1266 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init); 1267 } 1268 1269 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd) 1270 { 1271 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1272 1273 /* Set the data length. */ 1274 atmel_aes_write(dd, AES_AADLENR, dd->total); 1275 atmel_aes_write(dd, AES_CLENR, 0); 1276 1277 /* If needed, overwrite the GCM Intermediate Hash Word Registers */ 1278 if (ctx->ghash_in) 1279 atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in); 1280 1281 return atmel_aes_gcm_ghash_finalize(dd); 1282 } 1283 1284 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd) 1285 { 1286 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1287 u32 isr; 1288 1289 /* Write data into the Input Data Registers. */ 1290 while (dd->datalen > 0) { 1291 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 1292 dd->data += 4; 1293 dd->datalen -= AES_BLOCK_SIZE; 1294 1295 isr = atmel_aes_read(dd, AES_ISR); 1296 if (!(isr & AES_INT_DATARDY)) { 1297 dd->resume = atmel_aes_gcm_ghash_finalize; 1298 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 1299 return -EINPROGRESS; 1300 } 1301 } 1302 1303 /* Read the computed hash from GHASHRx. */ 1304 atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out); 1305 1306 return ctx->ghash_resume(dd); 1307 } 1308 1309 1310 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd) 1311 { 1312 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1313 struct aead_request *req = aead_request_cast(dd->areq); 1314 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1315 struct atmel_aes_reqctx *rctx = aead_request_ctx(req); 1316 size_t ivsize = crypto_aead_ivsize(tfm); 1317 size_t datalen, padlen; 1318 const void *iv = req->iv; 1319 u8 *data = dd->buf; 1320 int err; 1321 1322 atmel_aes_set_mode(dd, rctx); 1323 1324 err = atmel_aes_hw_init(dd); 1325 if (err) 1326 return atmel_aes_complete(dd, err); 1327 1328 if (likely(ivsize == GCM_AES_IV_SIZE)) { 1329 memcpy(ctx->j0, iv, ivsize); 1330 ctx->j0[3] = cpu_to_be32(1); 1331 return atmel_aes_gcm_process(dd); 1332 } 1333 1334 padlen = atmel_aes_padlen(ivsize, AES_BLOCK_SIZE); 1335 datalen = ivsize + padlen + AES_BLOCK_SIZE; 1336 if (datalen > dd->buflen) 1337 return atmel_aes_complete(dd, -EINVAL); 1338 1339 memcpy(data, iv, ivsize); 1340 memset(data + ivsize, 0, padlen + sizeof(u64)); 1341 ((__be64 *)(data + datalen))[-1] = cpu_to_be64(ivsize * 8); 1342 1343 return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen, 1344 NULL, ctx->j0, atmel_aes_gcm_process); 1345 } 1346 1347 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd) 1348 { 1349 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1350 struct aead_request *req = aead_request_cast(dd->areq); 1351 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1352 bool enc = atmel_aes_is_encrypt(dd); 1353 u32 authsize; 1354 1355 /* Compute text length. */ 1356 authsize = crypto_aead_authsize(tfm); 1357 ctx->textlen = req->cryptlen - (enc ? 0 : authsize); 1358 1359 /* 1360 * According to tcrypt test suite, the GCM Automatic Tag Generation 1361 * fails when both the message and its associated data are empty. 1362 */ 1363 if (likely(req->assoclen != 0 || ctx->textlen != 0)) 1364 dd->flags |= AES_FLAGS_GTAGEN; 1365 1366 atmel_aes_write_ctrl(dd, false, NULL); 1367 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length); 1368 } 1369 1370 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd) 1371 { 1372 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1373 struct aead_request *req = aead_request_cast(dd->areq); 1374 __be32 j0_lsw, *j0 = ctx->j0; 1375 size_t padlen; 1376 1377 /* Write incr32(J0) into IV. */ 1378 j0_lsw = j0[3]; 1379 be32_add_cpu(&j0[3], 1); 1380 atmel_aes_write_block(dd, AES_IVR(0), j0); 1381 j0[3] = j0_lsw; 1382 1383 /* Set aad and text lengths. */ 1384 atmel_aes_write(dd, AES_AADLENR, req->assoclen); 1385 atmel_aes_write(dd, AES_CLENR, ctx->textlen); 1386 1387 /* Check whether AAD are present. */ 1388 if (unlikely(req->assoclen == 0)) { 1389 dd->datalen = 0; 1390 return atmel_aes_gcm_data(dd); 1391 } 1392 1393 /* Copy assoc data and add padding. */ 1394 padlen = atmel_aes_padlen(req->assoclen, AES_BLOCK_SIZE); 1395 if (unlikely(req->assoclen + padlen > dd->buflen)) 1396 return atmel_aes_complete(dd, -EINVAL); 1397 sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen); 1398 1399 /* Write assoc data into the Input Data register. */ 1400 dd->data = (u32 *)dd->buf; 1401 dd->datalen = req->assoclen + padlen; 1402 return atmel_aes_gcm_data(dd); 1403 } 1404 1405 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd) 1406 { 1407 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1408 struct aead_request *req = aead_request_cast(dd->areq); 1409 bool use_dma = (ctx->textlen >= ATMEL_AES_DMA_THRESHOLD); 1410 struct scatterlist *src, *dst; 1411 u32 isr, mr; 1412 1413 /* Write AAD first. */ 1414 while (dd->datalen > 0) { 1415 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 1416 dd->data += 4; 1417 dd->datalen -= AES_BLOCK_SIZE; 1418 1419 isr = atmel_aes_read(dd, AES_ISR); 1420 if (!(isr & AES_INT_DATARDY)) { 1421 dd->resume = atmel_aes_gcm_data; 1422 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 1423 return -EINPROGRESS; 1424 } 1425 } 1426 1427 /* GMAC only. */ 1428 if (unlikely(ctx->textlen == 0)) 1429 return atmel_aes_gcm_tag_init(dd); 1430 1431 /* Prepare src and dst scatter lists to transfer cipher/plain texts */ 1432 src = scatterwalk_ffwd(ctx->src, req->src, req->assoclen); 1433 dst = ((req->src == req->dst) ? src : 1434 scatterwalk_ffwd(ctx->dst, req->dst, req->assoclen)); 1435 1436 if (use_dma) { 1437 /* Update the Mode Register for DMA transfers. */ 1438 mr = atmel_aes_read(dd, AES_MR); 1439 mr &= ~(AES_MR_SMOD_MASK | AES_MR_DUALBUFF); 1440 mr |= AES_MR_SMOD_IDATAR0; 1441 if (dd->caps.has_dualbuff) 1442 mr |= AES_MR_DUALBUFF; 1443 atmel_aes_write(dd, AES_MR, mr); 1444 1445 return atmel_aes_dma_start(dd, src, dst, ctx->textlen, 1446 atmel_aes_gcm_tag_init); 1447 } 1448 1449 return atmel_aes_cpu_start(dd, src, dst, ctx->textlen, 1450 atmel_aes_gcm_tag_init); 1451 } 1452 1453 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd) 1454 { 1455 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1456 struct aead_request *req = aead_request_cast(dd->areq); 1457 __be64 *data = dd->buf; 1458 1459 if (likely(dd->flags & AES_FLAGS_GTAGEN)) { 1460 if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) { 1461 dd->resume = atmel_aes_gcm_tag_init; 1462 atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY); 1463 return -EINPROGRESS; 1464 } 1465 1466 return atmel_aes_gcm_finalize(dd); 1467 } 1468 1469 /* Read the GCM Intermediate Hash Word Registers. */ 1470 atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash); 1471 1472 data[0] = cpu_to_be64(req->assoclen * 8); 1473 data[1] = cpu_to_be64(ctx->textlen * 8); 1474 1475 return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE, 1476 ctx->ghash, ctx->ghash, atmel_aes_gcm_tag); 1477 } 1478 1479 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd) 1480 { 1481 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1482 unsigned long flags; 1483 1484 /* 1485 * Change mode to CTR to complete the tag generation. 1486 * Use J0 as Initialization Vector. 1487 */ 1488 flags = dd->flags; 1489 dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN); 1490 dd->flags |= AES_FLAGS_CTR; 1491 atmel_aes_write_ctrl(dd, false, ctx->j0); 1492 dd->flags = flags; 1493 1494 atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash); 1495 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize); 1496 } 1497 1498 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd) 1499 { 1500 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1501 struct aead_request *req = aead_request_cast(dd->areq); 1502 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1503 bool enc = atmel_aes_is_encrypt(dd); 1504 u32 offset, authsize, itag[4], *otag = ctx->tag; 1505 int err; 1506 1507 /* Read the computed tag. */ 1508 if (likely(dd->flags & AES_FLAGS_GTAGEN)) 1509 atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag); 1510 else 1511 atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag); 1512 1513 offset = req->assoclen + ctx->textlen; 1514 authsize = crypto_aead_authsize(tfm); 1515 if (enc) { 1516 scatterwalk_map_and_copy(otag, req->dst, offset, authsize, 1); 1517 err = 0; 1518 } else { 1519 scatterwalk_map_and_copy(itag, req->src, offset, authsize, 0); 1520 err = crypto_memneq(itag, otag, authsize) ? -EBADMSG : 0; 1521 } 1522 1523 return atmel_aes_complete(dd, err); 1524 } 1525 1526 static int atmel_aes_gcm_crypt(struct aead_request *req, 1527 unsigned long mode) 1528 { 1529 struct atmel_aes_base_ctx *ctx; 1530 struct atmel_aes_reqctx *rctx; 1531 1532 ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1533 ctx->block_size = AES_BLOCK_SIZE; 1534 ctx->is_aead = true; 1535 1536 rctx = aead_request_ctx(req); 1537 rctx->mode = AES_FLAGS_GCM | mode; 1538 1539 return atmel_aes_handle_queue(ctx->dd, &req->base); 1540 } 1541 1542 static int atmel_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 1543 unsigned int keylen) 1544 { 1545 struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm); 1546 1547 if (keylen != AES_KEYSIZE_256 && 1548 keylen != AES_KEYSIZE_192 && 1549 keylen != AES_KEYSIZE_128) 1550 return -EINVAL; 1551 1552 memcpy(ctx->key, key, keylen); 1553 ctx->keylen = keylen; 1554 1555 return 0; 1556 } 1557 1558 static int atmel_aes_gcm_setauthsize(struct crypto_aead *tfm, 1559 unsigned int authsize) 1560 { 1561 return crypto_gcm_check_authsize(authsize); 1562 } 1563 1564 static int atmel_aes_gcm_encrypt(struct aead_request *req) 1565 { 1566 return atmel_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT); 1567 } 1568 1569 static int atmel_aes_gcm_decrypt(struct aead_request *req) 1570 { 1571 return atmel_aes_gcm_crypt(req, 0); 1572 } 1573 1574 static int atmel_aes_gcm_init(struct crypto_aead *tfm) 1575 { 1576 struct atmel_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 1577 struct atmel_aes_dev *dd; 1578 1579 dd = atmel_aes_dev_alloc(&ctx->base); 1580 if (!dd) 1581 return -ENODEV; 1582 1583 crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); 1584 ctx->base.dd = dd; 1585 ctx->base.start = atmel_aes_gcm_start; 1586 1587 return 0; 1588 } 1589 1590 static struct aead_alg aes_gcm_alg = { 1591 .setkey = atmel_aes_gcm_setkey, 1592 .setauthsize = atmel_aes_gcm_setauthsize, 1593 .encrypt = atmel_aes_gcm_encrypt, 1594 .decrypt = atmel_aes_gcm_decrypt, 1595 .init = atmel_aes_gcm_init, 1596 .ivsize = GCM_AES_IV_SIZE, 1597 .maxauthsize = AES_BLOCK_SIZE, 1598 1599 .base = { 1600 .cra_name = "gcm(aes)", 1601 .cra_driver_name = "atmel-gcm-aes", 1602 .cra_blocksize = 1, 1603 .cra_ctxsize = sizeof(struct atmel_aes_gcm_ctx), 1604 }, 1605 }; 1606 1607 1608 /* xts functions */ 1609 1610 static inline struct atmel_aes_xts_ctx * 1611 atmel_aes_xts_ctx_cast(struct atmel_aes_base_ctx *ctx) 1612 { 1613 return container_of(ctx, struct atmel_aes_xts_ctx, base); 1614 } 1615 1616 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd); 1617 1618 static int atmel_aes_xts_start(struct atmel_aes_dev *dd) 1619 { 1620 struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx); 1621 struct skcipher_request *req = skcipher_request_cast(dd->areq); 1622 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 1623 unsigned long flags; 1624 int err; 1625 1626 atmel_aes_set_mode(dd, rctx); 1627 1628 err = atmel_aes_hw_init(dd); 1629 if (err) 1630 return atmel_aes_complete(dd, err); 1631 1632 /* Compute the tweak value from req->iv with ecb(aes). */ 1633 flags = dd->flags; 1634 dd->flags &= ~AES_FLAGS_MODE_MASK; 1635 dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT); 1636 atmel_aes_write_ctrl_key(dd, false, NULL, 1637 ctx->key2, ctx->base.keylen); 1638 dd->flags = flags; 1639 1640 atmel_aes_write_block(dd, AES_IDATAR(0), req->iv); 1641 return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data); 1642 } 1643 1644 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd) 1645 { 1646 struct skcipher_request *req = skcipher_request_cast(dd->areq); 1647 bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD); 1648 u32 tweak[AES_BLOCK_SIZE / sizeof(u32)]; 1649 static const __le32 one[AES_BLOCK_SIZE / sizeof(u32)] = {cpu_to_le32(1), }; 1650 u8 *tweak_bytes = (u8 *)tweak; 1651 int i; 1652 1653 /* Read the computed ciphered tweak value. */ 1654 atmel_aes_read_block(dd, AES_ODATAR(0), tweak); 1655 /* 1656 * Hardware quirk: 1657 * the order of the ciphered tweak bytes need to be reversed before 1658 * writing them into the ODATARx registers. 1659 */ 1660 for (i = 0; i < AES_BLOCK_SIZE/2; ++i) 1661 swap(tweak_bytes[i], tweak_bytes[AES_BLOCK_SIZE - 1 - i]); 1662 1663 /* Process the data. */ 1664 atmel_aes_write_ctrl(dd, use_dma, NULL); 1665 atmel_aes_write_block(dd, AES_TWR(0), tweak); 1666 atmel_aes_write_block(dd, AES_ALPHAR(0), one); 1667 if (use_dma) 1668 return atmel_aes_dma_start(dd, req->src, req->dst, 1669 req->cryptlen, 1670 atmel_aes_transfer_complete); 1671 1672 return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen, 1673 atmel_aes_transfer_complete); 1674 } 1675 1676 static int atmel_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, 1677 unsigned int keylen) 1678 { 1679 struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 1680 int err; 1681 1682 err = xts_verify_key(tfm, key, keylen); 1683 if (err) 1684 return err; 1685 1686 crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK); 1687 crypto_skcipher_set_flags(ctx->fallback_tfm, tfm->base.crt_flags & 1688 CRYPTO_TFM_REQ_MASK); 1689 err = crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); 1690 if (err) 1691 return err; 1692 1693 memcpy(ctx->base.key, key, keylen/2); 1694 memcpy(ctx->key2, key + keylen/2, keylen/2); 1695 ctx->base.keylen = keylen/2; 1696 1697 return 0; 1698 } 1699 1700 static int atmel_aes_xts_encrypt(struct skcipher_request *req) 1701 { 1702 return atmel_aes_crypt(req, AES_FLAGS_XTS | AES_FLAGS_ENCRYPT); 1703 } 1704 1705 static int atmel_aes_xts_decrypt(struct skcipher_request *req) 1706 { 1707 return atmel_aes_crypt(req, AES_FLAGS_XTS); 1708 } 1709 1710 static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm) 1711 { 1712 struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 1713 struct atmel_aes_dev *dd; 1714 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 1715 1716 dd = atmel_aes_dev_alloc(&ctx->base); 1717 if (!dd) 1718 return -ENODEV; 1719 1720 ctx->fallback_tfm = crypto_alloc_skcipher(tfm_name, 0, 1721 CRYPTO_ALG_NEED_FALLBACK); 1722 if (IS_ERR(ctx->fallback_tfm)) 1723 return PTR_ERR(ctx->fallback_tfm); 1724 1725 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx) + 1726 crypto_skcipher_reqsize(ctx->fallback_tfm)); 1727 ctx->base.dd = dd; 1728 ctx->base.start = atmel_aes_xts_start; 1729 1730 return 0; 1731 } 1732 1733 static void atmel_aes_xts_exit_tfm(struct crypto_skcipher *tfm) 1734 { 1735 struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 1736 1737 crypto_free_skcipher(ctx->fallback_tfm); 1738 } 1739 1740 static struct skcipher_alg aes_xts_alg = { 1741 .base.cra_name = "xts(aes)", 1742 .base.cra_driver_name = "atmel-xts-aes", 1743 .base.cra_blocksize = AES_BLOCK_SIZE, 1744 .base.cra_ctxsize = sizeof(struct atmel_aes_xts_ctx), 1745 .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK | 1746 CRYPTO_ALG_KERN_DRIVER_ONLY, 1747 1748 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1749 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1750 .ivsize = AES_BLOCK_SIZE, 1751 .setkey = atmel_aes_xts_setkey, 1752 .encrypt = atmel_aes_xts_encrypt, 1753 .decrypt = atmel_aes_xts_decrypt, 1754 .init = atmel_aes_xts_init_tfm, 1755 .exit = atmel_aes_xts_exit_tfm, 1756 }; 1757 1758 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 1759 /* authenc aead functions */ 1760 1761 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd); 1762 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, 1763 bool is_async); 1764 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, 1765 bool is_async); 1766 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd); 1767 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, 1768 bool is_async); 1769 1770 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err) 1771 { 1772 struct aead_request *req = aead_request_cast(dd->areq); 1773 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1774 1775 if (err && (dd->flags & AES_FLAGS_OWN_SHA)) 1776 atmel_sha_authenc_abort(&rctx->auth_req); 1777 dd->flags &= ~AES_FLAGS_OWN_SHA; 1778 } 1779 1780 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd) 1781 { 1782 struct aead_request *req = aead_request_cast(dd->areq); 1783 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1784 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1785 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 1786 int err; 1787 1788 atmel_aes_set_mode(dd, &rctx->base); 1789 1790 err = atmel_aes_hw_init(dd); 1791 if (err) 1792 return atmel_aes_complete(dd, err); 1793 1794 return atmel_sha_authenc_schedule(&rctx->auth_req, ctx->auth, 1795 atmel_aes_authenc_init, dd); 1796 } 1797 1798 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, 1799 bool is_async) 1800 { 1801 struct aead_request *req = aead_request_cast(dd->areq); 1802 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1803 1804 if (is_async) 1805 dd->is_async = true; 1806 if (err) 1807 return atmel_aes_complete(dd, err); 1808 1809 /* If here, we've got the ownership of the SHA device. */ 1810 dd->flags |= AES_FLAGS_OWN_SHA; 1811 1812 /* Configure the SHA device. */ 1813 return atmel_sha_authenc_init(&rctx->auth_req, 1814 req->src, req->assoclen, 1815 rctx->textlen, 1816 atmel_aes_authenc_transfer, dd); 1817 } 1818 1819 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, 1820 bool is_async) 1821 { 1822 struct aead_request *req = aead_request_cast(dd->areq); 1823 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1824 bool enc = atmel_aes_is_encrypt(dd); 1825 struct scatterlist *src, *dst; 1826 __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; 1827 u32 emr; 1828 1829 if (is_async) 1830 dd->is_async = true; 1831 if (err) 1832 return atmel_aes_complete(dd, err); 1833 1834 /* Prepare src and dst scatter-lists to transfer cipher/plain texts. */ 1835 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); 1836 dst = src; 1837 1838 if (req->src != req->dst) 1839 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); 1840 1841 /* Configure the AES device. */ 1842 memcpy(iv, req->iv, sizeof(iv)); 1843 1844 /* 1845 * Here we always set the 2nd parameter of atmel_aes_write_ctrl() to 1846 * 'true' even if the data transfer is actually performed by the CPU (so 1847 * not by the DMA) because we must force the AES_MR_SMOD bitfield to the 1848 * value AES_MR_SMOD_IDATAR0. Indeed, both AES_MR_SMOD and SHA_MR_SMOD 1849 * must be set to *_MR_SMOD_IDATAR0. 1850 */ 1851 atmel_aes_write_ctrl(dd, true, iv); 1852 emr = AES_EMR_PLIPEN; 1853 if (!enc) 1854 emr |= AES_EMR_PLIPD; 1855 atmel_aes_write(dd, AES_EMR, emr); 1856 1857 /* Transfer data. */ 1858 return atmel_aes_dma_start(dd, src, dst, rctx->textlen, 1859 atmel_aes_authenc_digest); 1860 } 1861 1862 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd) 1863 { 1864 struct aead_request *req = aead_request_cast(dd->areq); 1865 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1866 1867 /* atmel_sha_authenc_final() releases the SHA device. */ 1868 dd->flags &= ~AES_FLAGS_OWN_SHA; 1869 return atmel_sha_authenc_final(&rctx->auth_req, 1870 rctx->digest, sizeof(rctx->digest), 1871 atmel_aes_authenc_final, dd); 1872 } 1873 1874 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, 1875 bool is_async) 1876 { 1877 struct aead_request *req = aead_request_cast(dd->areq); 1878 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1879 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1880 bool enc = atmel_aes_is_encrypt(dd); 1881 u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)], *odigest = rctx->digest; 1882 u32 offs, authsize; 1883 1884 if (is_async) 1885 dd->is_async = true; 1886 if (err) 1887 goto complete; 1888 1889 offs = req->assoclen + rctx->textlen; 1890 authsize = crypto_aead_authsize(tfm); 1891 if (enc) { 1892 scatterwalk_map_and_copy(odigest, req->dst, offs, authsize, 1); 1893 } else { 1894 scatterwalk_map_and_copy(idigest, req->src, offs, authsize, 0); 1895 if (crypto_memneq(idigest, odigest, authsize)) 1896 err = -EBADMSG; 1897 } 1898 1899 complete: 1900 return atmel_aes_complete(dd, err); 1901 } 1902 1903 static int atmel_aes_authenc_setkey(struct crypto_aead *tfm, const u8 *key, 1904 unsigned int keylen) 1905 { 1906 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 1907 struct crypto_authenc_keys keys; 1908 int err; 1909 1910 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1911 goto badkey; 1912 1913 if (keys.enckeylen > sizeof(ctx->base.key)) 1914 goto badkey; 1915 1916 /* Save auth key. */ 1917 err = atmel_sha_authenc_setkey(ctx->auth, 1918 keys.authkey, keys.authkeylen, 1919 crypto_aead_get_flags(tfm)); 1920 if (err) { 1921 memzero_explicit(&keys, sizeof(keys)); 1922 return err; 1923 } 1924 1925 /* Save enc key. */ 1926 ctx->base.keylen = keys.enckeylen; 1927 memcpy(ctx->base.key, keys.enckey, keys.enckeylen); 1928 1929 memzero_explicit(&keys, sizeof(keys)); 1930 return 0; 1931 1932 badkey: 1933 memzero_explicit(&keys, sizeof(keys)); 1934 return -EINVAL; 1935 } 1936 1937 static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm, 1938 unsigned long auth_mode) 1939 { 1940 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 1941 unsigned int auth_reqsize = atmel_sha_authenc_get_reqsize(); 1942 struct atmel_aes_dev *dd; 1943 1944 dd = atmel_aes_dev_alloc(&ctx->base); 1945 if (!dd) 1946 return -ENODEV; 1947 1948 ctx->auth = atmel_sha_authenc_spawn(auth_mode); 1949 if (IS_ERR(ctx->auth)) 1950 return PTR_ERR(ctx->auth); 1951 1952 crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) + 1953 auth_reqsize)); 1954 ctx->base.dd = dd; 1955 ctx->base.start = atmel_aes_authenc_start; 1956 1957 return 0; 1958 } 1959 1960 static int atmel_aes_authenc_hmac_sha1_init_tfm(struct crypto_aead *tfm) 1961 { 1962 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA1); 1963 } 1964 1965 static int atmel_aes_authenc_hmac_sha224_init_tfm(struct crypto_aead *tfm) 1966 { 1967 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA224); 1968 } 1969 1970 static int atmel_aes_authenc_hmac_sha256_init_tfm(struct crypto_aead *tfm) 1971 { 1972 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA256); 1973 } 1974 1975 static int atmel_aes_authenc_hmac_sha384_init_tfm(struct crypto_aead *tfm) 1976 { 1977 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA384); 1978 } 1979 1980 static int atmel_aes_authenc_hmac_sha512_init_tfm(struct crypto_aead *tfm) 1981 { 1982 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA512); 1983 } 1984 1985 static void atmel_aes_authenc_exit_tfm(struct crypto_aead *tfm) 1986 { 1987 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 1988 1989 atmel_sha_authenc_free(ctx->auth); 1990 } 1991 1992 static int atmel_aes_authenc_crypt(struct aead_request *req, 1993 unsigned long mode) 1994 { 1995 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1996 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1997 struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm); 1998 u32 authsize = crypto_aead_authsize(tfm); 1999 bool enc = (mode & AES_FLAGS_ENCRYPT); 2000 2001 /* Compute text length. */ 2002 if (!enc && req->cryptlen < authsize) 2003 return -EINVAL; 2004 rctx->textlen = req->cryptlen - (enc ? 0 : authsize); 2005 2006 /* 2007 * Currently, empty messages are not supported yet: 2008 * the SHA auto-padding can be used only on non-empty messages. 2009 * Hence a special case needs to be implemented for empty message. 2010 */ 2011 if (!rctx->textlen && !req->assoclen) 2012 return -EINVAL; 2013 2014 rctx->base.mode = mode; 2015 ctx->block_size = AES_BLOCK_SIZE; 2016 ctx->is_aead = true; 2017 2018 return atmel_aes_handle_queue(ctx->dd, &req->base); 2019 } 2020 2021 static int atmel_aes_authenc_cbc_aes_encrypt(struct aead_request *req) 2022 { 2023 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT); 2024 } 2025 2026 static int atmel_aes_authenc_cbc_aes_decrypt(struct aead_request *req) 2027 { 2028 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC); 2029 } 2030 2031 static struct aead_alg aes_authenc_algs[] = { 2032 { 2033 .setkey = atmel_aes_authenc_setkey, 2034 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2035 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2036 .init = atmel_aes_authenc_hmac_sha1_init_tfm, 2037 .exit = atmel_aes_authenc_exit_tfm, 2038 .ivsize = AES_BLOCK_SIZE, 2039 .maxauthsize = SHA1_DIGEST_SIZE, 2040 2041 .base = { 2042 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2043 .cra_driver_name = "atmel-authenc-hmac-sha1-cbc-aes", 2044 .cra_blocksize = AES_BLOCK_SIZE, 2045 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2046 }, 2047 }, 2048 { 2049 .setkey = atmel_aes_authenc_setkey, 2050 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2051 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2052 .init = atmel_aes_authenc_hmac_sha224_init_tfm, 2053 .exit = atmel_aes_authenc_exit_tfm, 2054 .ivsize = AES_BLOCK_SIZE, 2055 .maxauthsize = SHA224_DIGEST_SIZE, 2056 2057 .base = { 2058 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2059 .cra_driver_name = "atmel-authenc-hmac-sha224-cbc-aes", 2060 .cra_blocksize = AES_BLOCK_SIZE, 2061 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2062 }, 2063 }, 2064 { 2065 .setkey = atmel_aes_authenc_setkey, 2066 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2067 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2068 .init = atmel_aes_authenc_hmac_sha256_init_tfm, 2069 .exit = atmel_aes_authenc_exit_tfm, 2070 .ivsize = AES_BLOCK_SIZE, 2071 .maxauthsize = SHA256_DIGEST_SIZE, 2072 2073 .base = { 2074 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2075 .cra_driver_name = "atmel-authenc-hmac-sha256-cbc-aes", 2076 .cra_blocksize = AES_BLOCK_SIZE, 2077 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2078 }, 2079 }, 2080 { 2081 .setkey = atmel_aes_authenc_setkey, 2082 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2083 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2084 .init = atmel_aes_authenc_hmac_sha384_init_tfm, 2085 .exit = atmel_aes_authenc_exit_tfm, 2086 .ivsize = AES_BLOCK_SIZE, 2087 .maxauthsize = SHA384_DIGEST_SIZE, 2088 2089 .base = { 2090 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2091 .cra_driver_name = "atmel-authenc-hmac-sha384-cbc-aes", 2092 .cra_blocksize = AES_BLOCK_SIZE, 2093 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2094 }, 2095 }, 2096 { 2097 .setkey = atmel_aes_authenc_setkey, 2098 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2099 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2100 .init = atmel_aes_authenc_hmac_sha512_init_tfm, 2101 .exit = atmel_aes_authenc_exit_tfm, 2102 .ivsize = AES_BLOCK_SIZE, 2103 .maxauthsize = SHA512_DIGEST_SIZE, 2104 2105 .base = { 2106 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2107 .cra_driver_name = "atmel-authenc-hmac-sha512-cbc-aes", 2108 .cra_blocksize = AES_BLOCK_SIZE, 2109 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2110 }, 2111 }, 2112 }; 2113 #endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */ 2114 2115 /* Probe functions */ 2116 2117 static int atmel_aes_buff_init(struct atmel_aes_dev *dd) 2118 { 2119 dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER); 2120 dd->buflen = ATMEL_AES_BUFFER_SIZE; 2121 dd->buflen &= ~(AES_BLOCK_SIZE - 1); 2122 2123 if (!dd->buf) { 2124 dev_err(dd->dev, "unable to alloc pages.\n"); 2125 return -ENOMEM; 2126 } 2127 2128 return 0; 2129 } 2130 2131 static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd) 2132 { 2133 free_pages((unsigned long)dd->buf, ATMEL_AES_BUFFER_ORDER); 2134 } 2135 2136 static int atmel_aes_dma_init(struct atmel_aes_dev *dd) 2137 { 2138 int ret; 2139 2140 /* Try to grab 2 DMA channels */ 2141 dd->src.chan = dma_request_chan(dd->dev, "tx"); 2142 if (IS_ERR(dd->src.chan)) { 2143 ret = PTR_ERR(dd->src.chan); 2144 goto err_dma_in; 2145 } 2146 2147 dd->dst.chan = dma_request_chan(dd->dev, "rx"); 2148 if (IS_ERR(dd->dst.chan)) { 2149 ret = PTR_ERR(dd->dst.chan); 2150 goto err_dma_out; 2151 } 2152 2153 return 0; 2154 2155 err_dma_out: 2156 dma_release_channel(dd->src.chan); 2157 err_dma_in: 2158 dev_err(dd->dev, "no DMA channel available\n"); 2159 return ret; 2160 } 2161 2162 static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd) 2163 { 2164 dma_release_channel(dd->dst.chan); 2165 dma_release_channel(dd->src.chan); 2166 } 2167 2168 static void atmel_aes_queue_task(unsigned long data) 2169 { 2170 struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; 2171 2172 atmel_aes_handle_queue(dd, NULL); 2173 } 2174 2175 static void atmel_aes_done_task(unsigned long data) 2176 { 2177 struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; 2178 2179 dd->is_async = true; 2180 (void)dd->resume(dd); 2181 } 2182 2183 static irqreturn_t atmel_aes_irq(int irq, void *dev_id) 2184 { 2185 struct atmel_aes_dev *aes_dd = dev_id; 2186 u32 reg; 2187 2188 reg = atmel_aes_read(aes_dd, AES_ISR); 2189 if (reg & atmel_aes_read(aes_dd, AES_IMR)) { 2190 atmel_aes_write(aes_dd, AES_IDR, reg); 2191 if (AES_FLAGS_BUSY & aes_dd->flags) 2192 tasklet_schedule(&aes_dd->done_task); 2193 else 2194 dev_warn(aes_dd->dev, "AES interrupt when no active requests.\n"); 2195 return IRQ_HANDLED; 2196 } 2197 2198 return IRQ_NONE; 2199 } 2200 2201 static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) 2202 { 2203 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2204 if (dd->caps.has_authenc) 2205 crypto_unregister_aeads(aes_authenc_algs, 2206 ARRAY_SIZE(aes_authenc_algs)); 2207 #endif 2208 2209 if (dd->caps.has_xts) 2210 crypto_unregister_skcipher(&aes_xts_alg); 2211 2212 if (dd->caps.has_gcm) 2213 crypto_unregister_aead(&aes_gcm_alg); 2214 2215 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 2216 } 2217 2218 static void atmel_aes_crypto_alg_init(struct crypto_alg *alg) 2219 { 2220 alg->cra_flags |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2221 alg->cra_alignmask = 0xf; 2222 alg->cra_priority = ATMEL_AES_PRIORITY; 2223 alg->cra_module = THIS_MODULE; 2224 } 2225 2226 static int atmel_aes_register_algs(struct atmel_aes_dev *dd) 2227 { 2228 int err, i; 2229 2230 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 2231 atmel_aes_crypto_alg_init(&aes_algs[i].base); 2232 2233 err = crypto_register_skcipher(&aes_algs[i]); 2234 if (err) 2235 goto err_aes_algs; 2236 } 2237 2238 if (dd->caps.has_gcm) { 2239 atmel_aes_crypto_alg_init(&aes_gcm_alg.base); 2240 2241 err = crypto_register_aead(&aes_gcm_alg); 2242 if (err) 2243 goto err_aes_gcm_alg; 2244 } 2245 2246 if (dd->caps.has_xts) { 2247 atmel_aes_crypto_alg_init(&aes_xts_alg.base); 2248 2249 err = crypto_register_skcipher(&aes_xts_alg); 2250 if (err) 2251 goto err_aes_xts_alg; 2252 } 2253 2254 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2255 if (dd->caps.has_authenc) { 2256 for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) { 2257 atmel_aes_crypto_alg_init(&aes_authenc_algs[i].base); 2258 2259 err = crypto_register_aead(&aes_authenc_algs[i]); 2260 if (err) 2261 goto err_aes_authenc_alg; 2262 } 2263 } 2264 #endif 2265 2266 return 0; 2267 2268 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2269 /* i = ARRAY_SIZE(aes_authenc_algs); */ 2270 err_aes_authenc_alg: 2271 crypto_unregister_aeads(aes_authenc_algs, i); 2272 if (dd->caps.has_xts) 2273 crypto_unregister_skcipher(&aes_xts_alg); 2274 #endif 2275 err_aes_xts_alg: 2276 if (dd->caps.has_gcm) 2277 crypto_unregister_aead(&aes_gcm_alg); 2278 err_aes_gcm_alg: 2279 i = ARRAY_SIZE(aes_algs); 2280 err_aes_algs: 2281 crypto_unregister_skciphers(aes_algs, i); 2282 2283 return err; 2284 } 2285 2286 static void atmel_aes_get_cap(struct atmel_aes_dev *dd) 2287 { 2288 dd->caps.has_dualbuff = 0; 2289 dd->caps.has_gcm = 0; 2290 dd->caps.has_xts = 0; 2291 dd->caps.has_authenc = 0; 2292 dd->caps.max_burst_size = 1; 2293 2294 /* keep only major version number */ 2295 switch (dd->hw_version & 0xff0) { 2296 case 0x800: 2297 case 0x700: 2298 case 0x600: 2299 case 0x500: 2300 dd->caps.has_dualbuff = 1; 2301 dd->caps.has_gcm = 1; 2302 dd->caps.has_xts = 1; 2303 dd->caps.has_authenc = 1; 2304 dd->caps.max_burst_size = 4; 2305 break; 2306 case 0x200: 2307 dd->caps.has_dualbuff = 1; 2308 dd->caps.has_gcm = 1; 2309 dd->caps.max_burst_size = 4; 2310 break; 2311 case 0x130: 2312 dd->caps.has_dualbuff = 1; 2313 dd->caps.max_burst_size = 4; 2314 break; 2315 case 0x120: 2316 break; 2317 default: 2318 dev_warn(dd->dev, 2319 "Unmanaged aes version, set minimum capabilities\n"); 2320 break; 2321 } 2322 } 2323 2324 static const struct of_device_id atmel_aes_dt_ids[] = { 2325 { .compatible = "atmel,at91sam9g46-aes" }, 2326 { /* sentinel */ } 2327 }; 2328 MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids); 2329 2330 static int atmel_aes_probe(struct platform_device *pdev) 2331 { 2332 struct atmel_aes_dev *aes_dd; 2333 struct device *dev = &pdev->dev; 2334 struct resource *aes_res; 2335 int err; 2336 2337 aes_dd = devm_kzalloc(&pdev->dev, sizeof(*aes_dd), GFP_KERNEL); 2338 if (!aes_dd) 2339 return -ENOMEM; 2340 2341 aes_dd->dev = dev; 2342 2343 platform_set_drvdata(pdev, aes_dd); 2344 2345 INIT_LIST_HEAD(&aes_dd->list); 2346 spin_lock_init(&aes_dd->lock); 2347 2348 tasklet_init(&aes_dd->done_task, atmel_aes_done_task, 2349 (unsigned long)aes_dd); 2350 tasklet_init(&aes_dd->queue_task, atmel_aes_queue_task, 2351 (unsigned long)aes_dd); 2352 2353 crypto_init_queue(&aes_dd->queue, ATMEL_AES_QUEUE_LENGTH); 2354 2355 aes_dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &aes_res); 2356 if (IS_ERR(aes_dd->io_base)) { 2357 err = PTR_ERR(aes_dd->io_base); 2358 goto err_tasklet_kill; 2359 } 2360 aes_dd->phys_base = aes_res->start; 2361 2362 /* Get the IRQ */ 2363 aes_dd->irq = platform_get_irq(pdev, 0); 2364 if (aes_dd->irq < 0) { 2365 err = aes_dd->irq; 2366 goto err_tasklet_kill; 2367 } 2368 2369 err = devm_request_irq(&pdev->dev, aes_dd->irq, atmel_aes_irq, 2370 IRQF_SHARED, "atmel-aes", aes_dd); 2371 if (err) { 2372 dev_err(dev, "unable to request aes irq.\n"); 2373 goto err_tasklet_kill; 2374 } 2375 2376 /* Initializing the clock */ 2377 aes_dd->iclk = devm_clk_get_prepared(&pdev->dev, "aes_clk"); 2378 if (IS_ERR(aes_dd->iclk)) { 2379 dev_err(dev, "clock initialization failed.\n"); 2380 err = PTR_ERR(aes_dd->iclk); 2381 goto err_tasklet_kill; 2382 } 2383 2384 err = atmel_aes_hw_version_init(aes_dd); 2385 if (err) 2386 goto err_tasklet_kill; 2387 2388 atmel_aes_get_cap(aes_dd); 2389 2390 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2391 if (aes_dd->caps.has_authenc && !atmel_sha_authenc_is_ready()) { 2392 err = -EPROBE_DEFER; 2393 goto err_tasklet_kill; 2394 } 2395 #endif 2396 2397 err = atmel_aes_buff_init(aes_dd); 2398 if (err) 2399 goto err_tasklet_kill; 2400 2401 err = atmel_aes_dma_init(aes_dd); 2402 if (err) 2403 goto err_buff_cleanup; 2404 2405 spin_lock(&atmel_aes.lock); 2406 list_add_tail(&aes_dd->list, &atmel_aes.dev_list); 2407 spin_unlock(&atmel_aes.lock); 2408 2409 err = atmel_aes_register_algs(aes_dd); 2410 if (err) 2411 goto err_algs; 2412 2413 dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n", 2414 dma_chan_name(aes_dd->src.chan), 2415 dma_chan_name(aes_dd->dst.chan)); 2416 2417 return 0; 2418 2419 err_algs: 2420 spin_lock(&atmel_aes.lock); 2421 list_del(&aes_dd->list); 2422 spin_unlock(&atmel_aes.lock); 2423 atmel_aes_dma_cleanup(aes_dd); 2424 err_buff_cleanup: 2425 atmel_aes_buff_cleanup(aes_dd); 2426 err_tasklet_kill: 2427 tasklet_kill(&aes_dd->done_task); 2428 tasklet_kill(&aes_dd->queue_task); 2429 2430 return err; 2431 } 2432 2433 static void atmel_aes_remove(struct platform_device *pdev) 2434 { 2435 struct atmel_aes_dev *aes_dd; 2436 2437 aes_dd = platform_get_drvdata(pdev); 2438 2439 spin_lock(&atmel_aes.lock); 2440 list_del(&aes_dd->list); 2441 spin_unlock(&atmel_aes.lock); 2442 2443 atmel_aes_unregister_algs(aes_dd); 2444 2445 tasklet_kill(&aes_dd->done_task); 2446 tasklet_kill(&aes_dd->queue_task); 2447 2448 atmel_aes_dma_cleanup(aes_dd); 2449 atmel_aes_buff_cleanup(aes_dd); 2450 } 2451 2452 static struct platform_driver atmel_aes_driver = { 2453 .probe = atmel_aes_probe, 2454 .remove = atmel_aes_remove, 2455 .driver = { 2456 .name = "atmel_aes", 2457 .of_match_table = atmel_aes_dt_ids, 2458 }, 2459 }; 2460 2461 module_platform_driver(atmel_aes_driver); 2462 2463 MODULE_DESCRIPTION("Atmel AES hw acceleration support."); 2464 MODULE_LICENSE("GPL v2"); 2465 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 2466