1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Cryptographic API. 4 * 5 * Support for ATMEL AES HW acceleration. 6 * 7 * Copyright (c) 2012 Eukréa Electromatique - ATMEL 8 * Author: Nicolas Royer <nicolas@eukrea.com> 9 * 10 * Some ideas are from omap-aes.c driver. 11 */ 12 13 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/slab.h> 17 #include <linux/err.h> 18 #include <linux/clk.h> 19 #include <linux/io.h> 20 #include <linux/hw_random.h> 21 #include <linux/platform_device.h> 22 23 #include <linux/device.h> 24 #include <linux/dmaengine.h> 25 #include <linux/init.h> 26 #include <linux/errno.h> 27 #include <linux/interrupt.h> 28 #include <linux/irq.h> 29 #include <linux/scatterlist.h> 30 #include <linux/dma-mapping.h> 31 #include <linux/delay.h> 32 #include <linux/crypto.h> 33 #include <crypto/scatterwalk.h> 34 #include <crypto/algapi.h> 35 #include <crypto/aes.h> 36 #include <crypto/gcm.h> 37 #include <crypto/xts.h> 38 #include <crypto/internal/aead.h> 39 #include <crypto/internal/skcipher.h> 40 #include "atmel-aes-regs.h" 41 #include "atmel-authenc.h" 42 43 #define ATMEL_AES_PRIORITY 300 44 45 #define ATMEL_AES_BUFFER_ORDER 2 46 #define ATMEL_AES_BUFFER_SIZE (PAGE_SIZE << ATMEL_AES_BUFFER_ORDER) 47 48 #define SIZE_IN_WORDS(x) ((x) >> 2) 49 50 /* AES flags */ 51 /* Reserve bits [18:16] [14:12] [1:0] for mode (same as for AES_MR) */ 52 #define AES_FLAGS_ENCRYPT AES_MR_CYPHER_ENC 53 #define AES_FLAGS_GTAGEN AES_MR_GTAGEN 54 #define AES_FLAGS_OPMODE_MASK (AES_MR_OPMOD_MASK | AES_MR_CFBS_MASK) 55 #define AES_FLAGS_ECB AES_MR_OPMOD_ECB 56 #define AES_FLAGS_CBC AES_MR_OPMOD_CBC 57 #define AES_FLAGS_CTR AES_MR_OPMOD_CTR 58 #define AES_FLAGS_GCM AES_MR_OPMOD_GCM 59 #define AES_FLAGS_XTS AES_MR_OPMOD_XTS 60 61 #define AES_FLAGS_MODE_MASK (AES_FLAGS_OPMODE_MASK | \ 62 AES_FLAGS_ENCRYPT | \ 63 AES_FLAGS_GTAGEN) 64 65 #define AES_FLAGS_BUSY BIT(3) 66 #define AES_FLAGS_DUMP_REG BIT(4) 67 #define AES_FLAGS_OWN_SHA BIT(5) 68 69 #define AES_FLAGS_PERSISTENT AES_FLAGS_BUSY 70 71 #define ATMEL_AES_QUEUE_LENGTH 50 72 73 #define ATMEL_AES_DMA_THRESHOLD 256 74 75 76 struct atmel_aes_caps { 77 bool has_dualbuff; 78 bool has_gcm; 79 bool has_xts; 80 bool has_authenc; 81 u32 max_burst_size; 82 }; 83 84 struct atmel_aes_dev; 85 86 87 typedef int (*atmel_aes_fn_t)(struct atmel_aes_dev *); 88 89 90 struct atmel_aes_base_ctx { 91 struct atmel_aes_dev *dd; 92 atmel_aes_fn_t start; 93 int keylen; 94 u32 key[AES_KEYSIZE_256 / sizeof(u32)]; 95 u16 block_size; 96 bool is_aead; 97 }; 98 99 struct atmel_aes_ctx { 100 struct atmel_aes_base_ctx base; 101 }; 102 103 struct atmel_aes_ctr_ctx { 104 struct atmel_aes_base_ctx base; 105 106 __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; 107 size_t offset; 108 struct scatterlist src[2]; 109 struct scatterlist dst[2]; 110 u32 blocks; 111 }; 112 113 struct atmel_aes_gcm_ctx { 114 struct atmel_aes_base_ctx base; 115 116 struct scatterlist src[2]; 117 struct scatterlist dst[2]; 118 119 __be32 j0[AES_BLOCK_SIZE / sizeof(u32)]; 120 u32 tag[AES_BLOCK_SIZE / sizeof(u32)]; 121 __be32 ghash[AES_BLOCK_SIZE / sizeof(u32)]; 122 size_t textlen; 123 124 const __be32 *ghash_in; 125 __be32 *ghash_out; 126 atmel_aes_fn_t ghash_resume; 127 }; 128 129 struct atmel_aes_xts_ctx { 130 struct atmel_aes_base_ctx base; 131 132 u32 key2[AES_KEYSIZE_256 / sizeof(u32)]; 133 struct crypto_skcipher *fallback_tfm; 134 }; 135 136 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 137 struct atmel_aes_authenc_ctx { 138 struct atmel_aes_base_ctx base; 139 struct atmel_sha_authenc_ctx *auth; 140 }; 141 #endif 142 143 struct atmel_aes_reqctx { 144 unsigned long mode; 145 u8 lastc[AES_BLOCK_SIZE]; 146 struct skcipher_request fallback_req; 147 }; 148 149 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 150 struct atmel_aes_authenc_reqctx { 151 struct atmel_aes_reqctx base; 152 153 struct scatterlist src[2]; 154 struct scatterlist dst[2]; 155 size_t textlen; 156 u32 digest[SHA512_DIGEST_SIZE / sizeof(u32)]; 157 158 /* auth_req MUST be place last. */ 159 struct ahash_request auth_req; 160 }; 161 #endif 162 163 struct atmel_aes_dma { 164 struct dma_chan *chan; 165 struct scatterlist *sg; 166 int nents; 167 unsigned int remainder; 168 unsigned int sg_len; 169 }; 170 171 struct atmel_aes_dev { 172 struct list_head list; 173 unsigned long phys_base; 174 void __iomem *io_base; 175 176 struct crypto_async_request *areq; 177 struct atmel_aes_base_ctx *ctx; 178 179 bool is_async; 180 atmel_aes_fn_t resume; 181 atmel_aes_fn_t cpu_transfer_complete; 182 183 struct device *dev; 184 struct clk *iclk; 185 int irq; 186 187 unsigned long flags; 188 189 spinlock_t lock; 190 struct crypto_queue queue; 191 192 struct tasklet_struct done_task; 193 struct tasklet_struct queue_task; 194 195 size_t total; 196 size_t datalen; 197 u32 *data; 198 199 struct atmel_aes_dma src; 200 struct atmel_aes_dma dst; 201 202 size_t buflen; 203 void *buf; 204 struct scatterlist aligned_sg; 205 struct scatterlist *real_dst; 206 207 struct atmel_aes_caps caps; 208 209 u32 hw_version; 210 }; 211 212 struct atmel_aes_drv { 213 struct list_head dev_list; 214 spinlock_t lock; 215 }; 216 217 static struct atmel_aes_drv atmel_aes = { 218 .dev_list = LIST_HEAD_INIT(atmel_aes.dev_list), 219 .lock = __SPIN_LOCK_UNLOCKED(atmel_aes.lock), 220 }; 221 222 #ifdef VERBOSE_DEBUG 223 static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz) 224 { 225 switch (offset) { 226 case AES_CR: 227 return "CR"; 228 229 case AES_MR: 230 return "MR"; 231 232 case AES_ISR: 233 return "ISR"; 234 235 case AES_IMR: 236 return "IMR"; 237 238 case AES_IER: 239 return "IER"; 240 241 case AES_IDR: 242 return "IDR"; 243 244 case AES_KEYWR(0): 245 case AES_KEYWR(1): 246 case AES_KEYWR(2): 247 case AES_KEYWR(3): 248 case AES_KEYWR(4): 249 case AES_KEYWR(5): 250 case AES_KEYWR(6): 251 case AES_KEYWR(7): 252 snprintf(tmp, sz, "KEYWR[%u]", (offset - AES_KEYWR(0)) >> 2); 253 break; 254 255 case AES_IDATAR(0): 256 case AES_IDATAR(1): 257 case AES_IDATAR(2): 258 case AES_IDATAR(3): 259 snprintf(tmp, sz, "IDATAR[%u]", (offset - AES_IDATAR(0)) >> 2); 260 break; 261 262 case AES_ODATAR(0): 263 case AES_ODATAR(1): 264 case AES_ODATAR(2): 265 case AES_ODATAR(3): 266 snprintf(tmp, sz, "ODATAR[%u]", (offset - AES_ODATAR(0)) >> 2); 267 break; 268 269 case AES_IVR(0): 270 case AES_IVR(1): 271 case AES_IVR(2): 272 case AES_IVR(3): 273 snprintf(tmp, sz, "IVR[%u]", (offset - AES_IVR(0)) >> 2); 274 break; 275 276 case AES_AADLENR: 277 return "AADLENR"; 278 279 case AES_CLENR: 280 return "CLENR"; 281 282 case AES_GHASHR(0): 283 case AES_GHASHR(1): 284 case AES_GHASHR(2): 285 case AES_GHASHR(3): 286 snprintf(tmp, sz, "GHASHR[%u]", (offset - AES_GHASHR(0)) >> 2); 287 break; 288 289 case AES_TAGR(0): 290 case AES_TAGR(1): 291 case AES_TAGR(2): 292 case AES_TAGR(3): 293 snprintf(tmp, sz, "TAGR[%u]", (offset - AES_TAGR(0)) >> 2); 294 break; 295 296 case AES_CTRR: 297 return "CTRR"; 298 299 case AES_GCMHR(0): 300 case AES_GCMHR(1): 301 case AES_GCMHR(2): 302 case AES_GCMHR(3): 303 snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2); 304 break; 305 306 case AES_EMR: 307 return "EMR"; 308 309 case AES_TWR(0): 310 case AES_TWR(1): 311 case AES_TWR(2): 312 case AES_TWR(3): 313 snprintf(tmp, sz, "TWR[%u]", (offset - AES_TWR(0)) >> 2); 314 break; 315 316 case AES_ALPHAR(0): 317 case AES_ALPHAR(1): 318 case AES_ALPHAR(2): 319 case AES_ALPHAR(3): 320 snprintf(tmp, sz, "ALPHAR[%u]", (offset - AES_ALPHAR(0)) >> 2); 321 break; 322 323 default: 324 snprintf(tmp, sz, "0x%02x", offset); 325 break; 326 } 327 328 return tmp; 329 } 330 #endif /* VERBOSE_DEBUG */ 331 332 /* Shared functions */ 333 334 static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset) 335 { 336 u32 value = readl_relaxed(dd->io_base + offset); 337 338 #ifdef VERBOSE_DEBUG 339 if (dd->flags & AES_FLAGS_DUMP_REG) { 340 char tmp[16]; 341 342 dev_vdbg(dd->dev, "read 0x%08x from %s\n", value, 343 atmel_aes_reg_name(offset, tmp, sizeof(tmp))); 344 } 345 #endif /* VERBOSE_DEBUG */ 346 347 return value; 348 } 349 350 static inline void atmel_aes_write(struct atmel_aes_dev *dd, 351 u32 offset, u32 value) 352 { 353 #ifdef VERBOSE_DEBUG 354 if (dd->flags & AES_FLAGS_DUMP_REG) { 355 char tmp[16]; 356 357 dev_vdbg(dd->dev, "write 0x%08x into %s\n", value, 358 atmel_aes_reg_name(offset, tmp, sizeof(tmp))); 359 } 360 #endif /* VERBOSE_DEBUG */ 361 362 writel_relaxed(value, dd->io_base + offset); 363 } 364 365 static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset, 366 u32 *value, int count) 367 { 368 for (; count--; value++, offset += 4) 369 *value = atmel_aes_read(dd, offset); 370 } 371 372 static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset, 373 const u32 *value, int count) 374 { 375 for (; count--; value++, offset += 4) 376 atmel_aes_write(dd, offset, *value); 377 } 378 379 static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset, 380 void *value) 381 { 382 atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); 383 } 384 385 static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset, 386 const void *value) 387 { 388 atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); 389 } 390 391 static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd, 392 atmel_aes_fn_t resume) 393 { 394 u32 isr = atmel_aes_read(dd, AES_ISR); 395 396 if (unlikely(isr & AES_INT_DATARDY)) 397 return resume(dd); 398 399 dd->resume = resume; 400 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 401 return -EINPROGRESS; 402 } 403 404 static inline size_t atmel_aes_padlen(size_t len, size_t block_size) 405 { 406 len &= block_size - 1; 407 return len ? block_size - len : 0; 408 } 409 410 static struct atmel_aes_dev *atmel_aes_dev_alloc(struct atmel_aes_base_ctx *ctx) 411 { 412 struct atmel_aes_dev *aes_dd; 413 414 spin_lock_bh(&atmel_aes.lock); 415 /* One AES IP per SoC. */ 416 aes_dd = list_first_entry_or_null(&atmel_aes.dev_list, 417 struct atmel_aes_dev, list); 418 spin_unlock_bh(&atmel_aes.lock); 419 return aes_dd; 420 } 421 422 static int atmel_aes_hw_init(struct atmel_aes_dev *dd) 423 { 424 int err; 425 426 err = clk_enable(dd->iclk); 427 if (err) 428 return err; 429 430 atmel_aes_write(dd, AES_CR, AES_CR_SWRST); 431 atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); 432 433 return 0; 434 } 435 436 static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd) 437 { 438 return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff; 439 } 440 441 static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd) 442 { 443 int err; 444 445 err = atmel_aes_hw_init(dd); 446 if (err) 447 return err; 448 449 dd->hw_version = atmel_aes_get_version(dd); 450 451 dev_info(dd->dev, "version: 0x%x\n", dd->hw_version); 452 453 clk_disable(dd->iclk); 454 return 0; 455 } 456 457 static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd, 458 const struct atmel_aes_reqctx *rctx) 459 { 460 /* Clear all but persistent flags and set request flags. */ 461 dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode; 462 } 463 464 static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd) 465 { 466 return (dd->flags & AES_FLAGS_ENCRYPT); 467 } 468 469 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 470 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err); 471 #endif 472 473 static void atmel_aes_set_iv_as_last_ciphertext_block(struct atmel_aes_dev *dd) 474 { 475 struct skcipher_request *req = skcipher_request_cast(dd->areq); 476 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 477 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 478 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 479 480 if (req->cryptlen < ivsize) 481 return; 482 483 if (rctx->mode & AES_FLAGS_ENCRYPT) 484 scatterwalk_map_and_copy(req->iv, req->dst, 485 req->cryptlen - ivsize, ivsize, 0); 486 else 487 memcpy(req->iv, rctx->lastc, ivsize); 488 } 489 490 static inline struct atmel_aes_ctr_ctx * 491 atmel_aes_ctr_ctx_cast(struct atmel_aes_base_ctx *ctx) 492 { 493 return container_of(ctx, struct atmel_aes_ctr_ctx, base); 494 } 495 496 static void atmel_aes_ctr_update_req_iv(struct atmel_aes_dev *dd) 497 { 498 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); 499 struct skcipher_request *req = skcipher_request_cast(dd->areq); 500 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 501 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 502 int i; 503 504 /* 505 * The CTR transfer works in fragments of data of maximum 1 MByte 506 * because of the 16 bit CTR counter embedded in the IP. When reaching 507 * here, ctx->blocks contains the number of blocks of the last fragment 508 * processed, there is no need to explicit cast it to u16. 509 */ 510 for (i = 0; i < ctx->blocks; i++) 511 crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE); 512 513 memcpy(req->iv, ctx->iv, ivsize); 514 } 515 516 static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) 517 { 518 struct skcipher_request *req = skcipher_request_cast(dd->areq); 519 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 520 521 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 522 if (dd->ctx->is_aead) 523 atmel_aes_authenc_complete(dd, err); 524 #endif 525 526 clk_disable(dd->iclk); 527 dd->flags &= ~AES_FLAGS_BUSY; 528 529 if (!err && !dd->ctx->is_aead && 530 (rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_ECB) { 531 if ((rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_CTR) 532 atmel_aes_set_iv_as_last_ciphertext_block(dd); 533 else 534 atmel_aes_ctr_update_req_iv(dd); 535 } 536 537 if (dd->is_async) 538 crypto_request_complete(dd->areq, err); 539 540 tasklet_schedule(&dd->queue_task); 541 542 return err; 543 } 544 545 static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma, 546 const __be32 *iv, const u32 *key, int keylen) 547 { 548 u32 valmr = 0; 549 550 /* MR register must be set before IV registers */ 551 if (keylen == AES_KEYSIZE_128) 552 valmr |= AES_MR_KEYSIZE_128; 553 else if (keylen == AES_KEYSIZE_192) 554 valmr |= AES_MR_KEYSIZE_192; 555 else 556 valmr |= AES_MR_KEYSIZE_256; 557 558 valmr |= dd->flags & AES_FLAGS_MODE_MASK; 559 560 if (use_dma) { 561 valmr |= AES_MR_SMOD_IDATAR0; 562 if (dd->caps.has_dualbuff) 563 valmr |= AES_MR_DUALBUFF; 564 } else { 565 valmr |= AES_MR_SMOD_AUTO; 566 } 567 568 atmel_aes_write(dd, AES_MR, valmr); 569 570 atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen)); 571 572 if (iv && (valmr & AES_MR_OPMOD_MASK) != AES_MR_OPMOD_ECB) 573 atmel_aes_write_block(dd, AES_IVR(0), iv); 574 } 575 576 static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma, 577 const __be32 *iv) 578 579 { 580 atmel_aes_write_ctrl_key(dd, use_dma, iv, 581 dd->ctx->key, dd->ctx->keylen); 582 } 583 584 /* CPU transfer */ 585 586 static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd) 587 { 588 int err = 0; 589 u32 isr; 590 591 for (;;) { 592 atmel_aes_read_block(dd, AES_ODATAR(0), dd->data); 593 dd->data += 4; 594 dd->datalen -= AES_BLOCK_SIZE; 595 596 if (dd->datalen < AES_BLOCK_SIZE) 597 break; 598 599 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 600 601 isr = atmel_aes_read(dd, AES_ISR); 602 if (!(isr & AES_INT_DATARDY)) { 603 dd->resume = atmel_aes_cpu_transfer; 604 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 605 return -EINPROGRESS; 606 } 607 } 608 609 if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), 610 dd->buf, dd->total)) 611 err = -EINVAL; 612 613 if (err) 614 return atmel_aes_complete(dd, err); 615 616 return dd->cpu_transfer_complete(dd); 617 } 618 619 static int atmel_aes_cpu_start(struct atmel_aes_dev *dd, 620 struct scatterlist *src, 621 struct scatterlist *dst, 622 size_t len, 623 atmel_aes_fn_t resume) 624 { 625 size_t padlen = atmel_aes_padlen(len, AES_BLOCK_SIZE); 626 627 if (unlikely(len == 0)) 628 return -EINVAL; 629 630 sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); 631 632 dd->total = len; 633 dd->real_dst = dst; 634 dd->cpu_transfer_complete = resume; 635 dd->datalen = len + padlen; 636 dd->data = (u32 *)dd->buf; 637 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 638 return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer); 639 } 640 641 642 /* DMA transfer */ 643 644 static void atmel_aes_dma_callback(void *data); 645 646 static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd, 647 struct scatterlist *sg, 648 size_t len, 649 struct atmel_aes_dma *dma) 650 { 651 int nents; 652 653 if (!IS_ALIGNED(len, dd->ctx->block_size)) 654 return false; 655 656 for (nents = 0; sg; sg = sg_next(sg), ++nents) { 657 if (!IS_ALIGNED(sg->offset, sizeof(u32))) 658 return false; 659 660 if (len <= sg->length) { 661 if (!IS_ALIGNED(len, dd->ctx->block_size)) 662 return false; 663 664 dma->nents = nents+1; 665 dma->remainder = sg->length - len; 666 sg->length = len; 667 return true; 668 } 669 670 if (!IS_ALIGNED(sg->length, dd->ctx->block_size)) 671 return false; 672 673 len -= sg->length; 674 } 675 676 return false; 677 } 678 679 static inline void atmel_aes_restore_sg(const struct atmel_aes_dma *dma) 680 { 681 struct scatterlist *sg = dma->sg; 682 int nents = dma->nents; 683 684 if (!dma->remainder) 685 return; 686 687 while (--nents > 0 && sg) 688 sg = sg_next(sg); 689 690 if (!sg) 691 return; 692 693 sg->length += dma->remainder; 694 } 695 696 static int atmel_aes_map(struct atmel_aes_dev *dd, 697 struct scatterlist *src, 698 struct scatterlist *dst, 699 size_t len) 700 { 701 bool src_aligned, dst_aligned; 702 size_t padlen; 703 704 dd->total = len; 705 dd->src.sg = src; 706 dd->dst.sg = dst; 707 dd->real_dst = dst; 708 709 src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src); 710 if (src == dst) 711 dst_aligned = src_aligned; 712 else 713 dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst); 714 if (!src_aligned || !dst_aligned) { 715 padlen = atmel_aes_padlen(len, dd->ctx->block_size); 716 717 if (dd->buflen < len + padlen) 718 return -ENOMEM; 719 720 if (!src_aligned) { 721 sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); 722 dd->src.sg = &dd->aligned_sg; 723 dd->src.nents = 1; 724 dd->src.remainder = 0; 725 } 726 727 if (!dst_aligned) { 728 dd->dst.sg = &dd->aligned_sg; 729 dd->dst.nents = 1; 730 dd->dst.remainder = 0; 731 } 732 733 sg_init_table(&dd->aligned_sg, 1); 734 sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen); 735 } 736 737 if (dd->src.sg == dd->dst.sg) { 738 dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, 739 DMA_BIDIRECTIONAL); 740 dd->dst.sg_len = dd->src.sg_len; 741 if (!dd->src.sg_len) 742 return -EFAULT; 743 } else { 744 dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, 745 DMA_TO_DEVICE); 746 if (!dd->src.sg_len) 747 return -EFAULT; 748 749 dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents, 750 DMA_FROM_DEVICE); 751 if (!dd->dst.sg_len) { 752 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 753 DMA_TO_DEVICE); 754 return -EFAULT; 755 } 756 } 757 758 return 0; 759 } 760 761 static void atmel_aes_unmap(struct atmel_aes_dev *dd) 762 { 763 if (dd->src.sg == dd->dst.sg) { 764 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 765 DMA_BIDIRECTIONAL); 766 767 if (dd->src.sg != &dd->aligned_sg) 768 atmel_aes_restore_sg(&dd->src); 769 } else { 770 dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents, 771 DMA_FROM_DEVICE); 772 773 if (dd->dst.sg != &dd->aligned_sg) 774 atmel_aes_restore_sg(&dd->dst); 775 776 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 777 DMA_TO_DEVICE); 778 779 if (dd->src.sg != &dd->aligned_sg) 780 atmel_aes_restore_sg(&dd->src); 781 } 782 783 if (dd->dst.sg == &dd->aligned_sg) 784 sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), 785 dd->buf, dd->total); 786 } 787 788 static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd, 789 enum dma_slave_buswidth addr_width, 790 enum dma_transfer_direction dir, 791 u32 maxburst) 792 { 793 struct dma_async_tx_descriptor *desc; 794 struct dma_slave_config config; 795 dma_async_tx_callback callback; 796 struct atmel_aes_dma *dma; 797 798 memset(&config, 0, sizeof(config)); 799 config.src_addr_width = addr_width; 800 config.dst_addr_width = addr_width; 801 config.src_maxburst = maxburst; 802 config.dst_maxburst = maxburst; 803 804 switch (dir) { 805 case DMA_MEM_TO_DEV: 806 dma = &dd->src; 807 callback = NULL; 808 config.dst_addr = dd->phys_base + AES_IDATAR(0); 809 break; 810 811 case DMA_DEV_TO_MEM: 812 dma = &dd->dst; 813 callback = atmel_aes_dma_callback; 814 config.src_addr = dd->phys_base + AES_ODATAR(0); 815 break; 816 817 default: 818 return -EINVAL; 819 } 820 821 desc = dmaengine_prep_config_sg(dma->chan, dma->sg, dma->sg_len, dir, 822 DMA_PREP_INTERRUPT | DMA_CTRL_ACK, 823 &config); 824 if (!desc) 825 return -ENOMEM; 826 827 desc->callback = callback; 828 desc->callback_param = dd; 829 dmaengine_submit(desc); 830 dma_async_issue_pending(dma->chan); 831 832 return 0; 833 } 834 835 static int atmel_aes_dma_start(struct atmel_aes_dev *dd, 836 struct scatterlist *src, 837 struct scatterlist *dst, 838 size_t len, 839 atmel_aes_fn_t resume) 840 { 841 enum dma_slave_buswidth addr_width; 842 u32 maxburst; 843 int err; 844 845 switch (dd->ctx->block_size) { 846 case AES_BLOCK_SIZE: 847 addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 848 maxburst = dd->caps.max_burst_size; 849 break; 850 851 default: 852 err = -EINVAL; 853 goto exit; 854 } 855 856 err = atmel_aes_map(dd, src, dst, len); 857 if (err) 858 goto exit; 859 860 dd->resume = resume; 861 862 /* Set output DMA transfer first */ 863 err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM, 864 maxburst); 865 if (err) 866 goto unmap; 867 868 /* Then set input DMA transfer */ 869 err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV, 870 maxburst); 871 if (err) 872 goto output_transfer_stop; 873 874 return -EINPROGRESS; 875 876 output_transfer_stop: 877 dmaengine_terminate_sync(dd->dst.chan); 878 unmap: 879 atmel_aes_unmap(dd); 880 exit: 881 return atmel_aes_complete(dd, err); 882 } 883 884 static void atmel_aes_dma_callback(void *data) 885 { 886 struct atmel_aes_dev *dd = data; 887 888 atmel_aes_unmap(dd); 889 dd->is_async = true; 890 (void)dd->resume(dd); 891 } 892 893 static int atmel_aes_handle_queue(struct atmel_aes_dev *dd, 894 struct crypto_async_request *new_areq) 895 { 896 struct crypto_async_request *areq, *backlog; 897 struct atmel_aes_base_ctx *ctx; 898 unsigned long flags; 899 bool start_async; 900 int err, ret = 0; 901 902 spin_lock_irqsave(&dd->lock, flags); 903 if (new_areq) 904 ret = crypto_enqueue_request(&dd->queue, new_areq); 905 if (dd->flags & AES_FLAGS_BUSY) { 906 spin_unlock_irqrestore(&dd->lock, flags); 907 return ret; 908 } 909 backlog = crypto_get_backlog(&dd->queue); 910 areq = crypto_dequeue_request(&dd->queue); 911 if (areq) 912 dd->flags |= AES_FLAGS_BUSY; 913 spin_unlock_irqrestore(&dd->lock, flags); 914 915 if (!areq) 916 return ret; 917 918 if (backlog) 919 crypto_request_complete(backlog, -EINPROGRESS); 920 921 ctx = crypto_tfm_ctx(areq->tfm); 922 923 dd->areq = areq; 924 dd->ctx = ctx; 925 start_async = (areq != new_areq); 926 dd->is_async = start_async; 927 928 /* WARNING: ctx->start() MAY change dd->is_async. */ 929 err = ctx->start(dd); 930 return (start_async) ? ret : err; 931 } 932 933 934 /* AES async block ciphers */ 935 936 static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd) 937 { 938 return atmel_aes_complete(dd, 0); 939 } 940 941 static int atmel_aes_start(struct atmel_aes_dev *dd) 942 { 943 struct skcipher_request *req = skcipher_request_cast(dd->areq); 944 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 945 bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD || 946 dd->ctx->block_size != AES_BLOCK_SIZE); 947 int err; 948 949 atmel_aes_set_mode(dd, rctx); 950 951 err = atmel_aes_hw_init(dd); 952 if (err) 953 return atmel_aes_complete(dd, err); 954 955 atmel_aes_write_ctrl(dd, use_dma, (void *)req->iv); 956 if (use_dma) 957 return atmel_aes_dma_start(dd, req->src, req->dst, 958 req->cryptlen, 959 atmel_aes_transfer_complete); 960 961 return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen, 962 atmel_aes_transfer_complete); 963 } 964 965 static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd) 966 { 967 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); 968 struct skcipher_request *req = skcipher_request_cast(dd->areq); 969 struct scatterlist *src, *dst; 970 size_t datalen; 971 u32 ctr; 972 u16 start, end; 973 bool use_dma, fragmented = false; 974 975 /* Check for transfer completion. */ 976 ctx->offset += dd->total; 977 if (ctx->offset >= req->cryptlen) 978 return atmel_aes_transfer_complete(dd); 979 980 /* Compute data length. */ 981 datalen = req->cryptlen - ctx->offset; 982 ctx->blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE); 983 ctr = be32_to_cpu(ctx->iv[3]); 984 985 /* Check 16bit counter overflow. */ 986 start = ctr & 0xffff; 987 end = start + ctx->blocks - 1; 988 989 if (ctx->blocks >> 16 || end < start) { 990 ctr |= 0xffff; 991 datalen = AES_BLOCK_SIZE * (0x10000 - start); 992 fragmented = true; 993 } 994 995 use_dma = (datalen >= ATMEL_AES_DMA_THRESHOLD); 996 997 /* Jump to offset. */ 998 src = scatterwalk_ffwd(ctx->src, req->src, ctx->offset); 999 dst = ((req->src == req->dst) ? src : 1000 scatterwalk_ffwd(ctx->dst, req->dst, ctx->offset)); 1001 1002 /* Configure hardware. */ 1003 atmel_aes_write_ctrl(dd, use_dma, ctx->iv); 1004 if (unlikely(fragmented)) { 1005 /* 1006 * Increment the counter manually to cope with the hardware 1007 * counter overflow. 1008 */ 1009 ctx->iv[3] = cpu_to_be32(ctr); 1010 crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE); 1011 } 1012 1013 if (use_dma) 1014 return atmel_aes_dma_start(dd, src, dst, datalen, 1015 atmel_aes_ctr_transfer); 1016 1017 return atmel_aes_cpu_start(dd, src, dst, datalen, 1018 atmel_aes_ctr_transfer); 1019 } 1020 1021 static int atmel_aes_ctr_start(struct atmel_aes_dev *dd) 1022 { 1023 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); 1024 struct skcipher_request *req = skcipher_request_cast(dd->areq); 1025 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 1026 int err; 1027 1028 atmel_aes_set_mode(dd, rctx); 1029 1030 err = atmel_aes_hw_init(dd); 1031 if (err) 1032 return atmel_aes_complete(dd, err); 1033 1034 memcpy(ctx->iv, req->iv, AES_BLOCK_SIZE); 1035 ctx->offset = 0; 1036 dd->total = 0; 1037 return atmel_aes_ctr_transfer(dd); 1038 } 1039 1040 static int atmel_aes_xts_fallback(struct skcipher_request *req, bool enc) 1041 { 1042 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 1043 struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx( 1044 crypto_skcipher_reqtfm(req)); 1045 1046 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 1047 skcipher_request_set_callback(&rctx->fallback_req, req->base.flags, 1048 req->base.complete, req->base.data); 1049 skcipher_request_set_crypt(&rctx->fallback_req, req->src, req->dst, 1050 req->cryptlen, req->iv); 1051 1052 return enc ? crypto_skcipher_encrypt(&rctx->fallback_req) : 1053 crypto_skcipher_decrypt(&rctx->fallback_req); 1054 } 1055 1056 static int atmel_aes_crypt(struct skcipher_request *req, unsigned long mode) 1057 { 1058 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1059 struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(skcipher); 1060 struct atmel_aes_reqctx *rctx; 1061 u32 opmode = mode & AES_FLAGS_OPMODE_MASK; 1062 1063 if (opmode == AES_FLAGS_XTS) { 1064 if (req->cryptlen < XTS_BLOCK_SIZE) 1065 return -EINVAL; 1066 1067 if (!IS_ALIGNED(req->cryptlen, XTS_BLOCK_SIZE)) 1068 return atmel_aes_xts_fallback(req, 1069 mode & AES_FLAGS_ENCRYPT); 1070 } 1071 1072 /* 1073 * ECB, CBC or CTR mode require the plaintext and ciphertext 1074 * to have a positve integer length. 1075 */ 1076 if (!req->cryptlen && opmode != AES_FLAGS_XTS) 1077 return 0; 1078 1079 if ((opmode == AES_FLAGS_ECB || opmode == AES_FLAGS_CBC) && 1080 !IS_ALIGNED(req->cryptlen, crypto_skcipher_blocksize(skcipher))) 1081 return -EINVAL; 1082 1083 ctx->block_size = AES_BLOCK_SIZE; 1084 ctx->is_aead = false; 1085 1086 rctx = skcipher_request_ctx(req); 1087 rctx->mode = mode; 1088 1089 if (opmode != AES_FLAGS_ECB && 1090 !(mode & AES_FLAGS_ENCRYPT)) { 1091 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1092 1093 if (req->cryptlen >= ivsize) 1094 scatterwalk_map_and_copy(rctx->lastc, req->src, 1095 req->cryptlen - ivsize, 1096 ivsize, 0); 1097 } 1098 1099 return atmel_aes_handle_queue(ctx->dd, &req->base); 1100 } 1101 1102 static int atmel_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 1103 unsigned int keylen) 1104 { 1105 struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(tfm); 1106 1107 if (keylen != AES_KEYSIZE_128 && 1108 keylen != AES_KEYSIZE_192 && 1109 keylen != AES_KEYSIZE_256) 1110 return -EINVAL; 1111 1112 memcpy(ctx->key, key, keylen); 1113 ctx->keylen = keylen; 1114 1115 return 0; 1116 } 1117 1118 static int atmel_aes_ecb_encrypt(struct skcipher_request *req) 1119 { 1120 return atmel_aes_crypt(req, AES_FLAGS_ECB | AES_FLAGS_ENCRYPT); 1121 } 1122 1123 static int atmel_aes_ecb_decrypt(struct skcipher_request *req) 1124 { 1125 return atmel_aes_crypt(req, AES_FLAGS_ECB); 1126 } 1127 1128 static int atmel_aes_cbc_encrypt(struct skcipher_request *req) 1129 { 1130 return atmel_aes_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT); 1131 } 1132 1133 static int atmel_aes_cbc_decrypt(struct skcipher_request *req) 1134 { 1135 return atmel_aes_crypt(req, AES_FLAGS_CBC); 1136 } 1137 1138 static int atmel_aes_ctr_encrypt(struct skcipher_request *req) 1139 { 1140 return atmel_aes_crypt(req, AES_FLAGS_CTR | AES_FLAGS_ENCRYPT); 1141 } 1142 1143 static int atmel_aes_ctr_decrypt(struct skcipher_request *req) 1144 { 1145 return atmel_aes_crypt(req, AES_FLAGS_CTR); 1146 } 1147 1148 static int atmel_aes_init_tfm(struct crypto_skcipher *tfm) 1149 { 1150 struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 1151 struct atmel_aes_dev *dd; 1152 1153 dd = atmel_aes_dev_alloc(&ctx->base); 1154 if (!dd) 1155 return -ENODEV; 1156 1157 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); 1158 ctx->base.dd = dd; 1159 ctx->base.start = atmel_aes_start; 1160 1161 return 0; 1162 } 1163 1164 static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm) 1165 { 1166 struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 1167 struct atmel_aes_dev *dd; 1168 1169 dd = atmel_aes_dev_alloc(&ctx->base); 1170 if (!dd) 1171 return -ENODEV; 1172 1173 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); 1174 ctx->base.dd = dd; 1175 ctx->base.start = atmel_aes_ctr_start; 1176 1177 return 0; 1178 } 1179 1180 static struct skcipher_alg aes_algs[] = { 1181 { 1182 .base.cra_name = "ecb(aes)", 1183 .base.cra_driver_name = "atmel-ecb-aes", 1184 .base.cra_blocksize = AES_BLOCK_SIZE, 1185 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1186 1187 .init = atmel_aes_init_tfm, 1188 .min_keysize = AES_MIN_KEY_SIZE, 1189 .max_keysize = AES_MAX_KEY_SIZE, 1190 .setkey = atmel_aes_setkey, 1191 .encrypt = atmel_aes_ecb_encrypt, 1192 .decrypt = atmel_aes_ecb_decrypt, 1193 }, 1194 { 1195 .base.cra_name = "cbc(aes)", 1196 .base.cra_driver_name = "atmel-cbc-aes", 1197 .base.cra_blocksize = AES_BLOCK_SIZE, 1198 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1199 1200 .init = atmel_aes_init_tfm, 1201 .min_keysize = AES_MIN_KEY_SIZE, 1202 .max_keysize = AES_MAX_KEY_SIZE, 1203 .setkey = atmel_aes_setkey, 1204 .encrypt = atmel_aes_cbc_encrypt, 1205 .decrypt = atmel_aes_cbc_decrypt, 1206 .ivsize = AES_BLOCK_SIZE, 1207 }, 1208 { 1209 .base.cra_name = "ctr(aes)", 1210 .base.cra_driver_name = "atmel-ctr-aes", 1211 .base.cra_blocksize = 1, 1212 .base.cra_ctxsize = sizeof(struct atmel_aes_ctr_ctx), 1213 1214 .init = atmel_aes_ctr_init_tfm, 1215 .min_keysize = AES_MIN_KEY_SIZE, 1216 .max_keysize = AES_MAX_KEY_SIZE, 1217 .setkey = atmel_aes_setkey, 1218 .encrypt = atmel_aes_ctr_encrypt, 1219 .decrypt = atmel_aes_ctr_decrypt, 1220 .ivsize = AES_BLOCK_SIZE, 1221 }, 1222 }; 1223 1224 1225 /* gcm aead functions */ 1226 1227 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, 1228 const u32 *data, size_t datalen, 1229 const __be32 *ghash_in, __be32 *ghash_out, 1230 atmel_aes_fn_t resume); 1231 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd); 1232 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd); 1233 1234 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd); 1235 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd); 1236 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd); 1237 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd); 1238 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd); 1239 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd); 1240 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd); 1241 1242 static inline struct atmel_aes_gcm_ctx * 1243 atmel_aes_gcm_ctx_cast(struct atmel_aes_base_ctx *ctx) 1244 { 1245 return container_of(ctx, struct atmel_aes_gcm_ctx, base); 1246 } 1247 1248 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, 1249 const u32 *data, size_t datalen, 1250 const __be32 *ghash_in, __be32 *ghash_out, 1251 atmel_aes_fn_t resume) 1252 { 1253 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1254 1255 dd->data = (u32 *)data; 1256 dd->datalen = datalen; 1257 ctx->ghash_in = ghash_in; 1258 ctx->ghash_out = ghash_out; 1259 ctx->ghash_resume = resume; 1260 1261 atmel_aes_write_ctrl(dd, false, NULL); 1262 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init); 1263 } 1264 1265 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd) 1266 { 1267 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1268 1269 /* Set the data length. */ 1270 atmel_aes_write(dd, AES_AADLENR, dd->total); 1271 atmel_aes_write(dd, AES_CLENR, 0); 1272 1273 /* If needed, overwrite the GCM Intermediate Hash Word Registers */ 1274 if (ctx->ghash_in) 1275 atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in); 1276 1277 return atmel_aes_gcm_ghash_finalize(dd); 1278 } 1279 1280 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd) 1281 { 1282 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1283 u32 isr; 1284 1285 /* Write data into the Input Data Registers. */ 1286 while (dd->datalen > 0) { 1287 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 1288 dd->data += 4; 1289 dd->datalen -= AES_BLOCK_SIZE; 1290 1291 isr = atmel_aes_read(dd, AES_ISR); 1292 if (!(isr & AES_INT_DATARDY)) { 1293 dd->resume = atmel_aes_gcm_ghash_finalize; 1294 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 1295 return -EINPROGRESS; 1296 } 1297 } 1298 1299 /* Read the computed hash from GHASHRx. */ 1300 atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out); 1301 1302 return ctx->ghash_resume(dd); 1303 } 1304 1305 1306 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd) 1307 { 1308 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1309 struct aead_request *req = aead_request_cast(dd->areq); 1310 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1311 struct atmel_aes_reqctx *rctx = aead_request_ctx(req); 1312 size_t ivsize = crypto_aead_ivsize(tfm); 1313 size_t datalen, padlen; 1314 const void *iv = req->iv; 1315 u8 *data = dd->buf; 1316 int err; 1317 1318 atmel_aes_set_mode(dd, rctx); 1319 1320 err = atmel_aes_hw_init(dd); 1321 if (err) 1322 return atmel_aes_complete(dd, err); 1323 1324 if (likely(ivsize == GCM_AES_IV_SIZE)) { 1325 memcpy(ctx->j0, iv, ivsize); 1326 ctx->j0[3] = cpu_to_be32(1); 1327 return atmel_aes_gcm_process(dd); 1328 } 1329 1330 padlen = atmel_aes_padlen(ivsize, AES_BLOCK_SIZE); 1331 datalen = ivsize + padlen + AES_BLOCK_SIZE; 1332 if (datalen > dd->buflen) 1333 return atmel_aes_complete(dd, -EINVAL); 1334 1335 memcpy(data, iv, ivsize); 1336 memset(data + ivsize, 0, padlen + sizeof(u64)); 1337 ((__be64 *)(data + datalen))[-1] = cpu_to_be64(ivsize * 8); 1338 1339 return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen, 1340 NULL, ctx->j0, atmel_aes_gcm_process); 1341 } 1342 1343 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd) 1344 { 1345 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1346 struct aead_request *req = aead_request_cast(dd->areq); 1347 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1348 bool enc = atmel_aes_is_encrypt(dd); 1349 u32 authsize; 1350 1351 /* Compute text length. */ 1352 authsize = crypto_aead_authsize(tfm); 1353 ctx->textlen = req->cryptlen - (enc ? 0 : authsize); 1354 1355 /* 1356 * According to tcrypt test suite, the GCM Automatic Tag Generation 1357 * fails when both the message and its associated data are empty. 1358 */ 1359 if (likely(req->assoclen != 0 || ctx->textlen != 0)) 1360 dd->flags |= AES_FLAGS_GTAGEN; 1361 1362 atmel_aes_write_ctrl(dd, false, NULL); 1363 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length); 1364 } 1365 1366 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd) 1367 { 1368 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1369 struct aead_request *req = aead_request_cast(dd->areq); 1370 __be32 j0_lsw, *j0 = ctx->j0; 1371 size_t padlen; 1372 1373 /* Write incr32(J0) into IV. */ 1374 j0_lsw = j0[3]; 1375 be32_add_cpu(&j0[3], 1); 1376 atmel_aes_write_block(dd, AES_IVR(0), j0); 1377 j0[3] = j0_lsw; 1378 1379 /* Set aad and text lengths. */ 1380 atmel_aes_write(dd, AES_AADLENR, req->assoclen); 1381 atmel_aes_write(dd, AES_CLENR, ctx->textlen); 1382 1383 /* Check whether AAD are present. */ 1384 if (unlikely(req->assoclen == 0)) { 1385 dd->datalen = 0; 1386 return atmel_aes_gcm_data(dd); 1387 } 1388 1389 /* Copy assoc data and add padding. */ 1390 padlen = atmel_aes_padlen(req->assoclen, AES_BLOCK_SIZE); 1391 if (unlikely(req->assoclen + padlen > dd->buflen)) 1392 return atmel_aes_complete(dd, -EINVAL); 1393 sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen); 1394 1395 /* Write assoc data into the Input Data register. */ 1396 dd->data = (u32 *)dd->buf; 1397 dd->datalen = req->assoclen + padlen; 1398 return atmel_aes_gcm_data(dd); 1399 } 1400 1401 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd) 1402 { 1403 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1404 struct aead_request *req = aead_request_cast(dd->areq); 1405 bool use_dma = (ctx->textlen >= ATMEL_AES_DMA_THRESHOLD); 1406 struct scatterlist *src, *dst; 1407 u32 isr, mr; 1408 1409 /* Write AAD first. */ 1410 while (dd->datalen > 0) { 1411 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 1412 dd->data += 4; 1413 dd->datalen -= AES_BLOCK_SIZE; 1414 1415 isr = atmel_aes_read(dd, AES_ISR); 1416 if (!(isr & AES_INT_DATARDY)) { 1417 dd->resume = atmel_aes_gcm_data; 1418 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 1419 return -EINPROGRESS; 1420 } 1421 } 1422 1423 /* GMAC only. */ 1424 if (unlikely(ctx->textlen == 0)) 1425 return atmel_aes_gcm_tag_init(dd); 1426 1427 /* Prepare src and dst scatter lists to transfer cipher/plain texts */ 1428 src = scatterwalk_ffwd(ctx->src, req->src, req->assoclen); 1429 dst = ((req->src == req->dst) ? src : 1430 scatterwalk_ffwd(ctx->dst, req->dst, req->assoclen)); 1431 1432 if (use_dma) { 1433 /* Update the Mode Register for DMA transfers. */ 1434 mr = atmel_aes_read(dd, AES_MR); 1435 mr &= ~(AES_MR_SMOD_MASK | AES_MR_DUALBUFF); 1436 mr |= AES_MR_SMOD_IDATAR0; 1437 if (dd->caps.has_dualbuff) 1438 mr |= AES_MR_DUALBUFF; 1439 atmel_aes_write(dd, AES_MR, mr); 1440 1441 return atmel_aes_dma_start(dd, src, dst, ctx->textlen, 1442 atmel_aes_gcm_tag_init); 1443 } 1444 1445 return atmel_aes_cpu_start(dd, src, dst, ctx->textlen, 1446 atmel_aes_gcm_tag_init); 1447 } 1448 1449 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd) 1450 { 1451 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1452 struct aead_request *req = aead_request_cast(dd->areq); 1453 __be64 *data = dd->buf; 1454 1455 if (likely(dd->flags & AES_FLAGS_GTAGEN)) { 1456 if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) { 1457 dd->resume = atmel_aes_gcm_tag_init; 1458 atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY); 1459 return -EINPROGRESS; 1460 } 1461 1462 return atmel_aes_gcm_finalize(dd); 1463 } 1464 1465 /* Read the GCM Intermediate Hash Word Registers. */ 1466 atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash); 1467 1468 data[0] = cpu_to_be64(req->assoclen * 8); 1469 data[1] = cpu_to_be64(ctx->textlen * 8); 1470 1471 return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE, 1472 ctx->ghash, ctx->ghash, atmel_aes_gcm_tag); 1473 } 1474 1475 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd) 1476 { 1477 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1478 unsigned long flags; 1479 1480 /* 1481 * Change mode to CTR to complete the tag generation. 1482 * Use J0 as Initialization Vector. 1483 */ 1484 flags = dd->flags; 1485 dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN); 1486 dd->flags |= AES_FLAGS_CTR; 1487 atmel_aes_write_ctrl(dd, false, ctx->j0); 1488 dd->flags = flags; 1489 1490 atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash); 1491 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize); 1492 } 1493 1494 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd) 1495 { 1496 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1497 struct aead_request *req = aead_request_cast(dd->areq); 1498 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1499 bool enc = atmel_aes_is_encrypt(dd); 1500 u32 offset, authsize, itag[4], *otag = ctx->tag; 1501 int err; 1502 1503 /* Read the computed tag. */ 1504 if (likely(dd->flags & AES_FLAGS_GTAGEN)) 1505 atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag); 1506 else 1507 atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag); 1508 1509 offset = req->assoclen + ctx->textlen; 1510 authsize = crypto_aead_authsize(tfm); 1511 if (enc) { 1512 scatterwalk_map_and_copy(otag, req->dst, offset, authsize, 1); 1513 err = 0; 1514 } else { 1515 scatterwalk_map_and_copy(itag, req->src, offset, authsize, 0); 1516 err = crypto_memneq(itag, otag, authsize) ? -EBADMSG : 0; 1517 } 1518 1519 return atmel_aes_complete(dd, err); 1520 } 1521 1522 static int atmel_aes_gcm_crypt(struct aead_request *req, 1523 unsigned long mode) 1524 { 1525 struct atmel_aes_base_ctx *ctx; 1526 struct atmel_aes_reqctx *rctx; 1527 1528 ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1529 ctx->block_size = AES_BLOCK_SIZE; 1530 ctx->is_aead = true; 1531 1532 rctx = aead_request_ctx(req); 1533 rctx->mode = AES_FLAGS_GCM | mode; 1534 1535 return atmel_aes_handle_queue(ctx->dd, &req->base); 1536 } 1537 1538 static int atmel_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 1539 unsigned int keylen) 1540 { 1541 struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm); 1542 1543 if (keylen != AES_KEYSIZE_256 && 1544 keylen != AES_KEYSIZE_192 && 1545 keylen != AES_KEYSIZE_128) 1546 return -EINVAL; 1547 1548 memcpy(ctx->key, key, keylen); 1549 ctx->keylen = keylen; 1550 1551 return 0; 1552 } 1553 1554 static int atmel_aes_gcm_setauthsize(struct crypto_aead *tfm, 1555 unsigned int authsize) 1556 { 1557 return crypto_gcm_check_authsize(authsize); 1558 } 1559 1560 static int atmel_aes_gcm_encrypt(struct aead_request *req) 1561 { 1562 return atmel_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT); 1563 } 1564 1565 static int atmel_aes_gcm_decrypt(struct aead_request *req) 1566 { 1567 return atmel_aes_gcm_crypt(req, 0); 1568 } 1569 1570 static int atmel_aes_gcm_init(struct crypto_aead *tfm) 1571 { 1572 struct atmel_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 1573 struct atmel_aes_dev *dd; 1574 1575 dd = atmel_aes_dev_alloc(&ctx->base); 1576 if (!dd) 1577 return -ENODEV; 1578 1579 crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); 1580 ctx->base.dd = dd; 1581 ctx->base.start = atmel_aes_gcm_start; 1582 1583 return 0; 1584 } 1585 1586 static struct aead_alg aes_gcm_alg = { 1587 .setkey = atmel_aes_gcm_setkey, 1588 .setauthsize = atmel_aes_gcm_setauthsize, 1589 .encrypt = atmel_aes_gcm_encrypt, 1590 .decrypt = atmel_aes_gcm_decrypt, 1591 .init = atmel_aes_gcm_init, 1592 .ivsize = GCM_AES_IV_SIZE, 1593 .maxauthsize = AES_BLOCK_SIZE, 1594 1595 .base = { 1596 .cra_name = "gcm(aes)", 1597 .cra_driver_name = "atmel-gcm-aes", 1598 .cra_blocksize = 1, 1599 .cra_ctxsize = sizeof(struct atmel_aes_gcm_ctx), 1600 }, 1601 }; 1602 1603 1604 /* xts functions */ 1605 1606 static inline struct atmel_aes_xts_ctx * 1607 atmel_aes_xts_ctx_cast(struct atmel_aes_base_ctx *ctx) 1608 { 1609 return container_of(ctx, struct atmel_aes_xts_ctx, base); 1610 } 1611 1612 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd); 1613 1614 static int atmel_aes_xts_start(struct atmel_aes_dev *dd) 1615 { 1616 struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx); 1617 struct skcipher_request *req = skcipher_request_cast(dd->areq); 1618 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 1619 unsigned long flags; 1620 int err; 1621 1622 atmel_aes_set_mode(dd, rctx); 1623 1624 err = atmel_aes_hw_init(dd); 1625 if (err) 1626 return atmel_aes_complete(dd, err); 1627 1628 /* Compute the tweak value from req->iv with ecb(aes). */ 1629 flags = dd->flags; 1630 dd->flags &= ~AES_FLAGS_MODE_MASK; 1631 dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT); 1632 atmel_aes_write_ctrl_key(dd, false, NULL, 1633 ctx->key2, ctx->base.keylen); 1634 dd->flags = flags; 1635 1636 atmel_aes_write_block(dd, AES_IDATAR(0), req->iv); 1637 return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data); 1638 } 1639 1640 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd) 1641 { 1642 struct skcipher_request *req = skcipher_request_cast(dd->areq); 1643 bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD); 1644 u32 tweak[AES_BLOCK_SIZE / sizeof(u32)]; 1645 static const __le32 one[AES_BLOCK_SIZE / sizeof(u32)] = {cpu_to_le32(1), }; 1646 u8 *tweak_bytes = (u8 *)tweak; 1647 int i; 1648 1649 /* Read the computed ciphered tweak value. */ 1650 atmel_aes_read_block(dd, AES_ODATAR(0), tweak); 1651 /* 1652 * Hardware quirk: 1653 * the order of the ciphered tweak bytes need to be reversed before 1654 * writing them into the ODATARx registers. 1655 */ 1656 for (i = 0; i < AES_BLOCK_SIZE/2; ++i) 1657 swap(tweak_bytes[i], tweak_bytes[AES_BLOCK_SIZE - 1 - i]); 1658 1659 /* Process the data. */ 1660 atmel_aes_write_ctrl(dd, use_dma, NULL); 1661 atmel_aes_write_block(dd, AES_TWR(0), tweak); 1662 atmel_aes_write_block(dd, AES_ALPHAR(0), one); 1663 if (use_dma) 1664 return atmel_aes_dma_start(dd, req->src, req->dst, 1665 req->cryptlen, 1666 atmel_aes_transfer_complete); 1667 1668 return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen, 1669 atmel_aes_transfer_complete); 1670 } 1671 1672 static int atmel_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, 1673 unsigned int keylen) 1674 { 1675 struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 1676 int err; 1677 1678 err = xts_verify_key(tfm, key, keylen); 1679 if (err) 1680 return err; 1681 1682 crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK); 1683 crypto_skcipher_set_flags(ctx->fallback_tfm, tfm->base.crt_flags & 1684 CRYPTO_TFM_REQ_MASK); 1685 err = crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); 1686 if (err) 1687 return err; 1688 1689 memcpy(ctx->base.key, key, keylen/2); 1690 memcpy(ctx->key2, key + keylen/2, keylen/2); 1691 ctx->base.keylen = keylen/2; 1692 1693 return 0; 1694 } 1695 1696 static int atmel_aes_xts_encrypt(struct skcipher_request *req) 1697 { 1698 return atmel_aes_crypt(req, AES_FLAGS_XTS | AES_FLAGS_ENCRYPT); 1699 } 1700 1701 static int atmel_aes_xts_decrypt(struct skcipher_request *req) 1702 { 1703 return atmel_aes_crypt(req, AES_FLAGS_XTS); 1704 } 1705 1706 static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm) 1707 { 1708 struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 1709 struct atmel_aes_dev *dd; 1710 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 1711 1712 dd = atmel_aes_dev_alloc(&ctx->base); 1713 if (!dd) 1714 return -ENODEV; 1715 1716 ctx->fallback_tfm = crypto_alloc_skcipher(tfm_name, 0, 1717 CRYPTO_ALG_NEED_FALLBACK); 1718 if (IS_ERR(ctx->fallback_tfm)) 1719 return PTR_ERR(ctx->fallback_tfm); 1720 1721 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx) + 1722 crypto_skcipher_reqsize(ctx->fallback_tfm)); 1723 ctx->base.dd = dd; 1724 ctx->base.start = atmel_aes_xts_start; 1725 1726 return 0; 1727 } 1728 1729 static void atmel_aes_xts_exit_tfm(struct crypto_skcipher *tfm) 1730 { 1731 struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 1732 1733 crypto_free_skcipher(ctx->fallback_tfm); 1734 } 1735 1736 static struct skcipher_alg aes_xts_alg = { 1737 .base.cra_name = "xts(aes)", 1738 .base.cra_driver_name = "atmel-xts-aes", 1739 .base.cra_blocksize = AES_BLOCK_SIZE, 1740 .base.cra_ctxsize = sizeof(struct atmel_aes_xts_ctx), 1741 .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK | 1742 CRYPTO_ALG_KERN_DRIVER_ONLY, 1743 1744 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1745 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1746 .ivsize = AES_BLOCK_SIZE, 1747 .setkey = atmel_aes_xts_setkey, 1748 .encrypt = atmel_aes_xts_encrypt, 1749 .decrypt = atmel_aes_xts_decrypt, 1750 .init = atmel_aes_xts_init_tfm, 1751 .exit = atmel_aes_xts_exit_tfm, 1752 }; 1753 1754 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 1755 /* authenc aead functions */ 1756 1757 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd); 1758 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, 1759 bool is_async); 1760 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, 1761 bool is_async); 1762 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd); 1763 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, 1764 bool is_async); 1765 1766 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err) 1767 { 1768 struct aead_request *req = aead_request_cast(dd->areq); 1769 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1770 1771 if (err && (dd->flags & AES_FLAGS_OWN_SHA)) 1772 atmel_sha_authenc_abort(&rctx->auth_req); 1773 dd->flags &= ~AES_FLAGS_OWN_SHA; 1774 } 1775 1776 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd) 1777 { 1778 struct aead_request *req = aead_request_cast(dd->areq); 1779 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1780 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1781 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 1782 int err; 1783 1784 atmel_aes_set_mode(dd, &rctx->base); 1785 1786 err = atmel_aes_hw_init(dd); 1787 if (err) 1788 return atmel_aes_complete(dd, err); 1789 1790 return atmel_sha_authenc_schedule(&rctx->auth_req, ctx->auth, 1791 atmel_aes_authenc_init, dd); 1792 } 1793 1794 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, 1795 bool is_async) 1796 { 1797 struct aead_request *req = aead_request_cast(dd->areq); 1798 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1799 1800 if (is_async) 1801 dd->is_async = true; 1802 if (err) 1803 return atmel_aes_complete(dd, err); 1804 1805 /* If here, we've got the ownership of the SHA device. */ 1806 dd->flags |= AES_FLAGS_OWN_SHA; 1807 1808 /* Configure the SHA device. */ 1809 return atmel_sha_authenc_init(&rctx->auth_req, 1810 req->src, req->assoclen, 1811 rctx->textlen, 1812 atmel_aes_authenc_transfer, dd); 1813 } 1814 1815 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, 1816 bool is_async) 1817 { 1818 struct aead_request *req = aead_request_cast(dd->areq); 1819 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1820 bool enc = atmel_aes_is_encrypt(dd); 1821 struct scatterlist *src, *dst; 1822 __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; 1823 u32 emr; 1824 1825 if (is_async) 1826 dd->is_async = true; 1827 if (err) 1828 return atmel_aes_complete(dd, err); 1829 1830 /* Prepare src and dst scatter-lists to transfer cipher/plain texts. */ 1831 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); 1832 dst = src; 1833 1834 if (req->src != req->dst) 1835 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); 1836 1837 /* Configure the AES device. */ 1838 memcpy(iv, req->iv, sizeof(iv)); 1839 1840 /* 1841 * Here we always set the 2nd parameter of atmel_aes_write_ctrl() to 1842 * 'true' even if the data transfer is actually performed by the CPU (so 1843 * not by the DMA) because we must force the AES_MR_SMOD bitfield to the 1844 * value AES_MR_SMOD_IDATAR0. Indeed, both AES_MR_SMOD and SHA_MR_SMOD 1845 * must be set to *_MR_SMOD_IDATAR0. 1846 */ 1847 atmel_aes_write_ctrl(dd, true, iv); 1848 emr = AES_EMR_PLIPEN; 1849 if (!enc) 1850 emr |= AES_EMR_PLIPD; 1851 atmel_aes_write(dd, AES_EMR, emr); 1852 1853 /* Transfer data. */ 1854 return atmel_aes_dma_start(dd, src, dst, rctx->textlen, 1855 atmel_aes_authenc_digest); 1856 } 1857 1858 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd) 1859 { 1860 struct aead_request *req = aead_request_cast(dd->areq); 1861 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1862 1863 /* atmel_sha_authenc_final() releases the SHA device. */ 1864 dd->flags &= ~AES_FLAGS_OWN_SHA; 1865 return atmel_sha_authenc_final(&rctx->auth_req, 1866 rctx->digest, sizeof(rctx->digest), 1867 atmel_aes_authenc_final, dd); 1868 } 1869 1870 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, 1871 bool is_async) 1872 { 1873 struct aead_request *req = aead_request_cast(dd->areq); 1874 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1875 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1876 bool enc = atmel_aes_is_encrypt(dd); 1877 u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)], *odigest = rctx->digest; 1878 u32 offs, authsize; 1879 1880 if (is_async) 1881 dd->is_async = true; 1882 if (err) 1883 goto complete; 1884 1885 offs = req->assoclen + rctx->textlen; 1886 authsize = crypto_aead_authsize(tfm); 1887 if (enc) { 1888 scatterwalk_map_and_copy(odigest, req->dst, offs, authsize, 1); 1889 } else { 1890 scatterwalk_map_and_copy(idigest, req->src, offs, authsize, 0); 1891 if (crypto_memneq(idigest, odigest, authsize)) 1892 err = -EBADMSG; 1893 } 1894 1895 complete: 1896 return atmel_aes_complete(dd, err); 1897 } 1898 1899 static int atmel_aes_authenc_setkey(struct crypto_aead *tfm, const u8 *key, 1900 unsigned int keylen) 1901 { 1902 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 1903 struct crypto_authenc_keys keys; 1904 int err; 1905 1906 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1907 goto badkey; 1908 1909 if (keys.enckeylen > sizeof(ctx->base.key)) 1910 goto badkey; 1911 1912 /* Save auth key. */ 1913 err = atmel_sha_authenc_setkey(ctx->auth, 1914 keys.authkey, keys.authkeylen, 1915 crypto_aead_get_flags(tfm)); 1916 if (err) { 1917 memzero_explicit(&keys, sizeof(keys)); 1918 return err; 1919 } 1920 1921 /* Save enc key. */ 1922 ctx->base.keylen = keys.enckeylen; 1923 memcpy(ctx->base.key, keys.enckey, keys.enckeylen); 1924 1925 memzero_explicit(&keys, sizeof(keys)); 1926 return 0; 1927 1928 badkey: 1929 memzero_explicit(&keys, sizeof(keys)); 1930 return -EINVAL; 1931 } 1932 1933 static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm, 1934 unsigned long auth_mode) 1935 { 1936 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 1937 unsigned int auth_reqsize = atmel_sha_authenc_get_reqsize(); 1938 struct atmel_aes_dev *dd; 1939 1940 dd = atmel_aes_dev_alloc(&ctx->base); 1941 if (!dd) 1942 return -ENODEV; 1943 1944 ctx->auth = atmel_sha_authenc_spawn(auth_mode); 1945 if (IS_ERR(ctx->auth)) 1946 return PTR_ERR(ctx->auth); 1947 1948 crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) + 1949 auth_reqsize)); 1950 ctx->base.dd = dd; 1951 ctx->base.start = atmel_aes_authenc_start; 1952 1953 return 0; 1954 } 1955 1956 static int atmel_aes_authenc_hmac_sha1_init_tfm(struct crypto_aead *tfm) 1957 { 1958 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA1); 1959 } 1960 1961 static int atmel_aes_authenc_hmac_sha224_init_tfm(struct crypto_aead *tfm) 1962 { 1963 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA224); 1964 } 1965 1966 static int atmel_aes_authenc_hmac_sha256_init_tfm(struct crypto_aead *tfm) 1967 { 1968 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA256); 1969 } 1970 1971 static int atmel_aes_authenc_hmac_sha384_init_tfm(struct crypto_aead *tfm) 1972 { 1973 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA384); 1974 } 1975 1976 static int atmel_aes_authenc_hmac_sha512_init_tfm(struct crypto_aead *tfm) 1977 { 1978 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA512); 1979 } 1980 1981 static void atmel_aes_authenc_exit_tfm(struct crypto_aead *tfm) 1982 { 1983 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 1984 1985 atmel_sha_authenc_free(ctx->auth); 1986 } 1987 1988 static int atmel_aes_authenc_crypt(struct aead_request *req, 1989 unsigned long mode) 1990 { 1991 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1992 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1993 struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm); 1994 u32 authsize = crypto_aead_authsize(tfm); 1995 bool enc = (mode & AES_FLAGS_ENCRYPT); 1996 1997 /* Compute text length. */ 1998 if (!enc && req->cryptlen < authsize) 1999 return -EINVAL; 2000 rctx->textlen = req->cryptlen - (enc ? 0 : authsize); 2001 2002 /* 2003 * Currently, empty messages are not supported yet: 2004 * the SHA auto-padding can be used only on non-empty messages. 2005 * Hence a special case needs to be implemented for empty message. 2006 */ 2007 if (!rctx->textlen && !req->assoclen) 2008 return -EINVAL; 2009 2010 rctx->base.mode = mode; 2011 ctx->block_size = AES_BLOCK_SIZE; 2012 ctx->is_aead = true; 2013 2014 return atmel_aes_handle_queue(ctx->dd, &req->base); 2015 } 2016 2017 static int atmel_aes_authenc_cbc_aes_encrypt(struct aead_request *req) 2018 { 2019 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT); 2020 } 2021 2022 static int atmel_aes_authenc_cbc_aes_decrypt(struct aead_request *req) 2023 { 2024 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC); 2025 } 2026 2027 static struct aead_alg aes_authenc_algs[] = { 2028 { 2029 .setkey = atmel_aes_authenc_setkey, 2030 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2031 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2032 .init = atmel_aes_authenc_hmac_sha1_init_tfm, 2033 .exit = atmel_aes_authenc_exit_tfm, 2034 .ivsize = AES_BLOCK_SIZE, 2035 .maxauthsize = SHA1_DIGEST_SIZE, 2036 2037 .base = { 2038 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2039 .cra_driver_name = "atmel-authenc-hmac-sha1-cbc-aes", 2040 .cra_blocksize = AES_BLOCK_SIZE, 2041 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2042 }, 2043 }, 2044 { 2045 .setkey = atmel_aes_authenc_setkey, 2046 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2047 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2048 .init = atmel_aes_authenc_hmac_sha224_init_tfm, 2049 .exit = atmel_aes_authenc_exit_tfm, 2050 .ivsize = AES_BLOCK_SIZE, 2051 .maxauthsize = SHA224_DIGEST_SIZE, 2052 2053 .base = { 2054 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2055 .cra_driver_name = "atmel-authenc-hmac-sha224-cbc-aes", 2056 .cra_blocksize = AES_BLOCK_SIZE, 2057 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2058 }, 2059 }, 2060 { 2061 .setkey = atmel_aes_authenc_setkey, 2062 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2063 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2064 .init = atmel_aes_authenc_hmac_sha256_init_tfm, 2065 .exit = atmel_aes_authenc_exit_tfm, 2066 .ivsize = AES_BLOCK_SIZE, 2067 .maxauthsize = SHA256_DIGEST_SIZE, 2068 2069 .base = { 2070 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2071 .cra_driver_name = "atmel-authenc-hmac-sha256-cbc-aes", 2072 .cra_blocksize = AES_BLOCK_SIZE, 2073 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2074 }, 2075 }, 2076 { 2077 .setkey = atmel_aes_authenc_setkey, 2078 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2079 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2080 .init = atmel_aes_authenc_hmac_sha384_init_tfm, 2081 .exit = atmel_aes_authenc_exit_tfm, 2082 .ivsize = AES_BLOCK_SIZE, 2083 .maxauthsize = SHA384_DIGEST_SIZE, 2084 2085 .base = { 2086 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2087 .cra_driver_name = "atmel-authenc-hmac-sha384-cbc-aes", 2088 .cra_blocksize = AES_BLOCK_SIZE, 2089 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2090 }, 2091 }, 2092 { 2093 .setkey = atmel_aes_authenc_setkey, 2094 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2095 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2096 .init = atmel_aes_authenc_hmac_sha512_init_tfm, 2097 .exit = atmel_aes_authenc_exit_tfm, 2098 .ivsize = AES_BLOCK_SIZE, 2099 .maxauthsize = SHA512_DIGEST_SIZE, 2100 2101 .base = { 2102 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2103 .cra_driver_name = "atmel-authenc-hmac-sha512-cbc-aes", 2104 .cra_blocksize = AES_BLOCK_SIZE, 2105 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2106 }, 2107 }, 2108 }; 2109 #endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */ 2110 2111 /* Probe functions */ 2112 2113 static int atmel_aes_buff_init(struct atmel_aes_dev *dd) 2114 { 2115 dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER); 2116 dd->buflen = ATMEL_AES_BUFFER_SIZE; 2117 dd->buflen &= ~(AES_BLOCK_SIZE - 1); 2118 2119 if (!dd->buf) { 2120 dev_err(dd->dev, "unable to alloc pages.\n"); 2121 return -ENOMEM; 2122 } 2123 2124 return 0; 2125 } 2126 2127 static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd) 2128 { 2129 free_pages((unsigned long)dd->buf, ATMEL_AES_BUFFER_ORDER); 2130 } 2131 2132 static int atmel_aes_dma_init(struct atmel_aes_dev *dd) 2133 { 2134 int ret; 2135 2136 /* Try to grab 2 DMA channels */ 2137 dd->src.chan = dma_request_chan(dd->dev, "tx"); 2138 if (IS_ERR(dd->src.chan)) { 2139 ret = PTR_ERR(dd->src.chan); 2140 goto err_dma_in; 2141 } 2142 2143 dd->dst.chan = dma_request_chan(dd->dev, "rx"); 2144 if (IS_ERR(dd->dst.chan)) { 2145 ret = PTR_ERR(dd->dst.chan); 2146 goto err_dma_out; 2147 } 2148 2149 return 0; 2150 2151 err_dma_out: 2152 dma_release_channel(dd->src.chan); 2153 err_dma_in: 2154 dev_err(dd->dev, "no DMA channel available\n"); 2155 return ret; 2156 } 2157 2158 static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd) 2159 { 2160 dma_release_channel(dd->dst.chan); 2161 dma_release_channel(dd->src.chan); 2162 } 2163 2164 static void atmel_aes_queue_task(unsigned long data) 2165 { 2166 struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; 2167 2168 atmel_aes_handle_queue(dd, NULL); 2169 } 2170 2171 static void atmel_aes_done_task(unsigned long data) 2172 { 2173 struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; 2174 2175 dd->is_async = true; 2176 (void)dd->resume(dd); 2177 } 2178 2179 static irqreturn_t atmel_aes_irq(int irq, void *dev_id) 2180 { 2181 struct atmel_aes_dev *aes_dd = dev_id; 2182 u32 reg; 2183 2184 reg = atmel_aes_read(aes_dd, AES_ISR); 2185 if (reg & atmel_aes_read(aes_dd, AES_IMR)) { 2186 atmel_aes_write(aes_dd, AES_IDR, reg); 2187 if (AES_FLAGS_BUSY & aes_dd->flags) 2188 tasklet_schedule(&aes_dd->done_task); 2189 else 2190 dev_warn(aes_dd->dev, "AES interrupt when no active requests.\n"); 2191 return IRQ_HANDLED; 2192 } 2193 2194 return IRQ_NONE; 2195 } 2196 2197 static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) 2198 { 2199 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2200 if (dd->caps.has_authenc) 2201 crypto_unregister_aeads(aes_authenc_algs, 2202 ARRAY_SIZE(aes_authenc_algs)); 2203 #endif 2204 2205 if (dd->caps.has_xts) 2206 crypto_unregister_skcipher(&aes_xts_alg); 2207 2208 if (dd->caps.has_gcm) 2209 crypto_unregister_aead(&aes_gcm_alg); 2210 2211 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 2212 } 2213 2214 static void atmel_aes_crypto_alg_init(struct crypto_alg *alg) 2215 { 2216 alg->cra_flags |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2217 alg->cra_alignmask = 0xf; 2218 alg->cra_priority = ATMEL_AES_PRIORITY; 2219 alg->cra_module = THIS_MODULE; 2220 } 2221 2222 static int atmel_aes_register_algs(struct atmel_aes_dev *dd) 2223 { 2224 int err, i; 2225 2226 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 2227 atmel_aes_crypto_alg_init(&aes_algs[i].base); 2228 2229 err = crypto_register_skcipher(&aes_algs[i]); 2230 if (err) 2231 goto err_aes_algs; 2232 } 2233 2234 if (dd->caps.has_gcm) { 2235 atmel_aes_crypto_alg_init(&aes_gcm_alg.base); 2236 2237 err = crypto_register_aead(&aes_gcm_alg); 2238 if (err) 2239 goto err_aes_gcm_alg; 2240 } 2241 2242 if (dd->caps.has_xts) { 2243 atmel_aes_crypto_alg_init(&aes_xts_alg.base); 2244 2245 err = crypto_register_skcipher(&aes_xts_alg); 2246 if (err) 2247 goto err_aes_xts_alg; 2248 } 2249 2250 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2251 if (dd->caps.has_authenc) { 2252 for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) { 2253 atmel_aes_crypto_alg_init(&aes_authenc_algs[i].base); 2254 2255 err = crypto_register_aead(&aes_authenc_algs[i]); 2256 if (err) 2257 goto err_aes_authenc_alg; 2258 } 2259 } 2260 #endif 2261 2262 return 0; 2263 2264 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2265 /* i = ARRAY_SIZE(aes_authenc_algs); */ 2266 err_aes_authenc_alg: 2267 crypto_unregister_aeads(aes_authenc_algs, i); 2268 if (dd->caps.has_xts) 2269 crypto_unregister_skcipher(&aes_xts_alg); 2270 #endif 2271 err_aes_xts_alg: 2272 if (dd->caps.has_gcm) 2273 crypto_unregister_aead(&aes_gcm_alg); 2274 err_aes_gcm_alg: 2275 i = ARRAY_SIZE(aes_algs); 2276 err_aes_algs: 2277 crypto_unregister_skciphers(aes_algs, i); 2278 2279 return err; 2280 } 2281 2282 static void atmel_aes_get_cap(struct atmel_aes_dev *dd) 2283 { 2284 dd->caps.has_dualbuff = 0; 2285 dd->caps.has_gcm = 0; 2286 dd->caps.has_xts = 0; 2287 dd->caps.has_authenc = 0; 2288 dd->caps.max_burst_size = 1; 2289 2290 /* keep only major version number */ 2291 switch (dd->hw_version & 0xff0) { 2292 case 0x800: 2293 case 0x700: 2294 case 0x600: 2295 case 0x500: 2296 dd->caps.has_dualbuff = 1; 2297 dd->caps.has_gcm = 1; 2298 dd->caps.has_xts = 1; 2299 dd->caps.has_authenc = 1; 2300 dd->caps.max_burst_size = 4; 2301 break; 2302 case 0x200: 2303 dd->caps.has_dualbuff = 1; 2304 dd->caps.has_gcm = 1; 2305 dd->caps.max_burst_size = 4; 2306 break; 2307 case 0x130: 2308 dd->caps.has_dualbuff = 1; 2309 dd->caps.max_burst_size = 4; 2310 break; 2311 case 0x120: 2312 break; 2313 default: 2314 dev_warn(dd->dev, 2315 "Unmanaged aes version, set minimum capabilities\n"); 2316 break; 2317 } 2318 } 2319 2320 static const struct of_device_id atmel_aes_dt_ids[] = { 2321 { .compatible = "atmel,at91sam9g46-aes" }, 2322 { /* sentinel */ } 2323 }; 2324 MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids); 2325 2326 static int atmel_aes_probe(struct platform_device *pdev) 2327 { 2328 struct atmel_aes_dev *aes_dd; 2329 struct device *dev = &pdev->dev; 2330 struct resource *aes_res; 2331 int err; 2332 2333 aes_dd = devm_kzalloc(&pdev->dev, sizeof(*aes_dd), GFP_KERNEL); 2334 if (!aes_dd) 2335 return -ENOMEM; 2336 2337 aes_dd->dev = dev; 2338 2339 platform_set_drvdata(pdev, aes_dd); 2340 2341 INIT_LIST_HEAD(&aes_dd->list); 2342 spin_lock_init(&aes_dd->lock); 2343 2344 tasklet_init(&aes_dd->done_task, atmel_aes_done_task, 2345 (unsigned long)aes_dd); 2346 tasklet_init(&aes_dd->queue_task, atmel_aes_queue_task, 2347 (unsigned long)aes_dd); 2348 2349 crypto_init_queue(&aes_dd->queue, ATMEL_AES_QUEUE_LENGTH); 2350 2351 aes_dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &aes_res); 2352 if (IS_ERR(aes_dd->io_base)) { 2353 err = PTR_ERR(aes_dd->io_base); 2354 goto err_tasklet_kill; 2355 } 2356 aes_dd->phys_base = aes_res->start; 2357 2358 /* Get the IRQ */ 2359 aes_dd->irq = platform_get_irq(pdev, 0); 2360 if (aes_dd->irq < 0) { 2361 err = aes_dd->irq; 2362 goto err_tasklet_kill; 2363 } 2364 2365 err = devm_request_irq(&pdev->dev, aes_dd->irq, atmel_aes_irq, 2366 IRQF_SHARED, "atmel-aes", aes_dd); 2367 if (err) 2368 goto err_tasklet_kill; 2369 2370 /* Initializing the clock */ 2371 aes_dd->iclk = devm_clk_get_prepared(&pdev->dev, "aes_clk"); 2372 if (IS_ERR(aes_dd->iclk)) { 2373 dev_err(dev, "clock initialization failed.\n"); 2374 err = PTR_ERR(aes_dd->iclk); 2375 goto err_tasklet_kill; 2376 } 2377 2378 err = atmel_aes_hw_version_init(aes_dd); 2379 if (err) 2380 goto err_tasklet_kill; 2381 2382 atmel_aes_get_cap(aes_dd); 2383 2384 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2385 if (aes_dd->caps.has_authenc && !atmel_sha_authenc_is_ready()) { 2386 err = -EPROBE_DEFER; 2387 goto err_tasklet_kill; 2388 } 2389 #endif 2390 2391 err = atmel_aes_buff_init(aes_dd); 2392 if (err) 2393 goto err_tasklet_kill; 2394 2395 err = atmel_aes_dma_init(aes_dd); 2396 if (err) 2397 goto err_buff_cleanup; 2398 2399 spin_lock(&atmel_aes.lock); 2400 list_add_tail(&aes_dd->list, &atmel_aes.dev_list); 2401 spin_unlock(&atmel_aes.lock); 2402 2403 err = atmel_aes_register_algs(aes_dd); 2404 if (err) 2405 goto err_algs; 2406 2407 dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n", 2408 dma_chan_name(aes_dd->src.chan), 2409 dma_chan_name(aes_dd->dst.chan)); 2410 2411 return 0; 2412 2413 err_algs: 2414 spin_lock(&atmel_aes.lock); 2415 list_del(&aes_dd->list); 2416 spin_unlock(&atmel_aes.lock); 2417 atmel_aes_dma_cleanup(aes_dd); 2418 err_buff_cleanup: 2419 atmel_aes_buff_cleanup(aes_dd); 2420 err_tasklet_kill: 2421 tasklet_kill(&aes_dd->done_task); 2422 tasklet_kill(&aes_dd->queue_task); 2423 2424 return err; 2425 } 2426 2427 static void atmel_aes_remove(struct platform_device *pdev) 2428 { 2429 struct atmel_aes_dev *aes_dd; 2430 2431 aes_dd = platform_get_drvdata(pdev); 2432 2433 spin_lock(&atmel_aes.lock); 2434 list_del(&aes_dd->list); 2435 spin_unlock(&atmel_aes.lock); 2436 2437 atmel_aes_unregister_algs(aes_dd); 2438 2439 tasklet_kill(&aes_dd->done_task); 2440 tasklet_kill(&aes_dd->queue_task); 2441 2442 atmel_aes_dma_cleanup(aes_dd); 2443 atmel_aes_buff_cleanup(aes_dd); 2444 } 2445 2446 static struct platform_driver atmel_aes_driver = { 2447 .probe = atmel_aes_probe, 2448 .remove = atmel_aes_remove, 2449 .driver = { 2450 .name = "atmel_aes", 2451 .of_match_table = atmel_aes_dt_ids, 2452 }, 2453 }; 2454 2455 module_platform_driver(atmel_aes_driver); 2456 2457 MODULE_DESCRIPTION("Atmel AES hw acceleration support."); 2458 MODULE_LICENSE("GPL v2"); 2459 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 2460