1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api. 4 * 5 * Copyright (C) 2014-2017 Axis Communications AB 6 */ 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/bitfield.h> 10 #include <linux/crypto.h> 11 #include <linux/debugfs.h> 12 #include <linux/delay.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/fault-inject.h> 15 #include <linux/init.h> 16 #include <linux/interrupt.h> 17 #include <linux/kernel.h> 18 #include <linux/list.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/platform_device.h> 22 #include <linux/scatterlist.h> 23 #include <linux/slab.h> 24 25 #include <crypto/aes.h> 26 #include <crypto/gcm.h> 27 #include <crypto/internal/aead.h> 28 #include <crypto/internal/hash.h> 29 #include <crypto/internal/skcipher.h> 30 #include <crypto/scatterwalk.h> 31 #include <crypto/sha1.h> 32 #include <crypto/sha2.h> 33 #include <crypto/xts.h> 34 35 /* Max length of a line in all cache levels for Artpec SoCs. */ 36 #define ARTPEC_CACHE_LINE_MAX 32 37 38 #define PDMA_OUT_CFG 0x0000 39 #define PDMA_OUT_BUF_CFG 0x0004 40 #define PDMA_OUT_CMD 0x0008 41 #define PDMA_OUT_DESCRQ_PUSH 0x0010 42 #define PDMA_OUT_DESCRQ_STAT 0x0014 43 44 #define A6_PDMA_IN_CFG 0x0028 45 #define A6_PDMA_IN_BUF_CFG 0x002c 46 #define A6_PDMA_IN_CMD 0x0030 47 #define A6_PDMA_IN_STATQ_PUSH 0x0038 48 #define A6_PDMA_IN_DESCRQ_PUSH 0x0044 49 #define A6_PDMA_IN_DESCRQ_STAT 0x0048 50 #define A6_PDMA_INTR_MASK 0x0068 51 #define A6_PDMA_ACK_INTR 0x006c 52 #define A6_PDMA_MASKED_INTR 0x0074 53 54 #define A7_PDMA_IN_CFG 0x002c 55 #define A7_PDMA_IN_BUF_CFG 0x0030 56 #define A7_PDMA_IN_CMD 0x0034 57 #define A7_PDMA_IN_STATQ_PUSH 0x003c 58 #define A7_PDMA_IN_DESCRQ_PUSH 0x0048 59 #define A7_PDMA_IN_DESCRQ_STAT 0x004C 60 #define A7_PDMA_INTR_MASK 0x006c 61 #define A7_PDMA_ACK_INTR 0x0070 62 #define A7_PDMA_MASKED_INTR 0x0078 63 64 #define PDMA_OUT_CFG_EN BIT(0) 65 66 #define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) 67 #define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) 68 69 #define PDMA_OUT_CMD_START BIT(0) 70 #define A6_PDMA_OUT_CMD_STOP BIT(3) 71 #define A7_PDMA_OUT_CMD_STOP BIT(2) 72 73 #define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0) 74 #define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6) 75 76 #define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0) 77 #define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4) 78 79 #define PDMA_IN_CFG_EN BIT(0) 80 81 #define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) 82 #define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) 83 #define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10) 84 85 #define PDMA_IN_CMD_START BIT(0) 86 #define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2) 87 #define A6_PDMA_IN_CMD_STOP BIT(3) 88 #define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1) 89 #define A7_PDMA_IN_CMD_STOP BIT(2) 90 91 #define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0) 92 #define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6) 93 94 #define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0) 95 #define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6) 96 97 #define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0) 98 #define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4) 99 100 #define A6_PDMA_INTR_MASK_IN_DATA BIT(2) 101 #define A6_PDMA_INTR_MASK_IN_EOP BIT(3) 102 #define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4) 103 104 #define A7_PDMA_INTR_MASK_IN_DATA BIT(3) 105 #define A7_PDMA_INTR_MASK_IN_EOP BIT(4) 106 #define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5) 107 108 #define A6_CRY_MD_OPER GENMASK(19, 16) 109 110 #define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20) 111 #define A6_CRY_MD_HASH_HMAC_FIN BIT(23) 112 113 #define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20) 114 #define A6_CRY_MD_CIPHER_DECR BIT(22) 115 #define A6_CRY_MD_CIPHER_TWEAK BIT(23) 116 #define A6_CRY_MD_CIPHER_DSEQ BIT(24) 117 118 #define A7_CRY_MD_OPER GENMASK(11, 8) 119 120 #define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12) 121 #define A7_CRY_MD_HASH_HMAC_FIN BIT(15) 122 123 #define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12) 124 #define A7_CRY_MD_CIPHER_DECR BIT(14) 125 #define A7_CRY_MD_CIPHER_TWEAK BIT(15) 126 #define A7_CRY_MD_CIPHER_DSEQ BIT(16) 127 128 /* DMA metadata constants */ 129 #define regk_crypto_aes_cbc 0x00000002 130 #define regk_crypto_aes_ctr 0x00000003 131 #define regk_crypto_aes_ecb 0x00000001 132 #define regk_crypto_aes_gcm 0x00000004 133 #define regk_crypto_aes_xts 0x00000005 134 #define regk_crypto_cache 0x00000002 135 #define a6_regk_crypto_dlkey 0x0000000a 136 #define a7_regk_crypto_dlkey 0x0000000e 137 #define regk_crypto_ext 0x00000001 138 #define regk_crypto_hmac_sha1 0x00000007 139 #define regk_crypto_hmac_sha256 0x00000009 140 #define regk_crypto_init 0x00000000 141 #define regk_crypto_key_128 0x00000000 142 #define regk_crypto_key_192 0x00000001 143 #define regk_crypto_key_256 0x00000002 144 #define regk_crypto_null 0x00000000 145 #define regk_crypto_sha1 0x00000006 146 #define regk_crypto_sha256 0x00000008 147 148 /* DMA descriptor structures */ 149 struct pdma_descr_ctrl { 150 unsigned char short_descr : 1; 151 unsigned char pad1 : 1; 152 unsigned char eop : 1; 153 unsigned char intr : 1; 154 unsigned char short_len : 3; 155 unsigned char pad2 : 1; 156 } __packed; 157 158 struct pdma_data_descr { 159 unsigned int len : 24; 160 unsigned int buf : 32; 161 } __packed; 162 163 struct pdma_short_descr { 164 unsigned char data[7]; 165 } __packed; 166 167 struct pdma_descr { 168 struct pdma_descr_ctrl ctrl; 169 union { 170 struct pdma_data_descr data; 171 struct pdma_short_descr shrt; 172 }; 173 }; 174 175 struct pdma_stat_descr { 176 unsigned char pad1 : 1; 177 unsigned char pad2 : 1; 178 unsigned char eop : 1; 179 unsigned char pad3 : 5; 180 unsigned int len : 24; 181 }; 182 183 /* Each descriptor array can hold max 64 entries */ 184 #define PDMA_DESCR_COUNT 64 185 186 #define MODULE_NAME "Artpec-6 CA" 187 188 /* Hash modes (including HMAC variants) */ 189 #define ARTPEC6_CRYPTO_HASH_SHA1 1 190 #define ARTPEC6_CRYPTO_HASH_SHA256 2 191 192 /* Crypto modes */ 193 #define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1 194 #define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2 195 #define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3 196 #define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5 197 198 /* The PDMA is a DMA-engine tightly coupled with a ciphering engine. 199 * It operates on a descriptor array with up to 64 descriptor entries. 200 * The arrays must be 64 byte aligned in memory. 201 * 202 * The ciphering unit has no registers and is completely controlled by 203 * a 4-byte metadata that is inserted at the beginning of each dma packet. 204 * 205 * A dma packet is a sequence of descriptors terminated by setting the .eop 206 * field in the final descriptor of the packet. 207 * 208 * Multiple packets are used for providing context data, key data and 209 * the plain/ciphertext. 210 * 211 * PDMA Descriptors (Array) 212 * +------+------+------+~~+-------+------+---- 213 * | 0 | 1 | 2 |~~| 11 EOP| 12 | .... 214 * +--+---+--+---+----+-+~~+-------+----+-+---- 215 * | | | | | 216 * | | | | | 217 * __|__ +-------++-------++-------+ +----+ 218 * | MD | |Payload||Payload||Payload| | MD | 219 * +-----+ +-------++-------++-------+ +----+ 220 */ 221 222 struct artpec6_crypto_bounce_buffer { 223 struct list_head list; 224 size_t length; 225 struct scatterlist *sg; 226 size_t offset; 227 /* buf is aligned to ARTPEC_CACHE_LINE_MAX and 228 * holds up to ARTPEC_CACHE_LINE_MAX bytes data. 229 */ 230 void *buf; 231 }; 232 233 struct artpec6_crypto_dma_map { 234 dma_addr_t dma_addr; 235 size_t size; 236 enum dma_data_direction dir; 237 }; 238 239 struct artpec6_crypto_dma_descriptors { 240 struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64); 241 struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64); 242 u32 stat[PDMA_DESCR_COUNT] __aligned(64); 243 struct list_head bounce_buffers; 244 /* Enough maps for all out/in buffers, and all three descr. arrays */ 245 struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2]; 246 dma_addr_t out_dma_addr; 247 dma_addr_t in_dma_addr; 248 dma_addr_t stat_dma_addr; 249 size_t out_cnt; 250 size_t in_cnt; 251 size_t map_count; 252 }; 253 254 enum artpec6_crypto_variant { 255 ARTPEC6_CRYPTO = 1, 256 ARTPEC7_CRYPTO, 257 }; 258 259 struct artpec6_crypto { 260 void __iomem *base; 261 spinlock_t queue_lock; 262 struct list_head queue; /* waiting for pdma fifo space */ 263 struct list_head pending; /* submitted to pdma fifo */ 264 struct tasklet_struct task; 265 struct kmem_cache *dma_cache; 266 int pending_count; 267 struct timer_list timer; 268 enum artpec6_crypto_variant variant; 269 void *pad_buffer; /* cache-aligned block padding buffer */ 270 void *zero_buffer; 271 }; 272 273 enum artpec6_crypto_hash_flags { 274 HASH_FLAG_INIT_CTX = 2, 275 HASH_FLAG_UPDATE = 4, 276 HASH_FLAG_FINALIZE = 8, 277 HASH_FLAG_HMAC = 16, 278 HASH_FLAG_UPDATE_KEY = 32, 279 }; 280 281 struct artpec6_crypto_req_common { 282 struct list_head list; 283 struct list_head complete_in_progress; 284 struct artpec6_crypto_dma_descriptors *dma; 285 struct crypto_async_request *req; 286 void (*complete)(struct crypto_async_request *req); 287 gfp_t gfp_flags; 288 }; 289 290 struct artpec6_hash_request_context { 291 char partial_buffer[SHA256_BLOCK_SIZE]; 292 char partial_buffer_out[SHA256_BLOCK_SIZE]; 293 char key_buffer[SHA256_BLOCK_SIZE]; 294 char pad_buffer[SHA256_BLOCK_SIZE + 32]; 295 unsigned char digeststate[SHA256_DIGEST_SIZE]; 296 size_t partial_bytes; 297 u64 digcnt; 298 u32 key_md; 299 u32 hash_md; 300 enum artpec6_crypto_hash_flags hash_flags; 301 struct artpec6_crypto_req_common common; 302 }; 303 304 struct artpec6_hash_export_state { 305 char partial_buffer[SHA256_BLOCK_SIZE]; 306 unsigned char digeststate[SHA256_DIGEST_SIZE]; 307 size_t partial_bytes; 308 u64 digcnt; 309 int oper; 310 unsigned int hash_flags; 311 }; 312 313 struct artpec6_hashalg_context { 314 char hmac_key[SHA256_BLOCK_SIZE]; 315 size_t hmac_key_length; 316 struct crypto_shash *child_hash; 317 }; 318 319 struct artpec6_crypto_request_context { 320 u32 cipher_md; 321 bool decrypt; 322 struct artpec6_crypto_req_common common; 323 }; 324 325 struct artpec6_cryptotfm_context { 326 unsigned char aes_key[2*AES_MAX_KEY_SIZE]; 327 size_t key_length; 328 u32 key_md; 329 int crypto_type; 330 struct crypto_sync_skcipher *fallback; 331 }; 332 333 struct artpec6_crypto_aead_hw_ctx { 334 __be64 aad_length_bits; 335 __be64 text_length_bits; 336 __u8 J0[AES_BLOCK_SIZE]; 337 }; 338 339 struct artpec6_crypto_aead_req_ctx { 340 struct artpec6_crypto_aead_hw_ctx hw_ctx; 341 u32 cipher_md; 342 bool decrypt; 343 struct artpec6_crypto_req_common common; 344 __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned; 345 }; 346 347 /* The crypto framework makes it hard to avoid this global. */ 348 static struct device *artpec6_crypto_dev; 349 350 #ifdef CONFIG_FAULT_INJECTION 351 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read); 352 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full); 353 #endif 354 355 enum { 356 ARTPEC6_CRYPTO_PREPARE_HASH_NO_START, 357 ARTPEC6_CRYPTO_PREPARE_HASH_START, 358 }; 359 360 static int artpec6_crypto_prepare_aead(struct aead_request *areq); 361 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq); 362 static int artpec6_crypto_prepare_hash(struct ahash_request *areq); 363 364 static void 365 artpec6_crypto_complete_crypto(struct crypto_async_request *req); 366 static void 367 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req); 368 static void 369 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req); 370 static void 371 artpec6_crypto_complete_aead(struct crypto_async_request *req); 372 static void 373 artpec6_crypto_complete_hash(struct crypto_async_request *req); 374 375 static int 376 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common); 377 378 static void 379 artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common); 380 381 struct artpec6_crypto_walk { 382 struct scatterlist *sg; 383 size_t offset; 384 }; 385 386 static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk, 387 struct scatterlist *sg) 388 { 389 awalk->sg = sg; 390 awalk->offset = 0; 391 } 392 393 static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk, 394 size_t nbytes) 395 { 396 while (nbytes && awalk->sg) { 397 size_t piece; 398 399 WARN_ON(awalk->offset > awalk->sg->length); 400 401 piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset); 402 nbytes -= piece; 403 awalk->offset += piece; 404 if (awalk->offset == awalk->sg->length) { 405 awalk->sg = sg_next(awalk->sg); 406 awalk->offset = 0; 407 } 408 409 } 410 411 return nbytes; 412 } 413 414 static size_t 415 artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk) 416 { 417 WARN_ON(awalk->sg->length == awalk->offset); 418 419 return awalk->sg->length - awalk->offset; 420 } 421 422 static dma_addr_t 423 artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk) 424 { 425 return sg_phys(awalk->sg) + awalk->offset; 426 } 427 428 static void 429 artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common) 430 { 431 struct artpec6_crypto_dma_descriptors *dma = common->dma; 432 struct artpec6_crypto_bounce_buffer *b; 433 struct artpec6_crypto_bounce_buffer *next; 434 435 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { 436 pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n", 437 b, b->length, b->offset, b->buf); 438 sg_pcopy_from_buffer(b->sg, 439 1, 440 b->buf, 441 b->length, 442 b->offset); 443 444 list_del(&b->list); 445 kfree(b); 446 } 447 } 448 449 static inline bool artpec6_crypto_busy(void) 450 { 451 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 452 int fifo_count = ac->pending_count; 453 454 return fifo_count > 6; 455 } 456 457 static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req) 458 { 459 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 460 int ret = -EBUSY; 461 462 spin_lock_bh(&ac->queue_lock); 463 464 if (!artpec6_crypto_busy()) { 465 list_add_tail(&req->list, &ac->pending); 466 artpec6_crypto_start_dma(req); 467 ret = -EINPROGRESS; 468 } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { 469 list_add_tail(&req->list, &ac->queue); 470 } else { 471 artpec6_crypto_common_destroy(req); 472 } 473 474 spin_unlock_bh(&ac->queue_lock); 475 476 return ret; 477 } 478 479 static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common) 480 { 481 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 482 enum artpec6_crypto_variant variant = ac->variant; 483 void __iomem *base = ac->base; 484 struct artpec6_crypto_dma_descriptors *dma = common->dma; 485 u32 ind, statd, outd; 486 487 /* Make descriptor content visible to the DMA before starting it. */ 488 wmb(); 489 490 ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) | 491 FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6); 492 493 statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) | 494 FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6); 495 496 outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) | 497 FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6); 498 499 if (variant == ARTPEC6_CRYPTO) { 500 writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH); 501 writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH); 502 writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD); 503 } else { 504 writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH); 505 writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH); 506 writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD); 507 } 508 509 writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH); 510 writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD); 511 512 ac->pending_count++; 513 } 514 515 static void 516 artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common) 517 { 518 struct artpec6_crypto_dma_descriptors *dma = common->dma; 519 520 dma->out_cnt = 0; 521 dma->in_cnt = 0; 522 dma->map_count = 0; 523 INIT_LIST_HEAD(&dma->bounce_buffers); 524 } 525 526 static bool fault_inject_dma_descr(void) 527 { 528 #ifdef CONFIG_FAULT_INJECTION 529 return should_fail(&artpec6_crypto_fail_dma_array_full, 1); 530 #else 531 return false; 532 #endif 533 } 534 535 /** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a 536 * physical address 537 * 538 * @addr: The physical address of the data buffer 539 * @len: The length of the data buffer 540 * @eop: True if this is the last buffer in the packet 541 * 542 * @return 0 on success or -ENOSPC if there are no more descriptors available 543 */ 544 static int 545 artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common, 546 dma_addr_t addr, size_t len, bool eop) 547 { 548 struct artpec6_crypto_dma_descriptors *dma = common->dma; 549 struct pdma_descr *d; 550 551 if (dma->out_cnt >= PDMA_DESCR_COUNT || 552 fault_inject_dma_descr()) { 553 pr_err("No free OUT DMA descriptors available!\n"); 554 return -ENOSPC; 555 } 556 557 d = &dma->out[dma->out_cnt++]; 558 memset(d, 0, sizeof(*d)); 559 560 d->ctrl.short_descr = 0; 561 d->ctrl.eop = eop; 562 d->data.len = len; 563 d->data.buf = addr; 564 return 0; 565 } 566 567 /** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor 568 * 569 * @dst: The virtual address of the data 570 * @len: The length of the data, must be between 1 to 7 bytes 571 * @eop: True if this is the last buffer in the packet 572 * 573 * @return 0 on success 574 * -ENOSPC if no more descriptors are available 575 * -EINVAL if the data length exceeds 7 bytes 576 */ 577 static int 578 artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common, 579 void *dst, unsigned int len, bool eop) 580 { 581 struct artpec6_crypto_dma_descriptors *dma = common->dma; 582 struct pdma_descr *d; 583 584 if (dma->out_cnt >= PDMA_DESCR_COUNT || 585 fault_inject_dma_descr()) { 586 pr_err("No free OUT DMA descriptors available!\n"); 587 return -ENOSPC; 588 } else if (len > 7 || len < 1) { 589 return -EINVAL; 590 } 591 d = &dma->out[dma->out_cnt++]; 592 memset(d, 0, sizeof(*d)); 593 594 d->ctrl.short_descr = 1; 595 d->ctrl.short_len = len; 596 d->ctrl.eop = eop; 597 memcpy(d->shrt.data, dst, len); 598 return 0; 599 } 600 601 static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common, 602 struct page *page, size_t offset, 603 size_t size, 604 enum dma_data_direction dir, 605 dma_addr_t *dma_addr_out) 606 { 607 struct artpec6_crypto_dma_descriptors *dma = common->dma; 608 struct device *dev = artpec6_crypto_dev; 609 struct artpec6_crypto_dma_map *map; 610 dma_addr_t dma_addr; 611 612 *dma_addr_out = 0; 613 614 if (dma->map_count >= ARRAY_SIZE(dma->maps)) 615 return -ENOMEM; 616 617 dma_addr = dma_map_page(dev, page, offset, size, dir); 618 if (dma_mapping_error(dev, dma_addr)) 619 return -ENOMEM; 620 621 map = &dma->maps[dma->map_count++]; 622 map->size = size; 623 map->dma_addr = dma_addr; 624 map->dir = dir; 625 626 *dma_addr_out = dma_addr; 627 628 return 0; 629 } 630 631 static int 632 artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common, 633 void *ptr, size_t size, 634 enum dma_data_direction dir, 635 dma_addr_t *dma_addr_out) 636 { 637 struct page *page = virt_to_page(ptr); 638 size_t offset = (uintptr_t)ptr & ~PAGE_MASK; 639 640 return artpec6_crypto_dma_map_page(common, page, offset, size, dir, 641 dma_addr_out); 642 } 643 644 static int 645 artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common) 646 { 647 struct artpec6_crypto_dma_descriptors *dma = common->dma; 648 int ret; 649 650 ret = artpec6_crypto_dma_map_single(common, dma->in, 651 sizeof(dma->in[0]) * dma->in_cnt, 652 DMA_TO_DEVICE, &dma->in_dma_addr); 653 if (ret) 654 return ret; 655 656 ret = artpec6_crypto_dma_map_single(common, dma->out, 657 sizeof(dma->out[0]) * dma->out_cnt, 658 DMA_TO_DEVICE, &dma->out_dma_addr); 659 if (ret) 660 return ret; 661 662 /* We only read one stat descriptor */ 663 dma->stat[dma->in_cnt - 1] = 0; 664 665 /* 666 * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor 667 * to be written. 668 */ 669 return artpec6_crypto_dma_map_single(common, 670 dma->stat, 671 sizeof(dma->stat[0]) * dma->in_cnt, 672 DMA_BIDIRECTIONAL, 673 &dma->stat_dma_addr); 674 } 675 676 static void 677 artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common) 678 { 679 struct artpec6_crypto_dma_descriptors *dma = common->dma; 680 struct device *dev = artpec6_crypto_dev; 681 int i; 682 683 for (i = 0; i < dma->map_count; i++) { 684 struct artpec6_crypto_dma_map *map = &dma->maps[i]; 685 686 dma_unmap_page(dev, map->dma_addr, map->size, map->dir); 687 } 688 689 dma->map_count = 0; 690 } 691 692 /** artpec6_crypto_setup_out_descr - Setup an out descriptor 693 * 694 * @dst: The virtual address of the data 695 * @len: The length of the data 696 * @eop: True if this is the last buffer in the packet 697 * @use_short: If this is true and the data length is 7 bytes or less then 698 * a short descriptor will be used 699 * 700 * @return 0 on success 701 * Any errors from artpec6_crypto_setup_out_descr_short() or 702 * setup_out_descr_phys() 703 */ 704 static int 705 artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common, 706 void *dst, unsigned int len, bool eop, 707 bool use_short) 708 { 709 dma_addr_t dma_addr; 710 int ret; 711 712 if (use_short && len < 7) 713 return artpec6_crypto_setup_out_descr_short(common, dst, len, 714 eop); 715 716 ret = artpec6_crypto_dma_map_single(common, dst, len, DMA_TO_DEVICE, 717 &dma_addr); 718 if (ret) 719 return ret; 720 721 return artpec6_crypto_setup_out_descr_phys(common, dma_addr, len, eop); 722 } 723 724 /** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a 725 * physical address 726 * 727 * @addr: The physical address of the data buffer 728 * @len: The length of the data buffer 729 * @intr: True if an interrupt should be fired after HW processing of this 730 * descriptor 731 * 732 */ 733 static int 734 artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common, 735 dma_addr_t addr, unsigned int len, bool intr) 736 { 737 struct artpec6_crypto_dma_descriptors *dma = common->dma; 738 struct pdma_descr *d; 739 740 if (dma->in_cnt >= PDMA_DESCR_COUNT || 741 fault_inject_dma_descr()) { 742 pr_err("No free IN DMA descriptors available!\n"); 743 return -ENOSPC; 744 } 745 d = &dma->in[dma->in_cnt++]; 746 memset(d, 0, sizeof(*d)); 747 748 d->ctrl.intr = intr; 749 d->data.len = len; 750 d->data.buf = addr; 751 return 0; 752 } 753 754 /** artpec6_crypto_setup_in_descr - Setup an in channel descriptor 755 * 756 * @buffer: The virtual address to of the data buffer 757 * @len: The length of the data buffer 758 * @last: If this is the last data buffer in the request (i.e. an interrupt 759 * is needed 760 * 761 * Short descriptors are not used for the in channel 762 */ 763 static int 764 artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common, 765 void *buffer, unsigned int len, bool last) 766 { 767 dma_addr_t dma_addr; 768 int ret; 769 770 ret = artpec6_crypto_dma_map_single(common, buffer, len, 771 DMA_FROM_DEVICE, &dma_addr); 772 if (ret) 773 return ret; 774 775 return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last); 776 } 777 778 static struct artpec6_crypto_bounce_buffer * 779 artpec6_crypto_alloc_bounce(gfp_t flags) 780 { 781 void *base; 782 size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) + 783 2 * ARTPEC_CACHE_LINE_MAX; 784 struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags); 785 786 if (!bbuf) 787 return NULL; 788 789 base = bbuf + 1; 790 bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX); 791 return bbuf; 792 } 793 794 static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common, 795 struct artpec6_crypto_walk *walk, size_t size) 796 { 797 struct artpec6_crypto_bounce_buffer *bbuf; 798 int ret; 799 800 bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags); 801 if (!bbuf) 802 return -ENOMEM; 803 804 bbuf->length = size; 805 bbuf->sg = walk->sg; 806 bbuf->offset = walk->offset; 807 808 ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false); 809 if (ret) { 810 kfree(bbuf); 811 return ret; 812 } 813 814 pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset); 815 list_add_tail(&bbuf->list, &common->dma->bounce_buffers); 816 return 0; 817 } 818 819 static int 820 artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common, 821 struct artpec6_crypto_walk *walk, 822 size_t count) 823 { 824 size_t chunk; 825 int ret; 826 dma_addr_t addr; 827 828 while (walk->sg && count) { 829 chunk = min(count, artpec6_crypto_walk_chunklen(walk)); 830 addr = artpec6_crypto_walk_chunk_phys(walk); 831 832 /* When destination buffers are not aligned to the cache line 833 * size we need bounce buffers. The DMA-API requires that the 834 * entire line is owned by the DMA buffer and this holds also 835 * for the case when coherent DMA is used. 836 */ 837 if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) { 838 chunk = min_t(dma_addr_t, chunk, 839 ALIGN(addr, ARTPEC_CACHE_LINE_MAX) - 840 addr); 841 842 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); 843 ret = setup_bounce_buffer_in(common, walk, chunk); 844 } else if (chunk < ARTPEC_CACHE_LINE_MAX) { 845 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); 846 ret = setup_bounce_buffer_in(common, walk, chunk); 847 } else { 848 dma_addr_t dma_addr; 849 850 chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1); 851 852 pr_debug("CHUNK %pad:%zu\n", &addr, chunk); 853 854 ret = artpec6_crypto_dma_map_page(common, 855 sg_page(walk->sg), 856 walk->sg->offset + 857 walk->offset, 858 chunk, 859 DMA_FROM_DEVICE, 860 &dma_addr); 861 if (ret) 862 return ret; 863 864 ret = artpec6_crypto_setup_in_descr_phys(common, 865 dma_addr, 866 chunk, false); 867 } 868 869 if (ret) 870 return ret; 871 872 count = count - chunk; 873 artpec6_crypto_walk_advance(walk, chunk); 874 } 875 876 if (count) 877 pr_err("EOL unexpected %zu bytes left\n", count); 878 879 return count ? -EINVAL : 0; 880 } 881 882 static int 883 artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common, 884 struct artpec6_crypto_walk *walk, 885 size_t count) 886 { 887 size_t chunk; 888 int ret; 889 dma_addr_t addr; 890 891 while (walk->sg && count) { 892 chunk = min(count, artpec6_crypto_walk_chunklen(walk)); 893 addr = artpec6_crypto_walk_chunk_phys(walk); 894 895 pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk); 896 897 if (addr & 3) { 898 char buf[3]; 899 900 chunk = min_t(size_t, chunk, (4-(addr&3))); 901 902 sg_pcopy_to_buffer(walk->sg, 1, buf, chunk, 903 walk->offset); 904 905 ret = artpec6_crypto_setup_out_descr_short(common, buf, 906 chunk, 907 false); 908 } else { 909 dma_addr_t dma_addr; 910 911 ret = artpec6_crypto_dma_map_page(common, 912 sg_page(walk->sg), 913 walk->sg->offset + 914 walk->offset, 915 chunk, 916 DMA_TO_DEVICE, 917 &dma_addr); 918 if (ret) 919 return ret; 920 921 ret = artpec6_crypto_setup_out_descr_phys(common, 922 dma_addr, 923 chunk, false); 924 } 925 926 if (ret) 927 return ret; 928 929 count = count - chunk; 930 artpec6_crypto_walk_advance(walk, chunk); 931 } 932 933 if (count) 934 pr_err("EOL unexpected %zu bytes left\n", count); 935 936 return count ? -EINVAL : 0; 937 } 938 939 940 /** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor 941 * 942 * If the out descriptor list is non-empty, then the eop flag on the 943 * last used out descriptor will be set. 944 * 945 * @return 0 on success 946 * -EINVAL if the out descriptor is empty or has overflown 947 */ 948 static int 949 artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common) 950 { 951 struct artpec6_crypto_dma_descriptors *dma = common->dma; 952 struct pdma_descr *d; 953 954 if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) { 955 pr_err("%s: OUT descriptor list is %s\n", 956 MODULE_NAME, dma->out_cnt ? "empty" : "full"); 957 return -EINVAL; 958 959 } 960 961 d = &dma->out[dma->out_cnt-1]; 962 d->ctrl.eop = 1; 963 964 return 0; 965 } 966 967 /** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last 968 * in descriptor 969 * 970 * See artpec6_crypto_terminate_out_descrs() for return values 971 */ 972 static int 973 artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common) 974 { 975 struct artpec6_crypto_dma_descriptors *dma = common->dma; 976 struct pdma_descr *d; 977 978 if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) { 979 pr_err("%s: IN descriptor list is %s\n", 980 MODULE_NAME, dma->in_cnt ? "empty" : "full"); 981 return -EINVAL; 982 } 983 984 d = &dma->in[dma->in_cnt-1]; 985 d->ctrl.intr = 1; 986 return 0; 987 } 988 989 /** create_hash_pad - Create a Secure Hash conformant pad 990 * 991 * @dst: The destination buffer to write the pad. Must be at least 64 bytes 992 * @dgstlen: The total length of the hash digest in bytes 993 * @bitcount: The total length of the digest in bits 994 * 995 * @return The total number of padding bytes written to @dst 996 */ 997 static size_t 998 create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount) 999 { 1000 unsigned int mod, target, diff, pad_bytes, size_bytes; 1001 __be64 bits = __cpu_to_be64(bitcount); 1002 1003 switch (oper) { 1004 case regk_crypto_sha1: 1005 case regk_crypto_sha256: 1006 case regk_crypto_hmac_sha1: 1007 case regk_crypto_hmac_sha256: 1008 target = 448 / 8; 1009 mod = 512 / 8; 1010 size_bytes = 8; 1011 break; 1012 default: 1013 target = 896 / 8; 1014 mod = 1024 / 8; 1015 size_bytes = 16; 1016 break; 1017 } 1018 1019 target -= 1; 1020 diff = dgstlen & (mod - 1); 1021 pad_bytes = diff > target ? target + mod - diff : target - diff; 1022 1023 memset(dst + 1, 0, pad_bytes); 1024 dst[0] = 0x80; 1025 1026 if (size_bytes == 16) { 1027 memset(dst + 1 + pad_bytes, 0, 8); 1028 memcpy(dst + 1 + pad_bytes + 8, &bits, 8); 1029 } else { 1030 memcpy(dst + 1 + pad_bytes, &bits, 8); 1031 } 1032 1033 return pad_bytes + size_bytes + 1; 1034 } 1035 1036 static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common, 1037 struct crypto_async_request *parent, 1038 void (*complete)(struct crypto_async_request *req), 1039 struct scatterlist *dstsg, unsigned int nbytes) 1040 { 1041 gfp_t flags; 1042 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1043 1044 flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1045 GFP_KERNEL : GFP_ATOMIC; 1046 1047 common->gfp_flags = flags; 1048 common->dma = kmem_cache_alloc(ac->dma_cache, flags); 1049 if (!common->dma) 1050 return -ENOMEM; 1051 1052 common->req = parent; 1053 common->complete = complete; 1054 return 0; 1055 } 1056 1057 static void 1058 artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma) 1059 { 1060 struct artpec6_crypto_bounce_buffer *b; 1061 struct artpec6_crypto_bounce_buffer *next; 1062 1063 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { 1064 kfree(b); 1065 } 1066 } 1067 1068 static int 1069 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common) 1070 { 1071 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1072 1073 artpec6_crypto_dma_unmap_all(common); 1074 artpec6_crypto_bounce_destroy(common->dma); 1075 kmem_cache_free(ac->dma_cache, common->dma); 1076 common->dma = NULL; 1077 return 0; 1078 } 1079 1080 /* 1081 * Ciphering functions. 1082 */ 1083 static int artpec6_crypto_encrypt(struct skcipher_request *req) 1084 { 1085 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1086 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1087 struct artpec6_crypto_request_context *req_ctx = NULL; 1088 void (*complete)(struct crypto_async_request *req); 1089 int ret; 1090 1091 req_ctx = skcipher_request_ctx(req); 1092 1093 switch (ctx->crypto_type) { 1094 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1095 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1096 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1097 req_ctx->decrypt = 0; 1098 break; 1099 default: 1100 break; 1101 } 1102 1103 switch (ctx->crypto_type) { 1104 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1105 complete = artpec6_crypto_complete_cbc_encrypt; 1106 break; 1107 default: 1108 complete = artpec6_crypto_complete_crypto; 1109 break; 1110 } 1111 1112 ret = artpec6_crypto_common_init(&req_ctx->common, 1113 &req->base, 1114 complete, 1115 req->dst, req->cryptlen); 1116 if (ret) 1117 return ret; 1118 1119 ret = artpec6_crypto_prepare_crypto(req); 1120 if (ret) { 1121 artpec6_crypto_common_destroy(&req_ctx->common); 1122 return ret; 1123 } 1124 1125 return artpec6_crypto_submit(&req_ctx->common); 1126 } 1127 1128 static int artpec6_crypto_decrypt(struct skcipher_request *req) 1129 { 1130 int ret; 1131 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1132 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1133 struct artpec6_crypto_request_context *req_ctx = NULL; 1134 void (*complete)(struct crypto_async_request *req); 1135 1136 req_ctx = skcipher_request_ctx(req); 1137 1138 switch (ctx->crypto_type) { 1139 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1140 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1141 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1142 req_ctx->decrypt = 1; 1143 break; 1144 default: 1145 break; 1146 } 1147 1148 1149 switch (ctx->crypto_type) { 1150 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1151 complete = artpec6_crypto_complete_cbc_decrypt; 1152 break; 1153 default: 1154 complete = artpec6_crypto_complete_crypto; 1155 break; 1156 } 1157 1158 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, 1159 complete, 1160 req->dst, req->cryptlen); 1161 if (ret) 1162 return ret; 1163 1164 ret = artpec6_crypto_prepare_crypto(req); 1165 if (ret) { 1166 artpec6_crypto_common_destroy(&req_ctx->common); 1167 return ret; 1168 } 1169 1170 return artpec6_crypto_submit(&req_ctx->common); 1171 } 1172 1173 static int 1174 artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt) 1175 { 1176 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1177 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1178 size_t iv_len = crypto_skcipher_ivsize(cipher); 1179 unsigned int counter = be32_to_cpup((__be32 *) 1180 (req->iv + iv_len - 4)); 1181 unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / 1182 AES_BLOCK_SIZE; 1183 1184 /* 1185 * The hardware uses only the last 32-bits as the counter while the 1186 * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that 1187 * the whole IV is a counter. So fallback if the counter is going to 1188 * overlow. 1189 */ 1190 if (counter + nblks < counter) { 1191 int ret; 1192 1193 pr_debug("counter %x will overflow (nblks %u), falling back\n", 1194 counter, counter + nblks); 1195 1196 ret = crypto_sync_skcipher_setkey(ctx->fallback, ctx->aes_key, 1197 ctx->key_length); 1198 if (ret) 1199 return ret; 1200 1201 { 1202 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); 1203 1204 skcipher_request_set_sync_tfm(subreq, ctx->fallback); 1205 skcipher_request_set_callback(subreq, req->base.flags, 1206 NULL, NULL); 1207 skcipher_request_set_crypt(subreq, req->src, req->dst, 1208 req->cryptlen, req->iv); 1209 ret = encrypt ? crypto_skcipher_encrypt(subreq) 1210 : crypto_skcipher_decrypt(subreq); 1211 skcipher_request_zero(subreq); 1212 } 1213 return ret; 1214 } 1215 1216 return encrypt ? artpec6_crypto_encrypt(req) 1217 : artpec6_crypto_decrypt(req); 1218 } 1219 1220 static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req) 1221 { 1222 return artpec6_crypto_ctr_crypt(req, true); 1223 } 1224 1225 static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req) 1226 { 1227 return artpec6_crypto_ctr_crypt(req, false); 1228 } 1229 1230 /* 1231 * AEAD functions 1232 */ 1233 static int artpec6_crypto_aead_init(struct crypto_aead *tfm) 1234 { 1235 struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm); 1236 1237 memset(tfm_ctx, 0, sizeof(*tfm_ctx)); 1238 1239 crypto_aead_set_reqsize(tfm, 1240 sizeof(struct artpec6_crypto_aead_req_ctx)); 1241 1242 return 0; 1243 } 1244 1245 static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key, 1246 unsigned int len) 1247 { 1248 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base); 1249 1250 if (len != 16 && len != 24 && len != 32) 1251 return -EINVAL; 1252 1253 ctx->key_length = len; 1254 1255 memcpy(ctx->aes_key, key, len); 1256 return 0; 1257 } 1258 1259 static int artpec6_crypto_aead_encrypt(struct aead_request *req) 1260 { 1261 int ret; 1262 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); 1263 1264 req_ctx->decrypt = false; 1265 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, 1266 artpec6_crypto_complete_aead, 1267 NULL, 0); 1268 if (ret) 1269 return ret; 1270 1271 ret = artpec6_crypto_prepare_aead(req); 1272 if (ret) { 1273 artpec6_crypto_common_destroy(&req_ctx->common); 1274 return ret; 1275 } 1276 1277 return artpec6_crypto_submit(&req_ctx->common); 1278 } 1279 1280 static int artpec6_crypto_aead_decrypt(struct aead_request *req) 1281 { 1282 int ret; 1283 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); 1284 1285 req_ctx->decrypt = true; 1286 if (req->cryptlen < AES_BLOCK_SIZE) 1287 return -EINVAL; 1288 1289 ret = artpec6_crypto_common_init(&req_ctx->common, 1290 &req->base, 1291 artpec6_crypto_complete_aead, 1292 NULL, 0); 1293 if (ret) 1294 return ret; 1295 1296 ret = artpec6_crypto_prepare_aead(req); 1297 if (ret) { 1298 artpec6_crypto_common_destroy(&req_ctx->common); 1299 return ret; 1300 } 1301 1302 return artpec6_crypto_submit(&req_ctx->common); 1303 } 1304 1305 static int artpec6_crypto_prepare_hash(struct ahash_request *areq) 1306 { 1307 struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm); 1308 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq); 1309 size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); 1310 size_t contextsize = digestsize; 1311 size_t blocksize = crypto_tfm_alg_blocksize( 1312 crypto_ahash_tfm(crypto_ahash_reqtfm(areq))); 1313 struct artpec6_crypto_req_common *common = &req_ctx->common; 1314 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1315 enum artpec6_crypto_variant variant = ac->variant; 1316 u32 sel_ctx; 1317 bool ext_ctx = false; 1318 bool run_hw = false; 1319 int error = 0; 1320 1321 artpec6_crypto_init_dma_operation(common); 1322 1323 /* Upload HMAC key, it must be the first packet */ 1324 if (req_ctx->hash_flags & HASH_FLAG_HMAC) { 1325 if (variant == ARTPEC6_CRYPTO) { 1326 req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, 1327 a6_regk_crypto_dlkey); 1328 } else { 1329 req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, 1330 a7_regk_crypto_dlkey); 1331 } 1332 1333 memcpy_and_pad(req_ctx->key_buffer, blocksize, ctx->hmac_key, 1334 ctx->hmac_key_length, 0); 1335 1336 error = artpec6_crypto_setup_out_descr(common, 1337 (void *)&req_ctx->key_md, 1338 sizeof(req_ctx->key_md), false, false); 1339 if (error) 1340 return error; 1341 1342 error = artpec6_crypto_setup_out_descr(common, 1343 req_ctx->key_buffer, blocksize, 1344 true, false); 1345 if (error) 1346 return error; 1347 } 1348 1349 if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) { 1350 /* Restore context */ 1351 sel_ctx = regk_crypto_ext; 1352 ext_ctx = true; 1353 } else { 1354 sel_ctx = regk_crypto_init; 1355 } 1356 1357 if (variant == ARTPEC6_CRYPTO) { 1358 req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX; 1359 req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx); 1360 1361 /* If this is the final round, set the final flag */ 1362 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) 1363 req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN; 1364 } else { 1365 req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX; 1366 req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx); 1367 1368 /* If this is the final round, set the final flag */ 1369 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) 1370 req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN; 1371 } 1372 1373 /* Setup up metadata descriptors */ 1374 error = artpec6_crypto_setup_out_descr(common, 1375 (void *)&req_ctx->hash_md, 1376 sizeof(req_ctx->hash_md), false, false); 1377 if (error) 1378 return error; 1379 1380 error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1381 if (error) 1382 return error; 1383 1384 if (ext_ctx) { 1385 error = artpec6_crypto_setup_out_descr(common, 1386 req_ctx->digeststate, 1387 contextsize, false, false); 1388 1389 if (error) 1390 return error; 1391 } 1392 1393 if (req_ctx->hash_flags & HASH_FLAG_UPDATE) { 1394 size_t done_bytes = 0; 1395 size_t total_bytes = areq->nbytes + req_ctx->partial_bytes; 1396 size_t ready_bytes = round_down(total_bytes, blocksize); 1397 struct artpec6_crypto_walk walk; 1398 1399 run_hw = ready_bytes > 0; 1400 if (req_ctx->partial_bytes && ready_bytes) { 1401 /* We have a partial buffer and will at least some bytes 1402 * to the HW. Empty this partial buffer before tackling 1403 * the SG lists 1404 */ 1405 memcpy(req_ctx->partial_buffer_out, 1406 req_ctx->partial_buffer, 1407 req_ctx->partial_bytes); 1408 1409 error = artpec6_crypto_setup_out_descr(common, 1410 req_ctx->partial_buffer_out, 1411 req_ctx->partial_bytes, 1412 false, true); 1413 if (error) 1414 return error; 1415 1416 /* Reset partial buffer */ 1417 done_bytes += req_ctx->partial_bytes; 1418 req_ctx->partial_bytes = 0; 1419 } 1420 1421 artpec6_crypto_walk_init(&walk, areq->src); 1422 1423 error = artpec6_crypto_setup_sg_descrs_out(common, &walk, 1424 ready_bytes - 1425 done_bytes); 1426 if (error) 1427 return error; 1428 1429 if (walk.sg) { 1430 size_t sg_skip = ready_bytes - done_bytes; 1431 size_t sg_rem = areq->nbytes - sg_skip; 1432 1433 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 1434 req_ctx->partial_buffer + 1435 req_ctx->partial_bytes, 1436 sg_rem, sg_skip); 1437 1438 req_ctx->partial_bytes += sg_rem; 1439 } 1440 1441 req_ctx->digcnt += ready_bytes; 1442 req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE); 1443 } 1444 1445 /* Finalize */ 1446 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) { 1447 size_t hash_pad_len; 1448 u64 digest_bits; 1449 u32 oper; 1450 1451 if (variant == ARTPEC6_CRYPTO) 1452 oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md); 1453 else 1454 oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md); 1455 1456 /* Write out the partial buffer if present */ 1457 if (req_ctx->partial_bytes) { 1458 memcpy(req_ctx->partial_buffer_out, 1459 req_ctx->partial_buffer, 1460 req_ctx->partial_bytes); 1461 error = artpec6_crypto_setup_out_descr(common, 1462 req_ctx->partial_buffer_out, 1463 req_ctx->partial_bytes, 1464 false, true); 1465 if (error) 1466 return error; 1467 1468 req_ctx->digcnt += req_ctx->partial_bytes; 1469 req_ctx->partial_bytes = 0; 1470 } 1471 1472 if (req_ctx->hash_flags & HASH_FLAG_HMAC) 1473 digest_bits = 8 * (req_ctx->digcnt + blocksize); 1474 else 1475 digest_bits = 8 * req_ctx->digcnt; 1476 1477 /* Add the hash pad */ 1478 hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer, 1479 req_ctx->digcnt, digest_bits); 1480 error = artpec6_crypto_setup_out_descr(common, 1481 req_ctx->pad_buffer, 1482 hash_pad_len, false, 1483 true); 1484 req_ctx->digcnt = 0; 1485 1486 if (error) 1487 return error; 1488 1489 /* Descriptor for the final result */ 1490 error = artpec6_crypto_setup_in_descr(common, areq->result, 1491 digestsize, 1492 true); 1493 if (error) 1494 return error; 1495 1496 } else { /* This is not the final operation for this request */ 1497 if (!run_hw) 1498 return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START; 1499 1500 /* Save the result to the context */ 1501 error = artpec6_crypto_setup_in_descr(common, 1502 req_ctx->digeststate, 1503 contextsize, false); 1504 if (error) 1505 return error; 1506 /* fall through */ 1507 } 1508 1509 req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE | 1510 HASH_FLAG_FINALIZE); 1511 1512 error = artpec6_crypto_terminate_in_descrs(common); 1513 if (error) 1514 return error; 1515 1516 error = artpec6_crypto_terminate_out_descrs(common); 1517 if (error) 1518 return error; 1519 1520 error = artpec6_crypto_dma_map_descs(common); 1521 if (error) 1522 return error; 1523 1524 return ARTPEC6_CRYPTO_PREPARE_HASH_START; 1525 } 1526 1527 1528 static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm) 1529 { 1530 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1531 1532 crypto_skcipher_set_reqsize(tfm, 1533 sizeof(struct artpec6_crypto_request_context)); 1534 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB; 1535 1536 return 0; 1537 } 1538 1539 static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm) 1540 { 1541 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1542 1543 ctx->fallback = 1544 crypto_alloc_sync_skcipher(crypto_tfm_alg_name(&tfm->base), 1545 0, CRYPTO_ALG_NEED_FALLBACK); 1546 if (IS_ERR(ctx->fallback)) 1547 return PTR_ERR(ctx->fallback); 1548 1549 crypto_skcipher_set_reqsize(tfm, 1550 sizeof(struct artpec6_crypto_request_context)); 1551 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR; 1552 1553 return 0; 1554 } 1555 1556 static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm) 1557 { 1558 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1559 1560 crypto_skcipher_set_reqsize(tfm, 1561 sizeof(struct artpec6_crypto_request_context)); 1562 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC; 1563 1564 return 0; 1565 } 1566 1567 static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm) 1568 { 1569 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1570 1571 crypto_skcipher_set_reqsize(tfm, 1572 sizeof(struct artpec6_crypto_request_context)); 1573 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS; 1574 1575 return 0; 1576 } 1577 1578 static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm) 1579 { 1580 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1581 1582 memset(ctx, 0, sizeof(*ctx)); 1583 } 1584 1585 static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm) 1586 { 1587 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1588 1589 crypto_free_sync_skcipher(ctx->fallback); 1590 artpec6_crypto_aes_exit(tfm); 1591 } 1592 1593 static int 1594 artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key, 1595 unsigned int keylen) 1596 { 1597 struct artpec6_cryptotfm_context *ctx = 1598 crypto_skcipher_ctx(cipher); 1599 1600 switch (keylen) { 1601 case 16: 1602 case 24: 1603 case 32: 1604 break; 1605 default: 1606 return -EINVAL; 1607 } 1608 1609 memcpy(ctx->aes_key, key, keylen); 1610 ctx->key_length = keylen; 1611 return 0; 1612 } 1613 1614 static int 1615 artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key, 1616 unsigned int keylen) 1617 { 1618 struct artpec6_cryptotfm_context *ctx = 1619 crypto_skcipher_ctx(cipher); 1620 int ret; 1621 1622 ret = xts_verify_key(cipher, key, keylen); 1623 if (ret) 1624 return ret; 1625 1626 switch (keylen) { 1627 case 32: 1628 case 48: 1629 case 64: 1630 break; 1631 default: 1632 return -EINVAL; 1633 } 1634 1635 memcpy(ctx->aes_key, key, keylen); 1636 ctx->key_length = keylen; 1637 return 0; 1638 } 1639 1640 /** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request 1641 * 1642 * @req: The asynch request to process 1643 * 1644 * @return 0 if the dma job was successfully prepared 1645 * <0 on error 1646 * 1647 * This function sets up the PDMA descriptors for a block cipher request. 1648 * 1649 * The required padding is added for AES-CTR using a statically defined 1650 * buffer. 1651 * 1652 * The PDMA descriptor list will be as follows: 1653 * 1654 * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop> 1655 * IN: <CIPHER_MD><data_0>...[data_n]<intr> 1656 * 1657 */ 1658 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq) 1659 { 1660 int ret; 1661 struct artpec6_crypto_walk walk; 1662 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1663 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1664 struct artpec6_crypto_request_context *req_ctx = NULL; 1665 size_t iv_len = crypto_skcipher_ivsize(cipher); 1666 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1667 enum artpec6_crypto_variant variant = ac->variant; 1668 struct artpec6_crypto_req_common *common; 1669 bool cipher_decr = false; 1670 size_t cipher_klen; 1671 u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */ 1672 u32 oper; 1673 1674 req_ctx = skcipher_request_ctx(areq); 1675 common = &req_ctx->common; 1676 1677 artpec6_crypto_init_dma_operation(common); 1678 1679 if (variant == ARTPEC6_CRYPTO) 1680 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey); 1681 else 1682 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey); 1683 1684 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, 1685 sizeof(ctx->key_md), false, false); 1686 if (ret) 1687 return ret; 1688 1689 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, 1690 ctx->key_length, true, false); 1691 if (ret) 1692 return ret; 1693 1694 req_ctx->cipher_md = 0; 1695 1696 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) 1697 cipher_klen = ctx->key_length/2; 1698 else 1699 cipher_klen = ctx->key_length; 1700 1701 /* Metadata */ 1702 switch (cipher_klen) { 1703 case 16: 1704 cipher_len = regk_crypto_key_128; 1705 break; 1706 case 24: 1707 cipher_len = regk_crypto_key_192; 1708 break; 1709 case 32: 1710 cipher_len = regk_crypto_key_256; 1711 break; 1712 default: 1713 pr_err("%s: Invalid key length %zu!\n", 1714 MODULE_NAME, ctx->key_length); 1715 return -EINVAL; 1716 } 1717 1718 switch (ctx->crypto_type) { 1719 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1720 oper = regk_crypto_aes_ecb; 1721 cipher_decr = req_ctx->decrypt; 1722 break; 1723 1724 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1725 oper = regk_crypto_aes_cbc; 1726 cipher_decr = req_ctx->decrypt; 1727 break; 1728 1729 case ARTPEC6_CRYPTO_CIPHER_AES_CTR: 1730 oper = regk_crypto_aes_ctr; 1731 cipher_decr = false; 1732 break; 1733 1734 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1735 oper = regk_crypto_aes_xts; 1736 cipher_decr = req_ctx->decrypt; 1737 1738 if (variant == ARTPEC6_CRYPTO) 1739 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ; 1740 else 1741 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ; 1742 break; 1743 1744 default: 1745 pr_err("%s: Invalid cipher mode %d!\n", 1746 MODULE_NAME, ctx->crypto_type); 1747 return -EINVAL; 1748 } 1749 1750 if (variant == ARTPEC6_CRYPTO) { 1751 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper); 1752 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, 1753 cipher_len); 1754 if (cipher_decr) 1755 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; 1756 } else { 1757 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper); 1758 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, 1759 cipher_len); 1760 if (cipher_decr) 1761 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; 1762 } 1763 1764 ret = artpec6_crypto_setup_out_descr(common, 1765 &req_ctx->cipher_md, 1766 sizeof(req_ctx->cipher_md), 1767 false, false); 1768 if (ret) 1769 return ret; 1770 1771 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1772 if (ret) 1773 return ret; 1774 1775 if (iv_len) { 1776 ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len, 1777 false, false); 1778 if (ret) 1779 return ret; 1780 } 1781 /* Data out */ 1782 artpec6_crypto_walk_init(&walk, areq->src); 1783 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen); 1784 if (ret) 1785 return ret; 1786 1787 /* Data in */ 1788 artpec6_crypto_walk_init(&walk, areq->dst); 1789 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen); 1790 if (ret) 1791 return ret; 1792 1793 /* CTR-mode padding required by the HW. */ 1794 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR || 1795 ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) { 1796 size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) - 1797 areq->cryptlen; 1798 1799 if (pad) { 1800 ret = artpec6_crypto_setup_out_descr(common, 1801 ac->pad_buffer, 1802 pad, false, false); 1803 if (ret) 1804 return ret; 1805 1806 ret = artpec6_crypto_setup_in_descr(common, 1807 ac->pad_buffer, pad, 1808 false); 1809 if (ret) 1810 return ret; 1811 } 1812 } 1813 1814 ret = artpec6_crypto_terminate_out_descrs(common); 1815 if (ret) 1816 return ret; 1817 1818 ret = artpec6_crypto_terminate_in_descrs(common); 1819 if (ret) 1820 return ret; 1821 1822 return artpec6_crypto_dma_map_descs(common); 1823 } 1824 1825 static int artpec6_crypto_prepare_aead(struct aead_request *areq) 1826 { 1827 size_t count; 1828 int ret; 1829 size_t input_length; 1830 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm); 1831 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); 1832 struct crypto_aead *cipher = crypto_aead_reqtfm(areq); 1833 struct artpec6_crypto_req_common *common = &req_ctx->common; 1834 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1835 enum artpec6_crypto_variant variant = ac->variant; 1836 u32 md_cipher_len; 1837 1838 artpec6_crypto_init_dma_operation(common); 1839 1840 /* Key */ 1841 if (variant == ARTPEC6_CRYPTO) { 1842 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, 1843 a6_regk_crypto_dlkey); 1844 } else { 1845 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, 1846 a7_regk_crypto_dlkey); 1847 } 1848 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, 1849 sizeof(ctx->key_md), false, false); 1850 if (ret) 1851 return ret; 1852 1853 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, 1854 ctx->key_length, true, false); 1855 if (ret) 1856 return ret; 1857 1858 req_ctx->cipher_md = 0; 1859 1860 switch (ctx->key_length) { 1861 case 16: 1862 md_cipher_len = regk_crypto_key_128; 1863 break; 1864 case 24: 1865 md_cipher_len = regk_crypto_key_192; 1866 break; 1867 case 32: 1868 md_cipher_len = regk_crypto_key_256; 1869 break; 1870 default: 1871 return -EINVAL; 1872 } 1873 1874 if (variant == ARTPEC6_CRYPTO) { 1875 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, 1876 regk_crypto_aes_gcm); 1877 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, 1878 md_cipher_len); 1879 if (req_ctx->decrypt) 1880 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; 1881 } else { 1882 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, 1883 regk_crypto_aes_gcm); 1884 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, 1885 md_cipher_len); 1886 if (req_ctx->decrypt) 1887 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; 1888 } 1889 1890 ret = artpec6_crypto_setup_out_descr(common, 1891 (void *) &req_ctx->cipher_md, 1892 sizeof(req_ctx->cipher_md), false, 1893 false); 1894 if (ret) 1895 return ret; 1896 1897 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1898 if (ret) 1899 return ret; 1900 1901 /* For the decryption, cryptlen includes the tag. */ 1902 input_length = areq->cryptlen; 1903 if (req_ctx->decrypt) 1904 input_length -= crypto_aead_authsize(cipher); 1905 1906 /* Prepare the context buffer */ 1907 req_ctx->hw_ctx.aad_length_bits = 1908 __cpu_to_be64(8*areq->assoclen); 1909 1910 req_ctx->hw_ctx.text_length_bits = 1911 __cpu_to_be64(8*input_length); 1912 1913 memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher)); 1914 // The HW omits the initial increment of the counter field. 1915 memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4); 1916 1917 ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx, 1918 sizeof(struct artpec6_crypto_aead_hw_ctx), false, false); 1919 if (ret) 1920 return ret; 1921 1922 { 1923 struct artpec6_crypto_walk walk; 1924 1925 artpec6_crypto_walk_init(&walk, areq->src); 1926 1927 /* Associated data */ 1928 count = areq->assoclen; 1929 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); 1930 if (ret) 1931 return ret; 1932 1933 if (!IS_ALIGNED(areq->assoclen, 16)) { 1934 size_t assoc_pad = 16 - (areq->assoclen % 16); 1935 /* The HW mandates zero padding here */ 1936 ret = artpec6_crypto_setup_out_descr(common, 1937 ac->zero_buffer, 1938 assoc_pad, false, 1939 false); 1940 if (ret) 1941 return ret; 1942 } 1943 1944 /* Data to crypto */ 1945 count = input_length; 1946 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); 1947 if (ret) 1948 return ret; 1949 1950 if (!IS_ALIGNED(input_length, 16)) { 1951 size_t crypto_pad = 16 - (input_length % 16); 1952 /* The HW mandates zero padding here */ 1953 ret = artpec6_crypto_setup_out_descr(common, 1954 ac->zero_buffer, 1955 crypto_pad, 1956 false, 1957 false); 1958 if (ret) 1959 return ret; 1960 } 1961 } 1962 1963 /* Data from crypto */ 1964 { 1965 struct artpec6_crypto_walk walk; 1966 size_t output_len = areq->cryptlen; 1967 1968 if (req_ctx->decrypt) 1969 output_len -= crypto_aead_authsize(cipher); 1970 1971 artpec6_crypto_walk_init(&walk, areq->dst); 1972 1973 /* skip associated data in the output */ 1974 count = artpec6_crypto_walk_advance(&walk, areq->assoclen); 1975 if (count) 1976 return -EINVAL; 1977 1978 count = output_len; 1979 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count); 1980 if (ret) 1981 return ret; 1982 1983 /* Put padding between the cryptotext and the auth tag */ 1984 if (!IS_ALIGNED(output_len, 16)) { 1985 size_t crypto_pad = 16 - (output_len % 16); 1986 1987 ret = artpec6_crypto_setup_in_descr(common, 1988 ac->pad_buffer, 1989 crypto_pad, false); 1990 if (ret) 1991 return ret; 1992 } 1993 1994 /* The authentication tag shall follow immediately after 1995 * the output ciphertext. For decryption it is put in a context 1996 * buffer for later compare against the input tag. 1997 */ 1998 1999 if (req_ctx->decrypt) { 2000 ret = artpec6_crypto_setup_in_descr(common, 2001 req_ctx->decryption_tag, AES_BLOCK_SIZE, false); 2002 if (ret) 2003 return ret; 2004 2005 } else { 2006 /* For encryption the requested tag size may be smaller 2007 * than the hardware's generated tag. 2008 */ 2009 size_t authsize = crypto_aead_authsize(cipher); 2010 2011 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, 2012 authsize); 2013 if (ret) 2014 return ret; 2015 2016 if (authsize < AES_BLOCK_SIZE) { 2017 count = AES_BLOCK_SIZE - authsize; 2018 ret = artpec6_crypto_setup_in_descr(common, 2019 ac->pad_buffer, 2020 count, false); 2021 if (ret) 2022 return ret; 2023 } 2024 } 2025 2026 } 2027 2028 ret = artpec6_crypto_terminate_in_descrs(common); 2029 if (ret) 2030 return ret; 2031 2032 ret = artpec6_crypto_terminate_out_descrs(common); 2033 if (ret) 2034 return ret; 2035 2036 return artpec6_crypto_dma_map_descs(common); 2037 } 2038 2039 static void artpec6_crypto_process_queue(struct artpec6_crypto *ac, 2040 struct list_head *completions) 2041 { 2042 struct artpec6_crypto_req_common *req; 2043 2044 while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) { 2045 req = list_first_entry(&ac->queue, 2046 struct artpec6_crypto_req_common, 2047 list); 2048 list_move_tail(&req->list, &ac->pending); 2049 artpec6_crypto_start_dma(req); 2050 2051 list_add_tail(&req->complete_in_progress, completions); 2052 } 2053 2054 /* 2055 * In some cases, the hardware can raise an in_eop_flush interrupt 2056 * before actually updating the status, so we have an timer which will 2057 * recheck the status on timeout. Since the cases are expected to be 2058 * very rare, we use a relatively large timeout value. There should be 2059 * no noticeable negative effect if we timeout spuriously. 2060 */ 2061 if (ac->pending_count) 2062 mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100)); 2063 else 2064 timer_delete(&ac->timer); 2065 } 2066 2067 static void artpec6_crypto_timeout(struct timer_list *t) 2068 { 2069 struct artpec6_crypto *ac = timer_container_of(ac, t, timer); 2070 2071 dev_info_ratelimited(artpec6_crypto_dev, "timeout\n"); 2072 2073 tasklet_schedule(&ac->task); 2074 } 2075 2076 static void artpec6_crypto_task(unsigned long data) 2077 { 2078 struct artpec6_crypto *ac = (struct artpec6_crypto *)data; 2079 struct artpec6_crypto_req_common *req; 2080 struct artpec6_crypto_req_common *n; 2081 struct list_head complete_done; 2082 struct list_head complete_in_progress; 2083 2084 INIT_LIST_HEAD(&complete_done); 2085 INIT_LIST_HEAD(&complete_in_progress); 2086 2087 if (list_empty(&ac->pending)) { 2088 pr_debug("Spurious IRQ\n"); 2089 return; 2090 } 2091 2092 spin_lock(&ac->queue_lock); 2093 2094 list_for_each_entry_safe(req, n, &ac->pending, list) { 2095 struct artpec6_crypto_dma_descriptors *dma = req->dma; 2096 u32 stat; 2097 dma_addr_t stataddr; 2098 2099 stataddr = dma->stat_dma_addr + 4 * (req->dma->in_cnt - 1); 2100 dma_sync_single_for_cpu(artpec6_crypto_dev, 2101 stataddr, 2102 4, 2103 DMA_BIDIRECTIONAL); 2104 2105 stat = req->dma->stat[req->dma->in_cnt-1]; 2106 2107 /* A non-zero final status descriptor indicates 2108 * this job has finished. 2109 */ 2110 pr_debug("Request %p status is %X\n", req, stat); 2111 if (!stat) 2112 break; 2113 2114 /* Allow testing of timeout handling with fault injection */ 2115 #ifdef CONFIG_FAULT_INJECTION 2116 if (should_fail(&artpec6_crypto_fail_status_read, 1)) 2117 continue; 2118 #endif 2119 2120 pr_debug("Completing request %p\n", req); 2121 2122 list_move_tail(&req->list, &complete_done); 2123 2124 ac->pending_count--; 2125 } 2126 2127 artpec6_crypto_process_queue(ac, &complete_in_progress); 2128 2129 spin_unlock(&ac->queue_lock); 2130 2131 /* Perform the completion callbacks without holding the queue lock 2132 * to allow new request submissions from the callbacks. 2133 */ 2134 list_for_each_entry_safe(req, n, &complete_done, list) { 2135 artpec6_crypto_dma_unmap_all(req); 2136 artpec6_crypto_copy_bounce_buffers(req); 2137 artpec6_crypto_common_destroy(req); 2138 2139 req->complete(req->req); 2140 } 2141 2142 list_for_each_entry_safe(req, n, &complete_in_progress, 2143 complete_in_progress) { 2144 crypto_request_complete(req->req, -EINPROGRESS); 2145 } 2146 } 2147 2148 static void artpec6_crypto_complete_crypto(struct crypto_async_request *req) 2149 { 2150 crypto_request_complete(req, 0); 2151 } 2152 2153 static void 2154 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req) 2155 { 2156 struct skcipher_request *cipher_req = container_of(req, 2157 struct skcipher_request, base); 2158 2159 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src, 2160 cipher_req->cryptlen - AES_BLOCK_SIZE, 2161 AES_BLOCK_SIZE, 0); 2162 skcipher_request_complete(cipher_req, 0); 2163 } 2164 2165 static void 2166 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req) 2167 { 2168 struct skcipher_request *cipher_req = container_of(req, 2169 struct skcipher_request, base); 2170 2171 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst, 2172 cipher_req->cryptlen - AES_BLOCK_SIZE, 2173 AES_BLOCK_SIZE, 0); 2174 skcipher_request_complete(cipher_req, 0); 2175 } 2176 2177 static void artpec6_crypto_complete_aead(struct crypto_async_request *req) 2178 { 2179 int result = 0; 2180 2181 /* Verify GCM hashtag. */ 2182 struct aead_request *areq = container_of(req, 2183 struct aead_request, base); 2184 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 2185 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); 2186 2187 if (req_ctx->decrypt) { 2188 u8 input_tag[AES_BLOCK_SIZE]; 2189 unsigned int authsize = crypto_aead_authsize(aead); 2190 2191 sg_pcopy_to_buffer(areq->src, 2192 sg_nents(areq->src), 2193 input_tag, 2194 authsize, 2195 areq->assoclen + areq->cryptlen - 2196 authsize); 2197 2198 if (crypto_memneq(req_ctx->decryption_tag, 2199 input_tag, 2200 authsize)) { 2201 pr_debug("***EBADMSG:\n"); 2202 print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1, 2203 input_tag, authsize, true); 2204 print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1, 2205 req_ctx->decryption_tag, 2206 authsize, true); 2207 2208 result = -EBADMSG; 2209 } 2210 } 2211 2212 aead_request_complete(areq, result); 2213 } 2214 2215 static void artpec6_crypto_complete_hash(struct crypto_async_request *req) 2216 { 2217 crypto_request_complete(req, 0); 2218 } 2219 2220 2221 /*------------------- Hash functions -----------------------------------------*/ 2222 static int 2223 artpec6_crypto_hash_set_key(struct crypto_ahash *tfm, 2224 const u8 *key, unsigned int keylen) 2225 { 2226 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base); 2227 size_t blocksize; 2228 int ret; 2229 2230 if (!keylen) { 2231 pr_err("Invalid length (%d) of HMAC key\n", 2232 keylen); 2233 return -EINVAL; 2234 } 2235 2236 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); 2237 2238 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2239 2240 if (keylen > blocksize) { 2241 tfm_ctx->hmac_key_length = blocksize; 2242 2243 ret = crypto_shash_tfm_digest(tfm_ctx->child_hash, key, keylen, 2244 tfm_ctx->hmac_key); 2245 if (ret) 2246 return ret; 2247 } else { 2248 memcpy(tfm_ctx->hmac_key, key, keylen); 2249 tfm_ctx->hmac_key_length = keylen; 2250 } 2251 2252 return 0; 2253 } 2254 2255 static int 2256 artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac) 2257 { 2258 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2259 enum artpec6_crypto_variant variant = ac->variant; 2260 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2261 u32 oper; 2262 2263 memset(req_ctx, 0, sizeof(*req_ctx)); 2264 2265 req_ctx->hash_flags = HASH_FLAG_INIT_CTX; 2266 if (hmac) 2267 req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY); 2268 2269 switch (type) { 2270 case ARTPEC6_CRYPTO_HASH_SHA1: 2271 oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1; 2272 break; 2273 case ARTPEC6_CRYPTO_HASH_SHA256: 2274 oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256; 2275 break; 2276 default: 2277 pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type); 2278 return -EINVAL; 2279 } 2280 2281 if (variant == ARTPEC6_CRYPTO) 2282 req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper); 2283 else 2284 req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper); 2285 2286 return 0; 2287 } 2288 2289 static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req) 2290 { 2291 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2292 int ret; 2293 2294 if (!req_ctx->common.dma) { 2295 ret = artpec6_crypto_common_init(&req_ctx->common, 2296 &req->base, 2297 artpec6_crypto_complete_hash, 2298 NULL, 0); 2299 2300 if (ret) 2301 return ret; 2302 } 2303 2304 ret = artpec6_crypto_prepare_hash(req); 2305 switch (ret) { 2306 case ARTPEC6_CRYPTO_PREPARE_HASH_START: 2307 ret = artpec6_crypto_submit(&req_ctx->common); 2308 break; 2309 2310 case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START: 2311 ret = 0; 2312 fallthrough; 2313 2314 default: 2315 artpec6_crypto_common_destroy(&req_ctx->common); 2316 break; 2317 } 2318 2319 return ret; 2320 } 2321 2322 static int artpec6_crypto_hash_final(struct ahash_request *req) 2323 { 2324 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2325 2326 req_ctx->hash_flags |= HASH_FLAG_FINALIZE; 2327 2328 return artpec6_crypto_prepare_submit_hash(req); 2329 } 2330 2331 static int artpec6_crypto_hash_update(struct ahash_request *req) 2332 { 2333 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2334 2335 req_ctx->hash_flags |= HASH_FLAG_UPDATE; 2336 2337 return artpec6_crypto_prepare_submit_hash(req); 2338 } 2339 2340 static int artpec6_crypto_sha1_init(struct ahash_request *req) 2341 { 2342 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); 2343 } 2344 2345 static int artpec6_crypto_sha1_digest(struct ahash_request *req) 2346 { 2347 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2348 2349 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); 2350 2351 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2352 2353 return artpec6_crypto_prepare_submit_hash(req); 2354 } 2355 2356 static int artpec6_crypto_sha256_init(struct ahash_request *req) 2357 { 2358 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); 2359 } 2360 2361 static int artpec6_crypto_sha256_digest(struct ahash_request *req) 2362 { 2363 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2364 2365 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); 2366 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2367 2368 return artpec6_crypto_prepare_submit_hash(req); 2369 } 2370 2371 static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req) 2372 { 2373 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); 2374 } 2375 2376 static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req) 2377 { 2378 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2379 2380 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); 2381 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2382 2383 return artpec6_crypto_prepare_submit_hash(req); 2384 } 2385 2386 static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm, 2387 const char *base_hash_name) 2388 { 2389 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); 2390 2391 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2392 sizeof(struct artpec6_hash_request_context)); 2393 memset(tfm_ctx, 0, sizeof(*tfm_ctx)); 2394 2395 if (base_hash_name) { 2396 struct crypto_shash *child; 2397 2398 child = crypto_alloc_shash(base_hash_name, 0, 2399 CRYPTO_ALG_NEED_FALLBACK); 2400 2401 if (IS_ERR(child)) 2402 return PTR_ERR(child); 2403 2404 tfm_ctx->child_hash = child; 2405 } 2406 2407 return 0; 2408 } 2409 2410 static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm) 2411 { 2412 return artpec6_crypto_ahash_init_common(tfm, NULL); 2413 } 2414 2415 static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm) 2416 { 2417 return artpec6_crypto_ahash_init_common(tfm, "sha256"); 2418 } 2419 2420 static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm) 2421 { 2422 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); 2423 2424 if (tfm_ctx->child_hash) 2425 crypto_free_shash(tfm_ctx->child_hash); 2426 2427 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); 2428 tfm_ctx->hmac_key_length = 0; 2429 } 2430 2431 static int artpec6_crypto_hash_export(struct ahash_request *req, void *out) 2432 { 2433 const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); 2434 struct artpec6_hash_export_state *state = out; 2435 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2436 enum artpec6_crypto_variant variant = ac->variant; 2437 2438 BUILD_BUG_ON(sizeof(state->partial_buffer) != 2439 sizeof(ctx->partial_buffer)); 2440 BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate)); 2441 2442 state->digcnt = ctx->digcnt; 2443 state->partial_bytes = ctx->partial_bytes; 2444 state->hash_flags = ctx->hash_flags; 2445 2446 if (variant == ARTPEC6_CRYPTO) 2447 state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md); 2448 else 2449 state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md); 2450 2451 memcpy(state->partial_buffer, ctx->partial_buffer, 2452 sizeof(state->partial_buffer)); 2453 memcpy(state->digeststate, ctx->digeststate, 2454 sizeof(state->digeststate)); 2455 2456 return 0; 2457 } 2458 2459 static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in) 2460 { 2461 struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); 2462 const struct artpec6_hash_export_state *state = in; 2463 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2464 enum artpec6_crypto_variant variant = ac->variant; 2465 2466 memset(ctx, 0, sizeof(*ctx)); 2467 2468 ctx->digcnt = state->digcnt; 2469 ctx->partial_bytes = state->partial_bytes; 2470 ctx->hash_flags = state->hash_flags; 2471 2472 if (variant == ARTPEC6_CRYPTO) 2473 ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper); 2474 else 2475 ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper); 2476 2477 memcpy(ctx->partial_buffer, state->partial_buffer, 2478 sizeof(state->partial_buffer)); 2479 memcpy(ctx->digeststate, state->digeststate, 2480 sizeof(state->digeststate)); 2481 2482 return 0; 2483 } 2484 2485 static int init_crypto_hw(struct artpec6_crypto *ac) 2486 { 2487 enum artpec6_crypto_variant variant = ac->variant; 2488 void __iomem *base = ac->base; 2489 u32 out_descr_buf_size; 2490 u32 out_data_buf_size; 2491 u32 in_data_buf_size; 2492 u32 in_descr_buf_size; 2493 u32 in_stat_buf_size; 2494 u32 in, out; 2495 2496 /* 2497 * The PDMA unit contains 1984 bytes of internal memory for the OUT 2498 * channels and 1024 bytes for the IN channel. This is an elastic 2499 * memory used to internally store the descriptors and data. The values 2500 * ares specified in 64 byte incremements. Trustzone buffers are not 2501 * used at this stage. 2502 */ 2503 out_data_buf_size = 16; /* 1024 bytes for data */ 2504 out_descr_buf_size = 15; /* 960 bytes for descriptors */ 2505 in_data_buf_size = 8; /* 512 bytes for data */ 2506 in_descr_buf_size = 4; /* 256 bytes for descriptors */ 2507 in_stat_buf_size = 4; /* 256 bytes for stat descrs */ 2508 2509 BUILD_BUG_ON_MSG((out_data_buf_size 2510 + out_descr_buf_size) * 64 > 1984, 2511 "Invalid OUT configuration"); 2512 2513 BUILD_BUG_ON_MSG((in_data_buf_size 2514 + in_descr_buf_size 2515 + in_stat_buf_size) * 64 > 1024, 2516 "Invalid IN configuration"); 2517 2518 in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) | 2519 FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) | 2520 FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size); 2521 2522 out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) | 2523 FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size); 2524 2525 writel_relaxed(out, base + PDMA_OUT_BUF_CFG); 2526 writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG); 2527 2528 if (variant == ARTPEC6_CRYPTO) { 2529 writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG); 2530 writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG); 2531 writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA | 2532 A6_PDMA_INTR_MASK_IN_EOP_FLUSH, 2533 base + A6_PDMA_INTR_MASK); 2534 } else { 2535 writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG); 2536 writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG); 2537 writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA | 2538 A7_PDMA_INTR_MASK_IN_EOP_FLUSH, 2539 base + A7_PDMA_INTR_MASK); 2540 } 2541 2542 return 0; 2543 } 2544 2545 static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac) 2546 { 2547 enum artpec6_crypto_variant variant = ac->variant; 2548 void __iomem *base = ac->base; 2549 2550 if (variant == ARTPEC6_CRYPTO) { 2551 writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD); 2552 writel_relaxed(0, base + A6_PDMA_IN_CFG); 2553 writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); 2554 } else { 2555 writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD); 2556 writel_relaxed(0, base + A7_PDMA_IN_CFG); 2557 writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); 2558 } 2559 2560 writel_relaxed(0, base + PDMA_OUT_CFG); 2561 2562 } 2563 2564 static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id) 2565 { 2566 struct artpec6_crypto *ac = dev_id; 2567 enum artpec6_crypto_variant variant = ac->variant; 2568 void __iomem *base = ac->base; 2569 u32 mask_in_data, mask_in_eop_flush; 2570 u32 in_cmd_flush_stat, in_cmd_reg; 2571 u32 ack_intr_reg; 2572 u32 ack = 0; 2573 u32 intr; 2574 2575 if (variant == ARTPEC6_CRYPTO) { 2576 intr = readl_relaxed(base + A6_PDMA_MASKED_INTR); 2577 mask_in_data = A6_PDMA_INTR_MASK_IN_DATA; 2578 mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH; 2579 in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT; 2580 in_cmd_reg = A6_PDMA_IN_CMD; 2581 ack_intr_reg = A6_PDMA_ACK_INTR; 2582 } else { 2583 intr = readl_relaxed(base + A7_PDMA_MASKED_INTR); 2584 mask_in_data = A7_PDMA_INTR_MASK_IN_DATA; 2585 mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH; 2586 in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT; 2587 in_cmd_reg = A7_PDMA_IN_CMD; 2588 ack_intr_reg = A7_PDMA_ACK_INTR; 2589 } 2590 2591 /* We get two interrupt notifications from each job. 2592 * The in_data means all data was sent to memory and then 2593 * we request a status flush command to write the per-job 2594 * status to its status vector. This ensures that the 2595 * tasklet can detect exactly how many submitted jobs 2596 * that have finished. 2597 */ 2598 if (intr & mask_in_data) 2599 ack |= mask_in_data; 2600 2601 if (intr & mask_in_eop_flush) 2602 ack |= mask_in_eop_flush; 2603 else 2604 writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg); 2605 2606 writel_relaxed(ack, base + ack_intr_reg); 2607 2608 if (intr & mask_in_eop_flush) 2609 tasklet_schedule(&ac->task); 2610 2611 return IRQ_HANDLED; 2612 } 2613 2614 /*------------------- Algorithm definitions ----------------------------------*/ 2615 2616 /* Hashes */ 2617 static struct ahash_alg hash_algos[] = { 2618 /* SHA-1 */ 2619 { 2620 .init = artpec6_crypto_sha1_init, 2621 .update = artpec6_crypto_hash_update, 2622 .final = artpec6_crypto_hash_final, 2623 .digest = artpec6_crypto_sha1_digest, 2624 .import = artpec6_crypto_hash_import, 2625 .export = artpec6_crypto_hash_export, 2626 .halg.digestsize = SHA1_DIGEST_SIZE, 2627 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2628 .halg.base = { 2629 .cra_name = "sha1", 2630 .cra_driver_name = "artpec-sha1", 2631 .cra_priority = 300, 2632 .cra_flags = CRYPTO_ALG_ASYNC | 2633 CRYPTO_ALG_ALLOCATES_MEMORY, 2634 .cra_blocksize = SHA1_BLOCK_SIZE, 2635 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2636 .cra_module = THIS_MODULE, 2637 .cra_init = artpec6_crypto_ahash_init, 2638 .cra_exit = artpec6_crypto_ahash_exit, 2639 } 2640 }, 2641 /* SHA-256 */ 2642 { 2643 .init = artpec6_crypto_sha256_init, 2644 .update = artpec6_crypto_hash_update, 2645 .final = artpec6_crypto_hash_final, 2646 .digest = artpec6_crypto_sha256_digest, 2647 .import = artpec6_crypto_hash_import, 2648 .export = artpec6_crypto_hash_export, 2649 .halg.digestsize = SHA256_DIGEST_SIZE, 2650 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2651 .halg.base = { 2652 .cra_name = "sha256", 2653 .cra_driver_name = "artpec-sha256", 2654 .cra_priority = 300, 2655 .cra_flags = CRYPTO_ALG_ASYNC | 2656 CRYPTO_ALG_ALLOCATES_MEMORY, 2657 .cra_blocksize = SHA256_BLOCK_SIZE, 2658 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2659 .cra_module = THIS_MODULE, 2660 .cra_init = artpec6_crypto_ahash_init, 2661 .cra_exit = artpec6_crypto_ahash_exit, 2662 } 2663 }, 2664 /* HMAC SHA-256 */ 2665 { 2666 .init = artpec6_crypto_hmac_sha256_init, 2667 .update = artpec6_crypto_hash_update, 2668 .final = artpec6_crypto_hash_final, 2669 .digest = artpec6_crypto_hmac_sha256_digest, 2670 .import = artpec6_crypto_hash_import, 2671 .export = artpec6_crypto_hash_export, 2672 .setkey = artpec6_crypto_hash_set_key, 2673 .halg.digestsize = SHA256_DIGEST_SIZE, 2674 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2675 .halg.base = { 2676 .cra_name = "hmac(sha256)", 2677 .cra_driver_name = "artpec-hmac-sha256", 2678 .cra_priority = 300, 2679 .cra_flags = CRYPTO_ALG_ASYNC | 2680 CRYPTO_ALG_ALLOCATES_MEMORY, 2681 .cra_blocksize = SHA256_BLOCK_SIZE, 2682 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2683 .cra_module = THIS_MODULE, 2684 .cra_init = artpec6_crypto_ahash_init_hmac_sha256, 2685 .cra_exit = artpec6_crypto_ahash_exit, 2686 } 2687 }, 2688 }; 2689 2690 /* Crypto */ 2691 static struct skcipher_alg crypto_algos[] = { 2692 /* AES - ECB */ 2693 { 2694 .base = { 2695 .cra_name = "ecb(aes)", 2696 .cra_driver_name = "artpec6-ecb-aes", 2697 .cra_priority = 300, 2698 .cra_flags = CRYPTO_ALG_ASYNC | 2699 CRYPTO_ALG_ALLOCATES_MEMORY, 2700 .cra_blocksize = AES_BLOCK_SIZE, 2701 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2702 .cra_alignmask = 3, 2703 .cra_module = THIS_MODULE, 2704 }, 2705 .min_keysize = AES_MIN_KEY_SIZE, 2706 .max_keysize = AES_MAX_KEY_SIZE, 2707 .setkey = artpec6_crypto_cipher_set_key, 2708 .encrypt = artpec6_crypto_encrypt, 2709 .decrypt = artpec6_crypto_decrypt, 2710 .init = artpec6_crypto_aes_ecb_init, 2711 .exit = artpec6_crypto_aes_exit, 2712 }, 2713 /* AES - CTR */ 2714 { 2715 .base = { 2716 .cra_name = "ctr(aes)", 2717 .cra_driver_name = "artpec6-ctr-aes", 2718 .cra_priority = 300, 2719 .cra_flags = CRYPTO_ALG_ASYNC | 2720 CRYPTO_ALG_ALLOCATES_MEMORY | 2721 CRYPTO_ALG_NEED_FALLBACK, 2722 .cra_blocksize = 1, 2723 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2724 .cra_alignmask = 3, 2725 .cra_module = THIS_MODULE, 2726 }, 2727 .min_keysize = AES_MIN_KEY_SIZE, 2728 .max_keysize = AES_MAX_KEY_SIZE, 2729 .ivsize = AES_BLOCK_SIZE, 2730 .setkey = artpec6_crypto_cipher_set_key, 2731 .encrypt = artpec6_crypto_ctr_encrypt, 2732 .decrypt = artpec6_crypto_ctr_decrypt, 2733 .init = artpec6_crypto_aes_ctr_init, 2734 .exit = artpec6_crypto_aes_ctr_exit, 2735 }, 2736 /* AES - CBC */ 2737 { 2738 .base = { 2739 .cra_name = "cbc(aes)", 2740 .cra_driver_name = "artpec6-cbc-aes", 2741 .cra_priority = 300, 2742 .cra_flags = CRYPTO_ALG_ASYNC | 2743 CRYPTO_ALG_ALLOCATES_MEMORY, 2744 .cra_blocksize = AES_BLOCK_SIZE, 2745 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2746 .cra_alignmask = 3, 2747 .cra_module = THIS_MODULE, 2748 }, 2749 .min_keysize = AES_MIN_KEY_SIZE, 2750 .max_keysize = AES_MAX_KEY_SIZE, 2751 .ivsize = AES_BLOCK_SIZE, 2752 .setkey = artpec6_crypto_cipher_set_key, 2753 .encrypt = artpec6_crypto_encrypt, 2754 .decrypt = artpec6_crypto_decrypt, 2755 .init = artpec6_crypto_aes_cbc_init, 2756 .exit = artpec6_crypto_aes_exit 2757 }, 2758 /* AES - XTS */ 2759 { 2760 .base = { 2761 .cra_name = "xts(aes)", 2762 .cra_driver_name = "artpec6-xts-aes", 2763 .cra_priority = 300, 2764 .cra_flags = CRYPTO_ALG_ASYNC | 2765 CRYPTO_ALG_ALLOCATES_MEMORY, 2766 .cra_blocksize = 1, 2767 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2768 .cra_alignmask = 3, 2769 .cra_module = THIS_MODULE, 2770 }, 2771 .min_keysize = 2*AES_MIN_KEY_SIZE, 2772 .max_keysize = 2*AES_MAX_KEY_SIZE, 2773 .ivsize = 16, 2774 .setkey = artpec6_crypto_xts_set_key, 2775 .encrypt = artpec6_crypto_encrypt, 2776 .decrypt = artpec6_crypto_decrypt, 2777 .init = artpec6_crypto_aes_xts_init, 2778 .exit = artpec6_crypto_aes_exit, 2779 }, 2780 }; 2781 2782 static struct aead_alg aead_algos[] = { 2783 { 2784 .init = artpec6_crypto_aead_init, 2785 .setkey = artpec6_crypto_aead_set_key, 2786 .encrypt = artpec6_crypto_aead_encrypt, 2787 .decrypt = artpec6_crypto_aead_decrypt, 2788 .ivsize = GCM_AES_IV_SIZE, 2789 .maxauthsize = AES_BLOCK_SIZE, 2790 2791 .base = { 2792 .cra_name = "gcm(aes)", 2793 .cra_driver_name = "artpec-gcm-aes", 2794 .cra_priority = 300, 2795 .cra_flags = CRYPTO_ALG_ASYNC | 2796 CRYPTO_ALG_ALLOCATES_MEMORY | 2797 CRYPTO_ALG_KERN_DRIVER_ONLY, 2798 .cra_blocksize = 1, 2799 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2800 .cra_alignmask = 3, 2801 .cra_module = THIS_MODULE, 2802 }, 2803 } 2804 }; 2805 2806 #ifdef CONFIG_DEBUG_FS 2807 2808 static struct dentry *dbgfs_root; 2809 2810 static void artpec6_crypto_init_debugfs(void) 2811 { 2812 dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL); 2813 2814 #ifdef CONFIG_FAULT_INJECTION 2815 fault_create_debugfs_attr("fail_status_read", dbgfs_root, 2816 &artpec6_crypto_fail_status_read); 2817 2818 fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root, 2819 &artpec6_crypto_fail_dma_array_full); 2820 #endif 2821 } 2822 2823 static void artpec6_crypto_free_debugfs(void) 2824 { 2825 debugfs_remove_recursive(dbgfs_root); 2826 dbgfs_root = NULL; 2827 } 2828 #endif 2829 2830 static const struct of_device_id artpec6_crypto_of_match[] = { 2831 { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO }, 2832 { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO }, 2833 {} 2834 }; 2835 MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match); 2836 2837 static int artpec6_crypto_probe(struct platform_device *pdev) 2838 { 2839 enum artpec6_crypto_variant variant; 2840 struct artpec6_crypto *ac; 2841 struct device *dev = &pdev->dev; 2842 void __iomem *base; 2843 int irq; 2844 int err; 2845 2846 if (artpec6_crypto_dev) 2847 return -ENODEV; 2848 2849 variant = (enum artpec6_crypto_variant)of_device_get_match_data(dev); 2850 if (!variant) 2851 return -EINVAL; 2852 2853 base = devm_platform_ioremap_resource(pdev, 0); 2854 if (IS_ERR(base)) 2855 return PTR_ERR(base); 2856 2857 irq = platform_get_irq(pdev, 0); 2858 if (irq < 0) 2859 return -ENODEV; 2860 2861 ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto), 2862 GFP_KERNEL); 2863 if (!ac) 2864 return -ENOMEM; 2865 2866 platform_set_drvdata(pdev, ac); 2867 ac->variant = variant; 2868 2869 spin_lock_init(&ac->queue_lock); 2870 INIT_LIST_HEAD(&ac->queue); 2871 INIT_LIST_HEAD(&ac->pending); 2872 timer_setup(&ac->timer, artpec6_crypto_timeout, 0); 2873 2874 ac->base = base; 2875 2876 ac->dma_cache = kmem_cache_create("artpec6_crypto_dma", 2877 sizeof(struct artpec6_crypto_dma_descriptors), 2878 64, 2879 0, 2880 NULL); 2881 if (!ac->dma_cache) 2882 return -ENOMEM; 2883 2884 #ifdef CONFIG_DEBUG_FS 2885 artpec6_crypto_init_debugfs(); 2886 #endif 2887 2888 tasklet_init(&ac->task, artpec6_crypto_task, 2889 (unsigned long)ac); 2890 2891 ac->pad_buffer = devm_kcalloc(&pdev->dev, 2, ARTPEC_CACHE_LINE_MAX, 2892 GFP_KERNEL); 2893 if (!ac->pad_buffer) 2894 return -ENOMEM; 2895 ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX); 2896 2897 ac->zero_buffer = devm_kcalloc(&pdev->dev, 2, ARTPEC_CACHE_LINE_MAX, 2898 GFP_KERNEL); 2899 if (!ac->zero_buffer) 2900 return -ENOMEM; 2901 ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX); 2902 2903 err = init_crypto_hw(ac); 2904 if (err) 2905 goto free_cache; 2906 2907 err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0, 2908 "artpec6-crypto", ac); 2909 if (err) 2910 goto disable_hw; 2911 2912 artpec6_crypto_dev = &pdev->dev; 2913 2914 err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 2915 if (err) { 2916 dev_err(dev, "Failed to register ahashes\n"); 2917 goto disable_hw; 2918 } 2919 2920 err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 2921 if (err) { 2922 dev_err(dev, "Failed to register ciphers\n"); 2923 goto unregister_ahashes; 2924 } 2925 2926 err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos)); 2927 if (err) { 2928 dev_err(dev, "Failed to register aeads\n"); 2929 goto unregister_algs; 2930 } 2931 2932 return 0; 2933 2934 unregister_algs: 2935 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 2936 unregister_ahashes: 2937 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 2938 disable_hw: 2939 artpec6_crypto_disable_hw(ac); 2940 free_cache: 2941 kmem_cache_destroy(ac->dma_cache); 2942 return err; 2943 } 2944 2945 static void artpec6_crypto_remove(struct platform_device *pdev) 2946 { 2947 struct artpec6_crypto *ac = platform_get_drvdata(pdev); 2948 int irq = platform_get_irq(pdev, 0); 2949 2950 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 2951 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 2952 crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos)); 2953 2954 tasklet_disable(&ac->task); 2955 devm_free_irq(&pdev->dev, irq, ac); 2956 tasklet_kill(&ac->task); 2957 timer_delete_sync(&ac->timer); 2958 2959 artpec6_crypto_disable_hw(ac); 2960 2961 kmem_cache_destroy(ac->dma_cache); 2962 #ifdef CONFIG_DEBUG_FS 2963 artpec6_crypto_free_debugfs(); 2964 #endif 2965 } 2966 2967 static struct platform_driver artpec6_crypto_driver = { 2968 .probe = artpec6_crypto_probe, 2969 .remove = artpec6_crypto_remove, 2970 .driver = { 2971 .name = "artpec6-crypto", 2972 .of_match_table = artpec6_crypto_of_match, 2973 }, 2974 }; 2975 2976 module_platform_driver(artpec6_crypto_driver); 2977 2978 MODULE_AUTHOR("Axis Communications AB"); 2979 MODULE_DESCRIPTION("ARTPEC-6 Crypto driver"); 2980 MODULE_LICENSE("GPL"); 2981