1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api. 4 * 5 * Copyright (C) 2014-2017 Axis Communications AB 6 */ 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/bitfield.h> 10 #include <linux/crypto.h> 11 #include <linux/debugfs.h> 12 #include <linux/delay.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/fault-inject.h> 15 #include <linux/init.h> 16 #include <linux/interrupt.h> 17 #include <linux/kernel.h> 18 #include <linux/list.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/platform_device.h> 22 #include <linux/scatterlist.h> 23 #include <linux/slab.h> 24 25 #include <crypto/aes.h> 26 #include <crypto/gcm.h> 27 #include <crypto/internal/aead.h> 28 #include <crypto/internal/hash.h> 29 #include <crypto/internal/skcipher.h> 30 #include <crypto/scatterwalk.h> 31 #include <crypto/sha1.h> 32 #include <crypto/sha2.h> 33 #include <crypto/xts.h> 34 35 /* Max length of a line in all cache levels for Artpec SoCs. */ 36 #define ARTPEC_CACHE_LINE_MAX 32 37 38 #define PDMA_OUT_CFG 0x0000 39 #define PDMA_OUT_BUF_CFG 0x0004 40 #define PDMA_OUT_CMD 0x0008 41 #define PDMA_OUT_DESCRQ_PUSH 0x0010 42 #define PDMA_OUT_DESCRQ_STAT 0x0014 43 44 #define A6_PDMA_IN_CFG 0x0028 45 #define A6_PDMA_IN_BUF_CFG 0x002c 46 #define A6_PDMA_IN_CMD 0x0030 47 #define A6_PDMA_IN_STATQ_PUSH 0x0038 48 #define A6_PDMA_IN_DESCRQ_PUSH 0x0044 49 #define A6_PDMA_IN_DESCRQ_STAT 0x0048 50 #define A6_PDMA_INTR_MASK 0x0068 51 #define A6_PDMA_ACK_INTR 0x006c 52 #define A6_PDMA_MASKED_INTR 0x0074 53 54 #define A7_PDMA_IN_CFG 0x002c 55 #define A7_PDMA_IN_BUF_CFG 0x0030 56 #define A7_PDMA_IN_CMD 0x0034 57 #define A7_PDMA_IN_STATQ_PUSH 0x003c 58 #define A7_PDMA_IN_DESCRQ_PUSH 0x0048 59 #define A7_PDMA_IN_DESCRQ_STAT 0x004C 60 #define A7_PDMA_INTR_MASK 0x006c 61 #define A7_PDMA_ACK_INTR 0x0070 62 #define A7_PDMA_MASKED_INTR 0x0078 63 64 #define PDMA_OUT_CFG_EN BIT(0) 65 66 #define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) 67 #define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) 68 69 #define PDMA_OUT_CMD_START BIT(0) 70 #define A6_PDMA_OUT_CMD_STOP BIT(3) 71 #define A7_PDMA_OUT_CMD_STOP BIT(2) 72 73 #define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0) 74 #define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6) 75 76 #define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0) 77 #define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4) 78 79 #define PDMA_IN_CFG_EN BIT(0) 80 81 #define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) 82 #define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) 83 #define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10) 84 85 #define PDMA_IN_CMD_START BIT(0) 86 #define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2) 87 #define A6_PDMA_IN_CMD_STOP BIT(3) 88 #define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1) 89 #define A7_PDMA_IN_CMD_STOP BIT(2) 90 91 #define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0) 92 #define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6) 93 94 #define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0) 95 #define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6) 96 97 #define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0) 98 #define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4) 99 100 #define A6_PDMA_INTR_MASK_IN_DATA BIT(2) 101 #define A6_PDMA_INTR_MASK_IN_EOP BIT(3) 102 #define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4) 103 104 #define A7_PDMA_INTR_MASK_IN_DATA BIT(3) 105 #define A7_PDMA_INTR_MASK_IN_EOP BIT(4) 106 #define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5) 107 108 #define A6_CRY_MD_OPER GENMASK(19, 16) 109 110 #define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20) 111 #define A6_CRY_MD_HASH_HMAC_FIN BIT(23) 112 113 #define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20) 114 #define A6_CRY_MD_CIPHER_DECR BIT(22) 115 #define A6_CRY_MD_CIPHER_TWEAK BIT(23) 116 #define A6_CRY_MD_CIPHER_DSEQ BIT(24) 117 118 #define A7_CRY_MD_OPER GENMASK(11, 8) 119 120 #define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12) 121 #define A7_CRY_MD_HASH_HMAC_FIN BIT(15) 122 123 #define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12) 124 #define A7_CRY_MD_CIPHER_DECR BIT(14) 125 #define A7_CRY_MD_CIPHER_TWEAK BIT(15) 126 #define A7_CRY_MD_CIPHER_DSEQ BIT(16) 127 128 /* DMA metadata constants */ 129 #define regk_crypto_aes_cbc 0x00000002 130 #define regk_crypto_aes_ctr 0x00000003 131 #define regk_crypto_aes_ecb 0x00000001 132 #define regk_crypto_aes_gcm 0x00000004 133 #define regk_crypto_aes_xts 0x00000005 134 #define regk_crypto_cache 0x00000002 135 #define a6_regk_crypto_dlkey 0x0000000a 136 #define a7_regk_crypto_dlkey 0x0000000e 137 #define regk_crypto_ext 0x00000001 138 #define regk_crypto_hmac_sha1 0x00000007 139 #define regk_crypto_hmac_sha256 0x00000009 140 #define regk_crypto_init 0x00000000 141 #define regk_crypto_key_128 0x00000000 142 #define regk_crypto_key_192 0x00000001 143 #define regk_crypto_key_256 0x00000002 144 #define regk_crypto_null 0x00000000 145 #define regk_crypto_sha1 0x00000006 146 #define regk_crypto_sha256 0x00000008 147 148 /* DMA descriptor structures */ 149 struct pdma_descr_ctrl { 150 unsigned char short_descr : 1; 151 unsigned char pad1 : 1; 152 unsigned char eop : 1; 153 unsigned char intr : 1; 154 unsigned char short_len : 3; 155 unsigned char pad2 : 1; 156 } __packed; 157 158 struct pdma_data_descr { 159 unsigned int len : 24; 160 unsigned int buf : 32; 161 } __packed; 162 163 struct pdma_short_descr { 164 unsigned char data[7]; 165 } __packed; 166 167 struct pdma_descr { 168 struct pdma_descr_ctrl ctrl; 169 union { 170 struct pdma_data_descr data; 171 struct pdma_short_descr shrt; 172 }; 173 }; 174 175 struct pdma_stat_descr { 176 unsigned char pad1 : 1; 177 unsigned char pad2 : 1; 178 unsigned char eop : 1; 179 unsigned char pad3 : 5; 180 unsigned int len : 24; 181 }; 182 183 /* Each descriptor array can hold max 64 entries */ 184 #define PDMA_DESCR_COUNT 64 185 186 #define MODULE_NAME "Artpec-6 CA" 187 188 /* Hash modes (including HMAC variants) */ 189 #define ARTPEC6_CRYPTO_HASH_SHA1 1 190 #define ARTPEC6_CRYPTO_HASH_SHA256 2 191 192 /* Crypto modes */ 193 #define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1 194 #define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2 195 #define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3 196 #define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5 197 198 /* The PDMA is a DMA-engine tightly coupled with a ciphering engine. 199 * It operates on a descriptor array with up to 64 descriptor entries. 200 * The arrays must be 64 byte aligned in memory. 201 * 202 * The ciphering unit has no registers and is completely controlled by 203 * a 4-byte metadata that is inserted at the beginning of each dma packet. 204 * 205 * A dma packet is a sequence of descriptors terminated by setting the .eop 206 * field in the final descriptor of the packet. 207 * 208 * Multiple packets are used for providing context data, key data and 209 * the plain/ciphertext. 210 * 211 * PDMA Descriptors (Array) 212 * +------+------+------+~~+-------+------+---- 213 * | 0 | 1 | 2 |~~| 11 EOP| 12 | .... 214 * +--+---+--+---+----+-+~~+-------+----+-+---- 215 * | | | | | 216 * | | | | | 217 * __|__ +-------++-------++-------+ +----+ 218 * | MD | |Payload||Payload||Payload| | MD | 219 * +-----+ +-------++-------++-------+ +----+ 220 */ 221 222 struct artpec6_crypto_bounce_buffer { 223 struct list_head list; 224 size_t length; 225 struct scatterlist *sg; 226 size_t offset; 227 /* buf is aligned to ARTPEC_CACHE_LINE_MAX and 228 * holds up to ARTPEC_CACHE_LINE_MAX bytes data. 229 */ 230 void *buf; 231 }; 232 233 struct artpec6_crypto_dma_map { 234 dma_addr_t dma_addr; 235 size_t size; 236 enum dma_data_direction dir; 237 }; 238 239 struct artpec6_crypto_dma_descriptors { 240 struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64); 241 struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64); 242 u32 stat[PDMA_DESCR_COUNT] __aligned(64); 243 struct list_head bounce_buffers; 244 /* Enough maps for all out/in buffers, and all three descr. arrays */ 245 struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2]; 246 dma_addr_t out_dma_addr; 247 dma_addr_t in_dma_addr; 248 dma_addr_t stat_dma_addr; 249 size_t out_cnt; 250 size_t in_cnt; 251 size_t map_count; 252 }; 253 254 enum artpec6_crypto_variant { 255 ARTPEC6_CRYPTO, 256 ARTPEC7_CRYPTO, 257 }; 258 259 struct artpec6_crypto { 260 void __iomem *base; 261 spinlock_t queue_lock; 262 struct list_head queue; /* waiting for pdma fifo space */ 263 struct list_head pending; /* submitted to pdma fifo */ 264 struct tasklet_struct task; 265 struct kmem_cache *dma_cache; 266 int pending_count; 267 struct timer_list timer; 268 enum artpec6_crypto_variant variant; 269 void *pad_buffer; /* cache-aligned block padding buffer */ 270 void *zero_buffer; 271 }; 272 273 enum artpec6_crypto_hash_flags { 274 HASH_FLAG_INIT_CTX = 2, 275 HASH_FLAG_UPDATE = 4, 276 HASH_FLAG_FINALIZE = 8, 277 HASH_FLAG_HMAC = 16, 278 HASH_FLAG_UPDATE_KEY = 32, 279 }; 280 281 struct artpec6_crypto_req_common { 282 struct list_head list; 283 struct list_head complete_in_progress; 284 struct artpec6_crypto_dma_descriptors *dma; 285 struct crypto_async_request *req; 286 void (*complete)(struct crypto_async_request *req); 287 gfp_t gfp_flags; 288 }; 289 290 struct artpec6_hash_request_context { 291 char partial_buffer[SHA256_BLOCK_SIZE]; 292 char partial_buffer_out[SHA256_BLOCK_SIZE]; 293 char key_buffer[SHA256_BLOCK_SIZE]; 294 char pad_buffer[SHA256_BLOCK_SIZE + 32]; 295 unsigned char digeststate[SHA256_DIGEST_SIZE]; 296 size_t partial_bytes; 297 u64 digcnt; 298 u32 key_md; 299 u32 hash_md; 300 enum artpec6_crypto_hash_flags hash_flags; 301 struct artpec6_crypto_req_common common; 302 }; 303 304 struct artpec6_hash_export_state { 305 char partial_buffer[SHA256_BLOCK_SIZE]; 306 unsigned char digeststate[SHA256_DIGEST_SIZE]; 307 size_t partial_bytes; 308 u64 digcnt; 309 int oper; 310 unsigned int hash_flags; 311 }; 312 313 struct artpec6_hashalg_context { 314 char hmac_key[SHA256_BLOCK_SIZE]; 315 size_t hmac_key_length; 316 struct crypto_shash *child_hash; 317 }; 318 319 struct artpec6_crypto_request_context { 320 u32 cipher_md; 321 bool decrypt; 322 struct artpec6_crypto_req_common common; 323 }; 324 325 struct artpec6_cryptotfm_context { 326 unsigned char aes_key[2*AES_MAX_KEY_SIZE]; 327 size_t key_length; 328 u32 key_md; 329 int crypto_type; 330 struct crypto_sync_skcipher *fallback; 331 }; 332 333 struct artpec6_crypto_aead_hw_ctx { 334 __be64 aad_length_bits; 335 __be64 text_length_bits; 336 __u8 J0[AES_BLOCK_SIZE]; 337 }; 338 339 struct artpec6_crypto_aead_req_ctx { 340 struct artpec6_crypto_aead_hw_ctx hw_ctx; 341 u32 cipher_md; 342 bool decrypt; 343 struct artpec6_crypto_req_common common; 344 __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned; 345 }; 346 347 /* The crypto framework makes it hard to avoid this global. */ 348 static struct device *artpec6_crypto_dev; 349 350 #ifdef CONFIG_FAULT_INJECTION 351 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read); 352 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full); 353 #endif 354 355 enum { 356 ARTPEC6_CRYPTO_PREPARE_HASH_NO_START, 357 ARTPEC6_CRYPTO_PREPARE_HASH_START, 358 }; 359 360 static int artpec6_crypto_prepare_aead(struct aead_request *areq); 361 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq); 362 static int artpec6_crypto_prepare_hash(struct ahash_request *areq); 363 364 static void 365 artpec6_crypto_complete_crypto(struct crypto_async_request *req); 366 static void 367 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req); 368 static void 369 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req); 370 static void 371 artpec6_crypto_complete_aead(struct crypto_async_request *req); 372 static void 373 artpec6_crypto_complete_hash(struct crypto_async_request *req); 374 375 static int 376 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common); 377 378 static void 379 artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common); 380 381 struct artpec6_crypto_walk { 382 struct scatterlist *sg; 383 size_t offset; 384 }; 385 386 static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk, 387 struct scatterlist *sg) 388 { 389 awalk->sg = sg; 390 awalk->offset = 0; 391 } 392 393 static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk, 394 size_t nbytes) 395 { 396 while (nbytes && awalk->sg) { 397 size_t piece; 398 399 WARN_ON(awalk->offset > awalk->sg->length); 400 401 piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset); 402 nbytes -= piece; 403 awalk->offset += piece; 404 if (awalk->offset == awalk->sg->length) { 405 awalk->sg = sg_next(awalk->sg); 406 awalk->offset = 0; 407 } 408 409 } 410 411 return nbytes; 412 } 413 414 static size_t 415 artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk) 416 { 417 WARN_ON(awalk->sg->length == awalk->offset); 418 419 return awalk->sg->length - awalk->offset; 420 } 421 422 static dma_addr_t 423 artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk) 424 { 425 return sg_phys(awalk->sg) + awalk->offset; 426 } 427 428 static void 429 artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common) 430 { 431 struct artpec6_crypto_dma_descriptors *dma = common->dma; 432 struct artpec6_crypto_bounce_buffer *b; 433 struct artpec6_crypto_bounce_buffer *next; 434 435 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { 436 pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n", 437 b, b->length, b->offset, b->buf); 438 sg_pcopy_from_buffer(b->sg, 439 1, 440 b->buf, 441 b->length, 442 b->offset); 443 444 list_del(&b->list); 445 kfree(b); 446 } 447 } 448 449 static inline bool artpec6_crypto_busy(void) 450 { 451 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 452 int fifo_count = ac->pending_count; 453 454 return fifo_count > 6; 455 } 456 457 static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req) 458 { 459 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 460 int ret = -EBUSY; 461 462 spin_lock_bh(&ac->queue_lock); 463 464 if (!artpec6_crypto_busy()) { 465 list_add_tail(&req->list, &ac->pending); 466 artpec6_crypto_start_dma(req); 467 ret = -EINPROGRESS; 468 } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { 469 list_add_tail(&req->list, &ac->queue); 470 } else { 471 artpec6_crypto_common_destroy(req); 472 } 473 474 spin_unlock_bh(&ac->queue_lock); 475 476 return ret; 477 } 478 479 static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common) 480 { 481 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 482 enum artpec6_crypto_variant variant = ac->variant; 483 void __iomem *base = ac->base; 484 struct artpec6_crypto_dma_descriptors *dma = common->dma; 485 u32 ind, statd, outd; 486 487 /* Make descriptor content visible to the DMA before starting it. */ 488 wmb(); 489 490 ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) | 491 FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6); 492 493 statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) | 494 FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6); 495 496 outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) | 497 FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6); 498 499 if (variant == ARTPEC6_CRYPTO) { 500 writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH); 501 writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH); 502 writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD); 503 } else { 504 writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH); 505 writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH); 506 writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD); 507 } 508 509 writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH); 510 writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD); 511 512 ac->pending_count++; 513 } 514 515 static void 516 artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common) 517 { 518 struct artpec6_crypto_dma_descriptors *dma = common->dma; 519 520 dma->out_cnt = 0; 521 dma->in_cnt = 0; 522 dma->map_count = 0; 523 INIT_LIST_HEAD(&dma->bounce_buffers); 524 } 525 526 static bool fault_inject_dma_descr(void) 527 { 528 #ifdef CONFIG_FAULT_INJECTION 529 return should_fail(&artpec6_crypto_fail_dma_array_full, 1); 530 #else 531 return false; 532 #endif 533 } 534 535 /** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a 536 * physical address 537 * 538 * @addr: The physical address of the data buffer 539 * @len: The length of the data buffer 540 * @eop: True if this is the last buffer in the packet 541 * 542 * @return 0 on success or -ENOSPC if there are no more descriptors available 543 */ 544 static int 545 artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common, 546 dma_addr_t addr, size_t len, bool eop) 547 { 548 struct artpec6_crypto_dma_descriptors *dma = common->dma; 549 struct pdma_descr *d; 550 551 if (dma->out_cnt >= PDMA_DESCR_COUNT || 552 fault_inject_dma_descr()) { 553 pr_err("No free OUT DMA descriptors available!\n"); 554 return -ENOSPC; 555 } 556 557 d = &dma->out[dma->out_cnt++]; 558 memset(d, 0, sizeof(*d)); 559 560 d->ctrl.short_descr = 0; 561 d->ctrl.eop = eop; 562 d->data.len = len; 563 d->data.buf = addr; 564 return 0; 565 } 566 567 /** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor 568 * 569 * @dst: The virtual address of the data 570 * @len: The length of the data, must be between 1 to 7 bytes 571 * @eop: True if this is the last buffer in the packet 572 * 573 * @return 0 on success 574 * -ENOSPC if no more descriptors are available 575 * -EINVAL if the data length exceeds 7 bytes 576 */ 577 static int 578 artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common, 579 void *dst, unsigned int len, bool eop) 580 { 581 struct artpec6_crypto_dma_descriptors *dma = common->dma; 582 struct pdma_descr *d; 583 584 if (dma->out_cnt >= PDMA_DESCR_COUNT || 585 fault_inject_dma_descr()) { 586 pr_err("No free OUT DMA descriptors available!\n"); 587 return -ENOSPC; 588 } else if (len > 7 || len < 1) { 589 return -EINVAL; 590 } 591 d = &dma->out[dma->out_cnt++]; 592 memset(d, 0, sizeof(*d)); 593 594 d->ctrl.short_descr = 1; 595 d->ctrl.short_len = len; 596 d->ctrl.eop = eop; 597 memcpy(d->shrt.data, dst, len); 598 return 0; 599 } 600 601 static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common, 602 struct page *page, size_t offset, 603 size_t size, 604 enum dma_data_direction dir, 605 dma_addr_t *dma_addr_out) 606 { 607 struct artpec6_crypto_dma_descriptors *dma = common->dma; 608 struct device *dev = artpec6_crypto_dev; 609 struct artpec6_crypto_dma_map *map; 610 dma_addr_t dma_addr; 611 612 *dma_addr_out = 0; 613 614 if (dma->map_count >= ARRAY_SIZE(dma->maps)) 615 return -ENOMEM; 616 617 dma_addr = dma_map_page(dev, page, offset, size, dir); 618 if (dma_mapping_error(dev, dma_addr)) 619 return -ENOMEM; 620 621 map = &dma->maps[dma->map_count++]; 622 map->size = size; 623 map->dma_addr = dma_addr; 624 map->dir = dir; 625 626 *dma_addr_out = dma_addr; 627 628 return 0; 629 } 630 631 static int 632 artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common, 633 void *ptr, size_t size, 634 enum dma_data_direction dir, 635 dma_addr_t *dma_addr_out) 636 { 637 struct page *page = virt_to_page(ptr); 638 size_t offset = (uintptr_t)ptr & ~PAGE_MASK; 639 640 return artpec6_crypto_dma_map_page(common, page, offset, size, dir, 641 dma_addr_out); 642 } 643 644 static int 645 artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common) 646 { 647 struct artpec6_crypto_dma_descriptors *dma = common->dma; 648 int ret; 649 650 ret = artpec6_crypto_dma_map_single(common, dma->in, 651 sizeof(dma->in[0]) * dma->in_cnt, 652 DMA_TO_DEVICE, &dma->in_dma_addr); 653 if (ret) 654 return ret; 655 656 ret = artpec6_crypto_dma_map_single(common, dma->out, 657 sizeof(dma->out[0]) * dma->out_cnt, 658 DMA_TO_DEVICE, &dma->out_dma_addr); 659 if (ret) 660 return ret; 661 662 /* We only read one stat descriptor */ 663 dma->stat[dma->in_cnt - 1] = 0; 664 665 /* 666 * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor 667 * to be written. 668 */ 669 return artpec6_crypto_dma_map_single(common, 670 dma->stat, 671 sizeof(dma->stat[0]) * dma->in_cnt, 672 DMA_BIDIRECTIONAL, 673 &dma->stat_dma_addr); 674 } 675 676 static void 677 artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common) 678 { 679 struct artpec6_crypto_dma_descriptors *dma = common->dma; 680 struct device *dev = artpec6_crypto_dev; 681 int i; 682 683 for (i = 0; i < dma->map_count; i++) { 684 struct artpec6_crypto_dma_map *map = &dma->maps[i]; 685 686 dma_unmap_page(dev, map->dma_addr, map->size, map->dir); 687 } 688 689 dma->map_count = 0; 690 } 691 692 /** artpec6_crypto_setup_out_descr - Setup an out descriptor 693 * 694 * @dst: The virtual address of the data 695 * @len: The length of the data 696 * @eop: True if this is the last buffer in the packet 697 * @use_short: If this is true and the data length is 7 bytes or less then 698 * a short descriptor will be used 699 * 700 * @return 0 on success 701 * Any errors from artpec6_crypto_setup_out_descr_short() or 702 * setup_out_descr_phys() 703 */ 704 static int 705 artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common, 706 void *dst, unsigned int len, bool eop, 707 bool use_short) 708 { 709 if (use_short && len < 7) { 710 return artpec6_crypto_setup_out_descr_short(common, dst, len, 711 eop); 712 } else { 713 int ret; 714 dma_addr_t dma_addr; 715 716 ret = artpec6_crypto_dma_map_single(common, dst, len, 717 DMA_TO_DEVICE, 718 &dma_addr); 719 if (ret) 720 return ret; 721 722 return artpec6_crypto_setup_out_descr_phys(common, dma_addr, 723 len, eop); 724 } 725 } 726 727 /** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a 728 * physical address 729 * 730 * @addr: The physical address of the data buffer 731 * @len: The length of the data buffer 732 * @intr: True if an interrupt should be fired after HW processing of this 733 * descriptor 734 * 735 */ 736 static int 737 artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common, 738 dma_addr_t addr, unsigned int len, bool intr) 739 { 740 struct artpec6_crypto_dma_descriptors *dma = common->dma; 741 struct pdma_descr *d; 742 743 if (dma->in_cnt >= PDMA_DESCR_COUNT || 744 fault_inject_dma_descr()) { 745 pr_err("No free IN DMA descriptors available!\n"); 746 return -ENOSPC; 747 } 748 d = &dma->in[dma->in_cnt++]; 749 memset(d, 0, sizeof(*d)); 750 751 d->ctrl.intr = intr; 752 d->data.len = len; 753 d->data.buf = addr; 754 return 0; 755 } 756 757 /** artpec6_crypto_setup_in_descr - Setup an in channel descriptor 758 * 759 * @buffer: The virtual address to of the data buffer 760 * @len: The length of the data buffer 761 * @last: If this is the last data buffer in the request (i.e. an interrupt 762 * is needed 763 * 764 * Short descriptors are not used for the in channel 765 */ 766 static int 767 artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common, 768 void *buffer, unsigned int len, bool last) 769 { 770 dma_addr_t dma_addr; 771 int ret; 772 773 ret = artpec6_crypto_dma_map_single(common, buffer, len, 774 DMA_FROM_DEVICE, &dma_addr); 775 if (ret) 776 return ret; 777 778 return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last); 779 } 780 781 static struct artpec6_crypto_bounce_buffer * 782 artpec6_crypto_alloc_bounce(gfp_t flags) 783 { 784 void *base; 785 size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) + 786 2 * ARTPEC_CACHE_LINE_MAX; 787 struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags); 788 789 if (!bbuf) 790 return NULL; 791 792 base = bbuf + 1; 793 bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX); 794 return bbuf; 795 } 796 797 static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common, 798 struct artpec6_crypto_walk *walk, size_t size) 799 { 800 struct artpec6_crypto_bounce_buffer *bbuf; 801 int ret; 802 803 bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags); 804 if (!bbuf) 805 return -ENOMEM; 806 807 bbuf->length = size; 808 bbuf->sg = walk->sg; 809 bbuf->offset = walk->offset; 810 811 ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false); 812 if (ret) { 813 kfree(bbuf); 814 return ret; 815 } 816 817 pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset); 818 list_add_tail(&bbuf->list, &common->dma->bounce_buffers); 819 return 0; 820 } 821 822 static int 823 artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common, 824 struct artpec6_crypto_walk *walk, 825 size_t count) 826 { 827 size_t chunk; 828 int ret; 829 dma_addr_t addr; 830 831 while (walk->sg && count) { 832 chunk = min(count, artpec6_crypto_walk_chunklen(walk)); 833 addr = artpec6_crypto_walk_chunk_phys(walk); 834 835 /* When destination buffers are not aligned to the cache line 836 * size we need bounce buffers. The DMA-API requires that the 837 * entire line is owned by the DMA buffer and this holds also 838 * for the case when coherent DMA is used. 839 */ 840 if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) { 841 chunk = min_t(dma_addr_t, chunk, 842 ALIGN(addr, ARTPEC_CACHE_LINE_MAX) - 843 addr); 844 845 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); 846 ret = setup_bounce_buffer_in(common, walk, chunk); 847 } else if (chunk < ARTPEC_CACHE_LINE_MAX) { 848 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); 849 ret = setup_bounce_buffer_in(common, walk, chunk); 850 } else { 851 dma_addr_t dma_addr; 852 853 chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1); 854 855 pr_debug("CHUNK %pad:%zu\n", &addr, chunk); 856 857 ret = artpec6_crypto_dma_map_page(common, 858 sg_page(walk->sg), 859 walk->sg->offset + 860 walk->offset, 861 chunk, 862 DMA_FROM_DEVICE, 863 &dma_addr); 864 if (ret) 865 return ret; 866 867 ret = artpec6_crypto_setup_in_descr_phys(common, 868 dma_addr, 869 chunk, false); 870 } 871 872 if (ret) 873 return ret; 874 875 count = count - chunk; 876 artpec6_crypto_walk_advance(walk, chunk); 877 } 878 879 if (count) 880 pr_err("EOL unexpected %zu bytes left\n", count); 881 882 return count ? -EINVAL : 0; 883 } 884 885 static int 886 artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common, 887 struct artpec6_crypto_walk *walk, 888 size_t count) 889 { 890 size_t chunk; 891 int ret; 892 dma_addr_t addr; 893 894 while (walk->sg && count) { 895 chunk = min(count, artpec6_crypto_walk_chunklen(walk)); 896 addr = artpec6_crypto_walk_chunk_phys(walk); 897 898 pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk); 899 900 if (addr & 3) { 901 char buf[3]; 902 903 chunk = min_t(size_t, chunk, (4-(addr&3))); 904 905 sg_pcopy_to_buffer(walk->sg, 1, buf, chunk, 906 walk->offset); 907 908 ret = artpec6_crypto_setup_out_descr_short(common, buf, 909 chunk, 910 false); 911 } else { 912 dma_addr_t dma_addr; 913 914 ret = artpec6_crypto_dma_map_page(common, 915 sg_page(walk->sg), 916 walk->sg->offset + 917 walk->offset, 918 chunk, 919 DMA_TO_DEVICE, 920 &dma_addr); 921 if (ret) 922 return ret; 923 924 ret = artpec6_crypto_setup_out_descr_phys(common, 925 dma_addr, 926 chunk, false); 927 } 928 929 if (ret) 930 return ret; 931 932 count = count - chunk; 933 artpec6_crypto_walk_advance(walk, chunk); 934 } 935 936 if (count) 937 pr_err("EOL unexpected %zu bytes left\n", count); 938 939 return count ? -EINVAL : 0; 940 } 941 942 943 /** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor 944 * 945 * If the out descriptor list is non-empty, then the eop flag on the 946 * last used out descriptor will be set. 947 * 948 * @return 0 on success 949 * -EINVAL if the out descriptor is empty or has overflown 950 */ 951 static int 952 artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common) 953 { 954 struct artpec6_crypto_dma_descriptors *dma = common->dma; 955 struct pdma_descr *d; 956 957 if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) { 958 pr_err("%s: OUT descriptor list is %s\n", 959 MODULE_NAME, dma->out_cnt ? "empty" : "full"); 960 return -EINVAL; 961 962 } 963 964 d = &dma->out[dma->out_cnt-1]; 965 d->ctrl.eop = 1; 966 967 return 0; 968 } 969 970 /** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last 971 * in descriptor 972 * 973 * See artpec6_crypto_terminate_out_descrs() for return values 974 */ 975 static int 976 artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common) 977 { 978 struct artpec6_crypto_dma_descriptors *dma = common->dma; 979 struct pdma_descr *d; 980 981 if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) { 982 pr_err("%s: IN descriptor list is %s\n", 983 MODULE_NAME, dma->in_cnt ? "empty" : "full"); 984 return -EINVAL; 985 } 986 987 d = &dma->in[dma->in_cnt-1]; 988 d->ctrl.intr = 1; 989 return 0; 990 } 991 992 /** create_hash_pad - Create a Secure Hash conformant pad 993 * 994 * @dst: The destination buffer to write the pad. Must be at least 64 bytes 995 * @dgstlen: The total length of the hash digest in bytes 996 * @bitcount: The total length of the digest in bits 997 * 998 * @return The total number of padding bytes written to @dst 999 */ 1000 static size_t 1001 create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount) 1002 { 1003 unsigned int mod, target, diff, pad_bytes, size_bytes; 1004 __be64 bits = __cpu_to_be64(bitcount); 1005 1006 switch (oper) { 1007 case regk_crypto_sha1: 1008 case regk_crypto_sha256: 1009 case regk_crypto_hmac_sha1: 1010 case regk_crypto_hmac_sha256: 1011 target = 448 / 8; 1012 mod = 512 / 8; 1013 size_bytes = 8; 1014 break; 1015 default: 1016 target = 896 / 8; 1017 mod = 1024 / 8; 1018 size_bytes = 16; 1019 break; 1020 } 1021 1022 target -= 1; 1023 diff = dgstlen & (mod - 1); 1024 pad_bytes = diff > target ? target + mod - diff : target - diff; 1025 1026 memset(dst + 1, 0, pad_bytes); 1027 dst[0] = 0x80; 1028 1029 if (size_bytes == 16) { 1030 memset(dst + 1 + pad_bytes, 0, 8); 1031 memcpy(dst + 1 + pad_bytes + 8, &bits, 8); 1032 } else { 1033 memcpy(dst + 1 + pad_bytes, &bits, 8); 1034 } 1035 1036 return pad_bytes + size_bytes + 1; 1037 } 1038 1039 static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common, 1040 struct crypto_async_request *parent, 1041 void (*complete)(struct crypto_async_request *req), 1042 struct scatterlist *dstsg, unsigned int nbytes) 1043 { 1044 gfp_t flags; 1045 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1046 1047 flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1048 GFP_KERNEL : GFP_ATOMIC; 1049 1050 common->gfp_flags = flags; 1051 common->dma = kmem_cache_alloc(ac->dma_cache, flags); 1052 if (!common->dma) 1053 return -ENOMEM; 1054 1055 common->req = parent; 1056 common->complete = complete; 1057 return 0; 1058 } 1059 1060 static void 1061 artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma) 1062 { 1063 struct artpec6_crypto_bounce_buffer *b; 1064 struct artpec6_crypto_bounce_buffer *next; 1065 1066 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { 1067 kfree(b); 1068 } 1069 } 1070 1071 static int 1072 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common) 1073 { 1074 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1075 1076 artpec6_crypto_dma_unmap_all(common); 1077 artpec6_crypto_bounce_destroy(common->dma); 1078 kmem_cache_free(ac->dma_cache, common->dma); 1079 common->dma = NULL; 1080 return 0; 1081 } 1082 1083 /* 1084 * Ciphering functions. 1085 */ 1086 static int artpec6_crypto_encrypt(struct skcipher_request *req) 1087 { 1088 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1089 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1090 struct artpec6_crypto_request_context *req_ctx = NULL; 1091 void (*complete)(struct crypto_async_request *req); 1092 int ret; 1093 1094 req_ctx = skcipher_request_ctx(req); 1095 1096 switch (ctx->crypto_type) { 1097 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1098 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1099 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1100 req_ctx->decrypt = 0; 1101 break; 1102 default: 1103 break; 1104 } 1105 1106 switch (ctx->crypto_type) { 1107 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1108 complete = artpec6_crypto_complete_cbc_encrypt; 1109 break; 1110 default: 1111 complete = artpec6_crypto_complete_crypto; 1112 break; 1113 } 1114 1115 ret = artpec6_crypto_common_init(&req_ctx->common, 1116 &req->base, 1117 complete, 1118 req->dst, req->cryptlen); 1119 if (ret) 1120 return ret; 1121 1122 ret = artpec6_crypto_prepare_crypto(req); 1123 if (ret) { 1124 artpec6_crypto_common_destroy(&req_ctx->common); 1125 return ret; 1126 } 1127 1128 return artpec6_crypto_submit(&req_ctx->common); 1129 } 1130 1131 static int artpec6_crypto_decrypt(struct skcipher_request *req) 1132 { 1133 int ret; 1134 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1135 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1136 struct artpec6_crypto_request_context *req_ctx = NULL; 1137 void (*complete)(struct crypto_async_request *req); 1138 1139 req_ctx = skcipher_request_ctx(req); 1140 1141 switch (ctx->crypto_type) { 1142 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1143 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1144 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1145 req_ctx->decrypt = 1; 1146 break; 1147 default: 1148 break; 1149 } 1150 1151 1152 switch (ctx->crypto_type) { 1153 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1154 complete = artpec6_crypto_complete_cbc_decrypt; 1155 break; 1156 default: 1157 complete = artpec6_crypto_complete_crypto; 1158 break; 1159 } 1160 1161 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, 1162 complete, 1163 req->dst, req->cryptlen); 1164 if (ret) 1165 return ret; 1166 1167 ret = artpec6_crypto_prepare_crypto(req); 1168 if (ret) { 1169 artpec6_crypto_common_destroy(&req_ctx->common); 1170 return ret; 1171 } 1172 1173 return artpec6_crypto_submit(&req_ctx->common); 1174 } 1175 1176 static int 1177 artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt) 1178 { 1179 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1180 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1181 size_t iv_len = crypto_skcipher_ivsize(cipher); 1182 unsigned int counter = be32_to_cpup((__be32 *) 1183 (req->iv + iv_len - 4)); 1184 unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / 1185 AES_BLOCK_SIZE; 1186 1187 /* 1188 * The hardware uses only the last 32-bits as the counter while the 1189 * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that 1190 * the whole IV is a counter. So fallback if the counter is going to 1191 * overlow. 1192 */ 1193 if (counter + nblks < counter) { 1194 int ret; 1195 1196 pr_debug("counter %x will overflow (nblks %u), falling back\n", 1197 counter, counter + nblks); 1198 1199 ret = crypto_sync_skcipher_setkey(ctx->fallback, ctx->aes_key, 1200 ctx->key_length); 1201 if (ret) 1202 return ret; 1203 1204 { 1205 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); 1206 1207 skcipher_request_set_sync_tfm(subreq, ctx->fallback); 1208 skcipher_request_set_callback(subreq, req->base.flags, 1209 NULL, NULL); 1210 skcipher_request_set_crypt(subreq, req->src, req->dst, 1211 req->cryptlen, req->iv); 1212 ret = encrypt ? crypto_skcipher_encrypt(subreq) 1213 : crypto_skcipher_decrypt(subreq); 1214 skcipher_request_zero(subreq); 1215 } 1216 return ret; 1217 } 1218 1219 return encrypt ? artpec6_crypto_encrypt(req) 1220 : artpec6_crypto_decrypt(req); 1221 } 1222 1223 static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req) 1224 { 1225 return artpec6_crypto_ctr_crypt(req, true); 1226 } 1227 1228 static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req) 1229 { 1230 return artpec6_crypto_ctr_crypt(req, false); 1231 } 1232 1233 /* 1234 * AEAD functions 1235 */ 1236 static int artpec6_crypto_aead_init(struct crypto_aead *tfm) 1237 { 1238 struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm); 1239 1240 memset(tfm_ctx, 0, sizeof(*tfm_ctx)); 1241 1242 crypto_aead_set_reqsize(tfm, 1243 sizeof(struct artpec6_crypto_aead_req_ctx)); 1244 1245 return 0; 1246 } 1247 1248 static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key, 1249 unsigned int len) 1250 { 1251 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base); 1252 1253 if (len != 16 && len != 24 && len != 32) 1254 return -EINVAL; 1255 1256 ctx->key_length = len; 1257 1258 memcpy(ctx->aes_key, key, len); 1259 return 0; 1260 } 1261 1262 static int artpec6_crypto_aead_encrypt(struct aead_request *req) 1263 { 1264 int ret; 1265 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); 1266 1267 req_ctx->decrypt = false; 1268 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, 1269 artpec6_crypto_complete_aead, 1270 NULL, 0); 1271 if (ret) 1272 return ret; 1273 1274 ret = artpec6_crypto_prepare_aead(req); 1275 if (ret) { 1276 artpec6_crypto_common_destroy(&req_ctx->common); 1277 return ret; 1278 } 1279 1280 return artpec6_crypto_submit(&req_ctx->common); 1281 } 1282 1283 static int artpec6_crypto_aead_decrypt(struct aead_request *req) 1284 { 1285 int ret; 1286 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); 1287 1288 req_ctx->decrypt = true; 1289 if (req->cryptlen < AES_BLOCK_SIZE) 1290 return -EINVAL; 1291 1292 ret = artpec6_crypto_common_init(&req_ctx->common, 1293 &req->base, 1294 artpec6_crypto_complete_aead, 1295 NULL, 0); 1296 if (ret) 1297 return ret; 1298 1299 ret = artpec6_crypto_prepare_aead(req); 1300 if (ret) { 1301 artpec6_crypto_common_destroy(&req_ctx->common); 1302 return ret; 1303 } 1304 1305 return artpec6_crypto_submit(&req_ctx->common); 1306 } 1307 1308 static int artpec6_crypto_prepare_hash(struct ahash_request *areq) 1309 { 1310 struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm); 1311 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq); 1312 size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); 1313 size_t contextsize = digestsize; 1314 size_t blocksize = crypto_tfm_alg_blocksize( 1315 crypto_ahash_tfm(crypto_ahash_reqtfm(areq))); 1316 struct artpec6_crypto_req_common *common = &req_ctx->common; 1317 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1318 enum artpec6_crypto_variant variant = ac->variant; 1319 u32 sel_ctx; 1320 bool ext_ctx = false; 1321 bool run_hw = false; 1322 int error = 0; 1323 1324 artpec6_crypto_init_dma_operation(common); 1325 1326 /* Upload HMAC key, must be first the first packet */ 1327 if (req_ctx->hash_flags & HASH_FLAG_HMAC) { 1328 if (variant == ARTPEC6_CRYPTO) { 1329 req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, 1330 a6_regk_crypto_dlkey); 1331 } else { 1332 req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, 1333 a7_regk_crypto_dlkey); 1334 } 1335 1336 /* Copy and pad up the key */ 1337 memcpy(req_ctx->key_buffer, ctx->hmac_key, 1338 ctx->hmac_key_length); 1339 memset(req_ctx->key_buffer + ctx->hmac_key_length, 0, 1340 blocksize - ctx->hmac_key_length); 1341 1342 error = artpec6_crypto_setup_out_descr(common, 1343 (void *)&req_ctx->key_md, 1344 sizeof(req_ctx->key_md), false, false); 1345 if (error) 1346 return error; 1347 1348 error = artpec6_crypto_setup_out_descr(common, 1349 req_ctx->key_buffer, blocksize, 1350 true, false); 1351 if (error) 1352 return error; 1353 } 1354 1355 if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) { 1356 /* Restore context */ 1357 sel_ctx = regk_crypto_ext; 1358 ext_ctx = true; 1359 } else { 1360 sel_ctx = regk_crypto_init; 1361 } 1362 1363 if (variant == ARTPEC6_CRYPTO) { 1364 req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX; 1365 req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx); 1366 1367 /* If this is the final round, set the final flag */ 1368 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) 1369 req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN; 1370 } else { 1371 req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX; 1372 req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx); 1373 1374 /* If this is the final round, set the final flag */ 1375 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) 1376 req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN; 1377 } 1378 1379 /* Setup up metadata descriptors */ 1380 error = artpec6_crypto_setup_out_descr(common, 1381 (void *)&req_ctx->hash_md, 1382 sizeof(req_ctx->hash_md), false, false); 1383 if (error) 1384 return error; 1385 1386 error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1387 if (error) 1388 return error; 1389 1390 if (ext_ctx) { 1391 error = artpec6_crypto_setup_out_descr(common, 1392 req_ctx->digeststate, 1393 contextsize, false, false); 1394 1395 if (error) 1396 return error; 1397 } 1398 1399 if (req_ctx->hash_flags & HASH_FLAG_UPDATE) { 1400 size_t done_bytes = 0; 1401 size_t total_bytes = areq->nbytes + req_ctx->partial_bytes; 1402 size_t ready_bytes = round_down(total_bytes, blocksize); 1403 struct artpec6_crypto_walk walk; 1404 1405 run_hw = ready_bytes > 0; 1406 if (req_ctx->partial_bytes && ready_bytes) { 1407 /* We have a partial buffer and will at least some bytes 1408 * to the HW. Empty this partial buffer before tackling 1409 * the SG lists 1410 */ 1411 memcpy(req_ctx->partial_buffer_out, 1412 req_ctx->partial_buffer, 1413 req_ctx->partial_bytes); 1414 1415 error = artpec6_crypto_setup_out_descr(common, 1416 req_ctx->partial_buffer_out, 1417 req_ctx->partial_bytes, 1418 false, true); 1419 if (error) 1420 return error; 1421 1422 /* Reset partial buffer */ 1423 done_bytes += req_ctx->partial_bytes; 1424 req_ctx->partial_bytes = 0; 1425 } 1426 1427 artpec6_crypto_walk_init(&walk, areq->src); 1428 1429 error = artpec6_crypto_setup_sg_descrs_out(common, &walk, 1430 ready_bytes - 1431 done_bytes); 1432 if (error) 1433 return error; 1434 1435 if (walk.sg) { 1436 size_t sg_skip = ready_bytes - done_bytes; 1437 size_t sg_rem = areq->nbytes - sg_skip; 1438 1439 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 1440 req_ctx->partial_buffer + 1441 req_ctx->partial_bytes, 1442 sg_rem, sg_skip); 1443 1444 req_ctx->partial_bytes += sg_rem; 1445 } 1446 1447 req_ctx->digcnt += ready_bytes; 1448 req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE); 1449 } 1450 1451 /* Finalize */ 1452 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) { 1453 size_t hash_pad_len; 1454 u64 digest_bits; 1455 u32 oper; 1456 1457 if (variant == ARTPEC6_CRYPTO) 1458 oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md); 1459 else 1460 oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md); 1461 1462 /* Write out the partial buffer if present */ 1463 if (req_ctx->partial_bytes) { 1464 memcpy(req_ctx->partial_buffer_out, 1465 req_ctx->partial_buffer, 1466 req_ctx->partial_bytes); 1467 error = artpec6_crypto_setup_out_descr(common, 1468 req_ctx->partial_buffer_out, 1469 req_ctx->partial_bytes, 1470 false, true); 1471 if (error) 1472 return error; 1473 1474 req_ctx->digcnt += req_ctx->partial_bytes; 1475 req_ctx->partial_bytes = 0; 1476 } 1477 1478 if (req_ctx->hash_flags & HASH_FLAG_HMAC) 1479 digest_bits = 8 * (req_ctx->digcnt + blocksize); 1480 else 1481 digest_bits = 8 * req_ctx->digcnt; 1482 1483 /* Add the hash pad */ 1484 hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer, 1485 req_ctx->digcnt, digest_bits); 1486 error = artpec6_crypto_setup_out_descr(common, 1487 req_ctx->pad_buffer, 1488 hash_pad_len, false, 1489 true); 1490 req_ctx->digcnt = 0; 1491 1492 if (error) 1493 return error; 1494 1495 /* Descriptor for the final result */ 1496 error = artpec6_crypto_setup_in_descr(common, areq->result, 1497 digestsize, 1498 true); 1499 if (error) 1500 return error; 1501 1502 } else { /* This is not the final operation for this request */ 1503 if (!run_hw) 1504 return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START; 1505 1506 /* Save the result to the context */ 1507 error = artpec6_crypto_setup_in_descr(common, 1508 req_ctx->digeststate, 1509 contextsize, false); 1510 if (error) 1511 return error; 1512 /* fall through */ 1513 } 1514 1515 req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE | 1516 HASH_FLAG_FINALIZE); 1517 1518 error = artpec6_crypto_terminate_in_descrs(common); 1519 if (error) 1520 return error; 1521 1522 error = artpec6_crypto_terminate_out_descrs(common); 1523 if (error) 1524 return error; 1525 1526 error = artpec6_crypto_dma_map_descs(common); 1527 if (error) 1528 return error; 1529 1530 return ARTPEC6_CRYPTO_PREPARE_HASH_START; 1531 } 1532 1533 1534 static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm) 1535 { 1536 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1537 1538 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1539 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB; 1540 1541 return 0; 1542 } 1543 1544 static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm) 1545 { 1546 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1547 1548 ctx->fallback = 1549 crypto_alloc_sync_skcipher(crypto_tfm_alg_name(&tfm->base), 1550 0, CRYPTO_ALG_NEED_FALLBACK); 1551 if (IS_ERR(ctx->fallback)) 1552 return PTR_ERR(ctx->fallback); 1553 1554 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1555 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR; 1556 1557 return 0; 1558 } 1559 1560 static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm) 1561 { 1562 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1563 1564 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1565 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC; 1566 1567 return 0; 1568 } 1569 1570 static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm) 1571 { 1572 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1573 1574 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1575 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS; 1576 1577 return 0; 1578 } 1579 1580 static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm) 1581 { 1582 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1583 1584 memset(ctx, 0, sizeof(*ctx)); 1585 } 1586 1587 static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm) 1588 { 1589 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1590 1591 crypto_free_sync_skcipher(ctx->fallback); 1592 artpec6_crypto_aes_exit(tfm); 1593 } 1594 1595 static int 1596 artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key, 1597 unsigned int keylen) 1598 { 1599 struct artpec6_cryptotfm_context *ctx = 1600 crypto_skcipher_ctx(cipher); 1601 1602 switch (keylen) { 1603 case 16: 1604 case 24: 1605 case 32: 1606 break; 1607 default: 1608 return -EINVAL; 1609 } 1610 1611 memcpy(ctx->aes_key, key, keylen); 1612 ctx->key_length = keylen; 1613 return 0; 1614 } 1615 1616 static int 1617 artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key, 1618 unsigned int keylen) 1619 { 1620 struct artpec6_cryptotfm_context *ctx = 1621 crypto_skcipher_ctx(cipher); 1622 int ret; 1623 1624 ret = xts_check_key(&cipher->base, key, keylen); 1625 if (ret) 1626 return ret; 1627 1628 switch (keylen) { 1629 case 32: 1630 case 48: 1631 case 64: 1632 break; 1633 default: 1634 return -EINVAL; 1635 } 1636 1637 memcpy(ctx->aes_key, key, keylen); 1638 ctx->key_length = keylen; 1639 return 0; 1640 } 1641 1642 /** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request 1643 * 1644 * @req: The asynch request to process 1645 * 1646 * @return 0 if the dma job was successfully prepared 1647 * <0 on error 1648 * 1649 * This function sets up the PDMA descriptors for a block cipher request. 1650 * 1651 * The required padding is added for AES-CTR using a statically defined 1652 * buffer. 1653 * 1654 * The PDMA descriptor list will be as follows: 1655 * 1656 * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop> 1657 * IN: <CIPHER_MD><data_0>...[data_n]<intr> 1658 * 1659 */ 1660 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq) 1661 { 1662 int ret; 1663 struct artpec6_crypto_walk walk; 1664 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1665 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1666 struct artpec6_crypto_request_context *req_ctx = NULL; 1667 size_t iv_len = crypto_skcipher_ivsize(cipher); 1668 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1669 enum artpec6_crypto_variant variant = ac->variant; 1670 struct artpec6_crypto_req_common *common; 1671 bool cipher_decr = false; 1672 size_t cipher_klen; 1673 u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */ 1674 u32 oper; 1675 1676 req_ctx = skcipher_request_ctx(areq); 1677 common = &req_ctx->common; 1678 1679 artpec6_crypto_init_dma_operation(common); 1680 1681 if (variant == ARTPEC6_CRYPTO) 1682 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey); 1683 else 1684 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey); 1685 1686 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, 1687 sizeof(ctx->key_md), false, false); 1688 if (ret) 1689 return ret; 1690 1691 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, 1692 ctx->key_length, true, false); 1693 if (ret) 1694 return ret; 1695 1696 req_ctx->cipher_md = 0; 1697 1698 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) 1699 cipher_klen = ctx->key_length/2; 1700 else 1701 cipher_klen = ctx->key_length; 1702 1703 /* Metadata */ 1704 switch (cipher_klen) { 1705 case 16: 1706 cipher_len = regk_crypto_key_128; 1707 break; 1708 case 24: 1709 cipher_len = regk_crypto_key_192; 1710 break; 1711 case 32: 1712 cipher_len = regk_crypto_key_256; 1713 break; 1714 default: 1715 pr_err("%s: Invalid key length %zu!\n", 1716 MODULE_NAME, ctx->key_length); 1717 return -EINVAL; 1718 } 1719 1720 switch (ctx->crypto_type) { 1721 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1722 oper = regk_crypto_aes_ecb; 1723 cipher_decr = req_ctx->decrypt; 1724 break; 1725 1726 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1727 oper = regk_crypto_aes_cbc; 1728 cipher_decr = req_ctx->decrypt; 1729 break; 1730 1731 case ARTPEC6_CRYPTO_CIPHER_AES_CTR: 1732 oper = regk_crypto_aes_ctr; 1733 cipher_decr = false; 1734 break; 1735 1736 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1737 oper = regk_crypto_aes_xts; 1738 cipher_decr = req_ctx->decrypt; 1739 1740 if (variant == ARTPEC6_CRYPTO) 1741 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ; 1742 else 1743 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ; 1744 break; 1745 1746 default: 1747 pr_err("%s: Invalid cipher mode %d!\n", 1748 MODULE_NAME, ctx->crypto_type); 1749 return -EINVAL; 1750 } 1751 1752 if (variant == ARTPEC6_CRYPTO) { 1753 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper); 1754 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, 1755 cipher_len); 1756 if (cipher_decr) 1757 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; 1758 } else { 1759 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper); 1760 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, 1761 cipher_len); 1762 if (cipher_decr) 1763 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; 1764 } 1765 1766 ret = artpec6_crypto_setup_out_descr(common, 1767 &req_ctx->cipher_md, 1768 sizeof(req_ctx->cipher_md), 1769 false, false); 1770 if (ret) 1771 return ret; 1772 1773 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1774 if (ret) 1775 return ret; 1776 1777 if (iv_len) { 1778 ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len, 1779 false, false); 1780 if (ret) 1781 return ret; 1782 } 1783 /* Data out */ 1784 artpec6_crypto_walk_init(&walk, areq->src); 1785 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen); 1786 if (ret) 1787 return ret; 1788 1789 /* Data in */ 1790 artpec6_crypto_walk_init(&walk, areq->dst); 1791 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen); 1792 if (ret) 1793 return ret; 1794 1795 /* CTR-mode padding required by the HW. */ 1796 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR || 1797 ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) { 1798 size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) - 1799 areq->cryptlen; 1800 1801 if (pad) { 1802 ret = artpec6_crypto_setup_out_descr(common, 1803 ac->pad_buffer, 1804 pad, false, false); 1805 if (ret) 1806 return ret; 1807 1808 ret = artpec6_crypto_setup_in_descr(common, 1809 ac->pad_buffer, pad, 1810 false); 1811 if (ret) 1812 return ret; 1813 } 1814 } 1815 1816 ret = artpec6_crypto_terminate_out_descrs(common); 1817 if (ret) 1818 return ret; 1819 1820 ret = artpec6_crypto_terminate_in_descrs(common); 1821 if (ret) 1822 return ret; 1823 1824 return artpec6_crypto_dma_map_descs(common); 1825 } 1826 1827 static int artpec6_crypto_prepare_aead(struct aead_request *areq) 1828 { 1829 size_t count; 1830 int ret; 1831 size_t input_length; 1832 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm); 1833 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); 1834 struct crypto_aead *cipher = crypto_aead_reqtfm(areq); 1835 struct artpec6_crypto_req_common *common = &req_ctx->common; 1836 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1837 enum artpec6_crypto_variant variant = ac->variant; 1838 u32 md_cipher_len; 1839 1840 artpec6_crypto_init_dma_operation(common); 1841 1842 /* Key */ 1843 if (variant == ARTPEC6_CRYPTO) { 1844 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, 1845 a6_regk_crypto_dlkey); 1846 } else { 1847 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, 1848 a7_regk_crypto_dlkey); 1849 } 1850 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, 1851 sizeof(ctx->key_md), false, false); 1852 if (ret) 1853 return ret; 1854 1855 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, 1856 ctx->key_length, true, false); 1857 if (ret) 1858 return ret; 1859 1860 req_ctx->cipher_md = 0; 1861 1862 switch (ctx->key_length) { 1863 case 16: 1864 md_cipher_len = regk_crypto_key_128; 1865 break; 1866 case 24: 1867 md_cipher_len = regk_crypto_key_192; 1868 break; 1869 case 32: 1870 md_cipher_len = regk_crypto_key_256; 1871 break; 1872 default: 1873 return -EINVAL; 1874 } 1875 1876 if (variant == ARTPEC6_CRYPTO) { 1877 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, 1878 regk_crypto_aes_gcm); 1879 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, 1880 md_cipher_len); 1881 if (req_ctx->decrypt) 1882 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; 1883 } else { 1884 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, 1885 regk_crypto_aes_gcm); 1886 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, 1887 md_cipher_len); 1888 if (req_ctx->decrypt) 1889 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; 1890 } 1891 1892 ret = artpec6_crypto_setup_out_descr(common, 1893 (void *) &req_ctx->cipher_md, 1894 sizeof(req_ctx->cipher_md), false, 1895 false); 1896 if (ret) 1897 return ret; 1898 1899 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1900 if (ret) 1901 return ret; 1902 1903 /* For the decryption, cryptlen includes the tag. */ 1904 input_length = areq->cryptlen; 1905 if (req_ctx->decrypt) 1906 input_length -= crypto_aead_authsize(cipher); 1907 1908 /* Prepare the context buffer */ 1909 req_ctx->hw_ctx.aad_length_bits = 1910 __cpu_to_be64(8*areq->assoclen); 1911 1912 req_ctx->hw_ctx.text_length_bits = 1913 __cpu_to_be64(8*input_length); 1914 1915 memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher)); 1916 // The HW omits the initial increment of the counter field. 1917 memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4); 1918 1919 ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx, 1920 sizeof(struct artpec6_crypto_aead_hw_ctx), false, false); 1921 if (ret) 1922 return ret; 1923 1924 { 1925 struct artpec6_crypto_walk walk; 1926 1927 artpec6_crypto_walk_init(&walk, areq->src); 1928 1929 /* Associated data */ 1930 count = areq->assoclen; 1931 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); 1932 if (ret) 1933 return ret; 1934 1935 if (!IS_ALIGNED(areq->assoclen, 16)) { 1936 size_t assoc_pad = 16 - (areq->assoclen % 16); 1937 /* The HW mandates zero padding here */ 1938 ret = artpec6_crypto_setup_out_descr(common, 1939 ac->zero_buffer, 1940 assoc_pad, false, 1941 false); 1942 if (ret) 1943 return ret; 1944 } 1945 1946 /* Data to crypto */ 1947 count = input_length; 1948 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); 1949 if (ret) 1950 return ret; 1951 1952 if (!IS_ALIGNED(input_length, 16)) { 1953 size_t crypto_pad = 16 - (input_length % 16); 1954 /* The HW mandates zero padding here */ 1955 ret = artpec6_crypto_setup_out_descr(common, 1956 ac->zero_buffer, 1957 crypto_pad, 1958 false, 1959 false); 1960 if (ret) 1961 return ret; 1962 } 1963 } 1964 1965 /* Data from crypto */ 1966 { 1967 struct artpec6_crypto_walk walk; 1968 size_t output_len = areq->cryptlen; 1969 1970 if (req_ctx->decrypt) 1971 output_len -= crypto_aead_authsize(cipher); 1972 1973 artpec6_crypto_walk_init(&walk, areq->dst); 1974 1975 /* skip associated data in the output */ 1976 count = artpec6_crypto_walk_advance(&walk, areq->assoclen); 1977 if (count) 1978 return -EINVAL; 1979 1980 count = output_len; 1981 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count); 1982 if (ret) 1983 return ret; 1984 1985 /* Put padding between the cryptotext and the auth tag */ 1986 if (!IS_ALIGNED(output_len, 16)) { 1987 size_t crypto_pad = 16 - (output_len % 16); 1988 1989 ret = artpec6_crypto_setup_in_descr(common, 1990 ac->pad_buffer, 1991 crypto_pad, false); 1992 if (ret) 1993 return ret; 1994 } 1995 1996 /* The authentication tag shall follow immediately after 1997 * the output ciphertext. For decryption it is put in a context 1998 * buffer for later compare against the input tag. 1999 */ 2000 2001 if (req_ctx->decrypt) { 2002 ret = artpec6_crypto_setup_in_descr(common, 2003 req_ctx->decryption_tag, AES_BLOCK_SIZE, false); 2004 if (ret) 2005 return ret; 2006 2007 } else { 2008 /* For encryption the requested tag size may be smaller 2009 * than the hardware's generated tag. 2010 */ 2011 size_t authsize = crypto_aead_authsize(cipher); 2012 2013 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, 2014 authsize); 2015 if (ret) 2016 return ret; 2017 2018 if (authsize < AES_BLOCK_SIZE) { 2019 count = AES_BLOCK_SIZE - authsize; 2020 ret = artpec6_crypto_setup_in_descr(common, 2021 ac->pad_buffer, 2022 count, false); 2023 if (ret) 2024 return ret; 2025 } 2026 } 2027 2028 } 2029 2030 ret = artpec6_crypto_terminate_in_descrs(common); 2031 if (ret) 2032 return ret; 2033 2034 ret = artpec6_crypto_terminate_out_descrs(common); 2035 if (ret) 2036 return ret; 2037 2038 return artpec6_crypto_dma_map_descs(common); 2039 } 2040 2041 static void artpec6_crypto_process_queue(struct artpec6_crypto *ac, 2042 struct list_head *completions) 2043 { 2044 struct artpec6_crypto_req_common *req; 2045 2046 while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) { 2047 req = list_first_entry(&ac->queue, 2048 struct artpec6_crypto_req_common, 2049 list); 2050 list_move_tail(&req->list, &ac->pending); 2051 artpec6_crypto_start_dma(req); 2052 2053 list_add_tail(&req->complete_in_progress, completions); 2054 } 2055 2056 /* 2057 * In some cases, the hardware can raise an in_eop_flush interrupt 2058 * before actually updating the status, so we have an timer which will 2059 * recheck the status on timeout. Since the cases are expected to be 2060 * very rare, we use a relatively large timeout value. There should be 2061 * no noticeable negative effect if we timeout spuriously. 2062 */ 2063 if (ac->pending_count) 2064 mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100)); 2065 else 2066 del_timer(&ac->timer); 2067 } 2068 2069 static void artpec6_crypto_timeout(struct timer_list *t) 2070 { 2071 struct artpec6_crypto *ac = from_timer(ac, t, timer); 2072 2073 dev_info_ratelimited(artpec6_crypto_dev, "timeout\n"); 2074 2075 tasklet_schedule(&ac->task); 2076 } 2077 2078 static void artpec6_crypto_task(unsigned long data) 2079 { 2080 struct artpec6_crypto *ac = (struct artpec6_crypto *)data; 2081 struct artpec6_crypto_req_common *req; 2082 struct artpec6_crypto_req_common *n; 2083 struct list_head complete_done; 2084 struct list_head complete_in_progress; 2085 2086 INIT_LIST_HEAD(&complete_done); 2087 INIT_LIST_HEAD(&complete_in_progress); 2088 2089 if (list_empty(&ac->pending)) { 2090 pr_debug("Spurious IRQ\n"); 2091 return; 2092 } 2093 2094 spin_lock(&ac->queue_lock); 2095 2096 list_for_each_entry_safe(req, n, &ac->pending, list) { 2097 struct artpec6_crypto_dma_descriptors *dma = req->dma; 2098 u32 stat; 2099 dma_addr_t stataddr; 2100 2101 stataddr = dma->stat_dma_addr + 4 * (req->dma->in_cnt - 1); 2102 dma_sync_single_for_cpu(artpec6_crypto_dev, 2103 stataddr, 2104 4, 2105 DMA_BIDIRECTIONAL); 2106 2107 stat = req->dma->stat[req->dma->in_cnt-1]; 2108 2109 /* A non-zero final status descriptor indicates 2110 * this job has finished. 2111 */ 2112 pr_debug("Request %p status is %X\n", req, stat); 2113 if (!stat) 2114 break; 2115 2116 /* Allow testing of timeout handling with fault injection */ 2117 #ifdef CONFIG_FAULT_INJECTION 2118 if (should_fail(&artpec6_crypto_fail_status_read, 1)) 2119 continue; 2120 #endif 2121 2122 pr_debug("Completing request %p\n", req); 2123 2124 list_move_tail(&req->list, &complete_done); 2125 2126 ac->pending_count--; 2127 } 2128 2129 artpec6_crypto_process_queue(ac, &complete_in_progress); 2130 2131 spin_unlock(&ac->queue_lock); 2132 2133 /* Perform the completion callbacks without holding the queue lock 2134 * to allow new request submissions from the callbacks. 2135 */ 2136 list_for_each_entry_safe(req, n, &complete_done, list) { 2137 artpec6_crypto_dma_unmap_all(req); 2138 artpec6_crypto_copy_bounce_buffers(req); 2139 artpec6_crypto_common_destroy(req); 2140 2141 req->complete(req->req); 2142 } 2143 2144 list_for_each_entry_safe(req, n, &complete_in_progress, 2145 complete_in_progress) { 2146 req->req->complete(req->req, -EINPROGRESS); 2147 } 2148 } 2149 2150 static void artpec6_crypto_complete_crypto(struct crypto_async_request *req) 2151 { 2152 req->complete(req, 0); 2153 } 2154 2155 static void 2156 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req) 2157 { 2158 struct skcipher_request *cipher_req = container_of(req, 2159 struct skcipher_request, base); 2160 2161 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src, 2162 cipher_req->cryptlen - AES_BLOCK_SIZE, 2163 AES_BLOCK_SIZE, 0); 2164 req->complete(req, 0); 2165 } 2166 2167 static void 2168 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req) 2169 { 2170 struct skcipher_request *cipher_req = container_of(req, 2171 struct skcipher_request, base); 2172 2173 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst, 2174 cipher_req->cryptlen - AES_BLOCK_SIZE, 2175 AES_BLOCK_SIZE, 0); 2176 req->complete(req, 0); 2177 } 2178 2179 static void artpec6_crypto_complete_aead(struct crypto_async_request *req) 2180 { 2181 int result = 0; 2182 2183 /* Verify GCM hashtag. */ 2184 struct aead_request *areq = container_of(req, 2185 struct aead_request, base); 2186 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 2187 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); 2188 2189 if (req_ctx->decrypt) { 2190 u8 input_tag[AES_BLOCK_SIZE]; 2191 unsigned int authsize = crypto_aead_authsize(aead); 2192 2193 sg_pcopy_to_buffer(areq->src, 2194 sg_nents(areq->src), 2195 input_tag, 2196 authsize, 2197 areq->assoclen + areq->cryptlen - 2198 authsize); 2199 2200 if (crypto_memneq(req_ctx->decryption_tag, 2201 input_tag, 2202 authsize)) { 2203 pr_debug("***EBADMSG:\n"); 2204 print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1, 2205 input_tag, authsize, true); 2206 print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1, 2207 req_ctx->decryption_tag, 2208 authsize, true); 2209 2210 result = -EBADMSG; 2211 } 2212 } 2213 2214 req->complete(req, result); 2215 } 2216 2217 static void artpec6_crypto_complete_hash(struct crypto_async_request *req) 2218 { 2219 req->complete(req, 0); 2220 } 2221 2222 2223 /*------------------- Hash functions -----------------------------------------*/ 2224 static int 2225 artpec6_crypto_hash_set_key(struct crypto_ahash *tfm, 2226 const u8 *key, unsigned int keylen) 2227 { 2228 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base); 2229 size_t blocksize; 2230 int ret; 2231 2232 if (!keylen) { 2233 pr_err("Invalid length (%d) of HMAC key\n", 2234 keylen); 2235 return -EINVAL; 2236 } 2237 2238 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); 2239 2240 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2241 2242 if (keylen > blocksize) { 2243 tfm_ctx->hmac_key_length = blocksize; 2244 2245 ret = crypto_shash_tfm_digest(tfm_ctx->child_hash, key, keylen, 2246 tfm_ctx->hmac_key); 2247 if (ret) 2248 return ret; 2249 } else { 2250 memcpy(tfm_ctx->hmac_key, key, keylen); 2251 tfm_ctx->hmac_key_length = keylen; 2252 } 2253 2254 return 0; 2255 } 2256 2257 static int 2258 artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac) 2259 { 2260 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2261 enum artpec6_crypto_variant variant = ac->variant; 2262 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2263 u32 oper; 2264 2265 memset(req_ctx, 0, sizeof(*req_ctx)); 2266 2267 req_ctx->hash_flags = HASH_FLAG_INIT_CTX; 2268 if (hmac) 2269 req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY); 2270 2271 switch (type) { 2272 case ARTPEC6_CRYPTO_HASH_SHA1: 2273 oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1; 2274 break; 2275 case ARTPEC6_CRYPTO_HASH_SHA256: 2276 oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256; 2277 break; 2278 default: 2279 pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type); 2280 return -EINVAL; 2281 } 2282 2283 if (variant == ARTPEC6_CRYPTO) 2284 req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper); 2285 else 2286 req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper); 2287 2288 return 0; 2289 } 2290 2291 static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req) 2292 { 2293 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2294 int ret; 2295 2296 if (!req_ctx->common.dma) { 2297 ret = artpec6_crypto_common_init(&req_ctx->common, 2298 &req->base, 2299 artpec6_crypto_complete_hash, 2300 NULL, 0); 2301 2302 if (ret) 2303 return ret; 2304 } 2305 2306 ret = artpec6_crypto_prepare_hash(req); 2307 switch (ret) { 2308 case ARTPEC6_CRYPTO_PREPARE_HASH_START: 2309 ret = artpec6_crypto_submit(&req_ctx->common); 2310 break; 2311 2312 case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START: 2313 ret = 0; 2314 fallthrough; 2315 2316 default: 2317 artpec6_crypto_common_destroy(&req_ctx->common); 2318 break; 2319 } 2320 2321 return ret; 2322 } 2323 2324 static int artpec6_crypto_hash_final(struct ahash_request *req) 2325 { 2326 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2327 2328 req_ctx->hash_flags |= HASH_FLAG_FINALIZE; 2329 2330 return artpec6_crypto_prepare_submit_hash(req); 2331 } 2332 2333 static int artpec6_crypto_hash_update(struct ahash_request *req) 2334 { 2335 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2336 2337 req_ctx->hash_flags |= HASH_FLAG_UPDATE; 2338 2339 return artpec6_crypto_prepare_submit_hash(req); 2340 } 2341 2342 static int artpec6_crypto_sha1_init(struct ahash_request *req) 2343 { 2344 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); 2345 } 2346 2347 static int artpec6_crypto_sha1_digest(struct ahash_request *req) 2348 { 2349 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2350 2351 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); 2352 2353 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2354 2355 return artpec6_crypto_prepare_submit_hash(req); 2356 } 2357 2358 static int artpec6_crypto_sha256_init(struct ahash_request *req) 2359 { 2360 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); 2361 } 2362 2363 static int artpec6_crypto_sha256_digest(struct ahash_request *req) 2364 { 2365 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2366 2367 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); 2368 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2369 2370 return artpec6_crypto_prepare_submit_hash(req); 2371 } 2372 2373 static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req) 2374 { 2375 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); 2376 } 2377 2378 static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req) 2379 { 2380 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2381 2382 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); 2383 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2384 2385 return artpec6_crypto_prepare_submit_hash(req); 2386 } 2387 2388 static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm, 2389 const char *base_hash_name) 2390 { 2391 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); 2392 2393 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2394 sizeof(struct artpec6_hash_request_context)); 2395 memset(tfm_ctx, 0, sizeof(*tfm_ctx)); 2396 2397 if (base_hash_name) { 2398 struct crypto_shash *child; 2399 2400 child = crypto_alloc_shash(base_hash_name, 0, 2401 CRYPTO_ALG_NEED_FALLBACK); 2402 2403 if (IS_ERR(child)) 2404 return PTR_ERR(child); 2405 2406 tfm_ctx->child_hash = child; 2407 } 2408 2409 return 0; 2410 } 2411 2412 static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm) 2413 { 2414 return artpec6_crypto_ahash_init_common(tfm, NULL); 2415 } 2416 2417 static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm) 2418 { 2419 return artpec6_crypto_ahash_init_common(tfm, "sha256"); 2420 } 2421 2422 static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm) 2423 { 2424 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); 2425 2426 if (tfm_ctx->child_hash) 2427 crypto_free_shash(tfm_ctx->child_hash); 2428 2429 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); 2430 tfm_ctx->hmac_key_length = 0; 2431 } 2432 2433 static int artpec6_crypto_hash_export(struct ahash_request *req, void *out) 2434 { 2435 const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); 2436 struct artpec6_hash_export_state *state = out; 2437 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2438 enum artpec6_crypto_variant variant = ac->variant; 2439 2440 BUILD_BUG_ON(sizeof(state->partial_buffer) != 2441 sizeof(ctx->partial_buffer)); 2442 BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate)); 2443 2444 state->digcnt = ctx->digcnt; 2445 state->partial_bytes = ctx->partial_bytes; 2446 state->hash_flags = ctx->hash_flags; 2447 2448 if (variant == ARTPEC6_CRYPTO) 2449 state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md); 2450 else 2451 state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md); 2452 2453 memcpy(state->partial_buffer, ctx->partial_buffer, 2454 sizeof(state->partial_buffer)); 2455 memcpy(state->digeststate, ctx->digeststate, 2456 sizeof(state->digeststate)); 2457 2458 return 0; 2459 } 2460 2461 static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in) 2462 { 2463 struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); 2464 const struct artpec6_hash_export_state *state = in; 2465 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2466 enum artpec6_crypto_variant variant = ac->variant; 2467 2468 memset(ctx, 0, sizeof(*ctx)); 2469 2470 ctx->digcnt = state->digcnt; 2471 ctx->partial_bytes = state->partial_bytes; 2472 ctx->hash_flags = state->hash_flags; 2473 2474 if (variant == ARTPEC6_CRYPTO) 2475 ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper); 2476 else 2477 ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper); 2478 2479 memcpy(ctx->partial_buffer, state->partial_buffer, 2480 sizeof(state->partial_buffer)); 2481 memcpy(ctx->digeststate, state->digeststate, 2482 sizeof(state->digeststate)); 2483 2484 return 0; 2485 } 2486 2487 static int init_crypto_hw(struct artpec6_crypto *ac) 2488 { 2489 enum artpec6_crypto_variant variant = ac->variant; 2490 void __iomem *base = ac->base; 2491 u32 out_descr_buf_size; 2492 u32 out_data_buf_size; 2493 u32 in_data_buf_size; 2494 u32 in_descr_buf_size; 2495 u32 in_stat_buf_size; 2496 u32 in, out; 2497 2498 /* 2499 * The PDMA unit contains 1984 bytes of internal memory for the OUT 2500 * channels and 1024 bytes for the IN channel. This is an elastic 2501 * memory used to internally store the descriptors and data. The values 2502 * ares specified in 64 byte incremements. Trustzone buffers are not 2503 * used at this stage. 2504 */ 2505 out_data_buf_size = 16; /* 1024 bytes for data */ 2506 out_descr_buf_size = 15; /* 960 bytes for descriptors */ 2507 in_data_buf_size = 8; /* 512 bytes for data */ 2508 in_descr_buf_size = 4; /* 256 bytes for descriptors */ 2509 in_stat_buf_size = 4; /* 256 bytes for stat descrs */ 2510 2511 BUILD_BUG_ON_MSG((out_data_buf_size 2512 + out_descr_buf_size) * 64 > 1984, 2513 "Invalid OUT configuration"); 2514 2515 BUILD_BUG_ON_MSG((in_data_buf_size 2516 + in_descr_buf_size 2517 + in_stat_buf_size) * 64 > 1024, 2518 "Invalid IN configuration"); 2519 2520 in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) | 2521 FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) | 2522 FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size); 2523 2524 out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) | 2525 FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size); 2526 2527 writel_relaxed(out, base + PDMA_OUT_BUF_CFG); 2528 writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG); 2529 2530 if (variant == ARTPEC6_CRYPTO) { 2531 writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG); 2532 writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG); 2533 writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA | 2534 A6_PDMA_INTR_MASK_IN_EOP_FLUSH, 2535 base + A6_PDMA_INTR_MASK); 2536 } else { 2537 writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG); 2538 writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG); 2539 writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA | 2540 A7_PDMA_INTR_MASK_IN_EOP_FLUSH, 2541 base + A7_PDMA_INTR_MASK); 2542 } 2543 2544 return 0; 2545 } 2546 2547 static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac) 2548 { 2549 enum artpec6_crypto_variant variant = ac->variant; 2550 void __iomem *base = ac->base; 2551 2552 if (variant == ARTPEC6_CRYPTO) { 2553 writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD); 2554 writel_relaxed(0, base + A6_PDMA_IN_CFG); 2555 writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); 2556 } else { 2557 writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD); 2558 writel_relaxed(0, base + A7_PDMA_IN_CFG); 2559 writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); 2560 } 2561 2562 writel_relaxed(0, base + PDMA_OUT_CFG); 2563 2564 } 2565 2566 static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id) 2567 { 2568 struct artpec6_crypto *ac = dev_id; 2569 enum artpec6_crypto_variant variant = ac->variant; 2570 void __iomem *base = ac->base; 2571 u32 mask_in_data, mask_in_eop_flush; 2572 u32 in_cmd_flush_stat, in_cmd_reg; 2573 u32 ack_intr_reg; 2574 u32 ack = 0; 2575 u32 intr; 2576 2577 if (variant == ARTPEC6_CRYPTO) { 2578 intr = readl_relaxed(base + A6_PDMA_MASKED_INTR); 2579 mask_in_data = A6_PDMA_INTR_MASK_IN_DATA; 2580 mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH; 2581 in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT; 2582 in_cmd_reg = A6_PDMA_IN_CMD; 2583 ack_intr_reg = A6_PDMA_ACK_INTR; 2584 } else { 2585 intr = readl_relaxed(base + A7_PDMA_MASKED_INTR); 2586 mask_in_data = A7_PDMA_INTR_MASK_IN_DATA; 2587 mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH; 2588 in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT; 2589 in_cmd_reg = A7_PDMA_IN_CMD; 2590 ack_intr_reg = A7_PDMA_ACK_INTR; 2591 } 2592 2593 /* We get two interrupt notifications from each job. 2594 * The in_data means all data was sent to memory and then 2595 * we request a status flush command to write the per-job 2596 * status to its status vector. This ensures that the 2597 * tasklet can detect exactly how many submitted jobs 2598 * that have finished. 2599 */ 2600 if (intr & mask_in_data) 2601 ack |= mask_in_data; 2602 2603 if (intr & mask_in_eop_flush) 2604 ack |= mask_in_eop_flush; 2605 else 2606 writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg); 2607 2608 writel_relaxed(ack, base + ack_intr_reg); 2609 2610 if (intr & mask_in_eop_flush) 2611 tasklet_schedule(&ac->task); 2612 2613 return IRQ_HANDLED; 2614 } 2615 2616 /*------------------- Algorithm definitions ----------------------------------*/ 2617 2618 /* Hashes */ 2619 static struct ahash_alg hash_algos[] = { 2620 /* SHA-1 */ 2621 { 2622 .init = artpec6_crypto_sha1_init, 2623 .update = artpec6_crypto_hash_update, 2624 .final = artpec6_crypto_hash_final, 2625 .digest = artpec6_crypto_sha1_digest, 2626 .import = artpec6_crypto_hash_import, 2627 .export = artpec6_crypto_hash_export, 2628 .halg.digestsize = SHA1_DIGEST_SIZE, 2629 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2630 .halg.base = { 2631 .cra_name = "sha1", 2632 .cra_driver_name = "artpec-sha1", 2633 .cra_priority = 300, 2634 .cra_flags = CRYPTO_ALG_ASYNC | 2635 CRYPTO_ALG_ALLOCATES_MEMORY, 2636 .cra_blocksize = SHA1_BLOCK_SIZE, 2637 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2638 .cra_alignmask = 3, 2639 .cra_module = THIS_MODULE, 2640 .cra_init = artpec6_crypto_ahash_init, 2641 .cra_exit = artpec6_crypto_ahash_exit, 2642 } 2643 }, 2644 /* SHA-256 */ 2645 { 2646 .init = artpec6_crypto_sha256_init, 2647 .update = artpec6_crypto_hash_update, 2648 .final = artpec6_crypto_hash_final, 2649 .digest = artpec6_crypto_sha256_digest, 2650 .import = artpec6_crypto_hash_import, 2651 .export = artpec6_crypto_hash_export, 2652 .halg.digestsize = SHA256_DIGEST_SIZE, 2653 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2654 .halg.base = { 2655 .cra_name = "sha256", 2656 .cra_driver_name = "artpec-sha256", 2657 .cra_priority = 300, 2658 .cra_flags = CRYPTO_ALG_ASYNC | 2659 CRYPTO_ALG_ALLOCATES_MEMORY, 2660 .cra_blocksize = SHA256_BLOCK_SIZE, 2661 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2662 .cra_alignmask = 3, 2663 .cra_module = THIS_MODULE, 2664 .cra_init = artpec6_crypto_ahash_init, 2665 .cra_exit = artpec6_crypto_ahash_exit, 2666 } 2667 }, 2668 /* HMAC SHA-256 */ 2669 { 2670 .init = artpec6_crypto_hmac_sha256_init, 2671 .update = artpec6_crypto_hash_update, 2672 .final = artpec6_crypto_hash_final, 2673 .digest = artpec6_crypto_hmac_sha256_digest, 2674 .import = artpec6_crypto_hash_import, 2675 .export = artpec6_crypto_hash_export, 2676 .setkey = artpec6_crypto_hash_set_key, 2677 .halg.digestsize = SHA256_DIGEST_SIZE, 2678 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2679 .halg.base = { 2680 .cra_name = "hmac(sha256)", 2681 .cra_driver_name = "artpec-hmac-sha256", 2682 .cra_priority = 300, 2683 .cra_flags = CRYPTO_ALG_ASYNC | 2684 CRYPTO_ALG_ALLOCATES_MEMORY, 2685 .cra_blocksize = SHA256_BLOCK_SIZE, 2686 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2687 .cra_alignmask = 3, 2688 .cra_module = THIS_MODULE, 2689 .cra_init = artpec6_crypto_ahash_init_hmac_sha256, 2690 .cra_exit = artpec6_crypto_ahash_exit, 2691 } 2692 }, 2693 }; 2694 2695 /* Crypto */ 2696 static struct skcipher_alg crypto_algos[] = { 2697 /* AES - ECB */ 2698 { 2699 .base = { 2700 .cra_name = "ecb(aes)", 2701 .cra_driver_name = "artpec6-ecb-aes", 2702 .cra_priority = 300, 2703 .cra_flags = CRYPTO_ALG_ASYNC | 2704 CRYPTO_ALG_ALLOCATES_MEMORY, 2705 .cra_blocksize = AES_BLOCK_SIZE, 2706 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2707 .cra_alignmask = 3, 2708 .cra_module = THIS_MODULE, 2709 }, 2710 .min_keysize = AES_MIN_KEY_SIZE, 2711 .max_keysize = AES_MAX_KEY_SIZE, 2712 .setkey = artpec6_crypto_cipher_set_key, 2713 .encrypt = artpec6_crypto_encrypt, 2714 .decrypt = artpec6_crypto_decrypt, 2715 .init = artpec6_crypto_aes_ecb_init, 2716 .exit = artpec6_crypto_aes_exit, 2717 }, 2718 /* AES - CTR */ 2719 { 2720 .base = { 2721 .cra_name = "ctr(aes)", 2722 .cra_driver_name = "artpec6-ctr-aes", 2723 .cra_priority = 300, 2724 .cra_flags = CRYPTO_ALG_ASYNC | 2725 CRYPTO_ALG_ALLOCATES_MEMORY | 2726 CRYPTO_ALG_NEED_FALLBACK, 2727 .cra_blocksize = 1, 2728 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2729 .cra_alignmask = 3, 2730 .cra_module = THIS_MODULE, 2731 }, 2732 .min_keysize = AES_MIN_KEY_SIZE, 2733 .max_keysize = AES_MAX_KEY_SIZE, 2734 .ivsize = AES_BLOCK_SIZE, 2735 .setkey = artpec6_crypto_cipher_set_key, 2736 .encrypt = artpec6_crypto_ctr_encrypt, 2737 .decrypt = artpec6_crypto_ctr_decrypt, 2738 .init = artpec6_crypto_aes_ctr_init, 2739 .exit = artpec6_crypto_aes_ctr_exit, 2740 }, 2741 /* AES - CBC */ 2742 { 2743 .base = { 2744 .cra_name = "cbc(aes)", 2745 .cra_driver_name = "artpec6-cbc-aes", 2746 .cra_priority = 300, 2747 .cra_flags = CRYPTO_ALG_ASYNC | 2748 CRYPTO_ALG_ALLOCATES_MEMORY, 2749 .cra_blocksize = AES_BLOCK_SIZE, 2750 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2751 .cra_alignmask = 3, 2752 .cra_module = THIS_MODULE, 2753 }, 2754 .min_keysize = AES_MIN_KEY_SIZE, 2755 .max_keysize = AES_MAX_KEY_SIZE, 2756 .ivsize = AES_BLOCK_SIZE, 2757 .setkey = artpec6_crypto_cipher_set_key, 2758 .encrypt = artpec6_crypto_encrypt, 2759 .decrypt = artpec6_crypto_decrypt, 2760 .init = artpec6_crypto_aes_cbc_init, 2761 .exit = artpec6_crypto_aes_exit 2762 }, 2763 /* AES - XTS */ 2764 { 2765 .base = { 2766 .cra_name = "xts(aes)", 2767 .cra_driver_name = "artpec6-xts-aes", 2768 .cra_priority = 300, 2769 .cra_flags = CRYPTO_ALG_ASYNC | 2770 CRYPTO_ALG_ALLOCATES_MEMORY, 2771 .cra_blocksize = 1, 2772 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2773 .cra_alignmask = 3, 2774 .cra_module = THIS_MODULE, 2775 }, 2776 .min_keysize = 2*AES_MIN_KEY_SIZE, 2777 .max_keysize = 2*AES_MAX_KEY_SIZE, 2778 .ivsize = 16, 2779 .setkey = artpec6_crypto_xts_set_key, 2780 .encrypt = artpec6_crypto_encrypt, 2781 .decrypt = artpec6_crypto_decrypt, 2782 .init = artpec6_crypto_aes_xts_init, 2783 .exit = artpec6_crypto_aes_exit, 2784 }, 2785 }; 2786 2787 static struct aead_alg aead_algos[] = { 2788 { 2789 .init = artpec6_crypto_aead_init, 2790 .setkey = artpec6_crypto_aead_set_key, 2791 .encrypt = artpec6_crypto_aead_encrypt, 2792 .decrypt = artpec6_crypto_aead_decrypt, 2793 .ivsize = GCM_AES_IV_SIZE, 2794 .maxauthsize = AES_BLOCK_SIZE, 2795 2796 .base = { 2797 .cra_name = "gcm(aes)", 2798 .cra_driver_name = "artpec-gcm-aes", 2799 .cra_priority = 300, 2800 .cra_flags = CRYPTO_ALG_ASYNC | 2801 CRYPTO_ALG_ALLOCATES_MEMORY | 2802 CRYPTO_ALG_KERN_DRIVER_ONLY, 2803 .cra_blocksize = 1, 2804 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2805 .cra_alignmask = 3, 2806 .cra_module = THIS_MODULE, 2807 }, 2808 } 2809 }; 2810 2811 #ifdef CONFIG_DEBUG_FS 2812 2813 struct dbgfs_u32 { 2814 char *name; 2815 mode_t mode; 2816 u32 *flag; 2817 char *desc; 2818 }; 2819 2820 static struct dentry *dbgfs_root; 2821 2822 static void artpec6_crypto_init_debugfs(void) 2823 { 2824 dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL); 2825 2826 #ifdef CONFIG_FAULT_INJECTION 2827 fault_create_debugfs_attr("fail_status_read", dbgfs_root, 2828 &artpec6_crypto_fail_status_read); 2829 2830 fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root, 2831 &artpec6_crypto_fail_dma_array_full); 2832 #endif 2833 } 2834 2835 static void artpec6_crypto_free_debugfs(void) 2836 { 2837 debugfs_remove_recursive(dbgfs_root); 2838 dbgfs_root = NULL; 2839 } 2840 #endif 2841 2842 static const struct of_device_id artpec6_crypto_of_match[] = { 2843 { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO }, 2844 { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO }, 2845 {} 2846 }; 2847 MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match); 2848 2849 static int artpec6_crypto_probe(struct platform_device *pdev) 2850 { 2851 const struct of_device_id *match; 2852 enum artpec6_crypto_variant variant; 2853 struct artpec6_crypto *ac; 2854 struct device *dev = &pdev->dev; 2855 void __iomem *base; 2856 int irq; 2857 int err; 2858 2859 if (artpec6_crypto_dev) 2860 return -ENODEV; 2861 2862 match = of_match_node(artpec6_crypto_of_match, dev->of_node); 2863 if (!match) 2864 return -EINVAL; 2865 2866 variant = (enum artpec6_crypto_variant)match->data; 2867 2868 base = devm_platform_ioremap_resource(pdev, 0); 2869 if (IS_ERR(base)) 2870 return PTR_ERR(base); 2871 2872 irq = platform_get_irq(pdev, 0); 2873 if (irq < 0) 2874 return -ENODEV; 2875 2876 ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto), 2877 GFP_KERNEL); 2878 if (!ac) 2879 return -ENOMEM; 2880 2881 platform_set_drvdata(pdev, ac); 2882 ac->variant = variant; 2883 2884 spin_lock_init(&ac->queue_lock); 2885 INIT_LIST_HEAD(&ac->queue); 2886 INIT_LIST_HEAD(&ac->pending); 2887 timer_setup(&ac->timer, artpec6_crypto_timeout, 0); 2888 2889 ac->base = base; 2890 2891 ac->dma_cache = kmem_cache_create("artpec6_crypto_dma", 2892 sizeof(struct artpec6_crypto_dma_descriptors), 2893 64, 2894 0, 2895 NULL); 2896 if (!ac->dma_cache) 2897 return -ENOMEM; 2898 2899 #ifdef CONFIG_DEBUG_FS 2900 artpec6_crypto_init_debugfs(); 2901 #endif 2902 2903 tasklet_init(&ac->task, artpec6_crypto_task, 2904 (unsigned long)ac); 2905 2906 ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, 2907 GFP_KERNEL); 2908 if (!ac->pad_buffer) 2909 return -ENOMEM; 2910 ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX); 2911 2912 ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, 2913 GFP_KERNEL); 2914 if (!ac->zero_buffer) 2915 return -ENOMEM; 2916 ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX); 2917 2918 err = init_crypto_hw(ac); 2919 if (err) 2920 goto free_cache; 2921 2922 err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0, 2923 "artpec6-crypto", ac); 2924 if (err) 2925 goto disable_hw; 2926 2927 artpec6_crypto_dev = &pdev->dev; 2928 2929 err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 2930 if (err) { 2931 dev_err(dev, "Failed to register ahashes\n"); 2932 goto disable_hw; 2933 } 2934 2935 err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 2936 if (err) { 2937 dev_err(dev, "Failed to register ciphers\n"); 2938 goto unregister_ahashes; 2939 } 2940 2941 err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos)); 2942 if (err) { 2943 dev_err(dev, "Failed to register aeads\n"); 2944 goto unregister_algs; 2945 } 2946 2947 return 0; 2948 2949 unregister_algs: 2950 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 2951 unregister_ahashes: 2952 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 2953 disable_hw: 2954 artpec6_crypto_disable_hw(ac); 2955 free_cache: 2956 kmem_cache_destroy(ac->dma_cache); 2957 return err; 2958 } 2959 2960 static int artpec6_crypto_remove(struct platform_device *pdev) 2961 { 2962 struct artpec6_crypto *ac = platform_get_drvdata(pdev); 2963 int irq = platform_get_irq(pdev, 0); 2964 2965 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 2966 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 2967 crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos)); 2968 2969 tasklet_disable(&ac->task); 2970 devm_free_irq(&pdev->dev, irq, ac); 2971 tasklet_kill(&ac->task); 2972 del_timer_sync(&ac->timer); 2973 2974 artpec6_crypto_disable_hw(ac); 2975 2976 kmem_cache_destroy(ac->dma_cache); 2977 #ifdef CONFIG_DEBUG_FS 2978 artpec6_crypto_free_debugfs(); 2979 #endif 2980 return 0; 2981 } 2982 2983 static struct platform_driver artpec6_crypto_driver = { 2984 .probe = artpec6_crypto_probe, 2985 .remove = artpec6_crypto_remove, 2986 .driver = { 2987 .name = "artpec6-crypto", 2988 .of_match_table = artpec6_crypto_of_match, 2989 }, 2990 }; 2991 2992 module_platform_driver(artpec6_crypto_driver); 2993 2994 MODULE_AUTHOR("Axis Communications AB"); 2995 MODULE_DESCRIPTION("ARTPEC-6 Crypto driver"); 2996 MODULE_LICENSE("GPL"); 2997