1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #include <linux/module.h> 15 #include <linux/kernel.h> 16 #include <linux/pci.h> 17 #include <linux/interrupt.h> 18 #include <crypto/scatterwalk.h> 19 #include <linux/ccp.h> 20 21 #include "ccp-dev.h" 22 23 /* SHA initial context values */ 24 static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 25 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 26 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 27 cpu_to_be32(SHA1_H4), 0, 0, 0, 28 }; 29 30 static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 31 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 32 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 33 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 34 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 35 }; 36 37 static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 38 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 39 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 40 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 41 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 42 }; 43 44 static u32 ccp_gen_jobid(struct ccp_device *ccp) 45 { 46 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 47 } 48 49 static void ccp_sg_free(struct ccp_sg_workarea *wa) 50 { 51 if (wa->dma_count) 52 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); 53 54 wa->dma_count = 0; 55 } 56 57 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 58 struct scatterlist *sg, u64 len, 59 enum dma_data_direction dma_dir) 60 { 61 memset(wa, 0, sizeof(*wa)); 62 63 wa->sg = sg; 64 if (!sg) 65 return 0; 66 67 wa->nents = sg_nents_for_len(sg, len); 68 if (wa->nents < 0) 69 return wa->nents; 70 71 wa->bytes_left = len; 72 wa->sg_used = 0; 73 74 if (len == 0) 75 return 0; 76 77 if (dma_dir == DMA_NONE) 78 return 0; 79 80 wa->dma_sg = sg; 81 wa->dma_dev = dev; 82 wa->dma_dir = dma_dir; 83 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 84 if (!wa->dma_count) 85 return -ENOMEM; 86 87 return 0; 88 } 89 90 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 91 { 92 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 93 94 if (!wa->sg) 95 return; 96 97 wa->sg_used += nbytes; 98 wa->bytes_left -= nbytes; 99 if (wa->sg_used == wa->sg->length) { 100 wa->sg = sg_next(wa->sg); 101 wa->sg_used = 0; 102 } 103 } 104 105 static void ccp_dm_free(struct ccp_dm_workarea *wa) 106 { 107 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 108 if (wa->address) 109 dma_pool_free(wa->dma_pool, wa->address, 110 wa->dma.address); 111 } else { 112 if (wa->dma.address) 113 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 114 wa->dma.dir); 115 kfree(wa->address); 116 } 117 118 wa->address = NULL; 119 wa->dma.address = 0; 120 } 121 122 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 123 struct ccp_cmd_queue *cmd_q, 124 unsigned int len, 125 enum dma_data_direction dir) 126 { 127 memset(wa, 0, sizeof(*wa)); 128 129 if (!len) 130 return 0; 131 132 wa->dev = cmd_q->ccp->dev; 133 wa->length = len; 134 135 if (len <= CCP_DMAPOOL_MAX_SIZE) { 136 wa->dma_pool = cmd_q->dma_pool; 137 138 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, 139 &wa->dma.address); 140 if (!wa->address) 141 return -ENOMEM; 142 143 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 144 145 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); 146 } else { 147 wa->address = kzalloc(len, GFP_KERNEL); 148 if (!wa->address) 149 return -ENOMEM; 150 151 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 152 dir); 153 if (!wa->dma.address) 154 return -ENOMEM; 155 156 wa->dma.length = len; 157 } 158 wa->dma.dir = dir; 159 160 return 0; 161 } 162 163 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 164 struct scatterlist *sg, unsigned int sg_offset, 165 unsigned int len) 166 { 167 WARN_ON(!wa->address); 168 169 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 170 0); 171 } 172 173 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 174 struct scatterlist *sg, unsigned int sg_offset, 175 unsigned int len) 176 { 177 WARN_ON(!wa->address); 178 179 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 180 1); 181 } 182 183 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 184 struct scatterlist *sg, 185 unsigned int len, unsigned int se_len, 186 bool sign_extend) 187 { 188 unsigned int nbytes, sg_offset, dm_offset, sb_len, i; 189 u8 buffer[CCP_REVERSE_BUF_SIZE]; 190 191 if (WARN_ON(se_len > sizeof(buffer))) 192 return -EINVAL; 193 194 sg_offset = len; 195 dm_offset = 0; 196 nbytes = len; 197 while (nbytes) { 198 sb_len = min_t(unsigned int, nbytes, se_len); 199 sg_offset -= sb_len; 200 201 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 0); 202 for (i = 0; i < sb_len; i++) 203 wa->address[dm_offset + i] = buffer[sb_len - i - 1]; 204 205 dm_offset += sb_len; 206 nbytes -= sb_len; 207 208 if ((sb_len != se_len) && sign_extend) { 209 /* Must sign-extend to nearest sign-extend length */ 210 if (wa->address[dm_offset - 1] & 0x80) 211 memset(wa->address + dm_offset, 0xff, 212 se_len - sb_len); 213 } 214 } 215 216 return 0; 217 } 218 219 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 220 struct scatterlist *sg, 221 unsigned int len) 222 { 223 unsigned int nbytes, sg_offset, dm_offset, sb_len, i; 224 u8 buffer[CCP_REVERSE_BUF_SIZE]; 225 226 sg_offset = 0; 227 dm_offset = len; 228 nbytes = len; 229 while (nbytes) { 230 sb_len = min_t(unsigned int, nbytes, sizeof(buffer)); 231 dm_offset -= sb_len; 232 233 for (i = 0; i < sb_len; i++) 234 buffer[sb_len - i - 1] = wa->address[dm_offset + i]; 235 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 1); 236 237 sg_offset += sb_len; 238 nbytes -= sb_len; 239 } 240 } 241 242 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 243 { 244 ccp_dm_free(&data->dm_wa); 245 ccp_sg_free(&data->sg_wa); 246 } 247 248 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 249 struct scatterlist *sg, u64 sg_len, 250 unsigned int dm_len, 251 enum dma_data_direction dir) 252 { 253 int ret; 254 255 memset(data, 0, sizeof(*data)); 256 257 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 258 dir); 259 if (ret) 260 goto e_err; 261 262 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 263 if (ret) 264 goto e_err; 265 266 return 0; 267 268 e_err: 269 ccp_free_data(data, cmd_q); 270 271 return ret; 272 } 273 274 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 275 { 276 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 277 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 278 unsigned int buf_count, nbytes; 279 280 /* Clear the buffer if setting it */ 281 if (!from) 282 memset(dm_wa->address, 0, dm_wa->length); 283 284 if (!sg_wa->sg) 285 return 0; 286 287 /* Perform the copy operation 288 * nbytes will always be <= UINT_MAX because dm_wa->length is 289 * an unsigned int 290 */ 291 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 292 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 293 nbytes, from); 294 295 /* Update the structures and generate the count */ 296 buf_count = 0; 297 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 298 nbytes = min(sg_wa->sg->length - sg_wa->sg_used, 299 dm_wa->length - buf_count); 300 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 301 302 buf_count += nbytes; 303 ccp_update_sg_workarea(sg_wa, nbytes); 304 } 305 306 return buf_count; 307 } 308 309 static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 310 { 311 return ccp_queue_buf(data, 0); 312 } 313 314 static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 315 { 316 return ccp_queue_buf(data, 1); 317 } 318 319 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 320 struct ccp_op *op, unsigned int block_size, 321 bool blocksize_op) 322 { 323 unsigned int sg_src_len, sg_dst_len, op_len; 324 325 /* The CCP can only DMA from/to one address each per operation. This 326 * requires that we find the smallest DMA area between the source 327 * and destination. The resulting len values will always be <= UINT_MAX 328 * because the dma length is an unsigned int. 329 */ 330 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; 331 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 332 333 if (dst) { 334 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 335 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 336 op_len = min(sg_src_len, sg_dst_len); 337 } else { 338 op_len = sg_src_len; 339 } 340 341 /* The data operation length will be at least block_size in length 342 * or the smaller of available sg room remaining for the source or 343 * the destination 344 */ 345 op_len = max(op_len, block_size); 346 347 /* Unless we have to buffer data, there's no reason to wait */ 348 op->soc = 0; 349 350 if (sg_src_len < block_size) { 351 /* Not enough data in the sg element, so it 352 * needs to be buffered into a blocksize chunk 353 */ 354 int cp_len = ccp_fill_queue_buf(src); 355 356 op->soc = 1; 357 op->src.u.dma.address = src->dm_wa.dma.address; 358 op->src.u.dma.offset = 0; 359 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 360 } else { 361 /* Enough data in the sg element, but we need to 362 * adjust for any previously copied data 363 */ 364 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); 365 op->src.u.dma.offset = src->sg_wa.sg_used; 366 op->src.u.dma.length = op_len & ~(block_size - 1); 367 368 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 369 } 370 371 if (dst) { 372 if (sg_dst_len < block_size) { 373 /* Not enough room in the sg element or we're on the 374 * last piece of data (when using padding), so the 375 * output needs to be buffered into a blocksize chunk 376 */ 377 op->soc = 1; 378 op->dst.u.dma.address = dst->dm_wa.dma.address; 379 op->dst.u.dma.offset = 0; 380 op->dst.u.dma.length = op->src.u.dma.length; 381 } else { 382 /* Enough room in the sg element, but we need to 383 * adjust for any previously used area 384 */ 385 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); 386 op->dst.u.dma.offset = dst->sg_wa.sg_used; 387 op->dst.u.dma.length = op->src.u.dma.length; 388 } 389 } 390 } 391 392 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 393 struct ccp_op *op) 394 { 395 op->init = 0; 396 397 if (dst) { 398 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 399 ccp_empty_queue_buf(dst); 400 else 401 ccp_update_sg_workarea(&dst->sg_wa, 402 op->dst.u.dma.length); 403 } 404 } 405 406 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q, 407 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 408 u32 byte_swap, bool from) 409 { 410 struct ccp_op op; 411 412 memset(&op, 0, sizeof(op)); 413 414 op.cmd_q = cmd_q; 415 op.jobid = jobid; 416 op.eom = 1; 417 418 if (from) { 419 op.soc = 1; 420 op.src.type = CCP_MEMTYPE_SB; 421 op.src.u.sb = sb; 422 op.dst.type = CCP_MEMTYPE_SYSTEM; 423 op.dst.u.dma.address = wa->dma.address; 424 op.dst.u.dma.length = wa->length; 425 } else { 426 op.src.type = CCP_MEMTYPE_SYSTEM; 427 op.src.u.dma.address = wa->dma.address; 428 op.src.u.dma.length = wa->length; 429 op.dst.type = CCP_MEMTYPE_SB; 430 op.dst.u.sb = sb; 431 } 432 433 op.u.passthru.byte_swap = byte_swap; 434 435 return cmd_q->ccp->vdata->perform->passthru(&op); 436 } 437 438 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q, 439 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 440 u32 byte_swap) 441 { 442 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false); 443 } 444 445 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q, 446 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 447 u32 byte_swap) 448 { 449 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true); 450 } 451 452 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, 453 struct ccp_cmd *cmd) 454 { 455 struct ccp_aes_engine *aes = &cmd->u.aes; 456 struct ccp_dm_workarea key, ctx; 457 struct ccp_data src; 458 struct ccp_op op; 459 unsigned int dm_offset; 460 int ret; 461 462 if (!((aes->key_len == AES_KEYSIZE_128) || 463 (aes->key_len == AES_KEYSIZE_192) || 464 (aes->key_len == AES_KEYSIZE_256))) 465 return -EINVAL; 466 467 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 468 return -EINVAL; 469 470 if (aes->iv_len != AES_BLOCK_SIZE) 471 return -EINVAL; 472 473 if (!aes->key || !aes->iv || !aes->src) 474 return -EINVAL; 475 476 if (aes->cmac_final) { 477 if (aes->cmac_key_len != AES_BLOCK_SIZE) 478 return -EINVAL; 479 480 if (!aes->cmac_key) 481 return -EINVAL; 482 } 483 484 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 485 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 486 487 ret = -EIO; 488 memset(&op, 0, sizeof(op)); 489 op.cmd_q = cmd_q; 490 op.jobid = ccp_gen_jobid(cmd_q->ccp); 491 op.sb_key = cmd_q->sb_key; 492 op.sb_ctx = cmd_q->sb_ctx; 493 op.init = 1; 494 op.u.aes.type = aes->type; 495 op.u.aes.mode = aes->mode; 496 op.u.aes.action = aes->action; 497 498 /* All supported key sizes fit in a single (32-byte) SB entry 499 * and must be in little endian format. Use the 256-bit byte 500 * swap passthru option to convert from big endian to little 501 * endian. 502 */ 503 ret = ccp_init_dm_workarea(&key, cmd_q, 504 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 505 DMA_TO_DEVICE); 506 if (ret) 507 return ret; 508 509 dm_offset = CCP_SB_BYTES - aes->key_len; 510 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 511 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 512 CCP_PASSTHRU_BYTESWAP_256BIT); 513 if (ret) { 514 cmd->engine_error = cmd_q->cmd_error; 515 goto e_key; 516 } 517 518 /* The AES context fits in a single (32-byte) SB entry and 519 * must be in little endian format. Use the 256-bit byte swap 520 * passthru option to convert from big endian to little endian. 521 */ 522 ret = ccp_init_dm_workarea(&ctx, cmd_q, 523 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 524 DMA_BIDIRECTIONAL); 525 if (ret) 526 goto e_key; 527 528 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 529 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 530 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 531 CCP_PASSTHRU_BYTESWAP_256BIT); 532 if (ret) { 533 cmd->engine_error = cmd_q->cmd_error; 534 goto e_ctx; 535 } 536 537 /* Send data to the CCP AES engine */ 538 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 539 AES_BLOCK_SIZE, DMA_TO_DEVICE); 540 if (ret) 541 goto e_ctx; 542 543 while (src.sg_wa.bytes_left) { 544 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 545 if (aes->cmac_final && !src.sg_wa.bytes_left) { 546 op.eom = 1; 547 548 /* Push the K1/K2 key to the CCP now */ 549 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, 550 op.sb_ctx, 551 CCP_PASSTHRU_BYTESWAP_256BIT); 552 if (ret) { 553 cmd->engine_error = cmd_q->cmd_error; 554 goto e_src; 555 } 556 557 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 558 aes->cmac_key_len); 559 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 560 CCP_PASSTHRU_BYTESWAP_256BIT); 561 if (ret) { 562 cmd->engine_error = cmd_q->cmd_error; 563 goto e_src; 564 } 565 } 566 567 ret = cmd_q->ccp->vdata->perform->aes(&op); 568 if (ret) { 569 cmd->engine_error = cmd_q->cmd_error; 570 goto e_src; 571 } 572 573 ccp_process_data(&src, NULL, &op); 574 } 575 576 /* Retrieve the AES context - convert from LE to BE using 577 * 32-byte (256-bit) byteswapping 578 */ 579 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 580 CCP_PASSTHRU_BYTESWAP_256BIT); 581 if (ret) { 582 cmd->engine_error = cmd_q->cmd_error; 583 goto e_src; 584 } 585 586 /* ...but we only need AES_BLOCK_SIZE bytes */ 587 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 588 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 589 590 e_src: 591 ccp_free_data(&src, cmd_q); 592 593 e_ctx: 594 ccp_dm_free(&ctx); 595 596 e_key: 597 ccp_dm_free(&key); 598 599 return ret; 600 } 601 602 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 603 { 604 struct ccp_aes_engine *aes = &cmd->u.aes; 605 struct ccp_dm_workarea key, ctx; 606 struct ccp_data src, dst; 607 struct ccp_op op; 608 unsigned int dm_offset; 609 bool in_place = false; 610 int ret; 611 612 if (aes->mode == CCP_AES_MODE_CMAC) 613 return ccp_run_aes_cmac_cmd(cmd_q, cmd); 614 615 if (!((aes->key_len == AES_KEYSIZE_128) || 616 (aes->key_len == AES_KEYSIZE_192) || 617 (aes->key_len == AES_KEYSIZE_256))) 618 return -EINVAL; 619 620 if (((aes->mode == CCP_AES_MODE_ECB) || 621 (aes->mode == CCP_AES_MODE_CBC) || 622 (aes->mode == CCP_AES_MODE_CFB)) && 623 (aes->src_len & (AES_BLOCK_SIZE - 1))) 624 return -EINVAL; 625 626 if (!aes->key || !aes->src || !aes->dst) 627 return -EINVAL; 628 629 if (aes->mode != CCP_AES_MODE_ECB) { 630 if (aes->iv_len != AES_BLOCK_SIZE) 631 return -EINVAL; 632 633 if (!aes->iv) 634 return -EINVAL; 635 } 636 637 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 638 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 639 640 ret = -EIO; 641 memset(&op, 0, sizeof(op)); 642 op.cmd_q = cmd_q; 643 op.jobid = ccp_gen_jobid(cmd_q->ccp); 644 op.sb_key = cmd_q->sb_key; 645 op.sb_ctx = cmd_q->sb_ctx; 646 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 647 op.u.aes.type = aes->type; 648 op.u.aes.mode = aes->mode; 649 op.u.aes.action = aes->action; 650 651 /* All supported key sizes fit in a single (32-byte) SB entry 652 * and must be in little endian format. Use the 256-bit byte 653 * swap passthru option to convert from big endian to little 654 * endian. 655 */ 656 ret = ccp_init_dm_workarea(&key, cmd_q, 657 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 658 DMA_TO_DEVICE); 659 if (ret) 660 return ret; 661 662 dm_offset = CCP_SB_BYTES - aes->key_len; 663 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 664 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 665 CCP_PASSTHRU_BYTESWAP_256BIT); 666 if (ret) { 667 cmd->engine_error = cmd_q->cmd_error; 668 goto e_key; 669 } 670 671 /* The AES context fits in a single (32-byte) SB entry and 672 * must be in little endian format. Use the 256-bit byte swap 673 * passthru option to convert from big endian to little endian. 674 */ 675 ret = ccp_init_dm_workarea(&ctx, cmd_q, 676 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 677 DMA_BIDIRECTIONAL); 678 if (ret) 679 goto e_key; 680 681 if (aes->mode != CCP_AES_MODE_ECB) { 682 /* Load the AES context - conver to LE */ 683 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 684 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 685 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 686 CCP_PASSTHRU_BYTESWAP_256BIT); 687 if (ret) { 688 cmd->engine_error = cmd_q->cmd_error; 689 goto e_ctx; 690 } 691 } 692 693 /* Prepare the input and output data workareas. For in-place 694 * operations we need to set the dma direction to BIDIRECTIONAL 695 * and copy the src workarea to the dst workarea. 696 */ 697 if (sg_virt(aes->src) == sg_virt(aes->dst)) 698 in_place = true; 699 700 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 701 AES_BLOCK_SIZE, 702 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 703 if (ret) 704 goto e_ctx; 705 706 if (in_place) { 707 dst = src; 708 } else { 709 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 710 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 711 if (ret) 712 goto e_src; 713 } 714 715 /* Send data to the CCP AES engine */ 716 while (src.sg_wa.bytes_left) { 717 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 718 if (!src.sg_wa.bytes_left) { 719 op.eom = 1; 720 721 /* Since we don't retrieve the AES context in ECB 722 * mode we have to wait for the operation to complete 723 * on the last piece of data 724 */ 725 if (aes->mode == CCP_AES_MODE_ECB) 726 op.soc = 1; 727 } 728 729 ret = cmd_q->ccp->vdata->perform->aes(&op); 730 if (ret) { 731 cmd->engine_error = cmd_q->cmd_error; 732 goto e_dst; 733 } 734 735 ccp_process_data(&src, &dst, &op); 736 } 737 738 if (aes->mode != CCP_AES_MODE_ECB) { 739 /* Retrieve the AES context - convert from LE to BE using 740 * 32-byte (256-bit) byteswapping 741 */ 742 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 743 CCP_PASSTHRU_BYTESWAP_256BIT); 744 if (ret) { 745 cmd->engine_error = cmd_q->cmd_error; 746 goto e_dst; 747 } 748 749 /* ...but we only need AES_BLOCK_SIZE bytes */ 750 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 751 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 752 } 753 754 e_dst: 755 if (!in_place) 756 ccp_free_data(&dst, cmd_q); 757 758 e_src: 759 ccp_free_data(&src, cmd_q); 760 761 e_ctx: 762 ccp_dm_free(&ctx); 763 764 e_key: 765 ccp_dm_free(&key); 766 767 return ret; 768 } 769 770 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, 771 struct ccp_cmd *cmd) 772 { 773 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 774 struct ccp_dm_workarea key, ctx; 775 struct ccp_data src, dst; 776 struct ccp_op op; 777 unsigned int unit_size, dm_offset; 778 bool in_place = false; 779 int ret; 780 781 switch (xts->unit_size) { 782 case CCP_XTS_AES_UNIT_SIZE_16: 783 unit_size = 16; 784 break; 785 case CCP_XTS_AES_UNIT_SIZE_512: 786 unit_size = 512; 787 break; 788 case CCP_XTS_AES_UNIT_SIZE_1024: 789 unit_size = 1024; 790 break; 791 case CCP_XTS_AES_UNIT_SIZE_2048: 792 unit_size = 2048; 793 break; 794 case CCP_XTS_AES_UNIT_SIZE_4096: 795 unit_size = 4096; 796 break; 797 798 default: 799 return -EINVAL; 800 } 801 802 if (xts->key_len != AES_KEYSIZE_128) 803 return -EINVAL; 804 805 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 806 return -EINVAL; 807 808 if (xts->iv_len != AES_BLOCK_SIZE) 809 return -EINVAL; 810 811 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 812 return -EINVAL; 813 814 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1); 815 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1); 816 817 ret = -EIO; 818 memset(&op, 0, sizeof(op)); 819 op.cmd_q = cmd_q; 820 op.jobid = ccp_gen_jobid(cmd_q->ccp); 821 op.sb_key = cmd_q->sb_key; 822 op.sb_ctx = cmd_q->sb_ctx; 823 op.init = 1; 824 op.u.xts.action = xts->action; 825 op.u.xts.unit_size = xts->unit_size; 826 827 /* All supported key sizes fit in a single (32-byte) SB entry 828 * and must be in little endian format. Use the 256-bit byte 829 * swap passthru option to convert from big endian to little 830 * endian. 831 */ 832 ret = ccp_init_dm_workarea(&key, cmd_q, 833 CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES, 834 DMA_TO_DEVICE); 835 if (ret) 836 return ret; 837 838 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; 839 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 840 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); 841 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 842 CCP_PASSTHRU_BYTESWAP_256BIT); 843 if (ret) { 844 cmd->engine_error = cmd_q->cmd_error; 845 goto e_key; 846 } 847 848 /* The AES context fits in a single (32-byte) SB entry and 849 * for XTS is already in little endian format so no byte swapping 850 * is needed. 851 */ 852 ret = ccp_init_dm_workarea(&ctx, cmd_q, 853 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES, 854 DMA_BIDIRECTIONAL); 855 if (ret) 856 goto e_key; 857 858 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 859 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 860 CCP_PASSTHRU_BYTESWAP_NOOP); 861 if (ret) { 862 cmd->engine_error = cmd_q->cmd_error; 863 goto e_ctx; 864 } 865 866 /* Prepare the input and output data workareas. For in-place 867 * operations we need to set the dma direction to BIDIRECTIONAL 868 * and copy the src workarea to the dst workarea. 869 */ 870 if (sg_virt(xts->src) == sg_virt(xts->dst)) 871 in_place = true; 872 873 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 874 unit_size, 875 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 876 if (ret) 877 goto e_ctx; 878 879 if (in_place) { 880 dst = src; 881 } else { 882 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 883 unit_size, DMA_FROM_DEVICE); 884 if (ret) 885 goto e_src; 886 } 887 888 /* Send data to the CCP AES engine */ 889 while (src.sg_wa.bytes_left) { 890 ccp_prepare_data(&src, &dst, &op, unit_size, true); 891 if (!src.sg_wa.bytes_left) 892 op.eom = 1; 893 894 ret = cmd_q->ccp->vdata->perform->xts_aes(&op); 895 if (ret) { 896 cmd->engine_error = cmd_q->cmd_error; 897 goto e_dst; 898 } 899 900 ccp_process_data(&src, &dst, &op); 901 } 902 903 /* Retrieve the AES context - convert from LE to BE using 904 * 32-byte (256-bit) byteswapping 905 */ 906 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 907 CCP_PASSTHRU_BYTESWAP_256BIT); 908 if (ret) { 909 cmd->engine_error = cmd_q->cmd_error; 910 goto e_dst; 911 } 912 913 /* ...but we only need AES_BLOCK_SIZE bytes */ 914 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 915 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 916 917 e_dst: 918 if (!in_place) 919 ccp_free_data(&dst, cmd_q); 920 921 e_src: 922 ccp_free_data(&src, cmd_q); 923 924 e_ctx: 925 ccp_dm_free(&ctx); 926 927 e_key: 928 ccp_dm_free(&key); 929 930 return ret; 931 } 932 933 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 934 { 935 struct ccp_sha_engine *sha = &cmd->u.sha; 936 struct ccp_dm_workarea ctx; 937 struct ccp_data src; 938 struct ccp_op op; 939 int ret; 940 941 if (sha->ctx_len != CCP_SHA_CTXSIZE) 942 return -EINVAL; 943 944 if (!sha->ctx) 945 return -EINVAL; 946 947 if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1))) 948 return -EINVAL; 949 950 if (!sha->src_len) { 951 const u8 *sha_zero; 952 953 /* Not final, just return */ 954 if (!sha->final) 955 return 0; 956 957 /* CCP can't do a zero length sha operation so the caller 958 * must buffer the data. 959 */ 960 if (sha->msg_bits) 961 return -EINVAL; 962 963 /* The CCP cannot perform zero-length sha operations so the 964 * caller is required to buffer data for the final operation. 965 * However, a sha operation for a message with a total length 966 * of zero is valid so known values are required to supply 967 * the result. 968 */ 969 switch (sha->type) { 970 case CCP_SHA_TYPE_1: 971 sha_zero = sha1_zero_message_hash; 972 break; 973 case CCP_SHA_TYPE_224: 974 sha_zero = sha224_zero_message_hash; 975 break; 976 case CCP_SHA_TYPE_256: 977 sha_zero = sha256_zero_message_hash; 978 break; 979 default: 980 return -EINVAL; 981 } 982 983 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 984 sha->ctx_len, 1); 985 986 return 0; 987 } 988 989 if (!sha->src) 990 return -EINVAL; 991 992 BUILD_BUG_ON(CCP_SHA_SB_COUNT != 1); 993 994 memset(&op, 0, sizeof(op)); 995 op.cmd_q = cmd_q; 996 op.jobid = ccp_gen_jobid(cmd_q->ccp); 997 op.sb_ctx = cmd_q->sb_ctx; 998 op.u.sha.type = sha->type; 999 op.u.sha.msg_bits = sha->msg_bits; 1000 1001 /* The SHA context fits in a single (32-byte) SB entry and 1002 * must be in little endian format. Use the 256-bit byte swap 1003 * passthru option to convert from big endian to little endian. 1004 */ 1005 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1006 CCP_SHA_SB_COUNT * CCP_SB_BYTES, 1007 DMA_BIDIRECTIONAL); 1008 if (ret) 1009 return ret; 1010 1011 if (sha->first) { 1012 const __be32 *init; 1013 1014 switch (sha->type) { 1015 case CCP_SHA_TYPE_1: 1016 init = ccp_sha1_init; 1017 break; 1018 case CCP_SHA_TYPE_224: 1019 init = ccp_sha224_init; 1020 break; 1021 case CCP_SHA_TYPE_256: 1022 init = ccp_sha256_init; 1023 break; 1024 default: 1025 ret = -EINVAL; 1026 goto e_ctx; 1027 } 1028 memcpy(ctx.address, init, CCP_SHA_CTXSIZE); 1029 } else { 1030 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); 1031 } 1032 1033 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1034 CCP_PASSTHRU_BYTESWAP_256BIT); 1035 if (ret) { 1036 cmd->engine_error = cmd_q->cmd_error; 1037 goto e_ctx; 1038 } 1039 1040 /* Send data to the CCP SHA engine */ 1041 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1042 CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE); 1043 if (ret) 1044 goto e_ctx; 1045 1046 while (src.sg_wa.bytes_left) { 1047 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false); 1048 if (sha->final && !src.sg_wa.bytes_left) 1049 op.eom = 1; 1050 1051 ret = cmd_q->ccp->vdata->perform->sha(&op); 1052 if (ret) { 1053 cmd->engine_error = cmd_q->cmd_error; 1054 goto e_data; 1055 } 1056 1057 ccp_process_data(&src, NULL, &op); 1058 } 1059 1060 /* Retrieve the SHA context - convert from LE to BE using 1061 * 32-byte (256-bit) byteswapping to BE 1062 */ 1063 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1064 CCP_PASSTHRU_BYTESWAP_256BIT); 1065 if (ret) { 1066 cmd->engine_error = cmd_q->cmd_error; 1067 goto e_data; 1068 } 1069 1070 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); 1071 1072 if (sha->final && sha->opad) { 1073 /* HMAC operation, recursively perform final SHA */ 1074 struct ccp_cmd hmac_cmd; 1075 struct scatterlist sg; 1076 u64 block_size, digest_size; 1077 u8 *hmac_buf; 1078 1079 switch (sha->type) { 1080 case CCP_SHA_TYPE_1: 1081 block_size = SHA1_BLOCK_SIZE; 1082 digest_size = SHA1_DIGEST_SIZE; 1083 break; 1084 case CCP_SHA_TYPE_224: 1085 block_size = SHA224_BLOCK_SIZE; 1086 digest_size = SHA224_DIGEST_SIZE; 1087 break; 1088 case CCP_SHA_TYPE_256: 1089 block_size = SHA256_BLOCK_SIZE; 1090 digest_size = SHA256_DIGEST_SIZE; 1091 break; 1092 default: 1093 ret = -EINVAL; 1094 goto e_data; 1095 } 1096 1097 if (sha->opad_len != block_size) { 1098 ret = -EINVAL; 1099 goto e_data; 1100 } 1101 1102 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1103 if (!hmac_buf) { 1104 ret = -ENOMEM; 1105 goto e_data; 1106 } 1107 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1108 1109 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1110 memcpy(hmac_buf + block_size, ctx.address, digest_size); 1111 1112 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1113 hmac_cmd.engine = CCP_ENGINE_SHA; 1114 hmac_cmd.u.sha.type = sha->type; 1115 hmac_cmd.u.sha.ctx = sha->ctx; 1116 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1117 hmac_cmd.u.sha.src = &sg; 1118 hmac_cmd.u.sha.src_len = block_size + digest_size; 1119 hmac_cmd.u.sha.opad = NULL; 1120 hmac_cmd.u.sha.opad_len = 0; 1121 hmac_cmd.u.sha.first = 1; 1122 hmac_cmd.u.sha.final = 1; 1123 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1124 1125 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1126 if (ret) 1127 cmd->engine_error = hmac_cmd.engine_error; 1128 1129 kfree(hmac_buf); 1130 } 1131 1132 e_data: 1133 ccp_free_data(&src, cmd_q); 1134 1135 e_ctx: 1136 ccp_dm_free(&ctx); 1137 1138 return ret; 1139 } 1140 1141 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1142 { 1143 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1144 struct ccp_dm_workarea exp, src; 1145 struct ccp_data dst; 1146 struct ccp_op op; 1147 unsigned int sb_count, i_len, o_len; 1148 int ret; 1149 1150 if (rsa->key_size > CCP_RSA_MAX_WIDTH) 1151 return -EINVAL; 1152 1153 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1154 return -EINVAL; 1155 1156 /* The RSA modulus must precede the message being acted upon, so 1157 * it must be copied to a DMA area where the message and the 1158 * modulus can be concatenated. Therefore the input buffer 1159 * length required is twice the output buffer length (which 1160 * must be a multiple of 256-bits). 1161 */ 1162 o_len = ((rsa->key_size + 255) / 256) * 32; 1163 i_len = o_len * 2; 1164 1165 sb_count = o_len / CCP_SB_BYTES; 1166 1167 memset(&op, 0, sizeof(op)); 1168 op.cmd_q = cmd_q; 1169 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1170 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count); 1171 1172 if (!op.sb_key) 1173 return -EIO; 1174 1175 /* The RSA exponent may span multiple (32-byte) SB entries and must 1176 * be in little endian format. Reverse copy each 32-byte chunk 1177 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk) 1178 * and each byte within that chunk and do not perform any byte swap 1179 * operations on the passthru operation. 1180 */ 1181 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1182 if (ret) 1183 goto e_sb; 1184 1185 ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, 1186 CCP_SB_BYTES, false); 1187 if (ret) 1188 goto e_exp; 1189 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, 1190 CCP_PASSTHRU_BYTESWAP_NOOP); 1191 if (ret) { 1192 cmd->engine_error = cmd_q->cmd_error; 1193 goto e_exp; 1194 } 1195 1196 /* Concatenate the modulus and the message. Both the modulus and 1197 * the operands must be in little endian format. Since the input 1198 * is in big endian format it must be converted. 1199 */ 1200 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1201 if (ret) 1202 goto e_exp; 1203 1204 ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, 1205 CCP_SB_BYTES, false); 1206 if (ret) 1207 goto e_src; 1208 src.address += o_len; /* Adjust the address for the copy operation */ 1209 ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, 1210 CCP_SB_BYTES, false); 1211 if (ret) 1212 goto e_src; 1213 src.address -= o_len; /* Reset the address to original value */ 1214 1215 /* Prepare the output area for the operation */ 1216 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, 1217 o_len, DMA_FROM_DEVICE); 1218 if (ret) 1219 goto e_src; 1220 1221 op.soc = 1; 1222 op.src.u.dma.address = src.dma.address; 1223 op.src.u.dma.offset = 0; 1224 op.src.u.dma.length = i_len; 1225 op.dst.u.dma.address = dst.dm_wa.dma.address; 1226 op.dst.u.dma.offset = 0; 1227 op.dst.u.dma.length = o_len; 1228 1229 op.u.rsa.mod_size = rsa->key_size; 1230 op.u.rsa.input_len = i_len; 1231 1232 ret = cmd_q->ccp->vdata->perform->rsa(&op); 1233 if (ret) { 1234 cmd->engine_error = cmd_q->cmd_error; 1235 goto e_dst; 1236 } 1237 1238 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len); 1239 1240 e_dst: 1241 ccp_free_data(&dst, cmd_q); 1242 1243 e_src: 1244 ccp_dm_free(&src); 1245 1246 e_exp: 1247 ccp_dm_free(&exp); 1248 1249 e_sb: 1250 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); 1251 1252 return ret; 1253 } 1254 1255 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, 1256 struct ccp_cmd *cmd) 1257 { 1258 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1259 struct ccp_dm_workarea mask; 1260 struct ccp_data src, dst; 1261 struct ccp_op op; 1262 bool in_place = false; 1263 unsigned int i; 1264 int ret; 1265 1266 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1267 return -EINVAL; 1268 1269 if (!pt->src || !pt->dst) 1270 return -EINVAL; 1271 1272 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1273 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1274 return -EINVAL; 1275 if (!pt->mask) 1276 return -EINVAL; 1277 } 1278 1279 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1280 1281 memset(&op, 0, sizeof(op)); 1282 op.cmd_q = cmd_q; 1283 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1284 1285 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1286 /* Load the mask */ 1287 op.sb_key = cmd_q->sb_key; 1288 1289 ret = ccp_init_dm_workarea(&mask, cmd_q, 1290 CCP_PASSTHRU_SB_COUNT * 1291 CCP_SB_BYTES, 1292 DMA_TO_DEVICE); 1293 if (ret) 1294 return ret; 1295 1296 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 1297 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 1298 CCP_PASSTHRU_BYTESWAP_NOOP); 1299 if (ret) { 1300 cmd->engine_error = cmd_q->cmd_error; 1301 goto e_mask; 1302 } 1303 } 1304 1305 /* Prepare the input and output data workareas. For in-place 1306 * operations we need to set the dma direction to BIDIRECTIONAL 1307 * and copy the src workarea to the dst workarea. 1308 */ 1309 if (sg_virt(pt->src) == sg_virt(pt->dst)) 1310 in_place = true; 1311 1312 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 1313 CCP_PASSTHRU_MASKSIZE, 1314 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1315 if (ret) 1316 goto e_mask; 1317 1318 if (in_place) { 1319 dst = src; 1320 } else { 1321 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 1322 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 1323 if (ret) 1324 goto e_src; 1325 } 1326 1327 /* Send data to the CCP Passthru engine 1328 * Because the CCP engine works on a single source and destination 1329 * dma address at a time, each entry in the source scatterlist 1330 * (after the dma_map_sg call) must be less than or equal to the 1331 * (remaining) length in the destination scatterlist entry and the 1332 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 1333 */ 1334 dst.sg_wa.sg_used = 0; 1335 for (i = 1; i <= src.sg_wa.dma_count; i++) { 1336 if (!dst.sg_wa.sg || 1337 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { 1338 ret = -EINVAL; 1339 goto e_dst; 1340 } 1341 1342 if (i == src.sg_wa.dma_count) { 1343 op.eom = 1; 1344 op.soc = 1; 1345 } 1346 1347 op.src.type = CCP_MEMTYPE_SYSTEM; 1348 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 1349 op.src.u.dma.offset = 0; 1350 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 1351 1352 op.dst.type = CCP_MEMTYPE_SYSTEM; 1353 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 1354 op.dst.u.dma.offset = dst.sg_wa.sg_used; 1355 op.dst.u.dma.length = op.src.u.dma.length; 1356 1357 ret = cmd_q->ccp->vdata->perform->passthru(&op); 1358 if (ret) { 1359 cmd->engine_error = cmd_q->cmd_error; 1360 goto e_dst; 1361 } 1362 1363 dst.sg_wa.sg_used += src.sg_wa.sg->length; 1364 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { 1365 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 1366 dst.sg_wa.sg_used = 0; 1367 } 1368 src.sg_wa.sg = sg_next(src.sg_wa.sg); 1369 } 1370 1371 e_dst: 1372 if (!in_place) 1373 ccp_free_data(&dst, cmd_q); 1374 1375 e_src: 1376 ccp_free_data(&src, cmd_q); 1377 1378 e_mask: 1379 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 1380 ccp_dm_free(&mask); 1381 1382 return ret; 1383 } 1384 1385 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q, 1386 struct ccp_cmd *cmd) 1387 { 1388 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap; 1389 struct ccp_dm_workarea mask; 1390 struct ccp_op op; 1391 int ret; 1392 1393 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1394 return -EINVAL; 1395 1396 if (!pt->src_dma || !pt->dst_dma) 1397 return -EINVAL; 1398 1399 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1400 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1401 return -EINVAL; 1402 if (!pt->mask) 1403 return -EINVAL; 1404 } 1405 1406 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1407 1408 memset(&op, 0, sizeof(op)); 1409 op.cmd_q = cmd_q; 1410 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1411 1412 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1413 /* Load the mask */ 1414 op.sb_key = cmd_q->sb_key; 1415 1416 mask.length = pt->mask_len; 1417 mask.dma.address = pt->mask; 1418 mask.dma.length = pt->mask_len; 1419 1420 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 1421 CCP_PASSTHRU_BYTESWAP_NOOP); 1422 if (ret) { 1423 cmd->engine_error = cmd_q->cmd_error; 1424 return ret; 1425 } 1426 } 1427 1428 /* Send data to the CCP Passthru engine */ 1429 op.eom = 1; 1430 op.soc = 1; 1431 1432 op.src.type = CCP_MEMTYPE_SYSTEM; 1433 op.src.u.dma.address = pt->src_dma; 1434 op.src.u.dma.offset = 0; 1435 op.src.u.dma.length = pt->src_len; 1436 1437 op.dst.type = CCP_MEMTYPE_SYSTEM; 1438 op.dst.u.dma.address = pt->dst_dma; 1439 op.dst.u.dma.offset = 0; 1440 op.dst.u.dma.length = pt->src_len; 1441 1442 ret = cmd_q->ccp->vdata->perform->passthru(&op); 1443 if (ret) 1444 cmd->engine_error = cmd_q->cmd_error; 1445 1446 return ret; 1447 } 1448 1449 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1450 { 1451 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1452 struct ccp_dm_workarea src, dst; 1453 struct ccp_op op; 1454 int ret; 1455 u8 *save; 1456 1457 if (!ecc->u.mm.operand_1 || 1458 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 1459 return -EINVAL; 1460 1461 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 1462 if (!ecc->u.mm.operand_2 || 1463 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 1464 return -EINVAL; 1465 1466 if (!ecc->u.mm.result || 1467 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 1468 return -EINVAL; 1469 1470 memset(&op, 0, sizeof(op)); 1471 op.cmd_q = cmd_q; 1472 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1473 1474 /* Concatenate the modulus and the operands. Both the modulus and 1475 * the operands must be in little endian format. Since the input 1476 * is in big endian format it must be converted and placed in a 1477 * fixed length buffer. 1478 */ 1479 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1480 DMA_TO_DEVICE); 1481 if (ret) 1482 return ret; 1483 1484 /* Save the workarea address since it is updated in order to perform 1485 * the concatenation 1486 */ 1487 save = src.address; 1488 1489 /* Copy the ECC modulus */ 1490 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, 1491 CCP_ECC_OPERAND_SIZE, false); 1492 if (ret) 1493 goto e_src; 1494 src.address += CCP_ECC_OPERAND_SIZE; 1495 1496 /* Copy the first operand */ 1497 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1, 1498 ecc->u.mm.operand_1_len, 1499 CCP_ECC_OPERAND_SIZE, false); 1500 if (ret) 1501 goto e_src; 1502 src.address += CCP_ECC_OPERAND_SIZE; 1503 1504 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 1505 /* Copy the second operand */ 1506 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2, 1507 ecc->u.mm.operand_2_len, 1508 CCP_ECC_OPERAND_SIZE, false); 1509 if (ret) 1510 goto e_src; 1511 src.address += CCP_ECC_OPERAND_SIZE; 1512 } 1513 1514 /* Restore the workarea address */ 1515 src.address = save; 1516 1517 /* Prepare the output area for the operation */ 1518 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 1519 DMA_FROM_DEVICE); 1520 if (ret) 1521 goto e_src; 1522 1523 op.soc = 1; 1524 op.src.u.dma.address = src.dma.address; 1525 op.src.u.dma.offset = 0; 1526 op.src.u.dma.length = src.length; 1527 op.dst.u.dma.address = dst.dma.address; 1528 op.dst.u.dma.offset = 0; 1529 op.dst.u.dma.length = dst.length; 1530 1531 op.u.ecc.function = cmd->u.ecc.function; 1532 1533 ret = cmd_q->ccp->vdata->perform->ecc(&op); 1534 if (ret) { 1535 cmd->engine_error = cmd_q->cmd_error; 1536 goto e_dst; 1537 } 1538 1539 ecc->ecc_result = le16_to_cpup( 1540 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 1541 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 1542 ret = -EIO; 1543 goto e_dst; 1544 } 1545 1546 /* Save the ECC result */ 1547 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES); 1548 1549 e_dst: 1550 ccp_dm_free(&dst); 1551 1552 e_src: 1553 ccp_dm_free(&src); 1554 1555 return ret; 1556 } 1557 1558 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1559 { 1560 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1561 struct ccp_dm_workarea src, dst; 1562 struct ccp_op op; 1563 int ret; 1564 u8 *save; 1565 1566 if (!ecc->u.pm.point_1.x || 1567 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 1568 !ecc->u.pm.point_1.y || 1569 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 1570 return -EINVAL; 1571 1572 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1573 if (!ecc->u.pm.point_2.x || 1574 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 1575 !ecc->u.pm.point_2.y || 1576 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 1577 return -EINVAL; 1578 } else { 1579 if (!ecc->u.pm.domain_a || 1580 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 1581 return -EINVAL; 1582 1583 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 1584 if (!ecc->u.pm.scalar || 1585 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 1586 return -EINVAL; 1587 } 1588 1589 if (!ecc->u.pm.result.x || 1590 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 1591 !ecc->u.pm.result.y || 1592 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 1593 return -EINVAL; 1594 1595 memset(&op, 0, sizeof(op)); 1596 op.cmd_q = cmd_q; 1597 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1598 1599 /* Concatenate the modulus and the operands. Both the modulus and 1600 * the operands must be in little endian format. Since the input 1601 * is in big endian format it must be converted and placed in a 1602 * fixed length buffer. 1603 */ 1604 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1605 DMA_TO_DEVICE); 1606 if (ret) 1607 return ret; 1608 1609 /* Save the workarea address since it is updated in order to perform 1610 * the concatenation 1611 */ 1612 save = src.address; 1613 1614 /* Copy the ECC modulus */ 1615 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, 1616 CCP_ECC_OPERAND_SIZE, false); 1617 if (ret) 1618 goto e_src; 1619 src.address += CCP_ECC_OPERAND_SIZE; 1620 1621 /* Copy the first point X and Y coordinate */ 1622 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x, 1623 ecc->u.pm.point_1.x_len, 1624 CCP_ECC_OPERAND_SIZE, false); 1625 if (ret) 1626 goto e_src; 1627 src.address += CCP_ECC_OPERAND_SIZE; 1628 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y, 1629 ecc->u.pm.point_1.y_len, 1630 CCP_ECC_OPERAND_SIZE, false); 1631 if (ret) 1632 goto e_src; 1633 src.address += CCP_ECC_OPERAND_SIZE; 1634 1635 /* Set the first point Z coordianate to 1 */ 1636 *src.address = 0x01; 1637 src.address += CCP_ECC_OPERAND_SIZE; 1638 1639 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1640 /* Copy the second point X and Y coordinate */ 1641 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x, 1642 ecc->u.pm.point_2.x_len, 1643 CCP_ECC_OPERAND_SIZE, false); 1644 if (ret) 1645 goto e_src; 1646 src.address += CCP_ECC_OPERAND_SIZE; 1647 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y, 1648 ecc->u.pm.point_2.y_len, 1649 CCP_ECC_OPERAND_SIZE, false); 1650 if (ret) 1651 goto e_src; 1652 src.address += CCP_ECC_OPERAND_SIZE; 1653 1654 /* Set the second point Z coordianate to 1 */ 1655 *src.address = 0x01; 1656 src.address += CCP_ECC_OPERAND_SIZE; 1657 } else { 1658 /* Copy the Domain "a" parameter */ 1659 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a, 1660 ecc->u.pm.domain_a_len, 1661 CCP_ECC_OPERAND_SIZE, false); 1662 if (ret) 1663 goto e_src; 1664 src.address += CCP_ECC_OPERAND_SIZE; 1665 1666 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 1667 /* Copy the scalar value */ 1668 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar, 1669 ecc->u.pm.scalar_len, 1670 CCP_ECC_OPERAND_SIZE, 1671 false); 1672 if (ret) 1673 goto e_src; 1674 src.address += CCP_ECC_OPERAND_SIZE; 1675 } 1676 } 1677 1678 /* Restore the workarea address */ 1679 src.address = save; 1680 1681 /* Prepare the output area for the operation */ 1682 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 1683 DMA_FROM_DEVICE); 1684 if (ret) 1685 goto e_src; 1686 1687 op.soc = 1; 1688 op.src.u.dma.address = src.dma.address; 1689 op.src.u.dma.offset = 0; 1690 op.src.u.dma.length = src.length; 1691 op.dst.u.dma.address = dst.dma.address; 1692 op.dst.u.dma.offset = 0; 1693 op.dst.u.dma.length = dst.length; 1694 1695 op.u.ecc.function = cmd->u.ecc.function; 1696 1697 ret = cmd_q->ccp->vdata->perform->ecc(&op); 1698 if (ret) { 1699 cmd->engine_error = cmd_q->cmd_error; 1700 goto e_dst; 1701 } 1702 1703 ecc->ecc_result = le16_to_cpup( 1704 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 1705 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 1706 ret = -EIO; 1707 goto e_dst; 1708 } 1709 1710 /* Save the workarea address since it is updated as we walk through 1711 * to copy the point math result 1712 */ 1713 save = dst.address; 1714 1715 /* Save the ECC result X and Y coordinates */ 1716 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x, 1717 CCP_ECC_MODULUS_BYTES); 1718 dst.address += CCP_ECC_OUTPUT_SIZE; 1719 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y, 1720 CCP_ECC_MODULUS_BYTES); 1721 dst.address += CCP_ECC_OUTPUT_SIZE; 1722 1723 /* Restore the workarea address */ 1724 dst.address = save; 1725 1726 e_dst: 1727 ccp_dm_free(&dst); 1728 1729 e_src: 1730 ccp_dm_free(&src); 1731 1732 return ret; 1733 } 1734 1735 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1736 { 1737 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1738 1739 ecc->ecc_result = 0; 1740 1741 if (!ecc->mod || 1742 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 1743 return -EINVAL; 1744 1745 switch (ecc->function) { 1746 case CCP_ECC_FUNCTION_MMUL_384BIT: 1747 case CCP_ECC_FUNCTION_MADD_384BIT: 1748 case CCP_ECC_FUNCTION_MINV_384BIT: 1749 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 1750 1751 case CCP_ECC_FUNCTION_PADD_384BIT: 1752 case CCP_ECC_FUNCTION_PMUL_384BIT: 1753 case CCP_ECC_FUNCTION_PDBL_384BIT: 1754 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 1755 1756 default: 1757 return -EINVAL; 1758 } 1759 } 1760 1761 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1762 { 1763 int ret; 1764 1765 cmd->engine_error = 0; 1766 cmd_q->cmd_error = 0; 1767 cmd_q->int_rcvd = 0; 1768 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q); 1769 1770 switch (cmd->engine) { 1771 case CCP_ENGINE_AES: 1772 ret = ccp_run_aes_cmd(cmd_q, cmd); 1773 break; 1774 case CCP_ENGINE_XTS_AES_128: 1775 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 1776 break; 1777 case CCP_ENGINE_SHA: 1778 ret = ccp_run_sha_cmd(cmd_q, cmd); 1779 break; 1780 case CCP_ENGINE_RSA: 1781 ret = ccp_run_rsa_cmd(cmd_q, cmd); 1782 break; 1783 case CCP_ENGINE_PASSTHRU: 1784 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP) 1785 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd); 1786 else 1787 ret = ccp_run_passthru_cmd(cmd_q, cmd); 1788 break; 1789 case CCP_ENGINE_ECC: 1790 ret = ccp_run_ecc_cmd(cmd_q, cmd); 1791 break; 1792 default: 1793 ret = -EINVAL; 1794 } 1795 1796 return ret; 1797 } 1798